blob: 3fd1af90211236dfad7b488206bdf0b7214deaf3 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Kernel Probes (KProbes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright (C) IBM Corporation, 2002, 2004
19 *
20 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
21 * Probes initial implementation ( includes contributions from
22 * Rusty Russell).
23 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
24 * interface to access function arguments.
25 * 2004-Nov Ananth N Mavinakayanahalli <ananth@in.ibm.com> kprobes port
26 * for PPC64
27 */
28
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/kprobes.h>
30#include <linux/ptrace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070031#include <linux/preempt.h>
Prasanna S Panchamukhi50e21f22006-03-26 01:38:24 -080032#include <linux/module.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070033#include <linux/kdebug.h>
Rusty Lynch7e1048b2005-06-23 00:09:25 -070034#include <asm/cacheflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <asm/sstep.h>
Prasanna S Panchamukhi50e21f22006-03-26 01:38:24 -080036#include <asm/uaccess.h>
Kumar Galaf8279622008-06-26 02:01:37 -050037#include <asm/system.h>
38
Dave Kleikamp172ae2e2010-02-08 11:50:57 +000039#ifdef CONFIG_PPC_ADV_DEBUG_REGS
Kumar Galaf8279622008-06-26 02:01:37 -050040#define MSR_SINGLESTEP (MSR_DE)
41#else
42#define MSR_SINGLESTEP (MSR_SE)
43#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070044
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -080045DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
46DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
Linus Torvalds1da177e2005-04-16 15:20:36 -070047
Masami Hiramatsuf438d912007-10-16 01:27:49 -070048struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}};
49
Prasanna S Panchamukhibb144a82005-09-06 15:19:29 -070050int __kprobes arch_prepare_kprobe(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -070051{
Ananth N Mavinakayanahalli63224d1e82005-06-08 15:49:41 -070052 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053 kprobe_opcode_t insn = *p->addr;
54
Ananth N Mavinakayanahalli63224d1e82005-06-08 15:49:41 -070055 if ((unsigned long)p->addr & 0x03) {
56 printk("Attempt to register kprobe at an unaligned address\n");
57 ret = -EINVAL;
Kumar Gala82090032007-02-06 22:55:19 -060058 } else if (IS_MTMSRD(insn) || IS_RFID(insn) || IS_RFI(insn)) {
59 printk("Cannot register a kprobe on rfi/rfid or mtmsr[d]\n");
Ananth N Mavinakayanahalli63224d1e82005-06-08 15:49:41 -070060 ret = -EINVAL;
61 }
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070062
Kumar Galaf8279622008-06-26 02:01:37 -050063 /* insn must be on a special executable page on ppc64. This is
64 * not explicitly required on ppc32 (right now), but it doesn't hurt */
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070065 if (!ret) {
Ananth N Mavinakayanahalli2d8ab6a2005-10-01 13:14:17 -040066 p->ainsn.insn = get_insn_slot();
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070067 if (!p->ainsn.insn)
68 ret = -ENOMEM;
69 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070070
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -080071 if (!ret) {
Ananth N Mavinakayanahallie6349a952007-04-18 15:57:51 +100072 memcpy(p->ainsn.insn, p->addr,
73 MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -080074 p->opcode = *p->addr;
Ananth N Mavinakayanahalli83db3dd2006-08-11 17:01:34 +053075 flush_icache_range((unsigned long)p->ainsn.insn,
76 (unsigned long)p->ainsn.insn + sizeof(kprobe_opcode_t));
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -080077 }
78
Ananth N Mavinakayanahallie6349a952007-04-18 15:57:51 +100079 p->ainsn.boostable = 0;
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -080080 return ret;
Rusty Lynch7e1048b2005-06-23 00:09:25 -070081}
82
Prasanna S Panchamukhibb144a82005-09-06 15:19:29 -070083void __kprobes arch_arm_kprobe(struct kprobe *p)
Rusty Lynch7e1048b2005-06-23 00:09:25 -070084{
85 *p->addr = BREAKPOINT_INSTRUCTION;
86 flush_icache_range((unsigned long) p->addr,
87 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
88}
89
Prasanna S Panchamukhibb144a82005-09-06 15:19:29 -070090void __kprobes arch_disarm_kprobe(struct kprobe *p)
Rusty Lynch7e1048b2005-06-23 00:09:25 -070091{
92 *p->addr = p->opcode;
93 flush_icache_range((unsigned long) p->addr,
94 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -070095}
96
Ananth N Mavinakayanahalli0498b632006-01-09 20:52:46 -080097void __kprobes arch_remove_kprobe(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -070098{
Masami Hiramatsu12941562009-01-06 14:41:50 -080099 if (p->ainsn.insn) {
100 free_insn_slot(p->ainsn.insn, 0);
101 p->ainsn.insn = NULL;
102 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103}
104
Prasanna S Panchamukhi46dbe2f2006-04-18 22:22:01 -0700105static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106{
Kumar Galab76e59d2008-06-26 01:57:58 -0500107 /* We turn off async exceptions to ensure that the single step will
108 * be for the instruction we have the kprobe on, if we dont its
109 * possible we'd get the single step reported for an exception handler
110 * like Decrementer or External Interrupt */
111 regs->msr &= ~MSR_EE;
Kumar Galaf8279622008-06-26 02:01:37 -0500112 regs->msr |= MSR_SINGLESTEP;
Dave Kleikamp172ae2e2010-02-08 11:50:57 +0000113#ifdef CONFIG_PPC_ADV_DEBUG_REGS
Kumar Galaf8279622008-06-26 02:01:37 -0500114 regs->msr &= ~MSR_CE;
115 mtspr(SPRN_DBCR0, mfspr(SPRN_DBCR0) | DBCR0_IC | DBCR0_IDM);
116#endif
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700117
Ananth N Mavinakayanahalli0ccde0a2006-04-28 17:38:42 +0530118 /*
119 * On powerpc we should single step on the original
120 * instruction even if the probed insn is a trap
121 * variant as values in regs could play a part in
122 * if the trap is taken or not
123 */
124 regs->nip = (unsigned long)p->ainsn.insn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700125}
126
Prasanna S Panchamukhi46dbe2f2006-04-18 22:22:01 -0700127static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700128{
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800129 kcb->prev_kprobe.kp = kprobe_running();
130 kcb->prev_kprobe.status = kcb->kprobe_status;
131 kcb->prev_kprobe.saved_msr = kcb->kprobe_saved_msr;
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700132}
133
Prasanna S Panchamukhi46dbe2f2006-04-18 22:22:01 -0700134static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700135{
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800136 __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
137 kcb->kprobe_status = kcb->prev_kprobe.status;
138 kcb->kprobe_saved_msr = kcb->prev_kprobe.saved_msr;
139}
140
Prasanna S Panchamukhi46dbe2f2006-04-18 22:22:01 -0700141static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800142 struct kprobe_ctlblk *kcb)
143{
144 __get_cpu_var(current_kprobe) = p;
145 kcb->kprobe_saved_msr = regs->msr;
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700146}
147
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700148void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
Prasanna S Panchamukhibb144a82005-09-06 15:19:29 -0700149 struct pt_regs *regs)
Rusty Lynch97f79432005-06-27 15:17:15 -0700150{
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700151 ri->ret_addr = (kprobe_opcode_t *)regs->link;
Rusty Lynch97f79432005-06-27 15:17:15 -0700152
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700153 /* Replace the return addr with trampoline addr */
154 regs->link = (unsigned long)kretprobe_trampoline;
Rusty Lynch97f79432005-06-27 15:17:15 -0700155}
156
Prasanna S Panchamukhi46dbe2f2006-04-18 22:22:01 -0700157static int __kprobes kprobe_handler(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158{
159 struct kprobe *p;
160 int ret = 0;
161 unsigned int *addr = (unsigned int *)regs->nip;
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800162 struct kprobe_ctlblk *kcb;
163
164 /*
165 * We don't want to be preempted for the entire
166 * duration of kprobe processing
167 */
168 preempt_disable();
169 kcb = get_kprobe_ctlblk();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170
171 /* Check we're not actually recursing */
172 if (kprobe_running()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 p = get_kprobe(addr);
174 if (p) {
Keshavamurthy Anil Sdeac66a2005-09-06 15:19:35 -0700175 kprobe_opcode_t insn = *p->ainsn.insn;
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800176 if (kcb->kprobe_status == KPROBE_HIT_SS &&
Keshavamurthy Anil Sdeac66a2005-09-06 15:19:35 -0700177 is_trap(insn)) {
Kumar Galaf8279622008-06-26 02:01:37 -0500178 /* Turn off 'trace' bits */
179 regs->msr &= ~MSR_SINGLESTEP;
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800180 regs->msr |= kcb->kprobe_saved_msr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 goto no_kprobe;
182 }
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700183 /* We have reentered the kprobe_handler(), since
184 * another probe was hit while within the handler.
185 * We here save the original kprobes variables and
186 * just single step on the instruction of the new probe
187 * without calling any user handlers.
188 */
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800189 save_previous_kprobe(kcb);
190 set_current_kprobe(p, regs, kcb);
191 kcb->kprobe_saved_msr = regs->msr;
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -0800192 kprobes_inc_nmissed_count(p);
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700193 prepare_singlestep(p, regs);
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800194 kcb->kprobe_status = KPROBE_REENTER;
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700195 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196 } else {
Keshavamurthy Anil Seb3a7292006-01-11 12:17:42 -0800197 if (*addr != BREAKPOINT_INSTRUCTION) {
198 /* If trap variant, then it belongs not to us */
199 kprobe_opcode_t cur_insn = *addr;
200 if (is_trap(cur_insn))
201 goto no_kprobe;
202 /* The breakpoint instruction was removed by
203 * another cpu right after we hit, no further
204 * handling of this interrupt is appropriate
205 */
206 ret = 1;
207 goto no_kprobe;
208 }
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800209 p = __get_cpu_var(current_kprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210 if (p->break_handler && p->break_handler(p, regs)) {
211 goto ss_probe;
212 }
213 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214 goto no_kprobe;
215 }
216
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217 p = get_kprobe(addr);
218 if (!p) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700219 if (*addr != BREAKPOINT_INSTRUCTION) {
220 /*
221 * PowerPC has multiple variants of the "trap"
222 * instruction. If the current instruction is a
223 * trap variant, it could belong to someone else
224 */
225 kprobe_opcode_t cur_insn = *addr;
Keshavamurthy Anil Sdeac66a2005-09-06 15:19:35 -0700226 if (is_trap(cur_insn))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227 goto no_kprobe;
228 /*
229 * The breakpoint instruction was removed right
230 * after we hit it. Another cpu has removed
231 * either a probepoint or a debugger breakpoint
232 * at this address. In either case, no further
233 * handling of this interrupt is appropriate.
234 */
235 ret = 1;
236 }
237 /* Not one of ours: let kernel handle it */
238 goto no_kprobe;
239 }
240
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800241 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
242 set_current_kprobe(p, regs, kcb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 if (p->pre_handler && p->pre_handler(p, regs))
244 /* handler has already set things up, so skip ss setup */
245 return 1;
246
247ss_probe:
Ananth N Mavinakayanahallie6349a952007-04-18 15:57:51 +1000248 if (p->ainsn.boostable >= 0) {
249 unsigned int insn = *p->ainsn.insn;
250
251 /* regs->nip is also adjusted if emulate_step returns 1 */
252 ret = emulate_step(regs, insn);
253 if (ret > 0) {
254 /*
255 * Once this instruction has been boosted
256 * successfully, set the boostable flag
257 */
258 if (unlikely(p->ainsn.boostable == 0))
259 p->ainsn.boostable = 1;
260
261 if (p->post_handler)
262 p->post_handler(p, regs, 0);
263
264 kcb->kprobe_status = KPROBE_HIT_SSDONE;
265 reset_current_kprobe();
266 preempt_enable_no_resched();
267 return 1;
268 } else if (ret < 0) {
269 /*
270 * We don't allow kprobes on mtmsr(d)/rfi(d), etc.
271 * So, we should never get here... but, its still
272 * good to catch them, just in case...
273 */
274 printk("Can't step on instruction %x\n", insn);
275 BUG();
276 } else if (ret == 0)
277 /* This instruction can't be boosted */
278 p->ainsn.boostable = -1;
279 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 prepare_singlestep(p, regs);
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800281 kcb->kprobe_status = KPROBE_HIT_SS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 return 1;
283
284no_kprobe:
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800285 preempt_enable_no_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 return ret;
287}
288
289/*
Rusty Lynch97f79432005-06-27 15:17:15 -0700290 * Function return probe trampoline:
291 * - init_kprobes() establishes a probepoint here
292 * - When the probed function returns, this probe
293 * causes the handlers to fire
294 */
Ananth N Mavinakayanahalli66200ea2008-02-15 21:59:17 +1100295static void __used kretprobe_trampoline_holder(void)
Rusty Lynch97f79432005-06-27 15:17:15 -0700296{
297 asm volatile(".global kretprobe_trampoline\n"
298 "kretprobe_trampoline:\n"
299 "nop\n");
300}
301
302/*
303 * Called when the probe at kretprobe trampoline is hit
304 */
Ananth N Mavinakayanahalli66200ea2008-02-15 21:59:17 +1100305static int __kprobes trampoline_probe_handler(struct kprobe *p,
306 struct pt_regs *regs)
Rusty Lynch97f79432005-06-27 15:17:15 -0700307{
bibo,mao62c27be2006-10-02 02:17:33 -0700308 struct kretprobe_instance *ri = NULL;
bibo,mao99219a32006-10-02 02:17:35 -0700309 struct hlist_head *head, empty_rp;
bibo,mao62c27be2006-10-02 02:17:33 -0700310 struct hlist_node *node, *tmp;
Ananth N Mavinakayanahalli991a51d2005-11-07 01:00:14 -0800311 unsigned long flags, orig_ret_address = 0;
Rusty Lynch97f79432005-06-27 15:17:15 -0700312 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
313
bibo,mao99219a32006-10-02 02:17:35 -0700314 INIT_HLIST_HEAD(&empty_rp);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700315 kretprobe_hash_lock(current, &head, &flags);
Rusty Lynch97f79432005-06-27 15:17:15 -0700316
317 /*
318 * It is possible to have multiple instances associated with a given
319 * task either because an multiple functions in the call path
Frederik Schwarzer025dfda2008-10-16 19:02:37 +0200320 * have a return probe installed on them, and/or more than one return
Rusty Lynch97f79432005-06-27 15:17:15 -0700321 * return probe was registered for a target function.
322 *
323 * We can handle this because:
324 * - instances are always inserted at the head of the list
325 * - when multiple return probes are registered for the same
bibo,mao62c27be2006-10-02 02:17:33 -0700326 * function, the first instance's ret_addr will point to the
Rusty Lynch97f79432005-06-27 15:17:15 -0700327 * real return address, and all the rest will point to
328 * kretprobe_trampoline
329 */
330 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
bibo,mao62c27be2006-10-02 02:17:33 -0700331 if (ri->task != current)
Rusty Lynch97f79432005-06-27 15:17:15 -0700332 /* another task is sharing our hash bucket */
bibo,mao62c27be2006-10-02 02:17:33 -0700333 continue;
Rusty Lynch97f79432005-06-27 15:17:15 -0700334
335 if (ri->rp && ri->rp->handler)
336 ri->rp->handler(ri, regs);
337
338 orig_ret_address = (unsigned long)ri->ret_addr;
bibo,mao99219a32006-10-02 02:17:35 -0700339 recycle_rp_inst(ri, &empty_rp);
Rusty Lynch97f79432005-06-27 15:17:15 -0700340
341 if (orig_ret_address != trampoline_address)
342 /*
343 * This is the real return address. Any other
344 * instances associated with this task are for
345 * other calls deeper on the call stack
346 */
347 break;
348 }
349
Ananth N Mavinakayanahalli0f95b7f2007-05-08 00:28:27 -0700350 kretprobe_assert(ri, orig_ret_address, trampoline_address);
Rusty Lynch97f79432005-06-27 15:17:15 -0700351 regs->nip = orig_ret_address;
352
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800353 reset_current_kprobe();
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700354 kretprobe_hash_unlock(current, &flags);
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800355 preempt_enable_no_resched();
Rusty Lynch97f79432005-06-27 15:17:15 -0700356
bibo,mao99219a32006-10-02 02:17:35 -0700357 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
358 hlist_del(&ri->hlist);
359 kfree(ri);
360 }
bibo,mao62c27be2006-10-02 02:17:33 -0700361 /*
362 * By returning a non-zero value, we are telling
363 * kprobe_handler() that we don't want the post_handler
364 * to run (and have re-enabled preemption)
365 */
366 return 1;
Rusty Lynch97f79432005-06-27 15:17:15 -0700367}
368
369/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 * Called after single-stepping. p->addr is the address of the
371 * instruction whose first byte has been replaced by the "breakpoint"
372 * instruction. To avoid the SMP problems that can occur when we
373 * temporarily put back the original opcode to single-step, we
374 * single-stepped a copy of the instruction. The address of this
375 * copy is p->ainsn.insn.
376 */
Prasanna S Panchamukhibb144a82005-09-06 15:19:29 -0700377static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378{
379 int ret;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700380 unsigned int insn = *p->ainsn.insn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381
382 regs->nip = (unsigned long)p->addr;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700383 ret = emulate_step(regs, insn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384 if (ret == 0)
385 regs->nip = (unsigned long)p->addr + 4;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386}
387
Prasanna S Panchamukhi46dbe2f2006-04-18 22:22:01 -0700388static int __kprobes post_kprobe_handler(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389{
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800390 struct kprobe *cur = kprobe_running();
391 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
392
393 if (!cur)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394 return 0;
395
Kumar Galab76e59d2008-06-26 01:57:58 -0500396 /* make sure we got here for instruction we have a kprobe on */
397 if (((unsigned long)cur->ainsn.insn + 4) != regs->nip)
398 return 0;
399
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800400 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
401 kcb->kprobe_status = KPROBE_HIT_SSDONE;
402 cur->post_handler(cur, regs, 0);
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700403 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800405 resume_execution(cur, regs);
406 regs->msr |= kcb->kprobe_saved_msr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700407
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700408 /*Restore back the original saved kprobes variables and continue. */
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800409 if (kcb->kprobe_status == KPROBE_REENTER) {
410 restore_previous_kprobe(kcb);
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700411 goto out;
412 }
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800413 reset_current_kprobe();
Prasanna S Panchamukhi42cc2062005-06-23 00:09:38 -0700414out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 preempt_enable_no_resched();
416
417 /*
418 * if somebody else is singlestepping across a probe point, msr
Kumar Galaf8279622008-06-26 02:01:37 -0500419 * will have DE/SE set, in which case, continue the remaining processing
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 * of do_debug, as if this is not a probe hit.
421 */
Kumar Galaf8279622008-06-26 02:01:37 -0500422 if (regs->msr & MSR_SINGLESTEP)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 return 0;
424
425 return 1;
426}
427
Christoph Hellwig9f90b992007-04-30 11:56:46 +0100428int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700429{
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800430 struct kprobe *cur = kprobe_running();
431 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
Prasanna S Panchamukhi50e21f22006-03-26 01:38:24 -0800432 const struct exception_table_entry *entry;
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800433
Prasanna S Panchamukhi50e21f22006-03-26 01:38:24 -0800434 switch(kcb->kprobe_status) {
435 case KPROBE_HIT_SS:
436 case KPROBE_REENTER:
437 /*
438 * We are here because the instruction being single
439 * stepped caused a page fault. We reset the current
440 * kprobe and the nip points back to the probe address
441 * and allow the page fault handler to continue as a
442 * normal page fault.
443 */
444 regs->nip = (unsigned long)cur->addr;
Kumar Galaf8279622008-06-26 02:01:37 -0500445 regs->msr &= ~MSR_SINGLESTEP; /* Turn off 'trace' bits */
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800446 regs->msr |= kcb->kprobe_saved_msr;
Prasanna S Panchamukhi50e21f22006-03-26 01:38:24 -0800447 if (kcb->kprobe_status == KPROBE_REENTER)
448 restore_previous_kprobe(kcb);
449 else
450 reset_current_kprobe();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700451 preempt_enable_no_resched();
Prasanna S Panchamukhi50e21f22006-03-26 01:38:24 -0800452 break;
453 case KPROBE_HIT_ACTIVE:
454 case KPROBE_HIT_SSDONE:
455 /*
456 * We increment the nmissed count for accounting,
457 * we can also use npre/npostfault count for accouting
458 * these specific fault cases.
459 */
460 kprobes_inc_nmissed_count(cur);
461
462 /*
463 * We come here because instructions in the pre/post
464 * handler caused the page_fault, this could happen
465 * if handler tries to access user space by
466 * copy_from_user(), get_user() etc. Let the
467 * user-specified handler try to fix it first.
468 */
469 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
470 return 1;
471
472 /*
473 * In case the user-specified fault handler returned
474 * zero, try to fix up.
475 */
476 if ((entry = search_exception_tables(regs->nip)) != NULL) {
477 regs->nip = entry->fixup;
478 return 1;
479 }
480
481 /*
482 * fixup_exception() could not handle it,
483 * Let do_page_fault() fix it.
484 */
485 break;
486 default:
487 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488 }
489 return 0;
490}
491
492/*
493 * Wrapper routine to for handling exceptions.
494 */
Prasanna S Panchamukhibb144a82005-09-06 15:19:29 -0700495int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
496 unsigned long val, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700497{
498 struct die_args *args = (struct die_args *)data;
499 int ret = NOTIFY_DONE;
500
bibo,mao2326c772006-03-26 01:38:21 -0800501 if (args->regs && user_mode(args->regs))
502 return ret;
503
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 switch (val) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 case DIE_BPT:
506 if (kprobe_handler(args->regs))
507 ret = NOTIFY_STOP;
508 break;
509 case DIE_SSTEP:
510 if (post_kprobe_handler(args->regs))
511 ret = NOTIFY_STOP;
512 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 default:
514 break;
515 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516 return ret;
517}
518
Michael Ellerman3d7e3382007-07-19 01:48:11 -0700519#ifdef CONFIG_PPC64
520unsigned long arch_deref_entry_point(void *entry)
521{
Michael Ellerman80d267f2008-05-23 14:22:21 +1000522 return ((func_descr_t *)entry)->entry;
Michael Ellerman3d7e3382007-07-19 01:48:11 -0700523}
524#endif
525
Prasanna S Panchamukhibb144a82005-09-06 15:19:29 -0700526int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527{
528 struct jprobe *jp = container_of(p, struct jprobe, kp);
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800529 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800531 memcpy(&kcb->jprobe_saved_regs, regs, sizeof(struct pt_regs));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700532
533 /* setup return addr to the jprobe handler routine */
Michael Ellerman3d7e3382007-07-19 01:48:11 -0700534 regs->nip = arch_deref_entry_point(jp->entry);
Kumar Gala82090032007-02-06 22:55:19 -0600535#ifdef CONFIG_PPC64
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 regs->gpr[2] = (unsigned long)(((func_descr_t *)jp->entry)->toc);
Kumar Gala82090032007-02-06 22:55:19 -0600537#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538
539 return 1;
540}
541
Ananth N Mavinakayanahalli66200ea2008-02-15 21:59:17 +1100542void __used __kprobes jprobe_return(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543{
544 asm volatile("trap" ::: "memory");
545}
546
Ananth N Mavinakayanahalli66200ea2008-02-15 21:59:17 +1100547static void __used __kprobes jprobe_return_end(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700548{
549};
550
Prasanna S Panchamukhibb144a82005-09-06 15:19:29 -0700551int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552{
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800553 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
554
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 /*
556 * FIXME - we should ideally be validating that we got here 'cos
557 * of the "trap" in jprobe_return() above, before restoring the
558 * saved regs...
559 */
Ananth N Mavinakayanahalli0dc036c2005-11-07 01:00:10 -0800560 memcpy(regs, &kcb->jprobe_saved_regs, sizeof(struct pt_regs));
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800561 preempt_enable_no_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562 return 1;
563}
Rusty Lynch97f79432005-06-27 15:17:15 -0700564
565static struct kprobe trampoline_p = {
566 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
567 .pre_handler = trampoline_probe_handler
568};
569
Rusty Lynch67729262005-07-05 18:54:50 -0700570int __init arch_init_kprobes(void)
Rusty Lynch97f79432005-06-27 15:17:15 -0700571{
572 return register_kprobe(&trampoline_p);
573}
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -0700574
575int __kprobes arch_trampoline_kprobe(struct kprobe *p)
576{
577 if (p->addr == (kprobe_opcode_t *)&kretprobe_trampoline)
578 return 1;
579
580 return 0;
581}