blob: 32b0c24ab9a6dcca721414a7027635aa06d2c828 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Kernel Probes (KProbes)
3 * arch/i386/kernel/kprobes.c
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * Copyright (C) IBM Corporation, 2002, 2004
20 *
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation ( includes contributions from
23 * Rusty Russell).
24 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
25 * interface to access function arguments.
Hien Nguyenb94cce92005-06-23 00:09:19 -070026 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
27 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
28 * <prasanna@in.ibm.com> added function-return probes.
Linus Torvalds1da177e2005-04-16 15:20:36 -070029 */
30
31#include <linux/config.h>
32#include <linux/kprobes.h>
33#include <linux/ptrace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034#include <linux/preempt.h>
Rusty Lynch7e1048b2005-06-23 00:09:25 -070035#include <asm/cacheflush.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <asm/kdebug.h>
37#include <asm/desc.h>
38
Linus Torvalds1da177e2005-04-16 15:20:36 -070039void jprobe_return_end(void);
40
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -080041DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
42DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
43
Linus Torvalds1da177e2005-04-16 15:20:36 -070044/*
45 * returns non-zero if opcode modifies the interrupt flag.
46 */
47static inline int is_IF_modifier(kprobe_opcode_t opcode)
48{
49 switch (opcode) {
50 case 0xfa: /* cli */
51 case 0xfb: /* sti */
52 case 0xcf: /* iret/iretd */
53 case 0x9d: /* popf/popfd */
54 return 1;
55 }
56 return 0;
57}
58
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -070059int __kprobes arch_prepare_kprobe(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -070060{
61 return 0;
62}
63
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -070064void __kprobes arch_copy_kprobe(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -070065{
66 memcpy(p->ainsn.insn, p->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
Rusty Lynch7e1048b2005-06-23 00:09:25 -070067 p->opcode = *p->addr;
68}
69
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -070070void __kprobes arch_arm_kprobe(struct kprobe *p)
Rusty Lynch7e1048b2005-06-23 00:09:25 -070071{
72 *p->addr = BREAKPOINT_INSTRUCTION;
73 flush_icache_range((unsigned long) p->addr,
74 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
75}
76
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -070077void __kprobes arch_disarm_kprobe(struct kprobe *p)
Rusty Lynch7e1048b2005-06-23 00:09:25 -070078{
79 *p->addr = p->opcode;
80 flush_icache_range((unsigned long) p->addr,
81 (unsigned long) p->addr + sizeof(kprobe_opcode_t));
Linus Torvalds1da177e2005-04-16 15:20:36 -070082}
83
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -070084void __kprobes arch_remove_kprobe(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -070085{
86}
87
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -080088static inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -070089{
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -080090 kcb->prev_kprobe.kp = kprobe_running();
91 kcb->prev_kprobe.status = kcb->kprobe_status;
92 kcb->prev_kprobe.old_eflags = kcb->kprobe_old_eflags;
93 kcb->prev_kprobe.saved_eflags = kcb->kprobe_saved_eflags;
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -070094}
95
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -080096static inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb)
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -070097{
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -080098 __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
99 kcb->kprobe_status = kcb->prev_kprobe.status;
100 kcb->kprobe_old_eflags = kcb->prev_kprobe.old_eflags;
101 kcb->kprobe_saved_eflags = kcb->prev_kprobe.saved_eflags;
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700102}
103
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800104static inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
105 struct kprobe_ctlblk *kcb)
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700106{
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800107 __get_cpu_var(current_kprobe) = p;
108 kcb->kprobe_saved_eflags = kcb->kprobe_old_eflags
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700109 = (regs->eflags & (TF_MASK | IF_MASK));
110 if (is_IF_modifier(p->opcode))
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800111 kcb->kprobe_saved_eflags &= ~IF_MASK;
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700112}
113
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114static inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
115{
116 regs->eflags |= TF_MASK;
117 regs->eflags &= ~IF_MASK;
118 /*single step inline if the instruction is an int3*/
119 if (p->opcode == BREAKPOINT_INSTRUCTION)
120 regs->eip = (unsigned long)p->addr;
121 else
122 regs->eip = (unsigned long)&p->ainsn.insn;
123}
124
Ananth N Mavinakayanahalli991a51d2005-11-07 01:00:14 -0800125/* Called with kretprobe_lock held */
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -0700126void __kprobes arch_prepare_kretprobe(struct kretprobe *rp,
127 struct pt_regs *regs)
Hien Nguyenb94cce92005-06-23 00:09:19 -0700128{
129 unsigned long *sara = (unsigned long *)&regs->esp;
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700130 struct kretprobe_instance *ri;
Hien Nguyenb94cce92005-06-23 00:09:19 -0700131
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700132 if ((ri = get_free_rp_inst(rp)) != NULL) {
133 ri->rp = rp;
134 ri->task = current;
135 ri->ret_addr = (kprobe_opcode_t *) *sara;
Hien Nguyenb94cce92005-06-23 00:09:19 -0700136
Hien Nguyenb94cce92005-06-23 00:09:19 -0700137 /* Replace the return addr with trampoline addr */
138 *sara = (unsigned long) &kretprobe_trampoline;
Hien Nguyenb94cce92005-06-23 00:09:19 -0700139
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700140 add_rp_inst(ri);
141 } else {
142 rp->nmissed++;
143 }
Hien Nguyenb94cce92005-06-23 00:09:19 -0700144}
145
Linus Torvalds1da177e2005-04-16 15:20:36 -0700146/*
147 * Interrupts are disabled on entry as trap3 is an interrupt gate and they
148 * remain disabled thorough out this function.
149 */
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -0700150static int __kprobes kprobe_handler(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151{
152 struct kprobe *p;
153 int ret = 0;
154 kprobe_opcode_t *addr = NULL;
155 unsigned long *lp;
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800156 struct kprobe_ctlblk *kcb;
157
158 /*
159 * We don't want to be preempted for the entire
160 * duration of kprobe processing
161 */
162 preempt_disable();
163 kcb = get_kprobe_ctlblk();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700164
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 /* Check if the application is using LDT entry for its code segment and
166 * calculate the address by reading the base address from the LDT entry.
167 */
168 if ((regs->xcs & 4) && (current->mm)) {
169 lp = (unsigned long *) ((unsigned long)((regs->xcs >> 3) * 8)
170 + (char *) current->mm->context.ldt);
171 addr = (kprobe_opcode_t *) (get_desc_base(lp) + regs->eip -
172 sizeof(kprobe_opcode_t));
173 } else {
174 addr = (kprobe_opcode_t *)(regs->eip - sizeof(kprobe_opcode_t));
175 }
176 /* Check we're not actually recursing */
177 if (kprobe_running()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178 p = get_kprobe(addr);
179 if (p) {
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800180 if (kcb->kprobe_status == KPROBE_HIT_SS &&
Keshavamurthy Anil Sdeac66a2005-09-06 15:19:35 -0700181 *p->ainsn.insn == BREAKPOINT_INSTRUCTION) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 regs->eflags &= ~TF_MASK;
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800183 regs->eflags |= kcb->kprobe_saved_eflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184 goto no_kprobe;
185 }
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700186 /* We have reentered the kprobe_handler(), since
187 * another probe was hit while within the handler.
188 * We here save the original kprobes variables and
189 * just single step on the instruction of the new probe
190 * without calling any user handlers.
191 */
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800192 save_previous_kprobe(kcb);
193 set_current_kprobe(p, regs, kcb);
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700194 p->nmissed++;
195 prepare_singlestep(p, regs);
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800196 kcb->kprobe_status = KPROBE_REENTER;
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700197 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 } else {
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800199 p = __get_cpu_var(current_kprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 if (p->break_handler && p->break_handler(p, regs)) {
201 goto ss_probe;
202 }
203 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 goto no_kprobe;
205 }
206
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 p = get_kprobe(addr);
208 if (!p) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700209 if (regs->eflags & VM_MASK) {
210 /* We are in virtual-8086 mode. Return 0 */
211 goto no_kprobe;
212 }
213
214 if (*addr != BREAKPOINT_INSTRUCTION) {
215 /*
216 * The breakpoint instruction was removed right
217 * after we hit it. Another cpu has removed
218 * either a probepoint or a debugger breakpoint
219 * at this address. In either case, no further
220 * handling of this interrupt is appropriate.
Jim Kenistonbce06492005-09-06 15:19:34 -0700221 * Back up over the (now missing) int3 and run
222 * the original instruction.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223 */
Jim Kenistonbce06492005-09-06 15:19:34 -0700224 regs->eip -= sizeof(kprobe_opcode_t);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700225 ret = 1;
226 }
227 /* Not one of ours: let kernel handle it */
228 goto no_kprobe;
229 }
230
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800231 set_current_kprobe(p, regs, kcb);
232 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
234 if (p->pre_handler && p->pre_handler(p, regs))
235 /* handler has already set things up, so skip ss setup */
236 return 1;
237
238ss_probe:
239 prepare_singlestep(p, regs);
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800240 kcb->kprobe_status = KPROBE_HIT_SS;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241 return 1;
242
243no_kprobe:
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800244 preempt_enable_no_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245 return ret;
246}
247
248/*
Hien Nguyenb94cce92005-06-23 00:09:19 -0700249 * For function-return probes, init_kprobes() establishes a probepoint
250 * here. When a retprobed function returns, this probe is hit and
251 * trampoline_probe_handler() runs, calling the kretprobe's handler.
252 */
253 void kretprobe_trampoline_holder(void)
254 {
255 asm volatile ( ".global kretprobe_trampoline\n"
256 "kretprobe_trampoline: \n"
257 "nop\n");
258 }
259
260/*
261 * Called when we hit the probe point at kretprobe_trampoline
262 */
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -0700263int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
Hien Nguyenb94cce92005-06-23 00:09:19 -0700264{
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700265 struct kretprobe_instance *ri = NULL;
266 struct hlist_head *head;
267 struct hlist_node *node, *tmp;
Ananth N Mavinakayanahalli991a51d2005-11-07 01:00:14 -0800268 unsigned long flags, orig_ret_address = 0;
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700269 unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline;
Hien Nguyenb94cce92005-06-23 00:09:19 -0700270
Ananth N Mavinakayanahalli991a51d2005-11-07 01:00:14 -0800271 spin_lock_irqsave(&kretprobe_lock, flags);
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700272 head = kretprobe_inst_table_head(current);
Hien Nguyenb94cce92005-06-23 00:09:19 -0700273
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700274 /*
275 * It is possible to have multiple instances associated with a given
276 * task either because an multiple functions in the call path
277 * have a return probe installed on them, and/or more then one return
278 * return probe was registered for a target function.
279 *
280 * We can handle this because:
281 * - instances are always inserted at the head of the list
282 * - when multiple return probes are registered for the same
283 * function, the first instance's ret_addr will point to the
284 * real return address, and all the rest will point to
285 * kretprobe_trampoline
286 */
287 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
288 if (ri->task != current)
289 /* another task is sharing our hash bucket */
290 continue;
Hien Nguyenb94cce92005-06-23 00:09:19 -0700291
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700292 if (ri->rp && ri->rp->handler)
293 ri->rp->handler(ri, regs);
Hien Nguyenb94cce92005-06-23 00:09:19 -0700294
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700295 orig_ret_address = (unsigned long)ri->ret_addr;
Hien Nguyenb94cce92005-06-23 00:09:19 -0700296 recycle_rp_inst(ri);
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700297
298 if (orig_ret_address != trampoline_address)
299 /*
300 * This is the real return address. Any other
301 * instances associated with this task are for
302 * other calls deeper on the call stack
303 */
304 break;
Hien Nguyenb94cce92005-06-23 00:09:19 -0700305 }
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700306
307 BUG_ON(!orig_ret_address || (orig_ret_address == trampoline_address));
308 regs->eip = orig_ret_address;
309
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800310 reset_current_kprobe();
Ananth N Mavinakayanahalli991a51d2005-11-07 01:00:14 -0800311 spin_unlock_irqrestore(&kretprobe_lock, flags);
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700312 preempt_enable_no_resched();
313
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800314 /*
315 * By returning a non-zero value, we are telling
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800316 * kprobe_handler() that we don't want the post_handler
317 * to run (and have re-enabled preemption)
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800318 */
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700319 return 1;
Hien Nguyenb94cce92005-06-23 00:09:19 -0700320}
321
322/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700323 * Called after single-stepping. p->addr is the address of the
324 * instruction whose first byte has been replaced by the "int 3"
325 * instruction. To avoid the SMP problems that can occur when we
326 * temporarily put back the original opcode to single-step, we
327 * single-stepped a copy of the instruction. The address of this
328 * copy is p->ainsn.insn.
329 *
330 * This function prepares to return from the post-single-step
331 * interrupt. We have to fix up the stack as follows:
332 *
333 * 0) Except in the case of absolute or indirect jump or call instructions,
334 * the new eip is relative to the copied instruction. We need to make
335 * it relative to the original instruction.
336 *
337 * 1) If the single-stepped instruction was pushfl, then the TF and IF
338 * flags are set in the just-pushed eflags, and may need to be cleared.
339 *
340 * 2) If the single-stepped instruction was a call, the return address
341 * that is atop the stack is the address following the copied instruction.
342 * We need to make it the address following the original instruction.
343 */
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800344static void __kprobes resume_execution(struct kprobe *p,
345 struct pt_regs *regs, struct kprobe_ctlblk *kcb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346{
347 unsigned long *tos = (unsigned long *)&regs->esp;
348 unsigned long next_eip = 0;
349 unsigned long copy_eip = (unsigned long)&p->ainsn.insn;
350 unsigned long orig_eip = (unsigned long)p->addr;
351
352 switch (p->ainsn.insn[0]) {
353 case 0x9c: /* pushfl */
354 *tos &= ~(TF_MASK | IF_MASK);
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800355 *tos |= kcb->kprobe_old_eflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356 break;
Prasanna S Panchamukhi0b9e2ca2005-05-05 16:15:40 -0700357 case 0xc3: /* ret/lret */
358 case 0xcb:
359 case 0xc2:
360 case 0xca:
361 regs->eflags &= ~TF_MASK;
362 /* eip is already adjusted, no more changes required*/
363 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 case 0xe8: /* call relative - Fix return addr */
365 *tos = orig_eip + (*tos - copy_eip);
366 break;
367 case 0xff:
368 if ((p->ainsn.insn[1] & 0x30) == 0x10) {
369 /* call absolute, indirect */
370 /* Fix return addr; eip is correct. */
371 next_eip = regs->eip;
372 *tos = orig_eip + (*tos - copy_eip);
373 } else if (((p->ainsn.insn[1] & 0x31) == 0x20) || /* jmp near, absolute indirect */
374 ((p->ainsn.insn[1] & 0x31) == 0x21)) { /* jmp far, absolute indirect */
375 /* eip is correct. */
376 next_eip = regs->eip;
377 }
378 break;
379 case 0xea: /* jmp absolute -- eip is correct */
380 next_eip = regs->eip;
381 break;
382 default:
383 break;
384 }
385
386 regs->eflags &= ~TF_MASK;
387 if (next_eip) {
388 regs->eip = next_eip;
389 } else {
390 regs->eip = orig_eip + (regs->eip - copy_eip);
391 }
392}
393
394/*
395 * Interrupts are disabled on entry as trap1 is an interrupt gate and they
Ananth N Mavinakayanahalli991a51d2005-11-07 01:00:14 -0800396 * remain disabled thoroughout this function.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700397 */
398static inline int post_kprobe_handler(struct pt_regs *regs)
399{
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800400 struct kprobe *cur = kprobe_running();
401 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
402
403 if (!cur)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 return 0;
405
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800406 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
407 kcb->kprobe_status = KPROBE_HIT_SSDONE;
408 cur->post_handler(cur, regs, 0);
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700409 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800411 resume_execution(cur, regs, kcb);
412 regs->eflags |= kcb->kprobe_saved_eflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700414 /*Restore back the original saved kprobes variables and continue. */
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800415 if (kcb->kprobe_status == KPROBE_REENTER) {
416 restore_previous_kprobe(kcb);
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700417 goto out;
418 }
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800419 reset_current_kprobe();
Prasanna S Panchamukhi417c8da2005-06-23 00:09:37 -0700420out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421 preempt_enable_no_resched();
422
423 /*
424 * if somebody else is singlestepping across a probe point, eflags
425 * will have TF set, in which case, continue the remaining processing
426 * of do_debug, as if this is not a probe hit.
427 */
428 if (regs->eflags & TF_MASK)
429 return 0;
430
431 return 1;
432}
433
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434static inline int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
435{
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800436 struct kprobe *cur = kprobe_running();
437 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
438
439 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700440 return 1;
441
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800442 if (kcb->kprobe_status & KPROBE_HIT_SS) {
443 resume_execution(cur, regs, kcb);
444 regs->eflags |= kcb->kprobe_old_eflags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700445
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800446 reset_current_kprobe();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 preempt_enable_no_resched();
448 }
449 return 0;
450}
451
452/*
453 * Wrapper routine to for handling exceptions.
454 */
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -0700455int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
456 unsigned long val, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700457{
458 struct die_args *args = (struct die_args *)data;
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800459 int ret = NOTIFY_DONE;
460
Linus Torvalds1da177e2005-04-16 15:20:36 -0700461 switch (val) {
462 case DIE_INT3:
463 if (kprobe_handler(args->regs))
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800464 ret = NOTIFY_STOP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 break;
466 case DIE_DEBUG:
467 if (post_kprobe_handler(args->regs))
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800468 ret = NOTIFY_STOP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469 break;
470 case DIE_GPF:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471 case DIE_PAGE_FAULT:
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800472 /* kprobe_running() needs smp_processor_id() */
473 preempt_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474 if (kprobe_running() &&
475 kprobe_fault_handler(args->regs, args->trapnr))
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800476 ret = NOTIFY_STOP;
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800477 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478 break;
479 default:
480 break;
481 }
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800482 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700483}
484
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -0700485int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486{
487 struct jprobe *jp = container_of(p, struct jprobe, kp);
488 unsigned long addr;
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800489 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800491 kcb->jprobe_saved_regs = *regs;
492 kcb->jprobe_saved_esp = &regs->esp;
493 addr = (unsigned long)(kcb->jprobe_saved_esp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700494
495 /*
496 * TBD: As Linus pointed out, gcc assumes that the callee
497 * owns the argument space and could overwrite it, e.g.
498 * tailcall optimization. So, to be absolutely safe
499 * we also save and restore enough stack bytes to cover
500 * the argument area.
501 */
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800502 memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
503 MIN_STACK_SIZE(addr));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700504 regs->eflags &= ~IF_MASK;
505 regs->eip = (unsigned long)(jp->entry);
506 return 1;
507}
508
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -0700509void __kprobes jprobe_return(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510{
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800511 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
512
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 asm volatile (" xchgl %%ebx,%%esp \n"
514 " int3 \n"
515 " .globl jprobe_return_end \n"
516 " jprobe_return_end: \n"
517 " nop \n"::"b"
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800518 (kcb->jprobe_saved_esp):"memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700519}
520
Prasanna S Panchamukhi3d97ae52005-09-06 15:19:27 -0700521int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700522{
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800523 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524 u8 *addr = (u8 *) (regs->eip - 1);
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800525 unsigned long stack_addr = (unsigned long)(kcb->jprobe_saved_esp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 struct jprobe *jp = container_of(p, struct jprobe, kp);
527
528 if ((addr > (u8 *) jprobe_return) && (addr < (u8 *) jprobe_return_end)) {
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800529 if (&regs->esp != kcb->jprobe_saved_esp) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530 struct pt_regs *saved_regs =
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800531 container_of(kcb->jprobe_saved_esp,
532 struct pt_regs, esp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533 printk("current esp %p does not match saved esp %p\n",
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800534 &regs->esp, kcb->jprobe_saved_esp);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 printk("Saved registers for jprobe %p\n", jp);
536 show_registers(saved_regs);
537 printk("Current registers\n");
538 show_registers(regs);
539 BUG();
540 }
Ananth N Mavinakayanahalli9a0e3a82005-11-07 01:00:08 -0800541 *regs = kcb->jprobe_saved_regs;
542 memcpy((kprobe_opcode_t *) stack_addr, kcb->jprobes_stack,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 MIN_STACK_SIZE(stack_addr));
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800544 preempt_enable_no_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 return 1;
546 }
547 return 0;
548}
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700549
550static struct kprobe trampoline_p = {
551 .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
552 .pre_handler = trampoline_probe_handler
553};
554
Rusty Lynch67729262005-07-05 18:54:50 -0700555int __init arch_init_kprobes(void)
Rusty Lynch4bdbd372005-06-27 15:17:09 -0700556{
557 return register_kprobe(&trampoline_p);
558}