Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Kernel Probes (KProbes) |
| 3 | * arch/i386/kernel/kprobes.c |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify |
| 6 | * it under the terms of the GNU General Public License as published by |
| 7 | * the Free Software Foundation; either version 2 of the License, or |
| 8 | * (at your option) any later version. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, |
| 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 | * GNU General Public License for more details. |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License |
| 16 | * along with this program; if not, write to the Free Software |
| 17 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
| 18 | * |
| 19 | * Copyright (C) IBM Corporation, 2002, 2004 |
| 20 | * |
| 21 | * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel |
| 22 | * Probes initial implementation ( includes contributions from |
| 23 | * Rusty Russell). |
| 24 | * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes |
| 25 | * interface to access function arguments. |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 26 | * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston |
| 27 | * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi |
| 28 | * <prasanna@in.ibm.com> added function-return probes. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | */ |
| 30 | |
| 31 | #include <linux/config.h> |
| 32 | #include <linux/kprobes.h> |
| 33 | #include <linux/ptrace.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | #include <linux/preempt.h> |
Rusty Lynch | 7e1048b | 2005-06-23 00:09:25 -0700 | [diff] [blame] | 35 | #include <asm/cacheflush.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 36 | #include <asm/kdebug.h> |
| 37 | #include <asm/desc.h> |
| 38 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | void jprobe_return_end(void); |
| 40 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 41 | DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; |
| 42 | DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); |
| 43 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 44 | /* |
| 45 | * returns non-zero if opcode modifies the interrupt flag. |
| 46 | */ |
| 47 | static inline int is_IF_modifier(kprobe_opcode_t opcode) |
| 48 | { |
| 49 | switch (opcode) { |
| 50 | case 0xfa: /* cli */ |
| 51 | case 0xfb: /* sti */ |
| 52 | case 0xcf: /* iret/iretd */ |
| 53 | case 0x9d: /* popf/popfd */ |
| 54 | return 1; |
| 55 | } |
| 56 | return 0; |
| 57 | } |
| 58 | |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 59 | int __kprobes arch_prepare_kprobe(struct kprobe *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | { |
| 61 | return 0; |
| 62 | } |
| 63 | |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 64 | void __kprobes arch_copy_kprobe(struct kprobe *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 65 | { |
| 66 | memcpy(p->ainsn.insn, p->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t)); |
Rusty Lynch | 7e1048b | 2005-06-23 00:09:25 -0700 | [diff] [blame] | 67 | p->opcode = *p->addr; |
| 68 | } |
| 69 | |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 70 | void __kprobes arch_arm_kprobe(struct kprobe *p) |
Rusty Lynch | 7e1048b | 2005-06-23 00:09:25 -0700 | [diff] [blame] | 71 | { |
| 72 | *p->addr = BREAKPOINT_INSTRUCTION; |
| 73 | flush_icache_range((unsigned long) p->addr, |
| 74 | (unsigned long) p->addr + sizeof(kprobe_opcode_t)); |
| 75 | } |
| 76 | |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 77 | void __kprobes arch_disarm_kprobe(struct kprobe *p) |
Rusty Lynch | 7e1048b | 2005-06-23 00:09:25 -0700 | [diff] [blame] | 78 | { |
| 79 | *p->addr = p->opcode; |
| 80 | flush_icache_range((unsigned long) p->addr, |
| 81 | (unsigned long) p->addr + sizeof(kprobe_opcode_t)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 82 | } |
| 83 | |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 84 | void __kprobes arch_remove_kprobe(struct kprobe *p) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 85 | { |
| 86 | } |
| 87 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 88 | static inline void save_previous_kprobe(struct kprobe_ctlblk *kcb) |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 89 | { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 90 | kcb->prev_kprobe.kp = kprobe_running(); |
| 91 | kcb->prev_kprobe.status = kcb->kprobe_status; |
| 92 | kcb->prev_kprobe.old_eflags = kcb->kprobe_old_eflags; |
| 93 | kcb->prev_kprobe.saved_eflags = kcb->kprobe_saved_eflags; |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 94 | } |
| 95 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 96 | static inline void restore_previous_kprobe(struct kprobe_ctlblk *kcb) |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 97 | { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 98 | __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; |
| 99 | kcb->kprobe_status = kcb->prev_kprobe.status; |
| 100 | kcb->kprobe_old_eflags = kcb->prev_kprobe.old_eflags; |
| 101 | kcb->kprobe_saved_eflags = kcb->prev_kprobe.saved_eflags; |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 102 | } |
| 103 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 104 | static inline void set_current_kprobe(struct kprobe *p, struct pt_regs *regs, |
| 105 | struct kprobe_ctlblk *kcb) |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 106 | { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 107 | __get_cpu_var(current_kprobe) = p; |
| 108 | kcb->kprobe_saved_eflags = kcb->kprobe_old_eflags |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 109 | = (regs->eflags & (TF_MASK | IF_MASK)); |
| 110 | if (is_IF_modifier(p->opcode)) |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 111 | kcb->kprobe_saved_eflags &= ~IF_MASK; |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 112 | } |
| 113 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 114 | static inline void prepare_singlestep(struct kprobe *p, struct pt_regs *regs) |
| 115 | { |
| 116 | regs->eflags |= TF_MASK; |
| 117 | regs->eflags &= ~IF_MASK; |
| 118 | /*single step inline if the instruction is an int3*/ |
| 119 | if (p->opcode == BREAKPOINT_INSTRUCTION) |
| 120 | regs->eip = (unsigned long)p->addr; |
| 121 | else |
| 122 | regs->eip = (unsigned long)&p->ainsn.insn; |
| 123 | } |
| 124 | |
Ananth N Mavinakayanahalli | 991a51d | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 125 | /* Called with kretprobe_lock held */ |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 126 | void __kprobes arch_prepare_kretprobe(struct kretprobe *rp, |
| 127 | struct pt_regs *regs) |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 128 | { |
| 129 | unsigned long *sara = (unsigned long *)®s->esp; |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 130 | struct kretprobe_instance *ri; |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 131 | |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 132 | if ((ri = get_free_rp_inst(rp)) != NULL) { |
| 133 | ri->rp = rp; |
| 134 | ri->task = current; |
| 135 | ri->ret_addr = (kprobe_opcode_t *) *sara; |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 136 | |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 137 | /* Replace the return addr with trampoline addr */ |
| 138 | *sara = (unsigned long) &kretprobe_trampoline; |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 139 | |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 140 | add_rp_inst(ri); |
| 141 | } else { |
| 142 | rp->nmissed++; |
| 143 | } |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 144 | } |
| 145 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 146 | /* |
| 147 | * Interrupts are disabled on entry as trap3 is an interrupt gate and they |
| 148 | * remain disabled thorough out this function. |
| 149 | */ |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 150 | static int __kprobes kprobe_handler(struct pt_regs *regs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 151 | { |
| 152 | struct kprobe *p; |
| 153 | int ret = 0; |
| 154 | kprobe_opcode_t *addr = NULL; |
| 155 | unsigned long *lp; |
Ananth N Mavinakayanahalli | d217d54 | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 156 | struct kprobe_ctlblk *kcb; |
| 157 | |
| 158 | /* |
| 159 | * We don't want to be preempted for the entire |
| 160 | * duration of kprobe processing |
| 161 | */ |
| 162 | preempt_disable(); |
| 163 | kcb = get_kprobe_ctlblk(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 164 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 165 | /* Check if the application is using LDT entry for its code segment and |
| 166 | * calculate the address by reading the base address from the LDT entry. |
| 167 | */ |
| 168 | if ((regs->xcs & 4) && (current->mm)) { |
| 169 | lp = (unsigned long *) ((unsigned long)((regs->xcs >> 3) * 8) |
| 170 | + (char *) current->mm->context.ldt); |
| 171 | addr = (kprobe_opcode_t *) (get_desc_base(lp) + regs->eip - |
| 172 | sizeof(kprobe_opcode_t)); |
| 173 | } else { |
| 174 | addr = (kprobe_opcode_t *)(regs->eip - sizeof(kprobe_opcode_t)); |
| 175 | } |
| 176 | /* Check we're not actually recursing */ |
| 177 | if (kprobe_running()) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 178 | p = get_kprobe(addr); |
| 179 | if (p) { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 180 | if (kcb->kprobe_status == KPROBE_HIT_SS && |
Keshavamurthy Anil S | deac66a | 2005-09-06 15:19:35 -0700 | [diff] [blame] | 181 | *p->ainsn.insn == BREAKPOINT_INSTRUCTION) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 182 | regs->eflags &= ~TF_MASK; |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 183 | regs->eflags |= kcb->kprobe_saved_eflags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 184 | goto no_kprobe; |
| 185 | } |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 186 | /* We have reentered the kprobe_handler(), since |
| 187 | * another probe was hit while within the handler. |
| 188 | * We here save the original kprobes variables and |
| 189 | * just single step on the instruction of the new probe |
| 190 | * without calling any user handlers. |
| 191 | */ |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 192 | save_previous_kprobe(kcb); |
| 193 | set_current_kprobe(p, regs, kcb); |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 194 | p->nmissed++; |
| 195 | prepare_singlestep(p, regs); |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 196 | kcb->kprobe_status = KPROBE_REENTER; |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 197 | return 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 198 | } else { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 199 | p = __get_cpu_var(current_kprobe); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 200 | if (p->break_handler && p->break_handler(p, regs)) { |
| 201 | goto ss_probe; |
| 202 | } |
| 203 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 204 | goto no_kprobe; |
| 205 | } |
| 206 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 207 | p = get_kprobe(addr); |
| 208 | if (!p) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 209 | if (regs->eflags & VM_MASK) { |
| 210 | /* We are in virtual-8086 mode. Return 0 */ |
| 211 | goto no_kprobe; |
| 212 | } |
| 213 | |
| 214 | if (*addr != BREAKPOINT_INSTRUCTION) { |
| 215 | /* |
| 216 | * The breakpoint instruction was removed right |
| 217 | * after we hit it. Another cpu has removed |
| 218 | * either a probepoint or a debugger breakpoint |
| 219 | * at this address. In either case, no further |
| 220 | * handling of this interrupt is appropriate. |
Jim Keniston | bce0649 | 2005-09-06 15:19:34 -0700 | [diff] [blame] | 221 | * Back up over the (now missing) int3 and run |
| 222 | * the original instruction. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 223 | */ |
Jim Keniston | bce0649 | 2005-09-06 15:19:34 -0700 | [diff] [blame] | 224 | regs->eip -= sizeof(kprobe_opcode_t); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 225 | ret = 1; |
| 226 | } |
| 227 | /* Not one of ours: let kernel handle it */ |
| 228 | goto no_kprobe; |
| 229 | } |
| 230 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 231 | set_current_kprobe(p, regs, kcb); |
| 232 | kcb->kprobe_status = KPROBE_HIT_ACTIVE; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 233 | |
| 234 | if (p->pre_handler && p->pre_handler(p, regs)) |
| 235 | /* handler has already set things up, so skip ss setup */ |
| 236 | return 1; |
| 237 | |
| 238 | ss_probe: |
| 239 | prepare_singlestep(p, regs); |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 240 | kcb->kprobe_status = KPROBE_HIT_SS; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 241 | return 1; |
| 242 | |
| 243 | no_kprobe: |
Ananth N Mavinakayanahalli | d217d54 | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 244 | preempt_enable_no_resched(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 245 | return ret; |
| 246 | } |
| 247 | |
| 248 | /* |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 249 | * For function-return probes, init_kprobes() establishes a probepoint |
| 250 | * here. When a retprobed function returns, this probe is hit and |
| 251 | * trampoline_probe_handler() runs, calling the kretprobe's handler. |
| 252 | */ |
| 253 | void kretprobe_trampoline_holder(void) |
| 254 | { |
| 255 | asm volatile ( ".global kretprobe_trampoline\n" |
| 256 | "kretprobe_trampoline: \n" |
| 257 | "nop\n"); |
| 258 | } |
| 259 | |
| 260 | /* |
| 261 | * Called when we hit the probe point at kretprobe_trampoline |
| 262 | */ |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 263 | int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 264 | { |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 265 | struct kretprobe_instance *ri = NULL; |
| 266 | struct hlist_head *head; |
| 267 | struct hlist_node *node, *tmp; |
Ananth N Mavinakayanahalli | 991a51d | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 268 | unsigned long flags, orig_ret_address = 0; |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 269 | unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline; |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 270 | |
Ananth N Mavinakayanahalli | 991a51d | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 271 | spin_lock_irqsave(&kretprobe_lock, flags); |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 272 | head = kretprobe_inst_table_head(current); |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 273 | |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 274 | /* |
| 275 | * It is possible to have multiple instances associated with a given |
| 276 | * task either because an multiple functions in the call path |
| 277 | * have a return probe installed on them, and/or more then one return |
| 278 | * return probe was registered for a target function. |
| 279 | * |
| 280 | * We can handle this because: |
| 281 | * - instances are always inserted at the head of the list |
| 282 | * - when multiple return probes are registered for the same |
| 283 | * function, the first instance's ret_addr will point to the |
| 284 | * real return address, and all the rest will point to |
| 285 | * kretprobe_trampoline |
| 286 | */ |
| 287 | hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { |
| 288 | if (ri->task != current) |
| 289 | /* another task is sharing our hash bucket */ |
| 290 | continue; |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 291 | |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 292 | if (ri->rp && ri->rp->handler) |
| 293 | ri->rp->handler(ri, regs); |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 294 | |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 295 | orig_ret_address = (unsigned long)ri->ret_addr; |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 296 | recycle_rp_inst(ri); |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 297 | |
| 298 | if (orig_ret_address != trampoline_address) |
| 299 | /* |
| 300 | * This is the real return address. Any other |
| 301 | * instances associated with this task are for |
| 302 | * other calls deeper on the call stack |
| 303 | */ |
| 304 | break; |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 305 | } |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 306 | |
| 307 | BUG_ON(!orig_ret_address || (orig_ret_address == trampoline_address)); |
| 308 | regs->eip = orig_ret_address; |
| 309 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 310 | reset_current_kprobe(); |
Ananth N Mavinakayanahalli | 991a51d | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 311 | spin_unlock_irqrestore(&kretprobe_lock, flags); |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 312 | preempt_enable_no_resched(); |
| 313 | |
Ananth N Mavinakayanahalli | 66ff2d0 | 2005-11-07 01:00:07 -0800 | [diff] [blame] | 314 | /* |
| 315 | * By returning a non-zero value, we are telling |
Ananth N Mavinakayanahalli | d217d54 | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 316 | * kprobe_handler() that we don't want the post_handler |
| 317 | * to run (and have re-enabled preemption) |
Ananth N Mavinakayanahalli | 66ff2d0 | 2005-11-07 01:00:07 -0800 | [diff] [blame] | 318 | */ |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 319 | return 1; |
Hien Nguyen | b94cce9 | 2005-06-23 00:09:19 -0700 | [diff] [blame] | 320 | } |
| 321 | |
| 322 | /* |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 323 | * Called after single-stepping. p->addr is the address of the |
| 324 | * instruction whose first byte has been replaced by the "int 3" |
| 325 | * instruction. To avoid the SMP problems that can occur when we |
| 326 | * temporarily put back the original opcode to single-step, we |
| 327 | * single-stepped a copy of the instruction. The address of this |
| 328 | * copy is p->ainsn.insn. |
| 329 | * |
| 330 | * This function prepares to return from the post-single-step |
| 331 | * interrupt. We have to fix up the stack as follows: |
| 332 | * |
| 333 | * 0) Except in the case of absolute or indirect jump or call instructions, |
| 334 | * the new eip is relative to the copied instruction. We need to make |
| 335 | * it relative to the original instruction. |
| 336 | * |
| 337 | * 1) If the single-stepped instruction was pushfl, then the TF and IF |
| 338 | * flags are set in the just-pushed eflags, and may need to be cleared. |
| 339 | * |
| 340 | * 2) If the single-stepped instruction was a call, the return address |
| 341 | * that is atop the stack is the address following the copied instruction. |
| 342 | * We need to make it the address following the original instruction. |
| 343 | */ |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 344 | static void __kprobes resume_execution(struct kprobe *p, |
| 345 | struct pt_regs *regs, struct kprobe_ctlblk *kcb) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 346 | { |
| 347 | unsigned long *tos = (unsigned long *)®s->esp; |
| 348 | unsigned long next_eip = 0; |
| 349 | unsigned long copy_eip = (unsigned long)&p->ainsn.insn; |
| 350 | unsigned long orig_eip = (unsigned long)p->addr; |
| 351 | |
| 352 | switch (p->ainsn.insn[0]) { |
| 353 | case 0x9c: /* pushfl */ |
| 354 | *tos &= ~(TF_MASK | IF_MASK); |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 355 | *tos |= kcb->kprobe_old_eflags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 356 | break; |
Prasanna S Panchamukhi | 0b9e2ca | 2005-05-05 16:15:40 -0700 | [diff] [blame] | 357 | case 0xc3: /* ret/lret */ |
| 358 | case 0xcb: |
| 359 | case 0xc2: |
| 360 | case 0xca: |
| 361 | regs->eflags &= ~TF_MASK; |
| 362 | /* eip is already adjusted, no more changes required*/ |
| 363 | return; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 364 | case 0xe8: /* call relative - Fix return addr */ |
| 365 | *tos = orig_eip + (*tos - copy_eip); |
| 366 | break; |
| 367 | case 0xff: |
| 368 | if ((p->ainsn.insn[1] & 0x30) == 0x10) { |
| 369 | /* call absolute, indirect */ |
| 370 | /* Fix return addr; eip is correct. */ |
| 371 | next_eip = regs->eip; |
| 372 | *tos = orig_eip + (*tos - copy_eip); |
| 373 | } else if (((p->ainsn.insn[1] & 0x31) == 0x20) || /* jmp near, absolute indirect */ |
| 374 | ((p->ainsn.insn[1] & 0x31) == 0x21)) { /* jmp far, absolute indirect */ |
| 375 | /* eip is correct. */ |
| 376 | next_eip = regs->eip; |
| 377 | } |
| 378 | break; |
| 379 | case 0xea: /* jmp absolute -- eip is correct */ |
| 380 | next_eip = regs->eip; |
| 381 | break; |
| 382 | default: |
| 383 | break; |
| 384 | } |
| 385 | |
| 386 | regs->eflags &= ~TF_MASK; |
| 387 | if (next_eip) { |
| 388 | regs->eip = next_eip; |
| 389 | } else { |
| 390 | regs->eip = orig_eip + (regs->eip - copy_eip); |
| 391 | } |
| 392 | } |
| 393 | |
| 394 | /* |
| 395 | * Interrupts are disabled on entry as trap1 is an interrupt gate and they |
Ananth N Mavinakayanahalli | 991a51d | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 396 | * remain disabled thoroughout this function. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 397 | */ |
| 398 | static inline int post_kprobe_handler(struct pt_regs *regs) |
| 399 | { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 400 | struct kprobe *cur = kprobe_running(); |
| 401 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
| 402 | |
| 403 | if (!cur) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 404 | return 0; |
| 405 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 406 | if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { |
| 407 | kcb->kprobe_status = KPROBE_HIT_SSDONE; |
| 408 | cur->post_handler(cur, regs, 0); |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 409 | } |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 410 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 411 | resume_execution(cur, regs, kcb); |
| 412 | regs->eflags |= kcb->kprobe_saved_eflags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 413 | |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 414 | /*Restore back the original saved kprobes variables and continue. */ |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 415 | if (kcb->kprobe_status == KPROBE_REENTER) { |
| 416 | restore_previous_kprobe(kcb); |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 417 | goto out; |
| 418 | } |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 419 | reset_current_kprobe(); |
Prasanna S Panchamukhi | 417c8da | 2005-06-23 00:09:37 -0700 | [diff] [blame] | 420 | out: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 421 | preempt_enable_no_resched(); |
| 422 | |
| 423 | /* |
| 424 | * if somebody else is singlestepping across a probe point, eflags |
| 425 | * will have TF set, in which case, continue the remaining processing |
| 426 | * of do_debug, as if this is not a probe hit. |
| 427 | */ |
| 428 | if (regs->eflags & TF_MASK) |
| 429 | return 0; |
| 430 | |
| 431 | return 1; |
| 432 | } |
| 433 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 434 | static inline int kprobe_fault_handler(struct pt_regs *regs, int trapnr) |
| 435 | { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 436 | struct kprobe *cur = kprobe_running(); |
| 437 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
| 438 | |
| 439 | if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr)) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 440 | return 1; |
| 441 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 442 | if (kcb->kprobe_status & KPROBE_HIT_SS) { |
| 443 | resume_execution(cur, regs, kcb); |
| 444 | regs->eflags |= kcb->kprobe_old_eflags; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 445 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 446 | reset_current_kprobe(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 447 | preempt_enable_no_resched(); |
| 448 | } |
| 449 | return 0; |
| 450 | } |
| 451 | |
| 452 | /* |
| 453 | * Wrapper routine to for handling exceptions. |
| 454 | */ |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 455 | int __kprobes kprobe_exceptions_notify(struct notifier_block *self, |
| 456 | unsigned long val, void *data) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 457 | { |
| 458 | struct die_args *args = (struct die_args *)data; |
Ananth N Mavinakayanahalli | 66ff2d0 | 2005-11-07 01:00:07 -0800 | [diff] [blame] | 459 | int ret = NOTIFY_DONE; |
| 460 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 461 | switch (val) { |
| 462 | case DIE_INT3: |
| 463 | if (kprobe_handler(args->regs)) |
Ananth N Mavinakayanahalli | 66ff2d0 | 2005-11-07 01:00:07 -0800 | [diff] [blame] | 464 | ret = NOTIFY_STOP; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 465 | break; |
| 466 | case DIE_DEBUG: |
| 467 | if (post_kprobe_handler(args->regs)) |
Ananth N Mavinakayanahalli | 66ff2d0 | 2005-11-07 01:00:07 -0800 | [diff] [blame] | 468 | ret = NOTIFY_STOP; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 469 | break; |
| 470 | case DIE_GPF: |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 471 | case DIE_PAGE_FAULT: |
Ananth N Mavinakayanahalli | d217d54 | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 472 | /* kprobe_running() needs smp_processor_id() */ |
| 473 | preempt_disable(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 474 | if (kprobe_running() && |
| 475 | kprobe_fault_handler(args->regs, args->trapnr)) |
Ananth N Mavinakayanahalli | 66ff2d0 | 2005-11-07 01:00:07 -0800 | [diff] [blame] | 476 | ret = NOTIFY_STOP; |
Ananth N Mavinakayanahalli | d217d54 | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 477 | preempt_enable(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 478 | break; |
| 479 | default: |
| 480 | break; |
| 481 | } |
Ananth N Mavinakayanahalli | 66ff2d0 | 2005-11-07 01:00:07 -0800 | [diff] [blame] | 482 | return ret; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 483 | } |
| 484 | |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 485 | int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 486 | { |
| 487 | struct jprobe *jp = container_of(p, struct jprobe, kp); |
| 488 | unsigned long addr; |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 489 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 490 | |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 491 | kcb->jprobe_saved_regs = *regs; |
| 492 | kcb->jprobe_saved_esp = ®s->esp; |
| 493 | addr = (unsigned long)(kcb->jprobe_saved_esp); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 494 | |
| 495 | /* |
| 496 | * TBD: As Linus pointed out, gcc assumes that the callee |
| 497 | * owns the argument space and could overwrite it, e.g. |
| 498 | * tailcall optimization. So, to be absolutely safe |
| 499 | * we also save and restore enough stack bytes to cover |
| 500 | * the argument area. |
| 501 | */ |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 502 | memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr, |
| 503 | MIN_STACK_SIZE(addr)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 504 | regs->eflags &= ~IF_MASK; |
| 505 | regs->eip = (unsigned long)(jp->entry); |
| 506 | return 1; |
| 507 | } |
| 508 | |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 509 | void __kprobes jprobe_return(void) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 510 | { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 511 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
| 512 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 513 | asm volatile (" xchgl %%ebx,%%esp \n" |
| 514 | " int3 \n" |
| 515 | " .globl jprobe_return_end \n" |
| 516 | " jprobe_return_end: \n" |
| 517 | " nop \n"::"b" |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 518 | (kcb->jprobe_saved_esp):"memory"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 519 | } |
| 520 | |
Prasanna S Panchamukhi | 3d97ae5 | 2005-09-06 15:19:27 -0700 | [diff] [blame] | 521 | int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 522 | { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 523 | struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 524 | u8 *addr = (u8 *) (regs->eip - 1); |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 525 | unsigned long stack_addr = (unsigned long)(kcb->jprobe_saved_esp); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 526 | struct jprobe *jp = container_of(p, struct jprobe, kp); |
| 527 | |
| 528 | if ((addr > (u8 *) jprobe_return) && (addr < (u8 *) jprobe_return_end)) { |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 529 | if (®s->esp != kcb->jprobe_saved_esp) { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 530 | struct pt_regs *saved_regs = |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 531 | container_of(kcb->jprobe_saved_esp, |
| 532 | struct pt_regs, esp); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 533 | printk("current esp %p does not match saved esp %p\n", |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 534 | ®s->esp, kcb->jprobe_saved_esp); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 535 | printk("Saved registers for jprobe %p\n", jp); |
| 536 | show_registers(saved_regs); |
| 537 | printk("Current registers\n"); |
| 538 | show_registers(regs); |
| 539 | BUG(); |
| 540 | } |
Ananth N Mavinakayanahalli | 9a0e3a8 | 2005-11-07 01:00:08 -0800 | [diff] [blame] | 541 | *regs = kcb->jprobe_saved_regs; |
| 542 | memcpy((kprobe_opcode_t *) stack_addr, kcb->jprobes_stack, |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 543 | MIN_STACK_SIZE(stack_addr)); |
Ananth N Mavinakayanahalli | d217d54 | 2005-11-07 01:00:14 -0800 | [diff] [blame] | 544 | preempt_enable_no_resched(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 545 | return 1; |
| 546 | } |
| 547 | return 0; |
| 548 | } |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 549 | |
| 550 | static struct kprobe trampoline_p = { |
| 551 | .addr = (kprobe_opcode_t *) &kretprobe_trampoline, |
| 552 | .pre_handler = trampoline_probe_handler |
| 553 | }; |
| 554 | |
Rusty Lynch | 6772926 | 2005-07-05 18:54:50 -0700 | [diff] [blame] | 555 | int __init arch_init_kprobes(void) |
Rusty Lynch | 4bdbd37 | 2005-06-27 15:17:09 -0700 | [diff] [blame] | 556 | { |
| 557 | return register_kprobe(&trampoline_p); |
| 558 | } |