blob: 7b5169d2b00026272ed26874913c42ff315befb3 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Kernel Probes (KProbes)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright (C) IBM Corporation, 2002, 2004
19 *
20 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
21 * Probes initial implementation ( includes contributions from
22 * Rusty Russell).
23 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
24 * interface to access function arguments.
Masami Hiramatsud6be29b2008-01-30 13:31:21 +010025 * 2004-Oct Jim Keniston <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
26 * <prasanna@in.ibm.com> adapted for x86_64 from i386.
Linus Torvalds1da177e2005-04-16 15:20:36 -070027 * 2005-Mar Roland McGrath <roland@redhat.com>
28 * Fixed to handle %rip-relative addressing mode correctly.
Masami Hiramatsud6be29b2008-01-30 13:31:21 +010029 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
30 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
31 * <prasanna@in.ibm.com> added function-return probes.
32 * 2005-May Rusty Lynch <rusty.lynch@intel.com>
33 * Added function return probes functionality
34 * 2006-Feb Masami Hiramatsu <hiramatu@sdl.hitachi.co.jp> added
35 * kprobe-booster and kretprobe-booster for i386.
Masami Hiramatsuda07ab02008-01-30 13:31:21 +010036 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com> added kprobe-booster
37 * and kretprobe-booster for x86-64
Masami Hiramatsud6be29b2008-01-30 13:31:21 +010038 * 2007-Dec Masami Hiramatsu <mhiramat@redhat.com>, Arjan van de Ven
39 * <arjan@infradead.org> and Jim Keniston <jkenisto@us.ibm.com>
40 * unified x86 kprobes code.
Linus Torvalds1da177e2005-04-16 15:20:36 -070041 */
42
Linus Torvalds1da177e2005-04-16 15:20:36 -070043#include <linux/kprobes.h>
44#include <linux/ptrace.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070045#include <linux/string.h>
46#include <linux/slab.h>
Quentin Barnesb506a9d2008-01-30 13:32:32 +010047#include <linux/hardirq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070048#include <linux/preempt.h>
Prasanna S Panchamukhic28f8962006-03-26 01:38:23 -080049#include <linux/module.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070050#include <linux/kdebug.h>
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070051
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +010052#include <asm/cacheflush.h>
53#include <asm/desc.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070054#include <asm/pgtable.h>
Prasanna S Panchamukhic28f8962006-03-26 01:38:23 -080055#include <asm/uaccess.h>
Andi Kleen19d36cc2007-07-22 11:12:31 +020056#include <asm/alternative.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
Linus Torvalds1da177e2005-04-16 15:20:36 -070058void jprobe_return_end(void);
59
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -080060DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
61DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
Linus Torvalds1da177e2005-04-16 15:20:36 -070062
Masami Hiramatsud6be29b2008-01-30 13:31:21 +010063#ifdef CONFIG_X86_64
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +010064#define stack_addr(regs) ((unsigned long *)regs->sp)
Masami Hiramatsud6be29b2008-01-30 13:31:21 +010065#else
66/*
67 * "&regs->sp" looks wrong, but it's correct for x86_32. x86_32 CPUs
68 * don't save the ss and esp registers if the CPU is already in kernel
69 * mode when it traps. So for kprobes, regs->sp and regs->ss are not
70 * the [nonexistent] saved stack pointer and ss register, but rather
71 * the top 8 bytes of the pre-int3 stack. So &regs->sp happens to
72 * point to the top of the pre-int3 stack.
73 */
74#define stack_addr(regs) ((unsigned long *)&regs->sp)
75#endif
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +010076
77#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
78 (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
79 (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
80 (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \
81 (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \
82 << (row % 32))
83 /*
84 * Undefined/reserved opcodes, conditional jump, Opcode Extension
85 * Groups, and some special opcodes can not boost.
86 */
87static const u32 twobyte_is_boostable[256 / 32] = {
88 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
89 /* ---------------------------------------------- */
90 W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0) | /* 00 */
91 W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 10 */
92 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 20 */
93 W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
94 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
95 W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */
96 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1) | /* 60 */
97 W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
98 W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 80 */
99 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
100 W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* a0 */
101 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) , /* b0 */
102 W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
103 W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) , /* d0 */
104 W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1) | /* e0 */
105 W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0) /* f0 */
106 /* ----------------------------------------------- */
107 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
108};
109static const u32 onebyte_has_modrm[256 / 32] = {
110 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
111 /* ----------------------------------------------- */
112 W(0x00, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* 00 */
113 W(0x10, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) , /* 10 */
114 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* 20 */
115 W(0x30, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) , /* 30 */
116 W(0x40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 40 */
117 W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 50 */
118 W(0x60, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0) | /* 60 */
119 W(0x70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 70 */
120 W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
121 W(0x90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 90 */
122 W(0xa0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* a0 */
123 W(0xb0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* b0 */
124 W(0xc0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0) | /* c0 */
125 W(0xd0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
126 W(0xe0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* e0 */
127 W(0xf0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) /* f0 */
128 /* ----------------------------------------------- */
129 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
130};
131static const u32 twobyte_has_modrm[256 / 32] = {
132 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
133 /* ----------------------------------------------- */
134 W(0x00, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1) | /* 0f */
135 W(0x10, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0) , /* 1f */
136 W(0x20, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* 2f */
137 W(0x30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 3f */
138 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 4f */
139 W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 5f */
140 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 6f */
141 W(0x70, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1) , /* 7f */
142 W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) | /* 8f */
143 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 9f */
144 W(0xa0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1) | /* af */
145 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1) , /* bf */
146 W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0) | /* cf */
147 W(0xd0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* df */
148 W(0xe0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* ef */
149 W(0xf0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0) /* ff */
150 /* ----------------------------------------------- */
151 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
152};
153#undef W
154
Masami Hiramatsuf438d912007-10-16 01:27:49 -0700155struct kretprobe_blackpoint kretprobe_blacklist[] = {
156 {"__switch_to", }, /* This function switches only current task, but
157 doesn't switch kernel stack.*/
158 {NULL, NULL} /* Terminator */
159};
160const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);
161
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100162/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
Harvey Harrisone7b5e112008-01-30 13:31:43 +0100163static void __kprobes set_jmp_op(void *from, void *to)
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100164{
165 struct __arch_jmp_op {
166 char op;
167 s32 raddr;
168 } __attribute__((packed)) * jop;
169 jop = (struct __arch_jmp_op *)from;
170 jop->raddr = (s32)((long)(to) - ((long)(from) + 5));
171 jop->op = RELATIVEJUMP_INSTRUCTION;
172}
173
174/*
Harvey Harrison99309272008-01-30 13:32:14 +0100175 * Check for the REX prefix which can only exist on X86_64
176 * X86_32 always returns 0
177 */
178static int __kprobes is_REX_prefix(kprobe_opcode_t *insn)
179{
180#ifdef CONFIG_X86_64
181 if ((*insn & 0xf0) == 0x40)
182 return 1;
183#endif
184 return 0;
185}
186
187/*
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100188 * Returns non-zero if opcode is boostable.
189 * RIP relative instructions are adjusted at copying time in 64 bits mode
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100190 */
Harvey Harrisone7b5e112008-01-30 13:31:43 +0100191static int __kprobes can_boost(kprobe_opcode_t *opcodes)
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100192{
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100193 kprobe_opcode_t opcode;
194 kprobe_opcode_t *orig_opcodes = opcodes;
195
Jaswinder Singh Rajputcde5edb2009-03-18 17:37:45 +0530196 if (search_exception_tables((unsigned long)opcodes))
Masami Hiramatsu30390882009-03-16 18:57:22 -0400197 return 0; /* Page fault may occur on this address. */
198
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100199retry:
200 if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
201 return 0;
202 opcode = *(opcodes++);
203
204 /* 2nd-byte opcode */
205 if (opcode == 0x0f) {
206 if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
207 return 0;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100208 return test_bit(*opcodes,
209 (unsigned long *)twobyte_is_boostable);
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100210 }
211
212 switch (opcode & 0xf0) {
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100213#ifdef CONFIG_X86_64
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100214 case 0x40:
215 goto retry; /* REX prefix is boostable */
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100216#endif
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100217 case 0x60:
218 if (0x63 < opcode && opcode < 0x67)
219 goto retry; /* prefixes */
220 /* can't boost Address-size override and bound */
221 return (opcode != 0x62 && opcode != 0x67);
222 case 0x70:
223 return 0; /* can't boost conditional jump */
224 case 0xc0:
225 /* can't boost software-interruptions */
226 return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
227 case 0xd0:
228 /* can boost AA* and XLAT */
229 return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
230 case 0xe0:
231 /* can boost in/out and absolute jmps */
232 return ((opcode & 0x04) || opcode == 0xea);
233 case 0xf0:
234 if ((opcode & 0x0c) == 0 && opcode != 0xf1)
235 goto retry; /* lock/rep(ne) prefix */
236 /* clear and set flags are boostable */
237 return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
238 default:
239 /* segment override prefixes are boostable */
240 if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
241 goto retry; /* prefixes */
242 /* CS override prefix and call are not boostable */
243 return (opcode != 0x2e && opcode != 0x9a);
244 }
245}
246
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247/*
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100248 * Returns non-zero if opcode modifies the interrupt flag.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 */
Andrew Morton86454192007-11-26 20:42:19 +0100250static int __kprobes is_IF_modifier(kprobe_opcode_t *insn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251{
252 switch (*insn) {
253 case 0xfa: /* cli */
254 case 0xfb: /* sti */
255 case 0xcf: /* iret/iretd */
256 case 0x9d: /* popf/popfd */
257 return 1;
258 }
Harvey Harrison99309272008-01-30 13:32:14 +0100259
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100260 /*
Harvey Harrison99309272008-01-30 13:32:14 +0100261 * on X86_64, 0x40-0x4f are REX prefixes so we need to look
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100262 * at the next byte instead.. but of course not recurse infinitely
263 */
Harvey Harrison99309272008-01-30 13:32:14 +0100264 if (is_REX_prefix(insn))
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100265 return is_IF_modifier(++insn);
Harvey Harrison99309272008-01-30 13:32:14 +0100266
Linus Torvalds1da177e2005-04-16 15:20:36 -0700267 return 0;
268}
269
270/*
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100271 * Adjust the displacement if the instruction uses the %rip-relative
272 * addressing mode.
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100273 * If it does, Return the address of the 32-bit displacement word.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 * If not, return null.
Harvey Harrison31f80e42008-01-30 13:32:16 +0100275 * Only applicable to 64-bit x86.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 */
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100277static void __kprobes fix_riprel(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278{
Harvey Harrison31f80e42008-01-30 13:32:16 +0100279#ifdef CONFIG_X86_64
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100280 u8 *insn = p->ainsn.insn;
281 s64 disp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282 int need_modrm;
283
284 /* Skip legacy instruction prefixes. */
285 while (1) {
286 switch (*insn) {
287 case 0x66:
288 case 0x67:
289 case 0x2e:
290 case 0x3e:
291 case 0x26:
292 case 0x64:
293 case 0x65:
294 case 0x36:
295 case 0xf0:
296 case 0xf3:
297 case 0xf2:
298 ++insn;
299 continue;
300 }
301 break;
302 }
303
304 /* Skip REX instruction prefix. */
Harvey Harrison99309272008-01-30 13:32:14 +0100305 if (is_REX_prefix(insn))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 ++insn;
307
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100308 if (*insn == 0x0f) {
309 /* Two-byte opcode. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 ++insn;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100311 need_modrm = test_bit(*insn,
312 (unsigned long *)twobyte_has_modrm);
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100313 } else
314 /* One-byte opcode. */
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100315 need_modrm = test_bit(*insn,
316 (unsigned long *)onebyte_has_modrm);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317
318 if (need_modrm) {
319 u8 modrm = *++insn;
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100320 if ((modrm & 0xc7) == 0x05) {
321 /* %rip+disp32 addressing mode */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322 /* Displacement follows ModRM byte. */
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100323 ++insn;
324 /*
325 * The copied instruction uses the %rip-relative
326 * addressing mode. Adjust the displacement for the
327 * difference between the original location of this
328 * instruction and the location of the copy that will
329 * actually be run. The tricky bit here is making sure
330 * that the sign extension happens correctly in this
331 * calculation, since we need a signed 32-bit result to
332 * be sign-extended to 64 bits when it's added to the
333 * %rip value and yield the same 64-bit result that the
334 * sign-extension of the original signed 32-bit
335 * displacement would have given.
336 */
337 disp = (u8 *) p->addr + *((s32 *) insn) -
338 (u8 *) p->ainsn.insn;
339 BUG_ON((s64) (s32) disp != disp); /* Sanity check. */
340 *(s32 *)insn = (s32) disp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 }
342 }
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100343#endif
Harvey Harrison31f80e42008-01-30 13:32:16 +0100344}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345
Keshavamurthy Anil Sf709b122006-01-09 20:52:44 -0800346static void __kprobes arch_copy_kprobe(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347{
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100348 memcpy(p->ainsn.insn, p->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
Harvey Harrison31f80e42008-01-30 13:32:16 +0100349
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100350 fix_riprel(p);
Harvey Harrison31f80e42008-01-30 13:32:16 +0100351
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100352 if (can_boost(p->addr))
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100353 p->ainsn.boostable = 0;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100354 else
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100355 p->ainsn.boostable = -1;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100356
Rusty Lynch7e1048b2005-06-23 00:09:25 -0700357 p->opcode = *p->addr;
358}
359
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100360int __kprobes arch_prepare_kprobe(struct kprobe *p)
361{
362 /* insn: must be on special executable page on x86. */
363 p->ainsn.insn = get_insn_slot();
364 if (!p->ainsn.insn)
365 return -ENOMEM;
366 arch_copy_kprobe(p);
367 return 0;
368}
369
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700370void __kprobes arch_arm_kprobe(struct kprobe *p)
Rusty Lynch7e1048b2005-06-23 00:09:25 -0700371{
Andi Kleen19d36cc2007-07-22 11:12:31 +0200372 text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1);
Rusty Lynch7e1048b2005-06-23 00:09:25 -0700373}
374
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700375void __kprobes arch_disarm_kprobe(struct kprobe *p)
Rusty Lynch7e1048b2005-06-23 00:09:25 -0700376{
Andi Kleen19d36cc2007-07-22 11:12:31 +0200377 text_poke(p->addr, &p->opcode, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378}
379
Ananth N Mavinakayanahalli0498b632006-01-09 20:52:46 -0800380void __kprobes arch_remove_kprobe(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381{
Masami Hiramatsu12941562009-01-06 14:41:50 -0800382 if (p->ainsn.insn) {
383 free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
384 p->ainsn.insn = NULL;
385 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700386}
387
Prasanna S Panchamukhi3b602112006-04-18 22:22:00 -0700388static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700389{
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800390 kcb->prev_kprobe.kp = kprobe_running();
391 kcb->prev_kprobe.status = kcb->kprobe_status;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100392 kcb->prev_kprobe.old_flags = kcb->kprobe_old_flags;
393 kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700394}
395
Prasanna S Panchamukhi3b602112006-04-18 22:22:00 -0700396static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700397{
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800398 __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
399 kcb->kprobe_status = kcb->prev_kprobe.status;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100400 kcb->kprobe_old_flags = kcb->prev_kprobe.old_flags;
401 kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700402}
403
Prasanna S Panchamukhi3b602112006-04-18 22:22:00 -0700404static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800405 struct kprobe_ctlblk *kcb)
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700406{
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800407 __get_cpu_var(current_kprobe) = p;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100408 kcb->kprobe_saved_flags = kcb->kprobe_old_flags
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +0100409 = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700410 if (is_IF_modifier(p->ainsn.insn))
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +0100411 kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700412}
413
Harvey Harrisone7b5e112008-01-30 13:31:43 +0100414static void __kprobes clear_btf(void)
Roland McGrath1ecc7982008-01-30 13:30:54 +0100415{
416 if (test_thread_flag(TIF_DEBUGCTLMSR))
Jan Beulich5b0e5082008-03-10 13:11:17 +0000417 update_debugctlmsr(0);
Roland McGrath1ecc7982008-01-30 13:30:54 +0100418}
419
Harvey Harrisone7b5e112008-01-30 13:31:43 +0100420static void __kprobes restore_btf(void)
Roland McGrath1ecc7982008-01-30 13:30:54 +0100421{
422 if (test_thread_flag(TIF_DEBUGCTLMSR))
Jan Beulich5b0e5082008-03-10 13:11:17 +0000423 update_debugctlmsr(current->thread.debugctlmsr);
Roland McGrath1ecc7982008-01-30 13:30:54 +0100424}
425
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700426static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427{
Roland McGrath1ecc7982008-01-30 13:30:54 +0100428 clear_btf();
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +0100429 regs->flags |= X86_EFLAGS_TF;
430 regs->flags &= ~X86_EFLAGS_IF;
Harvey Harrisone7b5e112008-01-30 13:31:43 +0100431 /* single step inline if the instruction is an int3 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432 if (p->opcode == BREAKPOINT_INSTRUCTION)
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100433 regs->ip = (unsigned long)p->addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434 else
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100435 regs->ip = (unsigned long)p->ainsn.insn;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436}
437
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700438void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700439 struct pt_regs *regs)
Rusty Lynch73649da2005-06-23 00:09:23 -0700440{
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100441 unsigned long *sara = stack_addr(regs);
Rusty Lynch73649da2005-06-23 00:09:23 -0700442
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700443 ri->ret_addr = (kprobe_opcode_t *) *sara;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100444
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700445 /* Replace the return addr with trampoline addr */
446 *sara = (unsigned long) &kretprobe_trampoline;
Rusty Lynch73649da2005-06-23 00:09:23 -0700447}
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100448
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100449static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs,
450 struct kprobe_ctlblk *kcb)
451{
Masami Hiramatsu5a4ccaf2009-01-06 21:15:32 +0100452#if !defined(CONFIG_PREEMPT) || defined(CONFIG_FREEZER)
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100453 if (p->ainsn.boostable == 1 && !p->post_handler) {
454 /* Boost up -- we can execute copied instructions directly */
455 reset_current_kprobe();
456 regs->ip = (unsigned long)p->ainsn.insn;
457 preempt_enable_no_resched();
458 return;
459 }
460#endif
461 prepare_singlestep(p, regs);
462 kcb->kprobe_status = KPROBE_HIT_SS;
463}
464
Harvey Harrison40102d42008-01-30 13:32:02 +0100465/*
466 * We have reentered the kprobe_handler(), since another probe was hit while
467 * within the handler. We save the original kprobes variables and just single
468 * step on the instruction of the new probe without calling any user handlers.
469 */
Masami Hiramatsu59e87cd2008-01-30 13:32:02 +0100470static int __kprobes reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
471 struct kprobe_ctlblk *kcb)
Harvey Harrison40102d42008-01-30 13:32:02 +0100472{
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100473 switch (kcb->kprobe_status) {
474 case KPROBE_HIT_SSDONE:
Masami Hiramatsu59e87cd2008-01-30 13:32:02 +0100475#ifdef CONFIG_X86_64
Masami Hiramatsu59e87cd2008-01-30 13:32:02 +0100476 /* TODO: Provide re-entrancy from post_kprobes_handler() and
477 * avoid exception stack corruption while single-stepping on
478 * the instruction of the new probe.
479 */
480 arch_disarm_kprobe(p);
481 regs->ip = (unsigned long)p->addr;
482 reset_current_kprobe();
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100483 preempt_enable_no_resched();
484 break;
Masami Hiramatsu59e87cd2008-01-30 13:32:02 +0100485#endif
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100486 case KPROBE_HIT_ACTIVE:
Abhishek Sagarfb8830e2008-01-30 13:33:13 +0100487 save_previous_kprobe(kcb);
488 set_current_kprobe(p, regs, kcb);
489 kprobes_inc_nmissed_count(p);
490 prepare_singlestep(p, regs);
491 kcb->kprobe_status = KPROBE_REENTER;
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100492 break;
493 case KPROBE_HIT_SS:
Abhishek Sagarfb8830e2008-01-30 13:33:13 +0100494 if (p == kprobe_running()) {
gorcunov@gmail.coma5c15d42008-03-28 17:56:56 +0300495 regs->flags &= ~X86_EFLAGS_TF;
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100496 regs->flags |= kcb->kprobe_saved_flags;
497 return 0;
498 } else {
Abhishek Sagarfb8830e2008-01-30 13:33:13 +0100499 /* A probe has been hit in the codepath leading up
500 * to, or just after, single-stepping of a probed
501 * instruction. This entire codepath should strictly
502 * reside in .kprobes.text section. Raise a warning
503 * to highlight this peculiar case.
504 */
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100505 }
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100506 default:
507 /* impossible cases */
508 WARN_ON(1);
Abhishek Sagarfb8830e2008-01-30 13:33:13 +0100509 return 0;
Masami Hiramatsu59e87cd2008-01-30 13:32:02 +0100510 }
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100511
Masami Hiramatsu59e87cd2008-01-30 13:32:02 +0100512 return 1;
Harvey Harrison40102d42008-01-30 13:32:02 +0100513}
Rusty Lynch73649da2005-06-23 00:09:23 -0700514
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100515/*
516 * Interrupts are disabled on entry as trap3 is an interrupt gate and they
517 * remain disabled thorough out this function.
518 */
519static int __kprobes kprobe_handler(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520{
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100521 kprobe_opcode_t *addr;
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100522 struct kprobe *p;
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800523 struct kprobe_ctlblk *kcb;
524
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100525 addr = (kprobe_opcode_t *)(regs->ip - sizeof(kprobe_opcode_t));
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100526 if (*addr != BREAKPOINT_INSTRUCTION) {
527 /*
528 * The breakpoint instruction was removed right
529 * after we hit it. Another cpu has removed
530 * either a probepoint or a debugger breakpoint
531 * at this address. In either case, no further
532 * handling of this interrupt is appropriate.
533 * Back up over the (now missing) int3 and run
534 * the original instruction.
535 */
536 regs->ip = (unsigned long)addr;
537 return 1;
538 }
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100539
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800540 /*
541 * We don't want to be preempted for the entire
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100542 * duration of kprobe processing. We conditionally
543 * re-enable preemption at the end of this function,
544 * and also in reenter_kprobe() and setup_singlestep().
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800545 */
546 preempt_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100548 kcb = get_kprobe_ctlblk();
Harvey Harrisonb9760152008-01-30 13:32:19 +0100549 p = get_kprobe(addr);
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100550
Harvey Harrisonb9760152008-01-30 13:32:19 +0100551 if (p) {
Harvey Harrisonb9760152008-01-30 13:32:19 +0100552 if (kprobe_running()) {
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100553 if (reenter_kprobe(p, regs, kcb))
554 return 1;
Harvey Harrisonb9760152008-01-30 13:32:19 +0100555 } else {
556 set_current_kprobe(p, regs, kcb);
557 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100558
Linus Torvalds1da177e2005-04-16 15:20:36 -0700559 /*
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100560 * If we have no pre-handler or it returned 0, we
561 * continue with normal processing. If we have a
562 * pre-handler and it returned non-zero, it prepped
563 * for calling the break_handler below on re-entry
564 * for jprobe processing, so get out doing nothing
565 * more here.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566 */
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100567 if (!p->pre_handler || !p->pre_handler(p, regs))
568 setup_singlestep(p, regs, kcb);
569 return 1;
Harvey Harrisonb9760152008-01-30 13:32:19 +0100570 }
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100571 } else if (kprobe_running()) {
572 p = __get_cpu_var(current_kprobe);
573 if (p->break_handler && p->break_handler(p, regs)) {
574 setup_singlestep(p, regs, kcb);
575 return 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576 }
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100577 } /* else: not a kprobe fault; let the kernel handle it */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700578
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800579 preempt_enable_no_resched();
Abhishek Sagarf315dec2008-01-30 13:32:50 +0100580 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700581}
582
583/*
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100584 * When a retprobed function returns, this code saves registers and
585 * calls trampoline_handler() runs, which calls the kretprobe's handler.
Rusty Lynch73649da2005-06-23 00:09:23 -0700586 */
Harvey Harrisonf1452d42008-02-14 15:23:53 -0800587static void __used __kprobes kretprobe_trampoline_holder(void)
Harvey Harrison10175792008-01-30 13:33:01 +0100588{
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100589 asm volatile (
590 ".global kretprobe_trampoline\n"
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100591 "kretprobe_trampoline: \n"
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100592#ifdef CONFIG_X86_64
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100593 /* We don't bother saving the ss register */
594 " pushq %rsp\n"
595 " pushfq\n"
596 /*
597 * Skip cs, ip, orig_ax.
598 * trampoline_handler() will plug in these values
599 */
600 " subq $24, %rsp\n"
601 " pushq %rdi\n"
602 " pushq %rsi\n"
603 " pushq %rdx\n"
604 " pushq %rcx\n"
605 " pushq %rax\n"
606 " pushq %r8\n"
607 " pushq %r9\n"
608 " pushq %r10\n"
609 " pushq %r11\n"
610 " pushq %rbx\n"
611 " pushq %rbp\n"
612 " pushq %r12\n"
613 " pushq %r13\n"
614 " pushq %r14\n"
615 " pushq %r15\n"
616 " movq %rsp, %rdi\n"
617 " call trampoline_handler\n"
618 /* Replace saved sp with true return address. */
619 " movq %rax, 152(%rsp)\n"
620 " popq %r15\n"
621 " popq %r14\n"
622 " popq %r13\n"
623 " popq %r12\n"
624 " popq %rbp\n"
625 " popq %rbx\n"
626 " popq %r11\n"
627 " popq %r10\n"
628 " popq %r9\n"
629 " popq %r8\n"
630 " popq %rax\n"
631 " popq %rcx\n"
632 " popq %rdx\n"
633 " popq %rsi\n"
634 " popq %rdi\n"
635 /* Skip orig_ax, ip, cs */
636 " addq $24, %rsp\n"
637 " popfq\n"
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100638#else
639 " pushf\n"
640 /*
Masami Hiramatsufee039a2009-03-23 10:14:52 -0400641 * Skip cs, ip, orig_ax and gs.
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100642 * trampoline_handler() will plug in these values
643 */
Masami Hiramatsufee039a2009-03-23 10:14:52 -0400644 " subl $16, %esp\n"
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100645 " pushl %fs\n"
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100646 " pushl %es\n"
Masami Hiramatsufee039a2009-03-23 10:14:52 -0400647 " pushl %ds\n"
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100648 " pushl %eax\n"
649 " pushl %ebp\n"
650 " pushl %edi\n"
651 " pushl %esi\n"
652 " pushl %edx\n"
653 " pushl %ecx\n"
654 " pushl %ebx\n"
655 " movl %esp, %eax\n"
656 " call trampoline_handler\n"
657 /* Move flags to cs */
Masami Hiramatsufee039a2009-03-23 10:14:52 -0400658 " movl 56(%esp), %edx\n"
659 " movl %edx, 52(%esp)\n"
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100660 /* Replace saved flags with true return address. */
Masami Hiramatsufee039a2009-03-23 10:14:52 -0400661 " movl %eax, 56(%esp)\n"
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100662 " popl %ebx\n"
663 " popl %ecx\n"
664 " popl %edx\n"
665 " popl %esi\n"
666 " popl %edi\n"
667 " popl %ebp\n"
668 " popl %eax\n"
Masami Hiramatsufee039a2009-03-23 10:14:52 -0400669 /* Skip ds, es, fs, gs, orig_ax and ip */
670 " addl $24, %esp\n"
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100671 " popf\n"
672#endif
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100673 " ret\n");
Harvey Harrison10175792008-01-30 13:33:01 +0100674}
Rusty Lynch73649da2005-06-23 00:09:23 -0700675
676/*
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100677 * Called from kretprobe_trampoline
Rusty Lynch73649da2005-06-23 00:09:23 -0700678 */
Harvey Harrisonf1452d42008-02-14 15:23:53 -0800679static __used __kprobes void *trampoline_handler(struct pt_regs *regs)
Rusty Lynch73649da2005-06-23 00:09:23 -0700680{
bibo,mao62c27be2006-10-02 02:17:33 -0700681 struct kretprobe_instance *ri = NULL;
bibo,mao99219a32006-10-02 02:17:35 -0700682 struct hlist_head *head, empty_rp;
bibo,mao62c27be2006-10-02 02:17:33 -0700683 struct hlist_node *node, *tmp;
Ananth N Mavinakayanahalli991a51d2005-11-07 01:00:14 -0800684 unsigned long flags, orig_ret_address = 0;
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100685 unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline;
Rusty Lynch73649da2005-06-23 00:09:23 -0700686
bibo,mao99219a32006-10-02 02:17:35 -0700687 INIT_HLIST_HEAD(&empty_rp);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700688 kretprobe_hash_lock(current, &head, &flags);
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100689 /* fixup registers */
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100690#ifdef CONFIG_X86_64
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100691 regs->cs = __KERNEL_CS;
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100692#else
693 regs->cs = __KERNEL_CS | get_kernel_rpl();
Masami Hiramatsufee039a2009-03-23 10:14:52 -0400694 regs->gs = 0;
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100695#endif
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100696 regs->ip = trampoline_address;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100697 regs->orig_ax = ~0UL;
Rusty Lynch73649da2005-06-23 00:09:23 -0700698
Rusty Lynchba8af122005-06-27 15:17:10 -0700699 /*
700 * It is possible to have multiple instances associated with a given
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100701 * task either because multiple functions in the call path have
Frederik Schwarzer025dfda2008-10-16 19:02:37 +0200702 * return probes installed on them, and/or more than one
Rusty Lynchba8af122005-06-27 15:17:10 -0700703 * return probe was registered for a target function.
704 *
705 * We can handle this because:
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100706 * - instances are always pushed into the head of the list
Rusty Lynchba8af122005-06-27 15:17:10 -0700707 * - when multiple return probes are registered for the same
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100708 * function, the (chronologically) first instance's ret_addr
709 * will be the real return address, and all the rest will
710 * point to kretprobe_trampoline.
Rusty Lynchba8af122005-06-27 15:17:10 -0700711 */
712 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
bibo,mao62c27be2006-10-02 02:17:33 -0700713 if (ri->task != current)
Rusty Lynchba8af122005-06-27 15:17:10 -0700714 /* another task is sharing our hash bucket */
bibo,mao62c27be2006-10-02 02:17:33 -0700715 continue;
Rusty Lynch73649da2005-06-23 00:09:23 -0700716
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100717 if (ri->rp && ri->rp->handler) {
718 __get_cpu_var(current_kprobe) = &ri->rp->kp;
719 get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE;
Rusty Lynchba8af122005-06-27 15:17:10 -0700720 ri->rp->handler(ri, regs);
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100721 __get_cpu_var(current_kprobe) = NULL;
722 }
Rusty Lynch73649da2005-06-23 00:09:23 -0700723
Rusty Lynchba8af122005-06-27 15:17:10 -0700724 orig_ret_address = (unsigned long)ri->ret_addr;
bibo,mao99219a32006-10-02 02:17:35 -0700725 recycle_rp_inst(ri, &empty_rp);
Rusty Lynchba8af122005-06-27 15:17:10 -0700726
727 if (orig_ret_address != trampoline_address)
728 /*
729 * This is the real return address. Any other
730 * instances associated with this task are for
731 * other calls deeper on the call stack
732 */
733 break;
Rusty Lynch73649da2005-06-23 00:09:23 -0700734 }
Rusty Lynchba8af122005-06-27 15:17:10 -0700735
Ananth N Mavinakayanahalli0f95b7f2007-05-08 00:28:27 -0700736 kretprobe_assert(ri, orig_ret_address, trampoline_address);
Rusty Lynchba8af122005-06-27 15:17:10 -0700737
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700738 kretprobe_hash_unlock(current, &flags);
Rusty Lynchba8af122005-06-27 15:17:10 -0700739
bibo,mao99219a32006-10-02 02:17:35 -0700740 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
741 hlist_del(&ri->hlist);
742 kfree(ri);
743 }
Masami Hiramatsuda07ab02008-01-30 13:31:21 +0100744 return (void *)orig_ret_address;
Rusty Lynch73649da2005-06-23 00:09:23 -0700745}
746
747/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 * Called after single-stepping. p->addr is the address of the
749 * instruction whose first byte has been replaced by the "int 3"
750 * instruction. To avoid the SMP problems that can occur when we
751 * temporarily put back the original opcode to single-step, we
752 * single-stepped a copy of the instruction. The address of this
753 * copy is p->ainsn.insn.
754 *
755 * This function prepares to return from the post-single-step
756 * interrupt. We have to fix up the stack as follows:
757 *
758 * 0) Except in the case of absolute or indirect jump or call instructions,
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100759 * the new ip is relative to the copied instruction. We need to make
Linus Torvalds1da177e2005-04-16 15:20:36 -0700760 * it relative to the original instruction.
761 *
762 * 1) If the single-stepped instruction was pushfl, then the TF and IF
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100763 * flags are set in the just-pushed flags, and may need to be cleared.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764 *
765 * 2) If the single-stepped instruction was a call, the return address
766 * that is atop the stack is the address following the copied instruction.
767 * We need to make it the address following the original instruction.
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100768 *
769 * If this is the first time we've single-stepped the instruction at
770 * this probepoint, and the instruction is boostable, boost it: add a
771 * jump instruction after the copied instruction, that jumps to the next
772 * instruction after the probepoint.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773 */
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800774static void __kprobes resume_execution(struct kprobe *p,
775 struct pt_regs *regs, struct kprobe_ctlblk *kcb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776{
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100777 unsigned long *tos = stack_addr(regs);
778 unsigned long copy_ip = (unsigned long)p->ainsn.insn;
779 unsigned long orig_ip = (unsigned long)p->addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 kprobe_opcode_t *insn = p->ainsn.insn;
781
782 /*skip the REX prefix*/
Harvey Harrison99309272008-01-30 13:32:14 +0100783 if (is_REX_prefix(insn))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700784 insn++;
785
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +0100786 regs->flags &= ~X86_EFLAGS_TF;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700787 switch (*insn) {
Masami Hiramatsu0b0122f2007-12-18 18:05:58 +0100788 case 0x9c: /* pushfl */
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +0100789 *tos &= ~(X86_EFLAGS_TF | X86_EFLAGS_IF);
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100790 *tos |= kcb->kprobe_old_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791 break;
Masami Hiramatsu0b0122f2007-12-18 18:05:58 +0100792 case 0xc2: /* iret/ret/lret */
793 case 0xc3:
Prasanna S Panchamukhi0b9e2ca2005-05-05 16:15:40 -0700794 case 0xca:
Masami Hiramatsu0b0122f2007-12-18 18:05:58 +0100795 case 0xcb:
796 case 0xcf:
797 case 0xea: /* jmp absolute -- ip is correct */
798 /* ip is already adjusted, no more changes required */
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100799 p->ainsn.boostable = 1;
Masami Hiramatsu0b0122f2007-12-18 18:05:58 +0100800 goto no_change;
801 case 0xe8: /* call relative - Fix return addr */
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100802 *tos = orig_ip + (*tos - copy_ip);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 break;
Harvey Harrisone7b5e112008-01-30 13:31:43 +0100804#ifdef CONFIG_X86_32
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100805 case 0x9a: /* call absolute -- same as call absolute, indirect */
806 *tos = orig_ip + (*tos - copy_ip);
807 goto no_change;
808#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 case 0xff:
Satoshi Oshimadc49e342006-05-20 15:00:21 -0700810 if ((insn[1] & 0x30) == 0x10) {
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100811 /*
812 * call absolute, indirect
813 * Fix return addr; ip is correct.
814 * But this is not boostable
815 */
816 *tos = orig_ip + (*tos - copy_ip);
Masami Hiramatsu0b0122f2007-12-18 18:05:58 +0100817 goto no_change;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100818 } else if (((insn[1] & 0x31) == 0x20) ||
819 ((insn[1] & 0x31) == 0x21)) {
820 /*
821 * jmp near and far, absolute indirect
822 * ip is correct. And this is boostable
823 */
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100824 p->ainsn.boostable = 1;
Masami Hiramatsu0b0122f2007-12-18 18:05:58 +0100825 goto no_change;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700826 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700827 default:
828 break;
829 }
830
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100831 if (p->ainsn.boostable == 0) {
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100832 if ((regs->ip > copy_ip) &&
833 (regs->ip - copy_ip) + 5 < MAX_INSN_SIZE) {
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100834 /*
835 * These instructions can be executed directly if it
836 * jumps back to correct address.
837 */
838 set_jmp_op((void *)regs->ip,
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100839 (void *)orig_ip + (regs->ip - copy_ip));
Masami Hiramatsuaa470142008-01-30 13:31:21 +0100840 p->ainsn.boostable = 1;
841 } else {
842 p->ainsn.boostable = -1;
843 }
844 }
845
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100846 regs->ip += orig_ip - copy_ip;
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100847
Masami Hiramatsu0b0122f2007-12-18 18:05:58 +0100848no_change:
Roland McGrath1ecc7982008-01-30 13:30:54 +0100849 restore_btf();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700850}
851
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100852/*
853 * Interrupts are disabled on entry as trap1 is an interrupt gate and they
854 * remain disabled thoroughout this function.
855 */
856static int __kprobes post_kprobe_handler(struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700857{
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800858 struct kprobe *cur = kprobe_running();
859 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
860
861 if (!cur)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700862 return 0;
863
Yakov Lerneracb5b8a2008-03-16 03:21:21 -0500864 resume_execution(cur, regs, kcb);
865 regs->flags |= kcb->kprobe_saved_flags;
Yakov Lerneracb5b8a2008-03-16 03:21:21 -0500866
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800867 if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) {
868 kcb->kprobe_status = KPROBE_HIT_SSDONE;
869 cur->post_handler(cur, regs, 0);
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700870 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100872 /* Restore back the original saved kprobes variables and continue. */
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800873 if (kcb->kprobe_status == KPROBE_REENTER) {
874 restore_previous_kprobe(kcb);
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700875 goto out;
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700876 }
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800877 reset_current_kprobe();
Prasanna S Panchamukhiaa3d7e32005-06-23 00:09:37 -0700878out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700879 preempt_enable_no_resched();
880
881 /*
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100882 * if somebody else is singlestepping across a probe point, flags
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 * will have TF set, in which case, continue the remaining processing
884 * of do_debug, as if this is not a probe hit.
885 */
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +0100886 if (regs->flags & X86_EFLAGS_TF)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887 return 0;
888
889 return 1;
890}
891
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700892int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893{
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800894 struct kprobe *cur = kprobe_running();
895 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
896
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100897 switch (kcb->kprobe_status) {
Prasanna S Panchamukhic28f8962006-03-26 01:38:23 -0800898 case KPROBE_HIT_SS:
899 case KPROBE_REENTER:
900 /*
901 * We are here because the instruction being single
902 * stepped caused a page fault. We reset the current
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100903 * kprobe and the ip points back to the probe address
Prasanna S Panchamukhic28f8962006-03-26 01:38:23 -0800904 * and allow the page fault handler to continue as a
905 * normal page fault.
906 */
H. Peter Anvin65ea5b02008-01-30 13:30:56 +0100907 regs->ip = (unsigned long)cur->addr;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100908 regs->flags |= kcb->kprobe_old_flags;
Prasanna S Panchamukhic28f8962006-03-26 01:38:23 -0800909 if (kcb->kprobe_status == KPROBE_REENTER)
910 restore_previous_kprobe(kcb);
911 else
912 reset_current_kprobe();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 preempt_enable_no_resched();
Prasanna S Panchamukhic28f8962006-03-26 01:38:23 -0800914 break;
915 case KPROBE_HIT_ACTIVE:
916 case KPROBE_HIT_SSDONE:
917 /*
918 * We increment the nmissed count for accounting,
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100919 * we can also use npre/npostfault count for accounting
Prasanna S Panchamukhic28f8962006-03-26 01:38:23 -0800920 * these specific fault cases.
921 */
922 kprobes_inc_nmissed_count(cur);
923
924 /*
925 * We come here because instructions in the pre/post
926 * handler caused the page_fault, this could happen
927 * if handler tries to access user space by
928 * copy_from_user(), get_user() etc. Let the
929 * user-specified handler try to fix it first.
930 */
931 if (cur->fault_handler && cur->fault_handler(cur, regs, trapnr))
932 return 1;
933
934 /*
935 * In case the user-specified fault handler returned
936 * zero, try to fix up.
937 */
Masami Hiramatsud6be29b2008-01-30 13:31:21 +0100938 if (fixup_exception(regs))
939 return 1;
Harvey Harrison6d485832008-01-30 13:31:41 +0100940
Prasanna S Panchamukhic28f8962006-03-26 01:38:23 -0800941 /*
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100942 * fixup routine could not handle it,
Prasanna S Panchamukhic28f8962006-03-26 01:38:23 -0800943 * Let do_page_fault() fix it.
944 */
945 break;
946 default:
947 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948 }
949 return 0;
950}
951
952/*
953 * Wrapper routine for handling exceptions.
954 */
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700955int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
956 unsigned long val, void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957{
Jan Engelhardtade1af72008-01-30 13:33:23 +0100958 struct die_args *args = data;
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800959 int ret = NOTIFY_DONE;
960
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100961 if (args->regs && user_mode_vm(args->regs))
bibo,mao2326c772006-03-26 01:38:21 -0800962 return ret;
963
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 switch (val) {
965 case DIE_INT3:
966 if (kprobe_handler(args->regs))
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800967 ret = NOTIFY_STOP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 break;
969 case DIE_DEBUG:
970 if (post_kprobe_handler(args->regs))
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800971 ret = NOTIFY_STOP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972 break;
973 case DIE_GPF:
Quentin Barnesb506a9d2008-01-30 13:32:32 +0100974 /*
975 * To be potentially processing a kprobe fault and to
976 * trust the result from kprobe_running(), we have
977 * be non-preemptible.
978 */
979 if (!preemptible() && kprobe_running() &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980 kprobe_fault_handler(args->regs, args->trapnr))
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800981 ret = NOTIFY_STOP;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 break;
983 default:
984 break;
985 }
Ananth N Mavinakayanahalli66ff2d02005-11-07 01:00:07 -0800986 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987}
988
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -0700989int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700990{
991 struct jprobe *jp = container_of(p, struct jprobe, kp);
992 unsigned long addr;
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800993 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700994
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -0800995 kcb->jprobe_saved_regs = *regs;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +0100996 kcb->jprobe_saved_sp = stack_addr(regs);
997 addr = (unsigned long)(kcb->jprobe_saved_sp);
998
Linus Torvalds1da177e2005-04-16 15:20:36 -0700999 /*
1000 * As Linus pointed out, gcc assumes that the callee
1001 * owns the argument space and could overwrite it, e.g.
1002 * tailcall optimization. So, to be absolutely safe
1003 * we also save and restore enough stack bytes to cover
1004 * the argument area.
1005 */
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -08001006 memcpy(kcb->jprobes_stack, (kprobe_opcode_t *)addr,
Masami Hiramatsud6be29b2008-01-30 13:31:21 +01001007 MIN_STACK_SIZE(addr));
Glauber de Oliveira Costa053de042008-01-30 13:31:27 +01001008 regs->flags &= ~X86_EFLAGS_IF;
Peter Zijlstra58dfe882007-10-11 22:25:25 +02001009 trace_hardirqs_off();
H. Peter Anvin65ea5b02008-01-30 13:30:56 +01001010 regs->ip = (unsigned long)(jp->entry);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001011 return 1;
1012}
1013
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -07001014void __kprobes jprobe_return(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015{
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -08001016 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1017
Masami Hiramatsud6be29b2008-01-30 13:31:21 +01001018 asm volatile (
1019#ifdef CONFIG_X86_64
1020 " xchg %%rbx,%%rsp \n"
1021#else
1022 " xchgl %%ebx,%%esp \n"
1023#endif
1024 " int3 \n"
1025 " .globl jprobe_return_end\n"
1026 " jprobe_return_end: \n"
1027 " nop \n"::"b"
1028 (kcb->jprobe_saved_sp):"memory");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029}
1030
Prasanna S Panchamukhi0f2fbdc2005-09-06 15:19:28 -07001031int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001032{
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -08001033 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
H. Peter Anvin65ea5b02008-01-30 13:30:56 +01001034 u8 *addr = (u8 *) (regs->ip - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 struct jprobe *jp = container_of(p, struct jprobe, kp);
1036
Masami Hiramatsud6be29b2008-01-30 13:31:21 +01001037 if ((addr > (u8 *) jprobe_return) &&
1038 (addr < (u8 *) jprobe_return_end)) {
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +01001039 if (stack_addr(regs) != kcb->jprobe_saved_sp) {
Masami Hiramatsu29b6cd72007-12-18 18:05:58 +01001040 struct pt_regs *saved_regs = &kcb->jprobe_saved_regs;
Masami Hiramatsud6be29b2008-01-30 13:31:21 +01001041 printk(KERN_ERR
1042 "current sp %p does not match saved sp %p\n",
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +01001043 stack_addr(regs), kcb->jprobe_saved_sp);
Masami Hiramatsud6be29b2008-01-30 13:31:21 +01001044 printk(KERN_ERR "Saved registers for jprobe %p\n", jp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045 show_registers(saved_regs);
Masami Hiramatsud6be29b2008-01-30 13:31:21 +01001046 printk(KERN_ERR "Current registers\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001047 show_registers(regs);
1048 BUG();
1049 }
Ananth N Mavinakayanahallie7a510f2005-11-07 01:00:12 -08001050 *regs = kcb->jprobe_saved_regs;
Masami Hiramatsu8533bbe2008-01-30 13:31:21 +01001051 memcpy((kprobe_opcode_t *)(kcb->jprobe_saved_sp),
1052 kcb->jprobes_stack,
1053 MIN_STACK_SIZE(kcb->jprobe_saved_sp));
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -08001054 preempt_enable_no_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001055 return 1;
1056 }
1057 return 0;
1058}
Rusty Lynchba8af122005-06-27 15:17:10 -07001059
Rusty Lynch67729262005-07-05 18:54:50 -07001060int __init arch_init_kprobes(void)
Rusty Lynchba8af122005-06-27 15:17:10 -07001061{
Masami Hiramatsuda07ab02008-01-30 13:31:21 +01001062 return 0;
Rusty Lynchba8af122005-06-27 15:17:10 -07001063}
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001064
1065int __kprobes arch_trampoline_kprobe(struct kprobe *p)
1066{
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001067 return 0;
1068}