blob: 0540948e29abf0217fc7e4edded97874aa26605c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Kernel Probes (KProbes)
3 * kernel/kprobes.c
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * Copyright (C) IBM Corporation, 2002, 2004
20 *
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation (includes suggestions from
23 * Rusty Russell).
24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25 * hlists and exceptions notifier as suggested by Andi Kleen.
26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27 * interface to access function arguments.
28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29 * exceptions notifier to be first on the priority list.
Hien Nguyenb94cce92005-06-23 00:09:19 -070030 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32 * <prasanna@in.ibm.com> added function-return probes.
Linus Torvalds1da177e2005-04-16 15:20:36 -070033 */
34#include <linux/kprobes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <linux/hash.h>
36#include <linux/init.h>
Tim Schmielau4e57b682005-10-30 15:03:48 -080037#include <linux/slab.h>
Randy Dunlape3869792007-05-08 00:27:01 -070038#include <linux/stddef.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070039#include <linux/module.h>
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070040#include <linux/moduleloader.h>
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -070041#include <linux/kallsyms.h>
Masami Hiramatsub4c6c342006-12-06 20:38:11 -080042#include <linux/freezer.h>
Srinivasa Ds346fd592007-02-20 13:57:54 -080043#include <linux/seq_file.h>
44#include <linux/debugfs.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070045#include <linux/kdebug.h>
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -050046#include <linux/memory.h>
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070047
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -070048#include <asm-generic/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070049#include <asm/cacheflush.h>
50#include <asm/errno.h>
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070051#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
53#define KPROBE_HASH_BITS 6
54#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
55
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -070056
57/*
58 * Some oddball architectures like 64bit powerpc have function descriptors
59 * so this must be overridable.
60 */
61#ifndef kprobe_lookup_name
62#define kprobe_lookup_name(name, addr) \
63 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
64#endif
65
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070066static int kprobes_initialized;
Linus Torvalds1da177e2005-04-16 15:20:36 -070067static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
Hien Nguyenb94cce92005-06-23 00:09:19 -070068static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
Linus Torvalds1da177e2005-04-16 15:20:36 -070069
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070070/* NOTE: change this value only with kprobe_mutex held */
Masami Hiramatsue579abe2009-04-06 19:01:01 -070071static bool kprobes_all_disarmed;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070072
Masami Hiramatsu12941562009-01-06 14:41:50 -080073static DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -080074static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070075static struct {
Andrew Morton7e036d02008-11-12 13:26:57 -080076 spinlock_t lock ____cacheline_aligned_in_smp;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070077} kretprobe_table_locks[KPROBE_TABLE_SIZE];
78
79static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
80{
81 return &(kretprobe_table_locks[hash].lock);
82}
Linus Torvalds1da177e2005-04-16 15:20:36 -070083
Srinivasa Ds3d8d9962008-04-28 02:14:26 -070084/*
85 * Normally, functions that we'd want to prohibit kprobes in, are marked
86 * __kprobes. But, there are cases where such functions already belong to
87 * a different section (__sched for preempt_schedule)
88 *
89 * For such cases, we now have a blacklist
90 */
Daniel Guilak544304b2008-07-10 09:38:19 -070091static struct kprobe_blackpoint kprobe_blacklist[] = {
Srinivasa Ds3d8d9962008-04-28 02:14:26 -070092 {"preempt_schedule",},
93 {NULL} /* Terminator */
94};
95
Anil S Keshavamurthy2d14e392006-01-09 20:52:41 -080096#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070097/*
98 * kprobe->ainsn.insn points to the copy of the instruction to be
99 * single-stepped. x86_64, POWER4 and above have no-exec support and
100 * stepping on the instruction on a vmalloced/kmalloced/data page
101 * is a recipe for disaster
102 */
103#define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
104
105struct kprobe_insn_page {
106 struct hlist_node hlist;
107 kprobe_opcode_t *insns; /* Page of instruction slots */
108 char slot_used[INSNS_PER_PAGE];
109 int nused;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800110 int ngarbage;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700111};
112
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800113enum kprobe_slot_state {
114 SLOT_CLEAN = 0,
115 SLOT_DIRTY = 1,
116 SLOT_USED = 2,
117};
118
Masami Hiramatsu12941562009-01-06 14:41:50 -0800119static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_pages */
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700120static struct hlist_head kprobe_insn_pages;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800121static int kprobe_garbage_slots;
122static int collect_garbage_slots(void);
123
124static int __kprobes check_safety(void)
125{
126 int ret = 0;
Masami Hiramatsu5a4ccaf2009-01-06 21:15:32 +0100127#if defined(CONFIG_PREEMPT) && defined(CONFIG_FREEZER)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800128 ret = freeze_processes();
129 if (ret == 0) {
130 struct task_struct *p, *q;
131 do_each_thread(p, q) {
132 if (p != current && p->state == TASK_RUNNING &&
133 p->pid != 0) {
134 printk("Check failed: %s is running\n",p->comm);
135 ret = -1;
136 goto loop_end;
137 }
138 } while_each_thread(p, q);
139 }
140loop_end:
141 thaw_processes();
142#else
143 synchronize_sched();
144#endif
145 return ret;
146}
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700147
148/**
Masami Hiramatsu12941562009-01-06 14:41:50 -0800149 * __get_insn_slot() - Find a slot on an executable page for an instruction.
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700150 * We allocate an executable page if there's no room on existing ones.
151 */
Masami Hiramatsu12941562009-01-06 14:41:50 -0800152static kprobe_opcode_t __kprobes *__get_insn_slot(void)
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700153{
154 struct kprobe_insn_page *kip;
155 struct hlist_node *pos;
156
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700157 retry:
Christoph Hellwigb0bb5012007-05-08 00:34:11 -0700158 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700159 if (kip->nused < INSNS_PER_PAGE) {
160 int i;
161 for (i = 0; i < INSNS_PER_PAGE; i++) {
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800162 if (kip->slot_used[i] == SLOT_CLEAN) {
163 kip->slot_used[i] = SLOT_USED;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700164 kip->nused++;
165 return kip->insns + (i * MAX_INSN_SIZE);
166 }
167 }
168 /* Surprise! No unused slots. Fix kip->nused. */
169 kip->nused = INSNS_PER_PAGE;
170 }
171 }
172
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800173 /* If there are any garbage slots, collect it and try again. */
174 if (kprobe_garbage_slots && collect_garbage_slots() == 0) {
175 goto retry;
176 }
177 /* All out of space. Need to allocate a new page. Use slot 0. */
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700178 kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700179 if (!kip)
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700180 return NULL;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700181
182 /*
183 * Use module_alloc so this page is within +/- 2GB of where the
184 * kernel image and loaded module images reside. This is required
185 * so x86_64 can correctly handle the %rip-relative fixups.
186 */
187 kip->insns = module_alloc(PAGE_SIZE);
188 if (!kip->insns) {
189 kfree(kip);
190 return NULL;
191 }
192 INIT_HLIST_NODE(&kip->hlist);
193 hlist_add_head(&kip->hlist, &kprobe_insn_pages);
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800194 memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE);
195 kip->slot_used[0] = SLOT_USED;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700196 kip->nused = 1;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800197 kip->ngarbage = 0;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700198 return kip->insns;
199}
200
Masami Hiramatsu12941562009-01-06 14:41:50 -0800201kprobe_opcode_t __kprobes *get_insn_slot(void)
202{
203 kprobe_opcode_t *ret;
204 mutex_lock(&kprobe_insn_mutex);
205 ret = __get_insn_slot();
206 mutex_unlock(&kprobe_insn_mutex);
207 return ret;
208}
209
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800210/* Return 1 if all garbages are collected, otherwise 0. */
211static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
212{
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800213 kip->slot_used[idx] = SLOT_CLEAN;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800214 kip->nused--;
215 if (kip->nused == 0) {
216 /*
217 * Page is no longer in use. Free it unless
218 * it's the last one. We keep the last one
219 * so as not to have to set it up again the
220 * next time somebody inserts a probe.
221 */
222 hlist_del(&kip->hlist);
223 if (hlist_empty(&kprobe_insn_pages)) {
224 INIT_HLIST_NODE(&kip->hlist);
225 hlist_add_head(&kip->hlist,
226 &kprobe_insn_pages);
227 } else {
228 module_free(NULL, kip->insns);
229 kfree(kip);
230 }
231 return 1;
232 }
233 return 0;
234}
235
236static int __kprobes collect_garbage_slots(void)
237{
238 struct kprobe_insn_page *kip;
239 struct hlist_node *pos, *next;
240
241 /* Ensure no-one is preepmted on the garbages */
Masami Hiramatsu4a2bb6f2009-06-30 17:08:09 -0400242 if (check_safety())
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800243 return -EAGAIN;
244
Christoph Hellwigb0bb5012007-05-08 00:34:11 -0700245 hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) {
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800246 int i;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800247 if (kip->ngarbage == 0)
248 continue;
249 kip->ngarbage = 0; /* we will collect all garbages */
250 for (i = 0; i < INSNS_PER_PAGE; i++) {
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800251 if (kip->slot_used[i] == SLOT_DIRTY &&
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800252 collect_one_slot(kip, i))
253 break;
254 }
255 }
256 kprobe_garbage_slots = 0;
257 return 0;
258}
259
260void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700261{
262 struct kprobe_insn_page *kip;
263 struct hlist_node *pos;
264
Masami Hiramatsu12941562009-01-06 14:41:50 -0800265 mutex_lock(&kprobe_insn_mutex);
Christoph Hellwigb0bb5012007-05-08 00:34:11 -0700266 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700267 if (kip->insns <= slot &&
268 slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {
269 int i = (slot - kip->insns) / MAX_INSN_SIZE;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800270 if (dirty) {
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800271 kip->slot_used[i] = SLOT_DIRTY;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800272 kip->ngarbage++;
273 } else {
274 collect_one_slot(kip, i);
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700275 }
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800276 break;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700277 }
278 }
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700279
280 if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800281 collect_garbage_slots();
Masami Hiramatsu12941562009-01-06 14:41:50 -0800282
283 mutex_unlock(&kprobe_insn_mutex);
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700284}
Anil S Keshavamurthy2d14e392006-01-09 20:52:41 -0800285#endif
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700286
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800287/* We have preemption disabled.. so it is safe to use __ versions */
288static inline void set_kprobe_instance(struct kprobe *kp)
289{
290 __get_cpu_var(kprobe_instance) = kp;
291}
292
293static inline void reset_kprobe_instance(void)
294{
295 __get_cpu_var(kprobe_instance) = NULL;
296}
297
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800298/*
299 * This routine is called either:
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800300 * - under the kprobe_mutex - during kprobe_[un]register()
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800301 * OR
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800302 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800303 */
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700304struct kprobe __kprobes *get_kprobe(void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305{
306 struct hlist_head *head;
307 struct hlist_node *node;
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800308 struct kprobe *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309
310 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800311 hlist_for_each_entry_rcu(p, node, head, hlist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 if (p->addr == addr)
313 return p;
314 }
315 return NULL;
316}
317
Masami Hiramatsu201517a2009-05-07 16:31:26 -0400318/* Arm a kprobe with text_mutex */
319static void __kprobes arm_kprobe(struct kprobe *kp)
320{
321 mutex_lock(&text_mutex);
322 arch_arm_kprobe(kp);
323 mutex_unlock(&text_mutex);
324}
325
326/* Disarm a kprobe with text_mutex */
327static void __kprobes disarm_kprobe(struct kprobe *kp)
328{
329 mutex_lock(&text_mutex);
330 arch_disarm_kprobe(kp);
331 mutex_unlock(&text_mutex);
332}
333
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700334/*
335 * Aggregate handlers for multiple kprobes support - these handlers
336 * take care of invoking the individual kprobe handlers on p->list
337 */
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700338static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700339{
340 struct kprobe *kp;
341
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800342 list_for_each_entry_rcu(kp, &p->list, list) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700343 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800344 set_kprobe_instance(kp);
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700345 if (kp->pre_handler(kp, regs))
346 return 1;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700347 }
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800348 reset_kprobe_instance();
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700349 }
350 return 0;
351}
352
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700353static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
354 unsigned long flags)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700355{
356 struct kprobe *kp;
357
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800358 list_for_each_entry_rcu(kp, &p->list, list) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700359 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800360 set_kprobe_instance(kp);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700361 kp->post_handler(kp, regs, flags);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800362 reset_kprobe_instance();
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700363 }
364 }
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700365}
366
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700367static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
368 int trapnr)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700369{
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800370 struct kprobe *cur = __get_cpu_var(kprobe_instance);
371
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700372 /*
373 * if we faulted "during" the execution of a user specified
374 * probe handler, invoke just that probe's fault handler
375 */
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800376 if (cur && cur->fault_handler) {
377 if (cur->fault_handler(cur, regs, trapnr))
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700378 return 1;
379 }
380 return 0;
381}
382
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700383static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700384{
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800385 struct kprobe *cur = __get_cpu_var(kprobe_instance);
386 int ret = 0;
387
388 if (cur && cur->break_handler) {
389 if (cur->break_handler(cur, regs))
390 ret = 1;
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700391 }
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800392 reset_kprobe_instance();
393 return ret;
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700394}
395
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -0800396/* Walks the list and increments nmissed count for multiprobe case */
397void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
398{
399 struct kprobe *kp;
400 if (p->pre_handler != aggr_pre_handler) {
401 p->nmissed++;
402 } else {
403 list_for_each_entry_rcu(kp, &p->list, list)
404 kp->nmissed++;
405 }
406 return;
407}
408
bibo,mao99219a32006-10-02 02:17:35 -0700409void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
410 struct hlist_head *head)
Hien Nguyenb94cce92005-06-23 00:09:19 -0700411{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700412 struct kretprobe *rp = ri->rp;
413
Hien Nguyenb94cce92005-06-23 00:09:19 -0700414 /* remove rp inst off the rprobe_inst_table */
415 hlist_del(&ri->hlist);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700416 INIT_HLIST_NODE(&ri->hlist);
417 if (likely(rp)) {
418 spin_lock(&rp->lock);
419 hlist_add_head(&ri->hlist, &rp->free_instances);
420 spin_unlock(&rp->lock);
Hien Nguyenb94cce92005-06-23 00:09:19 -0700421 } else
422 /* Unregistering */
bibo,mao99219a32006-10-02 02:17:35 -0700423 hlist_add_head(&ri->hlist, head);
Hien Nguyenb94cce92005-06-23 00:09:19 -0700424}
425
Masami Hiramatsu017c39b2009-01-06 14:41:51 -0800426void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700427 struct hlist_head **head, unsigned long *flags)
Hien Nguyenb94cce92005-06-23 00:09:19 -0700428{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700429 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
430 spinlock_t *hlist_lock;
431
432 *head = &kretprobe_inst_table[hash];
433 hlist_lock = kretprobe_table_lock_ptr(hash);
434 spin_lock_irqsave(hlist_lock, *flags);
435}
436
Masami Hiramatsu017c39b2009-01-06 14:41:51 -0800437static void __kprobes kretprobe_table_lock(unsigned long hash,
438 unsigned long *flags)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700439{
440 spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
441 spin_lock_irqsave(hlist_lock, *flags);
442}
443
Masami Hiramatsu017c39b2009-01-06 14:41:51 -0800444void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
445 unsigned long *flags)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700446{
447 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
448 spinlock_t *hlist_lock;
449
450 hlist_lock = kretprobe_table_lock_ptr(hash);
451 spin_unlock_irqrestore(hlist_lock, *flags);
452}
453
Masami Hiramatsu017c39b2009-01-06 14:41:51 -0800454void __kprobes kretprobe_table_unlock(unsigned long hash, unsigned long *flags)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700455{
456 spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
457 spin_unlock_irqrestore(hlist_lock, *flags);
Hien Nguyenb94cce92005-06-23 00:09:19 -0700458}
459
Hien Nguyenb94cce92005-06-23 00:09:19 -0700460/*
bibo maoc6fd91f2006-03-26 01:38:20 -0800461 * This function is called from finish_task_switch when task tk becomes dead,
462 * so that we can recycle any function-return probe instances associated
463 * with this task. These left over instances represent probed functions
464 * that have been called but will never return.
Hien Nguyenb94cce92005-06-23 00:09:19 -0700465 */
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700466void __kprobes kprobe_flush_task(struct task_struct *tk)
Hien Nguyenb94cce92005-06-23 00:09:19 -0700467{
bibo,mao62c27be2006-10-02 02:17:33 -0700468 struct kretprobe_instance *ri;
bibo,mao99219a32006-10-02 02:17:35 -0700469 struct hlist_head *head, empty_rp;
Rusty Lynch802eae72005-06-27 15:17:08 -0700470 struct hlist_node *node, *tmp;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700471 unsigned long hash, flags = 0;
Rusty Lynch802eae72005-06-27 15:17:08 -0700472
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700473 if (unlikely(!kprobes_initialized))
474 /* Early boot. kretprobe_table_locks not yet initialized. */
475 return;
476
477 hash = hash_ptr(tk, KPROBE_HASH_BITS);
478 head = &kretprobe_inst_table[hash];
479 kretprobe_table_lock(hash, &flags);
bibo,mao62c27be2006-10-02 02:17:33 -0700480 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
481 if (ri->task == tk)
bibo,mao99219a32006-10-02 02:17:35 -0700482 recycle_rp_inst(ri, &empty_rp);
bibo,mao62c27be2006-10-02 02:17:33 -0700483 }
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700484 kretprobe_table_unlock(hash, &flags);
485 INIT_HLIST_HEAD(&empty_rp);
bibo,mao99219a32006-10-02 02:17:35 -0700486 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
487 hlist_del(&ri->hlist);
488 kfree(ri);
489 }
Hien Nguyenb94cce92005-06-23 00:09:19 -0700490}
491
Hien Nguyenb94cce92005-06-23 00:09:19 -0700492static inline void free_rp_inst(struct kretprobe *rp)
493{
494 struct kretprobe_instance *ri;
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700495 struct hlist_node *pos, *next;
496
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700497 hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
498 hlist_del(&ri->hlist);
Hien Nguyenb94cce92005-06-23 00:09:19 -0700499 kfree(ri);
500 }
501}
502
Masami Hiramatsu4a296e02008-04-28 02:14:29 -0700503static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
504{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700505 unsigned long flags, hash;
Masami Hiramatsu4a296e02008-04-28 02:14:29 -0700506 struct kretprobe_instance *ri;
507 struct hlist_node *pos, *next;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700508 struct hlist_head *head;
509
Masami Hiramatsu4a296e02008-04-28 02:14:29 -0700510 /* No race here */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700511 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
512 kretprobe_table_lock(hash, &flags);
513 head = &kretprobe_inst_table[hash];
514 hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
515 if (ri->rp == rp)
516 ri->rp = NULL;
517 }
518 kretprobe_table_unlock(hash, &flags);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -0700519 }
Masami Hiramatsu4a296e02008-04-28 02:14:29 -0700520 free_rp_inst(rp);
521}
522
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700523/*
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700524 * Keep all fields in the kprobe consistent
525 */
526static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
527{
528 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
529 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
530}
531
532/*
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700533* Add the new probe to ap->list. Fail if this is the
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700534* second jprobe at the address - two jprobes can't coexist
535*/
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700536static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700537{
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700538 BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700539 if (p->break_handler) {
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700540 if (ap->break_handler)
mao, bibo36721652006-06-26 00:25:22 -0700541 return -EEXIST;
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700542 list_add_tail_rcu(&p->list, &ap->list);
543 ap->break_handler = aggr_break_handler;
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700544 } else
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700545 list_add_rcu(&p->list, &ap->list);
546 if (p->post_handler && !ap->post_handler)
547 ap->post_handler = aggr_post_handler;
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700548
549 if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
550 ap->flags &= ~KPROBE_FLAG_DISABLED;
551 if (!kprobes_all_disarmed)
552 /* Arm the breakpoint again. */
Masami Hiramatsu201517a2009-05-07 16:31:26 -0400553 arm_kprobe(ap);
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700554 }
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700555 return 0;
556}
557
558/*
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700559 * Fill in the required fields of the "manager kprobe". Replace the
560 * earlier kprobe in the hlist with the manager kprobe
561 */
562static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
563{
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -0700564 copy_kprobe(p, ap);
bibo, maoa9ad9652006-07-30 03:03:26 -0700565 flush_insn_slot(ap);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700566 ap->addr = p->addr;
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700567 ap->flags = p->flags;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700568 ap->pre_handler = aggr_pre_handler;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700569 ap->fault_handler = aggr_fault_handler;
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800570 /* We don't care the kprobe which has gone. */
571 if (p->post_handler && !kprobe_gone(p))
mao, bibo36721652006-06-26 00:25:22 -0700572 ap->post_handler = aggr_post_handler;
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800573 if (p->break_handler && !kprobe_gone(p))
mao, bibo36721652006-06-26 00:25:22 -0700574 ap->break_handler = aggr_break_handler;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700575
576 INIT_LIST_HEAD(&ap->list);
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800577 list_add_rcu(&p->list, &ap->list);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700578
Keshavamurthy Anil Sadad0f32005-12-12 00:37:12 -0800579 hlist_replace_rcu(&p->hlist, &ap->hlist);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700580}
581
582/*
583 * This is the second or subsequent kprobe at the address - handle
584 * the intricacies
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700585 */
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700586static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
587 struct kprobe *p)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700588{
589 int ret = 0;
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700590 struct kprobe *ap = old_p;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700591
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700592 if (old_p->pre_handler != aggr_pre_handler) {
593 /* If old_p is not an aggr_probe, create new aggr_kprobe. */
594 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
595 if (!ap)
596 return -ENOMEM;
597 add_aggr_kprobe(ap, old_p);
598 }
599
600 if (kprobe_gone(ap)) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800601 /*
602 * Attempting to insert new probe at the same location that
603 * had a probe in the module vaddr area which already
604 * freed. So, the instruction slot has already been
605 * released. We need a new slot for the new probe.
606 */
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700607 ret = arch_prepare_kprobe(ap);
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800608 if (ret)
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700609 /*
610 * Even if fail to allocate new slot, don't need to
611 * free aggr_probe. It will be used next time, or
612 * freed by unregister_kprobe.
613 */
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800614 return ret;
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700615
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800616 /*
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700617 * Clear gone flag to prevent allocating new slot again, and
618 * set disabled flag because it is not armed yet.
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800619 */
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700620 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
621 | KPROBE_FLAG_DISABLED;
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800622 }
Masami Hiramatsub918e5e2009-04-06 19:00:58 -0700623
624 copy_kprobe(ap, p);
625 return add_new_kprobe(ap, p);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700626}
627
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700628/* Try to disable aggr_kprobe, and return 1 if succeeded.*/
629static int __kprobes try_to_disable_aggr_kprobe(struct kprobe *p)
630{
631 struct kprobe *kp;
632
633 list_for_each_entry_rcu(kp, &p->list, list) {
634 if (!kprobe_disabled(kp))
635 /*
636 * There is an active probe on the list.
637 * We can't disable aggr_kprobe.
638 */
639 return 0;
640 }
641 p->flags |= KPROBE_FLAG_DISABLED;
642 return 1;
643}
644
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700645static int __kprobes in_kprobes_functions(unsigned long addr)
646{
Srinivasa Ds3d8d9962008-04-28 02:14:26 -0700647 struct kprobe_blackpoint *kb;
648
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700649 if (addr >= (unsigned long)__kprobes_text_start &&
650 addr < (unsigned long)__kprobes_text_end)
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700651 return -EINVAL;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -0700652 /*
653 * If there exists a kprobe_blacklist, verify and
654 * fail any probe registration in the prohibited area
655 */
656 for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
657 if (kb->start_addr) {
658 if (addr >= kb->start_addr &&
659 addr < (kb->start_addr + kb->range))
660 return -EINVAL;
661 }
662 }
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700663 return 0;
664}
665
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -0800666/*
667 * If we have a symbol_name argument, look it up and add the offset field
668 * to it. This way, we can specify a relative address to a symbol.
669 */
670static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
671{
672 kprobe_opcode_t *addr = p->addr;
673 if (p->symbol_name) {
674 if (addr)
675 return NULL;
676 kprobe_lookup_name(p->symbol_name, addr);
677 }
678
679 if (!addr)
680 return NULL;
681 return (kprobe_opcode_t *)(((char *)addr) + p->offset);
682}
683
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -0800684int __kprobes register_kprobe(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685{
686 int ret = 0;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700687 struct kprobe *old_p;
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -0800688 struct module *probed_mod;
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -0800689 kprobe_opcode_t *addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -0800691 addr = kprobe_addr(p);
692 if (!addr)
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -0700693 return -EINVAL;
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -0800694 p->addr = addr;
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -0700695
Masami Hiramatsua189d032008-11-12 13:26:51 -0800696 preempt_disable();
Masami Hiramatsuec30c5f2009-07-28 19:47:23 -0400697 if (!kernel_text_address((unsigned long) p->addr) ||
Masami Hiramatsua189d032008-11-12 13:26:51 -0800698 in_kprobes_functions((unsigned long) p->addr)) {
699 preempt_enable();
Mao, Bibob3e55c72005-12-12 00:37:00 -0800700 return -EINVAL;
Masami Hiramatsua189d032008-11-12 13:26:51 -0800701 }
Mao, Bibob3e55c72005-12-12 00:37:00 -0800702
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700703 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
704 p->flags &= KPROBE_FLAG_DISABLED;
705
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700706 /*
707 * Check if are we probing a module.
708 */
Masami Hiramatsua189d032008-11-12 13:26:51 -0800709 probed_mod = __module_text_address((unsigned long) p->addr);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700710 if (probed_mod) {
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700711 /*
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800712 * We must hold a refcount of the probed module while updating
713 * its code to prohibit unexpected unloading.
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -0800714 */
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -0800715 if (unlikely(!try_module_get(probed_mod))) {
716 preempt_enable();
717 return -EINVAL;
718 }
Masami Hiramatsuf24659d2009-01-06 14:41:55 -0800719 /*
720 * If the module freed .init.text, we couldn't insert
721 * kprobes in there.
722 */
723 if (within_module_init((unsigned long)p->addr, probed_mod) &&
724 probed_mod->state != MODULE_STATE_COMING) {
725 module_put(probed_mod);
726 preempt_enable();
727 return -EINVAL;
728 }
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -0800729 }
Masami Hiramatsua189d032008-11-12 13:26:51 -0800730 preempt_enable();
Mao, Bibob3e55c72005-12-12 00:37:00 -0800731
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800732 p->nmissed = 0;
Masami Hiramatsu98616682008-04-28 02:14:28 -0700733 INIT_LIST_HEAD(&p->list);
Ingo Molnar7a7d1cf2006-03-23 03:00:35 -0800734 mutex_lock(&kprobe_mutex);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700735 old_p = get_kprobe(p->addr);
736 if (old_p) {
737 ret = register_aggr_kprobe(old_p, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738 goto out;
739 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700740
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -0500741 mutex_lock(&text_mutex);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700742 ret = arch_prepare_kprobe(p);
743 if (ret)
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -0500744 goto out_unlock_text;
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800745
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700746 INIT_HLIST_NODE(&p->hlist);
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800747 hlist_add_head_rcu(&p->hlist,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700748 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
749
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700750 if (!kprobes_all_disarmed && !kprobe_disabled(p))
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -0700751 arch_arm_kprobe(p);
Christoph Hellwig74a0b572007-10-16 01:24:07 -0700752
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -0500753out_unlock_text:
754 mutex_unlock(&text_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755out:
Ingo Molnar7a7d1cf2006-03-23 03:00:35 -0800756 mutex_unlock(&kprobe_mutex);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800757
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800758 if (probed_mod)
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -0800759 module_put(probed_mod);
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800760
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761 return ret;
762}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -0700763EXPORT_SYMBOL_GPL(register_kprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700765/* Check passed kprobe is valid and return kprobe in kprobe_table. */
766static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
767{
768 struct kprobe *old_p, *list_p;
769
770 old_p = get_kprobe(p->addr);
771 if (unlikely(!old_p))
772 return NULL;
773
774 if (p != old_p) {
775 list_for_each_entry_rcu(list_p, &old_p->list, list)
776 if (list_p == p)
777 /* kprobe p is a valid probe */
778 goto valid;
779 return NULL;
780 }
781valid:
782 return old_p;
783}
784
Masami Hiramatsu98616682008-04-28 02:14:28 -0700785/*
786 * Unregister a kprobe without a scheduler synchronization.
787 */
788static int __kprobes __unregister_kprobe_top(struct kprobe *p)
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -0800789{
Keshavamurthy Anil Sf709b122006-01-09 20:52:44 -0800790 struct kprobe *old_p, *list_p;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -0700791
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700792 old_p = __get_valid_kprobe(p);
793 if (old_p == NULL)
Masami Hiramatsu98616682008-04-28 02:14:28 -0700794 return -EINVAL;
795
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700796 if (old_p == p ||
797 (old_p->pre_handler == aggr_pre_handler &&
Masami Hiramatsu98616682008-04-28 02:14:28 -0700798 list_is_singular(&old_p->list))) {
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -0700799 /*
800 * Only probe on the hash list. Disarm only if kprobes are
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800801 * enabled and not gone - otherwise, the breakpoint would
802 * already have been removed. We save on flushing icache.
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -0700803 */
Masami Hiramatsu201517a2009-05-07 16:31:26 -0400804 if (!kprobes_all_disarmed && !kprobe_disabled(old_p))
805 disarm_kprobe(p);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800806 hlist_del_rcu(&old_p->hlist);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800807 } else {
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800808 if (p->break_handler && !kprobe_gone(p))
Masami Hiramatsu98616682008-04-28 02:14:28 -0700809 old_p->break_handler = NULL;
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800810 if (p->post_handler && !kprobe_gone(p)) {
Masami Hiramatsu98616682008-04-28 02:14:28 -0700811 list_for_each_entry_rcu(list_p, &old_p->list, list) {
812 if ((list_p != p) && (list_p->post_handler))
813 goto noclean;
814 }
815 old_p->post_handler = NULL;
816 }
817noclean:
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800818 list_del_rcu(&p->list);
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700819 if (!kprobe_disabled(old_p)) {
820 try_to_disable_aggr_kprobe(old_p);
821 if (!kprobes_all_disarmed && kprobe_disabled(old_p))
Masami Hiramatsu201517a2009-05-07 16:31:26 -0400822 disarm_kprobe(old_p);
Masami Hiramatsude5bd882009-04-06 19:01:02 -0700823 }
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800824 }
Masami Hiramatsu98616682008-04-28 02:14:28 -0700825 return 0;
826}
Mao, Bibob3e55c72005-12-12 00:37:00 -0800827
Masami Hiramatsu98616682008-04-28 02:14:28 -0700828static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
829{
Masami Hiramatsu98616682008-04-28 02:14:28 -0700830 struct kprobe *old_p;
Mao, Bibob3e55c72005-12-12 00:37:00 -0800831
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800832 if (list_empty(&p->list))
Ananth N Mavinakayanahalli0498b632006-01-09 20:52:46 -0800833 arch_remove_kprobe(p);
Masami Hiramatsue8386a02009-01-06 14:41:52 -0800834 else if (list_is_singular(&p->list)) {
835 /* "p" is the last child of an aggr_kprobe */
836 old_p = list_entry(p->list.next, struct kprobe, list);
837 list_del(&p->list);
838 arch_remove_kprobe(old_p);
839 kfree(old_p);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800840 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841}
842
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -0800843int __kprobes register_kprobes(struct kprobe **kps, int num)
Masami Hiramatsu98616682008-04-28 02:14:28 -0700844{
845 int i, ret = 0;
846
847 if (num <= 0)
848 return -EINVAL;
849 for (i = 0; i < num; i++) {
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -0800850 ret = register_kprobe(kps[i]);
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -0700851 if (ret < 0) {
852 if (i > 0)
853 unregister_kprobes(kps, i);
Masami Hiramatsu98616682008-04-28 02:14:28 -0700854 break;
855 }
856 }
857 return ret;
858}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -0700859EXPORT_SYMBOL_GPL(register_kprobes);
Masami Hiramatsu98616682008-04-28 02:14:28 -0700860
Masami Hiramatsu98616682008-04-28 02:14:28 -0700861void __kprobes unregister_kprobe(struct kprobe *p)
862{
863 unregister_kprobes(&p, 1);
864}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -0700865EXPORT_SYMBOL_GPL(unregister_kprobe);
Masami Hiramatsu98616682008-04-28 02:14:28 -0700866
Masami Hiramatsu98616682008-04-28 02:14:28 -0700867void __kprobes unregister_kprobes(struct kprobe **kps, int num)
868{
869 int i;
870
871 if (num <= 0)
872 return;
873 mutex_lock(&kprobe_mutex);
874 for (i = 0; i < num; i++)
875 if (__unregister_kprobe_top(kps[i]) < 0)
876 kps[i]->addr = NULL;
877 mutex_unlock(&kprobe_mutex);
878
879 synchronize_sched();
880 for (i = 0; i < num; i++)
881 if (kps[i]->addr)
882 __unregister_kprobe_bottom(kps[i]);
883}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -0700884EXPORT_SYMBOL_GPL(unregister_kprobes);
Masami Hiramatsu98616682008-04-28 02:14:28 -0700885
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886static struct notifier_block kprobe_exceptions_nb = {
887 .notifier_call = kprobe_exceptions_notify,
Anil S Keshavamurthy3d5631e2006-06-26 00:25:28 -0700888 .priority = 0x7fffffff /* we need to be notified first */
889};
890
Michael Ellerman3d7e3382007-07-19 01:48:11 -0700891unsigned long __weak arch_deref_entry_point(void *entry)
892{
893 return (unsigned long)entry;
894}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -0800896int __kprobes register_jprobes(struct jprobe **jps, int num)
Masami Hiramatsu26b31c12008-04-28 02:14:29 -0700897{
898 struct jprobe *jp;
899 int ret = 0, i;
900
901 if (num <= 0)
902 return -EINVAL;
903 for (i = 0; i < num; i++) {
904 unsigned long addr;
905 jp = jps[i];
906 addr = arch_deref_entry_point(jp->entry);
907
908 if (!kernel_text_address(addr))
909 ret = -EINVAL;
910 else {
911 /* Todo: Verify probepoint is a function entry point */
912 jp->kp.pre_handler = setjmp_pre_handler;
913 jp->kp.break_handler = longjmp_break_handler;
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -0800914 ret = register_kprobe(&jp->kp);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -0700915 }
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -0700916 if (ret < 0) {
917 if (i > 0)
918 unregister_jprobes(jps, i);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -0700919 break;
920 }
921 }
922 return ret;
923}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -0700924EXPORT_SYMBOL_GPL(register_jprobes);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -0700925
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700926int __kprobes register_jprobe(struct jprobe *jp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927{
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -0800928 return register_jprobes(&jp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -0700930EXPORT_SYMBOL_GPL(register_jprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -0700932void __kprobes unregister_jprobe(struct jprobe *jp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933{
Masami Hiramatsu26b31c12008-04-28 02:14:29 -0700934 unregister_jprobes(&jp, 1);
935}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -0700936EXPORT_SYMBOL_GPL(unregister_jprobe);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -0700937
Masami Hiramatsu26b31c12008-04-28 02:14:29 -0700938void __kprobes unregister_jprobes(struct jprobe **jps, int num)
939{
940 int i;
941
942 if (num <= 0)
943 return;
944 mutex_lock(&kprobe_mutex);
945 for (i = 0; i < num; i++)
946 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
947 jps[i]->kp.addr = NULL;
948 mutex_unlock(&kprobe_mutex);
949
950 synchronize_sched();
951 for (i = 0; i < num; i++) {
952 if (jps[i]->kp.addr)
953 __unregister_kprobe_bottom(&jps[i]->kp);
954 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -0700956EXPORT_SYMBOL_GPL(unregister_jprobes);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -0800958#ifdef CONFIG_KRETPROBES
Adrian Bunke65cefe2006-02-03 03:03:42 -0800959/*
960 * This kprobe pre_handler is registered with every kretprobe. When probe
961 * hits it will set up the return probe.
962 */
963static int __kprobes pre_handler_kretprobe(struct kprobe *p,
964 struct pt_regs *regs)
965{
966 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700967 unsigned long hash, flags = 0;
968 struct kretprobe_instance *ri;
Adrian Bunke65cefe2006-02-03 03:03:42 -0800969
970 /*TODO: consider to only swap the RA after the last pre_handler fired */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700971 hash = hash_ptr(current, KPROBE_HASH_BITS);
972 spin_lock_irqsave(&rp->lock, flags);
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700973 if (!hlist_empty(&rp->free_instances)) {
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700974 ri = hlist_entry(rp->free_instances.first,
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700975 struct kretprobe_instance, hlist);
976 hlist_del(&ri->hlist);
977 spin_unlock_irqrestore(&rp->lock, flags);
978
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700979 ri->rp = rp;
980 ri->task = current;
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -0800981
Ananth N Mavinakayanahallif02b8622009-03-18 17:06:21 +0530982 if (rp->entry_handler && rp->entry_handler(ri, regs))
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -0800983 return 0;
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -0800984
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700985 arch_prepare_kretprobe(ri, regs);
986
987 /* XXX(hch): why is there no hlist_move_head? */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700988 INIT_HLIST_NODE(&ri->hlist);
989 kretprobe_table_lock(hash, &flags);
990 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
991 kretprobe_table_unlock(hash, &flags);
992 } else {
Christoph Hellwig4c4308c2007-05-08 00:34:14 -0700993 rp->nmissed++;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -0700994 spin_unlock_irqrestore(&rp->lock, flags);
995 }
Adrian Bunke65cefe2006-02-03 03:03:42 -0800996 return 0;
997}
998
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -0800999int __kprobes register_kretprobe(struct kretprobe *rp)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001000{
1001 int ret = 0;
1002 struct kretprobe_instance *inst;
1003 int i;
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001004 void *addr;
Masami Hiramatsuf438d912007-10-16 01:27:49 -07001005
1006 if (kretprobe_blacklist_size) {
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001007 addr = kprobe_addr(&rp->kp);
1008 if (!addr)
1009 return -EINVAL;
Masami Hiramatsuf438d912007-10-16 01:27:49 -07001010
1011 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1012 if (kretprobe_blacklist[i].addr == addr)
1013 return -EINVAL;
1014 }
1015 }
Hien Nguyenb94cce92005-06-23 00:09:19 -07001016
1017 rp->kp.pre_handler = pre_handler_kretprobe;
Ananth N Mavinakayanahalli7522a842006-04-20 02:43:11 -07001018 rp->kp.post_handler = NULL;
1019 rp->kp.fault_handler = NULL;
1020 rp->kp.break_handler = NULL;
Hien Nguyenb94cce92005-06-23 00:09:19 -07001021
1022 /* Pre-allocate memory for max kretprobe instances */
1023 if (rp->maxactive <= 0) {
1024#ifdef CONFIG_PREEMPT
1025 rp->maxactive = max(10, 2 * NR_CPUS);
1026#else
1027 rp->maxactive = NR_CPUS;
1028#endif
1029 }
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001030 spin_lock_init(&rp->lock);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001031 INIT_HLIST_HEAD(&rp->free_instances);
1032 for (i = 0; i < rp->maxactive; i++) {
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001033 inst = kmalloc(sizeof(struct kretprobe_instance) +
1034 rp->data_size, GFP_KERNEL);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001035 if (inst == NULL) {
1036 free_rp_inst(rp);
1037 return -ENOMEM;
1038 }
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001039 INIT_HLIST_NODE(&inst->hlist);
1040 hlist_add_head(&inst->hlist, &rp->free_instances);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001041 }
1042
1043 rp->nmissed = 0;
1044 /* Establish function entry probe point */
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001045 ret = register_kprobe(&rp->kp);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001046 if (ret != 0)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001047 free_rp_inst(rp);
1048 return ret;
1049}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001050EXPORT_SYMBOL_GPL(register_kretprobe);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001051
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001052int __kprobes register_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001053{
1054 int ret = 0, i;
1055
1056 if (num <= 0)
1057 return -EINVAL;
1058 for (i = 0; i < num; i++) {
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001059 ret = register_kretprobe(rps[i]);
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -07001060 if (ret < 0) {
1061 if (i > 0)
1062 unregister_kretprobes(rps, i);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001063 break;
1064 }
1065 }
1066 return ret;
1067}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001068EXPORT_SYMBOL_GPL(register_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001069
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001070void __kprobes unregister_kretprobe(struct kretprobe *rp)
1071{
1072 unregister_kretprobes(&rp, 1);
1073}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001074EXPORT_SYMBOL_GPL(unregister_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001075
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001076void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1077{
1078 int i;
1079
1080 if (num <= 0)
1081 return;
1082 mutex_lock(&kprobe_mutex);
1083 for (i = 0; i < num; i++)
1084 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1085 rps[i]->kp.addr = NULL;
1086 mutex_unlock(&kprobe_mutex);
1087
1088 synchronize_sched();
1089 for (i = 0; i < num; i++) {
1090 if (rps[i]->kp.addr) {
1091 __unregister_kprobe_bottom(&rps[i]->kp);
1092 cleanup_rp_inst(rps[i]);
1093 }
1094 }
1095}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001096EXPORT_SYMBOL_GPL(unregister_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001097
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08001098#else /* CONFIG_KRETPROBES */
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001099int __kprobes register_kretprobe(struct kretprobe *rp)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001100{
1101 return -ENOSYS;
1102}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001103EXPORT_SYMBOL_GPL(register_kretprobe);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001104
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001105int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1106{
1107 return -ENOSYS;
1108}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001109EXPORT_SYMBOL_GPL(register_kretprobes);
1110
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001111void __kprobes unregister_kretprobe(struct kretprobe *rp)
1112{
1113}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001114EXPORT_SYMBOL_GPL(unregister_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001115
1116void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1117{
1118}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001119EXPORT_SYMBOL_GPL(unregister_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001120
Srinivasa Ds346fd592007-02-20 13:57:54 -08001121static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1122 struct pt_regs *regs)
1123{
1124 return 0;
1125}
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001126
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08001127#endif /* CONFIG_KRETPROBES */
Hien Nguyenb94cce92005-06-23 00:09:19 -07001128
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001129/* Set the kprobe gone and remove its instruction buffer. */
1130static void __kprobes kill_kprobe(struct kprobe *p)
1131{
1132 struct kprobe *kp;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001133
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001134 p->flags |= KPROBE_FLAG_GONE;
1135 if (p->pre_handler == aggr_pre_handler) {
1136 /*
1137 * If this is an aggr_kprobe, we have to list all the
1138 * chained probes and mark them GONE.
1139 */
1140 list_for_each_entry_rcu(kp, &p->list, list)
1141 kp->flags |= KPROBE_FLAG_GONE;
1142 p->post_handler = NULL;
1143 p->break_handler = NULL;
1144 }
1145 /*
1146 * Here, we can remove insn_slot safely, because no thread calls
1147 * the original probed function (which will be freed soon) any more.
1148 */
1149 arch_remove_kprobe(p);
1150}
1151
1152/* Module notifier call back, checking kprobes on the module */
1153static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1154 unsigned long val, void *data)
1155{
1156 struct module *mod = data;
1157 struct hlist_head *head;
1158 struct hlist_node *node;
1159 struct kprobe *p;
1160 unsigned int i;
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08001161 int checkcore = (val == MODULE_STATE_GOING);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001162
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08001163 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001164 return NOTIFY_DONE;
1165
1166 /*
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08001167 * When MODULE_STATE_GOING was notified, both of module .text and
1168 * .init.text sections would be freed. When MODULE_STATE_LIVE was
1169 * notified, only .init.text section would be freed. We need to
1170 * disable kprobes which have been inserted in the sections.
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001171 */
1172 mutex_lock(&kprobe_mutex);
1173 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1174 head = &kprobe_table[i];
1175 hlist_for_each_entry_rcu(p, node, head, hlist)
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08001176 if (within_module_init((unsigned long)p->addr, mod) ||
1177 (checkcore &&
1178 within_module_core((unsigned long)p->addr, mod))) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001179 /*
1180 * The vaddr this probe is installed will soon
1181 * be vfreed buy not synced to disk. Hence,
1182 * disarming the breakpoint isn't needed.
1183 */
1184 kill_kprobe(p);
1185 }
1186 }
1187 mutex_unlock(&kprobe_mutex);
1188 return NOTIFY_DONE;
1189}
1190
1191static struct notifier_block kprobe_module_nb = {
1192 .notifier_call = kprobes_module_callback,
1193 .priority = 0
1194};
1195
Linus Torvalds1da177e2005-04-16 15:20:36 -07001196static int __init init_kprobes(void)
1197{
1198 int i, err = 0;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001199 unsigned long offset = 0, size = 0;
1200 char *modname, namebuf[128];
1201 const char *symbol_name;
1202 void *addr;
1203 struct kprobe_blackpoint *kb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204
1205 /* FIXME allocate the probe table, currently defined statically */
1206 /* initialize all list heads */
Hien Nguyenb94cce92005-06-23 00:09:19 -07001207 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001208 INIT_HLIST_HEAD(&kprobe_table[i]);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001209 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001210 spin_lock_init(&(kretprobe_table_locks[i].lock));
Hien Nguyenb94cce92005-06-23 00:09:19 -07001211 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001213 /*
1214 * Lookup and populate the kprobe_blacklist.
1215 *
1216 * Unlike the kretprobe blacklist, we'll need to determine
1217 * the range of addresses that belong to the said functions,
1218 * since a kprobe need not necessarily be at the beginning
1219 * of a function.
1220 */
1221 for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1222 kprobe_lookup_name(kb->name, addr);
1223 if (!addr)
1224 continue;
1225
1226 kb->start_addr = (unsigned long)addr;
1227 symbol_name = kallsyms_lookup(kb->start_addr,
1228 &size, &offset, &modname, namebuf);
1229 if (!symbol_name)
1230 kb->range = 0;
1231 else
1232 kb->range = size;
1233 }
1234
Masami Hiramatsuf438d912007-10-16 01:27:49 -07001235 if (kretprobe_blacklist_size) {
1236 /* lookup the function address from its name */
1237 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1238 kprobe_lookup_name(kretprobe_blacklist[i].name,
1239 kretprobe_blacklist[i].addr);
1240 if (!kretprobe_blacklist[i].addr)
1241 printk("kretprobe: lookup failed: %s\n",
1242 kretprobe_blacklist[i].name);
1243 }
1244 }
1245
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001246 /* By default, kprobes are armed */
1247 kprobes_all_disarmed = false;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001248
Rusty Lynch67729262005-07-05 18:54:50 -07001249 err = arch_init_kprobes();
Rusty Lynch802eae72005-06-27 15:17:08 -07001250 if (!err)
1251 err = register_die_notifier(&kprobe_exceptions_nb);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001252 if (!err)
1253 err = register_module_notifier(&kprobe_module_nb);
1254
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001255 kprobes_initialized = (err == 0);
Rusty Lynch802eae72005-06-27 15:17:08 -07001256
Ananth N Mavinakayanahalli8c1c9352008-01-30 13:32:53 +01001257 if (!err)
1258 init_test_probes();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259 return err;
1260}
1261
Srinivasa Ds346fd592007-02-20 13:57:54 -08001262#ifdef CONFIG_DEBUG_FS
1263static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001264 const char *sym, int offset,char *modname)
Srinivasa Ds346fd592007-02-20 13:57:54 -08001265{
1266 char *kprobe_type;
1267
1268 if (p->pre_handler == pre_handler_kretprobe)
1269 kprobe_type = "r";
1270 else if (p->pre_handler == setjmp_pre_handler)
1271 kprobe_type = "j";
1272 else
1273 kprobe_type = "k";
1274 if (sym)
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001275 seq_printf(pi, "%p %s %s+0x%x %s %s%s\n",
1276 p->addr, kprobe_type, sym, offset,
1277 (modname ? modname : " "),
1278 (kprobe_gone(p) ? "[GONE]" : ""),
1279 ((kprobe_disabled(p) && !kprobe_gone(p)) ?
1280 "[DISABLED]" : ""));
Srinivasa Ds346fd592007-02-20 13:57:54 -08001281 else
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001282 seq_printf(pi, "%p %s %p %s%s\n",
1283 p->addr, kprobe_type, p->addr,
1284 (kprobe_gone(p) ? "[GONE]" : ""),
1285 ((kprobe_disabled(p) && !kprobe_gone(p)) ?
1286 "[DISABLED]" : ""));
Srinivasa Ds346fd592007-02-20 13:57:54 -08001287}
1288
1289static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
1290{
1291 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
1292}
1293
1294static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
1295{
1296 (*pos)++;
1297 if (*pos >= KPROBE_TABLE_SIZE)
1298 return NULL;
1299 return pos;
1300}
1301
1302static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
1303{
1304 /* Nothing to do */
1305}
1306
1307static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
1308{
1309 struct hlist_head *head;
1310 struct hlist_node *node;
1311 struct kprobe *p, *kp;
1312 const char *sym = NULL;
1313 unsigned int i = *(loff_t *) v;
Alexey Dobriyanffb45122007-05-08 00:28:41 -07001314 unsigned long offset = 0;
Srinivasa Ds346fd592007-02-20 13:57:54 -08001315 char *modname, namebuf[128];
1316
1317 head = &kprobe_table[i];
1318 preempt_disable();
1319 hlist_for_each_entry_rcu(p, node, head, hlist) {
Alexey Dobriyanffb45122007-05-08 00:28:41 -07001320 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
Srinivasa Ds346fd592007-02-20 13:57:54 -08001321 &offset, &modname, namebuf);
1322 if (p->pre_handler == aggr_pre_handler) {
1323 list_for_each_entry_rcu(kp, &p->list, list)
1324 report_probe(pi, kp, sym, offset, modname);
1325 } else
1326 report_probe(pi, p, sym, offset, modname);
1327 }
1328 preempt_enable();
1329 return 0;
1330}
1331
1332static struct seq_operations kprobes_seq_ops = {
1333 .start = kprobe_seq_start,
1334 .next = kprobe_seq_next,
1335 .stop = kprobe_seq_stop,
1336 .show = show_kprobe_addr
1337};
1338
1339static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
1340{
1341 return seq_open(filp, &kprobes_seq_ops);
1342}
1343
1344static struct file_operations debugfs_kprobes_operations = {
1345 .open = kprobes_open,
1346 .read = seq_read,
1347 .llseek = seq_lseek,
1348 .release = seq_release,
1349};
1350
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001351/* Disable one kprobe */
1352int __kprobes disable_kprobe(struct kprobe *kp)
1353{
1354 int ret = 0;
1355 struct kprobe *p;
1356
1357 mutex_lock(&kprobe_mutex);
1358
1359 /* Check whether specified probe is valid. */
1360 p = __get_valid_kprobe(kp);
1361 if (unlikely(p == NULL)) {
1362 ret = -EINVAL;
1363 goto out;
1364 }
1365
1366 /* If the probe is already disabled (or gone), just return */
1367 if (kprobe_disabled(kp))
1368 goto out;
1369
1370 kp->flags |= KPROBE_FLAG_DISABLED;
1371 if (p != kp)
1372 /* When kp != p, p is always enabled. */
1373 try_to_disable_aggr_kprobe(p);
1374
1375 if (!kprobes_all_disarmed && kprobe_disabled(p))
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001376 disarm_kprobe(p);
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001377out:
1378 mutex_unlock(&kprobe_mutex);
1379 return ret;
1380}
1381EXPORT_SYMBOL_GPL(disable_kprobe);
1382
1383/* Enable one kprobe */
1384int __kprobes enable_kprobe(struct kprobe *kp)
1385{
1386 int ret = 0;
1387 struct kprobe *p;
1388
1389 mutex_lock(&kprobe_mutex);
1390
1391 /* Check whether specified probe is valid. */
1392 p = __get_valid_kprobe(kp);
1393 if (unlikely(p == NULL)) {
1394 ret = -EINVAL;
1395 goto out;
1396 }
1397
1398 if (kprobe_gone(kp)) {
1399 /* This kprobe has gone, we couldn't enable it. */
1400 ret = -EINVAL;
1401 goto out;
1402 }
1403
1404 if (!kprobes_all_disarmed && kprobe_disabled(p))
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001405 arm_kprobe(p);
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001406
1407 p->flags &= ~KPROBE_FLAG_DISABLED;
1408 if (p != kp)
1409 kp->flags &= ~KPROBE_FLAG_DISABLED;
1410out:
1411 mutex_unlock(&kprobe_mutex);
1412 return ret;
1413}
1414EXPORT_SYMBOL_GPL(enable_kprobe);
1415
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001416static void __kprobes arm_all_kprobes(void)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001417{
1418 struct hlist_head *head;
1419 struct hlist_node *node;
1420 struct kprobe *p;
1421 unsigned int i;
1422
1423 mutex_lock(&kprobe_mutex);
1424
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001425 /* If kprobes are armed, just return */
1426 if (!kprobes_all_disarmed)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001427 goto already_enabled;
1428
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -05001429 mutex_lock(&text_mutex);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001430 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1431 head = &kprobe_table[i];
1432 hlist_for_each_entry_rcu(p, node, head, hlist)
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001433 if (!kprobe_disabled(p))
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001434 arch_arm_kprobe(p);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001435 }
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -05001436 mutex_unlock(&text_mutex);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001437
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001438 kprobes_all_disarmed = false;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001439 printk(KERN_INFO "Kprobes globally enabled\n");
1440
1441already_enabled:
1442 mutex_unlock(&kprobe_mutex);
1443 return;
1444}
1445
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001446static void __kprobes disarm_all_kprobes(void)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001447{
1448 struct hlist_head *head;
1449 struct hlist_node *node;
1450 struct kprobe *p;
1451 unsigned int i;
1452
1453 mutex_lock(&kprobe_mutex);
1454
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001455 /* If kprobes are already disarmed, just return */
1456 if (kprobes_all_disarmed)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001457 goto already_disabled;
1458
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001459 kprobes_all_disarmed = true;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001460 printk(KERN_INFO "Kprobes globally disabled\n");
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -05001461 mutex_lock(&text_mutex);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001462 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1463 head = &kprobe_table[i];
1464 hlist_for_each_entry_rcu(p, node, head, hlist) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001465 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001466 arch_disarm_kprobe(p);
1467 }
1468 }
1469
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -05001470 mutex_unlock(&text_mutex);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001471 mutex_unlock(&kprobe_mutex);
1472 /* Allow all currently running kprobes to complete */
1473 synchronize_sched();
Christoph Hellwig74a0b572007-10-16 01:24:07 -07001474 return;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001475
1476already_disabled:
1477 mutex_unlock(&kprobe_mutex);
1478 return;
1479}
1480
1481/*
1482 * XXX: The debugfs bool file interface doesn't allow for callbacks
1483 * when the bool state is switched. We can reuse that facility when
1484 * available
1485 */
1486static ssize_t read_enabled_file_bool(struct file *file,
1487 char __user *user_buf, size_t count, loff_t *ppos)
1488{
1489 char buf[3];
1490
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001491 if (!kprobes_all_disarmed)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001492 buf[0] = '1';
1493 else
1494 buf[0] = '0';
1495 buf[1] = '\n';
1496 buf[2] = 0x00;
1497 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1498}
1499
1500static ssize_t write_enabled_file_bool(struct file *file,
1501 const char __user *user_buf, size_t count, loff_t *ppos)
1502{
1503 char buf[32];
1504 int buf_size;
1505
1506 buf_size = min(count, (sizeof(buf)-1));
1507 if (copy_from_user(buf, user_buf, buf_size))
1508 return -EFAULT;
1509
1510 switch (buf[0]) {
1511 case 'y':
1512 case 'Y':
1513 case '1':
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001514 arm_all_kprobes();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001515 break;
1516 case 'n':
1517 case 'N':
1518 case '0':
Masami Hiramatsue579abe2009-04-06 19:01:01 -07001519 disarm_all_kprobes();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001520 break;
1521 }
1522
1523 return count;
1524}
1525
1526static struct file_operations fops_kp = {
1527 .read = read_enabled_file_bool,
1528 .write = write_enabled_file_bool,
1529};
1530
Srinivasa Ds346fd592007-02-20 13:57:54 -08001531static int __kprobes debugfs_kprobe_init(void)
1532{
1533 struct dentry *dir, *file;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001534 unsigned int value = 1;
Srinivasa Ds346fd592007-02-20 13:57:54 -08001535
1536 dir = debugfs_create_dir("kprobes", NULL);
1537 if (!dir)
1538 return -ENOMEM;
1539
Randy Dunlape3869792007-05-08 00:27:01 -07001540 file = debugfs_create_file("list", 0444, dir, NULL,
Srinivasa Ds346fd592007-02-20 13:57:54 -08001541 &debugfs_kprobes_operations);
1542 if (!file) {
1543 debugfs_remove(dir);
1544 return -ENOMEM;
1545 }
1546
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001547 file = debugfs_create_file("enabled", 0600, dir,
1548 &value, &fops_kp);
1549 if (!file) {
1550 debugfs_remove(dir);
1551 return -ENOMEM;
1552 }
1553
Srinivasa Ds346fd592007-02-20 13:57:54 -08001554 return 0;
1555}
1556
1557late_initcall(debugfs_kprobe_init);
1558#endif /* CONFIG_DEBUG_FS */
1559
1560module_init(init_kprobes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001561
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001562/* defined in arch/.../kernel/kprobes.c */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001563EXPORT_SYMBOL_GPL(jprobe_return);