blob: 2d2d3a568e4e8b3aeedb16711c90e660f698d6cf [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Kernel Probes (KProbes)
3 * kernel/kprobes.c
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * Copyright (C) IBM Corporation, 2002, 2004
20 *
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation (includes suggestions from
23 * Rusty Russell).
24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25 * hlists and exceptions notifier as suggested by Andi Kleen.
26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27 * interface to access function arguments.
28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29 * exceptions notifier to be first on the priority list.
Hien Nguyenb94cce92005-06-23 00:09:19 -070030 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32 * <prasanna@in.ibm.com> added function-return probes.
Linus Torvalds1da177e2005-04-16 15:20:36 -070033 */
34#include <linux/kprobes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <linux/hash.h>
36#include <linux/init.h>
Tim Schmielau4e57b682005-10-30 15:03:48 -080037#include <linux/slab.h>
Randy Dunlape3869792007-05-08 00:27:01 -070038#include <linux/stddef.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040039#include <linux/export.h>
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070040#include <linux/moduleloader.h>
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -070041#include <linux/kallsyms.h>
Masami Hiramatsub4c6c342006-12-06 20:38:11 -080042#include <linux/freezer.h>
Srinivasa Ds346fd592007-02-20 13:57:54 -080043#include <linux/seq_file.h>
44#include <linux/debugfs.h>
Masami Hiramatsub2be84d2010-02-25 08:34:15 -050045#include <linux/sysctl.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070046#include <linux/kdebug.h>
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -050047#include <linux/memory.h>
Masami Hiramatsu4554dbc2010-02-02 16:49:18 -050048#include <linux/ftrace.h>
Masami Hiramatsuafd66252010-02-25 08:34:07 -050049#include <linux/cpu.h>
Jason Baronbf5438fc2010-09-17 11:09:00 -040050#include <linux/jump_label.h>
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070051
Christoph Hellwigbfd45be2016-10-11 13:52:22 -070052#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070053#include <asm/cacheflush.h>
54#include <asm/errno.h>
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080055#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
57#define KPROBE_HASH_BITS 6
58#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
59
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -070060
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070061static int kprobes_initialized;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
Hien Nguyenb94cce92005-06-23 00:09:19 -070063static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
Linus Torvalds1da177e2005-04-16 15:20:36 -070064
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070065/* NOTE: change this value only with kprobe_mutex held */
Masami Hiramatsue579abe2009-04-06 19:01:01 -070066static bool kprobes_all_disarmed;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070067
Masami Hiramatsu43948f52010-10-25 22:18:01 +090068/* This protects kprobe_table and optimizing_list */
69static DEFINE_MUTEX(kprobe_mutex);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -080070static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070071static struct {
Thomas Gleixnerec4846082009-07-25 16:09:17 +020072 raw_spinlock_t lock ____cacheline_aligned_in_smp;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070073} kretprobe_table_locks[KPROBE_TABLE_SIZE];
74
Naveen N. Rao290e3072017-04-19 18:21:01 +053075kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
76 unsigned int __unused)
Naveen N. Rao49e0b462017-04-19 18:21:00 +053077{
78 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
79}
80
Thomas Gleixnerec4846082009-07-25 16:09:17 +020081static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070082{
83 return &(kretprobe_table_locks[hash].lock);
84}
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
Masami Hiramatsu376e2422014-04-17 17:17:05 +090086/* Blacklist -- list of struct kprobe_blacklist_entry */
87static LIST_HEAD(kprobe_blacklist);
Srinivasa Ds3d8d9962008-04-28 02:14:26 -070088
Anil S Keshavamurthy2d14e392006-01-09 20:52:41 -080089#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070090/*
91 * kprobe->ainsn.insn points to the copy of the instruction to be
92 * single-stepped. x86_64, POWER4 and above have no-exec support and
93 * stepping on the instruction on a vmalloced/kmalloced/data page
94 * is a recipe for disaster
95 */
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070096struct kprobe_insn_page {
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -040097 struct list_head list;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070098 kprobe_opcode_t *insns; /* Page of instruction slots */
Heiko Carstensaf963972013-09-11 14:24:13 -070099 struct kprobe_insn_cache *cache;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700100 int nused;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800101 int ngarbage;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500102 char slot_used[];
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700103};
104
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500105#define KPROBE_INSN_PAGE_SIZE(slots) \
106 (offsetof(struct kprobe_insn_page, slot_used) + \
107 (sizeof(char) * (slots)))
108
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500109static int slots_per_page(struct kprobe_insn_cache *c)
110{
111 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
112}
113
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800114enum kprobe_slot_state {
115 SLOT_CLEAN = 0,
116 SLOT_DIRTY = 1,
117 SLOT_USED = 2,
118};
119
Heiko Carstensaf963972013-09-11 14:24:13 -0700120static void *alloc_insn_page(void)
121{
122 return module_alloc(PAGE_SIZE);
123}
124
125static void free_insn_page(void *page)
126{
Rusty Russellbe1f2212015-01-20 09:07:05 +1030127 module_memfree(page);
Heiko Carstensaf963972013-09-11 14:24:13 -0700128}
129
Heiko Carstensc802d642013-09-11 14:24:11 -0700130struct kprobe_insn_cache kprobe_insn_slots = {
131 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
Heiko Carstensaf963972013-09-11 14:24:13 -0700132 .alloc = alloc_insn_page,
133 .free = free_insn_page,
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500134 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
135 .insn_size = MAX_INSN_SIZE,
136 .nr_garbage = 0,
137};
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900138static int collect_garbage_slots(struct kprobe_insn_cache *c);
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800139
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700140/**
Masami Hiramatsu12941562009-01-06 14:41:50 -0800141 * __get_insn_slot() - Find a slot on an executable page for an instruction.
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700142 * We allocate an executable page if there's no room on existing ones.
143 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900144kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700145{
146 struct kprobe_insn_page *kip;
Heiko Carstensc802d642013-09-11 14:24:11 -0700147 kprobe_opcode_t *slot = NULL;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700148
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900149 /* Since the slot array is not protected by rcu, we need a mutex */
Heiko Carstensc802d642013-09-11 14:24:11 -0700150 mutex_lock(&c->mutex);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700151 retry:
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900152 rcu_read_lock();
153 list_for_each_entry_rcu(kip, &c->pages, list) {
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500154 if (kip->nused < slots_per_page(c)) {
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700155 int i;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500156 for (i = 0; i < slots_per_page(c); i++) {
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800157 if (kip->slot_used[i] == SLOT_CLEAN) {
158 kip->slot_used[i] = SLOT_USED;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700159 kip->nused++;
Heiko Carstensc802d642013-09-11 14:24:11 -0700160 slot = kip->insns + (i * c->insn_size);
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900161 rcu_read_unlock();
Heiko Carstensc802d642013-09-11 14:24:11 -0700162 goto out;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700163 }
164 }
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500165 /* kip->nused is broken. Fix it. */
166 kip->nused = slots_per_page(c);
167 WARN_ON(1);
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700168 }
169 }
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900170 rcu_read_unlock();
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700171
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800172 /* If there are any garbage slots, collect it and try again. */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500173 if (c->nr_garbage && collect_garbage_slots(c) == 0)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800174 goto retry;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500175
176 /* All out of space. Need to allocate a new page. */
177 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700178 if (!kip)
Heiko Carstensc802d642013-09-11 14:24:11 -0700179 goto out;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700180
181 /*
182 * Use module_alloc so this page is within +/- 2GB of where the
183 * kernel image and loaded module images reside. This is required
184 * so x86_64 can correctly handle the %rip-relative fixups.
185 */
Heiko Carstensaf963972013-09-11 14:24:13 -0700186 kip->insns = c->alloc();
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700187 if (!kip->insns) {
188 kfree(kip);
Heiko Carstensc802d642013-09-11 14:24:11 -0700189 goto out;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700190 }
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -0400191 INIT_LIST_HEAD(&kip->list);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500192 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800193 kip->slot_used[0] = SLOT_USED;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700194 kip->nused = 1;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800195 kip->ngarbage = 0;
Heiko Carstensaf963972013-09-11 14:24:13 -0700196 kip->cache = c;
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900197 list_add_rcu(&kip->list, &c->pages);
Heiko Carstensc802d642013-09-11 14:24:11 -0700198 slot = kip->insns;
199out:
200 mutex_unlock(&c->mutex);
201 return slot;
Masami Hiramatsu12941562009-01-06 14:41:50 -0800202}
203
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800204/* Return 1 if all garbages are collected, otherwise 0. */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900205static int collect_one_slot(struct kprobe_insn_page *kip, int idx)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800206{
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800207 kip->slot_used[idx] = SLOT_CLEAN;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800208 kip->nused--;
209 if (kip->nused == 0) {
210 /*
211 * Page is no longer in use. Free it unless
212 * it's the last one. We keep the last one
213 * so as not to have to set it up again the
214 * next time somebody inserts a probe.
215 */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500216 if (!list_is_singular(&kip->list)) {
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900217 list_del_rcu(&kip->list);
218 synchronize_rcu();
Heiko Carstensaf963972013-09-11 14:24:13 -0700219 kip->cache->free(kip->insns);
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800220 kfree(kip);
221 }
222 return 1;
223 }
224 return 0;
225}
226
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900227static int collect_garbage_slots(struct kprobe_insn_cache *c)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800228{
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -0400229 struct kprobe_insn_page *kip, *next;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800230
Masami Hiramatsu615d0eb2010-02-02 16:49:04 -0500231 /* Ensure no-one is interrupted on the garbages */
232 synchronize_sched();
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800233
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500234 list_for_each_entry_safe(kip, next, &c->pages, list) {
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800235 int i;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800236 if (kip->ngarbage == 0)
237 continue;
238 kip->ngarbage = 0; /* we will collect all garbages */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500239 for (i = 0; i < slots_per_page(c); i++) {
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900240 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800241 break;
242 }
243 }
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500244 c->nr_garbage = 0;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800245 return 0;
246}
247
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900248void __free_insn_slot(struct kprobe_insn_cache *c,
249 kprobe_opcode_t *slot, int dirty)
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500250{
251 struct kprobe_insn_page *kip;
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900252 long idx;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500253
Heiko Carstensc802d642013-09-11 14:24:11 -0700254 mutex_lock(&c->mutex);
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900255 rcu_read_lock();
256 list_for_each_entry_rcu(kip, &c->pages, list) {
257 idx = ((long)slot - (long)kip->insns) /
258 (c->insn_size * sizeof(kprobe_opcode_t));
259 if (idx >= 0 && idx < slots_per_page(c))
Heiko Carstensc802d642013-09-11 14:24:11 -0700260 goto out;
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900261 }
262 /* Could not find this slot. */
263 WARN_ON(1);
264 kip = NULL;
265out:
266 rcu_read_unlock();
267 /* Mark and sweep: this may sleep */
268 if (kip) {
269 /* Check double free */
270 WARN_ON(kip->slot_used[idx] != SLOT_USED);
271 if (dirty) {
272 kip->slot_used[idx] = SLOT_DIRTY;
273 kip->ngarbage++;
274 if (++c->nr_garbage > slots_per_page(c))
275 collect_garbage_slots(c);
276 } else {
277 collect_one_slot(kip, idx);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500278 }
279 }
Heiko Carstensc802d642013-09-11 14:24:11 -0700280 mutex_unlock(&c->mutex);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500281}
282
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900283/*
284 * Check given address is on the page of kprobe instruction slots.
285 * This will be used for checking whether the address on a stack
286 * is on a text area or not.
287 */
288bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
289{
290 struct kprobe_insn_page *kip;
291 bool ret = false;
292
293 rcu_read_lock();
294 list_for_each_entry_rcu(kip, &c->pages, list) {
295 if (addr >= (unsigned long)kip->insns &&
296 addr < (unsigned long)kip->insns + PAGE_SIZE) {
297 ret = true;
298 break;
299 }
300 }
301 rcu_read_unlock();
302
303 return ret;
304}
305
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500306#ifdef CONFIG_OPTPROBES
307/* For optimized_kprobe buffer */
Heiko Carstensc802d642013-09-11 14:24:11 -0700308struct kprobe_insn_cache kprobe_optinsn_slots = {
309 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
Heiko Carstensaf963972013-09-11 14:24:13 -0700310 .alloc = alloc_insn_page,
311 .free = free_insn_page,
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500312 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
313 /* .insn_size is initialized later */
314 .nr_garbage = 0,
315};
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500316#endif
Anil S Keshavamurthy2d14e392006-01-09 20:52:41 -0800317#endif
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700318
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800319/* We have preemption disabled.. so it is safe to use __ versions */
320static inline void set_kprobe_instance(struct kprobe *kp)
321{
Christoph Lameterb76834b2010-12-06 11:16:25 -0600322 __this_cpu_write(kprobe_instance, kp);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800323}
324
325static inline void reset_kprobe_instance(void)
326{
Christoph Lameterb76834b2010-12-06 11:16:25 -0600327 __this_cpu_write(kprobe_instance, NULL);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800328}
329
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800330/*
331 * This routine is called either:
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800332 * - under the kprobe_mutex - during kprobe_[un]register()
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800333 * OR
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800334 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800335 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900336struct kprobe *get_kprobe(void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337{
338 struct hlist_head *head;
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800339 struct kprobe *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340
341 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
Sasha Levinb67bfe02013-02-27 17:06:00 -0800342 hlist_for_each_entry_rcu(p, head, hlist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 if (p->addr == addr)
344 return p;
345 }
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 return NULL;
348}
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900349NOKPROBE_SYMBOL(get_kprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900351static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500352
353/* Return true if the kprobe is an aggregator */
354static inline int kprobe_aggrprobe(struct kprobe *p)
355{
356 return p->pre_handler == aggr_pre_handler;
357}
358
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900359/* Return true(!0) if the kprobe is unused */
360static inline int kprobe_unused(struct kprobe *p)
361{
362 return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
363 list_empty(&p->list);
364}
365
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500366/*
367 * Keep all fields in the kprobe consistent
368 */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900369static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500370{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900371 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
372 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500373}
374
375#ifdef CONFIG_OPTPROBES
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500376/* NOTE: change this value only with kprobe_mutex held */
377static bool kprobes_allow_optimization;
378
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500379/*
380 * Call all pre_handler on the list, but ignores its return value.
381 * This must be called from arch-dep optimized caller.
382 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900383void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500384{
385 struct kprobe *kp;
386
387 list_for_each_entry_rcu(kp, &p->list, list) {
388 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
389 set_kprobe_instance(kp);
390 kp->pre_handler(kp, regs);
391 }
392 reset_kprobe_instance();
393 }
394}
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900395NOKPROBE_SYMBOL(opt_pre_handler);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500396
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900397/* Free optimized instructions and optimized_kprobe */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900398static void free_aggr_kprobe(struct kprobe *p)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900399{
400 struct optimized_kprobe *op;
401
402 op = container_of(p, struct optimized_kprobe, kp);
403 arch_remove_optimized_kprobe(op);
404 arch_remove_kprobe(p);
405 kfree(op);
406}
407
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500408/* Return true(!0) if the kprobe is ready for optimization. */
409static inline int kprobe_optready(struct kprobe *p)
410{
411 struct optimized_kprobe *op;
412
413 if (kprobe_aggrprobe(p)) {
414 op = container_of(p, struct optimized_kprobe, kp);
415 return arch_prepared_optinsn(&op->optinsn);
416 }
417
418 return 0;
419}
420
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900421/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */
422static inline int kprobe_disarmed(struct kprobe *p)
423{
424 struct optimized_kprobe *op;
425
426 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
427 if (!kprobe_aggrprobe(p))
428 return kprobe_disabled(p);
429
430 op = container_of(p, struct optimized_kprobe, kp);
431
432 return kprobe_disabled(p) && list_empty(&op->list);
433}
434
435/* Return true(!0) if the probe is queued on (un)optimizing lists */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900436static int kprobe_queued(struct kprobe *p)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900437{
438 struct optimized_kprobe *op;
439
440 if (kprobe_aggrprobe(p)) {
441 op = container_of(p, struct optimized_kprobe, kp);
442 if (!list_empty(&op->list))
443 return 1;
444 }
445 return 0;
446}
447
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500448/*
449 * Return an optimized kprobe whose optimizing code replaces
450 * instructions including addr (exclude breakpoint).
451 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900452static struct kprobe *get_optimized_kprobe(unsigned long addr)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500453{
454 int i;
455 struct kprobe *p = NULL;
456 struct optimized_kprobe *op;
457
458 /* Don't check i == 0, since that is a breakpoint case. */
459 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
460 p = get_kprobe((void *)(addr - i));
461
462 if (p && kprobe_optready(p)) {
463 op = container_of(p, struct optimized_kprobe, kp);
464 if (arch_within_optimized_kprobe(op, addr))
465 return p;
466 }
467
468 return NULL;
469}
470
471/* Optimization staging list, protected by kprobe_mutex */
472static LIST_HEAD(optimizing_list);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900473static LIST_HEAD(unoptimizing_list);
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900474static LIST_HEAD(freeing_list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500475
476static void kprobe_optimizer(struct work_struct *work);
477static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
478#define OPTIMIZE_DELAY 5
479
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900480/*
481 * Optimize (replace a breakpoint with a jump) kprobes listed on
482 * optimizing_list.
483 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900484static void do_optimize_kprobes(void)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500485{
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900486 /* Optimization never be done when disarmed */
487 if (kprobes_all_disarmed || !kprobes_allow_optimization ||
488 list_empty(&optimizing_list))
489 return;
490
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500491 /*
492 * The optimization/unoptimization refers online_cpus via
493 * stop_machine() and cpu-hotplug modifies online_cpus.
494 * And same time, text_mutex will be held in cpu-hotplug and here.
495 * This combination can cause a deadlock (cpu-hotplug try to lock
496 * text_mutex but stop_machine can not be done because online_cpus
497 * has been changed)
498 * To avoid this deadlock, we need to call get_online_cpus()
499 * for preventing cpu-hotplug outside of text_mutex locking.
500 */
501 get_online_cpus();
502 mutex_lock(&text_mutex);
Masami Hiramatsucd7ebe22010-12-03 18:54:28 +0900503 arch_optimize_kprobes(&optimizing_list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500504 mutex_unlock(&text_mutex);
505 put_online_cpus();
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900506}
507
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900508/*
509 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
510 * if need) kprobes listed on unoptimizing_list.
511 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900512static void do_unoptimize_kprobes(void)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900513{
514 struct optimized_kprobe *op, *tmp;
515
516 /* Unoptimization must be done anytime */
517 if (list_empty(&unoptimizing_list))
518 return;
519
520 /* Ditto to do_optimize_kprobes */
521 get_online_cpus();
522 mutex_lock(&text_mutex);
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900523 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
Masami Hiramatsuf984ba42010-12-03 18:54:34 +0900524 /* Loop free_list for disarming */
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900525 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900526 /* Disarm probes if marked disabled */
527 if (kprobe_disabled(&op->kp))
528 arch_disarm_kprobe(&op->kp);
529 if (kprobe_unused(&op->kp)) {
530 /*
531 * Remove unused probes from hash list. After waiting
532 * for synchronization, these probes are reclaimed.
533 * (reclaiming is done by do_free_cleaned_kprobes.)
534 */
535 hlist_del_rcu(&op->kp.hlist);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900536 } else
537 list_del_init(&op->list);
538 }
539 mutex_unlock(&text_mutex);
540 put_online_cpus();
541}
542
543/* Reclaim all kprobes on the free_list */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900544static void do_free_cleaned_kprobes(void)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900545{
546 struct optimized_kprobe *op, *tmp;
547
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900548 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900549 BUG_ON(!kprobe_unused(&op->kp));
550 list_del_init(&op->list);
551 free_aggr_kprobe(&op->kp);
552 }
553}
554
555/* Start optimizer after OPTIMIZE_DELAY passed */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900556static void kick_kprobe_optimizer(void)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900557{
Tejun Heoad72b3b2012-12-21 17:57:00 -0800558 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900559}
560
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900561/* Kprobe jump optimizer */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900562static void kprobe_optimizer(struct work_struct *work)
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900563{
Steven Rostedt72ef3792012-06-05 19:28:14 +0900564 mutex_lock(&kprobe_mutex);
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900565 /* Lock modules while optimizing kprobes */
566 mutex_lock(&module_mutex);
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900567
568 /*
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900569 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
570 * kprobes before waiting for quiesence period.
571 */
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900572 do_unoptimize_kprobes();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900573
574 /*
575 * Step 2: Wait for quiesence period to ensure all running interrupts
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900576 * are done. Because optprobe may modify multiple instructions
577 * there is a chance that Nth instruction is interrupted. In that
578 * case, running interrupt can return to 2nd-Nth byte of jump
579 * instruction. This wait is for avoiding it.
580 */
581 synchronize_sched();
582
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900583 /* Step 3: Optimize kprobes after quiesence period */
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900584 do_optimize_kprobes();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900585
586 /* Step 4: Free cleaned kprobes after quiesence period */
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900587 do_free_cleaned_kprobes();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900588
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500589 mutex_unlock(&module_mutex);
Steven Rostedt72ef3792012-06-05 19:28:14 +0900590 mutex_unlock(&kprobe_mutex);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900591
Masami Hiramatsucd7ebe22010-12-03 18:54:28 +0900592 /* Step 5: Kick optimizer again if needed */
Masami Hiramatsuf984ba42010-12-03 18:54:34 +0900593 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
Masami Hiramatsucd7ebe22010-12-03 18:54:28 +0900594 kick_kprobe_optimizer();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900595}
596
597/* Wait for completing optimization and unoptimization */
Thomas Gleixner30e7d892017-05-17 10:19:49 +0200598void wait_for_kprobe_optimizer(void)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900599{
Tejun Heoad72b3b2012-12-21 17:57:00 -0800600 mutex_lock(&kprobe_mutex);
601
602 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
603 mutex_unlock(&kprobe_mutex);
604
605 /* this will also make optimizing_work execute immmediately */
606 flush_delayed_work(&optimizing_work);
607 /* @optimizing_work might not have been queued yet, relax */
608 cpu_relax();
609
610 mutex_lock(&kprobe_mutex);
611 }
612
613 mutex_unlock(&kprobe_mutex);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500614}
615
616/* Optimize kprobe if p is ready to be optimized */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900617static void optimize_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500618{
619 struct optimized_kprobe *op;
620
621 /* Check if the kprobe is disabled or not ready for optimization. */
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500622 if (!kprobe_optready(p) || !kprobes_allow_optimization ||
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500623 (kprobe_disabled(p) || kprobes_all_disarmed))
624 return;
625
626 /* Both of break_handler and post_handler are not supported. */
627 if (p->break_handler || p->post_handler)
628 return;
629
630 op = container_of(p, struct optimized_kprobe, kp);
631
632 /* Check there is no other kprobes at the optimized instructions */
633 if (arch_check_optimized_kprobe(op) < 0)
634 return;
635
636 /* Check if it is already optimized. */
637 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
638 return;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500639 op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900640
641 if (!list_empty(&op->list))
642 /* This is under unoptimizing. Just dequeue the probe */
643 list_del_init(&op->list);
644 else {
645 list_add(&op->list, &optimizing_list);
646 kick_kprobe_optimizer();
647 }
648}
649
650/* Short cut to direct unoptimizing */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900651static void force_unoptimize_kprobe(struct optimized_kprobe *op)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900652{
653 get_online_cpus();
654 arch_unoptimize_kprobe(op);
655 put_online_cpus();
656 if (kprobe_disabled(&op->kp))
657 arch_disarm_kprobe(&op->kp);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500658}
659
660/* Unoptimize a kprobe if p is optimized */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900661static void unoptimize_kprobe(struct kprobe *p, bool force)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500662{
663 struct optimized_kprobe *op;
664
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900665 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
666 return; /* This is not an optprobe nor optimized */
667
668 op = container_of(p, struct optimized_kprobe, kp);
669 if (!kprobe_optimized(p)) {
670 /* Unoptimized or unoptimizing case */
671 if (force && !list_empty(&op->list)) {
672 /*
673 * Only if this is unoptimizing kprobe and forced,
674 * forcibly unoptimize it. (No need to unoptimize
675 * unoptimized kprobe again :)
676 */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500677 list_del_init(&op->list);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900678 force_unoptimize_kprobe(op);
679 }
680 return;
681 }
682
683 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
684 if (!list_empty(&op->list)) {
685 /* Dequeue from the optimization queue */
686 list_del_init(&op->list);
687 return;
688 }
689 /* Optimized kprobe case */
690 if (force)
691 /* Forcibly update the code: this is a special case */
692 force_unoptimize_kprobe(op);
693 else {
694 list_add(&op->list, &unoptimizing_list);
695 kick_kprobe_optimizer();
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500696 }
697}
698
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900699/* Cancel unoptimizing for reusing */
700static void reuse_unused_kprobe(struct kprobe *ap)
701{
702 struct optimized_kprobe *op;
703
704 BUG_ON(!kprobe_unused(ap));
705 /*
706 * Unused kprobe MUST be on the way of delayed unoptimizing (means
707 * there is still a relative jump) and disabled.
708 */
709 op = container_of(ap, struct optimized_kprobe, kp);
710 if (unlikely(list_empty(&op->list)))
711 printk(KERN_WARNING "Warning: found a stray unused "
712 "aggrprobe@%p\n", ap->addr);
713 /* Enable the probe again */
714 ap->flags &= ~KPROBE_FLAG_DISABLED;
715 /* Optimize it again (remove from op->list) */
716 BUG_ON(!kprobe_optready(ap));
717 optimize_kprobe(ap);
718}
719
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500720/* Remove optimized instructions */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900721static void kill_optimized_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500722{
723 struct optimized_kprobe *op;
724
725 op = container_of(p, struct optimized_kprobe, kp);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900726 if (!list_empty(&op->list))
727 /* Dequeue from the (un)optimization queue */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500728 list_del_init(&op->list);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900729 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900730
731 if (kprobe_unused(p)) {
732 /* Enqueue if it is unused */
733 list_add(&op->list, &freeing_list);
734 /*
735 * Remove unused probes from the hash list. After waiting
736 * for synchronization, this probe is reclaimed.
737 * (reclaiming is done by do_free_cleaned_kprobes().)
738 */
739 hlist_del_rcu(&op->kp.hlist);
740 }
741
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900742 /* Don't touch the code, because it is already freed. */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500743 arch_remove_optimized_kprobe(op);
744}
745
Masami Hiramatsua4602462017-04-19 18:22:25 +0530746static inline
747void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
748{
749 if (!kprobe_ftrace(p))
750 arch_prepare_optimized_kprobe(op, p);
751}
752
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500753/* Try to prepare optimized instructions */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900754static void prepare_optimized_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500755{
756 struct optimized_kprobe *op;
757
758 op = container_of(p, struct optimized_kprobe, kp);
Masami Hiramatsua4602462017-04-19 18:22:25 +0530759 __prepare_optimized_kprobe(op, p);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500760}
761
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500762/* Allocate new optimized_kprobe and try to prepare optimized instructions */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900763static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500764{
765 struct optimized_kprobe *op;
766
767 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
768 if (!op)
769 return NULL;
770
771 INIT_LIST_HEAD(&op->list);
772 op->kp.addr = p->addr;
Masami Hiramatsua4602462017-04-19 18:22:25 +0530773 __prepare_optimized_kprobe(op, p);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500774
775 return &op->kp;
776}
777
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900778static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500779
780/*
781 * Prepare an optimized_kprobe and optimize it
782 * NOTE: p must be a normal registered kprobe
783 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900784static void try_to_optimize_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500785{
786 struct kprobe *ap;
787 struct optimized_kprobe *op;
788
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900789 /* Impossible to optimize ftrace-based kprobe */
790 if (kprobe_ftrace(p))
791 return;
792
Masami Hiramatsu25764282012-06-05 19:28:26 +0900793 /* For preparing optimization, jump_label_text_reserved() is called */
794 jump_label_lock();
795 mutex_lock(&text_mutex);
796
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500797 ap = alloc_aggr_kprobe(p);
798 if (!ap)
Masami Hiramatsu25764282012-06-05 19:28:26 +0900799 goto out;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500800
801 op = container_of(ap, struct optimized_kprobe, kp);
802 if (!arch_prepared_optinsn(&op->optinsn)) {
803 /* If failed to setup optimizing, fallback to kprobe */
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900804 arch_remove_optimized_kprobe(op);
805 kfree(op);
Masami Hiramatsu25764282012-06-05 19:28:26 +0900806 goto out;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500807 }
808
809 init_aggr_kprobe(ap, p);
Masami Hiramatsu25764282012-06-05 19:28:26 +0900810 optimize_kprobe(ap); /* This just kicks optimizer thread */
811
812out:
813 mutex_unlock(&text_mutex);
814 jump_label_unlock();
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500815}
816
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500817#ifdef CONFIG_SYSCTL
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900818static void optimize_all_kprobes(void)
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500819{
820 struct hlist_head *head;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500821 struct kprobe *p;
822 unsigned int i;
823
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900824 mutex_lock(&kprobe_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500825 /* If optimization is already allowed, just return */
826 if (kprobes_allow_optimization)
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900827 goto out;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500828
829 kprobes_allow_optimization = true;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500830 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
831 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -0800832 hlist_for_each_entry_rcu(p, head, hlist)
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500833 if (!kprobe_disabled(p))
834 optimize_kprobe(p);
835 }
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500836 printk(KERN_INFO "Kprobes globally optimized\n");
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900837out:
838 mutex_unlock(&kprobe_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500839}
840
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900841static void unoptimize_all_kprobes(void)
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500842{
843 struct hlist_head *head;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500844 struct kprobe *p;
845 unsigned int i;
846
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900847 mutex_lock(&kprobe_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500848 /* If optimization is already prohibited, just return */
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900849 if (!kprobes_allow_optimization) {
850 mutex_unlock(&kprobe_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500851 return;
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900852 }
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500853
854 kprobes_allow_optimization = false;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500855 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
856 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -0800857 hlist_for_each_entry_rcu(p, head, hlist) {
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500858 if (!kprobe_disabled(p))
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900859 unoptimize_kprobe(p, false);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500860 }
861 }
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900862 mutex_unlock(&kprobe_mutex);
863
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900864 /* Wait for unoptimizing completion */
865 wait_for_kprobe_optimizer();
866 printk(KERN_INFO "Kprobes globally unoptimized\n");
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500867}
868
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900869static DEFINE_MUTEX(kprobe_sysctl_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500870int sysctl_kprobes_optimization;
871int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
872 void __user *buffer, size_t *length,
873 loff_t *ppos)
874{
875 int ret;
876
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900877 mutex_lock(&kprobe_sysctl_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500878 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
879 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
880
881 if (sysctl_kprobes_optimization)
882 optimize_all_kprobes();
883 else
884 unoptimize_all_kprobes();
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900885 mutex_unlock(&kprobe_sysctl_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500886
887 return ret;
888}
889#endif /* CONFIG_SYSCTL */
890
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900891/* Put a breakpoint for a probe. Must be called with text_mutex locked */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900892static void __arm_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500893{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900894 struct kprobe *_p;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500895
896 /* Check collision with other optimized kprobes */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900897 _p = get_optimized_kprobe((unsigned long)p->addr);
898 if (unlikely(_p))
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900899 /* Fallback to unoptimized kprobe */
900 unoptimize_kprobe(_p, true);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500901
902 arch_arm_kprobe(p);
903 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
904}
905
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900906/* Remove the breakpoint of a probe. Must be called with text_mutex locked */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900907static void __disarm_kprobe(struct kprobe *p, bool reopt)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500908{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900909 struct kprobe *_p;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500910
Wang Nan69d54b92015-02-13 14:40:26 -0800911 /* Try to unoptimize */
912 unoptimize_kprobe(p, kprobes_all_disarmed);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500913
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900914 if (!kprobe_queued(p)) {
915 arch_disarm_kprobe(p);
916 /* If another kprobe was blocked, optimize it. */
917 _p = get_optimized_kprobe((unsigned long)p->addr);
918 if (unlikely(_p) && reopt)
919 optimize_kprobe(_p);
920 }
921 /* TODO: reoptimize others after unoptimized this probe */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500922}
923
924#else /* !CONFIG_OPTPROBES */
925
926#define optimize_kprobe(p) do {} while (0)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900927#define unoptimize_kprobe(p, f) do {} while (0)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500928#define kill_optimized_kprobe(p) do {} while (0)
929#define prepare_optimized_kprobe(p) do {} while (0)
930#define try_to_optimize_kprobe(p) do {} while (0)
931#define __arm_kprobe(p) arch_arm_kprobe(p)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900932#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
933#define kprobe_disarmed(p) kprobe_disabled(p)
934#define wait_for_kprobe_optimizer() do {} while (0)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500935
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900936/* There should be no unused kprobes can be reused without optimization */
937static void reuse_unused_kprobe(struct kprobe *ap)
938{
939 printk(KERN_ERR "Error: There should be no unused kprobe here.\n");
940 BUG_ON(kprobe_unused(ap));
941}
942
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900943static void free_aggr_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500944{
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900945 arch_remove_kprobe(p);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500946 kfree(p);
947}
948
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900949static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500950{
951 return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
952}
953#endif /* CONFIG_OPTPROBES */
954
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +0900955#ifdef CONFIG_KPROBES_ON_FTRACE
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900956static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
Masami Hiramatsue5253892012-06-05 19:28:38 +0900957 .func = kprobe_ftrace_handler,
Masami Hiramatsu1d70be32014-11-21 05:25:23 -0500958 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900959};
960static int kprobe_ftrace_enabled;
961
962/* Must ensure p->addr is really on ftrace */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900963static int prepare_kprobe(struct kprobe *p)
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900964{
965 if (!kprobe_ftrace(p))
966 return arch_prepare_kprobe(p);
967
968 return arch_prepare_kprobe_ftrace(p);
969}
970
971/* Caller must lock kprobe_mutex */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900972static void arm_kprobe_ftrace(struct kprobe *p)
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900973{
974 int ret;
975
976 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
977 (unsigned long)p->addr, 0, 0);
978 WARN(ret < 0, "Failed to arm kprobe-ftrace at %p (%d)\n", p->addr, ret);
979 kprobe_ftrace_enabled++;
980 if (kprobe_ftrace_enabled == 1) {
981 ret = register_ftrace_function(&kprobe_ftrace_ops);
982 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret);
983 }
984}
985
986/* Caller must lock kprobe_mutex */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900987static void disarm_kprobe_ftrace(struct kprobe *p)
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900988{
989 int ret;
990
991 kprobe_ftrace_enabled--;
992 if (kprobe_ftrace_enabled == 0) {
993 ret = unregister_ftrace_function(&kprobe_ftrace_ops);
994 WARN(ret < 0, "Failed to init kprobe-ftrace (%d)\n", ret);
995 }
996 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
997 (unsigned long)p->addr, 1, 0);
998 WARN(ret < 0, "Failed to disarm kprobe-ftrace at %p (%d)\n", p->addr, ret);
999}
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +09001000#else /* !CONFIG_KPROBES_ON_FTRACE */
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001001#define prepare_kprobe(p) arch_prepare_kprobe(p)
1002#define arm_kprobe_ftrace(p) do {} while (0)
1003#define disarm_kprobe_ftrace(p) do {} while (0)
1004#endif
1005
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001006/* Arm a kprobe with text_mutex */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001007static void arm_kprobe(struct kprobe *kp)
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001008{
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001009 if (unlikely(kprobe_ftrace(kp))) {
1010 arm_kprobe_ftrace(kp);
1011 return;
1012 }
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001013 /*
1014 * Here, since __arm_kprobe() doesn't use stop_machine(),
1015 * this doesn't cause deadlock on text_mutex. So, we don't
1016 * need get_online_cpus().
1017 */
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001018 mutex_lock(&text_mutex);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001019 __arm_kprobe(kp);
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001020 mutex_unlock(&text_mutex);
1021}
1022
1023/* Disarm a kprobe with text_mutex */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001024static void disarm_kprobe(struct kprobe *kp, bool reopt)
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001025{
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001026 if (unlikely(kprobe_ftrace(kp))) {
1027 disarm_kprobe_ftrace(kp);
1028 return;
1029 }
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001030 /* Ditto */
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001031 mutex_lock(&text_mutex);
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001032 __disarm_kprobe(kp, reopt);
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001033 mutex_unlock(&text_mutex);
1034}
1035
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001036/*
1037 * Aggregate handlers for multiple kprobes support - these handlers
1038 * take care of invoking the individual kprobe handlers on p->list
1039 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001040static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001041{
1042 struct kprobe *kp;
1043
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001044 list_for_each_entry_rcu(kp, &p->list, list) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001045 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001046 set_kprobe_instance(kp);
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001047 if (kp->pre_handler(kp, regs))
1048 return 1;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001049 }
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001050 reset_kprobe_instance();
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001051 }
1052 return 0;
1053}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001054NOKPROBE_SYMBOL(aggr_pre_handler);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001055
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001056static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1057 unsigned long flags)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001058{
1059 struct kprobe *kp;
1060
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001061 list_for_each_entry_rcu(kp, &p->list, list) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001062 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001063 set_kprobe_instance(kp);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001064 kp->post_handler(kp, regs, flags);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001065 reset_kprobe_instance();
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001066 }
1067 }
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001068}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001069NOKPROBE_SYMBOL(aggr_post_handler);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001070
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001071static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
1072 int trapnr)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001073{
Christoph Lameterb76834b2010-12-06 11:16:25 -06001074 struct kprobe *cur = __this_cpu_read(kprobe_instance);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001075
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001076 /*
1077 * if we faulted "during" the execution of a user specified
1078 * probe handler, invoke just that probe's fault handler
1079 */
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001080 if (cur && cur->fault_handler) {
1081 if (cur->fault_handler(cur, regs, trapnr))
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001082 return 1;
1083 }
1084 return 0;
1085}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001086NOKPROBE_SYMBOL(aggr_fault_handler);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001087
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001088static int aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001089{
Christoph Lameterb76834b2010-12-06 11:16:25 -06001090 struct kprobe *cur = __this_cpu_read(kprobe_instance);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001091 int ret = 0;
1092
1093 if (cur && cur->break_handler) {
1094 if (cur->break_handler(cur, regs))
1095 ret = 1;
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001096 }
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001097 reset_kprobe_instance();
1098 return ret;
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001099}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001100NOKPROBE_SYMBOL(aggr_break_handler);
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001101
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001102/* Walks the list and increments nmissed count for multiprobe case */
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001103void kprobes_inc_nmissed_count(struct kprobe *p)
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001104{
1105 struct kprobe *kp;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001106 if (!kprobe_aggrprobe(p)) {
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001107 p->nmissed++;
1108 } else {
1109 list_for_each_entry_rcu(kp, &p->list, list)
1110 kp->nmissed++;
1111 }
1112 return;
1113}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001114NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001115
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001116void recycle_rp_inst(struct kretprobe_instance *ri,
1117 struct hlist_head *head)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001118{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001119 struct kretprobe *rp = ri->rp;
1120
Hien Nguyenb94cce92005-06-23 00:09:19 -07001121 /* remove rp inst off the rprobe_inst_table */
1122 hlist_del(&ri->hlist);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001123 INIT_HLIST_NODE(&ri->hlist);
1124 if (likely(rp)) {
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001125 raw_spin_lock(&rp->lock);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001126 hlist_add_head(&ri->hlist, &rp->free_instances);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001127 raw_spin_unlock(&rp->lock);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001128 } else
1129 /* Unregistering */
bibo,mao99219a32006-10-02 02:17:35 -07001130 hlist_add_head(&ri->hlist, head);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001131}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001132NOKPROBE_SYMBOL(recycle_rp_inst);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001133
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001134void kretprobe_hash_lock(struct task_struct *tsk,
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001135 struct hlist_head **head, unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001136__acquires(hlist_lock)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001137{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001138 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001139 raw_spinlock_t *hlist_lock;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001140
1141 *head = &kretprobe_inst_table[hash];
1142 hlist_lock = kretprobe_table_lock_ptr(hash);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001143 raw_spin_lock_irqsave(hlist_lock, *flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001144}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001145NOKPROBE_SYMBOL(kretprobe_hash_lock);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001146
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001147static void kretprobe_table_lock(unsigned long hash,
1148 unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001149__acquires(hlist_lock)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001150{
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001151 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1152 raw_spin_lock_irqsave(hlist_lock, *flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001153}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001154NOKPROBE_SYMBOL(kretprobe_table_lock);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001155
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001156void kretprobe_hash_unlock(struct task_struct *tsk,
1157 unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001158__releases(hlist_lock)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001159{
1160 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001161 raw_spinlock_t *hlist_lock;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001162
1163 hlist_lock = kretprobe_table_lock_ptr(hash);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001164 raw_spin_unlock_irqrestore(hlist_lock, *flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001165}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001166NOKPROBE_SYMBOL(kretprobe_hash_unlock);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001167
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001168static void kretprobe_table_unlock(unsigned long hash,
1169 unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001170__releases(hlist_lock)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001171{
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001172 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1173 raw_spin_unlock_irqrestore(hlist_lock, *flags);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001174}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001175NOKPROBE_SYMBOL(kretprobe_table_unlock);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001176
Hien Nguyenb94cce92005-06-23 00:09:19 -07001177/*
bibo maoc6fd91f2006-03-26 01:38:20 -08001178 * This function is called from finish_task_switch when task tk becomes dead,
1179 * so that we can recycle any function-return probe instances associated
1180 * with this task. These left over instances represent probed functions
1181 * that have been called but will never return.
Hien Nguyenb94cce92005-06-23 00:09:19 -07001182 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001183void kprobe_flush_task(struct task_struct *tk)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001184{
bibo,mao62c27be2006-10-02 02:17:33 -07001185 struct kretprobe_instance *ri;
bibo,mao99219a32006-10-02 02:17:35 -07001186 struct hlist_head *head, empty_rp;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001187 struct hlist_node *tmp;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001188 unsigned long hash, flags = 0;
Rusty Lynch802eae72005-06-27 15:17:08 -07001189
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001190 if (unlikely(!kprobes_initialized))
1191 /* Early boot. kretprobe_table_locks not yet initialized. */
1192 return;
1193
Ananth N Mavinakayanahallid496aab2012-01-20 14:34:04 -08001194 INIT_HLIST_HEAD(&empty_rp);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001195 hash = hash_ptr(tk, KPROBE_HASH_BITS);
1196 head = &kretprobe_inst_table[hash];
1197 kretprobe_table_lock(hash, &flags);
Sasha Levinb67bfe02013-02-27 17:06:00 -08001198 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
bibo,mao62c27be2006-10-02 02:17:33 -07001199 if (ri->task == tk)
bibo,mao99219a32006-10-02 02:17:35 -07001200 recycle_rp_inst(ri, &empty_rp);
bibo,mao62c27be2006-10-02 02:17:33 -07001201 }
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001202 kretprobe_table_unlock(hash, &flags);
Sasha Levinb67bfe02013-02-27 17:06:00 -08001203 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
bibo,mao99219a32006-10-02 02:17:35 -07001204 hlist_del(&ri->hlist);
1205 kfree(ri);
1206 }
Hien Nguyenb94cce92005-06-23 00:09:19 -07001207}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001208NOKPROBE_SYMBOL(kprobe_flush_task);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001209
Hien Nguyenb94cce92005-06-23 00:09:19 -07001210static inline void free_rp_inst(struct kretprobe *rp)
1211{
1212 struct kretprobe_instance *ri;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001213 struct hlist_node *next;
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001214
Sasha Levinb67bfe02013-02-27 17:06:00 -08001215 hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) {
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001216 hlist_del(&ri->hlist);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001217 kfree(ri);
1218 }
1219}
1220
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001221static void cleanup_rp_inst(struct kretprobe *rp)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001222{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001223 unsigned long flags, hash;
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001224 struct kretprobe_instance *ri;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001225 struct hlist_node *next;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001226 struct hlist_head *head;
1227
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001228 /* No race here */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001229 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
1230 kretprobe_table_lock(hash, &flags);
1231 head = &kretprobe_inst_table[hash];
Sasha Levinb67bfe02013-02-27 17:06:00 -08001232 hlist_for_each_entry_safe(ri, next, head, hlist) {
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001233 if (ri->rp == rp)
1234 ri->rp = NULL;
1235 }
1236 kretprobe_table_unlock(hash, &flags);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001237 }
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001238 free_rp_inst(rp);
1239}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001240NOKPROBE_SYMBOL(cleanup_rp_inst);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001241
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001242/*
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001243* Add the new probe to ap->list. Fail if this is the
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001244* second jprobe at the address - two jprobes can't coexist
1245*/
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001246static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001247{
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001248 BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001249
1250 if (p->break_handler || p->post_handler)
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001251 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001252
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001253 if (p->break_handler) {
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001254 if (ap->break_handler)
mao, bibo36721652006-06-26 00:25:22 -07001255 return -EEXIST;
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001256 list_add_tail_rcu(&p->list, &ap->list);
1257 ap->break_handler = aggr_break_handler;
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001258 } else
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001259 list_add_rcu(&p->list, &ap->list);
1260 if (p->post_handler && !ap->post_handler)
1261 ap->post_handler = aggr_post_handler;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001262
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001263 return 0;
1264}
1265
1266/*
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001267 * Fill in the required fields of the "manager kprobe". Replace the
1268 * earlier kprobe in the hlist with the manager kprobe
1269 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001270static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001271{
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001272 /* Copy p's insn slot to ap */
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001273 copy_kprobe(p, ap);
bibo, maoa9ad9652006-07-30 03:03:26 -07001274 flush_insn_slot(ap);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001275 ap->addr = p->addr;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001276 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001277 ap->pre_handler = aggr_pre_handler;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001278 ap->fault_handler = aggr_fault_handler;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001279 /* We don't care the kprobe which has gone. */
1280 if (p->post_handler && !kprobe_gone(p))
mao, bibo36721652006-06-26 00:25:22 -07001281 ap->post_handler = aggr_post_handler;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001282 if (p->break_handler && !kprobe_gone(p))
mao, bibo36721652006-06-26 00:25:22 -07001283 ap->break_handler = aggr_break_handler;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001284
1285 INIT_LIST_HEAD(&ap->list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001286 INIT_HLIST_NODE(&ap->hlist);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001287
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001288 list_add_rcu(&p->list, &ap->list);
Keshavamurthy Anil Sadad0f32005-12-12 00:37:12 -08001289 hlist_replace_rcu(&p->hlist, &ap->hlist);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001290}
1291
1292/*
1293 * This is the second or subsequent kprobe at the address - handle
1294 * the intricacies
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001295 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001296static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001297{
1298 int ret = 0;
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001299 struct kprobe *ap = orig_p;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001300
Masami Hiramatsu25764282012-06-05 19:28:26 +09001301 /* For preparing optimization, jump_label_text_reserved() is called */
1302 jump_label_lock();
1303 /*
1304 * Get online CPUs to avoid text_mutex deadlock.with stop machine,
1305 * which is invoked by unoptimize_kprobe() in add_new_kprobe()
1306 */
1307 get_online_cpus();
1308 mutex_lock(&text_mutex);
1309
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001310 if (!kprobe_aggrprobe(orig_p)) {
1311 /* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */
1312 ap = alloc_aggr_kprobe(orig_p);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001313 if (!ap) {
1314 ret = -ENOMEM;
1315 goto out;
1316 }
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001317 init_aggr_kprobe(ap, orig_p);
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001318 } else if (kprobe_unused(ap))
Masami Hiramatsu0490cd12010-12-03 18:54:16 +09001319 /* This probe is going to die. Rescue it */
1320 reuse_unused_kprobe(ap);
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001321
1322 if (kprobe_gone(ap)) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001323 /*
1324 * Attempting to insert new probe at the same location that
1325 * had a probe in the module vaddr area which already
1326 * freed. So, the instruction slot has already been
1327 * released. We need a new slot for the new probe.
1328 */
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001329 ret = arch_prepare_kprobe(ap);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001330 if (ret)
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001331 /*
1332 * Even if fail to allocate new slot, don't need to
1333 * free aggr_probe. It will be used next time, or
1334 * freed by unregister_kprobe.
1335 */
Masami Hiramatsu25764282012-06-05 19:28:26 +09001336 goto out;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001337
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001338 /* Prepare optimized instructions if possible. */
1339 prepare_optimized_kprobe(ap);
1340
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001341 /*
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001342 * Clear gone flag to prevent allocating new slot again, and
1343 * set disabled flag because it is not armed yet.
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001344 */
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001345 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1346 | KPROBE_FLAG_DISABLED;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001347 }
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001348
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001349 /* Copy ap's insn slot to p */
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001350 copy_kprobe(ap, p);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001351 ret = add_new_kprobe(ap, p);
1352
1353out:
1354 mutex_unlock(&text_mutex);
1355 put_online_cpus();
1356 jump_label_unlock();
1357
1358 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1359 ap->flags &= ~KPROBE_FLAG_DISABLED;
1360 if (!kprobes_all_disarmed)
1361 /* Arm the breakpoint again. */
1362 arm_kprobe(ap);
1363 }
1364 return ret;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001365}
1366
Masami Hiramatsube8f2742014-04-17 17:16:58 +09001367bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1368{
1369 /* The __kprobes marked functions and entry code must not be probed */
1370 return addr >= (unsigned long)__kprobes_text_start &&
1371 addr < (unsigned long)__kprobes_text_end;
1372}
1373
Andy Lutomirskie5779e82015-07-30 20:32:40 -07001374bool within_kprobe_blacklist(unsigned long addr)
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001375{
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001376 struct kprobe_blacklist_entry *ent;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001377
Masami Hiramatsube8f2742014-04-17 17:16:58 +09001378 if (arch_within_kprobe_blacklist(addr))
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001379 return true;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001380 /*
1381 * If there exists a kprobe_blacklist, verify and
1382 * fail any probe registration in the prohibited area
1383 */
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001384 list_for_each_entry(ent, &kprobe_blacklist, list) {
1385 if (addr >= ent->start_addr && addr < ent->end_addr)
1386 return true;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001387 }
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001388
1389 return false;
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001390}
1391
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001392/*
1393 * If we have a symbol_name argument, look it up and add the offset field
1394 * to it. This way, we can specify a relative address to a symbol.
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001395 * This returns encoded errors if it fails to look up symbol or invalid
1396 * combination of parameters.
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001397 */
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301398static kprobe_opcode_t *_kprobe_addr(kprobe_opcode_t *addr,
1399 const char *symbol_name, unsigned int offset)
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001400{
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301401 if ((symbol_name && addr) || (!symbol_name && !addr))
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001402 goto invalid;
1403
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301404 if (symbol_name) {
Linus Torvalds7246f602017-05-05 11:36:44 -07001405 addr = kprobe_lookup_name(symbol_name, offset);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001406 if (!addr)
1407 return ERR_PTR(-ENOENT);
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001408 }
1409
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301410 addr = (kprobe_opcode_t *)(((char *)addr) + offset);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001411 if (addr)
1412 return addr;
1413
1414invalid:
1415 return ERR_PTR(-EINVAL);
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001416}
1417
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301418static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
1419{
1420 return _kprobe_addr(p->addr, p->symbol_name, p->offset);
1421}
1422
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301423/* Check passed kprobe is valid and return kprobe in kprobe_table. */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001424static struct kprobe *__get_valid_kprobe(struct kprobe *p)
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301425{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001426 struct kprobe *ap, *list_p;
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301427
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001428 ap = get_kprobe(p->addr);
1429 if (unlikely(!ap))
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301430 return NULL;
1431
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001432 if (p != ap) {
1433 list_for_each_entry_rcu(list_p, &ap->list, list)
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301434 if (list_p == p)
1435 /* kprobe p is a valid probe */
1436 goto valid;
1437 return NULL;
1438 }
1439valid:
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001440 return ap;
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301441}
1442
1443/* Return error if the kprobe is being re-registered */
1444static inline int check_kprobe_rereg(struct kprobe *p)
1445{
1446 int ret = 0;
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301447
1448 mutex_lock(&kprobe_mutex);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001449 if (__get_valid_kprobe(p))
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301450 ret = -EINVAL;
1451 mutex_unlock(&kprobe_mutex);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001452
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301453 return ret;
1454}
1455
Heiko Carstensf7f242f2014-10-15 12:17:34 +02001456int __weak arch_check_ftrace_location(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001457{
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001458 unsigned long ftrace_addr;
1459
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001460 ftrace_addr = ftrace_location((unsigned long)p->addr);
1461 if (ftrace_addr) {
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +09001462#ifdef CONFIG_KPROBES_ON_FTRACE
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001463 /* Given address is not on the instruction boundary */
1464 if ((unsigned long)p->addr != ftrace_addr)
1465 return -EILSEQ;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001466 p->flags |= KPROBE_FLAG_FTRACE;
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +09001467#else /* !CONFIG_KPROBES_ON_FTRACE */
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001468 return -EINVAL;
1469#endif
1470 }
Heiko Carstensf7f242f2014-10-15 12:17:34 +02001471 return 0;
1472}
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001473
Heiko Carstensf7f242f2014-10-15 12:17:34 +02001474static int check_kprobe_address_safe(struct kprobe *p,
1475 struct module **probed_mod)
1476{
1477 int ret;
1478
1479 ret = arch_check_ftrace_location(p);
1480 if (ret)
1481 return ret;
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001482 jump_label_lock();
1483 preempt_disable();
1484
1485 /* Ensure it is not in reserved area nor out of text */
1486 if (!kernel_text_address((unsigned long) p->addr) ||
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001487 within_kprobe_blacklist((unsigned long) p->addr) ||
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001488 jump_label_text_reserved(p->addr, p->addr)) {
1489 ret = -EINVAL;
1490 goto out;
1491 }
1492
1493 /* Check if are we probing a module */
1494 *probed_mod = __module_text_address((unsigned long) p->addr);
1495 if (*probed_mod) {
1496 /*
1497 * We must hold a refcount of the probed module while updating
1498 * its code to prohibit unexpected unloading.
1499 */
1500 if (unlikely(!try_module_get(*probed_mod))) {
1501 ret = -ENOENT;
1502 goto out;
1503 }
1504
1505 /*
1506 * If the module freed .init.text, we couldn't insert
1507 * kprobes in there.
1508 */
1509 if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1510 (*probed_mod)->state != MODULE_STATE_COMING) {
1511 module_put(*probed_mod);
1512 *probed_mod = NULL;
1513 ret = -ENOENT;
1514 }
1515 }
1516out:
1517 preempt_enable();
1518 jump_label_unlock();
1519
1520 return ret;
1521}
1522
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001523int register_kprobe(struct kprobe *p)
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001524{
1525 int ret;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001526 struct kprobe *old_p;
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -08001527 struct module *probed_mod;
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001528 kprobe_opcode_t *addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001530 /* Adjust probe address from symbol */
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001531 addr = kprobe_addr(p);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001532 if (IS_ERR(addr))
1533 return PTR_ERR(addr);
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001534 p->addr = addr;
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -07001535
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301536 ret = check_kprobe_rereg(p);
1537 if (ret)
1538 return ret;
1539
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001540 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1541 p->flags &= KPROBE_FLAG_DISABLED;
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001542 p->nmissed = 0;
Masami Hiramatsu98616682008-04-28 02:14:28 -07001543 INIT_LIST_HEAD(&p->list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001544
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001545 ret = check_kprobe_address_safe(p, &probed_mod);
1546 if (ret)
1547 return ret;
1548
1549 mutex_lock(&kprobe_mutex);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001550
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001551 old_p = get_kprobe(p->addr);
1552 if (old_p) {
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001553 /* Since this may unoptimize old_p, locking text_mutex. */
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001554 ret = register_aggr_kprobe(old_p, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001555 goto out;
1556 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557
Masami Hiramatsu25764282012-06-05 19:28:26 +09001558 mutex_lock(&text_mutex); /* Avoiding text modification */
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001559 ret = prepare_kprobe(p);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001560 mutex_unlock(&text_mutex);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -07001561 if (ret)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001562 goto out;
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001563
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001564 INIT_HLIST_NODE(&p->hlist);
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001565 hlist_add_head_rcu(&p->hlist,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001566 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1567
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001568 if (!kprobes_all_disarmed && !kprobe_disabled(p))
Masami Hiramatsu25764282012-06-05 19:28:26 +09001569 arm_kprobe(p);
Christoph Hellwig74a0b572007-10-16 01:24:07 -07001570
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001571 /* Try to optimize kprobe */
1572 try_to_optimize_kprobe(p);
1573
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574out:
Ingo Molnar7a7d1cf2006-03-23 03:00:35 -08001575 mutex_unlock(&kprobe_mutex);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001576
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001577 if (probed_mod)
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -08001578 module_put(probed_mod);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001579
Linus Torvalds1da177e2005-04-16 15:20:36 -07001580 return ret;
1581}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001582EXPORT_SYMBOL_GPL(register_kprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001584/* Check if all probes on the aggrprobe are disabled */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001585static int aggr_kprobe_disabled(struct kprobe *ap)
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001586{
1587 struct kprobe *kp;
1588
1589 list_for_each_entry_rcu(kp, &ap->list, list)
1590 if (!kprobe_disabled(kp))
1591 /*
1592 * There is an active probe on the list.
1593 * We can't disable this ap.
1594 */
1595 return 0;
1596
1597 return 1;
1598}
1599
1600/* Disable one kprobe: Make sure called under kprobe_mutex is locked */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001601static struct kprobe *__disable_kprobe(struct kprobe *p)
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001602{
1603 struct kprobe *orig_p;
1604
1605 /* Get an original kprobe for return */
1606 orig_p = __get_valid_kprobe(p);
1607 if (unlikely(orig_p == NULL))
1608 return NULL;
1609
1610 if (!kprobe_disabled(p)) {
1611 /* Disable probe if it is a child probe */
1612 if (p != orig_p)
1613 p->flags |= KPROBE_FLAG_DISABLED;
1614
1615 /* Try to disarm and disable this/parent probe */
1616 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
Wang Nan69d54b92015-02-13 14:40:26 -08001617 /*
1618 * If kprobes_all_disarmed is set, orig_p
1619 * should have already been disarmed, so
1620 * skip unneed disarming process.
1621 */
1622 if (!kprobes_all_disarmed)
1623 disarm_kprobe(orig_p, true);
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001624 orig_p->flags |= KPROBE_FLAG_DISABLED;
1625 }
1626 }
1627
1628 return orig_p;
1629}
1630
Masami Hiramatsu98616682008-04-28 02:14:28 -07001631/*
1632 * Unregister a kprobe without a scheduler synchronization.
1633 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001634static int __unregister_kprobe_top(struct kprobe *p)
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -08001635{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001636 struct kprobe *ap, *list_p;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001637
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001638 /* Disable kprobe. This will disarm it if needed. */
1639 ap = __disable_kprobe(p);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001640 if (ap == NULL)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001641 return -EINVAL;
1642
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001643 if (ap == p)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001644 /*
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001645 * This probe is an independent(and non-optimized) kprobe
1646 * (not an aggrprobe). Remove from the hash list.
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001647 */
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001648 goto disarmed;
1649
1650 /* Following process expects this probe is an aggrprobe */
1651 WARN_ON(!kprobe_aggrprobe(ap));
1652
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001653 if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1654 /*
1655 * !disarmed could be happen if the probe is under delayed
1656 * unoptimizing.
1657 */
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001658 goto disarmed;
1659 else {
1660 /* If disabling probe has special handlers, update aggrprobe */
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001661 if (p->break_handler && !kprobe_gone(p))
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001662 ap->break_handler = NULL;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001663 if (p->post_handler && !kprobe_gone(p)) {
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001664 list_for_each_entry_rcu(list_p, &ap->list, list) {
Masami Hiramatsu98616682008-04-28 02:14:28 -07001665 if ((list_p != p) && (list_p->post_handler))
1666 goto noclean;
1667 }
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001668 ap->post_handler = NULL;
Masami Hiramatsu98616682008-04-28 02:14:28 -07001669 }
1670noclean:
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001671 /*
1672 * Remove from the aggrprobe: this path will do nothing in
1673 * __unregister_kprobe_bottom().
1674 */
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001675 list_del_rcu(&p->list);
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001676 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1677 /*
1678 * Try to optimize this probe again, because post
1679 * handler may have been changed.
1680 */
1681 optimize_kprobe(ap);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001682 }
Masami Hiramatsu98616682008-04-28 02:14:28 -07001683 return 0;
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001684
1685disarmed:
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001686 BUG_ON(!kprobe_disarmed(ap));
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001687 hlist_del_rcu(&ap->hlist);
1688 return 0;
Masami Hiramatsu98616682008-04-28 02:14:28 -07001689}
Mao, Bibob3e55c72005-12-12 00:37:00 -08001690
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001691static void __unregister_kprobe_bottom(struct kprobe *p)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001692{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001693 struct kprobe *ap;
Mao, Bibob3e55c72005-12-12 00:37:00 -08001694
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001695 if (list_empty(&p->list))
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001696 /* This is an independent kprobe */
Ananth N Mavinakayanahalli0498b632006-01-09 20:52:46 -08001697 arch_remove_kprobe(p);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001698 else if (list_is_singular(&p->list)) {
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001699 /* This is the last child of an aggrprobe */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001700 ap = list_entry(p->list.next, struct kprobe, list);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001701 list_del(&p->list);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001702 free_aggr_kprobe(ap);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001703 }
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001704 /* Otherwise, do nothing. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001705}
1706
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001707int register_kprobes(struct kprobe **kps, int num)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001708{
1709 int i, ret = 0;
1710
1711 if (num <= 0)
1712 return -EINVAL;
1713 for (i = 0; i < num; i++) {
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001714 ret = register_kprobe(kps[i]);
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -07001715 if (ret < 0) {
1716 if (i > 0)
1717 unregister_kprobes(kps, i);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001718 break;
1719 }
1720 }
1721 return ret;
1722}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001723EXPORT_SYMBOL_GPL(register_kprobes);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001724
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001725void unregister_kprobe(struct kprobe *p)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001726{
1727 unregister_kprobes(&p, 1);
1728}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001729EXPORT_SYMBOL_GPL(unregister_kprobe);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001730
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001731void unregister_kprobes(struct kprobe **kps, int num)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001732{
1733 int i;
1734
1735 if (num <= 0)
1736 return;
1737 mutex_lock(&kprobe_mutex);
1738 for (i = 0; i < num; i++)
1739 if (__unregister_kprobe_top(kps[i]) < 0)
1740 kps[i]->addr = NULL;
1741 mutex_unlock(&kprobe_mutex);
1742
1743 synchronize_sched();
1744 for (i = 0; i < num; i++)
1745 if (kps[i]->addr)
1746 __unregister_kprobe_bottom(kps[i]);
1747}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001748EXPORT_SYMBOL_GPL(unregister_kprobes);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001749
Naveen N. Rao5f6bee32017-03-08 22:34:15 +05301750int __weak kprobe_exceptions_notify(struct notifier_block *self,
1751 unsigned long val, void *data)
Naveen N. Raofc62d022017-02-08 01:24:14 +05301752{
1753 return NOTIFY_DONE;
1754}
Naveen N. Rao5f6bee32017-03-08 22:34:15 +05301755NOKPROBE_SYMBOL(kprobe_exceptions_notify);
Naveen N. Raofc62d022017-02-08 01:24:14 +05301756
Linus Torvalds1da177e2005-04-16 15:20:36 -07001757static struct notifier_block kprobe_exceptions_nb = {
1758 .notifier_call = kprobe_exceptions_notify,
Anil S Keshavamurthy3d5631e2006-06-26 00:25:28 -07001759 .priority = 0x7fffffff /* we need to be notified first */
1760};
1761
Michael Ellerman3d7e3382007-07-19 01:48:11 -07001762unsigned long __weak arch_deref_entry_point(void *entry)
1763{
1764 return (unsigned long)entry;
1765}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001766
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001767int register_jprobes(struct jprobe **jps, int num)
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001768{
1769 struct jprobe *jp;
1770 int ret = 0, i;
1771
1772 if (num <= 0)
1773 return -EINVAL;
1774 for (i = 0; i < num; i++) {
Namhyung Kim05662bd2010-09-15 10:04:27 +09001775 unsigned long addr, offset;
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001776 jp = jps[i];
1777 addr = arch_deref_entry_point(jp->entry);
1778
Namhyung Kim05662bd2010-09-15 10:04:27 +09001779 /* Verify probepoint is a function entry point */
1780 if (kallsyms_lookup_size_offset(addr, NULL, &offset) &&
1781 offset == 0) {
1782 jp->kp.pre_handler = setjmp_pre_handler;
1783 jp->kp.break_handler = longjmp_break_handler;
1784 ret = register_kprobe(&jp->kp);
1785 } else
1786 ret = -EINVAL;
Namhyung Kimedbaadb2010-09-15 10:04:26 +09001787
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -07001788 if (ret < 0) {
1789 if (i > 0)
1790 unregister_jprobes(jps, i);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001791 break;
1792 }
1793 }
1794 return ret;
1795}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001796EXPORT_SYMBOL_GPL(register_jprobes);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001797
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001798int register_jprobe(struct jprobe *jp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001799{
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001800 return register_jprobes(&jp, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001801}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001802EXPORT_SYMBOL_GPL(register_jprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001803
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001804void unregister_jprobe(struct jprobe *jp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805{
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001806 unregister_jprobes(&jp, 1);
1807}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001808EXPORT_SYMBOL_GPL(unregister_jprobe);
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001809
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001810void unregister_jprobes(struct jprobe **jps, int num)
Masami Hiramatsu26b31c12008-04-28 02:14:29 -07001811{
1812 int i;
1813
1814 if (num <= 0)
1815 return;
1816 mutex_lock(&kprobe_mutex);
1817 for (i = 0; i < num; i++)
1818 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1819 jps[i]->kp.addr = NULL;
1820 mutex_unlock(&kprobe_mutex);
1821
1822 synchronize_sched();
1823 for (i = 0; i < num; i++) {
1824 if (jps[i]->kp.addr)
1825 __unregister_kprobe_bottom(&jps[i]->kp);
1826 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001827}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001828EXPORT_SYMBOL_GPL(unregister_jprobes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08001830#ifdef CONFIG_KRETPROBES
Adrian Bunke65cefe2006-02-03 03:03:42 -08001831/*
1832 * This kprobe pre_handler is registered with every kretprobe. When probe
1833 * hits it will set up the return probe.
1834 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001835static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
Adrian Bunke65cefe2006-02-03 03:03:42 -08001836{
1837 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001838 unsigned long hash, flags = 0;
1839 struct kretprobe_instance *ri;
Adrian Bunke65cefe2006-02-03 03:03:42 -08001840
Masami Hiramatsuf96f5672014-08-04 03:10:16 +00001841 /*
1842 * To avoid deadlocks, prohibit return probing in NMI contexts,
1843 * just skip the probe and increase the (inexact) 'nmissed'
1844 * statistical counter, so that the user is informed that
1845 * something happened:
1846 */
1847 if (unlikely(in_nmi())) {
1848 rp->nmissed++;
1849 return 0;
1850 }
1851
1852 /* TODO: consider to only swap the RA after the last pre_handler fired */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001853 hash = hash_ptr(current, KPROBE_HASH_BITS);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001854 raw_spin_lock_irqsave(&rp->lock, flags);
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001855 if (!hlist_empty(&rp->free_instances)) {
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001856 ri = hlist_entry(rp->free_instances.first,
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001857 struct kretprobe_instance, hlist);
1858 hlist_del(&ri->hlist);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001859 raw_spin_unlock_irqrestore(&rp->lock, flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001860
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001861 ri->rp = rp;
1862 ri->task = current;
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001863
Jiang Liu55ca6142012-02-03 15:37:16 -08001864 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
1865 raw_spin_lock_irqsave(&rp->lock, flags);
1866 hlist_add_head(&ri->hlist, &rp->free_instances);
1867 raw_spin_unlock_irqrestore(&rp->lock, flags);
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001868 return 0;
Jiang Liu55ca6142012-02-03 15:37:16 -08001869 }
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001870
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001871 arch_prepare_kretprobe(ri, regs);
1872
1873 /* XXX(hch): why is there no hlist_move_head? */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001874 INIT_HLIST_NODE(&ri->hlist);
1875 kretprobe_table_lock(hash, &flags);
1876 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1877 kretprobe_table_unlock(hash, &flags);
1878 } else {
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001879 rp->nmissed++;
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001880 raw_spin_unlock_irqrestore(&rp->lock, flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001881 }
Adrian Bunke65cefe2006-02-03 03:03:42 -08001882 return 0;
1883}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001884NOKPROBE_SYMBOL(pre_handler_kretprobe);
Adrian Bunke65cefe2006-02-03 03:03:42 -08001885
Naveen N. Rao90ec5e82017-02-22 19:23:37 +05301886bool __weak arch_function_offset_within_entry(unsigned long offset)
1887{
1888 return !offset;
1889}
1890
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301891bool function_offset_within_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
1892{
1893 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
1894
1895 if (IS_ERR(kp_addr))
1896 return false;
1897
1898 if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) ||
1899 !arch_function_offset_within_entry(offset))
1900 return false;
1901
1902 return true;
1903}
1904
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001905int register_kretprobe(struct kretprobe *rp)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001906{
1907 int ret = 0;
1908 struct kretprobe_instance *inst;
1909 int i;
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001910 void *addr;
Naveen N. Rao90ec5e82017-02-22 19:23:37 +05301911
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301912 if (!function_offset_within_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset))
Naveen N. Rao90ec5e82017-02-22 19:23:37 +05301913 return -EINVAL;
Masami Hiramatsuf438d912007-10-16 01:27:49 -07001914
1915 if (kretprobe_blacklist_size) {
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001916 addr = kprobe_addr(&rp->kp);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001917 if (IS_ERR(addr))
1918 return PTR_ERR(addr);
Masami Hiramatsuf438d912007-10-16 01:27:49 -07001919
1920 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1921 if (kretprobe_blacklist[i].addr == addr)
1922 return -EINVAL;
1923 }
1924 }
Hien Nguyenb94cce92005-06-23 00:09:19 -07001925
1926 rp->kp.pre_handler = pre_handler_kretprobe;
Ananth N Mavinakayanahalli7522a842006-04-20 02:43:11 -07001927 rp->kp.post_handler = NULL;
1928 rp->kp.fault_handler = NULL;
1929 rp->kp.break_handler = NULL;
Hien Nguyenb94cce92005-06-23 00:09:19 -07001930
1931 /* Pre-allocate memory for max kretprobe instances */
1932 if (rp->maxactive <= 0) {
1933#ifdef CONFIG_PREEMPT
Heiko Carstensc2ef6662009-12-21 13:02:24 +01001934 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
Hien Nguyenb94cce92005-06-23 00:09:19 -07001935#else
Ananth N Mavinakayanahalli4dae5602009-10-30 19:23:10 +05301936 rp->maxactive = num_possible_cpus();
Hien Nguyenb94cce92005-06-23 00:09:19 -07001937#endif
1938 }
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001939 raw_spin_lock_init(&rp->lock);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001940 INIT_HLIST_HEAD(&rp->free_instances);
1941 for (i = 0; i < rp->maxactive; i++) {
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001942 inst = kmalloc(sizeof(struct kretprobe_instance) +
1943 rp->data_size, GFP_KERNEL);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001944 if (inst == NULL) {
1945 free_rp_inst(rp);
1946 return -ENOMEM;
1947 }
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001948 INIT_HLIST_NODE(&inst->hlist);
1949 hlist_add_head(&inst->hlist, &rp->free_instances);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001950 }
1951
1952 rp->nmissed = 0;
1953 /* Establish function entry probe point */
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001954 ret = register_kprobe(&rp->kp);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001955 if (ret != 0)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001956 free_rp_inst(rp);
1957 return ret;
1958}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001959EXPORT_SYMBOL_GPL(register_kretprobe);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001960
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001961int register_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001962{
1963 int ret = 0, i;
1964
1965 if (num <= 0)
1966 return -EINVAL;
1967 for (i = 0; i < num; i++) {
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001968 ret = register_kretprobe(rps[i]);
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -07001969 if (ret < 0) {
1970 if (i > 0)
1971 unregister_kretprobes(rps, i);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001972 break;
1973 }
1974 }
1975 return ret;
1976}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001977EXPORT_SYMBOL_GPL(register_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001978
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001979void unregister_kretprobe(struct kretprobe *rp)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001980{
1981 unregister_kretprobes(&rp, 1);
1982}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001983EXPORT_SYMBOL_GPL(unregister_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001984
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001985void unregister_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001986{
1987 int i;
1988
1989 if (num <= 0)
1990 return;
1991 mutex_lock(&kprobe_mutex);
1992 for (i = 0; i < num; i++)
1993 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1994 rps[i]->kp.addr = NULL;
1995 mutex_unlock(&kprobe_mutex);
1996
1997 synchronize_sched();
1998 for (i = 0; i < num; i++) {
1999 if (rps[i]->kp.addr) {
2000 __unregister_kprobe_bottom(&rps[i]->kp);
2001 cleanup_rp_inst(rps[i]);
2002 }
2003 }
2004}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07002005EXPORT_SYMBOL_GPL(unregister_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07002006
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08002007#else /* CONFIG_KRETPROBES */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002008int register_kretprobe(struct kretprobe *rp)
Hien Nguyenb94cce92005-06-23 00:09:19 -07002009{
2010 return -ENOSYS;
2011}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07002012EXPORT_SYMBOL_GPL(register_kretprobe);
Hien Nguyenb94cce92005-06-23 00:09:19 -07002013
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002014int register_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07002015{
2016 return -ENOSYS;
2017}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07002018EXPORT_SYMBOL_GPL(register_kretprobes);
2019
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002020void unregister_kretprobe(struct kretprobe *rp)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07002021{
2022}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07002023EXPORT_SYMBOL_GPL(unregister_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07002024
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002025void unregister_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07002026{
2027}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07002028EXPORT_SYMBOL_GPL(unregister_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07002029
Masami Hiramatsu820aede2014-04-17 17:18:21 +09002030static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002031{
2032 return 0;
2033}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09002034NOKPROBE_SYMBOL(pre_handler_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07002035
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08002036#endif /* CONFIG_KRETPROBES */
Hien Nguyenb94cce92005-06-23 00:09:19 -07002037
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002038/* Set the kprobe gone and remove its instruction buffer. */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002039static void kill_kprobe(struct kprobe *p)
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002040{
2041 struct kprobe *kp;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07002042
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002043 p->flags |= KPROBE_FLAG_GONE;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002044 if (kprobe_aggrprobe(p)) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002045 /*
2046 * If this is an aggr_kprobe, we have to list all the
2047 * chained probes and mark them GONE.
2048 */
2049 list_for_each_entry_rcu(kp, &p->list, list)
2050 kp->flags |= KPROBE_FLAG_GONE;
2051 p->post_handler = NULL;
2052 p->break_handler = NULL;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002053 kill_optimized_kprobe(p);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002054 }
2055 /*
2056 * Here, we can remove insn_slot safely, because no thread calls
2057 * the original probed function (which will be freed soon) any more.
2058 */
2059 arch_remove_kprobe(p);
2060}
2061
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002062/* Disable one kprobe */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002063int disable_kprobe(struct kprobe *kp)
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002064{
2065 int ret = 0;
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002066
2067 mutex_lock(&kprobe_mutex);
2068
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09002069 /* Disable this kprobe */
2070 if (__disable_kprobe(kp) == NULL)
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002071 ret = -EINVAL;
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002072
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002073 mutex_unlock(&kprobe_mutex);
2074 return ret;
2075}
2076EXPORT_SYMBOL_GPL(disable_kprobe);
2077
2078/* Enable one kprobe */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002079int enable_kprobe(struct kprobe *kp)
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002080{
2081 int ret = 0;
2082 struct kprobe *p;
2083
2084 mutex_lock(&kprobe_mutex);
2085
2086 /* Check whether specified probe is valid. */
2087 p = __get_valid_kprobe(kp);
2088 if (unlikely(p == NULL)) {
2089 ret = -EINVAL;
2090 goto out;
2091 }
2092
2093 if (kprobe_gone(kp)) {
2094 /* This kprobe has gone, we couldn't enable it. */
2095 ret = -EINVAL;
2096 goto out;
2097 }
2098
2099 if (p != kp)
2100 kp->flags &= ~KPROBE_FLAG_DISABLED;
2101
2102 if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2103 p->flags &= ~KPROBE_FLAG_DISABLED;
2104 arm_kprobe(p);
2105 }
2106out:
2107 mutex_unlock(&kprobe_mutex);
2108 return ret;
2109}
2110EXPORT_SYMBOL_GPL(enable_kprobe);
2111
Masami Hiramatsu820aede2014-04-17 17:18:21 +09002112void dump_kprobe(struct kprobe *kp)
Frederic Weisbecker24851d22009-08-26 23:38:30 +02002113{
2114 printk(KERN_WARNING "Dumping kprobe:\n");
2115 printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
2116 kp->symbol_name, kp->addr, kp->offset);
2117}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09002118NOKPROBE_SYMBOL(dump_kprobe);
Frederic Weisbecker24851d22009-08-26 23:38:30 +02002119
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002120/*
2121 * Lookup and populate the kprobe_blacklist.
2122 *
2123 * Unlike the kretprobe blacklist, we'll need to determine
2124 * the range of addresses that belong to the said functions,
2125 * since a kprobe need not necessarily be at the beginning
2126 * of a function.
2127 */
2128static int __init populate_kprobe_blacklist(unsigned long *start,
2129 unsigned long *end)
2130{
2131 unsigned long *iter;
2132 struct kprobe_blacklist_entry *ent;
Masami Hiramatsud81b4252014-07-17 11:44:11 +00002133 unsigned long entry, offset = 0, size = 0;
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002134
2135 for (iter = start; iter < end; iter++) {
Masami Hiramatsud81b4252014-07-17 11:44:11 +00002136 entry = arch_deref_entry_point((void *)*iter);
2137
2138 if (!kernel_text_address(entry) ||
2139 !kallsyms_lookup_size_offset(entry, &size, &offset)) {
2140 pr_err("Failed to find blacklist at %p\n",
2141 (void *)entry);
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002142 continue;
2143 }
2144
2145 ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2146 if (!ent)
2147 return -ENOMEM;
Masami Hiramatsud81b4252014-07-17 11:44:11 +00002148 ent->start_addr = entry;
2149 ent->end_addr = entry + size;
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002150 INIT_LIST_HEAD(&ent->list);
2151 list_add_tail(&ent->list, &kprobe_blacklist);
2152 }
2153 return 0;
2154}
2155
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002156/* Module notifier call back, checking kprobes on the module */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002157static int kprobes_module_callback(struct notifier_block *nb,
2158 unsigned long val, void *data)
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002159{
2160 struct module *mod = data;
2161 struct hlist_head *head;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002162 struct kprobe *p;
2163 unsigned int i;
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002164 int checkcore = (val == MODULE_STATE_GOING);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002165
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002166 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002167 return NOTIFY_DONE;
2168
2169 /*
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002170 * When MODULE_STATE_GOING was notified, both of module .text and
2171 * .init.text sections would be freed. When MODULE_STATE_LIVE was
2172 * notified, only .init.text section would be freed. We need to
2173 * disable kprobes which have been inserted in the sections.
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002174 */
2175 mutex_lock(&kprobe_mutex);
2176 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2177 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -08002178 hlist_for_each_entry_rcu(p, head, hlist)
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002179 if (within_module_init((unsigned long)p->addr, mod) ||
2180 (checkcore &&
2181 within_module_core((unsigned long)p->addr, mod))) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002182 /*
2183 * The vaddr this probe is installed will soon
2184 * be vfreed buy not synced to disk. Hence,
2185 * disarming the breakpoint isn't needed.
Steven Rostedt (VMware)545a0282017-05-16 14:58:35 -04002186 *
2187 * Note, this will also move any optimized probes
2188 * that are pending to be removed from their
2189 * corresponding lists to the freeing_list and
2190 * will not be touched by the delayed
2191 * kprobe_optimizer work handler.
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002192 */
2193 kill_kprobe(p);
2194 }
2195 }
2196 mutex_unlock(&kprobe_mutex);
2197 return NOTIFY_DONE;
2198}
2199
2200static struct notifier_block kprobe_module_nb = {
2201 .notifier_call = kprobes_module_callback,
2202 .priority = 0
2203};
2204
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002205/* Markers of _kprobe_blacklist section */
2206extern unsigned long __start_kprobe_blacklist[];
2207extern unsigned long __stop_kprobe_blacklist[];
2208
Linus Torvalds1da177e2005-04-16 15:20:36 -07002209static int __init init_kprobes(void)
2210{
2211 int i, err = 0;
2212
2213 /* FIXME allocate the probe table, currently defined statically */
2214 /* initialize all list heads */
Hien Nguyenb94cce92005-06-23 00:09:19 -07002215 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002216 INIT_HLIST_HEAD(&kprobe_table[i]);
Hien Nguyenb94cce92005-06-23 00:09:19 -07002217 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02002218 raw_spin_lock_init(&(kretprobe_table_locks[i].lock));
Hien Nguyenb94cce92005-06-23 00:09:19 -07002219 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002220
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002221 err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2222 __stop_kprobe_blacklist);
2223 if (err) {
2224 pr_err("kprobes: failed to populate blacklist: %d\n", err);
2225 pr_err("Please take care of using kprobes.\n");
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07002226 }
2227
Masami Hiramatsuf438d912007-10-16 01:27:49 -07002228 if (kretprobe_blacklist_size) {
2229 /* lookup the function address from its name */
2230 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
Naveen N. Rao49e0b462017-04-19 18:21:00 +05302231 kretprobe_blacklist[i].addr =
Naveen N. Rao290e3072017-04-19 18:21:01 +05302232 kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
Masami Hiramatsuf438d912007-10-16 01:27:49 -07002233 if (!kretprobe_blacklist[i].addr)
2234 printk("kretprobe: lookup failed: %s\n",
2235 kretprobe_blacklist[i].name);
2236 }
2237 }
2238
Masami Hiramatsub2be84d2010-02-25 08:34:15 -05002239#if defined(CONFIG_OPTPROBES)
2240#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002241 /* Init kprobe_optinsn_slots */
2242 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2243#endif
Masami Hiramatsub2be84d2010-02-25 08:34:15 -05002244 /* By default, kprobes can be optimized */
2245 kprobes_allow_optimization = true;
2246#endif
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002247
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002248 /* By default, kprobes are armed */
2249 kprobes_all_disarmed = false;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002250
Rusty Lynch67729262005-07-05 18:54:50 -07002251 err = arch_init_kprobes();
Rusty Lynch802eae72005-06-27 15:17:08 -07002252 if (!err)
2253 err = register_die_notifier(&kprobe_exceptions_nb);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002254 if (!err)
2255 err = register_module_notifier(&kprobe_module_nb);
2256
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07002257 kprobes_initialized = (err == 0);
Rusty Lynch802eae72005-06-27 15:17:08 -07002258
Ananth N Mavinakayanahalli8c1c9352008-01-30 13:32:53 +01002259 if (!err)
2260 init_test_probes();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002261 return err;
2262}
2263
Srinivasa Ds346fd592007-02-20 13:57:54 -08002264#ifdef CONFIG_DEBUG_FS
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002265static void report_probe(struct seq_file *pi, struct kprobe *p,
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002266 const char *sym, int offset, char *modname, struct kprobe *pp)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002267{
2268 char *kprobe_type;
2269
2270 if (p->pre_handler == pre_handler_kretprobe)
2271 kprobe_type = "r";
2272 else if (p->pre_handler == setjmp_pre_handler)
2273 kprobe_type = "j";
2274 else
2275 kprobe_type = "k";
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002276
Srinivasa Ds346fd592007-02-20 13:57:54 -08002277 if (sym)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002278 seq_printf(pi, "%p %s %s+0x%x %s ",
Masami Hiramatsude5bd882009-04-06 19:01:02 -07002279 p->addr, kprobe_type, sym, offset,
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002280 (modname ? modname : " "));
Srinivasa Ds346fd592007-02-20 13:57:54 -08002281 else
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002282 seq_printf(pi, "%p %s %p ",
2283 p->addr, kprobe_type, p->addr);
2284
2285 if (!pp)
2286 pp = p;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002287 seq_printf(pi, "%s%s%s%s\n",
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002288 (kprobe_gone(p) ? "[GONE]" : ""),
2289 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002290 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2291 (kprobe_ftrace(pp) ? "[FTRACE]" : ""));
Srinivasa Ds346fd592007-02-20 13:57:54 -08002292}
2293
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002294static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002295{
2296 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2297}
2298
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002299static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002300{
2301 (*pos)++;
2302 if (*pos >= KPROBE_TABLE_SIZE)
2303 return NULL;
2304 return pos;
2305}
2306
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002307static void kprobe_seq_stop(struct seq_file *f, void *v)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002308{
2309 /* Nothing to do */
2310}
2311
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002312static int show_kprobe_addr(struct seq_file *pi, void *v)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002313{
2314 struct hlist_head *head;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002315 struct kprobe *p, *kp;
2316 const char *sym = NULL;
2317 unsigned int i = *(loff_t *) v;
Alexey Dobriyanffb45122007-05-08 00:28:41 -07002318 unsigned long offset = 0;
Joe Marioab767862013-11-12 15:10:23 -08002319 char *modname, namebuf[KSYM_NAME_LEN];
Srinivasa Ds346fd592007-02-20 13:57:54 -08002320
2321 head = &kprobe_table[i];
2322 preempt_disable();
Sasha Levinb67bfe02013-02-27 17:06:00 -08002323 hlist_for_each_entry_rcu(p, head, hlist) {
Alexey Dobriyanffb45122007-05-08 00:28:41 -07002324 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
Srinivasa Ds346fd592007-02-20 13:57:54 -08002325 &offset, &modname, namebuf);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002326 if (kprobe_aggrprobe(p)) {
Srinivasa Ds346fd592007-02-20 13:57:54 -08002327 list_for_each_entry_rcu(kp, &p->list, list)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002328 report_probe(pi, kp, sym, offset, modname, p);
Srinivasa Ds346fd592007-02-20 13:57:54 -08002329 } else
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002330 report_probe(pi, p, sym, offset, modname, NULL);
Srinivasa Ds346fd592007-02-20 13:57:54 -08002331 }
2332 preempt_enable();
2333 return 0;
2334}
2335
James Morris88e9d342009-09-22 16:43:43 -07002336static const struct seq_operations kprobes_seq_ops = {
Srinivasa Ds346fd592007-02-20 13:57:54 -08002337 .start = kprobe_seq_start,
2338 .next = kprobe_seq_next,
2339 .stop = kprobe_seq_stop,
2340 .show = show_kprobe_addr
2341};
2342
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002343static int kprobes_open(struct inode *inode, struct file *filp)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002344{
2345 return seq_open(filp, &kprobes_seq_ops);
2346}
2347
Alexey Dobriyan828c0952009-10-01 15:43:56 -07002348static const struct file_operations debugfs_kprobes_operations = {
Srinivasa Ds346fd592007-02-20 13:57:54 -08002349 .open = kprobes_open,
2350 .read = seq_read,
2351 .llseek = seq_lseek,
2352 .release = seq_release,
2353};
2354
Masami Hiramatsu63724742014-04-17 17:18:49 +09002355/* kprobes/blacklist -- shows which functions can not be probed */
2356static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
2357{
2358 return seq_list_start(&kprobe_blacklist, *pos);
2359}
2360
2361static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
2362{
2363 return seq_list_next(v, &kprobe_blacklist, pos);
2364}
2365
2366static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
2367{
2368 struct kprobe_blacklist_entry *ent =
2369 list_entry(v, struct kprobe_blacklist_entry, list);
2370
2371 seq_printf(m, "0x%p-0x%p\t%ps\n", (void *)ent->start_addr,
2372 (void *)ent->end_addr, (void *)ent->start_addr);
2373 return 0;
2374}
2375
2376static const struct seq_operations kprobe_blacklist_seq_ops = {
2377 .start = kprobe_blacklist_seq_start,
2378 .next = kprobe_blacklist_seq_next,
2379 .stop = kprobe_seq_stop, /* Reuse void function */
2380 .show = kprobe_blacklist_seq_show,
2381};
2382
2383static int kprobe_blacklist_open(struct inode *inode, struct file *filp)
2384{
2385 return seq_open(filp, &kprobe_blacklist_seq_ops);
2386}
2387
2388static const struct file_operations debugfs_kprobe_blacklist_ops = {
2389 .open = kprobe_blacklist_open,
2390 .read = seq_read,
2391 .llseek = seq_lseek,
2392 .release = seq_release,
2393};
2394
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002395static void arm_all_kprobes(void)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002396{
2397 struct hlist_head *head;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002398 struct kprobe *p;
2399 unsigned int i;
2400
2401 mutex_lock(&kprobe_mutex);
2402
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002403 /* If kprobes are armed, just return */
2404 if (!kprobes_all_disarmed)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002405 goto already_enabled;
2406
Wang Nan977ad482015-02-13 14:40:24 -08002407 /*
2408 * optimize_kprobe() called by arm_kprobe() checks
2409 * kprobes_all_disarmed, so set kprobes_all_disarmed before
2410 * arm_kprobe.
2411 */
2412 kprobes_all_disarmed = false;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002413 /* Arming kprobes doesn't optimize kprobe itself */
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002414 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2415 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -08002416 hlist_for_each_entry_rcu(p, head, hlist)
Masami Hiramatsude5bd882009-04-06 19:01:02 -07002417 if (!kprobe_disabled(p))
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002418 arm_kprobe(p);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002419 }
2420
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002421 printk(KERN_INFO "Kprobes globally enabled\n");
2422
2423already_enabled:
2424 mutex_unlock(&kprobe_mutex);
2425 return;
2426}
2427
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002428static void disarm_all_kprobes(void)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002429{
2430 struct hlist_head *head;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002431 struct kprobe *p;
2432 unsigned int i;
2433
2434 mutex_lock(&kprobe_mutex);
2435
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002436 /* If kprobes are already disarmed, just return */
Masami Hiramatsu6274de42010-12-03 18:54:09 +09002437 if (kprobes_all_disarmed) {
2438 mutex_unlock(&kprobe_mutex);
2439 return;
2440 }
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002441
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002442 kprobes_all_disarmed = true;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002443 printk(KERN_INFO "Kprobes globally disabled\n");
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002444
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002445 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2446 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -08002447 hlist_for_each_entry_rcu(p, head, hlist) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -07002448 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002449 disarm_kprobe(p, false);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002450 }
2451 }
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002452 mutex_unlock(&kprobe_mutex);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002453
Masami Hiramatsu6274de42010-12-03 18:54:09 +09002454 /* Wait for disarming all kprobes by optimizer */
2455 wait_for_kprobe_optimizer();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002456}
2457
2458/*
2459 * XXX: The debugfs bool file interface doesn't allow for callbacks
2460 * when the bool state is switched. We can reuse that facility when
2461 * available
2462 */
2463static ssize_t read_enabled_file_bool(struct file *file,
2464 char __user *user_buf, size_t count, loff_t *ppos)
2465{
2466 char buf[3];
2467
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002468 if (!kprobes_all_disarmed)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002469 buf[0] = '1';
2470 else
2471 buf[0] = '0';
2472 buf[1] = '\n';
2473 buf[2] = 0x00;
2474 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2475}
2476
2477static ssize_t write_enabled_file_bool(struct file *file,
2478 const char __user *user_buf, size_t count, loff_t *ppos)
2479{
2480 char buf[32];
Stephen Boydefeb1562012-01-12 17:17:11 -08002481 size_t buf_size;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002482
2483 buf_size = min(count, (sizeof(buf)-1));
2484 if (copy_from_user(buf, user_buf, buf_size))
2485 return -EFAULT;
2486
Mathias Krause10fb46d2013-07-03 15:05:39 -07002487 buf[buf_size] = '\0';
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002488 switch (buf[0]) {
2489 case 'y':
2490 case 'Y':
2491 case '1':
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002492 arm_all_kprobes();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002493 break;
2494 case 'n':
2495 case 'N':
2496 case '0':
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002497 disarm_all_kprobes();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002498 break;
Mathias Krause10fb46d2013-07-03 15:05:39 -07002499 default:
2500 return -EINVAL;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002501 }
2502
2503 return count;
2504}
2505
Alexey Dobriyan828c0952009-10-01 15:43:56 -07002506static const struct file_operations fops_kp = {
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002507 .read = read_enabled_file_bool,
2508 .write = write_enabled_file_bool,
Arnd Bergmann6038f372010-08-15 18:52:59 +02002509 .llseek = default_llseek,
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002510};
2511
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002512static int __init debugfs_kprobe_init(void)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002513{
2514 struct dentry *dir, *file;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002515 unsigned int value = 1;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002516
2517 dir = debugfs_create_dir("kprobes", NULL);
2518 if (!dir)
2519 return -ENOMEM;
2520
Randy Dunlape3869792007-05-08 00:27:01 -07002521 file = debugfs_create_file("list", 0444, dir, NULL,
Srinivasa Ds346fd592007-02-20 13:57:54 -08002522 &debugfs_kprobes_operations);
Masami Hiramatsu63724742014-04-17 17:18:49 +09002523 if (!file)
2524 goto error;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002525
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002526 file = debugfs_create_file("enabled", 0600, dir,
2527 &value, &fops_kp);
Masami Hiramatsu63724742014-04-17 17:18:49 +09002528 if (!file)
2529 goto error;
2530
2531 file = debugfs_create_file("blacklist", 0444, dir, NULL,
2532 &debugfs_kprobe_blacklist_ops);
2533 if (!file)
2534 goto error;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002535
Srinivasa Ds346fd592007-02-20 13:57:54 -08002536 return 0;
Masami Hiramatsu63724742014-04-17 17:18:49 +09002537
2538error:
2539 debugfs_remove(dir);
2540 return -ENOMEM;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002541}
2542
2543late_initcall(debugfs_kprobe_init);
2544#endif /* CONFIG_DEBUG_FS */
2545
2546module_init(init_kprobes);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002547
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07002548/* defined in arch/.../kernel/kprobes.c */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002549EXPORT_SYMBOL_GPL(jprobe_return);