blob: c3e4575e7829e1e807652ff2bc86fd0220d1a8a8 [file] [log] [blame]
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +02001/*
2 * Infrastructure for profiling code inserted by 'gcc -pg'.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code in the latency_tracer, that is:
11 *
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
14 */
15
Steven Rostedt3d083392008-05-12 21:20:42 +020016#include <linux/stop_machine.h>
17#include <linux/clocksource.h>
18#include <linux/kallsyms.h>
Steven Rostedt5072c592008-05-12 21:20:43 +020019#include <linux/seq_file.h>
Frederic Weisbecker4a2b8dd2009-01-14 13:33:27 -080020#include <linux/suspend.h>
Steven Rostedt5072c592008-05-12 21:20:43 +020021#include <linux/debugfs.h>
Steven Rostedt3d083392008-05-12 21:20:42 +020022#include <linux/hardirq.h>
Ingo Molnar2d8b8202008-02-23 16:55:50 +010023#include <linux/kthread.h>
Steven Rostedt5072c592008-05-12 21:20:43 +020024#include <linux/uaccess.h>
Ingo Molnar2d8b8202008-02-23 16:55:50 +010025#include <linux/ftrace.h>
Steven Rostedtb0fc4942008-05-12 21:20:43 +020026#include <linux/sysctl.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090027#include <linux/slab.h>
Steven Rostedt5072c592008-05-12 21:20:43 +020028#include <linux/ctype.h>
Steven Rostedt3d083392008-05-12 21:20:42 +020029#include <linux/list.h>
Steven Rostedt59df055f2009-02-14 15:29:06 -050030#include <linux/hash.h>
Paul E. McKenney3f379b02010-03-05 15:03:25 -080031#include <linux/rcupdate.h>
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +020032
Steven Rostedtad8d75f2009-04-14 19:39:12 -040033#include <trace/events/sched.h>
Steven Rostedt8aef2d22009-03-24 01:10:15 -040034
Steven Rostedt2af15d62009-05-28 13:37:24 -040035#include <asm/setup.h>
Abhishek Sagar395a59d2008-06-21 23:47:27 +053036
Steven Rostedt0706f1c2009-03-23 23:12:58 -040037#include "trace_output.h"
Steven Rostedtbac429f2009-03-20 12:50:56 -040038#include "trace_stat.h"
Steven Rostedt3d083392008-05-12 21:20:42 +020039
Steven Rostedt69128962008-10-23 09:33:03 -040040#define FTRACE_WARN_ON(cond) \
Steven Rostedt0778d9a2011-04-29 10:36:31 -040041 ({ \
42 int ___r = cond; \
43 if (WARN_ON(___r)) \
Steven Rostedt69128962008-10-23 09:33:03 -040044 ftrace_kill(); \
Steven Rostedt0778d9a2011-04-29 10:36:31 -040045 ___r; \
46 })
Steven Rostedt69128962008-10-23 09:33:03 -040047
48#define FTRACE_WARN_ON_ONCE(cond) \
Steven Rostedt0778d9a2011-04-29 10:36:31 -040049 ({ \
50 int ___r = cond; \
51 if (WARN_ON_ONCE(___r)) \
Steven Rostedt69128962008-10-23 09:33:03 -040052 ftrace_kill(); \
Steven Rostedt0778d9a2011-04-29 10:36:31 -040053 ___r; \
54 })
Steven Rostedt69128962008-10-23 09:33:03 -040055
Steven Rostedt8fc0c702009-02-16 15:28:00 -050056/* hash bits for specific function selection */
57#define FTRACE_HASH_BITS 7
58#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
Steven Rostedt33dc9b12011-05-02 17:34:47 -040059#define FTRACE_HASH_DEFAULT_BITS 10
60#define FTRACE_HASH_MAX_BITS 12
Steven Rostedt8fc0c702009-02-16 15:28:00 -050061
Steven Rostedt4eebcc82008-05-12 21:20:48 +020062/* ftrace_enabled is a method to turn ftrace on or off */
63int ftrace_enabled __read_mostly;
Steven Rostedtd61f82d2008-05-12 21:20:43 +020064static int last_ftrace_enabled;
Steven Rostedtb0fc4942008-05-12 21:20:43 +020065
Steven Rostedt60a7ecf2008-11-05 16:05:44 -050066/* Quick disabling of function tracer. */
67int function_trace_stop;
68
jolsa@redhat.com756d17e2009-10-13 16:33:52 -040069/* List for set_ftrace_pid's pids. */
70LIST_HEAD(ftrace_pids);
71struct ftrace_pid {
72 struct list_head list;
73 struct pid *pid;
74};
75
Steven Rostedt4eebcc82008-05-12 21:20:48 +020076/*
77 * ftrace_disabled is set when an anomaly is discovered.
78 * ftrace_disabled is much stronger than ftrace_enabled.
79 */
80static int ftrace_disabled __read_mostly;
81
Steven Rostedt52baf112009-02-14 01:15:39 -050082static DEFINE_MUTEX(ftrace_lock);
Steven Rostedtb0fc4942008-05-12 21:20:43 +020083
Paul McQuadebd38c0e2011-05-31 20:51:55 +010084static struct ftrace_ops ftrace_list_end __read_mostly = {
Steven Rostedtfb9fb012009-03-25 13:26:41 -040085 .func = ftrace_stub,
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +020086};
87
Steven Rostedtb8489142011-05-04 09:27:52 -040088static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
89static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +020090ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
Steven Rostedt6331c282011-07-13 15:11:02 -040091static ftrace_func_t __ftrace_trace_function_delay __read_mostly = ftrace_stub;
Steven Rostedt60a7ecf2008-11-05 16:05:44 -050092ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
Steven Rostedtdf4fc312008-11-26 00:16:23 -050093ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
Steven Rostedt2b499382011-05-03 22:49:52 -040094static struct ftrace_ops global_ops;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +020095
Steven Rostedtb8489142011-05-04 09:27:52 -040096static void
97ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
98
Paul E. McKenney3f379b02010-03-05 15:03:25 -080099/*
Steven Rostedtb8489142011-05-04 09:27:52 -0400100 * Traverse the ftrace_global_list, invoking all entries. The reason that we
Paul E. McKenney3f379b02010-03-05 15:03:25 -0800101 * can use rcu_dereference_raw() is that elements removed from this list
102 * are simply leaked, so there is no need to interact with a grace-period
103 * mechanism. The rcu_dereference_raw() calls are needed to handle
Steven Rostedtb8489142011-05-04 09:27:52 -0400104 * concurrent insertions into the ftrace_global_list.
Paul E. McKenney3f379b02010-03-05 15:03:25 -0800105 *
106 * Silly Alpha and silly pointer-speculation compiler optimizations!
107 */
Steven Rostedtb8489142011-05-04 09:27:52 -0400108static void ftrace_global_list_func(unsigned long ip,
109 unsigned long parent_ip)
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200110{
Steven Rostedtb1cff0a2011-05-25 14:27:43 -0400111 struct ftrace_ops *op;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200112
Steven Rostedtb1cff0a2011-05-25 14:27:43 -0400113 if (unlikely(trace_recursion_test(TRACE_GLOBAL_BIT)))
114 return;
115
116 trace_recursion_set(TRACE_GLOBAL_BIT);
117 op = rcu_dereference_raw(ftrace_global_list); /*see above*/
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200118 while (op != &ftrace_list_end) {
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200119 op->func(ip, parent_ip);
Paul E. McKenney3f379b02010-03-05 15:03:25 -0800120 op = rcu_dereference_raw(op->next); /*see above*/
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200121 };
Steven Rostedtb1cff0a2011-05-25 14:27:43 -0400122 trace_recursion_clear(TRACE_GLOBAL_BIT);
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200123}
124
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500125static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
126{
Steven Rostedt0ef8cde2008-12-03 15:36:58 -0500127 if (!test_tsk_trace_trace(current))
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500128 return;
129
130 ftrace_pid_function(ip, parent_ip);
131}
132
133static void set_ftrace_pid_function(ftrace_func_t func)
134{
135 /* do not set ftrace_pid_function to itself! */
136 if (func != ftrace_pid_func)
137 ftrace_pid_function = func;
138}
139
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200140/**
Steven Rostedt3d083392008-05-12 21:20:42 +0200141 * clear_ftrace_function - reset the ftrace function
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200142 *
Steven Rostedt3d083392008-05-12 21:20:42 +0200143 * This NULLs the ftrace function and in essence stops
144 * tracing. There may be lag
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200145 */
Steven Rostedt3d083392008-05-12 21:20:42 +0200146void clear_ftrace_function(void)
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200147{
Steven Rostedt3d083392008-05-12 21:20:42 +0200148 ftrace_trace_function = ftrace_stub;
Steven Rostedt60a7ecf2008-11-05 16:05:44 -0500149 __ftrace_trace_function = ftrace_stub;
Steven Rostedt6331c282011-07-13 15:11:02 -0400150 __ftrace_trace_function_delay = ftrace_stub;
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500151 ftrace_pid_function = ftrace_stub;
Steven Rostedt3d083392008-05-12 21:20:42 +0200152}
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200153
Steven Rostedt6331c282011-07-13 15:11:02 -0400154#undef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
Steven Rostedt60a7ecf2008-11-05 16:05:44 -0500155#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
156/*
157 * For those archs that do not test ftrace_trace_stop in their
158 * mcount call site, we need to do it from C.
159 */
160static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
161{
162 if (function_trace_stop)
163 return;
164
165 __ftrace_trace_function(ip, parent_ip);
166}
167#endif
168
Steven Rostedt2b499382011-05-03 22:49:52 -0400169static void update_global_ops(void)
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400170{
171 ftrace_func_t func;
172
173 /*
174 * If there's only one function registered, then call that
175 * function directly. Otherwise, we need to iterate over the
176 * registered callers.
177 */
Steven Rostedtb8489142011-05-04 09:27:52 -0400178 if (ftrace_global_list == &ftrace_list_end ||
179 ftrace_global_list->next == &ftrace_list_end)
180 func = ftrace_global_list->func;
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400181 else
Steven Rostedtb8489142011-05-04 09:27:52 -0400182 func = ftrace_global_list_func;
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400183
184 /* If we filter on pids, update to use the pid function */
185 if (!list_empty(&ftrace_pids)) {
186 set_ftrace_pid_function(func);
187 func = ftrace_pid_func;
188 }
Steven Rostedt2b499382011-05-03 22:49:52 -0400189
190 global_ops.func = func;
191}
192
193static void update_ftrace_function(void)
194{
195 ftrace_func_t func;
196
197 update_global_ops();
198
Steven Rostedtcdbe61b2011-05-05 21:14:55 -0400199 /*
200 * If we are at the end of the list and this ops is
201 * not dynamic, then have the mcount trampoline call
202 * the function directly
203 */
Steven Rostedtb8489142011-05-04 09:27:52 -0400204 if (ftrace_ops_list == &ftrace_list_end ||
Steven Rostedtcdbe61b2011-05-05 21:14:55 -0400205 (ftrace_ops_list->next == &ftrace_list_end &&
206 !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC)))
Steven Rostedtb8489142011-05-04 09:27:52 -0400207 func = ftrace_ops_list->func;
208 else
209 func = ftrace_ops_list_func;
Steven Rostedt2b499382011-05-03 22:49:52 -0400210
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400211#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
212 ftrace_trace_function = func;
213#else
Steven Rostedt6331c282011-07-13 15:11:02 -0400214#ifdef CONFIG_DYNAMIC_FTRACE
215 /* do not update till all functions have been modified */
216 __ftrace_trace_function_delay = func;
217#else
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400218 __ftrace_trace_function = func;
Steven Rostedt6331c282011-07-13 15:11:02 -0400219#endif
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400220 ftrace_trace_function = ftrace_test_stop_func;
221#endif
222}
223
Steven Rostedt2b499382011-05-03 22:49:52 -0400224static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
Steven Rostedt3d083392008-05-12 21:20:42 +0200225{
Steven Rostedt2b499382011-05-03 22:49:52 -0400226 ops->next = *list;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200227 /*
Steven Rostedtb8489142011-05-04 09:27:52 -0400228 * We are entering ops into the list but another
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200229 * CPU might be walking that list. We need to make sure
230 * the ops->next pointer is valid before another CPU sees
Steven Rostedtb8489142011-05-04 09:27:52 -0400231 * the ops pointer included into the list.
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200232 */
Steven Rostedt2b499382011-05-03 22:49:52 -0400233 rcu_assign_pointer(*list, ops);
234}
Steven Rostedt3d083392008-05-12 21:20:42 +0200235
Steven Rostedt2b499382011-05-03 22:49:52 -0400236static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
237{
238 struct ftrace_ops **p;
239
240 /*
241 * If we are removing the last function, then simply point
242 * to the ftrace_stub.
243 */
244 if (*list == ops && ops->next == &ftrace_list_end) {
245 *list = &ftrace_list_end;
246 return 0;
247 }
248
249 for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
250 if (*p == ops)
251 break;
252
253 if (*p != ops)
254 return -1;
255
256 *p = (*p)->next;
257 return 0;
258}
259
260static int __register_ftrace_function(struct ftrace_ops *ops)
261{
262 if (ftrace_disabled)
263 return -ENODEV;
264
265 if (FTRACE_WARN_ON(ops == &global_ops))
266 return -EINVAL;
267
Steven Rostedtb8489142011-05-04 09:27:52 -0400268 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
269 return -EBUSY;
270
Steven Rostedtcdbe61b2011-05-05 21:14:55 -0400271 if (!core_kernel_data((unsigned long)ops))
272 ops->flags |= FTRACE_OPS_FL_DYNAMIC;
273
Steven Rostedtb8489142011-05-04 09:27:52 -0400274 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
275 int first = ftrace_global_list == &ftrace_list_end;
276 add_ftrace_ops(&ftrace_global_list, ops);
277 ops->flags |= FTRACE_OPS_FL_ENABLED;
278 if (first)
279 add_ftrace_ops(&ftrace_ops_list, &global_ops);
280 } else
281 add_ftrace_ops(&ftrace_ops_list, ops);
282
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400283 if (ftrace_enabled)
284 update_ftrace_function();
Steven Rostedt3d083392008-05-12 21:20:42 +0200285
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200286 return 0;
287}
288
Ingo Molnare309b412008-05-12 21:20:51 +0200289static int __unregister_ftrace_function(struct ftrace_ops *ops)
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200290{
Steven Rostedt2b499382011-05-03 22:49:52 -0400291 int ret;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200292
Steven Rostedt2b499382011-05-03 22:49:52 -0400293 if (ftrace_disabled)
294 return -ENODEV;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200295
Steven Rostedtb8489142011-05-04 09:27:52 -0400296 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
297 return -EBUSY;
298
Steven Rostedt2b499382011-05-03 22:49:52 -0400299 if (FTRACE_WARN_ON(ops == &global_ops))
300 return -EINVAL;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200301
Steven Rostedtb8489142011-05-04 09:27:52 -0400302 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
303 ret = remove_ftrace_ops(&ftrace_global_list, ops);
304 if (!ret && ftrace_global_list == &ftrace_list_end)
305 ret = remove_ftrace_ops(&ftrace_ops_list, &global_ops);
306 if (!ret)
307 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
308 } else
309 ret = remove_ftrace_ops(&ftrace_ops_list, ops);
310
Steven Rostedt2b499382011-05-03 22:49:52 -0400311 if (ret < 0)
312 return ret;
Steven Rostedtb8489142011-05-04 09:27:52 -0400313
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400314 if (ftrace_enabled)
315 update_ftrace_function();
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +0200316
Steven Rostedtcdbe61b2011-05-05 21:14:55 -0400317 /*
318 * Dynamic ops may be freed, we must make sure that all
319 * callers are done before leaving this function.
320 */
321 if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
322 synchronize_sched();
323
Steven Rostedte6ea44e2009-02-14 01:42:44 -0500324 return 0;
Steven Rostedt3d083392008-05-12 21:20:42 +0200325}
326
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500327static void ftrace_update_pid_func(void)
328{
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400329 /* Only do something if we are tracing something */
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500330 if (ftrace_trace_function == ftrace_stub)
KOSAKI Motohiro10dd3eb2009-03-06 15:29:04 +0900331 return;
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500332
Steven Rostedt491d0dc2011-04-27 21:43:36 -0400333 update_ftrace_function();
Steven Rostedtdf4fc312008-11-26 00:16:23 -0500334}
335
Steven Rostedt493762f2009-03-23 17:12:36 -0400336#ifdef CONFIG_FUNCTION_PROFILER
337struct ftrace_profile {
338 struct hlist_node node;
339 unsigned long ip;
340 unsigned long counter;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400341#ifdef CONFIG_FUNCTION_GRAPH_TRACER
342 unsigned long long time;
Chase Douglase330b3b2010-04-26 14:02:05 -0400343 unsigned long long time_squared;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400344#endif
Steven Rostedt493762f2009-03-23 17:12:36 -0400345};
346
347struct ftrace_profile_page {
348 struct ftrace_profile_page *next;
349 unsigned long index;
350 struct ftrace_profile records[];
351};
352
Steven Rostedtcafb1682009-03-24 20:50:39 -0400353struct ftrace_profile_stat {
354 atomic_t disabled;
355 struct hlist_head *hash;
356 struct ftrace_profile_page *pages;
357 struct ftrace_profile_page *start;
358 struct tracer_stat stat;
359};
360
Steven Rostedt493762f2009-03-23 17:12:36 -0400361#define PROFILE_RECORDS_SIZE \
362 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
363
364#define PROFILES_PER_PAGE \
365 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
366
Steven Rostedtfb9fb012009-03-25 13:26:41 -0400367static int ftrace_profile_bits __read_mostly;
368static int ftrace_profile_enabled __read_mostly;
369
370/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
Steven Rostedt493762f2009-03-23 17:12:36 -0400371static DEFINE_MUTEX(ftrace_profile_lock);
372
Steven Rostedtcafb1682009-03-24 20:50:39 -0400373static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
Steven Rostedt493762f2009-03-23 17:12:36 -0400374
375#define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
376
Steven Rostedt493762f2009-03-23 17:12:36 -0400377static void *
378function_stat_next(void *v, int idx)
379{
380 struct ftrace_profile *rec = v;
381 struct ftrace_profile_page *pg;
382
383 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
384
385 again:
Li Zefan0296e422009-06-26 11:15:37 +0800386 if (idx != 0)
387 rec++;
388
Steven Rostedt493762f2009-03-23 17:12:36 -0400389 if ((void *)rec >= (void *)&pg->records[pg->index]) {
390 pg = pg->next;
391 if (!pg)
392 return NULL;
393 rec = &pg->records[0];
394 if (!rec->counter)
395 goto again;
396 }
397
398 return rec;
399}
400
401static void *function_stat_start(struct tracer_stat *trace)
402{
Steven Rostedtcafb1682009-03-24 20:50:39 -0400403 struct ftrace_profile_stat *stat =
404 container_of(trace, struct ftrace_profile_stat, stat);
405
406 if (!stat || !stat->start)
407 return NULL;
408
409 return function_stat_next(&stat->start->records[0], 0);
Steven Rostedt493762f2009-03-23 17:12:36 -0400410}
411
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400412#ifdef CONFIG_FUNCTION_GRAPH_TRACER
413/* function graph compares on total time */
414static int function_stat_cmp(void *p1, void *p2)
415{
416 struct ftrace_profile *a = p1;
417 struct ftrace_profile *b = p2;
418
419 if (a->time < b->time)
420 return -1;
421 if (a->time > b->time)
422 return 1;
423 else
424 return 0;
425}
426#else
427/* not function graph compares against hits */
Steven Rostedt493762f2009-03-23 17:12:36 -0400428static int function_stat_cmp(void *p1, void *p2)
429{
430 struct ftrace_profile *a = p1;
431 struct ftrace_profile *b = p2;
432
433 if (a->counter < b->counter)
434 return -1;
435 if (a->counter > b->counter)
436 return 1;
437 else
438 return 0;
439}
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400440#endif
Steven Rostedt493762f2009-03-23 17:12:36 -0400441
442static int function_stat_headers(struct seq_file *m)
443{
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400444#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Steven Rostedt34886c82009-03-25 21:00:47 -0400445 seq_printf(m, " Function "
Chase Douglase330b3b2010-04-26 14:02:05 -0400446 "Hit Time Avg s^2\n"
Steven Rostedt34886c82009-03-25 21:00:47 -0400447 " -------- "
Chase Douglase330b3b2010-04-26 14:02:05 -0400448 "--- ---- --- ---\n");
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400449#else
Steven Rostedt493762f2009-03-23 17:12:36 -0400450 seq_printf(m, " Function Hit\n"
451 " -------- ---\n");
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400452#endif
Steven Rostedt493762f2009-03-23 17:12:36 -0400453 return 0;
454}
455
456static int function_stat_show(struct seq_file *m, void *v)
457{
458 struct ftrace_profile *rec = v;
459 char str[KSYM_SYMBOL_LEN];
Li Zefan3aaba202010-08-23 16:50:12 +0800460 int ret = 0;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400461#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Steven Rostedt34886c82009-03-25 21:00:47 -0400462 static struct trace_seq s;
463 unsigned long long avg;
Chase Douglase330b3b2010-04-26 14:02:05 -0400464 unsigned long long stddev;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400465#endif
Li Zefan3aaba202010-08-23 16:50:12 +0800466 mutex_lock(&ftrace_profile_lock);
467
468 /* we raced with function_profile_reset() */
469 if (unlikely(rec->counter == 0)) {
470 ret = -EBUSY;
471 goto out;
472 }
Steven Rostedt493762f2009-03-23 17:12:36 -0400473
474 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400475 seq_printf(m, " %-30.30s %10lu", str, rec->counter);
Steven Rostedt493762f2009-03-23 17:12:36 -0400476
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400477#ifdef CONFIG_FUNCTION_GRAPH_TRACER
478 seq_printf(m, " ");
Steven Rostedt34886c82009-03-25 21:00:47 -0400479 avg = rec->time;
480 do_div(avg, rec->counter);
481
Chase Douglase330b3b2010-04-26 14:02:05 -0400482 /* Sample standard deviation (s^2) */
483 if (rec->counter <= 1)
484 stddev = 0;
485 else {
486 stddev = rec->time_squared - rec->counter * avg * avg;
487 /*
488 * Divide only 1000 for ns^2 -> us^2 conversion.
489 * trace_print_graph_duration will divide 1000 again.
490 */
491 do_div(stddev, (rec->counter - 1) * 1000);
492 }
493
Steven Rostedt34886c82009-03-25 21:00:47 -0400494 trace_seq_init(&s);
495 trace_print_graph_duration(rec->time, &s);
496 trace_seq_puts(&s, " ");
497 trace_print_graph_duration(avg, &s);
Chase Douglase330b3b2010-04-26 14:02:05 -0400498 trace_seq_puts(&s, " ");
499 trace_print_graph_duration(stddev, &s);
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400500 trace_print_seq(m, &s);
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400501#endif
502 seq_putc(m, '\n');
Li Zefan3aaba202010-08-23 16:50:12 +0800503out:
504 mutex_unlock(&ftrace_profile_lock);
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400505
Li Zefan3aaba202010-08-23 16:50:12 +0800506 return ret;
Steven Rostedt493762f2009-03-23 17:12:36 -0400507}
508
Steven Rostedtcafb1682009-03-24 20:50:39 -0400509static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
Steven Rostedt493762f2009-03-23 17:12:36 -0400510{
511 struct ftrace_profile_page *pg;
512
Steven Rostedtcafb1682009-03-24 20:50:39 -0400513 pg = stat->pages = stat->start;
Steven Rostedt493762f2009-03-23 17:12:36 -0400514
515 while (pg) {
516 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
517 pg->index = 0;
518 pg = pg->next;
519 }
520
Steven Rostedtcafb1682009-03-24 20:50:39 -0400521 memset(stat->hash, 0,
Steven Rostedt493762f2009-03-23 17:12:36 -0400522 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
523}
524
Steven Rostedtcafb1682009-03-24 20:50:39 -0400525int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
Steven Rostedt493762f2009-03-23 17:12:36 -0400526{
527 struct ftrace_profile_page *pg;
Steven Rostedt318e0a72009-03-25 20:06:34 -0400528 int functions;
529 int pages;
Steven Rostedt493762f2009-03-23 17:12:36 -0400530 int i;
531
532 /* If we already allocated, do nothing */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400533 if (stat->pages)
Steven Rostedt493762f2009-03-23 17:12:36 -0400534 return 0;
535
Steven Rostedtcafb1682009-03-24 20:50:39 -0400536 stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
537 if (!stat->pages)
Steven Rostedt493762f2009-03-23 17:12:36 -0400538 return -ENOMEM;
539
Steven Rostedt318e0a72009-03-25 20:06:34 -0400540#ifdef CONFIG_DYNAMIC_FTRACE
541 functions = ftrace_update_tot_cnt;
542#else
543 /*
544 * We do not know the number of functions that exist because
545 * dynamic tracing is what counts them. With past experience
546 * we have around 20K functions. That should be more than enough.
547 * It is highly unlikely we will execute every function in
548 * the kernel.
549 */
550 functions = 20000;
551#endif
552
Steven Rostedtcafb1682009-03-24 20:50:39 -0400553 pg = stat->start = stat->pages;
Steven Rostedt493762f2009-03-23 17:12:36 -0400554
Steven Rostedt318e0a72009-03-25 20:06:34 -0400555 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
556
557 for (i = 0; i < pages; i++) {
Steven Rostedt493762f2009-03-23 17:12:36 -0400558 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
Steven Rostedt493762f2009-03-23 17:12:36 -0400559 if (!pg->next)
Steven Rostedt318e0a72009-03-25 20:06:34 -0400560 goto out_free;
Steven Rostedt493762f2009-03-23 17:12:36 -0400561 pg = pg->next;
562 }
563
564 return 0;
Steven Rostedt318e0a72009-03-25 20:06:34 -0400565
566 out_free:
567 pg = stat->start;
568 while (pg) {
569 unsigned long tmp = (unsigned long)pg;
570
571 pg = pg->next;
572 free_page(tmp);
573 }
574
575 free_page((unsigned long)stat->pages);
576 stat->pages = NULL;
577 stat->start = NULL;
578
579 return -ENOMEM;
Steven Rostedt493762f2009-03-23 17:12:36 -0400580}
581
Steven Rostedtcafb1682009-03-24 20:50:39 -0400582static int ftrace_profile_init_cpu(int cpu)
Steven Rostedt493762f2009-03-23 17:12:36 -0400583{
Steven Rostedtcafb1682009-03-24 20:50:39 -0400584 struct ftrace_profile_stat *stat;
Steven Rostedt493762f2009-03-23 17:12:36 -0400585 int size;
586
Steven Rostedtcafb1682009-03-24 20:50:39 -0400587 stat = &per_cpu(ftrace_profile_stats, cpu);
588
589 if (stat->hash) {
Steven Rostedt493762f2009-03-23 17:12:36 -0400590 /* If the profile is already created, simply reset it */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400591 ftrace_profile_reset(stat);
Steven Rostedt493762f2009-03-23 17:12:36 -0400592 return 0;
593 }
594
595 /*
596 * We are profiling all functions, but usually only a few thousand
597 * functions are hit. We'll make a hash of 1024 items.
598 */
599 size = FTRACE_PROFILE_HASH_SIZE;
600
Steven Rostedtcafb1682009-03-24 20:50:39 -0400601 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
Steven Rostedt493762f2009-03-23 17:12:36 -0400602
Steven Rostedtcafb1682009-03-24 20:50:39 -0400603 if (!stat->hash)
Steven Rostedt493762f2009-03-23 17:12:36 -0400604 return -ENOMEM;
605
Steven Rostedtcafb1682009-03-24 20:50:39 -0400606 if (!ftrace_profile_bits) {
607 size--;
Steven Rostedt493762f2009-03-23 17:12:36 -0400608
Steven Rostedtcafb1682009-03-24 20:50:39 -0400609 for (; size; size >>= 1)
610 ftrace_profile_bits++;
611 }
Steven Rostedt493762f2009-03-23 17:12:36 -0400612
Steven Rostedt318e0a72009-03-25 20:06:34 -0400613 /* Preallocate the function profiling pages */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400614 if (ftrace_profile_pages_init(stat) < 0) {
615 kfree(stat->hash);
616 stat->hash = NULL;
Steven Rostedt493762f2009-03-23 17:12:36 -0400617 return -ENOMEM;
618 }
619
620 return 0;
621}
622
Steven Rostedtcafb1682009-03-24 20:50:39 -0400623static int ftrace_profile_init(void)
624{
625 int cpu;
626 int ret = 0;
627
628 for_each_online_cpu(cpu) {
629 ret = ftrace_profile_init_cpu(cpu);
630 if (ret)
631 break;
632 }
633
634 return ret;
635}
636
Steven Rostedt493762f2009-03-23 17:12:36 -0400637/* interrupts must be disabled */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400638static struct ftrace_profile *
639ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
Steven Rostedt493762f2009-03-23 17:12:36 -0400640{
641 struct ftrace_profile *rec;
642 struct hlist_head *hhd;
643 struct hlist_node *n;
644 unsigned long key;
645
646 key = hash_long(ip, ftrace_profile_bits);
Steven Rostedtcafb1682009-03-24 20:50:39 -0400647 hhd = &stat->hash[key];
Steven Rostedt493762f2009-03-23 17:12:36 -0400648
649 if (hlist_empty(hhd))
650 return NULL;
651
652 hlist_for_each_entry_rcu(rec, n, hhd, node) {
653 if (rec->ip == ip)
654 return rec;
655 }
656
657 return NULL;
658}
659
Steven Rostedtcafb1682009-03-24 20:50:39 -0400660static void ftrace_add_profile(struct ftrace_profile_stat *stat,
661 struct ftrace_profile *rec)
Steven Rostedt493762f2009-03-23 17:12:36 -0400662{
663 unsigned long key;
664
665 key = hash_long(rec->ip, ftrace_profile_bits);
Steven Rostedtcafb1682009-03-24 20:50:39 -0400666 hlist_add_head_rcu(&rec->node, &stat->hash[key]);
Steven Rostedt493762f2009-03-23 17:12:36 -0400667}
668
Steven Rostedt318e0a72009-03-25 20:06:34 -0400669/*
670 * The memory is already allocated, this simply finds a new record to use.
671 */
Steven Rostedt493762f2009-03-23 17:12:36 -0400672static struct ftrace_profile *
Steven Rostedt318e0a72009-03-25 20:06:34 -0400673ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
Steven Rostedt493762f2009-03-23 17:12:36 -0400674{
675 struct ftrace_profile *rec = NULL;
676
Steven Rostedt318e0a72009-03-25 20:06:34 -0400677 /* prevent recursion (from NMIs) */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400678 if (atomic_inc_return(&stat->disabled) != 1)
Steven Rostedt493762f2009-03-23 17:12:36 -0400679 goto out;
680
Steven Rostedt493762f2009-03-23 17:12:36 -0400681 /*
Steven Rostedt318e0a72009-03-25 20:06:34 -0400682 * Try to find the function again since an NMI
683 * could have added it
Steven Rostedt493762f2009-03-23 17:12:36 -0400684 */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400685 rec = ftrace_find_profiled_func(stat, ip);
Steven Rostedt493762f2009-03-23 17:12:36 -0400686 if (rec)
Steven Rostedtcafb1682009-03-24 20:50:39 -0400687 goto out;
Steven Rostedt493762f2009-03-23 17:12:36 -0400688
Steven Rostedtcafb1682009-03-24 20:50:39 -0400689 if (stat->pages->index == PROFILES_PER_PAGE) {
690 if (!stat->pages->next)
691 goto out;
692 stat->pages = stat->pages->next;
Steven Rostedt493762f2009-03-23 17:12:36 -0400693 }
694
Steven Rostedtcafb1682009-03-24 20:50:39 -0400695 rec = &stat->pages->records[stat->pages->index++];
Steven Rostedt493762f2009-03-23 17:12:36 -0400696 rec->ip = ip;
Steven Rostedtcafb1682009-03-24 20:50:39 -0400697 ftrace_add_profile(stat, rec);
Steven Rostedt493762f2009-03-23 17:12:36 -0400698
Steven Rostedt493762f2009-03-23 17:12:36 -0400699 out:
Steven Rostedtcafb1682009-03-24 20:50:39 -0400700 atomic_dec(&stat->disabled);
Steven Rostedt493762f2009-03-23 17:12:36 -0400701
702 return rec;
703}
704
Steven Rostedt493762f2009-03-23 17:12:36 -0400705static void
706function_profile_call(unsigned long ip, unsigned long parent_ip)
707{
Steven Rostedtcafb1682009-03-24 20:50:39 -0400708 struct ftrace_profile_stat *stat;
Steven Rostedt493762f2009-03-23 17:12:36 -0400709 struct ftrace_profile *rec;
710 unsigned long flags;
Steven Rostedt493762f2009-03-23 17:12:36 -0400711
712 if (!ftrace_profile_enabled)
713 return;
714
Steven Rostedt493762f2009-03-23 17:12:36 -0400715 local_irq_save(flags);
Steven Rostedtcafb1682009-03-24 20:50:39 -0400716
717 stat = &__get_cpu_var(ftrace_profile_stats);
Steven Rostedt0f6ce3d2009-06-01 21:51:28 -0400718 if (!stat->hash || !ftrace_profile_enabled)
Steven Rostedtcafb1682009-03-24 20:50:39 -0400719 goto out;
720
721 rec = ftrace_find_profiled_func(stat, ip);
Steven Rostedt493762f2009-03-23 17:12:36 -0400722 if (!rec) {
Steven Rostedt318e0a72009-03-25 20:06:34 -0400723 rec = ftrace_profile_alloc(stat, ip);
Steven Rostedt493762f2009-03-23 17:12:36 -0400724 if (!rec)
725 goto out;
726 }
727
728 rec->counter++;
729 out:
730 local_irq_restore(flags);
731}
732
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400733#ifdef CONFIG_FUNCTION_GRAPH_TRACER
734static int profile_graph_entry(struct ftrace_graph_ent *trace)
735{
736 function_profile_call(trace->func, 0);
737 return 1;
738}
739
740static void profile_graph_return(struct ftrace_graph_ret *trace)
741{
Steven Rostedtcafb1682009-03-24 20:50:39 -0400742 struct ftrace_profile_stat *stat;
Steven Rostedta2a16d62009-03-24 23:17:58 -0400743 unsigned long long calltime;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400744 struct ftrace_profile *rec;
Steven Rostedtcafb1682009-03-24 20:50:39 -0400745 unsigned long flags;
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400746
747 local_irq_save(flags);
Steven Rostedtcafb1682009-03-24 20:50:39 -0400748 stat = &__get_cpu_var(ftrace_profile_stats);
Steven Rostedt0f6ce3d2009-06-01 21:51:28 -0400749 if (!stat->hash || !ftrace_profile_enabled)
Steven Rostedtcafb1682009-03-24 20:50:39 -0400750 goto out;
751
Steven Rostedt37e44bc2010-04-27 21:04:24 -0400752 /* If the calltime was zero'd ignore it */
753 if (!trace->calltime)
754 goto out;
755
Steven Rostedta2a16d62009-03-24 23:17:58 -0400756 calltime = trace->rettime - trace->calltime;
757
758 if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
759 int index;
760
761 index = trace->depth;
762
763 /* Append this call time to the parent time to subtract */
764 if (index)
765 current->ret_stack[index - 1].subtime += calltime;
766
767 if (current->ret_stack[index].subtime < calltime)
768 calltime -= current->ret_stack[index].subtime;
769 else
770 calltime = 0;
771 }
772
Steven Rostedtcafb1682009-03-24 20:50:39 -0400773 rec = ftrace_find_profiled_func(stat, trace->func);
Chase Douglase330b3b2010-04-26 14:02:05 -0400774 if (rec) {
Steven Rostedta2a16d62009-03-24 23:17:58 -0400775 rec->time += calltime;
Chase Douglase330b3b2010-04-26 14:02:05 -0400776 rec->time_squared += calltime * calltime;
777 }
Steven Rostedta2a16d62009-03-24 23:17:58 -0400778
Steven Rostedtcafb1682009-03-24 20:50:39 -0400779 out:
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400780 local_irq_restore(flags);
781}
782
783static int register_ftrace_profiler(void)
784{
785 return register_ftrace_graph(&profile_graph_return,
786 &profile_graph_entry);
787}
788
789static void unregister_ftrace_profiler(void)
790{
791 unregister_ftrace_graph();
792}
793#else
Paul McQuadebd38c0e2011-05-31 20:51:55 +0100794static struct ftrace_ops ftrace_profile_ops __read_mostly = {
Steven Rostedtfb9fb012009-03-25 13:26:41 -0400795 .func = function_profile_call,
Steven Rostedt493762f2009-03-23 17:12:36 -0400796};
797
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400798static int register_ftrace_profiler(void)
799{
800 return register_ftrace_function(&ftrace_profile_ops);
801}
802
803static void unregister_ftrace_profiler(void)
804{
805 unregister_ftrace_function(&ftrace_profile_ops);
806}
807#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
808
Steven Rostedt493762f2009-03-23 17:12:36 -0400809static ssize_t
810ftrace_profile_write(struct file *filp, const char __user *ubuf,
811 size_t cnt, loff_t *ppos)
812{
813 unsigned long val;
Steven Rostedt493762f2009-03-23 17:12:36 -0400814 int ret;
815
Peter Huewe22fe9b52011-06-07 21:58:27 +0200816 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
817 if (ret)
Steven Rostedt493762f2009-03-23 17:12:36 -0400818 return ret;
819
820 val = !!val;
821
822 mutex_lock(&ftrace_profile_lock);
823 if (ftrace_profile_enabled ^ val) {
824 if (val) {
825 ret = ftrace_profile_init();
826 if (ret < 0) {
827 cnt = ret;
828 goto out;
829 }
830
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400831 ret = register_ftrace_profiler();
832 if (ret < 0) {
833 cnt = ret;
834 goto out;
835 }
Steven Rostedt493762f2009-03-23 17:12:36 -0400836 ftrace_profile_enabled = 1;
837 } else {
838 ftrace_profile_enabled = 0;
Steven Rostedt0f6ce3d2009-06-01 21:51:28 -0400839 /*
840 * unregister_ftrace_profiler calls stop_machine
841 * so this acts like an synchronize_sched.
842 */
Steven Rostedt0706f1c2009-03-23 23:12:58 -0400843 unregister_ftrace_profiler();
Steven Rostedt493762f2009-03-23 17:12:36 -0400844 }
845 }
846 out:
847 mutex_unlock(&ftrace_profile_lock);
848
Jiri Olsacf8517c2009-10-23 19:36:16 -0400849 *ppos += cnt;
Steven Rostedt493762f2009-03-23 17:12:36 -0400850
851 return cnt;
852}
853
854static ssize_t
855ftrace_profile_read(struct file *filp, char __user *ubuf,
856 size_t cnt, loff_t *ppos)
857{
Steven Rostedtfb9fb012009-03-25 13:26:41 -0400858 char buf[64]; /* big enough to hold a number */
Steven Rostedt493762f2009-03-23 17:12:36 -0400859 int r;
860
861 r = sprintf(buf, "%u\n", ftrace_profile_enabled);
862 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
863}
864
865static const struct file_operations ftrace_profile_fops = {
866 .open = tracing_open_generic,
867 .read = ftrace_profile_read,
868 .write = ftrace_profile_write,
Arnd Bergmann6038f372010-08-15 18:52:59 +0200869 .llseek = default_llseek,
Steven Rostedt493762f2009-03-23 17:12:36 -0400870};
871
Steven Rostedtcafb1682009-03-24 20:50:39 -0400872/* used to initialize the real stat files */
873static struct tracer_stat function_stats __initdata = {
Steven Rostedtfb9fb012009-03-25 13:26:41 -0400874 .name = "functions",
875 .stat_start = function_stat_start,
876 .stat_next = function_stat_next,
877 .stat_cmp = function_stat_cmp,
878 .stat_headers = function_stat_headers,
879 .stat_show = function_stat_show
Steven Rostedtcafb1682009-03-24 20:50:39 -0400880};
881
Steven Rostedt6ab5d662009-06-04 00:55:45 -0400882static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
Steven Rostedt493762f2009-03-23 17:12:36 -0400883{
Steven Rostedtcafb1682009-03-24 20:50:39 -0400884 struct ftrace_profile_stat *stat;
Steven Rostedt493762f2009-03-23 17:12:36 -0400885 struct dentry *entry;
Steven Rostedtcafb1682009-03-24 20:50:39 -0400886 char *name;
Steven Rostedt493762f2009-03-23 17:12:36 -0400887 int ret;
Steven Rostedtcafb1682009-03-24 20:50:39 -0400888 int cpu;
Steven Rostedt493762f2009-03-23 17:12:36 -0400889
Steven Rostedtcafb1682009-03-24 20:50:39 -0400890 for_each_possible_cpu(cpu) {
891 stat = &per_cpu(ftrace_profile_stats, cpu);
892
893 /* allocate enough for function name + cpu number */
894 name = kmalloc(32, GFP_KERNEL);
895 if (!name) {
896 /*
897 * The files created are permanent, if something happens
898 * we still do not free memory.
899 */
Steven Rostedtcafb1682009-03-24 20:50:39 -0400900 WARN(1,
901 "Could not allocate stat file for cpu %d\n",
902 cpu);
903 return;
904 }
905 stat->stat = function_stats;
906 snprintf(name, 32, "function%d", cpu);
907 stat->stat.name = name;
908 ret = register_stat_tracer(&stat->stat);
909 if (ret) {
910 WARN(1,
911 "Could not register function stat for cpu %d\n",
912 cpu);
913 kfree(name);
914 return;
915 }
Steven Rostedt493762f2009-03-23 17:12:36 -0400916 }
917
918 entry = debugfs_create_file("function_profile_enabled", 0644,
919 d_tracer, NULL, &ftrace_profile_fops);
920 if (!entry)
921 pr_warning("Could not create debugfs "
922 "'function_profile_enabled' entry\n");
923}
924
925#else /* CONFIG_FUNCTION_PROFILER */
Steven Rostedt6ab5d662009-06-04 00:55:45 -0400926static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
Steven Rostedt493762f2009-03-23 17:12:36 -0400927{
928}
929#endif /* CONFIG_FUNCTION_PROFILER */
930
Ingo Molnar73d3fd92009-02-17 11:48:18 +0100931static struct pid * const ftrace_swapper_pid = &init_struct_pid;
932
Steven Rostedt3d083392008-05-12 21:20:42 +0200933#ifdef CONFIG_DYNAMIC_FTRACE
Ingo Molnar73d3fd92009-02-17 11:48:18 +0100934
Steven Rostedt99ecdc42008-08-15 21:40:05 -0400935#ifndef CONFIG_FTRACE_MCOUNT_RECORD
Steven Rostedtcb7be3b2008-10-23 09:33:05 -0400936# error Dynamic ftrace depends on MCOUNT_RECORD
Steven Rostedt99ecdc42008-08-15 21:40:05 -0400937#endif
938
Steven Rostedt8fc0c702009-02-16 15:28:00 -0500939static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
940
Steven Rostedtb6887d72009-02-17 12:32:04 -0500941struct ftrace_func_probe {
Steven Rostedt8fc0c702009-02-16 15:28:00 -0500942 struct hlist_node node;
Steven Rostedtb6887d72009-02-17 12:32:04 -0500943 struct ftrace_probe_ops *ops;
Steven Rostedt8fc0c702009-02-16 15:28:00 -0500944 unsigned long flags;
945 unsigned long ip;
946 void *data;
947 struct rcu_head rcu;
948};
949
Steven Rostedtd61f82d2008-05-12 21:20:43 +0200950enum {
951 FTRACE_ENABLE_CALLS = (1 << 0),
952 FTRACE_DISABLE_CALLS = (1 << 1),
953 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
Steven Rostedt79e406d2010-09-14 22:19:46 -0400954 FTRACE_START_FUNC_RET = (1 << 3),
955 FTRACE_STOP_FUNC_RET = (1 << 4),
Steven Rostedtd61f82d2008-05-12 21:20:43 +0200956};
Steven Rostedtb448c4e2011-04-29 15:12:32 -0400957struct ftrace_func_entry {
958 struct hlist_node hlist;
959 unsigned long ip;
960};
961
962struct ftrace_hash {
963 unsigned long size_bits;
964 struct hlist_head *buckets;
965 unsigned long count;
Steven Rostedt07fd5512011-05-05 18:03:47 -0400966 struct rcu_head rcu;
Steven Rostedtb448c4e2011-04-29 15:12:32 -0400967};
968
Steven Rostedt33dc9b12011-05-02 17:34:47 -0400969/*
970 * We make these constant because no one should touch them,
971 * but they are used as the default "empty hash", to avoid allocating
972 * it all the time. These are in a read only section such that if
973 * anyone does try to modify it, it will cause an exception.
974 */
975static const struct hlist_head empty_buckets[1];
976static const struct ftrace_hash empty_hash = {
977 .buckets = (struct hlist_head *)empty_buckets,
Steven Rostedtb448c4e2011-04-29 15:12:32 -0400978};
Steven Rostedt33dc9b12011-05-02 17:34:47 -0400979#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
Steven Rostedt5072c592008-05-12 21:20:43 +0200980
Steven Rostedt2b499382011-05-03 22:49:52 -0400981static struct ftrace_ops global_ops = {
Steven Rostedtf45948e2011-05-02 12:29:25 -0400982 .func = ftrace_stub,
Steven Rostedt33dc9b12011-05-02 17:34:47 -0400983 .notrace_hash = EMPTY_HASH,
984 .filter_hash = EMPTY_HASH,
Steven Rostedtf45948e2011-05-02 12:29:25 -0400985};
986
Lai Jiangshane94142a2009-03-13 17:51:27 +0800987static struct dyn_ftrace *ftrace_new_addrs;
Steven Rostedt3d083392008-05-12 21:20:42 +0200988
Steven Rostedt41c52c02008-05-22 11:46:33 -0400989static DEFINE_MUTEX(ftrace_regex_lock);
Steven Rostedt3d083392008-05-12 21:20:42 +0200990
Steven Rostedt3c1720f2008-05-12 21:20:43 +0200991struct ftrace_page {
992 struct ftrace_page *next;
Steven Rostedt431aa3f2009-01-06 12:43:01 -0500993 int index;
Steven Rostedt3c1720f2008-05-12 21:20:43 +0200994 struct dyn_ftrace records[];
David Milleraa5e5ce2008-05-13 22:06:56 -0700995};
Steven Rostedt3c1720f2008-05-12 21:20:43 +0200996
997#define ENTRIES_PER_PAGE \
998 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
999
1000/* estimate from running different kernels */
1001#define NR_TO_INIT 10000
1002
1003static struct ftrace_page *ftrace_pages_start;
1004static struct ftrace_page *ftrace_pages;
1005
Steven Rostedt37ad5082008-05-12 21:20:48 +02001006static struct dyn_ftrace *ftrace_free_records;
1007
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001008static struct ftrace_func_entry *
1009ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1010{
1011 unsigned long key;
1012 struct ftrace_func_entry *entry;
1013 struct hlist_head *hhd;
1014 struct hlist_node *n;
1015
1016 if (!hash->count)
1017 return NULL;
1018
1019 if (hash->size_bits > 0)
1020 key = hash_long(ip, hash->size_bits);
1021 else
1022 key = 0;
1023
1024 hhd = &hash->buckets[key];
1025
1026 hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
1027 if (entry->ip == ip)
1028 return entry;
1029 }
1030 return NULL;
1031}
1032
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001033static void __add_hash_entry(struct ftrace_hash *hash,
1034 struct ftrace_func_entry *entry)
1035{
1036 struct hlist_head *hhd;
1037 unsigned long key;
1038
1039 if (hash->size_bits)
1040 key = hash_long(entry->ip, hash->size_bits);
1041 else
1042 key = 0;
1043
1044 hhd = &hash->buckets[key];
1045 hlist_add_head(&entry->hlist, hhd);
1046 hash->count++;
1047}
1048
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001049static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1050{
1051 struct ftrace_func_entry *entry;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001052
1053 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1054 if (!entry)
1055 return -ENOMEM;
1056
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001057 entry->ip = ip;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001058 __add_hash_entry(hash, entry);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001059
1060 return 0;
1061}
1062
1063static void
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001064free_hash_entry(struct ftrace_hash *hash,
1065 struct ftrace_func_entry *entry)
1066{
1067 hlist_del(&entry->hlist);
1068 kfree(entry);
1069 hash->count--;
1070}
1071
1072static void
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001073remove_hash_entry(struct ftrace_hash *hash,
1074 struct ftrace_func_entry *entry)
1075{
1076 hlist_del(&entry->hlist);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001077 hash->count--;
1078}
1079
1080static void ftrace_hash_clear(struct ftrace_hash *hash)
1081{
1082 struct hlist_head *hhd;
1083 struct hlist_node *tp, *tn;
1084 struct ftrace_func_entry *entry;
1085 int size = 1 << hash->size_bits;
1086 int i;
1087
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001088 if (!hash->count)
1089 return;
1090
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001091 for (i = 0; i < size; i++) {
1092 hhd = &hash->buckets[i];
1093 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001094 free_hash_entry(hash, entry);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04001095 }
1096 FTRACE_WARN_ON(hash->count);
1097}
1098
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001099static void free_ftrace_hash(struct ftrace_hash *hash)
1100{
1101 if (!hash || hash == EMPTY_HASH)
1102 return;
1103 ftrace_hash_clear(hash);
1104 kfree(hash->buckets);
1105 kfree(hash);
1106}
1107
Steven Rostedt07fd5512011-05-05 18:03:47 -04001108static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1109{
1110 struct ftrace_hash *hash;
1111
1112 hash = container_of(rcu, struct ftrace_hash, rcu);
1113 free_ftrace_hash(hash);
1114}
1115
1116static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1117{
1118 if (!hash || hash == EMPTY_HASH)
1119 return;
1120 call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1121}
1122
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001123static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1124{
1125 struct ftrace_hash *hash;
1126 int size;
1127
1128 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1129 if (!hash)
1130 return NULL;
1131
1132 size = 1 << size_bits;
1133 hash->buckets = kzalloc(sizeof(*hash->buckets) * size, GFP_KERNEL);
1134
1135 if (!hash->buckets) {
1136 kfree(hash);
1137 return NULL;
1138 }
1139
1140 hash->size_bits = size_bits;
1141
1142 return hash;
1143}
1144
1145static struct ftrace_hash *
1146alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1147{
1148 struct ftrace_func_entry *entry;
1149 struct ftrace_hash *new_hash;
1150 struct hlist_node *tp;
1151 int size;
1152 int ret;
1153 int i;
1154
1155 new_hash = alloc_ftrace_hash(size_bits);
1156 if (!new_hash)
1157 return NULL;
1158
1159 /* Empty hash? */
1160 if (!hash || !hash->count)
1161 return new_hash;
1162
1163 size = 1 << hash->size_bits;
1164 for (i = 0; i < size; i++) {
1165 hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
1166 ret = add_hash_entry(new_hash, entry->ip);
1167 if (ret < 0)
1168 goto free_hash;
1169 }
1170 }
1171
1172 FTRACE_WARN_ON(new_hash->count != hash->count);
1173
1174 return new_hash;
1175
1176 free_hash:
1177 free_ftrace_hash(new_hash);
1178 return NULL;
1179}
1180
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001181static void
1182ftrace_hash_rec_disable(struct ftrace_ops *ops, int filter_hash);
1183static void
1184ftrace_hash_rec_enable(struct ftrace_ops *ops, int filter_hash);
1185
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001186static int
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001187ftrace_hash_move(struct ftrace_ops *ops, int enable,
1188 struct ftrace_hash **dst, struct ftrace_hash *src)
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001189{
1190 struct ftrace_func_entry *entry;
1191 struct hlist_node *tp, *tn;
1192 struct hlist_head *hhd;
Steven Rostedt07fd5512011-05-05 18:03:47 -04001193 struct ftrace_hash *old_hash;
1194 struct ftrace_hash *new_hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001195 unsigned long key;
1196 int size = src->count;
1197 int bits = 0;
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001198 int ret;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001199 int i;
1200
1201 /*
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001202 * Remove the current set, update the hash and add
1203 * them back.
1204 */
1205 ftrace_hash_rec_disable(ops, enable);
1206
1207 /*
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001208 * If the new source is empty, just free dst and assign it
1209 * the empty_hash.
1210 */
1211 if (!src->count) {
Steven Rostedt07fd5512011-05-05 18:03:47 -04001212 free_ftrace_hash_rcu(*dst);
1213 rcu_assign_pointer(*dst, EMPTY_HASH);
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001214 return 0;
1215 }
1216
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001217 /*
1218 * Make the hash size about 1/2 the # found
1219 */
1220 for (size /= 2; size; size >>= 1)
1221 bits++;
1222
1223 /* Don't allocate too much */
1224 if (bits > FTRACE_HASH_MAX_BITS)
1225 bits = FTRACE_HASH_MAX_BITS;
1226
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001227 ret = -ENOMEM;
Steven Rostedt07fd5512011-05-05 18:03:47 -04001228 new_hash = alloc_ftrace_hash(bits);
1229 if (!new_hash)
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001230 goto out;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001231
1232 size = 1 << src->size_bits;
1233 for (i = 0; i < size; i++) {
1234 hhd = &src->buckets[i];
1235 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
1236 if (bits > 0)
1237 key = hash_long(entry->ip, bits);
1238 else
1239 key = 0;
1240 remove_hash_entry(src, entry);
Steven Rostedt07fd5512011-05-05 18:03:47 -04001241 __add_hash_entry(new_hash, entry);
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001242 }
1243 }
1244
Steven Rostedt07fd5512011-05-05 18:03:47 -04001245 old_hash = *dst;
1246 rcu_assign_pointer(*dst, new_hash);
1247 free_ftrace_hash_rcu(old_hash);
1248
Steven Rostedt41fb61c2011-07-13 15:03:44 -04001249 ret = 0;
1250 out:
1251 /*
1252 * Enable regardless of ret:
1253 * On success, we enable the new hash.
1254 * On failure, we re-enable the original hash.
1255 */
1256 ftrace_hash_rec_enable(ops, enable);
1257
1258 return ret;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001259}
1260
Steven Rostedt265c8312009-02-13 12:43:56 -05001261/*
Steven Rostedtb8489142011-05-04 09:27:52 -04001262 * Test the hashes for this ops to see if we want to call
1263 * the ops->func or not.
1264 *
1265 * It's a match if the ip is in the ops->filter_hash or
1266 * the filter_hash does not exist or is empty,
1267 * AND
1268 * the ip is not in the ops->notrace_hash.
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04001269 *
1270 * This needs to be called with preemption disabled as
1271 * the hashes are freed with call_rcu_sched().
Steven Rostedtb8489142011-05-04 09:27:52 -04001272 */
1273static int
1274ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
1275{
1276 struct ftrace_hash *filter_hash;
1277 struct ftrace_hash *notrace_hash;
1278 int ret;
1279
Steven Rostedtb8489142011-05-04 09:27:52 -04001280 filter_hash = rcu_dereference_raw(ops->filter_hash);
1281 notrace_hash = rcu_dereference_raw(ops->notrace_hash);
1282
1283 if ((!filter_hash || !filter_hash->count ||
1284 ftrace_lookup_ip(filter_hash, ip)) &&
1285 (!notrace_hash || !notrace_hash->count ||
1286 !ftrace_lookup_ip(notrace_hash, ip)))
1287 ret = 1;
1288 else
1289 ret = 0;
Steven Rostedtb8489142011-05-04 09:27:52 -04001290
1291 return ret;
1292}
1293
1294/*
Steven Rostedt265c8312009-02-13 12:43:56 -05001295 * This is a double for. Do not use 'break' to break out of the loop,
1296 * you must use a goto.
1297 */
1298#define do_for_each_ftrace_rec(pg, rec) \
1299 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1300 int _____i; \
1301 for (_____i = 0; _____i < pg->index; _____i++) { \
1302 rec = &pg->records[_____i];
1303
1304#define while_for_each_ftrace_rec() \
1305 } \
1306 }
Abhishek Sagarecea6562008-06-21 23:47:53 +05301307
Steven Rostedted926f92011-05-03 13:25:24 -04001308static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1309 int filter_hash,
1310 bool inc)
1311{
1312 struct ftrace_hash *hash;
1313 struct ftrace_hash *other_hash;
1314 struct ftrace_page *pg;
1315 struct dyn_ftrace *rec;
1316 int count = 0;
1317 int all = 0;
1318
1319 /* Only update if the ops has been registered */
1320 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1321 return;
1322
1323 /*
1324 * In the filter_hash case:
1325 * If the count is zero, we update all records.
1326 * Otherwise we just update the items in the hash.
1327 *
1328 * In the notrace_hash case:
1329 * We enable the update in the hash.
1330 * As disabling notrace means enabling the tracing,
1331 * and enabling notrace means disabling, the inc variable
1332 * gets inversed.
1333 */
1334 if (filter_hash) {
1335 hash = ops->filter_hash;
1336 other_hash = ops->notrace_hash;
Steven Rostedtb8489142011-05-04 09:27:52 -04001337 if (!hash || !hash->count)
Steven Rostedted926f92011-05-03 13:25:24 -04001338 all = 1;
1339 } else {
1340 inc = !inc;
1341 hash = ops->notrace_hash;
1342 other_hash = ops->filter_hash;
1343 /*
1344 * If the notrace hash has no items,
1345 * then there's nothing to do.
1346 */
Steven Rostedtb8489142011-05-04 09:27:52 -04001347 if (hash && !hash->count)
Steven Rostedted926f92011-05-03 13:25:24 -04001348 return;
1349 }
1350
1351 do_for_each_ftrace_rec(pg, rec) {
1352 int in_other_hash = 0;
1353 int in_hash = 0;
1354 int match = 0;
1355
1356 if (all) {
1357 /*
1358 * Only the filter_hash affects all records.
1359 * Update if the record is not in the notrace hash.
1360 */
Steven Rostedtb8489142011-05-04 09:27:52 -04001361 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
Steven Rostedted926f92011-05-03 13:25:24 -04001362 match = 1;
1363 } else {
Steven Rostedtb8489142011-05-04 09:27:52 -04001364 in_hash = hash && !!ftrace_lookup_ip(hash, rec->ip);
1365 in_other_hash = other_hash && !!ftrace_lookup_ip(other_hash, rec->ip);
Steven Rostedted926f92011-05-03 13:25:24 -04001366
1367 /*
1368 *
1369 */
1370 if (filter_hash && in_hash && !in_other_hash)
1371 match = 1;
1372 else if (!filter_hash && in_hash &&
1373 (in_other_hash || !other_hash->count))
1374 match = 1;
1375 }
1376 if (!match)
1377 continue;
1378
1379 if (inc) {
1380 rec->flags++;
1381 if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1382 return;
1383 } else {
1384 if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1385 return;
1386 rec->flags--;
1387 }
1388 count++;
1389 /* Shortcut, if we handled all records, we are done. */
1390 if (!all && count == hash->count)
1391 return;
1392 } while_for_each_ftrace_rec();
1393}
1394
1395static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1396 int filter_hash)
1397{
1398 __ftrace_hash_rec_update(ops, filter_hash, 0);
1399}
1400
1401static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1402 int filter_hash)
1403{
1404 __ftrace_hash_rec_update(ops, filter_hash, 1);
1405}
1406
Ingo Molnare309b412008-05-12 21:20:51 +02001407static void ftrace_free_rec(struct dyn_ftrace *rec)
Steven Rostedt37ad5082008-05-12 21:20:48 +02001408{
Lai Jiangshanee000b72009-03-24 13:38:06 +08001409 rec->freelist = ftrace_free_records;
Steven Rostedt37ad5082008-05-12 21:20:48 +02001410 ftrace_free_records = rec;
1411 rec->flags |= FTRACE_FL_FREE;
1412}
1413
Ingo Molnare309b412008-05-12 21:20:51 +02001414static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001415{
Steven Rostedt37ad5082008-05-12 21:20:48 +02001416 struct dyn_ftrace *rec;
1417
1418 /* First check for freed records */
1419 if (ftrace_free_records) {
1420 rec = ftrace_free_records;
1421
Steven Rostedt37ad5082008-05-12 21:20:48 +02001422 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
Steven Rostedt69128962008-10-23 09:33:03 -04001423 FTRACE_WARN_ON_ONCE(1);
Steven Rostedt37ad5082008-05-12 21:20:48 +02001424 ftrace_free_records = NULL;
1425 return NULL;
1426 }
1427
Lai Jiangshanee000b72009-03-24 13:38:06 +08001428 ftrace_free_records = rec->freelist;
Steven Rostedt37ad5082008-05-12 21:20:48 +02001429 memset(rec, 0, sizeof(*rec));
1430 return rec;
1431 }
1432
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001433 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001434 if (!ftrace_pages->next) {
1435 /* allocate another page */
1436 ftrace_pages->next =
1437 (void *)get_zeroed_page(GFP_KERNEL);
1438 if (!ftrace_pages->next)
1439 return NULL;
1440 }
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001441 ftrace_pages = ftrace_pages->next;
1442 }
1443
1444 return &ftrace_pages->records[ftrace_pages->index++];
1445}
1446
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001447static struct dyn_ftrace *
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001448ftrace_record_ip(unsigned long ip)
Steven Rostedt3d083392008-05-12 21:20:42 +02001449{
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001450 struct dyn_ftrace *rec;
Steven Rostedt3d083392008-05-12 21:20:42 +02001451
Steven Rostedtf3c7ac42008-11-14 16:21:19 -08001452 if (ftrace_disabled)
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001453 return NULL;
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001454
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001455 rec = ftrace_alloc_dyn_node(ip);
1456 if (!rec)
1457 return NULL;
Steven Rostedt3d083392008-05-12 21:20:42 +02001458
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001459 rec->ip = ip;
Lai Jiangshanee000b72009-03-24 13:38:06 +08001460 rec->newlist = ftrace_new_addrs;
Lai Jiangshane94142a2009-03-13 17:51:27 +08001461 ftrace_new_addrs = rec;
Steven Rostedt3d083392008-05-12 21:20:42 +02001462
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001463 return rec;
Steven Rostedt3d083392008-05-12 21:20:42 +02001464}
1465
Steven Rostedt05736a42008-09-22 14:55:47 -07001466static void print_ip_ins(const char *fmt, unsigned char *p)
1467{
1468 int i;
1469
1470 printk(KERN_CONT "%s", fmt);
1471
1472 for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1473 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1474}
1475
Steven Rostedt31e88902008-11-14 16:21:19 -08001476static void ftrace_bug(int failed, unsigned long ip)
Steven Rostedtb17e8a32008-11-14 16:21:19 -08001477{
1478 switch (failed) {
1479 case -EFAULT:
1480 FTRACE_WARN_ON_ONCE(1);
1481 pr_info("ftrace faulted on modifying ");
1482 print_ip_sym(ip);
1483 break;
1484 case -EINVAL:
1485 FTRACE_WARN_ON_ONCE(1);
1486 pr_info("ftrace failed to modify ");
1487 print_ip_sym(ip);
Steven Rostedtb17e8a32008-11-14 16:21:19 -08001488 print_ip_ins(" actual: ", (unsigned char *)ip);
Steven Rostedtb17e8a32008-11-14 16:21:19 -08001489 printk(KERN_CONT "\n");
1490 break;
1491 case -EPERM:
1492 FTRACE_WARN_ON_ONCE(1);
1493 pr_info("ftrace faulted on writing ");
1494 print_ip_sym(ip);
1495 break;
1496 default:
1497 FTRACE_WARN_ON_ONCE(1);
1498 pr_info("ftrace faulted on unknown error ");
1499 print_ip_sym(ip);
1500 }
1501}
1502
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001503
Masami Hiramatsu2cfa1972010-02-02 16:49:11 -05001504/* Return 1 if the address range is reserved for ftrace */
1505int ftrace_text_reserved(void *start, void *end)
1506{
1507 struct dyn_ftrace *rec;
1508 struct ftrace_page *pg;
1509
1510 do_for_each_ftrace_rec(pg, rec) {
1511 if (rec->ip <= (unsigned long)end &&
1512 rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
1513 return 1;
1514 } while_for_each_ftrace_rec();
1515 return 0;
1516}
1517
1518
Abhishek Sagar492a7ea52008-05-25 00:10:04 +05301519static int
Steven Rostedt31e88902008-11-14 16:21:19 -08001520__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001521{
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01001522 unsigned long ftrace_addr;
Xiao Guangrong64fbcd12009-07-15 12:32:15 +08001523 unsigned long flag = 0UL;
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01001524
Shaohua Lif0001202009-01-09 11:29:42 +08001525 ftrace_addr = (unsigned long)FTRACE_ADDR;
Steven Rostedt5072c592008-05-12 21:20:43 +02001526
Steven Rostedt982c3502008-11-15 16:31:41 -05001527 /*
Steven Rostedted926f92011-05-03 13:25:24 -04001528 * If we are enabling tracing:
Steven Rostedt982c3502008-11-15 16:31:41 -05001529 *
Steven Rostedted926f92011-05-03 13:25:24 -04001530 * If the record has a ref count, then we need to enable it
1531 * because someone is using it.
Steven Rostedt982c3502008-11-15 16:31:41 -05001532 *
Steven Rostedted926f92011-05-03 13:25:24 -04001533 * Otherwise we make sure its disabled.
1534 *
1535 * If we are disabling tracing, then disable all records that
1536 * are enabled.
Steven Rostedt982c3502008-11-15 16:31:41 -05001537 */
Steven Rostedted926f92011-05-03 13:25:24 -04001538 if (enable && (rec->flags & ~FTRACE_FL_MASK))
1539 flag = FTRACE_FL_ENABLED;
Steven Rostedt5072c592008-05-12 21:20:43 +02001540
Xiao Guangrong64fbcd12009-07-15 12:32:15 +08001541 /* If the state of this record hasn't changed, then do nothing */
1542 if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1543 return 0;
1544
1545 if (flag) {
1546 rec->flags |= FTRACE_FL_ENABLED;
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01001547 return ftrace_make_call(rec, ftrace_addr);
Xiao Guangrong64fbcd12009-07-15 12:32:15 +08001548 }
1549
1550 rec->flags &= ~FTRACE_FL_ENABLED;
1551 return ftrace_make_nop(NULL, rec, ftrace_addr);
Steven Rostedt5072c592008-05-12 21:20:43 +02001552}
1553
1554static void ftrace_replace_code(int enable)
1555{
Steven Rostedt37ad5082008-05-12 21:20:48 +02001556 struct dyn_ftrace *rec;
1557 struct ftrace_page *pg;
Steven Rostedt6a24a242009-02-17 11:20:26 -05001558 int failed;
Steven Rostedt37ad5082008-05-12 21:20:48 +02001559
Steven Rostedt45a4a232011-04-21 23:16:46 -04001560 if (unlikely(ftrace_disabled))
1561 return;
1562
Steven Rostedt265c8312009-02-13 12:43:56 -05001563 do_for_each_ftrace_rec(pg, rec) {
Steven Rostedtd2c8c3e2011-04-25 14:32:42 -04001564 /* Skip over free records */
1565 if (rec->flags & FTRACE_FL_FREE)
Steven Rostedt265c8312009-02-13 12:43:56 -05001566 continue;
Steven Rostedt5072c592008-05-12 21:20:43 +02001567
Steven Rostedt265c8312009-02-13 12:43:56 -05001568 failed = __ftrace_replace_code(rec, enable);
Zhaoleifa9d13c2009-03-13 17:16:34 +08001569 if (failed) {
Steven Rostedt3279ba32009-10-07 16:57:56 -04001570 ftrace_bug(failed, rec->ip);
1571 /* Stop processing */
1572 return;
Steven Rostedt265c8312009-02-13 12:43:56 -05001573 }
1574 } while_for_each_ftrace_rec();
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001575}
1576
Ingo Molnare309b412008-05-12 21:20:51 +02001577static int
Steven Rostedt31e88902008-11-14 16:21:19 -08001578ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001579{
1580 unsigned long ip;
Steven Rostedt593eb8a2008-10-23 09:32:59 -04001581 int ret;
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001582
1583 ip = rec->ip;
1584
Steven Rostedt45a4a232011-04-21 23:16:46 -04001585 if (unlikely(ftrace_disabled))
1586 return 0;
1587
Shaohua Li25aac9d2009-01-09 11:29:40 +08001588 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
Steven Rostedt593eb8a2008-10-23 09:32:59 -04001589 if (ret) {
Steven Rostedt31e88902008-11-14 16:21:19 -08001590 ftrace_bug(ret, ip);
Abhishek Sagar492a7ea52008-05-25 00:10:04 +05301591 return 0;
Steven Rostedt37ad5082008-05-12 21:20:48 +02001592 }
Abhishek Sagar492a7ea52008-05-25 00:10:04 +05301593 return 1;
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001594}
1595
Steven Rostedt000ab692009-02-17 13:35:06 -05001596/*
1597 * archs can override this function if they must do something
1598 * before the modifying code is performed.
1599 */
1600int __weak ftrace_arch_code_modify_prepare(void)
1601{
1602 return 0;
1603}
1604
1605/*
1606 * archs can override this function if they must do something
1607 * after the modifying code is performed.
1608 */
1609int __weak ftrace_arch_code_modify_post_process(void)
1610{
1611 return 0;
1612}
1613
Ingo Molnare309b412008-05-12 21:20:51 +02001614static int __ftrace_modify_code(void *data)
Steven Rostedt3d083392008-05-12 21:20:42 +02001615{
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001616 int *command = data;
1617
Steven Rostedt6331c282011-07-13 15:11:02 -04001618 /*
1619 * Do not call function tracer while we update the code.
1620 * We are in stop machine, no worrying about races.
1621 */
1622 function_trace_stop++;
1623
Steven Rostedta3583242008-11-11 15:01:42 -05001624 if (*command & FTRACE_ENABLE_CALLS)
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001625 ftrace_replace_code(1);
Steven Rostedta3583242008-11-11 15:01:42 -05001626 else if (*command & FTRACE_DISABLE_CALLS)
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001627 ftrace_replace_code(0);
1628
1629 if (*command & FTRACE_UPDATE_TRACE_FUNC)
1630 ftrace_update_ftrace_func(ftrace_trace_function);
1631
Steven Rostedt5a45cfe2008-11-26 00:16:24 -05001632 if (*command & FTRACE_START_FUNC_RET)
1633 ftrace_enable_ftrace_graph_caller();
1634 else if (*command & FTRACE_STOP_FUNC_RET)
1635 ftrace_disable_ftrace_graph_caller();
1636
Steven Rostedt6331c282011-07-13 15:11:02 -04001637#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
1638 /*
1639 * For archs that call ftrace_test_stop_func(), we must
1640 * wait till after we update all the function callers
1641 * before we update the callback. This keeps different
1642 * ops that record different functions from corrupting
1643 * each other.
1644 */
1645 __ftrace_trace_function = __ftrace_trace_function_delay;
1646#endif
1647 function_trace_stop--;
1648
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001649 return 0;
Steven Rostedt3d083392008-05-12 21:20:42 +02001650}
1651
Ingo Molnare309b412008-05-12 21:20:51 +02001652static void ftrace_run_update_code(int command)
Steven Rostedt3d083392008-05-12 21:20:42 +02001653{
Steven Rostedt000ab692009-02-17 13:35:06 -05001654 int ret;
1655
1656 ret = ftrace_arch_code_modify_prepare();
1657 FTRACE_WARN_ON(ret);
1658 if (ret)
1659 return;
1660
Rusty Russell784e2d72008-07-28 12:16:31 -05001661 stop_machine(__ftrace_modify_code, &command, NULL);
Steven Rostedt000ab692009-02-17 13:35:06 -05001662
1663 ret = ftrace_arch_code_modify_post_process();
1664 FTRACE_WARN_ON(ret);
Steven Rostedt3d083392008-05-12 21:20:42 +02001665}
1666
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001667static ftrace_func_t saved_ftrace_func;
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05001668static int ftrace_start_up;
Steven Rostedtb8489142011-05-04 09:27:52 -04001669static int global_start_up;
Steven Rostedtdf4fc312008-11-26 00:16:23 -05001670
1671static void ftrace_startup_enable(int command)
1672{
1673 if (saved_ftrace_func != ftrace_trace_function) {
1674 saved_ftrace_func = ftrace_trace_function;
1675 command |= FTRACE_UPDATE_TRACE_FUNC;
1676 }
1677
1678 if (!command || !ftrace_enabled)
1679 return;
1680
1681 ftrace_run_update_code(command);
1682}
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001683
Steven Rostedta1cd6172011-05-23 15:24:25 -04001684static int ftrace_startup(struct ftrace_ops *ops, int command)
Steven Rostedt3d083392008-05-12 21:20:42 +02001685{
Steven Rostedtb8489142011-05-04 09:27:52 -04001686 bool hash_enable = true;
1687
Steven Rostedt4eebcc82008-05-12 21:20:48 +02001688 if (unlikely(ftrace_disabled))
Steven Rostedta1cd6172011-05-23 15:24:25 -04001689 return -ENODEV;
Steven Rostedt4eebcc82008-05-12 21:20:48 +02001690
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05001691 ftrace_start_up++;
Steven Rostedt982c3502008-11-15 16:31:41 -05001692 command |= FTRACE_ENABLE_CALLS;
Steven Rostedt3d083392008-05-12 21:20:42 +02001693
Steven Rostedtb8489142011-05-04 09:27:52 -04001694 /* ops marked global share the filter hashes */
1695 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1696 ops = &global_ops;
1697 /* Don't update hash if global is already set */
1698 if (global_start_up)
1699 hash_enable = false;
1700 global_start_up++;
1701 }
1702
Steven Rostedted926f92011-05-03 13:25:24 -04001703 ops->flags |= FTRACE_OPS_FL_ENABLED;
Steven Rostedtb8489142011-05-04 09:27:52 -04001704 if (hash_enable)
Steven Rostedted926f92011-05-03 13:25:24 -04001705 ftrace_hash_rec_enable(ops, 1);
1706
Steven Rostedtdf4fc312008-11-26 00:16:23 -05001707 ftrace_startup_enable(command);
Steven Rostedta1cd6172011-05-23 15:24:25 -04001708
1709 return 0;
Steven Rostedt3d083392008-05-12 21:20:42 +02001710}
1711
Steven Rostedtbd69c302011-05-03 21:55:54 -04001712static void ftrace_shutdown(struct ftrace_ops *ops, int command)
Steven Rostedt3d083392008-05-12 21:20:42 +02001713{
Steven Rostedtb8489142011-05-04 09:27:52 -04001714 bool hash_disable = true;
1715
Steven Rostedt4eebcc82008-05-12 21:20:48 +02001716 if (unlikely(ftrace_disabled))
1717 return;
1718
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05001719 ftrace_start_up--;
Frederic Weisbecker9ea1a152009-06-20 06:52:21 +02001720 /*
1721 * Just warn in case of unbalance, no need to kill ftrace, it's not
1722 * critical but the ftrace_call callers may be never nopped again after
1723 * further ftrace uses.
1724 */
1725 WARN_ON_ONCE(ftrace_start_up < 0);
1726
Steven Rostedtb8489142011-05-04 09:27:52 -04001727 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1728 ops = &global_ops;
1729 global_start_up--;
1730 WARN_ON_ONCE(global_start_up < 0);
1731 /* Don't update hash if global still has users */
1732 if (global_start_up) {
1733 WARN_ON_ONCE(!ftrace_start_up);
1734 hash_disable = false;
1735 }
1736 }
1737
1738 if (hash_disable)
Steven Rostedted926f92011-05-03 13:25:24 -04001739 ftrace_hash_rec_disable(ops, 1);
1740
Steven Rostedtb8489142011-05-04 09:27:52 -04001741 if (ops != &global_ops || !global_start_up)
Steven Rostedted926f92011-05-03 13:25:24 -04001742 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
Steven Rostedtb8489142011-05-04 09:27:52 -04001743
1744 if (!ftrace_start_up)
1745 command |= FTRACE_DISABLE_CALLS;
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001746
1747 if (saved_ftrace_func != ftrace_trace_function) {
1748 saved_ftrace_func = ftrace_trace_function;
1749 command |= FTRACE_UPDATE_TRACE_FUNC;
1750 }
1751
1752 if (!command || !ftrace_enabled)
Steven Rostedte6ea44e2009-02-14 01:42:44 -05001753 return;
Steven Rostedt3d083392008-05-12 21:20:42 +02001754
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001755 ftrace_run_update_code(command);
Steven Rostedt3d083392008-05-12 21:20:42 +02001756}
1757
Ingo Molnare309b412008-05-12 21:20:51 +02001758static void ftrace_startup_sysctl(void)
Steven Rostedtb0fc4942008-05-12 21:20:43 +02001759{
Steven Rostedt4eebcc82008-05-12 21:20:48 +02001760 if (unlikely(ftrace_disabled))
1761 return;
1762
Steven Rostedtd61f82d2008-05-12 21:20:43 +02001763 /* Force update next time */
1764 saved_ftrace_func = NULL;
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05001765 /* ftrace_start_up is true if we want ftrace running */
1766 if (ftrace_start_up)
Steven Rostedt79e406d2010-09-14 22:19:46 -04001767 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02001768}
1769
Ingo Molnare309b412008-05-12 21:20:51 +02001770static void ftrace_shutdown_sysctl(void)
Steven Rostedtb0fc4942008-05-12 21:20:43 +02001771{
Steven Rostedt4eebcc82008-05-12 21:20:48 +02001772 if (unlikely(ftrace_disabled))
1773 return;
1774
Steven Rostedt60a7ecf2008-11-05 16:05:44 -05001775 /* ftrace_start_up is true if ftrace is running */
1776 if (ftrace_start_up)
Steven Rostedt79e406d2010-09-14 22:19:46 -04001777 ftrace_run_update_code(FTRACE_DISABLE_CALLS);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02001778}
1779
Steven Rostedt3d083392008-05-12 21:20:42 +02001780static cycle_t ftrace_update_time;
1781static unsigned long ftrace_update_cnt;
1782unsigned long ftrace_update_tot_cnt;
1783
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04001784static int ops_traces_mod(struct ftrace_ops *ops)
1785{
1786 struct ftrace_hash *hash;
1787
1788 hash = ops->filter_hash;
1789 return !!(!hash || !hash->count);
1790}
1791
Steven Rostedt31e88902008-11-14 16:21:19 -08001792static int ftrace_update_code(struct module *mod)
Steven Rostedt3d083392008-05-12 21:20:42 +02001793{
Lai Jiangshane94142a2009-03-13 17:51:27 +08001794 struct dyn_ftrace *p;
Abhishek Sagarf22f9a82008-06-21 23:50:29 +05301795 cycle_t start, stop;
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04001796 unsigned long ref = 0;
1797
1798 /*
1799 * When adding a module, we need to check if tracers are
1800 * currently enabled and if they are set to trace all functions.
1801 * If they are, we need to enable the module functions as well
1802 * as update the reference counts for those function records.
1803 */
1804 if (mod) {
1805 struct ftrace_ops *ops;
1806
1807 for (ops = ftrace_ops_list;
1808 ops != &ftrace_list_end; ops = ops->next) {
1809 if (ops->flags & FTRACE_OPS_FL_ENABLED &&
1810 ops_traces_mod(ops))
1811 ref++;
1812 }
1813 }
Steven Rostedt3d083392008-05-12 21:20:42 +02001814
Ingo Molnar750ed1a2008-05-12 21:20:46 +02001815 start = ftrace_now(raw_smp_processor_id());
Steven Rostedt3d083392008-05-12 21:20:42 +02001816 ftrace_update_cnt = 0;
1817
Lai Jiangshane94142a2009-03-13 17:51:27 +08001818 while (ftrace_new_addrs) {
Abhishek Sagarf22f9a82008-06-21 23:50:29 +05301819
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001820 /* If something went wrong, bail without enabling anything */
1821 if (unlikely(ftrace_disabled))
1822 return -1;
Steven Rostedt3d083392008-05-12 21:20:42 +02001823
Lai Jiangshane94142a2009-03-13 17:51:27 +08001824 p = ftrace_new_addrs;
Lai Jiangshanee000b72009-03-24 13:38:06 +08001825 ftrace_new_addrs = p->newlist;
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04001826 p->flags = ref;
Abhishek Sagar0eb96702008-06-01 21:47:30 +05301827
Jiri Olsa5cb084b2009-10-13 16:33:53 -04001828 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001829 * Do the initial record conversion from mcount jump
Jiri Olsa5cb084b2009-10-13 16:33:53 -04001830 * to the NOP instructions.
1831 */
1832 if (!ftrace_code_disable(mod, p)) {
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001833 ftrace_free_rec(p);
Steven Rostedtd2c8c3e2011-04-25 14:32:42 -04001834 /* Game over */
1835 break;
Jiri Olsa5cb084b2009-10-13 16:33:53 -04001836 }
1837
Jiri Olsa5cb084b2009-10-13 16:33:53 -04001838 ftrace_update_cnt++;
1839
1840 /*
1841 * If the tracing is enabled, go ahead and enable the record.
1842 *
1843 * The reason not to enable the record immediatelly is the
1844 * inherent check of ftrace_make_nop/ftrace_make_call for
1845 * correct previous instructions. Making first the NOP
1846 * conversion puts the module to the correct state, thus
1847 * passing the ftrace_make_call check.
1848 */
Steven Rostedtf7bc8b62011-07-14 23:02:27 -04001849 if (ftrace_start_up && ref) {
Jiri Olsa5cb084b2009-10-13 16:33:53 -04001850 int failed = __ftrace_replace_code(p, 1);
1851 if (failed) {
1852 ftrace_bug(failed, p->ip);
1853 ftrace_free_rec(p);
1854 }
1855 }
Steven Rostedt3d083392008-05-12 21:20:42 +02001856 }
1857
Ingo Molnar750ed1a2008-05-12 21:20:46 +02001858 stop = ftrace_now(raw_smp_processor_id());
Steven Rostedt3d083392008-05-12 21:20:42 +02001859 ftrace_update_time = stop - start;
1860 ftrace_update_tot_cnt += ftrace_update_cnt;
1861
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +02001862 return 0;
1863}
1864
Steven Rostedt68bf21a2008-08-14 15:45:08 -04001865static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001866{
1867 struct ftrace_page *pg;
1868 int cnt;
1869 int i;
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001870
1871 /* allocate a few pages */
1872 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1873 if (!ftrace_pages_start)
1874 return -1;
1875
1876 /*
1877 * Allocate a few more pages.
1878 *
1879 * TODO: have some parser search vmlinux before
1880 * final linking to find all calls to ftrace.
1881 * Then we can:
1882 * a) know how many pages to allocate.
1883 * and/or
1884 * b) set up the table then.
1885 *
1886 * The dynamic code is still necessary for
1887 * modules.
1888 */
1889
1890 pg = ftrace_pages = ftrace_pages_start;
1891
Steven Rostedt68bf21a2008-08-14 15:45:08 -04001892 cnt = num_to_init / ENTRIES_PER_PAGE;
Steven Rostedt08f5ac902008-10-23 09:33:07 -04001893 pr_info("ftrace: allocating %ld entries in %d pages\n",
walimis5821e1b2008-11-15 15:19:06 +08001894 num_to_init, cnt + 1);
Steven Rostedt3c1720f2008-05-12 21:20:43 +02001895
1896 for (i = 0; i < cnt; i++) {
1897 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1898
1899 /* If we fail, we'll try later anyway */
1900 if (!pg->next)
1901 break;
1902
1903 pg = pg->next;
1904 }
1905
1906 return 0;
1907}
1908
Steven Rostedt5072c592008-05-12 21:20:43 +02001909enum {
1910 FTRACE_ITER_FILTER = (1 << 0),
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02001911 FTRACE_ITER_NOTRACE = (1 << 1),
Steven Rostedt3499e462011-04-21 22:59:12 -04001912 FTRACE_ITER_PRINTALL = (1 << 2),
1913 FTRACE_ITER_HASH = (1 << 3),
Steven Rostedt647bcd02011-05-03 14:39:21 -04001914 FTRACE_ITER_ENABLED = (1 << 4),
Steven Rostedt5072c592008-05-12 21:20:43 +02001915};
1916
1917#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1918
1919struct ftrace_iterator {
Steven Rostedt98c4fd02010-09-10 11:47:43 -04001920 loff_t pos;
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001921 loff_t func_pos;
1922 struct ftrace_page *pg;
1923 struct dyn_ftrace *func;
1924 struct ftrace_func_probe *probe;
1925 struct trace_parser parser;
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04001926 struct ftrace_hash *hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04001927 struct ftrace_ops *ops;
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001928 int hidx;
1929 int idx;
1930 unsigned flags;
Steven Rostedt5072c592008-05-12 21:20:43 +02001931};
1932
Ingo Molnare309b412008-05-12 21:20:51 +02001933static void *
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001934t_hash_next(struct seq_file *m, loff_t *pos)
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001935{
1936 struct ftrace_iterator *iter = m->private;
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001937 struct hlist_node *hnd = NULL;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001938 struct hlist_head *hhd;
1939
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001940 (*pos)++;
Steven Rostedt98c4fd02010-09-10 11:47:43 -04001941 iter->pos = *pos;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001942
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001943 if (iter->probe)
1944 hnd = &iter->probe->node;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001945 retry:
1946 if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1947 return NULL;
1948
1949 hhd = &ftrace_func_hash[iter->hidx];
1950
1951 if (hlist_empty(hhd)) {
1952 iter->hidx++;
1953 hnd = NULL;
1954 goto retry;
1955 }
1956
1957 if (!hnd)
1958 hnd = hhd->first;
1959 else {
1960 hnd = hnd->next;
1961 if (!hnd) {
1962 iter->hidx++;
1963 goto retry;
1964 }
1965 }
1966
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001967 if (WARN_ON_ONCE(!hnd))
1968 return NULL;
1969
1970 iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
1971
1972 return iter;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001973}
1974
1975static void *t_hash_start(struct seq_file *m, loff_t *pos)
1976{
1977 struct ftrace_iterator *iter = m->private;
1978 void *p = NULL;
Li Zefand82d6242009-06-24 09:54:54 +08001979 loff_t l;
1980
Steven Rostedt2bccfff2010-09-09 08:43:22 -04001981 if (iter->func_pos > *pos)
1982 return NULL;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001983
Li Zefand82d6242009-06-24 09:54:54 +08001984 iter->hidx = 0;
Steven Rostedt2bccfff2010-09-09 08:43:22 -04001985 for (l = 0; l <= (*pos - iter->func_pos); ) {
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001986 p = t_hash_next(m, &l);
Li Zefand82d6242009-06-24 09:54:54 +08001987 if (!p)
1988 break;
1989 }
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001990 if (!p)
1991 return NULL;
1992
Steven Rostedt98c4fd02010-09-10 11:47:43 -04001993 /* Only set this if we have an item */
1994 iter->flags |= FTRACE_ITER_HASH;
1995
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001996 return iter;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05001997}
1998
Steven Rostedt4aeb6962010-09-09 10:00:28 -04001999static int
2000t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002001{
Steven Rostedtb6887d72009-02-17 12:32:04 -05002002 struct ftrace_func_probe *rec;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002003
Steven Rostedt4aeb6962010-09-09 10:00:28 -04002004 rec = iter->probe;
2005 if (WARN_ON_ONCE(!rec))
2006 return -EIO;
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002007
Steven Rostedt809dcf22009-02-16 23:06:01 -05002008 if (rec->ops->print)
2009 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
2010
Steven Rostedtb375a112009-09-17 00:05:58 -04002011 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002012
2013 if (rec->data)
2014 seq_printf(m, ":%p", rec->data);
2015 seq_putc(m, '\n');
2016
2017 return 0;
2018}
2019
2020static void *
Steven Rostedt5072c592008-05-12 21:20:43 +02002021t_next(struct seq_file *m, void *v, loff_t *pos)
2022{
2023 struct ftrace_iterator *iter = m->private;
Steven Rostedtf45948e2011-05-02 12:29:25 -04002024 struct ftrace_ops *ops = &global_ops;
Steven Rostedt5072c592008-05-12 21:20:43 +02002025 struct dyn_ftrace *rec = NULL;
2026
Steven Rostedt45a4a232011-04-21 23:16:46 -04002027 if (unlikely(ftrace_disabled))
2028 return NULL;
2029
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002030 if (iter->flags & FTRACE_ITER_HASH)
Steven Rostedt4aeb6962010-09-09 10:00:28 -04002031 return t_hash_next(m, pos);
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002032
Steven Rostedt5072c592008-05-12 21:20:43 +02002033 (*pos)++;
Jiri Olsa1106b692011-02-16 17:35:34 +01002034 iter->pos = iter->func_pos = *pos;
Steven Rostedt5072c592008-05-12 21:20:43 +02002035
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05002036 if (iter->flags & FTRACE_ITER_PRINTALL)
Steven Rostedt57c072c2010-09-14 11:21:11 -04002037 return t_hash_start(m, pos);
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05002038
Steven Rostedt5072c592008-05-12 21:20:43 +02002039 retry:
2040 if (iter->idx >= iter->pg->index) {
2041 if (iter->pg->next) {
2042 iter->pg = iter->pg->next;
2043 iter->idx = 0;
2044 goto retry;
2045 }
2046 } else {
2047 rec = &iter->pg->records[iter->idx++];
Steven Rostedta9fdda32008-08-14 22:47:17 -04002048 if ((rec->flags & FTRACE_FL_FREE) ||
2049
Steven Rostedt0183fb12008-11-07 22:36:02 -05002050 ((iter->flags & FTRACE_ITER_FILTER) &&
Steven Rostedtf45948e2011-05-02 12:29:25 -04002051 !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
Steven Rostedt0183fb12008-11-07 22:36:02 -05002052
Steven Rostedt41c52c02008-05-22 11:46:33 -04002053 ((iter->flags & FTRACE_ITER_NOTRACE) &&
Steven Rostedt647bcd02011-05-03 14:39:21 -04002054 !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
2055
2056 ((iter->flags & FTRACE_ITER_ENABLED) &&
2057 !(rec->flags & ~FTRACE_FL_MASK))) {
2058
Steven Rostedt5072c592008-05-12 21:20:43 +02002059 rec = NULL;
2060 goto retry;
2061 }
2062 }
2063
Steven Rostedt4aeb6962010-09-09 10:00:28 -04002064 if (!rec)
Steven Rostedt57c072c2010-09-14 11:21:11 -04002065 return t_hash_start(m, pos);
Steven Rostedt4aeb6962010-09-09 10:00:28 -04002066
2067 iter->func = rec;
2068
2069 return iter;
Steven Rostedt5072c592008-05-12 21:20:43 +02002070}
2071
Steven Rostedt98c4fd02010-09-10 11:47:43 -04002072static void reset_iter_read(struct ftrace_iterator *iter)
2073{
2074 iter->pos = 0;
2075 iter->func_pos = 0;
2076 iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH);
Steven Rostedt5072c592008-05-12 21:20:43 +02002077}
2078
2079static void *t_start(struct seq_file *m, loff_t *pos)
2080{
2081 struct ftrace_iterator *iter = m->private;
Steven Rostedtf45948e2011-05-02 12:29:25 -04002082 struct ftrace_ops *ops = &global_ops;
Steven Rostedt5072c592008-05-12 21:20:43 +02002083 void *p = NULL;
Li Zefan694ce0a2009-06-24 09:54:19 +08002084 loff_t l;
Steven Rostedt5072c592008-05-12 21:20:43 +02002085
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002086 mutex_lock(&ftrace_lock);
Steven Rostedt45a4a232011-04-21 23:16:46 -04002087
2088 if (unlikely(ftrace_disabled))
2089 return NULL;
2090
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05002091 /*
Steven Rostedt98c4fd02010-09-10 11:47:43 -04002092 * If an lseek was done, then reset and start from beginning.
2093 */
2094 if (*pos < iter->pos)
2095 reset_iter_read(iter);
2096
2097 /*
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05002098 * For set_ftrace_filter reading, if we have the filter
2099 * off, we can short cut and just print out that all
2100 * functions are enabled.
2101 */
Steven Rostedtf45948e2011-05-02 12:29:25 -04002102 if (iter->flags & FTRACE_ITER_FILTER && !ops->filter_hash->count) {
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05002103 if (*pos > 0)
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002104 return t_hash_start(m, pos);
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05002105 iter->flags |= FTRACE_ITER_PRINTALL;
Chris Wrightdf091622010-09-09 16:34:59 -07002106 /* reset in case of seek/pread */
2107 iter->flags &= ~FTRACE_ITER_HASH;
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05002108 return iter;
2109 }
2110
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002111 if (iter->flags & FTRACE_ITER_HASH)
2112 return t_hash_start(m, pos);
2113
Steven Rostedt98c4fd02010-09-10 11:47:43 -04002114 /*
2115 * Unfortunately, we need to restart at ftrace_pages_start
2116 * every time we let go of the ftrace_mutex. This is because
2117 * those pointers can change without the lock.
2118 */
Li Zefan694ce0a2009-06-24 09:54:19 +08002119 iter->pg = ftrace_pages_start;
2120 iter->idx = 0;
2121 for (l = 0; l <= *pos; ) {
2122 p = t_next(m, p, &l);
2123 if (!p)
2124 break;
Liming Wang50cdaf02008-11-28 12:13:21 +08002125 }
walimis5821e1b2008-11-15 15:19:06 +08002126
Steven Rostedt4aeb6962010-09-09 10:00:28 -04002127 if (!p) {
2128 if (iter->flags & FTRACE_ITER_FILTER)
2129 return t_hash_start(m, pos);
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002130
Steven Rostedt4aeb6962010-09-09 10:00:28 -04002131 return NULL;
2132 }
2133
2134 return iter;
Steven Rostedt5072c592008-05-12 21:20:43 +02002135}
2136
2137static void t_stop(struct seq_file *m, void *p)
2138{
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002139 mutex_unlock(&ftrace_lock);
Steven Rostedt5072c592008-05-12 21:20:43 +02002140}
2141
2142static int t_show(struct seq_file *m, void *v)
2143{
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05002144 struct ftrace_iterator *iter = m->private;
Steven Rostedt4aeb6962010-09-09 10:00:28 -04002145 struct dyn_ftrace *rec;
Steven Rostedt5072c592008-05-12 21:20:43 +02002146
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002147 if (iter->flags & FTRACE_ITER_HASH)
Steven Rostedt4aeb6962010-09-09 10:00:28 -04002148 return t_hash_show(m, iter);
Steven Rostedt8fc0c702009-02-16 15:28:00 -05002149
Steven Rostedt0c75a3e2009-02-16 11:21:52 -05002150 if (iter->flags & FTRACE_ITER_PRINTALL) {
2151 seq_printf(m, "#### all functions enabled ####\n");
2152 return 0;
2153 }
2154
Steven Rostedt4aeb6962010-09-09 10:00:28 -04002155 rec = iter->func;
2156
Steven Rostedt5072c592008-05-12 21:20:43 +02002157 if (!rec)
2158 return 0;
2159
Steven Rostedt647bcd02011-05-03 14:39:21 -04002160 seq_printf(m, "%ps", (void *)rec->ip);
2161 if (iter->flags & FTRACE_ITER_ENABLED)
2162 seq_printf(m, " (%ld)",
2163 rec->flags & ~FTRACE_FL_MASK);
2164 seq_printf(m, "\n");
Steven Rostedt5072c592008-05-12 21:20:43 +02002165
2166 return 0;
2167}
2168
James Morris88e9d342009-09-22 16:43:43 -07002169static const struct seq_operations show_ftrace_seq_ops = {
Steven Rostedt5072c592008-05-12 21:20:43 +02002170 .start = t_start,
2171 .next = t_next,
2172 .stop = t_stop,
2173 .show = t_show,
2174};
2175
Ingo Molnare309b412008-05-12 21:20:51 +02002176static int
Steven Rostedt5072c592008-05-12 21:20:43 +02002177ftrace_avail_open(struct inode *inode, struct file *file)
2178{
2179 struct ftrace_iterator *iter;
2180 int ret;
2181
Steven Rostedt4eebcc82008-05-12 21:20:48 +02002182 if (unlikely(ftrace_disabled))
2183 return -ENODEV;
2184
Steven Rostedt5072c592008-05-12 21:20:43 +02002185 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2186 if (!iter)
2187 return -ENOMEM;
2188
2189 iter->pg = ftrace_pages_start;
Steven Rostedt5072c592008-05-12 21:20:43 +02002190
2191 ret = seq_open(file, &show_ftrace_seq_ops);
2192 if (!ret) {
2193 struct seq_file *m = file->private_data;
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002194
Steven Rostedt5072c592008-05-12 21:20:43 +02002195 m->private = iter;
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002196 } else {
Steven Rostedt5072c592008-05-12 21:20:43 +02002197 kfree(iter);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002198 }
Steven Rostedt5072c592008-05-12 21:20:43 +02002199
2200 return ret;
2201}
2202
Steven Rostedt647bcd02011-05-03 14:39:21 -04002203static int
2204ftrace_enabled_open(struct inode *inode, struct file *file)
2205{
2206 struct ftrace_iterator *iter;
2207 int ret;
2208
2209 if (unlikely(ftrace_disabled))
2210 return -ENODEV;
2211
2212 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2213 if (!iter)
2214 return -ENOMEM;
2215
2216 iter->pg = ftrace_pages_start;
2217 iter->flags = FTRACE_ITER_ENABLED;
2218
2219 ret = seq_open(file, &show_ftrace_seq_ops);
2220 if (!ret) {
2221 struct seq_file *m = file->private_data;
2222
2223 m->private = iter;
2224 } else {
2225 kfree(iter);
2226 }
2227
2228 return ret;
2229}
2230
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002231static void ftrace_filter_reset(struct ftrace_hash *hash)
Steven Rostedt5072c592008-05-12 21:20:43 +02002232{
Steven Rostedt52baf112009-02-14 01:15:39 -05002233 mutex_lock(&ftrace_lock);
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002234 ftrace_hash_clear(hash);
Steven Rostedt52baf112009-02-14 01:15:39 -05002235 mutex_unlock(&ftrace_lock);
Steven Rostedt5072c592008-05-12 21:20:43 +02002236}
2237
Ingo Molnare309b412008-05-12 21:20:51 +02002238static int
Steven Rostedtf45948e2011-05-02 12:29:25 -04002239ftrace_regex_open(struct ftrace_ops *ops, int flag,
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002240 struct inode *inode, struct file *file)
Steven Rostedt5072c592008-05-12 21:20:43 +02002241{
2242 struct ftrace_iterator *iter;
Steven Rostedtf45948e2011-05-02 12:29:25 -04002243 struct ftrace_hash *hash;
Steven Rostedt5072c592008-05-12 21:20:43 +02002244 int ret = 0;
2245
Steven Rostedt4eebcc82008-05-12 21:20:48 +02002246 if (unlikely(ftrace_disabled))
2247 return -ENODEV;
2248
Steven Rostedt5072c592008-05-12 21:20:43 +02002249 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2250 if (!iter)
2251 return -ENOMEM;
2252
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02002253 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
2254 kfree(iter);
2255 return -ENOMEM;
2256 }
2257
Steven Rostedtf45948e2011-05-02 12:29:25 -04002258 if (flag & FTRACE_ITER_NOTRACE)
2259 hash = ops->notrace_hash;
2260 else
2261 hash = ops->filter_hash;
2262
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002263 iter->ops = ops;
2264 iter->flags = flag;
2265
2266 if (file->f_mode & FMODE_WRITE) {
2267 mutex_lock(&ftrace_lock);
2268 iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
2269 mutex_unlock(&ftrace_lock);
2270
2271 if (!iter->hash) {
2272 trace_parser_put(&iter->parser);
2273 kfree(iter);
2274 return -ENOMEM;
2275 }
2276 }
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002277
Steven Rostedt41c52c02008-05-22 11:46:33 -04002278 mutex_lock(&ftrace_regex_lock);
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002279
Steven Rostedt5072c592008-05-12 21:20:43 +02002280 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04002281 (file->f_flags & O_TRUNC))
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002282 ftrace_filter_reset(iter->hash);
Steven Rostedt5072c592008-05-12 21:20:43 +02002283
2284 if (file->f_mode & FMODE_READ) {
2285 iter->pg = ftrace_pages_start;
Steven Rostedt5072c592008-05-12 21:20:43 +02002286
2287 ret = seq_open(file, &show_ftrace_seq_ops);
2288 if (!ret) {
2289 struct seq_file *m = file->private_data;
2290 m->private = iter;
Li Zefan79fe2492009-09-22 13:54:28 +08002291 } else {
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002292 /* Failed */
2293 free_ftrace_hash(iter->hash);
Li Zefan79fe2492009-09-22 13:54:28 +08002294 trace_parser_put(&iter->parser);
Steven Rostedt5072c592008-05-12 21:20:43 +02002295 kfree(iter);
Li Zefan79fe2492009-09-22 13:54:28 +08002296 }
Steven Rostedt5072c592008-05-12 21:20:43 +02002297 } else
2298 file->private_data = iter;
Steven Rostedt41c52c02008-05-22 11:46:33 -04002299 mutex_unlock(&ftrace_regex_lock);
Steven Rostedt5072c592008-05-12 21:20:43 +02002300
2301 return ret;
2302}
2303
Steven Rostedt41c52c02008-05-22 11:46:33 -04002304static int
2305ftrace_filter_open(struct inode *inode, struct file *file)
2306{
Steven Rostedtf45948e2011-05-02 12:29:25 -04002307 return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER,
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002308 inode, file);
Steven Rostedt41c52c02008-05-22 11:46:33 -04002309}
2310
2311static int
2312ftrace_notrace_open(struct inode *inode, struct file *file)
2313{
Steven Rostedtf45948e2011-05-02 12:29:25 -04002314 return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002315 inode, file);
Steven Rostedt41c52c02008-05-22 11:46:33 -04002316}
2317
Ingo Molnare309b412008-05-12 21:20:51 +02002318static loff_t
Steven Rostedt41c52c02008-05-22 11:46:33 -04002319ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
Steven Rostedt5072c592008-05-12 21:20:43 +02002320{
2321 loff_t ret;
2322
2323 if (file->f_mode & FMODE_READ)
2324 ret = seq_lseek(file, offset, origin);
2325 else
2326 file->f_pos = ret = 1;
2327
2328 return ret;
2329}
2330
Steven Rostedt64e7c442009-02-13 17:08:48 -05002331static int ftrace_match(char *str, char *regex, int len, int type)
Steven Rostedt9f4801e2009-02-13 15:56:43 -05002332{
Steven Rostedt9f4801e2009-02-13 15:56:43 -05002333 int matched = 0;
Li Zefan751e9982010-01-14 10:53:02 +08002334 int slen;
Steven Rostedt9f4801e2009-02-13 15:56:43 -05002335
Steven Rostedt9f4801e2009-02-13 15:56:43 -05002336 switch (type) {
2337 case MATCH_FULL:
2338 if (strcmp(str, regex) == 0)
2339 matched = 1;
2340 break;
2341 case MATCH_FRONT_ONLY:
2342 if (strncmp(str, regex, len) == 0)
2343 matched = 1;
2344 break;
2345 case MATCH_MIDDLE_ONLY:
2346 if (strstr(str, regex))
2347 matched = 1;
2348 break;
2349 case MATCH_END_ONLY:
Li Zefan751e9982010-01-14 10:53:02 +08002350 slen = strlen(str);
2351 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
Steven Rostedt9f4801e2009-02-13 15:56:43 -05002352 matched = 1;
2353 break;
2354 }
2355
2356 return matched;
2357}
2358
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002359static int
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002360enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
Steven Rostedt996e87b2011-04-26 16:11:03 -04002361{
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002362 struct ftrace_func_entry *entry;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002363 int ret = 0;
2364
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002365 entry = ftrace_lookup_ip(hash, rec->ip);
2366 if (not) {
2367 /* Do nothing if it doesn't exist */
2368 if (!entry)
2369 return 0;
2370
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002371 free_hash_entry(hash, entry);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002372 } else {
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002373 /* Do nothing if it exists */
2374 if (entry)
2375 return 0;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002376
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002377 ret = add_hash_entry(hash, rec->ip);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002378 }
2379 return ret;
Steven Rostedt996e87b2011-04-26 16:11:03 -04002380}
2381
Steven Rostedt64e7c442009-02-13 17:08:48 -05002382static int
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002383ftrace_match_record(struct dyn_ftrace *rec, char *mod,
2384 char *regex, int len, int type)
Steven Rostedt64e7c442009-02-13 17:08:48 -05002385{
2386 char str[KSYM_SYMBOL_LEN];
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002387 char *modname;
Steven Rostedt64e7c442009-02-13 17:08:48 -05002388
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002389 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
2390
2391 if (mod) {
2392 /* module lookup requires matching the module */
2393 if (!modname || strcmp(modname, mod))
2394 return 0;
2395
2396 /* blank search means to match all funcs in the mod */
2397 if (!len)
2398 return 1;
2399 }
2400
Steven Rostedt64e7c442009-02-13 17:08:48 -05002401 return ftrace_match(str, regex, len, type);
2402}
2403
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002404static int
2405match_records(struct ftrace_hash *hash, char *buff,
2406 int len, char *mod, int not)
Steven Rostedt9f4801e2009-02-13 15:56:43 -05002407{
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002408 unsigned search_len = 0;
Steven Rostedt9f4801e2009-02-13 15:56:43 -05002409 struct ftrace_page *pg;
2410 struct dyn_ftrace *rec;
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002411 int type = MATCH_FULL;
2412 char *search = buff;
Li Zefan311d16d2009-12-08 11:15:11 +08002413 int found = 0;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002414 int ret;
Steven Rostedt9f4801e2009-02-13 15:56:43 -05002415
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002416 if (len) {
2417 type = filter_parse_regex(buff, len, &search, &not);
2418 search_len = strlen(search);
2419 }
Steven Rostedt9f4801e2009-02-13 15:56:43 -05002420
Steven Rostedt52baf112009-02-14 01:15:39 -05002421 mutex_lock(&ftrace_lock);
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002422
2423 if (unlikely(ftrace_disabled))
2424 goto out_unlock;
2425
Steven Rostedt265c8312009-02-13 12:43:56 -05002426 do_for_each_ftrace_rec(pg, rec) {
Steven Rostedt5072c592008-05-12 21:20:43 +02002427
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002428 if (ftrace_match_record(rec, mod, search, search_len, type)) {
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002429 ret = enter_record(hash, rec, not);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002430 if (ret < 0) {
2431 found = ret;
2432 goto out_unlock;
2433 }
Li Zefan311d16d2009-12-08 11:15:11 +08002434 found = 1;
Steven Rostedt265c8312009-02-13 12:43:56 -05002435 }
2436 } while_for_each_ftrace_rec();
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002437 out_unlock:
Steven Rostedt52baf112009-02-14 01:15:39 -05002438 mutex_unlock(&ftrace_lock);
Li Zefan311d16d2009-12-08 11:15:11 +08002439
2440 return found;
Steven Rostedt5072c592008-05-12 21:20:43 +02002441}
2442
Steven Rostedt64e7c442009-02-13 17:08:48 -05002443static int
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002444ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
Steven Rostedt64e7c442009-02-13 17:08:48 -05002445{
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002446 return match_records(hash, buff, len, NULL, 0);
Steven Rostedt64e7c442009-02-13 17:08:48 -05002447}
2448
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002449static int
2450ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
Steven Rostedt64e7c442009-02-13 17:08:48 -05002451{
Steven Rostedt64e7c442009-02-13 17:08:48 -05002452 int not = 0;
Steven Rostedt6a24a242009-02-17 11:20:26 -05002453
Steven Rostedt64e7c442009-02-13 17:08:48 -05002454 /* blank or '*' mean the same */
2455 if (strcmp(buff, "*") == 0)
2456 buff[0] = 0;
2457
2458 /* handle the case of 'dont filter this module' */
2459 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
2460 buff[0] = 0;
2461 not = 1;
2462 }
2463
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002464 return match_records(hash, buff, strlen(buff), mod, not);
Steven Rostedt64e7c442009-02-13 17:08:48 -05002465}
2466
Steven Rostedtf6180772009-02-14 00:40:25 -05002467/*
2468 * We register the module command as a template to show others how
2469 * to register the a command as well.
2470 */
2471
2472static int
Steven Rostedt43dd61c2011-07-07 11:09:22 -04002473ftrace_mod_callback(struct ftrace_hash *hash,
2474 char *func, char *cmd, char *param, int enable)
Steven Rostedtf6180772009-02-14 00:40:25 -05002475{
2476 char *mod;
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002477 int ret = -EINVAL;
Steven Rostedtf6180772009-02-14 00:40:25 -05002478
2479 /*
2480 * cmd == 'mod' because we only registered this func
2481 * for the 'mod' ftrace_func_command.
2482 * But if you register one func with multiple commands,
2483 * you can tell which command was used by the cmd
2484 * parameter.
2485 */
2486
2487 /* we must have a module name */
2488 if (!param)
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002489 return ret;
Steven Rostedtf6180772009-02-14 00:40:25 -05002490
2491 mod = strsep(&param, ":");
2492 if (!strlen(mod))
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002493 return ret;
Steven Rostedtf6180772009-02-14 00:40:25 -05002494
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002495 ret = ftrace_match_module_records(hash, func, mod);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002496 if (!ret)
2497 ret = -EINVAL;
2498 if (ret < 0)
2499 return ret;
2500
2501 return 0;
Steven Rostedtf6180772009-02-14 00:40:25 -05002502}
2503
2504static struct ftrace_func_command ftrace_mod_cmd = {
2505 .name = "mod",
2506 .func = ftrace_mod_callback,
2507};
2508
2509static int __init ftrace_mod_cmd_init(void)
2510{
2511 return register_ftrace_command(&ftrace_mod_cmd);
2512}
2513device_initcall(ftrace_mod_cmd_init);
2514
Steven Rostedt59df055f2009-02-14 15:29:06 -05002515static void
Steven Rostedtb6887d72009-02-17 12:32:04 -05002516function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
Steven Rostedt59df055f2009-02-14 15:29:06 -05002517{
Steven Rostedtb6887d72009-02-17 12:32:04 -05002518 struct ftrace_func_probe *entry;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002519 struct hlist_head *hhd;
2520 struct hlist_node *n;
2521 unsigned long key;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002522
2523 key = hash_long(ip, FTRACE_HASH_BITS);
2524
2525 hhd = &ftrace_func_hash[key];
2526
2527 if (hlist_empty(hhd))
2528 return;
2529
2530 /*
2531 * Disable preemption for these calls to prevent a RCU grace
2532 * period. This syncs the hash iteration and freeing of items
2533 * on the hash. rcu_read_lock is too dangerous here.
2534 */
Steven Rostedt5168ae52010-06-03 09:36:50 -04002535 preempt_disable_notrace();
Steven Rostedt59df055f2009-02-14 15:29:06 -05002536 hlist_for_each_entry_rcu(entry, n, hhd, node) {
2537 if (entry->ip == ip)
2538 entry->ops->func(ip, parent_ip, &entry->data);
2539 }
Steven Rostedt5168ae52010-06-03 09:36:50 -04002540 preempt_enable_notrace();
Steven Rostedt59df055f2009-02-14 15:29:06 -05002541}
2542
Steven Rostedtb6887d72009-02-17 12:32:04 -05002543static struct ftrace_ops trace_probe_ops __read_mostly =
Steven Rostedt59df055f2009-02-14 15:29:06 -05002544{
Steven Rostedtfb9fb012009-03-25 13:26:41 -04002545 .func = function_trace_probe_call,
Steven Rostedt59df055f2009-02-14 15:29:06 -05002546};
2547
Steven Rostedtb6887d72009-02-17 12:32:04 -05002548static int ftrace_probe_registered;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002549
Steven Rostedtb6887d72009-02-17 12:32:04 -05002550static void __enable_ftrace_function_probe(void)
Steven Rostedt59df055f2009-02-14 15:29:06 -05002551{
Steven Rostedtb8489142011-05-04 09:27:52 -04002552 int ret;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002553 int i;
2554
Steven Rostedtb6887d72009-02-17 12:32:04 -05002555 if (ftrace_probe_registered)
Steven Rostedt59df055f2009-02-14 15:29:06 -05002556 return;
2557
2558 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2559 struct hlist_head *hhd = &ftrace_func_hash[i];
2560 if (hhd->first)
2561 break;
2562 }
2563 /* Nothing registered? */
2564 if (i == FTRACE_FUNC_HASHSIZE)
2565 return;
2566
Steven Rostedtb8489142011-05-04 09:27:52 -04002567 ret = __register_ftrace_function(&trace_probe_ops);
2568 if (!ret)
Steven Rostedta1cd6172011-05-23 15:24:25 -04002569 ret = ftrace_startup(&trace_probe_ops, 0);
Steven Rostedtb8489142011-05-04 09:27:52 -04002570
Steven Rostedtb6887d72009-02-17 12:32:04 -05002571 ftrace_probe_registered = 1;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002572}
2573
Steven Rostedtb6887d72009-02-17 12:32:04 -05002574static void __disable_ftrace_function_probe(void)
Steven Rostedt59df055f2009-02-14 15:29:06 -05002575{
Steven Rostedtb8489142011-05-04 09:27:52 -04002576 int ret;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002577 int i;
2578
Steven Rostedtb6887d72009-02-17 12:32:04 -05002579 if (!ftrace_probe_registered)
Steven Rostedt59df055f2009-02-14 15:29:06 -05002580 return;
2581
2582 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2583 struct hlist_head *hhd = &ftrace_func_hash[i];
2584 if (hhd->first)
2585 return;
2586 }
2587
2588 /* no more funcs left */
Steven Rostedtb8489142011-05-04 09:27:52 -04002589 ret = __unregister_ftrace_function(&trace_probe_ops);
2590 if (!ret)
2591 ftrace_shutdown(&trace_probe_ops, 0);
2592
Steven Rostedtb6887d72009-02-17 12:32:04 -05002593 ftrace_probe_registered = 0;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002594}
2595
2596
2597static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2598{
Steven Rostedtb6887d72009-02-17 12:32:04 -05002599 struct ftrace_func_probe *entry =
2600 container_of(rhp, struct ftrace_func_probe, rcu);
Steven Rostedt59df055f2009-02-14 15:29:06 -05002601
2602 if (entry->ops->free)
2603 entry->ops->free(&entry->data);
2604 kfree(entry);
2605}
2606
2607
2608int
Steven Rostedtb6887d72009-02-17 12:32:04 -05002609register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
Steven Rostedt59df055f2009-02-14 15:29:06 -05002610 void *data)
2611{
Steven Rostedtb6887d72009-02-17 12:32:04 -05002612 struct ftrace_func_probe *entry;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002613 struct ftrace_page *pg;
2614 struct dyn_ftrace *rec;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002615 int type, len, not;
Steven Rostedt6a24a242009-02-17 11:20:26 -05002616 unsigned long key;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002617 int count = 0;
2618 char *search;
2619
Frederic Weisbecker3f6fe062009-09-24 21:31:51 +02002620 type = filter_parse_regex(glob, strlen(glob), &search, &not);
Steven Rostedt59df055f2009-02-14 15:29:06 -05002621 len = strlen(search);
2622
Steven Rostedtb6887d72009-02-17 12:32:04 -05002623 /* we do not support '!' for function probes */
Steven Rostedt59df055f2009-02-14 15:29:06 -05002624 if (WARN_ON(not))
2625 return -EINVAL;
2626
2627 mutex_lock(&ftrace_lock);
Steven Rostedt59df055f2009-02-14 15:29:06 -05002628
Steven Rostedt45a4a232011-04-21 23:16:46 -04002629 if (unlikely(ftrace_disabled))
2630 goto out_unlock;
2631
Steven Rostedt59df055f2009-02-14 15:29:06 -05002632 do_for_each_ftrace_rec(pg, rec) {
2633
Steven Rostedtb9df92d2011-04-28 20:32:08 -04002634 if (!ftrace_match_record(rec, NULL, search, len, type))
Steven Rostedt59df055f2009-02-14 15:29:06 -05002635 continue;
2636
2637 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2638 if (!entry) {
Steven Rostedtb6887d72009-02-17 12:32:04 -05002639 /* If we did not process any, then return error */
Steven Rostedt59df055f2009-02-14 15:29:06 -05002640 if (!count)
2641 count = -ENOMEM;
2642 goto out_unlock;
2643 }
2644
2645 count++;
2646
2647 entry->data = data;
2648
2649 /*
2650 * The caller might want to do something special
2651 * for each function we find. We call the callback
2652 * to give the caller an opportunity to do so.
2653 */
2654 if (ops->callback) {
2655 if (ops->callback(rec->ip, &entry->data) < 0) {
2656 /* caller does not like this func */
2657 kfree(entry);
2658 continue;
2659 }
2660 }
2661
2662 entry->ops = ops;
2663 entry->ip = rec->ip;
2664
2665 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2666 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2667
2668 } while_for_each_ftrace_rec();
Steven Rostedtb6887d72009-02-17 12:32:04 -05002669 __enable_ftrace_function_probe();
Steven Rostedt59df055f2009-02-14 15:29:06 -05002670
2671 out_unlock:
2672 mutex_unlock(&ftrace_lock);
2673
2674 return count;
2675}
2676
2677enum {
Steven Rostedtb6887d72009-02-17 12:32:04 -05002678 PROBE_TEST_FUNC = 1,
2679 PROBE_TEST_DATA = 2
Steven Rostedt59df055f2009-02-14 15:29:06 -05002680};
2681
2682static void
Steven Rostedtb6887d72009-02-17 12:32:04 -05002683__unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
Steven Rostedt59df055f2009-02-14 15:29:06 -05002684 void *data, int flags)
2685{
Steven Rostedtb6887d72009-02-17 12:32:04 -05002686 struct ftrace_func_probe *entry;
Steven Rostedt59df055f2009-02-14 15:29:06 -05002687 struct hlist_node *n, *tmp;
2688 char str[KSYM_SYMBOL_LEN];
2689 int type = MATCH_FULL;
2690 int i, len = 0;
2691 char *search;
2692
Atsushi Tsujib36461d2009-09-15 19:06:30 +09002693 if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
Steven Rostedt59df055f2009-02-14 15:29:06 -05002694 glob = NULL;
Atsushi Tsujib36461d2009-09-15 19:06:30 +09002695 else if (glob) {
Steven Rostedt59df055f2009-02-14 15:29:06 -05002696 int not;
2697
Frederic Weisbecker3f6fe062009-09-24 21:31:51 +02002698 type = filter_parse_regex(glob, strlen(glob), &search, &not);
Steven Rostedt59df055f2009-02-14 15:29:06 -05002699 len = strlen(search);
2700
Steven Rostedtb6887d72009-02-17 12:32:04 -05002701 /* we do not support '!' for function probes */
Steven Rostedt59df055f2009-02-14 15:29:06 -05002702 if (WARN_ON(not))
2703 return;
2704 }
2705
2706 mutex_lock(&ftrace_lock);
2707 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2708 struct hlist_head *hhd = &ftrace_func_hash[i];
2709
2710 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2711
2712 /* break up if statements for readability */
Steven Rostedtb6887d72009-02-17 12:32:04 -05002713 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
Steven Rostedt59df055f2009-02-14 15:29:06 -05002714 continue;
2715
Steven Rostedtb6887d72009-02-17 12:32:04 -05002716 if ((flags & PROBE_TEST_DATA) && entry->data != data)
Steven Rostedt59df055f2009-02-14 15:29:06 -05002717 continue;
2718
2719 /* do this last, since it is the most expensive */
2720 if (glob) {
2721 kallsyms_lookup(entry->ip, NULL, NULL,
2722 NULL, str);
2723 if (!ftrace_match(str, glob, len, type))
2724 continue;
2725 }
2726
2727 hlist_del(&entry->node);
2728 call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2729 }
2730 }
Steven Rostedtb6887d72009-02-17 12:32:04 -05002731 __disable_ftrace_function_probe();
Steven Rostedt59df055f2009-02-14 15:29:06 -05002732 mutex_unlock(&ftrace_lock);
2733}
2734
2735void
Steven Rostedtb6887d72009-02-17 12:32:04 -05002736unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
Steven Rostedt59df055f2009-02-14 15:29:06 -05002737 void *data)
2738{
Steven Rostedtb6887d72009-02-17 12:32:04 -05002739 __unregister_ftrace_function_probe(glob, ops, data,
2740 PROBE_TEST_FUNC | PROBE_TEST_DATA);
Steven Rostedt59df055f2009-02-14 15:29:06 -05002741}
2742
2743void
Steven Rostedtb6887d72009-02-17 12:32:04 -05002744unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
Steven Rostedt59df055f2009-02-14 15:29:06 -05002745{
Steven Rostedtb6887d72009-02-17 12:32:04 -05002746 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
Steven Rostedt59df055f2009-02-14 15:29:06 -05002747}
2748
Steven Rostedtb6887d72009-02-17 12:32:04 -05002749void unregister_ftrace_function_probe_all(char *glob)
Steven Rostedt59df055f2009-02-14 15:29:06 -05002750{
Steven Rostedtb6887d72009-02-17 12:32:04 -05002751 __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
Steven Rostedt59df055f2009-02-14 15:29:06 -05002752}
2753
Steven Rostedtf6180772009-02-14 00:40:25 -05002754static LIST_HEAD(ftrace_commands);
2755static DEFINE_MUTEX(ftrace_cmd_mutex);
2756
2757int register_ftrace_command(struct ftrace_func_command *cmd)
2758{
2759 struct ftrace_func_command *p;
2760 int ret = 0;
2761
2762 mutex_lock(&ftrace_cmd_mutex);
2763 list_for_each_entry(p, &ftrace_commands, list) {
2764 if (strcmp(cmd->name, p->name) == 0) {
2765 ret = -EBUSY;
2766 goto out_unlock;
2767 }
2768 }
2769 list_add(&cmd->list, &ftrace_commands);
2770 out_unlock:
2771 mutex_unlock(&ftrace_cmd_mutex);
2772
2773 return ret;
2774}
2775
2776int unregister_ftrace_command(struct ftrace_func_command *cmd)
2777{
2778 struct ftrace_func_command *p, *n;
2779 int ret = -ENODEV;
2780
2781 mutex_lock(&ftrace_cmd_mutex);
2782 list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2783 if (strcmp(cmd->name, p->name) == 0) {
2784 ret = 0;
2785 list_del_init(&p->list);
2786 goto out_unlock;
2787 }
2788 }
2789 out_unlock:
2790 mutex_unlock(&ftrace_cmd_mutex);
2791
2792 return ret;
2793}
2794
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002795static int ftrace_process_regex(struct ftrace_hash *hash,
2796 char *buff, int len, int enable)
Steven Rostedt64e7c442009-02-13 17:08:48 -05002797{
Steven Rostedtf6180772009-02-14 00:40:25 -05002798 char *func, *command, *next = buff;
Steven Rostedt6a24a242009-02-17 11:20:26 -05002799 struct ftrace_func_command *p;
GuoWen Li0aff1c02011-06-01 19:18:47 +08002800 int ret = -EINVAL;
Steven Rostedt64e7c442009-02-13 17:08:48 -05002801
2802 func = strsep(&next, ":");
2803
2804 if (!next) {
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002805 ret = ftrace_match_records(hash, func, len);
Steven Rostedtb448c4e2011-04-29 15:12:32 -04002806 if (!ret)
2807 ret = -EINVAL;
2808 if (ret < 0)
2809 return ret;
2810 return 0;
Steven Rostedt64e7c442009-02-13 17:08:48 -05002811 }
2812
Steven Rostedtf6180772009-02-14 00:40:25 -05002813 /* command found */
Steven Rostedt64e7c442009-02-13 17:08:48 -05002814
2815 command = strsep(&next, ":");
2816
Steven Rostedtf6180772009-02-14 00:40:25 -05002817 mutex_lock(&ftrace_cmd_mutex);
2818 list_for_each_entry(p, &ftrace_commands, list) {
2819 if (strcmp(p->name, command) == 0) {
Steven Rostedt43dd61c2011-07-07 11:09:22 -04002820 ret = p->func(hash, func, command, next, enable);
Steven Rostedtf6180772009-02-14 00:40:25 -05002821 goto out_unlock;
2822 }
Steven Rostedt64e7c442009-02-13 17:08:48 -05002823 }
Steven Rostedtf6180772009-02-14 00:40:25 -05002824 out_unlock:
2825 mutex_unlock(&ftrace_cmd_mutex);
Steven Rostedt64e7c442009-02-13 17:08:48 -05002826
Steven Rostedtf6180772009-02-14 00:40:25 -05002827 return ret;
Steven Rostedt64e7c442009-02-13 17:08:48 -05002828}
2829
Ingo Molnare309b412008-05-12 21:20:51 +02002830static ssize_t
Steven Rostedt41c52c02008-05-22 11:46:33 -04002831ftrace_regex_write(struct file *file, const char __user *ubuf,
2832 size_t cnt, loff_t *ppos, int enable)
Steven Rostedt5072c592008-05-12 21:20:43 +02002833{
2834 struct ftrace_iterator *iter;
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02002835 struct trace_parser *parser;
2836 ssize_t ret, read;
Steven Rostedt5072c592008-05-12 21:20:43 +02002837
Li Zefan4ba79782009-09-22 13:52:20 +08002838 if (!cnt)
Steven Rostedt5072c592008-05-12 21:20:43 +02002839 return 0;
2840
Steven Rostedt41c52c02008-05-22 11:46:33 -04002841 mutex_lock(&ftrace_regex_lock);
Steven Rostedt5072c592008-05-12 21:20:43 +02002842
Steven Rostedt45a4a232011-04-21 23:16:46 -04002843 ret = -ENODEV;
2844 if (unlikely(ftrace_disabled))
2845 goto out_unlock;
2846
Steven Rostedt5072c592008-05-12 21:20:43 +02002847 if (file->f_mode & FMODE_READ) {
2848 struct seq_file *m = file->private_data;
2849 iter = m->private;
2850 } else
2851 iter = file->private_data;
2852
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02002853 parser = &iter->parser;
2854 read = trace_get_user(parser, ubuf, cnt, ppos);
Steven Rostedt5072c592008-05-12 21:20:43 +02002855
Li Zefan4ba79782009-09-22 13:52:20 +08002856 if (read >= 0 && trace_parser_loaded(parser) &&
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02002857 !trace_parser_cont(parser)) {
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002858 ret = ftrace_process_regex(iter->hash, parser->buffer,
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02002859 parser->idx, enable);
Li Zefan313254a2009-12-08 11:15:30 +08002860 trace_parser_clear(parser);
Steven Rostedt5072c592008-05-12 21:20:43 +02002861 if (ret)
Li Zefaned146b252009-11-03 08:55:38 +08002862 goto out_unlock;
Steven Rostedt5072c592008-05-12 21:20:43 +02002863 }
2864
Steven Rostedt5072c592008-05-12 21:20:43 +02002865 ret = read;
Li Zefaned146b252009-11-03 08:55:38 +08002866out_unlock:
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02002867 mutex_unlock(&ftrace_regex_lock);
Li Zefaned146b252009-11-03 08:55:38 +08002868
Steven Rostedt5072c592008-05-12 21:20:43 +02002869 return ret;
2870}
2871
Steven Rostedt41c52c02008-05-22 11:46:33 -04002872static ssize_t
2873ftrace_filter_write(struct file *file, const char __user *ubuf,
2874 size_t cnt, loff_t *ppos)
2875{
2876 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2877}
2878
2879static ssize_t
2880ftrace_notrace_write(struct file *file, const char __user *ubuf,
2881 size_t cnt, loff_t *ppos)
2882{
2883 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2884}
2885
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002886static int
Steven Rostedtf45948e2011-05-02 12:29:25 -04002887ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
2888 int reset, int enable)
Steven Rostedt41c52c02008-05-22 11:46:33 -04002889{
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002890 struct ftrace_hash **orig_hash;
Steven Rostedtf45948e2011-05-02 12:29:25 -04002891 struct ftrace_hash *hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002892 int ret;
Steven Rostedtf45948e2011-05-02 12:29:25 -04002893
Steven Rostedt936e0742011-05-05 22:54:01 -04002894 /* All global ops uses the global ops filters */
2895 if (ops->flags & FTRACE_OPS_FL_GLOBAL)
2896 ops = &global_ops;
2897
Steven Rostedt41c52c02008-05-22 11:46:33 -04002898 if (unlikely(ftrace_disabled))
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002899 return -ENODEV;
Steven Rostedt41c52c02008-05-22 11:46:33 -04002900
Steven Rostedtf45948e2011-05-02 12:29:25 -04002901 if (enable)
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002902 orig_hash = &ops->filter_hash;
Steven Rostedtf45948e2011-05-02 12:29:25 -04002903 else
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002904 orig_hash = &ops->notrace_hash;
2905
2906 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
2907 if (!hash)
2908 return -ENOMEM;
Steven Rostedtf45948e2011-05-02 12:29:25 -04002909
Steven Rostedt41c52c02008-05-22 11:46:33 -04002910 mutex_lock(&ftrace_regex_lock);
2911 if (reset)
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002912 ftrace_filter_reset(hash);
Steven Rostedt41c52c02008-05-22 11:46:33 -04002913 if (buf)
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04002914 ftrace_match_records(hash, buf, len);
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002915
2916 mutex_lock(&ftrace_lock);
Steven Rostedt41fb61c2011-07-13 15:03:44 -04002917 ret = ftrace_hash_move(ops, enable, orig_hash, hash);
Steven Rostedt072126f2011-07-13 15:08:31 -04002918 if (!ret && ops->flags & FTRACE_OPS_FL_ENABLED
2919 && ftrace_enabled)
2920 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2921
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002922 mutex_unlock(&ftrace_lock);
2923
Steven Rostedt41c52c02008-05-22 11:46:33 -04002924 mutex_unlock(&ftrace_regex_lock);
Steven Rostedt33dc9b12011-05-02 17:34:47 -04002925
2926 free_ftrace_hash(hash);
2927 return ret;
Steven Rostedt41c52c02008-05-22 11:46:33 -04002928}
2929
Steven Rostedt77a2b372008-05-12 21:20:45 +02002930/**
2931 * ftrace_set_filter - set a function to filter on in ftrace
Steven Rostedt936e0742011-05-05 22:54:01 -04002932 * @ops - the ops to set the filter with
Steven Rostedt77a2b372008-05-12 21:20:45 +02002933 * @buf - the string that holds the function filter text.
2934 * @len - the length of the string.
2935 * @reset - non zero to reset all filters before applying this filter.
2936 *
2937 * Filters denote which functions should be enabled when tracing is enabled.
2938 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2939 */
Steven Rostedt936e0742011-05-05 22:54:01 -04002940void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
2941 int len, int reset)
Steven Rostedt77a2b372008-05-12 21:20:45 +02002942{
Steven Rostedt936e0742011-05-05 22:54:01 -04002943 ftrace_set_regex(ops, buf, len, reset, 1);
Steven Rostedt41c52c02008-05-22 11:46:33 -04002944}
Steven Rostedt936e0742011-05-05 22:54:01 -04002945EXPORT_SYMBOL_GPL(ftrace_set_filter);
Steven Rostedt4eebcc82008-05-12 21:20:48 +02002946
Steven Rostedt41c52c02008-05-22 11:46:33 -04002947/**
2948 * ftrace_set_notrace - set a function to not trace in ftrace
Steven Rostedt936e0742011-05-05 22:54:01 -04002949 * @ops - the ops to set the notrace filter with
Steven Rostedt41c52c02008-05-22 11:46:33 -04002950 * @buf - the string that holds the function notrace text.
2951 * @len - the length of the string.
2952 * @reset - non zero to reset all filters before applying this filter.
2953 *
2954 * Notrace Filters denote which functions should not be enabled when tracing
2955 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2956 * for tracing.
2957 */
Steven Rostedt936e0742011-05-05 22:54:01 -04002958void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
2959 int len, int reset)
2960{
2961 ftrace_set_regex(ops, buf, len, reset, 0);
2962}
2963EXPORT_SYMBOL_GPL(ftrace_set_notrace);
2964/**
2965 * ftrace_set_filter - set a function to filter on in ftrace
2966 * @ops - the ops to set the filter with
2967 * @buf - the string that holds the function filter text.
2968 * @len - the length of the string.
2969 * @reset - non zero to reset all filters before applying this filter.
2970 *
2971 * Filters denote which functions should be enabled when tracing is enabled.
2972 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2973 */
2974void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
2975{
2976 ftrace_set_regex(&global_ops, buf, len, reset, 1);
2977}
2978EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
2979
2980/**
2981 * ftrace_set_notrace - set a function to not trace in ftrace
2982 * @ops - the ops to set the notrace filter with
2983 * @buf - the string that holds the function notrace text.
2984 * @len - the length of the string.
2985 * @reset - non zero to reset all filters before applying this filter.
2986 *
2987 * Notrace Filters denote which functions should not be enabled when tracing
2988 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2989 * for tracing.
2990 */
2991void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
Steven Rostedt41c52c02008-05-22 11:46:33 -04002992{
Steven Rostedtf45948e2011-05-02 12:29:25 -04002993 ftrace_set_regex(&global_ops, buf, len, reset, 0);
Steven Rostedt77a2b372008-05-12 21:20:45 +02002994}
Steven Rostedt936e0742011-05-05 22:54:01 -04002995EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
Steven Rostedt77a2b372008-05-12 21:20:45 +02002996
Steven Rostedt2af15d62009-05-28 13:37:24 -04002997/*
2998 * command line interface to allow users to set filters on boot up.
2999 */
3000#define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
3001static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
3002static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
3003
3004static int __init set_ftrace_notrace(char *str)
3005{
3006 strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
3007 return 1;
3008}
3009__setup("ftrace_notrace=", set_ftrace_notrace);
3010
3011static int __init set_ftrace_filter(char *str)
3012{
3013 strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
3014 return 1;
3015}
3016__setup("ftrace_filter=", set_ftrace_filter);
3017
Stefan Assmann369bc182009-10-12 22:17:21 +02003018#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Lai Jiangshanf6060f42009-11-05 11:16:17 +08003019static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
Steven Rostedt801c29f2010-03-05 20:02:19 -05003020static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
3021
Stefan Assmann369bc182009-10-12 22:17:21 +02003022static int __init set_graph_function(char *str)
3023{
Frederic Weisbecker06f43d62009-10-14 20:43:39 +02003024 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
Stefan Assmann369bc182009-10-12 22:17:21 +02003025 return 1;
3026}
3027__setup("ftrace_graph_filter=", set_graph_function);
3028
3029static void __init set_ftrace_early_graph(char *buf)
3030{
3031 int ret;
3032 char *func;
3033
3034 while (buf) {
3035 func = strsep(&buf, ",");
3036 /* we allow only one expression at a time */
3037 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3038 func);
3039 if (ret)
3040 printk(KERN_DEBUG "ftrace: function %s not "
3041 "traceable\n", func);
3042 }
3043}
3044#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3045
Steven Rostedtf45948e2011-05-02 12:29:25 -04003046static void __init
3047set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable)
Steven Rostedt2af15d62009-05-28 13:37:24 -04003048{
3049 char *func;
3050
3051 while (buf) {
3052 func = strsep(&buf, ",");
Steven Rostedtf45948e2011-05-02 12:29:25 -04003053 ftrace_set_regex(ops, func, strlen(func), 0, enable);
Steven Rostedt2af15d62009-05-28 13:37:24 -04003054 }
3055}
3056
3057static void __init set_ftrace_early_filters(void)
3058{
3059 if (ftrace_filter_buf[0])
Steven Rostedtf45948e2011-05-02 12:29:25 -04003060 set_ftrace_early_filter(&global_ops, ftrace_filter_buf, 1);
Steven Rostedt2af15d62009-05-28 13:37:24 -04003061 if (ftrace_notrace_buf[0])
Steven Rostedtf45948e2011-05-02 12:29:25 -04003062 set_ftrace_early_filter(&global_ops, ftrace_notrace_buf, 0);
Stefan Assmann369bc182009-10-12 22:17:21 +02003063#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3064 if (ftrace_graph_buf[0])
3065 set_ftrace_early_graph(ftrace_graph_buf);
3066#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
Steven Rostedt2af15d62009-05-28 13:37:24 -04003067}
3068
Ingo Molnare309b412008-05-12 21:20:51 +02003069static int
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04003070ftrace_regex_release(struct inode *inode, struct file *file)
Steven Rostedt5072c592008-05-12 21:20:43 +02003071{
3072 struct seq_file *m = (struct seq_file *)file->private_data;
3073 struct ftrace_iterator *iter;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003074 struct ftrace_hash **orig_hash;
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003075 struct trace_parser *parser;
Steven Rostedted926f92011-05-03 13:25:24 -04003076 int filter_hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003077 int ret;
Steven Rostedt5072c592008-05-12 21:20:43 +02003078
Steven Rostedt41c52c02008-05-22 11:46:33 -04003079 mutex_lock(&ftrace_regex_lock);
Steven Rostedt5072c592008-05-12 21:20:43 +02003080 if (file->f_mode & FMODE_READ) {
3081 iter = m->private;
3082
3083 seq_release(inode, file);
3084 } else
3085 iter = file->private_data;
3086
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003087 parser = &iter->parser;
3088 if (trace_parser_loaded(parser)) {
3089 parser->buffer[parser->idx] = 0;
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04003090 ftrace_match_records(iter->hash, parser->buffer, parser->idx);
Steven Rostedt5072c592008-05-12 21:20:43 +02003091 }
3092
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003093 trace_parser_put(parser);
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003094
Steven Rostedt058e2972011-04-29 22:35:33 -04003095 if (file->f_mode & FMODE_WRITE) {
Steven Rostedted926f92011-05-03 13:25:24 -04003096 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3097
3098 if (filter_hash)
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003099 orig_hash = &iter->ops->filter_hash;
Steven Rostedted926f92011-05-03 13:25:24 -04003100 else
3101 orig_hash = &iter->ops->notrace_hash;
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003102
Steven Rostedt058e2972011-04-29 22:35:33 -04003103 mutex_lock(&ftrace_lock);
Steven Rostedt41fb61c2011-07-13 15:03:44 -04003104 ret = ftrace_hash_move(iter->ops, filter_hash,
3105 orig_hash, iter->hash);
3106 if (!ret && (iter->ops->flags & FTRACE_OPS_FL_ENABLED)
3107 && ftrace_enabled)
3108 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
3109
Steven Rostedt058e2972011-04-29 22:35:33 -04003110 mutex_unlock(&ftrace_lock);
3111 }
Steven Rostedt33dc9b12011-05-02 17:34:47 -04003112 free_ftrace_hash(iter->hash);
3113 kfree(iter);
Steven Rostedt058e2972011-04-29 22:35:33 -04003114
Steven Rostedt41c52c02008-05-22 11:46:33 -04003115 mutex_unlock(&ftrace_regex_lock);
Steven Rostedt5072c592008-05-12 21:20:43 +02003116 return 0;
3117}
3118
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003119static const struct file_operations ftrace_avail_fops = {
Steven Rostedt5072c592008-05-12 21:20:43 +02003120 .open = ftrace_avail_open,
3121 .read = seq_read,
3122 .llseek = seq_lseek,
Li Zefan3be04b42009-08-17 16:54:03 +08003123 .release = seq_release_private,
Steven Rostedt5072c592008-05-12 21:20:43 +02003124};
3125
Steven Rostedt647bcd02011-05-03 14:39:21 -04003126static const struct file_operations ftrace_enabled_fops = {
3127 .open = ftrace_enabled_open,
3128 .read = seq_read,
3129 .llseek = seq_lseek,
3130 .release = seq_release_private,
3131};
3132
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003133static const struct file_operations ftrace_filter_fops = {
Steven Rostedt5072c592008-05-12 21:20:43 +02003134 .open = ftrace_filter_open,
Lai Jiangshan850a80c2009-03-13 17:47:23 +08003135 .read = seq_read,
Steven Rostedt5072c592008-05-12 21:20:43 +02003136 .write = ftrace_filter_write,
Steven Rostedt98c4fd02010-09-10 11:47:43 -04003137 .llseek = ftrace_regex_lseek,
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04003138 .release = ftrace_regex_release,
Steven Rostedt5072c592008-05-12 21:20:43 +02003139};
3140
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003141static const struct file_operations ftrace_notrace_fops = {
Steven Rostedt41c52c02008-05-22 11:46:33 -04003142 .open = ftrace_notrace_open,
Lai Jiangshan850a80c2009-03-13 17:47:23 +08003143 .read = seq_read,
Steven Rostedt41c52c02008-05-22 11:46:33 -04003144 .write = ftrace_notrace_write,
3145 .llseek = ftrace_regex_lseek,
Steven Rostedt1cf41dd2011-04-29 20:59:51 -04003146 .release = ftrace_regex_release,
Steven Rostedt41c52c02008-05-22 11:46:33 -04003147};
3148
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003149#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3150
3151static DEFINE_MUTEX(graph_lock);
3152
3153int ftrace_graph_count;
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003154int ftrace_graph_filter_enabled;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003155unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3156
3157static void *
Li Zefan85951842009-06-24 09:54:00 +08003158__g_next(struct seq_file *m, loff_t *pos)
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003159{
Li Zefan85951842009-06-24 09:54:00 +08003160 if (*pos >= ftrace_graph_count)
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003161 return NULL;
Li Zefana4ec5e02009-09-18 14:06:28 +08003162 return &ftrace_graph_funcs[*pos];
Li Zefan85951842009-06-24 09:54:00 +08003163}
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003164
Li Zefan85951842009-06-24 09:54:00 +08003165static void *
3166g_next(struct seq_file *m, void *v, loff_t *pos)
3167{
3168 (*pos)++;
3169 return __g_next(m, pos);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003170}
3171
3172static void *g_start(struct seq_file *m, loff_t *pos)
3173{
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003174 mutex_lock(&graph_lock);
3175
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003176 /* Nothing, tell g_show to print all functions are enabled */
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003177 if (!ftrace_graph_filter_enabled && !*pos)
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003178 return (void *)1;
3179
Li Zefan85951842009-06-24 09:54:00 +08003180 return __g_next(m, pos);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003181}
3182
3183static void g_stop(struct seq_file *m, void *p)
3184{
3185 mutex_unlock(&graph_lock);
3186}
3187
3188static int g_show(struct seq_file *m, void *v)
3189{
3190 unsigned long *ptr = v;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003191
3192 if (!ptr)
3193 return 0;
3194
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003195 if (ptr == (unsigned long *)1) {
3196 seq_printf(m, "#### all functions enabled ####\n");
3197 return 0;
3198 }
3199
Steven Rostedtb375a112009-09-17 00:05:58 -04003200 seq_printf(m, "%ps\n", (void *)*ptr);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003201
3202 return 0;
3203}
3204
James Morris88e9d342009-09-22 16:43:43 -07003205static const struct seq_operations ftrace_graph_seq_ops = {
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003206 .start = g_start,
3207 .next = g_next,
3208 .stop = g_stop,
3209 .show = g_show,
3210};
3211
3212static int
3213ftrace_graph_open(struct inode *inode, struct file *file)
3214{
3215 int ret = 0;
3216
3217 if (unlikely(ftrace_disabled))
3218 return -ENODEV;
3219
3220 mutex_lock(&graph_lock);
3221 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04003222 (file->f_flags & O_TRUNC)) {
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003223 ftrace_graph_filter_enabled = 0;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003224 ftrace_graph_count = 0;
3225 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
3226 }
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003227 mutex_unlock(&graph_lock);
3228
Li Zefana4ec5e02009-09-18 14:06:28 +08003229 if (file->f_mode & FMODE_READ)
3230 ret = seq_open(file, &ftrace_graph_seq_ops);
3231
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003232 return ret;
3233}
3234
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003235static int
Li Zefan87827112009-07-23 11:29:11 +08003236ftrace_graph_release(struct inode *inode, struct file *file)
3237{
3238 if (file->f_mode & FMODE_READ)
3239 seq_release(inode, file);
3240 return 0;
3241}
3242
3243static int
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003244ftrace_set_func(unsigned long *array, int *idx, char *buffer)
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003245{
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003246 struct dyn_ftrace *rec;
3247 struct ftrace_page *pg;
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003248 int search_len;
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003249 int fail = 1;
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003250 int type, not;
3251 char *search;
3252 bool exists;
3253 int i;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003254
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003255 /* decode regex */
Frederic Weisbecker3f6fe062009-09-24 21:31:51 +02003256 type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003257 if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
3258 return -EBUSY;
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003259
3260 search_len = strlen(search);
3261
Steven Rostedt52baf112009-02-14 01:15:39 -05003262 mutex_lock(&ftrace_lock);
Steven Rostedt45a4a232011-04-21 23:16:46 -04003263
3264 if (unlikely(ftrace_disabled)) {
3265 mutex_unlock(&ftrace_lock);
3266 return -ENODEV;
3267 }
3268
Steven Rostedt265c8312009-02-13 12:43:56 -05003269 do_for_each_ftrace_rec(pg, rec) {
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003270
Steven Rostedt45a4a232011-04-21 23:16:46 -04003271 if (rec->flags & FTRACE_FL_FREE)
Steven Rostedt265c8312009-02-13 12:43:56 -05003272 continue;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003273
Steven Rostedtb9df92d2011-04-28 20:32:08 -04003274 if (ftrace_match_record(rec, NULL, search, search_len, type)) {
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003275 /* if it is in the array */
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003276 exists = false;
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003277 for (i = 0; i < *idx; i++) {
Frederic Weisbeckerf9349a82009-02-19 21:13:12 +01003278 if (array[i] == rec->ip) {
3279 exists = true;
Steven Rostedt265c8312009-02-13 12:43:56 -05003280 break;
3281 }
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003282 }
3283
3284 if (!not) {
3285 fail = 0;
3286 if (!exists) {
3287 array[(*idx)++] = rec->ip;
3288 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
3289 goto out;
3290 }
3291 } else {
3292 if (exists) {
3293 array[i] = array[--(*idx)];
3294 array[*idx] = 0;
3295 fail = 0;
3296 }
3297 }
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003298 }
Steven Rostedt265c8312009-02-13 12:43:56 -05003299 } while_for_each_ftrace_rec();
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003300out:
Steven Rostedt52baf112009-02-14 01:15:39 -05003301 mutex_unlock(&ftrace_lock);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003302
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003303 if (fail)
3304 return -EINVAL;
3305
3306 ftrace_graph_filter_enabled = 1;
3307 return 0;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003308}
3309
3310static ssize_t
3311ftrace_graph_write(struct file *file, const char __user *ubuf,
3312 size_t cnt, loff_t *ppos)
3313{
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003314 struct trace_parser parser;
Li Zefan4ba79782009-09-22 13:52:20 +08003315 ssize_t read, ret;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003316
Li Zefanc7c6b1f2010-02-10 15:43:04 +08003317 if (!cnt)
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003318 return 0;
3319
3320 mutex_lock(&graph_lock);
3321
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003322 if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
3323 ret = -ENOMEM;
Li Zefan1eb90f12009-09-22 13:52:57 +08003324 goto out_unlock;
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003325 }
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003326
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003327 read = trace_get_user(&parser, ubuf, cnt, ppos);
3328
Li Zefan4ba79782009-09-22 13:52:20 +08003329 if (read >= 0 && trace_parser_loaded((&parser))) {
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003330 parser.buffer[parser.idx] = 0;
3331
3332 /* we allow only one expression at a time */
Li Zefana4ec5e02009-09-18 14:06:28 +08003333 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003334 parser.buffer);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003335 if (ret)
Li Zefan1eb90f12009-09-22 13:52:57 +08003336 goto out_free;
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003337 }
3338
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003339 ret = read;
Li Zefan1eb90f12009-09-22 13:52:57 +08003340
3341out_free:
jolsa@redhat.com689fd8b2009-09-11 17:29:29 +02003342 trace_parser_put(&parser);
Li Zefan1eb90f12009-09-22 13:52:57 +08003343out_unlock:
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003344 mutex_unlock(&graph_lock);
3345
3346 return ret;
3347}
3348
3349static const struct file_operations ftrace_graph_fops = {
Li Zefan87827112009-07-23 11:29:11 +08003350 .open = ftrace_graph_open,
3351 .read = seq_read,
3352 .write = ftrace_graph_write,
3353 .release = ftrace_graph_release,
Arnd Bergmann6038f372010-08-15 18:52:59 +02003354 .llseek = seq_lseek,
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003355};
3356#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3357
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003358static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
Steven Rostedt5072c592008-05-12 21:20:43 +02003359{
Steven Rostedt5072c592008-05-12 21:20:43 +02003360
Frederic Weisbecker5452af62009-03-27 00:25:38 +01003361 trace_create_file("available_filter_functions", 0444,
3362 d_tracer, NULL, &ftrace_avail_fops);
Steven Rostedt5072c592008-05-12 21:20:43 +02003363
Steven Rostedt647bcd02011-05-03 14:39:21 -04003364 trace_create_file("enabled_functions", 0444,
3365 d_tracer, NULL, &ftrace_enabled_fops);
3366
Frederic Weisbecker5452af62009-03-27 00:25:38 +01003367 trace_create_file("set_ftrace_filter", 0644, d_tracer,
3368 NULL, &ftrace_filter_fops);
Steven Rostedt41c52c02008-05-22 11:46:33 -04003369
Frederic Weisbecker5452af62009-03-27 00:25:38 +01003370 trace_create_file("set_ftrace_notrace", 0644, d_tracer,
Steven Rostedt41c52c02008-05-22 11:46:33 -04003371 NULL, &ftrace_notrace_fops);
Steven Rostedtad90c0e2008-05-27 20:48:37 -04003372
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003373#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Frederic Weisbecker5452af62009-03-27 00:25:38 +01003374 trace_create_file("set_graph_function", 0444, d_tracer,
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003375 NULL,
3376 &ftrace_graph_fops);
Steven Rostedtea4e2bc2008-12-03 15:36:57 -05003377#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3378
Steven Rostedt5072c592008-05-12 21:20:43 +02003379 return 0;
3380}
3381
Jiri Olsa5cb084b2009-10-13 16:33:53 -04003382static int ftrace_process_locs(struct module *mod,
Steven Rostedt31e88902008-11-14 16:21:19 -08003383 unsigned long *start,
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003384 unsigned long *end)
3385{
3386 unsigned long *p;
3387 unsigned long addr;
Steven Rostedt4376cac2011-06-24 23:28:13 -04003388 unsigned long flags = 0; /* Shut up gcc */
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003389
Steven Rostedte6ea44e2009-02-14 01:42:44 -05003390 mutex_lock(&ftrace_lock);
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003391 p = start;
3392 while (p < end) {
3393 addr = ftrace_call_adjust(*p++);
Steven Rostedt20e52272008-11-14 16:21:19 -08003394 /*
3395 * Some architecture linkers will pad between
3396 * the different mcount_loc sections of different
3397 * object files to satisfy alignments.
3398 * Skip any NULL pointers.
3399 */
3400 if (!addr)
3401 continue;
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003402 ftrace_record_ip(addr);
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003403 }
3404
Steven Rostedta4f18ed2011-06-07 09:26:46 -04003405 /*
Steven Rostedt4376cac2011-06-24 23:28:13 -04003406 * We only need to disable interrupts on start up
3407 * because we are modifying code that an interrupt
3408 * may execute, and the modification is not atomic.
3409 * But for modules, nothing runs the code we modify
3410 * until we are finished with it, and there's no
3411 * reason to cause large interrupt latencies while we do it.
Steven Rostedta4f18ed2011-06-07 09:26:46 -04003412 */
Steven Rostedt4376cac2011-06-24 23:28:13 -04003413 if (!mod)
3414 local_irq_save(flags);
Steven Rostedt31e88902008-11-14 16:21:19 -08003415 ftrace_update_code(mod);
Steven Rostedt4376cac2011-06-24 23:28:13 -04003416 if (!mod)
3417 local_irq_restore(flags);
Steven Rostedte6ea44e2009-02-14 01:42:44 -05003418 mutex_unlock(&ftrace_lock);
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003419
3420 return 0;
3421}
3422
Steven Rostedt93eb6772009-04-15 13:24:06 -04003423#ifdef CONFIG_MODULES
jolsa@redhat.come7247a12009-10-07 19:00:35 +02003424void ftrace_release_mod(struct module *mod)
Steven Rostedt93eb6772009-04-15 13:24:06 -04003425{
3426 struct dyn_ftrace *rec;
3427 struct ftrace_page *pg;
Steven Rostedt93eb6772009-04-15 13:24:06 -04003428
Steven Rostedt93eb6772009-04-15 13:24:06 -04003429 mutex_lock(&ftrace_lock);
Steven Rostedt45a4a232011-04-21 23:16:46 -04003430
3431 if (ftrace_disabled)
3432 goto out_unlock;
3433
Steven Rostedt93eb6772009-04-15 13:24:06 -04003434 do_for_each_ftrace_rec(pg, rec) {
jolsa@redhat.come7247a12009-10-07 19:00:35 +02003435 if (within_module_core(rec->ip, mod)) {
Steven Rostedt93eb6772009-04-15 13:24:06 -04003436 /*
3437 * rec->ip is changed in ftrace_free_rec()
3438 * It should not between s and e if record was freed.
3439 */
3440 FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
3441 ftrace_free_rec(rec);
3442 }
3443 } while_for_each_ftrace_rec();
Steven Rostedt45a4a232011-04-21 23:16:46 -04003444 out_unlock:
Steven Rostedt93eb6772009-04-15 13:24:06 -04003445 mutex_unlock(&ftrace_lock);
3446}
3447
3448static void ftrace_init_module(struct module *mod,
3449 unsigned long *start, unsigned long *end)
Steven Rostedt90d595f2008-08-14 15:45:09 -04003450{
Steven Rostedt00fd61a2008-08-15 21:40:04 -04003451 if (ftrace_disabled || start == end)
Steven Rostedtfed19392008-08-14 22:47:19 -04003452 return;
Jiri Olsa5cb084b2009-10-13 16:33:53 -04003453 ftrace_process_locs(mod, start, end);
Steven Rostedt90d595f2008-08-14 15:45:09 -04003454}
3455
Steven Rostedt93eb6772009-04-15 13:24:06 -04003456static int ftrace_module_notify(struct notifier_block *self,
3457 unsigned long val, void *data)
3458{
3459 struct module *mod = data;
3460
3461 switch (val) {
3462 case MODULE_STATE_COMING:
3463 ftrace_init_module(mod, mod->ftrace_callsites,
3464 mod->ftrace_callsites +
3465 mod->num_ftrace_callsites);
3466 break;
3467 case MODULE_STATE_GOING:
jolsa@redhat.come7247a12009-10-07 19:00:35 +02003468 ftrace_release_mod(mod);
Steven Rostedt93eb6772009-04-15 13:24:06 -04003469 break;
3470 }
3471
3472 return 0;
3473}
3474#else
3475static int ftrace_module_notify(struct notifier_block *self,
3476 unsigned long val, void *data)
3477{
3478 return 0;
3479}
3480#endif /* CONFIG_MODULES */
3481
3482struct notifier_block ftrace_module_nb = {
3483 .notifier_call = ftrace_module_notify,
3484 .priority = 0,
3485};
3486
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003487extern unsigned long __start_mcount_loc[];
3488extern unsigned long __stop_mcount_loc[];
3489
3490void __init ftrace_init(void)
3491{
3492 unsigned long count, addr, flags;
3493 int ret;
3494
3495 /* Keep the ftrace pointer to the stub */
3496 addr = (unsigned long)ftrace_stub;
3497
3498 local_irq_save(flags);
3499 ftrace_dyn_arch_init(&addr);
3500 local_irq_restore(flags);
3501
3502 /* ftrace_dyn_arch_init places the return code in addr */
3503 if (addr)
3504 goto failed;
3505
3506 count = __stop_mcount_loc - __start_mcount_loc;
3507
3508 ret = ftrace_dyn_table_alloc(count);
3509 if (ret)
3510 goto failed;
3511
3512 last_ftrace_enabled = ftrace_enabled = 1;
3513
Jiri Olsa5cb084b2009-10-13 16:33:53 -04003514 ret = ftrace_process_locs(NULL,
Steven Rostedt31e88902008-11-14 16:21:19 -08003515 __start_mcount_loc,
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003516 __stop_mcount_loc);
3517
Steven Rostedt93eb6772009-04-15 13:24:06 -04003518 ret = register_module_notifier(&ftrace_module_nb);
Ming Lei24ed0c42009-05-17 15:31:38 +08003519 if (ret)
Steven Rostedt93eb6772009-04-15 13:24:06 -04003520 pr_warning("Failed to register trace ftrace module notifier\n");
3521
Steven Rostedt2af15d62009-05-28 13:37:24 -04003522 set_ftrace_early_filters();
3523
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003524 return;
3525 failed:
3526 ftrace_disabled = 1;
3527}
Steven Rostedt68bf21a2008-08-14 15:45:08 -04003528
Steven Rostedt3d083392008-05-12 21:20:42 +02003529#else
Frederic Weisbecker0b6e4d52008-10-28 20:17:38 +01003530
Steven Rostedt2b499382011-05-03 22:49:52 -04003531static struct ftrace_ops global_ops = {
Steven Rostedtbd69c302011-05-03 21:55:54 -04003532 .func = ftrace_stub,
3533};
3534
Frederic Weisbecker0b6e4d52008-10-28 20:17:38 +01003535static int __init ftrace_nodyn_init(void)
3536{
3537 ftrace_enabled = 1;
3538 return 0;
3539}
3540device_initcall(ftrace_nodyn_init);
3541
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003542static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
3543static inline void ftrace_startup_enable(int command) { }
Steven Rostedt5a45cfe2008-11-26 00:16:24 -05003544/* Keep as macros so we do not need to define the commands */
Steven Rostedt3b6cfdb2011-05-23 15:33:49 -04003545# define ftrace_startup(ops, command) \
3546 ({ \
3547 (ops)->flags |= FTRACE_OPS_FL_ENABLED; \
3548 0; \
3549 })
Steven Rostedtbd69c302011-05-03 21:55:54 -04003550# define ftrace_shutdown(ops, command) do { } while (0)
Ingo Molnarc7aafc52008-05-12 21:20:45 +02003551# define ftrace_startup_sysctl() do { } while (0)
3552# define ftrace_shutdown_sysctl() do { } while (0)
Steven Rostedtb8489142011-05-04 09:27:52 -04003553
3554static inline int
3555ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
3556{
3557 return 1;
3558}
3559
Steven Rostedt3d083392008-05-12 21:20:42 +02003560#endif /* CONFIG_DYNAMIC_FTRACE */
3561
Steven Rostedtb8489142011-05-04 09:27:52 -04003562static void
3563ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
3564{
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04003565 struct ftrace_ops *op;
Steven Rostedtb8489142011-05-04 09:27:52 -04003566
Steven Rostedtb1cff0a2011-05-25 14:27:43 -04003567 if (unlikely(trace_recursion_test(TRACE_INTERNAL_BIT)))
3568 return;
3569
3570 trace_recursion_set(TRACE_INTERNAL_BIT);
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04003571 /*
3572 * Some of the ops may be dynamically allocated,
3573 * they must be freed after a synchronize_sched().
3574 */
3575 preempt_disable_notrace();
3576 op = rcu_dereference_raw(ftrace_ops_list);
Steven Rostedtb8489142011-05-04 09:27:52 -04003577 while (op != &ftrace_list_end) {
3578 if (ftrace_ops_test(op, ip))
3579 op->func(ip, parent_ip);
3580 op = rcu_dereference_raw(op->next);
3581 };
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04003582 preempt_enable_notrace();
Steven Rostedtb1cff0a2011-05-25 14:27:43 -04003583 trace_recursion_clear(TRACE_INTERNAL_BIT);
Steven Rostedtb8489142011-05-04 09:27:52 -04003584}
3585
Steven Rostedte32d8952008-12-04 00:26:41 -05003586static void clear_ftrace_swapper(void)
3587{
3588 struct task_struct *p;
3589 int cpu;
3590
3591 get_online_cpus();
3592 for_each_online_cpu(cpu) {
3593 p = idle_task(cpu);
3594 clear_tsk_trace_trace(p);
3595 }
3596 put_online_cpus();
3597}
3598
3599static void set_ftrace_swapper(void)
3600{
3601 struct task_struct *p;
3602 int cpu;
3603
3604 get_online_cpus();
3605 for_each_online_cpu(cpu) {
3606 p = idle_task(cpu);
3607 set_tsk_trace_trace(p);
3608 }
3609 put_online_cpus();
3610}
3611
3612static void clear_ftrace_pid(struct pid *pid)
Steven Rostedt978f3a42008-12-04 00:26:40 -05003613{
3614 struct task_struct *p;
3615
Oleg Nesterov229c4ef2009-02-03 20:39:04 +01003616 rcu_read_lock();
Steven Rostedte32d8952008-12-04 00:26:41 -05003617 do_each_pid_task(pid, PIDTYPE_PID, p) {
Steven Rostedt978f3a42008-12-04 00:26:40 -05003618 clear_tsk_trace_trace(p);
Steven Rostedte32d8952008-12-04 00:26:41 -05003619 } while_each_pid_task(pid, PIDTYPE_PID, p);
Oleg Nesterov229c4ef2009-02-03 20:39:04 +01003620 rcu_read_unlock();
3621
Steven Rostedte32d8952008-12-04 00:26:41 -05003622 put_pid(pid);
Steven Rostedt978f3a42008-12-04 00:26:40 -05003623}
3624
Steven Rostedte32d8952008-12-04 00:26:41 -05003625static void set_ftrace_pid(struct pid *pid)
Steven Rostedt978f3a42008-12-04 00:26:40 -05003626{
3627 struct task_struct *p;
3628
Oleg Nesterov229c4ef2009-02-03 20:39:04 +01003629 rcu_read_lock();
Steven Rostedt978f3a42008-12-04 00:26:40 -05003630 do_each_pid_task(pid, PIDTYPE_PID, p) {
3631 set_tsk_trace_trace(p);
3632 } while_each_pid_task(pid, PIDTYPE_PID, p);
Oleg Nesterov229c4ef2009-02-03 20:39:04 +01003633 rcu_read_unlock();
Steven Rostedt978f3a42008-12-04 00:26:40 -05003634}
3635
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003636static void clear_ftrace_pid_task(struct pid *pid)
Steven Rostedte32d8952008-12-04 00:26:41 -05003637{
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003638 if (pid == ftrace_swapper_pid)
Steven Rostedte32d8952008-12-04 00:26:41 -05003639 clear_ftrace_swapper();
3640 else
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003641 clear_ftrace_pid(pid);
Steven Rostedte32d8952008-12-04 00:26:41 -05003642}
3643
3644static void set_ftrace_pid_task(struct pid *pid)
3645{
3646 if (pid == ftrace_swapper_pid)
3647 set_ftrace_swapper();
3648 else
3649 set_ftrace_pid(pid);
3650}
3651
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003652static int ftrace_pid_add(int p)
3653{
3654 struct pid *pid;
3655 struct ftrace_pid *fpid;
3656 int ret = -EINVAL;
3657
3658 mutex_lock(&ftrace_lock);
3659
3660 if (!p)
3661 pid = ftrace_swapper_pid;
3662 else
3663 pid = find_get_pid(p);
3664
3665 if (!pid)
3666 goto out;
3667
3668 ret = 0;
3669
3670 list_for_each_entry(fpid, &ftrace_pids, list)
3671 if (fpid->pid == pid)
3672 goto out_put;
3673
3674 ret = -ENOMEM;
3675
3676 fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
3677 if (!fpid)
3678 goto out_put;
3679
3680 list_add(&fpid->list, &ftrace_pids);
3681 fpid->pid = pid;
3682
3683 set_ftrace_pid_task(pid);
3684
3685 ftrace_update_pid_func();
3686 ftrace_startup_enable(0);
3687
3688 mutex_unlock(&ftrace_lock);
3689 return 0;
3690
3691out_put:
3692 if (pid != ftrace_swapper_pid)
3693 put_pid(pid);
3694
3695out:
3696 mutex_unlock(&ftrace_lock);
3697 return ret;
3698}
3699
3700static void ftrace_pid_reset(void)
3701{
3702 struct ftrace_pid *fpid, *safe;
3703
3704 mutex_lock(&ftrace_lock);
3705 list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
3706 struct pid *pid = fpid->pid;
3707
3708 clear_ftrace_pid_task(pid);
3709
3710 list_del(&fpid->list);
3711 kfree(fpid);
3712 }
3713
3714 ftrace_update_pid_func();
3715 ftrace_startup_enable(0);
3716
3717 mutex_unlock(&ftrace_lock);
3718}
3719
3720static void *fpid_start(struct seq_file *m, loff_t *pos)
3721{
3722 mutex_lock(&ftrace_lock);
3723
3724 if (list_empty(&ftrace_pids) && (!*pos))
3725 return (void *) 1;
3726
3727 return seq_list_start(&ftrace_pids, *pos);
3728}
3729
3730static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
3731{
3732 if (v == (void *)1)
3733 return NULL;
3734
3735 return seq_list_next(v, &ftrace_pids, pos);
3736}
3737
3738static void fpid_stop(struct seq_file *m, void *p)
3739{
3740 mutex_unlock(&ftrace_lock);
3741}
3742
3743static int fpid_show(struct seq_file *m, void *v)
3744{
3745 const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
3746
3747 if (v == (void *)1) {
3748 seq_printf(m, "no pid\n");
3749 return 0;
3750 }
3751
3752 if (fpid->pid == ftrace_swapper_pid)
3753 seq_printf(m, "swapper tasks\n");
3754 else
3755 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
3756
3757 return 0;
3758}
3759
3760static const struct seq_operations ftrace_pid_sops = {
3761 .start = fpid_start,
3762 .next = fpid_next,
3763 .stop = fpid_stop,
3764 .show = fpid_show,
3765};
3766
3767static int
3768ftrace_pid_open(struct inode *inode, struct file *file)
3769{
3770 int ret = 0;
3771
3772 if ((file->f_mode & FMODE_WRITE) &&
3773 (file->f_flags & O_TRUNC))
3774 ftrace_pid_reset();
3775
3776 if (file->f_mode & FMODE_READ)
3777 ret = seq_open(file, &ftrace_pid_sops);
3778
3779 return ret;
3780}
3781
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003782static ssize_t
3783ftrace_pid_write(struct file *filp, const char __user *ubuf,
3784 size_t cnt, loff_t *ppos)
3785{
Ingo Molnar457dc922009-11-23 11:03:28 +01003786 char buf[64], *tmp;
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003787 long val;
3788 int ret;
3789
3790 if (cnt >= sizeof(buf))
3791 return -EINVAL;
3792
3793 if (copy_from_user(&buf, ubuf, cnt))
3794 return -EFAULT;
3795
3796 buf[cnt] = 0;
3797
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003798 /*
3799 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
3800 * to clean the filter quietly.
3801 */
Ingo Molnar457dc922009-11-23 11:03:28 +01003802 tmp = strstrip(buf);
3803 if (strlen(tmp) == 0)
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003804 return 1;
3805
Ingo Molnar457dc922009-11-23 11:03:28 +01003806 ret = strict_strtol(tmp, 10, &val);
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003807 if (ret < 0)
3808 return ret;
3809
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003810 ret = ftrace_pid_add(val);
Steven Rostedt978f3a42008-12-04 00:26:40 -05003811
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003812 return ret ? ret : cnt;
3813}
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003814
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003815static int
3816ftrace_pid_release(struct inode *inode, struct file *file)
3817{
3818 if (file->f_mode & FMODE_READ)
3819 seq_release(inode, file);
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003820
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003821 return 0;
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003822}
3823
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003824static const struct file_operations ftrace_pid_fops = {
jolsa@redhat.com756d17e2009-10-13 16:33:52 -04003825 .open = ftrace_pid_open,
3826 .write = ftrace_pid_write,
3827 .read = seq_read,
3828 .llseek = seq_lseek,
3829 .release = ftrace_pid_release,
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003830};
3831
3832static __init int ftrace_init_debugfs(void)
3833{
3834 struct dentry *d_tracer;
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003835
3836 d_tracer = tracing_init_dentry();
3837 if (!d_tracer)
3838 return 0;
3839
3840 ftrace_init_dyn_debugfs(d_tracer);
3841
Frederic Weisbecker5452af62009-03-27 00:25:38 +01003842 trace_create_file("set_ftrace_pid", 0644, d_tracer,
3843 NULL, &ftrace_pid_fops);
Steven Rostedt493762f2009-03-23 17:12:36 -04003844
3845 ftrace_profile_debugfs(d_tracer);
3846
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003847 return 0;
3848}
Steven Rostedtdf4fc312008-11-26 00:16:23 -05003849fs_initcall(ftrace_init_debugfs);
3850
Steven Rostedt3d083392008-05-12 21:20:42 +02003851/**
Steven Rostedt81adbdc2008-10-23 09:33:02 -04003852 * ftrace_kill - kill ftrace
Steven Rostedta2bb6a32008-07-10 20:58:15 -04003853 *
3854 * This function should be used by panic code. It stops ftrace
3855 * but in a not so nice way. If you need to simply kill ftrace
3856 * from a non-atomic section, use ftrace_kill.
3857 */
Steven Rostedt81adbdc2008-10-23 09:33:02 -04003858void ftrace_kill(void)
Steven Rostedta2bb6a32008-07-10 20:58:15 -04003859{
3860 ftrace_disabled = 1;
3861 ftrace_enabled = 0;
Steven Rostedta2bb6a32008-07-10 20:58:15 -04003862 clear_ftrace_function();
3863}
3864
3865/**
Steven Rostedt3d083392008-05-12 21:20:42 +02003866 * register_ftrace_function - register a function for profiling
3867 * @ops - ops structure that holds the function for profiling.
3868 *
3869 * Register a function to be called by all functions in the
3870 * kernel.
3871 *
3872 * Note: @ops->func and all the functions it calls must be labeled
3873 * with "notrace", otherwise it will go into a
3874 * recursive loop.
3875 */
3876int register_ftrace_function(struct ftrace_ops *ops)
3877{
Steven Rostedt45a4a232011-04-21 23:16:46 -04003878 int ret = -1;
Steven Rostedt4eebcc82008-05-12 21:20:48 +02003879
Steven Rostedte6ea44e2009-02-14 01:42:44 -05003880 mutex_lock(&ftrace_lock);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01003881
Steven Rostedt45a4a232011-04-21 23:16:46 -04003882 if (unlikely(ftrace_disabled))
3883 goto out_unlock;
3884
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003885 ret = __register_ftrace_function(ops);
Steven Rostedtb8489142011-05-04 09:27:52 -04003886 if (!ret)
Steven Rostedta1cd6172011-05-23 15:24:25 -04003887 ret = ftrace_startup(ops, 0);
Steven Rostedtb8489142011-05-04 09:27:52 -04003888
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003889
Steven Rostedt45a4a232011-04-21 23:16:46 -04003890 out_unlock:
Steven Rostedte6ea44e2009-02-14 01:42:44 -05003891 mutex_unlock(&ftrace_lock);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003892 return ret;
Steven Rostedt3d083392008-05-12 21:20:42 +02003893}
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04003894EXPORT_SYMBOL_GPL(register_ftrace_function);
Steven Rostedt3d083392008-05-12 21:20:42 +02003895
3896/**
Uwe Kleine-Koenig32632922009-01-12 23:35:50 +01003897 * unregister_ftrace_function - unregister a function for profiling.
Steven Rostedt3d083392008-05-12 21:20:42 +02003898 * @ops - ops structure that holds the function to unregister
3899 *
3900 * Unregister a function that was added to be called by ftrace profiling.
3901 */
3902int unregister_ftrace_function(struct ftrace_ops *ops)
3903{
3904 int ret;
3905
Steven Rostedte6ea44e2009-02-14 01:42:44 -05003906 mutex_lock(&ftrace_lock);
Steven Rostedt3d083392008-05-12 21:20:42 +02003907 ret = __unregister_ftrace_function(ops);
Steven Rostedtb8489142011-05-04 09:27:52 -04003908 if (!ret)
3909 ftrace_shutdown(ops, 0);
Steven Rostedte6ea44e2009-02-14 01:42:44 -05003910 mutex_unlock(&ftrace_lock);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003911
3912 return ret;
3913}
Steven Rostedtcdbe61b2011-05-05 21:14:55 -04003914EXPORT_SYMBOL_GPL(unregister_ftrace_function);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003915
Ingo Molnare309b412008-05-12 21:20:51 +02003916int
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003917ftrace_enable_sysctl(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07003918 void __user *buffer, size_t *lenp,
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003919 loff_t *ppos)
3920{
Steven Rostedt45a4a232011-04-21 23:16:46 -04003921 int ret = -ENODEV;
Steven Rostedt4eebcc82008-05-12 21:20:48 +02003922
Steven Rostedte6ea44e2009-02-14 01:42:44 -05003923 mutex_lock(&ftrace_lock);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003924
Steven Rostedt45a4a232011-04-21 23:16:46 -04003925 if (unlikely(ftrace_disabled))
3926 goto out;
3927
3928 ret = proc_dointvec(table, write, buffer, lenp, ppos);
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003929
Li Zefana32c7762009-06-26 16:55:51 +08003930 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003931 goto out;
3932
Li Zefana32c7762009-06-26 16:55:51 +08003933 last_ftrace_enabled = !!ftrace_enabled;
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003934
3935 if (ftrace_enabled) {
3936
3937 ftrace_startup_sysctl();
3938
3939 /* we are starting ftrace again */
Steven Rostedtb8489142011-05-04 09:27:52 -04003940 if (ftrace_ops_list != &ftrace_list_end) {
3941 if (ftrace_ops_list->next == &ftrace_list_end)
3942 ftrace_trace_function = ftrace_ops_list->func;
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003943 else
Steven Rostedtb8489142011-05-04 09:27:52 -04003944 ftrace_trace_function = ftrace_ops_list_func;
Steven Rostedtb0fc4942008-05-12 21:20:43 +02003945 }
3946
3947 } else {
3948 /* stopping ftrace calls (just send to ftrace_stub) */
3949 ftrace_trace_function = ftrace_stub;
3950
3951 ftrace_shutdown_sysctl();
3952 }
3953
3954 out:
Steven Rostedte6ea44e2009-02-14 01:42:44 -05003955 mutex_unlock(&ftrace_lock);
Steven Rostedt3d083392008-05-12 21:20:42 +02003956 return ret;
Arnaldo Carvalho de Melo16444a82008-05-12 21:20:42 +02003957}
Ingo Molnarf17845e2008-10-24 12:47:10 +02003958
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01003959#ifdef CONFIG_FUNCTION_GRAPH_TRACER
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01003960
Steven Rostedt597af812009-04-03 15:24:12 -04003961static int ftrace_graph_active;
Frederic Weisbecker4a2b8dd2009-01-14 13:33:27 -08003962static struct notifier_block ftrace_suspend_notifier;
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01003963
Steven Rostedte49dc192008-12-02 23:50:05 -05003964int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3965{
3966 return 0;
3967}
3968
Frederic Weisbecker287b6e62008-11-26 00:57:25 +01003969/* The callbacks that hook a function */
3970trace_func_graph_ret_t ftrace_graph_return =
3971 (trace_func_graph_ret_t)ftrace_stub;
Steven Rostedte49dc192008-12-02 23:50:05 -05003972trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01003973
3974/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3975static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3976{
3977 int i;
3978 int ret = 0;
3979 unsigned long flags;
3980 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3981 struct task_struct *g, *t;
3982
3983 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3984 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3985 * sizeof(struct ftrace_ret_stack),
3986 GFP_KERNEL);
3987 if (!ret_stack_list[i]) {
3988 start = 0;
3989 end = i;
3990 ret = -ENOMEM;
3991 goto free;
3992 }
3993 }
3994
3995 read_lock_irqsave(&tasklist_lock, flags);
3996 do_each_thread(g, t) {
3997 if (start == end) {
3998 ret = -EAGAIN;
3999 goto unlock;
4000 }
4001
4002 if (t->ret_stack == NULL) {
Frederic Weisbecker380c4b12008-12-06 03:43:41 +01004003 atomic_set(&t->tracing_graph_pause, 0);
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004004 atomic_set(&t->trace_overrun, 0);
Steven Rostedt26c01622009-06-02 14:01:19 -04004005 t->curr_ret_stack = -1;
4006 /* Make sure the tasks see the -1 first: */
4007 smp_wmb();
4008 t->ret_stack = ret_stack_list[start++];
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004009 }
4010 } while_each_thread(g, t);
4011
4012unlock:
4013 read_unlock_irqrestore(&tasklist_lock, flags);
4014free:
4015 for (i = start; i < end; i++)
4016 kfree(ret_stack_list[i]);
4017 return ret;
4018}
4019
Steven Rostedt8aef2d22009-03-24 01:10:15 -04004020static void
Steven Rostedt38516ab2010-04-20 17:04:50 -04004021ftrace_graph_probe_sched_switch(void *ignore,
4022 struct task_struct *prev, struct task_struct *next)
Steven Rostedt8aef2d22009-03-24 01:10:15 -04004023{
4024 unsigned long long timestamp;
4025 int index;
4026
Steven Rostedtbe6f1642009-03-24 11:06:24 -04004027 /*
4028 * Does the user want to count the time a function was asleep.
4029 * If so, do not update the time stamps.
4030 */
4031 if (trace_flags & TRACE_ITER_SLEEP_TIME)
4032 return;
4033
Steven Rostedt8aef2d22009-03-24 01:10:15 -04004034 timestamp = trace_clock_local();
4035
4036 prev->ftrace_timestamp = timestamp;
4037
4038 /* only process tasks that we timestamped */
4039 if (!next->ftrace_timestamp)
4040 return;
4041
4042 /*
4043 * Update all the counters in next to make up for the
4044 * time next was sleeping.
4045 */
4046 timestamp -= next->ftrace_timestamp;
4047
4048 for (index = next->curr_ret_stack; index >= 0; index--)
4049 next->ret_stack[index].calltime += timestamp;
4050}
4051
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004052/* Allocate a return stack for each task */
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01004053static int start_graph_tracing(void)
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004054{
4055 struct ftrace_ret_stack **ret_stack_list;
Frederic Weisbecker5b058bc2009-02-17 18:35:34 +01004056 int ret, cpu;
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004057
4058 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
4059 sizeof(struct ftrace_ret_stack *),
4060 GFP_KERNEL);
4061
4062 if (!ret_stack_list)
4063 return -ENOMEM;
4064
Frederic Weisbecker5b058bc2009-02-17 18:35:34 +01004065 /* The cpu_boot init_task->ret_stack will never be freed */
Steven Rostedt179c4982009-06-02 12:03:19 -04004066 for_each_online_cpu(cpu) {
4067 if (!idle_task(cpu)->ret_stack)
Steven Rostedt868baf02011-02-10 21:26:13 -05004068 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
Steven Rostedt179c4982009-06-02 12:03:19 -04004069 }
Frederic Weisbecker5b058bc2009-02-17 18:35:34 +01004070
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004071 do {
4072 ret = alloc_retstack_tasklist(ret_stack_list);
4073 } while (ret == -EAGAIN);
4074
Steven Rostedt8aef2d22009-03-24 01:10:15 -04004075 if (!ret) {
Steven Rostedt38516ab2010-04-20 17:04:50 -04004076 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
Steven Rostedt8aef2d22009-03-24 01:10:15 -04004077 if (ret)
4078 pr_info("ftrace_graph: Couldn't activate tracepoint"
4079 " probe to kernel_sched_switch\n");
4080 }
4081
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004082 kfree(ret_stack_list);
4083 return ret;
4084}
4085
Frederic Weisbecker4a2b8dd2009-01-14 13:33:27 -08004086/*
4087 * Hibernation protection.
4088 * The state of the current task is too much unstable during
4089 * suspend/restore to disk. We want to protect against that.
4090 */
4091static int
4092ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
4093 void *unused)
4094{
4095 switch (state) {
4096 case PM_HIBERNATION_PREPARE:
4097 pause_graph_tracing();
4098 break;
4099
4100 case PM_POST_HIBERNATION:
4101 unpause_graph_tracing();
4102 break;
4103 }
4104 return NOTIFY_DONE;
4105}
4106
Frederic Weisbecker287b6e62008-11-26 00:57:25 +01004107int register_ftrace_graph(trace_func_graph_ret_t retfunc,
4108 trace_func_graph_ent_t entryfunc)
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01004109{
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01004110 int ret = 0;
4111
Steven Rostedte6ea44e2009-02-14 01:42:44 -05004112 mutex_lock(&ftrace_lock);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01004113
Steven Rostedt05ce5812009-03-24 00:18:31 -04004114 /* we currently allow only one tracer registered at a time */
Steven Rostedt597af812009-04-03 15:24:12 -04004115 if (ftrace_graph_active) {
Steven Rostedt05ce5812009-03-24 00:18:31 -04004116 ret = -EBUSY;
4117 goto out;
4118 }
4119
Frederic Weisbecker4a2b8dd2009-01-14 13:33:27 -08004120 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
4121 register_pm_notifier(&ftrace_suspend_notifier);
4122
Steven Rostedt597af812009-04-03 15:24:12 -04004123 ftrace_graph_active++;
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01004124 ret = start_graph_tracing();
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004125 if (ret) {
Steven Rostedt597af812009-04-03 15:24:12 -04004126 ftrace_graph_active--;
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004127 goto out;
4128 }
Steven Rostedte53a6312008-11-26 00:16:25 -05004129
Frederic Weisbecker287b6e62008-11-26 00:57:25 +01004130 ftrace_graph_return = retfunc;
4131 ftrace_graph_entry = entryfunc;
Steven Rostedte53a6312008-11-26 00:16:25 -05004132
Steven Rostedta1cd6172011-05-23 15:24:25 -04004133 ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01004134
4135out:
Steven Rostedte6ea44e2009-02-14 01:42:44 -05004136 mutex_unlock(&ftrace_lock);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01004137 return ret;
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01004138}
4139
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01004140void unregister_ftrace_graph(void)
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01004141{
Steven Rostedte6ea44e2009-02-14 01:42:44 -05004142 mutex_lock(&ftrace_lock);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01004143
Steven Rostedt597af812009-04-03 15:24:12 -04004144 if (unlikely(!ftrace_graph_active))
Steven Rostedt2aad1b72009-03-30 11:11:28 -04004145 goto out;
4146
Steven Rostedt597af812009-04-03 15:24:12 -04004147 ftrace_graph_active--;
Frederic Weisbecker287b6e62008-11-26 00:57:25 +01004148 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
Steven Rostedte49dc192008-12-02 23:50:05 -05004149 ftrace_graph_entry = ftrace_graph_entry_stub;
Steven Rostedtbd69c302011-05-03 21:55:54 -04004150 ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
Frederic Weisbecker4a2b8dd2009-01-14 13:33:27 -08004151 unregister_pm_notifier(&ftrace_suspend_notifier);
Steven Rostedt38516ab2010-04-20 17:04:50 -04004152 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
Frederic Weisbeckere7d37372008-11-16 06:02:06 +01004153
Steven Rostedt2aad1b72009-03-30 11:11:28 -04004154 out:
Steven Rostedte6ea44e2009-02-14 01:42:44 -05004155 mutex_unlock(&ftrace_lock);
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01004156}
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004157
Steven Rostedt868baf02011-02-10 21:26:13 -05004158static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
4159
4160static void
4161graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
4162{
4163 atomic_set(&t->tracing_graph_pause, 0);
4164 atomic_set(&t->trace_overrun, 0);
4165 t->ftrace_timestamp = 0;
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004166 /* make curr_ret_stack visible before we add the ret_stack */
Steven Rostedt868baf02011-02-10 21:26:13 -05004167 smp_wmb();
4168 t->ret_stack = ret_stack;
4169}
4170
4171/*
4172 * Allocate a return stack for the idle task. May be the first
4173 * time through, or it may be done by CPU hotplug online.
4174 */
4175void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
4176{
4177 t->curr_ret_stack = -1;
4178 /*
4179 * The idle task has no parent, it either has its own
4180 * stack or no stack at all.
4181 */
4182 if (t->ret_stack)
4183 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
4184
4185 if (ftrace_graph_active) {
4186 struct ftrace_ret_stack *ret_stack;
4187
4188 ret_stack = per_cpu(idle_ret_stack, cpu);
4189 if (!ret_stack) {
4190 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4191 * sizeof(struct ftrace_ret_stack),
4192 GFP_KERNEL);
4193 if (!ret_stack)
4194 return;
4195 per_cpu(idle_ret_stack, cpu) = ret_stack;
4196 }
4197 graph_init_task(t, ret_stack);
4198 }
4199}
4200
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004201/* Allocate a return stack for newly created task */
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01004202void ftrace_graph_init_task(struct task_struct *t)
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004203{
Steven Rostedt84047e32009-06-02 16:51:55 -04004204 /* Make sure we do not use the parent ret_stack */
4205 t->ret_stack = NULL;
Steven Rostedtea14eb72010-03-12 19:41:23 -05004206 t->curr_ret_stack = -1;
Steven Rostedt84047e32009-06-02 16:51:55 -04004207
Steven Rostedt597af812009-04-03 15:24:12 -04004208 if (ftrace_graph_active) {
Steven Rostedt82310a32009-06-02 12:26:07 -04004209 struct ftrace_ret_stack *ret_stack;
4210
4211 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004212 * sizeof(struct ftrace_ret_stack),
4213 GFP_KERNEL);
Steven Rostedt82310a32009-06-02 12:26:07 -04004214 if (!ret_stack)
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004215 return;
Steven Rostedt868baf02011-02-10 21:26:13 -05004216 graph_init_task(t, ret_stack);
Steven Rostedt84047e32009-06-02 16:51:55 -04004217 }
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004218}
4219
Frederic Weisbeckerfb526072008-11-25 21:07:04 +01004220void ftrace_graph_exit_task(struct task_struct *t)
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004221{
Frederic Weisbeckereae849c2008-11-23 17:33:12 +01004222 struct ftrace_ret_stack *ret_stack = t->ret_stack;
4223
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004224 t->ret_stack = NULL;
Frederic Weisbeckereae849c2008-11-23 17:33:12 +01004225 /* NULL must become visible to IRQs before we free it: */
4226 barrier();
4227
4228 kfree(ret_stack);
Frederic Weisbeckerf201ae22008-11-23 06:22:56 +01004229}
Steven Rostedt14a866c2008-12-02 23:50:02 -05004230
4231void ftrace_graph_stop(void)
4232{
4233 ftrace_stop();
4234}
Frederic Weisbecker15e6cb32008-11-11 07:14:25 +01004235#endif