blob: f2bd275bb60f5c63630ee4483c02a0f33d53e46d [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040020#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050021#include <linux/irqflags.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020022#include <linux/debugfs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020023#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020024#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050027#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020028#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050031#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040032#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010033#include <linux/string.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080034#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090035#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020036#include <linux/ctype.h>
37#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020038#include <linux/poll.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020039#include <linux/fs.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020040
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020041#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050042#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020043
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +010044/*
Steven Rostedt73c51622009-03-11 13:42:01 -040045 * On boot up, the ring buffer is set to the minimum size, so that
46 * we do not waste memory on systems that are not using tracing.
47 */
Li Zefan020e5f82009-07-01 10:47:05 +080048int ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040049
50/*
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +010051 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010052 * A selftest will lurk into the ring-buffer to count the
53 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010054 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010055 * at the same time, giving false positive or negative results.
56 */
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +010057static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010058
Steven Rostedtb2821ae2009-02-02 21:38:32 -050059/*
60 * If a tracer is running, we do not want to run SELFTEST.
61 */
Li Zefan020e5f82009-07-01 10:47:05 +080062bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050063
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010064/* For tracers that don't implement custom flags */
65static struct tracer_opt dummy_tracer_opt[] = {
66 { }
67};
68
69static struct tracer_flags dummy_tracer_flags = {
70 .val = 0,
71 .opts = dummy_tracer_opt
72};
73
74static int dummy_set_flag(u32 old_flags, u32 bit, int set)
75{
76 return 0;
77}
Steven Rostedt0f048702008-11-05 16:05:44 -050078
79/*
80 * Kill all tracing for good (never come back).
81 * It is initialized to 1 but will turn to zero if the initialization
82 * of the tracer is successful. But that is the only place that sets
83 * this back to zero.
84 */
Hannes Eder4fd27352009-02-10 19:44:12 +010085static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -050086
Christoph Lameter9288f992009-10-07 19:17:45 -040087DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040088
89static inline void ftrace_disable_cpu(void)
90{
91 preempt_disable();
Rusty Russelldd17c8f2009-10-29 22:34:15 +090092 __this_cpu_inc(ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040093}
94
95static inline void ftrace_enable_cpu(void)
96{
Rusty Russelldd17c8f2009-10-29 22:34:15 +090097 __this_cpu_dec(ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040098 preempt_enable();
99}
100
Jason Wessel955b61e2010-08-05 09:22:23 -0500101cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +0200102
Steven Rostedt944ac422008-10-23 19:26:08 -0400103/*
104 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
105 *
106 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
107 * is set, then ftrace_dump is called. This will output the contents
108 * of the ftrace buffers to the console. This is very useful for
109 * capturing traces that lead to crashes and outputing it to a
110 * serial console.
111 *
112 * It is default off, but you can enable it with either specifying
113 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200114 * /proc/sys/kernel/ftrace_dump_on_oops
115 * Set 1 if you want to dump buffers of all CPUs
116 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400117 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200118
119enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400120
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500121static int tracing_set_tracer(const char *buf);
122
Li Zefanee6c2c12009-09-18 14:06:47 +0800123#define MAX_TRACER_SIZE 100
124static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500125static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100126
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200127static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100128{
Li Zefanee6c2c12009-09-18 14:06:47 +0800129 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500130 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400131 /* We are using ftrace early, expand it */
132 ring_buffer_expanded = 1;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100133 return 1;
134}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200135__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100136
Steven Rostedt944ac422008-10-23 19:26:08 -0400137static int __init set_ftrace_dump_on_oops(char *str)
138{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200139 if (*str++ != '=' || !*str) {
140 ftrace_dump_on_oops = DUMP_ALL;
141 return 1;
142 }
143
144 if (!strcmp("orig_cpu", str)) {
145 ftrace_dump_on_oops = DUMP_ORIG;
146 return 1;
147 }
148
149 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400150}
151__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200152
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800153unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200154{
155 nsec += 500;
156 do_div(nsec, 1000);
157 return nsec;
158}
159
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200160/*
161 * The global_trace is the descriptor that holds the tracing
162 * buffers for the live tracing. For each CPU, it contains
163 * a link list of pages that will store trace entries. The
164 * page descriptor of the pages in the memory is used to hold
165 * the link list by linking the lru item in the page descriptor
166 * to each of the pages in the buffer per CPU.
167 *
168 * For each active CPU there is a data field that holds the
169 * pages for the buffer for that CPU. Each CPU has the same number
170 * of pages allocated for its buffer.
171 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200172static struct trace_array global_trace;
173
174static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
175
Steven Rostedte77405a2009-09-02 14:17:06 -0400176int filter_current_check_discard(struct ring_buffer *buffer,
177 struct ftrace_event_call *call, void *rec,
Tom Zanussieb02ce02009-04-08 03:15:54 -0500178 struct ring_buffer_event *event)
179{
Steven Rostedte77405a2009-09-02 14:17:06 -0400180 return filter_check_discard(call, rec, buffer, event);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500181}
Steven Rostedt17c873e2009-04-10 18:12:50 -0400182EXPORT_SYMBOL_GPL(filter_current_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500183
Steven Rostedt37886f62009-03-17 17:22:06 -0400184cycle_t ftrace_now(int cpu)
185{
186 u64 ts;
187
188 /* Early boot up does not have a buffer yet */
189 if (!global_trace.buffer)
190 return trace_clock_local();
191
192 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
193 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
194
195 return ts;
196}
197
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200198/*
199 * The max_tr is used to snapshot the global_trace when a maximum
200 * latency is reached. Some tracers will use this to store a maximum
201 * trace while it continues examining live traces.
202 *
203 * The buffers for the max_tr are set up the same as the global_trace.
204 * When a snapshot is taken, the link list of the max_tr is swapped
205 * with the link list of the global_trace and the buffers are reset for
206 * the global_trace so the tracing can continue.
207 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200208static struct trace_array max_tr;
209
Tejun Heo9705f692009-10-29 22:34:13 +0900210static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200211
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200212/* tracer_enabled is used to toggle activation of a tracer */
Steven Rostedt26994ea2008-05-12 21:20:48 +0200213static int tracer_enabled = 1;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200214
Steven Rostedt90369902008-11-05 16:05:44 -0500215/**
216 * tracing_is_enabled - return tracer_enabled status
217 *
218 * This function is used by other tracers to know the status
219 * of the tracer_enabled flag. Tracers may use this function
220 * to know if it should enable their features when starting
221 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
222 */
223int tracing_is_enabled(void)
224{
225 return tracer_enabled;
226}
227
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200228/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400229 * trace_buf_size is the size in bytes that is allocated
230 * for a buffer. Note, the number of bytes is always rounded
231 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400232 *
233 * This number is purposely set to a low number of 16384.
234 * If the dump on oops happens, it will be much appreciated
235 * to not have to wait for all that output. Anyway this can be
236 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200237 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400238#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400239
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400240static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200241
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200242/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200243static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200244
245/* current_trace points to the tracer that is currently active */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200246static struct tracer *current_trace __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200247
248/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200249 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200250 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200251static DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200252
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800253/*
254 * serialize the access of the ring buffer
255 *
256 * ring buffer serializes readers, but it is low level protection.
257 * The validity of the events (which returns by ring_buffer_peek() ..etc)
258 * are not protected by ring buffer.
259 *
260 * The content of events may become garbage if we allow other process consumes
261 * these events concurrently:
262 * A) the page of the consumed events may become a normal page
263 * (not reader page) in ring buffer, and this page will be rewrited
264 * by events producer.
265 * B) The page of the consumed events may become a page for splice_read,
266 * and this page will be returned to system.
267 *
268 * These primitives allow multi process access to different cpu ring buffer
269 * concurrently.
270 *
271 * These primitives don't distinguish read-only and read-consume access.
272 * Multi read-only access are also serialized.
273 */
274
275#ifdef CONFIG_SMP
276static DECLARE_RWSEM(all_cpu_access_lock);
277static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
278
279static inline void trace_access_lock(int cpu)
280{
281 if (cpu == TRACE_PIPE_ALL_CPU) {
282 /* gain it for accessing the whole ring buffer. */
283 down_write(&all_cpu_access_lock);
284 } else {
285 /* gain it for accessing a cpu ring buffer. */
286
287 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
288 down_read(&all_cpu_access_lock);
289
290 /* Secondly block other access to this @cpu ring buffer. */
291 mutex_lock(&per_cpu(cpu_access_lock, cpu));
292 }
293}
294
295static inline void trace_access_unlock(int cpu)
296{
297 if (cpu == TRACE_PIPE_ALL_CPU) {
298 up_write(&all_cpu_access_lock);
299 } else {
300 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
301 up_read(&all_cpu_access_lock);
302 }
303}
304
305static inline void trace_access_lock_init(void)
306{
307 int cpu;
308
309 for_each_possible_cpu(cpu)
310 mutex_init(&per_cpu(cpu_access_lock, cpu));
311}
312
313#else
314
315static DEFINE_MUTEX(access_lock);
316
317static inline void trace_access_lock(int cpu)
318{
319 (void)cpu;
320 mutex_lock(&access_lock);
321}
322
323static inline void trace_access_unlock(int cpu)
324{
325 (void)cpu;
326 mutex_unlock(&access_lock);
327}
328
329static inline void trace_access_lock_init(void)
330{
331}
332
333#endif
334
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200335/* trace_wait is a waitqueue for tasks blocked on trace_poll */
Ingo Molnar4e655512008-05-12 21:20:52 +0200336static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
337
Steven Rostedtee6bce52008-11-12 17:52:37 -0500338/* trace_flags holds trace_options default values */
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500339unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
Steven Rostedta2a16d62009-03-24 23:17:58 -0400340 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
David Sharp750912f2010-12-08 13:46:47 -0800341 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE;
Ingo Molnar4e655512008-05-12 21:20:52 +0200342
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400343static int trace_stop_count;
Thomas Gleixner5389f6f2009-07-25 17:13:33 +0200344static DEFINE_RAW_SPINLOCK(tracing_start_lock);
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400345
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700346static void wakeup_work_handler(struct work_struct *work)
347{
348 wake_up(&trace_wait);
349}
350
351static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
352
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200353/**
354 * trace_wake_up - wake up tasks waiting for trace input
355 *
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700356 * Schedules a delayed work to wake up any task that is blocked on the
357 * trace_wait queue. These is used with trace_poll for tasks polling the
358 * trace.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200359 */
Ingo Molnar4e655512008-05-12 21:20:52 +0200360void trace_wake_up(void)
361{
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700362 const unsigned long delay = msecs_to_jiffies(2);
Andrew Morton89f19f02009-09-19 11:55:44 -0700363
364 if (trace_flags & TRACE_ITER_BLOCK)
365 return;
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700366 schedule_delayed_work(&wakeup_work, delay);
Ingo Molnar4e655512008-05-12 21:20:52 +0200367}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200368
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400369static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200370{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400371 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200372
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200373 if (!str)
374 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800375 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200376 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800377 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200378 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400379 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200380 return 1;
381}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400382__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200383
Tim Bird0e950172010-02-25 15:36:43 -0800384static int __init set_tracing_thresh(char *str)
385{
386 unsigned long threshhold;
387 int ret;
388
389 if (!str)
390 return 0;
391 ret = strict_strtoul(str, 0, &threshhold);
392 if (ret < 0)
393 return 0;
394 tracing_thresh = threshhold * 1000;
395 return 1;
396}
397__setup("tracing_thresh=", set_tracing_thresh);
398
Steven Rostedt57f50be2008-05-12 21:20:44 +0200399unsigned long nsecs_to_usecs(unsigned long nsecs)
400{
401 return nsecs / 1000;
402}
403
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200404/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200405static const char *trace_options[] = {
406 "print-parent",
407 "sym-offset",
408 "sym-addr",
409 "verbose",
Ingo Molnarf9896bf2008-05-12 21:20:47 +0200410 "raw",
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +0200411 "hex",
Ingo Molnarcb0f12a2008-05-12 21:20:47 +0200412 "bin",
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +0200413 "block",
Ingo Molnar86387f72008-05-12 21:20:51 +0200414 "stacktrace",
Ingo Molnar5e1607a2009-03-05 10:24:48 +0100415 "trace_printk",
Steven Rostedtb2a866f2008-11-03 23:15:57 -0500416 "ftrace_preempt",
Steven Rostedt9f029e82008-11-12 15:24:24 -0500417 "branch",
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500418 "annotate",
Török Edwin02b67512008-11-22 13:28:47 +0200419 "userstacktrace",
Török Edwinb54d3de2008-11-22 13:28:48 +0200420 "sym-userobj",
Frederic Weisbecker66896a82008-12-13 20:18:13 +0100421 "printk-msg-only",
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -0200422 "context-info",
Steven Rostedtc032ef642009-03-04 20:34:24 -0500423 "latency-format",
Steven Rostedtbe6f1642009-03-24 11:06:24 -0400424 "sleep-time",
Steven Rostedta2a16d62009-03-24 23:17:58 -0400425 "graph-time",
Li Zefane870e9a2010-07-02 11:07:32 +0800426 "record-cmd",
David Sharp750912f2010-12-08 13:46:47 -0800427 "overwrite",
Steven Rostedtcf30cf62011-06-14 22:44:07 -0400428 "disable_on_free",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200429 NULL
430};
431
Zhaolei5079f322009-08-25 16:12:56 +0800432static struct {
433 u64 (*func)(void);
434 const char *name;
435} trace_clocks[] = {
436 { trace_clock_local, "local" },
437 { trace_clock_global, "global" },
Steven Rostedt62496872011-09-19 11:35:58 -0400438 { trace_clock_counter, "counter" },
Zhaolei5079f322009-08-25 16:12:56 +0800439};
440
441int trace_clock_id;
442
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200443/*
444 * trace_parser_get_init - gets the buffer for trace parser
445 */
446int trace_parser_get_init(struct trace_parser *parser, int size)
447{
448 memset(parser, 0, sizeof(*parser));
449
450 parser->buffer = kmalloc(size, GFP_KERNEL);
451 if (!parser->buffer)
452 return 1;
453
454 parser->size = size;
455 return 0;
456}
457
458/*
459 * trace_parser_put - frees the buffer for trace parser
460 */
461void trace_parser_put(struct trace_parser *parser)
462{
463 kfree(parser->buffer);
464}
465
466/*
467 * trace_get_user - reads the user input string separated by space
468 * (matched by isspace(ch))
469 *
470 * For each string found the 'struct trace_parser' is updated,
471 * and the function returns.
472 *
473 * Returns number of bytes read.
474 *
475 * See kernel/trace/trace.h for 'struct trace_parser' details.
476 */
477int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
478 size_t cnt, loff_t *ppos)
479{
480 char ch;
481 size_t read = 0;
482 ssize_t ret;
483
484 if (!*ppos)
485 trace_parser_clear(parser);
486
487 ret = get_user(ch, ubuf++);
488 if (ret)
489 goto out;
490
491 read++;
492 cnt--;
493
494 /*
495 * The parser is not finished with the last write,
496 * continue reading the user input without skipping spaces.
497 */
498 if (!parser->cont) {
499 /* skip white space */
500 while (cnt && isspace(ch)) {
501 ret = get_user(ch, ubuf++);
502 if (ret)
503 goto out;
504 read++;
505 cnt--;
506 }
507
508 /* only spaces were written */
509 if (isspace(ch)) {
510 *ppos += read;
511 ret = read;
512 goto out;
513 }
514
515 parser->idx = 0;
516 }
517
518 /* read the non-space input */
519 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800520 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200521 parser->buffer[parser->idx++] = ch;
522 else {
523 ret = -EINVAL;
524 goto out;
525 }
526 ret = get_user(ch, ubuf++);
527 if (ret)
528 goto out;
529 read++;
530 cnt--;
531 }
532
533 /* We either got finished input or we have to wait for another call. */
534 if (isspace(ch)) {
535 parser->buffer[parser->idx] = 0;
536 parser->cont = false;
537 } else {
538 parser->cont = true;
539 parser->buffer[parser->idx++] = ch;
540 }
541
542 *ppos += read;
543 ret = read;
544
545out:
546 return ret;
547}
548
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200549ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
550{
551 int len;
552 int ret;
553
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500554 if (!cnt)
555 return 0;
556
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200557 if (s->len <= s->readpos)
558 return -EBUSY;
559
560 len = s->len - s->readpos;
561 if (cnt > len)
562 cnt = len;
563 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500564 if (ret == cnt)
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200565 return -EFAULT;
566
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500567 cnt -= ret;
568
Steven Rostedte74da522009-03-04 20:31:11 -0500569 s->readpos += cnt;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200570 return cnt;
Steven Rostedt214023c2008-05-12 21:20:46 +0200571}
572
Dmitri Vorobievb8b94262009-03-22 19:11:11 +0200573static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200574{
575 int len;
576 void *ret;
577
578 if (s->len <= s->readpos)
579 return -EBUSY;
580
581 len = s->len - s->readpos;
582 if (cnt > len)
583 cnt = len;
584 ret = memcpy(buf, s->buffer + s->readpos, cnt);
585 if (!ret)
586 return -EFAULT;
587
Steven Rostedte74da522009-03-04 20:31:11 -0500588 s->readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200589 return cnt;
590}
591
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400592/*
593 * ftrace_max_lock is used to protect the swapping of buffers
594 * when taking a max snapshot. The buffers themselves are
595 * protected by per_cpu spinlocks. But the action of the swap
596 * needs its own lock.
597 *
Thomas Gleixner445c8952009-12-02 19:49:50 +0100598 * This is defined as a arch_spinlock_t in order to help
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400599 * with performance when lockdep debugging is enabled.
600 *
601 * It is also used in other places outside the update_max_tr
602 * so it needs to be defined outside of the
603 * CONFIG_TRACER_MAX_TRACE.
604 */
Thomas Gleixner445c8952009-12-02 19:49:50 +0100605static arch_spinlock_t ftrace_max_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100606 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400607
Tim Bird0e950172010-02-25 15:36:43 -0800608unsigned long __read_mostly tracing_thresh;
609
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400610#ifdef CONFIG_TRACER_MAX_TRACE
611unsigned long __read_mostly tracing_max_latency;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400612
613/*
614 * Copy the new maximum trace into the separate maximum-trace
615 * structure. (this way the maximum trace is permanently saved,
616 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
617 */
618static void
619__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
620{
621 struct trace_array_cpu *data = tr->data[cpu];
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300622 struct trace_array_cpu *max_data;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400623
624 max_tr.cpu = cpu;
625 max_tr.time_start = data->preempt_timestamp;
626
Steven Rostedt8248ac02009-09-02 12:27:41 -0400627 max_data = max_tr.data[cpu];
628 max_data->saved_latency = tracing_max_latency;
629 max_data->critical_start = data->critical_start;
630 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400631
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300632 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -0400633 max_data->pid = tsk->pid;
634 max_data->uid = task_uid(tsk);
635 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
636 max_data->policy = tsk->policy;
637 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400638
639 /* record this tasks comm */
640 tracing_record_cmdline(tsk);
641}
642
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200643/**
644 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
645 * @tr: tracer
646 * @tsk: the task with the latency
647 * @cpu: The cpu that initiated the trace.
648 *
649 * Flip the buffers between the @tr and the max_tr and record information
650 * about which task was the cause of this latency.
651 */
Ingo Molnare309b412008-05-12 21:20:51 +0200652void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200653update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
654{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400655 struct ring_buffer *buf = tr->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200656
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400657 if (trace_stop_count)
658 return;
659
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200660 WARN_ON_ONCE(!irqs_disabled());
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900661 if (!current_trace->use_max_tr) {
662 WARN_ON_ONCE(1);
663 return;
664 }
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100665 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400666
667 tr->buffer = max_tr.buffer;
668 max_tr.buffer = buf;
669
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200670 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100671 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200672}
673
674/**
675 * update_max_tr_single - only copy one trace over, and reset the rest
676 * @tr - tracer
677 * @tsk - task with the latency
678 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200679 *
680 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200681 */
Ingo Molnare309b412008-05-12 21:20:51 +0200682void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200683update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
684{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400685 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200686
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400687 if (trace_stop_count)
688 return;
689
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200690 WARN_ON_ONCE(!irqs_disabled());
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900691 if (!current_trace->use_max_tr) {
692 WARN_ON_ONCE(1);
693 return;
694 }
695
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100696 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200697
Steven Rostedtd7690412008-10-01 00:29:53 -0400698 ftrace_disable_cpu();
699
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400700 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
701
Steven Rostedte8165db2009-09-03 19:13:05 -0400702 if (ret == -EBUSY) {
703 /*
704 * We failed to swap the buffer due to a commit taking
705 * place on this CPU. We fail to record, but we reset
706 * the max trace buffer (no one writes directly to it)
707 * and flag that it failed.
708 */
709 trace_array_printk(&max_tr, _THIS_IP_,
710 "Failed to swap buffers due to commit in progress\n");
711 }
712
Steven Rostedtd7690412008-10-01 00:29:53 -0400713 ftrace_enable_cpu();
714
Steven Rostedte8165db2009-09-03 19:13:05 -0400715 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200716
717 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100718 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200719}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400720#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200721
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200722/**
723 * register_tracer - register a tracer with the ftrace system.
724 * @type - the plugin for the tracer
725 *
726 * Register a new plugin tracer.
727 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200728int register_tracer(struct tracer *type)
Hannes Edere7669b82009-02-10 19:44:45 +0100729__releases(kernel_lock)
730__acquires(kernel_lock)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200731{
732 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200733 int ret = 0;
734
735 if (!type->name) {
736 pr_info("Tracer must have a name\n");
737 return -1;
738 }
739
Dan Carpenter24a461d2010-07-10 12:06:44 +0200740 if (strlen(type->name) >= MAX_TRACER_SIZE) {
Li Zefanee6c2c12009-09-18 14:06:47 +0800741 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
742 return -1;
743 }
744
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200745 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100746
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +0100747 tracing_selftest_running = true;
748
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200749 for (t = trace_types; t; t = t->next) {
750 if (strcmp(type->name, t->name) == 0) {
751 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +0800752 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200753 type->name);
754 ret = -1;
755 goto out;
756 }
757 }
758
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100759 if (!type->set_flag)
760 type->set_flag = &dummy_set_flag;
761 if (!type->flags)
762 type->flags = &dummy_tracer_flags;
763 else
764 if (!type->flags->opts)
765 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +0100766 if (!type->wait_pipe)
767 type->wait_pipe = default_wait_pipe;
768
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100769
Steven Rostedt60a11772008-05-12 21:20:44 +0200770#ifdef CONFIG_FTRACE_STARTUP_TEST
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500771 if (type->selftest && !tracing_selftest_disabled) {
Steven Rostedt60a11772008-05-12 21:20:44 +0200772 struct tracer *saved_tracer = current_trace;
Steven Rostedt60a11772008-05-12 21:20:44 +0200773 struct trace_array *tr = &global_trace;
Frederic Weisbeckerff325042008-12-04 23:47:35 +0100774
Steven Rostedt60a11772008-05-12 21:20:44 +0200775 /*
776 * Run a selftest on this tracer.
777 * Here we reset the trace buffer, and set the current
778 * tracer to be this tracer. The tracer can then run some
779 * internal tracing to verify that everything is in order.
780 * If we fail, we do not register this tracer.
781 */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400782 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100783
Steven Rostedt60a11772008-05-12 21:20:44 +0200784 current_trace = type;
Steven Rostedt4a0b1662011-03-09 20:09:26 -0500785
786 /* If we expanded the buffers, make sure the max is expanded too */
787 if (ring_buffer_expanded && type->use_max_tr)
788 ring_buffer_resize(max_tr.buffer, trace_buf_size);
789
Steven Rostedt60a11772008-05-12 21:20:44 +0200790 /* the test is responsible for initializing and enabling */
791 pr_info("Testing tracer %s: ", type->name);
792 ret = type->selftest(type, tr);
793 /* the test is responsible for resetting too */
794 current_trace = saved_tracer;
Steven Rostedt60a11772008-05-12 21:20:44 +0200795 if (ret) {
796 printk(KERN_CONT "FAILED!\n");
797 goto out;
798 }
Steven Rostedt1d4db002008-05-12 21:20:45 +0200799 /* Only reset on passing, to avoid touching corrupted buffers */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400800 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100801
Steven Rostedt4a0b1662011-03-09 20:09:26 -0500802 /* Shrink the max buffer again */
803 if (ring_buffer_expanded && type->use_max_tr)
804 ring_buffer_resize(max_tr.buffer, 1);
805
Steven Rostedt60a11772008-05-12 21:20:44 +0200806 printk(KERN_CONT "PASSED\n");
807 }
808#endif
809
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200810 type->next = trace_types;
811 trace_types = type;
Steven Rostedt60a11772008-05-12 21:20:44 +0200812
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200813 out:
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +0100814 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200815 mutex_unlock(&trace_types_lock);
816
Steven Rostedtdac74942009-02-05 01:13:38 -0500817 if (ret || !default_bootup_tracer)
818 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500819
Li Zefanee6c2c12009-09-18 14:06:47 +0800820 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -0500821 goto out_unlock;
822
823 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
824 /* Do we want this tracer to start on bootup? */
825 tracing_set_tracer(type->name);
826 default_bootup_tracer = NULL;
827 /* disable other selftests, since this will break it. */
828 tracing_selftest_disabled = 1;
829#ifdef CONFIG_FTRACE_STARTUP_TEST
830 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
831 type->name);
832#endif
833
834 out_unlock:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200835 return ret;
836}
837
838void unregister_tracer(struct tracer *type)
839{
840 struct tracer **t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200841
842 mutex_lock(&trace_types_lock);
843 for (t = &trace_types; *t; t = &(*t)->next) {
844 if (*t == type)
845 goto found;
846 }
Li Zefanee6c2c12009-09-18 14:06:47 +0800847 pr_info("Tracer %s not registered\n", type->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200848 goto out;
849
850 found:
851 *t = (*t)->next;
Arnaldo Carvalho de Melob5db03c2009-02-07 18:52:59 -0200852
853 if (type == current_trace && tracer_enabled) {
854 tracer_enabled = 0;
855 tracing_stop();
856 if (current_trace->stop)
857 current_trace->stop(&global_trace);
858 current_trace = &nop_trace;
859 }
Li Zefanee6c2c12009-09-18 14:06:47 +0800860out:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200861 mutex_unlock(&trace_types_lock);
862}
863
Steven Rostedt283740c2010-03-12 19:48:41 -0500864static void __tracing_reset(struct ring_buffer *buffer, int cpu)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200865{
Steven Rostedtd7690412008-10-01 00:29:53 -0400866 ftrace_disable_cpu();
Steven Rostedt283740c2010-03-12 19:48:41 -0500867 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtd7690412008-10-01 00:29:53 -0400868 ftrace_enable_cpu();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200869}
870
Steven Rostedtf6339032009-09-04 12:35:16 -0400871void tracing_reset(struct trace_array *tr, int cpu)
872{
873 struct ring_buffer *buffer = tr->buffer;
874
875 ring_buffer_record_disable(buffer);
876
877 /* Make sure all commits have finished */
878 synchronize_sched();
Steven Rostedt283740c2010-03-12 19:48:41 -0500879 __tracing_reset(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -0400880
881 ring_buffer_record_enable(buffer);
882}
883
Pekka J Enberg213cc062008-12-19 12:08:39 +0200884void tracing_reset_online_cpus(struct trace_array *tr)
885{
Steven Rostedt621968c2009-09-04 12:02:35 -0400886 struct ring_buffer *buffer = tr->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +0200887 int cpu;
888
Steven Rostedt621968c2009-09-04 12:02:35 -0400889 ring_buffer_record_disable(buffer);
890
891 /* Make sure all commits have finished */
892 synchronize_sched();
893
Pekka J Enberg213cc062008-12-19 12:08:39 +0200894 tr->time_start = ftrace_now(tr->cpu);
895
896 for_each_online_cpu(cpu)
Steven Rostedt283740c2010-03-12 19:48:41 -0500897 __tracing_reset(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -0400898
899 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +0200900}
901
Steven Rostedt9456f0f2009-05-06 21:54:09 -0400902void tracing_reset_current(int cpu)
903{
904 tracing_reset(&global_trace, cpu);
905}
906
907void tracing_reset_current_online_cpus(void)
908{
909 tracing_reset_online_cpus(&global_trace);
910}
911
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200912#define SAVED_CMDLINES 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100913#define NO_CMDLINE_MAP UINT_MAX
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200914static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
915static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
916static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
917static int cmdline_idx;
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100918static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt25b0b442008-05-12 21:21:00 +0200919
Steven Rostedt25b0b442008-05-12 21:21:00 +0200920/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +0100921static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200922
923static void trace_init_cmdlines(void)
924{
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100925 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
926 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200927 cmdline_idx = 0;
928}
929
Carsten Emdeb5130b12009-09-13 01:43:07 +0200930int is_tracing_stopped(void)
931{
932 return trace_stop_count;
933}
934
Steven Rostedt0f048702008-11-05 16:05:44 -0500935/**
Steven Rostedt69bb54e2008-11-21 12:59:38 -0500936 * ftrace_off_permanent - disable all ftrace code permanently
937 *
938 * This should only be called when a serious anomally has
939 * been detected. This will turn off the function tracing,
940 * ring buffers, and other tracing utilites. It takes no
941 * locks and can be called from any context.
942 */
943void ftrace_off_permanent(void)
944{
945 tracing_disabled = 1;
946 ftrace_stop();
947 tracing_off_permanent();
948}
949
950/**
Steven Rostedt0f048702008-11-05 16:05:44 -0500951 * tracing_start - quick start of the tracer
952 *
953 * If tracing is enabled but was stopped by tracing_stop,
954 * this will start the tracer back up.
955 */
956void tracing_start(void)
957{
958 struct ring_buffer *buffer;
959 unsigned long flags;
960
961 if (tracing_disabled)
962 return;
963
Thomas Gleixner5389f6f2009-07-25 17:13:33 +0200964 raw_spin_lock_irqsave(&tracing_start_lock, flags);
Steven Rostedtb06a8302009-01-22 14:26:15 -0500965 if (--trace_stop_count) {
966 if (trace_stop_count < 0) {
967 /* Someone screwed up their debugging */
968 WARN_ON_ONCE(1);
969 trace_stop_count = 0;
970 }
Steven Rostedt0f048702008-11-05 16:05:44 -0500971 goto out;
972 }
973
Steven Rostedta2f80712010-03-12 19:56:00 -0500974 /* Prevent the buffers from switching */
975 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -0500976
977 buffer = global_trace.buffer;
978 if (buffer)
979 ring_buffer_record_enable(buffer);
980
981 buffer = max_tr.buffer;
982 if (buffer)
983 ring_buffer_record_enable(buffer);
984
Steven Rostedta2f80712010-03-12 19:56:00 -0500985 arch_spin_unlock(&ftrace_max_lock);
986
Steven Rostedt0f048702008-11-05 16:05:44 -0500987 ftrace_start();
988 out:
Thomas Gleixner5389f6f2009-07-25 17:13:33 +0200989 raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -0500990}
991
992/**
993 * tracing_stop - quick stop of the tracer
994 *
995 * Light weight way to stop tracing. Use in conjunction with
996 * tracing_start.
997 */
998void tracing_stop(void)
999{
1000 struct ring_buffer *buffer;
1001 unsigned long flags;
1002
1003 ftrace_stop();
Thomas Gleixner5389f6f2009-07-25 17:13:33 +02001004 raw_spin_lock_irqsave(&tracing_start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001005 if (trace_stop_count++)
1006 goto out;
1007
Steven Rostedta2f80712010-03-12 19:56:00 -05001008 /* Prevent the buffers from switching */
1009 arch_spin_lock(&ftrace_max_lock);
1010
Steven Rostedt0f048702008-11-05 16:05:44 -05001011 buffer = global_trace.buffer;
1012 if (buffer)
1013 ring_buffer_record_disable(buffer);
1014
1015 buffer = max_tr.buffer;
1016 if (buffer)
1017 ring_buffer_record_disable(buffer);
1018
Steven Rostedta2f80712010-03-12 19:56:00 -05001019 arch_spin_unlock(&ftrace_max_lock);
1020
Steven Rostedt0f048702008-11-05 16:05:44 -05001021 out:
Thomas Gleixner5389f6f2009-07-25 17:13:33 +02001022 raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001023}
1024
Ingo Molnare309b412008-05-12 21:20:51 +02001025void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001026
Ingo Molnare309b412008-05-12 21:20:51 +02001027static void trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001028{
Carsten Emdea635cf02009-03-18 09:00:41 +01001029 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001030
1031 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1032 return;
1033
1034 /*
1035 * It's not the end of the world if we don't get
1036 * the lock, but we also don't want to spin
1037 * nor do we want to disable interrupts,
1038 * so if we miss here, then better luck next time.
1039 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001040 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001041 return;
1042
1043 idx = map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001044 if (idx == NO_CMDLINE_MAP) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001045 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1046
Carsten Emdea635cf02009-03-18 09:00:41 +01001047 /*
1048 * Check whether the cmdline buffer at idx has a pid
1049 * mapped. We are going to overwrite that entry so we
1050 * need to clear the map_pid_to_cmdline. Otherwise we
1051 * would read the new comm for the old pid.
1052 */
1053 pid = map_cmdline_to_pid[idx];
1054 if (pid != NO_CMDLINE_MAP)
1055 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001056
Carsten Emdea635cf02009-03-18 09:00:41 +01001057 map_cmdline_to_pid[idx] = tsk->pid;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001058 map_pid_to_cmdline[tsk->pid] = idx;
1059
1060 cmdline_idx = idx;
1061 }
1062
1063 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1064
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001065 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001066}
1067
Steven Rostedt4ca530852009-03-16 19:20:15 -04001068void trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001069{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001070 unsigned map;
1071
Steven Rostedt4ca530852009-03-16 19:20:15 -04001072 if (!pid) {
1073 strcpy(comm, "<idle>");
1074 return;
1075 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001076
Steven Rostedt74bf4072010-01-25 15:11:53 -05001077 if (WARN_ON_ONCE(pid < 0)) {
1078 strcpy(comm, "<XXX>");
1079 return;
1080 }
1081
Steven Rostedt4ca530852009-03-16 19:20:15 -04001082 if (pid > PID_MAX_DEFAULT) {
1083 strcpy(comm, "<...>");
1084 return;
1085 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001086
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001087 preempt_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001088 arch_spin_lock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001089 map = map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001090 if (map != NO_CMDLINE_MAP)
1091 strcpy(comm, saved_cmdlines[map]);
1092 else
1093 strcpy(comm, "<...>");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001094
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001095 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001096 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001097}
1098
Ingo Molnare309b412008-05-12 21:20:51 +02001099void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001100{
Thomas Gleixner18aecd32009-03-18 08:56:58 +01001101 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1102 !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001103 return;
1104
1105 trace_save_cmdline(tsk);
1106}
1107
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001108void
Steven Rostedt38697052008-10-01 13:14:09 -04001109tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1110 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001111{
1112 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001113
Steven Rostedt777e2082008-09-29 23:02:42 -04001114 entry->preempt_count = pc & 0xff;
1115 entry->pid = (tsk) ? tsk->pid : 0;
Arjan van de Vena3a4a5a2011-05-05 23:55:18 -04001116 entry->padding = 0;
Steven Rostedt777e2082008-09-29 23:02:42 -04001117 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001118#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001119 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001120#else
1121 TRACE_FLAG_IRQS_NOSUPPORT |
1122#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001123 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1124 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1125 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1126}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001127EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001128
Steven Rostedte77405a2009-09-02 14:17:06 -04001129struct ring_buffer_event *
1130trace_buffer_lock_reserve(struct ring_buffer *buffer,
1131 int type,
1132 unsigned long len,
1133 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001134{
1135 struct ring_buffer_event *event;
1136
Steven Rostedte77405a2009-09-02 14:17:06 -04001137 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001138 if (event != NULL) {
1139 struct trace_entry *ent = ring_buffer_event_data(event);
1140
1141 tracing_generic_entry_update(ent, flags, pc);
1142 ent->type = type;
1143 }
1144
1145 return event;
1146}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001147
Steven Rostedte77405a2009-09-02 14:17:06 -04001148static inline void
1149__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1150 struct ring_buffer_event *event,
1151 unsigned long flags, int pc,
1152 int wake)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001153{
Steven Rostedte77405a2009-09-02 14:17:06 -04001154 ring_buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001155
Steven Rostedte77405a2009-09-02 14:17:06 -04001156 ftrace_trace_stack(buffer, flags, 6, pc);
1157 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001158
1159 if (wake)
1160 trace_wake_up();
1161}
1162
Steven Rostedte77405a2009-09-02 14:17:06 -04001163void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1164 struct ring_buffer_event *event,
1165 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001166{
Steven Rostedte77405a2009-09-02 14:17:06 -04001167 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001168}
1169
Steven Rostedtef5580d2009-02-27 19:38:04 -05001170struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001171trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1172 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001173 unsigned long flags, int pc)
1174{
Steven Rostedte77405a2009-09-02 14:17:06 -04001175 *current_rb = global_trace.buffer;
1176 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001177 type, len, flags, pc);
1178}
Steven Rostedt94487d62009-05-05 19:22:53 -04001179EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001180
Steven Rostedte77405a2009-09-02 14:17:06 -04001181void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1182 struct ring_buffer_event *event,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001183 unsigned long flags, int pc)
1184{
Steven Rostedte77405a2009-09-02 14:17:06 -04001185 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001186}
Steven Rostedt94487d62009-05-05 19:22:53 -04001187EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001188
Steven Rostedte77405a2009-09-02 14:17:06 -04001189void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1190 struct ring_buffer_event *event,
1191 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001192{
Steven Rostedte77405a2009-09-02 14:17:06 -04001193 __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001194}
Steven Rostedt94487d62009-05-05 19:22:53 -04001195EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
Steven Rostedt77d9f462009-04-02 01:16:59 -04001196
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001197void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1198 struct ring_buffer_event *event,
1199 unsigned long flags, int pc,
1200 struct pt_regs *regs)
1201{
1202 ring_buffer_unlock_commit(buffer, event);
1203
1204 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1205 ftrace_trace_userstack(buffer, flags, pc);
1206}
1207EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1208
Steven Rostedte77405a2009-09-02 14:17:06 -04001209void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1210 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001211{
Steven Rostedte77405a2009-09-02 14:17:06 -04001212 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001213}
Steven Rostedt12acd472009-04-17 16:01:56 -04001214EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001215
Ingo Molnare309b412008-05-12 21:20:51 +02001216void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001217trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001218 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1219 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001220{
Tom Zanussie1112b42009-03-31 00:48:49 -05001221 struct ftrace_event_call *call = &event_function;
Steven Rostedte77405a2009-09-02 14:17:06 -04001222 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001223 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001224 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001225
Steven Rostedtd7690412008-10-01 00:29:53 -04001226 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001227 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001228 return;
1229
Steven Rostedte77405a2009-09-02 14:17:06 -04001230 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001231 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001232 if (!event)
1233 return;
1234 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001235 entry->ip = ip;
1236 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001237
Steven Rostedte77405a2009-09-02 14:17:06 -04001238 if (!filter_check_discard(call, entry, buffer, event))
1239 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001240}
1241
Ingo Molnare309b412008-05-12 21:20:51 +02001242void
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001243ftrace(struct trace_array *tr, struct trace_array_cpu *data,
Steven Rostedt38697052008-10-01 13:14:09 -04001244 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1245 int pc)
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001246{
1247 if (likely(!atomic_read(&data->disabled)))
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001248 trace_function(tr, ip, parent_ip, flags, pc);
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001249}
1250
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001251#ifdef CONFIG_STACKTRACE
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001252
1253#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1254struct ftrace_stack {
1255 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1256};
1257
1258static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1259static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1260
Steven Rostedte77405a2009-09-02 14:17:06 -04001261static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001262 unsigned long flags,
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001263 int skip, int pc, struct pt_regs *regs)
Ingo Molnar86387f72008-05-12 21:20:51 +02001264{
Tom Zanussie1112b42009-03-31 00:48:49 -05001265 struct ftrace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001266 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001267 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001268 struct stack_trace trace;
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001269 int use_stack;
1270 int size = FTRACE_STACK_ENTRIES;
Ingo Molnar86387f72008-05-12 21:20:51 +02001271
1272 trace.nr_entries = 0;
Ingo Molnar86387f72008-05-12 21:20:51 +02001273 trace.skip = skip;
Ingo Molnar86387f72008-05-12 21:20:51 +02001274
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001275 /*
1276 * Since events can happen in NMIs there's no safe way to
1277 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1278 * or NMI comes in, it will just have to use the default
1279 * FTRACE_STACK_SIZE.
1280 */
1281 preempt_disable_notrace();
1282
1283 use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1284 /*
1285 * We don't need any atomic variables, just a barrier.
1286 * If an interrupt comes in, we don't care, because it would
1287 * have exited and put the counter back to what we want.
1288 * We just need a barrier to keep gcc from moving things
1289 * around.
1290 */
1291 barrier();
1292 if (use_stack == 1) {
1293 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1294 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1295
1296 if (regs)
1297 save_stack_trace_regs(regs, &trace);
1298 else
1299 save_stack_trace(&trace);
1300
1301 if (trace.nr_entries > size)
1302 size = trace.nr_entries;
1303 } else
1304 /* From now on, use_stack is a boolean */
1305 use_stack = 0;
1306
1307 size *= sizeof(unsigned long);
1308
1309 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1310 sizeof(*entry) + size, flags, pc);
1311 if (!event)
1312 goto out;
1313 entry = ring_buffer_event_data(event);
1314
1315 memset(&entry->caller, 0, size);
1316
1317 if (use_stack)
1318 memcpy(&entry->caller, trace.entries,
1319 trace.nr_entries * sizeof(unsigned long));
1320 else {
1321 trace.max_entries = FTRACE_STACK_ENTRIES;
1322 trace.entries = entry->caller;
1323 if (regs)
1324 save_stack_trace_regs(regs, &trace);
1325 else
1326 save_stack_trace(&trace);
1327 }
1328
1329 entry->size = trace.nr_entries;
1330
Steven Rostedte77405a2009-09-02 14:17:06 -04001331 if (!filter_check_discard(call, entry, buffer, event))
1332 ring_buffer_unlock_commit(buffer, event);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001333
1334 out:
1335 /* Again, don't let gcc optimize things here */
1336 barrier();
1337 __get_cpu_var(ftrace_stack_reserve)--;
1338 preempt_enable_notrace();
1339
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001340}
1341
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001342void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1343 int skip, int pc, struct pt_regs *regs)
1344{
1345 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1346 return;
1347
1348 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1349}
1350
Steven Rostedte77405a2009-09-02 14:17:06 -04001351void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1352 int skip, int pc)
Steven Rostedt53614992009-01-15 19:12:40 -05001353{
1354 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1355 return;
1356
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001357 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
Steven Rostedt53614992009-01-15 19:12:40 -05001358}
1359
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001360void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1361 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001362{
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001363 __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
Steven Rostedt38697052008-10-01 13:14:09 -04001364}
1365
Steven Rostedt03889382009-12-11 09:48:22 -05001366/**
1367 * trace_dump_stack - record a stack back trace in the trace buffer
1368 */
1369void trace_dump_stack(void)
1370{
1371 unsigned long flags;
1372
1373 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001374 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001375
1376 local_save_flags(flags);
1377
1378 /* skipping 3 traces, seems to get us at the caller of this function */
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001379 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
Steven Rostedt03889382009-12-11 09:48:22 -05001380}
1381
Steven Rostedt91e86e52010-11-10 12:56:12 +01001382static DEFINE_PER_CPU(int, user_stack_count);
1383
Steven Rostedte77405a2009-09-02 14:17:06 -04001384void
1385ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001386{
Tom Zanussie1112b42009-03-31 00:48:49 -05001387 struct ftrace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001388 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001389 struct userstack_entry *entry;
1390 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001391
1392 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1393 return;
1394
Steven Rostedtb6345872010-03-12 20:03:30 -05001395 /*
1396 * NMIs can not handle page faults, even with fix ups.
1397 * The save user stack can (and often does) fault.
1398 */
1399 if (unlikely(in_nmi()))
1400 return;
1401
Steven Rostedt91e86e52010-11-10 12:56:12 +01001402 /*
1403 * prevent recursion, since the user stack tracing may
1404 * trigger other kernel events.
1405 */
1406 preempt_disable();
1407 if (__this_cpu_read(user_stack_count))
1408 goto out;
1409
1410 __this_cpu_inc(user_stack_count);
1411
Steven Rostedte77405a2009-09-02 14:17:06 -04001412 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001413 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001414 if (!event)
Li Zefan1dbd1952010-12-09 15:47:56 +08001415 goto out_drop_count;
Török Edwin02b67512008-11-22 13:28:47 +02001416 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001417
Steven Rostedt48659d32009-09-11 11:36:23 -04001418 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001419 memset(&entry->caller, 0, sizeof(entry->caller));
1420
1421 trace.nr_entries = 0;
1422 trace.max_entries = FTRACE_STACK_ENTRIES;
1423 trace.skip = 0;
1424 trace.entries = entry->caller;
1425
1426 save_stack_trace_user(&trace);
Steven Rostedte77405a2009-09-02 14:17:06 -04001427 if (!filter_check_discard(call, entry, buffer, event))
1428 ring_buffer_unlock_commit(buffer, event);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001429
Li Zefan1dbd1952010-12-09 15:47:56 +08001430 out_drop_count:
Steven Rostedt91e86e52010-11-10 12:56:12 +01001431 __this_cpu_dec(user_stack_count);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001432 out:
1433 preempt_enable();
Török Edwin02b67512008-11-22 13:28:47 +02001434}
1435
Hannes Eder4fd27352009-02-10 19:44:12 +01001436#ifdef UNUSED
1437static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001438{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001439 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001440}
Hannes Eder4fd27352009-02-10 19:44:12 +01001441#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001442
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001443#endif /* CONFIG_STACKTRACE */
1444
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001445/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001446 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001447 *
1448 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04001449int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001450{
Thomas Gleixner445c8952009-12-02 19:49:50 +01001451 static arch_spinlock_t trace_buf_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001452 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001453 static u32 trace_buf[TRACE_BUF_SIZE];
1454
Tom Zanussie1112b42009-03-31 00:48:49 -05001455 struct ftrace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001456 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001457 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001458 struct trace_array *tr = &global_trace;
1459 struct trace_array_cpu *data;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001460 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001461 unsigned long flags;
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001462 int disable;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001463 int cpu, len = 0, size, pc;
1464
1465 if (unlikely(tracing_selftest_running || tracing_disabled))
1466 return 0;
1467
1468 /* Don't pollute graph traces with trace_vprintk internals */
1469 pause_graph_tracing();
1470
1471 pc = preempt_count();
Steven Rostedt5168ae52010-06-03 09:36:50 -04001472 preempt_disable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001473 cpu = raw_smp_processor_id();
1474 data = tr->data[cpu];
1475
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001476 disable = atomic_inc_return(&data->disabled);
1477 if (unlikely(disable != 1))
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001478 goto out;
1479
Steven Rostedt80370cb2009-03-10 17:16:35 -04001480 /* Lockdep uses trace_printk for lock tracing */
1481 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001482 arch_spin_lock(&trace_buf_lock);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001483 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1484
1485 if (len > TRACE_BUF_SIZE || len < 0)
1486 goto out_unlock;
1487
1488 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedte77405a2009-09-02 14:17:06 -04001489 buffer = tr->buffer;
1490 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1491 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001492 if (!event)
1493 goto out_unlock;
1494 entry = ring_buffer_event_data(event);
1495 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001496 entry->fmt = fmt;
1497
1498 memcpy(entry->buf, trace_buf, sizeof(u32) * len);
Steven Rostedtd9313692010-01-06 17:27:11 -05001499 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedte77405a2009-09-02 14:17:06 -04001500 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05001501 ftrace_trace_stack(buffer, flags, 6, pc);
1502 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001503
1504out_unlock:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001505 arch_spin_unlock(&trace_buf_lock);
Steven Rostedt80370cb2009-03-10 17:16:35 -04001506 local_irq_restore(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001507
1508out:
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001509 atomic_dec_return(&data->disabled);
Steven Rostedt5168ae52010-06-03 09:36:50 -04001510 preempt_enable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001511 unpause_graph_tracing();
1512
1513 return len;
1514}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001515EXPORT_SYMBOL_GPL(trace_vbprintk);
1516
Steven Rostedt659372d2009-09-03 19:11:07 -04001517int trace_array_printk(struct trace_array *tr,
1518 unsigned long ip, const char *fmt, ...)
1519{
1520 int ret;
1521 va_list ap;
1522
1523 if (!(trace_flags & TRACE_ITER_PRINTK))
1524 return 0;
1525
1526 va_start(ap, fmt);
1527 ret = trace_array_vprintk(tr, ip, fmt, ap);
1528 va_end(ap);
1529 return ret;
1530}
1531
1532int trace_array_vprintk(struct trace_array *tr,
1533 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001534{
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001535 static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001536 static char trace_buf[TRACE_BUF_SIZE];
1537
Tom Zanussie1112b42009-03-31 00:48:49 -05001538 struct ftrace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001539 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001540 struct ring_buffer *buffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001541 struct trace_array_cpu *data;
1542 int cpu, len = 0, size, pc;
1543 struct print_entry *entry;
1544 unsigned long irq_flags;
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001545 int disable;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001546
1547 if (tracing_disabled || tracing_selftest_running)
1548 return 0;
1549
1550 pc = preempt_count();
1551 preempt_disable_notrace();
1552 cpu = raw_smp_processor_id();
1553 data = tr->data[cpu];
1554
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001555 disable = atomic_inc_return(&data->disabled);
1556 if (unlikely(disable != 1))
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001557 goto out;
1558
1559 pause_graph_tracing();
1560 raw_local_irq_save(irq_flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001561 arch_spin_lock(&trace_buf_lock);
Carsten Emdef2942482009-12-06 14:02:44 +01001562 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001563
1564 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04001565 buffer = tr->buffer;
1566 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1567 irq_flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001568 if (!event)
1569 goto out_unlock;
1570 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001571 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001572
1573 memcpy(&entry->buf, trace_buf, len);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001574 entry->buf[len] = '\0';
Steven Rostedtd9313692010-01-06 17:27:11 -05001575 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedte77405a2009-09-02 14:17:06 -04001576 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05001577 ftrace_trace_stack(buffer, irq_flags, 6, pc);
1578 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001579
1580 out_unlock:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001581 arch_spin_unlock(&trace_buf_lock);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001582 raw_local_irq_restore(irq_flags);
1583 unpause_graph_tracing();
1584 out:
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001585 atomic_dec_return(&data->disabled);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001586 preempt_enable_notrace();
1587
1588 return len;
1589}
Steven Rostedt659372d2009-09-03 19:11:07 -04001590
1591int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1592{
Steven Rostedta813a152009-10-09 01:41:35 -04001593 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04001594}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001595EXPORT_SYMBOL_GPL(trace_vprintk);
1596
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001597static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04001598{
Steven Rostedtd7690412008-10-01 00:29:53 -04001599 /* Don't allow ftrace to trace into the ring buffers */
1600 ftrace_disable_cpu();
1601
Steven Rostedt5a90f572008-09-03 17:42:51 -04001602 iter->idx++;
Steven Rostedtd7690412008-10-01 00:29:53 -04001603 if (iter->buffer_iter[iter->cpu])
1604 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1605
1606 ftrace_enable_cpu();
Steven Rostedt5a90f572008-09-03 17:42:51 -04001607}
1608
Ingo Molnare309b412008-05-12 21:20:51 +02001609static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001610peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1611 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001612{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001613 struct ring_buffer_event *event;
1614 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001615
Steven Rostedtd7690412008-10-01 00:29:53 -04001616 /* Don't allow ftrace to trace into the ring buffers */
1617 ftrace_disable_cpu();
1618
1619 if (buf_iter)
1620 event = ring_buffer_iter_peek(buf_iter, ts);
1621 else
Steven Rostedtbc21b472010-03-31 19:49:26 -04001622 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1623 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04001624
1625 ftrace_enable_cpu();
1626
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001627 if (event) {
1628 iter->ent_size = ring_buffer_event_length(event);
1629 return ring_buffer_event_data(event);
1630 }
1631 iter->ent_size = 0;
1632 return NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001633}
Steven Rostedtd7690412008-10-01 00:29:53 -04001634
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001635static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001636__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1637 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001638{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001639 struct ring_buffer *buffer = iter->tr->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001640 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08001641 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001642 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001643 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001644 int next_cpu = -1;
1645 int cpu;
1646
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001647 /*
1648 * If we are in a per_cpu trace file, don't bother by iterating over
1649 * all cpu and peek directly.
1650 */
1651 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1652 if (ring_buffer_empty_cpu(buffer, cpu_file))
1653 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001654 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001655 if (ent_cpu)
1656 *ent_cpu = cpu_file;
1657
1658 return ent;
1659 }
1660
Steven Rostedtab464282008-05-12 21:21:00 +02001661 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001662
1663 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001664 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001665
Steven Rostedtbc21b472010-03-31 19:49:26 -04001666 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001667
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02001668 /*
1669 * Pick the entry with the smallest timestamp:
1670 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001671 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001672 next = ent;
1673 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001674 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001675 next_lost = lost_events;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001676 }
1677 }
1678
1679 if (ent_cpu)
1680 *ent_cpu = next_cpu;
1681
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001682 if (ent_ts)
1683 *ent_ts = next_ts;
1684
Steven Rostedtbc21b472010-03-31 19:49:26 -04001685 if (missing_events)
1686 *missing_events = next_lost;
1687
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001688 return next;
1689}
1690
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001691/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001692struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1693 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001694{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001695 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001696}
Ingo Molnar8c523a92008-05-12 21:20:46 +02001697
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001698/* Find the next real entry, and increment the iterator to the next entry */
Jason Wessel955b61e2010-08-05 09:22:23 -05001699void *trace_find_next_entry_inc(struct trace_iterator *iter)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001700{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001701 iter->ent = __find_next_entry(iter, &iter->cpu,
1702 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02001703
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001704 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001705 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001706
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001707 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02001708}
1709
Ingo Molnare309b412008-05-12 21:20:51 +02001710static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001711{
Steven Rostedtd7690412008-10-01 00:29:53 -04001712 /* Don't allow ftrace to trace into the ring buffers */
1713 ftrace_disable_cpu();
Steven Rostedtbc21b472010-03-31 19:49:26 -04001714 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1715 &iter->lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04001716 ftrace_enable_cpu();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001717}
1718
Ingo Molnare309b412008-05-12 21:20:51 +02001719static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001720{
1721 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001722 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001723 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001724
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001725 WARN_ON_ONCE(iter->leftover);
1726
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001727 (*pos)++;
1728
1729 /* can't go backwards */
1730 if (iter->idx > i)
1731 return NULL;
1732
1733 if (iter->idx < 0)
Jason Wessel955b61e2010-08-05 09:22:23 -05001734 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001735 else
1736 ent = iter;
1737
1738 while (ent && iter->idx < i)
Jason Wessel955b61e2010-08-05 09:22:23 -05001739 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001740
1741 iter->pos = *pos;
1742
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001743 return ent;
1744}
1745
Jason Wessel955b61e2010-08-05 09:22:23 -05001746void tracing_iter_reset(struct trace_iterator *iter, int cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001747{
1748 struct trace_array *tr = iter->tr;
1749 struct ring_buffer_event *event;
1750 struct ring_buffer_iter *buf_iter;
1751 unsigned long entries = 0;
1752 u64 ts;
1753
1754 tr->data[cpu]->skipped_entries = 0;
1755
1756 if (!iter->buffer_iter[cpu])
1757 return;
1758
1759 buf_iter = iter->buffer_iter[cpu];
1760 ring_buffer_iter_reset(buf_iter);
1761
1762 /*
1763 * We could have the case with the max latency tracers
1764 * that a reset never took place on a cpu. This is evident
1765 * by the timestamp being before the start of the buffer.
1766 */
1767 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1768 if (ts >= iter->tr->time_start)
1769 break;
1770 entries++;
1771 ring_buffer_read(buf_iter, NULL);
1772 }
1773
1774 tr->data[cpu]->skipped_entries = entries;
1775}
1776
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001777/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001778 * The current tracer is copied to avoid a global locking
1779 * all around.
1780 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001781static void *s_start(struct seq_file *m, loff_t *pos)
1782{
1783 struct trace_iterator *iter = m->private;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001784 static struct tracer *old_tracer;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001785 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001786 void *p = NULL;
1787 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001788 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001789
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001790 /* copy the tracer to avoid using a global lock all around */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001791 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001792 if (unlikely(old_tracer != current_trace && current_trace)) {
1793 old_tracer = current_trace;
1794 *iter->trace = *current_trace;
Steven Rostedtd15f57f2008-05-12 21:20:56 +02001795 }
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001796 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001797
1798 atomic_inc(&trace_record_cmdline_disabled);
1799
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001800 if (*pos != iter->pos) {
1801 iter->ent = NULL;
1802 iter->cpu = 0;
1803 iter->idx = -1;
1804
Steven Rostedtd7690412008-10-01 00:29:53 -04001805 ftrace_disable_cpu();
1806
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001807 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1808 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001809 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001810 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001811 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001812
Steven Rostedtd7690412008-10-01 00:29:53 -04001813 ftrace_enable_cpu();
1814
Lai Jiangshanac91d852010-03-02 17:54:50 +08001815 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001816 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1817 ;
1818
1819 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001820 /*
1821 * If we overflowed the seq_file before, then we want
1822 * to just reuse the trace_seq buffer again.
1823 */
1824 if (iter->leftover)
1825 p = iter;
1826 else {
1827 l = *pos - 1;
1828 p = s_next(m, p, &l);
1829 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001830 }
1831
Lai Jiangshan4f535962009-05-18 19:35:34 +08001832 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001833 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001834 return p;
1835}
1836
1837static void s_stop(struct seq_file *m, void *p)
1838{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001839 struct trace_iterator *iter = m->private;
1840
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001841 atomic_dec(&trace_record_cmdline_disabled);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001842 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08001843 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001844}
1845
Ingo Molnare309b412008-05-12 21:20:51 +02001846static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001847{
Michael Ellermana6168352008-08-20 16:36:11 -07001848 seq_puts(m, "# _------=> CPU# \n");
1849 seq_puts(m, "# / _-----=> irqs-off \n");
1850 seq_puts(m, "# | / _----=> need-resched \n");
1851 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1852 seq_puts(m, "# ||| / _--=> preempt-depth \n");
Steven Rostedte6e1e252011-03-09 10:41:56 -05001853 seq_puts(m, "# |||| / delay \n");
1854 seq_puts(m, "# cmd pid ||||| time | caller \n");
1855 seq_puts(m, "# \\ / ||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001856}
1857
Ingo Molnare309b412008-05-12 21:20:51 +02001858static void print_func_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001859{
Michael Ellermana6168352008-08-20 16:36:11 -07001860 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1861 seq_puts(m, "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001862}
1863
1864
Jiri Olsa62b915f2010-04-02 19:01:22 +02001865void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001866print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1867{
1868 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1869 struct trace_array *tr = iter->tr;
1870 struct trace_array_cpu *data = tr->data[tr->cpu];
1871 struct tracer *type = current_trace;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001872 unsigned long entries = 0;
1873 unsigned long total = 0;
1874 unsigned long count;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001875 const char *name = "preemption";
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001876 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001877
1878 if (type)
1879 name = type->name;
1880
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001881
1882 for_each_tracing_cpu(cpu) {
1883 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1884 /*
1885 * If this buffer has skipped entries, then we hold all
1886 * entries for the trace and we need to ignore the
1887 * ones before the time stamp.
1888 */
1889 if (tr->data[cpu]->skipped_entries) {
1890 count -= tr->data[cpu]->skipped_entries;
1891 /* total is the same as the entries */
1892 total += count;
1893 } else
1894 total += count +
1895 ring_buffer_overrun_cpu(tr->buffer, cpu);
1896 entries += count;
1897 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001898
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001899 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001900 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001901 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001902 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001903 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001904 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02001905 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001906 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001907 total,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001908 tr->cpu,
1909#if defined(CONFIG_PREEMPT_NONE)
1910 "server",
1911#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1912 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04001913#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001914 "preempt",
1915#else
1916 "unknown",
1917#endif
1918 /* These are reserved for later use */
1919 0, 0, 0, 0);
1920#ifdef CONFIG_SMP
1921 seq_printf(m, " #P:%d)\n", num_online_cpus());
1922#else
1923 seq_puts(m, ")\n");
1924#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001925 seq_puts(m, "# -----------------\n");
1926 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001927 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1928 data->comm, data->pid, data->uid, data->nice,
1929 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001930 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001931
1932 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001933 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02001934 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1935 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001936 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02001937 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1938 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04001939 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001940 }
1941
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001942 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001943}
1944
Steven Rostedta3097202008-11-07 22:36:02 -05001945static void test_cpu_buff_start(struct trace_iterator *iter)
1946{
1947 struct trace_seq *s = &iter->seq;
1948
Steven Rostedt12ef7d42008-11-12 17:52:38 -05001949 if (!(trace_flags & TRACE_ITER_ANNOTATE))
1950 return;
1951
1952 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1953 return;
1954
Rusty Russell44623442009-01-01 10:12:23 +10301955 if (cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05001956 return;
1957
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001958 if (iter->tr->data[iter->cpu]->skipped_entries)
1959 return;
1960
Rusty Russell44623442009-01-01 10:12:23 +10301961 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02001962
1963 /* Don't print started cpu buffer for the first entry of the trace */
1964 if (iter->idx > 1)
1965 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1966 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05001967}
1968
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001969static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001970{
Steven Rostedt214023c2008-05-12 21:20:46 +02001971 struct trace_seq *s = &iter->seq;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001972 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001973 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05001974 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001975
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001976 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001977
Steven Rostedta3097202008-11-07 22:36:02 -05001978 test_cpu_buff_start(iter);
1979
Steven Rostedtf633cef2008-12-23 23:24:13 -05001980 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001981
1982 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt27d48be2009-03-04 21:57:29 -05001983 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1984 if (!trace_print_lat_context(iter))
1985 goto partial;
1986 } else {
1987 if (!trace_print_context(iter))
1988 goto partial;
1989 }
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001990 }
1991
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02001992 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04001993 return event->funcs->trace(iter, sym_flags, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001994
1995 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1996 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04001997
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001998 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001999partial:
2000 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002001}
2002
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002003static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002004{
2005 struct trace_seq *s = &iter->seq;
2006 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002007 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002008
2009 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002010
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002011 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002012 if (!trace_seq_printf(s, "%d %d %llu ",
2013 entry->pid, iter->cpu, iter->ts))
2014 goto partial;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002015 }
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002016
Steven Rostedtf633cef2008-12-23 23:24:13 -05002017 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002018 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002019 return event->funcs->raw(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002020
2021 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2022 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04002023
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002024 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002025partial:
2026 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002027}
2028
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002029static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002030{
2031 struct trace_seq *s = &iter->seq;
2032 unsigned char newline = '\n';
2033 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002034 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002035
2036 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002037
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002038 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2039 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2040 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2041 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2042 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002043
Steven Rostedtf633cef2008-12-23 23:24:13 -05002044 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002045 if (event) {
Steven Rostedta9a57762010-04-22 18:46:14 -04002046 enum print_line_t ret = event->funcs->hex(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002047 if (ret != TRACE_TYPE_HANDLED)
2048 return ret;
2049 }
Steven Rostedt7104f302008-10-01 10:52:51 -04002050
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002051 SEQ_PUT_FIELD_RET(s, newline);
2052
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002053 return TRACE_TYPE_HANDLED;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002054}
2055
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002056static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002057{
2058 struct trace_seq *s = &iter->seq;
2059 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002060 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002061
2062 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002063
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002064 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2065 SEQ_PUT_FIELD_RET(s, entry->pid);
Steven Rostedt1830b522009-02-07 19:38:43 -05002066 SEQ_PUT_FIELD_RET(s, iter->cpu);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002067 SEQ_PUT_FIELD_RET(s, iter->ts);
2068 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002069
Steven Rostedtf633cef2008-12-23 23:24:13 -05002070 event = ftrace_find_event(entry->type);
Steven Rostedta9a57762010-04-22 18:46:14 -04002071 return event ? event->funcs->binary(iter, 0, event) :
2072 TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002073}
2074
Jiri Olsa62b915f2010-04-02 19:01:22 +02002075int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002076{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002077 int cpu;
2078
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002079 /* If we are looking at one CPU buffer, only check that one */
2080 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2081 cpu = iter->cpu_file;
2082 if (iter->buffer_iter[cpu]) {
2083 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2084 return 0;
2085 } else {
2086 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2087 return 0;
2088 }
2089 return 1;
2090 }
2091
Steven Rostedtab464282008-05-12 21:21:00 +02002092 for_each_tracing_cpu(cpu) {
Steven Rostedtd7690412008-10-01 00:29:53 -04002093 if (iter->buffer_iter[cpu]) {
2094 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2095 return 0;
2096 } else {
2097 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2098 return 0;
2099 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002100 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002101
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002102 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002103}
2104
Lai Jiangshan4f535962009-05-18 19:35:34 +08002105/* Called with trace_event_read_lock() held. */
Jason Wessel955b61e2010-08-05 09:22:23 -05002106enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002107{
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002108 enum print_line_t ret;
2109
Jiri Olsaee5e51f2011-03-25 12:05:18 +01002110 if (iter->lost_events &&
2111 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2112 iter->cpu, iter->lost_events))
2113 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002114
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002115 if (iter->trace && iter->trace->print_line) {
2116 ret = iter->trace->print_line(iter);
2117 if (ret != TRACE_TYPE_UNHANDLED)
2118 return ret;
2119 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002120
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002121 if (iter->ent->type == TRACE_BPRINT &&
2122 trace_flags & TRACE_ITER_PRINTK &&
2123 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002124 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002125
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002126 if (iter->ent->type == TRACE_PRINT &&
2127 trace_flags & TRACE_ITER_PRINTK &&
2128 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002129 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002130
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002131 if (trace_flags & TRACE_ITER_BIN)
2132 return print_bin_fmt(iter);
2133
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002134 if (trace_flags & TRACE_ITER_HEX)
2135 return print_hex_fmt(iter);
2136
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002137 if (trace_flags & TRACE_ITER_RAW)
2138 return print_raw_fmt(iter);
2139
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002140 return print_trace_fmt(iter);
2141}
2142
Jiri Olsa62b915f2010-04-02 19:01:22 +02002143void trace_default_header(struct seq_file *m)
2144{
2145 struct trace_iterator *iter = m->private;
2146
Jiri Olsaf56e7f82011-06-03 16:58:49 +02002147 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2148 return;
2149
Jiri Olsa62b915f2010-04-02 19:01:22 +02002150 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2151 /* print nothing if the buffers are empty */
2152 if (trace_empty(iter))
2153 return;
2154 print_trace_header(m, iter);
2155 if (!(trace_flags & TRACE_ITER_VERBOSE))
2156 print_lat_help_header(m);
2157 } else {
2158 if (!(trace_flags & TRACE_ITER_VERBOSE))
2159 print_func_help_header(m);
2160 }
2161}
2162
Steven Rostedte0a413f2011-09-29 21:26:16 -04002163static void test_ftrace_alive(struct seq_file *m)
2164{
2165 if (!ftrace_is_dead())
2166 return;
2167 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2168 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2169}
2170
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002171static int s_show(struct seq_file *m, void *v)
2172{
2173 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002174 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002175
2176 if (iter->ent == NULL) {
2177 if (iter->tr) {
2178 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2179 seq_puts(m, "#\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002180 test_ftrace_alive(m);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002181 }
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002182 if (iter->trace && iter->trace->print_header)
2183 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002184 else
2185 trace_default_header(m);
2186
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002187 } else if (iter->leftover) {
2188 /*
2189 * If we filled the seq_file buffer earlier, we
2190 * want to just show it now.
2191 */
2192 ret = trace_print_seq(m, &iter->seq);
2193
2194 /* ret should this time be zero, but you never know */
2195 iter->leftover = ret;
2196
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002197 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002198 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002199 ret = trace_print_seq(m, &iter->seq);
2200 /*
2201 * If we overflow the seq_file buffer, then it will
2202 * ask us for this data again at start up.
2203 * Use that instead.
2204 * ret is 0 if seq_file write succeeded.
2205 * -1 otherwise.
2206 */
2207 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002208 }
2209
2210 return 0;
2211}
2212
James Morris88e9d342009-09-22 16:43:43 -07002213static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002214 .start = s_start,
2215 .next = s_next,
2216 .stop = s_stop,
2217 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002218};
2219
Ingo Molnare309b412008-05-12 21:20:51 +02002220static struct trace_iterator *
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002221__tracing_open(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002222{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002223 long cpu_file = (long) inode->i_private;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002224 void *fail_ret = ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002225 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002226 struct seq_file *m;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002227 int cpu, ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002228
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002229 if (tracing_disabled)
2230 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02002231
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002232 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002233 if (!iter)
2234 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002235
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002236 /*
2237 * We make a copy of the current tracer to avoid concurrent
2238 * changes on it while we are reading.
2239 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002240 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002241 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002242 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002243 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002244
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002245 if (current_trace)
2246 *iter->trace = *current_trace;
2247
Li Zefan79f55992009-06-15 14:58:26 +08002248 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002249 goto fail;
2250
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002251 if (current_trace && current_trace->print_max)
2252 iter->tr = &max_tr;
2253 else
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002254 iter->tr = &global_trace;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002255 iter->pos = -1;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002256 mutex_init(&iter->mutex);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002257 iter->cpu_file = cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002258
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002259 /* Notify the tracer early; before we stop tracing. */
2260 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01002261 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002262
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002263 /* Annotate start of buffers if we had overruns */
2264 if (ring_buffer_overruns(iter->tr->buffer))
2265 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2266
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002267 /* stop the trace while dumping */
2268 tracing_stop();
2269
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002270 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2271 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002272 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002273 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2274 }
2275 ring_buffer_read_prepare_sync();
2276 for_each_tracing_cpu(cpu) {
2277 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002278 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002279 }
2280 } else {
2281 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002282 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002283 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2284 ring_buffer_read_prepare_sync();
2285 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002286 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002287 }
2288
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002289 ret = seq_open(file, &tracer_seq_ops);
2290 if (ret < 0) {
2291 fail_ret = ERR_PTR(ret);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002292 goto fail_buffer;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002293 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002294
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002295 m = file->private_data;
2296 m->private = iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002297
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002298 mutex_unlock(&trace_types_lock);
2299
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002300 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002301
2302 fail_buffer:
2303 for_each_tracing_cpu(cpu) {
2304 if (iter->buffer_iter[cpu])
2305 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2306 }
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002307 free_cpumask_var(iter->started);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002308 tracing_start();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002309 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002310 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002311 kfree(iter->trace);
Julia Lawall0bb943c2008-11-14 19:05:31 +01002312 kfree(iter);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002313
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002314 return fail_ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002315}
2316
2317int tracing_open_generic(struct inode *inode, struct file *filp)
2318{
Steven Rostedt60a11772008-05-12 21:20:44 +02002319 if (tracing_disabled)
2320 return -ENODEV;
2321
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002322 filp->private_data = inode->i_private;
2323 return 0;
2324}
2325
Hannes Eder4fd27352009-02-10 19:44:12 +01002326static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002327{
matt mooney907f2782010-09-27 19:04:53 -07002328 struct seq_file *m = file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002329 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002330 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002331
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002332 if (!(file->f_mode & FMODE_READ))
2333 return 0;
2334
2335 iter = m->private;
2336
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002337 mutex_lock(&trace_types_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002338 for_each_tracing_cpu(cpu) {
2339 if (iter->buffer_iter[cpu])
2340 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2341 }
2342
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002343 if (iter->trace && iter->trace->close)
2344 iter->trace->close(iter);
2345
2346 /* reenable tracing if it was previously enabled */
Steven Rostedt90369902008-11-05 16:05:44 -05002347 tracing_start();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002348 mutex_unlock(&trace_types_lock);
2349
2350 seq_release(inode, file);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002351 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002352 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002353 kfree(iter->trace);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002354 kfree(iter);
2355 return 0;
2356}
2357
2358static int tracing_open(struct inode *inode, struct file *file)
2359{
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002360 struct trace_iterator *iter;
2361 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002362
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002363 /* If this file was open for write, then erase contents */
2364 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04002365 (file->f_flags & O_TRUNC)) {
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002366 long cpu = (long) inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002367
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002368 if (cpu == TRACE_PIPE_ALL_CPU)
2369 tracing_reset_online_cpus(&global_trace);
2370 else
2371 tracing_reset(&global_trace, cpu);
2372 }
2373
2374 if (file->f_mode & FMODE_READ) {
2375 iter = __tracing_open(inode, file);
2376 if (IS_ERR(iter))
2377 ret = PTR_ERR(iter);
2378 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2379 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2380 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002381 return ret;
2382}
2383
Ingo Molnare309b412008-05-12 21:20:51 +02002384static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002385t_next(struct seq_file *m, void *v, loff_t *pos)
2386{
Li Zefanf129e962009-06-24 09:53:44 +08002387 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002388
2389 (*pos)++;
2390
2391 if (t)
2392 t = t->next;
2393
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002394 return t;
2395}
2396
2397static void *t_start(struct seq_file *m, loff_t *pos)
2398{
Li Zefanf129e962009-06-24 09:53:44 +08002399 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002400 loff_t l = 0;
2401
2402 mutex_lock(&trace_types_lock);
Li Zefanf129e962009-06-24 09:53:44 +08002403 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002404 ;
2405
2406 return t;
2407}
2408
2409static void t_stop(struct seq_file *m, void *p)
2410{
2411 mutex_unlock(&trace_types_lock);
2412}
2413
2414static int t_show(struct seq_file *m, void *v)
2415{
2416 struct tracer *t = v;
2417
2418 if (!t)
2419 return 0;
2420
2421 seq_printf(m, "%s", t->name);
2422 if (t->next)
2423 seq_putc(m, ' ');
2424 else
2425 seq_putc(m, '\n');
2426
2427 return 0;
2428}
2429
James Morris88e9d342009-09-22 16:43:43 -07002430static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002431 .start = t_start,
2432 .next = t_next,
2433 .stop = t_stop,
2434 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002435};
2436
2437static int show_traces_open(struct inode *inode, struct file *file)
2438{
Steven Rostedt60a11772008-05-12 21:20:44 +02002439 if (tracing_disabled)
2440 return -ENODEV;
2441
Li Zefanf129e962009-06-24 09:53:44 +08002442 return seq_open(file, &show_traces_seq_ops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002443}
2444
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002445static ssize_t
2446tracing_write_stub(struct file *filp, const char __user *ubuf,
2447 size_t count, loff_t *ppos)
2448{
2449 return count;
2450}
2451
Slava Pestov364829b2010-11-24 15:13:16 -08002452static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2453{
2454 if (file->f_mode & FMODE_READ)
2455 return seq_lseek(file, offset, origin);
2456 else
2457 return 0;
2458}
2459
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002460static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002461 .open = tracing_open,
2462 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002463 .write = tracing_write_stub,
Slava Pestov364829b2010-11-24 15:13:16 -08002464 .llseek = tracing_seek,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002465 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002466};
2467
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002468static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002469 .open = show_traces_open,
2470 .read = seq_read,
2471 .release = seq_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002472 .llseek = seq_lseek,
Ingo Molnarc7078de2008-05-12 21:20:52 +02002473};
2474
Ingo Molnar36dfe922008-05-12 21:20:52 +02002475/*
2476 * Only trace on a CPU if the bitmask is set:
2477 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302478static cpumask_var_t tracing_cpumask;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002479
2480/*
2481 * The tracer itself will not take this lock, but still we want
2482 * to provide a consistent cpumask to user-space:
2483 */
2484static DEFINE_MUTEX(tracing_cpumask_update_lock);
2485
2486/*
2487 * Temporary storage for the character representation of the
2488 * CPU bitmask (and one more byte for the newline):
2489 */
2490static char mask_str[NR_CPUS + 1];
2491
Ingo Molnarc7078de2008-05-12 21:20:52 +02002492static ssize_t
2493tracing_cpumask_read(struct file *filp, char __user *ubuf,
2494 size_t count, loff_t *ppos)
2495{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002496 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002497
2498 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002499
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302500 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002501 if (count - len < 2) {
2502 count = -EINVAL;
2503 goto out_err;
2504 }
2505 len += sprintf(mask_str + len, "\n");
2506 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2507
2508out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02002509 mutex_unlock(&tracing_cpumask_update_lock);
2510
2511 return count;
2512}
2513
2514static ssize_t
2515tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2516 size_t count, loff_t *ppos)
2517{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002518 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302519 cpumask_var_t tracing_cpumask_new;
2520
2521 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2522 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002523
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302524 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002525 if (err)
2526 goto err_unlock;
2527
Li Zefan215368e2009-06-15 10:56:42 +08002528 mutex_lock(&tracing_cpumask_update_lock);
2529
Steven Rostedta5e25882008-12-02 15:34:05 -05002530 local_irq_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002531 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02002532 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002533 /*
2534 * Increase/decrease the disabled counter if we are
2535 * about to flip a bit in the cpumask:
2536 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302537 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2538 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002539 atomic_inc(&global_trace.data[cpu]->disabled);
2540 }
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302541 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2542 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002543 atomic_dec(&global_trace.data[cpu]->disabled);
2544 }
2545 }
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002546 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05002547 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02002548
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302549 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002550
Ingo Molnarc7078de2008-05-12 21:20:52 +02002551 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302552 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02002553
Ingo Molnarc7078de2008-05-12 21:20:52 +02002554 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002555
2556err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08002557 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002558
2559 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002560}
2561
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002562static const struct file_operations tracing_cpumask_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002563 .open = tracing_open_generic,
2564 .read = tracing_cpumask_read,
2565 .write = tracing_cpumask_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002566 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002567};
2568
Li Zefanfdb372e2009-12-08 11:15:59 +08002569static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002570{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002571 struct tracer_opt *trace_opts;
2572 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002573 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002574
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002575 mutex_lock(&trace_types_lock);
2576 tracer_flags = current_trace->flags->val;
2577 trace_opts = current_trace->flags->opts;
2578
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002579 for (i = 0; trace_options[i]; i++) {
2580 if (trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08002581 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002582 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002583 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002584 }
2585
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002586 for (i = 0; trace_opts[i].name; i++) {
2587 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08002588 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002589 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002590 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002591 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002592 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002593
Li Zefanfdb372e2009-12-08 11:15:59 +08002594 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002595}
2596
Li Zefan8d18eaa2009-12-08 11:17:06 +08002597static int __set_tracer_option(struct tracer *trace,
2598 struct tracer_flags *tracer_flags,
2599 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002600{
Li Zefan8d18eaa2009-12-08 11:17:06 +08002601 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002602
Li Zefan8d18eaa2009-12-08 11:17:06 +08002603 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002604 if (ret)
2605 return ret;
2606
2607 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08002608 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002609 else
Zhaolei77708412009-08-07 18:53:21 +08002610 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002611 return 0;
2612}
2613
Li Zefan8d18eaa2009-12-08 11:17:06 +08002614/* Try to assign a tracer specific option */
2615static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2616{
2617 struct tracer_flags *tracer_flags = trace->flags;
2618 struct tracer_opt *opts = NULL;
2619 int i;
2620
2621 for (i = 0; tracer_flags->opts[i].name; i++) {
2622 opts = &tracer_flags->opts[i];
2623
2624 if (strcmp(cmp, opts->name) == 0)
2625 return __set_tracer_option(trace, trace->flags,
2626 opts, neg);
2627 }
2628
2629 return -EINVAL;
2630}
2631
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002632static void set_tracer_flags(unsigned int mask, int enabled)
2633{
2634 /* do nothing if flag is already set */
2635 if (!!(trace_flags & mask) == !!enabled)
2636 return;
2637
2638 if (enabled)
2639 trace_flags |= mask;
2640 else
2641 trace_flags &= ~mask;
Li Zefane870e9a2010-07-02 11:07:32 +08002642
2643 if (mask == TRACE_ITER_RECORD_CMD)
2644 trace_event_enable_cmd_record(enabled);
David Sharp750912f2010-12-08 13:46:47 -08002645
2646 if (mask == TRACE_ITER_OVERWRITE)
2647 ring_buffer_change_overwrite(global_trace.buffer, enabled);
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002648}
2649
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002650static ssize_t
Steven Rostedtee6bce52008-11-12 17:52:37 -05002651tracing_trace_options_write(struct file *filp, const char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002652 size_t cnt, loff_t *ppos)
2653{
2654 char buf[64];
Li Zefan8d18eaa2009-12-08 11:17:06 +08002655 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002656 int neg = 0;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002657 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002658 int i;
2659
Steven Rostedtcffae432008-05-12 21:21:00 +02002660 if (cnt >= sizeof(buf))
2661 return -EINVAL;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002662
2663 if (copy_from_user(&buf, ubuf, cnt))
2664 return -EFAULT;
2665
2666 buf[cnt] = 0;
Li Zefan8d18eaa2009-12-08 11:17:06 +08002667 cmp = strstrip(buf);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002668
Li Zefan8d18eaa2009-12-08 11:17:06 +08002669 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002670 neg = 1;
2671 cmp += 2;
2672 }
2673
2674 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08002675 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002676 set_tracer_flags(1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002677 break;
2678 }
2679 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002680
2681 /* If no option could be set, test the specific tracer options */
2682 if (!trace_options[i]) {
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002683 mutex_lock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002684 ret = set_tracer_option(current_trace, cmp, neg);
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002685 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002686 if (ret)
2687 return ret;
2688 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002689
Jiri Olsacf8517c2009-10-23 19:36:16 -04002690 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002691
2692 return cnt;
2693}
2694
Li Zefanfdb372e2009-12-08 11:15:59 +08002695static int tracing_trace_options_open(struct inode *inode, struct file *file)
2696{
2697 if (tracing_disabled)
2698 return -ENODEV;
2699 return single_open(file, tracing_trace_options_show, NULL);
2700}
2701
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002702static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08002703 .open = tracing_trace_options_open,
2704 .read = seq_read,
2705 .llseek = seq_lseek,
2706 .release = single_release,
Steven Rostedtee6bce52008-11-12 17:52:37 -05002707 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002708};
2709
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002710static const char readme_msg[] =
2711 "tracing mini-HOWTO:\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002712 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2713 "# cat /sys/kernel/debug/tracing/available_tracers\n"
Nikanth Karthikesanbc2b6872009-03-23 11:58:31 +05302714 "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002715 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Nikanth Karthikesanbc2b6872009-03-23 11:58:31 +05302716 "nop\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002717 "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2718 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002719 "sched_switch\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002720 "# cat /sys/kernel/debug/tracing/trace_options\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002721 "noprint-parent nosym-offset nosym-addr noverbose\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002722 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
Geunsik Lim9b5f8b32011-08-12 14:30:22 +09002723 "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002724 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
Geunsik Lim9b5f8b32011-08-12 14:30:22 +09002725 "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002726;
2727
2728static ssize_t
2729tracing_readme_read(struct file *filp, char __user *ubuf,
2730 size_t cnt, loff_t *ppos)
2731{
2732 return simple_read_from_buffer(ubuf, cnt, ppos,
2733 readme_msg, strlen(readme_msg));
2734}
2735
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002736static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002737 .open = tracing_open_generic,
2738 .read = tracing_readme_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002739 .llseek = generic_file_llseek,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002740};
2741
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002742static ssize_t
Avadh Patel69abe6a2009-04-10 16:04:48 -04002743tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2744 size_t cnt, loff_t *ppos)
2745{
2746 char *buf_comm;
2747 char *file_buf;
2748 char *buf;
2749 int len = 0;
2750 int pid;
2751 int i;
2752
2753 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2754 if (!file_buf)
2755 return -ENOMEM;
2756
2757 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2758 if (!buf_comm) {
2759 kfree(file_buf);
2760 return -ENOMEM;
2761 }
2762
2763 buf = file_buf;
2764
2765 for (i = 0; i < SAVED_CMDLINES; i++) {
2766 int r;
2767
2768 pid = map_cmdline_to_pid[i];
2769 if (pid == -1 || pid == NO_CMDLINE_MAP)
2770 continue;
2771
2772 trace_find_cmdline(pid, buf_comm);
2773 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2774 buf += r;
2775 len += r;
2776 }
2777
2778 len = simple_read_from_buffer(ubuf, cnt, ppos,
2779 file_buf, len);
2780
2781 kfree(file_buf);
2782 kfree(buf_comm);
2783
2784 return len;
2785}
2786
2787static const struct file_operations tracing_saved_cmdlines_fops = {
2788 .open = tracing_open_generic,
2789 .read = tracing_saved_cmdlines_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002790 .llseek = generic_file_llseek,
Avadh Patel69abe6a2009-04-10 16:04:48 -04002791};
2792
2793static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002794tracing_ctrl_read(struct file *filp, char __user *ubuf,
2795 size_t cnt, loff_t *ppos)
2796{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002797 char buf[64];
2798 int r;
2799
Steven Rostedt90369902008-11-05 16:05:44 -05002800 r = sprintf(buf, "%u\n", tracer_enabled);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002801 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002802}
2803
2804static ssize_t
2805tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2806 size_t cnt, loff_t *ppos)
2807{
2808 struct trace_array *tr = filp->private_data;
Hannes Eder5e398412009-02-10 19:44:34 +01002809 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002810 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002811
Peter Huewe22fe9b52011-06-07 21:58:27 +02002812 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2813 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002814 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002815
2816 val = !!val;
2817
2818 mutex_lock(&trace_types_lock);
Steven Rostedt90369902008-11-05 16:05:44 -05002819 if (tracer_enabled ^ val) {
Steven Rostedt6752ab42011-02-08 13:54:06 -05002820
2821 /* Only need to warn if this is used to change the state */
2822 WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2823
Steven Rostedt90369902008-11-05 16:05:44 -05002824 if (val) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002825 tracer_enabled = 1;
Steven Rostedt90369902008-11-05 16:05:44 -05002826 if (current_trace->start)
2827 current_trace->start(tr);
2828 tracing_start();
2829 } else {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002830 tracer_enabled = 0;
Steven Rostedt90369902008-11-05 16:05:44 -05002831 tracing_stop();
2832 if (current_trace->stop)
2833 current_trace->stop(tr);
2834 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002835 }
2836 mutex_unlock(&trace_types_lock);
2837
Jiri Olsacf8517c2009-10-23 19:36:16 -04002838 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002839
2840 return cnt;
2841}
2842
2843static ssize_t
2844tracing_set_trace_read(struct file *filp, char __user *ubuf,
2845 size_t cnt, loff_t *ppos)
2846{
Li Zefanee6c2c12009-09-18 14:06:47 +08002847 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002848 int r;
2849
2850 mutex_lock(&trace_types_lock);
2851 if (current_trace)
2852 r = sprintf(buf, "%s\n", current_trace->name);
2853 else
2854 r = sprintf(buf, "\n");
2855 mutex_unlock(&trace_types_lock);
2856
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002857 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002858}
2859
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02002860int tracer_init(struct tracer *t, struct trace_array *tr)
2861{
2862 tracing_reset_online_cpus(tr);
2863 return t->init(tr);
2864}
2865
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07002866static int __tracing_resize_ring_buffer(unsigned long size)
Steven Rostedt73c51622009-03-11 13:42:01 -04002867{
2868 int ret;
2869
2870 /*
2871 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04002872 * we use the size that was given, and we can forget about
2873 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04002874 */
2875 ring_buffer_expanded = 1;
2876
2877 ret = ring_buffer_resize(global_trace.buffer, size);
2878 if (ret < 0)
2879 return ret;
2880
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09002881 if (!current_trace->use_max_tr)
2882 goto out;
2883
Steven Rostedt73c51622009-03-11 13:42:01 -04002884 ret = ring_buffer_resize(max_tr.buffer, size);
2885 if (ret < 0) {
2886 int r;
2887
2888 r = ring_buffer_resize(global_trace.buffer,
2889 global_trace.entries);
2890 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04002891 /*
2892 * AARGH! We are left with different
2893 * size max buffer!!!!
2894 * The max buffer is our "snapshot" buffer.
2895 * When a tracer needs a snapshot (one of the
2896 * latency tracers), it swaps the max buffer
2897 * with the saved snap shot. We succeeded to
2898 * update the size of the main buffer, but failed to
2899 * update the size of the max buffer. But when we tried
2900 * to reset the main buffer to the original size, we
2901 * failed there too. This is very unlikely to
2902 * happen, but if it does, warn and kill all
2903 * tracing.
2904 */
Steven Rostedt73c51622009-03-11 13:42:01 -04002905 WARN_ON(1);
2906 tracing_disabled = 1;
2907 }
2908 return ret;
2909 }
2910
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09002911 max_tr.entries = size;
2912 out:
Steven Rostedt73c51622009-03-11 13:42:01 -04002913 global_trace.entries = size;
2914
2915 return ret;
2916}
2917
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07002918static ssize_t tracing_resize_ring_buffer(unsigned long size)
2919{
2920 int cpu, ret = size;
2921
2922 mutex_lock(&trace_types_lock);
2923
2924 tracing_stop();
2925
2926 /* disable all cpu buffers */
2927 for_each_tracing_cpu(cpu) {
2928 if (global_trace.data[cpu])
2929 atomic_inc(&global_trace.data[cpu]->disabled);
2930 if (max_tr.data[cpu])
2931 atomic_inc(&max_tr.data[cpu]->disabled);
2932 }
2933
2934 if (size != global_trace.entries)
2935 ret = __tracing_resize_ring_buffer(size);
2936
2937 if (ret < 0)
2938 ret = -ENOMEM;
2939
2940 for_each_tracing_cpu(cpu) {
2941 if (global_trace.data[cpu])
2942 atomic_dec(&global_trace.data[cpu]->disabled);
2943 if (max_tr.data[cpu])
2944 atomic_dec(&max_tr.data[cpu]->disabled);
2945 }
2946
2947 tracing_start();
2948 mutex_unlock(&trace_types_lock);
2949
2950 return ret;
2951}
2952
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09002953
Steven Rostedt1852fcc2009-03-11 14:33:00 -04002954/**
2955 * tracing_update_buffers - used by tracing facility to expand ring buffers
2956 *
2957 * To save on memory when the tracing is never used on a system with it
2958 * configured in. The ring buffers are set to a minimum size. But once
2959 * a user starts to use the tracing facility, then they need to grow
2960 * to their default size.
2961 *
2962 * This function is to be called when a tracer is about to be used.
2963 */
2964int tracing_update_buffers(void)
2965{
2966 int ret = 0;
2967
Steven Rostedt1027fcb2009-03-12 11:33:20 -04002968 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04002969 if (!ring_buffer_expanded)
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07002970 ret = __tracing_resize_ring_buffer(trace_buf_size);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04002971 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04002972
2973 return ret;
2974}
2975
Steven Rostedt577b7852009-02-26 23:43:05 -05002976struct trace_option_dentry;
2977
2978static struct trace_option_dentry *
2979create_trace_option_files(struct tracer *tracer);
2980
2981static void
2982destroy_trace_option_files(struct trace_option_dentry *topts);
2983
Steven Rostedtb2821ae2009-02-02 21:38:32 -05002984static int tracing_set_tracer(const char *buf)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002985{
Steven Rostedt577b7852009-02-26 23:43:05 -05002986 static struct trace_option_dentry *topts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002987 struct trace_array *tr = &global_trace;
2988 struct tracer *t;
Peter Zijlstrad9e54072008-11-01 19:57:37 +01002989 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002990
Steven Rostedt1027fcb2009-03-12 11:33:20 -04002991 mutex_lock(&trace_types_lock);
2992
Steven Rostedt73c51622009-03-11 13:42:01 -04002993 if (!ring_buffer_expanded) {
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07002994 ret = __tracing_resize_ring_buffer(trace_buf_size);
Steven Rostedt73c51622009-03-11 13:42:01 -04002995 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01002996 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04002997 ret = 0;
2998 }
2999
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003000 for (t = trace_types; t; t = t->next) {
3001 if (strcmp(t->name, buf) == 0)
3002 break;
3003 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003004 if (!t) {
3005 ret = -EINVAL;
3006 goto out;
3007 }
3008 if (t == current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003009 goto out;
3010
Steven Rostedt9f029e82008-11-12 15:24:24 -05003011 trace_branch_disable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003012 if (current_trace && current_trace->reset)
3013 current_trace->reset(tr);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003014 if (current_trace && current_trace->use_max_tr) {
3015 /*
3016 * We don't free the ring buffer. instead, resize it because
3017 * The max_tr ring buffer has some state (e.g. ring->clock) and
3018 * we want preserve it.
3019 */
3020 ring_buffer_resize(max_tr.buffer, 1);
3021 max_tr.entries = 1;
3022 }
Steven Rostedt577b7852009-02-26 23:43:05 -05003023 destroy_trace_option_files(topts);
3024
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003025 current_trace = t;
Steven Rostedt577b7852009-02-26 23:43:05 -05003026
3027 topts = create_trace_option_files(current_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003028 if (current_trace->use_max_tr) {
3029 ret = ring_buffer_resize(max_tr.buffer, global_trace.entries);
3030 if (ret < 0)
3031 goto out;
3032 max_tr.entries = global_trace.entries;
3033 }
Steven Rostedt577b7852009-02-26 23:43:05 -05003034
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003035 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003036 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003037 if (ret)
3038 goto out;
3039 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003040
Steven Rostedt9f029e82008-11-12 15:24:24 -05003041 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003042 out:
3043 mutex_unlock(&trace_types_lock);
3044
Peter Zijlstrad9e54072008-11-01 19:57:37 +01003045 return ret;
3046}
3047
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003048static ssize_t
3049tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3050 size_t cnt, loff_t *ppos)
3051{
Li Zefanee6c2c12009-09-18 14:06:47 +08003052 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003053 int i;
3054 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003055 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003056
Steven Rostedt60063a62008-10-28 10:44:24 -04003057 ret = cnt;
3058
Li Zefanee6c2c12009-09-18 14:06:47 +08003059 if (cnt > MAX_TRACER_SIZE)
3060 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003061
3062 if (copy_from_user(&buf, ubuf, cnt))
3063 return -EFAULT;
3064
3065 buf[cnt] = 0;
3066
3067 /* strip ending whitespace. */
3068 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3069 buf[i] = 0;
3070
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003071 err = tracing_set_tracer(buf);
3072 if (err)
3073 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003074
Jiri Olsacf8517c2009-10-23 19:36:16 -04003075 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003076
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003077 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003078}
3079
3080static ssize_t
3081tracing_max_lat_read(struct file *filp, char __user *ubuf,
3082 size_t cnt, loff_t *ppos)
3083{
3084 unsigned long *ptr = filp->private_data;
3085 char buf[64];
3086 int r;
3087
Steven Rostedtcffae432008-05-12 21:21:00 +02003088 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003089 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02003090 if (r > sizeof(buf))
3091 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003092 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003093}
3094
3095static ssize_t
3096tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3097 size_t cnt, loff_t *ppos)
3098{
Hannes Eder5e398412009-02-10 19:44:34 +01003099 unsigned long *ptr = filp->private_data;
Hannes Eder5e398412009-02-10 19:44:34 +01003100 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003101 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003102
Peter Huewe22fe9b52011-06-07 21:58:27 +02003103 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3104 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003105 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003106
3107 *ptr = val * 1000;
3108
3109 return cnt;
3110}
3111
Steven Rostedtb3806b42008-05-12 21:20:46 +02003112static int tracing_open_pipe(struct inode *inode, struct file *filp)
3113{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003114 long cpu_file = (long) inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003115 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003116 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003117
3118 if (tracing_disabled)
3119 return -ENODEV;
3120
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003121 mutex_lock(&trace_types_lock);
3122
Steven Rostedtb3806b42008-05-12 21:20:46 +02003123 /* create a buffer to store the information to pass to userspace */
3124 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003125 if (!iter) {
3126 ret = -ENOMEM;
3127 goto out;
3128 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003129
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003130 /*
3131 * We make a copy of the current tracer to avoid concurrent
3132 * changes on it while we are reading.
3133 */
3134 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3135 if (!iter->trace) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003136 ret = -ENOMEM;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003137 goto fail;
3138 }
3139 if (current_trace)
3140 *iter->trace = *current_trace;
3141
3142 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3143 ret = -ENOMEM;
3144 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10303145 }
3146
Steven Rostedta3097202008-11-07 22:36:02 -05003147 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10303148 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05003149
Steven Rostedt112f38a72009-06-01 15:16:05 -04003150 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3151 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3152
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003153 iter->cpu_file = cpu_file;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003154 iter->tr = &global_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003155 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003156 filp->private_data = iter;
3157
Steven Rostedt107bad82008-05-12 21:21:01 +02003158 if (iter->trace->pipe_open)
3159 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02003160
Arnd Bergmannb4447862010-07-07 23:40:11 +02003161 nonseekable_open(inode, filp);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003162out:
3163 mutex_unlock(&trace_types_lock);
3164 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003165
3166fail:
3167 kfree(iter->trace);
3168 kfree(iter);
3169 mutex_unlock(&trace_types_lock);
3170 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003171}
3172
3173static int tracing_release_pipe(struct inode *inode, struct file *file)
3174{
3175 struct trace_iterator *iter = file->private_data;
3176
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003177 mutex_lock(&trace_types_lock);
3178
Steven Rostedt29bf4a52009-12-09 12:37:43 -05003179 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05003180 iter->trace->pipe_close(iter);
3181
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003182 mutex_unlock(&trace_types_lock);
3183
Rusty Russell44623442009-01-01 10:12:23 +10303184 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003185 mutex_destroy(&iter->mutex);
3186 kfree(iter->trace);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003187 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003188
3189 return 0;
3190}
3191
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003192static unsigned int
3193tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3194{
3195 struct trace_iterator *iter = filp->private_data;
3196
3197 if (trace_flags & TRACE_ITER_BLOCK) {
3198 /*
3199 * Always select as readable when in blocking mode
3200 */
3201 return POLLIN | POLLRDNORM;
Ingo Molnarafc2abc2008-05-12 21:21:00 +02003202 } else {
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003203 if (!trace_empty(iter))
3204 return POLLIN | POLLRDNORM;
3205 poll_wait(filp, &trace_wait, poll_table);
3206 if (!trace_empty(iter))
3207 return POLLIN | POLLRDNORM;
3208
3209 return 0;
3210 }
3211}
3212
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003213
3214void default_wait_pipe(struct trace_iterator *iter)
3215{
3216 DEFINE_WAIT(wait);
3217
3218 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3219
3220 if (trace_empty(iter))
3221 schedule();
3222
3223 finish_wait(&trace_wait, &wait);
3224}
3225
3226/*
3227 * This is a make-shift waitqueue.
3228 * A tracer might use this callback on some rare cases:
3229 *
3230 * 1) the current tracer might hold the runqueue lock when it wakes up
3231 * a reader, hence a deadlock (sched, function, and function graph tracers)
3232 * 2) the function tracers, trace all functions, we don't want
3233 * the overhead of calling wake_up and friends
3234 * (and tracing them too)
3235 *
3236 * Anyway, this is really very primitive wakeup.
3237 */
3238void poll_wait_pipe(struct trace_iterator *iter)
3239{
3240 set_current_state(TASK_INTERRUPTIBLE);
3241 /* sleep for 100 msecs, and try again. */
3242 schedule_timeout(HZ / 10);
3243}
3244
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003245/* Must be called with trace_types_lock mutex held. */
3246static int tracing_wait_pipe(struct file *filp)
3247{
3248 struct trace_iterator *iter = filp->private_data;
3249
3250 while (trace_empty(iter)) {
3251
3252 if ((filp->f_flags & O_NONBLOCK)) {
3253 return -EAGAIN;
3254 }
3255
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003256 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003257
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003258 iter->trace->wait_pipe(iter);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003259
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003260 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003261
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003262 if (signal_pending(current))
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003263 return -EINTR;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003264
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003265 /*
3266 * We block until we read something and tracing is disabled.
3267 * We still block if tracing is disabled, but we have never
3268 * read anything. This allows a user to cat this file, and
3269 * then enable tracing. But after we have read something,
3270 * we give an EOF when tracing is again disabled.
3271 *
3272 * iter->pos will be 0 if we haven't read anything.
3273 */
3274 if (!tracer_enabled && iter->pos)
3275 break;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003276 }
3277
3278 return 1;
3279}
3280
Steven Rostedtb3806b42008-05-12 21:20:46 +02003281/*
3282 * Consumer reader.
3283 */
3284static ssize_t
3285tracing_read_pipe(struct file *filp, char __user *ubuf,
3286 size_t cnt, loff_t *ppos)
3287{
3288 struct trace_iterator *iter = filp->private_data;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003289 static struct tracer *old_tracer;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003290 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003291
3292 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003293 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3294 if (sret != -EBUSY)
3295 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003296
Steven Rostedtf9520752009-03-02 14:04:40 -05003297 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003298
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003299 /* copy the tracer to avoid using a global lock all around */
Steven Rostedt107bad82008-05-12 21:21:01 +02003300 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003301 if (unlikely(old_tracer != current_trace && current_trace)) {
3302 old_tracer = current_trace;
3303 *iter->trace = *current_trace;
3304 }
3305 mutex_unlock(&trace_types_lock);
3306
3307 /*
3308 * Avoid more than one consumer on a single file descriptor
3309 * This is just a matter of traces coherency, the ring buffer itself
3310 * is protected.
3311 */
3312 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003313 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003314 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3315 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02003316 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02003317 }
3318
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003319waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003320 sret = tracing_wait_pipe(filp);
3321 if (sret <= 0)
3322 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003323
3324 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003325 if (trace_empty(iter)) {
3326 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02003327 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003328 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003329
3330 if (cnt >= PAGE_SIZE)
3331 cnt = PAGE_SIZE - 1;
3332
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003333 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003334 memset(&iter->seq, 0,
3335 sizeof(struct trace_iterator) -
3336 offsetof(struct trace_iterator, seq));
Steven Rostedt4823ed72008-05-12 21:21:01 +02003337 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003338
Lai Jiangshan4f535962009-05-18 19:35:34 +08003339 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003340 trace_access_lock(iter->cpu_file);
Jason Wessel955b61e2010-08-05 09:22:23 -05003341 while (trace_find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003342 enum print_line_t ret;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003343 int len = iter->seq.len;
3344
Ingo Molnarf9896bf2008-05-12 21:20:47 +02003345 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003346 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02003347 /* don't print partial lines */
3348 iter->seq.len = len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003349 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003350 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01003351 if (ret != TRACE_TYPE_NO_CONSUME)
3352 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003353
3354 if (iter->seq.len >= cnt)
3355 break;
Jiri Olsaee5e51f2011-03-25 12:05:18 +01003356
3357 /*
3358 * Setting the full flag means we reached the trace_seq buffer
3359 * size and we should leave by partial output condition above.
3360 * One of the trace_seq_* functions is not used properly.
3361 */
3362 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3363 iter->ent->type);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003364 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003365 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003366 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02003367
Steven Rostedtb3806b42008-05-12 21:20:46 +02003368 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003369 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3370 if (iter->seq.readpos >= iter->seq.len)
Steven Rostedtf9520752009-03-02 14:04:40 -05003371 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003372
3373 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003374 * If there was nothing to send to user, in spite of consuming trace
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003375 * entries, go back to wait for more entries.
3376 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003377 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003378 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003379
Steven Rostedt107bad82008-05-12 21:21:01 +02003380out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003381 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003382
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003383 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003384}
3385
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003386static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3387 struct pipe_buffer *buf)
3388{
3389 __free_page(buf->page);
3390}
3391
3392static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3393 unsigned int idx)
3394{
3395 __free_page(spd->pages[idx]);
3396}
3397
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08003398static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003399 .can_merge = 0,
3400 .map = generic_pipe_buf_map,
3401 .unmap = generic_pipe_buf_unmap,
3402 .confirm = generic_pipe_buf_confirm,
3403 .release = tracing_pipe_buf_release,
3404 .steal = generic_pipe_buf_steal,
3405 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003406};
3407
Steven Rostedt34cd4992009-02-09 12:06:29 -05003408static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003409tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003410{
3411 size_t count;
3412 int ret;
3413
3414 /* Seq buffer is page-sized, exactly what we need. */
3415 for (;;) {
3416 count = iter->seq.len;
3417 ret = print_trace_line(iter);
3418 count = iter->seq.len - count;
3419 if (rem < count) {
3420 rem = 0;
3421 iter->seq.len -= count;
3422 break;
3423 }
3424 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3425 iter->seq.len -= count;
3426 break;
3427 }
3428
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08003429 if (ret != TRACE_TYPE_NO_CONSUME)
3430 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05003431 rem -= count;
Jason Wessel955b61e2010-08-05 09:22:23 -05003432 if (!trace_find_next_entry_inc(iter)) {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003433 rem = 0;
3434 iter->ent = NULL;
3435 break;
3436 }
3437 }
3438
3439 return rem;
3440}
3441
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003442static ssize_t tracing_splice_read_pipe(struct file *filp,
3443 loff_t *ppos,
3444 struct pipe_inode_info *pipe,
3445 size_t len,
3446 unsigned int flags)
3447{
Jens Axboe35f3d142010-05-20 10:43:18 +02003448 struct page *pages_def[PIPE_DEF_BUFFERS];
3449 struct partial_page partial_def[PIPE_DEF_BUFFERS];
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003450 struct trace_iterator *iter = filp->private_data;
3451 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02003452 .pages = pages_def,
3453 .partial = partial_def,
Steven Rostedt34cd4992009-02-09 12:06:29 -05003454 .nr_pages = 0, /* This gets updated below. */
3455 .flags = flags,
3456 .ops = &tracing_pipe_buf_ops,
3457 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003458 };
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003459 static struct tracer *old_tracer;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003460 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003461 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003462 unsigned int i;
3463
Jens Axboe35f3d142010-05-20 10:43:18 +02003464 if (splice_grow_spd(pipe, &spd))
3465 return -ENOMEM;
3466
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003467 /* copy the tracer to avoid using a global lock all around */
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003468 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003469 if (unlikely(old_tracer != current_trace && current_trace)) {
3470 old_tracer = current_trace;
3471 *iter->trace = *current_trace;
3472 }
3473 mutex_unlock(&trace_types_lock);
3474
3475 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003476
3477 if (iter->trace->splice_read) {
3478 ret = iter->trace->splice_read(iter, filp,
3479 ppos, pipe, len, flags);
3480 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003481 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003482 }
3483
3484 ret = tracing_wait_pipe(filp);
3485 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003486 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003487
Jason Wessel955b61e2010-08-05 09:22:23 -05003488 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003489 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003490 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003491 }
3492
Lai Jiangshan4f535962009-05-18 19:35:34 +08003493 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003494 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003495
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003496 /* Fill as many pages as possible. */
Jens Axboe35f3d142010-05-20 10:43:18 +02003497 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3498 spd.pages[i] = alloc_page(GFP_KERNEL);
3499 if (!spd.pages[i])
Steven Rostedt34cd4992009-02-09 12:06:29 -05003500 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003501
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003502 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003503
3504 /* Copy the data into the page, so we can start over. */
3505 ret = trace_seq_to_buffer(&iter->seq,
Jens Axboe35f3d142010-05-20 10:43:18 +02003506 page_address(spd.pages[i]),
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003507 iter->seq.len);
3508 if (ret < 0) {
Jens Axboe35f3d142010-05-20 10:43:18 +02003509 __free_page(spd.pages[i]);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003510 break;
3511 }
Jens Axboe35f3d142010-05-20 10:43:18 +02003512 spd.partial[i].offset = 0;
3513 spd.partial[i].len = iter->seq.len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003514
Steven Rostedtf9520752009-03-02 14:04:40 -05003515 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003516 }
3517
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003518 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003519 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003520 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003521
3522 spd.nr_pages = i;
3523
Jens Axboe35f3d142010-05-20 10:43:18 +02003524 ret = splice_to_pipe(pipe, &spd);
3525out:
3526 splice_shrink_spd(pipe, &spd);
3527 return ret;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003528
Steven Rostedt34cd4992009-02-09 12:06:29 -05003529out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003530 mutex_unlock(&iter->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +02003531 goto out;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003532}
3533
Steven Rostedta98a3c32008-05-12 21:20:59 +02003534static ssize_t
3535tracing_entries_read(struct file *filp, char __user *ubuf,
3536 size_t cnt, loff_t *ppos)
3537{
3538 struct trace_array *tr = filp->private_data;
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003539 char buf[96];
Steven Rostedta98a3c32008-05-12 21:20:59 +02003540 int r;
3541
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003542 mutex_lock(&trace_types_lock);
3543 if (!ring_buffer_expanded)
3544 r = sprintf(buf, "%lu (expanded: %lu)\n",
3545 tr->entries >> 10,
3546 trace_buf_size >> 10);
3547 else
3548 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3549 mutex_unlock(&trace_types_lock);
3550
Steven Rostedta98a3c32008-05-12 21:20:59 +02003551 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3552}
3553
3554static ssize_t
3555tracing_entries_write(struct file *filp, const char __user *ubuf,
3556 size_t cnt, loff_t *ppos)
3557{
3558 unsigned long val;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003559 int ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003560
Peter Huewe22fe9b52011-06-07 21:58:27 +02003561 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3562 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003563 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003564
3565 /* must have at least 1 entry */
3566 if (!val)
3567 return -EINVAL;
3568
Steven Rostedt1696b2b2008-11-13 00:09:35 -05003569 /* value is in KB */
3570 val <<= 10;
3571
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003572 ret = tracing_resize_ring_buffer(val);
3573 if (ret < 0)
3574 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003575
Jiri Olsacf8517c2009-10-23 19:36:16 -04003576 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003577
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003578 return cnt;
3579}
Steven Rostedtbf5e6512008-11-10 21:46:00 -05003580
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003581static ssize_t
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07003582tracing_total_entries_read(struct file *filp, char __user *ubuf,
3583 size_t cnt, loff_t *ppos)
3584{
3585 struct trace_array *tr = filp->private_data;
3586 char buf[64];
3587 int r, cpu;
3588 unsigned long size = 0, expanded_size = 0;
3589
3590 mutex_lock(&trace_types_lock);
3591 for_each_tracing_cpu(cpu) {
3592 size += tr->entries >> 10;
3593 if (!ring_buffer_expanded)
3594 expanded_size += trace_buf_size >> 10;
3595 }
3596 if (ring_buffer_expanded)
3597 r = sprintf(buf, "%lu\n", size);
3598 else
3599 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3600 mutex_unlock(&trace_types_lock);
3601
3602 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3603}
3604
3605static ssize_t
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003606tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
3607 size_t cnt, loff_t *ppos)
3608{
3609 /*
3610 * There is no need to read what the user has written, this function
3611 * is just to make sure that there is no error when "echo" is used
3612 */
3613
3614 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003615
3616 return cnt;
3617}
3618
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003619static int
3620tracing_free_buffer_release(struct inode *inode, struct file *filp)
3621{
Steven Rostedtcf30cf62011-06-14 22:44:07 -04003622 /* disable tracing ? */
3623 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3624 tracing_off();
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003625 /* resize the ring buffer to 0 */
3626 tracing_resize_ring_buffer(0);
3627
3628 return 0;
3629}
3630
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003631static ssize_t
3632tracing_mark_write(struct file *filp, const char __user *ubuf,
3633 size_t cnt, loff_t *fpos)
3634{
Steven Rostedtd696b582011-09-22 11:50:27 -04003635 unsigned long addr = (unsigned long)ubuf;
3636 struct ring_buffer_event *event;
3637 struct ring_buffer *buffer;
3638 struct print_entry *entry;
3639 unsigned long irq_flags;
3640 struct page *pages[2];
3641 int nr_pages = 1;
3642 ssize_t written;
3643 void *page1;
3644 void *page2;
3645 int offset;
3646 int size;
3647 int len;
3648 int ret;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003649
Steven Rostedtc76f0692008-11-07 22:36:02 -05003650 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003651 return -EINVAL;
3652
3653 if (cnt > TRACE_BUF_SIZE)
3654 cnt = TRACE_BUF_SIZE;
3655
Steven Rostedtd696b582011-09-22 11:50:27 -04003656 /*
3657 * Userspace is injecting traces into the kernel trace buffer.
3658 * We want to be as non intrusive as possible.
3659 * To do so, we do not want to allocate any special buffers
3660 * or take any locks, but instead write the userspace data
3661 * straight into the ring buffer.
3662 *
3663 * First we need to pin the userspace buffer into memory,
3664 * which, most likely it is, because it just referenced it.
3665 * But there's no guarantee that it is. By using get_user_pages_fast()
3666 * and kmap_atomic/kunmap_atomic() we can get access to the
3667 * pages directly. We then write the data directly into the
3668 * ring buffer.
3669 */
3670 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003671
Steven Rostedtd696b582011-09-22 11:50:27 -04003672 /* check if we cross pages */
3673 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
3674 nr_pages = 2;
3675
3676 offset = addr & (PAGE_SIZE - 1);
3677 addr &= PAGE_MASK;
3678
3679 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
3680 if (ret < nr_pages) {
3681 while (--ret >= 0)
3682 put_page(pages[ret]);
3683 written = -EFAULT;
3684 goto out;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003685 }
3686
Steven Rostedtd696b582011-09-22 11:50:27 -04003687 page1 = kmap_atomic(pages[0]);
3688 if (nr_pages == 2)
3689 page2 = kmap_atomic(pages[1]);
3690
3691 local_save_flags(irq_flags);
3692 size = sizeof(*entry) + cnt + 2; /* possible \n added */
3693 buffer = global_trace.buffer;
3694 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3695 irq_flags, preempt_count());
3696 if (!event) {
3697 /* Ring buffer disabled, return as if not open for write */
3698 written = -EBADF;
3699 goto out_unlock;
3700 }
3701
3702 entry = ring_buffer_event_data(event);
3703 entry->ip = _THIS_IP_;
3704
3705 if (nr_pages == 2) {
3706 len = PAGE_SIZE - offset;
3707 memcpy(&entry->buf, page1 + offset, len);
3708 memcpy(&entry->buf[len], page2, cnt - len);
3709 } else
3710 memcpy(&entry->buf, page1 + offset, cnt);
3711
3712 if (entry->buf[cnt - 1] != '\n') {
3713 entry->buf[cnt] = '\n';
3714 entry->buf[cnt + 1] = '\0';
3715 } else
3716 entry->buf[cnt] = '\0';
3717
3718 ring_buffer_unlock_commit(buffer, event);
3719
3720 written = cnt;
3721
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02003722 *fpos += written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003723
Steven Rostedtd696b582011-09-22 11:50:27 -04003724 out_unlock:
3725 if (nr_pages == 2)
3726 kunmap_atomic(page2);
3727 kunmap_atomic(page1);
3728 while (nr_pages > 0)
3729 put_page(pages[--nr_pages]);
3730 out:
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02003731 return written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003732}
3733
Li Zefan13f16d22009-12-08 11:16:11 +08003734static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08003735{
Zhaolei5079f322009-08-25 16:12:56 +08003736 int i;
3737
3738 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08003739 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08003740 "%s%s%s%s", i ? " " : "",
3741 i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3742 i == trace_clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08003743 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08003744
Li Zefan13f16d22009-12-08 11:16:11 +08003745 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08003746}
3747
3748static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3749 size_t cnt, loff_t *fpos)
3750{
3751 char buf[64];
3752 const char *clockstr;
3753 int i;
3754
3755 if (cnt >= sizeof(buf))
3756 return -EINVAL;
3757
3758 if (copy_from_user(&buf, ubuf, cnt))
3759 return -EFAULT;
3760
3761 buf[cnt] = 0;
3762
3763 clockstr = strstrip(buf);
3764
3765 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3766 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3767 break;
3768 }
3769 if (i == ARRAY_SIZE(trace_clocks))
3770 return -EINVAL;
3771
3772 trace_clock_id = i;
3773
3774 mutex_lock(&trace_types_lock);
3775
3776 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3777 if (max_tr.buffer)
3778 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3779
3780 mutex_unlock(&trace_types_lock);
3781
3782 *fpos += cnt;
3783
3784 return cnt;
3785}
3786
Li Zefan13f16d22009-12-08 11:16:11 +08003787static int tracing_clock_open(struct inode *inode, struct file *file)
3788{
3789 if (tracing_disabled)
3790 return -ENODEV;
3791 return single_open(file, tracing_clock_show, NULL);
3792}
3793
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003794static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003795 .open = tracing_open_generic,
3796 .read = tracing_max_lat_read,
3797 .write = tracing_max_lat_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003798 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003799};
3800
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003801static const struct file_operations tracing_ctrl_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003802 .open = tracing_open_generic,
3803 .read = tracing_ctrl_read,
3804 .write = tracing_ctrl_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003805 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003806};
3807
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003808static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003809 .open = tracing_open_generic,
3810 .read = tracing_set_trace_read,
3811 .write = tracing_set_trace_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003812 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003813};
3814
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003815static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003816 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003817 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003818 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003819 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003820 .release = tracing_release_pipe,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003821 .llseek = no_llseek,
Steven Rostedtb3806b42008-05-12 21:20:46 +02003822};
3823
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003824static const struct file_operations tracing_entries_fops = {
Steven Rostedta98a3c32008-05-12 21:20:59 +02003825 .open = tracing_open_generic,
3826 .read = tracing_entries_read,
3827 .write = tracing_entries_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003828 .llseek = generic_file_llseek,
Steven Rostedta98a3c32008-05-12 21:20:59 +02003829};
3830
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07003831static const struct file_operations tracing_total_entries_fops = {
3832 .open = tracing_open_generic,
3833 .read = tracing_total_entries_read,
3834 .llseek = generic_file_llseek,
3835};
3836
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003837static const struct file_operations tracing_free_buffer_fops = {
3838 .write = tracing_free_buffer_write,
3839 .release = tracing_free_buffer_release,
3840};
3841
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003842static const struct file_operations tracing_mark_fops = {
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02003843 .open = tracing_open_generic,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003844 .write = tracing_mark_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003845 .llseek = generic_file_llseek,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003846};
3847
Zhaolei5079f322009-08-25 16:12:56 +08003848static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08003849 .open = tracing_clock_open,
3850 .read = seq_read,
3851 .llseek = seq_lseek,
3852 .release = single_release,
Zhaolei5079f322009-08-25 16:12:56 +08003853 .write = tracing_clock_write,
3854};
3855
Steven Rostedt2cadf912008-12-01 22:20:19 -05003856struct ftrace_buffer_info {
3857 struct trace_array *tr;
3858 void *spare;
3859 int cpu;
3860 unsigned int read;
3861};
3862
3863static int tracing_buffers_open(struct inode *inode, struct file *filp)
3864{
3865 int cpu = (int)(long)inode->i_private;
3866 struct ftrace_buffer_info *info;
3867
3868 if (tracing_disabled)
3869 return -ENODEV;
3870
3871 info = kzalloc(sizeof(*info), GFP_KERNEL);
3872 if (!info)
3873 return -ENOMEM;
3874
3875 info->tr = &global_trace;
3876 info->cpu = cpu;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003877 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003878 /* Force reading ring buffer for first read */
3879 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003880
3881 filp->private_data = info;
3882
Lai Jiangshand1e7e022009-04-02 15:16:56 +08003883 return nonseekable_open(inode, filp);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003884}
3885
3886static ssize_t
3887tracing_buffers_read(struct file *filp, char __user *ubuf,
3888 size_t count, loff_t *ppos)
3889{
3890 struct ftrace_buffer_info *info = filp->private_data;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003891 ssize_t ret;
3892 size_t size;
3893
Steven Rostedt2dc5d122009-03-04 19:10:05 -05003894 if (!count)
3895 return 0;
3896
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003897 if (!info->spare)
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07003898 info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003899 if (!info->spare)
3900 return -ENOMEM;
3901
Steven Rostedt2cadf912008-12-01 22:20:19 -05003902 /* Do we have previous read data to read? */
3903 if (info->read < PAGE_SIZE)
3904 goto read;
3905
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003906 trace_access_lock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003907 ret = ring_buffer_read_page(info->tr->buffer,
3908 &info->spare,
3909 count,
3910 info->cpu, 0);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003911 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003912 if (ret < 0)
3913 return 0;
3914
Steven Rostedt436fc282011-10-14 10:44:25 -04003915 info->read = 0;
3916
Steven Rostedt2cadf912008-12-01 22:20:19 -05003917read:
3918 size = PAGE_SIZE - info->read;
3919 if (size > count)
3920 size = count;
3921
3922 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt2dc5d122009-03-04 19:10:05 -05003923 if (ret == size)
Steven Rostedt2cadf912008-12-01 22:20:19 -05003924 return -EFAULT;
Steven Rostedt2dc5d122009-03-04 19:10:05 -05003925 size -= ret;
3926
Steven Rostedt2cadf912008-12-01 22:20:19 -05003927 *ppos += size;
3928 info->read += size;
3929
3930 return size;
3931}
3932
3933static int tracing_buffers_release(struct inode *inode, struct file *file)
3934{
3935 struct ftrace_buffer_info *info = file->private_data;
3936
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003937 if (info->spare)
3938 ring_buffer_free_read_page(info->tr->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003939 kfree(info);
3940
3941 return 0;
3942}
3943
3944struct buffer_ref {
3945 struct ring_buffer *buffer;
3946 void *page;
3947 int ref;
3948};
3949
3950static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3951 struct pipe_buffer *buf)
3952{
3953 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3954
3955 if (--ref->ref)
3956 return;
3957
3958 ring_buffer_free_read_page(ref->buffer, ref->page);
3959 kfree(ref);
3960 buf->private = 0;
3961}
3962
3963static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3964 struct pipe_buffer *buf)
3965{
3966 return 1;
3967}
3968
3969static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3970 struct pipe_buffer *buf)
3971{
3972 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3973
3974 ref->ref++;
3975}
3976
3977/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08003978static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05003979 .can_merge = 0,
3980 .map = generic_pipe_buf_map,
3981 .unmap = generic_pipe_buf_unmap,
3982 .confirm = generic_pipe_buf_confirm,
3983 .release = buffer_pipe_buf_release,
3984 .steal = buffer_pipe_buf_steal,
3985 .get = buffer_pipe_buf_get,
3986};
3987
3988/*
3989 * Callback from splice_to_pipe(), if we need to release some pages
3990 * at the end of the spd in case we error'ed out in filling the pipe.
3991 */
3992static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3993{
3994 struct buffer_ref *ref =
3995 (struct buffer_ref *)spd->partial[i].private;
3996
3997 if (--ref->ref)
3998 return;
3999
4000 ring_buffer_free_read_page(ref->buffer, ref->page);
4001 kfree(ref);
4002 spd->partial[i].private = 0;
4003}
4004
4005static ssize_t
4006tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4007 struct pipe_inode_info *pipe, size_t len,
4008 unsigned int flags)
4009{
4010 struct ftrace_buffer_info *info = file->private_data;
Jens Axboe35f3d142010-05-20 10:43:18 +02004011 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4012 struct page *pages_def[PIPE_DEF_BUFFERS];
Steven Rostedt2cadf912008-12-01 22:20:19 -05004013 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02004014 .pages = pages_def,
4015 .partial = partial_def,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004016 .flags = flags,
4017 .ops = &buffer_pipe_buf_ops,
4018 .spd_release = buffer_spd_release,
4019 };
4020 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04004021 int entries, size, i;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004022 size_t ret;
4023
Jens Axboe35f3d142010-05-20 10:43:18 +02004024 if (splice_grow_spd(pipe, &spd))
4025 return -ENOMEM;
4026
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004027 if (*ppos & (PAGE_SIZE - 1)) {
4028 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
Jens Axboe35f3d142010-05-20 10:43:18 +02004029 ret = -EINVAL;
4030 goto out;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004031 }
4032
4033 if (len & (PAGE_SIZE - 1)) {
4034 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
Jens Axboe35f3d142010-05-20 10:43:18 +02004035 if (len < PAGE_SIZE) {
4036 ret = -EINVAL;
4037 goto out;
4038 }
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004039 len &= PAGE_MASK;
4040 }
4041
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004042 trace_access_lock(info->cpu);
Steven Rostedt93459c62009-04-29 00:23:13 -04004043 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4044
Jens Axboe35f3d142010-05-20 10:43:18 +02004045 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05004046 struct page *page;
4047 int r;
4048
4049 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4050 if (!ref)
4051 break;
4052
Steven Rostedt7267fa62009-04-29 00:16:21 -04004053 ref->ref = 1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004054 ref->buffer = info->tr->buffer;
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07004055 ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004056 if (!ref->page) {
4057 kfree(ref);
4058 break;
4059 }
4060
4061 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtf2957f12009-04-29 00:26:30 -04004062 len, info->cpu, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004063 if (r < 0) {
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07004064 ring_buffer_free_read_page(ref->buffer, ref->page);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004065 kfree(ref);
4066 break;
4067 }
4068
4069 /*
4070 * zero out any left over data, this is going to
4071 * user land.
4072 */
4073 size = ring_buffer_page_len(ref->page);
4074 if (size < PAGE_SIZE)
4075 memset(ref->page + size, 0, PAGE_SIZE - size);
4076
4077 page = virt_to_page(ref->page);
4078
4079 spd.pages[i] = page;
4080 spd.partial[i].len = PAGE_SIZE;
4081 spd.partial[i].offset = 0;
4082 spd.partial[i].private = (unsigned long)ref;
4083 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004084 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04004085
4086 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004087 }
4088
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004089 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004090 spd.nr_pages = i;
4091
4092 /* did we read anything? */
4093 if (!spd.nr_pages) {
4094 if (flags & SPLICE_F_NONBLOCK)
4095 ret = -EAGAIN;
4096 else
4097 ret = 0;
4098 /* TODO: block */
Jens Axboe35f3d142010-05-20 10:43:18 +02004099 goto out;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004100 }
4101
4102 ret = splice_to_pipe(pipe, &spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02004103 splice_shrink_spd(pipe, &spd);
4104out:
Steven Rostedt2cadf912008-12-01 22:20:19 -05004105 return ret;
4106}
4107
4108static const struct file_operations tracing_buffers_fops = {
4109 .open = tracing_buffers_open,
4110 .read = tracing_buffers_read,
4111 .release = tracing_buffers_release,
4112 .splice_read = tracing_buffers_splice_read,
4113 .llseek = no_llseek,
4114};
4115
Steven Rostedtc8d77182009-04-29 18:03:45 -04004116static ssize_t
4117tracing_stats_read(struct file *filp, char __user *ubuf,
4118 size_t count, loff_t *ppos)
4119{
4120 unsigned long cpu = (unsigned long)filp->private_data;
4121 struct trace_array *tr = &global_trace;
4122 struct trace_seq *s;
4123 unsigned long cnt;
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07004124 unsigned long long t;
4125 unsigned long usec_rem;
Steven Rostedtc8d77182009-04-29 18:03:45 -04004126
Li Zefane4f2d102009-06-15 10:57:28 +08004127 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04004128 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01004129 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04004130
4131 trace_seq_init(s);
4132
4133 cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4134 trace_seq_printf(s, "entries: %ld\n", cnt);
4135
4136 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4137 trace_seq_printf(s, "overrun: %ld\n", cnt);
4138
4139 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4140 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4141
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07004142 cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4143 trace_seq_printf(s, "bytes: %ld\n", cnt);
4144
4145 t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4146 usec_rem = do_div(t, USEC_PER_SEC);
4147 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);
4148
4149 t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4150 usec_rem = do_div(t, USEC_PER_SEC);
4151 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4152
Steven Rostedtc8d77182009-04-29 18:03:45 -04004153 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4154
4155 kfree(s);
4156
4157 return count;
4158}
4159
4160static const struct file_operations tracing_stats_fops = {
4161 .open = tracing_open_generic,
4162 .read = tracing_stats_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004163 .llseek = generic_file_llseek,
Steven Rostedtc8d77182009-04-29 18:03:45 -04004164};
4165
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004166#ifdef CONFIG_DYNAMIC_FTRACE
4167
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004168int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004169{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004170 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004171}
4172
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004173static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004174tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004175 size_t cnt, loff_t *ppos)
4176{
Steven Rostedta26a2a22008-10-31 00:03:22 -04004177 static char ftrace_dyn_info_buffer[1024];
4178 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004179 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004180 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04004181 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004182 int r;
4183
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004184 mutex_lock(&dyn_info_mutex);
4185 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004186
Steven Rostedta26a2a22008-10-31 00:03:22 -04004187 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004188 buf[r++] = '\n';
4189
4190 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4191
4192 mutex_unlock(&dyn_info_mutex);
4193
4194 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004195}
4196
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004197static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004198 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004199 .read = tracing_read_dyn_info,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004200 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004201};
4202#endif
4203
4204static struct dentry *d_tracer;
4205
4206struct dentry *tracing_init_dentry(void)
4207{
4208 static int once;
4209
4210 if (d_tracer)
4211 return d_tracer;
4212
Frederic Weisbecker3e1f60b2009-03-22 23:10:45 +01004213 if (!debugfs_initialized())
4214 return NULL;
4215
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004216 d_tracer = debugfs_create_dir("tracing", NULL);
4217
4218 if (!d_tracer && !once) {
4219 once = 1;
4220 pr_warning("Could not create debugfs directory 'tracing'\n");
4221 return NULL;
4222 }
4223
4224 return d_tracer;
4225}
4226
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004227static struct dentry *d_percpu;
4228
4229struct dentry *tracing_dentry_percpu(void)
4230{
4231 static int once;
4232 struct dentry *d_tracer;
4233
4234 if (d_percpu)
4235 return d_percpu;
4236
4237 d_tracer = tracing_init_dentry();
4238
4239 if (!d_tracer)
4240 return NULL;
4241
4242 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4243
4244 if (!d_percpu && !once) {
4245 once = 1;
4246 pr_warning("Could not create debugfs directory 'per_cpu'\n");
4247 return NULL;
4248 }
4249
4250 return d_percpu;
4251}
4252
4253static void tracing_init_debugfs_percpu(long cpu)
4254{
4255 struct dentry *d_percpu = tracing_dentry_percpu();
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004256 struct dentry *d_cpu;
Steven Rostedtdd49a382010-10-20 21:51:26 -04004257 char cpu_dir[30]; /* 30 characters should be more than enough */
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004258
Steven Rostedtdd49a382010-10-20 21:51:26 -04004259 snprintf(cpu_dir, 30, "cpu%ld", cpu);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004260 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4261 if (!d_cpu) {
4262 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4263 return;
4264 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004265
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004266 /* per cpu trace_pipe */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004267 trace_create_file("trace_pipe", 0444, d_cpu,
4268 (void *) cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004269
4270 /* per cpu trace */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004271 trace_create_file("trace", 0644, d_cpu,
4272 (void *) cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04004273
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004274 trace_create_file("trace_pipe_raw", 0444, d_cpu,
4275 (void *) cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04004276
4277 trace_create_file("stats", 0444, d_cpu,
4278 (void *) cpu, &tracing_stats_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004279}
4280
Steven Rostedt60a11772008-05-12 21:20:44 +02004281#ifdef CONFIG_FTRACE_SELFTEST
4282/* Let selftest have access to static functions in this file */
4283#include "trace_selftest.c"
4284#endif
4285
Steven Rostedt577b7852009-02-26 23:43:05 -05004286struct trace_option_dentry {
4287 struct tracer_opt *opt;
4288 struct tracer_flags *flags;
4289 struct dentry *entry;
4290};
4291
4292static ssize_t
4293trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4294 loff_t *ppos)
4295{
4296 struct trace_option_dentry *topt = filp->private_data;
4297 char *buf;
4298
4299 if (topt->flags->val & topt->opt->bit)
4300 buf = "1\n";
4301 else
4302 buf = "0\n";
4303
4304 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4305}
4306
4307static ssize_t
4308trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4309 loff_t *ppos)
4310{
4311 struct trace_option_dentry *topt = filp->private_data;
4312 unsigned long val;
Steven Rostedt577b7852009-02-26 23:43:05 -05004313 int ret;
4314
Peter Huewe22fe9b52011-06-07 21:58:27 +02004315 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4316 if (ret)
Steven Rostedt577b7852009-02-26 23:43:05 -05004317 return ret;
4318
Li Zefan8d18eaa2009-12-08 11:17:06 +08004319 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05004320 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08004321
4322 if (!!(topt->flags->val & topt->opt->bit) != val) {
4323 mutex_lock(&trace_types_lock);
4324 ret = __set_tracer_option(current_trace, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05004325 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08004326 mutex_unlock(&trace_types_lock);
4327 if (ret)
4328 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05004329 }
4330
4331 *ppos += cnt;
4332
4333 return cnt;
4334}
4335
4336
4337static const struct file_operations trace_options_fops = {
4338 .open = tracing_open_generic,
4339 .read = trace_options_read,
4340 .write = trace_options_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004341 .llseek = generic_file_llseek,
Steven Rostedt577b7852009-02-26 23:43:05 -05004342};
4343
Steven Rostedta8259072009-02-26 22:19:12 -05004344static ssize_t
4345trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4346 loff_t *ppos)
4347{
4348 long index = (long)filp->private_data;
4349 char *buf;
4350
4351 if (trace_flags & (1 << index))
4352 buf = "1\n";
4353 else
4354 buf = "0\n";
4355
4356 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4357}
4358
4359static ssize_t
4360trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4361 loff_t *ppos)
4362{
4363 long index = (long)filp->private_data;
Steven Rostedta8259072009-02-26 22:19:12 -05004364 unsigned long val;
4365 int ret;
4366
Peter Huewe22fe9b52011-06-07 21:58:27 +02004367 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4368 if (ret)
Steven Rostedta8259072009-02-26 22:19:12 -05004369 return ret;
4370
Zhaoleif2d84b62009-08-07 18:55:48 +08004371 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05004372 return -EINVAL;
Zhaoleif2d84b62009-08-07 18:55:48 +08004373 set_tracer_flags(1 << index, val);
Steven Rostedta8259072009-02-26 22:19:12 -05004374
4375 *ppos += cnt;
4376
4377 return cnt;
4378}
4379
Steven Rostedta8259072009-02-26 22:19:12 -05004380static const struct file_operations trace_options_core_fops = {
4381 .open = tracing_open_generic,
4382 .read = trace_options_core_read,
4383 .write = trace_options_core_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004384 .llseek = generic_file_llseek,
Steven Rostedta8259072009-02-26 22:19:12 -05004385};
4386
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004387struct dentry *trace_create_file(const char *name,
4388 mode_t mode,
4389 struct dentry *parent,
4390 void *data,
4391 const struct file_operations *fops)
4392{
4393 struct dentry *ret;
4394
4395 ret = debugfs_create_file(name, mode, parent, data, fops);
4396 if (!ret)
4397 pr_warning("Could not create debugfs '%s' entry\n", name);
4398
4399 return ret;
4400}
4401
4402
Steven Rostedta8259072009-02-26 22:19:12 -05004403static struct dentry *trace_options_init_dentry(void)
4404{
4405 struct dentry *d_tracer;
4406 static struct dentry *t_options;
4407
4408 if (t_options)
4409 return t_options;
4410
4411 d_tracer = tracing_init_dentry();
4412 if (!d_tracer)
4413 return NULL;
4414
4415 t_options = debugfs_create_dir("options", d_tracer);
4416 if (!t_options) {
4417 pr_warning("Could not create debugfs directory 'options'\n");
4418 return NULL;
4419 }
4420
4421 return t_options;
4422}
4423
Steven Rostedt577b7852009-02-26 23:43:05 -05004424static void
4425create_trace_option_file(struct trace_option_dentry *topt,
4426 struct tracer_flags *flags,
4427 struct tracer_opt *opt)
4428{
4429 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05004430
4431 t_options = trace_options_init_dentry();
4432 if (!t_options)
4433 return;
4434
4435 topt->flags = flags;
4436 topt->opt = opt;
4437
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004438 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05004439 &trace_options_fops);
4440
Steven Rostedt577b7852009-02-26 23:43:05 -05004441}
4442
4443static struct trace_option_dentry *
4444create_trace_option_files(struct tracer *tracer)
4445{
4446 struct trace_option_dentry *topts;
4447 struct tracer_flags *flags;
4448 struct tracer_opt *opts;
4449 int cnt;
4450
4451 if (!tracer)
4452 return NULL;
4453
4454 flags = tracer->flags;
4455
4456 if (!flags || !flags->opts)
4457 return NULL;
4458
4459 opts = flags->opts;
4460
4461 for (cnt = 0; opts[cnt].name; cnt++)
4462 ;
4463
Steven Rostedt0cfe8242009-02-27 10:51:10 -05004464 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05004465 if (!topts)
4466 return NULL;
4467
4468 for (cnt = 0; opts[cnt].name; cnt++)
4469 create_trace_option_file(&topts[cnt], flags,
4470 &opts[cnt]);
4471
4472 return topts;
4473}
4474
4475static void
4476destroy_trace_option_files(struct trace_option_dentry *topts)
4477{
4478 int cnt;
4479
4480 if (!topts)
4481 return;
4482
4483 for (cnt = 0; topts[cnt].opt; cnt++) {
4484 if (topts[cnt].entry)
4485 debugfs_remove(topts[cnt].entry);
4486 }
4487
4488 kfree(topts);
4489}
4490
Steven Rostedta8259072009-02-26 22:19:12 -05004491static struct dentry *
4492create_trace_option_core_file(const char *option, long index)
4493{
4494 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004495
4496 t_options = trace_options_init_dentry();
4497 if (!t_options)
4498 return NULL;
4499
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004500 return trace_create_file(option, 0644, t_options, (void *)index,
Steven Rostedta8259072009-02-26 22:19:12 -05004501 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05004502}
4503
4504static __init void create_trace_options_dir(void)
4505{
4506 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004507 int i;
4508
4509 t_options = trace_options_init_dentry();
4510 if (!t_options)
4511 return;
4512
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004513 for (i = 0; trace_options[i]; i++)
4514 create_trace_option_core_file(trace_options[i], i);
Steven Rostedta8259072009-02-26 22:19:12 -05004515}
4516
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004517static __init int tracer_init_debugfs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004518{
4519 struct dentry *d_tracer;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004520 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004521
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004522 trace_access_lock_init();
4523
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004524 d_tracer = tracing_init_dentry();
4525
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004526 trace_create_file("tracing_enabled", 0644, d_tracer,
4527 &global_trace, &tracing_ctrl_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004528
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004529 trace_create_file("trace_options", 0644, d_tracer,
4530 NULL, &tracing_iter_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004531
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004532 trace_create_file("tracing_cpumask", 0644, d_tracer,
4533 NULL, &tracing_cpumask_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05004534
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004535 trace_create_file("trace", 0644, d_tracer,
4536 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
Ingo Molnarc7078de2008-05-12 21:20:52 +02004537
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004538 trace_create_file("available_tracers", 0444, d_tracer,
4539 &global_trace, &show_traces_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004540
Li Zefan339ae5d2009-04-17 10:34:30 +08004541 trace_create_file("current_tracer", 0644, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004542 &global_trace, &set_tracer_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004543
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04004544#ifdef CONFIG_TRACER_MAX_TRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004545 trace_create_file("tracing_max_latency", 0644, d_tracer,
4546 &tracing_max_latency, &tracing_max_lat_fops);
Tim Bird0e950172010-02-25 15:36:43 -08004547#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004548
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004549 trace_create_file("tracing_thresh", 0644, d_tracer,
4550 &tracing_thresh, &tracing_max_lat_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004551
Li Zefan339ae5d2009-04-17 10:34:30 +08004552 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004553 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02004554
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004555 trace_create_file("trace_pipe", 0444, d_tracer,
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004556 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004557
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004558 trace_create_file("buffer_size_kb", 0644, d_tracer,
4559 &global_trace, &tracing_entries_fops);
Steven Rostedta98a3c32008-05-12 21:20:59 +02004560
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004561 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4562 &global_trace, &tracing_total_entries_fops);
4563
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004564 trace_create_file("free_buffer", 0644, d_tracer,
4565 &global_trace, &tracing_free_buffer_fops);
4566
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004567 trace_create_file("trace_marker", 0220, d_tracer,
4568 NULL, &tracing_mark_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004569
Avadh Patel69abe6a2009-04-10 16:04:48 -04004570 trace_create_file("saved_cmdlines", 0444, d_tracer,
4571 NULL, &tracing_saved_cmdlines_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004572
Zhaolei5079f322009-08-25 16:12:56 +08004573 trace_create_file("trace_clock", 0644, d_tracer, NULL,
4574 &trace_clock_fops);
4575
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004576#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004577 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4578 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004579#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004580
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004581 create_trace_options_dir();
4582
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004583 for_each_tracing_cpu(cpu)
4584 tracing_init_debugfs_percpu(cpu);
4585
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004586 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004587}
4588
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004589static int trace_panic_handler(struct notifier_block *this,
4590 unsigned long event, void *unused)
4591{
Steven Rostedt944ac422008-10-23 19:26:08 -04004592 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004593 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004594 return NOTIFY_OK;
4595}
4596
4597static struct notifier_block trace_panic_notifier = {
4598 .notifier_call = trace_panic_handler,
4599 .next = NULL,
4600 .priority = 150 /* priority: INT_MAX >= x >= 0 */
4601};
4602
4603static int trace_die_handler(struct notifier_block *self,
4604 unsigned long val,
4605 void *data)
4606{
4607 switch (val) {
4608 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04004609 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004610 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004611 break;
4612 default:
4613 break;
4614 }
4615 return NOTIFY_OK;
4616}
4617
4618static struct notifier_block trace_die_notifier = {
4619 .notifier_call = trace_die_handler,
4620 .priority = 200
4621};
4622
4623/*
4624 * printk is set to max of 1024, we really don't need it that big.
4625 * Nothing should be printing 1000 characters anyway.
4626 */
4627#define TRACE_MAX_PRINT 1000
4628
4629/*
4630 * Define here KERN_TRACE so that we have one place to modify
4631 * it if we decide to change what log level the ftrace dump
4632 * should be at.
4633 */
Steven Rostedt428aee12009-01-14 12:24:42 -05004634#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004635
Jason Wessel955b61e2010-08-05 09:22:23 -05004636void
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004637trace_printk_seq(struct trace_seq *s)
4638{
4639 /* Probably should print a warning here. */
4640 if (s->len >= 1000)
4641 s->len = 1000;
4642
4643 /* should be zero ended, but we are paranoid. */
4644 s->buffer[s->len] = 0;
4645
4646 printk(KERN_TRACE "%s", s->buffer);
4647
Steven Rostedtf9520752009-03-02 14:04:40 -05004648 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004649}
4650
Jason Wessel955b61e2010-08-05 09:22:23 -05004651void trace_init_global_iter(struct trace_iterator *iter)
4652{
4653 iter->tr = &global_trace;
4654 iter->trace = current_trace;
4655 iter->cpu_file = TRACE_PIPE_ALL_CPU;
4656}
4657
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004658static void
4659__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004660{
Thomas Gleixner445c8952009-12-02 19:49:50 +01004661 static arch_spinlock_t ftrace_dump_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01004662 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004663 /* use static because iter can be a bit big for the stack */
4664 static struct trace_iterator iter;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004665 unsigned int old_userobj;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004666 static int dump_ran;
Steven Rostedtd7690412008-10-01 00:29:53 -04004667 unsigned long flags;
4668 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004669
4670 /* only one dump */
Steven Rostedtcd891ae2009-04-28 11:39:34 -04004671 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01004672 arch_spin_lock(&ftrace_dump_lock);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004673 if (dump_ran)
4674 goto out;
4675
4676 dump_ran = 1;
4677
Steven Rostedt0ee6b6c2009-01-14 14:50:19 -05004678 tracing_off();
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004679
Steven Rostedte0a413f2011-09-29 21:26:16 -04004680 /* Did function tracer already get disabled? */
4681 if (ftrace_is_dead()) {
4682 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
4683 printk("# MAY BE MISSING FUNCTION EVENTS\n");
4684 }
4685
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004686 if (disable_tracing)
4687 ftrace_kill();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004688
Jason Wessel955b61e2010-08-05 09:22:23 -05004689 trace_init_global_iter(&iter);
4690
Steven Rostedtd7690412008-10-01 00:29:53 -04004691 for_each_tracing_cpu(cpu) {
Jason Wessel955b61e2010-08-05 09:22:23 -05004692 atomic_inc(&iter.tr->data[cpu]->disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -04004693 }
4694
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004695 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4696
Török Edwinb54d3de2008-11-22 13:28:48 +02004697 /* don't look at user memory in panic mode */
4698 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4699
Steven Rostedte543ad72009-03-04 18:20:36 -05004700 /* Simulate the iterator */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004701 iter.tr = &global_trace;
4702 iter.trace = current_trace;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004703
4704 switch (oops_dump_mode) {
4705 case DUMP_ALL:
4706 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4707 break;
4708 case DUMP_ORIG:
4709 iter.cpu_file = raw_smp_processor_id();
4710 break;
4711 case DUMP_NONE:
4712 goto out_enable;
4713 default:
4714 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
4715 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4716 }
4717
4718 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004719
4720 /*
4721 * We need to stop all tracing on all CPUS to read the
4722 * the next buffer. This is a bit expensive, but is
4723 * not done often. We fill all what we can read,
4724 * and then release the locks again.
4725 */
4726
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004727 while (!trace_empty(&iter)) {
4728
4729 if (!cnt)
4730 printk(KERN_TRACE "---------------------------------\n");
4731
4732 cnt++;
4733
4734 /* reset all but tr, trace, and overruns */
4735 memset(&iter.seq, 0,
4736 sizeof(struct trace_iterator) -
4737 offsetof(struct trace_iterator, seq));
4738 iter.iter_flags |= TRACE_FILE_LAT_FMT;
4739 iter.pos = -1;
4740
Jason Wessel955b61e2010-08-05 09:22:23 -05004741 if (trace_find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08004742 int ret;
4743
4744 ret = print_trace_line(&iter);
4745 if (ret != TRACE_TYPE_NO_CONSUME)
4746 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004747 }
4748
4749 trace_printk_seq(&iter.seq);
4750 }
4751
4752 if (!cnt)
4753 printk(KERN_TRACE " (ftrace buffer empty)\n");
4754 else
4755 printk(KERN_TRACE "---------------------------------\n");
4756
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004757 out_enable:
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004758 /* Re-enable tracing if requested */
4759 if (!disable_tracing) {
4760 trace_flags |= old_userobj;
4761
4762 for_each_tracing_cpu(cpu) {
Jason Wessel955b61e2010-08-05 09:22:23 -05004763 atomic_dec(&iter.tr->data[cpu]->disabled);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004764 }
4765 tracing_on();
4766 }
4767
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004768 out:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01004769 arch_spin_unlock(&ftrace_dump_lock);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04004770 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004771}
4772
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004773/* By default: disable tracing after the dump */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004774void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004775{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004776 __ftrace_dump(true, oops_dump_mode);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004777}
4778
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004779__init static int tracer_alloc_buffers(void)
4780{
Steven Rostedt73c51622009-03-11 13:42:01 -04004781 int ring_buf_size;
David Sharp750912f2010-12-08 13:46:47 -08004782 enum ring_buffer_flags rb_flags;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004783 int i;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304784 int ret = -ENOMEM;
4785
David Sharp750912f2010-12-08 13:46:47 -08004786
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304787 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4788 goto out;
4789
4790 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4791 goto out_free_buffer_mask;
4792
Steven Rostedt73c51622009-03-11 13:42:01 -04004793 /* To save memory, keep the ring buffer size to its minimum */
4794 if (ring_buffer_expanded)
4795 ring_buf_size = trace_buf_size;
4796 else
4797 ring_buf_size = 1;
4798
David Sharp750912f2010-12-08 13:46:47 -08004799 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
4800
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304801 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4802 cpumask_copy(tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004803
Steven Rostedtab464282008-05-12 21:21:00 +02004804 /* TODO: make the number of buffers hot pluggable with CPUS */
David Sharp750912f2010-12-08 13:46:47 -08004805 global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004806 if (!global_trace.buffer) {
4807 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4808 WARN_ON(1);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304809 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004810 }
4811 global_trace.entries = ring_buffer_size(global_trace.buffer);
4812
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304813
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004814#ifdef CONFIG_TRACER_MAX_TRACE
David Sharp750912f2010-12-08 13:46:47 -08004815 max_tr.buffer = ring_buffer_alloc(1, rb_flags);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004816 if (!max_tr.buffer) {
4817 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4818 WARN_ON(1);
4819 ring_buffer_free(global_trace.buffer);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304820 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004821 }
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004822 max_tr.entries = 1;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004823#endif
4824
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02004825 /* Allocate the first page for all buffers */
Steven Rostedtab464282008-05-12 21:21:00 +02004826 for_each_tracing_cpu(i) {
jolsa@redhat.com566b0aa2009-07-16 21:44:26 +02004827 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
Tejun Heo9705f692009-10-29 22:34:13 +09004828 max_tr.data[i] = &per_cpu(max_tr_data, i);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004829 }
4830
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004831 trace_init_cmdlines();
4832
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02004833 register_tracer(&nop_trace);
Steven Rostedt79fb0762009-02-02 21:38:33 -05004834 current_trace = &nop_trace;
Steven Rostedt60a11772008-05-12 21:20:44 +02004835 /* All seems OK, enable tracing */
4836 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004837
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004838 atomic_notifier_chain_register(&panic_notifier_list,
4839 &trace_panic_notifier);
4840
4841 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01004842
4843 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004844
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304845out_free_cpumask:
4846 free_cpumask_var(tracing_cpumask);
4847out_free_buffer_mask:
4848 free_cpumask_var(tracing_buffer_mask);
4849out:
4850 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004851}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05004852
4853__init static int clear_boot_tracer(void)
4854{
4855 /*
4856 * The default tracer at boot buffer is an init section.
4857 * This function is called in lateinit. If we did not
4858 * find the boot tracer, then clear it out, to prevent
4859 * later registration from accessing the buffer that is
4860 * about to be freed.
4861 */
4862 if (!default_bootup_tracer)
4863 return 0;
4864
4865 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4866 default_bootup_tracer);
4867 default_bootup_tracer = NULL;
4868
4869 return 0;
4870}
4871
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004872early_initcall(tracer_alloc_buffers);
4873fs_initcall(tracer_init_debugfs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05004874late_initcall(clear_boot_tracer);