blob: 756d7283318bd24e4998ca93d72d1edf62f8bef1 [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Alexey Dobriyan405f5572009-07-11 22:08:37 +040020#include <linux/smp_lock.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040021#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050022#include <linux/irqflags.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020023#include <linux/debugfs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020024#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020025#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050028#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020029#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050032#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040033#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010034#include <linux/string.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080035#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090036#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020037#include <linux/ctype.h>
38#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020039#include <linux/poll.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020040#include <linux/fs.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020041
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020042#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050043#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020044
Steven Rostedt3928a8a2008-09-29 23:02:41 -040045#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
46
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +010047/*
Steven Rostedt73c51622009-03-11 13:42:01 -040048 * On boot up, the ring buffer is set to the minimum size, so that
49 * we do not waste memory on systems that are not using tracing.
50 */
Li Zefan020e5f82009-07-01 10:47:05 +080051int ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040052
53/*
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +010054 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010055 * A selftest will lurk into the ring-buffer to count the
56 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010057 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010058 * at the same time, giving false positive or negative results.
59 */
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +010060static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010061
Steven Rostedtb2821ae2009-02-02 21:38:32 -050062/*
63 * If a tracer is running, we do not want to run SELFTEST.
64 */
Li Zefan020e5f82009-07-01 10:47:05 +080065bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050066
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010067/* For tracers that don't implement custom flags */
68static struct tracer_opt dummy_tracer_opt[] = {
69 { }
70};
71
72static struct tracer_flags dummy_tracer_flags = {
73 .val = 0,
74 .opts = dummy_tracer_opt
75};
76
77static int dummy_set_flag(u32 old_flags, u32 bit, int set)
78{
79 return 0;
80}
Steven Rostedt0f048702008-11-05 16:05:44 -050081
82/*
83 * Kill all tracing for good (never come back).
84 * It is initialized to 1 but will turn to zero if the initialization
85 * of the tracer is successful. But that is the only place that sets
86 * this back to zero.
87 */
Hannes Eder4fd27352009-02-10 19:44:12 +010088static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -050089
Christoph Lameter9288f992009-10-07 19:17:45 -040090DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040091
92static inline void ftrace_disable_cpu(void)
93{
94 preempt_disable();
Rusty Russelldd17c8f2009-10-29 22:34:15 +090095 __this_cpu_inc(ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040096}
97
98static inline void ftrace_enable_cpu(void)
99{
Rusty Russelldd17c8f2009-10-29 22:34:15 +0900100 __this_cpu_dec(ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -0400101 preempt_enable();
102}
103
Rusty Russell9e01c1b2009-01-01 10:12:22 +1030104static cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +0200105
106#define for_each_tracing_cpu(cpu) \
Rusty Russell9e01c1b2009-01-01 10:12:22 +1030107 for_each_cpu(cpu, tracing_buffer_mask)
Steven Rostedtab464282008-05-12 21:21:00 +0200108
Steven Rostedt944ac422008-10-23 19:26:08 -0400109/*
110 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111 *
112 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
113 * is set, then ftrace_dump is called. This will output the contents
114 * of the ftrace buffers to the console. This is very useful for
115 * capturing traces that lead to crashes and outputing it to a
116 * serial console.
117 *
118 * It is default off, but you can enable it with either specifying
119 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200120 * /proc/sys/kernel/ftrace_dump_on_oops
121 * Set 1 if you want to dump buffers of all CPUs
122 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400123 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200124
125enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400126
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500127static int tracing_set_tracer(const char *buf);
128
Li Zefanee6c2c12009-09-18 14:06:47 +0800129#define MAX_TRACER_SIZE 100
130static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500131static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100132
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200133static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100134{
Li Zefanee6c2c12009-09-18 14:06:47 +0800135 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500136 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400137 /* We are using ftrace early, expand it */
138 ring_buffer_expanded = 1;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100139 return 1;
140}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200141__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100142
Steven Rostedt944ac422008-10-23 19:26:08 -0400143static int __init set_ftrace_dump_on_oops(char *str)
144{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200145 if (*str++ != '=' || !*str) {
146 ftrace_dump_on_oops = DUMP_ALL;
147 return 1;
148 }
149
150 if (!strcmp("orig_cpu", str)) {
151 ftrace_dump_on_oops = DUMP_ORIG;
152 return 1;
153 }
154
155 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400156}
157__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200158
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800159unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200160{
161 nsec += 500;
162 do_div(nsec, 1000);
163 return nsec;
164}
165
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200166/*
167 * The global_trace is the descriptor that holds the tracing
168 * buffers for the live tracing. For each CPU, it contains
169 * a link list of pages that will store trace entries. The
170 * page descriptor of the pages in the memory is used to hold
171 * the link list by linking the lru item in the page descriptor
172 * to each of the pages in the buffer per CPU.
173 *
174 * For each active CPU there is a data field that holds the
175 * pages for the buffer for that CPU. Each CPU has the same number
176 * of pages allocated for its buffer.
177 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200178static struct trace_array global_trace;
179
180static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
181
Steven Rostedte77405a2009-09-02 14:17:06 -0400182int filter_current_check_discard(struct ring_buffer *buffer,
183 struct ftrace_event_call *call, void *rec,
Tom Zanussieb02ce02009-04-08 03:15:54 -0500184 struct ring_buffer_event *event)
185{
Steven Rostedte77405a2009-09-02 14:17:06 -0400186 return filter_check_discard(call, rec, buffer, event);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500187}
Steven Rostedt17c873e2009-04-10 18:12:50 -0400188EXPORT_SYMBOL_GPL(filter_current_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500189
Steven Rostedt37886f62009-03-17 17:22:06 -0400190cycle_t ftrace_now(int cpu)
191{
192 u64 ts;
193
194 /* Early boot up does not have a buffer yet */
195 if (!global_trace.buffer)
196 return trace_clock_local();
197
198 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
199 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
200
201 return ts;
202}
203
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200204/*
205 * The max_tr is used to snapshot the global_trace when a maximum
206 * latency is reached. Some tracers will use this to store a maximum
207 * trace while it continues examining live traces.
208 *
209 * The buffers for the max_tr are set up the same as the global_trace.
210 * When a snapshot is taken, the link list of the max_tr is swapped
211 * with the link list of the global_trace and the buffers are reset for
212 * the global_trace so the tracing can continue.
213 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200214static struct trace_array max_tr;
215
Tejun Heo9705f692009-10-29 22:34:13 +0900216static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200217
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200218/* tracer_enabled is used to toggle activation of a tracer */
Steven Rostedt26994ea2008-05-12 21:20:48 +0200219static int tracer_enabled = 1;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200220
Steven Rostedt90369902008-11-05 16:05:44 -0500221/**
222 * tracing_is_enabled - return tracer_enabled status
223 *
224 * This function is used by other tracers to know the status
225 * of the tracer_enabled flag. Tracers may use this function
226 * to know if it should enable their features when starting
227 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
228 */
229int tracing_is_enabled(void)
230{
231 return tracer_enabled;
232}
233
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200234/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400235 * trace_buf_size is the size in bytes that is allocated
236 * for a buffer. Note, the number of bytes is always rounded
237 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400238 *
239 * This number is purposely set to a low number of 16384.
240 * If the dump on oops happens, it will be much appreciated
241 * to not have to wait for all that output. Anyway this can be
242 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200243 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400244#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400245
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400246static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200247
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200248/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200249static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200250
251/* current_trace points to the tracer that is currently active */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200252static struct tracer *current_trace __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200253
254/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200255 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200256 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200257static DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200258
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800259/*
260 * serialize the access of the ring buffer
261 *
262 * ring buffer serializes readers, but it is low level protection.
263 * The validity of the events (which returns by ring_buffer_peek() ..etc)
264 * are not protected by ring buffer.
265 *
266 * The content of events may become garbage if we allow other process consumes
267 * these events concurrently:
268 * A) the page of the consumed events may become a normal page
269 * (not reader page) in ring buffer, and this page will be rewrited
270 * by events producer.
271 * B) The page of the consumed events may become a page for splice_read,
272 * and this page will be returned to system.
273 *
274 * These primitives allow multi process access to different cpu ring buffer
275 * concurrently.
276 *
277 * These primitives don't distinguish read-only and read-consume access.
278 * Multi read-only access are also serialized.
279 */
280
281#ifdef CONFIG_SMP
282static DECLARE_RWSEM(all_cpu_access_lock);
283static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
284
285static inline void trace_access_lock(int cpu)
286{
287 if (cpu == TRACE_PIPE_ALL_CPU) {
288 /* gain it for accessing the whole ring buffer. */
289 down_write(&all_cpu_access_lock);
290 } else {
291 /* gain it for accessing a cpu ring buffer. */
292
293 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
294 down_read(&all_cpu_access_lock);
295
296 /* Secondly block other access to this @cpu ring buffer. */
297 mutex_lock(&per_cpu(cpu_access_lock, cpu));
298 }
299}
300
301static inline void trace_access_unlock(int cpu)
302{
303 if (cpu == TRACE_PIPE_ALL_CPU) {
304 up_write(&all_cpu_access_lock);
305 } else {
306 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
307 up_read(&all_cpu_access_lock);
308 }
309}
310
311static inline void trace_access_lock_init(void)
312{
313 int cpu;
314
315 for_each_possible_cpu(cpu)
316 mutex_init(&per_cpu(cpu_access_lock, cpu));
317}
318
319#else
320
321static DEFINE_MUTEX(access_lock);
322
323static inline void trace_access_lock(int cpu)
324{
325 (void)cpu;
326 mutex_lock(&access_lock);
327}
328
329static inline void trace_access_unlock(int cpu)
330{
331 (void)cpu;
332 mutex_unlock(&access_lock);
333}
334
335static inline void trace_access_lock_init(void)
336{
337}
338
339#endif
340
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200341/* trace_wait is a waitqueue for tasks blocked on trace_poll */
Ingo Molnar4e655512008-05-12 21:20:52 +0200342static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
343
Steven Rostedtee6bce52008-11-12 17:52:37 -0500344/* trace_flags holds trace_options default values */
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500345unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
Steven Rostedta2a16d62009-03-24 23:17:58 -0400346 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
347 TRACE_ITER_GRAPH_TIME;
Ingo Molnar4e655512008-05-12 21:20:52 +0200348
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400349static int trace_stop_count;
350static DEFINE_SPINLOCK(tracing_start_lock);
351
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200352/**
353 * trace_wake_up - wake up tasks waiting for trace input
354 *
355 * Simply wakes up any task that is blocked on the trace_wait
356 * queue. These is used with trace_poll for tasks polling the trace.
357 */
Ingo Molnar4e655512008-05-12 21:20:52 +0200358void trace_wake_up(void)
359{
Andrew Morton89f19f02009-09-19 11:55:44 -0700360 int cpu;
361
362 if (trace_flags & TRACE_ITER_BLOCK)
363 return;
Ingo Molnar017730c2008-05-12 21:20:52 +0200364 /*
365 * The runqueue_is_locked() can fail, but this is the best we
366 * have for now:
367 */
Andrew Morton89f19f02009-09-19 11:55:44 -0700368 cpu = get_cpu();
369 if (!runqueue_is_locked(cpu))
Ingo Molnar4e655512008-05-12 21:20:52 +0200370 wake_up(&trace_wait);
Andrew Morton89f19f02009-09-19 11:55:44 -0700371 put_cpu();
Ingo Molnar4e655512008-05-12 21:20:52 +0200372}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200373
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400374static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200375{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400376 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200377
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200378 if (!str)
379 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800380 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200381 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800382 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200383 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400384 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200385 return 1;
386}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400387__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200388
Tim Bird0e950172010-02-25 15:36:43 -0800389static int __init set_tracing_thresh(char *str)
390{
391 unsigned long threshhold;
392 int ret;
393
394 if (!str)
395 return 0;
396 ret = strict_strtoul(str, 0, &threshhold);
397 if (ret < 0)
398 return 0;
399 tracing_thresh = threshhold * 1000;
400 return 1;
401}
402__setup("tracing_thresh=", set_tracing_thresh);
403
Steven Rostedt57f50be2008-05-12 21:20:44 +0200404unsigned long nsecs_to_usecs(unsigned long nsecs)
405{
406 return nsecs / 1000;
407}
408
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200409/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200410static const char *trace_options[] = {
411 "print-parent",
412 "sym-offset",
413 "sym-addr",
414 "verbose",
Ingo Molnarf9896bf2008-05-12 21:20:47 +0200415 "raw",
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +0200416 "hex",
Ingo Molnarcb0f12a2008-05-12 21:20:47 +0200417 "bin",
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +0200418 "block",
Ingo Molnar86387f72008-05-12 21:20:51 +0200419 "stacktrace",
Ingo Molnar5e1607a2009-03-05 10:24:48 +0100420 "trace_printk",
Steven Rostedtb2a866f2008-11-03 23:15:57 -0500421 "ftrace_preempt",
Steven Rostedt9f029e82008-11-12 15:24:24 -0500422 "branch",
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500423 "annotate",
Török Edwin02b67512008-11-22 13:28:47 +0200424 "userstacktrace",
Török Edwinb54d3de2008-11-22 13:28:48 +0200425 "sym-userobj",
Frederic Weisbecker66896a82008-12-13 20:18:13 +0100426 "printk-msg-only",
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -0200427 "context-info",
Steven Rostedtc032ef642009-03-04 20:34:24 -0500428 "latency-format",
Steven Rostedtbe6f1642009-03-24 11:06:24 -0400429 "sleep-time",
Steven Rostedta2a16d62009-03-24 23:17:58 -0400430 "graph-time",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200431 NULL
432};
433
Zhaolei5079f322009-08-25 16:12:56 +0800434static struct {
435 u64 (*func)(void);
436 const char *name;
437} trace_clocks[] = {
438 { trace_clock_local, "local" },
439 { trace_clock_global, "global" },
440};
441
442int trace_clock_id;
443
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200444/*
445 * trace_parser_get_init - gets the buffer for trace parser
446 */
447int trace_parser_get_init(struct trace_parser *parser, int size)
448{
449 memset(parser, 0, sizeof(*parser));
450
451 parser->buffer = kmalloc(size, GFP_KERNEL);
452 if (!parser->buffer)
453 return 1;
454
455 parser->size = size;
456 return 0;
457}
458
459/*
460 * trace_parser_put - frees the buffer for trace parser
461 */
462void trace_parser_put(struct trace_parser *parser)
463{
464 kfree(parser->buffer);
465}
466
467/*
468 * trace_get_user - reads the user input string separated by space
469 * (matched by isspace(ch))
470 *
471 * For each string found the 'struct trace_parser' is updated,
472 * and the function returns.
473 *
474 * Returns number of bytes read.
475 *
476 * See kernel/trace/trace.h for 'struct trace_parser' details.
477 */
478int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
479 size_t cnt, loff_t *ppos)
480{
481 char ch;
482 size_t read = 0;
483 ssize_t ret;
484
485 if (!*ppos)
486 trace_parser_clear(parser);
487
488 ret = get_user(ch, ubuf++);
489 if (ret)
490 goto out;
491
492 read++;
493 cnt--;
494
495 /*
496 * The parser is not finished with the last write,
497 * continue reading the user input without skipping spaces.
498 */
499 if (!parser->cont) {
500 /* skip white space */
501 while (cnt && isspace(ch)) {
502 ret = get_user(ch, ubuf++);
503 if (ret)
504 goto out;
505 read++;
506 cnt--;
507 }
508
509 /* only spaces were written */
510 if (isspace(ch)) {
511 *ppos += read;
512 ret = read;
513 goto out;
514 }
515
516 parser->idx = 0;
517 }
518
519 /* read the non-space input */
520 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800521 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200522 parser->buffer[parser->idx++] = ch;
523 else {
524 ret = -EINVAL;
525 goto out;
526 }
527 ret = get_user(ch, ubuf++);
528 if (ret)
529 goto out;
530 read++;
531 cnt--;
532 }
533
534 /* We either got finished input or we have to wait for another call. */
535 if (isspace(ch)) {
536 parser->buffer[parser->idx] = 0;
537 parser->cont = false;
538 } else {
539 parser->cont = true;
540 parser->buffer[parser->idx++] = ch;
541 }
542
543 *ppos += read;
544 ret = read;
545
546out:
547 return ret;
548}
549
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200550ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
551{
552 int len;
553 int ret;
554
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500555 if (!cnt)
556 return 0;
557
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200558 if (s->len <= s->readpos)
559 return -EBUSY;
560
561 len = s->len - s->readpos;
562 if (cnt > len)
563 cnt = len;
564 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500565 if (ret == cnt)
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200566 return -EFAULT;
567
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500568 cnt -= ret;
569
Steven Rostedte74da522009-03-04 20:31:11 -0500570 s->readpos += cnt;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200571 return cnt;
Steven Rostedt214023c2008-05-12 21:20:46 +0200572}
573
Dmitri Vorobievb8b94262009-03-22 19:11:11 +0200574static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200575{
576 int len;
577 void *ret;
578
579 if (s->len <= s->readpos)
580 return -EBUSY;
581
582 len = s->len - s->readpos;
583 if (cnt > len)
584 cnt = len;
585 ret = memcpy(buf, s->buffer + s->readpos, cnt);
586 if (!ret)
587 return -EFAULT;
588
Steven Rostedte74da522009-03-04 20:31:11 -0500589 s->readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200590 return cnt;
591}
592
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400593/*
594 * ftrace_max_lock is used to protect the swapping of buffers
595 * when taking a max snapshot. The buffers themselves are
596 * protected by per_cpu spinlocks. But the action of the swap
597 * needs its own lock.
598 *
Thomas Gleixner445c8952009-12-02 19:49:50 +0100599 * This is defined as a arch_spinlock_t in order to help
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400600 * with performance when lockdep debugging is enabled.
601 *
602 * It is also used in other places outside the update_max_tr
603 * so it needs to be defined outside of the
604 * CONFIG_TRACER_MAX_TRACE.
605 */
Thomas Gleixner445c8952009-12-02 19:49:50 +0100606static arch_spinlock_t ftrace_max_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100607 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400608
Tim Bird0e950172010-02-25 15:36:43 -0800609unsigned long __read_mostly tracing_thresh;
610
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400611#ifdef CONFIG_TRACER_MAX_TRACE
612unsigned long __read_mostly tracing_max_latency;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400613
614/*
615 * Copy the new maximum trace into the separate maximum-trace
616 * structure. (this way the maximum trace is permanently saved,
617 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
618 */
619static void
620__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
621{
622 struct trace_array_cpu *data = tr->data[cpu];
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300623 struct trace_array_cpu *max_data;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400624
625 max_tr.cpu = cpu;
626 max_tr.time_start = data->preempt_timestamp;
627
Steven Rostedt8248ac02009-09-02 12:27:41 -0400628 max_data = max_tr.data[cpu];
629 max_data->saved_latency = tracing_max_latency;
630 max_data->critical_start = data->critical_start;
631 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400632
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300633 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -0400634 max_data->pid = tsk->pid;
635 max_data->uid = task_uid(tsk);
636 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
637 max_data->policy = tsk->policy;
638 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400639
640 /* record this tasks comm */
641 tracing_record_cmdline(tsk);
642}
643
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200644/**
645 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
646 * @tr: tracer
647 * @tsk: the task with the latency
648 * @cpu: The cpu that initiated the trace.
649 *
650 * Flip the buffers between the @tr and the max_tr and record information
651 * about which task was the cause of this latency.
652 */
Ingo Molnare309b412008-05-12 21:20:51 +0200653void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200654update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
655{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400656 struct ring_buffer *buf = tr->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200657
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400658 if (trace_stop_count)
659 return;
660
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200661 WARN_ON_ONCE(!irqs_disabled());
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100662 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400663
664 tr->buffer = max_tr.buffer;
665 max_tr.buffer = buf;
666
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200667 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100668 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200669}
670
671/**
672 * update_max_tr_single - only copy one trace over, and reset the rest
673 * @tr - tracer
674 * @tsk - task with the latency
675 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200676 *
677 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200678 */
Ingo Molnare309b412008-05-12 21:20:51 +0200679void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200680update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
681{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400682 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200683
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400684 if (trace_stop_count)
685 return;
686
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200687 WARN_ON_ONCE(!irqs_disabled());
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100688 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200689
Steven Rostedtd7690412008-10-01 00:29:53 -0400690 ftrace_disable_cpu();
691
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400692 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
693
Steven Rostedte8165db2009-09-03 19:13:05 -0400694 if (ret == -EBUSY) {
695 /*
696 * We failed to swap the buffer due to a commit taking
697 * place on this CPU. We fail to record, but we reset
698 * the max trace buffer (no one writes directly to it)
699 * and flag that it failed.
700 */
701 trace_array_printk(&max_tr, _THIS_IP_,
702 "Failed to swap buffers due to commit in progress\n");
703 }
704
Steven Rostedtd7690412008-10-01 00:29:53 -0400705 ftrace_enable_cpu();
706
Steven Rostedte8165db2009-09-03 19:13:05 -0400707 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200708
709 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100710 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200711}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400712#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200713
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200714/**
715 * register_tracer - register a tracer with the ftrace system.
716 * @type - the plugin for the tracer
717 *
718 * Register a new plugin tracer.
719 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200720int register_tracer(struct tracer *type)
Hannes Edere7669b82009-02-10 19:44:45 +0100721__releases(kernel_lock)
722__acquires(kernel_lock)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200723{
724 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200725 int ret = 0;
726
727 if (!type->name) {
728 pr_info("Tracer must have a name\n");
729 return -1;
730 }
731
Li Zefanee6c2c12009-09-18 14:06:47 +0800732 if (strlen(type->name) > MAX_TRACER_SIZE) {
733 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
734 return -1;
735 }
736
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100737 /*
738 * When this gets called we hold the BKL which means that
739 * preemption is disabled. Various trace selftests however
740 * need to disable and enable preemption for successful tests.
741 * So we drop the BKL here and grab it after the tests again.
742 */
743 unlock_kernel();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200744 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100745
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +0100746 tracing_selftest_running = true;
747
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200748 for (t = trace_types; t; t = t->next) {
749 if (strcmp(type->name, t->name) == 0) {
750 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +0800751 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200752 type->name);
753 ret = -1;
754 goto out;
755 }
756 }
757
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100758 if (!type->set_flag)
759 type->set_flag = &dummy_set_flag;
760 if (!type->flags)
761 type->flags = &dummy_tracer_flags;
762 else
763 if (!type->flags->opts)
764 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +0100765 if (!type->wait_pipe)
766 type->wait_pipe = default_wait_pipe;
767
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100768
Steven Rostedt60a11772008-05-12 21:20:44 +0200769#ifdef CONFIG_FTRACE_STARTUP_TEST
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500770 if (type->selftest && !tracing_selftest_disabled) {
Steven Rostedt60a11772008-05-12 21:20:44 +0200771 struct tracer *saved_tracer = current_trace;
Steven Rostedt60a11772008-05-12 21:20:44 +0200772 struct trace_array *tr = &global_trace;
Frederic Weisbeckerff325042008-12-04 23:47:35 +0100773
Steven Rostedt60a11772008-05-12 21:20:44 +0200774 /*
775 * Run a selftest on this tracer.
776 * Here we reset the trace buffer, and set the current
777 * tracer to be this tracer. The tracer can then run some
778 * internal tracing to verify that everything is in order.
779 * If we fail, we do not register this tracer.
780 */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400781 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100782
Steven Rostedt60a11772008-05-12 21:20:44 +0200783 current_trace = type;
Steven Rostedt60a11772008-05-12 21:20:44 +0200784 /* the test is responsible for initializing and enabling */
785 pr_info("Testing tracer %s: ", type->name);
786 ret = type->selftest(type, tr);
787 /* the test is responsible for resetting too */
788 current_trace = saved_tracer;
Steven Rostedt60a11772008-05-12 21:20:44 +0200789 if (ret) {
790 printk(KERN_CONT "FAILED!\n");
791 goto out;
792 }
Steven Rostedt1d4db002008-05-12 21:20:45 +0200793 /* Only reset on passing, to avoid touching corrupted buffers */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400794 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100795
Steven Rostedt60a11772008-05-12 21:20:44 +0200796 printk(KERN_CONT "PASSED\n");
797 }
798#endif
799
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200800 type->next = trace_types;
801 trace_types = type;
Steven Rostedt60a11772008-05-12 21:20:44 +0200802
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200803 out:
Frederic Weisbecker8e1b82e02008-12-06 03:41:33 +0100804 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200805 mutex_unlock(&trace_types_lock);
806
Steven Rostedtdac74942009-02-05 01:13:38 -0500807 if (ret || !default_bootup_tracer)
808 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500809
Li Zefanee6c2c12009-09-18 14:06:47 +0800810 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -0500811 goto out_unlock;
812
813 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
814 /* Do we want this tracer to start on bootup? */
815 tracing_set_tracer(type->name);
816 default_bootup_tracer = NULL;
817 /* disable other selftests, since this will break it. */
818 tracing_selftest_disabled = 1;
819#ifdef CONFIG_FTRACE_STARTUP_TEST
820 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
821 type->name);
822#endif
823
824 out_unlock:
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500825 lock_kernel();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200826 return ret;
827}
828
829void unregister_tracer(struct tracer *type)
830{
831 struct tracer **t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200832
833 mutex_lock(&trace_types_lock);
834 for (t = &trace_types; *t; t = &(*t)->next) {
835 if (*t == type)
836 goto found;
837 }
Li Zefanee6c2c12009-09-18 14:06:47 +0800838 pr_info("Tracer %s not registered\n", type->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200839 goto out;
840
841 found:
842 *t = (*t)->next;
Arnaldo Carvalho de Melob5db03c2009-02-07 18:52:59 -0200843
844 if (type == current_trace && tracer_enabled) {
845 tracer_enabled = 0;
846 tracing_stop();
847 if (current_trace->stop)
848 current_trace->stop(&global_trace);
849 current_trace = &nop_trace;
850 }
Li Zefanee6c2c12009-09-18 14:06:47 +0800851out:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200852 mutex_unlock(&trace_types_lock);
853}
854
Steven Rostedt283740c2010-03-12 19:48:41 -0500855static void __tracing_reset(struct ring_buffer *buffer, int cpu)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200856{
Steven Rostedtd7690412008-10-01 00:29:53 -0400857 ftrace_disable_cpu();
Steven Rostedt283740c2010-03-12 19:48:41 -0500858 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtd7690412008-10-01 00:29:53 -0400859 ftrace_enable_cpu();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200860}
861
Steven Rostedtf6339032009-09-04 12:35:16 -0400862void tracing_reset(struct trace_array *tr, int cpu)
863{
864 struct ring_buffer *buffer = tr->buffer;
865
866 ring_buffer_record_disable(buffer);
867
868 /* Make sure all commits have finished */
869 synchronize_sched();
Steven Rostedt283740c2010-03-12 19:48:41 -0500870 __tracing_reset(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -0400871
872 ring_buffer_record_enable(buffer);
873}
874
Pekka J Enberg213cc062008-12-19 12:08:39 +0200875void tracing_reset_online_cpus(struct trace_array *tr)
876{
Steven Rostedt621968c2009-09-04 12:02:35 -0400877 struct ring_buffer *buffer = tr->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +0200878 int cpu;
879
Steven Rostedt621968c2009-09-04 12:02:35 -0400880 ring_buffer_record_disable(buffer);
881
882 /* Make sure all commits have finished */
883 synchronize_sched();
884
Pekka J Enberg213cc062008-12-19 12:08:39 +0200885 tr->time_start = ftrace_now(tr->cpu);
886
887 for_each_online_cpu(cpu)
Steven Rostedt283740c2010-03-12 19:48:41 -0500888 __tracing_reset(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -0400889
890 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +0200891}
892
Steven Rostedt9456f0f2009-05-06 21:54:09 -0400893void tracing_reset_current(int cpu)
894{
895 tracing_reset(&global_trace, cpu);
896}
897
898void tracing_reset_current_online_cpus(void)
899{
900 tracing_reset_online_cpus(&global_trace);
901}
902
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200903#define SAVED_CMDLINES 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100904#define NO_CMDLINE_MAP UINT_MAX
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200905static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
906static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
907static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
908static int cmdline_idx;
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100909static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt25b0b442008-05-12 21:21:00 +0200910
Steven Rostedt25b0b442008-05-12 21:21:00 +0200911/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +0100912static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200913
914static void trace_init_cmdlines(void)
915{
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100916 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
917 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200918 cmdline_idx = 0;
919}
920
Carsten Emdeb5130b12009-09-13 01:43:07 +0200921int is_tracing_stopped(void)
922{
923 return trace_stop_count;
924}
925
Steven Rostedt0f048702008-11-05 16:05:44 -0500926/**
Steven Rostedt69bb54e2008-11-21 12:59:38 -0500927 * ftrace_off_permanent - disable all ftrace code permanently
928 *
929 * This should only be called when a serious anomally has
930 * been detected. This will turn off the function tracing,
931 * ring buffers, and other tracing utilites. It takes no
932 * locks and can be called from any context.
933 */
934void ftrace_off_permanent(void)
935{
936 tracing_disabled = 1;
937 ftrace_stop();
938 tracing_off_permanent();
939}
940
941/**
Steven Rostedt0f048702008-11-05 16:05:44 -0500942 * tracing_start - quick start of the tracer
943 *
944 * If tracing is enabled but was stopped by tracing_stop,
945 * this will start the tracer back up.
946 */
947void tracing_start(void)
948{
949 struct ring_buffer *buffer;
950 unsigned long flags;
951
952 if (tracing_disabled)
953 return;
954
955 spin_lock_irqsave(&tracing_start_lock, flags);
Steven Rostedtb06a8302009-01-22 14:26:15 -0500956 if (--trace_stop_count) {
957 if (trace_stop_count < 0) {
958 /* Someone screwed up their debugging */
959 WARN_ON_ONCE(1);
960 trace_stop_count = 0;
961 }
Steven Rostedt0f048702008-11-05 16:05:44 -0500962 goto out;
963 }
964
Steven Rostedta2f80712010-03-12 19:56:00 -0500965 /* Prevent the buffers from switching */
966 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -0500967
968 buffer = global_trace.buffer;
969 if (buffer)
970 ring_buffer_record_enable(buffer);
971
972 buffer = max_tr.buffer;
973 if (buffer)
974 ring_buffer_record_enable(buffer);
975
Steven Rostedta2f80712010-03-12 19:56:00 -0500976 arch_spin_unlock(&ftrace_max_lock);
977
Steven Rostedt0f048702008-11-05 16:05:44 -0500978 ftrace_start();
979 out:
980 spin_unlock_irqrestore(&tracing_start_lock, flags);
981}
982
983/**
984 * tracing_stop - quick stop of the tracer
985 *
986 * Light weight way to stop tracing. Use in conjunction with
987 * tracing_start.
988 */
989void tracing_stop(void)
990{
991 struct ring_buffer *buffer;
992 unsigned long flags;
993
994 ftrace_stop();
995 spin_lock_irqsave(&tracing_start_lock, flags);
996 if (trace_stop_count++)
997 goto out;
998
Steven Rostedta2f80712010-03-12 19:56:00 -0500999 /* Prevent the buffers from switching */
1000 arch_spin_lock(&ftrace_max_lock);
1001
Steven Rostedt0f048702008-11-05 16:05:44 -05001002 buffer = global_trace.buffer;
1003 if (buffer)
1004 ring_buffer_record_disable(buffer);
1005
1006 buffer = max_tr.buffer;
1007 if (buffer)
1008 ring_buffer_record_disable(buffer);
1009
Steven Rostedta2f80712010-03-12 19:56:00 -05001010 arch_spin_unlock(&ftrace_max_lock);
1011
Steven Rostedt0f048702008-11-05 16:05:44 -05001012 out:
1013 spin_unlock_irqrestore(&tracing_start_lock, flags);
1014}
1015
Ingo Molnare309b412008-05-12 21:20:51 +02001016void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001017
Ingo Molnare309b412008-05-12 21:20:51 +02001018static void trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001019{
Carsten Emdea635cf02009-03-18 09:00:41 +01001020 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001021
1022 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1023 return;
1024
1025 /*
1026 * It's not the end of the world if we don't get
1027 * the lock, but we also don't want to spin
1028 * nor do we want to disable interrupts,
1029 * so if we miss here, then better luck next time.
1030 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001031 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001032 return;
1033
1034 idx = map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001035 if (idx == NO_CMDLINE_MAP) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001036 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1037
Carsten Emdea635cf02009-03-18 09:00:41 +01001038 /*
1039 * Check whether the cmdline buffer at idx has a pid
1040 * mapped. We are going to overwrite that entry so we
1041 * need to clear the map_pid_to_cmdline. Otherwise we
1042 * would read the new comm for the old pid.
1043 */
1044 pid = map_cmdline_to_pid[idx];
1045 if (pid != NO_CMDLINE_MAP)
1046 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001047
Carsten Emdea635cf02009-03-18 09:00:41 +01001048 map_cmdline_to_pid[idx] = tsk->pid;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001049 map_pid_to_cmdline[tsk->pid] = idx;
1050
1051 cmdline_idx = idx;
1052 }
1053
1054 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1055
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001056 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001057}
1058
Steven Rostedt4ca530852009-03-16 19:20:15 -04001059void trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001060{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001061 unsigned map;
1062
Steven Rostedt4ca530852009-03-16 19:20:15 -04001063 if (!pid) {
1064 strcpy(comm, "<idle>");
1065 return;
1066 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001067
Steven Rostedt74bf4072010-01-25 15:11:53 -05001068 if (WARN_ON_ONCE(pid < 0)) {
1069 strcpy(comm, "<XXX>");
1070 return;
1071 }
1072
Steven Rostedt4ca530852009-03-16 19:20:15 -04001073 if (pid > PID_MAX_DEFAULT) {
1074 strcpy(comm, "<...>");
1075 return;
1076 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001077
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001078 preempt_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001079 arch_spin_lock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001080 map = map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001081 if (map != NO_CMDLINE_MAP)
1082 strcpy(comm, saved_cmdlines[map]);
1083 else
1084 strcpy(comm, "<...>");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001085
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001086 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001087 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001088}
1089
Ingo Molnare309b412008-05-12 21:20:51 +02001090void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001091{
Thomas Gleixner18aecd32009-03-18 08:56:58 +01001092 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1093 !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001094 return;
1095
1096 trace_save_cmdline(tsk);
1097}
1098
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001099void
Steven Rostedt38697052008-10-01 13:14:09 -04001100tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1101 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001102{
1103 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001104
Steven Rostedt777e2082008-09-29 23:02:42 -04001105 entry->preempt_count = pc & 0xff;
1106 entry->pid = (tsk) ? tsk->pid : 0;
Steven Rostedt637e7e82009-09-11 13:55:35 -04001107 entry->lock_depth = (tsk) ? tsk->lock_depth : 0;
Steven Rostedt777e2082008-09-29 23:02:42 -04001108 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001109#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001110 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001111#else
1112 TRACE_FLAG_IRQS_NOSUPPORT |
1113#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001114 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1115 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1116 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1117}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001118EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001119
Steven Rostedte77405a2009-09-02 14:17:06 -04001120struct ring_buffer_event *
1121trace_buffer_lock_reserve(struct ring_buffer *buffer,
1122 int type,
1123 unsigned long len,
1124 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001125{
1126 struct ring_buffer_event *event;
1127
Steven Rostedte77405a2009-09-02 14:17:06 -04001128 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001129 if (event != NULL) {
1130 struct trace_entry *ent = ring_buffer_event_data(event);
1131
1132 tracing_generic_entry_update(ent, flags, pc);
1133 ent->type = type;
1134 }
1135
1136 return event;
1137}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001138
Steven Rostedte77405a2009-09-02 14:17:06 -04001139static inline void
1140__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1141 struct ring_buffer_event *event,
1142 unsigned long flags, int pc,
1143 int wake)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001144{
Steven Rostedte77405a2009-09-02 14:17:06 -04001145 ring_buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001146
Steven Rostedte77405a2009-09-02 14:17:06 -04001147 ftrace_trace_stack(buffer, flags, 6, pc);
1148 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001149
1150 if (wake)
1151 trace_wake_up();
1152}
1153
Steven Rostedte77405a2009-09-02 14:17:06 -04001154void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1155 struct ring_buffer_event *event,
1156 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001157{
Steven Rostedte77405a2009-09-02 14:17:06 -04001158 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001159}
1160
Steven Rostedtef5580d2009-02-27 19:38:04 -05001161struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001162trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1163 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001164 unsigned long flags, int pc)
1165{
Steven Rostedte77405a2009-09-02 14:17:06 -04001166 *current_rb = global_trace.buffer;
1167 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001168 type, len, flags, pc);
1169}
Steven Rostedt94487d62009-05-05 19:22:53 -04001170EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001171
Steven Rostedte77405a2009-09-02 14:17:06 -04001172void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1173 struct ring_buffer_event *event,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001174 unsigned long flags, int pc)
1175{
Steven Rostedte77405a2009-09-02 14:17:06 -04001176 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001177}
Steven Rostedt94487d62009-05-05 19:22:53 -04001178EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001179
Steven Rostedte77405a2009-09-02 14:17:06 -04001180void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1181 struct ring_buffer_event *event,
1182 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001183{
Steven Rostedte77405a2009-09-02 14:17:06 -04001184 __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001185}
Steven Rostedt94487d62009-05-05 19:22:53 -04001186EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
Steven Rostedt77d9f462009-04-02 01:16:59 -04001187
Steven Rostedte77405a2009-09-02 14:17:06 -04001188void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1189 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001190{
Steven Rostedte77405a2009-09-02 14:17:06 -04001191 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001192}
Steven Rostedt12acd472009-04-17 16:01:56 -04001193EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001194
Ingo Molnare309b412008-05-12 21:20:51 +02001195void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001196trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001197 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1198 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001199{
Tom Zanussie1112b42009-03-31 00:48:49 -05001200 struct ftrace_event_call *call = &event_function;
Steven Rostedte77405a2009-09-02 14:17:06 -04001201 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001202 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001203 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001204
Steven Rostedtd7690412008-10-01 00:29:53 -04001205 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001206 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001207 return;
1208
Steven Rostedte77405a2009-09-02 14:17:06 -04001209 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001210 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001211 if (!event)
1212 return;
1213 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001214 entry->ip = ip;
1215 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001216
Steven Rostedte77405a2009-09-02 14:17:06 -04001217 if (!filter_check_discard(call, entry, buffer, event))
1218 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001219}
1220
Ingo Molnare309b412008-05-12 21:20:51 +02001221void
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001222ftrace(struct trace_array *tr, struct trace_array_cpu *data,
Steven Rostedt38697052008-10-01 13:14:09 -04001223 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1224 int pc)
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001225{
1226 if (likely(!atomic_read(&data->disabled)))
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001227 trace_function(tr, ip, parent_ip, flags, pc);
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001228}
1229
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001230#ifdef CONFIG_STACKTRACE
Steven Rostedte77405a2009-09-02 14:17:06 -04001231static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001232 unsigned long flags,
1233 int skip, int pc)
Ingo Molnar86387f72008-05-12 21:20:51 +02001234{
Tom Zanussie1112b42009-03-31 00:48:49 -05001235 struct ftrace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001236 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001237 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001238 struct stack_trace trace;
1239
Steven Rostedte77405a2009-09-02 14:17:06 -04001240 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001241 sizeof(*entry), flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001242 if (!event)
1243 return;
1244 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001245 memset(&entry->caller, 0, sizeof(entry->caller));
Ingo Molnar86387f72008-05-12 21:20:51 +02001246
1247 trace.nr_entries = 0;
1248 trace.max_entries = FTRACE_STACK_ENTRIES;
1249 trace.skip = skip;
Steven Rostedt777e2082008-09-29 23:02:42 -04001250 trace.entries = entry->caller;
Ingo Molnar86387f72008-05-12 21:20:51 +02001251
1252 save_stack_trace(&trace);
Steven Rostedte77405a2009-09-02 14:17:06 -04001253 if (!filter_check_discard(call, entry, buffer, event))
1254 ring_buffer_unlock_commit(buffer, event);
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001255}
1256
Steven Rostedte77405a2009-09-02 14:17:06 -04001257void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1258 int skip, int pc)
Steven Rostedt53614992009-01-15 19:12:40 -05001259{
1260 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1261 return;
1262
Steven Rostedte77405a2009-09-02 14:17:06 -04001263 __ftrace_trace_stack(buffer, flags, skip, pc);
Steven Rostedt53614992009-01-15 19:12:40 -05001264}
1265
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001266void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1267 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001268{
Steven Rostedte77405a2009-09-02 14:17:06 -04001269 __ftrace_trace_stack(tr->buffer, flags, skip, pc);
Steven Rostedt38697052008-10-01 13:14:09 -04001270}
1271
Steven Rostedt03889382009-12-11 09:48:22 -05001272/**
1273 * trace_dump_stack - record a stack back trace in the trace buffer
1274 */
1275void trace_dump_stack(void)
1276{
1277 unsigned long flags;
1278
1279 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001280 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001281
1282 local_save_flags(flags);
1283
1284 /* skipping 3 traces, seems to get us at the caller of this function */
1285 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count());
1286}
1287
Steven Rostedte77405a2009-09-02 14:17:06 -04001288void
1289ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001290{
Tom Zanussie1112b42009-03-31 00:48:49 -05001291 struct ftrace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001292 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001293 struct userstack_entry *entry;
1294 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001295
1296 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1297 return;
1298
Steven Rostedtb6345872010-03-12 20:03:30 -05001299 /*
1300 * NMIs can not handle page faults, even with fix ups.
1301 * The save user stack can (and often does) fault.
1302 */
1303 if (unlikely(in_nmi()))
1304 return;
1305
Steven Rostedte77405a2009-09-02 14:17:06 -04001306 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001307 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001308 if (!event)
1309 return;
1310 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001311
Steven Rostedt48659d32009-09-11 11:36:23 -04001312 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001313 memset(&entry->caller, 0, sizeof(entry->caller));
1314
1315 trace.nr_entries = 0;
1316 trace.max_entries = FTRACE_STACK_ENTRIES;
1317 trace.skip = 0;
1318 trace.entries = entry->caller;
1319
1320 save_stack_trace_user(&trace);
Steven Rostedte77405a2009-09-02 14:17:06 -04001321 if (!filter_check_discard(call, entry, buffer, event))
1322 ring_buffer_unlock_commit(buffer, event);
Török Edwin02b67512008-11-22 13:28:47 +02001323}
1324
Hannes Eder4fd27352009-02-10 19:44:12 +01001325#ifdef UNUSED
1326static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001327{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001328 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001329}
Hannes Eder4fd27352009-02-10 19:44:12 +01001330#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001331
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001332#endif /* CONFIG_STACKTRACE */
1333
Steven Rostedt38697052008-10-01 13:14:09 -04001334static void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001335ftrace_trace_special(void *__tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001336 unsigned long arg1, unsigned long arg2, unsigned long arg3,
1337 int pc)
Ingo Molnara4feb8342008-05-12 21:21:02 +02001338{
Steven Rostedt60ba7702009-09-12 23:34:04 -04001339 struct ftrace_event_call *call = &event_special;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001340 struct ring_buffer_event *event;
Ingo Molnara4feb8342008-05-12 21:21:02 +02001341 struct trace_array *tr = __tr;
Steven Rostedte77405a2009-09-02 14:17:06 -04001342 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt777e2082008-09-29 23:02:42 -04001343 struct special_entry *entry;
Ingo Molnara4feb8342008-05-12 21:21:02 +02001344
Steven Rostedte77405a2009-09-02 14:17:06 -04001345 event = trace_buffer_lock_reserve(buffer, TRACE_SPECIAL,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001346 sizeof(*entry), 0, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001347 if (!event)
1348 return;
1349 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001350 entry->arg1 = arg1;
1351 entry->arg2 = arg2;
1352 entry->arg3 = arg3;
Steven Rostedt60ba7702009-09-12 23:34:04 -04001353
1354 if (!filter_check_discard(call, entry, buffer, event))
1355 trace_buffer_unlock_commit(buffer, event, 0, pc);
Ingo Molnara4feb8342008-05-12 21:21:02 +02001356}
1357
1358void
Steven Rostedt38697052008-10-01 13:14:09 -04001359__trace_special(void *__tr, void *__data,
1360 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1361{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001362 ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
Steven Rostedt38697052008-10-01 13:14:09 -04001363}
1364
1365void
Steven Rostedt4902f882008-05-22 00:22:18 -04001366ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1367{
1368 struct trace_array *tr = &global_trace;
1369 struct trace_array_cpu *data;
Steven Rostedt5aa1ba62008-11-10 23:07:30 -05001370 unsigned long flags;
Steven Rostedt4902f882008-05-22 00:22:18 -04001371 int cpu;
Steven Rostedt38697052008-10-01 13:14:09 -04001372 int pc;
Steven Rostedt4902f882008-05-22 00:22:18 -04001373
Steven Rostedtc76f0692008-11-07 22:36:02 -05001374 if (tracing_disabled)
Steven Rostedt4902f882008-05-22 00:22:18 -04001375 return;
1376
Steven Rostedt38697052008-10-01 13:14:09 -04001377 pc = preempt_count();
Steven Rostedt5aa1ba62008-11-10 23:07:30 -05001378 local_irq_save(flags);
Steven Rostedt4902f882008-05-22 00:22:18 -04001379 cpu = raw_smp_processor_id();
1380 data = tr->data[cpu];
Steven Rostedt4902f882008-05-22 00:22:18 -04001381
Steven Rostedt5aa1ba62008-11-10 23:07:30 -05001382 if (likely(atomic_inc_return(&data->disabled) == 1))
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001383 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
Steven Rostedt4902f882008-05-22 00:22:18 -04001384
Steven Rostedt5aa1ba62008-11-10 23:07:30 -05001385 atomic_dec(&data->disabled);
1386 local_irq_restore(flags);
Steven Rostedt4902f882008-05-22 00:22:18 -04001387}
1388
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001389/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001390 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001391 *
1392 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04001393int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001394{
Thomas Gleixner445c8952009-12-02 19:49:50 +01001395 static arch_spinlock_t trace_buf_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001396 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001397 static u32 trace_buf[TRACE_BUF_SIZE];
1398
Tom Zanussie1112b42009-03-31 00:48:49 -05001399 struct ftrace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001400 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001401 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001402 struct trace_array *tr = &global_trace;
1403 struct trace_array_cpu *data;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001404 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001405 unsigned long flags;
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001406 int disable;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001407 int resched;
1408 int cpu, len = 0, size, pc;
1409
1410 if (unlikely(tracing_selftest_running || tracing_disabled))
1411 return 0;
1412
1413 /* Don't pollute graph traces with trace_vprintk internals */
1414 pause_graph_tracing();
1415
1416 pc = preempt_count();
1417 resched = ftrace_preempt_disable();
1418 cpu = raw_smp_processor_id();
1419 data = tr->data[cpu];
1420
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001421 disable = atomic_inc_return(&data->disabled);
1422 if (unlikely(disable != 1))
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001423 goto out;
1424
Steven Rostedt80370cb2009-03-10 17:16:35 -04001425 /* Lockdep uses trace_printk for lock tracing */
1426 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001427 arch_spin_lock(&trace_buf_lock);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001428 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1429
1430 if (len > TRACE_BUF_SIZE || len < 0)
1431 goto out_unlock;
1432
1433 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedte77405a2009-09-02 14:17:06 -04001434 buffer = tr->buffer;
1435 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1436 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001437 if (!event)
1438 goto out_unlock;
1439 entry = ring_buffer_event_data(event);
1440 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001441 entry->fmt = fmt;
1442
1443 memcpy(entry->buf, trace_buf, sizeof(u32) * len);
Steven Rostedtd9313692010-01-06 17:27:11 -05001444 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedte77405a2009-09-02 14:17:06 -04001445 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05001446 ftrace_trace_stack(buffer, flags, 6, pc);
1447 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001448
1449out_unlock:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001450 arch_spin_unlock(&trace_buf_lock);
Steven Rostedt80370cb2009-03-10 17:16:35 -04001451 local_irq_restore(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001452
1453out:
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001454 atomic_dec_return(&data->disabled);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001455 ftrace_preempt_enable(resched);
1456 unpause_graph_tracing();
1457
1458 return len;
1459}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001460EXPORT_SYMBOL_GPL(trace_vbprintk);
1461
Steven Rostedt659372d2009-09-03 19:11:07 -04001462int trace_array_printk(struct trace_array *tr,
1463 unsigned long ip, const char *fmt, ...)
1464{
1465 int ret;
1466 va_list ap;
1467
1468 if (!(trace_flags & TRACE_ITER_PRINTK))
1469 return 0;
1470
1471 va_start(ap, fmt);
1472 ret = trace_array_vprintk(tr, ip, fmt, ap);
1473 va_end(ap);
1474 return ret;
1475}
1476
1477int trace_array_vprintk(struct trace_array *tr,
1478 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001479{
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001480 static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001481 static char trace_buf[TRACE_BUF_SIZE];
1482
Tom Zanussie1112b42009-03-31 00:48:49 -05001483 struct ftrace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001484 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001485 struct ring_buffer *buffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001486 struct trace_array_cpu *data;
1487 int cpu, len = 0, size, pc;
1488 struct print_entry *entry;
1489 unsigned long irq_flags;
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001490 int disable;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001491
1492 if (tracing_disabled || tracing_selftest_running)
1493 return 0;
1494
1495 pc = preempt_count();
1496 preempt_disable_notrace();
1497 cpu = raw_smp_processor_id();
1498 data = tr->data[cpu];
1499
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001500 disable = atomic_inc_return(&data->disabled);
1501 if (unlikely(disable != 1))
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001502 goto out;
1503
1504 pause_graph_tracing();
1505 raw_local_irq_save(irq_flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001506 arch_spin_lock(&trace_buf_lock);
Carsten Emdef2942482009-12-06 14:02:44 +01001507 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001508
1509 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04001510 buffer = tr->buffer;
1511 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1512 irq_flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001513 if (!event)
1514 goto out_unlock;
1515 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001516 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001517
1518 memcpy(&entry->buf, trace_buf, len);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001519 entry->buf[len] = '\0';
Steven Rostedtd9313692010-01-06 17:27:11 -05001520 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedte77405a2009-09-02 14:17:06 -04001521 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05001522 ftrace_trace_stack(buffer, irq_flags, 6, pc);
1523 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001524
1525 out_unlock:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001526 arch_spin_unlock(&trace_buf_lock);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001527 raw_local_irq_restore(irq_flags);
1528 unpause_graph_tracing();
1529 out:
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001530 atomic_dec_return(&data->disabled);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001531 preempt_enable_notrace();
1532
1533 return len;
1534}
Steven Rostedt659372d2009-09-03 19:11:07 -04001535
1536int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1537{
Steven Rostedta813a152009-10-09 01:41:35 -04001538 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04001539}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001540EXPORT_SYMBOL_GPL(trace_vprintk);
1541
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001542enum trace_file_type {
1543 TRACE_FILE_LAT_FMT = 1,
Steven Rostedt12ef7d42008-11-12 17:52:38 -05001544 TRACE_FILE_ANNOTATE = 2,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001545};
1546
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001547static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04001548{
Steven Rostedtd7690412008-10-01 00:29:53 -04001549 /* Don't allow ftrace to trace into the ring buffers */
1550 ftrace_disable_cpu();
1551
Steven Rostedt5a90f572008-09-03 17:42:51 -04001552 iter->idx++;
Steven Rostedtd7690412008-10-01 00:29:53 -04001553 if (iter->buffer_iter[iter->cpu])
1554 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1555
1556 ftrace_enable_cpu();
Steven Rostedt5a90f572008-09-03 17:42:51 -04001557}
1558
Ingo Molnare309b412008-05-12 21:20:51 +02001559static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001560peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1561 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001562{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001563 struct ring_buffer_event *event;
1564 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001565
Steven Rostedtd7690412008-10-01 00:29:53 -04001566 /* Don't allow ftrace to trace into the ring buffers */
1567 ftrace_disable_cpu();
1568
1569 if (buf_iter)
1570 event = ring_buffer_iter_peek(buf_iter, ts);
1571 else
Steven Rostedtbc21b472010-03-31 19:49:26 -04001572 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1573 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04001574
1575 ftrace_enable_cpu();
1576
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001577 return event ? ring_buffer_event_data(event) : NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001578}
Steven Rostedtd7690412008-10-01 00:29:53 -04001579
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001580static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001581__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1582 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001583{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001584 struct ring_buffer *buffer = iter->tr->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001585 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08001586 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001587 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001588 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001589 int next_cpu = -1;
1590 int cpu;
1591
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001592 /*
1593 * If we are in a per_cpu trace file, don't bother by iterating over
1594 * all cpu and peek directly.
1595 */
1596 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1597 if (ring_buffer_empty_cpu(buffer, cpu_file))
1598 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001599 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001600 if (ent_cpu)
1601 *ent_cpu = cpu_file;
1602
1603 return ent;
1604 }
1605
Steven Rostedtab464282008-05-12 21:21:00 +02001606 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001607
1608 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001609 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001610
Steven Rostedtbc21b472010-03-31 19:49:26 -04001611 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001612
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02001613 /*
1614 * Pick the entry with the smallest timestamp:
1615 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001616 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001617 next = ent;
1618 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001619 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001620 next_lost = lost_events;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001621 }
1622 }
1623
1624 if (ent_cpu)
1625 *ent_cpu = next_cpu;
1626
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001627 if (ent_ts)
1628 *ent_ts = next_ts;
1629
Steven Rostedtbc21b472010-03-31 19:49:26 -04001630 if (missing_events)
1631 *missing_events = next_lost;
1632
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001633 return next;
1634}
1635
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001636/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001637struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1638 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001639{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001640 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001641}
Ingo Molnar8c523a92008-05-12 21:20:46 +02001642
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001643/* Find the next real entry, and increment the iterator to the next entry */
1644static void *find_next_entry_inc(struct trace_iterator *iter)
1645{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001646 iter->ent = __find_next_entry(iter, &iter->cpu,
1647 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02001648
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001649 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001650 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001651
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001652 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02001653}
1654
Ingo Molnare309b412008-05-12 21:20:51 +02001655static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001656{
Steven Rostedtd7690412008-10-01 00:29:53 -04001657 /* Don't allow ftrace to trace into the ring buffers */
1658 ftrace_disable_cpu();
Steven Rostedtbc21b472010-03-31 19:49:26 -04001659 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1660 &iter->lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04001661 ftrace_enable_cpu();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001662}
1663
Ingo Molnare309b412008-05-12 21:20:51 +02001664static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001665{
1666 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001667 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001668 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001669
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001670 WARN_ON_ONCE(iter->leftover);
1671
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001672 (*pos)++;
1673
1674 /* can't go backwards */
1675 if (iter->idx > i)
1676 return NULL;
1677
1678 if (iter->idx < 0)
1679 ent = find_next_entry_inc(iter);
1680 else
1681 ent = iter;
1682
1683 while (ent && iter->idx < i)
1684 ent = find_next_entry_inc(iter);
1685
1686 iter->pos = *pos;
1687
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001688 return ent;
1689}
1690
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001691static void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1692{
1693 struct trace_array *tr = iter->tr;
1694 struct ring_buffer_event *event;
1695 struct ring_buffer_iter *buf_iter;
1696 unsigned long entries = 0;
1697 u64 ts;
1698
1699 tr->data[cpu]->skipped_entries = 0;
1700
1701 if (!iter->buffer_iter[cpu])
1702 return;
1703
1704 buf_iter = iter->buffer_iter[cpu];
1705 ring_buffer_iter_reset(buf_iter);
1706
1707 /*
1708 * We could have the case with the max latency tracers
1709 * that a reset never took place on a cpu. This is evident
1710 * by the timestamp being before the start of the buffer.
1711 */
1712 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1713 if (ts >= iter->tr->time_start)
1714 break;
1715 entries++;
1716 ring_buffer_read(buf_iter, NULL);
1717 }
1718
1719 tr->data[cpu]->skipped_entries = entries;
1720}
1721
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001722/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001723 * The current tracer is copied to avoid a global locking
1724 * all around.
1725 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001726static void *s_start(struct seq_file *m, loff_t *pos)
1727{
1728 struct trace_iterator *iter = m->private;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001729 static struct tracer *old_tracer;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001730 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001731 void *p = NULL;
1732 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001733 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001734
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001735 /* copy the tracer to avoid using a global lock all around */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001736 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001737 if (unlikely(old_tracer != current_trace && current_trace)) {
1738 old_tracer = current_trace;
1739 *iter->trace = *current_trace;
Steven Rostedtd15f57f2008-05-12 21:20:56 +02001740 }
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001741 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001742
1743 atomic_inc(&trace_record_cmdline_disabled);
1744
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001745 if (*pos != iter->pos) {
1746 iter->ent = NULL;
1747 iter->cpu = 0;
1748 iter->idx = -1;
1749
Steven Rostedtd7690412008-10-01 00:29:53 -04001750 ftrace_disable_cpu();
1751
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001752 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1753 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001754 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001755 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001756 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001757
Steven Rostedtd7690412008-10-01 00:29:53 -04001758 ftrace_enable_cpu();
1759
Lai Jiangshanac91d852010-03-02 17:54:50 +08001760 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001761 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1762 ;
1763
1764 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001765 /*
1766 * If we overflowed the seq_file before, then we want
1767 * to just reuse the trace_seq buffer again.
1768 */
1769 if (iter->leftover)
1770 p = iter;
1771 else {
1772 l = *pos - 1;
1773 p = s_next(m, p, &l);
1774 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001775 }
1776
Lai Jiangshan4f535962009-05-18 19:35:34 +08001777 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001778 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001779 return p;
1780}
1781
1782static void s_stop(struct seq_file *m, void *p)
1783{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001784 struct trace_iterator *iter = m->private;
1785
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001786 atomic_dec(&trace_record_cmdline_disabled);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001787 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08001788 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001789}
1790
Ingo Molnare309b412008-05-12 21:20:51 +02001791static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001792{
Michael Ellermana6168352008-08-20 16:36:11 -07001793 seq_puts(m, "# _------=> CPU# \n");
1794 seq_puts(m, "# / _-----=> irqs-off \n");
1795 seq_puts(m, "# | / _----=> need-resched \n");
1796 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1797 seq_puts(m, "# ||| / _--=> preempt-depth \n");
Steven Rostedt637e7e82009-09-11 13:55:35 -04001798 seq_puts(m, "# |||| /_--=> lock-depth \n");
1799 seq_puts(m, "# |||||/ delay \n");
1800 seq_puts(m, "# cmd pid |||||| time | caller \n");
1801 seq_puts(m, "# \\ / |||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001802}
1803
Ingo Molnare309b412008-05-12 21:20:51 +02001804static void print_func_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001805{
Michael Ellermana6168352008-08-20 16:36:11 -07001806 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1807 seq_puts(m, "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001808}
1809
1810
Jiri Olsa62b915f2010-04-02 19:01:22 +02001811void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001812print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1813{
1814 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1815 struct trace_array *tr = iter->tr;
1816 struct trace_array_cpu *data = tr->data[tr->cpu];
1817 struct tracer *type = current_trace;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001818 unsigned long entries = 0;
1819 unsigned long total = 0;
1820 unsigned long count;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001821 const char *name = "preemption";
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001822 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001823
1824 if (type)
1825 name = type->name;
1826
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001827
1828 for_each_tracing_cpu(cpu) {
1829 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1830 /*
1831 * If this buffer has skipped entries, then we hold all
1832 * entries for the trace and we need to ignore the
1833 * ones before the time stamp.
1834 */
1835 if (tr->data[cpu]->skipped_entries) {
1836 count -= tr->data[cpu]->skipped_entries;
1837 /* total is the same as the entries */
1838 total += count;
1839 } else
1840 total += count +
1841 ring_buffer_overrun_cpu(tr->buffer, cpu);
1842 entries += count;
1843 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001844
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001845 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001846 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001847 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001848 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001849 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001850 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02001851 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001852 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001853 total,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001854 tr->cpu,
1855#if defined(CONFIG_PREEMPT_NONE)
1856 "server",
1857#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1858 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04001859#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001860 "preempt",
1861#else
1862 "unknown",
1863#endif
1864 /* These are reserved for later use */
1865 0, 0, 0, 0);
1866#ifdef CONFIG_SMP
1867 seq_printf(m, " #P:%d)\n", num_online_cpus());
1868#else
1869 seq_puts(m, ")\n");
1870#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001871 seq_puts(m, "# -----------------\n");
1872 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001873 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1874 data->comm, data->pid, data->uid, data->nice,
1875 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001876 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001877
1878 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001879 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02001880 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1881 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001882 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02001883 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1884 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04001885 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001886 }
1887
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001888 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001889}
1890
Steven Rostedta3097202008-11-07 22:36:02 -05001891static void test_cpu_buff_start(struct trace_iterator *iter)
1892{
1893 struct trace_seq *s = &iter->seq;
1894
Steven Rostedt12ef7d42008-11-12 17:52:38 -05001895 if (!(trace_flags & TRACE_ITER_ANNOTATE))
1896 return;
1897
1898 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1899 return;
1900
Rusty Russell44623442009-01-01 10:12:23 +10301901 if (cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05001902 return;
1903
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001904 if (iter->tr->data[iter->cpu]->skipped_entries)
1905 return;
1906
Rusty Russell44623442009-01-01 10:12:23 +10301907 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02001908
1909 /* Don't print started cpu buffer for the first entry of the trace */
1910 if (iter->idx > 1)
1911 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1912 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05001913}
1914
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001915static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001916{
Steven Rostedt214023c2008-05-12 21:20:46 +02001917 struct trace_seq *s = &iter->seq;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001918 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001919 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05001920 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001921
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001922 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001923
Steven Rostedta3097202008-11-07 22:36:02 -05001924 test_cpu_buff_start(iter);
1925
Steven Rostedtf633cef2008-12-23 23:24:13 -05001926 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001927
1928 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt27d48be2009-03-04 21:57:29 -05001929 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1930 if (!trace_print_lat_context(iter))
1931 goto partial;
1932 } else {
1933 if (!trace_print_context(iter))
1934 goto partial;
1935 }
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001936 }
1937
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02001938 if (event)
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001939 return event->trace(iter, sym_flags);
1940
1941 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1942 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04001943
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001944 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001945partial:
1946 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001947}
1948
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001949static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02001950{
1951 struct trace_seq *s = &iter->seq;
1952 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05001953 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02001954
1955 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001956
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001957 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001958 if (!trace_seq_printf(s, "%d %d %llu ",
1959 entry->pid, iter->cpu, iter->ts))
1960 goto partial;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001961 }
Ingo Molnarf9896bf2008-05-12 21:20:47 +02001962
Steven Rostedtf633cef2008-12-23 23:24:13 -05001963 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02001964 if (event)
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001965 return event->raw(iter, 0);
1966
1967 if (!trace_seq_printf(s, "%d ?\n", entry->type))
1968 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04001969
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001970 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001971partial:
1972 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02001973}
1974
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001975static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02001976{
1977 struct trace_seq *s = &iter->seq;
1978 unsigned char newline = '\n';
1979 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05001980 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02001981
1982 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001983
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001984 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1985 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1986 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1987 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1988 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02001989
Steven Rostedtf633cef2008-12-23 23:24:13 -05001990 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02001991 if (event) {
Arnaldo Carvalho de Meloae7462b2009-02-03 22:05:50 -02001992 enum print_line_t ret = event->hex(iter, 0);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001993 if (ret != TRACE_TYPE_HANDLED)
1994 return ret;
1995 }
Steven Rostedt7104f302008-10-01 10:52:51 -04001996
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02001997 SEQ_PUT_FIELD_RET(s, newline);
1998
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001999 return TRACE_TYPE_HANDLED;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002000}
2001
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002002static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002003{
2004 struct trace_seq *s = &iter->seq;
2005 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002006 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002007
2008 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002009
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002010 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2011 SEQ_PUT_FIELD_RET(s, entry->pid);
Steven Rostedt1830b522009-02-07 19:38:43 -05002012 SEQ_PUT_FIELD_RET(s, iter->cpu);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002013 SEQ_PUT_FIELD_RET(s, iter->ts);
2014 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002015
Steven Rostedtf633cef2008-12-23 23:24:13 -05002016 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002017 return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002018}
2019
Jiri Olsa62b915f2010-04-02 19:01:22 +02002020int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002021{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002022 int cpu;
2023
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002024 /* If we are looking at one CPU buffer, only check that one */
2025 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2026 cpu = iter->cpu_file;
2027 if (iter->buffer_iter[cpu]) {
2028 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2029 return 0;
2030 } else {
2031 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2032 return 0;
2033 }
2034 return 1;
2035 }
2036
Steven Rostedtab464282008-05-12 21:21:00 +02002037 for_each_tracing_cpu(cpu) {
Steven Rostedtd7690412008-10-01 00:29:53 -04002038 if (iter->buffer_iter[cpu]) {
2039 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2040 return 0;
2041 } else {
2042 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2043 return 0;
2044 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002045 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002046
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002047 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002048}
2049
Lai Jiangshan4f535962009-05-18 19:35:34 +08002050/* Called with trace_event_read_lock() held. */
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002051static enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002052{
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002053 enum print_line_t ret;
2054
Steven Rostedtbc21b472010-03-31 19:49:26 -04002055 if (iter->lost_events)
2056 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2057 iter->cpu, iter->lost_events);
2058
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002059 if (iter->trace && iter->trace->print_line) {
2060 ret = iter->trace->print_line(iter);
2061 if (ret != TRACE_TYPE_UNHANDLED)
2062 return ret;
2063 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002064
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002065 if (iter->ent->type == TRACE_BPRINT &&
2066 trace_flags & TRACE_ITER_PRINTK &&
2067 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002068 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002069
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002070 if (iter->ent->type == TRACE_PRINT &&
2071 trace_flags & TRACE_ITER_PRINTK &&
2072 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002073 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002074
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002075 if (trace_flags & TRACE_ITER_BIN)
2076 return print_bin_fmt(iter);
2077
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002078 if (trace_flags & TRACE_ITER_HEX)
2079 return print_hex_fmt(iter);
2080
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002081 if (trace_flags & TRACE_ITER_RAW)
2082 return print_raw_fmt(iter);
2083
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002084 return print_trace_fmt(iter);
2085}
2086
Jiri Olsa62b915f2010-04-02 19:01:22 +02002087void trace_default_header(struct seq_file *m)
2088{
2089 struct trace_iterator *iter = m->private;
2090
2091 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2092 /* print nothing if the buffers are empty */
2093 if (trace_empty(iter))
2094 return;
2095 print_trace_header(m, iter);
2096 if (!(trace_flags & TRACE_ITER_VERBOSE))
2097 print_lat_help_header(m);
2098 } else {
2099 if (!(trace_flags & TRACE_ITER_VERBOSE))
2100 print_func_help_header(m);
2101 }
2102}
2103
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002104static int s_show(struct seq_file *m, void *v)
2105{
2106 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002107 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002108
2109 if (iter->ent == NULL) {
2110 if (iter->tr) {
2111 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2112 seq_puts(m, "#\n");
2113 }
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002114 if (iter->trace && iter->trace->print_header)
2115 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002116 else
2117 trace_default_header(m);
2118
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002119 } else if (iter->leftover) {
2120 /*
2121 * If we filled the seq_file buffer earlier, we
2122 * want to just show it now.
2123 */
2124 ret = trace_print_seq(m, &iter->seq);
2125
2126 /* ret should this time be zero, but you never know */
2127 iter->leftover = ret;
2128
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002129 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002130 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002131 ret = trace_print_seq(m, &iter->seq);
2132 /*
2133 * If we overflow the seq_file buffer, then it will
2134 * ask us for this data again at start up.
2135 * Use that instead.
2136 * ret is 0 if seq_file write succeeded.
2137 * -1 otherwise.
2138 */
2139 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002140 }
2141
2142 return 0;
2143}
2144
James Morris88e9d342009-09-22 16:43:43 -07002145static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002146 .start = s_start,
2147 .next = s_next,
2148 .stop = s_stop,
2149 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002150};
2151
Ingo Molnare309b412008-05-12 21:20:51 +02002152static struct trace_iterator *
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002153__tracing_open(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002154{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002155 long cpu_file = (long) inode->i_private;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002156 void *fail_ret = ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002157 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002158 struct seq_file *m;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002159 int cpu, ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002160
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002161 if (tracing_disabled)
2162 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02002163
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002164 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002165 if (!iter)
2166 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002167
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002168 /*
2169 * We make a copy of the current tracer to avoid concurrent
2170 * changes on it while we are reading.
2171 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002172 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002173 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002174 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002175 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002176
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002177 if (current_trace)
2178 *iter->trace = *current_trace;
2179
Li Zefan79f55992009-06-15 14:58:26 +08002180 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002181 goto fail;
2182
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002183 if (current_trace && current_trace->print_max)
2184 iter->tr = &max_tr;
2185 else
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002186 iter->tr = &global_trace;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002187 iter->pos = -1;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002188 mutex_init(&iter->mutex);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002189 iter->cpu_file = cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002190
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002191 /* Notify the tracer early; before we stop tracing. */
2192 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01002193 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002194
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002195 /* Annotate start of buffers if we had overruns */
2196 if (ring_buffer_overruns(iter->tr->buffer))
2197 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2198
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002199 /* stop the trace while dumping */
2200 tracing_stop();
2201
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002202 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2203 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002204 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002205 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2206 }
2207 ring_buffer_read_prepare_sync();
2208 for_each_tracing_cpu(cpu) {
2209 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002210 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002211 }
2212 } else {
2213 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002214 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002215 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2216 ring_buffer_read_prepare_sync();
2217 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002218 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002219 }
2220
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002221 ret = seq_open(file, &tracer_seq_ops);
2222 if (ret < 0) {
2223 fail_ret = ERR_PTR(ret);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002224 goto fail_buffer;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002225 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002226
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002227 m = file->private_data;
2228 m->private = iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002229
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002230 mutex_unlock(&trace_types_lock);
2231
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002232 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002233
2234 fail_buffer:
2235 for_each_tracing_cpu(cpu) {
2236 if (iter->buffer_iter[cpu])
2237 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2238 }
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002239 free_cpumask_var(iter->started);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002240 tracing_start();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002241 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002242 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002243 kfree(iter->trace);
Julia Lawall0bb943c2008-11-14 19:05:31 +01002244 kfree(iter);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002245
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002246 return fail_ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002247}
2248
2249int tracing_open_generic(struct inode *inode, struct file *filp)
2250{
Steven Rostedt60a11772008-05-12 21:20:44 +02002251 if (tracing_disabled)
2252 return -ENODEV;
2253
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002254 filp->private_data = inode->i_private;
2255 return 0;
2256}
2257
Hannes Eder4fd27352009-02-10 19:44:12 +01002258static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002259{
2260 struct seq_file *m = (struct seq_file *)file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002261 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002262 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002263
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002264 if (!(file->f_mode & FMODE_READ))
2265 return 0;
2266
2267 iter = m->private;
2268
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002269 mutex_lock(&trace_types_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002270 for_each_tracing_cpu(cpu) {
2271 if (iter->buffer_iter[cpu])
2272 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2273 }
2274
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002275 if (iter->trace && iter->trace->close)
2276 iter->trace->close(iter);
2277
2278 /* reenable tracing if it was previously enabled */
Steven Rostedt90369902008-11-05 16:05:44 -05002279 tracing_start();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002280 mutex_unlock(&trace_types_lock);
2281
2282 seq_release(inode, file);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002283 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002284 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002285 kfree(iter->trace);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002286 kfree(iter);
2287 return 0;
2288}
2289
2290static int tracing_open(struct inode *inode, struct file *file)
2291{
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002292 struct trace_iterator *iter;
2293 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002294
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002295 /* If this file was open for write, then erase contents */
2296 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04002297 (file->f_flags & O_TRUNC)) {
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002298 long cpu = (long) inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002299
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002300 if (cpu == TRACE_PIPE_ALL_CPU)
2301 tracing_reset_online_cpus(&global_trace);
2302 else
2303 tracing_reset(&global_trace, cpu);
2304 }
2305
2306 if (file->f_mode & FMODE_READ) {
2307 iter = __tracing_open(inode, file);
2308 if (IS_ERR(iter))
2309 ret = PTR_ERR(iter);
2310 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2311 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2312 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002313 return ret;
2314}
2315
Ingo Molnare309b412008-05-12 21:20:51 +02002316static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002317t_next(struct seq_file *m, void *v, loff_t *pos)
2318{
Li Zefanf129e962009-06-24 09:53:44 +08002319 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002320
2321 (*pos)++;
2322
2323 if (t)
2324 t = t->next;
2325
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002326 return t;
2327}
2328
2329static void *t_start(struct seq_file *m, loff_t *pos)
2330{
Li Zefanf129e962009-06-24 09:53:44 +08002331 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002332 loff_t l = 0;
2333
2334 mutex_lock(&trace_types_lock);
Li Zefanf129e962009-06-24 09:53:44 +08002335 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002336 ;
2337
2338 return t;
2339}
2340
2341static void t_stop(struct seq_file *m, void *p)
2342{
2343 mutex_unlock(&trace_types_lock);
2344}
2345
2346static int t_show(struct seq_file *m, void *v)
2347{
2348 struct tracer *t = v;
2349
2350 if (!t)
2351 return 0;
2352
2353 seq_printf(m, "%s", t->name);
2354 if (t->next)
2355 seq_putc(m, ' ');
2356 else
2357 seq_putc(m, '\n');
2358
2359 return 0;
2360}
2361
James Morris88e9d342009-09-22 16:43:43 -07002362static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002363 .start = t_start,
2364 .next = t_next,
2365 .stop = t_stop,
2366 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002367};
2368
2369static int show_traces_open(struct inode *inode, struct file *file)
2370{
Steven Rostedt60a11772008-05-12 21:20:44 +02002371 if (tracing_disabled)
2372 return -ENODEV;
2373
Li Zefanf129e962009-06-24 09:53:44 +08002374 return seq_open(file, &show_traces_seq_ops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002375}
2376
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002377static ssize_t
2378tracing_write_stub(struct file *filp, const char __user *ubuf,
2379 size_t count, loff_t *ppos)
2380{
2381 return count;
2382}
2383
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002384static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002385 .open = tracing_open,
2386 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002387 .write = tracing_write_stub,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002388 .llseek = seq_lseek,
2389 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002390};
2391
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002392static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002393 .open = show_traces_open,
2394 .read = seq_read,
2395 .release = seq_release,
2396};
2397
Ingo Molnar36dfe922008-05-12 21:20:52 +02002398/*
2399 * Only trace on a CPU if the bitmask is set:
2400 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302401static cpumask_var_t tracing_cpumask;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002402
2403/*
2404 * The tracer itself will not take this lock, but still we want
2405 * to provide a consistent cpumask to user-space:
2406 */
2407static DEFINE_MUTEX(tracing_cpumask_update_lock);
2408
2409/*
2410 * Temporary storage for the character representation of the
2411 * CPU bitmask (and one more byte for the newline):
2412 */
2413static char mask_str[NR_CPUS + 1];
2414
Ingo Molnarc7078de2008-05-12 21:20:52 +02002415static ssize_t
2416tracing_cpumask_read(struct file *filp, char __user *ubuf,
2417 size_t count, loff_t *ppos)
2418{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002419 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002420
2421 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002422
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302423 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002424 if (count - len < 2) {
2425 count = -EINVAL;
2426 goto out_err;
2427 }
2428 len += sprintf(mask_str + len, "\n");
2429 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2430
2431out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02002432 mutex_unlock(&tracing_cpumask_update_lock);
2433
2434 return count;
2435}
2436
2437static ssize_t
2438tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2439 size_t count, loff_t *ppos)
2440{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002441 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302442 cpumask_var_t tracing_cpumask_new;
2443
2444 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2445 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002446
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302447 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002448 if (err)
2449 goto err_unlock;
2450
Li Zefan215368e2009-06-15 10:56:42 +08002451 mutex_lock(&tracing_cpumask_update_lock);
2452
Steven Rostedta5e25882008-12-02 15:34:05 -05002453 local_irq_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002454 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02002455 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002456 /*
2457 * Increase/decrease the disabled counter if we are
2458 * about to flip a bit in the cpumask:
2459 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302460 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2461 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002462 atomic_inc(&global_trace.data[cpu]->disabled);
2463 }
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302464 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2465 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002466 atomic_dec(&global_trace.data[cpu]->disabled);
2467 }
2468 }
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002469 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05002470 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02002471
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302472 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002473
Ingo Molnarc7078de2008-05-12 21:20:52 +02002474 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302475 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02002476
Ingo Molnarc7078de2008-05-12 21:20:52 +02002477 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002478
2479err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08002480 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002481
2482 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002483}
2484
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002485static const struct file_operations tracing_cpumask_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002486 .open = tracing_open_generic,
2487 .read = tracing_cpumask_read,
2488 .write = tracing_cpumask_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002489};
2490
Li Zefanfdb372e2009-12-08 11:15:59 +08002491static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002492{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002493 struct tracer_opt *trace_opts;
2494 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002495 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002496
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002497 mutex_lock(&trace_types_lock);
2498 tracer_flags = current_trace->flags->val;
2499 trace_opts = current_trace->flags->opts;
2500
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002501 for (i = 0; trace_options[i]; i++) {
2502 if (trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08002503 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002504 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002505 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002506 }
2507
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002508 for (i = 0; trace_opts[i].name; i++) {
2509 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08002510 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002511 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002512 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002513 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002514 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002515
Li Zefanfdb372e2009-12-08 11:15:59 +08002516 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002517}
2518
Li Zefan8d18eaa2009-12-08 11:17:06 +08002519static int __set_tracer_option(struct tracer *trace,
2520 struct tracer_flags *tracer_flags,
2521 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002522{
Li Zefan8d18eaa2009-12-08 11:17:06 +08002523 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002524
Li Zefan8d18eaa2009-12-08 11:17:06 +08002525 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002526 if (ret)
2527 return ret;
2528
2529 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08002530 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002531 else
Zhaolei77708412009-08-07 18:53:21 +08002532 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002533 return 0;
2534}
2535
Li Zefan8d18eaa2009-12-08 11:17:06 +08002536/* Try to assign a tracer specific option */
2537static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2538{
2539 struct tracer_flags *tracer_flags = trace->flags;
2540 struct tracer_opt *opts = NULL;
2541 int i;
2542
2543 for (i = 0; tracer_flags->opts[i].name; i++) {
2544 opts = &tracer_flags->opts[i];
2545
2546 if (strcmp(cmp, opts->name) == 0)
2547 return __set_tracer_option(trace, trace->flags,
2548 opts, neg);
2549 }
2550
2551 return -EINVAL;
2552}
2553
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002554static void set_tracer_flags(unsigned int mask, int enabled)
2555{
2556 /* do nothing if flag is already set */
2557 if (!!(trace_flags & mask) == !!enabled)
2558 return;
2559
2560 if (enabled)
2561 trace_flags |= mask;
2562 else
2563 trace_flags &= ~mask;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002564}
2565
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002566static ssize_t
Steven Rostedtee6bce52008-11-12 17:52:37 -05002567tracing_trace_options_write(struct file *filp, const char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002568 size_t cnt, loff_t *ppos)
2569{
2570 char buf[64];
Li Zefan8d18eaa2009-12-08 11:17:06 +08002571 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002572 int neg = 0;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002573 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002574 int i;
2575
Steven Rostedtcffae432008-05-12 21:21:00 +02002576 if (cnt >= sizeof(buf))
2577 return -EINVAL;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002578
2579 if (copy_from_user(&buf, ubuf, cnt))
2580 return -EFAULT;
2581
2582 buf[cnt] = 0;
Li Zefan8d18eaa2009-12-08 11:17:06 +08002583 cmp = strstrip(buf);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002584
Li Zefan8d18eaa2009-12-08 11:17:06 +08002585 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002586 neg = 1;
2587 cmp += 2;
2588 }
2589
2590 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08002591 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002592 set_tracer_flags(1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002593 break;
2594 }
2595 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002596
2597 /* If no option could be set, test the specific tracer options */
2598 if (!trace_options[i]) {
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002599 mutex_lock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002600 ret = set_tracer_option(current_trace, cmp, neg);
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002601 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002602 if (ret)
2603 return ret;
2604 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002605
Jiri Olsacf8517c2009-10-23 19:36:16 -04002606 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002607
2608 return cnt;
2609}
2610
Li Zefanfdb372e2009-12-08 11:15:59 +08002611static int tracing_trace_options_open(struct inode *inode, struct file *file)
2612{
2613 if (tracing_disabled)
2614 return -ENODEV;
2615 return single_open(file, tracing_trace_options_show, NULL);
2616}
2617
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002618static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08002619 .open = tracing_trace_options_open,
2620 .read = seq_read,
2621 .llseek = seq_lseek,
2622 .release = single_release,
Steven Rostedtee6bce52008-11-12 17:52:37 -05002623 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002624};
2625
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002626static const char readme_msg[] =
2627 "tracing mini-HOWTO:\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002628 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2629 "# cat /sys/kernel/debug/tracing/available_tracers\n"
Nikanth Karthikesanbc2b6872009-03-23 11:58:31 +05302630 "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002631 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Nikanth Karthikesanbc2b6872009-03-23 11:58:31 +05302632 "nop\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002633 "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2634 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002635 "sched_switch\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002636 "# cat /sys/kernel/debug/tracing/trace_options\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002637 "noprint-parent nosym-offset nosym-addr noverbose\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002638 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2639 "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
2640 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2641 "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002642;
2643
2644static ssize_t
2645tracing_readme_read(struct file *filp, char __user *ubuf,
2646 size_t cnt, loff_t *ppos)
2647{
2648 return simple_read_from_buffer(ubuf, cnt, ppos,
2649 readme_msg, strlen(readme_msg));
2650}
2651
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002652static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002653 .open = tracing_open_generic,
2654 .read = tracing_readme_read,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002655};
2656
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002657static ssize_t
Avadh Patel69abe6a2009-04-10 16:04:48 -04002658tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2659 size_t cnt, loff_t *ppos)
2660{
2661 char *buf_comm;
2662 char *file_buf;
2663 char *buf;
2664 int len = 0;
2665 int pid;
2666 int i;
2667
2668 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2669 if (!file_buf)
2670 return -ENOMEM;
2671
2672 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2673 if (!buf_comm) {
2674 kfree(file_buf);
2675 return -ENOMEM;
2676 }
2677
2678 buf = file_buf;
2679
2680 for (i = 0; i < SAVED_CMDLINES; i++) {
2681 int r;
2682
2683 pid = map_cmdline_to_pid[i];
2684 if (pid == -1 || pid == NO_CMDLINE_MAP)
2685 continue;
2686
2687 trace_find_cmdline(pid, buf_comm);
2688 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2689 buf += r;
2690 len += r;
2691 }
2692
2693 len = simple_read_from_buffer(ubuf, cnt, ppos,
2694 file_buf, len);
2695
2696 kfree(file_buf);
2697 kfree(buf_comm);
2698
2699 return len;
2700}
2701
2702static const struct file_operations tracing_saved_cmdlines_fops = {
2703 .open = tracing_open_generic,
2704 .read = tracing_saved_cmdlines_read,
2705};
2706
2707static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002708tracing_ctrl_read(struct file *filp, char __user *ubuf,
2709 size_t cnt, loff_t *ppos)
2710{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002711 char buf[64];
2712 int r;
2713
Steven Rostedt90369902008-11-05 16:05:44 -05002714 r = sprintf(buf, "%u\n", tracer_enabled);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002715 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002716}
2717
2718static ssize_t
2719tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2720 size_t cnt, loff_t *ppos)
2721{
2722 struct trace_array *tr = filp->private_data;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002723 char buf[64];
Hannes Eder5e398412009-02-10 19:44:34 +01002724 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002725 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002726
Steven Rostedtcffae432008-05-12 21:21:00 +02002727 if (cnt >= sizeof(buf))
2728 return -EINVAL;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002729
2730 if (copy_from_user(&buf, ubuf, cnt))
2731 return -EFAULT;
2732
2733 buf[cnt] = 0;
2734
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002735 ret = strict_strtoul(buf, 10, &val);
2736 if (ret < 0)
2737 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002738
2739 val = !!val;
2740
2741 mutex_lock(&trace_types_lock);
Steven Rostedt90369902008-11-05 16:05:44 -05002742 if (tracer_enabled ^ val) {
2743 if (val) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002744 tracer_enabled = 1;
Steven Rostedt90369902008-11-05 16:05:44 -05002745 if (current_trace->start)
2746 current_trace->start(tr);
2747 tracing_start();
2748 } else {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002749 tracer_enabled = 0;
Steven Rostedt90369902008-11-05 16:05:44 -05002750 tracing_stop();
2751 if (current_trace->stop)
2752 current_trace->stop(tr);
2753 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002754 }
2755 mutex_unlock(&trace_types_lock);
2756
Jiri Olsacf8517c2009-10-23 19:36:16 -04002757 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002758
2759 return cnt;
2760}
2761
2762static ssize_t
2763tracing_set_trace_read(struct file *filp, char __user *ubuf,
2764 size_t cnt, loff_t *ppos)
2765{
Li Zefanee6c2c12009-09-18 14:06:47 +08002766 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002767 int r;
2768
2769 mutex_lock(&trace_types_lock);
2770 if (current_trace)
2771 r = sprintf(buf, "%s\n", current_trace->name);
2772 else
2773 r = sprintf(buf, "\n");
2774 mutex_unlock(&trace_types_lock);
2775
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002776 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002777}
2778
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02002779int tracer_init(struct tracer *t, struct trace_array *tr)
2780{
2781 tracing_reset_online_cpus(tr);
2782 return t->init(tr);
2783}
2784
Steven Rostedt73c51622009-03-11 13:42:01 -04002785static int tracing_resize_ring_buffer(unsigned long size)
2786{
2787 int ret;
2788
2789 /*
2790 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04002791 * we use the size that was given, and we can forget about
2792 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04002793 */
2794 ring_buffer_expanded = 1;
2795
2796 ret = ring_buffer_resize(global_trace.buffer, size);
2797 if (ret < 0)
2798 return ret;
2799
2800 ret = ring_buffer_resize(max_tr.buffer, size);
2801 if (ret < 0) {
2802 int r;
2803
2804 r = ring_buffer_resize(global_trace.buffer,
2805 global_trace.entries);
2806 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04002807 /*
2808 * AARGH! We are left with different
2809 * size max buffer!!!!
2810 * The max buffer is our "snapshot" buffer.
2811 * When a tracer needs a snapshot (one of the
2812 * latency tracers), it swaps the max buffer
2813 * with the saved snap shot. We succeeded to
2814 * update the size of the main buffer, but failed to
2815 * update the size of the max buffer. But when we tried
2816 * to reset the main buffer to the original size, we
2817 * failed there too. This is very unlikely to
2818 * happen, but if it does, warn and kill all
2819 * tracing.
2820 */
Steven Rostedt73c51622009-03-11 13:42:01 -04002821 WARN_ON(1);
2822 tracing_disabled = 1;
2823 }
2824 return ret;
2825 }
2826
2827 global_trace.entries = size;
2828
2829 return ret;
2830}
2831
Steven Rostedt1852fcc2009-03-11 14:33:00 -04002832/**
2833 * tracing_update_buffers - used by tracing facility to expand ring buffers
2834 *
2835 * To save on memory when the tracing is never used on a system with it
2836 * configured in. The ring buffers are set to a minimum size. But once
2837 * a user starts to use the tracing facility, then they need to grow
2838 * to their default size.
2839 *
2840 * This function is to be called when a tracer is about to be used.
2841 */
2842int tracing_update_buffers(void)
2843{
2844 int ret = 0;
2845
Steven Rostedt1027fcb2009-03-12 11:33:20 -04002846 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04002847 if (!ring_buffer_expanded)
2848 ret = tracing_resize_ring_buffer(trace_buf_size);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04002849 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04002850
2851 return ret;
2852}
2853
Steven Rostedt577b7852009-02-26 23:43:05 -05002854struct trace_option_dentry;
2855
2856static struct trace_option_dentry *
2857create_trace_option_files(struct tracer *tracer);
2858
2859static void
2860destroy_trace_option_files(struct trace_option_dentry *topts);
2861
Steven Rostedtb2821ae2009-02-02 21:38:32 -05002862static int tracing_set_tracer(const char *buf)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002863{
Steven Rostedt577b7852009-02-26 23:43:05 -05002864 static struct trace_option_dentry *topts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002865 struct trace_array *tr = &global_trace;
2866 struct tracer *t;
Peter Zijlstrad9e54072008-11-01 19:57:37 +01002867 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002868
Steven Rostedt1027fcb2009-03-12 11:33:20 -04002869 mutex_lock(&trace_types_lock);
2870
Steven Rostedt73c51622009-03-11 13:42:01 -04002871 if (!ring_buffer_expanded) {
2872 ret = tracing_resize_ring_buffer(trace_buf_size);
2873 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01002874 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04002875 ret = 0;
2876 }
2877
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002878 for (t = trace_types; t; t = t->next) {
2879 if (strcmp(t->name, buf) == 0)
2880 break;
2881 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02002882 if (!t) {
2883 ret = -EINVAL;
2884 goto out;
2885 }
2886 if (t == current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002887 goto out;
2888
Steven Rostedt9f029e82008-11-12 15:24:24 -05002889 trace_branch_disable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002890 if (current_trace && current_trace->reset)
2891 current_trace->reset(tr);
2892
Steven Rostedt577b7852009-02-26 23:43:05 -05002893 destroy_trace_option_files(topts);
2894
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002895 current_trace = t;
Steven Rostedt577b7852009-02-26 23:43:05 -05002896
2897 topts = create_trace_option_files(current_trace);
2898
Frederic Weisbecker1c800252008-11-16 05:57:26 +01002899 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02002900 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01002901 if (ret)
2902 goto out;
2903 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002904
Steven Rostedt9f029e82008-11-12 15:24:24 -05002905 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002906 out:
2907 mutex_unlock(&trace_types_lock);
2908
Peter Zijlstrad9e54072008-11-01 19:57:37 +01002909 return ret;
2910}
2911
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002912static ssize_t
2913tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2914 size_t cnt, loff_t *ppos)
2915{
Li Zefanee6c2c12009-09-18 14:06:47 +08002916 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002917 int i;
2918 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01002919 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002920
Steven Rostedt60063a62008-10-28 10:44:24 -04002921 ret = cnt;
2922
Li Zefanee6c2c12009-09-18 14:06:47 +08002923 if (cnt > MAX_TRACER_SIZE)
2924 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002925
2926 if (copy_from_user(&buf, ubuf, cnt))
2927 return -EFAULT;
2928
2929 buf[cnt] = 0;
2930
2931 /* strip ending whitespace. */
2932 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2933 buf[i] = 0;
2934
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01002935 err = tracing_set_tracer(buf);
2936 if (err)
2937 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002938
Jiri Olsacf8517c2009-10-23 19:36:16 -04002939 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002940
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02002941 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002942}
2943
2944static ssize_t
2945tracing_max_lat_read(struct file *filp, char __user *ubuf,
2946 size_t cnt, loff_t *ppos)
2947{
2948 unsigned long *ptr = filp->private_data;
2949 char buf[64];
2950 int r;
2951
Steven Rostedtcffae432008-05-12 21:21:00 +02002952 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002953 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02002954 if (r > sizeof(buf))
2955 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002956 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002957}
2958
2959static ssize_t
2960tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2961 size_t cnt, loff_t *ppos)
2962{
Hannes Eder5e398412009-02-10 19:44:34 +01002963 unsigned long *ptr = filp->private_data;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002964 char buf[64];
Hannes Eder5e398412009-02-10 19:44:34 +01002965 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002966 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002967
Steven Rostedtcffae432008-05-12 21:21:00 +02002968 if (cnt >= sizeof(buf))
2969 return -EINVAL;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002970
2971 if (copy_from_user(&buf, ubuf, cnt))
2972 return -EFAULT;
2973
2974 buf[cnt] = 0;
2975
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002976 ret = strict_strtoul(buf, 10, &val);
2977 if (ret < 0)
2978 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002979
2980 *ptr = val * 1000;
2981
2982 return cnt;
2983}
2984
Steven Rostedtb3806b42008-05-12 21:20:46 +02002985static int tracing_open_pipe(struct inode *inode, struct file *filp)
2986{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002987 long cpu_file = (long) inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02002988 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002989 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02002990
2991 if (tracing_disabled)
2992 return -ENODEV;
2993
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002994 mutex_lock(&trace_types_lock);
2995
Steven Rostedtb3806b42008-05-12 21:20:46 +02002996 /* create a buffer to store the information to pass to userspace */
2997 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002998 if (!iter) {
2999 ret = -ENOMEM;
3000 goto out;
3001 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003002
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003003 /*
3004 * We make a copy of the current tracer to avoid concurrent
3005 * changes on it while we are reading.
3006 */
3007 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3008 if (!iter->trace) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003009 ret = -ENOMEM;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003010 goto fail;
3011 }
3012 if (current_trace)
3013 *iter->trace = *current_trace;
3014
3015 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3016 ret = -ENOMEM;
3017 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10303018 }
3019
Steven Rostedta3097202008-11-07 22:36:02 -05003020 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10303021 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05003022
Steven Rostedt112f38a72009-06-01 15:16:05 -04003023 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3024 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3025
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003026 iter->cpu_file = cpu_file;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003027 iter->tr = &global_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003028 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003029 filp->private_data = iter;
3030
Steven Rostedt107bad82008-05-12 21:21:01 +02003031 if (iter->trace->pipe_open)
3032 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02003033
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003034out:
3035 mutex_unlock(&trace_types_lock);
3036 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003037
3038fail:
3039 kfree(iter->trace);
3040 kfree(iter);
3041 mutex_unlock(&trace_types_lock);
3042 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003043}
3044
3045static int tracing_release_pipe(struct inode *inode, struct file *file)
3046{
3047 struct trace_iterator *iter = file->private_data;
3048
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003049 mutex_lock(&trace_types_lock);
3050
Steven Rostedt29bf4a52009-12-09 12:37:43 -05003051 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05003052 iter->trace->pipe_close(iter);
3053
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003054 mutex_unlock(&trace_types_lock);
3055
Rusty Russell44623442009-01-01 10:12:23 +10303056 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003057 mutex_destroy(&iter->mutex);
3058 kfree(iter->trace);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003059 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003060
3061 return 0;
3062}
3063
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003064static unsigned int
3065tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3066{
3067 struct trace_iterator *iter = filp->private_data;
3068
3069 if (trace_flags & TRACE_ITER_BLOCK) {
3070 /*
3071 * Always select as readable when in blocking mode
3072 */
3073 return POLLIN | POLLRDNORM;
Ingo Molnarafc2abc2008-05-12 21:21:00 +02003074 } else {
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003075 if (!trace_empty(iter))
3076 return POLLIN | POLLRDNORM;
3077 poll_wait(filp, &trace_wait, poll_table);
3078 if (!trace_empty(iter))
3079 return POLLIN | POLLRDNORM;
3080
3081 return 0;
3082 }
3083}
3084
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003085
3086void default_wait_pipe(struct trace_iterator *iter)
3087{
3088 DEFINE_WAIT(wait);
3089
3090 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3091
3092 if (trace_empty(iter))
3093 schedule();
3094
3095 finish_wait(&trace_wait, &wait);
3096}
3097
3098/*
3099 * This is a make-shift waitqueue.
3100 * A tracer might use this callback on some rare cases:
3101 *
3102 * 1) the current tracer might hold the runqueue lock when it wakes up
3103 * a reader, hence a deadlock (sched, function, and function graph tracers)
3104 * 2) the function tracers, trace all functions, we don't want
3105 * the overhead of calling wake_up and friends
3106 * (and tracing them too)
3107 *
3108 * Anyway, this is really very primitive wakeup.
3109 */
3110void poll_wait_pipe(struct trace_iterator *iter)
3111{
3112 set_current_state(TASK_INTERRUPTIBLE);
3113 /* sleep for 100 msecs, and try again. */
3114 schedule_timeout(HZ / 10);
3115}
3116
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003117/* Must be called with trace_types_lock mutex held. */
3118static int tracing_wait_pipe(struct file *filp)
3119{
3120 struct trace_iterator *iter = filp->private_data;
3121
3122 while (trace_empty(iter)) {
3123
3124 if ((filp->f_flags & O_NONBLOCK)) {
3125 return -EAGAIN;
3126 }
3127
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003128 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003129
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003130 iter->trace->wait_pipe(iter);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003131
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003132 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003133
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003134 if (signal_pending(current))
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003135 return -EINTR;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003136
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003137 /*
3138 * We block until we read something and tracing is disabled.
3139 * We still block if tracing is disabled, but we have never
3140 * read anything. This allows a user to cat this file, and
3141 * then enable tracing. But after we have read something,
3142 * we give an EOF when tracing is again disabled.
3143 *
3144 * iter->pos will be 0 if we haven't read anything.
3145 */
3146 if (!tracer_enabled && iter->pos)
3147 break;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003148 }
3149
3150 return 1;
3151}
3152
Steven Rostedtb3806b42008-05-12 21:20:46 +02003153/*
3154 * Consumer reader.
3155 */
3156static ssize_t
3157tracing_read_pipe(struct file *filp, char __user *ubuf,
3158 size_t cnt, loff_t *ppos)
3159{
3160 struct trace_iterator *iter = filp->private_data;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003161 static struct tracer *old_tracer;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003162 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003163
3164 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003165 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3166 if (sret != -EBUSY)
3167 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003168
Steven Rostedtf9520752009-03-02 14:04:40 -05003169 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003170
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003171 /* copy the tracer to avoid using a global lock all around */
Steven Rostedt107bad82008-05-12 21:21:01 +02003172 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003173 if (unlikely(old_tracer != current_trace && current_trace)) {
3174 old_tracer = current_trace;
3175 *iter->trace = *current_trace;
3176 }
3177 mutex_unlock(&trace_types_lock);
3178
3179 /*
3180 * Avoid more than one consumer on a single file descriptor
3181 * This is just a matter of traces coherency, the ring buffer itself
3182 * is protected.
3183 */
3184 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003185 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003186 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3187 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02003188 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02003189 }
3190
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003191waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003192 sret = tracing_wait_pipe(filp);
3193 if (sret <= 0)
3194 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003195
3196 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003197 if (trace_empty(iter)) {
3198 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02003199 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003200 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003201
3202 if (cnt >= PAGE_SIZE)
3203 cnt = PAGE_SIZE - 1;
3204
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003205 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003206 memset(&iter->seq, 0,
3207 sizeof(struct trace_iterator) -
3208 offsetof(struct trace_iterator, seq));
Steven Rostedt4823ed72008-05-12 21:21:01 +02003209 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003210
Lai Jiangshan4f535962009-05-18 19:35:34 +08003211 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003212 trace_access_lock(iter->cpu_file);
Steven Rostedt088b1e422008-05-12 21:20:48 +02003213 while (find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003214 enum print_line_t ret;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003215 int len = iter->seq.len;
3216
Ingo Molnarf9896bf2008-05-12 21:20:47 +02003217 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003218 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02003219 /* don't print partial lines */
3220 iter->seq.len = len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003221 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003222 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01003223 if (ret != TRACE_TYPE_NO_CONSUME)
3224 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003225
3226 if (iter->seq.len >= cnt)
3227 break;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003228 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003229 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003230 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02003231
Steven Rostedtb3806b42008-05-12 21:20:46 +02003232 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003233 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3234 if (iter->seq.readpos >= iter->seq.len)
Steven Rostedtf9520752009-03-02 14:04:40 -05003235 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003236
3237 /*
3238 * If there was nothing to send to user, inspite of consuming trace
3239 * entries, go back to wait for more entries.
3240 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003241 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003242 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003243
Steven Rostedt107bad82008-05-12 21:21:01 +02003244out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003245 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003246
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003247 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003248}
3249
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003250static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3251 struct pipe_buffer *buf)
3252{
3253 __free_page(buf->page);
3254}
3255
3256static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3257 unsigned int idx)
3258{
3259 __free_page(spd->pages[idx]);
3260}
3261
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08003262static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003263 .can_merge = 0,
3264 .map = generic_pipe_buf_map,
3265 .unmap = generic_pipe_buf_unmap,
3266 .confirm = generic_pipe_buf_confirm,
3267 .release = tracing_pipe_buf_release,
3268 .steal = generic_pipe_buf_steal,
3269 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003270};
3271
Steven Rostedt34cd4992009-02-09 12:06:29 -05003272static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003273tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003274{
3275 size_t count;
3276 int ret;
3277
3278 /* Seq buffer is page-sized, exactly what we need. */
3279 for (;;) {
3280 count = iter->seq.len;
3281 ret = print_trace_line(iter);
3282 count = iter->seq.len - count;
3283 if (rem < count) {
3284 rem = 0;
3285 iter->seq.len -= count;
3286 break;
3287 }
3288 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3289 iter->seq.len -= count;
3290 break;
3291 }
3292
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08003293 if (ret != TRACE_TYPE_NO_CONSUME)
3294 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05003295 rem -= count;
3296 if (!find_next_entry_inc(iter)) {
3297 rem = 0;
3298 iter->ent = NULL;
3299 break;
3300 }
3301 }
3302
3303 return rem;
3304}
3305
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003306static ssize_t tracing_splice_read_pipe(struct file *filp,
3307 loff_t *ppos,
3308 struct pipe_inode_info *pipe,
3309 size_t len,
3310 unsigned int flags)
3311{
3312 struct page *pages[PIPE_BUFFERS];
3313 struct partial_page partial[PIPE_BUFFERS];
3314 struct trace_iterator *iter = filp->private_data;
3315 struct splice_pipe_desc spd = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003316 .pages = pages,
3317 .partial = partial,
3318 .nr_pages = 0, /* This gets updated below. */
3319 .flags = flags,
3320 .ops = &tracing_pipe_buf_ops,
3321 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003322 };
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003323 static struct tracer *old_tracer;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003324 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003325 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003326 unsigned int i;
3327
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003328 /* copy the tracer to avoid using a global lock all around */
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003329 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003330 if (unlikely(old_tracer != current_trace && current_trace)) {
3331 old_tracer = current_trace;
3332 *iter->trace = *current_trace;
3333 }
3334 mutex_unlock(&trace_types_lock);
3335
3336 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003337
3338 if (iter->trace->splice_read) {
3339 ret = iter->trace->splice_read(iter, filp,
3340 ppos, pipe, len, flags);
3341 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003342 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003343 }
3344
3345 ret = tracing_wait_pipe(filp);
3346 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003347 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003348
3349 if (!iter->ent && !find_next_entry_inc(iter)) {
3350 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003351 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003352 }
3353
Lai Jiangshan4f535962009-05-18 19:35:34 +08003354 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003355 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003356
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003357 /* Fill as many pages as possible. */
3358 for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
3359 pages[i] = alloc_page(GFP_KERNEL);
Steven Rostedt34cd4992009-02-09 12:06:29 -05003360 if (!pages[i])
3361 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003362
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003363 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003364
3365 /* Copy the data into the page, so we can start over. */
3366 ret = trace_seq_to_buffer(&iter->seq,
3367 page_address(pages[i]),
3368 iter->seq.len);
3369 if (ret < 0) {
3370 __free_page(pages[i]);
3371 break;
3372 }
3373 partial[i].offset = 0;
3374 partial[i].len = iter->seq.len;
3375
Steven Rostedtf9520752009-03-02 14:04:40 -05003376 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003377 }
3378
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003379 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003380 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003381 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003382
3383 spd.nr_pages = i;
3384
3385 return splice_to_pipe(pipe, &spd);
3386
Steven Rostedt34cd4992009-02-09 12:06:29 -05003387out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003388 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003389
3390 return ret;
3391}
3392
Steven Rostedta98a3c32008-05-12 21:20:59 +02003393static ssize_t
3394tracing_entries_read(struct file *filp, char __user *ubuf,
3395 size_t cnt, loff_t *ppos)
3396{
3397 struct trace_array *tr = filp->private_data;
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003398 char buf[96];
Steven Rostedta98a3c32008-05-12 21:20:59 +02003399 int r;
3400
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003401 mutex_lock(&trace_types_lock);
3402 if (!ring_buffer_expanded)
3403 r = sprintf(buf, "%lu (expanded: %lu)\n",
3404 tr->entries >> 10,
3405 trace_buf_size >> 10);
3406 else
3407 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3408 mutex_unlock(&trace_types_lock);
3409
Steven Rostedta98a3c32008-05-12 21:20:59 +02003410 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3411}
3412
3413static ssize_t
3414tracing_entries_write(struct file *filp, const char __user *ubuf,
3415 size_t cnt, loff_t *ppos)
3416{
3417 unsigned long val;
3418 char buf[64];
Steven Rostedtbf5e6512008-11-10 21:46:00 -05003419 int ret, cpu;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003420
Steven Rostedtcffae432008-05-12 21:21:00 +02003421 if (cnt >= sizeof(buf))
3422 return -EINVAL;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003423
3424 if (copy_from_user(&buf, ubuf, cnt))
3425 return -EFAULT;
3426
3427 buf[cnt] = 0;
3428
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003429 ret = strict_strtoul(buf, 10, &val);
3430 if (ret < 0)
3431 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003432
3433 /* must have at least 1 entry */
3434 if (!val)
3435 return -EINVAL;
3436
3437 mutex_lock(&trace_types_lock);
3438
Steven Rostedtc76f0692008-11-07 22:36:02 -05003439 tracing_stop();
Steven Rostedta98a3c32008-05-12 21:20:59 +02003440
Steven Rostedtbf5e6512008-11-10 21:46:00 -05003441 /* disable all cpu buffers */
3442 for_each_tracing_cpu(cpu) {
3443 if (global_trace.data[cpu])
3444 atomic_inc(&global_trace.data[cpu]->disabled);
3445 if (max_tr.data[cpu])
3446 atomic_inc(&max_tr.data[cpu]->disabled);
3447 }
3448
Steven Rostedt1696b2b2008-11-13 00:09:35 -05003449 /* value is in KB */
3450 val <<= 10;
3451
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003452 if (val != global_trace.entries) {
Steven Rostedt73c51622009-03-11 13:42:01 -04003453 ret = tracing_resize_ring_buffer(val);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003454 if (ret < 0) {
3455 cnt = ret;
Steven Rostedt3eefae92008-05-12 21:21:04 +02003456 goto out;
3457 }
Steven Rostedta98a3c32008-05-12 21:20:59 +02003458 }
3459
Jiri Olsacf8517c2009-10-23 19:36:16 -04003460 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003461
Steven Rostedt19384c02008-05-22 00:22:16 -04003462 /* If check pages failed, return ENOMEM */
3463 if (tracing_disabled)
3464 cnt = -ENOMEM;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003465 out:
Steven Rostedtbf5e6512008-11-10 21:46:00 -05003466 for_each_tracing_cpu(cpu) {
3467 if (global_trace.data[cpu])
3468 atomic_dec(&global_trace.data[cpu]->disabled);
3469 if (max_tr.data[cpu])
3470 atomic_dec(&max_tr.data[cpu]->disabled);
3471 }
3472
Steven Rostedtc76f0692008-11-07 22:36:02 -05003473 tracing_start();
Steven Rostedta98a3c32008-05-12 21:20:59 +02003474 max_tr.entries = global_trace.entries;
3475 mutex_unlock(&trace_types_lock);
3476
3477 return cnt;
3478}
3479
Carsten Emdef2942482009-12-06 14:02:44 +01003480static int mark_printk(const char *fmt, ...)
3481{
3482 int ret;
3483 va_list args;
3484 va_start(args, fmt);
3485 ret = trace_vprintk(0, fmt, args);
3486 va_end(args);
3487 return ret;
3488}
3489
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003490static ssize_t
3491tracing_mark_write(struct file *filp, const char __user *ubuf,
3492 size_t cnt, loff_t *fpos)
3493{
3494 char *buf;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003495
Steven Rostedtc76f0692008-11-07 22:36:02 -05003496 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003497 return -EINVAL;
3498
3499 if (cnt > TRACE_BUF_SIZE)
3500 cnt = TRACE_BUF_SIZE;
3501
Carsten Emdec13d2f72009-11-16 20:56:13 +01003502 buf = kmalloc(cnt + 2, GFP_KERNEL);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003503 if (buf == NULL)
3504 return -ENOMEM;
3505
3506 if (copy_from_user(buf, ubuf, cnt)) {
3507 kfree(buf);
3508 return -EFAULT;
3509 }
Carsten Emdec13d2f72009-11-16 20:56:13 +01003510 if (buf[cnt-1] != '\n') {
3511 buf[cnt] = '\n';
3512 buf[cnt+1] = '\0';
3513 } else
3514 buf[cnt] = '\0';
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003515
Carsten Emdef2942482009-12-06 14:02:44 +01003516 cnt = mark_printk("%s", buf);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003517 kfree(buf);
3518 *fpos += cnt;
3519
3520 return cnt;
3521}
3522
Li Zefan13f16d22009-12-08 11:16:11 +08003523static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08003524{
Zhaolei5079f322009-08-25 16:12:56 +08003525 int i;
3526
3527 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08003528 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08003529 "%s%s%s%s", i ? " " : "",
3530 i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3531 i == trace_clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08003532 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08003533
Li Zefan13f16d22009-12-08 11:16:11 +08003534 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08003535}
3536
3537static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3538 size_t cnt, loff_t *fpos)
3539{
3540 char buf[64];
3541 const char *clockstr;
3542 int i;
3543
3544 if (cnt >= sizeof(buf))
3545 return -EINVAL;
3546
3547 if (copy_from_user(&buf, ubuf, cnt))
3548 return -EFAULT;
3549
3550 buf[cnt] = 0;
3551
3552 clockstr = strstrip(buf);
3553
3554 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3555 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3556 break;
3557 }
3558 if (i == ARRAY_SIZE(trace_clocks))
3559 return -EINVAL;
3560
3561 trace_clock_id = i;
3562
3563 mutex_lock(&trace_types_lock);
3564
3565 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3566 if (max_tr.buffer)
3567 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3568
3569 mutex_unlock(&trace_types_lock);
3570
3571 *fpos += cnt;
3572
3573 return cnt;
3574}
3575
Li Zefan13f16d22009-12-08 11:16:11 +08003576static int tracing_clock_open(struct inode *inode, struct file *file)
3577{
3578 if (tracing_disabled)
3579 return -ENODEV;
3580 return single_open(file, tracing_clock_show, NULL);
3581}
3582
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003583static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003584 .open = tracing_open_generic,
3585 .read = tracing_max_lat_read,
3586 .write = tracing_max_lat_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003587};
3588
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003589static const struct file_operations tracing_ctrl_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003590 .open = tracing_open_generic,
3591 .read = tracing_ctrl_read,
3592 .write = tracing_ctrl_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003593};
3594
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003595static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003596 .open = tracing_open_generic,
3597 .read = tracing_set_trace_read,
3598 .write = tracing_set_trace_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003599};
3600
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003601static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003602 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003603 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003604 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003605 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003606 .release = tracing_release_pipe,
Steven Rostedtb3806b42008-05-12 21:20:46 +02003607};
3608
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003609static const struct file_operations tracing_entries_fops = {
Steven Rostedta98a3c32008-05-12 21:20:59 +02003610 .open = tracing_open_generic,
3611 .read = tracing_entries_read,
3612 .write = tracing_entries_write,
3613};
3614
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003615static const struct file_operations tracing_mark_fops = {
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02003616 .open = tracing_open_generic,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003617 .write = tracing_mark_write,
3618};
3619
Zhaolei5079f322009-08-25 16:12:56 +08003620static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08003621 .open = tracing_clock_open,
3622 .read = seq_read,
3623 .llseek = seq_lseek,
3624 .release = single_release,
Zhaolei5079f322009-08-25 16:12:56 +08003625 .write = tracing_clock_write,
3626};
3627
Steven Rostedt2cadf912008-12-01 22:20:19 -05003628struct ftrace_buffer_info {
3629 struct trace_array *tr;
3630 void *spare;
3631 int cpu;
3632 unsigned int read;
3633};
3634
3635static int tracing_buffers_open(struct inode *inode, struct file *filp)
3636{
3637 int cpu = (int)(long)inode->i_private;
3638 struct ftrace_buffer_info *info;
3639
3640 if (tracing_disabled)
3641 return -ENODEV;
3642
3643 info = kzalloc(sizeof(*info), GFP_KERNEL);
3644 if (!info)
3645 return -ENOMEM;
3646
3647 info->tr = &global_trace;
3648 info->cpu = cpu;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003649 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003650 /* Force reading ring buffer for first read */
3651 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003652
3653 filp->private_data = info;
3654
Lai Jiangshand1e7e022009-04-02 15:16:56 +08003655 return nonseekable_open(inode, filp);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003656}
3657
3658static ssize_t
3659tracing_buffers_read(struct file *filp, char __user *ubuf,
3660 size_t count, loff_t *ppos)
3661{
3662 struct ftrace_buffer_info *info = filp->private_data;
3663 unsigned int pos;
3664 ssize_t ret;
3665 size_t size;
3666
Steven Rostedt2dc5d122009-03-04 19:10:05 -05003667 if (!count)
3668 return 0;
3669
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003670 if (!info->spare)
3671 info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3672 if (!info->spare)
3673 return -ENOMEM;
3674
Steven Rostedt2cadf912008-12-01 22:20:19 -05003675 /* Do we have previous read data to read? */
3676 if (info->read < PAGE_SIZE)
3677 goto read;
3678
3679 info->read = 0;
3680
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003681 trace_access_lock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003682 ret = ring_buffer_read_page(info->tr->buffer,
3683 &info->spare,
3684 count,
3685 info->cpu, 0);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003686 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003687 if (ret < 0)
3688 return 0;
3689
3690 pos = ring_buffer_page_len(info->spare);
3691
3692 if (pos < PAGE_SIZE)
3693 memset(info->spare + pos, 0, PAGE_SIZE - pos);
3694
3695read:
3696 size = PAGE_SIZE - info->read;
3697 if (size > count)
3698 size = count;
3699
3700 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt2dc5d122009-03-04 19:10:05 -05003701 if (ret == size)
Steven Rostedt2cadf912008-12-01 22:20:19 -05003702 return -EFAULT;
Steven Rostedt2dc5d122009-03-04 19:10:05 -05003703 size -= ret;
3704
Steven Rostedt2cadf912008-12-01 22:20:19 -05003705 *ppos += size;
3706 info->read += size;
3707
3708 return size;
3709}
3710
3711static int tracing_buffers_release(struct inode *inode, struct file *file)
3712{
3713 struct ftrace_buffer_info *info = file->private_data;
3714
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003715 if (info->spare)
3716 ring_buffer_free_read_page(info->tr->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003717 kfree(info);
3718
3719 return 0;
3720}
3721
3722struct buffer_ref {
3723 struct ring_buffer *buffer;
3724 void *page;
3725 int ref;
3726};
3727
3728static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3729 struct pipe_buffer *buf)
3730{
3731 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3732
3733 if (--ref->ref)
3734 return;
3735
3736 ring_buffer_free_read_page(ref->buffer, ref->page);
3737 kfree(ref);
3738 buf->private = 0;
3739}
3740
3741static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3742 struct pipe_buffer *buf)
3743{
3744 return 1;
3745}
3746
3747static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3748 struct pipe_buffer *buf)
3749{
3750 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3751
3752 ref->ref++;
3753}
3754
3755/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08003756static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05003757 .can_merge = 0,
3758 .map = generic_pipe_buf_map,
3759 .unmap = generic_pipe_buf_unmap,
3760 .confirm = generic_pipe_buf_confirm,
3761 .release = buffer_pipe_buf_release,
3762 .steal = buffer_pipe_buf_steal,
3763 .get = buffer_pipe_buf_get,
3764};
3765
3766/*
3767 * Callback from splice_to_pipe(), if we need to release some pages
3768 * at the end of the spd in case we error'ed out in filling the pipe.
3769 */
3770static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3771{
3772 struct buffer_ref *ref =
3773 (struct buffer_ref *)spd->partial[i].private;
3774
3775 if (--ref->ref)
3776 return;
3777
3778 ring_buffer_free_read_page(ref->buffer, ref->page);
3779 kfree(ref);
3780 spd->partial[i].private = 0;
3781}
3782
3783static ssize_t
3784tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3785 struct pipe_inode_info *pipe, size_t len,
3786 unsigned int flags)
3787{
3788 struct ftrace_buffer_info *info = file->private_data;
3789 struct partial_page partial[PIPE_BUFFERS];
3790 struct page *pages[PIPE_BUFFERS];
3791 struct splice_pipe_desc spd = {
3792 .pages = pages,
3793 .partial = partial,
3794 .flags = flags,
3795 .ops = &buffer_pipe_buf_ops,
3796 .spd_release = buffer_spd_release,
3797 };
3798 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04003799 int entries, size, i;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003800 size_t ret;
3801
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08003802 if (*ppos & (PAGE_SIZE - 1)) {
3803 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
3804 return -EINVAL;
3805 }
3806
3807 if (len & (PAGE_SIZE - 1)) {
3808 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
3809 if (len < PAGE_SIZE)
3810 return -EINVAL;
3811 len &= PAGE_MASK;
3812 }
3813
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003814 trace_access_lock(info->cpu);
Steven Rostedt93459c62009-04-29 00:23:13 -04003815 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3816
3817 for (i = 0; i < PIPE_BUFFERS && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05003818 struct page *page;
3819 int r;
3820
3821 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3822 if (!ref)
3823 break;
3824
Steven Rostedt7267fa62009-04-29 00:16:21 -04003825 ref->ref = 1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003826 ref->buffer = info->tr->buffer;
3827 ref->page = ring_buffer_alloc_read_page(ref->buffer);
3828 if (!ref->page) {
3829 kfree(ref);
3830 break;
3831 }
3832
3833 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtf2957f12009-04-29 00:26:30 -04003834 len, info->cpu, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003835 if (r < 0) {
3836 ring_buffer_free_read_page(ref->buffer,
3837 ref->page);
3838 kfree(ref);
3839 break;
3840 }
3841
3842 /*
3843 * zero out any left over data, this is going to
3844 * user land.
3845 */
3846 size = ring_buffer_page_len(ref->page);
3847 if (size < PAGE_SIZE)
3848 memset(ref->page + size, 0, PAGE_SIZE - size);
3849
3850 page = virt_to_page(ref->page);
3851
3852 spd.pages[i] = page;
3853 spd.partial[i].len = PAGE_SIZE;
3854 spd.partial[i].offset = 0;
3855 spd.partial[i].private = (unsigned long)ref;
3856 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08003857 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04003858
3859 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003860 }
3861
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003862 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003863 spd.nr_pages = i;
3864
3865 /* did we read anything? */
3866 if (!spd.nr_pages) {
3867 if (flags & SPLICE_F_NONBLOCK)
3868 ret = -EAGAIN;
3869 else
3870 ret = 0;
3871 /* TODO: block */
3872 return ret;
3873 }
3874
3875 ret = splice_to_pipe(pipe, &spd);
3876
3877 return ret;
3878}
3879
3880static const struct file_operations tracing_buffers_fops = {
3881 .open = tracing_buffers_open,
3882 .read = tracing_buffers_read,
3883 .release = tracing_buffers_release,
3884 .splice_read = tracing_buffers_splice_read,
3885 .llseek = no_llseek,
3886};
3887
Steven Rostedtc8d77182009-04-29 18:03:45 -04003888static ssize_t
3889tracing_stats_read(struct file *filp, char __user *ubuf,
3890 size_t count, loff_t *ppos)
3891{
3892 unsigned long cpu = (unsigned long)filp->private_data;
3893 struct trace_array *tr = &global_trace;
3894 struct trace_seq *s;
3895 unsigned long cnt;
3896
Li Zefane4f2d102009-06-15 10:57:28 +08003897 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04003898 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01003899 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04003900
3901 trace_seq_init(s);
3902
3903 cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
3904 trace_seq_printf(s, "entries: %ld\n", cnt);
3905
3906 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
3907 trace_seq_printf(s, "overrun: %ld\n", cnt);
3908
3909 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
3910 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
3911
Steven Rostedtc8d77182009-04-29 18:03:45 -04003912 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
3913
3914 kfree(s);
3915
3916 return count;
3917}
3918
3919static const struct file_operations tracing_stats_fops = {
3920 .open = tracing_open_generic,
3921 .read = tracing_stats_read,
3922};
3923
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003924#ifdef CONFIG_DYNAMIC_FTRACE
3925
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003926int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003927{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003928 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003929}
3930
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003931static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003932tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003933 size_t cnt, loff_t *ppos)
3934{
Steven Rostedta26a2a22008-10-31 00:03:22 -04003935 static char ftrace_dyn_info_buffer[1024];
3936 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003937 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003938 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04003939 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003940 int r;
3941
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003942 mutex_lock(&dyn_info_mutex);
3943 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003944
Steven Rostedta26a2a22008-10-31 00:03:22 -04003945 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003946 buf[r++] = '\n';
3947
3948 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3949
3950 mutex_unlock(&dyn_info_mutex);
3951
3952 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003953}
3954
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003955static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003956 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003957 .read = tracing_read_dyn_info,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003958};
3959#endif
3960
3961static struct dentry *d_tracer;
3962
3963struct dentry *tracing_init_dentry(void)
3964{
3965 static int once;
3966
3967 if (d_tracer)
3968 return d_tracer;
3969
Frederic Weisbecker3e1f60b2009-03-22 23:10:45 +01003970 if (!debugfs_initialized())
3971 return NULL;
3972
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003973 d_tracer = debugfs_create_dir("tracing", NULL);
3974
3975 if (!d_tracer && !once) {
3976 once = 1;
3977 pr_warning("Could not create debugfs directory 'tracing'\n");
3978 return NULL;
3979 }
3980
3981 return d_tracer;
3982}
3983
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003984static struct dentry *d_percpu;
3985
3986struct dentry *tracing_dentry_percpu(void)
3987{
3988 static int once;
3989 struct dentry *d_tracer;
3990
3991 if (d_percpu)
3992 return d_percpu;
3993
3994 d_tracer = tracing_init_dentry();
3995
3996 if (!d_tracer)
3997 return NULL;
3998
3999 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4000
4001 if (!d_percpu && !once) {
4002 once = 1;
4003 pr_warning("Could not create debugfs directory 'per_cpu'\n");
4004 return NULL;
4005 }
4006
4007 return d_percpu;
4008}
4009
4010static void tracing_init_debugfs_percpu(long cpu)
4011{
4012 struct dentry *d_percpu = tracing_dentry_percpu();
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004013 struct dentry *d_cpu;
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004014 /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
4015 char cpu_dir[7];
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004016
4017 if (cpu > 999 || cpu < 0)
4018 return;
4019
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004020 sprintf(cpu_dir, "cpu%ld", cpu);
4021 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4022 if (!d_cpu) {
4023 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4024 return;
4025 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004026
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004027 /* per cpu trace_pipe */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004028 trace_create_file("trace_pipe", 0444, d_cpu,
4029 (void *) cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004030
4031 /* per cpu trace */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004032 trace_create_file("trace", 0644, d_cpu,
4033 (void *) cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04004034
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004035 trace_create_file("trace_pipe_raw", 0444, d_cpu,
4036 (void *) cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04004037
4038 trace_create_file("stats", 0444, d_cpu,
4039 (void *) cpu, &tracing_stats_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004040}
4041
Steven Rostedt60a11772008-05-12 21:20:44 +02004042#ifdef CONFIG_FTRACE_SELFTEST
4043/* Let selftest have access to static functions in this file */
4044#include "trace_selftest.c"
4045#endif
4046
Steven Rostedt577b7852009-02-26 23:43:05 -05004047struct trace_option_dentry {
4048 struct tracer_opt *opt;
4049 struct tracer_flags *flags;
4050 struct dentry *entry;
4051};
4052
4053static ssize_t
4054trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4055 loff_t *ppos)
4056{
4057 struct trace_option_dentry *topt = filp->private_data;
4058 char *buf;
4059
4060 if (topt->flags->val & topt->opt->bit)
4061 buf = "1\n";
4062 else
4063 buf = "0\n";
4064
4065 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4066}
4067
4068static ssize_t
4069trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4070 loff_t *ppos)
4071{
4072 struct trace_option_dentry *topt = filp->private_data;
4073 unsigned long val;
4074 char buf[64];
4075 int ret;
4076
4077 if (cnt >= sizeof(buf))
4078 return -EINVAL;
4079
4080 if (copy_from_user(&buf, ubuf, cnt))
4081 return -EFAULT;
4082
4083 buf[cnt] = 0;
4084
4085 ret = strict_strtoul(buf, 10, &val);
4086 if (ret < 0)
4087 return ret;
4088
Li Zefan8d18eaa2009-12-08 11:17:06 +08004089 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05004090 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08004091
4092 if (!!(topt->flags->val & topt->opt->bit) != val) {
4093 mutex_lock(&trace_types_lock);
4094 ret = __set_tracer_option(current_trace, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05004095 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08004096 mutex_unlock(&trace_types_lock);
4097 if (ret)
4098 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05004099 }
4100
4101 *ppos += cnt;
4102
4103 return cnt;
4104}
4105
4106
4107static const struct file_operations trace_options_fops = {
4108 .open = tracing_open_generic,
4109 .read = trace_options_read,
4110 .write = trace_options_write,
4111};
4112
Steven Rostedta8259072009-02-26 22:19:12 -05004113static ssize_t
4114trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4115 loff_t *ppos)
4116{
4117 long index = (long)filp->private_data;
4118 char *buf;
4119
4120 if (trace_flags & (1 << index))
4121 buf = "1\n";
4122 else
4123 buf = "0\n";
4124
4125 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4126}
4127
4128static ssize_t
4129trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4130 loff_t *ppos)
4131{
4132 long index = (long)filp->private_data;
4133 char buf[64];
4134 unsigned long val;
4135 int ret;
4136
4137 if (cnt >= sizeof(buf))
4138 return -EINVAL;
4139
4140 if (copy_from_user(&buf, ubuf, cnt))
4141 return -EFAULT;
4142
4143 buf[cnt] = 0;
4144
4145 ret = strict_strtoul(buf, 10, &val);
4146 if (ret < 0)
4147 return ret;
4148
Zhaoleif2d84b62009-08-07 18:55:48 +08004149 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05004150 return -EINVAL;
Zhaoleif2d84b62009-08-07 18:55:48 +08004151 set_tracer_flags(1 << index, val);
Steven Rostedta8259072009-02-26 22:19:12 -05004152
4153 *ppos += cnt;
4154
4155 return cnt;
4156}
4157
Steven Rostedta8259072009-02-26 22:19:12 -05004158static const struct file_operations trace_options_core_fops = {
4159 .open = tracing_open_generic,
4160 .read = trace_options_core_read,
4161 .write = trace_options_core_write,
4162};
4163
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004164struct dentry *trace_create_file(const char *name,
4165 mode_t mode,
4166 struct dentry *parent,
4167 void *data,
4168 const struct file_operations *fops)
4169{
4170 struct dentry *ret;
4171
4172 ret = debugfs_create_file(name, mode, parent, data, fops);
4173 if (!ret)
4174 pr_warning("Could not create debugfs '%s' entry\n", name);
4175
4176 return ret;
4177}
4178
4179
Steven Rostedta8259072009-02-26 22:19:12 -05004180static struct dentry *trace_options_init_dentry(void)
4181{
4182 struct dentry *d_tracer;
4183 static struct dentry *t_options;
4184
4185 if (t_options)
4186 return t_options;
4187
4188 d_tracer = tracing_init_dentry();
4189 if (!d_tracer)
4190 return NULL;
4191
4192 t_options = debugfs_create_dir("options", d_tracer);
4193 if (!t_options) {
4194 pr_warning("Could not create debugfs directory 'options'\n");
4195 return NULL;
4196 }
4197
4198 return t_options;
4199}
4200
Steven Rostedt577b7852009-02-26 23:43:05 -05004201static void
4202create_trace_option_file(struct trace_option_dentry *topt,
4203 struct tracer_flags *flags,
4204 struct tracer_opt *opt)
4205{
4206 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05004207
4208 t_options = trace_options_init_dentry();
4209 if (!t_options)
4210 return;
4211
4212 topt->flags = flags;
4213 topt->opt = opt;
4214
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004215 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05004216 &trace_options_fops);
4217
Steven Rostedt577b7852009-02-26 23:43:05 -05004218}
4219
4220static struct trace_option_dentry *
4221create_trace_option_files(struct tracer *tracer)
4222{
4223 struct trace_option_dentry *topts;
4224 struct tracer_flags *flags;
4225 struct tracer_opt *opts;
4226 int cnt;
4227
4228 if (!tracer)
4229 return NULL;
4230
4231 flags = tracer->flags;
4232
4233 if (!flags || !flags->opts)
4234 return NULL;
4235
4236 opts = flags->opts;
4237
4238 for (cnt = 0; opts[cnt].name; cnt++)
4239 ;
4240
Steven Rostedt0cfe8242009-02-27 10:51:10 -05004241 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05004242 if (!topts)
4243 return NULL;
4244
4245 for (cnt = 0; opts[cnt].name; cnt++)
4246 create_trace_option_file(&topts[cnt], flags,
4247 &opts[cnt]);
4248
4249 return topts;
4250}
4251
4252static void
4253destroy_trace_option_files(struct trace_option_dentry *topts)
4254{
4255 int cnt;
4256
4257 if (!topts)
4258 return;
4259
4260 for (cnt = 0; topts[cnt].opt; cnt++) {
4261 if (topts[cnt].entry)
4262 debugfs_remove(topts[cnt].entry);
4263 }
4264
4265 kfree(topts);
4266}
4267
Steven Rostedta8259072009-02-26 22:19:12 -05004268static struct dentry *
4269create_trace_option_core_file(const char *option, long index)
4270{
4271 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004272
4273 t_options = trace_options_init_dentry();
4274 if (!t_options)
4275 return NULL;
4276
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004277 return trace_create_file(option, 0644, t_options, (void *)index,
Steven Rostedta8259072009-02-26 22:19:12 -05004278 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05004279}
4280
4281static __init void create_trace_options_dir(void)
4282{
4283 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004284 int i;
4285
4286 t_options = trace_options_init_dentry();
4287 if (!t_options)
4288 return;
4289
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004290 for (i = 0; trace_options[i]; i++)
4291 create_trace_option_core_file(trace_options[i], i);
Steven Rostedta8259072009-02-26 22:19:12 -05004292}
4293
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004294static __init int tracer_init_debugfs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004295{
4296 struct dentry *d_tracer;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004297 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004298
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004299 trace_access_lock_init();
4300
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004301 d_tracer = tracing_init_dentry();
4302
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004303 trace_create_file("tracing_enabled", 0644, d_tracer,
4304 &global_trace, &tracing_ctrl_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004305
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004306 trace_create_file("trace_options", 0644, d_tracer,
4307 NULL, &tracing_iter_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004308
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004309 trace_create_file("tracing_cpumask", 0644, d_tracer,
4310 NULL, &tracing_cpumask_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05004311
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004312 trace_create_file("trace", 0644, d_tracer,
4313 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
Ingo Molnarc7078de2008-05-12 21:20:52 +02004314
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004315 trace_create_file("available_tracers", 0444, d_tracer,
4316 &global_trace, &show_traces_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004317
Li Zefan339ae5d2009-04-17 10:34:30 +08004318 trace_create_file("current_tracer", 0644, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004319 &global_trace, &set_tracer_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004320
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04004321#ifdef CONFIG_TRACER_MAX_TRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004322 trace_create_file("tracing_max_latency", 0644, d_tracer,
4323 &tracing_max_latency, &tracing_max_lat_fops);
Tim Bird0e950172010-02-25 15:36:43 -08004324#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004325
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004326 trace_create_file("tracing_thresh", 0644, d_tracer,
4327 &tracing_thresh, &tracing_max_lat_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004328
Li Zefan339ae5d2009-04-17 10:34:30 +08004329 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004330 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02004331
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004332 trace_create_file("trace_pipe", 0444, d_tracer,
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004333 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004334
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004335 trace_create_file("buffer_size_kb", 0644, d_tracer,
4336 &global_trace, &tracing_entries_fops);
Steven Rostedta98a3c32008-05-12 21:20:59 +02004337
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004338 trace_create_file("trace_marker", 0220, d_tracer,
4339 NULL, &tracing_mark_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004340
Avadh Patel69abe6a2009-04-10 16:04:48 -04004341 trace_create_file("saved_cmdlines", 0444, d_tracer,
4342 NULL, &tracing_saved_cmdlines_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004343
Zhaolei5079f322009-08-25 16:12:56 +08004344 trace_create_file("trace_clock", 0644, d_tracer, NULL,
4345 &trace_clock_fops);
4346
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004347#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004348 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4349 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004350#endif
Ingo Molnard618b3e2008-05-12 21:20:49 +02004351#ifdef CONFIG_SYSPROF_TRACER
4352 init_tracer_sysprof_debugfs(d_tracer);
4353#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004354
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004355 create_trace_options_dir();
4356
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004357 for_each_tracing_cpu(cpu)
4358 tracing_init_debugfs_percpu(cpu);
4359
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004360 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004361}
4362
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004363static int trace_panic_handler(struct notifier_block *this,
4364 unsigned long event, void *unused)
4365{
Steven Rostedt944ac422008-10-23 19:26:08 -04004366 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004367 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004368 return NOTIFY_OK;
4369}
4370
4371static struct notifier_block trace_panic_notifier = {
4372 .notifier_call = trace_panic_handler,
4373 .next = NULL,
4374 .priority = 150 /* priority: INT_MAX >= x >= 0 */
4375};
4376
4377static int trace_die_handler(struct notifier_block *self,
4378 unsigned long val,
4379 void *data)
4380{
4381 switch (val) {
4382 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04004383 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004384 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004385 break;
4386 default:
4387 break;
4388 }
4389 return NOTIFY_OK;
4390}
4391
4392static struct notifier_block trace_die_notifier = {
4393 .notifier_call = trace_die_handler,
4394 .priority = 200
4395};
4396
4397/*
4398 * printk is set to max of 1024, we really don't need it that big.
4399 * Nothing should be printing 1000 characters anyway.
4400 */
4401#define TRACE_MAX_PRINT 1000
4402
4403/*
4404 * Define here KERN_TRACE so that we have one place to modify
4405 * it if we decide to change what log level the ftrace dump
4406 * should be at.
4407 */
Steven Rostedt428aee12009-01-14 12:24:42 -05004408#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004409
4410static void
4411trace_printk_seq(struct trace_seq *s)
4412{
4413 /* Probably should print a warning here. */
4414 if (s->len >= 1000)
4415 s->len = 1000;
4416
4417 /* should be zero ended, but we are paranoid. */
4418 s->buffer[s->len] = 0;
4419
4420 printk(KERN_TRACE "%s", s->buffer);
4421
Steven Rostedtf9520752009-03-02 14:04:40 -05004422 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004423}
4424
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004425static void
4426__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004427{
Thomas Gleixner445c8952009-12-02 19:49:50 +01004428 static arch_spinlock_t ftrace_dump_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01004429 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004430 /* use static because iter can be a bit big for the stack */
4431 static struct trace_iterator iter;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004432 unsigned int old_userobj;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004433 static int dump_ran;
Steven Rostedtd7690412008-10-01 00:29:53 -04004434 unsigned long flags;
4435 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004436
4437 /* only one dump */
Steven Rostedtcd891ae2009-04-28 11:39:34 -04004438 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01004439 arch_spin_lock(&ftrace_dump_lock);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004440 if (dump_ran)
4441 goto out;
4442
4443 dump_ran = 1;
4444
Steven Rostedt0ee6b6c2009-01-14 14:50:19 -05004445 tracing_off();
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004446
4447 if (disable_tracing)
4448 ftrace_kill();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004449
Steven Rostedtd7690412008-10-01 00:29:53 -04004450 for_each_tracing_cpu(cpu) {
4451 atomic_inc(&global_trace.data[cpu]->disabled);
4452 }
4453
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004454 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4455
Török Edwinb54d3de2008-11-22 13:28:48 +02004456 /* don't look at user memory in panic mode */
4457 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4458
Steven Rostedte543ad72009-03-04 18:20:36 -05004459 /* Simulate the iterator */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004460 iter.tr = &global_trace;
4461 iter.trace = current_trace;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004462
4463 switch (oops_dump_mode) {
4464 case DUMP_ALL:
4465 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4466 break;
4467 case DUMP_ORIG:
4468 iter.cpu_file = raw_smp_processor_id();
4469 break;
4470 case DUMP_NONE:
4471 goto out_enable;
4472 default:
4473 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
4474 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4475 }
4476
4477 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004478
4479 /*
4480 * We need to stop all tracing on all CPUS to read the
4481 * the next buffer. This is a bit expensive, but is
4482 * not done often. We fill all what we can read,
4483 * and then release the locks again.
4484 */
4485
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004486 while (!trace_empty(&iter)) {
4487
4488 if (!cnt)
4489 printk(KERN_TRACE "---------------------------------\n");
4490
4491 cnt++;
4492
4493 /* reset all but tr, trace, and overruns */
4494 memset(&iter.seq, 0,
4495 sizeof(struct trace_iterator) -
4496 offsetof(struct trace_iterator, seq));
4497 iter.iter_flags |= TRACE_FILE_LAT_FMT;
4498 iter.pos = -1;
4499
4500 if (find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08004501 int ret;
4502
4503 ret = print_trace_line(&iter);
4504 if (ret != TRACE_TYPE_NO_CONSUME)
4505 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004506 }
4507
4508 trace_printk_seq(&iter.seq);
4509 }
4510
4511 if (!cnt)
4512 printk(KERN_TRACE " (ftrace buffer empty)\n");
4513 else
4514 printk(KERN_TRACE "---------------------------------\n");
4515
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004516 out_enable:
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004517 /* Re-enable tracing if requested */
4518 if (!disable_tracing) {
4519 trace_flags |= old_userobj;
4520
4521 for_each_tracing_cpu(cpu) {
4522 atomic_dec(&global_trace.data[cpu]->disabled);
4523 }
4524 tracing_on();
4525 }
4526
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004527 out:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01004528 arch_spin_unlock(&ftrace_dump_lock);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04004529 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004530}
4531
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004532/* By default: disable tracing after the dump */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004533void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004534{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004535 __ftrace_dump(true, oops_dump_mode);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004536}
4537
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004538__init static int tracer_alloc_buffers(void)
4539{
Steven Rostedt73c51622009-03-11 13:42:01 -04004540 int ring_buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004541 int i;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304542 int ret = -ENOMEM;
4543
4544 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4545 goto out;
4546
4547 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4548 goto out_free_buffer_mask;
4549
Steven Rostedt73c51622009-03-11 13:42:01 -04004550 /* To save memory, keep the ring buffer size to its minimum */
4551 if (ring_buffer_expanded)
4552 ring_buf_size = trace_buf_size;
4553 else
4554 ring_buf_size = 1;
4555
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304556 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4557 cpumask_copy(tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004558
Steven Rostedtab464282008-05-12 21:21:00 +02004559 /* TODO: make the number of buffers hot pluggable with CPUS */
Steven Rostedt73c51622009-03-11 13:42:01 -04004560 global_trace.buffer = ring_buffer_alloc(ring_buf_size,
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004561 TRACE_BUFFER_FLAGS);
4562 if (!global_trace.buffer) {
4563 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4564 WARN_ON(1);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304565 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004566 }
4567 global_trace.entries = ring_buffer_size(global_trace.buffer);
4568
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304569
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004570#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt73c51622009-03-11 13:42:01 -04004571 max_tr.buffer = ring_buffer_alloc(ring_buf_size,
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004572 TRACE_BUFFER_FLAGS);
4573 if (!max_tr.buffer) {
4574 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4575 WARN_ON(1);
4576 ring_buffer_free(global_trace.buffer);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304577 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004578 }
4579 max_tr.entries = ring_buffer_size(max_tr.buffer);
4580 WARN_ON(max_tr.entries != global_trace.entries);
4581#endif
4582
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02004583 /* Allocate the first page for all buffers */
Steven Rostedtab464282008-05-12 21:21:00 +02004584 for_each_tracing_cpu(i) {
jolsa@redhat.com566b0aa2009-07-16 21:44:26 +02004585 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
Tejun Heo9705f692009-10-29 22:34:13 +09004586 max_tr.data[i] = &per_cpu(max_tr_data, i);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004587 }
4588
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004589 trace_init_cmdlines();
4590
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02004591 register_tracer(&nop_trace);
Steven Rostedt79fb0762009-02-02 21:38:33 -05004592 current_trace = &nop_trace;
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004593#ifdef CONFIG_BOOT_TRACER
4594 register_tracer(&boot_tracer);
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004595#endif
Steven Rostedt60a11772008-05-12 21:20:44 +02004596 /* All seems OK, enable tracing */
4597 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004598
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004599 atomic_notifier_chain_register(&panic_notifier_list,
4600 &trace_panic_notifier);
4601
4602 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01004603
4604 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004605
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304606out_free_cpumask:
4607 free_cpumask_var(tracing_cpumask);
4608out_free_buffer_mask:
4609 free_cpumask_var(tracing_buffer_mask);
4610out:
4611 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004612}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05004613
4614__init static int clear_boot_tracer(void)
4615{
4616 /*
4617 * The default tracer at boot buffer is an init section.
4618 * This function is called in lateinit. If we did not
4619 * find the boot tracer, then clear it out, to prevent
4620 * later registration from accessing the buffer that is
4621 * about to be freed.
4622 */
4623 if (!default_bootup_tracer)
4624 return 0;
4625
4626 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4627 default_bootup_tracer);
4628 default_bootup_tracer = NULL;
4629
4630 return 0;
4631}
4632
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004633early_initcall(tracer_alloc_buffers);
4634fs_initcall(tracer_init_debugfs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05004635late_initcall(clear_boot_tracer);