blob: 26185d727676b0620496a9eb5a4832e9c3fdbd01 [file] [log] [blame]
Steven Rostedt352ad252008-05-12 21:20:42 +02001/*
2 * trace task wakeup timings
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Based on code from the latency_tracer, that is:
8 *
9 * Copyright (C) 2004-2006 Ingo Molnar
10 * Copyright (C) 2004 William Lee Irwin III
11 */
12#include <linux/module.h>
13#include <linux/fs.h>
14#include <linux/debugfs.h>
15#include <linux/kallsyms.h>
16#include <linux/uaccess.h>
17#include <linux/ftrace.h>
Steven Rostedtad8d75f2009-04-14 19:39:12 -040018#include <trace/events/sched.h>
Steven Rostedt352ad252008-05-12 21:20:42 +020019
20#include "trace.h"
21
22static struct trace_array *wakeup_trace;
23static int __read_mostly tracer_enabled;
24
25static struct task_struct *wakeup_task;
26static int wakeup_cpu;
Steven Rostedt478142c32009-09-09 10:36:01 -040027static int wakeup_current_cpu;
Steven Rostedt352ad252008-05-12 21:20:42 +020028static unsigned wakeup_prio = -1;
Steven Rostedt32443512009-01-21 16:24:46 -050029static int wakeup_rt;
Steven Rostedt352ad252008-05-12 21:20:42 +020030
Steven Rostedte59494f2008-07-16 00:13:45 -040031static raw_spinlock_t wakeup_lock =
32 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
Steven Rostedt352ad252008-05-12 21:20:42 +020033
Ingo Molnare309b412008-05-12 21:20:51 +020034static void __wakeup_reset(struct trace_array *tr);
Steven Rostedt352ad252008-05-12 21:20:42 +020035
Steven Rostedte9d25fe2009-03-04 22:15:30 -050036static int save_lat_flag;
37
Steven Rostedt606576c2008-10-06 19:06:12 -040038#ifdef CONFIG_FUNCTION_TRACER
Steven Rostedt7e18d8e2008-05-22 00:22:19 -040039/*
40 * irqsoff uses its own tracer function to keep the overhead down:
41 */
42static void
43wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
44{
45 struct trace_array *tr = wakeup_trace;
46 struct trace_array_cpu *data;
47 unsigned long flags;
48 long disabled;
49 int resched;
50 int cpu;
Steven Rostedt38697052008-10-01 13:14:09 -040051 int pc;
Steven Rostedt7e18d8e2008-05-22 00:22:19 -040052
53 if (likely(!wakeup_task))
54 return;
55
Steven Rostedt38697052008-10-01 13:14:09 -040056 pc = preempt_count();
Steven Rostedt182e9f52008-11-03 23:15:56 -050057 resched = ftrace_preempt_disable();
Steven Rostedt7e18d8e2008-05-22 00:22:19 -040058
59 cpu = raw_smp_processor_id();
Steven Rostedt478142c32009-09-09 10:36:01 -040060 if (cpu != wakeup_current_cpu)
61 goto out_enable;
62
Steven Rostedt7e18d8e2008-05-22 00:22:19 -040063 data = tr->data[cpu];
64 disabled = atomic_inc_return(&data->disabled);
65 if (unlikely(disabled != 1))
66 goto out;
67
Steven Rostedte59494f2008-07-16 00:13:45 -040068 local_irq_save(flags);
Steven Rostedt7e18d8e2008-05-22 00:22:19 -040069
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -050070 trace_function(tr, ip, parent_ip, flags, pc);
Steven Rostedt7e18d8e2008-05-22 00:22:19 -040071
Steven Rostedte59494f2008-07-16 00:13:45 -040072 local_irq_restore(flags);
Steven Rostedt7e18d8e2008-05-22 00:22:19 -040073
74 out:
75 atomic_dec(&data->disabled);
Steven Rostedt478142c32009-09-09 10:36:01 -040076 out_enable:
Steven Rostedt182e9f52008-11-03 23:15:56 -050077 ftrace_preempt_enable(resched);
Steven Rostedt7e18d8e2008-05-22 00:22:19 -040078}
79
80static struct ftrace_ops trace_ops __read_mostly =
81{
82 .func = wakeup_tracer_call,
83};
Steven Rostedt606576c2008-10-06 19:06:12 -040084#endif /* CONFIG_FUNCTION_TRACER */
Steven Rostedt7e18d8e2008-05-22 00:22:19 -040085
Steven Rostedt352ad252008-05-12 21:20:42 +020086/*
87 * Should this new latency be reported/recorded?
88 */
Ingo Molnare309b412008-05-12 21:20:51 +020089static int report_latency(cycle_t delta)
Steven Rostedt352ad252008-05-12 21:20:42 +020090{
91 if (tracing_thresh) {
92 if (delta < tracing_thresh)
93 return 0;
94 } else {
95 if (delta <= tracing_max_latency)
96 return 0;
97 }
98 return 1;
99}
100
Steven Rostedt478142c32009-09-09 10:36:01 -0400101static void probe_wakeup_migrate_task(struct task_struct *task, int cpu)
102{
103 if (task != wakeup_task)
104 return;
105
106 wakeup_current_cpu = cpu;
107}
108
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200109static void notrace
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400110probe_wakeup_sched_switch(struct rq *rq, struct task_struct *prev,
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200111 struct task_struct *next)
Steven Rostedt352ad252008-05-12 21:20:42 +0200112{
Steven Rostedt352ad252008-05-12 21:20:42 +0200113 struct trace_array_cpu *data;
114 cycle_t T0, T1, delta;
115 unsigned long flags;
116 long disabled;
117 int cpu;
Steven Rostedt38697052008-10-01 13:14:09 -0400118 int pc;
Steven Rostedt352ad252008-05-12 21:20:42 +0200119
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400120 tracing_record_cmdline(prev);
121
Steven Rostedt352ad252008-05-12 21:20:42 +0200122 if (unlikely(!tracer_enabled))
123 return;
124
125 /*
126 * When we start a new trace, we set wakeup_task to NULL
127 * and then set tracer_enabled = 1. We want to make sure
128 * that another CPU does not see the tracer_enabled = 1
129 * and the wakeup_task with an older task, that might
130 * actually be the same as next.
131 */
132 smp_rmb();
133
134 if (next != wakeup_task)
135 return;
136
Steven Rostedt38697052008-10-01 13:14:09 -0400137 pc = preempt_count();
138
Steven Rostedt352ad252008-05-12 21:20:42 +0200139 /* disable local data, not wakeup_cpu data */
140 cpu = raw_smp_processor_id();
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400141 disabled = atomic_inc_return(&wakeup_trace->data[cpu]->disabled);
Steven Rostedt352ad252008-05-12 21:20:42 +0200142 if (likely(disabled != 1))
143 goto out;
144
Steven Rostedte59494f2008-07-16 00:13:45 -0400145 local_irq_save(flags);
146 __raw_spin_lock(&wakeup_lock);
Steven Rostedt352ad252008-05-12 21:20:42 +0200147
148 /* We could race with grabbing wakeup_lock */
149 if (unlikely(!tracer_enabled || next != wakeup_task))
150 goto out_unlock;
151
Steven Rostedt9be24412009-03-26 10:25:24 -0400152 /* The task we are waiting for is waking up */
153 data = wakeup_trace->data[wakeup_cpu];
154
Steven Rostedt301fd742009-04-03 11:12:23 -0400155 trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc);
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -0500156 tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc);
Steven Rostedt352ad252008-05-12 21:20:42 +0200157
Steven Rostedt352ad252008-05-12 21:20:42 +0200158 T0 = data->preempt_timestamp;
Ingo Molnar750ed1a2008-05-12 21:20:46 +0200159 T1 = ftrace_now(cpu);
Steven Rostedt352ad252008-05-12 21:20:42 +0200160 delta = T1-T0;
161
162 if (!report_latency(delta))
163 goto out_unlock;
164
Carsten Emdeb5130b12009-09-13 01:43:07 +0200165 if (likely(!is_tracing_stopped())) {
166 tracing_max_latency = delta;
167 update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu);
168 }
Steven Rostedt352ad252008-05-12 21:20:42 +0200169
Steven Rostedt352ad252008-05-12 21:20:42 +0200170out_unlock:
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400171 __wakeup_reset(wakeup_trace);
Steven Rostedte59494f2008-07-16 00:13:45 -0400172 __raw_spin_unlock(&wakeup_lock);
173 local_irq_restore(flags);
Steven Rostedt352ad252008-05-12 21:20:42 +0200174out:
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400175 atomic_dec(&wakeup_trace->data[cpu]->disabled);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200176}
177
Ingo Molnare309b412008-05-12 21:20:51 +0200178static void __wakeup_reset(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200179{
Steven Rostedt352ad252008-05-12 21:20:42 +0200180 wakeup_cpu = -1;
181 wakeup_prio = -1;
182
183 if (wakeup_task)
184 put_task_struct(wakeup_task);
185
186 wakeup_task = NULL;
187}
188
Ingo Molnare309b412008-05-12 21:20:51 +0200189static void wakeup_reset(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200190{
191 unsigned long flags;
192
Steven Rostedt2f26ebd2009-09-01 11:06:29 -0400193 tracing_reset_online_cpus(tr);
194
Steven Rostedte59494f2008-07-16 00:13:45 -0400195 local_irq_save(flags);
196 __raw_spin_lock(&wakeup_lock);
Steven Rostedt352ad252008-05-12 21:20:42 +0200197 __wakeup_reset(tr);
Steven Rostedte59494f2008-07-16 00:13:45 -0400198 __raw_spin_unlock(&wakeup_lock);
199 local_irq_restore(flags);
Steven Rostedt352ad252008-05-12 21:20:42 +0200200}
201
Ingo Molnare309b412008-05-12 21:20:51 +0200202static void
Peter Zijlstra468a15b2008-12-16 08:07:03 +0100203probe_wakeup(struct rq *rq, struct task_struct *p, int success)
Steven Rostedt352ad252008-05-12 21:20:42 +0200204{
Steven Rostedtf8ec1062009-01-21 17:17:04 -0500205 struct trace_array_cpu *data;
Steven Rostedt352ad252008-05-12 21:20:42 +0200206 int cpu = smp_processor_id();
207 unsigned long flags;
208 long disabled;
Steven Rostedt38697052008-10-01 13:14:09 -0400209 int pc;
Steven Rostedt352ad252008-05-12 21:20:42 +0200210
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400211 if (likely(!tracer_enabled))
Steven Rostedt352ad252008-05-12 21:20:42 +0200212 return;
213
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400214 tracing_record_cmdline(p);
215 tracing_record_cmdline(current);
216
Steven Rostedt32443512009-01-21 16:24:46 -0500217 if ((wakeup_rt && !rt_task(p)) ||
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400218 p->prio >= wakeup_prio ||
219 p->prio >= current->prio)
220 return;
221
Steven Rostedt38697052008-10-01 13:14:09 -0400222 pc = preempt_count();
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400223 disabled = atomic_inc_return(&wakeup_trace->data[cpu]->disabled);
Steven Rostedt352ad252008-05-12 21:20:42 +0200224 if (unlikely(disabled != 1))
225 goto out;
226
227 /* interrupts should be off from try_to_wake_up */
Steven Rostedte59494f2008-07-16 00:13:45 -0400228 __raw_spin_lock(&wakeup_lock);
Steven Rostedt352ad252008-05-12 21:20:42 +0200229
230 /* check for races. */
231 if (!tracer_enabled || p->prio >= wakeup_prio)
232 goto out_locked;
233
234 /* reset the trace */
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400235 __wakeup_reset(wakeup_trace);
Steven Rostedt352ad252008-05-12 21:20:42 +0200236
237 wakeup_cpu = task_cpu(p);
Steven Rostedt478142c32009-09-09 10:36:01 -0400238 wakeup_current_cpu = wakeup_cpu;
Steven Rostedt352ad252008-05-12 21:20:42 +0200239 wakeup_prio = p->prio;
240
241 wakeup_task = p;
242 get_task_struct(wakeup_task);
243
244 local_save_flags(flags);
245
Steven Rostedtf8ec1062009-01-21 17:17:04 -0500246 data = wakeup_trace->data[wakeup_cpu];
247 data->preempt_timestamp = ftrace_now(cpu);
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -0500248 tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc);
Steven Rostedt301fd742009-04-03 11:12:23 -0400249
250 /*
251 * We must be careful in using CALLER_ADDR2. But since wake_up
252 * is not called by an assembly function (where as schedule is)
253 * it should be safe to use it here.
254 */
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -0500255 trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc);
Steven Rostedt352ad252008-05-12 21:20:42 +0200256
257out_locked:
Steven Rostedte59494f2008-07-16 00:13:45 -0400258 __raw_spin_unlock(&wakeup_lock);
Steven Rostedt352ad252008-05-12 21:20:42 +0200259out:
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400260 atomic_dec(&wakeup_trace->data[cpu]->disabled);
Steven Rostedt352ad252008-05-12 21:20:42 +0200261}
262
Ingo Molnare309b412008-05-12 21:20:51 +0200263static void start_wakeup_tracer(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200264{
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200265 int ret;
266
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400267 ret = register_trace_sched_wakeup(probe_wakeup);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200268 if (ret) {
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400269 pr_info("wakeup trace: Couldn't activate tracepoint"
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200270 " probe to kernel_sched_wakeup\n");
271 return;
272 }
273
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400274 ret = register_trace_sched_wakeup_new(probe_wakeup);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200275 if (ret) {
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400276 pr_info("wakeup trace: Couldn't activate tracepoint"
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200277 " probe to kernel_sched_wakeup_new\n");
278 goto fail_deprobe;
279 }
280
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400281 ret = register_trace_sched_switch(probe_wakeup_sched_switch);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200282 if (ret) {
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400283 pr_info("sched trace: Couldn't activate tracepoint"
Wenji Huang73d8b8b2009-02-17 01:10:02 -0500284 " probe to kernel_sched_switch\n");
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200285 goto fail_deprobe_wake_new;
286 }
287
Steven Rostedt478142c32009-09-09 10:36:01 -0400288 ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task);
289 if (ret) {
290 pr_info("wakeup trace: Couldn't activate tracepoint"
291 " probe to kernel_sched_migrate_task\n");
292 return;
293 }
294
Steven Rostedt352ad252008-05-12 21:20:42 +0200295 wakeup_reset(tr);
296
297 /*
298 * Don't let the tracer_enabled = 1 show up before
299 * the wakeup_task is reset. This may be overkill since
300 * wakeup_reset does a spin_unlock after setting the
301 * wakeup_task to NULL, but I want to be safe.
302 * This is a slow path anyway.
303 */
304 smp_wmb();
305
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400306 register_ftrace_function(&trace_ops);
Steven Rostedt352ad252008-05-12 21:20:42 +0200307
Steven Rostedt5bc45642009-01-21 14:36:52 -0500308 if (tracing_is_enabled())
Steven Rostedt90369902008-11-05 16:05:44 -0500309 tracer_enabled = 1;
Steven Rostedt5bc45642009-01-21 14:36:52 -0500310 else
Steven Rostedt90369902008-11-05 16:05:44 -0500311 tracer_enabled = 0;
Steven Rostedtad591242008-07-10 20:58:13 -0400312
Steven Rostedt352ad252008-05-12 21:20:42 +0200313 return;
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200314fail_deprobe_wake_new:
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400315 unregister_trace_sched_wakeup_new(probe_wakeup);
Mathieu Desnoyers5b82a1b2008-05-12 21:21:10 +0200316fail_deprobe:
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400317 unregister_trace_sched_wakeup(probe_wakeup);
Steven Rostedt352ad252008-05-12 21:20:42 +0200318}
319
Ingo Molnare309b412008-05-12 21:20:51 +0200320static void stop_wakeup_tracer(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200321{
322 tracer_enabled = 0;
Steven Rostedt7e18d8e2008-05-22 00:22:19 -0400323 unregister_ftrace_function(&trace_ops);
Mathieu Desnoyersb07c3f12008-07-18 12:16:17 -0400324 unregister_trace_sched_switch(probe_wakeup_sched_switch);
325 unregister_trace_sched_wakeup_new(probe_wakeup);
326 unregister_trace_sched_wakeup(probe_wakeup);
Steven Rostedt478142c32009-09-09 10:36:01 -0400327 unregister_trace_sched_migrate_task(probe_wakeup_migrate_task);
Steven Rostedt352ad252008-05-12 21:20:42 +0200328}
329
Steven Rostedt32443512009-01-21 16:24:46 -0500330static int __wakeup_tracer_init(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200331{
Steven Rostedte9d25fe2009-03-04 22:15:30 -0500332 save_lat_flag = trace_flags & TRACE_ITER_LATENCY_FMT;
333 trace_flags |= TRACE_ITER_LATENCY_FMT;
334
Steven Rostedt745b1622009-01-15 23:40:11 -0500335 tracing_max_latency = 0;
Steven Rostedt352ad252008-05-12 21:20:42 +0200336 wakeup_trace = tr;
Steven Rostedtc76f0692008-11-07 22:36:02 -0500337 start_wakeup_tracer(tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +0100338 return 0;
Steven Rostedt352ad252008-05-12 21:20:42 +0200339}
340
Steven Rostedt32443512009-01-21 16:24:46 -0500341static int wakeup_tracer_init(struct trace_array *tr)
342{
343 wakeup_rt = 0;
344 return __wakeup_tracer_init(tr);
345}
346
347static int wakeup_rt_tracer_init(struct trace_array *tr)
348{
349 wakeup_rt = 1;
350 return __wakeup_tracer_init(tr);
351}
352
Ingo Molnare309b412008-05-12 21:20:51 +0200353static void wakeup_tracer_reset(struct trace_array *tr)
Steven Rostedt352ad252008-05-12 21:20:42 +0200354{
Steven Rostedtc76f0692008-11-07 22:36:02 -0500355 stop_wakeup_tracer(tr);
356 /* make sure we put back any tasks we are tracing */
357 wakeup_reset(tr);
Steven Rostedte9d25fe2009-03-04 22:15:30 -0500358
359 if (!save_lat_flag)
360 trace_flags &= ~TRACE_ITER_LATENCY_FMT;
Steven Rostedt352ad252008-05-12 21:20:42 +0200361}
362
Steven Rostedt90369902008-11-05 16:05:44 -0500363static void wakeup_tracer_start(struct trace_array *tr)
364{
365 wakeup_reset(tr);
366 tracer_enabled = 1;
Steven Rostedt90369902008-11-05 16:05:44 -0500367}
368
369static void wakeup_tracer_stop(struct trace_array *tr)
370{
371 tracer_enabled = 0;
Steven Rostedt352ad252008-05-12 21:20:42 +0200372}
373
374static struct tracer wakeup_tracer __read_mostly =
375{
376 .name = "wakeup",
377 .init = wakeup_tracer_init,
378 .reset = wakeup_tracer_reset,
Steven Rostedt90369902008-11-05 16:05:44 -0500379 .start = wakeup_tracer_start,
380 .stop = wakeup_tracer_stop,
Steven Rostedt352ad252008-05-12 21:20:42 +0200381 .print_max = 1,
Steven Rostedt60a11772008-05-12 21:20:44 +0200382#ifdef CONFIG_FTRACE_SELFTEST
383 .selftest = trace_selftest_startup_wakeup,
384#endif
Steven Rostedt352ad252008-05-12 21:20:42 +0200385};
386
Steven Rostedt32443512009-01-21 16:24:46 -0500387static struct tracer wakeup_rt_tracer __read_mostly =
388{
389 .name = "wakeup_rt",
390 .init = wakeup_rt_tracer_init,
391 .reset = wakeup_tracer_reset,
392 .start = wakeup_tracer_start,
393 .stop = wakeup_tracer_stop,
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +0100394 .wait_pipe = poll_wait_pipe,
Steven Rostedt32443512009-01-21 16:24:46 -0500395 .print_max = 1,
396#ifdef CONFIG_FTRACE_SELFTEST
397 .selftest = trace_selftest_startup_wakeup,
398#endif
399};
400
Steven Rostedt352ad252008-05-12 21:20:42 +0200401__init static int init_wakeup_tracer(void)
402{
403 int ret;
404
405 ret = register_tracer(&wakeup_tracer);
406 if (ret)
407 return ret;
408
Steven Rostedt32443512009-01-21 16:24:46 -0500409 ret = register_tracer(&wakeup_rt_tracer);
410 if (ret)
411 return ret;
412
Steven Rostedt352ad252008-05-12 21:20:42 +0200413 return 0;
414}
415device_initcall(init_wakeup_tracer);