blob: 258885a543db60692a695aa682abb6c95f3ffcdf [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/softirq.c
3 *
4 * Copyright (C) 1992 Linus Torvalds
5 *
Pavel Machekb10db7f2008-01-30 13:30:00 +01006 * Distribute under GPLv2.
7 *
8 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
David S. Miller54514a72008-09-23 22:15:57 -07009 *
10 * Remote softirq infrastructure is by Jens Axboe.
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 */
12
13#include <linux/module.h>
14#include <linux/kernel_stat.h>
15#include <linux/interrupt.h>
16#include <linux/init.h>
17#include <linux/mm.h>
18#include <linux/notifier.h>
19#include <linux/percpu.h>
20#include <linux/cpu.h>
Rafael J. Wysocki83144182007-07-17 04:03:35 -070021#include <linux/freezer.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/kthread.h>
23#include <linux/rcupdate.h>
Steven Rostedt7e49fcc2009-01-22 19:01:40 -050024#include <linux/ftrace.h>
Andrew Morton78eef012006-03-22 00:08:16 -080025#include <linux/smp.h>
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -080026#include <linux/tick.h>
Heiko Carstensa0e39ed2009-04-29 13:51:39 +020027
28#define CREATE_TRACE_POINTS
Steven Rostedtad8d75f2009-04-14 19:39:12 -040029#include <trace/events/irq.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
31#include <asm/irq.h>
32/*
33 - No shared variables, all the data are CPU local.
34 - If a softirq needs serialization, let it serialize itself
35 by its own spinlocks.
36 - Even if softirq is serialized, only local cpu is marked for
37 execution. Hence, we get something sort of weak cpu binding.
38 Though it is still not clear, will it result in better locality
39 or will not.
40
41 Examples:
42 - NET RX softirq. It is multithreaded and does not require
43 any global serialization.
44 - NET TX softirq. It kicks software netdevice queues, hence
45 it is logically serialized per device, but this serialization
46 is invisible to common code.
47 - Tasklets: serialized wrt itself.
48 */
49
50#ifndef __ARCH_IRQ_STAT
51irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
52EXPORT_SYMBOL(irq_stat);
53#endif
54
Alexey Dobriyan978b0112008-09-06 20:04:36 +020055static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
57static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
58
Jason Baron5d592b42009-03-12 14:33:36 -040059char *softirq_to_name[NR_SOFTIRQS] = {
Steven Rostedt899039e2009-03-13 00:43:33 -040060 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK",
61 "TASKLET", "SCHED", "HRTIMER", "RCU"
Jason Baron5d592b42009-03-12 14:33:36 -040062};
63
Linus Torvalds1da177e2005-04-16 15:20:36 -070064/*
65 * we cannot loop indefinitely here to avoid userspace starvation,
66 * but we also don't want to introduce a worst case 1/HZ latency
67 * to the pending events, so lets the scheduler to balance
68 * the softirq load for us.
69 */
Peter Zijlstra7f1e2ca2009-03-13 12:21:27 +010070void wakeup_softirqd(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -070071{
72 /* Interrupts are disabled: no need to stop preemption */
73 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
74
75 if (tsk && tsk->state != TASK_RUNNING)
76 wake_up_process(tsk);
77}
78
79/*
Ingo Molnarde30a2b2006-07-03 00:24:42 -070080 * This one is for softirq.c-internal use,
81 * where hardirqs are disabled legitimately:
82 */
Tim Chen3c829c32006-07-30 03:04:02 -070083#ifdef CONFIG_TRACE_IRQFLAGS
Ingo Molnarde30a2b2006-07-03 00:24:42 -070084static void __local_bh_disable(unsigned long ip)
85{
86 unsigned long flags;
87
88 WARN_ON_ONCE(in_irq());
89
90 raw_local_irq_save(flags);
Steven Rostedt7e49fcc2009-01-22 19:01:40 -050091 /*
92 * The preempt tracer hooks into add_preempt_count and will break
93 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
94 * is set and before current->softirq_enabled is cleared.
95 * We must manually increment preempt_count here and manually
96 * call the trace_preempt_off later.
97 */
98 preempt_count() += SOFTIRQ_OFFSET;
Ingo Molnarde30a2b2006-07-03 00:24:42 -070099 /*
100 * Were softirqs turned off above:
101 */
102 if (softirq_count() == SOFTIRQ_OFFSET)
103 trace_softirqs_off(ip);
104 raw_local_irq_restore(flags);
Steven Rostedt7e49fcc2009-01-22 19:01:40 -0500105
106 if (preempt_count() == SOFTIRQ_OFFSET)
107 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700108}
Tim Chen3c829c32006-07-30 03:04:02 -0700109#else /* !CONFIG_TRACE_IRQFLAGS */
110static inline void __local_bh_disable(unsigned long ip)
111{
112 add_preempt_count(SOFTIRQ_OFFSET);
113 barrier();
114}
115#endif /* CONFIG_TRACE_IRQFLAGS */
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700116
117void local_bh_disable(void)
118{
119 __local_bh_disable((unsigned long)__builtin_return_address(0));
120}
121
122EXPORT_SYMBOL(local_bh_disable);
123
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700124/*
125 * Special-case - softirqs can safely be enabled in
126 * cond_resched_softirq(), or by __do_softirq(),
127 * without processing still-pending softirqs:
128 */
129void _local_bh_enable(void)
130{
131 WARN_ON_ONCE(in_irq());
132 WARN_ON_ONCE(!irqs_disabled());
133
134 if (softirq_count() == SOFTIRQ_OFFSET)
135 trace_softirqs_on((unsigned long)__builtin_return_address(0));
136 sub_preempt_count(SOFTIRQ_OFFSET);
137}
138
139EXPORT_SYMBOL(_local_bh_enable);
140
Johannes Berg0f476b6d2008-06-18 09:29:37 +0200141static inline void _local_bh_enable_ip(unsigned long ip)
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700142{
Johannes Berg0f476b6d2008-06-18 09:29:37 +0200143 WARN_ON_ONCE(in_irq() || irqs_disabled());
Tim Chen3c829c32006-07-30 03:04:02 -0700144#ifdef CONFIG_TRACE_IRQFLAGS
Johannes Berg0f476b6d2008-06-18 09:29:37 +0200145 local_irq_disable();
Tim Chen3c829c32006-07-30 03:04:02 -0700146#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700147 /*
148 * Are softirqs going to be turned on now:
149 */
150 if (softirq_count() == SOFTIRQ_OFFSET)
151 trace_softirqs_on(ip);
152 /*
153 * Keep preemption disabled until we are done with
154 * softirq processing:
155 */
156 sub_preempt_count(SOFTIRQ_OFFSET - 1);
157
158 if (unlikely(!in_interrupt() && local_softirq_pending()))
159 do_softirq();
160
161 dec_preempt_count();
Tim Chen3c829c32006-07-30 03:04:02 -0700162#ifdef CONFIG_TRACE_IRQFLAGS
Johannes Berg0f476b6d2008-06-18 09:29:37 +0200163 local_irq_enable();
Tim Chen3c829c32006-07-30 03:04:02 -0700164#endif
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700165 preempt_check_resched();
166}
Johannes Berg0f476b6d2008-06-18 09:29:37 +0200167
168void local_bh_enable(void)
169{
170 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
171}
172EXPORT_SYMBOL(local_bh_enable);
173
174void local_bh_enable_ip(unsigned long ip)
175{
176 _local_bh_enable_ip(ip);
177}
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700178EXPORT_SYMBOL(local_bh_enable_ip);
179
180/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
182 * and we fall back to softirqd after that.
183 *
184 * This number has been established via experimentation.
185 * The two things to balance is latency against fairness -
186 * we want to handle softirqs as soon as possible, but they
187 * should not be able to lock up the box.
188 */
189#define MAX_SOFTIRQ_RESTART 10
190
191asmlinkage void __do_softirq(void)
192{
193 struct softirq_action *h;
194 __u32 pending;
195 int max_restart = MAX_SOFTIRQ_RESTART;
196 int cpu;
197
198 pending = local_softirq_pending();
Paul Mackerras829035fd2006-07-03 00:25:40 -0700199 account_system_vtime(current);
200
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700201 __local_bh_disable((unsigned long)__builtin_return_address(0));
Ingo Molnard820ac42009-03-13 01:30:40 +0100202 lockdep_softirq_enter();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203
Linus Torvalds1da177e2005-04-16 15:20:36 -0700204 cpu = smp_processor_id();
205restart:
206 /* Reset the pending bitmask before enabling irqs */
Andi Kleen3f744782005-09-12 18:49:24 +0200207 set_softirq_pending(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700208
Andrew Mortonc70f5d62005-07-30 10:22:49 -0700209 local_irq_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700210
211 h = softirq_vec;
212
213 do {
214 if (pending & 1) {
Thomas Gleixner8e85b4b2008-10-02 10:50:53 +0200215 int prev_count = preempt_count();
216
Jason Baron39842322009-03-12 14:36:03 -0400217 trace_softirq_entry(h, softirq_vec);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218 h->action(h);
Jason Baron39842322009-03-12 14:36:03 -0400219 trace_softirq_exit(h, softirq_vec);
Thomas Gleixner8e85b4b2008-10-02 10:50:53 +0200220 if (unlikely(prev_count != preempt_count())) {
Jason Baron5d592b42009-03-12 14:33:36 -0400221 printk(KERN_ERR "huh, entered softirq %td %s %p"
Thomas Gleixner8e85b4b2008-10-02 10:50:53 +0200222 "with preempt_count %08x,"
223 " exited with %08x?\n", h - softirq_vec,
Jason Baron5d592b42009-03-12 14:33:36 -0400224 softirq_to_name[h - softirq_vec],
Thomas Gleixner8e85b4b2008-10-02 10:50:53 +0200225 h->action, prev_count, preempt_count());
226 preempt_count() = prev_count;
227 }
228
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 rcu_bh_qsctr_inc(cpu);
230 }
231 h++;
232 pending >>= 1;
233 } while (pending);
234
Andrew Mortonc70f5d62005-07-30 10:22:49 -0700235 local_irq_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700236
237 pending = local_softirq_pending();
238 if (pending && --max_restart)
239 goto restart;
240
241 if (pending)
242 wakeup_softirqd();
243
Ingo Molnard820ac42009-03-13 01:30:40 +0100244 lockdep_softirq_exit();
Paul Mackerras829035fd2006-07-03 00:25:40 -0700245
246 account_system_vtime(current);
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700247 _local_bh_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248}
249
250#ifndef __ARCH_HAS_DO_SOFTIRQ
251
252asmlinkage void do_softirq(void)
253{
254 __u32 pending;
255 unsigned long flags;
256
257 if (in_interrupt())
258 return;
259
260 local_irq_save(flags);
261
262 pending = local_softirq_pending();
263
264 if (pending)
265 __do_softirq();
266
267 local_irq_restore(flags);
268}
269
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270#endif
271
Ingo Molnardde4b2b2007-02-16 01:27:45 -0800272/*
273 * Enter an interrupt context.
274 */
275void irq_enter(void)
276{
Venki Pallipadi6378ddb2008-01-30 13:30:04 +0100277 int cpu = smp_processor_id();
Thomas Gleixner719254f2008-10-17 09:59:47 +0200278
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100279 rcu_irq_enter();
Thomas Gleixneree5f80a2008-11-07 11:06:00 +0100280 if (idle_cpu(cpu) && !in_interrupt()) {
281 __irq_enter();
Thomas Gleixner719254f2008-10-17 09:59:47 +0200282 tick_check_idle(cpu);
Thomas Gleixneree5f80a2008-11-07 11:06:00 +0100283 } else
284 __irq_enter();
Ingo Molnardde4b2b2007-02-16 01:27:45 -0800285}
286
Linus Torvalds1da177e2005-04-16 15:20:36 -0700287#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
288# define invoke_softirq() __do_softirq()
289#else
290# define invoke_softirq() do_softirq()
291#endif
292
293/*
294 * Exit an interrupt context. Process softirqs if needed and possible:
295 */
296void irq_exit(void)
297{
298 account_system_vtime(current);
Ingo Molnarde30a2b2006-07-03 00:24:42 -0700299 trace_hardirq_exit();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 sub_preempt_count(IRQ_EXIT_OFFSET);
301 if (!in_interrupt() && local_softirq_pending())
302 invoke_softirq();
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800303
304#ifdef CONFIG_NO_HZ
305 /* Make sure that timer wheel updates are propagated */
Steven Rostedt2232c2d2008-02-29 18:46:50 +0100306 rcu_irq_exit();
Paul E. McKenney64db4cf2008-12-18 21:55:32 +0100307 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
308 tick_nohz_stop_sched_tick(0);
Thomas Gleixner79bf2bb2007-02-16 01:28:03 -0800309#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 preempt_enable_no_resched();
311}
312
313/*
314 * This function must run with irqs disabled!
315 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800316inline void raise_softirq_irqoff(unsigned int nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317{
318 __raise_softirq_irqoff(nr);
319
320 /*
321 * If we're in an interrupt or softirq, we're done
322 * (this also catches softirq-disabled code). We will
323 * actually run the softirq once we return from
324 * the irq or softirq.
325 *
326 * Otherwise we wake up ksoftirqd to make sure we
327 * schedule the softirq soon.
328 */
329 if (!in_interrupt())
330 wakeup_softirqd();
331}
332
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800333void raise_softirq(unsigned int nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334{
335 unsigned long flags;
336
337 local_irq_save(flags);
338 raise_softirq_irqoff(nr);
339 local_irq_restore(flags);
340}
341
Carlos R. Mafra962cf362008-05-15 11:15:37 -0300342void open_softirq(int nr, void (*action)(struct softirq_action *))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 softirq_vec[nr].action = action;
345}
346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347/* Tasklets */
348struct tasklet_head
349{
Olof Johansson48f20a92008-03-04 15:23:25 -0800350 struct tasklet_struct *head;
351 struct tasklet_struct **tail;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352};
353
Vegard Nossum4620b492008-06-12 23:21:53 +0200354static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
355static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700356
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800357void __tasklet_schedule(struct tasklet_struct *t)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358{
359 unsigned long flags;
360
361 local_irq_save(flags);
Olof Johansson48f20a92008-03-04 15:23:25 -0800362 t->next = NULL;
363 *__get_cpu_var(tasklet_vec).tail = t;
364 __get_cpu_var(tasklet_vec).tail = &(t->next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 raise_softirq_irqoff(TASKLET_SOFTIRQ);
366 local_irq_restore(flags);
367}
368
369EXPORT_SYMBOL(__tasklet_schedule);
370
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800371void __tasklet_hi_schedule(struct tasklet_struct *t)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372{
373 unsigned long flags;
374
375 local_irq_save(flags);
Olof Johansson48f20a92008-03-04 15:23:25 -0800376 t->next = NULL;
377 *__get_cpu_var(tasklet_hi_vec).tail = t;
378 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 raise_softirq_irqoff(HI_SOFTIRQ);
380 local_irq_restore(flags);
381}
382
383EXPORT_SYMBOL(__tasklet_hi_schedule);
384
385static void tasklet_action(struct softirq_action *a)
386{
387 struct tasklet_struct *list;
388
389 local_irq_disable();
Olof Johansson48f20a92008-03-04 15:23:25 -0800390 list = __get_cpu_var(tasklet_vec).head;
391 __get_cpu_var(tasklet_vec).head = NULL;
392 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393 local_irq_enable();
394
395 while (list) {
396 struct tasklet_struct *t = list;
397
398 list = list->next;
399
400 if (tasklet_trylock(t)) {
401 if (!atomic_read(&t->count)) {
402 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
403 BUG();
404 t->func(t->data);
405 tasklet_unlock(t);
406 continue;
407 }
408 tasklet_unlock(t);
409 }
410
411 local_irq_disable();
Olof Johansson48f20a92008-03-04 15:23:25 -0800412 t->next = NULL;
413 *__get_cpu_var(tasklet_vec).tail = t;
414 __get_cpu_var(tasklet_vec).tail = &(t->next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
416 local_irq_enable();
417 }
418}
419
420static void tasklet_hi_action(struct softirq_action *a)
421{
422 struct tasklet_struct *list;
423
424 local_irq_disable();
Olof Johansson48f20a92008-03-04 15:23:25 -0800425 list = __get_cpu_var(tasklet_hi_vec).head;
426 __get_cpu_var(tasklet_hi_vec).head = NULL;
427 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428 local_irq_enable();
429
430 while (list) {
431 struct tasklet_struct *t = list;
432
433 list = list->next;
434
435 if (tasklet_trylock(t)) {
436 if (!atomic_read(&t->count)) {
437 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
438 BUG();
439 t->func(t->data);
440 tasklet_unlock(t);
441 continue;
442 }
443 tasklet_unlock(t);
444 }
445
446 local_irq_disable();
Olof Johansson48f20a92008-03-04 15:23:25 -0800447 t->next = NULL;
448 *__get_cpu_var(tasklet_hi_vec).tail = t;
449 __get_cpu_var(tasklet_hi_vec).tail = &(t->next);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700450 __raise_softirq_irqoff(HI_SOFTIRQ);
451 local_irq_enable();
452 }
453}
454
455
456void tasklet_init(struct tasklet_struct *t,
457 void (*func)(unsigned long), unsigned long data)
458{
459 t->next = NULL;
460 t->state = 0;
461 atomic_set(&t->count, 0);
462 t->func = func;
463 t->data = data;
464}
465
466EXPORT_SYMBOL(tasklet_init);
467
468void tasklet_kill(struct tasklet_struct *t)
469{
470 if (in_interrupt())
471 printk("Attempt to kill tasklet from interrupt\n");
472
473 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
H Hartley Sweeten79d381c2009-04-16 19:30:18 -0400474 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700475 yield();
H Hartley Sweeten79d381c2009-04-16 19:30:18 -0400476 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700477 }
478 tasklet_unlock_wait(t);
479 clear_bit(TASKLET_STATE_SCHED, &t->state);
480}
481
482EXPORT_SYMBOL(tasklet_kill);
483
David S. Miller54514a72008-09-23 22:15:57 -0700484DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
485EXPORT_PER_CPU_SYMBOL(softirq_work_list);
486
487static void __local_trigger(struct call_single_data *cp, int softirq)
488{
489 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
490
491 list_add_tail(&cp->list, head);
492
493 /* Trigger the softirq only if the list was previously empty. */
494 if (head->next == &cp->list)
495 raise_softirq_irqoff(softirq);
496}
497
498#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
499static void remote_softirq_receive(void *data)
500{
501 struct call_single_data *cp = data;
502 unsigned long flags;
503 int softirq;
504
505 softirq = cp->priv;
506
507 local_irq_save(flags);
508 __local_trigger(cp, softirq);
509 local_irq_restore(flags);
510}
511
512static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
513{
514 if (cpu_online(cpu)) {
515 cp->func = remote_softirq_receive;
516 cp->info = cp;
517 cp->flags = 0;
518 cp->priv = softirq;
519
Peter Zijlstra6e275632009-02-25 13:59:48 +0100520 __smp_call_function_single(cpu, cp, 0);
David S. Miller54514a72008-09-23 22:15:57 -0700521 return 0;
522 }
523 return 1;
524}
525#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
526static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
527{
528 return 1;
529}
530#endif
531
532/**
533 * __send_remote_softirq - try to schedule softirq work on a remote cpu
534 * @cp: private SMP call function data area
535 * @cpu: the remote cpu
536 * @this_cpu: the currently executing cpu
537 * @softirq: the softirq for the work
538 *
539 * Attempt to schedule softirq work on a remote cpu. If this cannot be
540 * done, the work is instead queued up on the local cpu.
541 *
542 * Interrupts must be disabled.
543 */
544void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
545{
546 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
547 __local_trigger(cp, softirq);
548}
549EXPORT_SYMBOL(__send_remote_softirq);
550
551/**
552 * send_remote_softirq - try to schedule softirq work on a remote cpu
553 * @cp: private SMP call function data area
554 * @cpu: the remote cpu
555 * @softirq: the softirq for the work
556 *
557 * Like __send_remote_softirq except that disabling interrupts and
558 * computing the current cpu is done for the caller.
559 */
560void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
561{
562 unsigned long flags;
563 int this_cpu;
564
565 local_irq_save(flags);
566 this_cpu = smp_processor_id();
567 __send_remote_softirq(cp, cpu, this_cpu, softirq);
568 local_irq_restore(flags);
569}
570EXPORT_SYMBOL(send_remote_softirq);
571
572static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
573 unsigned long action, void *hcpu)
574{
575 /*
576 * If a CPU goes away, splice its entries to the current CPU
577 * and trigger a run of the softirq
578 */
579 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
580 int cpu = (unsigned long) hcpu;
581 int i;
582
583 local_irq_disable();
584 for (i = 0; i < NR_SOFTIRQS; i++) {
585 struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
586 struct list_head *local_head;
587
588 if (list_empty(head))
589 continue;
590
591 local_head = &__get_cpu_var(softirq_work_list[i]);
592 list_splice_init(head, local_head);
593 raise_softirq_irqoff(i);
594 }
595 local_irq_enable();
596 }
597
598 return NOTIFY_OK;
599}
600
601static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
602 .notifier_call = remote_softirq_cpu_notify,
603};
604
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605void __init softirq_init(void)
606{
Olof Johansson48f20a92008-03-04 15:23:25 -0800607 int cpu;
608
609 for_each_possible_cpu(cpu) {
David S. Miller54514a72008-09-23 22:15:57 -0700610 int i;
611
Olof Johansson48f20a92008-03-04 15:23:25 -0800612 per_cpu(tasklet_vec, cpu).tail =
613 &per_cpu(tasklet_vec, cpu).head;
614 per_cpu(tasklet_hi_vec, cpu).tail =
615 &per_cpu(tasklet_hi_vec, cpu).head;
David S. Miller54514a72008-09-23 22:15:57 -0700616 for (i = 0; i < NR_SOFTIRQS; i++)
617 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
Olof Johansson48f20a92008-03-04 15:23:25 -0800618 }
619
David S. Miller54514a72008-09-23 22:15:57 -0700620 register_hotcpu_notifier(&remote_softirq_cpu_notifier);
621
Carlos R. Mafra962cf362008-05-15 11:15:37 -0300622 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
623 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624}
625
626static int ksoftirqd(void * __bind_cpu)
627{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628 set_current_state(TASK_INTERRUPTIBLE);
629
630 while (!kthread_should_stop()) {
631 preempt_disable();
632 if (!local_softirq_pending()) {
633 preempt_enable_no_resched();
634 schedule();
635 preempt_disable();
636 }
637
638 __set_current_state(TASK_RUNNING);
639
640 while (local_softirq_pending()) {
641 /* Preempt disable stops cpu going offline.
642 If already offline, we'll be on wrong CPU:
643 don't process */
644 if (cpu_is_offline((long)__bind_cpu))
645 goto wait_to_die;
646 do_softirq();
647 preempt_enable_no_resched();
648 cond_resched();
649 preempt_disable();
Eric Dumazet64ca5ab2009-03-04 12:11:56 -0800650 rcu_qsctr_inc((long)__bind_cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651 }
652 preempt_enable();
653 set_current_state(TASK_INTERRUPTIBLE);
654 }
655 __set_current_state(TASK_RUNNING);
656 return 0;
657
658wait_to_die:
659 preempt_enable();
660 /* Wait for kthread_stop */
661 set_current_state(TASK_INTERRUPTIBLE);
662 while (!kthread_should_stop()) {
663 schedule();
664 set_current_state(TASK_INTERRUPTIBLE);
665 }
666 __set_current_state(TASK_RUNNING);
667 return 0;
668}
669
670#ifdef CONFIG_HOTPLUG_CPU
671/*
672 * tasklet_kill_immediate is called to remove a tasklet which can already be
673 * scheduled for execution on @cpu.
674 *
675 * Unlike tasklet_kill, this function removes the tasklet
676 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
677 *
678 * When this function is called, @cpu must be in the CPU_DEAD state.
679 */
680void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
681{
682 struct tasklet_struct **i;
683
684 BUG_ON(cpu_online(cpu));
685 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
686
687 if (!test_bit(TASKLET_STATE_SCHED, &t->state))
688 return;
689
690 /* CPU is dead, so no lock needed. */
Olof Johansson48f20a92008-03-04 15:23:25 -0800691 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692 if (*i == t) {
693 *i = t->next;
Olof Johansson48f20a92008-03-04 15:23:25 -0800694 /* If this was the tail element, move the tail ptr */
695 if (*i == NULL)
696 per_cpu(tasklet_vec, cpu).tail = i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700697 return;
698 }
699 }
700 BUG();
701}
702
703static void takeover_tasklets(unsigned int cpu)
704{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705 /* CPU is dead, so no lock needed. */
706 local_irq_disable();
707
708 /* Find end, append list for that CPU. */
Christian Borntraegere5e41722008-05-01 04:34:23 -0700709 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
710 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head;
711 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail;
712 per_cpu(tasklet_vec, cpu).head = NULL;
713 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
714 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700715 raise_softirq_irqoff(TASKLET_SOFTIRQ);
716
Christian Borntraegere5e41722008-05-01 04:34:23 -0700717 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
718 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head;
719 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail;
720 per_cpu(tasklet_hi_vec, cpu).head = NULL;
721 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
722 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 raise_softirq_irqoff(HI_SOFTIRQ);
724
725 local_irq_enable();
726}
727#endif /* CONFIG_HOTPLUG_CPU */
728
Chandra Seetharaman8c78f302006-07-30 03:03:35 -0700729static int __cpuinit cpu_callback(struct notifier_block *nfb,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 unsigned long action,
731 void *hcpu)
732{
733 int hotcpu = (unsigned long)hcpu;
734 struct task_struct *p;
735
736 switch (action) {
737 case CPU_UP_PREPARE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700738 case CPU_UP_PREPARE_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739 p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
740 if (IS_ERR(p)) {
741 printk("ksoftirqd for %i failed\n", hotcpu);
742 return NOTIFY_BAD;
743 }
744 kthread_bind(p, hotcpu);
745 per_cpu(ksoftirqd, hotcpu) = p;
746 break;
747 case CPU_ONLINE:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700748 case CPU_ONLINE_FROZEN:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700749 wake_up_process(per_cpu(ksoftirqd, hotcpu));
750 break;
751#ifdef CONFIG_HOTPLUG_CPU
752 case CPU_UP_CANCELED:
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700753 case CPU_UP_CANCELED_FROZEN:
Heiko Carstensfc75cdf2006-06-25 05:49:10 -0700754 if (!per_cpu(ksoftirqd, hotcpu))
755 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 /* Unbind so it can run. Fall thru. */
Heiko Carstensa4c4af72005-11-07 00:58:38 -0800757 kthread_bind(per_cpu(ksoftirqd, hotcpu),
Rusty Russellf1fc0572009-01-01 10:12:23 +1030758 cpumask_any(cpu_online_mask));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 case CPU_DEAD:
Satoru Takeuchi1c6b4aa2007-07-15 23:39:48 -0700760 case CPU_DEAD_FROZEN: {
761 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
762
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763 p = per_cpu(ksoftirqd, hotcpu);
764 per_cpu(ksoftirqd, hotcpu) = NULL;
Rusty Russell961ccdd2008-06-23 13:55:38 +1000765 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766 kthread_stop(p);
767 takeover_tasklets(hotcpu);
768 break;
Satoru Takeuchi1c6b4aa2007-07-15 23:39:48 -0700769 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700770#endif /* CONFIG_HOTPLUG_CPU */
771 }
772 return NOTIFY_OK;
773}
774
Chandra Seetharaman8c78f302006-07-30 03:03:35 -0700775static struct notifier_block __cpuinitdata cpu_nfb = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700776 .notifier_call = cpu_callback
777};
778
Eduard - Gabriel Munteanu7babe8d2008-07-25 19:45:11 -0700779static __init int spawn_ksoftirqd(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780{
781 void *cpu = (void *)(long)smp_processor_id();
Akinobu Mita07dccf32006-09-29 02:00:22 -0700782 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
783
784 BUG_ON(err == NOTIFY_BAD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
786 register_cpu_notifier(&cpu_nfb);
787 return 0;
788}
Eduard - Gabriel Munteanu7babe8d2008-07-25 19:45:11 -0700789early_initcall(spawn_ksoftirqd);
Andrew Morton78eef012006-03-22 00:08:16 -0800790
791#ifdef CONFIG_SMP
792/*
793 * Call a function on all processors
794 */
Jens Axboe15c8b6c2008-05-09 09:39:44 +0200795int on_each_cpu(void (*func) (void *info), void *info, int wait)
Andrew Morton78eef012006-03-22 00:08:16 -0800796{
797 int ret = 0;
798
799 preempt_disable();
Jens Axboe8691e5a2008-06-06 11:18:06 +0200800 ret = smp_call_function(func, info, wait);
Andrew Morton78eef012006-03-22 00:08:16 -0800801 local_irq_disable();
802 func(info);
803 local_irq_enable();
804 preempt_enable();
805 return ret;
806}
807EXPORT_SYMBOL(on_each_cpu);
808#endif
Yinghai Lu43a25632008-12-28 16:01:13 -0800809
810/*
811 * [ These __weak aliases are kept in a separate compilation unit, so that
812 * GCC does not inline them incorrectly. ]
813 */
814
815int __init __weak early_irq_init(void)
816{
817 return 0;
818}
819
Yinghai Lu4a046d12009-01-12 17:39:24 -0800820int __init __weak arch_probe_nr_irqs(void)
821{
822 return 0;
823}
824
Yinghai Lu43a25632008-12-28 16:01:13 -0800825int __init __weak arch_early_irq_init(void)
826{
827 return 0;
828}
829
Yinghai Lu85ac16d2009-04-27 18:00:38 -0700830int __weak arch_init_chip_data(struct irq_desc *desc, int node)
Yinghai Lu43a25632008-12-28 16:01:13 -0800831{
832 return 0;
833}