blob: dee48658805c4e58d1b4fc5776a165cc18e8c464 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/workqueue.c
3 *
4 * Generic mechanism for defining kernel helper threads for running
5 * arbitrary tasks in process context.
6 *
7 * Started by Ingo Molnar, Copyright (C) 2002
8 *
9 * Derived from the taskqueue/keventd code by:
10 *
11 * David Woodhouse <dwmw2@infradead.org>
Francois Camie1f8e872008-10-15 22:01:59 -070012 * Andrew Morton
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
14 * Theodore Ts'o <tytso@mit.edu>
Christoph Lameter89ada672005-10-30 15:01:59 -080015 *
Christoph Lametercde53532008-07-04 09:59:22 -070016 * Made to use alloc_percpu by Christoph Lameter.
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
18
19#include <linux/module.h>
20#include <linux/kernel.h>
21#include <linux/sched.h>
22#include <linux/init.h>
23#include <linux/signal.h>
24#include <linux/completion.h>
25#include <linux/workqueue.h>
26#include <linux/slab.h>
27#include <linux/cpu.h>
28#include <linux/notifier.h>
29#include <linux/kthread.h>
James Bottomley1fa44ec2006-02-23 12:43:43 -060030#include <linux/hardirq.h>
Christoph Lameter46934022006-10-11 01:21:26 -070031#include <linux/mempolicy.h>
Rafael J. Wysocki341a5952006-12-06 20:34:49 -080032#include <linux/freezer.h>
Peter Zijlstrad5abe662006-12-06 20:37:26 -080033#include <linux/kallsyms.h>
34#include <linux/debug_locks.h>
Johannes Berg4e6045f2007-10-18 23:39:55 -070035#include <linux/lockdep.h>
Zhaoleifb391252009-04-17 15:15:51 +080036#define CREATE_TRACE_POINTS
37#include <trace/events/workqueue.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038
39/*
Nathan Lynchf756d5e2006-01-08 01:05:12 -080040 * The per-CPU workqueue (if single thread, we always use the first
41 * possible cpu).
Linus Torvalds1da177e2005-04-16 15:20:36 -070042 */
43struct cpu_workqueue_struct {
44
45 spinlock_t lock;
46
Linus Torvalds1da177e2005-04-16 15:20:36 -070047 struct list_head worklist;
48 wait_queue_head_t more_work;
Oleg Nesterov3af244332007-05-09 02:34:09 -070049 struct work_struct *current_work;
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
51 struct workqueue_struct *wq;
Ingo Molnar36c8b582006-07-03 00:25:41 -070052 struct task_struct *thread;
Linus Torvalds1da177e2005-04-16 15:20:36 -070053} ____cacheline_aligned;
54
55/*
56 * The externally visible workqueue abstraction is an array of
57 * per-CPU workqueues:
58 */
59struct workqueue_struct {
Christoph Lameter89ada672005-10-30 15:01:59 -080060 struct cpu_workqueue_struct *cpu_wq;
Oleg Nesterovcce1a162007-05-09 02:34:13 -070061 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 const char *name;
Oleg Nesterovcce1a162007-05-09 02:34:13 -070063 int singlethread;
Oleg Nesterov319c2a92007-05-09 02:34:06 -070064 int freezeable; /* Freeze threads during suspend */
Heiko Carstens0d557dc2008-10-13 23:50:09 +020065 int rt;
Johannes Berg4e6045f2007-10-18 23:39:55 -070066#ifdef CONFIG_LOCKDEP
67 struct lockdep_map lockdep_map;
68#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070069};
70
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +090071#ifdef CONFIG_DEBUG_OBJECTS_WORK
72
73static struct debug_obj_descr work_debug_descr;
74
75/*
76 * fixup_init is called when:
77 * - an active object is initialized
78 */
79static int work_fixup_init(void *addr, enum debug_obj_state state)
80{
81 struct work_struct *work = addr;
82
83 switch (state) {
84 case ODEBUG_STATE_ACTIVE:
85 cancel_work_sync(work);
86 debug_object_init(work, &work_debug_descr);
87 return 1;
88 default:
89 return 0;
90 }
91}
92
93/*
94 * fixup_activate is called when:
95 * - an active object is activated
96 * - an unknown object is activated (might be a statically initialized object)
97 */
98static int work_fixup_activate(void *addr, enum debug_obj_state state)
99{
100 struct work_struct *work = addr;
101
102 switch (state) {
103
104 case ODEBUG_STATE_NOTAVAILABLE:
105 /*
106 * This is not really a fixup. The work struct was
107 * statically initialized. We just make sure that it
108 * is tracked in the object tracker.
109 */
110 if (test_bit(WORK_STRUCT_STATIC, work_data_bits(work))) {
111 debug_object_init(work, &work_debug_descr);
112 debug_object_activate(work, &work_debug_descr);
113 return 0;
114 }
115 WARN_ON_ONCE(1);
116 return 0;
117
118 case ODEBUG_STATE_ACTIVE:
119 WARN_ON(1);
120
121 default:
122 return 0;
123 }
124}
125
126/*
127 * fixup_free is called when:
128 * - an active object is freed
129 */
130static int work_fixup_free(void *addr, enum debug_obj_state state)
131{
132 struct work_struct *work = addr;
133
134 switch (state) {
135 case ODEBUG_STATE_ACTIVE:
136 cancel_work_sync(work);
137 debug_object_free(work, &work_debug_descr);
138 return 1;
139 default:
140 return 0;
141 }
142}
143
144static struct debug_obj_descr work_debug_descr = {
145 .name = "work_struct",
146 .fixup_init = work_fixup_init,
147 .fixup_activate = work_fixup_activate,
148 .fixup_free = work_fixup_free,
149};
150
151static inline void debug_work_activate(struct work_struct *work)
152{
153 debug_object_activate(work, &work_debug_descr);
154}
155
156static inline void debug_work_deactivate(struct work_struct *work)
157{
158 debug_object_deactivate(work, &work_debug_descr);
159}
160
161void __init_work(struct work_struct *work, int onstack)
162{
163 if (onstack)
164 debug_object_init_on_stack(work, &work_debug_descr);
165 else
166 debug_object_init(work, &work_debug_descr);
167}
168EXPORT_SYMBOL_GPL(__init_work);
169
170void destroy_work_on_stack(struct work_struct *work)
171{
172 debug_object_free(work, &work_debug_descr);
173}
174EXPORT_SYMBOL_GPL(destroy_work_on_stack);
175
176#else
177static inline void debug_work_activate(struct work_struct *work) { }
178static inline void debug_work_deactivate(struct work_struct *work) { }
179#endif
180
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100181/* Serializes the accesses to the list of workqueues. */
182static DEFINE_SPINLOCK(workqueue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183static LIST_HEAD(workqueues);
184
Oleg Nesterov3af244332007-05-09 02:34:09 -0700185static int singlethread_cpu __read_mostly;
Rusty Russelle7577c52009-01-01 10:12:25 +1030186static const struct cpumask *cpu_singlethread_map __read_mostly;
Oleg Nesterov14441962007-05-23 13:57:57 -0700187/*
188 * _cpu_down() first removes CPU from cpu_online_map, then CPU_DEAD
189 * flushes cwq->worklist. This means that flush_workqueue/wait_on_work
190 * which comes in between can't use for_each_online_cpu(). We could
191 * use cpu_possible_map, the cpumask below is more a documentation
192 * than optimization.
193 */
Rusty Russelle7577c52009-01-01 10:12:25 +1030194static cpumask_var_t cpu_populated_map __read_mostly;
Nathan Lynchf756d5e2006-01-08 01:05:12 -0800195
Linus Torvalds1da177e2005-04-16 15:20:36 -0700196/* If it's single threaded, it isn't in the list of workqueues. */
David Howells6cc88bc2008-11-14 10:39:21 +1100197static inline int is_wq_single_threaded(struct workqueue_struct *wq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198{
Oleg Nesterovcce1a162007-05-09 02:34:13 -0700199 return wq->singlethread;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200}
201
Rusty Russelle7577c52009-01-01 10:12:25 +1030202static const struct cpumask *wq_cpu_map(struct workqueue_struct *wq)
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700203{
David Howells6cc88bc2008-11-14 10:39:21 +1100204 return is_wq_single_threaded(wq)
Rusty Russelle7577c52009-01-01 10:12:25 +1030205 ? cpu_singlethread_map : cpu_populated_map;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700206}
207
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700208static
209struct cpu_workqueue_struct *wq_per_cpu(struct workqueue_struct *wq, int cpu)
210{
David Howells6cc88bc2008-11-14 10:39:21 +1100211 if (unlikely(is_wq_single_threaded(wq)))
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700212 cpu = singlethread_cpu;
213 return per_cpu_ptr(wq->cpu_wq, cpu);
214}
215
David Howells4594bf12006-12-07 11:33:26 +0000216/*
217 * Set the workqueue on which a work item is to be run
218 * - Must *only* be called if the pending flag is set
219 */
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700220static inline void set_wq_data(struct work_struct *work,
221 struct cpu_workqueue_struct *cwq)
David Howells365970a2006-11-22 14:54:49 +0000222{
David Howells4594bf12006-12-07 11:33:26 +0000223 unsigned long new;
David Howells365970a2006-11-22 14:54:49 +0000224
David Howells4594bf12006-12-07 11:33:26 +0000225 BUG_ON(!work_pending(work));
226
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700227 new = (unsigned long) cwq | (1UL << WORK_STRUCT_PENDING);
Linus Torvaldsa08727b2006-12-16 09:53:50 -0800228 new |= WORK_STRUCT_FLAG_MASK & *work_data_bits(work);
229 atomic_long_set(&work->data, new);
David Howells365970a2006-11-22 14:54:49 +0000230}
231
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700232static inline
233struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
David Howells365970a2006-11-22 14:54:49 +0000234{
Linus Torvaldsa08727b2006-12-16 09:53:50 -0800235 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
David Howells365970a2006-11-22 14:54:49 +0000236}
237
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700238static void insert_work(struct cpu_workqueue_struct *cwq,
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700239 struct work_struct *work, struct list_head *head)
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700240{
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +0100241 trace_workqueue_insertion(cwq->thread, work);
242
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700243 set_wq_data(work, cwq);
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700244 /*
245 * Ensure that we get the right work->data if we see the
246 * result of list_add() below, see try_to_grab_pending().
247 */
248 smp_wmb();
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700249 list_add_tail(&work->entry, head);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700250 wake_up(&cwq->more_work);
251}
252
Linus Torvalds1da177e2005-04-16 15:20:36 -0700253static void __queue_work(struct cpu_workqueue_struct *cwq,
254 struct work_struct *work)
255{
256 unsigned long flags;
257
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900258 debug_work_activate(work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 spin_lock_irqsave(&cwq->lock, flags);
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700260 insert_work(cwq, work, &cwq->worklist);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700261 spin_unlock_irqrestore(&cwq->lock, flags);
262}
263
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700264/**
265 * queue_work - queue work on a workqueue
266 * @wq: workqueue to use
267 * @work: work to queue
268 *
Alan Stern057647f2006-10-28 10:38:58 -0700269 * Returns 0 if @work was already on a queue, non-zero otherwise.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270 *
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -0700271 * We queue the work to the CPU on which it was submitted, but if the CPU dies
272 * it can be processed by another CPU.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800274int queue_work(struct workqueue_struct *wq, struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275{
Oleg Nesterovef1ca232008-07-25 01:47:53 -0700276 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700277
Oleg Nesterovef1ca232008-07-25 01:47:53 -0700278 ret = queue_work_on(get_cpu(), wq, work);
279 put_cpu();
280
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281 return ret;
282}
Dave Jonesae90dd52006-06-30 01:40:45 -0400283EXPORT_SYMBOL_GPL(queue_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700284
Zhang Ruic1a220e2008-07-23 21:28:39 -0700285/**
286 * queue_work_on - queue work on specific cpu
287 * @cpu: CPU number to execute work on
288 * @wq: workqueue to use
289 * @work: work to queue
290 *
291 * Returns 0 if @work was already on a queue, non-zero otherwise.
292 *
293 * We queue the work to a specific CPU, the caller must ensure it
294 * can't go away.
295 */
296int
297queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work)
298{
299 int ret = 0;
300
301 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
302 BUG_ON(!list_empty(&work->entry));
303 __queue_work(wq_per_cpu(wq, cpu), work);
304 ret = 1;
305 }
306 return ret;
307}
308EXPORT_SYMBOL_GPL(queue_work_on);
309
Li Zefan6d141c32008-02-08 04:21:09 -0800310static void delayed_work_timer_fn(unsigned long __data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311{
David Howells52bad642006-11-22 14:54:01 +0000312 struct delayed_work *dwork = (struct delayed_work *)__data;
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700313 struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work);
314 struct workqueue_struct *wq = cwq->wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700316 __queue_work(wq_per_cpu(wq, smp_processor_id()), &dwork->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317}
318
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700319/**
320 * queue_delayed_work - queue work on a workqueue after delay
321 * @wq: workqueue to use
Randy Dunlapaf9997e2006-12-22 01:06:52 -0800322 * @dwork: delayable work to queue
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700323 * @delay: number of jiffies to wait before queueing
324 *
Alan Stern057647f2006-10-28 10:38:58 -0700325 * Returns 0 if @work was already on a queue, non-zero otherwise.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700326 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800327int queue_delayed_work(struct workqueue_struct *wq,
David Howells52bad642006-11-22 14:54:01 +0000328 struct delayed_work *dwork, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329{
David Howells52bad642006-11-22 14:54:01 +0000330 if (delay == 0)
Oleg Nesterov63bc0362007-05-09 02:34:16 -0700331 return queue_work(wq, &dwork->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332
Oleg Nesterov63bc0362007-05-09 02:34:16 -0700333 return queue_delayed_work_on(-1, wq, dwork, delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334}
Dave Jonesae90dd52006-06-30 01:40:45 -0400335EXPORT_SYMBOL_GPL(queue_delayed_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700337/**
338 * queue_delayed_work_on - queue work on specific CPU after delay
339 * @cpu: CPU number to execute work on
340 * @wq: workqueue to use
Randy Dunlapaf9997e2006-12-22 01:06:52 -0800341 * @dwork: work to queue
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700342 * @delay: number of jiffies to wait before queueing
343 *
Alan Stern057647f2006-10-28 10:38:58 -0700344 * Returns 0 if @work was already on a queue, non-zero otherwise.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700345 */
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700346int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
David Howells52bad642006-11-22 14:54:01 +0000347 struct delayed_work *dwork, unsigned long delay)
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700348{
349 int ret = 0;
David Howells52bad642006-11-22 14:54:01 +0000350 struct timer_list *timer = &dwork->timer;
351 struct work_struct *work = &dwork->work;
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700352
Linus Torvaldsa08727b2006-12-16 09:53:50 -0800353 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700354 BUG_ON(timer_pending(timer));
355 BUG_ON(!list_empty(&work->entry));
356
Andrew Liu8a3e77c2008-05-01 04:35:14 -0700357 timer_stats_timer_set_start_info(&dwork->timer);
358
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700359 /* This stores cwq for the moment, for the timer_fn */
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700360 set_wq_data(work, wq_per_cpu(wq, raw_smp_processor_id()));
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700361 timer->expires = jiffies + delay;
David Howells52bad642006-11-22 14:54:01 +0000362 timer->data = (unsigned long)dwork;
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700363 timer->function = delayed_work_timer_fn;
Oleg Nesterov63bc0362007-05-09 02:34:16 -0700364
365 if (unlikely(cpu >= 0))
366 add_timer_on(timer, cpu);
367 else
368 add_timer(timer);
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700369 ret = 1;
370 }
371 return ret;
372}
Dave Jonesae90dd52006-06-30 01:40:45 -0400373EXPORT_SYMBOL_GPL(queue_delayed_work_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374
Arjan van de Ven858119e2006-01-14 13:20:43 -0800375static void run_workqueue(struct cpu_workqueue_struct *cwq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376{
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700377 spin_lock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 while (!list_empty(&cwq->worklist)) {
379 struct work_struct *work = list_entry(cwq->worklist.next,
380 struct work_struct, entry);
David Howells6bb49e52006-11-22 14:54:45 +0000381 work_func_t f = work->func;
Johannes Berg4e6045f2007-10-18 23:39:55 -0700382#ifdef CONFIG_LOCKDEP
383 /*
384 * It is permissible to free the struct work_struct
385 * from inside the function that is called from it,
386 * this we need to take into account for lockdep too.
387 * To avoid bogus "held lock freed" warnings as well
388 * as problems when looking into work->lockdep_map,
389 * make a copy and use that here.
390 */
391 struct lockdep_map lockdep_map = work->lockdep_map;
392#endif
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +0100393 trace_workqueue_execution(cwq->thread, work);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900394 debug_work_deactivate(work);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700395 cwq->current_work = work;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 list_del_init(cwq->worklist.next);
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700397 spin_unlock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398
David Howells365970a2006-11-22 14:54:49 +0000399 BUG_ON(get_wq_data(work) != cwq);
Oleg Nesterov23b2e592007-05-09 02:34:19 -0700400 work_clear_pending(work);
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200401 lock_map_acquire(&cwq->wq->lockdep_map);
402 lock_map_acquire(&lockdep_map);
David Howells65f27f32006-11-22 14:55:48 +0000403 f(work);
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200404 lock_map_release(&lockdep_map);
405 lock_map_release(&cwq->wq->lockdep_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406
Peter Zijlstrad5abe662006-12-06 20:37:26 -0800407 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
408 printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
409 "%s/0x%08x/%d\n",
410 current->comm, preempt_count(),
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700411 task_pid_nr(current));
Peter Zijlstrad5abe662006-12-06 20:37:26 -0800412 printk(KERN_ERR " last function: ");
413 print_symbol("%s\n", (unsigned long)f);
414 debug_show_held_locks(current);
415 dump_stack();
416 }
417
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700418 spin_lock_irq(&cwq->lock);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700419 cwq->current_work = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 }
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700421 spin_unlock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422}
423
424static int worker_thread(void *__cwq)
425{
426 struct cpu_workqueue_struct *cwq = __cwq;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700427 DEFINE_WAIT(wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700428
Rafael J. Wysocki83144182007-07-17 04:03:35 -0700429 if (cwq->wq->freezeable)
430 set_freezable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700431
Oleg Nesterov3af244332007-05-09 02:34:09 -0700432 for (;;) {
Oleg Nesterov3af244332007-05-09 02:34:09 -0700433 prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
Oleg Nesterov14441962007-05-23 13:57:57 -0700434 if (!freezing(current) &&
435 !kthread_should_stop() &&
436 list_empty(&cwq->worklist))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 schedule();
Oleg Nesterov3af244332007-05-09 02:34:09 -0700438 finish_wait(&cwq->more_work, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700439
Oleg Nesterov85f41862007-05-09 02:34:20 -0700440 try_to_freeze();
441
Oleg Nesterov14441962007-05-23 13:57:57 -0700442 if (kthread_should_stop())
Oleg Nesterov3af244332007-05-09 02:34:09 -0700443 break;
444
445 run_workqueue(cwq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700446 }
Oleg Nesterov3af244332007-05-09 02:34:09 -0700447
Linus Torvalds1da177e2005-04-16 15:20:36 -0700448 return 0;
449}
450
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700451struct wq_barrier {
452 struct work_struct work;
453 struct completion done;
454};
455
456static void wq_barrier_func(struct work_struct *work)
457{
458 struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
459 complete(&barr->done);
460}
461
Oleg Nesterov83c22522007-05-09 02:33:54 -0700462static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700463 struct wq_barrier *barr, struct list_head *head)
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700464{
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900465 /*
466 * debugobject calls are safe here even with cwq->lock locked
467 * as we know for sure that this will not trigger any of the
468 * checks and call back into the fixup functions where we
469 * might deadlock.
470 */
471 INIT_WORK_ON_STACK(&barr->work, wq_barrier_func);
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700472 __set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work));
473
474 init_completion(&barr->done);
Oleg Nesterov83c22522007-05-09 02:33:54 -0700475
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900476 debug_work_activate(&barr->work);
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700477 insert_work(cwq, &barr->work, head);
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700478}
479
Oleg Nesterov14441962007-05-23 13:57:57 -0700480static int flush_cpu_workqueue(struct cpu_workqueue_struct *cwq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481{
Lai Jiangshan2355b702009-04-02 16:58:24 -0700482 int active = 0;
483 struct wq_barrier barr;
Oleg Nesterov14441962007-05-23 13:57:57 -0700484
Lai Jiangshan2355b702009-04-02 16:58:24 -0700485 WARN_ON(cwq->thread == current);
486
487 spin_lock_irq(&cwq->lock);
488 if (!list_empty(&cwq->worklist) || cwq->current_work != NULL) {
489 insert_wq_barrier(cwq, &barr, &cwq->worklist);
Oleg Nesterov14441962007-05-23 13:57:57 -0700490 active = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 }
Lai Jiangshan2355b702009-04-02 16:58:24 -0700492 spin_unlock_irq(&cwq->lock);
493
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900494 if (active) {
Lai Jiangshan2355b702009-04-02 16:58:24 -0700495 wait_for_completion(&barr.done);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900496 destroy_work_on_stack(&barr.work);
497 }
Oleg Nesterov14441962007-05-23 13:57:57 -0700498
499 return active;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500}
501
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700502/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700503 * flush_workqueue - ensure that any scheduled work has run to completion.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700504 * @wq: workqueue to flush
Linus Torvalds1da177e2005-04-16 15:20:36 -0700505 *
506 * Forces execution of the workqueue and blocks until its completion.
507 * This is typically used in driver shutdown handlers.
508 *
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700509 * We sleep until all works which were queued on entry have been handled,
510 * but we are not livelocked by new incoming ones.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511 *
512 * This function used to run the workqueues itself. Now we just wait for the
513 * helper threads to do it.
514 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800515void flush_workqueue(struct workqueue_struct *wq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700516{
Rusty Russelle7577c52009-01-01 10:12:25 +1030517 const struct cpumask *cpu_map = wq_cpu_map(wq);
Oleg Nesterovcce1a162007-05-09 02:34:13 -0700518 int cpu;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700519
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700520 might_sleep();
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200521 lock_map_acquire(&wq->lockdep_map);
522 lock_map_release(&wq->lockdep_map);
Rusty Russellaa85ea52009-03-30 22:05:15 -0600523 for_each_cpu(cpu, cpu_map)
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700524 flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700525}
Dave Jonesae90dd52006-06-30 01:40:45 -0400526EXPORT_SYMBOL_GPL(flush_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527
Oleg Nesterovdb700892008-07-25 01:47:49 -0700528/**
529 * flush_work - block until a work_struct's callback has terminated
530 * @work: the work which is to be flushed
531 *
Oleg Nesterova67da702008-07-25 01:47:52 -0700532 * Returns false if @work has already terminated.
533 *
Oleg Nesterovdb700892008-07-25 01:47:49 -0700534 * It is expected that, prior to calling flush_work(), the caller has
535 * arranged for the work to not be requeued, otherwise it doesn't make
536 * sense to use this function.
537 */
538int flush_work(struct work_struct *work)
539{
540 struct cpu_workqueue_struct *cwq;
541 struct list_head *prev;
542 struct wq_barrier barr;
543
544 might_sleep();
545 cwq = get_wq_data(work);
546 if (!cwq)
547 return 0;
548
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200549 lock_map_acquire(&cwq->wq->lockdep_map);
550 lock_map_release(&cwq->wq->lockdep_map);
Oleg Nesterova67da702008-07-25 01:47:52 -0700551
Oleg Nesterovdb700892008-07-25 01:47:49 -0700552 prev = NULL;
553 spin_lock_irq(&cwq->lock);
554 if (!list_empty(&work->entry)) {
555 /*
556 * See the comment near try_to_grab_pending()->smp_rmb().
557 * If it was re-queued under us we are not going to wait.
558 */
559 smp_rmb();
560 if (unlikely(cwq != get_wq_data(work)))
561 goto out;
562 prev = &work->entry;
563 } else {
564 if (cwq->current_work != work)
565 goto out;
566 prev = &cwq->worklist;
567 }
568 insert_wq_barrier(cwq, &barr, prev->next);
569out:
570 spin_unlock_irq(&cwq->lock);
571 if (!prev)
572 return 0;
573
574 wait_for_completion(&barr.done);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900575 destroy_work_on_stack(&barr.work);
Oleg Nesterovdb700892008-07-25 01:47:49 -0700576 return 1;
577}
578EXPORT_SYMBOL_GPL(flush_work);
579
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700580/*
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700581 * Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit,
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700582 * so this work can't be re-armed in any way.
583 */
584static int try_to_grab_pending(struct work_struct *work)
585{
586 struct cpu_workqueue_struct *cwq;
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700587 int ret = -1;
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700588
589 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work)))
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700590 return 0;
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700591
592 /*
593 * The queueing is in progress, or it is already queued. Try to
594 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
595 */
596
597 cwq = get_wq_data(work);
598 if (!cwq)
599 return ret;
600
601 spin_lock_irq(&cwq->lock);
602 if (!list_empty(&work->entry)) {
603 /*
604 * This work is queued, but perhaps we locked the wrong cwq.
605 * In that case we must see the new value after rmb(), see
606 * insert_work()->wmb().
607 */
608 smp_rmb();
609 if (cwq == get_wq_data(work)) {
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900610 debug_work_deactivate(work);
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700611 list_del_init(&work->entry);
612 ret = 1;
613 }
614 }
615 spin_unlock_irq(&cwq->lock);
616
617 return ret;
618}
619
620static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq,
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700621 struct work_struct *work)
622{
623 struct wq_barrier barr;
624 int running = 0;
625
626 spin_lock_irq(&cwq->lock);
627 if (unlikely(cwq->current_work == work)) {
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700628 insert_wq_barrier(cwq, &barr, cwq->worklist.next);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700629 running = 1;
630 }
631 spin_unlock_irq(&cwq->lock);
632
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900633 if (unlikely(running)) {
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700634 wait_for_completion(&barr.done);
Thomas Gleixnerdc186ad2009-11-16 01:09:48 +0900635 destroy_work_on_stack(&barr.work);
636 }
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700637}
638
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700639static void wait_on_work(struct work_struct *work)
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700640{
641 struct cpu_workqueue_struct *cwq;
Oleg Nesterov28e53bd2007-05-09 02:34:22 -0700642 struct workqueue_struct *wq;
Rusty Russelle7577c52009-01-01 10:12:25 +1030643 const struct cpumask *cpu_map;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700644 int cpu;
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700645
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700646 might_sleep();
647
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200648 lock_map_acquire(&work->lockdep_map);
649 lock_map_release(&work->lockdep_map);
Johannes Berg4e6045f2007-10-18 23:39:55 -0700650
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700651 cwq = get_wq_data(work);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700652 if (!cwq)
Oleg Nesterov3af244332007-05-09 02:34:09 -0700653 return;
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700654
Oleg Nesterov28e53bd2007-05-09 02:34:22 -0700655 wq = cwq->wq;
656 cpu_map = wq_cpu_map(wq);
657
Rusty Russellaa85ea52009-03-30 22:05:15 -0600658 for_each_cpu(cpu, cpu_map)
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700659 wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work);
660}
661
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700662static int __cancel_work_timer(struct work_struct *work,
663 struct timer_list* timer)
664{
665 int ret;
666
667 do {
668 ret = (timer && likely(del_timer(timer)));
669 if (!ret)
670 ret = try_to_grab_pending(work);
671 wait_on_work(work);
672 } while (unlikely(ret < 0));
673
674 work_clear_pending(work);
675 return ret;
676}
677
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700678/**
679 * cancel_work_sync - block until a work_struct's callback has terminated
680 * @work: the work which is to be flushed
681 *
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700682 * Returns true if @work was pending.
683 *
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700684 * cancel_work_sync() will cancel the work if it is queued. If the work's
685 * callback appears to be running, cancel_work_sync() will block until it
686 * has completed.
687 *
688 * It is possible to use this function if the work re-queues itself. It can
689 * cancel the work even if it migrates to another workqueue, however in that
690 * case it only guarantees that work->func() has completed on the last queued
691 * workqueue.
692 *
693 * cancel_work_sync(&delayed_work->work) should be used only if ->timer is not
694 * pending, otherwise it goes into a busy-wait loop until the timer expires.
695 *
696 * The caller must ensure that workqueue_struct on which this work was last
697 * queued can't be destroyed before this function returns.
698 */
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700699int cancel_work_sync(struct work_struct *work)
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700700{
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700701 return __cancel_work_timer(work, NULL);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700702}
Oleg Nesterov28e53bd2007-05-09 02:34:22 -0700703EXPORT_SYMBOL_GPL(cancel_work_sync);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700704
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700705/**
Oleg Nesterovf5a421a2007-07-15 23:41:44 -0700706 * cancel_delayed_work_sync - reliably kill off a delayed work.
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700707 * @dwork: the delayed work struct
708 *
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700709 * Returns true if @dwork was pending.
710 *
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700711 * It is possible to use this function if @dwork rearms itself via queue_work()
712 * or queue_delayed_work(). See also the comment for cancel_work_sync().
713 */
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700714int cancel_delayed_work_sync(struct delayed_work *dwork)
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700715{
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700716 return __cancel_work_timer(&dwork->work, &dwork->timer);
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700717}
Oleg Nesterovf5a421a2007-07-15 23:41:44 -0700718EXPORT_SYMBOL(cancel_delayed_work_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700720static struct workqueue_struct *keventd_wq __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700722/**
723 * schedule_work - put work task in global workqueue
724 * @work: job to be done
725 *
Bart Van Assche5b0f437d2009-07-30 19:00:53 +0200726 * Returns zero if @work was already on the kernel-global workqueue and
727 * non-zero otherwise.
728 *
729 * This puts a job in the kernel-global workqueue if it was not already
730 * queued and leaves it in the same position on the kernel-global
731 * workqueue otherwise.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700732 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800733int schedule_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734{
735 return queue_work(keventd_wq, work);
736}
Dave Jonesae90dd52006-06-30 01:40:45 -0400737EXPORT_SYMBOL(schedule_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738
Zhang Ruic1a220e2008-07-23 21:28:39 -0700739/*
740 * schedule_work_on - put work task on a specific cpu
741 * @cpu: cpu to put the work task on
742 * @work: job to be done
743 *
744 * This puts a job on a specific cpu
745 */
746int schedule_work_on(int cpu, struct work_struct *work)
747{
748 return queue_work_on(cpu, keventd_wq, work);
749}
750EXPORT_SYMBOL(schedule_work_on);
751
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700752/**
753 * schedule_delayed_work - put work task in global workqueue after delay
David Howells52bad642006-11-22 14:54:01 +0000754 * @dwork: job to be done
755 * @delay: number of jiffies to wait or 0 for immediate execution
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700756 *
757 * After waiting for a given time this puts a job in the kernel-global
758 * workqueue.
759 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800760int schedule_delayed_work(struct delayed_work *dwork,
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800761 unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762{
David Howells52bad642006-11-22 14:54:01 +0000763 return queue_delayed_work(keventd_wq, dwork, delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700764}
Dave Jonesae90dd52006-06-30 01:40:45 -0400765EXPORT_SYMBOL(schedule_delayed_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700766
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700767/**
Linus Torvalds8c53e462009-10-14 09:16:42 -0700768 * flush_delayed_work - block until a dwork_struct's callback has terminated
769 * @dwork: the delayed work which is to be flushed
770 *
771 * Any timeout is cancelled, and any pending work is run immediately.
772 */
773void flush_delayed_work(struct delayed_work *dwork)
774{
775 if (del_timer_sync(&dwork->timer)) {
776 struct cpu_workqueue_struct *cwq;
777 cwq = wq_per_cpu(keventd_wq, get_cpu());
778 __queue_work(cwq, &dwork->work);
779 put_cpu();
780 }
781 flush_work(&dwork->work);
782}
783EXPORT_SYMBOL(flush_delayed_work);
784
785/**
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700786 * schedule_delayed_work_on - queue work in global workqueue on CPU after delay
787 * @cpu: cpu to use
David Howells52bad642006-11-22 14:54:01 +0000788 * @dwork: job to be done
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700789 * @delay: number of jiffies to wait
790 *
791 * After waiting for a given time this puts a job in the kernel-global
792 * workqueue on the specified CPU.
793 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794int schedule_delayed_work_on(int cpu,
David Howells52bad642006-11-22 14:54:01 +0000795 struct delayed_work *dwork, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700796{
David Howells52bad642006-11-22 14:54:01 +0000797 return queue_delayed_work_on(cpu, keventd_wq, dwork, delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798}
Dave Jonesae90dd52006-06-30 01:40:45 -0400799EXPORT_SYMBOL(schedule_delayed_work_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800
Andrew Mortonb6136772006-06-25 05:47:49 -0700801/**
802 * schedule_on_each_cpu - call a function on each online CPU from keventd
803 * @func: the function to call
Andrew Mortonb6136772006-06-25 05:47:49 -0700804 *
805 * Returns zero on success.
806 * Returns -ve errno on failure.
807 *
Andrew Mortonb6136772006-06-25 05:47:49 -0700808 * schedule_on_each_cpu() is very slow.
809 */
David Howells65f27f32006-11-22 14:55:48 +0000810int schedule_on_each_cpu(work_func_t func)
Christoph Lameter15316ba2006-01-08 01:00:43 -0800811{
812 int cpu;
Andi Kleen65a64462009-10-14 06:22:47 +0200813 int orig = -1;
Andrew Mortonb6136772006-06-25 05:47:49 -0700814 struct work_struct *works;
Christoph Lameter15316ba2006-01-08 01:00:43 -0800815
Andrew Mortonb6136772006-06-25 05:47:49 -0700816 works = alloc_percpu(struct work_struct);
817 if (!works)
Christoph Lameter15316ba2006-01-08 01:00:43 -0800818 return -ENOMEM;
Andrew Mortonb6136772006-06-25 05:47:49 -0700819
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100820 get_online_cpus();
Tejun Heo93981802009-11-17 14:06:20 -0800821
822 /*
823 * When running in keventd don't schedule a work item on
824 * itself. Can just call directly because the work queue is
825 * already bound. This also is faster.
826 */
827 if (current_is_keventd())
828 orig = raw_smp_processor_id();
829
Christoph Lameter15316ba2006-01-08 01:00:43 -0800830 for_each_online_cpu(cpu) {
Ingo Molnar9bfb1832006-12-18 20:05:09 +0100831 struct work_struct *work = per_cpu_ptr(works, cpu);
832
833 INIT_WORK(work, func);
Andi Kleen65a64462009-10-14 06:22:47 +0200834 if (cpu != orig)
Tejun Heo93981802009-11-17 14:06:20 -0800835 schedule_work_on(cpu, work);
Andi Kleen65a64462009-10-14 06:22:47 +0200836 }
Tejun Heo93981802009-11-17 14:06:20 -0800837 if (orig >= 0)
838 func(per_cpu_ptr(works, orig));
839
840 for_each_online_cpu(cpu)
841 flush_work(per_cpu_ptr(works, cpu));
842
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100843 put_online_cpus();
Andrew Mortonb6136772006-06-25 05:47:49 -0700844 free_percpu(works);
Christoph Lameter15316ba2006-01-08 01:00:43 -0800845 return 0;
846}
847
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848void flush_scheduled_work(void)
849{
850 flush_workqueue(keventd_wq);
851}
Dave Jonesae90dd52006-06-30 01:40:45 -0400852EXPORT_SYMBOL(flush_scheduled_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853
854/**
James Bottomley1fa44ec2006-02-23 12:43:43 -0600855 * execute_in_process_context - reliably execute the routine with user context
856 * @fn: the function to execute
James Bottomley1fa44ec2006-02-23 12:43:43 -0600857 * @ew: guaranteed storage for the execute work structure (must
858 * be available when the work executes)
859 *
860 * Executes the function immediately if process context is available,
861 * otherwise schedules the function for delayed execution.
862 *
863 * Returns: 0 - function was executed
864 * 1 - function was scheduled for execution
865 */
David Howells65f27f32006-11-22 14:55:48 +0000866int execute_in_process_context(work_func_t fn, struct execute_work *ew)
James Bottomley1fa44ec2006-02-23 12:43:43 -0600867{
868 if (!in_interrupt()) {
David Howells65f27f32006-11-22 14:55:48 +0000869 fn(&ew->work);
James Bottomley1fa44ec2006-02-23 12:43:43 -0600870 return 0;
871 }
872
David Howells65f27f32006-11-22 14:55:48 +0000873 INIT_WORK(&ew->work, fn);
James Bottomley1fa44ec2006-02-23 12:43:43 -0600874 schedule_work(&ew->work);
875
876 return 1;
877}
878EXPORT_SYMBOL_GPL(execute_in_process_context);
879
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880int keventd_up(void)
881{
882 return keventd_wq != NULL;
883}
884
885int current_is_keventd(void)
886{
887 struct cpu_workqueue_struct *cwq;
Hugh Dickinsd2437692007-08-27 16:06:19 +0100888 int cpu = raw_smp_processor_id(); /* preempt-safe: keventd is per-cpu */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 int ret = 0;
890
891 BUG_ON(!keventd_wq);
892
Christoph Lameter89ada672005-10-30 15:01:59 -0800893 cwq = per_cpu_ptr(keventd_wq->cpu_wq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894 if (current == cwq->thread)
895 ret = 1;
896
897 return ret;
898
899}
900
Oleg Nesterov3af244332007-05-09 02:34:09 -0700901static struct cpu_workqueue_struct *
902init_cpu_workqueue(struct workqueue_struct *wq, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903{
Christoph Lameter89ada672005-10-30 15:01:59 -0800904 struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905
Oleg Nesterov3af244332007-05-09 02:34:09 -0700906 cwq->wq = wq;
907 spin_lock_init(&cwq->lock);
908 INIT_LIST_HEAD(&cwq->worklist);
909 init_waitqueue_head(&cwq->more_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700910
Oleg Nesterov3af244332007-05-09 02:34:09 -0700911 return cwq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700912}
913
Oleg Nesterov3af244332007-05-09 02:34:09 -0700914static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915{
Heiko Carstens0d557dc2008-10-13 23:50:09 +0200916 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
Oleg Nesterov3af244332007-05-09 02:34:09 -0700917 struct workqueue_struct *wq = cwq->wq;
David Howells6cc88bc2008-11-14 10:39:21 +1100918 const char *fmt = is_wq_single_threaded(wq) ? "%s" : "%s/%d";
Oleg Nesterov3af244332007-05-09 02:34:09 -0700919 struct task_struct *p;
920
921 p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu);
922 /*
923 * Nobody can add the work_struct to this cwq,
924 * if (caller is __create_workqueue)
925 * nobody should see this wq
926 * else // caller is CPU_UP_PREPARE
927 * cpu is not on cpu_online_map
928 * so we can abort safely.
929 */
930 if (IS_ERR(p))
931 return PTR_ERR(p);
Heiko Carstens0d557dc2008-10-13 23:50:09 +0200932 if (cwq->wq->rt)
933 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700934 cwq->thread = p;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700935
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +0100936 trace_workqueue_creation(cwq->thread, cpu);
937
Oleg Nesterov3af244332007-05-09 02:34:09 -0700938 return 0;
939}
940
Oleg Nesterov06ba38a2007-05-09 02:34:15 -0700941static void start_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
942{
943 struct task_struct *p = cwq->thread;
944
945 if (p != NULL) {
946 if (cpu >= 0)
947 kthread_bind(p, cpu);
948 wake_up_process(p);
949 }
950}
951
Johannes Berg4e6045f2007-10-18 23:39:55 -0700952struct workqueue_struct *__create_workqueue_key(const char *name,
953 int singlethread,
954 int freezeable,
Heiko Carstens0d557dc2008-10-13 23:50:09 +0200955 int rt,
Johannes Bergeb13ba82008-01-16 09:51:58 +0100956 struct lock_class_key *key,
957 const char *lock_name)
Oleg Nesterov3af244332007-05-09 02:34:09 -0700958{
959 struct workqueue_struct *wq;
960 struct cpu_workqueue_struct *cwq;
961 int err = 0, cpu;
962
963 wq = kzalloc(sizeof(*wq), GFP_KERNEL);
964 if (!wq)
965 return NULL;
966
967 wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct);
968 if (!wq->cpu_wq) {
969 kfree(wq);
970 return NULL;
971 }
972
973 wq->name = name;
Johannes Bergeb13ba82008-01-16 09:51:58 +0100974 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
Oleg Nesterovcce1a162007-05-09 02:34:13 -0700975 wq->singlethread = singlethread;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700976 wq->freezeable = freezeable;
Heiko Carstens0d557dc2008-10-13 23:50:09 +0200977 wq->rt = rt;
Oleg Nesterovcce1a162007-05-09 02:34:13 -0700978 INIT_LIST_HEAD(&wq->list);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700979
980 if (singlethread) {
Oleg Nesterov3af244332007-05-09 02:34:09 -0700981 cwq = init_cpu_workqueue(wq, singlethread_cpu);
982 err = create_workqueue_thread(cwq, singlethread_cpu);
Oleg Nesterov06ba38a2007-05-09 02:34:15 -0700983 start_workqueue_thread(cwq, -1);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700984 } else {
Oleg Nesterov3da1c842008-07-25 01:47:50 -0700985 cpu_maps_update_begin();
Oleg Nesterov6af8bf32008-07-29 22:33:49 -0700986 /*
987 * We must place this wq on list even if the code below fails.
988 * cpu_down(cpu) can remove cpu from cpu_populated_map before
989 * destroy_workqueue() takes the lock, in that case we leak
990 * cwq[cpu]->thread.
991 */
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100992 spin_lock(&workqueue_lock);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700993 list_add(&wq->list, &workqueues);
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100994 spin_unlock(&workqueue_lock);
Oleg Nesterov6af8bf32008-07-29 22:33:49 -0700995 /*
996 * We must initialize cwqs for each possible cpu even if we
997 * are going to call destroy_workqueue() finally. Otherwise
998 * cpu_up() can hit the uninitialized cwq once we drop the
999 * lock.
1000 */
Oleg Nesterov3af244332007-05-09 02:34:09 -07001001 for_each_possible_cpu(cpu) {
1002 cwq = init_cpu_workqueue(wq, cpu);
1003 if (err || !cpu_online(cpu))
1004 continue;
1005 err = create_workqueue_thread(cwq, cpu);
Oleg Nesterov06ba38a2007-05-09 02:34:15 -07001006 start_workqueue_thread(cwq, cpu);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001007 }
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001008 cpu_maps_update_done();
Oleg Nesterov3af244332007-05-09 02:34:09 -07001009 }
1010
1011 if (err) {
1012 destroy_workqueue(wq);
1013 wq = NULL;
1014 }
1015 return wq;
1016}
Johannes Berg4e6045f2007-10-18 23:39:55 -07001017EXPORT_SYMBOL_GPL(__create_workqueue_key);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001018
Oleg Nesterov1e35eaa2008-04-29 01:00:28 -07001019static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
Oleg Nesterov3af244332007-05-09 02:34:09 -07001020{
Oleg Nesterov14441962007-05-23 13:57:57 -07001021 /*
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001022 * Our caller is either destroy_workqueue() or CPU_POST_DEAD,
1023 * cpu_add_remove_lock protects cwq->thread.
Oleg Nesterov14441962007-05-23 13:57:57 -07001024 */
1025 if (cwq->thread == NULL)
1026 return;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001027
Ingo Molnar3295f0e2008-08-11 10:30:30 +02001028 lock_map_acquire(&cwq->wq->lockdep_map);
1029 lock_map_release(&cwq->wq->lockdep_map);
Johannes Berg4e6045f2007-10-18 23:39:55 -07001030
Oleg Nesterov13c22162007-07-17 04:03:55 -07001031 flush_cpu_workqueue(cwq);
Oleg Nesterov14441962007-05-23 13:57:57 -07001032 /*
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001033 * If the caller is CPU_POST_DEAD and cwq->worklist was not empty,
Oleg Nesterov13c22162007-07-17 04:03:55 -07001034 * a concurrent flush_workqueue() can insert a barrier after us.
1035 * However, in that case run_workqueue() won't return and check
1036 * kthread_should_stop() until it flushes all work_struct's.
Oleg Nesterov14441962007-05-23 13:57:57 -07001037 * When ->worklist becomes empty it is safe to exit because no
1038 * more work_structs can be queued on this cwq: flush_workqueue
1039 * checks list_empty(), and a "normal" queue_work() can't use
1040 * a dead CPU.
1041 */
Frederic Weisbeckere1d8aa92009-01-12 23:15:46 +01001042 trace_workqueue_destruction(cwq->thread);
Oleg Nesterov14441962007-05-23 13:57:57 -07001043 kthread_stop(cwq->thread);
1044 cwq->thread = NULL;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001045}
1046
1047/**
1048 * destroy_workqueue - safely terminate a workqueue
1049 * @wq: target workqueue
1050 *
1051 * Safely destroy a workqueue. All work currently pending will be done first.
1052 */
1053void destroy_workqueue(struct workqueue_struct *wq)
1054{
Rusty Russelle7577c52009-01-01 10:12:25 +10301055 const struct cpumask *cpu_map = wq_cpu_map(wq);
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -07001056 int cpu;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001057
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001058 cpu_maps_update_begin();
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001059 spin_lock(&workqueue_lock);
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -07001060 list_del(&wq->list);
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001061 spin_unlock(&workqueue_lock);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001062
Rusty Russellaa85ea52009-03-30 22:05:15 -06001063 for_each_cpu(cpu, cpu_map)
Oleg Nesterov1e35eaa2008-04-29 01:00:28 -07001064 cleanup_workqueue_thread(per_cpu_ptr(wq->cpu_wq, cpu));
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001065 cpu_maps_update_done();
Oleg Nesterov3af244332007-05-09 02:34:09 -07001066
1067 free_percpu(wq->cpu_wq);
1068 kfree(wq);
1069}
1070EXPORT_SYMBOL_GPL(destroy_workqueue);
1071
1072static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
1073 unsigned long action,
1074 void *hcpu)
1075{
1076 unsigned int cpu = (unsigned long)hcpu;
1077 struct cpu_workqueue_struct *cwq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078 struct workqueue_struct *wq;
Oleg Nesterov84485022008-07-25 01:47:54 -07001079 int ret = NOTIFY_OK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07001081 action &= ~CPU_TASKS_FROZEN;
1082
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083 switch (action) {
1084 case CPU_UP_PREPARE:
Rusty Russelle7577c52009-01-01 10:12:25 +10301085 cpumask_set_cpu(cpu, cpu_populated_map);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001086 }
Oleg Nesterov84485022008-07-25 01:47:54 -07001087undo:
Oleg Nesterov3af244332007-05-09 02:34:09 -07001088 list_for_each_entry(wq, &workqueues, list) {
1089 cwq = per_cpu_ptr(wq->cpu_wq, cpu);
Christoph Lameter89ada672005-10-30 15:01:59 -08001090
Oleg Nesterov3af244332007-05-09 02:34:09 -07001091 switch (action) {
1092 case CPU_UP_PREPARE:
1093 if (!create_workqueue_thread(cwq, cpu))
1094 break;
Gautham R Shenoy95402b32008-01-25 21:08:02 +01001095 printk(KERN_ERR "workqueue [%s] for %i failed\n",
1096 wq->name, cpu);
Oleg Nesterov84485022008-07-25 01:47:54 -07001097 action = CPU_UP_CANCELED;
1098 ret = NOTIFY_BAD;
1099 goto undo;
Oleg Nesterov3af244332007-05-09 02:34:09 -07001100
1101 case CPU_ONLINE:
Oleg Nesterov06ba38a2007-05-09 02:34:15 -07001102 start_workqueue_thread(cwq, cpu);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001103 break;
1104
1105 case CPU_UP_CANCELED:
Oleg Nesterov06ba38a2007-05-09 02:34:15 -07001106 start_workqueue_thread(cwq, -1);
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001107 case CPU_POST_DEAD:
Oleg Nesterov1e35eaa2008-04-29 01:00:28 -07001108 cleanup_workqueue_thread(cwq);
Oleg Nesterov3af244332007-05-09 02:34:09 -07001109 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001110 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001111 }
1112
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -07001113 switch (action) {
1114 case CPU_UP_CANCELED:
Oleg Nesterov3da1c842008-07-25 01:47:50 -07001115 case CPU_POST_DEAD:
Rusty Russelle7577c52009-01-01 10:12:25 +10301116 cpumask_clear_cpu(cpu, cpu_populated_map);
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -07001117 }
1118
Oleg Nesterov84485022008-07-25 01:47:54 -07001119 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001120}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001121
Rusty Russell2d3854a2008-11-05 13:39:10 +11001122#ifdef CONFIG_SMP
Rusty Russell8ccad402009-01-16 15:31:15 -08001123
Rusty Russell2d3854a2008-11-05 13:39:10 +11001124struct work_for_cpu {
Andrew Morton6b44003e2009-04-09 09:50:37 -06001125 struct completion completion;
Rusty Russell2d3854a2008-11-05 13:39:10 +11001126 long (*fn)(void *);
1127 void *arg;
1128 long ret;
1129};
1130
Andrew Morton6b44003e2009-04-09 09:50:37 -06001131static int do_work_for_cpu(void *_wfc)
Rusty Russell2d3854a2008-11-05 13:39:10 +11001132{
Andrew Morton6b44003e2009-04-09 09:50:37 -06001133 struct work_for_cpu *wfc = _wfc;
Rusty Russell2d3854a2008-11-05 13:39:10 +11001134 wfc->ret = wfc->fn(wfc->arg);
Andrew Morton6b44003e2009-04-09 09:50:37 -06001135 complete(&wfc->completion);
1136 return 0;
Rusty Russell2d3854a2008-11-05 13:39:10 +11001137}
1138
1139/**
1140 * work_on_cpu - run a function in user context on a particular cpu
1141 * @cpu: the cpu to run on
1142 * @fn: the function to run
1143 * @arg: the function arg
1144 *
Rusty Russell31ad9082009-01-16 15:31:15 -08001145 * This will return the value @fn returns.
1146 * It is up to the caller to ensure that the cpu doesn't go offline.
Andrew Morton6b44003e2009-04-09 09:50:37 -06001147 * The caller must not hold any locks which would prevent @fn from completing.
Rusty Russell2d3854a2008-11-05 13:39:10 +11001148 */
1149long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg)
1150{
Andrew Morton6b44003e2009-04-09 09:50:37 -06001151 struct task_struct *sub_thread;
1152 struct work_for_cpu wfc = {
1153 .completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion),
1154 .fn = fn,
1155 .arg = arg,
1156 };
Rusty Russell2d3854a2008-11-05 13:39:10 +11001157
Andrew Morton6b44003e2009-04-09 09:50:37 -06001158 sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu");
1159 if (IS_ERR(sub_thread))
1160 return PTR_ERR(sub_thread);
1161 kthread_bind(sub_thread, cpu);
1162 wake_up_process(sub_thread);
1163 wait_for_completion(&wfc.completion);
Rusty Russell2d3854a2008-11-05 13:39:10 +11001164 return wfc.ret;
1165}
1166EXPORT_SYMBOL_GPL(work_on_cpu);
1167#endif /* CONFIG_SMP */
1168
Oleg Nesterovc12920d2007-05-09 02:34:14 -07001169void __init init_workqueues(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170{
Rusty Russelle7577c52009-01-01 10:12:25 +10301171 alloc_cpumask_var(&cpu_populated_map, GFP_KERNEL);
1172
1173 cpumask_copy(cpu_populated_map, cpu_online_mask);
1174 singlethread_cpu = cpumask_first(cpu_possible_mask);
1175 cpu_singlethread_map = cpumask_of(singlethread_cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176 hotcpu_notifier(workqueue_cpu_callback, 0);
1177 keventd_wq = create_workqueue("events");
1178 BUG_ON(!keventd_wq);
1179}