blob: 4048e92aa04f21e7e105563fac7bf24e1be5411f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/kernel/workqueue.c
3 *
4 * Generic mechanism for defining kernel helper threads for running
5 * arbitrary tasks in process context.
6 *
7 * Started by Ingo Molnar, Copyright (C) 2002
8 *
9 * Derived from the taskqueue/keventd code by:
10 *
11 * David Woodhouse <dwmw2@infradead.org>
12 * Andrew Morton <andrewm@uow.edu.au>
13 * Kai Petzke <wpp@marie.physik.tu-berlin.de>
14 * Theodore Ts'o <tytso@mit.edu>
Christoph Lameter89ada672005-10-30 15:01:59 -080015 *
Christoph Lametercde53532008-07-04 09:59:22 -070016 * Made to use alloc_percpu by Christoph Lameter.
Linus Torvalds1da177e2005-04-16 15:20:36 -070017 */
18
19#include <linux/module.h>
20#include <linux/kernel.h>
21#include <linux/sched.h>
22#include <linux/init.h>
23#include <linux/signal.h>
24#include <linux/completion.h>
25#include <linux/workqueue.h>
26#include <linux/slab.h>
27#include <linux/cpu.h>
28#include <linux/notifier.h>
29#include <linux/kthread.h>
James Bottomley1fa44ec2006-02-23 12:43:43 -060030#include <linux/hardirq.h>
Christoph Lameter46934022006-10-11 01:21:26 -070031#include <linux/mempolicy.h>
Rafael J. Wysocki341a5952006-12-06 20:34:49 -080032#include <linux/freezer.h>
Peter Zijlstrad5abe662006-12-06 20:37:26 -080033#include <linux/kallsyms.h>
34#include <linux/debug_locks.h>
Johannes Berg4e6045f2007-10-18 23:39:55 -070035#include <linux/lockdep.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036
37/*
Nathan Lynchf756d5e2006-01-08 01:05:12 -080038 * The per-CPU workqueue (if single thread, we always use the first
39 * possible cpu).
Linus Torvalds1da177e2005-04-16 15:20:36 -070040 */
41struct cpu_workqueue_struct {
42
43 spinlock_t lock;
44
Linus Torvalds1da177e2005-04-16 15:20:36 -070045 struct list_head worklist;
46 wait_queue_head_t more_work;
Oleg Nesterov3af244332007-05-09 02:34:09 -070047 struct work_struct *current_work;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
49 struct workqueue_struct *wq;
Ingo Molnar36c8b582006-07-03 00:25:41 -070050 struct task_struct *thread;
Linus Torvalds1da177e2005-04-16 15:20:36 -070051
52 int run_depth; /* Detect run_workqueue() recursion depth */
53} ____cacheline_aligned;
54
55/*
56 * The externally visible workqueue abstraction is an array of
57 * per-CPU workqueues:
58 */
59struct workqueue_struct {
Christoph Lameter89ada672005-10-30 15:01:59 -080060 struct cpu_workqueue_struct *cpu_wq;
Oleg Nesterovcce1a162007-05-09 02:34:13 -070061 struct list_head list;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062 const char *name;
Oleg Nesterovcce1a162007-05-09 02:34:13 -070063 int singlethread;
Oleg Nesterov319c2a92007-05-09 02:34:06 -070064 int freezeable; /* Freeze threads during suspend */
Johannes Berg4e6045f2007-10-18 23:39:55 -070065#ifdef CONFIG_LOCKDEP
66 struct lockdep_map lockdep_map;
67#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070068};
69
Gautham R Shenoy95402b32008-01-25 21:08:02 +010070/* Serializes the accesses to the list of workqueues. */
71static DEFINE_SPINLOCK(workqueue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070072static LIST_HEAD(workqueues);
73
Oleg Nesterov3af244332007-05-09 02:34:09 -070074static int singlethread_cpu __read_mostly;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -070075static cpumask_t cpu_singlethread_map __read_mostly;
Oleg Nesterov14441962007-05-23 13:57:57 -070076/*
77 * _cpu_down() first removes CPU from cpu_online_map, then CPU_DEAD
78 * flushes cwq->worklist. This means that flush_workqueue/wait_on_work
79 * which comes in between can't use for_each_online_cpu(). We could
80 * use cpu_possible_map, the cpumask below is more a documentation
81 * than optimization.
82 */
Oleg Nesterov3af244332007-05-09 02:34:09 -070083static cpumask_t cpu_populated_map __read_mostly;
Nathan Lynchf756d5e2006-01-08 01:05:12 -080084
Linus Torvalds1da177e2005-04-16 15:20:36 -070085/* If it's single threaded, it isn't in the list of workqueues. */
86static inline int is_single_threaded(struct workqueue_struct *wq)
87{
Oleg Nesterovcce1a162007-05-09 02:34:13 -070088 return wq->singlethread;
Linus Torvalds1da177e2005-04-16 15:20:36 -070089}
90
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -070091static const cpumask_t *wq_cpu_map(struct workqueue_struct *wq)
92{
93 return is_single_threaded(wq)
94 ? &cpu_singlethread_map : &cpu_populated_map;
95}
96
Oleg Nesterova848e3b2007-05-09 02:34:17 -070097static
98struct cpu_workqueue_struct *wq_per_cpu(struct workqueue_struct *wq, int cpu)
99{
100 if (unlikely(is_single_threaded(wq)))
101 cpu = singlethread_cpu;
102 return per_cpu_ptr(wq->cpu_wq, cpu);
103}
104
David Howells4594bf12006-12-07 11:33:26 +0000105/*
106 * Set the workqueue on which a work item is to be run
107 * - Must *only* be called if the pending flag is set
108 */
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700109static inline void set_wq_data(struct work_struct *work,
110 struct cpu_workqueue_struct *cwq)
David Howells365970a2006-11-22 14:54:49 +0000111{
David Howells4594bf12006-12-07 11:33:26 +0000112 unsigned long new;
David Howells365970a2006-11-22 14:54:49 +0000113
David Howells4594bf12006-12-07 11:33:26 +0000114 BUG_ON(!work_pending(work));
115
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700116 new = (unsigned long) cwq | (1UL << WORK_STRUCT_PENDING);
Linus Torvaldsa08727b2006-12-16 09:53:50 -0800117 new |= WORK_STRUCT_FLAG_MASK & *work_data_bits(work);
118 atomic_long_set(&work->data, new);
David Howells365970a2006-11-22 14:54:49 +0000119}
120
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700121static inline
122struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
David Howells365970a2006-11-22 14:54:49 +0000123{
Linus Torvaldsa08727b2006-12-16 09:53:50 -0800124 return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
David Howells365970a2006-11-22 14:54:49 +0000125}
126
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700127static void insert_work(struct cpu_workqueue_struct *cwq,
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700128 struct work_struct *work, struct list_head *head)
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700129{
130 set_wq_data(work, cwq);
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700131 /*
132 * Ensure that we get the right work->data if we see the
133 * result of list_add() below, see try_to_grab_pending().
134 */
135 smp_wmb();
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700136 list_add_tail(&work->entry, head);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700137 wake_up(&cwq->more_work);
138}
139
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140static void __queue_work(struct cpu_workqueue_struct *cwq,
141 struct work_struct *work)
142{
143 unsigned long flags;
144
145 spin_lock_irqsave(&cwq->lock, flags);
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700146 insert_work(cwq, work, &cwq->worklist);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700147 spin_unlock_irqrestore(&cwq->lock, flags);
148}
149
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700150/**
151 * queue_work - queue work on a workqueue
152 * @wq: workqueue to use
153 * @work: work to queue
154 *
Alan Stern057647f2006-10-28 10:38:58 -0700155 * Returns 0 if @work was already on a queue, non-zero otherwise.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700156 *
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -0700157 * We queue the work to the CPU on which it was submitted, but if the CPU dies
158 * it can be processed by another CPU.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700159 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800160int queue_work(struct workqueue_struct *wq, struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161{
Oleg Nesterovef1ca232008-07-25 01:47:53 -0700162 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163
Oleg Nesterovef1ca232008-07-25 01:47:53 -0700164 ret = queue_work_on(get_cpu(), wq, work);
165 put_cpu();
166
Linus Torvalds1da177e2005-04-16 15:20:36 -0700167 return ret;
168}
Dave Jonesae90dd52006-06-30 01:40:45 -0400169EXPORT_SYMBOL_GPL(queue_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170
Zhang Ruic1a220e2008-07-23 21:28:39 -0700171/**
172 * queue_work_on - queue work on specific cpu
173 * @cpu: CPU number to execute work on
174 * @wq: workqueue to use
175 * @work: work to queue
176 *
177 * Returns 0 if @work was already on a queue, non-zero otherwise.
178 *
179 * We queue the work to a specific CPU, the caller must ensure it
180 * can't go away.
181 */
182int
183queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work)
184{
185 int ret = 0;
186
187 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
188 BUG_ON(!list_empty(&work->entry));
189 __queue_work(wq_per_cpu(wq, cpu), work);
190 ret = 1;
191 }
192 return ret;
193}
194EXPORT_SYMBOL_GPL(queue_work_on);
195
Li Zefan6d141c32008-02-08 04:21:09 -0800196static void delayed_work_timer_fn(unsigned long __data)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197{
David Howells52bad642006-11-22 14:54:01 +0000198 struct delayed_work *dwork = (struct delayed_work *)__data;
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700199 struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work);
200 struct workqueue_struct *wq = cwq->wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700202 __queue_work(wq_per_cpu(wq, smp_processor_id()), &dwork->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203}
204
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700205/**
206 * queue_delayed_work - queue work on a workqueue after delay
207 * @wq: workqueue to use
Randy Dunlapaf9997e2006-12-22 01:06:52 -0800208 * @dwork: delayable work to queue
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700209 * @delay: number of jiffies to wait before queueing
210 *
Alan Stern057647f2006-10-28 10:38:58 -0700211 * Returns 0 if @work was already on a queue, non-zero otherwise.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700212 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800213int queue_delayed_work(struct workqueue_struct *wq,
David Howells52bad642006-11-22 14:54:01 +0000214 struct delayed_work *dwork, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700215{
David Howells52bad642006-11-22 14:54:01 +0000216 if (delay == 0)
Oleg Nesterov63bc0362007-05-09 02:34:16 -0700217 return queue_work(wq, &dwork->work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700218
Oleg Nesterov63bc0362007-05-09 02:34:16 -0700219 return queue_delayed_work_on(-1, wq, dwork, delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220}
Dave Jonesae90dd52006-06-30 01:40:45 -0400221EXPORT_SYMBOL_GPL(queue_delayed_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700222
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700223/**
224 * queue_delayed_work_on - queue work on specific CPU after delay
225 * @cpu: CPU number to execute work on
226 * @wq: workqueue to use
Randy Dunlapaf9997e2006-12-22 01:06:52 -0800227 * @dwork: work to queue
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700228 * @delay: number of jiffies to wait before queueing
229 *
Alan Stern057647f2006-10-28 10:38:58 -0700230 * Returns 0 if @work was already on a queue, non-zero otherwise.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700231 */
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700232int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
David Howells52bad642006-11-22 14:54:01 +0000233 struct delayed_work *dwork, unsigned long delay)
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700234{
235 int ret = 0;
David Howells52bad642006-11-22 14:54:01 +0000236 struct timer_list *timer = &dwork->timer;
237 struct work_struct *work = &dwork->work;
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700238
Linus Torvaldsa08727b2006-12-16 09:53:50 -0800239 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700240 BUG_ON(timer_pending(timer));
241 BUG_ON(!list_empty(&work->entry));
242
Andrew Liu8a3e77c2008-05-01 04:35:14 -0700243 timer_stats_timer_set_start_info(&dwork->timer);
244
Oleg Nesteroved7c0fe2007-05-09 02:34:16 -0700245 /* This stores cwq for the moment, for the timer_fn */
Oleg Nesterova848e3b2007-05-09 02:34:17 -0700246 set_wq_data(work, wq_per_cpu(wq, raw_smp_processor_id()));
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700247 timer->expires = jiffies + delay;
David Howells52bad642006-11-22 14:54:01 +0000248 timer->data = (unsigned long)dwork;
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700249 timer->function = delayed_work_timer_fn;
Oleg Nesterov63bc0362007-05-09 02:34:16 -0700250
251 if (unlikely(cpu >= 0))
252 add_timer_on(timer, cpu);
253 else
254 add_timer(timer);
Venkatesh Pallipadi7a6bc1c2006-06-28 13:50:33 -0700255 ret = 1;
256 }
257 return ret;
258}
Dave Jonesae90dd52006-06-30 01:40:45 -0400259EXPORT_SYMBOL_GPL(queue_delayed_work_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260
Arjan van de Ven858119e2006-01-14 13:20:43 -0800261static void run_workqueue(struct cpu_workqueue_struct *cwq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700262{
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700263 spin_lock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700264 cwq->run_depth++;
265 if (cwq->run_depth > 3) {
266 /* morton gets to eat his hat */
267 printk("%s: recursion depth exceeded: %d\n",
Harvey Harrisonaf1f16d2008-04-30 00:55:08 -0700268 __func__, cwq->run_depth);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 dump_stack();
270 }
271 while (!list_empty(&cwq->worklist)) {
272 struct work_struct *work = list_entry(cwq->worklist.next,
273 struct work_struct, entry);
David Howells6bb49e52006-11-22 14:54:45 +0000274 work_func_t f = work->func;
Johannes Berg4e6045f2007-10-18 23:39:55 -0700275#ifdef CONFIG_LOCKDEP
276 /*
277 * It is permissible to free the struct work_struct
278 * from inside the function that is called from it,
279 * this we need to take into account for lockdep too.
280 * To avoid bogus "held lock freed" warnings as well
281 * as problems when looking into work->lockdep_map,
282 * make a copy and use that here.
283 */
284 struct lockdep_map lockdep_map = work->lockdep_map;
285#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700287 cwq->current_work = work;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 list_del_init(cwq->worklist.next);
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700289 spin_unlock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290
David Howells365970a2006-11-22 14:54:49 +0000291 BUG_ON(get_wq_data(work) != cwq);
Oleg Nesterov23b2e592007-05-09 02:34:19 -0700292 work_clear_pending(work);
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200293 lock_map_acquire(&cwq->wq->lockdep_map);
294 lock_map_acquire(&lockdep_map);
David Howells65f27f32006-11-22 14:55:48 +0000295 f(work);
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200296 lock_map_release(&lockdep_map);
297 lock_map_release(&cwq->wq->lockdep_map);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298
Peter Zijlstrad5abe662006-12-06 20:37:26 -0800299 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
300 printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
301 "%s/0x%08x/%d\n",
302 current->comm, preempt_count(),
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700303 task_pid_nr(current));
Peter Zijlstrad5abe662006-12-06 20:37:26 -0800304 printk(KERN_ERR " last function: ");
305 print_symbol("%s\n", (unsigned long)f);
306 debug_show_held_locks(current);
307 dump_stack();
308 }
309
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700310 spin_lock_irq(&cwq->lock);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700311 cwq->current_work = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 }
313 cwq->run_depth--;
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700314 spin_unlock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315}
316
317static int worker_thread(void *__cwq)
318{
319 struct cpu_workqueue_struct *cwq = __cwq;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700320 DEFINE_WAIT(wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321
Rafael J. Wysocki83144182007-07-17 04:03:35 -0700322 if (cwq->wq->freezeable)
323 set_freezable();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324
325 set_user_nice(current, -5);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326
Oleg Nesterov3af244332007-05-09 02:34:09 -0700327 for (;;) {
Oleg Nesterov3af244332007-05-09 02:34:09 -0700328 prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
Oleg Nesterov14441962007-05-23 13:57:57 -0700329 if (!freezing(current) &&
330 !kthread_should_stop() &&
331 list_empty(&cwq->worklist))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 schedule();
Oleg Nesterov3af244332007-05-09 02:34:09 -0700333 finish_wait(&cwq->more_work, &wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700334
Oleg Nesterov85f41862007-05-09 02:34:20 -0700335 try_to_freeze();
336
Oleg Nesterov14441962007-05-23 13:57:57 -0700337 if (kthread_should_stop())
Oleg Nesterov3af244332007-05-09 02:34:09 -0700338 break;
339
340 run_workqueue(cwq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341 }
Oleg Nesterov3af244332007-05-09 02:34:09 -0700342
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 return 0;
344}
345
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700346struct wq_barrier {
347 struct work_struct work;
348 struct completion done;
349};
350
351static void wq_barrier_func(struct work_struct *work)
352{
353 struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
354 complete(&barr->done);
355}
356
Oleg Nesterov83c22522007-05-09 02:33:54 -0700357static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700358 struct wq_barrier *barr, struct list_head *head)
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700359{
360 INIT_WORK(&barr->work, wq_barrier_func);
361 __set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work));
362
363 init_completion(&barr->done);
Oleg Nesterov83c22522007-05-09 02:33:54 -0700364
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700365 insert_work(cwq, &barr->work, head);
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700366}
367
Oleg Nesterov14441962007-05-23 13:57:57 -0700368static int flush_cpu_workqueue(struct cpu_workqueue_struct *cwq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369{
Oleg Nesterov14441962007-05-23 13:57:57 -0700370 int active;
371
Linus Torvalds1da177e2005-04-16 15:20:36 -0700372 if (cwq->thread == current) {
373 /*
374 * Probably keventd trying to flush its own queue. So simply run
375 * it by hand rather than deadlocking.
376 */
377 run_workqueue(cwq);
Oleg Nesterov14441962007-05-23 13:57:57 -0700378 active = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 } else {
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700380 struct wq_barrier barr;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381
Oleg Nesterov14441962007-05-23 13:57:57 -0700382 active = 0;
Oleg Nesterov83c22522007-05-09 02:33:54 -0700383 spin_lock_irq(&cwq->lock);
384 if (!list_empty(&cwq->worklist) || cwq->current_work != NULL) {
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700385 insert_wq_barrier(cwq, &barr, &cwq->worklist);
Oleg Nesterov83c22522007-05-09 02:33:54 -0700386 active = 1;
387 }
388 spin_unlock_irq(&cwq->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389
Oleg Nesterovd7213042007-05-09 02:34:07 -0700390 if (active)
Oleg Nesterov83c22522007-05-09 02:33:54 -0700391 wait_for_completion(&barr.done);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 }
Oleg Nesterov14441962007-05-23 13:57:57 -0700393
394 return active;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700395}
396
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700397/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398 * flush_workqueue - ensure that any scheduled work has run to completion.
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700399 * @wq: workqueue to flush
Linus Torvalds1da177e2005-04-16 15:20:36 -0700400 *
401 * Forces execution of the workqueue and blocks until its completion.
402 * This is typically used in driver shutdown handlers.
403 *
Oleg Nesterovfc2e4d72007-05-09 02:33:51 -0700404 * We sleep until all works which were queued on entry have been handled,
405 * but we are not livelocked by new incoming ones.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700406 *
407 * This function used to run the workqueues itself. Now we just wait for the
408 * helper threads to do it.
409 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800410void flush_workqueue(struct workqueue_struct *wq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411{
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700412 const cpumask_t *cpu_map = wq_cpu_map(wq);
Oleg Nesterovcce1a162007-05-09 02:34:13 -0700413 int cpu;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700414
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700415 might_sleep();
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200416 lock_map_acquire(&wq->lockdep_map);
417 lock_map_release(&wq->lockdep_map);
Mike Travis363ab6f2008-05-12 21:21:13 +0200418 for_each_cpu_mask_nr(cpu, *cpu_map)
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700419 flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420}
Dave Jonesae90dd52006-06-30 01:40:45 -0400421EXPORT_SYMBOL_GPL(flush_workqueue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422
Oleg Nesterovdb700892008-07-25 01:47:49 -0700423/**
424 * flush_work - block until a work_struct's callback has terminated
425 * @work: the work which is to be flushed
426 *
Oleg Nesterova67da702008-07-25 01:47:52 -0700427 * Returns false if @work has already terminated.
428 *
Oleg Nesterovdb700892008-07-25 01:47:49 -0700429 * It is expected that, prior to calling flush_work(), the caller has
430 * arranged for the work to not be requeued, otherwise it doesn't make
431 * sense to use this function.
432 */
433int flush_work(struct work_struct *work)
434{
435 struct cpu_workqueue_struct *cwq;
436 struct list_head *prev;
437 struct wq_barrier barr;
438
439 might_sleep();
440 cwq = get_wq_data(work);
441 if (!cwq)
442 return 0;
443
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200444 lock_map_acquire(&cwq->wq->lockdep_map);
445 lock_map_release(&cwq->wq->lockdep_map);
Oleg Nesterova67da702008-07-25 01:47:52 -0700446
Oleg Nesterovdb700892008-07-25 01:47:49 -0700447 prev = NULL;
448 spin_lock_irq(&cwq->lock);
449 if (!list_empty(&work->entry)) {
450 /*
451 * See the comment near try_to_grab_pending()->smp_rmb().
452 * If it was re-queued under us we are not going to wait.
453 */
454 smp_rmb();
455 if (unlikely(cwq != get_wq_data(work)))
456 goto out;
457 prev = &work->entry;
458 } else {
459 if (cwq->current_work != work)
460 goto out;
461 prev = &cwq->worklist;
462 }
463 insert_wq_barrier(cwq, &barr, prev->next);
464out:
465 spin_unlock_irq(&cwq->lock);
466 if (!prev)
467 return 0;
468
469 wait_for_completion(&barr.done);
470 return 1;
471}
472EXPORT_SYMBOL_GPL(flush_work);
473
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700474/*
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700475 * Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit,
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700476 * so this work can't be re-armed in any way.
477 */
478static int try_to_grab_pending(struct work_struct *work)
479{
480 struct cpu_workqueue_struct *cwq;
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700481 int ret = -1;
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700482
483 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work)))
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700484 return 0;
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700485
486 /*
487 * The queueing is in progress, or it is already queued. Try to
488 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
489 */
490
491 cwq = get_wq_data(work);
492 if (!cwq)
493 return ret;
494
495 spin_lock_irq(&cwq->lock);
496 if (!list_empty(&work->entry)) {
497 /*
498 * This work is queued, but perhaps we locked the wrong cwq.
499 * In that case we must see the new value after rmb(), see
500 * insert_work()->wmb().
501 */
502 smp_rmb();
503 if (cwq == get_wq_data(work)) {
504 list_del_init(&work->entry);
505 ret = 1;
506 }
507 }
508 spin_unlock_irq(&cwq->lock);
509
510 return ret;
511}
512
513static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq,
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700514 struct work_struct *work)
515{
516 struct wq_barrier barr;
517 int running = 0;
518
519 spin_lock_irq(&cwq->lock);
520 if (unlikely(cwq->current_work == work)) {
Oleg Nesterov1a4d9b02008-07-25 01:47:47 -0700521 insert_wq_barrier(cwq, &barr, cwq->worklist.next);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700522 running = 1;
523 }
524 spin_unlock_irq(&cwq->lock);
525
Oleg Nesterov3af244332007-05-09 02:34:09 -0700526 if (unlikely(running))
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700527 wait_for_completion(&barr.done);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700528}
529
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700530static void wait_on_work(struct work_struct *work)
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700531{
532 struct cpu_workqueue_struct *cwq;
Oleg Nesterov28e53bd2007-05-09 02:34:22 -0700533 struct workqueue_struct *wq;
534 const cpumask_t *cpu_map;
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700535 int cpu;
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700536
Oleg Nesterovf293ea92007-05-09 02:34:10 -0700537 might_sleep();
538
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200539 lock_map_acquire(&work->lockdep_map);
540 lock_map_release(&work->lockdep_map);
Johannes Berg4e6045f2007-10-18 23:39:55 -0700541
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700542 cwq = get_wq_data(work);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700543 if (!cwq)
Oleg Nesterov3af244332007-05-09 02:34:09 -0700544 return;
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700545
Oleg Nesterov28e53bd2007-05-09 02:34:22 -0700546 wq = cwq->wq;
547 cpu_map = wq_cpu_map(wq);
548
Mike Travis363ab6f2008-05-12 21:21:13 +0200549 for_each_cpu_mask_nr(cpu, *cpu_map)
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700550 wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work);
551}
552
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700553static int __cancel_work_timer(struct work_struct *work,
554 struct timer_list* timer)
555{
556 int ret;
557
558 do {
559 ret = (timer && likely(del_timer(timer)));
560 if (!ret)
561 ret = try_to_grab_pending(work);
562 wait_on_work(work);
563 } while (unlikely(ret < 0));
564
565 work_clear_pending(work);
566 return ret;
567}
568
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700569/**
570 * cancel_work_sync - block until a work_struct's callback has terminated
571 * @work: the work which is to be flushed
572 *
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700573 * Returns true if @work was pending.
574 *
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700575 * cancel_work_sync() will cancel the work if it is queued. If the work's
576 * callback appears to be running, cancel_work_sync() will block until it
577 * has completed.
578 *
579 * It is possible to use this function if the work re-queues itself. It can
580 * cancel the work even if it migrates to another workqueue, however in that
581 * case it only guarantees that work->func() has completed on the last queued
582 * workqueue.
583 *
584 * cancel_work_sync(&delayed_work->work) should be used only if ->timer is not
585 * pending, otherwise it goes into a busy-wait loop until the timer expires.
586 *
587 * The caller must ensure that workqueue_struct on which this work was last
588 * queued can't be destroyed before this function returns.
589 */
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700590int cancel_work_sync(struct work_struct *work)
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700591{
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700592 return __cancel_work_timer(work, NULL);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700593}
Oleg Nesterov28e53bd2007-05-09 02:34:22 -0700594EXPORT_SYMBOL_GPL(cancel_work_sync);
Oleg Nesterovb89deed2007-05-09 02:33:52 -0700595
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700596/**
Oleg Nesterovf5a421a2007-07-15 23:41:44 -0700597 * cancel_delayed_work_sync - reliably kill off a delayed work.
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700598 * @dwork: the delayed work struct
599 *
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700600 * Returns true if @dwork was pending.
601 *
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700602 * It is possible to use this function if @dwork rearms itself via queue_work()
603 * or queue_delayed_work(). See also the comment for cancel_work_sync().
604 */
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700605int cancel_delayed_work_sync(struct delayed_work *dwork)
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700606{
Oleg Nesterov1f1f6422007-07-15 23:41:44 -0700607 return __cancel_work_timer(&dwork->work, &dwork->timer);
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700608}
Oleg Nesterovf5a421a2007-07-15 23:41:44 -0700609EXPORT_SYMBOL(cancel_delayed_work_sync);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700610
Oleg Nesterov6e84d642007-05-09 02:34:46 -0700611static struct workqueue_struct *keventd_wq __read_mostly;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700613/**
614 * schedule_work - put work task in global workqueue
615 * @work: job to be done
616 *
617 * This puts a job in the kernel-global workqueue.
618 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800619int schedule_work(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620{
621 return queue_work(keventd_wq, work);
622}
Dave Jonesae90dd52006-06-30 01:40:45 -0400623EXPORT_SYMBOL(schedule_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624
Zhang Ruic1a220e2008-07-23 21:28:39 -0700625/*
626 * schedule_work_on - put work task on a specific cpu
627 * @cpu: cpu to put the work task on
628 * @work: job to be done
629 *
630 * This puts a job on a specific cpu
631 */
632int schedule_work_on(int cpu, struct work_struct *work)
633{
634 return queue_work_on(cpu, keventd_wq, work);
635}
636EXPORT_SYMBOL(schedule_work_on);
637
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700638/**
639 * schedule_delayed_work - put work task in global workqueue after delay
David Howells52bad642006-11-22 14:54:01 +0000640 * @dwork: job to be done
641 * @delay: number of jiffies to wait or 0 for immediate execution
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700642 *
643 * After waiting for a given time this puts a job in the kernel-global
644 * workqueue.
645 */
Harvey Harrison7ad5b3a2008-02-08 04:19:53 -0800646int schedule_delayed_work(struct delayed_work *dwork,
Ingo Molnar82f67cd2007-02-16 01:28:13 -0800647 unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700648{
David Howells52bad642006-11-22 14:54:01 +0000649 return queue_delayed_work(keventd_wq, dwork, delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650}
Dave Jonesae90dd52006-06-30 01:40:45 -0400651EXPORT_SYMBOL(schedule_delayed_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700653/**
654 * schedule_delayed_work_on - queue work in global workqueue on CPU after delay
655 * @cpu: cpu to use
David Howells52bad642006-11-22 14:54:01 +0000656 * @dwork: job to be done
Rolf Eike Beer0fcb78c2006-07-30 03:03:42 -0700657 * @delay: number of jiffies to wait
658 *
659 * After waiting for a given time this puts a job in the kernel-global
660 * workqueue on the specified CPU.
661 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662int schedule_delayed_work_on(int cpu,
David Howells52bad642006-11-22 14:54:01 +0000663 struct delayed_work *dwork, unsigned long delay)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664{
David Howells52bad642006-11-22 14:54:01 +0000665 return queue_delayed_work_on(cpu, keventd_wq, dwork, delay);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666}
Dave Jonesae90dd52006-06-30 01:40:45 -0400667EXPORT_SYMBOL(schedule_delayed_work_on);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668
Andrew Mortonb6136772006-06-25 05:47:49 -0700669/**
670 * schedule_on_each_cpu - call a function on each online CPU from keventd
671 * @func: the function to call
Andrew Mortonb6136772006-06-25 05:47:49 -0700672 *
673 * Returns zero on success.
674 * Returns -ve errno on failure.
675 *
Andrew Mortonb6136772006-06-25 05:47:49 -0700676 * schedule_on_each_cpu() is very slow.
677 */
David Howells65f27f32006-11-22 14:55:48 +0000678int schedule_on_each_cpu(work_func_t func)
Christoph Lameter15316ba2006-01-08 01:00:43 -0800679{
680 int cpu;
Andrew Mortonb6136772006-06-25 05:47:49 -0700681 struct work_struct *works;
Christoph Lameter15316ba2006-01-08 01:00:43 -0800682
Andrew Mortonb6136772006-06-25 05:47:49 -0700683 works = alloc_percpu(struct work_struct);
684 if (!works)
Christoph Lameter15316ba2006-01-08 01:00:43 -0800685 return -ENOMEM;
Andrew Mortonb6136772006-06-25 05:47:49 -0700686
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100687 get_online_cpus();
Christoph Lameter15316ba2006-01-08 01:00:43 -0800688 for_each_online_cpu(cpu) {
Ingo Molnar9bfb1832006-12-18 20:05:09 +0100689 struct work_struct *work = per_cpu_ptr(works, cpu);
690
691 INIT_WORK(work, func);
Oleg Nesterov8de6d302008-07-25 01:47:53 -0700692 schedule_work_on(cpu, work);
Christoph Lameter15316ba2006-01-08 01:00:43 -0800693 }
Oleg Nesterov8616a892008-07-25 01:47:49 -0700694 for_each_online_cpu(cpu)
695 flush_work(per_cpu_ptr(works, cpu));
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100696 put_online_cpus();
Andrew Mortonb6136772006-06-25 05:47:49 -0700697 free_percpu(works);
Christoph Lameter15316ba2006-01-08 01:00:43 -0800698 return 0;
699}
700
Linus Torvalds1da177e2005-04-16 15:20:36 -0700701void flush_scheduled_work(void)
702{
703 flush_workqueue(keventd_wq);
704}
Dave Jonesae90dd52006-06-30 01:40:45 -0400705EXPORT_SYMBOL(flush_scheduled_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706
707/**
James Bottomley1fa44ec2006-02-23 12:43:43 -0600708 * execute_in_process_context - reliably execute the routine with user context
709 * @fn: the function to execute
James Bottomley1fa44ec2006-02-23 12:43:43 -0600710 * @ew: guaranteed storage for the execute work structure (must
711 * be available when the work executes)
712 *
713 * Executes the function immediately if process context is available,
714 * otherwise schedules the function for delayed execution.
715 *
716 * Returns: 0 - function was executed
717 * 1 - function was scheduled for execution
718 */
David Howells65f27f32006-11-22 14:55:48 +0000719int execute_in_process_context(work_func_t fn, struct execute_work *ew)
James Bottomley1fa44ec2006-02-23 12:43:43 -0600720{
721 if (!in_interrupt()) {
David Howells65f27f32006-11-22 14:55:48 +0000722 fn(&ew->work);
James Bottomley1fa44ec2006-02-23 12:43:43 -0600723 return 0;
724 }
725
David Howells65f27f32006-11-22 14:55:48 +0000726 INIT_WORK(&ew->work, fn);
James Bottomley1fa44ec2006-02-23 12:43:43 -0600727 schedule_work(&ew->work);
728
729 return 1;
730}
731EXPORT_SYMBOL_GPL(execute_in_process_context);
732
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733int keventd_up(void)
734{
735 return keventd_wq != NULL;
736}
737
738int current_is_keventd(void)
739{
740 struct cpu_workqueue_struct *cwq;
Hugh Dickinsd2437692007-08-27 16:06:19 +0100741 int cpu = raw_smp_processor_id(); /* preempt-safe: keventd is per-cpu */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 int ret = 0;
743
744 BUG_ON(!keventd_wq);
745
Christoph Lameter89ada672005-10-30 15:01:59 -0800746 cwq = per_cpu_ptr(keventd_wq->cpu_wq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747 if (current == cwq->thread)
748 ret = 1;
749
750 return ret;
751
752}
753
Oleg Nesterov3af244332007-05-09 02:34:09 -0700754static struct cpu_workqueue_struct *
755init_cpu_workqueue(struct workqueue_struct *wq, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756{
Christoph Lameter89ada672005-10-30 15:01:59 -0800757 struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758
Oleg Nesterov3af244332007-05-09 02:34:09 -0700759 cwq->wq = wq;
760 spin_lock_init(&cwq->lock);
761 INIT_LIST_HEAD(&cwq->worklist);
762 init_waitqueue_head(&cwq->more_work);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763
Oleg Nesterov3af244332007-05-09 02:34:09 -0700764 return cwq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765}
766
Oleg Nesterov3af244332007-05-09 02:34:09 -0700767static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768{
Oleg Nesterov3af244332007-05-09 02:34:09 -0700769 struct workqueue_struct *wq = cwq->wq;
770 const char *fmt = is_single_threaded(wq) ? "%s" : "%s/%d";
771 struct task_struct *p;
772
773 p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu);
774 /*
775 * Nobody can add the work_struct to this cwq,
776 * if (caller is __create_workqueue)
777 * nobody should see this wq
778 * else // caller is CPU_UP_PREPARE
779 * cpu is not on cpu_online_map
780 * so we can abort safely.
781 */
782 if (IS_ERR(p))
783 return PTR_ERR(p);
784
785 cwq->thread = p;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700786
787 return 0;
788}
789
Oleg Nesterov06ba38a2007-05-09 02:34:15 -0700790static void start_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
791{
792 struct task_struct *p = cwq->thread;
793
794 if (p != NULL) {
795 if (cpu >= 0)
796 kthread_bind(p, cpu);
797 wake_up_process(p);
798 }
799}
800
Johannes Berg4e6045f2007-10-18 23:39:55 -0700801struct workqueue_struct *__create_workqueue_key(const char *name,
802 int singlethread,
803 int freezeable,
Johannes Bergeb13ba82008-01-16 09:51:58 +0100804 struct lock_class_key *key,
805 const char *lock_name)
Oleg Nesterov3af244332007-05-09 02:34:09 -0700806{
807 struct workqueue_struct *wq;
808 struct cpu_workqueue_struct *cwq;
809 int err = 0, cpu;
810
811 wq = kzalloc(sizeof(*wq), GFP_KERNEL);
812 if (!wq)
813 return NULL;
814
815 wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct);
816 if (!wq->cpu_wq) {
817 kfree(wq);
818 return NULL;
819 }
820
821 wq->name = name;
Johannes Bergeb13ba82008-01-16 09:51:58 +0100822 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
Oleg Nesterovcce1a162007-05-09 02:34:13 -0700823 wq->singlethread = singlethread;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700824 wq->freezeable = freezeable;
Oleg Nesterovcce1a162007-05-09 02:34:13 -0700825 INIT_LIST_HEAD(&wq->list);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700826
827 if (singlethread) {
Oleg Nesterov3af244332007-05-09 02:34:09 -0700828 cwq = init_cpu_workqueue(wq, singlethread_cpu);
829 err = create_workqueue_thread(cwq, singlethread_cpu);
Oleg Nesterov06ba38a2007-05-09 02:34:15 -0700830 start_workqueue_thread(cwq, -1);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700831 } else {
Oleg Nesterov3da1c842008-07-25 01:47:50 -0700832 cpu_maps_update_begin();
Oleg Nesterov6af8bf32008-07-29 22:33:49 -0700833 /*
834 * We must place this wq on list even if the code below fails.
835 * cpu_down(cpu) can remove cpu from cpu_populated_map before
836 * destroy_workqueue() takes the lock, in that case we leak
837 * cwq[cpu]->thread.
838 */
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100839 spin_lock(&workqueue_lock);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700840 list_add(&wq->list, &workqueues);
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100841 spin_unlock(&workqueue_lock);
Oleg Nesterov6af8bf32008-07-29 22:33:49 -0700842 /*
843 * We must initialize cwqs for each possible cpu even if we
844 * are going to call destroy_workqueue() finally. Otherwise
845 * cpu_up() can hit the uninitialized cwq once we drop the
846 * lock.
847 */
Oleg Nesterov3af244332007-05-09 02:34:09 -0700848 for_each_possible_cpu(cpu) {
849 cwq = init_cpu_workqueue(wq, cpu);
850 if (err || !cpu_online(cpu))
851 continue;
852 err = create_workqueue_thread(cwq, cpu);
Oleg Nesterov06ba38a2007-05-09 02:34:15 -0700853 start_workqueue_thread(cwq, cpu);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700854 }
Oleg Nesterov3da1c842008-07-25 01:47:50 -0700855 cpu_maps_update_done();
Oleg Nesterov3af244332007-05-09 02:34:09 -0700856 }
857
858 if (err) {
859 destroy_workqueue(wq);
860 wq = NULL;
861 }
862 return wq;
863}
Johannes Berg4e6045f2007-10-18 23:39:55 -0700864EXPORT_SYMBOL_GPL(__create_workqueue_key);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700865
Oleg Nesterov1e35eaa2008-04-29 01:00:28 -0700866static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
Oleg Nesterov3af244332007-05-09 02:34:09 -0700867{
Oleg Nesterov14441962007-05-23 13:57:57 -0700868 /*
Oleg Nesterov3da1c842008-07-25 01:47:50 -0700869 * Our caller is either destroy_workqueue() or CPU_POST_DEAD,
870 * cpu_add_remove_lock protects cwq->thread.
Oleg Nesterov14441962007-05-23 13:57:57 -0700871 */
872 if (cwq->thread == NULL)
873 return;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700874
Ingo Molnar3295f0e2008-08-11 10:30:30 +0200875 lock_map_acquire(&cwq->wq->lockdep_map);
876 lock_map_release(&cwq->wq->lockdep_map);
Johannes Berg4e6045f2007-10-18 23:39:55 -0700877
Oleg Nesterov13c22162007-07-17 04:03:55 -0700878 flush_cpu_workqueue(cwq);
Oleg Nesterov14441962007-05-23 13:57:57 -0700879 /*
Oleg Nesterov3da1c842008-07-25 01:47:50 -0700880 * If the caller is CPU_POST_DEAD and cwq->worklist was not empty,
Oleg Nesterov13c22162007-07-17 04:03:55 -0700881 * a concurrent flush_workqueue() can insert a barrier after us.
882 * However, in that case run_workqueue() won't return and check
883 * kthread_should_stop() until it flushes all work_struct's.
Oleg Nesterov14441962007-05-23 13:57:57 -0700884 * When ->worklist becomes empty it is safe to exit because no
885 * more work_structs can be queued on this cwq: flush_workqueue
886 * checks list_empty(), and a "normal" queue_work() can't use
887 * a dead CPU.
888 */
Oleg Nesterov14441962007-05-23 13:57:57 -0700889 kthread_stop(cwq->thread);
890 cwq->thread = NULL;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700891}
892
893/**
894 * destroy_workqueue - safely terminate a workqueue
895 * @wq: target workqueue
896 *
897 * Safely destroy a workqueue. All work currently pending will be done first.
898 */
899void destroy_workqueue(struct workqueue_struct *wq)
900{
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700901 const cpumask_t *cpu_map = wq_cpu_map(wq);
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700902 int cpu;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700903
Oleg Nesterov3da1c842008-07-25 01:47:50 -0700904 cpu_maps_update_begin();
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100905 spin_lock(&workqueue_lock);
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700906 list_del(&wq->list);
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100907 spin_unlock(&workqueue_lock);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700908
Mike Travis363ab6f2008-05-12 21:21:13 +0200909 for_each_cpu_mask_nr(cpu, *cpu_map)
Oleg Nesterov1e35eaa2008-04-29 01:00:28 -0700910 cleanup_workqueue_thread(per_cpu_ptr(wq->cpu_wq, cpu));
Oleg Nesterov3da1c842008-07-25 01:47:50 -0700911 cpu_maps_update_done();
Oleg Nesterov3af244332007-05-09 02:34:09 -0700912
913 free_percpu(wq->cpu_wq);
914 kfree(wq);
915}
916EXPORT_SYMBOL_GPL(destroy_workqueue);
917
918static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
919 unsigned long action,
920 void *hcpu)
921{
922 unsigned int cpu = (unsigned long)hcpu;
923 struct cpu_workqueue_struct *cwq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924 struct workqueue_struct *wq;
Oleg Nesterov84485022008-07-25 01:47:54 -0700925 int ret = NOTIFY_OK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -0700927 action &= ~CPU_TASKS_FROZEN;
928
Linus Torvalds1da177e2005-04-16 15:20:36 -0700929 switch (action) {
930 case CPU_UP_PREPARE:
Oleg Nesterov3af244332007-05-09 02:34:09 -0700931 cpu_set(cpu, cpu_populated_map);
932 }
Oleg Nesterov84485022008-07-25 01:47:54 -0700933undo:
Oleg Nesterov3af244332007-05-09 02:34:09 -0700934 list_for_each_entry(wq, &workqueues, list) {
935 cwq = per_cpu_ptr(wq->cpu_wq, cpu);
Christoph Lameter89ada672005-10-30 15:01:59 -0800936
Oleg Nesterov3af244332007-05-09 02:34:09 -0700937 switch (action) {
938 case CPU_UP_PREPARE:
939 if (!create_workqueue_thread(cwq, cpu))
940 break;
Gautham R Shenoy95402b32008-01-25 21:08:02 +0100941 printk(KERN_ERR "workqueue [%s] for %i failed\n",
942 wq->name, cpu);
Oleg Nesterov84485022008-07-25 01:47:54 -0700943 action = CPU_UP_CANCELED;
944 ret = NOTIFY_BAD;
945 goto undo;
Oleg Nesterov3af244332007-05-09 02:34:09 -0700946
947 case CPU_ONLINE:
Oleg Nesterov06ba38a2007-05-09 02:34:15 -0700948 start_workqueue_thread(cwq, cpu);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700949 break;
950
951 case CPU_UP_CANCELED:
Oleg Nesterov06ba38a2007-05-09 02:34:15 -0700952 start_workqueue_thread(cwq, -1);
Oleg Nesterov3da1c842008-07-25 01:47:50 -0700953 case CPU_POST_DEAD:
Oleg Nesterov1e35eaa2008-04-29 01:00:28 -0700954 cleanup_workqueue_thread(cwq);
Oleg Nesterov3af244332007-05-09 02:34:09 -0700955 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700956 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957 }
958
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -0700959 switch (action) {
960 case CPU_UP_CANCELED:
Oleg Nesterov3da1c842008-07-25 01:47:50 -0700961 case CPU_POST_DEAD:
Oleg Nesterov00dfcaf2008-04-29 01:00:27 -0700962 cpu_clear(cpu, cpu_populated_map);
963 }
964
Oleg Nesterov84485022008-07-25 01:47:54 -0700965 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700966}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700967
Oleg Nesterovc12920d2007-05-09 02:34:14 -0700968void __init init_workqueues(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969{
Oleg Nesterov3af244332007-05-09 02:34:09 -0700970 cpu_populated_map = cpu_online_map;
Nathan Lynchf756d5e2006-01-08 01:05:12 -0800971 singlethread_cpu = first_cpu(cpu_possible_map);
Oleg Nesterovb1f4ec12007-05-09 02:34:12 -0700972 cpu_singlethread_map = cpumask_of_cpu(singlethread_cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 hotcpu_notifier(workqueue_cpu_callback, 0);
974 keventd_wq = create_workqueue("events");
975 BUG_ON(!keventd_wq);
976}