blob: 38729d3cd23692887afac015b442c052a150f38c [file] [log] [blame]
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -07001/*
2 * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright IBM Corporation, 2008
19 *
20 * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
21 *
22 * For detailed explanation of Read-Copy Update mechanism see -
Ingo Molnar4ce5b902009-10-26 07:55:55 +010023 * Documentation/RCU
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070024 */
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070025#include <linux/moduleparam.h>
Ingo Molnar4ce5b902009-10-26 07:55:55 +010026#include <linux/completion.h>
27#include <linux/interrupt.h>
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070028#include <linux/notifier.h>
Ingo Molnar4ce5b902009-10-26 07:55:55 +010029#include <linux/rcupdate.h>
30#include <linux/kernel.h>
31#include <linux/module.h>
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070032#include <linux/mutex.h>
Ingo Molnar4ce5b902009-10-26 07:55:55 +010033#include <linux/sched.h>
34#include <linux/types.h>
35#include <linux/init.h>
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070036#include <linux/time.h>
Ingo Molnar4ce5b902009-10-26 07:55:55 +010037#include <linux/cpu.h>
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070038
39/* Global control variables for rcupdate callback mechanism. */
40struct rcu_ctrlblk {
41 struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */
42 struct rcu_head **donetail; /* ->next pointer of last "done" CB. */
43 struct rcu_head **curtail; /* ->next pointer of last CB. */
44};
45
46/* Definition for rcupdate control block. */
Paul E. McKenney99652b52010-03-30 15:50:01 -070047static struct rcu_ctrlblk rcu_sched_ctrlblk = {
48 .donetail = &rcu_sched_ctrlblk.rcucblist,
49 .curtail = &rcu_sched_ctrlblk.rcucblist,
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070050};
Ingo Molnar4ce5b902009-10-26 07:55:55 +010051
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070052static struct rcu_ctrlblk rcu_bh_ctrlblk = {
Ingo Molnar4ce5b902009-10-26 07:55:55 +010053 .donetail = &rcu_bh_ctrlblk.rcucblist,
54 .curtail = &rcu_bh_ctrlblk.rcucblist,
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070055};
56
Paul E. McKenneybbad9372010-04-02 16:17:17 -070057#ifdef CONFIG_DEBUG_LOCK_ALLOC
58int rcu_scheduler_active __read_mostly;
59EXPORT_SYMBOL_GPL(rcu_scheduler_active);
60#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
61
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070062#ifdef CONFIG_NO_HZ
63
64static long rcu_dynticks_nesting = 1;
65
66/*
67 * Enter dynticks-idle mode, which is an extended quiescent state
68 * if we have fully entered that mode (i.e., if the new value of
69 * dynticks_nesting is zero).
70 */
71void rcu_enter_nohz(void)
72{
73 if (--rcu_dynticks_nesting == 0)
74 rcu_sched_qs(0); /* implies rcu_bh_qsctr_inc(0) */
75}
76
77/*
78 * Exit dynticks-idle mode, so that we are no longer in an extended
79 * quiescent state.
80 */
81void rcu_exit_nohz(void)
82{
83 rcu_dynticks_nesting++;
84}
85
86#endif /* #ifdef CONFIG_NO_HZ */
87
88/*
89 * Helper function for rcu_qsctr_inc() and rcu_bh_qsctr_inc().
Ingo Molnar4ce5b902009-10-26 07:55:55 +010090 * Also disable irqs to avoid confusion due to interrupt handlers
91 * invoking call_rcu().
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -070092 */
93static int rcu_qsctr_help(struct rcu_ctrlblk *rcp)
94{
95 unsigned long flags;
96
97 local_irq_save(flags);
98 if (rcp->rcucblist != NULL &&
99 rcp->donetail != rcp->curtail) {
100 rcp->donetail = rcp->curtail;
101 local_irq_restore(flags);
102 return 1;
103 }
104 local_irq_restore(flags);
Ingo Molnar4ce5b902009-10-26 07:55:55 +0100105
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700106 return 0;
107}
108
109/*
110 * Record an rcu quiescent state. And an rcu_bh quiescent state while we
111 * are at it, given that any rcu quiescent state is also an rcu_bh
112 * quiescent state. Use "+" instead of "||" to defeat short circuiting.
113 */
114void rcu_sched_qs(int cpu)
115{
Paul E. McKenney99652b52010-03-30 15:50:01 -0700116 if (rcu_qsctr_help(&rcu_sched_ctrlblk) +
117 rcu_qsctr_help(&rcu_bh_ctrlblk))
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700118 raise_softirq(RCU_SOFTIRQ);
119}
120
121/*
122 * Record an rcu_bh quiescent state.
123 */
124void rcu_bh_qs(int cpu)
125{
126 if (rcu_qsctr_help(&rcu_bh_ctrlblk))
127 raise_softirq(RCU_SOFTIRQ);
128}
129
130/*
131 * Check to see if the scheduling-clock interrupt came from an extended
132 * quiescent state, and, if so, tell RCU about it.
133 */
134void rcu_check_callbacks(int cpu, int user)
135{
136 if (user ||
137 (idle_cpu(cpu) &&
138 !in_softirq() &&
139 hardirq_count() <= (1 << HARDIRQ_SHIFT)))
140 rcu_sched_qs(cpu);
141 else if (!in_softirq())
142 rcu_bh_qs(cpu);
143}
144
145/*
146 * Helper function for rcu_process_callbacks() that operates on the
147 * specified rcu_ctrlkblk structure.
148 */
149static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp)
150{
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700151 struct rcu_head *next, *list;
Ingo Molnar4ce5b902009-10-26 07:55:55 +0100152 unsigned long flags;
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700153
154 /* If no RCU callbacks ready to invoke, just return. */
155 if (&rcp->rcucblist == rcp->donetail)
156 return;
157
158 /* Move the ready-to-invoke callbacks to a local list. */
159 local_irq_save(flags);
160 list = rcp->rcucblist;
161 rcp->rcucblist = *rcp->donetail;
162 *rcp->donetail = NULL;
163 if (rcp->curtail == rcp->donetail)
164 rcp->curtail = &rcp->rcucblist;
165 rcp->donetail = &rcp->rcucblist;
166 local_irq_restore(flags);
167
168 /* Invoke the callbacks on the local list. */
169 while (list) {
170 next = list->next;
171 prefetch(next);
172 list->func(list);
173 list = next;
174 }
175}
176
177/*
178 * Invoke any callbacks whose grace period has completed.
179 */
180static void rcu_process_callbacks(struct softirq_action *unused)
181{
Paul E. McKenney99652b52010-03-30 15:50:01 -0700182 __rcu_process_callbacks(&rcu_sched_ctrlblk);
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700183 __rcu_process_callbacks(&rcu_bh_ctrlblk);
184}
185
186/*
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700187 * Wait for a grace period to elapse. But it is illegal to invoke
188 * synchronize_sched() from within an RCU read-side critical section.
189 * Therefore, any legal call to synchronize_sched() is a quiescent
190 * state, and so on a UP system, synchronize_sched() need do nothing.
191 * Ditto for synchronize_rcu_bh(). (But Lai Jiangshan points out the
192 * benefits of doing might_sleep() to reduce latency.)
193 *
194 * Cool, huh? (Due to Josh Triplett.)
195 *
Paul E. McKenneyda848c42010-03-30 15:46:01 -0700196 * But we want to make this a static inline later. The cond_resched()
197 * currently makes this problematic.
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700198 */
199void synchronize_sched(void)
200{
201 cond_resched();
202}
203EXPORT_SYMBOL_GPL(synchronize_sched);
204
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700205/*
206 * Helper function for call_rcu() and call_rcu_bh().
207 */
208static void __call_rcu(struct rcu_head *head,
209 void (*func)(struct rcu_head *rcu),
210 struct rcu_ctrlblk *rcp)
211{
212 unsigned long flags;
213
214 head->func = func;
215 head->next = NULL;
Ingo Molnar4ce5b902009-10-26 07:55:55 +0100216
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700217 local_irq_save(flags);
218 *rcp->curtail = head;
219 rcp->curtail = &head->next;
220 local_irq_restore(flags);
221}
222
223/*
224 * Post an RCU callback to be invoked after the end of an RCU grace
225 * period. But since we have but one CPU, that would be after any
226 * quiescent state.
227 */
Ingo Molnar4ce5b902009-10-26 07:55:55 +0100228void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700229{
Paul E. McKenney99652b52010-03-30 15:50:01 -0700230 __call_rcu(head, func, &rcu_sched_ctrlblk);
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700231}
232EXPORT_SYMBOL_GPL(call_rcu);
233
234/*
235 * Post an RCU bottom-half callback to be invoked after any subsequent
236 * quiescent state.
237 */
Ingo Molnar4ce5b902009-10-26 07:55:55 +0100238void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700239{
240 __call_rcu(head, func, &rcu_bh_ctrlblk);
241}
242EXPORT_SYMBOL_GPL(call_rcu_bh);
243
244void rcu_barrier(void)
245{
246 struct rcu_synchronize rcu;
247
Paul E. McKenney72d5a9f2010-05-10 17:12:17 -0700248 init_rcu_head_on_stack(&rcu.head);
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700249 init_completion(&rcu.completion);
250 /* Will wake me after RCU finished. */
251 call_rcu(&rcu.head, wakeme_after_rcu);
252 /* Wait for it. */
253 wait_for_completion(&rcu.completion);
Paul E. McKenney72d5a9f2010-05-10 17:12:17 -0700254 destroy_rcu_head_on_stack(&rcu.head);
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700255}
256EXPORT_SYMBOL_GPL(rcu_barrier);
257
258void rcu_barrier_bh(void)
259{
260 struct rcu_synchronize rcu;
261
Paul E. McKenney72d5a9f2010-05-10 17:12:17 -0700262 init_rcu_head_on_stack(&rcu.head);
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700263 init_completion(&rcu.completion);
264 /* Will wake me after RCU finished. */
265 call_rcu_bh(&rcu.head, wakeme_after_rcu);
266 /* Wait for it. */
267 wait_for_completion(&rcu.completion);
Paul E. McKenney72d5a9f2010-05-10 17:12:17 -0700268 destroy_rcu_head_on_stack(&rcu.head);
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700269}
270EXPORT_SYMBOL_GPL(rcu_barrier_bh);
271
272void rcu_barrier_sched(void)
273{
274 struct rcu_synchronize rcu;
275
Paul E. McKenney72d5a9f2010-05-10 17:12:17 -0700276 init_rcu_head_on_stack(&rcu.head);
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700277 init_completion(&rcu.completion);
278 /* Will wake me after RCU finished. */
279 call_rcu_sched(&rcu.head, wakeme_after_rcu);
280 /* Wait for it. */
281 wait_for_completion(&rcu.completion);
Paul E. McKenney72d5a9f2010-05-10 17:12:17 -0700282 destroy_rcu_head_on_stack(&rcu.head);
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700283}
284EXPORT_SYMBOL_GPL(rcu_barrier_sched);
285
Paul E. McKenney9f680ab2009-11-22 08:53:49 -0800286void __init rcu_init(void)
Paul E. McKenney9b1d82f2009-10-25 19:03:50 -0700287{
288 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
289}
Paul E. McKenneybbad9372010-04-02 16:17:17 -0700290
291#include "rcutiny_plugin.h"