blob: 258cdf0a91eb78efcda53b91aaa845f7045066b7 [file] [log] [blame]
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001/*
Paul E. McKenney29766f12006-06-27 02:54:02 -07002 * Read-Copy Update module-based torture test facility
Paul E. McKenneya241ec62005-10-30 15:03:12 -08003 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
Josh Triplettb772e1d2006-10-04 02:17:13 -070018 * Copyright (C) IBM Corporation, 2005, 2006
Paul E. McKenneya241ec62005-10-30 15:03:12 -080019 *
20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
Paul E. McKenneya71fca52009-09-18 10:28:19 -070021 * Josh Triplett <josh@freedesktop.org>
Paul E. McKenneya241ec62005-10-30 15:03:12 -080022 *
23 * See also: Documentation/RCU/torture.txt
24 */
25#include <linux/types.h>
26#include <linux/kernel.h>
27#include <linux/init.h>
28#include <linux/module.h>
29#include <linux/kthread.h>
30#include <linux/err.h>
31#include <linux/spinlock.h>
32#include <linux/smp.h>
33#include <linux/rcupdate.h>
34#include <linux/interrupt.h>
35#include <linux/sched.h>
36#include <asm/atomic.h>
37#include <linux/bitops.h>
Paul E. McKenneya241ec62005-10-30 15:03:12 -080038#include <linux/completion.h>
39#include <linux/moduleparam.h>
40#include <linux/percpu.h>
41#include <linux/notifier.h>
Paul E. McKenney343e9092008-12-15 16:13:07 -080042#include <linux/reboot.h>
Rafael J. Wysocki83144182007-07-17 04:03:35 -070043#include <linux/freezer.h>
Paul E. McKenneya241ec62005-10-30 15:03:12 -080044#include <linux/cpu.h>
Paul E. McKenneya241ec62005-10-30 15:03:12 -080045#include <linux/delay.h>
Paul E. McKenneya241ec62005-10-30 15:03:12 -080046#include <linux/stat.h>
Paul E. McKenneyb2896d22006-10-04 02:17:03 -070047#include <linux/srcu.h>
Robert P. J. Day1aeb2722008-04-29 00:59:25 -070048#include <linux/slab.h>
Harvey Harrisonf07767f2008-10-20 10:23:38 -070049#include <asm/byteorder.h>
Paul E. McKenneya241ec62005-10-30 15:03:12 -080050
51MODULE_LICENSE("GPL");
Josh Triplettb772e1d2006-10-04 02:17:13 -070052MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "
Paul E. McKenneya71fca52009-09-18 10:28:19 -070053 "Josh Triplett <josh@freedesktop.org>");
Paul E. McKenneya241ec62005-10-30 15:03:12 -080054
Josh Triplett48022112006-10-03 23:26:16 +020055static int nreaders = -1; /* # reader threads, defaults to 2*ncpus */
Josh Triplettb772e1d2006-10-04 02:17:13 -070056static int nfakewriters = 4; /* # fake writer threads */
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -080057static int stat_interval; /* Interval between stats, in seconds. */
Paul E. McKenneya241ec62005-10-30 15:03:12 -080058 /* Defaults to "only at end of test". */
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -080059static int verbose; /* Print more debug info. */
60static int test_no_idle_hz; /* Test RCU's support for tickless idle CPUs. */
Paul E. McKenneyd120f652008-06-18 05:21:44 -070061static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
62static int stutter = 5; /* Start/stop testing interval (in sec) */
Paul E. McKenney0729fbf2008-06-25 12:24:52 -070063static int irqreader = 1; /* RCU readers from irq (timers). */
Paul E. McKenneybf66f182010-01-04 15:09:10 -080064static int fqs_duration = 0; /* Duration of bursts (us), 0 to disable. */
65static int fqs_holdoff = 0; /* Hold time within burst (us). */
66static int fqs_stutter = 3; /* Wait time between bursts (s). */
Josh Triplett20d2e422006-10-04 02:17:15 -070067static char *torture_type = "rcu"; /* What RCU implementation to torture. */
Paul E. McKenneya241ec62005-10-30 15:03:12 -080068
Josh Triplettd6ad6712007-03-06 01:42:13 -080069module_param(nreaders, int, 0444);
Paul E. McKenneya241ec62005-10-30 15:03:12 -080070MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
Josh Triplettd6ad6712007-03-06 01:42:13 -080071module_param(nfakewriters, int, 0444);
Josh Triplettb772e1d2006-10-04 02:17:13 -070072MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
Josh Triplettd6ad6712007-03-06 01:42:13 -080073module_param(stat_interval, int, 0444);
Paul E. McKenneya241ec62005-10-30 15:03:12 -080074MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
Josh Triplettd6ad6712007-03-06 01:42:13 -080075module_param(verbose, bool, 0444);
Paul E. McKenneya241ec62005-10-30 15:03:12 -080076MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
Josh Triplettd6ad6712007-03-06 01:42:13 -080077module_param(test_no_idle_hz, bool, 0444);
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -080078MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
Josh Triplettd6ad6712007-03-06 01:42:13 -080079module_param(shuffle_interval, int, 0444);
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -080080MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
Paul E. McKenneyd120f652008-06-18 05:21:44 -070081module_param(stutter, int, 0444);
82MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
Paul E. McKenney0729fbf2008-06-25 12:24:52 -070083module_param(irqreader, int, 0444);
84MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
Paul E. McKenneybf66f182010-01-04 15:09:10 -080085module_param(fqs_duration, int, 0444);
86MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)");
87module_param(fqs_holdoff, int, 0444);
88MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
89module_param(fqs_stutter, int, 0444);
90MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
Josh Triplettd6ad6712007-03-06 01:42:13 -080091module_param(torture_type, charp, 0444);
Paul E. McKenneyb2896d22006-10-04 02:17:03 -070092MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
Paul E. McKenney72e9bb52006-06-27 02:54:03 -070093
94#define TORTURE_FLAG "-torture:"
Paul E. McKenneya241ec62005-10-30 15:03:12 -080095#define PRINTK_STRING(s) \
Paul E. McKenney72e9bb52006-06-27 02:54:03 -070096 do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
Paul E. McKenneya241ec62005-10-30 15:03:12 -080097#define VERBOSE_PRINTK_STRING(s) \
Paul E. McKenney72e9bb52006-06-27 02:54:03 -070098 do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
Paul E. McKenneya241ec62005-10-30 15:03:12 -080099#define VERBOSE_PRINTK_ERRSTRING(s) \
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700100 do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800101
102static char printk_buf[4096];
103
104static int nrealreaders;
105static struct task_struct *writer_task;
Josh Triplettb772e1d2006-10-04 02:17:13 -0700106static struct task_struct **fakewriter_tasks;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800107static struct task_struct **reader_tasks;
108static struct task_struct *stats_task;
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -0800109static struct task_struct *shuffler_task;
Paul E. McKenneyd120f652008-06-18 05:21:44 -0700110static struct task_struct *stutter_task;
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800111static struct task_struct *fqs_task;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800112
113#define RCU_TORTURE_PIPE_LEN 10
114
115struct rcu_torture {
116 struct rcu_head rtort_rcu;
117 int rtort_pipe_count;
118 struct list_head rtort_free;
Paul E. McKenney996417d2005-11-18 01:10:50 -0800119 int rtort_mbtest;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800120};
121
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800122static LIST_HEAD(rcu_torture_freelist);
Paul E. McKenneya71fca52009-09-18 10:28:19 -0700123static struct rcu_torture *rcu_torture_current;
124static long rcu_torture_current_version;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800125static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
126static DEFINE_SPINLOCK(rcu_torture_lock);
127static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
128 { 0 };
129static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
130 { 0 };
131static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700132static atomic_t n_rcu_torture_alloc;
133static atomic_t n_rcu_torture_alloc_fail;
134static atomic_t n_rcu_torture_free;
135static atomic_t n_rcu_torture_mberror;
136static atomic_t n_rcu_torture_error;
Paul E. McKenneya71fca52009-09-18 10:28:19 -0700137static long n_rcu_torture_timers;
Josh Triplette3033732006-10-04 02:17:14 -0700138static struct list_head rcu_torture_removed;
Rusty Russell73d0a4b2009-03-30 22:05:16 -0600139static cpumask_var_t shuffle_tmp_mask;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800140
Paul E. McKenneya71fca52009-09-18 10:28:19 -0700141static int stutter_pause_test;
Paul E. McKenneyd120f652008-06-18 05:21:44 -0700142
Paul E. McKenney31a72bc2008-06-18 09:26:49 -0700143#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
144#define RCUTORTURE_RUNNABLE_INIT 1
145#else
146#define RCUTORTURE_RUNNABLE_INIT 0
147#endif
148int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
149
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800150/* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */
151
152#define FULLSTOP_DONTSTOP 0 /* Normal operation. */
153#define FULLSTOP_SHUTDOWN 1 /* System shutdown with rcutorture running. */
154#define FULLSTOP_RMMOD 2 /* Normal rmmod of rcutorture. */
155static int fullstop = FULLSTOP_RMMOD;
156DEFINE_MUTEX(fullstop_mutex); /* Protect fullstop transitions and spawning */
157 /* of kthreads. */
Paul E. McKenney343e9092008-12-15 16:13:07 -0800158
159/*
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800160 * Detect and respond to a system shutdown.
Paul E. McKenney343e9092008-12-15 16:13:07 -0800161 */
162static int
163rcutorture_shutdown_notify(struct notifier_block *unused1,
164 unsigned long unused2, void *unused3)
165{
Paul E. McKenneyc59ab972009-01-04 18:28:27 -0800166 mutex_lock(&fullstop_mutex);
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800167 if (fullstop == FULLSTOP_DONTSTOP)
Paul E. McKenneyc59ab972009-01-04 18:28:27 -0800168 fullstop = FULLSTOP_SHUTDOWN;
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800169 else
170 printk(KERN_WARNING /* but going down anyway, so... */
171 "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
Paul E. McKenneyc59ab972009-01-04 18:28:27 -0800172 mutex_unlock(&fullstop_mutex);
Paul E. McKenney343e9092008-12-15 16:13:07 -0800173 return NOTIFY_DONE;
174}
175
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800176/*
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800177 * Absorb kthreads into a kernel function that won't return, so that
178 * they won't ever access module text or data again.
179 */
180static void rcutorture_shutdown_absorb(char *title)
181{
182 if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
183 printk(KERN_NOTICE
184 "rcutorture thread %s parking due to system shutdown\n",
185 title);
186 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
187 }
188}
189
190/*
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800191 * Allocate an element from the rcu_tortures pool.
192 */
Adrian Bunk97a41e22006-01-08 01:02:17 -0800193static struct rcu_torture *
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800194rcu_torture_alloc(void)
195{
196 struct list_head *p;
197
Ingo Molnaradac1662006-01-25 19:50:12 +0100198 spin_lock_bh(&rcu_torture_lock);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800199 if (list_empty(&rcu_torture_freelist)) {
200 atomic_inc(&n_rcu_torture_alloc_fail);
Ingo Molnaradac1662006-01-25 19:50:12 +0100201 spin_unlock_bh(&rcu_torture_lock);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800202 return NULL;
203 }
204 atomic_inc(&n_rcu_torture_alloc);
205 p = rcu_torture_freelist.next;
206 list_del_init(p);
Ingo Molnaradac1662006-01-25 19:50:12 +0100207 spin_unlock_bh(&rcu_torture_lock);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800208 return container_of(p, struct rcu_torture, rtort_free);
209}
210
211/*
212 * Free an element to the rcu_tortures pool.
213 */
214static void
215rcu_torture_free(struct rcu_torture *p)
216{
217 atomic_inc(&n_rcu_torture_free);
Ingo Molnaradac1662006-01-25 19:50:12 +0100218 spin_lock_bh(&rcu_torture_lock);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800219 list_add_tail(&p->rtort_free, &rcu_torture_freelist);
Ingo Molnaradac1662006-01-25 19:50:12 +0100220 spin_unlock_bh(&rcu_torture_lock);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800221}
222
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800223struct rcu_random_state {
224 unsigned long rrs_state;
Josh Triplett75cfef32006-10-04 02:17:12 -0700225 long rrs_count;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800226};
227
228#define RCU_RANDOM_MULT 39916801 /* prime */
229#define RCU_RANDOM_ADD 479001701 /* prime */
230#define RCU_RANDOM_REFRESH 10000
231
232#define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
233
234/*
235 * Crude but fast random-number generator. Uses a linear congruential
Paul E. McKenneyc17ac852007-10-16 23:27:19 -0700236 * generator, with occasional help from cpu_clock().
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800237 */
Josh Triplett75cfef32006-10-04 02:17:12 -0700238static unsigned long
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800239rcu_random(struct rcu_random_state *rrsp)
240{
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800241 if (--rrsp->rrs_count < 0) {
Paul E. McKenneyc17ac852007-10-16 23:27:19 -0700242 rrsp->rrs_state +=
243 (unsigned long)cpu_clock(raw_smp_processor_id());
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800244 rrsp->rrs_count = RCU_RANDOM_REFRESH;
245 }
246 rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
247 return swahw32(rrsp->rrs_state);
248}
249
Paul E. McKenneyd120f652008-06-18 05:21:44 -0700250static void
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800251rcu_stutter_wait(char *title)
Paul E. McKenneyd120f652008-06-18 05:21:44 -0700252{
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800253 while (stutter_pause_test || !rcutorture_runnable) {
Paul E. McKenneye3d7be22008-06-22 13:06:38 -0700254 if (rcutorture_runnable)
255 schedule_timeout_interruptible(1);
256 else
Paul E. McKenney3ccf79f2008-06-22 14:02:55 -0700257 schedule_timeout_interruptible(round_jiffies_relative(HZ));
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800258 rcutorture_shutdown_absorb(title);
Paul E. McKenney343e9092008-12-15 16:13:07 -0800259 }
Paul E. McKenneyd120f652008-06-18 05:21:44 -0700260}
261
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800262/*
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700263 * Operations vector for selecting different types of tests.
264 */
265
266struct rcu_torture_ops {
267 void (*init)(void);
268 void (*cleanup)(void);
269 int (*readlock)(void);
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700270 void (*read_delay)(struct rcu_random_state *rrsp);
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700271 void (*readunlock)(int idx);
272 int (*completed)(void);
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700273 void (*deferred_free)(struct rcu_torture *p);
Josh Triplettb772e1d2006-10-04 02:17:13 -0700274 void (*sync)(void);
Paul E. McKenney23269742008-05-12 21:21:05 +0200275 void (*cb_barrier)(void);
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800276 void (*fqs)(void);
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700277 int (*stats)(char *page);
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700278 int irq_capable;
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700279 char *name;
280};
Paul E. McKenneya71fca52009-09-18 10:28:19 -0700281
282static struct rcu_torture_ops *cur_ops;
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700283
284/*
285 * Definitions for rcu torture testing.
286 */
287
Josh Tripletta49a4af2006-09-29 01:59:30 -0700288static int rcu_torture_read_lock(void) __acquires(RCU)
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700289{
290 rcu_read_lock();
291 return 0;
292}
293
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700294static void rcu_read_delay(struct rcu_random_state *rrsp)
295{
Josh Triplettb8d57a72009-09-08 15:54:35 -0700296 const unsigned long shortdelay_us = 200;
297 const unsigned long longdelay_ms = 50;
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700298
Josh Triplettb8d57a72009-09-08 15:54:35 -0700299 /* We want a short delay sometimes to make a reader delay the grace
300 * period, and we want a long delay occasionally to trigger
301 * force_quiescent_state. */
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700302
Josh Triplettb8d57a72009-09-08 15:54:35 -0700303 if (!(rcu_random(rrsp) % (nrealreaders * 2000 * longdelay_ms)))
304 mdelay(longdelay_ms);
305 if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
306 udelay(shortdelay_us);
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700307}
308
Josh Tripletta49a4af2006-09-29 01:59:30 -0700309static void rcu_torture_read_unlock(int idx) __releases(RCU)
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700310{
311 rcu_read_unlock();
312}
313
314static int rcu_torture_completed(void)
315{
316 return rcu_batches_completed();
317}
318
319static void
320rcu_torture_cb(struct rcu_head *p)
321{
322 int i;
323 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
324
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800325 if (fullstop != FULLSTOP_DONTSTOP) {
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700326 /* Test is ending, just drop callbacks on the floor. */
327 /* The next initialization will pick up the pieces. */
328 return;
329 }
330 i = rp->rtort_pipe_count;
331 if (i > RCU_TORTURE_PIPE_LEN)
332 i = RCU_TORTURE_PIPE_LEN;
333 atomic_inc(&rcu_torture_wcount[i]);
334 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
335 rp->rtort_mbtest = 0;
336 rcu_torture_free(rp);
337 } else
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700338 cur_ops->deferred_free(rp);
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700339}
340
Paul E. McKenneyd9a3da02009-12-02 12:10:15 -0800341static int rcu_no_completed(void)
342{
343 return 0;
344}
345
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700346static void rcu_torture_deferred_free(struct rcu_torture *p)
347{
348 call_rcu(&p->rtort_rcu, rcu_torture_cb);
349}
350
351static struct rcu_torture_ops rcu_ops = {
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700352 .init = NULL,
353 .cleanup = NULL,
354 .readlock = rcu_torture_read_lock,
355 .read_delay = rcu_read_delay,
356 .readunlock = rcu_torture_read_unlock,
357 .completed = rcu_torture_completed,
358 .deferred_free = rcu_torture_deferred_free,
359 .sync = synchronize_rcu,
360 .cb_barrier = rcu_barrier,
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800361 .fqs = rcu_force_quiescent_state,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700362 .stats = NULL,
Paul E. McKenneya71fca52009-09-18 10:28:19 -0700363 .irq_capable = 1,
364 .name = "rcu"
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700365};
366
Josh Triplette3033732006-10-04 02:17:14 -0700367static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
368{
369 int i;
370 struct rcu_torture *rp;
371 struct rcu_torture *rp1;
372
373 cur_ops->sync();
374 list_add(&p->rtort_free, &rcu_torture_removed);
375 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
376 i = rp->rtort_pipe_count;
377 if (i > RCU_TORTURE_PIPE_LEN)
378 i = RCU_TORTURE_PIPE_LEN;
379 atomic_inc(&rcu_torture_wcount[i]);
380 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
381 rp->rtort_mbtest = 0;
382 list_del(&rp->rtort_free);
383 rcu_torture_free(rp);
384 }
385 }
386}
387
388static void rcu_sync_torture_init(void)
389{
390 INIT_LIST_HEAD(&rcu_torture_removed);
391}
392
Josh Triplett20d2e422006-10-04 02:17:15 -0700393static struct rcu_torture_ops rcu_sync_ops = {
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700394 .init = rcu_sync_torture_init,
395 .cleanup = NULL,
396 .readlock = rcu_torture_read_lock,
397 .read_delay = rcu_read_delay,
398 .readunlock = rcu_torture_read_unlock,
399 .completed = rcu_torture_completed,
400 .deferred_free = rcu_sync_torture_deferred_free,
401 .sync = synchronize_rcu,
402 .cb_barrier = NULL,
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800403 .fqs = rcu_force_quiescent_state,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700404 .stats = NULL,
405 .irq_capable = 1,
406 .name = "rcu_sync"
Josh Triplett20d2e422006-10-04 02:17:15 -0700407};
408
Paul E. McKenneyd9a3da02009-12-02 12:10:15 -0800409static struct rcu_torture_ops rcu_expedited_ops = {
410 .init = rcu_sync_torture_init,
411 .cleanup = NULL,
412 .readlock = rcu_torture_read_lock,
413 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
414 .readunlock = rcu_torture_read_unlock,
415 .completed = rcu_no_completed,
416 .deferred_free = rcu_sync_torture_deferred_free,
417 .sync = synchronize_rcu_expedited,
418 .cb_barrier = NULL,
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800419 .fqs = rcu_force_quiescent_state,
Paul E. McKenneyd9a3da02009-12-02 12:10:15 -0800420 .stats = NULL,
421 .irq_capable = 1,
422 .name = "rcu_expedited"
423};
424
Paul E. McKenneyc32e0662006-06-27 02:54:04 -0700425/*
426 * Definitions for rcu_bh torture testing.
427 */
428
Josh Tripletta49a4af2006-09-29 01:59:30 -0700429static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
Paul E. McKenneyc32e0662006-06-27 02:54:04 -0700430{
431 rcu_read_lock_bh();
432 return 0;
433}
434
Josh Tripletta49a4af2006-09-29 01:59:30 -0700435static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
Paul E. McKenneyc32e0662006-06-27 02:54:04 -0700436{
437 rcu_read_unlock_bh();
438}
439
440static int rcu_bh_torture_completed(void)
441{
442 return rcu_batches_completed_bh();
443}
444
445static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
446{
447 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
448}
449
Josh Triplettb772e1d2006-10-04 02:17:13 -0700450struct rcu_bh_torture_synchronize {
451 struct rcu_head head;
452 struct completion completion;
453};
454
455static void rcu_bh_torture_wakeme_after_cb(struct rcu_head *head)
456{
457 struct rcu_bh_torture_synchronize *rcu;
458
459 rcu = container_of(head, struct rcu_bh_torture_synchronize, head);
460 complete(&rcu->completion);
461}
462
463static void rcu_bh_torture_synchronize(void)
464{
465 struct rcu_bh_torture_synchronize rcu;
466
467 init_completion(&rcu.completion);
468 call_rcu_bh(&rcu.head, rcu_bh_torture_wakeme_after_cb);
469 wait_for_completion(&rcu.completion);
470}
471
Paul E. McKenneyc32e0662006-06-27 02:54:04 -0700472static struct rcu_torture_ops rcu_bh_ops = {
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700473 .init = NULL,
474 .cleanup = NULL,
475 .readlock = rcu_bh_torture_read_lock,
476 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
477 .readunlock = rcu_bh_torture_read_unlock,
478 .completed = rcu_bh_torture_completed,
479 .deferred_free = rcu_bh_torture_deferred_free,
480 .sync = rcu_bh_torture_synchronize,
481 .cb_barrier = rcu_barrier_bh,
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800482 .fqs = rcu_bh_force_quiescent_state,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700483 .stats = NULL,
484 .irq_capable = 1,
485 .name = "rcu_bh"
Paul E. McKenneyc32e0662006-06-27 02:54:04 -0700486};
487
Josh Triplett11a14702006-10-04 02:17:16 -0700488static struct rcu_torture_ops rcu_bh_sync_ops = {
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700489 .init = rcu_sync_torture_init,
490 .cleanup = NULL,
491 .readlock = rcu_bh_torture_read_lock,
492 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
493 .readunlock = rcu_bh_torture_read_unlock,
494 .completed = rcu_bh_torture_completed,
495 .deferred_free = rcu_sync_torture_deferred_free,
496 .sync = rcu_bh_torture_synchronize,
497 .cb_barrier = NULL,
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800498 .fqs = rcu_bh_force_quiescent_state,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700499 .stats = NULL,
500 .irq_capable = 1,
501 .name = "rcu_bh_sync"
Josh Triplett11a14702006-10-04 02:17:16 -0700502};
503
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700504/*
505 * Definitions for srcu torture testing.
506 */
507
508static struct srcu_struct srcu_ctl;
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700509
510static void srcu_torture_init(void)
511{
512 init_srcu_struct(&srcu_ctl);
Josh Triplette3033732006-10-04 02:17:14 -0700513 rcu_sync_torture_init();
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700514}
515
516static void srcu_torture_cleanup(void)
517{
518 synchronize_srcu(&srcu_ctl);
519 cleanup_srcu_struct(&srcu_ctl);
520}
521
Josh Triplett012d3ca2006-12-06 20:40:19 -0800522static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700523{
524 return srcu_read_lock(&srcu_ctl);
525}
526
527static void srcu_read_delay(struct rcu_random_state *rrsp)
528{
529 long delay;
530 const long uspertick = 1000000 / HZ;
531 const long longdelay = 10;
532
533 /* We want there to be long-running readers, but not all the time. */
534
535 delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
536 if (!delay)
537 schedule_timeout_interruptible(longdelay);
538}
539
Josh Triplett012d3ca2006-12-06 20:40:19 -0800540static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700541{
542 srcu_read_unlock(&srcu_ctl, idx);
543}
544
545static int srcu_torture_completed(void)
546{
547 return srcu_batches_completed(&srcu_ctl);
548}
549
Josh Triplettb772e1d2006-10-04 02:17:13 -0700550static void srcu_torture_synchronize(void)
551{
552 synchronize_srcu(&srcu_ctl);
553}
554
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700555static int srcu_torture_stats(char *page)
556{
557 int cnt = 0;
558 int cpu;
559 int idx = srcu_ctl.completed & 0x1;
560
561 cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
562 torture_type, TORTURE_FLAG, idx);
563 for_each_possible_cpu(cpu) {
564 cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,
565 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
566 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
567 }
568 cnt += sprintf(&page[cnt], "\n");
569 return cnt;
570}
571
572static struct rcu_torture_ops srcu_ops = {
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700573 .init = srcu_torture_init,
574 .cleanup = srcu_torture_cleanup,
575 .readlock = srcu_torture_read_lock,
576 .read_delay = srcu_read_delay,
577 .readunlock = srcu_torture_read_unlock,
578 .completed = srcu_torture_completed,
579 .deferred_free = rcu_sync_torture_deferred_free,
580 .sync = srcu_torture_synchronize,
581 .cb_barrier = NULL,
582 .stats = srcu_torture_stats,
583 .name = "srcu"
Paul E. McKenneyb2896d22006-10-04 02:17:03 -0700584};
585
Paul E. McKenney804bb832009-10-25 19:03:52 -0700586static void srcu_torture_synchronize_expedited(void)
587{
588 synchronize_srcu_expedited(&srcu_ctl);
589}
590
591static struct rcu_torture_ops srcu_expedited_ops = {
592 .init = srcu_torture_init,
593 .cleanup = srcu_torture_cleanup,
594 .readlock = srcu_torture_read_lock,
595 .read_delay = srcu_read_delay,
596 .readunlock = srcu_torture_read_unlock,
597 .completed = srcu_torture_completed,
598 .deferred_free = rcu_sync_torture_deferred_free,
599 .sync = srcu_torture_synchronize_expedited,
600 .cb_barrier = NULL,
601 .stats = srcu_torture_stats,
602 .name = "srcu_expedited"
603};
604
Josh Triplett4b6c2cc2006-10-04 02:17:16 -0700605/*
606 * Definitions for sched torture testing.
607 */
608
609static int sched_torture_read_lock(void)
610{
611 preempt_disable();
612 return 0;
613}
614
615static void sched_torture_read_unlock(int idx)
616{
617 preempt_enable();
618}
619
Paul E. McKenney23269742008-05-12 21:21:05 +0200620static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
621{
622 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
623}
624
Josh Triplett4b6c2cc2006-10-04 02:17:16 -0700625static void sched_torture_synchronize(void)
626{
627 synchronize_sched();
628}
629
630static struct rcu_torture_ops sched_ops = {
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700631 .init = rcu_sync_torture_init,
632 .cleanup = NULL,
633 .readlock = sched_torture_read_lock,
634 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
635 .readunlock = sched_torture_read_unlock,
Paul E. McKenneyd9a3da02009-12-02 12:10:15 -0800636 .completed = rcu_no_completed,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700637 .deferred_free = rcu_sched_torture_deferred_free,
638 .sync = sched_torture_synchronize,
639 .cb_barrier = rcu_barrier_sched,
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800640 .fqs = rcu_sched_force_quiescent_state,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700641 .stats = NULL,
642 .irq_capable = 1,
643 .name = "sched"
Josh Triplett4b6c2cc2006-10-04 02:17:16 -0700644};
645
Paul E. McKenney804bb832009-10-25 19:03:52 -0700646static struct rcu_torture_ops sched_sync_ops = {
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700647 .init = rcu_sync_torture_init,
648 .cleanup = NULL,
649 .readlock = sched_torture_read_lock,
650 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
651 .readunlock = sched_torture_read_unlock,
Paul E. McKenneyd9a3da02009-12-02 12:10:15 -0800652 .completed = rcu_no_completed,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700653 .deferred_free = rcu_sync_torture_deferred_free,
654 .sync = sched_torture_synchronize,
655 .cb_barrier = NULL,
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800656 .fqs = rcu_sched_force_quiescent_state,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700657 .stats = NULL,
658 .name = "sched_sync"
659};
660
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700661static struct rcu_torture_ops sched_expedited_ops = {
662 .init = rcu_sync_torture_init,
663 .cleanup = NULL,
664 .readlock = sched_torture_read_lock,
665 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
666 .readunlock = sched_torture_read_unlock,
Paul E. McKenneyd9a3da02009-12-02 12:10:15 -0800667 .completed = rcu_no_completed,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700668 .deferred_free = rcu_sync_torture_deferred_free,
669 .sync = synchronize_sched_expedited,
670 .cb_barrier = NULL,
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800671 .fqs = rcu_sched_force_quiescent_state,
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700672 .stats = rcu_expedited_torture_stats,
673 .irq_capable = 1,
674 .name = "sched_expedited"
Paul E. McKenney23269742008-05-12 21:21:05 +0200675};
676
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700677/*
Paul E. McKenneybf66f182010-01-04 15:09:10 -0800678 * RCU torture force-quiescent-state kthread. Repeatedly induces
679 * bursts of calls to force_quiescent_state(), increasing the probability
680 * of occurrence of some important types of race conditions.
681 */
682static int
683rcu_torture_fqs(void *arg)
684{
685 unsigned long fqs_resume_time;
686 int fqs_burst_remaining;
687
688 VERBOSE_PRINTK_STRING("rcu_torture_fqs task started");
689 do {
690 fqs_resume_time = jiffies + fqs_stutter * HZ;
691 while (jiffies - fqs_resume_time > LONG_MAX) {
692 schedule_timeout_interruptible(1);
693 }
694 fqs_burst_remaining = fqs_duration;
695 while (fqs_burst_remaining > 0) {
696 cur_ops->fqs();
697 udelay(fqs_holdoff);
698 fqs_burst_remaining -= fqs_holdoff;
699 }
700 rcu_stutter_wait("rcu_torture_fqs");
701 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
702 VERBOSE_PRINTK_STRING("rcu_torture_fqs task stopping");
703 rcutorture_shutdown_absorb("rcu_torture_fqs");
704 while (!kthread_should_stop())
705 schedule_timeout_uninterruptible(1);
706 return 0;
707}
708
709/*
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800710 * RCU torture writer kthread. Repeatedly substitutes a new structure
711 * for that pointed to by rcu_torture_current, freeing the old structure
712 * after a series of grace periods (the "pipeline").
713 */
714static int
715rcu_torture_writer(void *arg)
716{
717 int i;
718 long oldbatch = rcu_batches_completed();
719 struct rcu_torture *rp;
720 struct rcu_torture *old_rp;
721 static DEFINE_RCU_RANDOM(rand);
722
723 VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
Ingo Molnardbdf65b2005-11-13 16:07:22 -0800724 set_user_nice(current, 19);
725
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800726 do {
727 schedule_timeout_uninterruptible(1);
Paul E. McKenneya71fca52009-09-18 10:28:19 -0700728 rp = rcu_torture_alloc();
729 if (rp == NULL)
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800730 continue;
731 rp->rtort_pipe_count = 0;
732 udelay(rcu_random(&rand) & 0x3ff);
733 old_rp = rcu_torture_current;
Paul E. McKenney996417d2005-11-18 01:10:50 -0800734 rp->rtort_mbtest = 1;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800735 rcu_assign_pointer(rcu_torture_current, rp);
Paul E. McKenney9b2619a2009-09-23 09:50:43 -0700736 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
Josh Triplettc8e5b162007-05-08 00:33:20 -0700737 if (old_rp) {
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800738 i = old_rp->rtort_pipe_count;
739 if (i > RCU_TORTURE_PIPE_LEN)
740 i = RCU_TORTURE_PIPE_LEN;
741 atomic_inc(&rcu_torture_wcount[i]);
742 old_rp->rtort_pipe_count++;
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700743 cur_ops->deferred_free(old_rp);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800744 }
745 rcu_torture_current_version++;
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700746 oldbatch = cur_ops->completed();
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800747 rcu_stutter_wait("rcu_torture_writer");
748 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800749 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800750 rcutorture_shutdown_absorb("rcu_torture_writer");
751 while (!kthread_should_stop())
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800752 schedule_timeout_uninterruptible(1);
753 return 0;
754}
755
756/*
Josh Triplettb772e1d2006-10-04 02:17:13 -0700757 * RCU torture fake writer kthread. Repeatedly calls sync, with a random
758 * delay between calls.
759 */
760static int
761rcu_torture_fakewriter(void *arg)
762{
763 DEFINE_RCU_RANDOM(rand);
764
765 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started");
766 set_user_nice(current, 19);
767
768 do {
769 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
770 udelay(rcu_random(&rand) & 0x3ff);
771 cur_ops->sync();
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800772 rcu_stutter_wait("rcu_torture_fakewriter");
773 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
Josh Triplettb772e1d2006-10-04 02:17:13 -0700774
775 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800776 rcutorture_shutdown_absorb("rcu_torture_fakewriter");
777 while (!kthread_should_stop())
Josh Triplettb772e1d2006-10-04 02:17:13 -0700778 schedule_timeout_uninterruptible(1);
779 return 0;
780}
781
782/*
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700783 * RCU torture reader from timer handler. Dereferences rcu_torture_current,
784 * incrementing the corresponding element of the pipeline array. The
785 * counter in the element should never be greater than 1, otherwise, the
786 * RCU implementation is broken.
787 */
788static void rcu_torture_timer(unsigned long unused)
789{
790 int idx;
791 int completed;
792 static DEFINE_RCU_RANDOM(rand);
793 static DEFINE_SPINLOCK(rand_lock);
794 struct rcu_torture *p;
795 int pipe_count;
796
797 idx = cur_ops->readlock();
798 completed = cur_ops->completed();
Paul E. McKenney632ee202010-02-22 17:04:45 -0800799 p = rcu_dereference_check(rcu_torture_current,
800 rcu_read_lock_held() ||
801 rcu_read_lock_bh_held() ||
802 rcu_read_lock_sched_held() ||
803 srcu_read_lock_held(&srcu_ctl));
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700804 if (p == NULL) {
805 /* Leave because rcu_torture_writer is not yet underway */
806 cur_ops->readunlock(idx);
807 return;
808 }
809 if (p->rtort_mbtest == 0)
810 atomic_inc(&n_rcu_torture_mberror);
811 spin_lock(&rand_lock);
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700812 cur_ops->read_delay(&rand);
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700813 n_rcu_torture_timers++;
814 spin_unlock(&rand_lock);
815 preempt_disable();
816 pipe_count = p->rtort_pipe_count;
817 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
818 /* Should not happen, but... */
819 pipe_count = RCU_TORTURE_PIPE_LEN;
820 }
Christoph Lametere8008792009-10-03 19:48:23 +0900821 __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]);
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700822 completed = cur_ops->completed() - completed;
823 if (completed > RCU_TORTURE_PIPE_LEN) {
824 /* Should not happen, but... */
825 completed = RCU_TORTURE_PIPE_LEN;
826 }
Christoph Lametere8008792009-10-03 19:48:23 +0900827 __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]);
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700828 preempt_enable();
829 cur_ops->readunlock(idx);
830}
831
832/*
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800833 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
834 * incrementing the corresponding element of the pipeline array. The
835 * counter in the element should never be greater than 1, otherwise, the
836 * RCU implementation is broken.
837 */
838static int
839rcu_torture_reader(void *arg)
840{
841 int completed;
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700842 int idx;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800843 DEFINE_RCU_RANDOM(rand);
844 struct rcu_torture *p;
845 int pipe_count;
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700846 struct timer_list t;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800847
848 VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
Ingo Molnardbdf65b2005-11-13 16:07:22 -0800849 set_user_nice(current, 19);
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700850 if (irqreader && cur_ops->irq_capable)
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700851 setup_timer_on_stack(&t, rcu_torture_timer, 0);
Ingo Molnardbdf65b2005-11-13 16:07:22 -0800852
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800853 do {
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700854 if (irqreader && cur_ops->irq_capable) {
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700855 if (!timer_pending(&t))
Paul E. McKenney6155fec2010-02-22 17:05:04 -0800856 mod_timer(&t, jiffies + 1);
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700857 }
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700858 idx = cur_ops->readlock();
859 completed = cur_ops->completed();
Paul E. McKenney632ee202010-02-22 17:04:45 -0800860 p = rcu_dereference_check(rcu_torture_current,
861 rcu_read_lock_held() ||
862 rcu_read_lock_bh_held() ||
863 rcu_read_lock_sched_held() ||
864 srcu_read_lock_held(&srcu_ctl));
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800865 if (p == NULL) {
866 /* Wait for rcu_torture_writer to get underway */
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700867 cur_ops->readunlock(idx);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800868 schedule_timeout_interruptible(HZ);
869 continue;
870 }
Paul E. McKenney996417d2005-11-18 01:10:50 -0800871 if (p->rtort_mbtest == 0)
872 atomic_inc(&n_rcu_torture_mberror);
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700873 cur_ops->read_delay(&rand);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800874 preempt_disable();
875 pipe_count = p->rtort_pipe_count;
876 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
877 /* Should not happen, but... */
878 pipe_count = RCU_TORTURE_PIPE_LEN;
879 }
Christoph Lametere8008792009-10-03 19:48:23 +0900880 __this_cpu_inc(per_cpu_var(rcu_torture_count)[pipe_count]);
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700881 completed = cur_ops->completed() - completed;
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800882 if (completed > RCU_TORTURE_PIPE_LEN) {
883 /* Should not happen, but... */
884 completed = RCU_TORTURE_PIPE_LEN;
885 }
Christoph Lametere8008792009-10-03 19:48:23 +0900886 __this_cpu_inc(per_cpu_var(rcu_torture_batch)[completed]);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800887 preempt_enable();
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700888 cur_ops->readunlock(idx);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800889 schedule();
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800890 rcu_stutter_wait("rcu_torture_reader");
891 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800892 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800893 rcutorture_shutdown_absorb("rcu_torture_reader");
Paul E. McKenney0acc5122009-06-25 09:08:17 -0700894 if (irqreader && cur_ops->irq_capable)
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700895 del_timer_sync(&t);
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800896 while (!kthread_should_stop())
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800897 schedule_timeout_uninterruptible(1);
898 return 0;
899}
900
901/*
902 * Create an RCU-torture statistics message in the specified buffer.
903 */
904static int
905rcu_torture_printk(char *page)
906{
907 int cnt = 0;
908 int cpu;
909 int i;
910 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
911 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
912
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -0800913 for_each_possible_cpu(cpu) {
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800914 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
915 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
916 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
917 }
918 }
919 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
920 if (pipesummary[i] != 0)
921 break;
922 }
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700923 cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800924 cnt += sprintf(&page[cnt],
Paul E. McKenney996417d2005-11-18 01:10:50 -0800925 "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700926 "rtmbe: %d nt: %ld",
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800927 rcu_torture_current,
928 rcu_torture_current_version,
929 list_empty(&rcu_torture_freelist),
930 atomic_read(&n_rcu_torture_alloc),
931 atomic_read(&n_rcu_torture_alloc_fail),
Paul E. McKenney996417d2005-11-18 01:10:50 -0800932 atomic_read(&n_rcu_torture_free),
Paul E. McKenney0729fbf2008-06-25 12:24:52 -0700933 atomic_read(&n_rcu_torture_mberror),
934 n_rcu_torture_timers);
Paul E. McKenney996417d2005-11-18 01:10:50 -0800935 if (atomic_read(&n_rcu_torture_mberror) != 0)
936 cnt += sprintf(&page[cnt], " !!!");
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700937 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
Paul E. McKenney996417d2005-11-18 01:10:50 -0800938 if (i > 1) {
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800939 cnt += sprintf(&page[cnt], "!!! ");
Paul E. McKenney996417d2005-11-18 01:10:50 -0800940 atomic_inc(&n_rcu_torture_error);
Ingo Molnar5af970a2008-06-18 10:09:48 +0200941 WARN_ON_ONCE(1);
Paul E. McKenney996417d2005-11-18 01:10:50 -0800942 }
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800943 cnt += sprintf(&page[cnt], "Reader Pipe: ");
944 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
945 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700946 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800947 cnt += sprintf(&page[cnt], "Reader Batch: ");
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700948 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800949 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700950 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800951 cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
952 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
953 cnt += sprintf(&page[cnt], " %d",
954 atomic_read(&rcu_torture_wcount[i]));
955 }
956 cnt += sprintf(&page[cnt], "\n");
Josh Triplettc8e5b162007-05-08 00:33:20 -0700957 if (cur_ops->stats)
Paul E. McKenney72e9bb52006-06-27 02:54:03 -0700958 cnt += cur_ops->stats(&page[cnt]);
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800959 return cnt;
960}
961
962/*
963 * Print torture statistics. Caller must ensure that there is only
964 * one call to this function at a given time!!! This is normally
965 * accomplished by relying on the module system to only have one copy
966 * of the module loaded, and then by giving the rcu_torture_stats
967 * kthread full control (or the init/cleanup functions when rcu_torture_stats
968 * thread is not running).
969 */
970static void
971rcu_torture_stats_print(void)
972{
973 int cnt;
974
975 cnt = rcu_torture_printk(printk_buf);
976 printk(KERN_ALERT "%s", printk_buf);
977}
978
979/*
980 * Periodically prints torture statistics, if periodic statistics printing
981 * was specified via the stat_interval module parameter.
982 *
983 * No need to worry about fullstop here, since this one doesn't reference
984 * volatile state or register callbacks.
985 */
986static int
987rcu_torture_stats(void *arg)
988{
989 VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
990 do {
991 schedule_timeout_interruptible(stat_interval * HZ);
992 rcu_torture_stats_print();
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -0800993 rcutorture_shutdown_absorb("rcu_torture_stats");
994 } while (!kthread_should_stop());
Paul E. McKenneya241ec62005-10-30 15:03:12 -0800995 VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
996 return 0;
997}
998
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -0800999static int rcu_idle_cpu; /* Force all torture tasks off this CPU */
1000
1001/* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
1002 * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
1003 */
Paul E. McKenneyb2896d22006-10-04 02:17:03 -07001004static void rcu_torture_shuffle_tasks(void)
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001005{
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001006 int i;
1007
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001008 cpumask_setall(shuffle_tmp_mask);
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +01001009 get_online_cpus();
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001010
1011 /* No point in shuffling if there is only one online CPU (ex: UP) */
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -08001012 if (num_online_cpus() == 1) {
1013 put_online_cpus();
1014 return;
1015 }
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001016
1017 if (rcu_idle_cpu != -1)
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001018 cpumask_clear_cpu(rcu_idle_cpu, shuffle_tmp_mask);
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001019
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001020 set_cpus_allowed_ptr(current, shuffle_tmp_mask);
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001021
Josh Triplettc8e5b162007-05-08 00:33:20 -07001022 if (reader_tasks) {
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001023 for (i = 0; i < nrealreaders; i++)
1024 if (reader_tasks[i])
Mike Travisf70316d2008-04-04 18:11:06 -07001025 set_cpus_allowed_ptr(reader_tasks[i],
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001026 shuffle_tmp_mask);
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001027 }
1028
Josh Triplettc8e5b162007-05-08 00:33:20 -07001029 if (fakewriter_tasks) {
Josh Triplettb772e1d2006-10-04 02:17:13 -07001030 for (i = 0; i < nfakewriters; i++)
1031 if (fakewriter_tasks[i])
Mike Travisf70316d2008-04-04 18:11:06 -07001032 set_cpus_allowed_ptr(fakewriter_tasks[i],
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001033 shuffle_tmp_mask);
Josh Triplettb772e1d2006-10-04 02:17:13 -07001034 }
1035
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001036 if (writer_task)
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001037 set_cpus_allowed_ptr(writer_task, shuffle_tmp_mask);
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001038
1039 if (stats_task)
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001040 set_cpus_allowed_ptr(stats_task, shuffle_tmp_mask);
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001041
1042 if (rcu_idle_cpu == -1)
1043 rcu_idle_cpu = num_online_cpus() - 1;
1044 else
1045 rcu_idle_cpu--;
1046
Gautham R Shenoy86ef5c92008-01-25 21:08:02 +01001047 put_online_cpus();
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001048}
1049
1050/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
1051 * system to become idle at a time and cut off its timer ticks. This is meant
1052 * to test the support for such tickless idle CPU in RCU.
1053 */
1054static int
1055rcu_torture_shuffle(void *arg)
1056{
1057 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
1058 do {
1059 schedule_timeout_interruptible(shuffle_interval * HZ);
1060 rcu_torture_shuffle_tasks();
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -08001061 rcutorture_shutdown_absorb("rcu_torture_shuffle");
1062 } while (!kthread_should_stop());
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001063 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
1064 return 0;
1065}
1066
Paul E. McKenneyd120f652008-06-18 05:21:44 -07001067/* Cause the rcutorture test to "stutter", starting and stopping all
1068 * threads periodically.
1069 */
1070static int
1071rcu_torture_stutter(void *arg)
1072{
1073 VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
1074 do {
1075 schedule_timeout_interruptible(stutter * HZ);
1076 stutter_pause_test = 1;
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -08001077 if (!kthread_should_stop())
Paul E. McKenneyd120f652008-06-18 05:21:44 -07001078 schedule_timeout_interruptible(stutter * HZ);
1079 stutter_pause_test = 0;
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -08001080 rcutorture_shutdown_absorb("rcu_torture_stutter");
1081 } while (!kthread_should_stop());
Paul E. McKenneyd120f652008-06-18 05:21:44 -07001082 VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
1083 return 0;
1084}
1085
Paul E. McKenney95c38322006-03-24 03:15:58 -08001086static inline void
1087rcu_torture_print_module_parms(char *tag)
1088{
Josh Triplettb772e1d2006-10-04 02:17:13 -07001089 printk(KERN_ALERT "%s" TORTURE_FLAG
1090 "--- %s: nreaders=%d nfakewriters=%d "
Paul E. McKenney95c38322006-03-24 03:15:58 -08001091 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
Paul E. McKenneybf66f182010-01-04 15:09:10 -08001092 "shuffle_interval=%d stutter=%d irqreader=%d "
1093 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d\n",
Josh Triplettb772e1d2006-10-04 02:17:13 -07001094 torture_type, tag, nrealreaders, nfakewriters,
Paul E. McKenneyd120f652008-06-18 05:21:44 -07001095 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
Paul E. McKenneybf66f182010-01-04 15:09:10 -08001096 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter);
Paul E. McKenney95c38322006-03-24 03:15:58 -08001097}
1098
Paul E. McKenney343e9092008-12-15 16:13:07 -08001099static struct notifier_block rcutorture_nb = {
1100 .notifier_call = rcutorture_shutdown_notify,
1101};
1102
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001103static void
1104rcu_torture_cleanup(void)
1105{
1106 int i;
1107
Paul E. McKenney343e9092008-12-15 16:13:07 -08001108 mutex_lock(&fullstop_mutex);
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -08001109 if (fullstop == FULLSTOP_SHUTDOWN) {
1110 printk(KERN_WARNING /* but going down anyway, so... */
1111 "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
Paul E. McKenney343e9092008-12-15 16:13:07 -08001112 mutex_unlock(&fullstop_mutex);
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -08001113 schedule_timeout_uninterruptible(10);
Paul E. McKenney343e9092008-12-15 16:13:07 -08001114 if (cur_ops->cb_barrier != NULL)
1115 cur_ops->cb_barrier();
1116 return;
1117 }
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -08001118 fullstop = FULLSTOP_RMMOD;
Paul E. McKenney343e9092008-12-15 16:13:07 -08001119 mutex_unlock(&fullstop_mutex);
1120 unregister_reboot_notifier(&rcutorture_nb);
Paul E. McKenneyd120f652008-06-18 05:21:44 -07001121 if (stutter_task) {
1122 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
1123 kthread_stop(stutter_task);
1124 }
1125 stutter_task = NULL;
Josh Triplettc8e5b162007-05-08 00:33:20 -07001126 if (shuffler_task) {
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001127 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
1128 kthread_stop(shuffler_task);
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001129 free_cpumask_var(shuffle_tmp_mask);
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001130 }
1131 shuffler_task = NULL;
1132
Josh Triplettc8e5b162007-05-08 00:33:20 -07001133 if (writer_task) {
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001134 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
1135 kthread_stop(writer_task);
1136 }
1137 writer_task = NULL;
1138
Josh Triplettc8e5b162007-05-08 00:33:20 -07001139 if (reader_tasks) {
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001140 for (i = 0; i < nrealreaders; i++) {
Josh Triplettc8e5b162007-05-08 00:33:20 -07001141 if (reader_tasks[i]) {
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001142 VERBOSE_PRINTK_STRING(
1143 "Stopping rcu_torture_reader task");
1144 kthread_stop(reader_tasks[i]);
1145 }
1146 reader_tasks[i] = NULL;
1147 }
1148 kfree(reader_tasks);
1149 reader_tasks = NULL;
1150 }
1151 rcu_torture_current = NULL;
1152
Josh Triplettc8e5b162007-05-08 00:33:20 -07001153 if (fakewriter_tasks) {
Josh Triplettb772e1d2006-10-04 02:17:13 -07001154 for (i = 0; i < nfakewriters; i++) {
Josh Triplettc8e5b162007-05-08 00:33:20 -07001155 if (fakewriter_tasks[i]) {
Josh Triplettb772e1d2006-10-04 02:17:13 -07001156 VERBOSE_PRINTK_STRING(
1157 "Stopping rcu_torture_fakewriter task");
1158 kthread_stop(fakewriter_tasks[i]);
1159 }
1160 fakewriter_tasks[i] = NULL;
1161 }
1162 kfree(fakewriter_tasks);
1163 fakewriter_tasks = NULL;
1164 }
1165
Josh Triplettc8e5b162007-05-08 00:33:20 -07001166 if (stats_task) {
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001167 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
1168 kthread_stop(stats_task);
1169 }
1170 stats_task = NULL;
1171
Paul E. McKenneybf66f182010-01-04 15:09:10 -08001172 if (fqs_task) {
1173 VERBOSE_PRINTK_STRING("Stopping rcu_torture_fqs task");
1174 kthread_stop(fqs_task);
1175 }
1176 fqs_task = NULL;
1177
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001178 /* Wait for all RCU callbacks to fire. */
Paul E. McKenney23269742008-05-12 21:21:05 +02001179
1180 if (cur_ops->cb_barrier != NULL)
1181 cur_ops->cb_barrier();
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001182
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001183 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
Paul E. McKenney72e9bb52006-06-27 02:54:03 -07001184
Josh Triplettc8e5b162007-05-08 00:33:20 -07001185 if (cur_ops->cleanup)
Paul E. McKenney72e9bb52006-06-27 02:54:03 -07001186 cur_ops->cleanup();
Paul E. McKenney95c38322006-03-24 03:15:58 -08001187 if (atomic_read(&n_rcu_torture_error))
1188 rcu_torture_print_module_parms("End of test: FAILURE");
1189 else
1190 rcu_torture_print_module_parms("End of test: SUCCESS");
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001191}
1192
Josh Triplett6f8bc502007-05-08 00:25:24 -07001193static int __init
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001194rcu_torture_init(void)
1195{
1196 int i;
1197 int cpu;
1198 int firsterr = 0;
Josh Triplettade5fb82007-05-08 00:33:22 -07001199 static struct rcu_torture_ops *torture_ops[] =
Paul E. McKenneyd9a3da02009-12-02 12:10:15 -08001200 { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
1201 &rcu_bh_ops, &rcu_bh_sync_ops,
Paul E. McKenney804bb832009-10-25 19:03:52 -07001202 &srcu_ops, &srcu_expedited_ops,
1203 &sched_ops, &sched_sync_ops, &sched_expedited_ops, };
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001204
Paul E. McKenney343e9092008-12-15 16:13:07 -08001205 mutex_lock(&fullstop_mutex);
1206
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001207 /* Process args and tell the world that the torturer is on the job. */
Josh Triplettade5fb82007-05-08 00:33:22 -07001208 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
Paul E. McKenney72e9bb52006-06-27 02:54:03 -07001209 cur_ops = torture_ops[i];
Josh Triplettade5fb82007-05-08 00:33:22 -07001210 if (strcmp(torture_type, cur_ops->name) == 0)
Paul E. McKenney72e9bb52006-06-27 02:54:03 -07001211 break;
Paul E. McKenney72e9bb52006-06-27 02:54:03 -07001212 }
Josh Triplettade5fb82007-05-08 00:33:22 -07001213 if (i == ARRAY_SIZE(torture_ops)) {
Paul E. McKenneycf886c42009-10-25 19:03:54 -07001214 printk(KERN_ALERT "rcu-torture: invalid torture type: \"%s\"\n",
Paul E. McKenney72e9bb52006-06-27 02:54:03 -07001215 torture_type);
Paul E. McKenneycf886c42009-10-25 19:03:54 -07001216 printk(KERN_ALERT "rcu-torture types:");
1217 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1218 printk(KERN_ALERT " %s", torture_ops[i]->name);
1219 printk(KERN_ALERT "\n");
Paul E. McKenney343e9092008-12-15 16:13:07 -08001220 mutex_unlock(&fullstop_mutex);
Paul E. McKenneya71fca52009-09-18 10:28:19 -07001221 return -EINVAL;
Paul E. McKenney72e9bb52006-06-27 02:54:03 -07001222 }
Paul E. McKenneybf66f182010-01-04 15:09:10 -08001223 if (cur_ops->fqs == NULL && fqs_duration != 0) {
1224 printk(KERN_ALERT "rcu-torture: ->fqs NULL and non-zero "
1225 "fqs_duration, fqs disabled.\n");
1226 fqs_duration = 0;
1227 }
Josh Triplettc8e5b162007-05-08 00:33:20 -07001228 if (cur_ops->init)
Paul E. McKenney72e9bb52006-06-27 02:54:03 -07001229 cur_ops->init(); /* no "goto unwind" prior to this point!!! */
1230
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001231 if (nreaders >= 0)
1232 nrealreaders = nreaders;
1233 else
1234 nrealreaders = 2 * num_online_cpus();
Paul E. McKenney95c38322006-03-24 03:15:58 -08001235 rcu_torture_print_module_parms("Start of test");
Paul E. McKenneyc9d557c2009-01-07 14:33:30 -08001236 fullstop = FULLSTOP_DONTSTOP;
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001237
1238 /* Set up the freelist. */
1239
1240 INIT_LIST_HEAD(&rcu_torture_freelist);
Ahmed S. Darwish788e7702007-05-08 00:33:14 -07001241 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
Paul E. McKenney996417d2005-11-18 01:10:50 -08001242 rcu_tortures[i].rtort_mbtest = 0;
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001243 list_add_tail(&rcu_tortures[i].rtort_free,
1244 &rcu_torture_freelist);
1245 }
1246
1247 /* Initialize the statistics so that each run gets its own numbers. */
1248
1249 rcu_torture_current = NULL;
1250 rcu_torture_current_version = 0;
1251 atomic_set(&n_rcu_torture_alloc, 0);
1252 atomic_set(&n_rcu_torture_alloc_fail, 0);
1253 atomic_set(&n_rcu_torture_free, 0);
Paul E. McKenney996417d2005-11-18 01:10:50 -08001254 atomic_set(&n_rcu_torture_mberror, 0);
1255 atomic_set(&n_rcu_torture_error, 0);
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001256 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1257 atomic_set(&rcu_torture_wcount[i], 0);
KAMEZAWA Hiroyuki0a945022006-03-28 01:56:37 -08001258 for_each_possible_cpu(cpu) {
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001259 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1260 per_cpu(rcu_torture_count, cpu)[i] = 0;
1261 per_cpu(rcu_torture_batch, cpu)[i] = 0;
1262 }
1263 }
1264
1265 /* Start up the kthreads. */
1266
1267 VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
1268 writer_task = kthread_run(rcu_torture_writer, NULL,
1269 "rcu_torture_writer");
1270 if (IS_ERR(writer_task)) {
1271 firsterr = PTR_ERR(writer_task);
1272 VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
1273 writer_task = NULL;
1274 goto unwind;
1275 }
Josh Triplettb772e1d2006-10-04 02:17:13 -07001276 fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
Paul E. McKenneya71fca52009-09-18 10:28:19 -07001277 GFP_KERNEL);
Josh Triplettb772e1d2006-10-04 02:17:13 -07001278 if (fakewriter_tasks == NULL) {
1279 VERBOSE_PRINTK_ERRSTRING("out of memory");
1280 firsterr = -ENOMEM;
1281 goto unwind;
1282 }
1283 for (i = 0; i < nfakewriters; i++) {
1284 VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
1285 fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
Paul E. McKenneya71fca52009-09-18 10:28:19 -07001286 "rcu_torture_fakewriter");
Josh Triplettb772e1d2006-10-04 02:17:13 -07001287 if (IS_ERR(fakewriter_tasks[i])) {
1288 firsterr = PTR_ERR(fakewriter_tasks[i]);
1289 VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");
1290 fakewriter_tasks[i] = NULL;
1291 goto unwind;
1292 }
1293 }
Josh Triplett2860aab2006-10-04 02:17:11 -07001294 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001295 GFP_KERNEL);
1296 if (reader_tasks == NULL) {
1297 VERBOSE_PRINTK_ERRSTRING("out of memory");
1298 firsterr = -ENOMEM;
1299 goto unwind;
1300 }
1301 for (i = 0; i < nrealreaders; i++) {
1302 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
1303 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
1304 "rcu_torture_reader");
1305 if (IS_ERR(reader_tasks[i])) {
1306 firsterr = PTR_ERR(reader_tasks[i]);
1307 VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
1308 reader_tasks[i] = NULL;
1309 goto unwind;
1310 }
1311 }
1312 if (stat_interval > 0) {
1313 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
1314 stats_task = kthread_run(rcu_torture_stats, NULL,
1315 "rcu_torture_stats");
1316 if (IS_ERR(stats_task)) {
1317 firsterr = PTR_ERR(stats_task);
1318 VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
1319 stats_task = NULL;
1320 goto unwind;
1321 }
1322 }
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001323 if (test_no_idle_hz) {
1324 rcu_idle_cpu = num_online_cpus() - 1;
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001325
1326 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
1327 firsterr = -ENOMEM;
1328 VERBOSE_PRINTK_ERRSTRING("Failed to alloc mask");
1329 goto unwind;
1330 }
1331
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001332 /* Create the shuffler thread */
1333 shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
1334 "rcu_torture_shuffle");
1335 if (IS_ERR(shuffler_task)) {
Rusty Russell73d0a4b2009-03-30 22:05:16 -06001336 free_cpumask_var(shuffle_tmp_mask);
Srivatsa Vaddagirid84f5202006-01-08 01:03:42 -08001337 firsterr = PTR_ERR(shuffler_task);
1338 VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
1339 shuffler_task = NULL;
1340 goto unwind;
1341 }
1342 }
Paul E. McKenneyd120f652008-06-18 05:21:44 -07001343 if (stutter < 0)
1344 stutter = 0;
1345 if (stutter) {
1346 /* Create the stutter thread */
1347 stutter_task = kthread_run(rcu_torture_stutter, NULL,
1348 "rcu_torture_stutter");
1349 if (IS_ERR(stutter_task)) {
1350 firsterr = PTR_ERR(stutter_task);
1351 VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
1352 stutter_task = NULL;
1353 goto unwind;
1354 }
1355 }
Paul E. McKenneybf66f182010-01-04 15:09:10 -08001356 if (fqs_duration < 0)
1357 fqs_duration = 0;
1358 if (fqs_duration) {
1359 /* Create the stutter thread */
1360 fqs_task = kthread_run(rcu_torture_fqs, NULL,
1361 "rcu_torture_fqs");
1362 if (IS_ERR(fqs_task)) {
1363 firsterr = PTR_ERR(fqs_task);
1364 VERBOSE_PRINTK_ERRSTRING("Failed to create fqs");
1365 fqs_task = NULL;
1366 goto unwind;
1367 }
1368 }
Paul E. McKenney343e9092008-12-15 16:13:07 -08001369 register_reboot_notifier(&rcutorture_nb);
1370 mutex_unlock(&fullstop_mutex);
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001371 return 0;
1372
1373unwind:
Paul E. McKenney343e9092008-12-15 16:13:07 -08001374 mutex_unlock(&fullstop_mutex);
Paul E. McKenneya241ec62005-10-30 15:03:12 -08001375 rcu_torture_cleanup();
1376 return firsterr;
1377}
1378
1379module_init(rcu_torture_init);
1380module_exit(rcu_torture_cleanup);