blob: 88d0d4420ad2e3e47e71129d361b153c96a49e18 [file] [log] [blame]
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07001/*
2 * kernel/lockdep.c
3 *
4 * Runtime locking correctness validator
5 *
6 * Started by Ingo Molnar:
7 *
Peter Zijlstra4b32d0a2007-07-19 01:48:59 -07008 * Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
9 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
Ingo Molnarfbb9ce952006-07-03 00:24:50 -070010 *
11 * this code maps all the lock dependencies as they occur in a live kernel
12 * and will warn about the following classes of locking bugs:
13 *
14 * - lock inversion scenarios
15 * - circular lock dependencies
16 * - hardirq/softirq safe/unsafe locking bugs
17 *
18 * Bugs are reported even if the current locking scenario does not cause
19 * any deadlock at this point.
20 *
21 * I.e. if anytime in the past two locks were taken in a different order,
22 * even if it happened for another task, even if those were different
23 * locks (but of the same class as this lock), this code will detect it.
24 *
25 * Thanks to Arjan van de Ven for coming up with the initial idea of
26 * mapping lock dependencies runtime.
27 */
Steven Rostedta5e25882008-12-02 15:34:05 -050028#define DISABLE_BRANCH_PROFILING
Ingo Molnarfbb9ce952006-07-03 00:24:50 -070029#include <linux/mutex.h>
30#include <linux/sched.h>
31#include <linux/delay.h>
32#include <linux/module.h>
33#include <linux/proc_fs.h>
34#include <linux/seq_file.h>
35#include <linux/spinlock.h>
36#include <linux/kallsyms.h>
37#include <linux/interrupt.h>
38#include <linux/stacktrace.h>
39#include <linux/debug_locks.h>
40#include <linux/irqflags.h>
Dave Jones99de0552006-09-29 02:00:10 -070041#include <linux/utsname.h>
Peter Zijlstra4b32d0a2007-07-19 01:48:59 -070042#include <linux/hash.h>
Steven Rostedt81d68a92008-05-12 21:20:42 +020043#include <linux/ftrace.h>
Peter Zijlstrab4b136f2009-01-29 14:50:36 +010044#include <linux/stringify.h>
Ming Leid588e462009-07-16 15:44:29 +020045#include <linux/bitops.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090046#include <linux/gfp.h>
Yong Zhangd3d03d42011-11-09 16:04:51 +080047#include <linux/kmemcheck.h>
Peter Zijlstraaf012962009-07-16 15:44:29 +020048
Ingo Molnarfbb9ce952006-07-03 00:24:50 -070049#include <asm/sections.h>
50
51#include "lockdep_internals.h"
52
Steven Rostedta8d154b2009-04-10 09:36:00 -040053#define CREATE_TRACE_POINTS
Frederic Weisbecker67178762009-11-13 10:06:34 +010054#include <trace/events/lock.h>
Steven Rostedta8d154b2009-04-10 09:36:00 -040055
Peter Zijlstraf20786f2007-07-19 01:48:56 -070056#ifdef CONFIG_PROVE_LOCKING
57int prove_locking = 1;
58module_param(prove_locking, int, 0644);
59#else
60#define prove_locking 0
61#endif
62
63#ifdef CONFIG_LOCK_STAT
64int lock_stat = 1;
65module_param(lock_stat, int, 0644);
66#else
67#define lock_stat 0
68#endif
69
Ingo Molnarfbb9ce952006-07-03 00:24:50 -070070/*
Ingo Molnar74c383f2006-12-13 00:34:43 -080071 * lockdep_lock: protects the lockdep graph, the hashes and the
72 * class/list/hash allocators.
Ingo Molnarfbb9ce952006-07-03 00:24:50 -070073 *
74 * This is one of the rare exceptions where it's justified
75 * to use a raw spinlock - we really dont want the spinlock
Ingo Molnar74c383f2006-12-13 00:34:43 -080076 * code to recurse back into the lockdep code...
Ingo Molnarfbb9ce952006-07-03 00:24:50 -070077 */
Thomas Gleixneredc35bd2009-12-03 12:38:57 +010078static arch_spinlock_t lockdep_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Ingo Molnar74c383f2006-12-13 00:34:43 -080079
80static int graph_lock(void)
81{
Thomas Gleixner0199c4e2009-12-02 20:01:25 +010082 arch_spin_lock(&lockdep_lock);
Ingo Molnar74c383f2006-12-13 00:34:43 -080083 /*
84 * Make sure that if another CPU detected a bug while
85 * walking the graph we dont change it (while the other
86 * CPU is busy printing out stuff with the graph lock
87 * dropped already)
88 */
89 if (!debug_locks) {
Thomas Gleixner0199c4e2009-12-02 20:01:25 +010090 arch_spin_unlock(&lockdep_lock);
Ingo Molnar74c383f2006-12-13 00:34:43 -080091 return 0;
92 }
Steven Rostedtbb065af2008-05-12 21:21:00 +020093 /* prevent any recursions within lockdep from causing deadlocks */
94 current->lockdep_recursion++;
Ingo Molnar74c383f2006-12-13 00:34:43 -080095 return 1;
96}
97
98static inline int graph_unlock(void)
99{
Peter Zijlstra0119fee2011-09-02 01:30:29 +0200100 if (debug_locks && !arch_spin_is_locked(&lockdep_lock)) {
101 /*
102 * The lockdep graph lock isn't locked while we expect it to
103 * be, we're confused now, bye!
104 */
Jarek Poplawski381a2292007-02-10 01:44:58 -0800105 return DEBUG_LOCKS_WARN_ON(1);
Peter Zijlstra0119fee2011-09-02 01:30:29 +0200106 }
Jarek Poplawski381a2292007-02-10 01:44:58 -0800107
Steven Rostedtbb065af2008-05-12 21:21:00 +0200108 current->lockdep_recursion--;
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100109 arch_spin_unlock(&lockdep_lock);
Ingo Molnar74c383f2006-12-13 00:34:43 -0800110 return 0;
111}
112
113/*
114 * Turn lock debugging off and return with 0 if it was off already,
115 * and also release the graph lock:
116 */
117static inline int debug_locks_off_graph_unlock(void)
118{
119 int ret = debug_locks_off();
120
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100121 arch_spin_unlock(&lockdep_lock);
Ingo Molnar74c383f2006-12-13 00:34:43 -0800122
123 return ret;
124}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700125
126static int lockdep_initialized;
127
128unsigned long nr_list_entries;
Peter Zijlstraaf012962009-07-16 15:44:29 +0200129static struct lock_list list_entries[MAX_LOCKDEP_ENTRIES];
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700130
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700131/*
132 * All data structures here are protected by the global debug_lock.
133 *
134 * Mutex key structs only get allocated, once during bootup, and never
135 * get freed - this significantly simplifies the debugging code.
136 */
137unsigned long nr_lock_classes;
138static struct lock_class lock_classes[MAX_LOCKDEP_KEYS];
139
Dave Jonesf82b2172008-08-11 09:30:23 +0200140static inline struct lock_class *hlock_class(struct held_lock *hlock)
141{
142 if (!hlock->class_idx) {
Peter Zijlstra0119fee2011-09-02 01:30:29 +0200143 /*
144 * Someone passed in garbage, we give up.
145 */
Dave Jonesf82b2172008-08-11 09:30:23 +0200146 DEBUG_LOCKS_WARN_ON(1);
147 return NULL;
148 }
149 return lock_classes + hlock->class_idx - 1;
150}
151
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700152#ifdef CONFIG_LOCK_STAT
Tejun Heo1871e522009-10-29 22:34:13 +0900153static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS],
154 cpu_lock_stats);
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700155
Peter Zijlstra3365e7792009-10-09 10:12:41 +0200156static inline u64 lockstat_clock(void)
157{
Peter Zijlstrac6763292010-05-25 10:48:51 +0200158 return local_clock();
Peter Zijlstra3365e7792009-10-09 10:12:41 +0200159}
160
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +0200161static int lock_point(unsigned long points[], unsigned long ip)
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700162{
163 int i;
164
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +0200165 for (i = 0; i < LOCKSTAT_POINTS; i++) {
166 if (points[i] == 0) {
167 points[i] = ip;
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700168 break;
169 }
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +0200170 if (points[i] == ip)
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700171 break;
172 }
173
174 return i;
175}
176
Peter Zijlstra3365e7792009-10-09 10:12:41 +0200177static void lock_time_inc(struct lock_time *lt, u64 time)
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700178{
179 if (time > lt->max)
180 lt->max = time;
181
Frank Rowand109d71c2009-11-19 13:42:06 -0800182 if (time < lt->min || !lt->nr)
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700183 lt->min = time;
184
185 lt->total += time;
186 lt->nr++;
187}
188
Peter Zijlstrac46261d2007-07-19 01:48:57 -0700189static inline void lock_time_add(struct lock_time *src, struct lock_time *dst)
190{
Frank Rowand109d71c2009-11-19 13:42:06 -0800191 if (!src->nr)
192 return;
193
194 if (src->max > dst->max)
195 dst->max = src->max;
196
197 if (src->min < dst->min || !dst->nr)
198 dst->min = src->min;
199
Peter Zijlstrac46261d2007-07-19 01:48:57 -0700200 dst->total += src->total;
201 dst->nr += src->nr;
202}
203
204struct lock_class_stats lock_stats(struct lock_class *class)
205{
206 struct lock_class_stats stats;
207 int cpu, i;
208
209 memset(&stats, 0, sizeof(struct lock_class_stats));
210 for_each_possible_cpu(cpu) {
211 struct lock_class_stats *pcs =
Tejun Heo1871e522009-10-29 22:34:13 +0900212 &per_cpu(cpu_lock_stats, cpu)[class - lock_classes];
Peter Zijlstrac46261d2007-07-19 01:48:57 -0700213
214 for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)
215 stats.contention_point[i] += pcs->contention_point[i];
216
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +0200217 for (i = 0; i < ARRAY_SIZE(stats.contending_point); i++)
218 stats.contending_point[i] += pcs->contending_point[i];
219
Peter Zijlstrac46261d2007-07-19 01:48:57 -0700220 lock_time_add(&pcs->read_waittime, &stats.read_waittime);
221 lock_time_add(&pcs->write_waittime, &stats.write_waittime);
222
223 lock_time_add(&pcs->read_holdtime, &stats.read_holdtime);
224 lock_time_add(&pcs->write_holdtime, &stats.write_holdtime);
Peter Zijlstra96645672007-07-19 01:49:00 -0700225
226 for (i = 0; i < ARRAY_SIZE(stats.bounces); i++)
227 stats.bounces[i] += pcs->bounces[i];
Peter Zijlstrac46261d2007-07-19 01:48:57 -0700228 }
229
230 return stats;
231}
232
233void clear_lock_stats(struct lock_class *class)
234{
235 int cpu;
236
237 for_each_possible_cpu(cpu) {
238 struct lock_class_stats *cpu_stats =
Tejun Heo1871e522009-10-29 22:34:13 +0900239 &per_cpu(cpu_lock_stats, cpu)[class - lock_classes];
Peter Zijlstrac46261d2007-07-19 01:48:57 -0700240
241 memset(cpu_stats, 0, sizeof(struct lock_class_stats));
242 }
243 memset(class->contention_point, 0, sizeof(class->contention_point));
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +0200244 memset(class->contending_point, 0, sizeof(class->contending_point));
Peter Zijlstrac46261d2007-07-19 01:48:57 -0700245}
246
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700247static struct lock_class_stats *get_lock_stats(struct lock_class *class)
248{
Tejun Heo1871e522009-10-29 22:34:13 +0900249 return &get_cpu_var(cpu_lock_stats)[class - lock_classes];
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700250}
251
252static void put_lock_stats(struct lock_class_stats *stats)
253{
Tejun Heo1871e522009-10-29 22:34:13 +0900254 put_cpu_var(cpu_lock_stats);
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700255}
256
257static void lock_release_holdtime(struct held_lock *hlock)
258{
259 struct lock_class_stats *stats;
Peter Zijlstra3365e7792009-10-09 10:12:41 +0200260 u64 holdtime;
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700261
262 if (!lock_stat)
263 return;
264
Peter Zijlstra3365e7792009-10-09 10:12:41 +0200265 holdtime = lockstat_clock() - hlock->holdtime_stamp;
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700266
Dave Jonesf82b2172008-08-11 09:30:23 +0200267 stats = get_lock_stats(hlock_class(hlock));
Peter Zijlstraf20786f2007-07-19 01:48:56 -0700268 if (hlock->read)
269 lock_time_inc(&stats->read_holdtime, holdtime);
270 else
271 lock_time_inc(&stats->write_holdtime, holdtime);
272 put_lock_stats(stats);
273}
274#else
275static inline void lock_release_holdtime(struct held_lock *hlock)
276{
277}
278#endif
279
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700280/*
281 * We keep a global list of all lock classes. The list only grows,
282 * never shrinks. The list is only accessed with the lockdep
283 * spinlock lock held.
284 */
285LIST_HEAD(all_lock_classes);
286
287/*
288 * The lockdep classes are in a hash-table as well, for fast lookup:
289 */
290#define CLASSHASH_BITS (MAX_LOCKDEP_KEYS_BITS - 1)
291#define CLASSHASH_SIZE (1UL << CLASSHASH_BITS)
Peter Zijlstra4b32d0a2007-07-19 01:48:59 -0700292#define __classhashfn(key) hash_long((unsigned long)key, CLASSHASH_BITS)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700293#define classhashentry(key) (classhash_table + __classhashfn((key)))
294
295static struct list_head classhash_table[CLASSHASH_SIZE];
296
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700297/*
298 * We put the lock dependency chains into a hash-table as well, to cache
299 * their existence:
300 */
301#define CHAINHASH_BITS (MAX_LOCKDEP_CHAINS_BITS-1)
302#define CHAINHASH_SIZE (1UL << CHAINHASH_BITS)
Peter Zijlstra4b32d0a2007-07-19 01:48:59 -0700303#define __chainhashfn(chain) hash_long(chain, CHAINHASH_BITS)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700304#define chainhashentry(chain) (chainhash_table + __chainhashfn((chain)))
305
306static struct list_head chainhash_table[CHAINHASH_SIZE];
307
308/*
309 * The hash key of the lock dependency chains is a hash itself too:
310 * it's a hash of all locks taken up to that lock, including that lock.
311 * It's a 64-bit hash, because it's important for the keys to be
312 * unique.
313 */
314#define iterate_chain_key(key1, key2) \
Ingo Molnar03cbc352006-09-29 02:01:46 -0700315 (((key1) << MAX_LOCKDEP_KEYS_BITS) ^ \
316 ((key1) >> (64-MAX_LOCKDEP_KEYS_BITS)) ^ \
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700317 (key2))
318
Steven Rostedt1d09daa2008-05-12 21:20:55 +0200319void lockdep_off(void)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700320{
321 current->lockdep_recursion++;
322}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700323EXPORT_SYMBOL(lockdep_off);
324
Steven Rostedt1d09daa2008-05-12 21:20:55 +0200325void lockdep_on(void)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700326{
327 current->lockdep_recursion--;
328}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700329EXPORT_SYMBOL(lockdep_on);
330
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700331/*
332 * Debugging switches:
333 */
334
335#define VERBOSE 0
Ingo Molnar33e94e92006-12-13 00:34:41 -0800336#define VERY_VERBOSE 0
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700337
338#if VERBOSE
339# define HARDIRQ_VERBOSE 1
340# define SOFTIRQ_VERBOSE 1
Nick Piggincf40bd12009-01-21 08:12:39 +0100341# define RECLAIM_VERBOSE 1
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700342#else
343# define HARDIRQ_VERBOSE 0
344# define SOFTIRQ_VERBOSE 0
Nick Piggincf40bd12009-01-21 08:12:39 +0100345# define RECLAIM_VERBOSE 0
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700346#endif
347
Nick Piggincf40bd12009-01-21 08:12:39 +0100348#if VERBOSE || HARDIRQ_VERBOSE || SOFTIRQ_VERBOSE || RECLAIM_VERBOSE
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700349/*
350 * Quick filtering for interesting events:
351 */
352static int class_filter(struct lock_class *class)
353{
Andi Kleenf9829cc2006-07-10 04:44:01 -0700354#if 0
355 /* Example */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700356 if (class->name_version == 1 &&
Andi Kleenf9829cc2006-07-10 04:44:01 -0700357 !strcmp(class->name, "lockname"))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700358 return 1;
359 if (class->name_version == 1 &&
Andi Kleenf9829cc2006-07-10 04:44:01 -0700360 !strcmp(class->name, "&struct->lockfield"))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700361 return 1;
Andi Kleenf9829cc2006-07-10 04:44:01 -0700362#endif
Ingo Molnara6640892006-12-13 00:34:39 -0800363 /* Filter everything else. 1 would be to allow everything else */
364 return 0;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700365}
366#endif
367
368static int verbose(struct lock_class *class)
369{
370#if VERBOSE
371 return class_filter(class);
372#endif
373 return 0;
374}
375
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700376/*
377 * Stack-trace: tightly packed array of stack backtrace
Ingo Molnar74c383f2006-12-13 00:34:43 -0800378 * addresses. Protected by the graph_lock.
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700379 */
380unsigned long nr_stack_trace_entries;
381static unsigned long stack_trace[MAX_STACK_TRACE_ENTRIES];
382
Dave Jones2c522832013-04-25 13:40:02 -0400383static void print_lockdep_off(const char *bug_msg)
384{
385 printk(KERN_DEBUG "%s\n", bug_msg);
386 printk(KERN_DEBUG "turning off the locking correctness validator.\n");
Andreas Gruenbacheracf59372014-07-15 21:10:52 +0200387#ifdef CONFIG_LOCK_STAT
Dave Jones2c522832013-04-25 13:40:02 -0400388 printk(KERN_DEBUG "Please attach the output of /proc/lock_stat to the bug report\n");
Andreas Gruenbacheracf59372014-07-15 21:10:52 +0200389#endif
Dave Jones2c522832013-04-25 13:40:02 -0400390}
391
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700392static int save_trace(struct stack_trace *trace)
393{
394 trace->nr_entries = 0;
395 trace->max_entries = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries;
396 trace->entries = stack_trace + nr_stack_trace_entries;
397
Andi Kleen5a1b3992006-09-26 10:52:34 +0200398 trace->skip = 3;
Andi Kleen5a1b3992006-09-26 10:52:34 +0200399
Christoph Hellwigab1b6f02007-05-08 00:23:29 -0700400 save_stack_trace(trace);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700401
Peter Zijlstra4f84f432009-07-20 15:27:04 +0200402 /*
403 * Some daft arches put -1 at the end to indicate its a full trace.
404 *
405 * <rant> this is buggy anyway, since it takes a whole extra entry so a
406 * complete trace that maxes out the entries provided will be reported
407 * as incomplete, friggin useless </rant>
408 */
Luck, Tonyea5b41f2009-12-09 14:29:36 -0800409 if (trace->nr_entries != 0 &&
410 trace->entries[trace->nr_entries-1] == ULONG_MAX)
Peter Zijlstra4f84f432009-07-20 15:27:04 +0200411 trace->nr_entries--;
412
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700413 trace->max_entries = trace->nr_entries;
414
415 nr_stack_trace_entries += trace->nr_entries;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700416
Peter Zijlstra4f84f432009-07-20 15:27:04 +0200417 if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES-1) {
Ingo Molnar74c383f2006-12-13 00:34:43 -0800418 if (!debug_locks_off_graph_unlock())
419 return 0;
420
Dave Jones2c522832013-04-25 13:40:02 -0400421 print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");
Ingo Molnar74c383f2006-12-13 00:34:43 -0800422 dump_stack();
423
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700424 return 0;
425 }
426
427 return 1;
428}
429
430unsigned int nr_hardirq_chains;
431unsigned int nr_softirq_chains;
432unsigned int nr_process_chains;
433unsigned int max_lockdep_depth;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700434
435#ifdef CONFIG_DEBUG_LOCKDEP
436/*
437 * We cannot printk in early bootup code. Not even early_printk()
438 * might work. So we mark any initialization errors and printk
439 * about it later on, in lockdep_info().
440 */
441static int lockdep_init_error;
Ming Lei81140ac2011-11-17 13:34:32 +0800442static const char *lock_init_error;
Johannes Bergc71063c2007-07-19 01:49:02 -0700443static unsigned long lockdep_init_trace_data[20];
444static struct stack_trace lockdep_init_trace = {
445 .max_entries = ARRAY_SIZE(lockdep_init_trace_data),
446 .entries = lockdep_init_trace_data,
447};
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700448
449/*
450 * Various lockdep statistics:
451 */
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +0200452DEFINE_PER_CPU(struct lockdep_stats, lockdep_stats);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700453#endif
454
455/*
456 * Locking printouts:
457 */
458
Peter Zijlstrafabe9c42009-01-22 14:51:01 +0100459#define __USAGE(__STATE) \
Peter Zijlstrab4b136f2009-01-29 14:50:36 +0100460 [LOCK_USED_IN_##__STATE] = "IN-"__stringify(__STATE)"-W", \
461 [LOCK_ENABLED_##__STATE] = __stringify(__STATE)"-ON-W", \
462 [LOCK_USED_IN_##__STATE##_READ] = "IN-"__stringify(__STATE)"-R",\
463 [LOCK_ENABLED_##__STATE##_READ] = __stringify(__STATE)"-ON-R",
Peter Zijlstrafabe9c42009-01-22 14:51:01 +0100464
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700465static const char *usage_str[] =
466{
Peter Zijlstrafabe9c42009-01-22 14:51:01 +0100467#define LOCKDEP_STATE(__STATE) __USAGE(__STATE)
468#include "lockdep_states.h"
469#undef LOCKDEP_STATE
470 [LOCK_USED] = "INITIAL USE",
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700471};
472
473const char * __get_key_name(struct lockdep_subclass_key *key, char *str)
474{
Alexey Dobriyanffb45122007-05-08 00:28:41 -0700475 return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700476}
477
Peter Zijlstra3ff176c2009-01-22 17:40:42 +0100478static inline unsigned long lock_flag(enum lock_usage_bit bit)
479{
480 return 1UL << bit;
481}
482
483static char get_usage_char(struct lock_class *class, enum lock_usage_bit bit)
484{
485 char c = '.';
486
487 if (class->usage_mask & lock_flag(bit + 2))
488 c = '+';
489 if (class->usage_mask & lock_flag(bit)) {
490 c = '-';
491 if (class->usage_mask & lock_flag(bit + 2))
492 c = '?';
493 }
494
495 return c;
496}
497
Peter Zijlstraf510b232009-01-22 17:53:47 +0100498void get_usage_chars(struct lock_class *class, char usage[LOCK_USAGE_CHARS])
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700499{
Peter Zijlstraf510b232009-01-22 17:53:47 +0100500 int i = 0;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700501
Peter Zijlstraf510b232009-01-22 17:53:47 +0100502#define LOCKDEP_STATE(__STATE) \
503 usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE); \
504 usage[i++] = get_usage_char(class, LOCK_USED_IN_##__STATE##_READ);
505#include "lockdep_states.h"
506#undef LOCKDEP_STATE
507
508 usage[i] = '\0';
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700509}
510
Steven Rostedte5e78d02011-11-02 20:24:16 -0400511static void __print_lock_name(struct lock_class *class)
Steven Rostedt3003eba2011-04-20 21:41:54 -0400512{
513 char str[KSYM_NAME_LEN];
514 const char *name;
515
516 name = class->name;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700517 if (!name) {
518 name = __get_key_name(class->key, str);
Steven Rostedte5e78d02011-11-02 20:24:16 -0400519 printk("%s", name);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700520 } else {
Steven Rostedte5e78d02011-11-02 20:24:16 -0400521 printk("%s", name);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700522 if (class->name_version > 1)
523 printk("#%d", class->name_version);
524 if (class->subclass)
525 printk("/%d", class->subclass);
526 }
Steven Rostedte5e78d02011-11-02 20:24:16 -0400527}
528
529static void print_lock_name(struct lock_class *class)
530{
531 char usage[LOCK_USAGE_CHARS];
532
533 get_usage_chars(class, usage);
534
535 printk(" (");
536 __print_lock_name(class);
Peter Zijlstraf510b232009-01-22 17:53:47 +0100537 printk("){%s}", usage);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700538}
539
540static void print_lockdep_cache(struct lockdep_map *lock)
541{
542 const char *name;
Tejun Heo9281ace2007-07-17 04:03:51 -0700543 char str[KSYM_NAME_LEN];
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700544
545 name = lock->name;
546 if (!name)
547 name = __get_key_name(lock->key->subkeys, str);
548
549 printk("%s", name);
550}
551
552static void print_lock(struct held_lock *hlock)
553{
Dave Jonesf82b2172008-08-11 09:30:23 +0200554 print_lock_name(hlock_class(hlock));
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700555 printk(", at: ");
556 print_ip_sym(hlock->acquire_ip);
557}
558
559static void lockdep_print_held_locks(struct task_struct *curr)
560{
561 int i, depth = curr->lockdep_depth;
562
563 if (!depth) {
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700564 printk("no locks held by %s/%d.\n", curr->comm, task_pid_nr(curr));
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700565 return;
566 }
567 printk("%d lock%s held by %s/%d:\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -0700568 depth, depth > 1 ? "s" : "", curr->comm, task_pid_nr(curr));
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700569
570 for (i = 0; i < depth; i++) {
571 printk(" #%d: ", i);
572 print_lock(curr->held_locks + i);
573 }
574}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700575
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +0100576static void print_kernel_ident(void)
Dave Jones99de0552006-09-29 02:00:10 -0700577{
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +0100578 printk("%s %.*s %s\n", init_utsname()->release,
Serge E. Hallyn96b644b2006-10-02 02:18:13 -0700579 (int)strcspn(init_utsname()->version, " "),
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +0100580 init_utsname()->version,
581 print_tainted());
Dave Jones99de0552006-09-29 02:00:10 -0700582}
583
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700584static int very_verbose(struct lock_class *class)
585{
586#if VERY_VERBOSE
587 return class_filter(class);
588#endif
589 return 0;
590}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700591
592/*
593 * Is this the address of a static object:
594 */
Sasha Levin8dce7a92013-06-13 18:41:16 -0400595#ifdef __KERNEL__
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700596static int static_obj(void *obj)
597{
598 unsigned long start = (unsigned long) &_stext,
599 end = (unsigned long) &_end,
600 addr = (unsigned long) obj;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700601
602 /*
603 * static variable?
604 */
605 if ((addr >= start) && (addr < end))
606 return 1;
607
Mike Frysinger2a9ad182009-09-22 16:44:16 -0700608 if (arch_is_kernel_data(addr))
609 return 1;
610
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700611 /*
Tejun Heo10fad5e2010-03-10 18:57:54 +0900612 * in-kernel percpu var?
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700613 */
Tejun Heo10fad5e2010-03-10 18:57:54 +0900614 if (is_kernel_percpu_address(addr))
615 return 1;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700616
617 /*
Tejun Heo10fad5e2010-03-10 18:57:54 +0900618 * module static or percpu var?
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700619 */
Tejun Heo10fad5e2010-03-10 18:57:54 +0900620 return is_module_address(addr) || is_module_percpu_address(addr);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700621}
Sasha Levin8dce7a92013-06-13 18:41:16 -0400622#endif
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700623
624/*
625 * To make lock name printouts unique, we calculate a unique
626 * class->name_version generation counter:
627 */
628static int count_matching_names(struct lock_class *new_class)
629{
630 struct lock_class *class;
631 int count = 0;
632
633 if (!new_class->name)
634 return 0;
635
636 list_for_each_entry(class, &all_lock_classes, lock_entry) {
637 if (new_class->key - new_class->subclass == class->key)
638 return class->name_version;
639 if (class->name && !strcmp(class->name, new_class->name))
640 count = max(count, class->name_version);
641 }
642
643 return count + 1;
644}
645
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700646/*
647 * Register a lock's class in the hash-table, if the class is not present
648 * yet. Otherwise we look it up. We cache the result in the lock object
649 * itself, so actual lookup of the hash should be once per lock object.
650 */
651static inline struct lock_class *
Ingo Molnard6d897c2006-07-10 04:44:04 -0700652look_up_lock_class(struct lockdep_map *lock, unsigned int subclass)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700653{
654 struct lockdep_subclass_key *key;
655 struct list_head *hash_head;
656 struct lock_class *class;
657
658#ifdef CONFIG_DEBUG_LOCKDEP
659 /*
660 * If the architecture calls into lockdep before initializing
661 * the hashes then we'll warn about it later. (we cannot printk
662 * right now)
663 */
664 if (unlikely(!lockdep_initialized)) {
665 lockdep_init();
666 lockdep_init_error = 1;
Ming Lei81140ac2011-11-17 13:34:32 +0800667 lock_init_error = lock->name;
Johannes Bergc71063c2007-07-19 01:49:02 -0700668 save_stack_trace(&lockdep_init_trace);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700669 }
670#endif
671
Hitoshi Mitake4ba053c2010-10-13 17:30:26 +0900672 if (unlikely(subclass >= MAX_LOCKDEP_SUBCLASSES)) {
673 debug_locks_off();
674 printk(KERN_ERR
675 "BUG: looking up invalid subclass: %u\n", subclass);
676 printk(KERN_ERR
677 "turning off the locking correctness validator.\n");
678 dump_stack();
679 return NULL;
680 }
681
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700682 /*
683 * Static locks do not have their class-keys yet - for them the key
684 * is the lock object itself:
685 */
686 if (unlikely(!lock->key))
687 lock->key = (void *)lock;
688
689 /*
690 * NOTE: the class-key must be unique. For dynamic locks, a static
691 * lock_class_key variable is passed in through the mutex_init()
692 * (or spin_lock_init()) call - which acts as the key. For static
693 * locks we use the lock object itself as the key.
694 */
Peter Zijlstra4b32d0a2007-07-19 01:48:59 -0700695 BUILD_BUG_ON(sizeof(struct lock_class_key) >
696 sizeof(struct lockdep_map));
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700697
698 key = lock->key->subkeys + subclass;
699
700 hash_head = classhashentry(key);
701
702 /*
703 * We can walk the hash lockfree, because the hash only
704 * grows, and we are careful when adding entries to the end:
705 */
Peter Zijlstra4b32d0a2007-07-19 01:48:59 -0700706 list_for_each_entry(class, hash_head, hash_entry) {
707 if (class->key == key) {
Peter Zijlstra0119fee2011-09-02 01:30:29 +0200708 /*
709 * Huh! same key, different name? Did someone trample
710 * on some memory? We're most confused.
711 */
Peter Zijlstra4b32d0a2007-07-19 01:48:59 -0700712 WARN_ON_ONCE(class->name != lock->name);
Ingo Molnard6d897c2006-07-10 04:44:04 -0700713 return class;
Peter Zijlstra4b32d0a2007-07-19 01:48:59 -0700714 }
715 }
Ingo Molnard6d897c2006-07-10 04:44:04 -0700716
717 return NULL;
718}
719
720/*
721 * Register a lock's class in the hash-table, if the class is not present
722 * yet. Otherwise we look it up. We cache the result in the lock object
723 * itself, so actual lookup of the hash should be once per lock object.
724 */
725static inline struct lock_class *
Peter Zijlstra4dfbb9d2006-10-11 01:45:14 -0400726register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force)
Ingo Molnard6d897c2006-07-10 04:44:04 -0700727{
728 struct lockdep_subclass_key *key;
729 struct list_head *hash_head;
730 struct lock_class *class;
Ingo Molnar70e45062006-12-06 20:40:50 -0800731 unsigned long flags;
Ingo Molnard6d897c2006-07-10 04:44:04 -0700732
733 class = look_up_lock_class(lock, subclass);
734 if (likely(class))
Yong Zhang87cdee72011-11-09 16:07:14 +0800735 goto out_set_class_cache;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700736
737 /*
738 * Debug-check: all keys must be persistent!
739 */
740 if (!static_obj(lock->key)) {
741 debug_locks_off();
742 printk("INFO: trying to register non-static key.\n");
743 printk("the code is fine but needs lockdep annotation.\n");
744 printk("turning off the locking correctness validator.\n");
745 dump_stack();
746
747 return NULL;
748 }
749
Ingo Molnard6d897c2006-07-10 04:44:04 -0700750 key = lock->key->subkeys + subclass;
751 hash_head = classhashentry(key);
752
Ingo Molnar70e45062006-12-06 20:40:50 -0800753 raw_local_irq_save(flags);
Ingo Molnar74c383f2006-12-13 00:34:43 -0800754 if (!graph_lock()) {
755 raw_local_irq_restore(flags);
756 return NULL;
757 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700758 /*
759 * We have to do the hash-walk again, to avoid races
760 * with another CPU:
761 */
762 list_for_each_entry(class, hash_head, hash_entry)
763 if (class->key == key)
764 goto out_unlock_set;
765 /*
766 * Allocate a new key from the static array, and add it to
767 * the hash:
768 */
769 if (nr_lock_classes >= MAX_LOCKDEP_KEYS) {
Ingo Molnar74c383f2006-12-13 00:34:43 -0800770 if (!debug_locks_off_graph_unlock()) {
771 raw_local_irq_restore(flags);
772 return NULL;
773 }
Ingo Molnar70e45062006-12-06 20:40:50 -0800774 raw_local_irq_restore(flags);
Ingo Molnar74c383f2006-12-13 00:34:43 -0800775
Dave Jones2c522832013-04-25 13:40:02 -0400776 print_lockdep_off("BUG: MAX_LOCKDEP_KEYS too low!");
Peter Zijlstraeedeeab2009-03-18 12:38:47 +0100777 dump_stack();
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700778 return NULL;
779 }
780 class = lock_classes + nr_lock_classes++;
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +0200781 debug_atomic_inc(nr_unused_locks);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700782 class->key = key;
783 class->name = lock->name;
784 class->subclass = subclass;
785 INIT_LIST_HEAD(&class->lock_entry);
786 INIT_LIST_HEAD(&class->locks_before);
787 INIT_LIST_HEAD(&class->locks_after);
788 class->name_version = count_matching_names(class);
789 /*
790 * We use RCU's safe list-add method to make
791 * parallel walking of the hash-list safe:
792 */
793 list_add_tail_rcu(&class->hash_entry, hash_head);
Dale Farnsworth14811972008-02-25 23:03:02 +0100794 /*
795 * Add it to the global list of classes:
796 */
797 list_add_tail_rcu(&class->lock_entry, &all_lock_classes);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700798
799 if (verbose(class)) {
Ingo Molnar74c383f2006-12-13 00:34:43 -0800800 graph_unlock();
Ingo Molnar70e45062006-12-06 20:40:50 -0800801 raw_local_irq_restore(flags);
Ingo Molnar74c383f2006-12-13 00:34:43 -0800802
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700803 printk("\nnew class %p: %s", class->key, class->name);
804 if (class->name_version > 1)
805 printk("#%d", class->name_version);
806 printk("\n");
807 dump_stack();
Ingo Molnar74c383f2006-12-13 00:34:43 -0800808
Ingo Molnar70e45062006-12-06 20:40:50 -0800809 raw_local_irq_save(flags);
Ingo Molnar74c383f2006-12-13 00:34:43 -0800810 if (!graph_lock()) {
811 raw_local_irq_restore(flags);
812 return NULL;
813 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700814 }
815out_unlock_set:
Ingo Molnar74c383f2006-12-13 00:34:43 -0800816 graph_unlock();
Ingo Molnar70e45062006-12-06 20:40:50 -0800817 raw_local_irq_restore(flags);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700818
Yong Zhang87cdee72011-11-09 16:07:14 +0800819out_set_class_cache:
Peter Zijlstra4dfbb9d2006-10-11 01:45:14 -0400820 if (!subclass || force)
Hitoshi Mitake62016252010-10-05 18:01:51 +0900821 lock->class_cache[0] = class;
822 else if (subclass < NR_LOCKDEP_CACHING_CLASSES)
823 lock->class_cache[subclass] = class;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700824
Peter Zijlstra0119fee2011-09-02 01:30:29 +0200825 /*
826 * Hash collision, did we smoke some? We found a class with a matching
827 * hash but the subclass -- which is hashed in -- didn't match.
828 */
Jarek Poplawski381a2292007-02-10 01:44:58 -0800829 if (DEBUG_LOCKS_WARN_ON(class->subclass != subclass))
830 return NULL;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700831
832 return class;
833}
834
Peter Zijlstraca58abc2007-07-19 01:48:53 -0700835#ifdef CONFIG_PROVE_LOCKING
Ingo Molnarfbb9ce952006-07-03 00:24:50 -0700836/*
Peter Zijlstra8e182572007-07-19 01:48:54 -0700837 * Allocate a lockdep entry. (assumes the graph_lock held, returns
838 * with NULL on failure)
839 */
840static struct lock_list *alloc_list_entry(void)
841{
842 if (nr_list_entries >= MAX_LOCKDEP_ENTRIES) {
843 if (!debug_locks_off_graph_unlock())
844 return NULL;
845
Dave Jones2c522832013-04-25 13:40:02 -0400846 print_lockdep_off("BUG: MAX_LOCKDEP_ENTRIES too low!");
Peter Zijlstraeedeeab2009-03-18 12:38:47 +0100847 dump_stack();
Peter Zijlstra8e182572007-07-19 01:48:54 -0700848 return NULL;
849 }
850 return list_entries + nr_list_entries++;
851}
852
853/*
854 * Add a new dependency to the head of the list:
855 */
856static int add_lock_to_list(struct lock_class *class, struct lock_class *this,
Yong Zhang4726f2a2010-05-04 14:16:48 +0800857 struct list_head *head, unsigned long ip,
858 int distance, struct stack_trace *trace)
Peter Zijlstra8e182572007-07-19 01:48:54 -0700859{
860 struct lock_list *entry;
861 /*
862 * Lock not present yet - get a new dependency struct and
863 * add it to the list:
864 */
865 entry = alloc_list_entry();
866 if (!entry)
867 return 0;
868
Zhu Yi74870172008-08-27 14:33:00 +0800869 entry->class = this;
870 entry->distance = distance;
Yong Zhang4726f2a2010-05-04 14:16:48 +0800871 entry->trace = *trace;
Peter Zijlstra8e182572007-07-19 01:48:54 -0700872 /*
873 * Since we never remove from the dependency list, the list can
874 * be walked lockless by other CPUs, it's only allocation
875 * that must be protected by the spinlock. But this also means
876 * we must make new entries visible only once writes to the
877 * entry become visible - hence the RCU op:
878 */
879 list_add_tail_rcu(&entry->entry, head);
880
881 return 1;
882}
883
Peter Zijlstra98c33ed2009-07-21 13:19:07 +0200884/*
885 * For good efficiency of modular, we use power of 2
886 */
Peter Zijlstraaf012962009-07-16 15:44:29 +0200887#define MAX_CIRCULAR_QUEUE_SIZE 4096UL
888#define CQ_MASK (MAX_CIRCULAR_QUEUE_SIZE-1)
889
Peter Zijlstra98c33ed2009-07-21 13:19:07 +0200890/*
891 * The circular_queue and helpers is used to implement the
Peter Zijlstraaf012962009-07-16 15:44:29 +0200892 * breadth-first search(BFS)algorithem, by which we can build
893 * the shortest path from the next lock to be acquired to the
894 * previous held lock if there is a circular between them.
Peter Zijlstra98c33ed2009-07-21 13:19:07 +0200895 */
Peter Zijlstraaf012962009-07-16 15:44:29 +0200896struct circular_queue {
897 unsigned long element[MAX_CIRCULAR_QUEUE_SIZE];
898 unsigned int front, rear;
899};
900
901static struct circular_queue lock_cq;
Peter Zijlstraaf012962009-07-16 15:44:29 +0200902
Ming Lei12f3dfd2009-07-16 15:44:29 +0200903unsigned int max_bfs_queue_depth;
Peter Zijlstraaf012962009-07-16 15:44:29 +0200904
Ming Leie351b662009-07-22 22:48:09 +0800905static unsigned int lockdep_dependency_gen_id;
906
Peter Zijlstraaf012962009-07-16 15:44:29 +0200907static inline void __cq_init(struct circular_queue *cq)
908{
909 cq->front = cq->rear = 0;
Ming Leie351b662009-07-22 22:48:09 +0800910 lockdep_dependency_gen_id++;
Peter Zijlstraaf012962009-07-16 15:44:29 +0200911}
912
913static inline int __cq_empty(struct circular_queue *cq)
914{
915 return (cq->front == cq->rear);
916}
917
918static inline int __cq_full(struct circular_queue *cq)
919{
920 return ((cq->rear + 1) & CQ_MASK) == cq->front;
921}
922
923static inline int __cq_enqueue(struct circular_queue *cq, unsigned long elem)
924{
925 if (__cq_full(cq))
926 return -1;
927
928 cq->element[cq->rear] = elem;
929 cq->rear = (cq->rear + 1) & CQ_MASK;
930 return 0;
931}
932
933static inline int __cq_dequeue(struct circular_queue *cq, unsigned long *elem)
934{
935 if (__cq_empty(cq))
936 return -1;
937
938 *elem = cq->element[cq->front];
939 cq->front = (cq->front + 1) & CQ_MASK;
940 return 0;
941}
942
943static inline unsigned int __cq_get_elem_count(struct circular_queue *cq)
944{
945 return (cq->rear - cq->front) & CQ_MASK;
946}
947
948static inline void mark_lock_accessed(struct lock_list *lock,
949 struct lock_list *parent)
950{
951 unsigned long nr;
Peter Zijlstra98c33ed2009-07-21 13:19:07 +0200952
Peter Zijlstraaf012962009-07-16 15:44:29 +0200953 nr = lock - list_entries;
Peter Zijlstra0119fee2011-09-02 01:30:29 +0200954 WARN_ON(nr >= nr_list_entries); /* Out-of-bounds, input fail */
Peter Zijlstraaf012962009-07-16 15:44:29 +0200955 lock->parent = parent;
Ming Leie351b662009-07-22 22:48:09 +0800956 lock->class->dep_gen_id = lockdep_dependency_gen_id;
Peter Zijlstraaf012962009-07-16 15:44:29 +0200957}
958
959static inline unsigned long lock_accessed(struct lock_list *lock)
960{
961 unsigned long nr;
Peter Zijlstra98c33ed2009-07-21 13:19:07 +0200962
Peter Zijlstraaf012962009-07-16 15:44:29 +0200963 nr = lock - list_entries;
Peter Zijlstra0119fee2011-09-02 01:30:29 +0200964 WARN_ON(nr >= nr_list_entries); /* Out-of-bounds, input fail */
Ming Leie351b662009-07-22 22:48:09 +0800965 return lock->class->dep_gen_id == lockdep_dependency_gen_id;
Peter Zijlstraaf012962009-07-16 15:44:29 +0200966}
967
968static inline struct lock_list *get_lock_parent(struct lock_list *child)
969{
970 return child->parent;
971}
972
973static inline int get_lock_depth(struct lock_list *child)
974{
975 int depth = 0;
976 struct lock_list *parent;
977
978 while ((parent = get_lock_parent(child))) {
979 child = parent;
980 depth++;
981 }
982 return depth;
983}
984
Ming Lei9e2d5512009-07-16 15:44:29 +0200985static int __bfs(struct lock_list *source_entry,
Peter Zijlstraaf012962009-07-16 15:44:29 +0200986 void *data,
987 int (*match)(struct lock_list *entry, void *data),
988 struct lock_list **target_entry,
989 int forward)
Ming Leic94aa5c2009-07-16 15:44:29 +0200990{
991 struct lock_list *entry;
Ming Leid588e462009-07-16 15:44:29 +0200992 struct list_head *head;
Ming Leic94aa5c2009-07-16 15:44:29 +0200993 struct circular_queue *cq = &lock_cq;
994 int ret = 1;
995
Ming Lei9e2d5512009-07-16 15:44:29 +0200996 if (match(source_entry, data)) {
Ming Leic94aa5c2009-07-16 15:44:29 +0200997 *target_entry = source_entry;
998 ret = 0;
999 goto exit;
1000 }
1001
Ming Leid588e462009-07-16 15:44:29 +02001002 if (forward)
1003 head = &source_entry->class->locks_after;
1004 else
1005 head = &source_entry->class->locks_before;
1006
1007 if (list_empty(head))
1008 goto exit;
1009
1010 __cq_init(cq);
Ming Leic94aa5c2009-07-16 15:44:29 +02001011 __cq_enqueue(cq, (unsigned long)source_entry);
1012
1013 while (!__cq_empty(cq)) {
1014 struct lock_list *lock;
Ming Leic94aa5c2009-07-16 15:44:29 +02001015
1016 __cq_dequeue(cq, (unsigned long *)&lock);
1017
1018 if (!lock->class) {
1019 ret = -2;
1020 goto exit;
1021 }
1022
1023 if (forward)
1024 head = &lock->class->locks_after;
1025 else
1026 head = &lock->class->locks_before;
1027
1028 list_for_each_entry(entry, head, entry) {
1029 if (!lock_accessed(entry)) {
Ming Lei12f3dfd2009-07-16 15:44:29 +02001030 unsigned int cq_depth;
Ming Leic94aa5c2009-07-16 15:44:29 +02001031 mark_lock_accessed(entry, lock);
Ming Lei9e2d5512009-07-16 15:44:29 +02001032 if (match(entry, data)) {
Ming Leic94aa5c2009-07-16 15:44:29 +02001033 *target_entry = entry;
1034 ret = 0;
1035 goto exit;
1036 }
1037
1038 if (__cq_enqueue(cq, (unsigned long)entry)) {
1039 ret = -1;
1040 goto exit;
1041 }
Ming Lei12f3dfd2009-07-16 15:44:29 +02001042 cq_depth = __cq_get_elem_count(cq);
1043 if (max_bfs_queue_depth < cq_depth)
1044 max_bfs_queue_depth = cq_depth;
Ming Leic94aa5c2009-07-16 15:44:29 +02001045 }
1046 }
1047 }
1048exit:
1049 return ret;
1050}
1051
Ming Leid7aaba12009-07-16 15:44:29 +02001052static inline int __bfs_forwards(struct lock_list *src_entry,
Ming Lei9e2d5512009-07-16 15:44:29 +02001053 void *data,
1054 int (*match)(struct lock_list *entry, void *data),
1055 struct lock_list **target_entry)
Ming Leic94aa5c2009-07-16 15:44:29 +02001056{
Ming Lei9e2d5512009-07-16 15:44:29 +02001057 return __bfs(src_entry, data, match, target_entry, 1);
Ming Leic94aa5c2009-07-16 15:44:29 +02001058
1059}
1060
Ming Leid7aaba12009-07-16 15:44:29 +02001061static inline int __bfs_backwards(struct lock_list *src_entry,
Ming Lei9e2d5512009-07-16 15:44:29 +02001062 void *data,
1063 int (*match)(struct lock_list *entry, void *data),
1064 struct lock_list **target_entry)
Ming Leic94aa5c2009-07-16 15:44:29 +02001065{
Ming Lei9e2d5512009-07-16 15:44:29 +02001066 return __bfs(src_entry, data, match, target_entry, 0);
Ming Leic94aa5c2009-07-16 15:44:29 +02001067
1068}
1069
Peter Zijlstra8e182572007-07-19 01:48:54 -07001070/*
1071 * Recursive, forwards-direction lock-dependency checking, used for
1072 * both noncyclic checking and for hardirq-unsafe/softirq-unsafe
1073 * checking.
Peter Zijlstra8e182572007-07-19 01:48:54 -07001074 */
Peter Zijlstra8e182572007-07-19 01:48:54 -07001075
1076/*
1077 * Print a dependency chain entry (this is only done when a deadlock
1078 * has been detected):
1079 */
1080static noinline int
Ming Lei24208ca2009-07-16 15:44:29 +02001081print_circular_bug_entry(struct lock_list *target, int depth)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001082{
1083 if (debug_locks_silent)
1084 return 0;
1085 printk("\n-> #%u", depth);
1086 print_lock_name(target->class);
1087 printk(":\n");
1088 print_stack_trace(&target->trace, 6);
1089
1090 return 0;
1091}
1092
Steven Rostedtf4185812011-04-20 21:41:55 -04001093static void
1094print_circular_lock_scenario(struct held_lock *src,
1095 struct held_lock *tgt,
1096 struct lock_list *prt)
1097{
1098 struct lock_class *source = hlock_class(src);
1099 struct lock_class *target = hlock_class(tgt);
1100 struct lock_class *parent = prt->class;
1101
1102 /*
1103 * A direct locking problem where unsafe_class lock is taken
1104 * directly by safe_class lock, then all we need to show
1105 * is the deadlock scenario, as it is obvious that the
1106 * unsafe lock is taken under the safe lock.
1107 *
1108 * But if there is a chain instead, where the safe lock takes
1109 * an intermediate lock (middle_class) where this lock is
1110 * not the same as the safe lock, then the lock chain is
1111 * used to describe the problem. Otherwise we would need
1112 * to show a different CPU case for each link in the chain
1113 * from the safe_class lock to the unsafe_class lock.
1114 */
1115 if (parent != source) {
1116 printk("Chain exists of:\n ");
1117 __print_lock_name(source);
1118 printk(" --> ");
1119 __print_lock_name(parent);
1120 printk(" --> ");
1121 __print_lock_name(target);
1122 printk("\n\n");
1123 }
1124
1125 printk(" Possible unsafe locking scenario:\n\n");
1126 printk(" CPU0 CPU1\n");
1127 printk(" ---- ----\n");
1128 printk(" lock(");
1129 __print_lock_name(target);
1130 printk(");\n");
1131 printk(" lock(");
1132 __print_lock_name(parent);
1133 printk(");\n");
1134 printk(" lock(");
1135 __print_lock_name(target);
1136 printk(");\n");
1137 printk(" lock(");
1138 __print_lock_name(source);
1139 printk(");\n");
1140 printk("\n *** DEADLOCK ***\n\n");
1141}
1142
Peter Zijlstra8e182572007-07-19 01:48:54 -07001143/*
1144 * When a circular dependency is detected, print the
1145 * header first:
1146 */
1147static noinline int
Ming Leidb0002a2009-07-16 15:44:29 +02001148print_circular_bug_header(struct lock_list *entry, unsigned int depth,
1149 struct held_lock *check_src,
1150 struct held_lock *check_tgt)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001151{
1152 struct task_struct *curr = current;
1153
Ming Leic94aa5c2009-07-16 15:44:29 +02001154 if (debug_locks_silent)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001155 return 0;
1156
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07001157 printk("\n");
1158 printk("======================================================\n");
1159 printk("[ INFO: possible circular locking dependency detected ]\n");
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01001160 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07001161 printk("-------------------------------------------------------\n");
Peter Zijlstra8e182572007-07-19 01:48:54 -07001162 printk("%s/%d is trying to acquire lock:\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07001163 curr->comm, task_pid_nr(curr));
Ming Leidb0002a2009-07-16 15:44:29 +02001164 print_lock(check_src);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001165 printk("\nbut task is already holding lock:\n");
Ming Leidb0002a2009-07-16 15:44:29 +02001166 print_lock(check_tgt);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001167 printk("\nwhich lock already depends on the new lock.\n\n");
1168 printk("\nthe existing dependency chain (in reverse order) is:\n");
1169
1170 print_circular_bug_entry(entry, depth);
1171
1172 return 0;
1173}
1174
Ming Lei9e2d5512009-07-16 15:44:29 +02001175static inline int class_equal(struct lock_list *entry, void *data)
1176{
1177 return entry->class == data;
1178}
1179
Ming Leidb0002a2009-07-16 15:44:29 +02001180static noinline int print_circular_bug(struct lock_list *this,
1181 struct lock_list *target,
1182 struct held_lock *check_src,
1183 struct held_lock *check_tgt)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001184{
1185 struct task_struct *curr = current;
Ming Leic94aa5c2009-07-16 15:44:29 +02001186 struct lock_list *parent;
Steven Rostedtf4185812011-04-20 21:41:55 -04001187 struct lock_list *first_parent;
Ming Lei24208ca2009-07-16 15:44:29 +02001188 int depth;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001189
Ming Leic94aa5c2009-07-16 15:44:29 +02001190 if (!debug_locks_off_graph_unlock() || debug_locks_silent)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001191 return 0;
1192
Ming Leidb0002a2009-07-16 15:44:29 +02001193 if (!save_trace(&this->trace))
Peter Zijlstra8e182572007-07-19 01:48:54 -07001194 return 0;
1195
Ming Leic94aa5c2009-07-16 15:44:29 +02001196 depth = get_lock_depth(target);
1197
Ming Leidb0002a2009-07-16 15:44:29 +02001198 print_circular_bug_header(target, depth, check_src, check_tgt);
Ming Leic94aa5c2009-07-16 15:44:29 +02001199
1200 parent = get_lock_parent(target);
Steven Rostedtf4185812011-04-20 21:41:55 -04001201 first_parent = parent;
Ming Leic94aa5c2009-07-16 15:44:29 +02001202
1203 while (parent) {
1204 print_circular_bug_entry(parent, --depth);
1205 parent = get_lock_parent(parent);
1206 }
Peter Zijlstra8e182572007-07-19 01:48:54 -07001207
1208 printk("\nother info that might help us debug this:\n\n");
Steven Rostedtf4185812011-04-20 21:41:55 -04001209 print_circular_lock_scenario(check_src, check_tgt,
1210 first_parent);
1211
Peter Zijlstra8e182572007-07-19 01:48:54 -07001212 lockdep_print_held_locks(curr);
1213
1214 printk("\nstack backtrace:\n");
1215 dump_stack();
1216
1217 return 0;
1218}
1219
Ming Leidb0002a2009-07-16 15:44:29 +02001220static noinline int print_bfs_bug(int ret)
1221{
1222 if (!debug_locks_off_graph_unlock())
1223 return 0;
1224
Peter Zijlstra0119fee2011-09-02 01:30:29 +02001225 /*
1226 * Breadth-first-search failed, graph got corrupted?
1227 */
Ming Leidb0002a2009-07-16 15:44:29 +02001228 WARN(1, "lockdep bfs error:%d\n", ret);
1229
1230 return 0;
1231}
1232
Ming Leief681022009-07-16 15:44:29 +02001233static int noop_count(struct lock_list *entry, void *data)
David Miller419ca3f2008-07-29 21:45:03 -07001234{
Ming Leief681022009-07-16 15:44:29 +02001235 (*(unsigned long *)data)++;
1236 return 0;
David Miller419ca3f2008-07-29 21:45:03 -07001237}
1238
Fengguang Wu5216d532013-11-09 00:55:35 +08001239static unsigned long __lockdep_count_forward_deps(struct lock_list *this)
Ming Leief681022009-07-16 15:44:29 +02001240{
1241 unsigned long count = 0;
1242 struct lock_list *uninitialized_var(target_entry);
1243
1244 __bfs_forwards(this, (void *)&count, noop_count, &target_entry);
1245
1246 return count;
1247}
David Miller419ca3f2008-07-29 21:45:03 -07001248unsigned long lockdep_count_forward_deps(struct lock_class *class)
1249{
1250 unsigned long ret, flags;
Ming Leief681022009-07-16 15:44:29 +02001251 struct lock_list this;
1252
1253 this.parent = NULL;
1254 this.class = class;
David Miller419ca3f2008-07-29 21:45:03 -07001255
1256 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001257 arch_spin_lock(&lockdep_lock);
Ming Leief681022009-07-16 15:44:29 +02001258 ret = __lockdep_count_forward_deps(&this);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001259 arch_spin_unlock(&lockdep_lock);
David Miller419ca3f2008-07-29 21:45:03 -07001260 local_irq_restore(flags);
1261
1262 return ret;
1263}
1264
Fengguang Wu5216d532013-11-09 00:55:35 +08001265static unsigned long __lockdep_count_backward_deps(struct lock_list *this)
David Miller419ca3f2008-07-29 21:45:03 -07001266{
Ming Leief681022009-07-16 15:44:29 +02001267 unsigned long count = 0;
1268 struct lock_list *uninitialized_var(target_entry);
David Miller419ca3f2008-07-29 21:45:03 -07001269
Ming Leief681022009-07-16 15:44:29 +02001270 __bfs_backwards(this, (void *)&count, noop_count, &target_entry);
David Miller419ca3f2008-07-29 21:45:03 -07001271
Ming Leief681022009-07-16 15:44:29 +02001272 return count;
David Miller419ca3f2008-07-29 21:45:03 -07001273}
1274
1275unsigned long lockdep_count_backward_deps(struct lock_class *class)
1276{
1277 unsigned long ret, flags;
Ming Leief681022009-07-16 15:44:29 +02001278 struct lock_list this;
1279
1280 this.parent = NULL;
1281 this.class = class;
David Miller419ca3f2008-07-29 21:45:03 -07001282
1283 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001284 arch_spin_lock(&lockdep_lock);
Ming Leief681022009-07-16 15:44:29 +02001285 ret = __lockdep_count_backward_deps(&this);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001286 arch_spin_unlock(&lockdep_lock);
David Miller419ca3f2008-07-29 21:45:03 -07001287 local_irq_restore(flags);
1288
1289 return ret;
1290}
1291
Peter Zijlstra8e182572007-07-19 01:48:54 -07001292/*
1293 * Prove that the dependency graph starting at <entry> can not
1294 * lead to <target>. Print an error and return 0 if it does.
1295 */
1296static noinline int
Ming Leidb0002a2009-07-16 15:44:29 +02001297check_noncircular(struct lock_list *root, struct lock_class *target,
1298 struct lock_list **target_entry)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001299{
Ming Leidb0002a2009-07-16 15:44:29 +02001300 int result;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001301
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02001302 debug_atomic_inc(nr_cyclic_checks);
David Miller419ca3f2008-07-29 21:45:03 -07001303
Ming Leid7aaba12009-07-16 15:44:29 +02001304 result = __bfs_forwards(root, target, class_equal, target_entry);
Ming Leidb0002a2009-07-16 15:44:29 +02001305
1306 return result;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001307}
1308
Steven Rostedt81d68a92008-05-12 21:20:42 +02001309#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001310/*
1311 * Forwards and backwards subgraph searching, for the purposes of
1312 * proving that two subgraphs can be connected by a new dependency
1313 * without creating any illegal irq-safe -> irq-unsafe lock dependency.
1314 */
Peter Zijlstra8e182572007-07-19 01:48:54 -07001315
Ming Leid7aaba12009-07-16 15:44:29 +02001316static inline int usage_match(struct lock_list *entry, void *bit)
1317{
1318 return entry->class->usage_mask & (1 << (enum lock_usage_bit)bit);
1319}
1320
1321
1322
Peter Zijlstra8e182572007-07-19 01:48:54 -07001323/*
1324 * Find a node in the forwards-direction dependency sub-graph starting
Ming Leid7aaba12009-07-16 15:44:29 +02001325 * at @root->class that matches @bit.
Peter Zijlstra8e182572007-07-19 01:48:54 -07001326 *
Ming Leid7aaba12009-07-16 15:44:29 +02001327 * Return 0 if such a node exists in the subgraph, and put that node
1328 * into *@target_entry.
Peter Zijlstra8e182572007-07-19 01:48:54 -07001329 *
Ming Leid7aaba12009-07-16 15:44:29 +02001330 * Return 1 otherwise and keep *@target_entry unchanged.
1331 * Return <0 on error.
Peter Zijlstra8e182572007-07-19 01:48:54 -07001332 */
Ming Leid7aaba12009-07-16 15:44:29 +02001333static int
1334find_usage_forwards(struct lock_list *root, enum lock_usage_bit bit,
1335 struct lock_list **target_entry)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001336{
Ming Leid7aaba12009-07-16 15:44:29 +02001337 int result;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001338
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02001339 debug_atomic_inc(nr_find_usage_forwards_checks);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001340
Ming Leid7aaba12009-07-16 15:44:29 +02001341 result = __bfs_forwards(root, (void *)bit, usage_match, target_entry);
1342
1343 return result;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001344}
1345
1346/*
1347 * Find a node in the backwards-direction dependency sub-graph starting
Ming Leid7aaba12009-07-16 15:44:29 +02001348 * at @root->class that matches @bit.
Peter Zijlstra8e182572007-07-19 01:48:54 -07001349 *
Ming Leid7aaba12009-07-16 15:44:29 +02001350 * Return 0 if such a node exists in the subgraph, and put that node
1351 * into *@target_entry.
Peter Zijlstra8e182572007-07-19 01:48:54 -07001352 *
Ming Leid7aaba12009-07-16 15:44:29 +02001353 * Return 1 otherwise and keep *@target_entry unchanged.
1354 * Return <0 on error.
Peter Zijlstra8e182572007-07-19 01:48:54 -07001355 */
Ming Leid7aaba12009-07-16 15:44:29 +02001356static int
1357find_usage_backwards(struct lock_list *root, enum lock_usage_bit bit,
1358 struct lock_list **target_entry)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001359{
Ming Leid7aaba12009-07-16 15:44:29 +02001360 int result;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001361
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02001362 debug_atomic_inc(nr_find_usage_backwards_checks);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001363
Ming Leid7aaba12009-07-16 15:44:29 +02001364 result = __bfs_backwards(root, (void *)bit, usage_match, target_entry);
Dave Jonesf82b2172008-08-11 09:30:23 +02001365
Ming Leid7aaba12009-07-16 15:44:29 +02001366 return result;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001367}
1368
Peter Zijlstraaf012962009-07-16 15:44:29 +02001369static void print_lock_class_header(struct lock_class *class, int depth)
1370{
1371 int bit;
1372
1373 printk("%*s->", depth, "");
1374 print_lock_name(class);
1375 printk(" ops: %lu", class->ops);
1376 printk(" {\n");
1377
1378 for (bit = 0; bit < LOCK_USAGE_STATES; bit++) {
1379 if (class->usage_mask & (1 << bit)) {
1380 int len = depth;
1381
1382 len += printk("%*s %s", depth, "", usage_str[bit]);
1383 len += printk(" at:\n");
1384 print_stack_trace(class->usage_traces + bit, len);
1385 }
1386 }
1387 printk("%*s }\n", depth, "");
1388
1389 printk("%*s ... key at: ",depth,"");
1390 print_ip_sym((unsigned long)class->key);
1391}
1392
1393/*
1394 * printk the shortest lock dependencies from @start to @end in reverse order:
1395 */
1396static void __used
1397print_shortest_lock_dependencies(struct lock_list *leaf,
1398 struct lock_list *root)
1399{
1400 struct lock_list *entry = leaf;
1401 int depth;
1402
1403 /*compute depth from generated tree by BFS*/
1404 depth = get_lock_depth(leaf);
1405
1406 do {
1407 print_lock_class_header(entry->class, depth);
1408 printk("%*s ... acquired at:\n", depth, "");
1409 print_stack_trace(&entry->trace, 2);
1410 printk("\n");
1411
1412 if (depth == 0 && (entry != root)) {
Steven Rostedt6be8c392011-04-20 21:41:58 -04001413 printk("lockdep:%s bad path found in chain graph\n", __func__);
Peter Zijlstraaf012962009-07-16 15:44:29 +02001414 break;
1415 }
1416
1417 entry = get_lock_parent(entry);
1418 depth--;
1419 } while (entry && (depth >= 0));
1420
1421 return;
1422}
Ming Leid7aaba12009-07-16 15:44:29 +02001423
Steven Rostedt3003eba2011-04-20 21:41:54 -04001424static void
1425print_irq_lock_scenario(struct lock_list *safe_entry,
1426 struct lock_list *unsafe_entry,
Steven Rostedtdad3d742011-04-20 21:41:57 -04001427 struct lock_class *prev_class,
1428 struct lock_class *next_class)
Steven Rostedt3003eba2011-04-20 21:41:54 -04001429{
1430 struct lock_class *safe_class = safe_entry->class;
1431 struct lock_class *unsafe_class = unsafe_entry->class;
Steven Rostedtdad3d742011-04-20 21:41:57 -04001432 struct lock_class *middle_class = prev_class;
Steven Rostedt3003eba2011-04-20 21:41:54 -04001433
1434 if (middle_class == safe_class)
Steven Rostedtdad3d742011-04-20 21:41:57 -04001435 middle_class = next_class;
Steven Rostedt3003eba2011-04-20 21:41:54 -04001436
1437 /*
1438 * A direct locking problem where unsafe_class lock is taken
1439 * directly by safe_class lock, then all we need to show
1440 * is the deadlock scenario, as it is obvious that the
1441 * unsafe lock is taken under the safe lock.
1442 *
1443 * But if there is a chain instead, where the safe lock takes
1444 * an intermediate lock (middle_class) where this lock is
1445 * not the same as the safe lock, then the lock chain is
1446 * used to describe the problem. Otherwise we would need
1447 * to show a different CPU case for each link in the chain
1448 * from the safe_class lock to the unsafe_class lock.
1449 */
1450 if (middle_class != unsafe_class) {
1451 printk("Chain exists of:\n ");
1452 __print_lock_name(safe_class);
1453 printk(" --> ");
1454 __print_lock_name(middle_class);
1455 printk(" --> ");
1456 __print_lock_name(unsafe_class);
1457 printk("\n\n");
1458 }
1459
1460 printk(" Possible interrupt unsafe locking scenario:\n\n");
1461 printk(" CPU0 CPU1\n");
1462 printk(" ---- ----\n");
1463 printk(" lock(");
1464 __print_lock_name(unsafe_class);
1465 printk(");\n");
1466 printk(" local_irq_disable();\n");
1467 printk(" lock(");
1468 __print_lock_name(safe_class);
1469 printk(");\n");
1470 printk(" lock(");
1471 __print_lock_name(middle_class);
1472 printk(");\n");
1473 printk(" <Interrupt>\n");
1474 printk(" lock(");
1475 __print_lock_name(safe_class);
1476 printk(");\n");
1477 printk("\n *** DEADLOCK ***\n\n");
1478}
1479
Peter Zijlstra8e182572007-07-19 01:48:54 -07001480static int
1481print_bad_irq_dependency(struct task_struct *curr,
Ming Lei24208ca2009-07-16 15:44:29 +02001482 struct lock_list *prev_root,
1483 struct lock_list *next_root,
1484 struct lock_list *backwards_entry,
1485 struct lock_list *forwards_entry,
Peter Zijlstra8e182572007-07-19 01:48:54 -07001486 struct held_lock *prev,
1487 struct held_lock *next,
1488 enum lock_usage_bit bit1,
1489 enum lock_usage_bit bit2,
1490 const char *irqclass)
1491{
1492 if (!debug_locks_off_graph_unlock() || debug_locks_silent)
1493 return 0;
1494
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07001495 printk("\n");
1496 printk("======================================================\n");
1497 printk("[ INFO: %s-safe -> %s-unsafe lock order detected ]\n",
Peter Zijlstra8e182572007-07-19 01:48:54 -07001498 irqclass, irqclass);
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01001499 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07001500 printk("------------------------------------------------------\n");
Peter Zijlstra8e182572007-07-19 01:48:54 -07001501 printk("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] is trying to acquire:\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07001502 curr->comm, task_pid_nr(curr),
Peter Zijlstra8e182572007-07-19 01:48:54 -07001503 curr->hardirq_context, hardirq_count() >> HARDIRQ_SHIFT,
1504 curr->softirq_context, softirq_count() >> SOFTIRQ_SHIFT,
1505 curr->hardirqs_enabled,
1506 curr->softirqs_enabled);
1507 print_lock(next);
1508
1509 printk("\nand this task is already holding:\n");
1510 print_lock(prev);
1511 printk("which would create a new lock dependency:\n");
Dave Jonesf82b2172008-08-11 09:30:23 +02001512 print_lock_name(hlock_class(prev));
Peter Zijlstra8e182572007-07-19 01:48:54 -07001513 printk(" ->");
Dave Jonesf82b2172008-08-11 09:30:23 +02001514 print_lock_name(hlock_class(next));
Peter Zijlstra8e182572007-07-19 01:48:54 -07001515 printk("\n");
1516
1517 printk("\nbut this new dependency connects a %s-irq-safe lock:\n",
1518 irqclass);
Ming Lei24208ca2009-07-16 15:44:29 +02001519 print_lock_name(backwards_entry->class);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001520 printk("\n... which became %s-irq-safe at:\n", irqclass);
1521
Ming Lei24208ca2009-07-16 15:44:29 +02001522 print_stack_trace(backwards_entry->class->usage_traces + bit1, 1);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001523
1524 printk("\nto a %s-irq-unsafe lock:\n", irqclass);
Ming Lei24208ca2009-07-16 15:44:29 +02001525 print_lock_name(forwards_entry->class);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001526 printk("\n... which became %s-irq-unsafe at:\n", irqclass);
1527 printk("...");
1528
Ming Lei24208ca2009-07-16 15:44:29 +02001529 print_stack_trace(forwards_entry->class->usage_traces + bit2, 1);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001530
1531 printk("\nother info that might help us debug this:\n\n");
Steven Rostedtdad3d742011-04-20 21:41:57 -04001532 print_irq_lock_scenario(backwards_entry, forwards_entry,
1533 hlock_class(prev), hlock_class(next));
Steven Rostedt3003eba2011-04-20 21:41:54 -04001534
Peter Zijlstra8e182572007-07-19 01:48:54 -07001535 lockdep_print_held_locks(curr);
1536
Ming Lei24208ca2009-07-16 15:44:29 +02001537 printk("\nthe dependencies between %s-irq-safe lock", irqclass);
1538 printk(" and the holding lock:\n");
1539 if (!save_trace(&prev_root->trace))
1540 return 0;
1541 print_shortest_lock_dependencies(backwards_entry, prev_root);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001542
Ming Lei24208ca2009-07-16 15:44:29 +02001543 printk("\nthe dependencies between the lock to be acquired");
1544 printk(" and %s-irq-unsafe lock:\n", irqclass);
1545 if (!save_trace(&next_root->trace))
1546 return 0;
1547 print_shortest_lock_dependencies(forwards_entry, next_root);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001548
1549 printk("\nstack backtrace:\n");
1550 dump_stack();
1551
1552 return 0;
1553}
1554
1555static int
1556check_usage(struct task_struct *curr, struct held_lock *prev,
1557 struct held_lock *next, enum lock_usage_bit bit_backwards,
1558 enum lock_usage_bit bit_forwards, const char *irqclass)
1559{
1560 int ret;
Ming Lei24208ca2009-07-16 15:44:29 +02001561 struct lock_list this, that;
Ming Leid7aaba12009-07-16 15:44:29 +02001562 struct lock_list *uninitialized_var(target_entry);
Ming Lei24208ca2009-07-16 15:44:29 +02001563 struct lock_list *uninitialized_var(target_entry1);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001564
Ming Leid7aaba12009-07-16 15:44:29 +02001565 this.parent = NULL;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001566
Ming Leid7aaba12009-07-16 15:44:29 +02001567 this.class = hlock_class(prev);
1568 ret = find_usage_backwards(&this, bit_backwards, &target_entry);
Peter Zijlstraaf012962009-07-16 15:44:29 +02001569 if (ret < 0)
1570 return print_bfs_bug(ret);
1571 if (ret == 1)
1572 return ret;
Ming Leid7aaba12009-07-16 15:44:29 +02001573
Ming Lei24208ca2009-07-16 15:44:29 +02001574 that.parent = NULL;
1575 that.class = hlock_class(next);
1576 ret = find_usage_forwards(&that, bit_forwards, &target_entry1);
Peter Zijlstraaf012962009-07-16 15:44:29 +02001577 if (ret < 0)
1578 return print_bfs_bug(ret);
1579 if (ret == 1)
1580 return ret;
Ming Leid7aaba12009-07-16 15:44:29 +02001581
Ming Lei24208ca2009-07-16 15:44:29 +02001582 return print_bad_irq_dependency(curr, &this, &that,
1583 target_entry, target_entry1,
1584 prev, next,
Peter Zijlstra8e182572007-07-19 01:48:54 -07001585 bit_backwards, bit_forwards, irqclass);
1586}
1587
Peter Zijlstra4f367d82009-01-22 18:10:42 +01001588static const char *state_names[] = {
1589#define LOCKDEP_STATE(__STATE) \
Peter Zijlstrab4b136f2009-01-29 14:50:36 +01001590 __stringify(__STATE),
Peter Zijlstra4f367d82009-01-22 18:10:42 +01001591#include "lockdep_states.h"
1592#undef LOCKDEP_STATE
1593};
1594
1595static const char *state_rnames[] = {
1596#define LOCKDEP_STATE(__STATE) \
Peter Zijlstrab4b136f2009-01-29 14:50:36 +01001597 __stringify(__STATE)"-READ",
Peter Zijlstra4f367d82009-01-22 18:10:42 +01001598#include "lockdep_states.h"
1599#undef LOCKDEP_STATE
1600};
1601
1602static inline const char *state_name(enum lock_usage_bit bit)
1603{
1604 return (bit & 1) ? state_rnames[bit >> 2] : state_names[bit >> 2];
1605}
1606
1607static int exclusive_bit(int new_bit)
1608{
1609 /*
1610 * USED_IN
1611 * USED_IN_READ
1612 * ENABLED
1613 * ENABLED_READ
1614 *
1615 * bit 0 - write/read
1616 * bit 1 - used_in/enabled
1617 * bit 2+ state
1618 */
1619
1620 int state = new_bit & ~3;
1621 int dir = new_bit & 2;
1622
1623 /*
1624 * keep state, bit flip the direction and strip read.
1625 */
1626 return state | (dir ^ 2);
1627}
1628
1629static int check_irq_usage(struct task_struct *curr, struct held_lock *prev,
1630 struct held_lock *next, enum lock_usage_bit bit)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001631{
1632 /*
1633 * Prove that the new dependency does not connect a hardirq-safe
1634 * lock with a hardirq-unsafe lock - to achieve this we search
1635 * the backwards-subgraph starting at <prev>, and the
1636 * forwards-subgraph starting at <next>:
1637 */
Peter Zijlstra4f367d82009-01-22 18:10:42 +01001638 if (!check_usage(curr, prev, next, bit,
1639 exclusive_bit(bit), state_name(bit)))
Peter Zijlstra8e182572007-07-19 01:48:54 -07001640 return 0;
1641
Peter Zijlstra4f367d82009-01-22 18:10:42 +01001642 bit++; /* _READ */
1643
Peter Zijlstra8e182572007-07-19 01:48:54 -07001644 /*
1645 * Prove that the new dependency does not connect a hardirq-safe-read
1646 * lock with a hardirq-unsafe lock - to achieve this we search
1647 * the backwards-subgraph starting at <prev>, and the
1648 * forwards-subgraph starting at <next>:
1649 */
Peter Zijlstra4f367d82009-01-22 18:10:42 +01001650 if (!check_usage(curr, prev, next, bit,
1651 exclusive_bit(bit), state_name(bit)))
Peter Zijlstra8e182572007-07-19 01:48:54 -07001652 return 0;
1653
Peter Zijlstra4f367d82009-01-22 18:10:42 +01001654 return 1;
1655}
Peter Zijlstra8e182572007-07-19 01:48:54 -07001656
Peter Zijlstra4f367d82009-01-22 18:10:42 +01001657static int
1658check_prev_add_irq(struct task_struct *curr, struct held_lock *prev,
1659 struct held_lock *next)
1660{
1661#define LOCKDEP_STATE(__STATE) \
1662 if (!check_irq_usage(curr, prev, next, LOCK_USED_IN_##__STATE)) \
Nick Piggincf40bd12009-01-21 08:12:39 +01001663 return 0;
Peter Zijlstra4f367d82009-01-22 18:10:42 +01001664#include "lockdep_states.h"
1665#undef LOCKDEP_STATE
Nick Piggincf40bd12009-01-21 08:12:39 +01001666
Peter Zijlstra8e182572007-07-19 01:48:54 -07001667 return 1;
1668}
1669
1670static void inc_chains(void)
1671{
1672 if (current->hardirq_context)
1673 nr_hardirq_chains++;
1674 else {
1675 if (current->softirq_context)
1676 nr_softirq_chains++;
1677 else
1678 nr_process_chains++;
1679 }
1680}
1681
1682#else
1683
1684static inline int
1685check_prev_add_irq(struct task_struct *curr, struct held_lock *prev,
1686 struct held_lock *next)
1687{
1688 return 1;
1689}
1690
1691static inline void inc_chains(void)
1692{
1693 nr_process_chains++;
1694}
1695
1696#endif
1697
Steven Rostedt48702ec2011-04-20 21:41:56 -04001698static void
1699print_deadlock_scenario(struct held_lock *nxt,
1700 struct held_lock *prv)
1701{
1702 struct lock_class *next = hlock_class(nxt);
1703 struct lock_class *prev = hlock_class(prv);
1704
1705 printk(" Possible unsafe locking scenario:\n\n");
1706 printk(" CPU0\n");
1707 printk(" ----\n");
1708 printk(" lock(");
1709 __print_lock_name(prev);
1710 printk(");\n");
1711 printk(" lock(");
1712 __print_lock_name(next);
1713 printk(");\n");
1714 printk("\n *** DEADLOCK ***\n\n");
1715 printk(" May be due to missing lock nesting notation\n\n");
1716}
1717
Peter Zijlstra8e182572007-07-19 01:48:54 -07001718static int
1719print_deadlock_bug(struct task_struct *curr, struct held_lock *prev,
1720 struct held_lock *next)
1721{
1722 if (!debug_locks_off_graph_unlock() || debug_locks_silent)
1723 return 0;
1724
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07001725 printk("\n");
1726 printk("=============================================\n");
1727 printk("[ INFO: possible recursive locking detected ]\n");
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01001728 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07001729 printk("---------------------------------------------\n");
Peter Zijlstra8e182572007-07-19 01:48:54 -07001730 printk("%s/%d is trying to acquire lock:\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07001731 curr->comm, task_pid_nr(curr));
Peter Zijlstra8e182572007-07-19 01:48:54 -07001732 print_lock(next);
1733 printk("\nbut task is already holding lock:\n");
1734 print_lock(prev);
1735
1736 printk("\nother info that might help us debug this:\n");
Steven Rostedt48702ec2011-04-20 21:41:56 -04001737 print_deadlock_scenario(next, prev);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001738 lockdep_print_held_locks(curr);
1739
1740 printk("\nstack backtrace:\n");
1741 dump_stack();
1742
1743 return 0;
1744}
1745
1746/*
1747 * Check whether we are holding such a class already.
1748 *
1749 * (Note that this has to be done separately, because the graph cannot
1750 * detect such classes of deadlocks.)
1751 *
1752 * Returns: 0 on deadlock detected, 1 on OK, 2 on recursive read
1753 */
1754static int
1755check_deadlock(struct task_struct *curr, struct held_lock *next,
1756 struct lockdep_map *next_instance, int read)
1757{
1758 struct held_lock *prev;
Peter Zijlstra7531e2f2008-08-11 09:30:24 +02001759 struct held_lock *nest = NULL;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001760 int i;
1761
1762 for (i = 0; i < curr->lockdep_depth; i++) {
1763 prev = curr->held_locks + i;
Peter Zijlstra7531e2f2008-08-11 09:30:24 +02001764
1765 if (prev->instance == next->nest_lock)
1766 nest = prev;
1767
Dave Jonesf82b2172008-08-11 09:30:23 +02001768 if (hlock_class(prev) != hlock_class(next))
Peter Zijlstra8e182572007-07-19 01:48:54 -07001769 continue;
Peter Zijlstra7531e2f2008-08-11 09:30:24 +02001770
Peter Zijlstra8e182572007-07-19 01:48:54 -07001771 /*
1772 * Allow read-after-read recursion of the same
1773 * lock class (i.e. read_lock(lock)+read_lock(lock)):
1774 */
1775 if ((read == 2) && prev->read)
1776 return 2;
Peter Zijlstra7531e2f2008-08-11 09:30:24 +02001777
1778 /*
1779 * We're holding the nest_lock, which serializes this lock's
1780 * nesting behaviour.
1781 */
1782 if (nest)
1783 return 2;
1784
Peter Zijlstra8e182572007-07-19 01:48:54 -07001785 return print_deadlock_bug(curr, prev, next);
1786 }
1787 return 1;
1788}
1789
1790/*
1791 * There was a chain-cache miss, and we are about to add a new dependency
1792 * to a previous lock. We recursively validate the following rules:
1793 *
1794 * - would the adding of the <prev> -> <next> dependency create a
1795 * circular dependency in the graph? [== circular deadlock]
1796 *
1797 * - does the new prev->next dependency connect any hardirq-safe lock
1798 * (in the full backwards-subgraph starting at <prev>) with any
1799 * hardirq-unsafe lock (in the full forwards-subgraph starting at
1800 * <next>)? [== illegal lock inversion with hardirq contexts]
1801 *
1802 * - does the new prev->next dependency connect any softirq-safe lock
1803 * (in the full backwards-subgraph starting at <prev>) with any
1804 * softirq-unsafe lock (in the full forwards-subgraph starting at
1805 * <next>)? [== illegal lock inversion with softirq contexts]
1806 *
1807 * any of these scenarios could lead to a deadlock.
1808 *
1809 * Then if all the validations pass, we add the forwards and backwards
1810 * dependency.
1811 */
1812static int
1813check_prev_add(struct task_struct *curr, struct held_lock *prev,
Yong Zhang4726f2a2010-05-04 14:16:48 +08001814 struct held_lock *next, int distance, int trylock_loop)
Peter Zijlstra8e182572007-07-19 01:48:54 -07001815{
1816 struct lock_list *entry;
1817 int ret;
Ming Leidb0002a2009-07-16 15:44:29 +02001818 struct lock_list this;
1819 struct lock_list *uninitialized_var(target_entry);
Yong Zhang4726f2a2010-05-04 14:16:48 +08001820 /*
1821 * Static variable, serialized by the graph_lock().
1822 *
1823 * We use this static variable to save the stack trace in case
1824 * we call into this function multiple times due to encountering
1825 * trylocks in the held lock stack.
1826 */
1827 static struct stack_trace trace;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001828
1829 /*
1830 * Prove that the new <prev> -> <next> dependency would not
1831 * create a circular dependency in the graph. (We do this by
1832 * forward-recursing into the graph starting at <next>, and
1833 * checking whether we can reach <prev>.)
1834 *
1835 * We are using global variables to control the recursion, to
1836 * keep the stackframe size of the recursive functions low:
1837 */
Ming Leidb0002a2009-07-16 15:44:29 +02001838 this.class = hlock_class(next);
1839 this.parent = NULL;
1840 ret = check_noncircular(&this, hlock_class(prev), &target_entry);
1841 if (unlikely(!ret))
1842 return print_circular_bug(&this, target_entry, next, prev);
1843 else if (unlikely(ret < 0))
1844 return print_bfs_bug(ret);
Ming Leic94aa5c2009-07-16 15:44:29 +02001845
Peter Zijlstra8e182572007-07-19 01:48:54 -07001846 if (!check_prev_add_irq(curr, prev, next))
1847 return 0;
1848
1849 /*
1850 * For recursive read-locks we do all the dependency checks,
1851 * but we dont store read-triggered dependencies (only
1852 * write-triggered dependencies). This ensures that only the
1853 * write-side dependencies matter, and that if for example a
1854 * write-lock never takes any other locks, then the reads are
1855 * equivalent to a NOP.
1856 */
1857 if (next->read == 2 || prev->read == 2)
1858 return 1;
1859 /*
1860 * Is the <prev> -> <next> dependency already present?
1861 *
1862 * (this may occur even though this is a new chain: consider
1863 * e.g. the L1 -> L2 -> L3 -> L4 and the L5 -> L1 -> L2 -> L3
1864 * chains - the second one will be new, but L1 already has
1865 * L2 added to its dependency list, due to the first chain.)
1866 */
Dave Jonesf82b2172008-08-11 09:30:23 +02001867 list_for_each_entry(entry, &hlock_class(prev)->locks_after, entry) {
1868 if (entry->class == hlock_class(next)) {
Peter Zijlstra8e182572007-07-19 01:48:54 -07001869 if (distance == 1)
1870 entry->distance = 1;
1871 return 2;
1872 }
1873 }
1874
Yong Zhang4726f2a2010-05-04 14:16:48 +08001875 if (!trylock_loop && !save_trace(&trace))
1876 return 0;
1877
Peter Zijlstra8e182572007-07-19 01:48:54 -07001878 /*
1879 * Ok, all validations passed, add the new lock
1880 * to the previous lock's dependency list:
1881 */
Dave Jonesf82b2172008-08-11 09:30:23 +02001882 ret = add_lock_to_list(hlock_class(prev), hlock_class(next),
1883 &hlock_class(prev)->locks_after,
Yong Zhang4726f2a2010-05-04 14:16:48 +08001884 next->acquire_ip, distance, &trace);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001885
1886 if (!ret)
1887 return 0;
1888
Dave Jonesf82b2172008-08-11 09:30:23 +02001889 ret = add_lock_to_list(hlock_class(next), hlock_class(prev),
1890 &hlock_class(next)->locks_before,
Yong Zhang4726f2a2010-05-04 14:16:48 +08001891 next->acquire_ip, distance, &trace);
Peter Zijlstra8e182572007-07-19 01:48:54 -07001892 if (!ret)
1893 return 0;
1894
1895 /*
1896 * Debugging printouts:
1897 */
Dave Jonesf82b2172008-08-11 09:30:23 +02001898 if (verbose(hlock_class(prev)) || verbose(hlock_class(next))) {
Peter Zijlstra8e182572007-07-19 01:48:54 -07001899 graph_unlock();
1900 printk("\n new dependency: ");
Dave Jonesf82b2172008-08-11 09:30:23 +02001901 print_lock_name(hlock_class(prev));
Peter Zijlstra8e182572007-07-19 01:48:54 -07001902 printk(" => ");
Dave Jonesf82b2172008-08-11 09:30:23 +02001903 print_lock_name(hlock_class(next));
Peter Zijlstra8e182572007-07-19 01:48:54 -07001904 printk("\n");
1905 dump_stack();
1906 return graph_lock();
1907 }
1908 return 1;
1909}
1910
1911/*
1912 * Add the dependency to all directly-previous locks that are 'relevant'.
1913 * The ones that are relevant are (in increasing distance from curr):
1914 * all consecutive trylock entries and the final non-trylock entry - or
1915 * the end of this context's lock-chain - whichever comes first.
1916 */
1917static int
1918check_prevs_add(struct task_struct *curr, struct held_lock *next)
1919{
1920 int depth = curr->lockdep_depth;
Yong Zhang4726f2a2010-05-04 14:16:48 +08001921 int trylock_loop = 0;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001922 struct held_lock *hlock;
1923
1924 /*
1925 * Debugging checks.
1926 *
1927 * Depth must not be zero for a non-head lock:
1928 */
1929 if (!depth)
1930 goto out_bug;
1931 /*
1932 * At least two relevant locks must exist for this
1933 * to be a head:
1934 */
1935 if (curr->held_locks[depth].irq_context !=
1936 curr->held_locks[depth-1].irq_context)
1937 goto out_bug;
1938
1939 for (;;) {
1940 int distance = curr->lockdep_depth - depth + 1;
Oleg Nesterov1b5ff812014-01-20 19:20:10 +01001941 hlock = curr->held_locks + depth - 1;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001942 /*
1943 * Only non-recursive-read entries get new dependencies
1944 * added:
1945 */
Oleg Nesterov1b5ff812014-01-20 19:20:10 +01001946 if (hlock->read != 2 && hlock->check) {
Yong Zhang4726f2a2010-05-04 14:16:48 +08001947 if (!check_prev_add(curr, hlock, next,
1948 distance, trylock_loop))
Peter Zijlstra8e182572007-07-19 01:48:54 -07001949 return 0;
1950 /*
1951 * Stop after the first non-trylock entry,
1952 * as non-trylock entries have added their
1953 * own direct dependencies already, so this
1954 * lock is connected to them indirectly:
1955 */
1956 if (!hlock->trylock)
1957 break;
1958 }
1959 depth--;
1960 /*
1961 * End of lock-stack?
1962 */
1963 if (!depth)
1964 break;
1965 /*
1966 * Stop the search if we cross into another context:
1967 */
1968 if (curr->held_locks[depth].irq_context !=
1969 curr->held_locks[depth-1].irq_context)
1970 break;
Yong Zhang4726f2a2010-05-04 14:16:48 +08001971 trylock_loop = 1;
Peter Zijlstra8e182572007-07-19 01:48:54 -07001972 }
1973 return 1;
1974out_bug:
1975 if (!debug_locks_off_graph_unlock())
1976 return 0;
1977
Peter Zijlstra0119fee2011-09-02 01:30:29 +02001978 /*
1979 * Clearly we all shouldn't be here, but since we made it we
1980 * can reliable say we messed up our state. See the above two
1981 * gotos for reasons why we could possibly end up here.
1982 */
Peter Zijlstra8e182572007-07-19 01:48:54 -07001983 WARN_ON(1);
1984
1985 return 0;
1986}
1987
1988unsigned long nr_lock_chains;
Huang, Ying443cd502008-06-20 16:39:21 +08001989struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS];
Huang, Yingcd1a28e2008-06-23 11:20:54 +08001990int nr_chain_hlocks;
Huang, Ying443cd502008-06-20 16:39:21 +08001991static u16 chain_hlocks[MAX_LOCKDEP_CHAIN_HLOCKS];
1992
1993struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i)
1994{
1995 return lock_classes + chain_hlocks[chain->base + i];
1996}
Peter Zijlstra8e182572007-07-19 01:48:54 -07001997
1998/*
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07001999 * Look up a dependency chain. If the key is not present yet then
Jarek Poplawski9e860d02007-05-08 00:30:12 -07002000 * add it and return 1 - in this case the new dependency chain is
2001 * validated. If the key is already hashed, return 0.
2002 * (On return with 1 graph_lock is held.)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002003 */
Huang, Ying443cd502008-06-20 16:39:21 +08002004static inline int lookup_chain_cache(struct task_struct *curr,
2005 struct held_lock *hlock,
2006 u64 chain_key)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002007{
Dave Jonesf82b2172008-08-11 09:30:23 +02002008 struct lock_class *class = hlock_class(hlock);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002009 struct list_head *hash_head = chainhashentry(chain_key);
2010 struct lock_chain *chain;
Hong Zhiguobfaf4af2013-04-04 15:01:21 +08002011 struct held_lock *hlock_curr;
Steven Rostedte0944ee2011-04-20 21:42:00 -04002012 int i, j;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002013
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002014 /*
2015 * We might need to take the graph lock, ensure we've got IRQs
2016 * disabled to make this an IRQ-safe lock.. for recursion reasons
2017 * lockdep won't complain about its own locking errors.
2018 */
Jarek Poplawski381a2292007-02-10 01:44:58 -08002019 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
2020 return 0;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002021 /*
2022 * We can walk it lock-free, because entries only get added
2023 * to the hash:
2024 */
2025 list_for_each_entry(chain, hash_head, entry) {
2026 if (chain->chain_key == chain_key) {
2027cache_hit:
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002028 debug_atomic_inc(chain_lookup_hits);
Ingo Molnar81fc6852006-12-13 00:34:40 -08002029 if (very_verbose(class))
Andrew Morton755cd902006-12-29 16:49:14 -08002030 printk("\nhash chain already cached, key: "
2031 "%016Lx tail class: [%p] %s\n",
2032 (unsigned long long)chain_key,
2033 class->key, class->name);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002034 return 0;
2035 }
2036 }
Ingo Molnar81fc6852006-12-13 00:34:40 -08002037 if (very_verbose(class))
Andrew Morton755cd902006-12-29 16:49:14 -08002038 printk("\nnew hash chain, key: %016Lx tail class: [%p] %s\n",
2039 (unsigned long long)chain_key, class->key, class->name);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002040 /*
2041 * Allocate a new chain entry from the static array, and add
2042 * it to the hash:
2043 */
Ingo Molnar74c383f2006-12-13 00:34:43 -08002044 if (!graph_lock())
2045 return 0;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002046 /*
2047 * We have to walk the chain again locked - to avoid duplicates:
2048 */
2049 list_for_each_entry(chain, hash_head, entry) {
2050 if (chain->chain_key == chain_key) {
Ingo Molnar74c383f2006-12-13 00:34:43 -08002051 graph_unlock();
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002052 goto cache_hit;
2053 }
2054 }
2055 if (unlikely(nr_lock_chains >= MAX_LOCKDEP_CHAINS)) {
Ingo Molnar74c383f2006-12-13 00:34:43 -08002056 if (!debug_locks_off_graph_unlock())
2057 return 0;
2058
Dave Jones2c522832013-04-25 13:40:02 -04002059 print_lockdep_off("BUG: MAX_LOCKDEP_CHAINS too low!");
Peter Zijlstraeedeeab2009-03-18 12:38:47 +01002060 dump_stack();
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002061 return 0;
2062 }
2063 chain = lock_chains + nr_lock_chains++;
2064 chain->chain_key = chain_key;
Huang, Ying443cd502008-06-20 16:39:21 +08002065 chain->irq_context = hlock->irq_context;
2066 /* Find the first held_lock of current chain */
Huang, Ying443cd502008-06-20 16:39:21 +08002067 for (i = curr->lockdep_depth - 1; i >= 0; i--) {
2068 hlock_curr = curr->held_locks + i;
Hong Zhiguobfaf4af2013-04-04 15:01:21 +08002069 if (hlock_curr->irq_context != hlock->irq_context)
Huang, Ying443cd502008-06-20 16:39:21 +08002070 break;
Huang, Ying443cd502008-06-20 16:39:21 +08002071 }
2072 i++;
2073 chain->depth = curr->lockdep_depth + 1 - i;
Steven Rostedte0944ee2011-04-20 21:42:00 -04002074 if (likely(nr_chain_hlocks + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS)) {
2075 chain->base = nr_chain_hlocks;
2076 nr_chain_hlocks += chain->depth;
Huang, Ying443cd502008-06-20 16:39:21 +08002077 for (j = 0; j < chain->depth - 1; j++, i++) {
Dave Jonesf82b2172008-08-11 09:30:23 +02002078 int lock_id = curr->held_locks[i].class_idx - 1;
Huang, Ying443cd502008-06-20 16:39:21 +08002079 chain_hlocks[chain->base + j] = lock_id;
2080 }
2081 chain_hlocks[chain->base + j] = class - lock_classes;
2082 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002083 list_add_tail_rcu(&chain->entry, hash_head);
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002084 debug_atomic_inc(chain_lookup_misses);
Peter Zijlstra8e182572007-07-19 01:48:54 -07002085 inc_chains();
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002086
2087 return 1;
2088}
Peter Zijlstra8e182572007-07-19 01:48:54 -07002089
2090static int validate_chain(struct task_struct *curr, struct lockdep_map *lock,
Johannes Berg4e6045f2007-10-18 23:39:55 -07002091 struct held_lock *hlock, int chain_head, u64 chain_key)
Peter Zijlstra8e182572007-07-19 01:48:54 -07002092{
2093 /*
2094 * Trylock needs to maintain the stack of held locks, but it
2095 * does not add new dependencies, because trylock can be done
2096 * in any order.
2097 *
2098 * We look up the chain_key and do the O(N^2) check and update of
2099 * the dependencies only if this is a new dependency chain.
2100 * (If lookup_chain_cache() returns with 1 it acquires
2101 * graph_lock for us)
2102 */
Oleg Nesterovfb9edbe2014-01-20 19:20:06 +01002103 if (!hlock->trylock && hlock->check &&
Huang, Ying443cd502008-06-20 16:39:21 +08002104 lookup_chain_cache(curr, hlock, chain_key)) {
Peter Zijlstra8e182572007-07-19 01:48:54 -07002105 /*
2106 * Check whether last held lock:
2107 *
2108 * - is irq-safe, if this lock is irq-unsafe
2109 * - is softirq-safe, if this lock is hardirq-unsafe
2110 *
2111 * And check whether the new lock's dependency graph
2112 * could lead back to the previous lock.
2113 *
2114 * any of these scenarios could lead to a deadlock. If
2115 * All validations
2116 */
2117 int ret = check_deadlock(curr, hlock, lock, hlock->read);
2118
2119 if (!ret)
2120 return 0;
2121 /*
2122 * Mark recursive read, as we jump over it when
2123 * building dependencies (just like we jump over
2124 * trylock entries):
2125 */
2126 if (ret == 2)
2127 hlock->read = 2;
2128 /*
2129 * Add dependency only if this lock is not the head
2130 * of the chain, and if it's not a secondary read-lock:
2131 */
2132 if (!chain_head && ret != 2)
2133 if (!check_prevs_add(curr, hlock))
2134 return 0;
2135 graph_unlock();
2136 } else
2137 /* after lookup_chain_cache(): */
2138 if (unlikely(!debug_locks))
2139 return 0;
2140
2141 return 1;
2142}
2143#else
2144static inline int validate_chain(struct task_struct *curr,
2145 struct lockdep_map *lock, struct held_lock *hlock,
Gregory Haskins3aa416b2007-10-11 22:11:11 +02002146 int chain_head, u64 chain_key)
Peter Zijlstra8e182572007-07-19 01:48:54 -07002147{
2148 return 1;
2149}
Peter Zijlstraca58abc2007-07-19 01:48:53 -07002150#endif
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002151
2152/*
2153 * We are building curr_chain_key incrementally, so double-check
2154 * it from scratch, to make sure that it's done correctly:
2155 */
Steven Rostedt1d09daa2008-05-12 21:20:55 +02002156static void check_chain_key(struct task_struct *curr)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002157{
2158#ifdef CONFIG_DEBUG_LOCKDEP
2159 struct held_lock *hlock, *prev_hlock = NULL;
2160 unsigned int i, id;
2161 u64 chain_key = 0;
2162
2163 for (i = 0; i < curr->lockdep_depth; i++) {
2164 hlock = curr->held_locks + i;
2165 if (chain_key != hlock->prev_chain_key) {
2166 debug_locks_off();
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002167 /*
2168 * We got mighty confused, our chain keys don't match
2169 * with what we expect, someone trample on our task state?
2170 */
Arjan van de Ven2df8b1d2008-07-30 12:43:11 -07002171 WARN(1, "hm#1, depth: %u [%u], %016Lx != %016Lx\n",
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002172 curr->lockdep_depth, i,
2173 (unsigned long long)chain_key,
2174 (unsigned long long)hlock->prev_chain_key);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002175 return;
2176 }
Dave Jonesf82b2172008-08-11 09:30:23 +02002177 id = hlock->class_idx - 1;
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002178 /*
2179 * Whoops ran out of static storage again?
2180 */
Jarek Poplawski381a2292007-02-10 01:44:58 -08002181 if (DEBUG_LOCKS_WARN_ON(id >= MAX_LOCKDEP_KEYS))
2182 return;
2183
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002184 if (prev_hlock && (prev_hlock->irq_context !=
2185 hlock->irq_context))
2186 chain_key = 0;
2187 chain_key = iterate_chain_key(chain_key, id);
2188 prev_hlock = hlock;
2189 }
2190 if (chain_key != curr->curr_chain_key) {
2191 debug_locks_off();
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002192 /*
2193 * More smoking hash instead of calculating it, damn see these
2194 * numbers float.. I bet that a pink elephant stepped on my memory.
2195 */
Arjan van de Ven2df8b1d2008-07-30 12:43:11 -07002196 WARN(1, "hm#2, depth: %u [%u], %016Lx != %016Lx\n",
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002197 curr->lockdep_depth, i,
2198 (unsigned long long)chain_key,
2199 (unsigned long long)curr->curr_chain_key);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002200 }
2201#endif
2202}
2203
Steven Rostedt282b5c22011-04-20 21:41:59 -04002204static void
2205print_usage_bug_scenario(struct held_lock *lock)
2206{
2207 struct lock_class *class = hlock_class(lock);
2208
2209 printk(" Possible unsafe locking scenario:\n\n");
2210 printk(" CPU0\n");
2211 printk(" ----\n");
2212 printk(" lock(");
2213 __print_lock_name(class);
2214 printk(");\n");
2215 printk(" <Interrupt>\n");
2216 printk(" lock(");
2217 __print_lock_name(class);
2218 printk(");\n");
2219 printk("\n *** DEADLOCK ***\n\n");
2220}
2221
Peter Zijlstra8e182572007-07-19 01:48:54 -07002222static int
2223print_usage_bug(struct task_struct *curr, struct held_lock *this,
2224 enum lock_usage_bit prev_bit, enum lock_usage_bit new_bit)
2225{
2226 if (!debug_locks_off_graph_unlock() || debug_locks_silent)
2227 return 0;
2228
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07002229 printk("\n");
2230 printk("=================================\n");
2231 printk("[ INFO: inconsistent lock state ]\n");
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01002232 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07002233 printk("---------------------------------\n");
Peter Zijlstra8e182572007-07-19 01:48:54 -07002234
2235 printk("inconsistent {%s} -> {%s} usage.\n",
2236 usage_str[prev_bit], usage_str[new_bit]);
2237
2238 printk("%s/%d [HC%u[%lu]:SC%u[%lu]:HE%u:SE%u] takes:\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07002239 curr->comm, task_pid_nr(curr),
Peter Zijlstra8e182572007-07-19 01:48:54 -07002240 trace_hardirq_context(curr), hardirq_count() >> HARDIRQ_SHIFT,
2241 trace_softirq_context(curr), softirq_count() >> SOFTIRQ_SHIFT,
2242 trace_hardirqs_enabled(curr),
2243 trace_softirqs_enabled(curr));
2244 print_lock(this);
2245
2246 printk("{%s} state was registered at:\n", usage_str[prev_bit]);
Dave Jonesf82b2172008-08-11 09:30:23 +02002247 print_stack_trace(hlock_class(this)->usage_traces + prev_bit, 1);
Peter Zijlstra8e182572007-07-19 01:48:54 -07002248
2249 print_irqtrace_events(curr);
2250 printk("\nother info that might help us debug this:\n");
Steven Rostedt282b5c22011-04-20 21:41:59 -04002251 print_usage_bug_scenario(this);
2252
Peter Zijlstra8e182572007-07-19 01:48:54 -07002253 lockdep_print_held_locks(curr);
2254
2255 printk("\nstack backtrace:\n");
2256 dump_stack();
2257
2258 return 0;
2259}
2260
2261/*
2262 * Print out an error if an invalid bit is set:
2263 */
2264static inline int
2265valid_state(struct task_struct *curr, struct held_lock *this,
2266 enum lock_usage_bit new_bit, enum lock_usage_bit bad_bit)
2267{
Dave Jonesf82b2172008-08-11 09:30:23 +02002268 if (unlikely(hlock_class(this)->usage_mask & (1 << bad_bit)))
Peter Zijlstra8e182572007-07-19 01:48:54 -07002269 return print_usage_bug(curr, this, bad_bit, new_bit);
2270 return 1;
2271}
2272
2273static int mark_lock(struct task_struct *curr, struct held_lock *this,
2274 enum lock_usage_bit new_bit);
2275
Steven Rostedt81d68a92008-05-12 21:20:42 +02002276#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002277
2278/*
2279 * print irq inversion bug:
2280 */
2281static int
Ming Lei24208ca2009-07-16 15:44:29 +02002282print_irq_inversion_bug(struct task_struct *curr,
2283 struct lock_list *root, struct lock_list *other,
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002284 struct held_lock *this, int forwards,
2285 const char *irqclass)
2286{
Steven Rostedtdad3d742011-04-20 21:41:57 -04002287 struct lock_list *entry = other;
2288 struct lock_list *middle = NULL;
2289 int depth;
2290
Ingo Molnar74c383f2006-12-13 00:34:43 -08002291 if (!debug_locks_off_graph_unlock() || debug_locks_silent)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002292 return 0;
2293
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07002294 printk("\n");
2295 printk("=========================================================\n");
2296 printk("[ INFO: possible irq lock inversion dependency detected ]\n");
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01002297 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07002298 printk("---------------------------------------------------------\n");
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002299 printk("%s/%d just changed the state of lock:\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07002300 curr->comm, task_pid_nr(curr));
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002301 print_lock(this);
2302 if (forwards)
Peter Zijlstra26575e22009-03-04 14:53:24 +01002303 printk("but this lock took another, %s-unsafe lock in the past:\n", irqclass);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002304 else
Peter Zijlstra26575e22009-03-04 14:53:24 +01002305 printk("but this lock was taken by another, %s-safe lock in the past:\n", irqclass);
Ming Lei24208ca2009-07-16 15:44:29 +02002306 print_lock_name(other->class);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002307 printk("\n\nand interrupts could create inverse lock ordering between them.\n\n");
2308
2309 printk("\nother info that might help us debug this:\n");
Steven Rostedtdad3d742011-04-20 21:41:57 -04002310
2311 /* Find a middle lock (if one exists) */
2312 depth = get_lock_depth(other);
2313 do {
2314 if (depth == 0 && (entry != root)) {
2315 printk("lockdep:%s bad path found in chain graph\n", __func__);
2316 break;
2317 }
2318 middle = entry;
2319 entry = get_lock_parent(entry);
2320 depth--;
2321 } while (entry && entry != root && (depth >= 0));
2322 if (forwards)
2323 print_irq_lock_scenario(root, other,
2324 middle ? middle->class : root->class, other->class);
2325 else
2326 print_irq_lock_scenario(other, root,
2327 middle ? middle->class : other->class, root->class);
2328
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002329 lockdep_print_held_locks(curr);
2330
Ming Lei24208ca2009-07-16 15:44:29 +02002331 printk("\nthe shortest dependencies between 2nd lock and 1st lock:\n");
2332 if (!save_trace(&root->trace))
2333 return 0;
2334 print_shortest_lock_dependencies(other, root);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002335
2336 printk("\nstack backtrace:\n");
2337 dump_stack();
2338
2339 return 0;
2340}
2341
2342/*
2343 * Prove that in the forwards-direction subgraph starting at <this>
2344 * there is no lock matching <mask>:
2345 */
2346static int
2347check_usage_forwards(struct task_struct *curr, struct held_lock *this,
2348 enum lock_usage_bit bit, const char *irqclass)
2349{
2350 int ret;
Ming Leid7aaba12009-07-16 15:44:29 +02002351 struct lock_list root;
2352 struct lock_list *uninitialized_var(target_entry);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002353
Ming Leid7aaba12009-07-16 15:44:29 +02002354 root.parent = NULL;
2355 root.class = hlock_class(this);
2356 ret = find_usage_forwards(&root, bit, &target_entry);
Peter Zijlstraaf012962009-07-16 15:44:29 +02002357 if (ret < 0)
2358 return print_bfs_bug(ret);
2359 if (ret == 1)
2360 return ret;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002361
Ming Lei24208ca2009-07-16 15:44:29 +02002362 return print_irq_inversion_bug(curr, &root, target_entry,
Ming Leid7aaba12009-07-16 15:44:29 +02002363 this, 1, irqclass);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002364}
2365
2366/*
2367 * Prove that in the backwards-direction subgraph starting at <this>
2368 * there is no lock matching <mask>:
2369 */
2370static int
2371check_usage_backwards(struct task_struct *curr, struct held_lock *this,
2372 enum lock_usage_bit bit, const char *irqclass)
2373{
2374 int ret;
Ming Leid7aaba12009-07-16 15:44:29 +02002375 struct lock_list root;
2376 struct lock_list *uninitialized_var(target_entry);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002377
Ming Leid7aaba12009-07-16 15:44:29 +02002378 root.parent = NULL;
2379 root.class = hlock_class(this);
2380 ret = find_usage_backwards(&root, bit, &target_entry);
Peter Zijlstraaf012962009-07-16 15:44:29 +02002381 if (ret < 0)
2382 return print_bfs_bug(ret);
2383 if (ret == 1)
2384 return ret;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002385
Ming Lei24208ca2009-07-16 15:44:29 +02002386 return print_irq_inversion_bug(curr, &root, target_entry,
Oleg Nesterov48d50672010-01-26 19:16:41 +01002387 this, 0, irqclass);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002388}
2389
Ingo Molnar3117df02006-12-13 00:34:43 -08002390void print_irqtrace_events(struct task_struct *curr)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002391{
2392 printk("irq event stamp: %u\n", curr->irq_events);
2393 printk("hardirqs last enabled at (%u): ", curr->hardirq_enable_event);
2394 print_ip_sym(curr->hardirq_enable_ip);
2395 printk("hardirqs last disabled at (%u): ", curr->hardirq_disable_event);
2396 print_ip_sym(curr->hardirq_disable_ip);
2397 printk("softirqs last enabled at (%u): ", curr->softirq_enable_event);
2398 print_ip_sym(curr->softirq_enable_ip);
2399 printk("softirqs last disabled at (%u): ", curr->softirq_disable_event);
2400 print_ip_sym(curr->softirq_disable_ip);
2401}
2402
Peter Zijlstracd953022009-01-22 16:38:21 +01002403static int HARDIRQ_verbose(struct lock_class *class)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002404{
Peter Zijlstra8e182572007-07-19 01:48:54 -07002405#if HARDIRQ_VERBOSE
2406 return class_filter(class);
2407#endif
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002408 return 0;
2409}
2410
Peter Zijlstracd953022009-01-22 16:38:21 +01002411static int SOFTIRQ_verbose(struct lock_class *class)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002412{
Peter Zijlstra8e182572007-07-19 01:48:54 -07002413#if SOFTIRQ_VERBOSE
2414 return class_filter(class);
2415#endif
2416 return 0;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002417}
2418
Peter Zijlstracd953022009-01-22 16:38:21 +01002419static int RECLAIM_FS_verbose(struct lock_class *class)
Nick Piggincf40bd12009-01-21 08:12:39 +01002420{
2421#if RECLAIM_VERBOSE
2422 return class_filter(class);
2423#endif
2424 return 0;
2425}
2426
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002427#define STRICT_READ_CHECKS 1
2428
Peter Zijlstracd953022009-01-22 16:38:21 +01002429static int (*state_verbose_f[])(struct lock_class *class) = {
2430#define LOCKDEP_STATE(__STATE) \
2431 __STATE##_verbose,
2432#include "lockdep_states.h"
2433#undef LOCKDEP_STATE
2434};
2435
2436static inline int state_verbose(enum lock_usage_bit bit,
2437 struct lock_class *class)
2438{
2439 return state_verbose_f[bit >> 2](class);
2440}
2441
Peter Zijlstra42c50d52009-01-22 16:58:16 +01002442typedef int (*check_usage_f)(struct task_struct *, struct held_lock *,
2443 enum lock_usage_bit bit, const char *name);
2444
Peter Zijlstra6a6904d2009-01-22 16:07:44 +01002445static int
Peter Zijlstra1c21f142009-03-04 13:51:13 +01002446mark_lock_irq(struct task_struct *curr, struct held_lock *this,
2447 enum lock_usage_bit new_bit)
Peter Zijlstra6a6904d2009-01-22 16:07:44 +01002448{
Peter Zijlstraf9892092009-01-22 16:09:59 +01002449 int excl_bit = exclusive_bit(new_bit);
Peter Zijlstra9d3651a2009-01-22 17:18:32 +01002450 int read = new_bit & 1;
Peter Zijlstra42c50d52009-01-22 16:58:16 +01002451 int dir = new_bit & 2;
2452
Peter Zijlstra38aa2712009-01-27 14:53:50 +01002453 /*
2454 * mark USED_IN has to look forwards -- to ensure no dependency
2455 * has ENABLED state, which would allow recursion deadlocks.
2456 *
2457 * mark ENABLED has to look backwards -- to ensure no dependee
2458 * has USED_IN state, which, again, would allow recursion deadlocks.
2459 */
Peter Zijlstra42c50d52009-01-22 16:58:16 +01002460 check_usage_f usage = dir ?
2461 check_usage_backwards : check_usage_forwards;
Peter Zijlstraf9892092009-01-22 16:09:59 +01002462
Peter Zijlstra38aa2712009-01-27 14:53:50 +01002463 /*
2464 * Validate that this particular lock does not have conflicting
2465 * usage states.
2466 */
Peter Zijlstra6a6904d2009-01-22 16:07:44 +01002467 if (!valid_state(curr, this, new_bit, excl_bit))
2468 return 0;
Peter Zijlstra9d3651a2009-01-22 17:18:32 +01002469
Peter Zijlstra38aa2712009-01-27 14:53:50 +01002470 /*
2471 * Validate that the lock dependencies don't have conflicting usage
2472 * states.
2473 */
2474 if ((!read || !dir || STRICT_READ_CHECKS) &&
Peter Zijlstra1c21f142009-03-04 13:51:13 +01002475 !usage(curr, this, excl_bit, state_name(new_bit & ~1)))
Peter Zijlstra6a6904d2009-01-22 16:07:44 +01002476 return 0;
Peter Zijlstra780e8202009-01-22 16:51:29 +01002477
Peter Zijlstra38aa2712009-01-27 14:53:50 +01002478 /*
2479 * Check for read in write conflicts
2480 */
2481 if (!read) {
2482 if (!valid_state(curr, this, new_bit, excl_bit + 1))
2483 return 0;
2484
2485 if (STRICT_READ_CHECKS &&
Peter Zijlstra4f367d82009-01-22 18:10:42 +01002486 !usage(curr, this, excl_bit + 1,
2487 state_name(new_bit + 1)))
Peter Zijlstra38aa2712009-01-27 14:53:50 +01002488 return 0;
2489 }
Peter Zijlstra780e8202009-01-22 16:51:29 +01002490
Peter Zijlstracd953022009-01-22 16:38:21 +01002491 if (state_verbose(new_bit, hlock_class(this)))
Peter Zijlstra6a6904d2009-01-22 16:07:44 +01002492 return 2;
2493
2494 return 1;
2495}
2496
Nick Piggincf40bd12009-01-21 08:12:39 +01002497enum mark_type {
Peter Zijlstra36bfb9b2009-01-22 14:12:41 +01002498#define LOCKDEP_STATE(__STATE) __STATE,
2499#include "lockdep_states.h"
2500#undef LOCKDEP_STATE
Nick Piggincf40bd12009-01-21 08:12:39 +01002501};
2502
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002503/*
2504 * Mark all held locks with a usage bit:
2505 */
Steven Rostedt1d09daa2008-05-12 21:20:55 +02002506static int
Nick Piggincf40bd12009-01-21 08:12:39 +01002507mark_held_locks(struct task_struct *curr, enum mark_type mark)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002508{
2509 enum lock_usage_bit usage_bit;
2510 struct held_lock *hlock;
2511 int i;
2512
2513 for (i = 0; i < curr->lockdep_depth; i++) {
2514 hlock = curr->held_locks + i;
2515
Peter Zijlstracf2ad4d2009-01-27 13:58:08 +01002516 usage_bit = 2 + (mark << 2); /* ENABLED */
2517 if (hlock->read)
2518 usage_bit += 1; /* READ */
2519
2520 BUG_ON(usage_bit >= LOCK_USAGE_STATES);
Nick Piggincf40bd12009-01-21 08:12:39 +01002521
Oleg Nesterov34d0ed52014-01-20 19:20:13 +01002522 if (!hlock->check)
Peter Zijlstraefbe2ee2011-07-07 11:39:45 +02002523 continue;
2524
Jarek Poplawski4ff773bb2007-05-08 00:31:00 -07002525 if (!mark_lock(curr, hlock, usage_bit))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002526 return 0;
2527 }
2528
2529 return 1;
2530}
2531
2532/*
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002533 * Hardirqs will be enabled:
2534 */
Peter Zijlstradd4e5d32011-06-21 17:17:27 +02002535static void __trace_hardirqs_on_caller(unsigned long ip)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002536{
2537 struct task_struct *curr = current;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002538
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002539 /* we'll do an OFF -> ON transition: */
2540 curr->hardirqs_enabled = 1;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002541
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002542 /*
2543 * We are going to turn hardirqs on, so set the
2544 * usage bit for all held locks:
2545 */
Nick Piggincf40bd12009-01-21 08:12:39 +01002546 if (!mark_held_locks(curr, HARDIRQ))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002547 return;
2548 /*
2549 * If we have softirqs enabled, then set the usage
2550 * bit for all held locks. (disabled hardirqs prevented
2551 * this bit from being set before)
2552 */
2553 if (curr->softirqs_enabled)
Nick Piggincf40bd12009-01-21 08:12:39 +01002554 if (!mark_held_locks(curr, SOFTIRQ))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002555 return;
2556
2557 curr->hardirq_enable_ip = ip;
2558 curr->hardirq_enable_event = ++curr->irq_events;
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002559 debug_atomic_inc(hardirqs_on_events);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002560}
Peter Zijlstradd4e5d32011-06-21 17:17:27 +02002561
Andi Kleenb35f8302014-02-08 08:52:02 +01002562__visible void trace_hardirqs_on_caller(unsigned long ip)
Peter Zijlstradd4e5d32011-06-21 17:17:27 +02002563{
2564 time_hardirqs_on(CALLER_ADDR0, ip);
2565
2566 if (unlikely(!debug_locks || current->lockdep_recursion))
2567 return;
2568
Peter Zijlstra7d36b262011-07-26 13:13:44 +02002569 if (unlikely(current->hardirqs_enabled)) {
2570 /*
2571 * Neither irq nor preemption are disabled here
2572 * so this is racy by nature but losing one hit
2573 * in a stat is not a big deal.
2574 */
2575 __debug_atomic_inc(redundant_hardirqs_on);
2576 return;
2577 }
2578
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002579 /*
2580 * We're enabling irqs and according to our state above irqs weren't
2581 * already enabled, yet we find the hardware thinks they are in fact
2582 * enabled.. someone messed up their IRQ state tracing.
2583 */
Peter Zijlstradd4e5d32011-06-21 17:17:27 +02002584 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
2585 return;
2586
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002587 /*
2588 * See the fine text that goes along with this variable definition.
2589 */
Peter Zijlstra7d36b262011-07-26 13:13:44 +02002590 if (DEBUG_LOCKS_WARN_ON(unlikely(early_boot_irqs_disabled)))
2591 return;
2592
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002593 /*
2594 * Can't allow enabling interrupts while in an interrupt handler,
2595 * that's general bad form and such. Recursion, limited stack etc..
2596 */
Peter Zijlstra7d36b262011-07-26 13:13:44 +02002597 if (DEBUG_LOCKS_WARN_ON(current->hardirq_context))
2598 return;
2599
Peter Zijlstradd4e5d32011-06-21 17:17:27 +02002600 current->lockdep_recursion = 1;
2601 __trace_hardirqs_on_caller(ip);
2602 current->lockdep_recursion = 0;
2603}
Steven Rostedt81d68a92008-05-12 21:20:42 +02002604EXPORT_SYMBOL(trace_hardirqs_on_caller);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002605
Steven Rostedt1d09daa2008-05-12 21:20:55 +02002606void trace_hardirqs_on(void)
Steven Rostedt81d68a92008-05-12 21:20:42 +02002607{
2608 trace_hardirqs_on_caller(CALLER_ADDR0);
2609}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002610EXPORT_SYMBOL(trace_hardirqs_on);
2611
2612/*
2613 * Hardirqs were disabled:
2614 */
Andi Kleenb35f8302014-02-08 08:52:02 +01002615__visible void trace_hardirqs_off_caller(unsigned long ip)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002616{
2617 struct task_struct *curr = current;
2618
Heiko Carstens6afe40b2008-10-28 11:14:58 +01002619 time_hardirqs_off(CALLER_ADDR0, ip);
Steven Rostedt81d68a92008-05-12 21:20:42 +02002620
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002621 if (unlikely(!debug_locks || current->lockdep_recursion))
2622 return;
2623
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002624 /*
2625 * So we're supposed to get called after you mask local IRQs, but for
2626 * some reason the hardware doesn't quite think you did a proper job.
2627 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002628 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
2629 return;
2630
2631 if (curr->hardirqs_enabled) {
2632 /*
2633 * We have done an ON -> OFF transition:
2634 */
2635 curr->hardirqs_enabled = 0;
Heiko Carstens6afe40b2008-10-28 11:14:58 +01002636 curr->hardirq_disable_ip = ip;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002637 curr->hardirq_disable_event = ++curr->irq_events;
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002638 debug_atomic_inc(hardirqs_off_events);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002639 } else
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002640 debug_atomic_inc(redundant_hardirqs_off);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002641}
Steven Rostedt81d68a92008-05-12 21:20:42 +02002642EXPORT_SYMBOL(trace_hardirqs_off_caller);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002643
Steven Rostedt1d09daa2008-05-12 21:20:55 +02002644void trace_hardirqs_off(void)
Steven Rostedt81d68a92008-05-12 21:20:42 +02002645{
2646 trace_hardirqs_off_caller(CALLER_ADDR0);
2647}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002648EXPORT_SYMBOL(trace_hardirqs_off);
2649
2650/*
2651 * Softirqs will be enabled:
2652 */
2653void trace_softirqs_on(unsigned long ip)
2654{
2655 struct task_struct *curr = current;
2656
Peter Zijlstradd4e5d32011-06-21 17:17:27 +02002657 if (unlikely(!debug_locks || current->lockdep_recursion))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002658 return;
2659
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002660 /*
2661 * We fancy IRQs being disabled here, see softirq.c, avoids
2662 * funny state and nesting things.
2663 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002664 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
2665 return;
2666
2667 if (curr->softirqs_enabled) {
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002668 debug_atomic_inc(redundant_softirqs_on);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002669 return;
2670 }
2671
Peter Zijlstradd4e5d32011-06-21 17:17:27 +02002672 current->lockdep_recursion = 1;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002673 /*
2674 * We'll do an OFF -> ON transition:
2675 */
2676 curr->softirqs_enabled = 1;
2677 curr->softirq_enable_ip = ip;
2678 curr->softirq_enable_event = ++curr->irq_events;
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002679 debug_atomic_inc(softirqs_on_events);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002680 /*
2681 * We are going to turn softirqs on, so set the
2682 * usage bit for all held locks, if hardirqs are
2683 * enabled too:
2684 */
2685 if (curr->hardirqs_enabled)
Nick Piggincf40bd12009-01-21 08:12:39 +01002686 mark_held_locks(curr, SOFTIRQ);
Peter Zijlstradd4e5d32011-06-21 17:17:27 +02002687 current->lockdep_recursion = 0;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002688}
2689
2690/*
2691 * Softirqs were disabled:
2692 */
2693void trace_softirqs_off(unsigned long ip)
2694{
2695 struct task_struct *curr = current;
2696
Peter Zijlstradd4e5d32011-06-21 17:17:27 +02002697 if (unlikely(!debug_locks || current->lockdep_recursion))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002698 return;
2699
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002700 /*
2701 * We fancy IRQs being disabled here, see softirq.c
2702 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002703 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
2704 return;
2705
2706 if (curr->softirqs_enabled) {
2707 /*
2708 * We have done an ON -> OFF transition:
2709 */
2710 curr->softirqs_enabled = 0;
2711 curr->softirq_disable_ip = ip;
2712 curr->softirq_disable_event = ++curr->irq_events;
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002713 debug_atomic_inc(softirqs_off_events);
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002714 /*
2715 * Whoops, we wanted softirqs off, so why aren't they?
2716 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002717 DEBUG_LOCKS_WARN_ON(!softirq_count());
2718 } else
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002719 debug_atomic_inc(redundant_softirqs_off);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002720}
2721
Peter Zijlstra2f850182009-03-20 11:13:20 +01002722static void __lockdep_trace_alloc(gfp_t gfp_mask, unsigned long flags)
Nick Piggincf40bd12009-01-21 08:12:39 +01002723{
2724 struct task_struct *curr = current;
2725
2726 if (unlikely(!debug_locks))
2727 return;
2728
2729 /* no reclaim without waiting on it */
2730 if (!(gfp_mask & __GFP_WAIT))
2731 return;
2732
2733 /* this guy won't enter reclaim */
2734 if ((curr->flags & PF_MEMALLOC) && !(gfp_mask & __GFP_NOMEMALLOC))
2735 return;
2736
2737 /* We're only interested __GFP_FS allocations for now */
2738 if (!(gfp_mask & __GFP_FS))
2739 return;
2740
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002741 /*
2742 * Oi! Can't be having __GFP_FS allocations with IRQs disabled.
2743 */
Peter Zijlstra2f850182009-03-20 11:13:20 +01002744 if (DEBUG_LOCKS_WARN_ON(irqs_disabled_flags(flags)))
Nick Piggincf40bd12009-01-21 08:12:39 +01002745 return;
2746
2747 mark_held_locks(curr, RECLAIM_FS);
2748}
2749
Peter Zijlstra2f850182009-03-20 11:13:20 +01002750static void check_flags(unsigned long flags);
2751
2752void lockdep_trace_alloc(gfp_t gfp_mask)
2753{
2754 unsigned long flags;
2755
2756 if (unlikely(current->lockdep_recursion))
2757 return;
2758
2759 raw_local_irq_save(flags);
2760 check_flags(flags);
2761 current->lockdep_recursion = 1;
2762 __lockdep_trace_alloc(gfp_mask, flags);
2763 current->lockdep_recursion = 0;
2764 raw_local_irq_restore(flags);
2765}
2766
Peter Zijlstra8e182572007-07-19 01:48:54 -07002767static int mark_irqflags(struct task_struct *curr, struct held_lock *hlock)
2768{
2769 /*
2770 * If non-trylock use in a hardirq or softirq context, then
2771 * mark the lock as used in these contexts:
2772 */
2773 if (!hlock->trylock) {
2774 if (hlock->read) {
2775 if (curr->hardirq_context)
2776 if (!mark_lock(curr, hlock,
2777 LOCK_USED_IN_HARDIRQ_READ))
2778 return 0;
2779 if (curr->softirq_context)
2780 if (!mark_lock(curr, hlock,
2781 LOCK_USED_IN_SOFTIRQ_READ))
2782 return 0;
2783 } else {
2784 if (curr->hardirq_context)
2785 if (!mark_lock(curr, hlock, LOCK_USED_IN_HARDIRQ))
2786 return 0;
2787 if (curr->softirq_context)
2788 if (!mark_lock(curr, hlock, LOCK_USED_IN_SOFTIRQ))
2789 return 0;
2790 }
2791 }
2792 if (!hlock->hardirqs_off) {
2793 if (hlock->read) {
2794 if (!mark_lock(curr, hlock,
Peter Zijlstra4fc95e82009-01-22 13:10:52 +01002795 LOCK_ENABLED_HARDIRQ_READ))
Peter Zijlstra8e182572007-07-19 01:48:54 -07002796 return 0;
2797 if (curr->softirqs_enabled)
2798 if (!mark_lock(curr, hlock,
Peter Zijlstra4fc95e82009-01-22 13:10:52 +01002799 LOCK_ENABLED_SOFTIRQ_READ))
Peter Zijlstra8e182572007-07-19 01:48:54 -07002800 return 0;
2801 } else {
2802 if (!mark_lock(curr, hlock,
Peter Zijlstra4fc95e82009-01-22 13:10:52 +01002803 LOCK_ENABLED_HARDIRQ))
Peter Zijlstra8e182572007-07-19 01:48:54 -07002804 return 0;
2805 if (curr->softirqs_enabled)
2806 if (!mark_lock(curr, hlock,
Peter Zijlstra4fc95e82009-01-22 13:10:52 +01002807 LOCK_ENABLED_SOFTIRQ))
Peter Zijlstra8e182572007-07-19 01:48:54 -07002808 return 0;
2809 }
2810 }
2811
Nick Piggincf40bd12009-01-21 08:12:39 +01002812 /*
2813 * We reuse the irq context infrastructure more broadly as a general
2814 * context checking code. This tests GFP_FS recursion (a lock taken
2815 * during reclaim for a GFP_FS allocation is held over a GFP_FS
2816 * allocation).
2817 */
2818 if (!hlock->trylock && (curr->lockdep_reclaim_gfp & __GFP_FS)) {
2819 if (hlock->read) {
2820 if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS_READ))
2821 return 0;
2822 } else {
2823 if (!mark_lock(curr, hlock, LOCK_USED_IN_RECLAIM_FS))
2824 return 0;
2825 }
2826 }
2827
Peter Zijlstra8e182572007-07-19 01:48:54 -07002828 return 1;
2829}
2830
2831static int separate_irq_context(struct task_struct *curr,
2832 struct held_lock *hlock)
2833{
2834 unsigned int depth = curr->lockdep_depth;
2835
2836 /*
2837 * Keep track of points where we cross into an interrupt context:
2838 */
2839 hlock->irq_context = 2*(curr->hardirq_context ? 1 : 0) +
2840 curr->softirq_context;
2841 if (depth) {
2842 struct held_lock *prev_hlock;
2843
2844 prev_hlock = curr->held_locks + depth-1;
2845 /*
2846 * If we cross into another context, reset the
2847 * hash key (this also prevents the checking and the
2848 * adding of the dependency to 'prev'):
2849 */
2850 if (prev_hlock->irq_context != hlock->irq_context)
2851 return 1;
2852 }
2853 return 0;
2854}
2855
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002856#else /* defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING) */
Peter Zijlstra8e182572007-07-19 01:48:54 -07002857
2858static inline
2859int mark_lock_irq(struct task_struct *curr, struct held_lock *this,
2860 enum lock_usage_bit new_bit)
2861{
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002862 WARN_ON(1); /* Impossible innit? when we don't have TRACE_IRQFLAG */
Peter Zijlstra8e182572007-07-19 01:48:54 -07002863 return 1;
2864}
2865
2866static inline int mark_irqflags(struct task_struct *curr,
2867 struct held_lock *hlock)
2868{
2869 return 1;
2870}
2871
2872static inline int separate_irq_context(struct task_struct *curr,
2873 struct held_lock *hlock)
2874{
2875 return 0;
2876}
2877
Peter Zijlstra868a23a2009-02-15 00:25:21 +01002878void lockdep_trace_alloc(gfp_t gfp_mask)
2879{
2880}
2881
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002882#endif /* defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING) */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002883
2884/*
Peter Zijlstra8e182572007-07-19 01:48:54 -07002885 * Mark a lock with a usage bit, and validate the state transition:
2886 */
Steven Rostedt1d09daa2008-05-12 21:20:55 +02002887static int mark_lock(struct task_struct *curr, struct held_lock *this,
Steven Rostedt0764d232008-05-12 21:20:44 +02002888 enum lock_usage_bit new_bit)
Peter Zijlstra8e182572007-07-19 01:48:54 -07002889{
2890 unsigned int new_mask = 1 << new_bit, ret = 1;
2891
2892 /*
2893 * If already set then do not dirty the cacheline,
2894 * nor do any checks:
2895 */
Dave Jonesf82b2172008-08-11 09:30:23 +02002896 if (likely(hlock_class(this)->usage_mask & new_mask))
Peter Zijlstra8e182572007-07-19 01:48:54 -07002897 return 1;
2898
2899 if (!graph_lock())
2900 return 0;
2901 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002902 * Make sure we didn't race:
Peter Zijlstra8e182572007-07-19 01:48:54 -07002903 */
Dave Jonesf82b2172008-08-11 09:30:23 +02002904 if (unlikely(hlock_class(this)->usage_mask & new_mask)) {
Peter Zijlstra8e182572007-07-19 01:48:54 -07002905 graph_unlock();
2906 return 1;
2907 }
2908
Dave Jonesf82b2172008-08-11 09:30:23 +02002909 hlock_class(this)->usage_mask |= new_mask;
Peter Zijlstra8e182572007-07-19 01:48:54 -07002910
Dave Jonesf82b2172008-08-11 09:30:23 +02002911 if (!save_trace(hlock_class(this)->usage_traces + new_bit))
Peter Zijlstra8e182572007-07-19 01:48:54 -07002912 return 0;
2913
2914 switch (new_bit) {
Peter Zijlstra53464172009-01-22 14:15:53 +01002915#define LOCKDEP_STATE(__STATE) \
2916 case LOCK_USED_IN_##__STATE: \
2917 case LOCK_USED_IN_##__STATE##_READ: \
2918 case LOCK_ENABLED_##__STATE: \
2919 case LOCK_ENABLED_##__STATE##_READ:
2920#include "lockdep_states.h"
2921#undef LOCKDEP_STATE
Peter Zijlstra8e182572007-07-19 01:48:54 -07002922 ret = mark_lock_irq(curr, this, new_bit);
2923 if (!ret)
2924 return 0;
2925 break;
2926 case LOCK_USED:
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02002927 debug_atomic_dec(nr_unused_locks);
Peter Zijlstra8e182572007-07-19 01:48:54 -07002928 break;
2929 default:
2930 if (!debug_locks_off_graph_unlock())
2931 return 0;
2932 WARN_ON(1);
2933 return 0;
2934 }
2935
2936 graph_unlock();
2937
2938 /*
2939 * We must printk outside of the graph_lock:
2940 */
2941 if (ret == 2) {
2942 printk("\nmarked lock as {%s}:\n", usage_str[new_bit]);
2943 print_lock(this);
2944 print_irqtrace_events(curr);
2945 dump_stack();
2946 }
2947
2948 return ret;
2949}
2950
2951/*
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002952 * Initialize a lock instance's lock-class mapping info:
2953 */
2954void lockdep_init_map(struct lockdep_map *lock, const char *name,
Peter Zijlstra4dfbb9d2006-10-11 01:45:14 -04002955 struct lock_class_key *key, int subclass)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002956{
Yong Zhangd3d03d42011-11-09 16:04:51 +08002957 int i;
2958
2959 kmemcheck_mark_initialized(lock, sizeof(*lock));
2960
2961 for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++)
2962 lock->class_cache[i] = NULL;
Hitoshi Mitake62016252010-10-05 18:01:51 +09002963
Peter Zijlstrac8a25002009-04-17 09:40:49 +02002964#ifdef CONFIG_LOCK_STAT
2965 lock->cpu = raw_smp_processor_id();
2966#endif
2967
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002968 /*
2969 * Can't be having no nameless bastards around this place!
2970 */
Peter Zijlstrac8a25002009-04-17 09:40:49 +02002971 if (DEBUG_LOCKS_WARN_ON(!name)) {
2972 lock->name = "NULL";
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002973 return;
Peter Zijlstrac8a25002009-04-17 09:40:49 +02002974 }
2975
2976 lock->name = name;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002977
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002978 /*
2979 * No key, no joy, we need to hash something.
2980 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002981 if (DEBUG_LOCKS_WARN_ON(!key))
2982 return;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002983 /*
2984 * Sanity check, the lock-class key must be persistent:
2985 */
2986 if (!static_obj(key)) {
2987 printk("BUG: key %p not in .data!\n", key);
Peter Zijlstra0119fee2011-09-02 01:30:29 +02002988 /*
2989 * What it says above ^^^^^, I suggest you read it.
2990 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002991 DEBUG_LOCKS_WARN_ON(1);
2992 return;
2993 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002994 lock->key = key;
Peter Zijlstrac8a25002009-04-17 09:40:49 +02002995
2996 if (unlikely(!debug_locks))
2997 return;
2998
Peter Zijlstra4dfbb9d2006-10-11 01:45:14 -04002999 if (subclass)
3000 register_lock_class(lock, subclass, 1);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003001}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003002EXPORT_SYMBOL_GPL(lockdep_init_map);
3003
Peter Zijlstra1704f472010-03-19 01:37:42 +01003004struct lock_class_key __lockdep_no_validate__;
Kent Overstreetea6749c2012-12-27 22:21:58 -08003005EXPORT_SYMBOL_GPL(__lockdep_no_validate__);
Peter Zijlstra1704f472010-03-19 01:37:42 +01003006
Maarten Lankhorstd0945952012-09-13 11:39:51 +02003007static int
3008print_lock_nested_lock_not_held(struct task_struct *curr,
3009 struct held_lock *hlock,
3010 unsigned long ip)
3011{
3012 if (!debug_locks_off())
3013 return 0;
3014 if (debug_locks_silent)
3015 return 0;
3016
3017 printk("\n");
3018 printk("==================================\n");
3019 printk("[ BUG: Nested lock was not taken ]\n");
3020 print_kernel_ident();
3021 printk("----------------------------------\n");
3022
3023 printk("%s/%d is trying to lock:\n", curr->comm, task_pid_nr(curr));
3024 print_lock(hlock);
3025
3026 printk("\nbut this task is not holding:\n");
3027 printk("%s\n", hlock->nest_lock->name);
3028
3029 printk("\nstack backtrace:\n");
3030 dump_stack();
3031
3032 printk("\nother info that might help us debug this:\n");
3033 lockdep_print_held_locks(curr);
3034
3035 printk("\nstack backtrace:\n");
3036 dump_stack();
3037
3038 return 0;
3039}
3040
3041static int __lock_is_held(struct lockdep_map *lock);
3042
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003043/*
3044 * This gets called for every mutex_lock*()/spin_lock*() operation.
3045 * We maintain the dependency maps and validate the locking attempt:
3046 */
3047static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
3048 int trylock, int read, int check, int hardirqs_off,
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003049 struct lockdep_map *nest_lock, unsigned long ip,
3050 int references)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003051{
3052 struct task_struct *curr = current;
Ingo Molnard6d897c2006-07-10 04:44:04 -07003053 struct lock_class *class = NULL;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003054 struct held_lock *hlock;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003055 unsigned int depth, id;
3056 int chain_head = 0;
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003057 int class_idx;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003058 u64 chain_key;
3059
3060 if (unlikely(!debug_locks))
3061 return 0;
3062
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003063 /*
3064 * Lockdep should run with IRQs disabled, otherwise we could
3065 * get an interrupt which would want to take locks, which would
3066 * end up in lockdep and have you got a head-ache already?
3067 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003068 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
3069 return 0;
3070
Oleg Nesterovfb9edbe2014-01-20 19:20:06 +01003071 if (!prove_locking || lock->key == &__lockdep_no_validate__)
3072 check = 0;
Peter Zijlstra1704f472010-03-19 01:37:42 +01003073
Hitoshi Mitake62016252010-10-05 18:01:51 +09003074 if (subclass < NR_LOCKDEP_CACHING_CLASSES)
3075 class = lock->class_cache[subclass];
Ingo Molnard6d897c2006-07-10 04:44:04 -07003076 /*
Hitoshi Mitake62016252010-10-05 18:01:51 +09003077 * Not cached?
Ingo Molnard6d897c2006-07-10 04:44:04 -07003078 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003079 if (unlikely(!class)) {
Peter Zijlstra4dfbb9d2006-10-11 01:45:14 -04003080 class = register_lock_class(lock, subclass, 0);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003081 if (!class)
3082 return 0;
3083 }
Frederic Weisbeckerbd6d29c2010-04-06 00:10:17 +02003084 atomic_inc((atomic_t *)&class->ops);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003085 if (very_verbose(class)) {
3086 printk("\nacquire class [%p] %s", class->key, class->name);
3087 if (class->name_version > 1)
3088 printk("#%d", class->name_version);
3089 printk("\n");
3090 dump_stack();
3091 }
3092
3093 /*
3094 * Add the lock to the list of currently held locks.
3095 * (we dont increase the depth just yet, up until the
3096 * dependency checks are done)
3097 */
3098 depth = curr->lockdep_depth;
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003099 /*
3100 * Ran out of static storage for our per-task lock stack again have we?
3101 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003102 if (DEBUG_LOCKS_WARN_ON(depth >= MAX_LOCK_DEPTH))
3103 return 0;
3104
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003105 class_idx = class - lock_classes + 1;
3106
3107 if (depth) {
3108 hlock = curr->held_locks + depth - 1;
3109 if (hlock->class_idx == class_idx && nest_lock) {
3110 if (hlock->references)
3111 hlock->references++;
3112 else
3113 hlock->references = 2;
3114
3115 return 1;
3116 }
3117 }
3118
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003119 hlock = curr->held_locks + depth;
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003120 /*
3121 * Plain impossible, we just registered it and checked it weren't no
3122 * NULL like.. I bet this mushroom I ate was good!
3123 */
Dave Jonesf82b2172008-08-11 09:30:23 +02003124 if (DEBUG_LOCKS_WARN_ON(!class))
3125 return 0;
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003126 hlock->class_idx = class_idx;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003127 hlock->acquire_ip = ip;
3128 hlock->instance = lock;
Peter Zijlstra7531e2f2008-08-11 09:30:24 +02003129 hlock->nest_lock = nest_lock;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003130 hlock->trylock = trylock;
3131 hlock->read = read;
3132 hlock->check = check;
Dmitry Baryshkov6951b122008-08-18 04:26:37 +04003133 hlock->hardirqs_off = !!hardirqs_off;
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003134 hlock->references = references;
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003135#ifdef CONFIG_LOCK_STAT
3136 hlock->waittime_stamp = 0;
Peter Zijlstra3365e7792009-10-09 10:12:41 +02003137 hlock->holdtime_stamp = lockstat_clock();
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003138#endif
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003139
Oleg Nesterovfb9edbe2014-01-20 19:20:06 +01003140 if (check && !mark_irqflags(curr, hlock))
Peter Zijlstra8e182572007-07-19 01:48:54 -07003141 return 0;
3142
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003143 /* mark it as used: */
Jarek Poplawski4ff773bb2007-05-08 00:31:00 -07003144 if (!mark_lock(curr, hlock, LOCK_USED))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003145 return 0;
Peter Zijlstra8e182572007-07-19 01:48:54 -07003146
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003147 /*
Gautham R Shenoy17aacfb2007-10-28 20:47:01 +01003148 * Calculate the chain hash: it's the combined hash of all the
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003149 * lock keys along the dependency chain. We save the hash value
3150 * at every step so that we can get the current hash easily
3151 * after unlock. The chain hash is then used to cache dependency
3152 * results.
3153 *
3154 * The 'key ID' is what is the most compact key value to drive
3155 * the hash, not class->key.
3156 */
3157 id = class - lock_classes;
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003158 /*
3159 * Whoops, we did it again.. ran straight out of our static allocation.
3160 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003161 if (DEBUG_LOCKS_WARN_ON(id >= MAX_LOCKDEP_KEYS))
3162 return 0;
3163
3164 chain_key = curr->curr_chain_key;
3165 if (!depth) {
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003166 /*
3167 * How can we have a chain hash when we ain't got no keys?!
3168 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003169 if (DEBUG_LOCKS_WARN_ON(chain_key != 0))
3170 return 0;
3171 chain_head = 1;
3172 }
3173
3174 hlock->prev_chain_key = chain_key;
Peter Zijlstra8e182572007-07-19 01:48:54 -07003175 if (separate_irq_context(curr, hlock)) {
3176 chain_key = 0;
3177 chain_head = 1;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003178 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003179 chain_key = iterate_chain_key(chain_key, id);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003180
Maarten Lankhorstd0945952012-09-13 11:39:51 +02003181 if (nest_lock && !__lock_is_held(nest_lock))
3182 return print_lock_nested_lock_not_held(curr, hlock, ip);
3183
Gregory Haskins3aa416b2007-10-11 22:11:11 +02003184 if (!validate_chain(curr, lock, hlock, chain_head, chain_key))
Peter Zijlstra8e182572007-07-19 01:48:54 -07003185 return 0;
Jarek Poplawski381a2292007-02-10 01:44:58 -08003186
Gregory Haskins3aa416b2007-10-11 22:11:11 +02003187 curr->curr_chain_key = chain_key;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003188 curr->lockdep_depth++;
3189 check_chain_key(curr);
Jarek Poplawski60e114d2007-02-20 13:58:00 -08003190#ifdef CONFIG_DEBUG_LOCKDEP
3191 if (unlikely(!debug_locks))
3192 return 0;
3193#endif
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003194 if (unlikely(curr->lockdep_depth >= MAX_LOCK_DEPTH)) {
3195 debug_locks_off();
Dave Jones2c522832013-04-25 13:40:02 -04003196 print_lockdep_off("BUG: MAX_LOCK_DEPTH too low!");
3197 printk(KERN_DEBUG "depth: %i max: %lu!\n",
Ben Greearc0540602013-02-06 10:56:19 -08003198 curr->lockdep_depth, MAX_LOCK_DEPTH);
Ben Greearc0540602013-02-06 10:56:19 -08003199
3200 lockdep_print_held_locks(current);
3201 debug_show_all_locks();
Peter Zijlstraeedeeab2009-03-18 12:38:47 +01003202 dump_stack();
Ben Greearc0540602013-02-06 10:56:19 -08003203
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003204 return 0;
3205 }
Jarek Poplawski381a2292007-02-10 01:44:58 -08003206
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003207 if (unlikely(curr->lockdep_depth > max_lockdep_depth))
3208 max_lockdep_depth = curr->lockdep_depth;
3209
3210 return 1;
3211}
3212
3213static int
Srivatsa S. Bhatf86f7552013-01-08 18:35:58 +05303214print_unlock_imbalance_bug(struct task_struct *curr, struct lockdep_map *lock,
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003215 unsigned long ip)
3216{
3217 if (!debug_locks_off())
3218 return 0;
3219 if (debug_locks_silent)
3220 return 0;
3221
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07003222 printk("\n");
3223 printk("=====================================\n");
3224 printk("[ BUG: bad unlock balance detected! ]\n");
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01003225 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07003226 printk("-------------------------------------\n");
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003227 printk("%s/%d is trying to release lock (",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07003228 curr->comm, task_pid_nr(curr));
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003229 print_lockdep_cache(lock);
3230 printk(") at:\n");
3231 print_ip_sym(ip);
3232 printk("but there are no more locks to release!\n");
3233 printk("\nother info that might help us debug this:\n");
3234 lockdep_print_held_locks(curr);
3235
3236 printk("\nstack backtrace:\n");
3237 dump_stack();
3238
3239 return 0;
3240}
3241
3242/*
3243 * Common debugging checks for both nested and non-nested unlock:
3244 */
3245static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,
3246 unsigned long ip)
3247{
3248 if (unlikely(!debug_locks))
3249 return 0;
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003250 /*
3251 * Lockdep should run with IRQs disabled, recursion, head-ache, etc..
3252 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003253 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
3254 return 0;
3255
3256 if (curr->lockdep_depth <= 0)
Srivatsa S. Bhatf86f7552013-01-08 18:35:58 +05303257 return print_unlock_imbalance_bug(curr, lock, ip);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003258
3259 return 1;
3260}
3261
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003262static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)
3263{
3264 if (hlock->instance == lock)
3265 return 1;
3266
3267 if (hlock->references) {
Hitoshi Mitake62016252010-10-05 18:01:51 +09003268 struct lock_class *class = lock->class_cache[0];
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003269
3270 if (!class)
3271 class = look_up_lock_class(lock, 0);
3272
Peter Zijlstra80e04012011-08-05 14:26:17 +02003273 /*
3274 * If look_up_lock_class() failed to find a class, we're trying
3275 * to test if we hold a lock that has never yet been acquired.
3276 * Clearly if the lock hasn't been acquired _ever_, we're not
3277 * holding it either, so report failure.
3278 */
3279 if (!class)
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003280 return 0;
3281
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003282 /*
3283 * References, but not a lock we're actually ref-counting?
3284 * State got messed up, follow the sites that change ->references
3285 * and try to make sense of it.
3286 */
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003287 if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))
3288 return 0;
3289
3290 if (hlock->class_idx == class - lock_classes + 1)
3291 return 1;
3292 }
3293
3294 return 0;
3295}
3296
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003297static int
Peter Zijlstra00ef9f72008-12-04 09:00:17 +01003298__lock_set_class(struct lockdep_map *lock, const char *name,
3299 struct lock_class_key *key, unsigned int subclass,
3300 unsigned long ip)
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003301{
3302 struct task_struct *curr = current;
3303 struct held_lock *hlock, *prev_hlock;
3304 struct lock_class *class;
3305 unsigned int depth;
3306 int i;
3307
3308 depth = curr->lockdep_depth;
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003309 /*
3310 * This function is about (re)setting the class of a held lock,
3311 * yet we're not actually holding any locks. Naughty user!
3312 */
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003313 if (DEBUG_LOCKS_WARN_ON(!depth))
3314 return 0;
3315
3316 prev_hlock = NULL;
3317 for (i = depth-1; i >= 0; i--) {
3318 hlock = curr->held_locks + i;
3319 /*
3320 * We must not cross into another context:
3321 */
3322 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
3323 break;
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003324 if (match_held_lock(hlock, lock))
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003325 goto found_it;
3326 prev_hlock = hlock;
3327 }
Srivatsa S. Bhatf86f7552013-01-08 18:35:58 +05303328 return print_unlock_imbalance_bug(curr, lock, ip);
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003329
3330found_it:
Peter Zijlstra00ef9f72008-12-04 09:00:17 +01003331 lockdep_init_map(lock, name, key, 0);
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003332 class = register_lock_class(lock, subclass, 0);
Dave Jonesf82b2172008-08-11 09:30:23 +02003333 hlock->class_idx = class - lock_classes + 1;
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003334
3335 curr->lockdep_depth = i;
3336 curr->curr_chain_key = hlock->prev_chain_key;
3337
3338 for (; i < depth; i++) {
3339 hlock = curr->held_locks + i;
3340 if (!__lock_acquire(hlock->instance,
Dave Jonesf82b2172008-08-11 09:30:23 +02003341 hlock_class(hlock)->subclass, hlock->trylock,
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003342 hlock->read, hlock->check, hlock->hardirqs_off,
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003343 hlock->nest_lock, hlock->acquire_ip,
3344 hlock->references))
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003345 return 0;
3346 }
3347
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003348 /*
3349 * I took it apart and put it back together again, except now I have
3350 * these 'spare' parts.. where shall I put them.
3351 */
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003352 if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth))
3353 return 0;
3354 return 1;
3355}
3356
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003357/*
3358 * Remove the lock to the list of currently held locks in a
3359 * potentially non-nested (out of order) manner. This is a
3360 * relatively rare operation, as all the unlock APIs default
3361 * to nested mode (which uses lock_release()):
3362 */
3363static int
3364lock_release_non_nested(struct task_struct *curr,
3365 struct lockdep_map *lock, unsigned long ip)
3366{
3367 struct held_lock *hlock, *prev_hlock;
3368 unsigned int depth;
3369 int i;
3370
3371 /*
3372 * Check whether the lock exists in the current stack
3373 * of held locks:
3374 */
3375 depth = curr->lockdep_depth;
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003376 /*
3377 * So we're all set to release this lock.. wait what lock? We don't
3378 * own any locks, you've been drinking again?
3379 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003380 if (DEBUG_LOCKS_WARN_ON(!depth))
3381 return 0;
3382
3383 prev_hlock = NULL;
3384 for (i = depth-1; i >= 0; i--) {
3385 hlock = curr->held_locks + i;
3386 /*
3387 * We must not cross into another context:
3388 */
3389 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
3390 break;
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003391 if (match_held_lock(hlock, lock))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003392 goto found_it;
3393 prev_hlock = hlock;
3394 }
Srivatsa S. Bhatf86f7552013-01-08 18:35:58 +05303395 return print_unlock_imbalance_bug(curr, lock, ip);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003396
3397found_it:
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003398 if (hlock->instance == lock)
3399 lock_release_holdtime(hlock);
3400
3401 if (hlock->references) {
3402 hlock->references--;
3403 if (hlock->references) {
3404 /*
3405 * We had, and after removing one, still have
3406 * references, the current lock stack is still
3407 * valid. We're done!
3408 */
3409 return 1;
3410 }
3411 }
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003412
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003413 /*
3414 * We have the right lock to unlock, 'hlock' points to it.
3415 * Now we remove it from the stack, and add back the other
3416 * entries (if any), recalculating the hash along the way:
3417 */
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003418
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003419 curr->lockdep_depth = i;
3420 curr->curr_chain_key = hlock->prev_chain_key;
3421
3422 for (i++; i < depth; i++) {
3423 hlock = curr->held_locks + i;
3424 if (!__lock_acquire(hlock->instance,
Dave Jonesf82b2172008-08-11 09:30:23 +02003425 hlock_class(hlock)->subclass, hlock->trylock,
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003426 hlock->read, hlock->check, hlock->hardirqs_off,
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003427 hlock->nest_lock, hlock->acquire_ip,
3428 hlock->references))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003429 return 0;
3430 }
3431
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003432 /*
3433 * We had N bottles of beer on the wall, we drank one, but now
3434 * there's not N-1 bottles of beer left on the wall...
3435 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003436 if (DEBUG_LOCKS_WARN_ON(curr->lockdep_depth != depth - 1))
3437 return 0;
3438 return 1;
3439}
3440
3441/*
3442 * Remove the lock to the list of currently held locks - this gets
3443 * called on mutex_unlock()/spin_unlock*() (or on a failed
3444 * mutex_lock_interruptible()). This is done for unlocks that nest
3445 * perfectly. (i.e. the current top of the lock-stack is unlocked)
3446 */
3447static int lock_release_nested(struct task_struct *curr,
3448 struct lockdep_map *lock, unsigned long ip)
3449{
3450 struct held_lock *hlock;
3451 unsigned int depth;
3452
3453 /*
3454 * Pop off the top of the lock stack:
3455 */
3456 depth = curr->lockdep_depth - 1;
3457 hlock = curr->held_locks + depth;
3458
3459 /*
3460 * Is the unlock non-nested:
3461 */
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003462 if (hlock->instance != lock || hlock->references)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003463 return lock_release_non_nested(curr, lock, ip);
3464 curr->lockdep_depth--;
3465
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003466 /*
3467 * No more locks, but somehow we've got hash left over, who left it?
3468 */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003469 if (DEBUG_LOCKS_WARN_ON(!depth && (hlock->prev_chain_key != 0)))
3470 return 0;
3471
3472 curr->curr_chain_key = hlock->prev_chain_key;
3473
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003474 lock_release_holdtime(hlock);
3475
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003476#ifdef CONFIG_DEBUG_LOCKDEP
3477 hlock->prev_chain_key = 0;
Dave Jonesf82b2172008-08-11 09:30:23 +02003478 hlock->class_idx = 0;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003479 hlock->acquire_ip = 0;
3480 hlock->irq_context = 0;
3481#endif
3482 return 1;
3483}
3484
3485/*
3486 * Remove the lock to the list of currently held locks - this gets
3487 * called on mutex_unlock()/spin_unlock*() (or on a failed
3488 * mutex_lock_interruptible()). This is done for unlocks that nest
3489 * perfectly. (i.e. the current top of the lock-stack is unlocked)
3490 */
3491static void
3492__lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
3493{
3494 struct task_struct *curr = current;
3495
3496 if (!check_unlock(curr, lock, ip))
3497 return;
3498
3499 if (nested) {
3500 if (!lock_release_nested(curr, lock, ip))
3501 return;
3502 } else {
3503 if (!lock_release_non_nested(curr, lock, ip))
3504 return;
3505 }
3506
3507 check_chain_key(curr);
3508}
3509
Peter Zijlstraf607c662009-07-20 19:16:29 +02003510static int __lock_is_held(struct lockdep_map *lock)
3511{
3512 struct task_struct *curr = current;
3513 int i;
3514
3515 for (i = 0; i < curr->lockdep_depth; i++) {
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003516 struct held_lock *hlock = curr->held_locks + i;
3517
3518 if (match_held_lock(hlock, lock))
Peter Zijlstraf607c662009-07-20 19:16:29 +02003519 return 1;
3520 }
3521
3522 return 0;
3523}
3524
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003525/*
3526 * Check whether we follow the irq-flags state precisely:
3527 */
Steven Rostedt1d09daa2008-05-12 21:20:55 +02003528static void check_flags(unsigned long flags)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003529{
Ingo Molnar992860e2008-07-14 10:28:38 +02003530#if defined(CONFIG_PROVE_LOCKING) && defined(CONFIG_DEBUG_LOCKDEP) && \
3531 defined(CONFIG_TRACE_IRQFLAGS)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003532 if (!debug_locks)
3533 return;
3534
Ingo Molnar5f9fa8a2007-12-07 19:02:47 +01003535 if (irqs_disabled_flags(flags)) {
3536 if (DEBUG_LOCKS_WARN_ON(current->hardirqs_enabled)) {
3537 printk("possible reason: unannotated irqs-off.\n");
3538 }
3539 } else {
3540 if (DEBUG_LOCKS_WARN_ON(!current->hardirqs_enabled)) {
3541 printk("possible reason: unannotated irqs-on.\n");
3542 }
3543 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003544
3545 /*
3546 * We dont accurately track softirq state in e.g.
3547 * hardirq contexts (such as on 4KSTACKS), so only
3548 * check if not in hardirq contexts:
3549 */
3550 if (!hardirq_count()) {
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003551 if (softirq_count()) {
3552 /* like the above, but with softirqs */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003553 DEBUG_LOCKS_WARN_ON(current->softirqs_enabled);
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003554 } else {
3555 /* lick the above, does it taste good? */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003556 DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled);
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003557 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003558 }
3559
3560 if (!debug_locks)
3561 print_irqtrace_events(current);
3562#endif
3563}
3564
Peter Zijlstra00ef9f72008-12-04 09:00:17 +01003565void lock_set_class(struct lockdep_map *lock, const char *name,
3566 struct lock_class_key *key, unsigned int subclass,
3567 unsigned long ip)
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003568{
3569 unsigned long flags;
3570
3571 if (unlikely(current->lockdep_recursion))
3572 return;
3573
3574 raw_local_irq_save(flags);
3575 current->lockdep_recursion = 1;
3576 check_flags(flags);
Peter Zijlstra00ef9f72008-12-04 09:00:17 +01003577 if (__lock_set_class(lock, name, key, subclass, ip))
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003578 check_chain_key(current);
3579 current->lockdep_recursion = 0;
3580 raw_local_irq_restore(flags);
3581}
Peter Zijlstra00ef9f72008-12-04 09:00:17 +01003582EXPORT_SYMBOL_GPL(lock_set_class);
Peter Zijlstra64aa3482008-08-11 09:30:21 +02003583
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003584/*
3585 * We are not always called with irqs disabled - do that here,
3586 * and also avoid lockdep recursion:
3587 */
Steven Rostedt1d09daa2008-05-12 21:20:55 +02003588void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
Peter Zijlstra7531e2f2008-08-11 09:30:24 +02003589 int trylock, int read, int check,
3590 struct lockdep_map *nest_lock, unsigned long ip)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003591{
3592 unsigned long flags;
3593
3594 if (unlikely(current->lockdep_recursion))
3595 return;
3596
3597 raw_local_irq_save(flags);
3598 check_flags(flags);
3599
3600 current->lockdep_recursion = 1;
Frederic Weisbeckerdb2c4c72010-02-02 23:34:40 +01003601 trace_lock_acquire(lock, subclass, trylock, read, check, nest_lock, ip);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003602 __lock_acquire(lock, subclass, trylock, read, check,
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003603 irqs_disabled_flags(flags), nest_lock, ip, 0);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003604 current->lockdep_recursion = 0;
3605 raw_local_irq_restore(flags);
3606}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003607EXPORT_SYMBOL_GPL(lock_acquire);
3608
Steven Rostedt1d09daa2008-05-12 21:20:55 +02003609void lock_release(struct lockdep_map *lock, int nested,
Steven Rostedt0764d232008-05-12 21:20:44 +02003610 unsigned long ip)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003611{
3612 unsigned long flags;
3613
3614 if (unlikely(current->lockdep_recursion))
3615 return;
3616
3617 raw_local_irq_save(flags);
3618 check_flags(flags);
3619 current->lockdep_recursion = 1;
Frederic Weisbecker93135432010-05-08 06:24:25 +02003620 trace_lock_release(lock, ip);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003621 __lock_release(lock, nested, ip);
3622 current->lockdep_recursion = 0;
3623 raw_local_irq_restore(flags);
3624}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003625EXPORT_SYMBOL_GPL(lock_release);
3626
Peter Zijlstraf607c662009-07-20 19:16:29 +02003627int lock_is_held(struct lockdep_map *lock)
3628{
3629 unsigned long flags;
3630 int ret = 0;
3631
3632 if (unlikely(current->lockdep_recursion))
Peter Zijlstraf2513cd2011-06-06 12:32:43 +02003633 return 1; /* avoid false negative lockdep_assert_held() */
Peter Zijlstraf607c662009-07-20 19:16:29 +02003634
3635 raw_local_irq_save(flags);
3636 check_flags(flags);
3637
3638 current->lockdep_recursion = 1;
3639 ret = __lock_is_held(lock);
3640 current->lockdep_recursion = 0;
3641 raw_local_irq_restore(flags);
3642
3643 return ret;
3644}
3645EXPORT_SYMBOL_GPL(lock_is_held);
3646
Nick Piggincf40bd12009-01-21 08:12:39 +01003647void lockdep_set_current_reclaim_state(gfp_t gfp_mask)
3648{
3649 current->lockdep_reclaim_gfp = gfp_mask;
3650}
3651
3652void lockdep_clear_current_reclaim_state(void)
3653{
3654 current->lockdep_reclaim_gfp = 0;
3655}
3656
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003657#ifdef CONFIG_LOCK_STAT
3658static int
3659print_lock_contention_bug(struct task_struct *curr, struct lockdep_map *lock,
3660 unsigned long ip)
3661{
3662 if (!debug_locks_off())
3663 return 0;
3664 if (debug_locks_silent)
3665 return 0;
3666
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07003667 printk("\n");
3668 printk("=================================\n");
3669 printk("[ BUG: bad contention detected! ]\n");
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01003670 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07003671 printk("---------------------------------\n");
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003672 printk("%s/%d is trying to contend lock (",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07003673 curr->comm, task_pid_nr(curr));
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003674 print_lockdep_cache(lock);
3675 printk(") at:\n");
3676 print_ip_sym(ip);
3677 printk("but there are no locks held!\n");
3678 printk("\nother info that might help us debug this:\n");
3679 lockdep_print_held_locks(curr);
3680
3681 printk("\nstack backtrace:\n");
3682 dump_stack();
3683
3684 return 0;
3685}
3686
3687static void
3688__lock_contended(struct lockdep_map *lock, unsigned long ip)
3689{
3690 struct task_struct *curr = current;
3691 struct held_lock *hlock, *prev_hlock;
3692 struct lock_class_stats *stats;
3693 unsigned int depth;
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +02003694 int i, contention_point, contending_point;
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003695
3696 depth = curr->lockdep_depth;
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003697 /*
3698 * Whee, we contended on this lock, except it seems we're not
3699 * actually trying to acquire anything much at all..
3700 */
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003701 if (DEBUG_LOCKS_WARN_ON(!depth))
3702 return;
3703
3704 prev_hlock = NULL;
3705 for (i = depth-1; i >= 0; i--) {
3706 hlock = curr->held_locks + i;
3707 /*
3708 * We must not cross into another context:
3709 */
3710 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
3711 break;
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003712 if (match_held_lock(hlock, lock))
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003713 goto found_it;
3714 prev_hlock = hlock;
3715 }
3716 print_lock_contention_bug(curr, lock, ip);
3717 return;
3718
3719found_it:
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003720 if (hlock->instance != lock)
3721 return;
3722
Peter Zijlstra3365e7792009-10-09 10:12:41 +02003723 hlock->waittime_stamp = lockstat_clock();
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003724
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +02003725 contention_point = lock_point(hlock_class(hlock)->contention_point, ip);
3726 contending_point = lock_point(hlock_class(hlock)->contending_point,
3727 lock->ip);
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003728
Dave Jonesf82b2172008-08-11 09:30:23 +02003729 stats = get_lock_stats(hlock_class(hlock));
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +02003730 if (contention_point < LOCKSTAT_POINTS)
3731 stats->contention_point[contention_point]++;
3732 if (contending_point < LOCKSTAT_POINTS)
3733 stats->contending_point[contending_point]++;
Peter Zijlstra96645672007-07-19 01:49:00 -07003734 if (lock->cpu != smp_processor_id())
3735 stats->bounces[bounce_contended + !!hlock->read]++;
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003736 put_lock_stats(stats);
3737}
3738
3739static void
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +02003740__lock_acquired(struct lockdep_map *lock, unsigned long ip)
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003741{
3742 struct task_struct *curr = current;
3743 struct held_lock *hlock, *prev_hlock;
3744 struct lock_class_stats *stats;
3745 unsigned int depth;
Peter Zijlstra3365e7792009-10-09 10:12:41 +02003746 u64 now, waittime = 0;
Peter Zijlstra96645672007-07-19 01:49:00 -07003747 int i, cpu;
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003748
3749 depth = curr->lockdep_depth;
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003750 /*
3751 * Yay, we acquired ownership of this lock we didn't try to
3752 * acquire, how the heck did that happen?
3753 */
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003754 if (DEBUG_LOCKS_WARN_ON(!depth))
3755 return;
3756
3757 prev_hlock = NULL;
3758 for (i = depth-1; i >= 0; i--) {
3759 hlock = curr->held_locks + i;
3760 /*
3761 * We must not cross into another context:
3762 */
3763 if (prev_hlock && prev_hlock->irq_context != hlock->irq_context)
3764 break;
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003765 if (match_held_lock(hlock, lock))
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003766 goto found_it;
3767 prev_hlock = hlock;
3768 }
3769 print_lock_contention_bug(curr, lock, _RET_IP_);
3770 return;
3771
3772found_it:
Peter Zijlstrabb97a912009-07-20 19:15:35 +02003773 if (hlock->instance != lock)
3774 return;
3775
Peter Zijlstra96645672007-07-19 01:49:00 -07003776 cpu = smp_processor_id();
3777 if (hlock->waittime_stamp) {
Peter Zijlstra3365e7792009-10-09 10:12:41 +02003778 now = lockstat_clock();
Peter Zijlstra96645672007-07-19 01:49:00 -07003779 waittime = now - hlock->waittime_stamp;
3780 hlock->holdtime_stamp = now;
3781 }
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003782
Frederic Weisbecker883a2a32010-05-08 06:16:11 +02003783 trace_lock_acquired(lock, ip);
Frederic Weisbecker20625012009-04-06 01:49:33 +02003784
Dave Jonesf82b2172008-08-11 09:30:23 +02003785 stats = get_lock_stats(hlock_class(hlock));
Peter Zijlstra96645672007-07-19 01:49:00 -07003786 if (waittime) {
3787 if (hlock->read)
3788 lock_time_inc(&stats->read_waittime, waittime);
3789 else
3790 lock_time_inc(&stats->write_waittime, waittime);
3791 }
3792 if (lock->cpu != cpu)
3793 stats->bounces[bounce_acquired + !!hlock->read]++;
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003794 put_lock_stats(stats);
Peter Zijlstra96645672007-07-19 01:49:00 -07003795
3796 lock->cpu = cpu;
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +02003797 lock->ip = ip;
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003798}
3799
3800void lock_contended(struct lockdep_map *lock, unsigned long ip)
3801{
3802 unsigned long flags;
3803
3804 if (unlikely(!lock_stat))
3805 return;
3806
3807 if (unlikely(current->lockdep_recursion))
3808 return;
3809
3810 raw_local_irq_save(flags);
3811 check_flags(flags);
3812 current->lockdep_recursion = 1;
Frederic Weisbeckerdb2c4c72010-02-02 23:34:40 +01003813 trace_lock_contended(lock, ip);
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003814 __lock_contended(lock, ip);
3815 current->lockdep_recursion = 0;
3816 raw_local_irq_restore(flags);
3817}
3818EXPORT_SYMBOL_GPL(lock_contended);
3819
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +02003820void lock_acquired(struct lockdep_map *lock, unsigned long ip)
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003821{
3822 unsigned long flags;
3823
3824 if (unlikely(!lock_stat))
3825 return;
3826
3827 if (unlikely(current->lockdep_recursion))
3828 return;
3829
3830 raw_local_irq_save(flags);
3831 check_flags(flags);
3832 current->lockdep_recursion = 1;
Peter Zijlstrac7e78cf2008-10-16 23:17:09 +02003833 __lock_acquired(lock, ip);
Peter Zijlstraf20786f2007-07-19 01:48:56 -07003834 current->lockdep_recursion = 0;
3835 raw_local_irq_restore(flags);
3836}
3837EXPORT_SYMBOL_GPL(lock_acquired);
3838#endif
3839
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003840/*
3841 * Used by the testsuite, sanitize the validator state
3842 * after a simulated failure:
3843 */
3844
3845void lockdep_reset(void)
3846{
3847 unsigned long flags;
Ingo Molnar23d95a02006-12-13 00:34:40 -08003848 int i;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003849
3850 raw_local_irq_save(flags);
3851 current->curr_chain_key = 0;
3852 current->lockdep_depth = 0;
3853 current->lockdep_recursion = 0;
3854 memset(current->held_locks, 0, MAX_LOCK_DEPTH*sizeof(struct held_lock));
3855 nr_hardirq_chains = 0;
3856 nr_softirq_chains = 0;
3857 nr_process_chains = 0;
3858 debug_locks = 1;
Ingo Molnar23d95a02006-12-13 00:34:40 -08003859 for (i = 0; i < CHAINHASH_SIZE; i++)
3860 INIT_LIST_HEAD(chainhash_table + i);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003861 raw_local_irq_restore(flags);
3862}
3863
3864static void zap_class(struct lock_class *class)
3865{
3866 int i;
3867
3868 /*
3869 * Remove all dependencies this lock is
3870 * involved in:
3871 */
3872 for (i = 0; i < nr_list_entries; i++) {
3873 if (list_entries[i].class == class)
3874 list_del_rcu(&list_entries[i].entry);
3875 }
3876 /*
3877 * Unhash the class and remove it from the all_lock_classes list:
3878 */
3879 list_del_rcu(&class->hash_entry);
3880 list_del_rcu(&class->lock_entry);
3881
Rabin Vincent8bfe0292008-08-11 09:30:26 +02003882 class->key = NULL;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003883}
3884
Arjan van de Venfabe8742008-01-24 07:00:45 +01003885static inline int within(const void *addr, void *start, unsigned long size)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003886{
3887 return addr >= start && addr < start + size;
3888}
3889
3890void lockdep_free_key_range(void *start, unsigned long size)
3891{
3892 struct lock_class *class, *next;
3893 struct list_head *head;
3894 unsigned long flags;
3895 int i;
Nick Piggin5a26db52008-01-16 09:51:58 +01003896 int locked;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003897
3898 raw_local_irq_save(flags);
Nick Piggin5a26db52008-01-16 09:51:58 +01003899 locked = graph_lock();
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003900
3901 /*
3902 * Unhash all classes that were created by this module:
3903 */
3904 for (i = 0; i < CLASSHASH_SIZE; i++) {
3905 head = classhash_table + i;
3906 if (list_empty(head))
3907 continue;
Arjan van de Venfabe8742008-01-24 07:00:45 +01003908 list_for_each_entry_safe(class, next, head, hash_entry) {
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003909 if (within(class->key, start, size))
3910 zap_class(class);
Arjan van de Venfabe8742008-01-24 07:00:45 +01003911 else if (within(class->name, start, size))
3912 zap_class(class);
3913 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003914 }
3915
Nick Piggin5a26db52008-01-16 09:51:58 +01003916 if (locked)
3917 graph_unlock();
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003918 raw_local_irq_restore(flags);
3919}
3920
3921void lockdep_reset_lock(struct lockdep_map *lock)
3922{
Ingo Molnard6d897c2006-07-10 04:44:04 -07003923 struct lock_class *class, *next;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003924 struct list_head *head;
3925 unsigned long flags;
3926 int i, j;
Nick Piggin5a26db52008-01-16 09:51:58 +01003927 int locked;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003928
3929 raw_local_irq_save(flags);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003930
3931 /*
Ingo Molnard6d897c2006-07-10 04:44:04 -07003932 * Remove all classes this lock might have:
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003933 */
Ingo Molnard6d897c2006-07-10 04:44:04 -07003934 for (j = 0; j < MAX_LOCKDEP_SUBCLASSES; j++) {
3935 /*
3936 * If the class exists we look it up and zap it:
3937 */
3938 class = look_up_lock_class(lock, j);
3939 if (class)
3940 zap_class(class);
3941 }
3942 /*
3943 * Debug check: in the end all mapped classes should
3944 * be gone.
3945 */
Nick Piggin5a26db52008-01-16 09:51:58 +01003946 locked = graph_lock();
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003947 for (i = 0; i < CLASSHASH_SIZE; i++) {
3948 head = classhash_table + i;
3949 if (list_empty(head))
3950 continue;
3951 list_for_each_entry_safe(class, next, head, hash_entry) {
Hitoshi Mitake62016252010-10-05 18:01:51 +09003952 int match = 0;
3953
3954 for (j = 0; j < NR_LOCKDEP_CACHING_CLASSES; j++)
3955 match |= class == lock->class_cache[j];
3956
3957 if (unlikely(match)) {
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003958 if (debug_locks_off_graph_unlock()) {
3959 /*
3960 * We all just reset everything, how did it match?
3961 */
Ingo Molnar74c383f2006-12-13 00:34:43 -08003962 WARN_ON(1);
Peter Zijlstra0119fee2011-09-02 01:30:29 +02003963 }
Ingo Molnard6d897c2006-07-10 04:44:04 -07003964 goto out_restore;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003965 }
3966 }
3967 }
Nick Piggin5a26db52008-01-16 09:51:58 +01003968 if (locked)
3969 graph_unlock();
Ingo Molnard6d897c2006-07-10 04:44:04 -07003970
3971out_restore:
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003972 raw_local_irq_restore(flags);
3973}
3974
Sam Ravnborg14999932007-02-28 20:12:31 -08003975void lockdep_init(void)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07003976{
3977 int i;
3978
3979 /*
3980 * Some architectures have their own start_kernel()
3981 * code which calls lockdep_init(), while we also
3982 * call lockdep_init() from the start_kernel() itself,
3983 * and we want to initialize the hashes only once:
3984 */
3985 if (lockdep_initialized)
3986 return;
3987
3988 for (i = 0; i < CLASSHASH_SIZE; i++)
3989 INIT_LIST_HEAD(classhash_table + i);
3990
3991 for (i = 0; i < CHAINHASH_SIZE; i++)
3992 INIT_LIST_HEAD(chainhash_table + i);
3993
3994 lockdep_initialized = 1;
3995}
3996
3997void __init lockdep_info(void)
3998{
3999 printk("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n");
4000
Li Zefanb0788ca2008-11-21 15:57:32 +08004001 printk("... MAX_LOCKDEP_SUBCLASSES: %lu\n", MAX_LOCKDEP_SUBCLASSES);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004002 printk("... MAX_LOCK_DEPTH: %lu\n", MAX_LOCK_DEPTH);
4003 printk("... MAX_LOCKDEP_KEYS: %lu\n", MAX_LOCKDEP_KEYS);
Li Zefanb0788ca2008-11-21 15:57:32 +08004004 printk("... CLASSHASH_SIZE: %lu\n", CLASSHASH_SIZE);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004005 printk("... MAX_LOCKDEP_ENTRIES: %lu\n", MAX_LOCKDEP_ENTRIES);
4006 printk("... MAX_LOCKDEP_CHAINS: %lu\n", MAX_LOCKDEP_CHAINS);
4007 printk("... CHAINHASH_SIZE: %lu\n", CHAINHASH_SIZE);
4008
4009 printk(" memory used by lock dependency info: %lu kB\n",
4010 (sizeof(struct lock_class) * MAX_LOCKDEP_KEYS +
4011 sizeof(struct list_head) * CLASSHASH_SIZE +
4012 sizeof(struct lock_list) * MAX_LOCKDEP_ENTRIES +
4013 sizeof(struct lock_chain) * MAX_LOCKDEP_CHAINS +
Ming Lei90629202009-08-02 21:43:36 +08004014 sizeof(struct list_head) * CHAINHASH_SIZE
Ming Lei4dd861d2009-07-16 15:44:29 +02004015#ifdef CONFIG_PROVE_LOCKING
Ming Leie351b662009-07-22 22:48:09 +08004016 + sizeof(struct circular_queue)
Ming Lei4dd861d2009-07-16 15:44:29 +02004017#endif
Ming Lei90629202009-08-02 21:43:36 +08004018 ) / 1024
Ming Lei4dd861d2009-07-16 15:44:29 +02004019 );
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004020
4021 printk(" per task-struct memory footprint: %lu bytes\n",
4022 sizeof(struct held_lock) * MAX_LOCK_DEPTH);
4023
4024#ifdef CONFIG_DEBUG_LOCKDEP
Johannes Bergc71063c2007-07-19 01:49:02 -07004025 if (lockdep_init_error) {
Ming Lei81140ac2011-11-17 13:34:32 +08004026 printk("WARNING: lockdep init error! lock-%s was acquired"
4027 "before lockdep_init\n", lock_init_error);
Johannes Bergc71063c2007-07-19 01:49:02 -07004028 printk("Call stack leading to lockdep invocation was:\n");
4029 print_stack_trace(&lockdep_init_trace, 0);
4030 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004031#endif
4032}
4033
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004034static void
4035print_freed_lock_bug(struct task_struct *curr, const void *mem_from,
Arjan van de Ven55794a42006-07-10 04:44:03 -07004036 const void *mem_to, struct held_lock *hlock)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004037{
4038 if (!debug_locks_off())
4039 return;
4040 if (debug_locks_silent)
4041 return;
4042
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004043 printk("\n");
4044 printk("=========================\n");
4045 printk("[ BUG: held lock freed! ]\n");
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01004046 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004047 printk("-------------------------\n");
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004048 printk("%s/%d is freeing memory %p-%p, with a lock still held there!\n",
Pavel Emelyanovba25f9d2007-10-18 23:40:40 -07004049 curr->comm, task_pid_nr(curr), mem_from, mem_to-1);
Arjan van de Ven55794a42006-07-10 04:44:03 -07004050 print_lock(hlock);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004051 lockdep_print_held_locks(curr);
4052
4053 printk("\nstack backtrace:\n");
4054 dump_stack();
4055}
4056
Oleg Nesterov54561782007-12-05 15:46:09 +01004057static inline int not_in_range(const void* mem_from, unsigned long mem_len,
4058 const void* lock_from, unsigned long lock_len)
4059{
4060 return lock_from + lock_len <= mem_from ||
4061 mem_from + mem_len <= lock_from;
4062}
4063
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004064/*
4065 * Called when kernel memory is freed (or unmapped), or if a lock
4066 * is destroyed or reinitialized - this code checks whether there is
4067 * any held lock in the memory range of <from> to <to>:
4068 */
4069void debug_check_no_locks_freed(const void *mem_from, unsigned long mem_len)
4070{
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004071 struct task_struct *curr = current;
4072 struct held_lock *hlock;
4073 unsigned long flags;
4074 int i;
4075
4076 if (unlikely(!debug_locks))
4077 return;
4078
4079 local_irq_save(flags);
4080 for (i = 0; i < curr->lockdep_depth; i++) {
4081 hlock = curr->held_locks + i;
4082
Oleg Nesterov54561782007-12-05 15:46:09 +01004083 if (not_in_range(mem_from, mem_len, hlock->instance,
4084 sizeof(*hlock->instance)))
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004085 continue;
4086
Oleg Nesterov54561782007-12-05 15:46:09 +01004087 print_freed_lock_bug(curr, mem_from, mem_from + mem_len, hlock);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004088 break;
4089 }
4090 local_irq_restore(flags);
4091}
Peter Zijlstraed075362006-12-06 20:35:24 -08004092EXPORT_SYMBOL_GPL(debug_check_no_locks_freed);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004093
Colin Cross1b1d2fb2013-05-06 23:50:08 +00004094static void print_held_locks_bug(void)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004095{
4096 if (!debug_locks_off())
4097 return;
4098 if (debug_locks_silent)
4099 return;
4100
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004101 printk("\n");
4102 printk("=====================================\n");
Colin Cross1b1d2fb2013-05-06 23:50:08 +00004103 printk("[ BUG: %s/%d still has locks held! ]\n",
4104 current->comm, task_pid_nr(current));
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01004105 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004106 printk("-------------------------------------\n");
Colin Cross1b1d2fb2013-05-06 23:50:08 +00004107 lockdep_print_held_locks(current);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004108 printk("\nstack backtrace:\n");
4109 dump_stack();
4110}
4111
Colin Cross1b1d2fb2013-05-06 23:50:08 +00004112void debug_check_no_locks_held(void)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004113{
Colin Cross1b1d2fb2013-05-06 23:50:08 +00004114 if (unlikely(current->lockdep_depth > 0))
4115 print_held_locks_bug();
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004116}
Colin Cross1b1d2fb2013-05-06 23:50:08 +00004117EXPORT_SYMBOL_GPL(debug_check_no_locks_held);
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004118
Sasha Levin8dce7a92013-06-13 18:41:16 -04004119#ifdef __KERNEL__
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004120void debug_show_all_locks(void)
4121{
4122 struct task_struct *g, *p;
4123 int count = 10;
4124 int unlock = 1;
4125
Jarek Poplawski9c35dd72007-03-22 00:11:28 -08004126 if (unlikely(!debug_locks)) {
4127 printk("INFO: lockdep is turned off.\n");
4128 return;
4129 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004130 printk("\nShowing all locks held in the system:\n");
4131
4132 /*
4133 * Here we try to get the tasklist_lock as hard as possible,
4134 * if not successful after 2 seconds we ignore it (but keep
4135 * trying). This is to enable a debug printout even if a
4136 * tasklist_lock-holding task deadlocks or crashes.
4137 */
4138retry:
4139 if (!read_trylock(&tasklist_lock)) {
4140 if (count == 10)
4141 printk("hm, tasklist_lock locked, retrying... ");
4142 if (count) {
4143 count--;
4144 printk(" #%d", 10-count);
4145 mdelay(200);
4146 goto retry;
4147 }
4148 printk(" ignoring it.\n");
4149 unlock = 0;
qinghuang feng46fec7a2008-10-28 17:24:28 +08004150 } else {
4151 if (count != 10)
4152 printk(KERN_CONT " locked it.\n");
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004153 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004154
4155 do_each_thread(g, p) {
Ingo Molnar85684872007-12-05 15:46:09 +01004156 /*
4157 * It's not reliable to print a task's held locks
4158 * if it's not sleeping (or if it's not the current
4159 * task):
4160 */
4161 if (p->state == TASK_RUNNING && p != current)
4162 continue;
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004163 if (p->lockdep_depth)
4164 lockdep_print_held_locks(p);
4165 if (!unlock)
4166 if (read_trylock(&tasklist_lock))
4167 unlock = 1;
4168 } while_each_thread(g, p);
4169
4170 printk("\n");
4171 printk("=============================================\n\n");
4172
4173 if (unlock)
4174 read_unlock(&tasklist_lock);
4175}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004176EXPORT_SYMBOL_GPL(debug_show_all_locks);
Sasha Levin8dce7a92013-06-13 18:41:16 -04004177#endif
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004178
Ingo Molnar82a1fcb2008-01-25 21:08:02 +01004179/*
4180 * Careful: only use this function if you are sure that
4181 * the task cannot run in parallel!
4182 */
John Kacurf1b499f2010-08-05 17:10:53 +02004183void debug_show_held_locks(struct task_struct *task)
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004184{
Jarek Poplawski9c35dd72007-03-22 00:11:28 -08004185 if (unlikely(!debug_locks)) {
4186 printk("INFO: lockdep is turned off.\n");
4187 return;
4188 }
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004189 lockdep_print_held_locks(task);
4190}
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07004191EXPORT_SYMBOL_GPL(debug_show_held_locks);
Peter Zijlstrab351d162007-10-11 22:11:12 +02004192
Andi Kleen722a9f92014-05-02 00:44:38 +02004193asmlinkage __visible void lockdep_sys_exit(void)
Peter Zijlstrab351d162007-10-11 22:11:12 +02004194{
4195 struct task_struct *curr = current;
4196
4197 if (unlikely(curr->lockdep_depth)) {
4198 if (!debug_locks_off())
4199 return;
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004200 printk("\n");
4201 printk("================================================\n");
4202 printk("[ BUG: lock held when returning to user space! ]\n");
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01004203 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004204 printk("------------------------------------------------\n");
Peter Zijlstrab351d162007-10-11 22:11:12 +02004205 printk("%s/%d is leaving the kernel with locks still held!\n",
4206 curr->comm, curr->pid);
4207 lockdep_print_held_locks(curr);
4208 }
4209}
Paul E. McKenney0632eb32010-02-22 17:04:47 -08004210
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004211void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
Paul E. McKenney0632eb32010-02-22 17:04:47 -08004212{
4213 struct task_struct *curr = current;
4214
Lai Jiangshan2b3fc352010-04-20 16:23:07 +08004215#ifndef CONFIG_PROVE_RCU_REPEATEDLY
Paul E. McKenney0632eb32010-02-22 17:04:47 -08004216 if (!debug_locks_off())
4217 return;
Lai Jiangshan2b3fc352010-04-20 16:23:07 +08004218#endif /* #ifdef CONFIG_PROVE_RCU_REPEATEDLY */
4219 /* Note: the following can be executed concurrently, so be careful. */
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004220 printk("\n");
4221 printk("===============================\n");
4222 printk("[ INFO: suspicious RCU usage. ]\n");
Ben Hutchingsfbdc4b92011-10-28 04:36:55 +01004223 print_kernel_ident();
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004224 printk("-------------------------------\n");
4225 printk("%s:%d %s!\n", file, line, s);
Paul E. McKenney0632eb32010-02-22 17:04:47 -08004226 printk("\nother info that might help us debug this:\n\n");
Paul E. McKenneyc5fdcec2012-01-30 08:46:32 -08004227 printk("\n%srcu_scheduler_active = %d, debug_locks = %d\n",
4228 !rcu_lockdep_current_cpu_online()
4229 ? "RCU used illegally from offline CPU!\n"
Paul E. McKenney5c173eb2013-09-13 17:20:11 -07004230 : !rcu_is_watching()
Paul E. McKenneyc5fdcec2012-01-30 08:46:32 -08004231 ? "RCU used illegally from idle CPU!\n"
4232 : "",
4233 rcu_scheduler_active, debug_locks);
Frederic Weisbecker0464e932011-10-07 18:22:01 +02004234
4235 /*
4236 * If a CPU is in the RCU-free window in idle (ie: in the section
4237 * between rcu_idle_enter() and rcu_idle_exit(), then RCU
4238 * considers that CPU to be in an "extended quiescent state",
4239 * which means that RCU will be completely ignoring that CPU.
4240 * Therefore, rcu_read_lock() and friends have absolutely no
4241 * effect on a CPU running in that state. In other words, even if
4242 * such an RCU-idle CPU has called rcu_read_lock(), RCU might well
4243 * delete data structures out from under it. RCU really has no
4244 * choice here: we need to keep an RCU-free window in idle where
4245 * the CPU may possibly enter into low power mode. This way we can
4246 * notice an extended quiescent state to other CPUs that started a grace
4247 * period. Otherwise we would delay any grace period as long as we run
4248 * in the idle task.
4249 *
4250 * So complain bitterly if someone does call rcu_read_lock(),
4251 * rcu_read_lock_bh() and so on from extended quiescent states.
4252 */
Paul E. McKenney5c173eb2013-09-13 17:20:11 -07004253 if (!rcu_is_watching())
Frederic Weisbecker0464e932011-10-07 18:22:01 +02004254 printk("RCU used illegally from extended quiescent state!\n");
4255
Paul E. McKenney0632eb32010-02-22 17:04:47 -08004256 lockdep_print_held_locks(curr);
4257 printk("\nstack backtrace:\n");
4258 dump_stack();
4259}
Paul E. McKenneyb3fbab02011-05-24 08:31:09 -07004260EXPORT_SYMBOL_GPL(lockdep_rcu_suspicious);