Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for i486+ |
| 2 | * |
| 3 | * Written by David Howells (dhowells@redhat.com). |
| 4 | * |
Dave Jones | 99122a3 | 2008-01-30 13:30:28 +0100 | [diff] [blame] | 5 | * Derived from asm-x86/semaphore.h |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 6 | * |
| 7 | * |
| 8 | * The MSW of the count is the negated number of active writers and waiting |
| 9 | * lockers, and the LSW is the total number of active locks |
| 10 | * |
| 11 | * The lock count is initialized to 0 (no active and no waiting lockers). |
| 12 | * |
| 13 | * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an |
| 14 | * uncontended lock. This can be determined because XADD returns the old value. |
| 15 | * Readers increment by 1 and see a positive value when uncontended, negative |
| 16 | * if there are writers (and maybe) readers waiting (in which case it goes to |
| 17 | * sleep). |
| 18 | * |
| 19 | * The value of WAITING_BIAS supports up to 32766 waiting processes. This can |
| 20 | * be extended to 65534 by manually checking the whole MSW rather than relying |
| 21 | * on the S flag. |
| 22 | * |
| 23 | * The value of ACTIVE_BIAS supports up to 65535 active processes. |
| 24 | * |
| 25 | * This should be totally fair - if anything is waiting, a process that wants a |
| 26 | * lock will go to the back of the queue. When the currently active lock is |
| 27 | * released, if there's a writer at the front of the queue, then that and only |
Adam Buchbinder | 6a6256f | 2016-02-23 15:34:30 -0800 | [diff] [blame] | 28 | * that will be woken up; if there's a bunch of consecutive readers at the |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 29 | * front, then they'll all be woken up, but no other readers will be. |
| 30 | */ |
| 31 | |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 32 | #ifndef _ASM_X86_RWSEM_H |
| 33 | #define _ASM_X86_RWSEM_H |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 34 | |
| 35 | #ifndef _LINUX_RWSEM_H |
| 36 | #error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead" |
| 37 | #endif |
| 38 | |
| 39 | #ifdef __KERNEL__ |
H. Peter Anvin | 1838ef1 | 2010-01-18 14:00:34 -0800 | [diff] [blame] | 40 | #include <asm/asm.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 41 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 42 | /* |
H. Peter Anvin | 1838ef1 | 2010-01-18 14:00:34 -0800 | [diff] [blame] | 43 | * The bias values and the counter type limits the number of |
| 44 | * potential readers/writers to 32767 for 32 bits and 2147483647 |
| 45 | * for 64 bits. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 46 | */ |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 47 | |
H. Peter Anvin | 1838ef1 | 2010-01-18 14:00:34 -0800 | [diff] [blame] | 48 | #ifdef CONFIG_X86_64 |
| 49 | # define RWSEM_ACTIVE_MASK 0xffffffffL |
| 50 | #else |
| 51 | # define RWSEM_ACTIVE_MASK 0x0000ffffL |
| 52 | #endif |
| 53 | |
| 54 | #define RWSEM_UNLOCKED_VALUE 0x00000000L |
| 55 | #define RWSEM_ACTIVE_BIAS 0x00000001L |
| 56 | #define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 57 | #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS |
| 58 | #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 59 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 60 | /* |
| 61 | * lock for reading |
| 62 | */ |
| 63 | static inline void __down_read(struct rw_semaphore *sem) |
| 64 | { |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 65 | asm volatile("# beginning down_read\n\t" |
H. Peter Anvin | 1838ef1 | 2010-01-18 14:00:34 -0800 | [diff] [blame] | 66 | LOCK_PREFIX _ASM_INC "(%1)\n\t" |
Michel Lespinasse | b4bcb4c | 2010-07-20 15:19:45 -0700 | [diff] [blame] | 67 | /* adds 0x00000001 */ |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 68 | " jns 1f\n" |
| 69 | " call call_rwsem_down_read_failed\n" |
| 70 | "1:\n\t" |
| 71 | "# ending down_read\n\t" |
| 72 | : "+m" (sem->count) |
| 73 | : "a" (sem) |
| 74 | : "memory", "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 75 | } |
| 76 | |
| 77 | /* |
| 78 | * trylock for reading -- returns 1 if successful, 0 if contention |
| 79 | */ |
| 80 | static inline int __down_read_trylock(struct rw_semaphore *sem) |
| 81 | { |
Thomas Gleixner | bde11ef | 2011-01-26 20:05:53 +0000 | [diff] [blame] | 82 | long result, tmp; |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 83 | asm volatile("# beginning __down_read_trylock\n\t" |
Linus Torvalds | 59c33fa | 2010-01-12 16:21:09 -0800 | [diff] [blame] | 84 | " mov %0,%1\n\t" |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 85 | "1:\n\t" |
Linus Torvalds | 59c33fa | 2010-01-12 16:21:09 -0800 | [diff] [blame] | 86 | " mov %1,%2\n\t" |
| 87 | " add %3,%2\n\t" |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 88 | " jle 2f\n\t" |
Linus Torvalds | 59c33fa | 2010-01-12 16:21:09 -0800 | [diff] [blame] | 89 | LOCK_PREFIX " cmpxchg %2,%0\n\t" |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 90 | " jnz 1b\n\t" |
| 91 | "2:\n\t" |
| 92 | "# ending __down_read_trylock\n\t" |
| 93 | : "+m" (sem->count), "=&a" (result), "=&r" (tmp) |
| 94 | : "i" (RWSEM_ACTIVE_READ_BIAS) |
| 95 | : "memory", "cc"); |
| 96 | return result >= 0 ? 1 : 0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 97 | } |
| 98 | |
| 99 | /* |
| 100 | * lock for writing |
| 101 | */ |
Ingo Molnar | 4ea2176 | 2006-07-03 00:24:53 -0700 | [diff] [blame] | 102 | static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 103 | { |
Thomas Gleixner | bde11ef | 2011-01-26 20:05:53 +0000 | [diff] [blame] | 104 | long tmp; |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 105 | asm volatile("# beginning down_write\n\t" |
Linus Torvalds | 59c33fa | 2010-01-12 16:21:09 -0800 | [diff] [blame] | 106 | LOCK_PREFIX " xadd %1,(%2)\n\t" |
Michel Lespinasse | b4bcb4c | 2010-07-20 15:19:45 -0700 | [diff] [blame] | 107 | /* adds 0xffff0001, returns the old value */ |
Michel Lespinasse | a31a369 | 2013-05-07 06:46:01 -0700 | [diff] [blame] | 108 | " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" |
| 109 | /* was the active mask 0 before? */ |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 110 | " jz 1f\n" |
| 111 | " call call_rwsem_down_write_failed\n" |
| 112 | "1:\n" |
| 113 | "# ending down_write" |
| 114 | : "+m" (sem->count), "=d" (tmp) |
Michel Lespinasse | b4bcb4c | 2010-07-20 15:19:45 -0700 | [diff] [blame] | 115 | : "a" (sem), "1" (RWSEM_ACTIVE_WRITE_BIAS) |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 116 | : "memory", "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 117 | } |
| 118 | |
Ingo Molnar | 4ea2176 | 2006-07-03 00:24:53 -0700 | [diff] [blame] | 119 | static inline void __down_write(struct rw_semaphore *sem) |
| 120 | { |
| 121 | __down_write_nested(sem, 0); |
| 122 | } |
| 123 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 124 | /* |
| 125 | * trylock for writing -- returns 1 if successful, 0 if contention |
| 126 | */ |
| 127 | static inline int __down_write_trylock(struct rw_semaphore *sem) |
| 128 | { |
Michel Lespinasse | a31a369 | 2013-05-07 06:46:01 -0700 | [diff] [blame] | 129 | long result, tmp; |
| 130 | asm volatile("# beginning __down_write_trylock\n\t" |
| 131 | " mov %0,%1\n\t" |
| 132 | "1:\n\t" |
| 133 | " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t" |
| 134 | /* was the active mask 0 before? */ |
| 135 | " jnz 2f\n\t" |
| 136 | " mov %1,%2\n\t" |
| 137 | " add %3,%2\n\t" |
| 138 | LOCK_PREFIX " cmpxchg %2,%0\n\t" |
| 139 | " jnz 1b\n\t" |
| 140 | "2:\n\t" |
| 141 | " sete %b1\n\t" |
| 142 | " movzbl %b1, %k1\n\t" |
| 143 | "# ending __down_write_trylock\n\t" |
| 144 | : "+m" (sem->count), "=&a" (result), "=&r" (tmp) |
| 145 | : "er" (RWSEM_ACTIVE_WRITE_BIAS) |
| 146 | : "memory", "cc"); |
| 147 | return result; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 148 | } |
| 149 | |
| 150 | /* |
| 151 | * unlock after reading |
| 152 | */ |
| 153 | static inline void __up_read(struct rw_semaphore *sem) |
| 154 | { |
Thomas Gleixner | bde11ef | 2011-01-26 20:05:53 +0000 | [diff] [blame] | 155 | long tmp; |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 156 | asm volatile("# beginning __up_read\n\t" |
Linus Torvalds | 59c33fa | 2010-01-12 16:21:09 -0800 | [diff] [blame] | 157 | LOCK_PREFIX " xadd %1,(%2)\n\t" |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 158 | /* subtracts 1, returns the old value */ |
| 159 | " jns 1f\n\t" |
Michel Lespinasse | b4bcb4c | 2010-07-20 15:19:45 -0700 | [diff] [blame] | 160 | " call call_rwsem_wake\n" /* expects old value in %edx */ |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 161 | "1:\n" |
| 162 | "# ending __up_read\n" |
| 163 | : "+m" (sem->count), "=d" (tmp) |
Michel Lespinasse | b4bcb4c | 2010-07-20 15:19:45 -0700 | [diff] [blame] | 164 | : "a" (sem), "1" (-RWSEM_ACTIVE_READ_BIAS) |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 165 | : "memory", "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 166 | } |
| 167 | |
| 168 | /* |
| 169 | * unlock after writing |
| 170 | */ |
| 171 | static inline void __up_write(struct rw_semaphore *sem) |
| 172 | { |
Thomas Gleixner | bde11ef | 2011-01-26 20:05:53 +0000 | [diff] [blame] | 173 | long tmp; |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 174 | asm volatile("# beginning __up_write\n\t" |
Linus Torvalds | 59c33fa | 2010-01-12 16:21:09 -0800 | [diff] [blame] | 175 | LOCK_PREFIX " xadd %1,(%2)\n\t" |
Michel Lespinasse | a751bd8 | 2010-07-20 15:19:45 -0700 | [diff] [blame] | 176 | /* subtracts 0xffff0001, returns the old value */ |
| 177 | " jns 1f\n\t" |
Michel Lespinasse | b4bcb4c | 2010-07-20 15:19:45 -0700 | [diff] [blame] | 178 | " call call_rwsem_wake\n" /* expects old value in %edx */ |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 179 | "1:\n\t" |
| 180 | "# ending __up_write\n" |
Linus Torvalds | 59c33fa | 2010-01-12 16:21:09 -0800 | [diff] [blame] | 181 | : "+m" (sem->count), "=d" (tmp) |
| 182 | : "a" (sem), "1" (-RWSEM_ACTIVE_WRITE_BIAS) |
| 183 | : "memory", "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 184 | } |
| 185 | |
| 186 | /* |
| 187 | * downgrade write lock to read lock |
| 188 | */ |
| 189 | static inline void __downgrade_write(struct rw_semaphore *sem) |
| 190 | { |
H. Peter Anvin | 1838ef1 | 2010-01-18 14:00:34 -0800 | [diff] [blame] | 191 | asm volatile("# beginning __downgrade_write\n\t" |
| 192 | LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t" |
Avi Kivity | 0d1622d | 2010-02-13 10:33:12 +0200 | [diff] [blame] | 193 | /* |
| 194 | * transitions 0xZZZZ0001 -> 0xYYYY0001 (i386) |
| 195 | * 0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64) |
| 196 | */ |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 197 | " jns 1f\n\t" |
| 198 | " call call_rwsem_downgrade_wake\n" |
| 199 | "1:\n\t" |
| 200 | "# ending __downgrade_write\n" |
| 201 | : "+m" (sem->count) |
Avi Kivity | 0d1622d | 2010-02-13 10:33:12 +0200 | [diff] [blame] | 202 | : "a" (sem), "er" (-RWSEM_WAITING_BIAS) |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 203 | : "memory", "cc"); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 204 | } |
| 205 | |
| 206 | /* |
| 207 | * implement atomic add functionality |
| 208 | */ |
Thomas Gleixner | bde11ef | 2011-01-26 20:05:53 +0000 | [diff] [blame] | 209 | static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 210 | { |
H. Peter Anvin | 1838ef1 | 2010-01-18 14:00:34 -0800 | [diff] [blame] | 211 | asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0" |
Joe Perches | 6e5609a | 2008-03-23 01:03:21 -0700 | [diff] [blame] | 212 | : "+m" (sem->count) |
H. Peter Anvin | 1838ef1 | 2010-01-18 14:00:34 -0800 | [diff] [blame] | 213 | : "er" (delta)); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 214 | } |
| 215 | |
| 216 | /* |
| 217 | * implement exchange and add functionality |
| 218 | */ |
Thomas Gleixner | bde11ef | 2011-01-26 20:05:53 +0000 | [diff] [blame] | 219 | static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 220 | { |
Jeremy Fitzhardinge | 8b8bc2f | 2011-08-23 16:59:58 -0700 | [diff] [blame] | 221 | return delta + xadd(&sem->count, delta); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 222 | } |
| 223 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 224 | #endif /* __KERNEL__ */ |
H. Peter Anvin | 1965aae | 2008-10-22 22:26:29 -0700 | [diff] [blame] | 225 | #endif /* _ASM_X86_RWSEM_H */ |