Peter Zijlstra | b8a2162 | 2013-10-04 22:06:53 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Generic wait-for-completion handler; |
| 3 | * |
| 4 | * It differs from semaphores in that their default case is the opposite, |
| 5 | * wait_for_completion default blocks whereas semaphore default non-block. The |
| 6 | * interface also makes it easy to 'complete' multiple waiting threads, |
| 7 | * something which isn't entirely natural for semaphores. |
| 8 | * |
| 9 | * But more importantly, the primitive documents the usage. Semaphores would |
| 10 | * typically be used for exclusion which gives rise to priority inversion. |
| 11 | * Waiting for completion is a typically sync point, but not an exclusion point. |
| 12 | */ |
| 13 | |
| 14 | #include <linux/sched.h> |
| 15 | #include <linux/completion.h> |
| 16 | |
| 17 | /** |
| 18 | * complete: - signals a single thread waiting on this completion |
| 19 | * @x: holds the state of this particular completion |
| 20 | * |
| 21 | * This will wake up a single thread waiting on this completion. Threads will be |
| 22 | * awakened in the same order in which they were queued. |
| 23 | * |
| 24 | * See also complete_all(), wait_for_completion() and related routines. |
| 25 | * |
| 26 | * It may be assumed that this function implies a write memory barrier before |
| 27 | * changing the task state if and only if any tasks are woken up. |
| 28 | */ |
| 29 | void complete(struct completion *x) |
| 30 | { |
| 31 | unsigned long flags; |
| 32 | |
| 33 | spin_lock_irqsave(&x->wait.lock, flags); |
| 34 | x->done++; |
| 35 | __wake_up_locked(&x->wait, TASK_NORMAL, 1); |
| 36 | spin_unlock_irqrestore(&x->wait.lock, flags); |
| 37 | } |
| 38 | EXPORT_SYMBOL(complete); |
| 39 | |
| 40 | /** |
| 41 | * complete_all: - signals all threads waiting on this completion |
| 42 | * @x: holds the state of this particular completion |
| 43 | * |
| 44 | * This will wake up all threads waiting on this particular completion event. |
| 45 | * |
| 46 | * It may be assumed that this function implies a write memory barrier before |
| 47 | * changing the task state if and only if any tasks are woken up. |
| 48 | */ |
| 49 | void complete_all(struct completion *x) |
| 50 | { |
| 51 | unsigned long flags; |
| 52 | |
| 53 | spin_lock_irqsave(&x->wait.lock, flags); |
| 54 | x->done += UINT_MAX/2; |
| 55 | __wake_up_locked(&x->wait, TASK_NORMAL, 0); |
| 56 | spin_unlock_irqrestore(&x->wait.lock, flags); |
| 57 | } |
| 58 | EXPORT_SYMBOL(complete_all); |
| 59 | |
| 60 | static inline long __sched |
| 61 | do_wait_for_common(struct completion *x, |
| 62 | long (*action)(long), long timeout, int state) |
| 63 | { |
| 64 | if (!x->done) { |
| 65 | DECLARE_WAITQUEUE(wait, current); |
| 66 | |
| 67 | __add_wait_queue_tail_exclusive(&x->wait, &wait); |
| 68 | do { |
| 69 | if (signal_pending_state(state, current)) { |
| 70 | timeout = -ERESTARTSYS; |
| 71 | break; |
| 72 | } |
| 73 | __set_current_state(state); |
| 74 | spin_unlock_irq(&x->wait.lock); |
| 75 | timeout = action(timeout); |
| 76 | spin_lock_irq(&x->wait.lock); |
| 77 | } while (!x->done && timeout); |
| 78 | __remove_wait_queue(&x->wait, &wait); |
| 79 | if (!x->done) |
| 80 | return timeout; |
| 81 | } |
| 82 | x->done--; |
| 83 | return timeout ?: 1; |
| 84 | } |
| 85 | |
| 86 | static inline long __sched |
| 87 | __wait_for_common(struct completion *x, |
| 88 | long (*action)(long), long timeout, int state) |
| 89 | { |
| 90 | might_sleep(); |
| 91 | |
| 92 | spin_lock_irq(&x->wait.lock); |
| 93 | timeout = do_wait_for_common(x, action, timeout, state); |
| 94 | spin_unlock_irq(&x->wait.lock); |
| 95 | return timeout; |
| 96 | } |
| 97 | |
| 98 | static long __sched |
| 99 | wait_for_common(struct completion *x, long timeout, int state) |
| 100 | { |
| 101 | return __wait_for_common(x, schedule_timeout, timeout, state); |
| 102 | } |
| 103 | |
| 104 | static long __sched |
| 105 | wait_for_common_io(struct completion *x, long timeout, int state) |
| 106 | { |
| 107 | return __wait_for_common(x, io_schedule_timeout, timeout, state); |
| 108 | } |
| 109 | |
| 110 | /** |
| 111 | * wait_for_completion: - waits for completion of a task |
| 112 | * @x: holds the state of this particular completion |
| 113 | * |
| 114 | * This waits to be signaled for completion of a specific task. It is NOT |
| 115 | * interruptible and there is no timeout. |
| 116 | * |
| 117 | * See also similar routines (i.e. wait_for_completion_timeout()) with timeout |
| 118 | * and interrupt capability. Also see complete(). |
| 119 | */ |
| 120 | void __sched wait_for_completion(struct completion *x) |
| 121 | { |
| 122 | wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); |
| 123 | } |
| 124 | EXPORT_SYMBOL(wait_for_completion); |
| 125 | |
| 126 | /** |
| 127 | * wait_for_completion_timeout: - waits for completion of a task (w/timeout) |
| 128 | * @x: holds the state of this particular completion |
| 129 | * @timeout: timeout value in jiffies |
| 130 | * |
| 131 | * This waits for either a completion of a specific task to be signaled or for a |
| 132 | * specified timeout to expire. The timeout is in jiffies. It is not |
| 133 | * interruptible. |
| 134 | * |
| 135 | * Return: 0 if timed out, and positive (at least 1, or number of jiffies left |
| 136 | * till timeout) if completed. |
| 137 | */ |
| 138 | unsigned long __sched |
| 139 | wait_for_completion_timeout(struct completion *x, unsigned long timeout) |
| 140 | { |
| 141 | return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); |
| 142 | } |
| 143 | EXPORT_SYMBOL(wait_for_completion_timeout); |
| 144 | |
| 145 | /** |
| 146 | * wait_for_completion_io: - waits for completion of a task |
| 147 | * @x: holds the state of this particular completion |
| 148 | * |
| 149 | * This waits to be signaled for completion of a specific task. It is NOT |
| 150 | * interruptible and there is no timeout. The caller is accounted as waiting |
Wolfram Sang | a1bd537 | 2014-11-04 12:01:41 +0100 | [diff] [blame] | 151 | * for IO (which traditionally means blkio only). |
Peter Zijlstra | b8a2162 | 2013-10-04 22:06:53 +0200 | [diff] [blame] | 152 | */ |
| 153 | void __sched wait_for_completion_io(struct completion *x) |
| 154 | { |
| 155 | wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); |
| 156 | } |
| 157 | EXPORT_SYMBOL(wait_for_completion_io); |
| 158 | |
| 159 | /** |
| 160 | * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) |
| 161 | * @x: holds the state of this particular completion |
| 162 | * @timeout: timeout value in jiffies |
| 163 | * |
| 164 | * This waits for either a completion of a specific task to be signaled or for a |
| 165 | * specified timeout to expire. The timeout is in jiffies. It is not |
Wolfram Sang | a1bd537 | 2014-11-04 12:01:41 +0100 | [diff] [blame] | 166 | * interruptible. The caller is accounted as waiting for IO (which traditionally |
| 167 | * means blkio only). |
Peter Zijlstra | b8a2162 | 2013-10-04 22:06:53 +0200 | [diff] [blame] | 168 | * |
| 169 | * Return: 0 if timed out, and positive (at least 1, or number of jiffies left |
| 170 | * till timeout) if completed. |
| 171 | */ |
| 172 | unsigned long __sched |
| 173 | wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) |
| 174 | { |
| 175 | return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); |
| 176 | } |
| 177 | EXPORT_SYMBOL(wait_for_completion_io_timeout); |
| 178 | |
| 179 | /** |
| 180 | * wait_for_completion_interruptible: - waits for completion of a task (w/intr) |
| 181 | * @x: holds the state of this particular completion |
| 182 | * |
| 183 | * This waits for completion of a specific task to be signaled. It is |
| 184 | * interruptible. |
| 185 | * |
| 186 | * Return: -ERESTARTSYS if interrupted, 0 if completed. |
| 187 | */ |
| 188 | int __sched wait_for_completion_interruptible(struct completion *x) |
| 189 | { |
| 190 | long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); |
| 191 | if (t == -ERESTARTSYS) |
| 192 | return t; |
| 193 | return 0; |
| 194 | } |
| 195 | EXPORT_SYMBOL(wait_for_completion_interruptible); |
| 196 | |
| 197 | /** |
| 198 | * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) |
| 199 | * @x: holds the state of this particular completion |
| 200 | * @timeout: timeout value in jiffies |
| 201 | * |
| 202 | * This waits for either a completion of a specific task to be signaled or for a |
| 203 | * specified timeout to expire. It is interruptible. The timeout is in jiffies. |
| 204 | * |
| 205 | * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, |
| 206 | * or number of jiffies left till timeout) if completed. |
| 207 | */ |
| 208 | long __sched |
| 209 | wait_for_completion_interruptible_timeout(struct completion *x, |
| 210 | unsigned long timeout) |
| 211 | { |
| 212 | return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); |
| 213 | } |
| 214 | EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); |
| 215 | |
| 216 | /** |
| 217 | * wait_for_completion_killable: - waits for completion of a task (killable) |
| 218 | * @x: holds the state of this particular completion |
| 219 | * |
| 220 | * This waits to be signaled for completion of a specific task. It can be |
| 221 | * interrupted by a kill signal. |
| 222 | * |
| 223 | * Return: -ERESTARTSYS if interrupted, 0 if completed. |
| 224 | */ |
| 225 | int __sched wait_for_completion_killable(struct completion *x) |
| 226 | { |
| 227 | long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); |
| 228 | if (t == -ERESTARTSYS) |
| 229 | return t; |
| 230 | return 0; |
| 231 | } |
| 232 | EXPORT_SYMBOL(wait_for_completion_killable); |
| 233 | |
| 234 | /** |
| 235 | * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) |
| 236 | * @x: holds the state of this particular completion |
| 237 | * @timeout: timeout value in jiffies |
| 238 | * |
| 239 | * This waits for either a completion of a specific task to be |
| 240 | * signaled or for a specified timeout to expire. It can be |
| 241 | * interrupted by a kill signal. The timeout is in jiffies. |
| 242 | * |
| 243 | * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, |
| 244 | * or number of jiffies left till timeout) if completed. |
| 245 | */ |
| 246 | long __sched |
| 247 | wait_for_completion_killable_timeout(struct completion *x, |
| 248 | unsigned long timeout) |
| 249 | { |
| 250 | return wait_for_common(x, timeout, TASK_KILLABLE); |
| 251 | } |
| 252 | EXPORT_SYMBOL(wait_for_completion_killable_timeout); |
| 253 | |
| 254 | /** |
| 255 | * try_wait_for_completion - try to decrement a completion without blocking |
| 256 | * @x: completion structure |
| 257 | * |
| 258 | * Return: 0 if a decrement cannot be done without blocking |
| 259 | * 1 if a decrement succeeded. |
| 260 | * |
| 261 | * If a completion is being used as a counting completion, |
| 262 | * attempt to decrement the counter without blocking. This |
| 263 | * enables us to avoid waiting if the resource the completion |
| 264 | * is protecting is not available. |
| 265 | */ |
| 266 | bool try_wait_for_completion(struct completion *x) |
| 267 | { |
| 268 | unsigned long flags; |
| 269 | int ret = 1; |
| 270 | |
Nicholas Mc Guire | 7c34e31 | 2015-01-23 12:41:47 +0100 | [diff] [blame] | 271 | /* |
| 272 | * Since x->done will need to be locked only |
| 273 | * in the non-blocking case, we check x->done |
| 274 | * first without taking the lock so we can |
| 275 | * return early in the blocking case. |
| 276 | */ |
Oleg Nesterov | bc95601 | 2015-02-12 20:59:13 +0100 | [diff] [blame] | 277 | if (!READ_ONCE(x->done)) |
Nicholas Mc Guire | 7c34e31 | 2015-01-23 12:41:47 +0100 | [diff] [blame] | 278 | return 0; |
| 279 | |
Peter Zijlstra | b8a2162 | 2013-10-04 22:06:53 +0200 | [diff] [blame] | 280 | spin_lock_irqsave(&x->wait.lock, flags); |
| 281 | if (!x->done) |
| 282 | ret = 0; |
| 283 | else |
| 284 | x->done--; |
| 285 | spin_unlock_irqrestore(&x->wait.lock, flags); |
| 286 | return ret; |
| 287 | } |
| 288 | EXPORT_SYMBOL(try_wait_for_completion); |
| 289 | |
| 290 | /** |
| 291 | * completion_done - Test to see if a completion has any waiters |
| 292 | * @x: completion structure |
| 293 | * |
| 294 | * Return: 0 if there are waiters (wait_for_completion() in progress) |
| 295 | * 1 if there are no waiters. |
| 296 | * |
| 297 | */ |
| 298 | bool completion_done(struct completion *x) |
| 299 | { |
Oleg Nesterov | bc95601 | 2015-02-12 20:59:13 +0100 | [diff] [blame] | 300 | if (!READ_ONCE(x->done)) |
| 301 | return false; |
| 302 | |
| 303 | /* |
| 304 | * If ->done, we need to wait for complete() to release ->wait.lock |
| 305 | * otherwise we can end up freeing the completion before complete() |
| 306 | * is done referencing it. |
| 307 | * |
| 308 | * The RMB pairs with complete()'s RELEASE of ->wait.lock and orders |
| 309 | * the loads of ->done and ->wait.lock such that we cannot observe |
| 310 | * the lock before complete() acquires it while observing the ->done |
| 311 | * after it's acquired the lock. |
| 312 | */ |
| 313 | smp_rmb(); |
| 314 | spin_unlock_wait(&x->wait.lock); |
| 315 | return true; |
Peter Zijlstra | b8a2162 | 2013-10-04 22:06:53 +0200 | [diff] [blame] | 316 | } |
| 317 | EXPORT_SYMBOL(completion_done); |