blob: 536ef0bef154e507aae4f738c01c52e7d166d798 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Copyright (C) 2003 Sistina Software Limited.
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01003 * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * This file is released under the GPL.
6 */
7
Linus Torvalds1da177e2005-04-16 15:20:36 -07008#include "dm-bio-list.h"
Jonathan Brassow06386bb2008-02-08 02:11:37 +00009#include "dm-bio-record.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070010
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/init.h>
12#include <linux/mempool.h>
13#include <linux/module.h>
14#include <linux/pagemap.h>
15#include <linux/slab.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070016#include <linux/workqueue.h>
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010017#include <linux/device-mapper.h>
Alasdair G Kergona765e202008-04-24 22:02:01 +010018#include <linux/dm-io.h>
19#include <linux/dm-dirty-log.h>
20#include <linux/dm-kcopyd.h>
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010021#include <linux/dm-region-hash.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
Alasdair G Kergon72d94862006-06-26 00:27:35 -070023#define DM_MSG_PREFIX "raid1"
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010024
25#define MAX_RECOVERY 1 /* Maximum number of regions recovered in parallel. */
Milan Broz88be1632007-05-09 02:33:04 -070026#define DM_IO_PAGES 64
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010027#define DM_KCOPYD_PAGES 64
Alasdair G Kergon72d94862006-06-26 00:27:35 -070028
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -070029#define DM_RAID1_HANDLE_ERRORS 0x01
Jonathan Brassowf44db672007-07-12 17:29:04 +010030#define errors_handled(p) ((p)->features & DM_RAID1_HANDLE_ERRORS)
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -070031
Jonathan E Brassow33184042006-11-08 17:44:44 -080032static DECLARE_WAIT_QUEUE_HEAD(_kmirrord_recovery_stopped);
Linus Torvalds1da177e2005-04-16 15:20:36 -070033
Linus Torvalds1da177e2005-04-16 15:20:36 -070034/*-----------------------------------------------------------------
Neil Browne4c8b3b2006-06-26 00:27:26 -070035 * Mirror set structures.
36 *---------------------------------------------------------------*/
Jonathan Brassow72f4b312008-02-08 02:11:29 +000037enum dm_raid1_error {
38 DM_RAID1_WRITE_ERROR,
39 DM_RAID1_SYNC_ERROR,
40 DM_RAID1_READ_ERROR
41};
42
Neil Browne4c8b3b2006-06-26 00:27:26 -070043struct mirror {
Jonathan Brassowaa5617c2007-10-19 22:47:58 +010044 struct mirror_set *ms;
Neil Browne4c8b3b2006-06-26 00:27:26 -070045 atomic_t error_count;
Al Viro39ed7ad2008-02-13 03:53:00 +000046 unsigned long error_type;
Neil Browne4c8b3b2006-06-26 00:27:26 -070047 struct dm_dev *dev;
48 sector_t offset;
49};
50
51struct mirror_set {
52 struct dm_target *ti;
53 struct list_head list;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010054
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -070055 uint64_t features;
Neil Browne4c8b3b2006-06-26 00:27:26 -070056
Jonathan Brassow72f4b312008-02-08 02:11:29 +000057 spinlock_t lock; /* protects the lists */
Neil Browne4c8b3b2006-06-26 00:27:26 -070058 struct bio_list reads;
59 struct bio_list writes;
Jonathan Brassow72f4b312008-02-08 02:11:29 +000060 struct bio_list failures;
Neil Browne4c8b3b2006-06-26 00:27:26 -070061
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010062 struct dm_region_hash *rh;
63 struct dm_kcopyd_client *kcopyd_client;
Milan Broz88be1632007-05-09 02:33:04 -070064 struct dm_io_client *io_client;
Jonathan Brassow06386bb2008-02-08 02:11:37 +000065 mempool_t *read_record_pool;
Milan Broz88be1632007-05-09 02:33:04 -070066
Neil Browne4c8b3b2006-06-26 00:27:26 -070067 /* recovery */
68 region_t nr_regions;
69 int in_sync;
Jonathan Brassowfc1ff952007-07-12 17:29:15 +010070 int log_failure;
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +000071 atomic_t suspend;
Neil Browne4c8b3b2006-06-26 00:27:26 -070072
Jonathan Brassow72f4b312008-02-08 02:11:29 +000073 atomic_t default_mirror; /* Default mirror */
Neil Browne4c8b3b2006-06-26 00:27:26 -070074
Holger Smolinski6ad36fe2007-05-09 02:32:50 -070075 struct workqueue_struct *kmirrord_wq;
76 struct work_struct kmirrord_work;
Mikulas Patockaa2aebe02008-04-24 22:10:42 +010077 struct timer_list timer;
78 unsigned long timer_pending;
79
Jonathan Brassow72f4b312008-02-08 02:11:29 +000080 struct work_struct trigger_event;
Holger Smolinski6ad36fe2007-05-09 02:32:50 -070081
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010082 unsigned nr_mirrors;
Neil Browne4c8b3b2006-06-26 00:27:26 -070083 struct mirror mirror[0];
84};
85
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010086static void wakeup_mirrord(void *context)
Linus Torvalds1da177e2005-04-16 15:20:36 -070087{
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010088 struct mirror_set *ms = context;
Linus Torvalds1da177e2005-04-16 15:20:36 -070089
Holger Smolinski6ad36fe2007-05-09 02:32:50 -070090 queue_work(ms->kmirrord_wq, &ms->kmirrord_work);
91}
92
Mikulas Patockaa2aebe02008-04-24 22:10:42 +010093static void delayed_wake_fn(unsigned long data)
94{
95 struct mirror_set *ms = (struct mirror_set *) data;
96
97 clear_bit(0, &ms->timer_pending);
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +010098 wakeup_mirrord(ms);
Mikulas Patockaa2aebe02008-04-24 22:10:42 +010099}
100
101static void delayed_wake(struct mirror_set *ms)
102{
103 if (test_and_set_bit(0, &ms->timer_pending))
104 return;
105
106 ms->timer.expires = jiffies + HZ / 5;
107 ms->timer.data = (unsigned long) ms;
108 ms->timer.function = delayed_wake_fn;
109 add_timer(&ms->timer);
110}
111
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100112static void wakeup_all_recovery_waiters(void *context)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700113{
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100114 wake_up_all(&_kmirrord_recovery_stopped);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700115}
116
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100117static void queue_bio(struct mirror_set *ms, struct bio *bio, int rw)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700118{
119 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120 int should_wake = 0;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100121 struct bio_list *bl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700122
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100123 bl = (rw == WRITE) ? &ms->writes : &ms->reads;
124 spin_lock_irqsave(&ms->lock, flags);
125 should_wake = !(bl->head);
126 bio_list_add(bl, bio);
127 spin_unlock_irqrestore(&ms->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700128
129 if (should_wake)
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100130 wakeup_mirrord(ms);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131}
132
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100133static void dispatch_bios(void *context, struct bio_list *bio_list)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700134{
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100135 struct mirror_set *ms = context;
136 struct bio *bio;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100138 while ((bio = bio_list_pop(bio_list)))
139 queue_bio(ms, bio, WRITE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140}
141
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000142#define MIN_READ_RECORDS 20
143struct dm_raid1_read_record {
144 struct mirror *m;
145 struct dm_bio_details details;
146};
147
Mikulas Patocka95f8fac2009-04-02 19:55:24 +0100148static struct kmem_cache *_dm_raid1_read_record_cache;
149
Linus Torvalds1da177e2005-04-16 15:20:36 -0700150/*
151 * Every mirror should look like this one.
152 */
153#define DEFAULT_MIRROR 0
154
155/*
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000156 * This is yucky. We squirrel the mirror struct away inside
157 * bi_next for read/write buffers. This is safe since the bh
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158 * doesn't get submitted to the lower levels of block layer.
159 */
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000160static struct mirror *bio_get_m(struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161{
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000162 return (struct mirror *) bio->bi_next;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163}
164
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000165static void bio_set_m(struct bio *bio, struct mirror *m)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166{
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000167 bio->bi_next = (struct bio *) m;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168}
169
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000170static struct mirror *get_default_mirror(struct mirror_set *ms)
171{
172 return &ms->mirror[atomic_read(&ms->default_mirror)];
173}
174
175static void set_default_mirror(struct mirror *m)
176{
177 struct mirror_set *ms = m->ms;
178 struct mirror *m0 = &(ms->mirror[0]);
179
180 atomic_set(&ms->default_mirror, m - m0);
181}
182
183/* fail_mirror
184 * @m: mirror device to fail
185 * @error_type: one of the enum's, DM_RAID1_*_ERROR
186 *
187 * If errors are being handled, record the type of
188 * error encountered for this device. If this type
189 * of error has already been recorded, we can return;
190 * otherwise, we must signal userspace by triggering
191 * an event. Additionally, if the device is the
192 * primary device, we must choose a new primary, but
193 * only if the mirror is in-sync.
194 *
195 * This function must not block.
196 */
197static void fail_mirror(struct mirror *m, enum dm_raid1_error error_type)
198{
199 struct mirror_set *ms = m->ms;
200 struct mirror *new;
201
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000202 /*
203 * error_count is used for nothing more than a
204 * simple way to tell if a device has encountered
205 * errors.
206 */
207 atomic_inc(&m->error_count);
208
209 if (test_and_set_bit(error_type, &m->error_type))
210 return;
211
Jonathan Brassowd460c652009-01-06 03:04:57 +0000212 if (!errors_handled(ms))
213 return;
214
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000215 if (m != get_default_mirror(ms))
216 goto out;
217
218 if (!ms->in_sync) {
219 /*
220 * Better to issue requests to same failing device
221 * than to risk returning corrupt data.
222 */
223 DMERR("Primary mirror (%s) failed while out-of-sync: "
224 "Reads may fail.", m->dev->name);
225 goto out;
226 }
227
228 for (new = ms->mirror; new < ms->mirror + ms->nr_mirrors; new++)
229 if (!atomic_read(&new->error_count)) {
230 set_default_mirror(new);
231 break;
232 }
233
234 if (unlikely(new == ms->mirror + ms->nr_mirrors))
235 DMWARN("All sides of mirror have failed.");
236
237out:
238 schedule_work(&ms->trigger_event);
239}
240
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241/*-----------------------------------------------------------------
242 * Recovery.
243 *
244 * When a mirror is first activated we may find that some regions
245 * are in the no-sync state. We have to recover these by
246 * recopying from the default mirror to all the others.
247 *---------------------------------------------------------------*/
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700248static void recovery_complete(int read_err, unsigned long write_err,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249 void *context)
250{
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100251 struct dm_region *reg = context;
252 struct mirror_set *ms = dm_rh_region_context(reg);
Jonathan Brassow8f0205b2008-02-08 02:11:32 +0000253 int m, bit = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254
Jonathan Brassow8f0205b2008-02-08 02:11:32 +0000255 if (read_err) {
Jonathan Brassowf44db672007-07-12 17:29:04 +0100256 /* Read error means the failure of default mirror. */
257 DMERR_LIMIT("Unable to read primary mirror during recovery");
Jonathan Brassow8f0205b2008-02-08 02:11:32 +0000258 fail_mirror(get_default_mirror(ms), DM_RAID1_SYNC_ERROR);
259 }
Jonathan Brassowf44db672007-07-12 17:29:04 +0100260
Jonathan Brassow8f0205b2008-02-08 02:11:32 +0000261 if (write_err) {
Alasdair G Kergon4cdc1d12008-03-28 14:16:10 -0700262 DMERR_LIMIT("Write error during recovery (error = 0x%lx)",
Jonathan Brassowf44db672007-07-12 17:29:04 +0100263 write_err);
Jonathan Brassow8f0205b2008-02-08 02:11:32 +0000264 /*
265 * Bits correspond to devices (excluding default mirror).
266 * The default mirror cannot change during recovery.
267 */
268 for (m = 0; m < ms->nr_mirrors; m++) {
269 if (&ms->mirror[m] == get_default_mirror(ms))
270 continue;
271 if (test_bit(bit, &write_err))
272 fail_mirror(ms->mirror + m,
273 DM_RAID1_SYNC_ERROR);
274 bit++;
275 }
276 }
Jonathan Brassowf44db672007-07-12 17:29:04 +0100277
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100278 dm_rh_recovery_end(reg, !(read_err || write_err));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279}
280
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100281static int recover(struct mirror_set *ms, struct dm_region *reg)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282{
283 int r;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100284 unsigned i;
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100285 struct dm_io_region from, to[DM_KCOPYD_MAX_REGIONS], *dest;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 struct mirror *m;
287 unsigned long flags = 0;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100288 region_t key = dm_rh_get_region_key(reg);
289 sector_t region_size = dm_rh_get_region_size(ms->rh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290
291 /* fill in the source */
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000292 m = get_default_mirror(ms);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293 from.bdev = m->dev->bdev;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100294 from.sector = m->offset + dm_rh_region_to_sector(ms->rh, key);
295 if (key == (ms->nr_regions - 1)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296 /*
297 * The final region may be smaller than
298 * region_size.
299 */
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100300 from.count = ms->ti->len & (region_size - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 if (!from.count)
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100302 from.count = region_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 } else
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100304 from.count = region_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305
306 /* fill in the destinations */
307 for (i = 0, dest = to; i < ms->nr_mirrors; i++) {
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000308 if (&ms->mirror[i] == get_default_mirror(ms))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 continue;
310
311 m = ms->mirror + i;
312 dest->bdev = m->dev->bdev;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100313 dest->sector = m->offset + dm_rh_region_to_sector(ms->rh, key);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700314 dest->count = from.count;
315 dest++;
316 }
317
318 /* hand to kcopyd */
Jonathan Brassowf7c83e22008-10-10 13:36:59 +0100319 if (!errors_handled(ms))
320 set_bit(DM_KCOPYD_IGNORE_ERROR, &flags);
321
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100322 r = dm_kcopyd_copy(ms->kcopyd_client, &from, ms->nr_mirrors - 1, to,
323 flags, recovery_complete, reg);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324
325 return r;
326}
327
328static void do_recovery(struct mirror_set *ms)
329{
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100330 struct dm_region *reg;
331 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332 int r;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333
334 /*
335 * Start quiescing some regions.
336 */
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100337 dm_rh_recovery_prepare(ms->rh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338
339 /*
340 * Copy any already quiesced regions.
341 */
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100342 while ((reg = dm_rh_recovery_start(ms->rh))) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 r = recover(ms, reg);
344 if (r)
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100345 dm_rh_recovery_end(reg, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 }
347
348 /*
349 * Update the in sync flag.
350 */
351 if (!ms->in_sync &&
352 (log->type->get_sync_count(log) == ms->nr_regions)) {
353 /* the sync is complete */
354 dm_table_event(ms->ti->table);
355 ms->in_sync = 1;
356 }
357}
358
359/*-----------------------------------------------------------------
360 * Reads
361 *---------------------------------------------------------------*/
362static struct mirror *choose_mirror(struct mirror_set *ms, sector_t sector)
363{
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000364 struct mirror *m = get_default_mirror(ms);
365
366 do {
367 if (likely(!atomic_read(&m->error_count)))
368 return m;
369
370 if (m-- == ms->mirror)
371 m += ms->nr_mirrors;
372 } while (m != get_default_mirror(ms));
373
374 return NULL;
375}
376
377static int default_ok(struct mirror *m)
378{
379 struct mirror *default_mirror = get_default_mirror(m->ms);
380
381 return !atomic_read(&default_mirror->error_count);
382}
383
384static int mirror_available(struct mirror_set *ms, struct bio *bio)
385{
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100386 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
387 region_t region = dm_rh_bio_to_region(ms->rh, bio);
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000388
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100389 if (log->type->in_sync(log, region, 0))
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000390 return choose_mirror(ms, bio->bi_sector) ? 1 : 0;
391
392 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393}
394
395/*
396 * remap a buffer to a particular mirror.
397 */
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000398static sector_t map_sector(struct mirror *m, struct bio *bio)
399{
400 return m->offset + (bio->bi_sector - m->ms->ti->begin);
401}
402
403static void map_bio(struct mirror *m, struct bio *bio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404{
405 bio->bi_bdev = m->dev->bdev;
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000406 bio->bi_sector = map_sector(m, bio);
407}
408
Heinz Mauelshagen22a1ceb2008-04-24 21:43:17 +0100409static void map_region(struct dm_io_region *io, struct mirror *m,
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000410 struct bio *bio)
411{
412 io->bdev = m->dev->bdev;
413 io->sector = map_sector(m, bio);
414 io->count = bio->bi_size >> 9;
415}
416
417/*-----------------------------------------------------------------
418 * Reads
419 *---------------------------------------------------------------*/
420static void read_callback(unsigned long error, void *context)
421{
422 struct bio *bio = context;
423 struct mirror *m;
424
425 m = bio_get_m(bio);
426 bio_set_m(bio, NULL);
427
428 if (likely(!error)) {
429 bio_endio(bio, 0);
430 return;
431 }
432
433 fail_mirror(m, DM_RAID1_READ_ERROR);
434
435 if (likely(default_ok(m)) || mirror_available(m->ms, bio)) {
436 DMWARN_LIMIT("Read failure on mirror device %s. "
437 "Trying alternative device.",
438 m->dev->name);
439 queue_bio(m->ms, bio, bio_rw(bio));
440 return;
441 }
442
443 DMERR_LIMIT("Read failure on mirror device %s. Failing I/O.",
444 m->dev->name);
445 bio_endio(bio, -EIO);
446}
447
448/* Asynchronous read. */
449static void read_async_bio(struct mirror *m, struct bio *bio)
450{
Heinz Mauelshagen22a1ceb2008-04-24 21:43:17 +0100451 struct dm_io_region io;
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000452 struct dm_io_request io_req = {
453 .bi_rw = READ,
454 .mem.type = DM_IO_BVEC,
455 .mem.ptr.bvec = bio->bi_io_vec + bio->bi_idx,
456 .notify.fn = read_callback,
457 .notify.context = bio,
458 .client = m->ms->io_client,
459 };
460
461 map_region(&io, m, bio);
462 bio_set_m(bio, m);
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100463 BUG_ON(dm_io(&io_req, 1, &io, NULL));
464}
465
466static inline int region_in_sync(struct mirror_set *ms, region_t region,
467 int may_block)
468{
469 int state = dm_rh_get_state(ms->rh, region, may_block);
470 return state == DM_RH_CLEAN || state == DM_RH_DIRTY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700471}
472
473static void do_reads(struct mirror_set *ms, struct bio_list *reads)
474{
475 region_t region;
476 struct bio *bio;
477 struct mirror *m;
478
479 while ((bio = bio_list_pop(reads))) {
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100480 region = dm_rh_bio_to_region(ms->rh, bio);
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000481 m = get_default_mirror(ms);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700482
483 /*
484 * We can only read balance if the region is in sync.
485 */
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100486 if (likely(region_in_sync(ms, region, 1)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700487 m = choose_mirror(ms, bio->bi_sector);
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000488 else if (m && atomic_read(&m->error_count))
489 m = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700490
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000491 if (likely(m))
492 read_async_bio(m, bio);
493 else
494 bio_endio(bio, -EIO);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700495 }
496}
497
498/*-----------------------------------------------------------------
499 * Writes.
500 *
501 * We do different things with the write io depending on the
502 * state of the region that it's in:
503 *
504 * SYNC: increment pending, use kcopyd to write to *all* mirrors
505 * RECOVERING: delay the io until recovery completes
506 * NOSYNC: increment pending, just write to the default mirror
507 *---------------------------------------------------------------*/
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000508
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000509
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510static void write_callback(unsigned long error, void *context)
511{
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000512 unsigned i, ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513 struct bio *bio = (struct bio *) context;
514 struct mirror_set *ms;
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000515 int uptodate = 0;
516 int should_wake = 0;
517 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000519 ms = bio_get_m(bio)->ms;
520 bio_set_m(bio, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700521
522 /*
523 * NOTE: We don't decrement the pending count here,
524 * instead it is done by the targets endio function.
525 * This way we handle both writes to SYNC and NOSYNC
526 * regions with the same code.
527 */
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000528 if (likely(!error))
529 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000531 for (i = 0; i < ms->nr_mirrors; i++)
532 if (test_bit(i, &error))
533 fail_mirror(ms->mirror + i, DM_RAID1_WRITE_ERROR);
534 else
535 uptodate = 1;
536
537 if (unlikely(!uptodate)) {
538 DMERR("All replicated volumes dead, failing I/O");
539 /* None of the writes succeeded, fail the I/O. */
540 ret = -EIO;
541 } else if (errors_handled(ms)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542 /*
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000543 * Need to raise event. Since raising
544 * events can block, we need to do it in
545 * the main thread.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546 */
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000547 spin_lock_irqsave(&ms->lock, flags);
548 if (!ms->failures.head)
549 should_wake = 1;
550 bio_list_add(&ms->failures, bio);
551 spin_unlock_irqrestore(&ms->lock, flags);
552 if (should_wake)
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100553 wakeup_mirrord(ms);
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000554 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 }
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000556out:
557 bio_endio(bio, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700558}
559
560static void do_write(struct mirror_set *ms, struct bio *bio)
561{
562 unsigned int i;
Heinz Mauelshagen22a1ceb2008-04-24 21:43:17 +0100563 struct dm_io_region io[ms->nr_mirrors], *dest = io;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 struct mirror *m;
Milan Broz88be1632007-05-09 02:33:04 -0700565 struct dm_io_request io_req = {
566 .bi_rw = WRITE,
567 .mem.type = DM_IO_BVEC,
568 .mem.ptr.bvec = bio->bi_io_vec + bio->bi_idx,
569 .notify.fn = write_callback,
570 .notify.context = bio,
571 .client = ms->io_client,
572 };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000574 for (i = 0, m = ms->mirror; i < ms->nr_mirrors; i++, m++)
575 map_region(dest++, m, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000577 /*
578 * Use default mirror because we only need it to retrieve the reference
579 * to the mirror set in write_callback().
580 */
581 bio_set_m(bio, get_default_mirror(ms));
Milan Broz88be1632007-05-09 02:33:04 -0700582
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100583 BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584}
585
586static void do_writes(struct mirror_set *ms, struct bio_list *writes)
587{
588 int state;
589 struct bio *bio;
590 struct bio_list sync, nosync, recover, *this_list = NULL;
Jonathan Brassow7513c2a2009-04-02 19:55:30 +0100591 struct bio_list requeue;
592 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
593 region_t region;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594
595 if (!writes->head)
596 return;
597
598 /*
599 * Classify each write.
600 */
601 bio_list_init(&sync);
602 bio_list_init(&nosync);
603 bio_list_init(&recover);
Jonathan Brassow7513c2a2009-04-02 19:55:30 +0100604 bio_list_init(&requeue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605
606 while ((bio = bio_list_pop(writes))) {
Jonathan Brassow7513c2a2009-04-02 19:55:30 +0100607 region = dm_rh_bio_to_region(ms->rh, bio);
608
609 if (log->type->is_remote_recovering &&
610 log->type->is_remote_recovering(log, region)) {
611 bio_list_add(&requeue, bio);
612 continue;
613 }
614
615 state = dm_rh_get_state(ms->rh, region, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 switch (state) {
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100617 case DM_RH_CLEAN:
618 case DM_RH_DIRTY:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 this_list = &sync;
620 break;
621
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100622 case DM_RH_NOSYNC:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 this_list = &nosync;
624 break;
625
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100626 case DM_RH_RECOVERING:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627 this_list = &recover;
628 break;
629 }
630
631 bio_list_add(this_list, bio);
632 }
633
634 /*
Jonathan Brassow7513c2a2009-04-02 19:55:30 +0100635 * Add bios that are delayed due to remote recovery
636 * back on to the write queue
637 */
638 if (unlikely(requeue.head)) {
639 spin_lock_irq(&ms->lock);
640 bio_list_merge(&ms->writes, &requeue);
641 spin_unlock_irq(&ms->lock);
642 }
643
644 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 * Increment the pending counts for any regions that will
646 * be written to (writes to recover regions are going to
647 * be delayed).
648 */
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100649 dm_rh_inc_pending(ms->rh, &sync);
650 dm_rh_inc_pending(ms->rh, &nosync);
651 ms->log_failure = dm_rh_flush(ms->rh) ? 1 : 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652
653 /*
654 * Dispatch io.
655 */
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +0000656 if (unlikely(ms->log_failure)) {
657 spin_lock_irq(&ms->lock);
658 bio_list_merge(&ms->failures, &sync);
659 spin_unlock_irq(&ms->lock);
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100660 wakeup_mirrord(ms);
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +0000661 } else
Jonathan Brassowfc1ff952007-07-12 17:29:15 +0100662 while ((bio = bio_list_pop(&sync)))
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +0000663 do_write(ms, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700664
665 while ((bio = bio_list_pop(&recover)))
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100666 dm_rh_delay(ms->rh, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700667
668 while ((bio = bio_list_pop(&nosync))) {
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000669 map_bio(get_default_mirror(ms), bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670 generic_make_request(bio);
671 }
672}
673
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000674static void do_failures(struct mirror_set *ms, struct bio_list *failures)
675{
676 struct bio *bio;
677
678 if (!failures->head)
679 return;
680
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +0000681 if (!ms->log_failure) {
Ilpo Jarvinenb34578a2008-10-30 13:33:07 +0000682 while ((bio = bio_list_pop(failures))) {
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100683 ms->in_sync = 0;
684 dm_rh_mark_nosync(ms->rh, bio, bio->bi_size, 0);
Ilpo Jarvinenb34578a2008-10-30 13:33:07 +0000685 }
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +0000686 return;
687 }
688
689 /*
690 * If the log has failed, unattempted writes are being
691 * put on the failures list. We can't issue those writes
692 * until a log has been marked, so we must store them.
693 *
694 * If a 'noflush' suspend is in progress, we can requeue
695 * the I/O's to the core. This give userspace a chance
696 * to reconfigure the mirror, at which point the core
697 * will reissue the writes. If the 'noflush' flag is
698 * not set, we have no choice but to return errors.
699 *
700 * Some writes on the failures list may have been
701 * submitted before the log failure and represent a
702 * failure to write to one of the devices. It is ok
703 * for us to treat them the same and requeue them
704 * as well.
705 */
706 if (dm_noflush_suspending(ms->ti)) {
707 while ((bio = bio_list_pop(failures)))
708 bio_endio(bio, DM_ENDIO_REQUEUE);
709 return;
710 }
711
712 if (atomic_read(&ms->suspend)) {
713 while ((bio = bio_list_pop(failures)))
714 bio_endio(bio, -EIO);
715 return;
716 }
717
718 spin_lock_irq(&ms->lock);
719 bio_list_merge(&ms->failures, failures);
720 spin_unlock_irq(&ms->lock);
721
Mikulas Patockaa2aebe02008-04-24 22:10:42 +0100722 delayed_wake(ms);
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000723}
724
725static void trigger_event(struct work_struct *work)
726{
727 struct mirror_set *ms =
728 container_of(work, struct mirror_set, trigger_event);
729
730 dm_table_event(ms->ti->table);
731}
732
Linus Torvalds1da177e2005-04-16 15:20:36 -0700733/*-----------------------------------------------------------------
734 * kmirrord
735 *---------------------------------------------------------------*/
Mikulas Patockaa2aebe02008-04-24 22:10:42 +0100736static void do_mirror(struct work_struct *work)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737{
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100738 struct mirror_set *ms = container_of(work, struct mirror_set,
739 kmirrord_work);
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000740 struct bio_list reads, writes, failures;
741 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000743 spin_lock_irqsave(&ms->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700744 reads = ms->reads;
745 writes = ms->writes;
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000746 failures = ms->failures;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747 bio_list_init(&ms->reads);
748 bio_list_init(&ms->writes);
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000749 bio_list_init(&ms->failures);
750 spin_unlock_irqrestore(&ms->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100752 dm_rh_update_states(ms->rh, errors_handled(ms));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 do_recovery(ms);
754 do_reads(ms, &reads);
755 do_writes(ms, &writes);
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000756 do_failures(ms, &failures);
Mikulas Patocka7ff14a32008-04-24 22:10:47 +0100757
758 dm_table_unplug_all(ms->ti->table);
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000759}
760
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761/*-----------------------------------------------------------------
762 * Target functions
763 *---------------------------------------------------------------*/
764static struct mirror_set *alloc_context(unsigned int nr_mirrors,
765 uint32_t region_size,
766 struct dm_target *ti,
Heinz Mauelshagen416cd172008-04-24 21:43:35 +0100767 struct dm_dirty_log *dl)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768{
769 size_t len;
770 struct mirror_set *ms = NULL;
771
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 len = sizeof(*ms) + (sizeof(ms->mirror[0]) * nr_mirrors);
773
Yoann Padioleaudd00cc42007-07-19 01:49:03 -0700774 ms = kzalloc(len, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 if (!ms) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700776 ti->error = "Cannot allocate mirror context";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700777 return NULL;
778 }
779
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780 spin_lock_init(&ms->lock);
781
782 ms->ti = ti;
783 ms->nr_mirrors = nr_mirrors;
784 ms->nr_regions = dm_sector_div_up(ti->len, region_size);
785 ms->in_sync = 0;
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +0000786 ms->log_failure = 0;
787 atomic_set(&ms->suspend, 0);
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000788 atomic_set(&ms->default_mirror, DEFAULT_MIRROR);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700789
Mikulas Patocka95f8fac2009-04-02 19:55:24 +0100790 ms->read_record_pool = mempool_create_slab_pool(MIN_READ_RECORDS,
791 _dm_raid1_read_record_cache);
792
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000793 if (!ms->read_record_pool) {
794 ti->error = "Error creating mirror read_record_pool";
795 kfree(ms);
796 return NULL;
797 }
798
Milan Broz88be1632007-05-09 02:33:04 -0700799 ms->io_client = dm_io_client_create(DM_IO_PAGES);
800 if (IS_ERR(ms->io_client)) {
801 ti->error = "Error creating dm_io client";
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000802 mempool_destroy(ms->read_record_pool);
Milan Broz88be1632007-05-09 02:33:04 -0700803 kfree(ms);
804 return NULL;
805 }
806
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100807 ms->rh = dm_region_hash_create(ms, dispatch_bios, wakeup_mirrord,
808 wakeup_all_recovery_waiters,
809 ms->ti->begin, MAX_RECOVERY,
810 dl, region_size, ms->nr_regions);
811 if (IS_ERR(ms->rh)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700812 ti->error = "Error creating dirty region hash";
Dmitry Monakhova72cf732007-10-19 22:38:39 +0100813 dm_io_client_destroy(ms->io_client);
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000814 mempool_destroy(ms->read_record_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700815 kfree(ms);
816 return NULL;
817 }
818
819 return ms;
820}
821
822static void free_context(struct mirror_set *ms, struct dm_target *ti,
823 unsigned int m)
824{
825 while (m--)
826 dm_put_device(ti, ms->mirror[m].dev);
827
Milan Broz88be1632007-05-09 02:33:04 -0700828 dm_io_client_destroy(ms->io_client);
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100829 dm_region_hash_destroy(ms->rh);
Jonathan Brassow06386bb2008-02-08 02:11:37 +0000830 mempool_destroy(ms->read_record_pool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831 kfree(ms);
832}
833
Linus Torvalds1da177e2005-04-16 15:20:36 -0700834static int get_mirror(struct mirror_set *ms, struct dm_target *ti,
835 unsigned int mirror, char **argv)
836{
Andrew Morton4ee218c2006-03-27 01:17:48 -0800837 unsigned long long offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838
Andrew Morton4ee218c2006-03-27 01:17:48 -0800839 if (sscanf(argv[1], "%llu", &offset) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700840 ti->error = "Invalid offset";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700841 return -EINVAL;
842 }
843
844 if (dm_get_device(ti, argv[0], offset, ti->len,
845 dm_table_get_mode(ti->table),
846 &ms->mirror[mirror].dev)) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700847 ti->error = "Device lookup failure";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848 return -ENXIO;
849 }
850
Jonathan Brassowaa5617c2007-10-19 22:47:58 +0100851 ms->mirror[mirror].ms = ms;
Jonathan Brassow72f4b312008-02-08 02:11:29 +0000852 atomic_set(&(ms->mirror[mirror].error_count), 0);
853 ms->mirror[mirror].error_type = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854 ms->mirror[mirror].offset = offset;
855
856 return 0;
857}
858
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859/*
860 * Create dirty log: log_type #log_params <log_params>
861 */
Heinz Mauelshagen416cd172008-04-24 21:43:35 +0100862static struct dm_dirty_log *create_dirty_log(struct dm_target *ti,
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100863 unsigned argc, char **argv,
864 unsigned *args_used)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700865{
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100866 unsigned param_count;
Heinz Mauelshagen416cd172008-04-24 21:43:35 +0100867 struct dm_dirty_log *dl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868
869 if (argc < 2) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700870 ti->error = "Insufficient mirror log arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 return NULL;
872 }
873
874 if (sscanf(argv[1], "%u", &param_count) != 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700875 ti->error = "Invalid mirror log argument count";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700876 return NULL;
877 }
878
879 *args_used = 2 + param_count;
880
881 if (argc < *args_used) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700882 ti->error = "Insufficient mirror log arguments";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 return NULL;
884 }
885
Heinz Mauelshagen416cd172008-04-24 21:43:35 +0100886 dl = dm_dirty_log_create(argv[0], ti, param_count, argv + 2);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887 if (!dl) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700888 ti->error = "Error creating mirror dirty log";
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 return NULL;
890 }
891
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892 return dl;
893}
894
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -0700895static int parse_features(struct mirror_set *ms, unsigned argc, char **argv,
896 unsigned *args_used)
897{
898 unsigned num_features;
899 struct dm_target *ti = ms->ti;
900
901 *args_used = 0;
902
903 if (!argc)
904 return 0;
905
906 if (sscanf(argv[0], "%u", &num_features) != 1) {
907 ti->error = "Invalid number of features";
908 return -EINVAL;
909 }
910
911 argc--;
912 argv++;
913 (*args_used)++;
914
915 if (num_features > argc) {
916 ti->error = "Not enough arguments to support feature count";
917 return -EINVAL;
918 }
919
920 if (!strcmp("handle_errors", argv[0]))
921 ms->features |= DM_RAID1_HANDLE_ERRORS;
922 else {
923 ti->error = "Unrecognised feature requested";
924 return -EINVAL;
925 }
926
927 (*args_used)++;
928
929 return 0;
930}
931
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932/*
933 * Construct a mirror mapping:
934 *
935 * log_type #log_params <log_params>
936 * #mirrors [mirror_path offset]{2,}
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -0700937 * [#features <features>]
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938 *
939 * log_type is "core" or "disk"
940 * #log_params is between 1 and 3
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -0700941 *
942 * If present, features must be "handle_errors".
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944static int mirror_ctr(struct dm_target *ti, unsigned int argc, char **argv)
945{
946 int r;
947 unsigned int nr_mirrors, m, args_used;
948 struct mirror_set *ms;
Heinz Mauelshagen416cd172008-04-24 21:43:35 +0100949 struct dm_dirty_log *dl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950
951 dl = create_dirty_log(ti, argc, argv, &args_used);
952 if (!dl)
953 return -EINVAL;
954
955 argv += args_used;
956 argc -= args_used;
957
958 if (!argc || sscanf(argv[0], "%u", &nr_mirrors) != 1 ||
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +0100959 nr_mirrors < 2 || nr_mirrors > DM_KCOPYD_MAX_REGIONS + 1) {
Alasdair G Kergon72d94862006-06-26 00:27:35 -0700960 ti->error = "Invalid number of mirrors";
Heinz Mauelshagen416cd172008-04-24 21:43:35 +0100961 dm_dirty_log_destroy(dl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 return -EINVAL;
963 }
964
965 argv++, argc--;
966
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -0700967 if (argc < nr_mirrors * 2) {
968 ti->error = "Too few mirror arguments";
Heinz Mauelshagen416cd172008-04-24 21:43:35 +0100969 dm_dirty_log_destroy(dl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970 return -EINVAL;
971 }
972
973 ms = alloc_context(nr_mirrors, dl->type->get_region_size(dl), ti, dl);
974 if (!ms) {
Heinz Mauelshagen416cd172008-04-24 21:43:35 +0100975 dm_dirty_log_destroy(dl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700976 return -ENOMEM;
977 }
978
979 /* Get the mirror parameter sets */
980 for (m = 0; m < nr_mirrors; m++) {
981 r = get_mirror(ms, ti, m, argv);
982 if (r) {
983 free_context(ms, ti, m);
984 return r;
985 }
986 argv += 2;
987 argc -= 2;
988 }
989
990 ti->private = ms;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +0100991 ti->split_io = dm_rh_get_region_size(ms->rh);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992
Holger Smolinski6ad36fe2007-05-09 02:32:50 -0700993 ms->kmirrord_wq = create_singlethread_workqueue("kmirrord");
994 if (!ms->kmirrord_wq) {
995 DMERR("couldn't start kmirrord");
Dmitry Monakhova72cf732007-10-19 22:38:39 +0100996 r = -ENOMEM;
997 goto err_free_context;
Holger Smolinski6ad36fe2007-05-09 02:32:50 -0700998 }
999 INIT_WORK(&ms->kmirrord_work, do_mirror);
Mikulas Patockaa2aebe02008-04-24 22:10:42 +01001000 init_timer(&ms->timer);
1001 ms->timer_pending = 0;
Jonathan Brassow72f4b312008-02-08 02:11:29 +00001002 INIT_WORK(&ms->trigger_event, trigger_event);
Holger Smolinski6ad36fe2007-05-09 02:32:50 -07001003
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -07001004 r = parse_features(ms, argc, argv, &args_used);
Dmitry Monakhova72cf732007-10-19 22:38:39 +01001005 if (r)
1006 goto err_destroy_wq;
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -07001007
1008 argv += args_used;
1009 argc -= args_used;
1010
Jonathan Brassowf44db672007-07-12 17:29:04 +01001011 /*
1012 * Any read-balancing addition depends on the
1013 * DM_RAID1_HANDLE_ERRORS flag being present.
1014 * This is because the decision to balance depends
1015 * on the sync state of a region. If the above
1016 * flag is not present, we ignore errors; and
1017 * the sync state may be inaccurate.
1018 */
1019
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -07001020 if (argc) {
1021 ti->error = "Too many mirror arguments";
Dmitry Monakhova72cf732007-10-19 22:38:39 +01001022 r = -EINVAL;
1023 goto err_destroy_wq;
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -07001024 }
1025
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001026 r = dm_kcopyd_client_create(DM_KCOPYD_PAGES, &ms->kcopyd_client);
Dmitry Monakhova72cf732007-10-19 22:38:39 +01001027 if (r)
1028 goto err_destroy_wq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001030 wakeup_mirrord(ms);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031 return 0;
Dmitry Monakhova72cf732007-10-19 22:38:39 +01001032
1033err_destroy_wq:
1034 destroy_workqueue(ms->kmirrord_wq);
1035err_free_context:
1036 free_context(ms, ti, ms->nr_mirrors);
1037 return r;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001038}
1039
1040static void mirror_dtr(struct dm_target *ti)
1041{
1042 struct mirror_set *ms = (struct mirror_set *) ti->private;
1043
Mikulas Patockaa2aebe02008-04-24 22:10:42 +01001044 del_timer_sync(&ms->timer);
Holger Smolinski6ad36fe2007-05-09 02:32:50 -07001045 flush_workqueue(ms->kmirrord_wq);
Mikulas Patocka18776c72008-11-13 23:38:52 +00001046 flush_scheduled_work();
Heinz Mauelshageneb69aca2008-04-24 21:43:19 +01001047 dm_kcopyd_client_destroy(ms->kcopyd_client);
Holger Smolinski6ad36fe2007-05-09 02:32:50 -07001048 destroy_workqueue(ms->kmirrord_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001049 free_context(ms, ti, ms->nr_mirrors);
1050}
1051
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052/*
1053 * Mirror mapping function
1054 */
1055static int mirror_map(struct dm_target *ti, struct bio *bio,
1056 union map_info *map_context)
1057{
1058 int r, rw = bio_rw(bio);
1059 struct mirror *m;
1060 struct mirror_set *ms = ti->private;
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001061 struct dm_raid1_read_record *read_record = NULL;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001062 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001063
1064 if (rw == WRITE) {
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001065 /* Save region for mirror_end_io() handler */
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001066 map_context->ll = dm_rh_bio_to_region(ms->rh, bio);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001067 queue_bio(ms, bio, rw);
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001068 return DM_MAPIO_SUBMITTED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069 }
1070
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001071 r = log->type->in_sync(log, dm_rh_bio_to_region(ms->rh, bio), 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072 if (r < 0 && r != -EWOULDBLOCK)
1073 return r;
1074
Linus Torvalds1da177e2005-04-16 15:20:36 -07001075 /*
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001076 * If region is not in-sync queue the bio.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001077 */
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001078 if (!r || (r == -EWOULDBLOCK)) {
1079 if (rw == READA)
1080 return -EWOULDBLOCK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001081
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082 queue_bio(ms, bio, rw);
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001083 return DM_MAPIO_SUBMITTED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001084 }
1085
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001086 /*
1087 * The region is in-sync and we can perform reads directly.
1088 * Store enough information so we can retry if it fails.
1089 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090 m = choose_mirror(ms, bio->bi_sector);
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001091 if (unlikely(!m))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001092 return -EIO;
1093
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001094 read_record = mempool_alloc(ms->read_record_pool, GFP_NOIO);
1095 if (likely(read_record)) {
1096 dm_bio_record(&read_record->details, bio);
1097 map_context->ptr = read_record;
1098 read_record->m = m;
1099 }
1100
1101 map_bio(m, bio);
1102
Kiyoshi Uedad2a7ad22006-12-08 02:41:06 -08001103 return DM_MAPIO_REMAPPED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104}
1105
1106static int mirror_end_io(struct dm_target *ti, struct bio *bio,
1107 int error, union map_info *map_context)
1108{
1109 int rw = bio_rw(bio);
1110 struct mirror_set *ms = (struct mirror_set *) ti->private;
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001111 struct mirror *m = NULL;
1112 struct dm_bio_details *bd = NULL;
1113 struct dm_raid1_read_record *read_record = map_context->ptr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001114
1115 /*
1116 * We need to dec pending if this was a write.
1117 */
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001118 if (rw == WRITE) {
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001119 dm_rh_dec(ms->rh, map_context->ll);
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001120 return error;
1121 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001122
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001123 if (error == -EOPNOTSUPP)
1124 goto out;
1125
1126 if ((error == -EWOULDBLOCK) && bio_rw_ahead(bio))
1127 goto out;
1128
1129 if (unlikely(error)) {
1130 if (!read_record) {
1131 /*
1132 * There wasn't enough memory to record necessary
1133 * information for a retry or there was no other
1134 * mirror in-sync.
1135 */
Adrian Bunke03f1a82008-02-19 19:44:19 +00001136 DMERR_LIMIT("Mirror read failed.");
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001137 return -EIO;
1138 }
Adrian Bunke03f1a82008-02-19 19:44:19 +00001139
1140 m = read_record->m;
1141
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001142 DMERR("Mirror read failed from %s. Trying alternative device.",
1143 m->dev->name);
1144
Jonathan Brassow06386bb2008-02-08 02:11:37 +00001145 fail_mirror(m, DM_RAID1_READ_ERROR);
1146
1147 /*
1148 * A failed read is requeued for another attempt using an intact
1149 * mirror.
1150 */
1151 if (default_ok(m) || mirror_available(ms, bio)) {
1152 bd = &read_record->details;
1153
1154 dm_bio_restore(bd, bio);
1155 mempool_free(read_record, ms->read_record_pool);
1156 map_context->ptr = NULL;
1157 queue_bio(ms, bio, rw);
1158 return 1;
1159 }
1160 DMERR("All replicated volumes dead, failing I/O");
1161 }
1162
1163out:
1164 if (read_record) {
1165 mempool_free(read_record, ms->read_record_pool);
1166 map_context->ptr = NULL;
1167 }
1168
1169 return error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001170}
1171
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +00001172static void mirror_presuspend(struct dm_target *ti)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001173{
1174 struct mirror_set *ms = (struct mirror_set *) ti->private;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001175 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001176
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +00001177 atomic_set(&ms->suspend, 1);
1178
1179 /*
1180 * We must finish up all the work that we've
1181 * generated (i.e. recovery work).
1182 */
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001183 dm_rh_stop_recovery(ms->rh);
Jonathan E Brassow33184042006-11-08 17:44:44 -08001184
Jonathan E Brassow33184042006-11-08 17:44:44 -08001185 wait_event(_kmirrord_recovery_stopped,
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001186 !dm_rh_recovery_in_flight(ms->rh));
Jonathan E Brassow33184042006-11-08 17:44:44 -08001187
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +00001188 if (log->type->presuspend && log->type->presuspend(log))
1189 /* FIXME: need better error handling */
1190 DMWARN("log presuspend failed");
1191
1192 /*
1193 * Now that recovery is complete/stopped and the
1194 * delayed bios are queued, we need to wait for
1195 * the worker thread to complete. This way,
1196 * we know that all of our I/O has been pushed.
1197 */
1198 flush_workqueue(ms->kmirrord_wq);
1199}
1200
1201static void mirror_postsuspend(struct dm_target *ti)
1202{
1203 struct mirror_set *ms = ti->private;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001204 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +00001205
Jonathan Brassow6b3df0d2007-10-19 22:47:57 +01001206 if (log->type->postsuspend && log->type->postsuspend(log))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207 /* FIXME: need better error handling */
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +00001208 DMWARN("log postsuspend failed");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209}
1210
1211static void mirror_resume(struct dm_target *ti)
1212{
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +00001213 struct mirror_set *ms = ti->private;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001214 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +00001215
1216 atomic_set(&ms->suspend, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217 if (log->type->resume && log->type->resume(log))
1218 /* FIXME: need better error handling */
1219 DMWARN("log resume failed");
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001220 dm_rh_start_recovery(ms->rh);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221}
1222
Jonathan Brassowaf195ac2008-02-08 02:11:39 +00001223/*
1224 * device_status_char
1225 * @m: mirror device/leg we want the status of
1226 *
1227 * We return one character representing the most severe error
1228 * we have encountered.
1229 * A => Alive - No failures
1230 * D => Dead - A write failure occurred leaving mirror out-of-sync
1231 * S => Sync - A sychronization failure occurred, mirror out-of-sync
1232 * R => Read - A read failure occurred, mirror data unaffected
1233 *
1234 * Returns: <char>
1235 */
1236static char device_status_char(struct mirror *m)
1237{
1238 if (!atomic_read(&(m->error_count)))
1239 return 'A';
1240
1241 return (test_bit(DM_RAID1_WRITE_ERROR, &(m->error_type))) ? 'D' :
1242 (test_bit(DM_RAID1_SYNC_ERROR, &(m->error_type))) ? 'S' :
1243 (test_bit(DM_RAID1_READ_ERROR, &(m->error_type))) ? 'R' : 'U';
1244}
1245
1246
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247static int mirror_status(struct dm_target *ti, status_type_t type,
1248 char *result, unsigned int maxlen)
1249{
Jonathan E Brassow315dcc22007-05-09 02:32:58 -07001250 unsigned int m, sz = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001251 struct mirror_set *ms = (struct mirror_set *) ti->private;
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001252 struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh);
Jonathan Brassowaf195ac2008-02-08 02:11:39 +00001253 char buffer[ms->nr_mirrors + 1];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001254
Linus Torvalds1da177e2005-04-16 15:20:36 -07001255 switch (type) {
1256 case STATUSTYPE_INFO:
1257 DMEMIT("%d ", ms->nr_mirrors);
Jonathan Brassowaf195ac2008-02-08 02:11:39 +00001258 for (m = 0; m < ms->nr_mirrors; m++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259 DMEMIT("%s ", ms->mirror[m].dev->name);
Jonathan Brassowaf195ac2008-02-08 02:11:39 +00001260 buffer[m] = device_status_char(&(ms->mirror[m]));
1261 }
1262 buffer[m] = '\0';
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263
Jonathan Brassowaf195ac2008-02-08 02:11:39 +00001264 DMEMIT("%llu/%llu 1 %s ",
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001265 (unsigned long long)log->type->get_sync_count(log),
Jonathan Brassowaf195ac2008-02-08 02:11:39 +00001266 (unsigned long long)ms->nr_regions, buffer);
Jonathan E Brassow315dcc22007-05-09 02:32:58 -07001267
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001268 sz += log->type->status(log, type, result+sz, maxlen-sz);
Jonathan E Brassow315dcc22007-05-09 02:32:58 -07001269
Linus Torvalds1da177e2005-04-16 15:20:36 -07001270 break;
1271
1272 case STATUSTYPE_TABLE:
Heinz Mauelshagen1f965b12008-10-21 17:45:06 +01001273 sz = log->type->status(log, type, result, maxlen);
Jonathan E Brassow315dcc22007-05-09 02:32:58 -07001274
Jonathan Brassowe52b8f62006-10-03 01:15:32 -07001275 DMEMIT("%d", ms->nr_mirrors);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276 for (m = 0; m < ms->nr_mirrors; m++)
Jonathan Brassowe52b8f62006-10-03 01:15:32 -07001277 DMEMIT(" %s %llu", ms->mirror[m].dev->name,
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +00001278 (unsigned long long)ms->mirror[m].offset);
Jonathan E Brassowa8e6afa2007-05-09 02:32:59 -07001279
1280 if (ms->features & DM_RAID1_HANDLE_ERRORS)
1281 DMEMIT(" 1 handle_errors");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001282 }
1283
1284 return 0;
1285}
1286
1287static struct target_type mirror_target = {
1288 .name = "mirror",
Jonathan Brassowaf195ac2008-02-08 02:11:39 +00001289 .version = {1, 0, 20},
Linus Torvalds1da177e2005-04-16 15:20:36 -07001290 .module = THIS_MODULE,
1291 .ctr = mirror_ctr,
1292 .dtr = mirror_dtr,
1293 .map = mirror_map,
1294 .end_io = mirror_end_io,
Jonathan Brassowb80aa7a2008-02-08 02:11:35 +00001295 .presuspend = mirror_presuspend,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001296 .postsuspend = mirror_postsuspend,
1297 .resume = mirror_resume,
1298 .status = mirror_status,
1299};
1300
1301static int __init dm_mirror_init(void)
1302{
1303 int r;
1304
Mikulas Patocka95f8fac2009-04-02 19:55:24 +01001305 _dm_raid1_read_record_cache = KMEM_CACHE(dm_raid1_read_record, 0);
1306 if (!_dm_raid1_read_record_cache) {
1307 DMERR("Can't allocate dm_raid1_read_record cache");
1308 r = -ENOMEM;
1309 goto bad_cache;
1310 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001311
Mikulas Patocka95f8fac2009-04-02 19:55:24 +01001312 r = dm_register_target(&mirror_target);
1313 if (r < 0) {
1314 DMERR("Failed to register mirror target");
1315 goto bad_target;
1316 }
1317
1318 return 0;
1319
1320bad_target:
1321 kmem_cache_destroy(_dm_raid1_read_record_cache);
1322bad_cache:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323 return r;
1324}
1325
1326static void __exit dm_mirror_exit(void)
1327{
Mikulas Patocka10d3bd02009-01-06 03:04:58 +00001328 dm_unregister_target(&mirror_target);
Mikulas Patocka95f8fac2009-04-02 19:55:24 +01001329 kmem_cache_destroy(_dm_raid1_read_record_cache);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001330}
1331
1332/* Module hooks */
1333module_init(dm_mirror_init);
1334module_exit(dm_mirror_exit);
1335
1336MODULE_DESCRIPTION(DM_NAME " mirror target");
1337MODULE_AUTHOR("Joe Thornber");
1338MODULE_LICENSE("GPL");