blob: e31be14da8ea1a7e55277665943a8ae29b417716 [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Jens Axboe86db1e22008-01-29 14:53:40 +01002/*
3 * Functions related to mapping data to requests
4 */
5#include <linux/kernel.h>
Ingo Molnar68db0cf2017-02-08 18:51:37 +01006#include <linux/sched/task_stack.h>
Jens Axboe86db1e22008-01-29 14:53:40 +01007#include <linux/module.h>
8#include <linux/bio.h>
9#include <linux/blkdev.h>
Kent Overstreet26e49cf2015-01-18 16:16:31 +010010#include <linux/uio.h>
Jens Axboe86db1e22008-01-29 14:53:40 +010011
12#include "blk.h"
13
Christoph Hellwig98d61d52016-07-19 11:31:51 +020014/*
Jens Axboe88da0282017-12-18 15:40:44 +080015 * Append a bio to a passthrough request. Only works if the bio can be merged
16 * into the request based on the driver constraints.
Christoph Hellwig98d61d52016-07-19 11:31:51 +020017 */
Jens Axboe88da0282017-12-18 15:40:44 +080018int blk_rq_append_bio(struct request *rq, struct bio **bio)
Jens Axboe86db1e22008-01-29 14:53:40 +010019{
Jens Axboe88da0282017-12-18 15:40:44 +080020 struct bio *orig_bio = *bio;
21
22 blk_queue_bounce(rq->q, bio);
Christoph Hellwigcaa4b0242017-06-27 12:13:21 -060023
Christoph Hellwig98d61d52016-07-19 11:31:51 +020024 if (!rq->bio) {
Jens Axboe88da0282017-12-18 15:40:44 +080025 blk_rq_bio_prep(rq->q, rq, *bio);
Christoph Hellwig98d61d52016-07-19 11:31:51 +020026 } else {
Jens Axboe88da0282017-12-18 15:40:44 +080027 if (!ll_back_merge_fn(rq->q, rq, *bio)) {
28 if (orig_bio != *bio) {
29 bio_put(*bio);
30 *bio = orig_bio;
31 }
Christoph Hellwig98d61d52016-07-19 11:31:51 +020032 return -EINVAL;
Jens Axboe88da0282017-12-18 15:40:44 +080033 }
Christoph Hellwig98d61d52016-07-19 11:31:51 +020034
Jens Axboe88da0282017-12-18 15:40:44 +080035 rq->biotail->bi_next = *bio;
36 rq->biotail = *bio;
37 rq->__data_len += (*bio)->bi_iter.bi_size;
Jens Axboe86db1e22008-01-29 14:53:40 +010038 }
Christoph Hellwig98d61d52016-07-19 11:31:51 +020039
Jens Axboe86db1e22008-01-29 14:53:40 +010040 return 0;
41}
Christoph Hellwig98d61d52016-07-19 11:31:51 +020042EXPORT_SYMBOL(blk_rq_append_bio);
Jens Axboe86db1e22008-01-29 14:53:40 +010043
44static int __blk_rq_unmap_user(struct bio *bio)
45{
46 int ret = 0;
47
48 if (bio) {
49 if (bio_flagged(bio, BIO_USER_MAPPED))
50 bio_unmap_user(bio);
51 else
52 ret = bio_uncopy_user(bio);
53 }
54
55 return ret;
56}
57
Christoph Hellwig4d6af732016-03-02 18:07:14 +010058static int __blk_rq_map_user_iov(struct request *rq,
59 struct rq_map_data *map_data, struct iov_iter *iter,
60 gfp_t gfp_mask, bool copy)
61{
62 struct request_queue *q = rq->q;
63 struct bio *bio, *orig_bio;
64 int ret;
65
66 if (copy)
67 bio = bio_copy_user_iov(q, map_data, iter, gfp_mask);
68 else
69 bio = bio_map_user_iov(q, iter, gfp_mask);
70
71 if (IS_ERR(bio))
72 return PTR_ERR(bio);
73
Christoph Hellwigaebf5262017-01-31 16:57:31 +010074 bio->bi_opf &= ~REQ_OP_MASK;
75 bio->bi_opf |= req_op(rq);
76
Christoph Hellwig4d6af732016-03-02 18:07:14 +010077 if (map_data && map_data->null_mapped)
78 bio_set_flag(bio, BIO_NULL_MAPPED);
79
80 iov_iter_advance(iter, bio->bi_iter.bi_size);
81 if (map_data)
82 map_data->offset += bio->bi_iter.bi_size;
83
84 orig_bio = bio;
Christoph Hellwig4d6af732016-03-02 18:07:14 +010085
86 /*
87 * We link the bounce buffer in and could have to traverse it
88 * later so we have to get a ref to prevent it from being freed
89 */
Jens Axboe88da0282017-12-18 15:40:44 +080090 ret = blk_rq_append_bio(rq, &bio);
Christoph Hellwig4d6af732016-03-02 18:07:14 +010091 if (ret) {
Christoph Hellwig4d6af732016-03-02 18:07:14 +010092 __blk_rq_unmap_user(orig_bio);
Christoph Hellwig4d6af732016-03-02 18:07:14 +010093 return ret;
94 }
Jens Axboe88da0282017-12-18 15:40:44 +080095 bio_get(bio);
Christoph Hellwig4d6af732016-03-02 18:07:14 +010096
97 return 0;
98}
99
Jens Axboe86db1e22008-01-29 14:53:40 +0100100/**
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100101 * blk_rq_map_user_iov - map user data to a request, for passthrough requests
Jens Axboe86db1e22008-01-29 14:53:40 +0100102 * @q: request queue where request should be inserted
103 * @rq: request to map data to
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900104 * @map_data: pointer to the rq_map_data holding pages (if necessary)
Kent Overstreet26e49cf2015-01-18 16:16:31 +0100105 * @iter: iovec iterator
FUJITA Tomonoria3bce902008-08-28 16:17:05 +0900106 * @gfp_mask: memory allocation flags
Jens Axboe86db1e22008-01-29 14:53:40 +0100107 *
108 * Description:
Randy Dunlap710027a2008-08-19 20:13:11 +0200109 * Data will be mapped directly for zero copy I/O, if possible. Otherwise
Jens Axboe86db1e22008-01-29 14:53:40 +0100110 * a kernel bounce buffer is used.
111 *
Randy Dunlap710027a2008-08-19 20:13:11 +0200112 * A matching blk_rq_unmap_user() must be issued at the end of I/O, while
Jens Axboe86db1e22008-01-29 14:53:40 +0100113 * still in process context.
114 *
115 * Note: The mapped bio may need to be bounced through blk_queue_bounce()
116 * before being submitted to the device, as pages mapped may be out of
117 * reach. It's the callers responsibility to make sure this happens. The
118 * original bio must be passed back in to blk_rq_unmap_user() for proper
119 * unmapping.
120 */
121int blk_rq_map_user_iov(struct request_queue *q, struct request *rq,
Kent Overstreet26e49cf2015-01-18 16:16:31 +0100122 struct rq_map_data *map_data,
123 const struct iov_iter *iter, gfp_t gfp_mask)
Jens Axboe86db1e22008-01-29 14:53:40 +0100124{
Al Viro357f4352016-04-08 19:05:19 -0400125 bool copy = false;
126 unsigned long align = q->dma_pad_mask | queue_dma_alignment(q);
Christoph Hellwig4d6af732016-03-02 18:07:14 +0100127 struct bio *bio = NULL;
128 struct iov_iter i;
Douglas Gilbert2cc50a12018-01-14 17:00:48 -0500129 int ret = -EINVAL;
Jens Axboe86db1e22008-01-29 14:53:40 +0100130
Linus Torvaldsa0ac4022016-12-06 16:18:14 -0800131 if (!iter_is_iovec(iter))
132 goto fail;
133
Al Viro357f4352016-04-08 19:05:19 -0400134 if (map_data)
135 copy = true;
136 else if (iov_iter_alignment(iter) & align)
137 copy = true;
138 else if (queue_virt_boundary(q))
139 copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter);
FUJITA Tomonoriafdc1a72008-04-11 12:56:51 +0200140
Christoph Hellwig4d6af732016-03-02 18:07:14 +0100141 i = *iter;
142 do {
143 ret =__blk_rq_map_user_iov(rq, map_data, &i, gfp_mask, copy);
144 if (ret)
145 goto unmap_rq;
146 if (!bio)
147 bio = rq->bio;
148 } while (iov_iter_count(&i));
Jens Axboe86db1e22008-01-29 14:53:40 +0100149
FUJITA Tomonorif18573a2008-04-11 12:56:52 +0200150 if (!bio_flagged(bio, BIO_USER_MAPPED))
Christoph Hellwige8064022016-10-20 15:12:13 +0200151 rq->rq_flags |= RQF_COPY_USER;
Jens Axboe86db1e22008-01-29 14:53:40 +0100152 return 0;
Christoph Hellwig4d6af732016-03-02 18:07:14 +0100153
154unmap_rq:
155 __blk_rq_unmap_user(bio);
Linus Torvaldsa0ac4022016-12-06 16:18:14 -0800156fail:
Christoph Hellwig4d6af732016-03-02 18:07:14 +0100157 rq->bio = NULL;
Douglas Gilbert2cc50a12018-01-14 17:00:48 -0500158 return ret;
Jens Axboe86db1e22008-01-29 14:53:40 +0100159}
FUJITA Tomonori152e2832008-08-28 16:17:06 +0900160EXPORT_SYMBOL(blk_rq_map_user_iov);
Jens Axboe86db1e22008-01-29 14:53:40 +0100161
Christoph Hellwigddad8dd2015-01-18 16:16:29 +0100162int blk_rq_map_user(struct request_queue *q, struct request *rq,
163 struct rq_map_data *map_data, void __user *ubuf,
164 unsigned long len, gfp_t gfp_mask)
165{
Kent Overstreet26e49cf2015-01-18 16:16:31 +0100166 struct iovec iov;
167 struct iov_iter i;
Al Viro8f7e8852015-03-21 20:06:04 -0400168 int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i);
Christoph Hellwigddad8dd2015-01-18 16:16:29 +0100169
Al Viro8f7e8852015-03-21 20:06:04 -0400170 if (unlikely(ret < 0))
171 return ret;
Christoph Hellwigddad8dd2015-01-18 16:16:29 +0100172
Kent Overstreet26e49cf2015-01-18 16:16:31 +0100173 return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask);
Christoph Hellwigddad8dd2015-01-18 16:16:29 +0100174}
175EXPORT_SYMBOL(blk_rq_map_user);
176
Jens Axboe86db1e22008-01-29 14:53:40 +0100177/**
178 * blk_rq_unmap_user - unmap a request with user data
179 * @bio: start of bio list
180 *
181 * Description:
182 * Unmap a rq previously mapped by blk_rq_map_user(). The caller must
183 * supply the original rq->bio from the blk_rq_map_user() return, since
Randy Dunlap710027a2008-08-19 20:13:11 +0200184 * the I/O completion may have changed rq->bio.
Jens Axboe86db1e22008-01-29 14:53:40 +0100185 */
186int blk_rq_unmap_user(struct bio *bio)
187{
188 struct bio *mapped_bio;
189 int ret = 0, ret2;
190
191 while (bio) {
192 mapped_bio = bio;
193 if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
194 mapped_bio = bio->bi_private;
195
196 ret2 = __blk_rq_unmap_user(mapped_bio);
197 if (ret2 && !ret)
198 ret = ret2;
199
200 mapped_bio = bio;
201 bio = bio->bi_next;
202 bio_put(mapped_bio);
203 }
204
205 return ret;
206}
Jens Axboe86db1e22008-01-29 14:53:40 +0100207EXPORT_SYMBOL(blk_rq_unmap_user);
208
209/**
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100210 * blk_rq_map_kern - map kernel data to a request, for passthrough requests
Jens Axboe86db1e22008-01-29 14:53:40 +0100211 * @q: request queue where request should be inserted
212 * @rq: request to fill
213 * @kbuf: the kernel buffer
214 * @len: length of user data
215 * @gfp_mask: memory allocation flags
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200216 *
217 * Description:
218 * Data will be mapped directly if possible. Otherwise a bounce
Masanari Iidae2278672014-02-18 22:54:36 +0900219 * buffer is used. Can be called multiple times to append multiple
James Bottomley3a5a3922009-05-17 18:55:18 +0300220 * buffers.
Jens Axboe86db1e22008-01-29 14:53:40 +0100221 */
222int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
223 unsigned int len, gfp_t gfp_mask)
224{
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200225 int reading = rq_data_dir(rq) == READ;
Namhyung Kim14417792010-09-15 13:08:27 +0200226 unsigned long addr = (unsigned long) kbuf;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200227 int do_copy = 0;
Jens Axboe88da0282017-12-18 15:40:44 +0800228 struct bio *bio, *orig_bio;
James Bottomley3a5a3922009-05-17 18:55:18 +0300229 int ret;
Jens Axboe86db1e22008-01-29 14:53:40 +0100230
Martin K. Petersenae03bf62009-05-22 17:17:50 -0400231 if (len > (queue_max_hw_sectors(q) << 9))
Jens Axboe86db1e22008-01-29 14:53:40 +0100232 return -EINVAL;
233 if (!len || !kbuf)
234 return -EINVAL;
235
Namhyung Kim14417792010-09-15 13:08:27 +0200236 do_copy = !blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf);
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200237 if (do_copy)
238 bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading);
239 else
240 bio = bio_map_kern(q, kbuf, len, gfp_mask);
241
Jens Axboe86db1e22008-01-29 14:53:40 +0100242 if (IS_ERR(bio))
243 return PTR_ERR(bio);
244
Christoph Hellwigaebf5262017-01-31 16:57:31 +0100245 bio->bi_opf &= ~REQ_OP_MASK;
246 bio->bi_opf |= req_op(rq);
Jens Axboe86db1e22008-01-29 14:53:40 +0100247
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200248 if (do_copy)
Christoph Hellwige8064022016-10-20 15:12:13 +0200249 rq->rq_flags |= RQF_COPY_USER;
FUJITA Tomonori68154e92008-04-25 12:47:50 +0200250
Jens Axboe88da0282017-12-18 15:40:44 +0800251 orig_bio = bio;
252 ret = blk_rq_append_bio(rq, &bio);
James Bottomley3a5a3922009-05-17 18:55:18 +0300253 if (unlikely(ret)) {
254 /* request is too big */
Jens Axboe88da0282017-12-18 15:40:44 +0800255 bio_put(orig_bio);
James Bottomley3a5a3922009-05-17 18:55:18 +0300256 return ret;
257 }
258
Jens Axboe86db1e22008-01-29 14:53:40 +0100259 return 0;
260}
Jens Axboe86db1e22008-01-29 14:53:40 +0100261EXPORT_SYMBOL(blk_rq_map_kern);