blob: 7a8326bc59039e487802c596a6c26375f35310fa [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/read_write.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7#include <linux/slab.h>
8#include <linux/stat.h>
9#include <linux/fcntl.h>
10#include <linux/file.h>
11#include <linux/uio.h>
12#include <linux/smp_lock.h>
Robert Love0eeca282005-07-12 17:06:03 -040013#include <linux/fsnotify.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/security.h>
15#include <linux/module.h>
16#include <linux/syscalls.h>
Linus Torvaldse28cc712006-01-04 16:20:40 -080017#include <linux/pagemap.h>
Jens Axboed6b29d72007-06-04 09:59:47 +020018#include <linux/splice.h>
Badari Pulavartyee0b3e62006-09-30 23:28:47 -070019#include "read_write.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
21#include <asm/uaccess.h>
22#include <asm/unistd.h>
23
Arjan van de Ven4b6f5d22006-03-28 01:56:42 -080024const struct file_operations generic_ro_fops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070025 .llseek = generic_file_llseek,
Badari Pulavarty543ade12006-09-30 23:28:48 -070026 .read = do_sync_read,
27 .aio_read = generic_file_aio_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070028 .mmap = generic_file_readonly_mmap,
Jens Axboe534f2aa2007-06-01 14:52:37 +020029 .splice_read = generic_file_splice_read,
Linus Torvalds1da177e2005-04-16 15:20:36 -070030};
31
32EXPORT_SYMBOL(generic_ro_fops);
33
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020034/**
35 * generic_file_llseek_unlocked - lockless generic llseek implementation
36 * @file: file structure to seek on
37 * @offset: file offset to seek to
38 * @origin: type of seek
39 *
40 * Updates the file offset to the value specified by @offset and @origin.
41 * Locking must be provided by the caller.
42 */
Andi Kleen9465efc2008-06-27 11:05:24 +020043loff_t
44generic_file_llseek_unlocked(struct file *file, loff_t offset, int origin)
Linus Torvalds1da177e2005-04-16 15:20:36 -070045{
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 struct inode *inode = file->f_mapping->host;
47
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 switch (origin) {
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020049 case SEEK_END:
50 offset += inode->i_size;
51 break;
52 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -080053 /*
54 * Here we special-case the lseek(fd, 0, SEEK_CUR)
55 * position-querying operation. Avoid rewriting the "same"
56 * f_pos value back to the file because a concurrent read(),
57 * write() or lseek() might have altered it
58 */
59 if (offset == 0)
60 return file->f_pos;
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020061 offset += file->f_pos;
62 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -070063 }
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020064
65 if (offset < 0 || offset > inode->i_sb->s_maxbytes)
66 return -EINVAL;
67
68 /* Special lock needed here? */
69 if (offset != file->f_pos) {
70 file->f_pos = offset;
71 file->f_version = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 }
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020073
74 return offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -070075}
Andi Kleen9465efc2008-06-27 11:05:24 +020076EXPORT_SYMBOL(generic_file_llseek_unlocked);
Linus Torvalds1da177e2005-04-16 15:20:36 -070077
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020078/**
79 * generic_file_llseek - generic llseek implementation for regular files
80 * @file: file structure to seek on
81 * @offset: file offset to seek to
82 * @origin: type of seek
83 *
84 * This is a generic implemenation of ->llseek useable for all normal local
85 * filesystems. It just updates the file offset to the value specified by
86 * @offset and @origin under i_mutex.
87 */
Andi Kleen9465efc2008-06-27 11:05:24 +020088loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
Linus Torvalds1da177e2005-04-16 15:20:36 -070089{
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020090 loff_t rval;
91
Andi Kleen9465efc2008-06-27 11:05:24 +020092 mutex_lock(&file->f_dentry->d_inode->i_mutex);
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020093 rval = generic_file_llseek_unlocked(file, offset, origin);
Andi Kleen9465efc2008-06-27 11:05:24 +020094 mutex_unlock(&file->f_dentry->d_inode->i_mutex);
Christoph Hellwig3a8cff42008-08-11 15:37:17 +020095
96 return rval;
Linus Torvalds1da177e2005-04-16 15:20:36 -070097}
Andi Kleen9465efc2008-06-27 11:05:24 +020098EXPORT_SYMBOL(generic_file_llseek);
Linus Torvalds1da177e2005-04-16 15:20:36 -070099
100loff_t no_llseek(struct file *file, loff_t offset, int origin)
101{
102 return -ESPIPE;
103}
104EXPORT_SYMBOL(no_llseek);
105
106loff_t default_llseek(struct file *file, loff_t offset, int origin)
107{
David Sterba16abef02008-04-22 15:09:22 +0200108 loff_t retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700109
110 lock_kernel();
111 switch (origin) {
Chris Snook7b8e8922007-05-08 00:24:13 -0700112 case SEEK_END:
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800113 offset += i_size_read(file->f_path.dentry->d_inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114 break;
Chris Snook7b8e8922007-05-08 00:24:13 -0700115 case SEEK_CUR:
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800116 if (offset == 0) {
117 retval = file->f_pos;
118 goto out;
119 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120 offset += file->f_pos;
121 }
122 retval = -EINVAL;
123 if (offset >= 0) {
124 if (offset != file->f_pos) {
125 file->f_pos = offset;
126 file->f_version = 0;
127 }
128 retval = offset;
129 }
Alain Knaff5b6f1eb2008-11-10 17:08:08 -0800130out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131 unlock_kernel();
132 return retval;
133}
134EXPORT_SYMBOL(default_llseek);
135
136loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
137{
138 loff_t (*fn)(struct file *, loff_t, int);
139
140 fn = no_llseek;
141 if (file->f_mode & FMODE_LSEEK) {
142 fn = default_llseek;
143 if (file->f_op && file->f_op->llseek)
144 fn = file->f_op->llseek;
145 }
146 return fn(file, offset, origin);
147}
148EXPORT_SYMBOL(vfs_llseek);
149
Heiko Carstens2ed7c032009-01-14 14:13:54 +0100150asmlinkage long sys_lseek(unsigned int fd, off_t offset, unsigned int origin)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151{
152 off_t retval;
153 struct file * file;
154 int fput_needed;
155
156 retval = -EBADF;
157 file = fget_light(fd, &fput_needed);
158 if (!file)
159 goto bad;
160
161 retval = -EINVAL;
Chris Snook1ae70752007-05-08 00:24:15 -0700162 if (origin <= SEEK_MAX) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163 loff_t res = vfs_llseek(file, offset, origin);
164 retval = res;
165 if (res != (loff_t)retval)
166 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
167 }
168 fput_light(file, fput_needed);
169bad:
170 return retval;
171}
172
173#ifdef __ARCH_WANT_SYS_LLSEEK
174asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
175 unsigned long offset_low, loff_t __user * result,
176 unsigned int origin)
177{
178 int retval;
179 struct file * file;
180 loff_t offset;
181 int fput_needed;
182
183 retval = -EBADF;
184 file = fget_light(fd, &fput_needed);
185 if (!file)
186 goto bad;
187
188 retval = -EINVAL;
Chris Snook1ae70752007-05-08 00:24:15 -0700189 if (origin > SEEK_MAX)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190 goto out_putf;
191
192 offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
193 origin);
194
195 retval = (int)offset;
196 if (offset >= 0) {
197 retval = -EFAULT;
198 if (!copy_to_user(result, &offset, sizeof(offset)))
199 retval = 0;
200 }
201out_putf:
202 fput_light(file, fput_needed);
203bad:
204 return retval;
205}
206#endif
207
Linus Torvaldse28cc712006-01-04 16:20:40 -0800208/*
209 * rw_verify_area doesn't like huge counts. We limit
210 * them to something that fits in "int" so that others
211 * won't have to do range checks all the time.
212 */
213#define MAX_RW_COUNT (INT_MAX & PAGE_CACHE_MASK)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214
215int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
216{
217 struct inode *inode;
218 loff_t pos;
James Morrisc43e2592008-01-12 22:05:48 +1100219 int retval = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700220
Eric Dumazet163da952007-02-12 00:52:24 -0800221 inode = file->f_path.dentry->d_inode;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800222 if (unlikely((ssize_t) count < 0))
James Morrisc43e2592008-01-12 22:05:48 +1100223 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224 pos = *ppos;
225 if (unlikely((pos < 0) || (loff_t) (pos + count) < 0))
James Morrisc43e2592008-01-12 22:05:48 +1100226 return retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700227
Pavel Emelyanova16877c2007-10-01 14:41:11 -0700228 if (unlikely(inode->i_flock && mandatory_lock(inode))) {
James Morrisc43e2592008-01-12 22:05:48 +1100229 retval = locks_mandatory_area(
Linus Torvaldse28cc712006-01-04 16:20:40 -0800230 read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
231 inode, file, pos, count);
232 if (retval < 0)
233 return retval;
234 }
James Morrisc43e2592008-01-12 22:05:48 +1100235 retval = security_file_permission(file,
236 read_write == READ ? MAY_READ : MAY_WRITE);
237 if (retval)
238 return retval;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800239 return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240}
241
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700242static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
243{
244 set_current_state(TASK_UNINTERRUPTIBLE);
245 if (!kiocbIsKicked(iocb))
246 schedule();
247 else
248 kiocbClearKicked(iocb);
249 __set_current_state(TASK_RUNNING);
250}
251
Linus Torvalds1da177e2005-04-16 15:20:36 -0700252ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
253{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700254 struct iovec iov = { .iov_base = buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255 struct kiocb kiocb;
256 ssize_t ret;
257
258 init_sync_kiocb(&kiocb, filp);
259 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700260 kiocb.ki_left = len;
261
262 for (;;) {
263 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
264 if (ret != -EIOCBRETRY)
265 break;
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700266 wait_on_retry_sync_kiocb(&kiocb);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700267 }
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700268
Linus Torvalds1da177e2005-04-16 15:20:36 -0700269 if (-EIOCBQUEUED == ret)
270 ret = wait_on_sync_kiocb(&kiocb);
271 *ppos = kiocb.ki_pos;
272 return ret;
273}
274
275EXPORT_SYMBOL(do_sync_read);
276
277ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
278{
279 ssize_t ret;
280
281 if (!(file->f_mode & FMODE_READ))
282 return -EBADF;
283 if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
284 return -EINVAL;
285 if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
286 return -EFAULT;
287
288 ret = rw_verify_area(READ, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800289 if (ret >= 0) {
290 count = ret;
James Morrisc43e2592008-01-12 22:05:48 +1100291 if (file->f_op->read)
292 ret = file->f_op->read(file, buf, count, pos);
293 else
294 ret = do_sync_read(file, buf, count, pos);
295 if (ret > 0) {
296 fsnotify_access(file->f_path.dentry);
297 add_rchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298 }
James Morrisc43e2592008-01-12 22:05:48 +1100299 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300 }
301
302 return ret;
303}
304
305EXPORT_SYMBOL(vfs_read);
306
307ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
308{
Badari Pulavarty027445c2006-09-30 23:28:46 -0700309 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 struct kiocb kiocb;
311 ssize_t ret;
312
313 init_sync_kiocb(&kiocb, filp);
314 kiocb.ki_pos = *ppos;
Badari Pulavarty027445c2006-09-30 23:28:46 -0700315 kiocb.ki_left = len;
316
317 for (;;) {
318 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
319 if (ret != -EIOCBRETRY)
320 break;
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700321 wait_on_retry_sync_kiocb(&kiocb);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700322 }
Benjamin LaHaise63e68802005-06-23 00:10:27 -0700323
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 if (-EIOCBQUEUED == ret)
325 ret = wait_on_sync_kiocb(&kiocb);
326 *ppos = kiocb.ki_pos;
327 return ret;
328}
329
330EXPORT_SYMBOL(do_sync_write);
331
332ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
333{
334 ssize_t ret;
335
336 if (!(file->f_mode & FMODE_WRITE))
337 return -EBADF;
338 if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
339 return -EINVAL;
340 if (unlikely(!access_ok(VERIFY_READ, buf, count)))
341 return -EFAULT;
342
343 ret = rw_verify_area(WRITE, file, pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800344 if (ret >= 0) {
345 count = ret;
James Morrisc43e2592008-01-12 22:05:48 +1100346 if (file->f_op->write)
347 ret = file->f_op->write(file, buf, count, pos);
348 else
349 ret = do_sync_write(file, buf, count, pos);
350 if (ret > 0) {
351 fsnotify_modify(file->f_path.dentry);
352 add_wchar(current, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 }
James Morrisc43e2592008-01-12 22:05:48 +1100354 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 }
356
357 return ret;
358}
359
360EXPORT_SYMBOL(vfs_write);
361
362static inline loff_t file_pos_read(struct file *file)
363{
364 return file->f_pos;
365}
366
367static inline void file_pos_write(struct file *file, loff_t pos)
368{
369 file->f_pos = pos;
370}
371
Heiko Carstens2ed7c032009-01-14 14:13:54 +0100372asmlinkage long sys_read(unsigned int fd, char __user * buf, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373{
374 struct file *file;
375 ssize_t ret = -EBADF;
376 int fput_needed;
377
378 file = fget_light(fd, &fput_needed);
379 if (file) {
380 loff_t pos = file_pos_read(file);
381 ret = vfs_read(file, buf, count, &pos);
382 file_pos_write(file, pos);
383 fput_light(file, fput_needed);
384 }
385
386 return ret;
387}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388
Heiko Carstens2ed7c032009-01-14 14:13:54 +0100389asmlinkage long sys_write(unsigned int fd, const char __user * buf, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390{
391 struct file *file;
392 ssize_t ret = -EBADF;
393 int fput_needed;
394
395 file = fget_light(fd, &fput_needed);
396 if (file) {
397 loff_t pos = file_pos_read(file);
398 ret = vfs_write(file, buf, count, &pos);
399 file_pos_write(file, pos);
400 fput_light(file, fput_needed);
401 }
402
403 return ret;
404}
405
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100406SYSCALL_DEFINE(pread64)(unsigned int fd, char __user *buf,
407 size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408{
409 struct file *file;
410 ssize_t ret = -EBADF;
411 int fput_needed;
412
413 if (pos < 0)
414 return -EINVAL;
415
416 file = fget_light(fd, &fput_needed);
417 if (file) {
418 ret = -ESPIPE;
419 if (file->f_mode & FMODE_PREAD)
420 ret = vfs_read(file, buf, count, &pos);
421 fput_light(file, fput_needed);
422 }
423
424 return ret;
425}
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100426#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
427asmlinkage long SyS_pread64(long fd, long buf, long count, loff_t pos)
428{
429 return SYSC_pread64((unsigned int) fd, (char __user *) buf,
430 (size_t) count, pos);
431}
432SYSCALL_ALIAS(sys_pread64, SyS_pread64);
433#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700434
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100435SYSCALL_DEFINE(pwrite64)(unsigned int fd, const char __user *buf,
436 size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437{
438 struct file *file;
439 ssize_t ret = -EBADF;
440 int fput_needed;
441
442 if (pos < 0)
443 return -EINVAL;
444
445 file = fget_light(fd, &fput_needed);
446 if (file) {
447 ret = -ESPIPE;
448 if (file->f_mode & FMODE_PWRITE)
449 ret = vfs_write(file, buf, count, &pos);
450 fput_light(file, fput_needed);
451 }
452
453 return ret;
454}
Heiko Carstens6673e0c2009-01-14 14:14:02 +0100455#ifdef CONFIG_HAVE_SYSCALL_WRAPPERS
456asmlinkage long SyS_pwrite64(long fd, long buf, long count, loff_t pos)
457{
458 return SYSC_pwrite64((unsigned int) fd, (const char __user *) buf,
459 (size_t) count, pos);
460}
461SYSCALL_ALIAS(sys_pwrite64, SyS_pwrite64);
462#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463
464/*
465 * Reduce an iovec's length in-place. Return the resulting number of segments
466 */
467unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
468{
469 unsigned long seg = 0;
470 size_t len = 0;
471
472 while (seg < nr_segs) {
473 seg++;
474 if (len + iov->iov_len >= to) {
475 iov->iov_len = to - len;
476 break;
477 }
478 len += iov->iov_len;
479 iov++;
480 }
481 return seg;
482}
Eric Sandeen19295522008-01-28 23:58:27 -0500483EXPORT_SYMBOL(iov_shorten);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700485ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
486 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
487{
488 struct kiocb kiocb;
489 ssize_t ret;
490
491 init_sync_kiocb(&kiocb, filp);
492 kiocb.ki_pos = *ppos;
493 kiocb.ki_left = len;
494 kiocb.ki_nbytes = len;
495
496 for (;;) {
497 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
498 if (ret != -EIOCBRETRY)
499 break;
500 wait_on_retry_sync_kiocb(&kiocb);
501 }
502
503 if (ret == -EIOCBQUEUED)
504 ret = wait_on_sync_kiocb(&kiocb);
505 *ppos = kiocb.ki_pos;
506 return ret;
507}
508
509/* Do it by hand, with file-ops */
510ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
511 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
512{
513 struct iovec *vector = iov;
514 ssize_t ret = 0;
515
516 while (nr_segs > 0) {
517 void __user *base;
518 size_t len;
519 ssize_t nr;
520
521 base = vector->iov_base;
522 len = vector->iov_len;
523 vector++;
524 nr_segs--;
525
526 nr = fn(filp, base, len, ppos);
527
528 if (nr < 0) {
529 if (!ret)
530 ret = nr;
531 break;
532 }
533 ret += nr;
534 if (nr != len)
535 break;
536 }
537
538 return ret;
539}
540
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541/* A write operation does a read from user space and vice versa */
542#define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
543
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700544ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
545 unsigned long nr_segs, unsigned long fast_segs,
546 struct iovec *fast_pointer,
547 struct iovec **ret_pointer)
548 {
549 unsigned long seg;
550 ssize_t ret;
551 struct iovec *iov = fast_pointer;
552
553 /*
554 * SuS says "The readv() function *may* fail if the iovcnt argument
555 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has
556 * traditionally returned zero for zero segments, so...
557 */
558 if (nr_segs == 0) {
559 ret = 0;
560 goto out;
561 }
562
563 /*
564 * First get the "struct iovec" from user memory and
565 * verify all the pointers
566 */
567 if (nr_segs > UIO_MAXIOV) {
568 ret = -EINVAL;
569 goto out;
570 }
571 if (nr_segs > fast_segs) {
572 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
573 if (iov == NULL) {
574 ret = -ENOMEM;
575 goto out;
576 }
577 }
578 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
579 ret = -EFAULT;
580 goto out;
581 }
582
583 /*
584 * According to the Single Unix Specification we should return EINVAL
585 * if an element length is < 0 when cast to ssize_t or if the
586 * total length would overflow the ssize_t return value of the
587 * system call.
588 */
589 ret = 0;
590 for (seg = 0; seg < nr_segs; seg++) {
591 void __user *buf = iov[seg].iov_base;
592 ssize_t len = (ssize_t)iov[seg].iov_len;
593
594 /* see if we we're about to use an invalid len or if
595 * it's about to overflow ssize_t */
596 if (len < 0 || (ret + len < ret)) {
597 ret = -EINVAL;
598 goto out;
599 }
600 if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
601 ret = -EFAULT;
602 goto out;
603 }
604
605 ret += len;
606 }
607out:
608 *ret_pointer = iov;
609 return ret;
610}
611
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612static ssize_t do_readv_writev(int type, struct file *file,
613 const struct iovec __user * uvector,
614 unsigned long nr_segs, loff_t *pos)
615{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 size_t tot_len;
617 struct iovec iovstack[UIO_FASTIOV];
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700618 struct iovec *iov = iovstack;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700620 io_fn_t fn;
621 iov_fn_t fnv;
622
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700623 if (!file->f_op) {
624 ret = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 goto out;
626 }
627
Badari Pulavartyeed4e512006-09-30 23:28:49 -0700628 ret = rw_copy_check_uvector(type, uvector, nr_segs,
629 ARRAY_SIZE(iovstack), iovstack, &iov);
630 if (ret <= 0)
631 goto out;
632
633 tot_len = ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 ret = rw_verify_area(type, file, pos, tot_len);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800635 if (ret < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700636 goto out;
637
638 fnv = NULL;
639 if (type == READ) {
640 fn = file->f_op->read;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700641 fnv = file->f_op->aio_read;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700642 } else {
643 fn = (io_fn_t)file->f_op->write;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700644 fnv = file->f_op->aio_write;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645 }
646
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700647 if (fnv)
648 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
649 pos, fnv);
650 else
651 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700652
Linus Torvalds1da177e2005-04-16 15:20:36 -0700653out:
654 if (iov != iovstack)
655 kfree(iov);
Robert Love0eeca282005-07-12 17:06:03 -0400656 if ((ret + (type == READ)) > 0) {
657 if (type == READ)
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800658 fsnotify_access(file->f_path.dentry);
Robert Love0eeca282005-07-12 17:06:03 -0400659 else
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800660 fsnotify_modify(file->f_path.dentry);
Robert Love0eeca282005-07-12 17:06:03 -0400661 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700663}
664
665ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
666 unsigned long vlen, loff_t *pos)
667{
668 if (!(file->f_mode & FMODE_READ))
669 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700670 if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 return -EINVAL;
672
673 return do_readv_writev(READ, file, vec, vlen, pos);
674}
675
676EXPORT_SYMBOL(vfs_readv);
677
678ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
679 unsigned long vlen, loff_t *pos)
680{
681 if (!(file->f_mode & FMODE_WRITE))
682 return -EBADF;
Badari Pulavartyee0b3e62006-09-30 23:28:47 -0700683 if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 return -EINVAL;
685
686 return do_readv_writev(WRITE, file, vec, vlen, pos);
687}
688
689EXPORT_SYMBOL(vfs_writev);
690
Heiko Carstens2ed7c032009-01-14 14:13:54 +0100691asmlinkage long
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692sys_readv(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
693{
694 struct file *file;
695 ssize_t ret = -EBADF;
696 int fput_needed;
697
698 file = fget_light(fd, &fput_needed);
699 if (file) {
700 loff_t pos = file_pos_read(file);
701 ret = vfs_readv(file, vec, vlen, &pos);
702 file_pos_write(file, pos);
703 fput_light(file, fput_needed);
704 }
705
706 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800707 add_rchar(current, ret);
708 inc_syscr(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709 return ret;
710}
711
Heiko Carstens2ed7c032009-01-14 14:13:54 +0100712asmlinkage long
Linus Torvalds1da177e2005-04-16 15:20:36 -0700713sys_writev(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
714{
715 struct file *file;
716 ssize_t ret = -EBADF;
717 int fput_needed;
718
719 file = fget_light(fd, &fput_needed);
720 if (file) {
721 loff_t pos = file_pos_read(file);
722 ret = vfs_writev(file, vec, vlen, &pos);
723 file_pos_write(file, pos);
724 fput_light(file, fput_needed);
725 }
726
727 if (ret > 0)
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800728 add_wchar(current, ret);
729 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700730 return ret;
731}
732
733static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
734 size_t count, loff_t max)
735{
736 struct file * in_file, * out_file;
737 struct inode * in_inode, * out_inode;
738 loff_t pos;
739 ssize_t retval;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200740 int fput_needed_in, fput_needed_out, fl;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741
742 /*
743 * Get input file, and verify that it is ok..
744 */
745 retval = -EBADF;
746 in_file = fget_light(in_fd, &fput_needed_in);
747 if (!in_file)
748 goto out;
749 if (!(in_file->f_mode & FMODE_READ))
750 goto fput_in;
751 retval = -EINVAL;
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800752 in_inode = in_file->f_path.dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 if (!in_inode)
754 goto fput_in;
Jens Axboed96e6e72007-06-11 12:18:52 +0200755 if (!in_file->f_op || !in_file->f_op->splice_read)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756 goto fput_in;
757 retval = -ESPIPE;
758 if (!ppos)
759 ppos = &in_file->f_pos;
760 else
761 if (!(in_file->f_mode & FMODE_PREAD))
762 goto fput_in;
763 retval = rw_verify_area(READ, in_file, ppos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800764 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765 goto fput_in;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800766 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767
Linus Torvalds1da177e2005-04-16 15:20:36 -0700768 /*
769 * Get output file, and verify that it is ok..
770 */
771 retval = -EBADF;
772 out_file = fget_light(out_fd, &fput_needed_out);
773 if (!out_file)
774 goto fput_in;
775 if (!(out_file->f_mode & FMODE_WRITE))
776 goto fput_out;
777 retval = -EINVAL;
778 if (!out_file->f_op || !out_file->f_op->sendpage)
779 goto fput_out;
Josef "Jeff" Sipek0f7fc9e2006-12-08 02:36:35 -0800780 out_inode = out_file->f_path.dentry->d_inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700781 retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
Linus Torvaldse28cc712006-01-04 16:20:40 -0800782 if (retval < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783 goto fput_out;
Linus Torvaldse28cc712006-01-04 16:20:40 -0800784 count = retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785
Linus Torvalds1da177e2005-04-16 15:20:36 -0700786 if (!max)
787 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
788
789 pos = *ppos;
790 retval = -EINVAL;
791 if (unlikely(pos < 0))
792 goto fput_out;
793 if (unlikely(pos + count > max)) {
794 retval = -EOVERFLOW;
795 if (pos >= max)
796 goto fput_out;
797 count = max - pos;
798 }
799
Jens Axboed96e6e72007-06-11 12:18:52 +0200800 fl = 0;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200801#if 0
Jens Axboed96e6e72007-06-11 12:18:52 +0200802 /*
803 * We need to debate whether we can enable this or not. The
804 * man page documents EAGAIN return for the output at least,
805 * and the application is arguably buggy if it doesn't expect
806 * EAGAIN on a non-blocking file descriptor.
807 */
808 if (in_file->f_flags & O_NONBLOCK)
809 fl = SPLICE_F_NONBLOCK;
Jens Axboe534f2aa2007-06-01 14:52:37 +0200810#endif
Jens Axboed96e6e72007-06-11 12:18:52 +0200811 retval = do_splice_direct(in_file, ppos, out_file, count, fl);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812
813 if (retval > 0) {
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800814 add_rchar(current, retval);
815 add_wchar(current, retval);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700817
Alexey Dobriyan4b98d112007-02-10 01:46:45 -0800818 inc_syscr(current);
819 inc_syscw(current);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 if (*ppos > max)
821 retval = -EOVERFLOW;
822
823fput_out:
824 fput_light(out_file, fput_needed_out);
825fput_in:
826 fput_light(in_file, fput_needed_in);
827out:
828 return retval;
829}
830
Heiko Carstens2ed7c032009-01-14 14:13:54 +0100831asmlinkage long sys_sendfile(int out_fd, int in_fd, off_t __user *offset, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832{
833 loff_t pos;
834 off_t off;
835 ssize_t ret;
836
837 if (offset) {
838 if (unlikely(get_user(off, offset)))
839 return -EFAULT;
840 pos = off;
841 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
842 if (unlikely(put_user(pos, offset)))
843 return -EFAULT;
844 return ret;
845 }
846
847 return do_sendfile(out_fd, in_fd, NULL, count, 0);
848}
849
Heiko Carstens2ed7c032009-01-14 14:13:54 +0100850asmlinkage long sys_sendfile64(int out_fd, int in_fd, loff_t __user *offset, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851{
852 loff_t pos;
853 ssize_t ret;
854
855 if (offset) {
856 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
857 return -EFAULT;
858 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
859 if (unlikely(put_user(pos, offset)))
860 return -EFAULT;
861 return ret;
862 }
863
864 return do_sendfile(out_fd, in_fd, NULL, count, 0);
865}