blob: e2e738cc08b1d1fa88f74c8897253893e93fff4a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/filemap.c
3 *
4 * Copyright (C) 1994-1999 Linus Torvalds
5 */
6
7/*
8 * This file handles the generic file mmap semantics used by
9 * most "normal" filesystems (but you don't /have/ to use this:
10 * the NFS filesystem used to do this differently, for example)
11 */
Paul Gortmakerb95f1b312011-10-16 02:01:52 -040012#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/compiler.h>
Ross Zwislerf9fe48b2016-01-22 15:10:40 -080014#include <linux/dax.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015#include <linux/fs.h>
Ingo Molnar3f07c012017-02-08 18:51:30 +010016#include <linux/sched/signal.h>
Hiro Yoshiokac22ce142006-06-23 02:04:16 -070017#include <linux/uaccess.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080018#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <linux/kernel_stat.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090020#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/mm.h>
22#include <linux/swap.h>
23#include <linux/mman.h>
24#include <linux/pagemap.h>
25#include <linux/file.h>
26#include <linux/uio.h>
27#include <linux/hash.h>
28#include <linux/writeback.h>
Linus Torvalds53253382007-10-18 14:47:32 -070029#include <linux/backing-dev.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <linux/pagevec.h>
31#include <linux/blkdev.h>
32#include <linux/security.h>
Paul Jackson44110fe2006-03-24 03:16:04 -080033#include <linux/cpuset.h>
Nick Piggin2f718ff2007-10-16 01:24:59 -070034#include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */
Johannes Weiner00501b52014-08-08 14:19:20 -070035#include <linux/hugetlb.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080036#include <linux/memcontrol.h>
Dan Magenheimerc515e1f2011-05-26 10:01:43 -060037#include <linux/cleancache.h>
Kirill A. Shutemovf1820362014-04-07 15:37:19 -070038#include <linux/rmap.h>
Nick Piggin0f8053a2006-03-22 00:08:33 -080039#include "internal.h"
40
Robert Jarzmikfe0bfaa2013-04-29 15:06:10 -070041#define CREATE_TRACE_POINTS
42#include <trace/events/filemap.h>
43
Linus Torvalds1da177e2005-04-16 15:20:36 -070044/*
Linus Torvalds1da177e2005-04-16 15:20:36 -070045 * FIXME: remove all knowledge of the buffer layer from the core VM
46 */
Jan Kara148f9482009-08-17 19:52:36 +020047#include <linux/buffer_head.h> /* for try_to_free_buffers */
Linus Torvalds1da177e2005-04-16 15:20:36 -070048
Linus Torvalds1da177e2005-04-16 15:20:36 -070049#include <asm/mman.h>
50
51/*
52 * Shared mappings implemented 30.11.1994. It's not fully working yet,
53 * though.
54 *
55 * Shared mappings now work. 15.8.1995 Bruno.
56 *
57 * finished 'unifying' the page and buffer cache and SMP-threaded the
58 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
59 *
60 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
61 */
62
63/*
64 * Lock ordering:
65 *
Davidlohr Buesoc8c06ef2014-12-12 16:54:24 -080066 * ->i_mmap_rwsem (truncate_pagecache)
Linus Torvalds1da177e2005-04-16 15:20:36 -070067 * ->private_lock (__free_pte->__set_page_dirty_buffers)
Hugh Dickins5d337b92005-09-03 15:54:41 -070068 * ->swap_lock (exclusive_swap_page, others)
69 * ->mapping->tree_lock
Linus Torvalds1da177e2005-04-16 15:20:36 -070070 *
Jes Sorensen1b1dcc12006-01-09 15:59:24 -080071 * ->i_mutex
Davidlohr Buesoc8c06ef2014-12-12 16:54:24 -080072 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
Linus Torvalds1da177e2005-04-16 15:20:36 -070073 *
74 * ->mmap_sem
Davidlohr Buesoc8c06ef2014-12-12 16:54:24 -080075 * ->i_mmap_rwsem
Hugh Dickinsb8072f02005-10-29 18:16:41 -070076 * ->page_table_lock or pte_lock (various, mainly in memory.c)
Linus Torvalds1da177e2005-04-16 15:20:36 -070077 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
78 *
79 * ->mmap_sem
80 * ->lock_page (access_process_vm)
81 *
Al Viroccad2362014-02-11 22:36:48 -050082 * ->i_mutex (generic_perform_write)
Nick Piggin82591e62006-10-19 23:29:10 -070083 * ->mmap_sem (fault_in_pages_readable->do_page_fault)
Linus Torvalds1da177e2005-04-16 15:20:36 -070084 *
Christoph Hellwigf758eea2011-04-21 18:19:44 -060085 * bdi->wb.list_lock
Dave Chinnera66979a2011-03-22 22:23:41 +110086 * sb_lock (fs/fs-writeback.c)
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 * ->mapping->tree_lock (__sync_single_inode)
88 *
Davidlohr Buesoc8c06ef2014-12-12 16:54:24 -080089 * ->i_mmap_rwsem
Linus Torvalds1da177e2005-04-16 15:20:36 -070090 * ->anon_vma.lock (vma_adjust)
91 *
92 * ->anon_vma.lock
Hugh Dickinsb8072f02005-10-29 18:16:41 -070093 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
Linus Torvalds1da177e2005-04-16 15:20:36 -070094 *
Hugh Dickinsb8072f02005-10-29 18:16:41 -070095 * ->page_table_lock or pte_lock
Hugh Dickins5d337b92005-09-03 15:54:41 -070096 * ->swap_lock (try_to_unmap_one)
Linus Torvalds1da177e2005-04-16 15:20:36 -070097 * ->private_lock (try_to_unmap_one)
98 * ->tree_lock (try_to_unmap_one)
Mel Gormana52633d2016-07-28 15:45:28 -070099 * ->zone_lru_lock(zone) (follow_page->mark_page_accessed)
100 * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101 * ->private_lock (page_remove_rmap->set_page_dirty)
102 * ->tree_lock (page_remove_rmap->set_page_dirty)
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600103 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
Dave Chinner250df6e2011-03-22 22:23:36 +1100104 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
Johannes Weiner81f8c3a2016-03-15 14:57:04 -0700105 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
Christoph Hellwigf758eea2011-04-21 18:19:44 -0600106 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
Dave Chinner250df6e2011-03-22 22:23:36 +1100107 * ->inode->i_lock (zap_pte_range->set_page_dirty)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700108 * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
109 *
Davidlohr Buesoc8c06ef2014-12-12 16:54:24 -0800110 * ->i_mmap_rwsem
Andi Kleen9a3c5312012-03-21 16:34:09 -0700111 * ->tasklist_lock (memory_failure, collect_procs_ao)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 */
113
Johannes Weiner22f2ac52016-09-30 15:11:29 -0700114static int page_cache_tree_insert(struct address_space *mapping,
115 struct page *page, void **shadowp)
116{
117 struct radix_tree_node *node;
118 void **slot;
119 int error;
120
121 error = __radix_tree_create(&mapping->page_tree, page->index, 0,
122 &node, &slot);
123 if (error)
124 return error;
125 if (*slot) {
126 void *p;
127
128 p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
129 if (!radix_tree_exceptional_entry(p))
130 return -EEXIST;
131
132 mapping->nrexceptional--;
Ross Zwislerd01ad192017-09-06 16:18:47 -0700133 if (shadowp)
134 *shadowp = p;
Johannes Weiner22f2ac52016-09-30 15:11:29 -0700135 }
Johannes Weiner14b46872016-12-12 16:43:52 -0800136 __radix_tree_replace(&mapping->page_tree, node, slot, page,
137 workingset_update_node, mapping);
Johannes Weiner22f2ac52016-09-30 15:11:29 -0700138 mapping->nrpages++;
Johannes Weiner22f2ac52016-09-30 15:11:29 -0700139 return 0;
140}
141
Johannes Weiner91b0abe2014-04-03 14:47:49 -0700142static void page_cache_tree_delete(struct address_space *mapping,
143 struct page *page, void *shadow)
144{
Kirill A. Shutemovc70b6472016-12-12 16:43:17 -0800145 int i, nr;
146
147 /* hugetlb pages are represented by one entry in the radix tree */
148 nr = PageHuge(page) ? 1 : hpage_nr_pages(page);
Johannes Weiner91b0abe2014-04-03 14:47:49 -0700149
Kirill A. Shutemov83929372016-07-26 15:26:04 -0700150 VM_BUG_ON_PAGE(!PageLocked(page), page);
151 VM_BUG_ON_PAGE(PageTail(page), page);
152 VM_BUG_ON_PAGE(nr != 1 && shadow, page);
Johannes Weiner449dd692014-04-03 14:47:56 -0700153
Kirill A. Shutemov83929372016-07-26 15:26:04 -0700154 for (i = 0; i < nr; i++) {
Johannes Weinerd3798ae2016-10-04 22:02:08 +0200155 struct radix_tree_node *node;
156 void **slot;
157
158 __radix_tree_lookup(&mapping->page_tree, page->index + i,
159 &node, &slot);
160
Johannes Weinerdbc446b2016-12-12 16:43:55 -0800161 VM_BUG_ON_PAGE(!node && nr != 1, page);
Johannes Weiner449dd692014-04-03 14:47:56 -0700162
Johannes Weiner14b46872016-12-12 16:43:52 -0800163 radix_tree_clear_tags(&mapping->page_tree, node, slot);
164 __radix_tree_replace(&mapping->page_tree, node, slot, shadow,
165 workingset_update_node, mapping);
Johannes Weiner449dd692014-04-03 14:47:56 -0700166 }
Johannes Weinerd3798ae2016-10-04 22:02:08 +0200167
168 if (shadow) {
169 mapping->nrexceptional += nr;
170 /*
171 * Make sure the nrexceptional update is committed before
172 * the nrpages update so that final truncate racing
173 * with reclaim does not see both counters 0 at the
174 * same time and miss a shadow entry.
175 */
176 smp_wmb();
177 }
178 mapping->nrpages -= nr;
Johannes Weiner91b0abe2014-04-03 14:47:49 -0700179}
180
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181/*
Minchan Kime64a7822011-03-22 16:32:44 -0700182 * Delete a page from the page cache and free it. Caller has to make
Linus Torvalds1da177e2005-04-16 15:20:36 -0700183 * sure the page is locked and that nobody else uses it - or that usage
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700184 * is safe. The caller must hold the mapping's tree_lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185 */
Johannes Weiner62cccb82016-03-15 14:57:22 -0700186void __delete_from_page_cache(struct page *page, void *shadow)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187{
188 struct address_space *mapping = page->mapping;
Kirill A. Shutemov83929372016-07-26 15:26:04 -0700189 int nr = hpage_nr_pages(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190
Robert Jarzmikfe0bfaa2013-04-29 15:06:10 -0700191 trace_mm_filemap_delete_from_page_cache(page);
Dan Magenheimerc515e1f2011-05-26 10:01:43 -0600192 /*
193 * if we're uptodate, flush out into the cleancache, otherwise
194 * invalidate any existing cleancache entries. We can't leave
195 * stale data around in the cleancache once our page is gone
196 */
197 if (PageUptodate(page) && PageMappedToDisk(page))
198 cleancache_put_page(page);
199 else
Dan Magenheimer31677602011-09-21 11:56:28 -0400200 cleancache_invalidate_page(mapping, page);
Dan Magenheimerc515e1f2011-05-26 10:01:43 -0600201
Kirill A. Shutemov83929372016-07-26 15:26:04 -0700202 VM_BUG_ON_PAGE(PageTail(page), page);
Hugh Dickins06b241f2016-03-09 14:08:07 -0800203 VM_BUG_ON_PAGE(page_mapped(page), page);
204 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(page_mapped(page))) {
205 int mapcount;
206
207 pr_alert("BUG: Bad page cache in process %s pfn:%05lx\n",
208 current->comm, page_to_pfn(page));
209 dump_page(page, "still mapped when deleted");
210 dump_stack();
211 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
212
213 mapcount = page_mapcount(page);
214 if (mapping_exiting(mapping) &&
215 page_count(page) >= mapcount + 2) {
216 /*
217 * All vmas have already been torn down, so it's
218 * a good bet that actually the page is unmapped,
219 * and we'd prefer not to leak it: if we're wrong,
220 * some other bad page check should catch it later.
221 */
222 page_mapcount_reset(page);
Joonsoo Kim6d061f92016-05-19 17:10:46 -0700223 page_ref_sub(page, mapcount);
Hugh Dickins06b241f2016-03-09 14:08:07 -0800224 }
225 }
226
Johannes Weiner91b0abe2014-04-03 14:47:49 -0700227 page_cache_tree_delete(mapping, page, shadow);
228
Linus Torvalds1da177e2005-04-16 15:20:36 -0700229 page->mapping = NULL;
Hugh Dickinsb85e0ef2011-07-25 17:12:25 -0700230 /* Leave page->index set: truncation lookup relies upon it */
Johannes Weiner91b0abe2014-04-03 14:47:49 -0700231
Michal Hocko4165b9b2015-06-24 16:57:24 -0700232 /* hugetlb pages do not participate in page cache accounting. */
Naoya Horiguchi09612fa2017-07-10 15:47:35 -0700233 if (PageHuge(page))
234 return;
235
236 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr);
Kirill A. Shutemov800d8c62016-07-26 15:26:18 -0700237 if (PageSwapBacked(page)) {
Mel Gorman11fb9982016-07-28 15:46:20 -0700238 __mod_node_page_state(page_pgdat(page), NR_SHMEM, -nr);
Kirill A. Shutemov800d8c62016-07-26 15:26:18 -0700239 if (PageTransHuge(page))
Mel Gorman11fb9982016-07-28 15:46:20 -0700240 __dec_node_page_state(page, NR_SHMEM_THPS);
Kirill A. Shutemov800d8c62016-07-26 15:26:18 -0700241 } else {
Naoya Horiguchi09612fa2017-07-10 15:47:35 -0700242 VM_BUG_ON_PAGE(PageTransHuge(page), page);
Kirill A. Shutemov800d8c62016-07-26 15:26:18 -0700243 }
Linus Torvalds3a692792007-12-19 14:05:13 -0800244
245 /*
Konstantin Khlebnikovb9ea2512015-04-14 15:45:27 -0700246 * At this point page must be either written or cleaned by truncate.
247 * Dirty page here signals a bug and loss of unwritten data.
Linus Torvalds3a692792007-12-19 14:05:13 -0800248 *
Konstantin Khlebnikovb9ea2512015-04-14 15:45:27 -0700249 * This fixes dirty accounting after removing the page entirely but
250 * leaves PageDirty set: it has no effect for truncated page and
251 * anyway will be cleared before returning page into buddy allocator.
Linus Torvalds3a692792007-12-19 14:05:13 -0800252 */
Konstantin Khlebnikovb9ea2512015-04-14 15:45:27 -0700253 if (WARN_ON_ONCE(PageDirty(page)))
Johannes Weiner62cccb82016-03-15 14:57:22 -0700254 account_page_cleaned(page, mapping, inode_to_wb(mapping->host));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700255}
256
Minchan Kim702cfbf2011-03-22 16:32:43 -0700257/**
258 * delete_from_page_cache - delete page from page cache
259 * @page: the page which the kernel is trying to remove from page cache
260 *
261 * This must be called only on pages that have been verified to be in the page
262 * cache and locked. It will never put the page into the free list, the caller
263 * has a reference on the page.
264 */
265void delete_from_page_cache(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266{
Kirill A. Shutemov83929372016-07-26 15:26:04 -0700267 struct address_space *mapping = page_mapping(page);
Greg Thelenc4843a72015-05-22 17:13:16 -0400268 unsigned long flags;
Linus Torvalds6072d132010-12-01 13:35:19 -0500269 void (*freepage)(struct page *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270
Matt Mackallcd7619d2005-05-01 08:59:01 -0700271 BUG_ON(!PageLocked(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272
Linus Torvalds6072d132010-12-01 13:35:19 -0500273 freepage = mapping->a_ops->freepage;
Greg Thelenc4843a72015-05-22 17:13:16 -0400274
Greg Thelenc4843a72015-05-22 17:13:16 -0400275 spin_lock_irqsave(&mapping->tree_lock, flags);
Johannes Weiner62cccb82016-03-15 14:57:22 -0700276 __delete_from_page_cache(page, NULL);
Greg Thelenc4843a72015-05-22 17:13:16 -0400277 spin_unlock_irqrestore(&mapping->tree_lock, flags);
Linus Torvalds6072d132010-12-01 13:35:19 -0500278
279 if (freepage)
280 freepage(page);
Kirill A. Shutemov83929372016-07-26 15:26:04 -0700281
282 if (PageTransHuge(page) && !PageHuge(page)) {
283 page_ref_sub(page, HPAGE_PMD_NR);
284 VM_BUG_ON_PAGE(page_count(page) <= 0, page);
285 } else {
286 put_page(page);
287 }
Minchan Kim97cecb52011-03-22 16:30:53 -0700288}
289EXPORT_SYMBOL(delete_from_page_cache);
290
Miklos Szeredid72d9e22016-07-29 14:10:57 +0200291int filemap_check_errors(struct address_space *mapping)
Dmitry Monakhov865ffef32013-04-29 15:08:42 -0700292{
293 int ret = 0;
294 /* Check for outstanding write errors */
Jens Axboe7fcbbaf2014-05-22 11:54:16 -0700295 if (test_bit(AS_ENOSPC, &mapping->flags) &&
296 test_and_clear_bit(AS_ENOSPC, &mapping->flags))
Dmitry Monakhov865ffef32013-04-29 15:08:42 -0700297 ret = -ENOSPC;
Jens Axboe7fcbbaf2014-05-22 11:54:16 -0700298 if (test_bit(AS_EIO, &mapping->flags) &&
299 test_and_clear_bit(AS_EIO, &mapping->flags))
Dmitry Monakhov865ffef32013-04-29 15:08:42 -0700300 ret = -EIO;
301 return ret;
302}
Miklos Szeredid72d9e22016-07-29 14:10:57 +0200303EXPORT_SYMBOL(filemap_check_errors);
Dmitry Monakhov865ffef32013-04-29 15:08:42 -0700304
Jeff Layton76341ca2017-07-06 07:02:22 -0400305static int filemap_check_and_keep_errors(struct address_space *mapping)
306{
307 /* Check for outstanding write errors */
308 if (test_bit(AS_EIO, &mapping->flags))
309 return -EIO;
310 if (test_bit(AS_ENOSPC, &mapping->flags))
311 return -ENOSPC;
312 return 0;
313}
314
Linus Torvalds1da177e2005-04-16 15:20:36 -0700315/**
Randy Dunlap485bb992006-06-23 02:03:49 -0700316 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
Martin Waitz67be2dd2005-05-01 08:59:26 -0700317 * @mapping: address space structure to write
318 * @start: offset in bytes where the range starts
Andrew Morton469eb4d2006-03-24 03:17:45 -0800319 * @end: offset in bytes where the range ends (inclusive)
Martin Waitz67be2dd2005-05-01 08:59:26 -0700320 * @sync_mode: enable synchronous operation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321 *
Randy Dunlap485bb992006-06-23 02:03:49 -0700322 * Start writeback against all of a mapping's dirty pages that lie
323 * within the byte offsets <start, end> inclusive.
324 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
Randy Dunlap485bb992006-06-23 02:03:49 -0700326 * opposed to a regular memory cleansing writeback. The difference between
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327 * these two operations is that if a dirty page/buffer is encountered, it must
328 * be waited upon, and not just skipped over.
329 */
Andrew Mortonebcf28e2006-03-24 03:18:04 -0800330int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
331 loff_t end, int sync_mode)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700332{
333 int ret;
334 struct writeback_control wbc = {
335 .sync_mode = sync_mode,
Nick Piggin05fe4782009-01-06 14:39:08 -0800336 .nr_to_write = LONG_MAX,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700337 .range_start = start,
338 .range_end = end,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339 };
340
341 if (!mapping_cap_writeback_dirty(mapping))
342 return 0;
343
Tejun Heob16b1de2015-06-02 08:39:48 -0600344 wbc_attach_fdatawrite_inode(&wbc, mapping->host);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345 ret = do_writepages(mapping, &wbc);
Tejun Heob16b1de2015-06-02 08:39:48 -0600346 wbc_detach_inode(&wbc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 return ret;
348}
349
350static inline int __filemap_fdatawrite(struct address_space *mapping,
351 int sync_mode)
352{
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700353 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354}
355
356int filemap_fdatawrite(struct address_space *mapping)
357{
358 return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
359}
360EXPORT_SYMBOL(filemap_fdatawrite);
361
Jan Karaf4c0a0f2008-07-11 19:27:31 -0400362int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
Andrew Mortonebcf28e2006-03-24 03:18:04 -0800363 loff_t end)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364{
365 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
366}
Jan Karaf4c0a0f2008-07-11 19:27:31 -0400367EXPORT_SYMBOL(filemap_fdatawrite_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368
Randy Dunlap485bb992006-06-23 02:03:49 -0700369/**
370 * filemap_flush - mostly a non-blocking flush
371 * @mapping: target address_space
372 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 * This is a mostly non-blocking flush. Not suitable for data-integrity
374 * purposes - I/O may not be started against all dirty pages.
375 */
376int filemap_flush(struct address_space *mapping)
377{
378 return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
379}
380EXPORT_SYMBOL(filemap_flush);
381
Goldwyn Rodrigues7fc9e472017-06-20 07:05:41 -0500382/**
383 * filemap_range_has_page - check if a page exists in range.
384 * @mapping: address space within which to check
385 * @start_byte: offset in bytes where the range starts
386 * @end_byte: offset in bytes where the range ends (inclusive)
387 *
388 * Find at least one page in the range supplied, usually used to check if
389 * direct writing in this range will trigger a writeback.
390 */
391bool filemap_range_has_page(struct address_space *mapping,
392 loff_t start_byte, loff_t end_byte)
393{
394 pgoff_t index = start_byte >> PAGE_SHIFT;
395 pgoff_t end = end_byte >> PAGE_SHIFT;
Jan Karaf7b68042017-09-06 16:21:40 -0700396 struct page *page;
Goldwyn Rodrigues7fc9e472017-06-20 07:05:41 -0500397
398 if (end_byte < start_byte)
399 return false;
400
401 if (mapping->nrpages == 0)
402 return false;
403
Jan Karaf7b68042017-09-06 16:21:40 -0700404 if (!find_get_pages_range(mapping, &index, end, 1, &page))
Goldwyn Rodrigues7fc9e472017-06-20 07:05:41 -0500405 return false;
Jan Karaf7b68042017-09-06 16:21:40 -0700406 put_page(page);
407 return true;
Goldwyn Rodrigues7fc9e472017-06-20 07:05:41 -0500408}
409EXPORT_SYMBOL(filemap_range_has_page);
410
Jeff Layton5e8fcc12017-07-06 07:02:24 -0400411static void __filemap_fdatawait_range(struct address_space *mapping,
Junichi Nomuraaa750fd2015-11-05 18:47:23 -0800412 loff_t start_byte, loff_t end_byte)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300414 pgoff_t index = start_byte >> PAGE_SHIFT;
415 pgoff_t end = end_byte >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 struct pagevec pvec;
417 int nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418
Christoph Hellwig94004ed2009-09-30 22:16:33 +0200419 if (end_byte < start_byte)
Jeff Layton5e8fcc12017-07-06 07:02:24 -0400420 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700421
422 pagevec_init(&pvec, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700423 while ((index <= end) &&
424 (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
425 PAGECACHE_TAG_WRITEBACK,
426 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
427 unsigned i;
428
429 for (i = 0; i < nr_pages; i++) {
430 struct page *page = pvec.pages[i];
431
432 /* until radix tree lookup accepts end_index */
433 if (page->index > end)
434 continue;
435
436 wait_on_page_writeback(page);
Jeff Layton5e8fcc12017-07-06 07:02:24 -0400437 ClearPageError(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700438 }
439 pagevec_release(&pvec);
440 cond_resched();
441 }
Junichi Nomuraaa750fd2015-11-05 18:47:23 -0800442}
443
444/**
445 * filemap_fdatawait_range - wait for writeback to complete
446 * @mapping: address space structure to wait for
447 * @start_byte: offset in bytes where the range starts
448 * @end_byte: offset in bytes where the range ends (inclusive)
449 *
450 * Walk the list of under-writeback pages of the given address space
451 * in the given range and wait for all of them. Check error status of
452 * the address space and return it.
453 *
454 * Since the error status of the address space is cleared by this function,
455 * callers are responsible for checking the return value and handling and/or
456 * reporting the error.
457 */
458int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
459 loff_t end_byte)
460{
Jeff Layton5e8fcc12017-07-06 07:02:24 -0400461 __filemap_fdatawait_range(mapping, start_byte, end_byte);
462 return filemap_check_errors(mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700463}
Jan Karad3bccb6f2009-08-17 19:30:27 +0200464EXPORT_SYMBOL(filemap_fdatawait_range);
465
466/**
Jeff Laytona823e452017-07-28 07:24:43 -0400467 * file_fdatawait_range - wait for writeback to complete
468 * @file: file pointing to address space structure to wait for
469 * @start_byte: offset in bytes where the range starts
470 * @end_byte: offset in bytes where the range ends (inclusive)
471 *
472 * Walk the list of under-writeback pages of the address space that file
473 * refers to, in the given range and wait for all of them. Check error
474 * status of the address space vs. the file->f_wb_err cursor and return it.
475 *
476 * Since the error status of the file is advanced by this function,
477 * callers are responsible for checking the return value and handling and/or
478 * reporting the error.
479 */
480int file_fdatawait_range(struct file *file, loff_t start_byte, loff_t end_byte)
481{
482 struct address_space *mapping = file->f_mapping;
483
484 __filemap_fdatawait_range(mapping, start_byte, end_byte);
485 return file_check_and_advance_wb_err(file);
486}
487EXPORT_SYMBOL(file_fdatawait_range);
488
489/**
Junichi Nomuraaa750fd2015-11-05 18:47:23 -0800490 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
491 * @mapping: address space structure to wait for
492 *
493 * Walk the list of under-writeback pages of the given address space
494 * and wait for all of them. Unlike filemap_fdatawait(), this function
495 * does not clear error status of the address space.
496 *
497 * Use this function if callers don't handle errors themselves. Expected
498 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
499 * fsfreeze(8)
500 */
Jeff Layton76341ca2017-07-06 07:02:22 -0400501int filemap_fdatawait_keep_errors(struct address_space *mapping)
Junichi Nomuraaa750fd2015-11-05 18:47:23 -0800502{
Jeff Laytonffb959b2017-07-31 10:29:38 -0400503 __filemap_fdatawait_range(mapping, 0, LLONG_MAX);
Jeff Layton76341ca2017-07-06 07:02:22 -0400504 return filemap_check_and_keep_errors(mapping);
Junichi Nomuraaa750fd2015-11-05 18:47:23 -0800505}
Jeff Layton76341ca2017-07-06 07:02:22 -0400506EXPORT_SYMBOL(filemap_fdatawait_keep_errors);
Junichi Nomuraaa750fd2015-11-05 18:47:23 -0800507
Jeff Layton9326c9b2017-07-26 10:21:11 -0400508static bool mapping_needs_writeback(struct address_space *mapping)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700509{
Jeff Layton9326c9b2017-07-26 10:21:11 -0400510 return (!dax_mapping(mapping) && mapping->nrpages) ||
511 (dax_mapping(mapping) && mapping->nrexceptional);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700512}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513
514int filemap_write_and_wait(struct address_space *mapping)
515{
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800516 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517
Jeff Layton9326c9b2017-07-26 10:21:11 -0400518 if (mapping_needs_writeback(mapping)) {
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800519 err = filemap_fdatawrite(mapping);
520 /*
521 * Even if the above returned error, the pages may be
522 * written partially (e.g. -ENOSPC), so we wait for it.
523 * But the -EIO is special case, it may indicate the worst
524 * thing (e.g. bug) happened, so we avoid waiting for it.
525 */
526 if (err != -EIO) {
527 int err2 = filemap_fdatawait(mapping);
528 if (!err)
529 err = err2;
Jeff Laytoncbeaf952017-07-06 07:02:23 -0400530 } else {
531 /* Clear any previously stored errors */
532 filemap_check_errors(mapping);
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800533 }
Dmitry Monakhov865ffef32013-04-29 15:08:42 -0700534 } else {
535 err = filemap_check_errors(mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700536 }
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800537 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538}
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800539EXPORT_SYMBOL(filemap_write_and_wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540
Randy Dunlap485bb992006-06-23 02:03:49 -0700541/**
542 * filemap_write_and_wait_range - write out & wait on a file range
543 * @mapping: the address_space for the pages
544 * @lstart: offset in bytes where the range starts
545 * @lend: offset in bytes where the range ends (inclusive)
546 *
Andrew Morton469eb4d2006-03-24 03:17:45 -0800547 * Write out and wait upon file offsets lstart->lend, inclusive.
548 *
mchehab@s-opensource.com0e056eb2017-03-30 17:11:36 -0300549 * Note that @lend is inclusive (describes the last byte to be written) so
Andrew Morton469eb4d2006-03-24 03:17:45 -0800550 * that this function can be used to write to the very end-of-file (end = -1).
551 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552int filemap_write_and_wait_range(struct address_space *mapping,
553 loff_t lstart, loff_t lend)
554{
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800555 int err = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556
Jeff Layton9326c9b2017-07-26 10:21:11 -0400557 if (mapping_needs_writeback(mapping)) {
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800558 err = __filemap_fdatawrite_range(mapping, lstart, lend,
559 WB_SYNC_ALL);
560 /* See comment of filemap_write_and_wait() */
561 if (err != -EIO) {
Christoph Hellwig94004ed2009-09-30 22:16:33 +0200562 int err2 = filemap_fdatawait_range(mapping,
563 lstart, lend);
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800564 if (!err)
565 err = err2;
Jeff Laytoncbeaf952017-07-06 07:02:23 -0400566 } else {
567 /* Clear any previously stored errors */
568 filemap_check_errors(mapping);
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800569 }
Dmitry Monakhov865ffef32013-04-29 15:08:42 -0700570 } else {
571 err = filemap_check_errors(mapping);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 }
OGAWA Hirofumi28fd1292006-01-08 01:02:14 -0800573 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574}
Chris Masonf6995582009-04-15 13:22:37 -0400575EXPORT_SYMBOL(filemap_write_and_wait_range);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576
Jeff Layton5660e132017-07-06 07:02:25 -0400577void __filemap_set_wb_err(struct address_space *mapping, int err)
578{
Jeff Layton3acdfd22017-07-24 06:22:15 -0400579 errseq_t eseq = errseq_set(&mapping->wb_err, err);
Jeff Layton5660e132017-07-06 07:02:25 -0400580
581 trace_filemap_set_wb_err(mapping, eseq);
582}
583EXPORT_SYMBOL(__filemap_set_wb_err);
584
585/**
586 * file_check_and_advance_wb_err - report wb error (if any) that was previously
587 * and advance wb_err to current one
588 * @file: struct file on which the error is being reported
589 *
590 * When userland calls fsync (or something like nfsd does the equivalent), we
591 * want to report any writeback errors that occurred since the last fsync (or
592 * since the file was opened if there haven't been any).
593 *
594 * Grab the wb_err from the mapping. If it matches what we have in the file,
595 * then just quickly return 0. The file is all caught up.
596 *
597 * If it doesn't match, then take the mapping value, set the "seen" flag in
598 * it and try to swap it into place. If it works, or another task beat us
599 * to it with the new value, then update the f_wb_err and return the error
600 * portion. The error at this point must be reported via proper channels
601 * (a'la fsync, or NFS COMMIT operation, etc.).
602 *
603 * While we handle mapping->wb_err with atomic operations, the f_wb_err
604 * value is protected by the f_lock since we must ensure that it reflects
605 * the latest value swapped in for this file descriptor.
606 */
607int file_check_and_advance_wb_err(struct file *file)
608{
609 int err = 0;
610 errseq_t old = READ_ONCE(file->f_wb_err);
611 struct address_space *mapping = file->f_mapping;
612
613 /* Locklessly handle the common case where nothing has changed */
614 if (errseq_check(&mapping->wb_err, old)) {
615 /* Something changed, must use slow path */
616 spin_lock(&file->f_lock);
617 old = file->f_wb_err;
618 err = errseq_check_and_advance(&mapping->wb_err,
619 &file->f_wb_err);
620 trace_file_check_and_advance_wb_err(file, old);
621 spin_unlock(&file->f_lock);
622 }
Jeff Laytonf4e222c2017-10-03 16:15:25 -0700623
624 /*
625 * We're mostly using this function as a drop in replacement for
626 * filemap_check_errors. Clear AS_EIO/AS_ENOSPC to emulate the effect
627 * that the legacy code would have had on these flags.
628 */
629 clear_bit(AS_EIO, &mapping->flags);
630 clear_bit(AS_ENOSPC, &mapping->flags);
Jeff Layton5660e132017-07-06 07:02:25 -0400631 return err;
632}
633EXPORT_SYMBOL(file_check_and_advance_wb_err);
634
635/**
636 * file_write_and_wait_range - write out & wait on a file range
637 * @file: file pointing to address_space with pages
638 * @lstart: offset in bytes where the range starts
639 * @lend: offset in bytes where the range ends (inclusive)
640 *
641 * Write out and wait upon file offsets lstart->lend, inclusive.
642 *
643 * Note that @lend is inclusive (describes the last byte to be written) so
644 * that this function can be used to write to the very end-of-file (end = -1).
645 *
646 * After writing out and waiting on the data, we check and advance the
647 * f_wb_err cursor to the latest value, and return any errors detected there.
648 */
649int file_write_and_wait_range(struct file *file, loff_t lstart, loff_t lend)
650{
651 int err = 0, err2;
652 struct address_space *mapping = file->f_mapping;
653
Jeff Layton9326c9b2017-07-26 10:21:11 -0400654 if (mapping_needs_writeback(mapping)) {
Jeff Layton5660e132017-07-06 07:02:25 -0400655 err = __filemap_fdatawrite_range(mapping, lstart, lend,
656 WB_SYNC_ALL);
657 /* See comment of filemap_write_and_wait() */
658 if (err != -EIO)
659 __filemap_fdatawait_range(mapping, lstart, lend);
660 }
661 err2 = file_check_and_advance_wb_err(file);
662 if (!err)
663 err = err2;
664 return err;
665}
666EXPORT_SYMBOL(file_write_and_wait_range);
667
Randy Dunlap485bb992006-06-23 02:03:49 -0700668/**
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700669 * replace_page_cache_page - replace a pagecache page with a new one
670 * @old: page to be replaced
671 * @new: page to replace with
672 * @gfp_mask: allocation mode
673 *
674 * This function replaces a page in the pagecache with a new one. On
675 * success it acquires the pagecache reference for the new page and
676 * drops it for the old page. Both the old and new pages must be
677 * locked. This function does not add the new page to the LRU, the
678 * caller must do that.
679 *
680 * The remove + add is atomic. The only way this function can fail is
681 * memory allocation failure.
682 */
683int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
684{
685 int error;
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700686
Sasha Levin309381fea2014-01-23 15:52:54 -0800687 VM_BUG_ON_PAGE(!PageLocked(old), old);
688 VM_BUG_ON_PAGE(!PageLocked(new), new);
689 VM_BUG_ON_PAGE(new->mapping, new);
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700690
Matthew Wilcox90a32d12018-04-20 14:56:20 -0700691 error = radix_tree_preload(gfp_mask & GFP_RECLAIM_MASK);
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700692 if (!error) {
693 struct address_space *mapping = old->mapping;
694 void (*freepage)(struct page *);
Greg Thelenc4843a72015-05-22 17:13:16 -0400695 unsigned long flags;
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700696
697 pgoff_t offset = old->index;
698 freepage = mapping->a_ops->freepage;
699
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300700 get_page(new);
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700701 new->mapping = mapping;
702 new->index = offset;
703
Greg Thelenc4843a72015-05-22 17:13:16 -0400704 spin_lock_irqsave(&mapping->tree_lock, flags);
Johannes Weiner62cccb82016-03-15 14:57:22 -0700705 __delete_from_page_cache(old, NULL);
Johannes Weiner22f2ac52016-09-30 15:11:29 -0700706 error = page_cache_tree_insert(mapping, new, NULL);
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700707 BUG_ON(error);
Michal Hocko4165b9b2015-06-24 16:57:24 -0700708
709 /*
710 * hugetlb pages do not participate in page cache accounting.
711 */
712 if (!PageHuge(new))
Mel Gorman11fb9982016-07-28 15:46:20 -0700713 __inc_node_page_state(new, NR_FILE_PAGES);
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700714 if (PageSwapBacked(new))
Mel Gorman11fb9982016-07-28 15:46:20 -0700715 __inc_node_page_state(new, NR_SHMEM);
Greg Thelenc4843a72015-05-22 17:13:16 -0400716 spin_unlock_irqrestore(&mapping->tree_lock, flags);
Johannes Weiner6a93ca82016-03-15 14:57:19 -0700717 mem_cgroup_migrate(old, new);
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700718 radix_tree_preload_end();
719 if (freepage)
720 freepage(old);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300721 put_page(old);
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700722 }
723
724 return error;
725}
726EXPORT_SYMBOL_GPL(replace_page_cache_page);
727
Johannes Weinera5289102014-04-03 14:47:51 -0700728static int __add_to_page_cache_locked(struct page *page,
729 struct address_space *mapping,
730 pgoff_t offset, gfp_t gfp_mask,
731 void **shadowp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732{
Johannes Weiner00501b52014-08-08 14:19:20 -0700733 int huge = PageHuge(page);
734 struct mem_cgroup *memcg;
Nick Piggine2867812008-07-25 19:45:30 -0700735 int error;
736
Sasha Levin309381fea2014-01-23 15:52:54 -0800737 VM_BUG_ON_PAGE(!PageLocked(page), page);
738 VM_BUG_ON_PAGE(PageSwapBacked(page), page);
Nick Piggine2867812008-07-25 19:45:30 -0700739
Johannes Weiner00501b52014-08-08 14:19:20 -0700740 if (!huge) {
741 error = mem_cgroup_try_charge(page, current->mm,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800742 gfp_mask, &memcg, false);
Johannes Weiner00501b52014-08-08 14:19:20 -0700743 if (error)
744 return error;
745 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746
Matthew Wilcox90a32d12018-04-20 14:56:20 -0700747 error = radix_tree_maybe_preload(gfp_mask & GFP_RECLAIM_MASK);
Kirill A. Shutemov66a0c8e2013-09-12 15:13:59 -0700748 if (error) {
Johannes Weiner00501b52014-08-08 14:19:20 -0700749 if (!huge)
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800750 mem_cgroup_cancel_charge(page, memcg, false);
Kirill A. Shutemov66a0c8e2013-09-12 15:13:59 -0700751 return error;
752 }
753
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300754 get_page(page);
Kirill A. Shutemov66a0c8e2013-09-12 15:13:59 -0700755 page->mapping = mapping;
756 page->index = offset;
757
758 spin_lock_irq(&mapping->tree_lock);
Johannes Weinera5289102014-04-03 14:47:51 -0700759 error = page_cache_tree_insert(mapping, page, shadowp);
Kirill A. Shutemov66a0c8e2013-09-12 15:13:59 -0700760 radix_tree_preload_end();
761 if (unlikely(error))
762 goto err_insert;
Michal Hocko4165b9b2015-06-24 16:57:24 -0700763
764 /* hugetlb pages do not participate in page cache accounting. */
765 if (!huge)
Mel Gorman11fb9982016-07-28 15:46:20 -0700766 __inc_node_page_state(page, NR_FILE_PAGES);
Kirill A. Shutemov66a0c8e2013-09-12 15:13:59 -0700767 spin_unlock_irq(&mapping->tree_lock);
Johannes Weiner00501b52014-08-08 14:19:20 -0700768 if (!huge)
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800769 mem_cgroup_commit_charge(page, memcg, false, false);
Kirill A. Shutemov66a0c8e2013-09-12 15:13:59 -0700770 trace_mm_filemap_add_to_page_cache(page);
771 return 0;
772err_insert:
773 page->mapping = NULL;
774 /* Leave page->index set: truncation relies upon it */
775 spin_unlock_irq(&mapping->tree_lock);
Johannes Weiner00501b52014-08-08 14:19:20 -0700776 if (!huge)
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800777 mem_cgroup_cancel_charge(page, memcg, false);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300778 put_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779 return error;
780}
Johannes Weinera5289102014-04-03 14:47:51 -0700781
782/**
783 * add_to_page_cache_locked - add a locked page to the pagecache
784 * @page: page to add
785 * @mapping: the page's address_space
786 * @offset: page index
787 * @gfp_mask: page allocation mode
788 *
789 * This function is used to add a page to the pagecache. It must be locked.
790 * This function does not add the page to the LRU. The caller must do that.
791 */
792int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
793 pgoff_t offset, gfp_t gfp_mask)
794{
795 return __add_to_page_cache_locked(page, mapping, offset,
796 gfp_mask, NULL);
797}
Nick Piggine2867812008-07-25 19:45:30 -0700798EXPORT_SYMBOL(add_to_page_cache_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700799
800int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
Al Viro6daa0e22005-10-21 03:18:50 -0400801 pgoff_t offset, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802{
Johannes Weinera5289102014-04-03 14:47:51 -0700803 void *shadow = NULL;
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700804 int ret;
805
Kirill A. Shutemov48c935a2016-01-15 16:51:24 -0800806 __SetPageLocked(page);
Johannes Weinera5289102014-04-03 14:47:51 -0700807 ret = __add_to_page_cache_locked(page, mapping, offset,
808 gfp_mask, &shadow);
809 if (unlikely(ret))
Kirill A. Shutemov48c935a2016-01-15 16:51:24 -0800810 __ClearPageLocked(page);
Johannes Weinera5289102014-04-03 14:47:51 -0700811 else {
812 /*
813 * The page might have been evicted from cache only
814 * recently, in which case it should be activated like
815 * any other repeatedly accessed page.
Rik van Rielf0281a02016-05-20 16:56:25 -0700816 * The exception is pages getting rewritten; evicting other
817 * data from the working set, only to cache data that will
818 * get overwritten with something else, is a waste of memory.
Johannes Weinera5289102014-04-03 14:47:51 -0700819 */
Rik van Rielf0281a02016-05-20 16:56:25 -0700820 if (!(gfp_mask & __GFP_WRITE) &&
821 shadow && workingset_refault(shadow)) {
Johannes Weinera5289102014-04-03 14:47:51 -0700822 SetPageActive(page);
823 workingset_activation(page);
824 } else
825 ClearPageActive(page);
826 lru_cache_add(page);
827 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828 return ret;
829}
Evgeniy Polyakov18bc0bb2009-02-09 17:02:42 +0300830EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700831
Paul Jackson44110fe2006-03-24 03:16:04 -0800832#ifdef CONFIG_NUMA
Nick Piggin2ae88142006-10-28 10:38:23 -0700833struct page *__page_cache_alloc(gfp_t gfp)
Paul Jackson44110fe2006-03-24 03:16:04 -0800834{
Miao Xiec0ff7452010-05-24 14:32:08 -0700835 int n;
836 struct page *page;
837
Paul Jackson44110fe2006-03-24 03:16:04 -0800838 if (cpuset_do_page_mem_spread()) {
Mel Gormancc9a6c82012-03-21 16:34:11 -0700839 unsigned int cpuset_mems_cookie;
840 do {
Mel Gormand26914d2014-04-03 14:47:24 -0700841 cpuset_mems_cookie = read_mems_allowed_begin();
Mel Gormancc9a6c82012-03-21 16:34:11 -0700842 n = cpuset_mem_spread_node();
Vlastimil Babka96db8002015-09-08 15:03:50 -0700843 page = __alloc_pages_node(n, gfp, 0);
Mel Gormand26914d2014-04-03 14:47:24 -0700844 } while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
Mel Gormancc9a6c82012-03-21 16:34:11 -0700845
Miao Xiec0ff7452010-05-24 14:32:08 -0700846 return page;
Paul Jackson44110fe2006-03-24 03:16:04 -0800847 }
Nick Piggin2ae88142006-10-28 10:38:23 -0700848 return alloc_pages(gfp, 0);
Paul Jackson44110fe2006-03-24 03:16:04 -0800849}
Nick Piggin2ae88142006-10-28 10:38:23 -0700850EXPORT_SYMBOL(__page_cache_alloc);
Paul Jackson44110fe2006-03-24 03:16:04 -0800851#endif
852
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853/*
854 * In order to wait for pages to become available there must be
855 * waitqueues associated with pages. By using a hash table of
856 * waitqueues where the bucket discipline is to maintain all
857 * waiters on the same queue and wake all when any of the pages
858 * become available, and for the woken contexts to check to be
859 * sure the appropriate page became available, this saves space
860 * at a cost of "thundering herd" phenomena during rare hash
861 * collisions.
862 */
Nicholas Piggin62906022016-12-25 13:00:30 +1000863#define PAGE_WAIT_TABLE_BITS 8
864#define PAGE_WAIT_TABLE_SIZE (1 << PAGE_WAIT_TABLE_BITS)
865static wait_queue_head_t page_wait_table[PAGE_WAIT_TABLE_SIZE] __cacheline_aligned;
866
867static wait_queue_head_t *page_waitqueue(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868{
Nicholas Piggin62906022016-12-25 13:00:30 +1000869 return &page_wait_table[hash_ptr(page, PAGE_WAIT_TABLE_BITS)];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870}
Nicholas Piggin62906022016-12-25 13:00:30 +1000871
872void __init pagecache_init(void)
873{
874 int i;
875
876 for (i = 0; i < PAGE_WAIT_TABLE_SIZE; i++)
877 init_waitqueue_head(&page_wait_table[i]);
878
879 page_writeback_init();
880}
881
Linus Torvalds3510ca22017-08-27 13:55:12 -0700882/* This has the same layout as wait_bit_key - see fs/cachefiles/rdwr.c */
Nicholas Piggin62906022016-12-25 13:00:30 +1000883struct wait_page_key {
884 struct page *page;
885 int bit_nr;
886 int page_match;
887};
888
889struct wait_page_queue {
890 struct page *page;
891 int bit_nr;
Ingo Molnarac6424b2017-06-20 12:06:13 +0200892 wait_queue_entry_t wait;
Nicholas Piggin62906022016-12-25 13:00:30 +1000893};
894
Ingo Molnarac6424b2017-06-20 12:06:13 +0200895static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *arg)
Nicholas Piggin62906022016-12-25 13:00:30 +1000896{
897 struct wait_page_key *key = arg;
898 struct wait_page_queue *wait_page
899 = container_of(wait, struct wait_page_queue, wait);
900
901 if (wait_page->page != key->page)
902 return 0;
903 key->page_match = 1;
904
905 if (wait_page->bit_nr != key->bit_nr)
906 return 0;
Linus Torvalds3510ca22017-08-27 13:55:12 -0700907
908 /* Stop walking if it's locked */
Nicholas Piggin62906022016-12-25 13:00:30 +1000909 if (test_bit(key->bit_nr, &key->page->flags))
Linus Torvalds3510ca22017-08-27 13:55:12 -0700910 return -1;
Nicholas Piggin62906022016-12-25 13:00:30 +1000911
912 return autoremove_wake_function(wait, mode, sync, key);
913}
914
Nicholas Piggin74d81bf2017-02-22 15:44:41 -0800915static void wake_up_page_bit(struct page *page, int bit_nr)
Nicholas Piggin62906022016-12-25 13:00:30 +1000916{
917 wait_queue_head_t *q = page_waitqueue(page);
918 struct wait_page_key key;
919 unsigned long flags;
Tim Chen11a19c72017-08-25 09:13:55 -0700920 wait_queue_entry_t bookmark;
Nicholas Piggin62906022016-12-25 13:00:30 +1000921
922 key.page = page;
923 key.bit_nr = bit_nr;
924 key.page_match = 0;
925
Tim Chen11a19c72017-08-25 09:13:55 -0700926 bookmark.flags = 0;
927 bookmark.private = NULL;
928 bookmark.func = NULL;
929 INIT_LIST_HEAD(&bookmark.entry);
930
Nicholas Piggin62906022016-12-25 13:00:30 +1000931 spin_lock_irqsave(&q->lock, flags);
Tim Chen11a19c72017-08-25 09:13:55 -0700932 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
933
934 while (bookmark.flags & WQ_FLAG_BOOKMARK) {
935 /*
936 * Take a breather from holding the lock,
937 * allow pages that finish wake up asynchronously
938 * to acquire the lock and remove themselves
939 * from wait queue
940 */
941 spin_unlock_irqrestore(&q->lock, flags);
942 cpu_relax();
943 spin_lock_irqsave(&q->lock, flags);
944 __wake_up_locked_key_bookmark(q, TASK_NORMAL, &key, &bookmark);
945 }
946
Nicholas Piggin62906022016-12-25 13:00:30 +1000947 /*
948 * It is possible for other pages to have collided on the waitqueue
949 * hash, so in that case check for a page match. That prevents a long-
950 * term waiter
951 *
952 * It is still possible to miss a case here, when we woke page waiters
953 * and removed them from the waitqueue, but there are still other
954 * page waiters.
955 */
956 if (!waitqueue_active(q) || !key.page_match) {
957 ClearPageWaiters(page);
958 /*
959 * It's possible to miss clearing Waiters here, when we woke
960 * our page waiters, but the hashed waitqueue has waiters for
961 * other pages on it.
962 *
963 * That's okay, it's a rare case. The next waker will clear it.
964 */
965 }
966 spin_unlock_irqrestore(&q->lock, flags);
967}
Nicholas Piggin74d81bf2017-02-22 15:44:41 -0800968
969static void wake_up_page(struct page *page, int bit)
970{
971 if (!PageWaiters(page))
972 return;
973 wake_up_page_bit(page, bit);
974}
Nicholas Piggin62906022016-12-25 13:00:30 +1000975
976static inline int wait_on_page_bit_common(wait_queue_head_t *q,
977 struct page *page, int bit_nr, int state, bool lock)
978{
979 struct wait_page_queue wait_page;
Ingo Molnarac6424b2017-06-20 12:06:13 +0200980 wait_queue_entry_t *wait = &wait_page.wait;
Nicholas Piggin62906022016-12-25 13:00:30 +1000981 int ret = 0;
982
983 init_wait(wait);
Linus Torvalds3510ca22017-08-27 13:55:12 -0700984 wait->flags = lock ? WQ_FLAG_EXCLUSIVE : 0;
Nicholas Piggin62906022016-12-25 13:00:30 +1000985 wait->func = wake_page_function;
986 wait_page.page = page;
987 wait_page.bit_nr = bit_nr;
988
989 for (;;) {
990 spin_lock_irq(&q->lock);
991
Ingo Molnar2055da92017-06-20 12:06:46 +0200992 if (likely(list_empty(&wait->entry))) {
Linus Torvalds3510ca22017-08-27 13:55:12 -0700993 __add_wait_queue_entry_tail(q, wait);
Nicholas Piggin62906022016-12-25 13:00:30 +1000994 SetPageWaiters(page);
995 }
996
997 set_current_state(state);
998
999 spin_unlock_irq(&q->lock);
1000
1001 if (likely(test_bit(bit_nr, &page->flags))) {
1002 io_schedule();
Nicholas Piggin62906022016-12-25 13:00:30 +10001003 }
1004
1005 if (lock) {
1006 if (!test_and_set_bit_lock(bit_nr, &page->flags))
1007 break;
1008 } else {
1009 if (!test_bit(bit_nr, &page->flags))
1010 break;
1011 }
Linus Torvaldsa8b169a2017-08-27 16:25:09 -07001012
1013 if (unlikely(signal_pending_state(state, current))) {
1014 ret = -EINTR;
1015 break;
1016 }
Nicholas Piggin62906022016-12-25 13:00:30 +10001017 }
1018
1019 finish_wait(q, wait);
1020
1021 /*
1022 * A signal could leave PageWaiters set. Clearing it here if
1023 * !waitqueue_active would be possible (by open-coding finish_wait),
1024 * but still fail to catch it in the case of wait hash collision. We
1025 * already can fail to clear wait hash collision cases, so don't
1026 * bother with signals either.
1027 */
1028
1029 return ret;
1030}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001031
Harvey Harrison920c7a52008-02-04 22:29:26 -08001032void wait_on_page_bit(struct page *page, int bit_nr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033{
Nicholas Piggin62906022016-12-25 13:00:30 +10001034 wait_queue_head_t *q = page_waitqueue(page);
1035 wait_on_page_bit_common(q, page, bit_nr, TASK_UNINTERRUPTIBLE, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001036}
1037EXPORT_SYMBOL(wait_on_page_bit);
1038
KOSAKI Motohirof62e00c2011-05-24 17:11:29 -07001039int wait_on_page_bit_killable(struct page *page, int bit_nr)
1040{
Nicholas Piggin62906022016-12-25 13:00:30 +10001041 wait_queue_head_t *q = page_waitqueue(page);
1042 return wait_on_page_bit_common(q, page, bit_nr, TASK_KILLABLE, false);
KOSAKI Motohirof62e00c2011-05-24 17:11:29 -07001043}
1044
Linus Torvalds1da177e2005-04-16 15:20:36 -07001045/**
David Howells385e1ca5f2009-04-03 16:42:39 +01001046 * add_page_wait_queue - Add an arbitrary waiter to a page's wait queue
Randy Dunlap697f6192009-04-13 14:39:54 -07001047 * @page: Page defining the wait queue of interest
1048 * @waiter: Waiter to add to the queue
David Howells385e1ca5f2009-04-03 16:42:39 +01001049 *
1050 * Add an arbitrary @waiter to the wait queue for the nominated @page.
1051 */
Ingo Molnarac6424b2017-06-20 12:06:13 +02001052void add_page_wait_queue(struct page *page, wait_queue_entry_t *waiter)
David Howells385e1ca5f2009-04-03 16:42:39 +01001053{
1054 wait_queue_head_t *q = page_waitqueue(page);
1055 unsigned long flags;
1056
1057 spin_lock_irqsave(&q->lock, flags);
Linus Torvalds9c3a8152017-08-28 16:45:40 -07001058 __add_wait_queue_entry_tail(q, waiter);
Nicholas Piggin62906022016-12-25 13:00:30 +10001059 SetPageWaiters(page);
David Howells385e1ca5f2009-04-03 16:42:39 +01001060 spin_unlock_irqrestore(&q->lock, flags);
1061}
1062EXPORT_SYMBOL_GPL(add_page_wait_queue);
1063
Linus Torvaldsb91e1302016-12-27 11:40:38 -08001064#ifndef clear_bit_unlock_is_negative_byte
1065
1066/*
1067 * PG_waiters is the high bit in the same byte as PG_lock.
1068 *
1069 * On x86 (and on many other architectures), we can clear PG_lock and
1070 * test the sign bit at the same time. But if the architecture does
1071 * not support that special operation, we just do this all by hand
1072 * instead.
1073 *
1074 * The read of PG_waiters has to be after (or concurrently with) PG_locked
1075 * being cleared, but a memory barrier should be unneccssary since it is
1076 * in the same byte as PG_locked.
1077 */
1078static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile void *mem)
1079{
1080 clear_bit_unlock(nr, mem);
1081 /* smp_mb__after_atomic(); */
Olof Johansson98473f92016-12-29 14:16:07 -08001082 return test_bit(PG_waiters, mem);
Linus Torvaldsb91e1302016-12-27 11:40:38 -08001083}
1084
1085#endif
1086
David Howells385e1ca5f2009-04-03 16:42:39 +01001087/**
Randy Dunlap485bb992006-06-23 02:03:49 -07001088 * unlock_page - unlock a locked page
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089 * @page: the page
1090 *
1091 * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
1092 * Also wakes sleepers in wait_on_page_writeback() because the wakeup
Masanari Iidada3dae52014-09-09 01:27:23 +09001093 * mechanism between PageLocked pages and PageWriteback pages is shared.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001094 * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
1095 *
Linus Torvaldsb91e1302016-12-27 11:40:38 -08001096 * Note that this depends on PG_waiters being the sign bit in the byte
1097 * that contains PG_locked - thus the BUILD_BUG_ON(). That allows us to
1098 * clear the PG_locked bit and test PG_waiters at the same time fairly
1099 * portably (architectures that do LL/SC can test any bit, while x86 can
1100 * test the sign bit).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001101 */
Harvey Harrison920c7a52008-02-04 22:29:26 -08001102void unlock_page(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001103{
Linus Torvaldsb91e1302016-12-27 11:40:38 -08001104 BUILD_BUG_ON(PG_waiters != 7);
Kirill A. Shutemov48c935a2016-01-15 16:51:24 -08001105 page = compound_head(page);
Sasha Levin309381fea2014-01-23 15:52:54 -08001106 VM_BUG_ON_PAGE(!PageLocked(page), page);
Linus Torvaldsb91e1302016-12-27 11:40:38 -08001107 if (clear_bit_unlock_is_negative_byte(PG_locked, &page->flags))
1108 wake_up_page_bit(page, PG_locked);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109}
1110EXPORT_SYMBOL(unlock_page);
1111
Randy Dunlap485bb992006-06-23 02:03:49 -07001112/**
1113 * end_page_writeback - end writeback against a page
1114 * @page: the page
Linus Torvalds1da177e2005-04-16 15:20:36 -07001115 */
1116void end_page_writeback(struct page *page)
1117{
Mel Gorman888cf2d2014-06-04 16:10:34 -07001118 /*
1119 * TestClearPageReclaim could be used here but it is an atomic
1120 * operation and overkill in this particular case. Failing to
1121 * shuffle a page marked for immediate reclaim is too mild to
1122 * justify taking an atomic operation penalty at the end of
1123 * ever page writeback.
1124 */
1125 if (PageReclaim(page)) {
1126 ClearPageReclaim(page);
Miklos Szerediac6aadb2008-04-28 02:12:38 -07001127 rotate_reclaimable_page(page);
Mel Gorman888cf2d2014-06-04 16:10:34 -07001128 }
Miklos Szerediac6aadb2008-04-28 02:12:38 -07001129
1130 if (!test_clear_page_writeback(page))
1131 BUG();
1132
Peter Zijlstra4e857c52014-03-17 18:06:10 +01001133 smp_mb__after_atomic();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134 wake_up_page(page, PG_writeback);
1135}
1136EXPORT_SYMBOL(end_page_writeback);
1137
Matthew Wilcox57d99842014-06-04 16:07:45 -07001138/*
1139 * After completing I/O on a page, call this routine to update the page
1140 * flags appropriately
1141 */
Jens Axboec11f0c02016-08-05 08:11:04 -06001142void page_endio(struct page *page, bool is_write, int err)
Matthew Wilcox57d99842014-06-04 16:07:45 -07001143{
Jens Axboec11f0c02016-08-05 08:11:04 -06001144 if (!is_write) {
Matthew Wilcox57d99842014-06-04 16:07:45 -07001145 if (!err) {
1146 SetPageUptodate(page);
1147 } else {
1148 ClearPageUptodate(page);
1149 SetPageError(page);
1150 }
1151 unlock_page(page);
Mike Christieabf54542016-08-04 14:23:34 -06001152 } else {
Matthew Wilcox57d99842014-06-04 16:07:45 -07001153 if (err) {
Minchan Kimdd8416c2017-02-24 14:59:59 -08001154 struct address_space *mapping;
1155
Matthew Wilcox57d99842014-06-04 16:07:45 -07001156 SetPageError(page);
Minchan Kimdd8416c2017-02-24 14:59:59 -08001157 mapping = page_mapping(page);
1158 if (mapping)
1159 mapping_set_error(mapping, err);
Matthew Wilcox57d99842014-06-04 16:07:45 -07001160 }
1161 end_page_writeback(page);
1162 }
1163}
1164EXPORT_SYMBOL_GPL(page_endio);
1165
Randy Dunlap485bb992006-06-23 02:03:49 -07001166/**
1167 * __lock_page - get a lock on the page, assuming we need to sleep to get it
Randy Dunlap87066752017-02-22 15:44:44 -08001168 * @__page: the page to lock
Linus Torvalds1da177e2005-04-16 15:20:36 -07001169 */
Nicholas Piggin62906022016-12-25 13:00:30 +10001170void __lock_page(struct page *__page)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171{
Nicholas Piggin62906022016-12-25 13:00:30 +10001172 struct page *page = compound_head(__page);
1173 wait_queue_head_t *q = page_waitqueue(page);
1174 wait_on_page_bit_common(q, page, PG_locked, TASK_UNINTERRUPTIBLE, true);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175}
1176EXPORT_SYMBOL(__lock_page);
1177
Nicholas Piggin62906022016-12-25 13:00:30 +10001178int __lock_page_killable(struct page *__page)
Matthew Wilcox2687a352007-12-06 11:18:49 -05001179{
Nicholas Piggin62906022016-12-25 13:00:30 +10001180 struct page *page = compound_head(__page);
1181 wait_queue_head_t *q = page_waitqueue(page);
1182 return wait_on_page_bit_common(q, page, PG_locked, TASK_KILLABLE, true);
Matthew Wilcox2687a352007-12-06 11:18:49 -05001183}
Evgeniy Polyakov18bc0bb2009-02-09 17:02:42 +03001184EXPORT_SYMBOL_GPL(__lock_page_killable);
Matthew Wilcox2687a352007-12-06 11:18:49 -05001185
Paul Cassella9a95f3c2014-08-06 16:07:24 -07001186/*
1187 * Return values:
1188 * 1 - page is locked; mmap_sem is still held.
1189 * 0 - page is not locked.
1190 * mmap_sem has been released (up_read()), unless flags had both
1191 * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in
1192 * which case mmap_sem is still held.
1193 *
1194 * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1
1195 * with the page locked and the mmap_sem unperturbed.
1196 */
Michel Lespinassed065bd82010-10-26 14:21:57 -07001197int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
1198 unsigned int flags)
1199{
KOSAKI Motohiro37b23e02011-05-24 17:11:30 -07001200 if (flags & FAULT_FLAG_ALLOW_RETRY) {
1201 /*
1202 * CAUTION! In this case, mmap_sem is not released
1203 * even though return 0.
1204 */
1205 if (flags & FAULT_FLAG_RETRY_NOWAIT)
1206 return 0;
1207
1208 up_read(&mm->mmap_sem);
1209 if (flags & FAULT_FLAG_KILLABLE)
1210 wait_on_page_locked_killable(page);
1211 else
Gleb Natapov318b2752011-03-22 16:30:51 -07001212 wait_on_page_locked(page);
Michel Lespinassed065bd82010-10-26 14:21:57 -07001213 return 0;
KOSAKI Motohiro37b23e02011-05-24 17:11:30 -07001214 } else {
1215 if (flags & FAULT_FLAG_KILLABLE) {
1216 int ret;
1217
1218 ret = __lock_page_killable(page);
1219 if (ret) {
1220 up_read(&mm->mmap_sem);
1221 return 0;
1222 }
1223 } else
1224 __lock_page(page);
1225 return 1;
Michel Lespinassed065bd82010-10-26 14:21:57 -07001226 }
1227}
1228
Randy Dunlap485bb992006-06-23 02:03:49 -07001229/**
Johannes Weinere7b563b2014-04-03 14:47:44 -07001230 * page_cache_next_hole - find the next hole (not-present entry)
1231 * @mapping: mapping
1232 * @index: index
1233 * @max_scan: maximum range to search
1234 *
1235 * Search the set [index, min(index+max_scan-1, MAX_INDEX)] for the
1236 * lowest indexed hole.
1237 *
1238 * Returns: the index of the hole if found, otherwise returns an index
1239 * outside of the set specified (in which case 'return - index >=
1240 * max_scan' will be true). In rare cases of index wrap-around, 0 will
1241 * be returned.
1242 *
1243 * page_cache_next_hole may be called under rcu_read_lock. However,
1244 * like radix_tree_gang_lookup, this will not atomically search a
1245 * snapshot of the tree at a single point in time. For example, if a
1246 * hole is created at index 5, then subsequently a hole is created at
1247 * index 10, page_cache_next_hole covering both indexes may return 10
1248 * if called under rcu_read_lock.
1249 */
1250pgoff_t page_cache_next_hole(struct address_space *mapping,
1251 pgoff_t index, unsigned long max_scan)
1252{
1253 unsigned long i;
1254
1255 for (i = 0; i < max_scan; i++) {
Johannes Weiner0cd61442014-04-03 14:47:46 -07001256 struct page *page;
1257
1258 page = radix_tree_lookup(&mapping->page_tree, index);
1259 if (!page || radix_tree_exceptional_entry(page))
Johannes Weinere7b563b2014-04-03 14:47:44 -07001260 break;
1261 index++;
1262 if (index == 0)
1263 break;
1264 }
1265
1266 return index;
1267}
1268EXPORT_SYMBOL(page_cache_next_hole);
1269
1270/**
1271 * page_cache_prev_hole - find the prev hole (not-present entry)
1272 * @mapping: mapping
1273 * @index: index
1274 * @max_scan: maximum range to search
1275 *
1276 * Search backwards in the range [max(index-max_scan+1, 0), index] for
1277 * the first hole.
1278 *
1279 * Returns: the index of the hole if found, otherwise returns an index
1280 * outside of the set specified (in which case 'index - return >=
1281 * max_scan' will be true). In rare cases of wrap-around, ULONG_MAX
1282 * will be returned.
1283 *
1284 * page_cache_prev_hole may be called under rcu_read_lock. However,
1285 * like radix_tree_gang_lookup, this will not atomically search a
1286 * snapshot of the tree at a single point in time. For example, if a
1287 * hole is created at index 10, then subsequently a hole is created at
1288 * index 5, page_cache_prev_hole covering both indexes may return 5 if
1289 * called under rcu_read_lock.
1290 */
1291pgoff_t page_cache_prev_hole(struct address_space *mapping,
1292 pgoff_t index, unsigned long max_scan)
1293{
1294 unsigned long i;
1295
1296 for (i = 0; i < max_scan; i++) {
Johannes Weiner0cd61442014-04-03 14:47:46 -07001297 struct page *page;
1298
1299 page = radix_tree_lookup(&mapping->page_tree, index);
1300 if (!page || radix_tree_exceptional_entry(page))
Johannes Weinere7b563b2014-04-03 14:47:44 -07001301 break;
1302 index--;
1303 if (index == ULONG_MAX)
1304 break;
1305 }
1306
1307 return index;
1308}
1309EXPORT_SYMBOL(page_cache_prev_hole);
1310
1311/**
Johannes Weiner0cd61442014-04-03 14:47:46 -07001312 * find_get_entry - find and get a page cache entry
Randy Dunlap485bb992006-06-23 02:03:49 -07001313 * @mapping: the address_space to search
Johannes Weiner0cd61442014-04-03 14:47:46 -07001314 * @offset: the page cache index
Randy Dunlap485bb992006-06-23 02:03:49 -07001315 *
Johannes Weiner0cd61442014-04-03 14:47:46 -07001316 * Looks up the page cache slot at @mapping & @offset. If there is a
1317 * page cache page, it is returned with an increased refcount.
1318 *
Johannes Weiner139b6a62014-05-06 12:50:05 -07001319 * If the slot holds a shadow entry of a previously evicted page, or a
1320 * swap entry from shmem/tmpfs, it is returned.
Johannes Weiner0cd61442014-04-03 14:47:46 -07001321 *
1322 * Otherwise, %NULL is returned.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001323 */
Johannes Weiner0cd61442014-04-03 14:47:46 -07001324struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001325{
Nick Piggina60637c2008-07-25 19:45:31 -07001326 void **pagep;
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001327 struct page *head, *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328
Nick Piggina60637c2008-07-25 19:45:31 -07001329 rcu_read_lock();
1330repeat:
1331 page = NULL;
1332 pagep = radix_tree_lookup_slot(&mapping->page_tree, offset);
1333 if (pagep) {
1334 page = radix_tree_deref_slot(pagep);
Nick Piggin27d20fd2010-11-11 14:05:19 -08001335 if (unlikely(!page))
1336 goto out;
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001337 if (radix_tree_exception(page)) {
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001338 if (radix_tree_deref_retry(page))
1339 goto repeat;
1340 /*
Johannes Weiner139b6a62014-05-06 12:50:05 -07001341 * A shadow entry of a recently evicted page,
1342 * or a swap entry from shmem/tmpfs. Return
1343 * it without attempting to raise page count.
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001344 */
1345 goto out;
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001346 }
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001347
1348 head = compound_head(page);
1349 if (!page_cache_get_speculative(head))
Nick Piggina60637c2008-07-25 19:45:31 -07001350 goto repeat;
1351
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001352 /* The page was split under us? */
1353 if (compound_head(page) != head) {
1354 put_page(head);
1355 goto repeat;
1356 }
1357
Nick Piggina60637c2008-07-25 19:45:31 -07001358 /*
1359 * Has the page moved?
1360 * This is part of the lockless pagecache protocol. See
1361 * include/linux/pagemap.h for details.
1362 */
1363 if (unlikely(page != *pagep)) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001364 put_page(head);
Nick Piggina60637c2008-07-25 19:45:31 -07001365 goto repeat;
1366 }
1367 }
Nick Piggin27d20fd2010-11-11 14:05:19 -08001368out:
Nick Piggina60637c2008-07-25 19:45:31 -07001369 rcu_read_unlock();
1370
Linus Torvalds1da177e2005-04-16 15:20:36 -07001371 return page;
1372}
Johannes Weiner0cd61442014-04-03 14:47:46 -07001373EXPORT_SYMBOL(find_get_entry);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001374
Randy Dunlap485bb992006-06-23 02:03:49 -07001375/**
Johannes Weiner0cd61442014-04-03 14:47:46 -07001376 * find_lock_entry - locate, pin and lock a page cache entry
1377 * @mapping: the address_space to search
1378 * @offset: the page cache index
1379 *
1380 * Looks up the page cache slot at @mapping & @offset. If there is a
1381 * page cache page, it is returned locked and with an increased
1382 * refcount.
1383 *
Johannes Weiner139b6a62014-05-06 12:50:05 -07001384 * If the slot holds a shadow entry of a previously evicted page, or a
1385 * swap entry from shmem/tmpfs, it is returned.
Johannes Weiner0cd61442014-04-03 14:47:46 -07001386 *
1387 * Otherwise, %NULL is returned.
1388 *
1389 * find_lock_entry() may sleep.
1390 */
1391struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392{
1393 struct page *page;
1394
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395repeat:
Johannes Weiner0cd61442014-04-03 14:47:46 -07001396 page = find_get_entry(mapping, offset);
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001397 if (page && !radix_tree_exception(page)) {
Nick Piggina60637c2008-07-25 19:45:31 -07001398 lock_page(page);
1399 /* Has the page been truncated? */
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001400 if (unlikely(page_mapping(page) != mapping)) {
Nick Piggina60637c2008-07-25 19:45:31 -07001401 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001402 put_page(page);
Nick Piggina60637c2008-07-25 19:45:31 -07001403 goto repeat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001404 }
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001405 VM_BUG_ON_PAGE(page_to_pgoff(page) != offset, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001406 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001407 return page;
1408}
Johannes Weiner0cd61442014-04-03 14:47:46 -07001409EXPORT_SYMBOL(find_lock_entry);
1410
1411/**
Mel Gorman2457aec2014-06-04 16:10:31 -07001412 * pagecache_get_page - find and get a page reference
Johannes Weiner0cd61442014-04-03 14:47:46 -07001413 * @mapping: the address_space to search
1414 * @offset: the page index
Mel Gorman2457aec2014-06-04 16:10:31 -07001415 * @fgp_flags: PCG flags
Michal Hocko45f87de2014-12-29 20:30:35 +01001416 * @gfp_mask: gfp mask to use for the page cache data page allocation
Johannes Weiner0cd61442014-04-03 14:47:46 -07001417 *
Mel Gorman2457aec2014-06-04 16:10:31 -07001418 * Looks up the page cache slot at @mapping & @offset.
Johannes Weiner0cd61442014-04-03 14:47:46 -07001419 *
Randy Dunlap75325182014-07-30 16:08:37 -07001420 * PCG flags modify how the page is returned.
Johannes Weiner0cd61442014-04-03 14:47:46 -07001421 *
mchehab@s-opensource.com0e056eb2017-03-30 17:11:36 -03001422 * @fgp_flags can be:
1423 *
1424 * - FGP_ACCESSED: the page will be marked accessed
1425 * - FGP_LOCK: Page is return locked
1426 * - FGP_CREAT: If page is not present then a new page is allocated using
1427 * @gfp_mask and added to the page cache and the VM's LRU
1428 * list. The page is returned locked and with an increased
1429 * refcount. Otherwise, NULL is returned.
Mel Gorman2457aec2014-06-04 16:10:31 -07001430 *
1431 * If FGP_LOCK or FGP_CREAT are specified then the function may sleep even
1432 * if the GFP flags specified for FGP_CREAT are atomic.
1433 *
1434 * If there is a page cache page, it is returned with an increased refcount.
Johannes Weiner0cd61442014-04-03 14:47:46 -07001435 */
Mel Gorman2457aec2014-06-04 16:10:31 -07001436struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
Michal Hocko45f87de2014-12-29 20:30:35 +01001437 int fgp_flags, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001438{
Nick Piggineb2be182007-10-16 01:24:57 -07001439 struct page *page;
Mel Gorman2457aec2014-06-04 16:10:31 -07001440
Linus Torvalds1da177e2005-04-16 15:20:36 -07001441repeat:
Mel Gorman2457aec2014-06-04 16:10:31 -07001442 page = find_get_entry(mapping, offset);
1443 if (radix_tree_exceptional_entry(page))
1444 page = NULL;
1445 if (!page)
1446 goto no_page;
1447
1448 if (fgp_flags & FGP_LOCK) {
1449 if (fgp_flags & FGP_NOWAIT) {
1450 if (!trylock_page(page)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001451 put_page(page);
Mel Gorman2457aec2014-06-04 16:10:31 -07001452 return NULL;
1453 }
1454 } else {
1455 lock_page(page);
1456 }
1457
1458 /* Has the page been truncated? */
1459 if (unlikely(page->mapping != mapping)) {
1460 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001461 put_page(page);
Mel Gorman2457aec2014-06-04 16:10:31 -07001462 goto repeat;
1463 }
1464 VM_BUG_ON_PAGE(page->index != offset, page);
1465 }
1466
1467 if (page && (fgp_flags & FGP_ACCESSED))
1468 mark_page_accessed(page);
1469
1470no_page:
1471 if (!page && (fgp_flags & FGP_CREAT)) {
1472 int err;
1473 if ((fgp_flags & FGP_WRITE) && mapping_cap_account_dirty(mapping))
Michal Hocko45f87de2014-12-29 20:30:35 +01001474 gfp_mask |= __GFP_WRITE;
1475 if (fgp_flags & FGP_NOFS)
1476 gfp_mask &= ~__GFP_FS;
Mel Gorman2457aec2014-06-04 16:10:31 -07001477
Michal Hocko45f87de2014-12-29 20:30:35 +01001478 page = __page_cache_alloc(gfp_mask);
Nick Piggineb2be182007-10-16 01:24:57 -07001479 if (!page)
1480 return NULL;
Mel Gorman2457aec2014-06-04 16:10:31 -07001481
1482 if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK)))
1483 fgp_flags |= FGP_LOCK;
1484
Hugh Dickinseb39d612014-08-06 16:06:43 -07001485 /* Init accessed so avoid atomic mark_page_accessed later */
Mel Gorman2457aec2014-06-04 16:10:31 -07001486 if (fgp_flags & FGP_ACCESSED)
Hugh Dickinseb39d612014-08-06 16:06:43 -07001487 __SetPageReferenced(page);
Mel Gorman2457aec2014-06-04 16:10:31 -07001488
Matthew Wilcox90a32d12018-04-20 14:56:20 -07001489 err = add_to_page_cache_lru(page, mapping, offset, gfp_mask);
Nick Piggineb2be182007-10-16 01:24:57 -07001490 if (unlikely(err)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001491 put_page(page);
Nick Piggineb2be182007-10-16 01:24:57 -07001492 page = NULL;
1493 if (err == -EEXIST)
1494 goto repeat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001496 }
Mel Gorman2457aec2014-06-04 16:10:31 -07001497
Linus Torvalds1da177e2005-04-16 15:20:36 -07001498 return page;
1499}
Mel Gorman2457aec2014-06-04 16:10:31 -07001500EXPORT_SYMBOL(pagecache_get_page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001501
1502/**
Johannes Weiner0cd61442014-04-03 14:47:46 -07001503 * find_get_entries - gang pagecache lookup
1504 * @mapping: The address_space to search
1505 * @start: The starting page cache index
1506 * @nr_entries: The maximum number of entries
1507 * @entries: Where the resulting entries are placed
1508 * @indices: The cache indices corresponding to the entries in @entries
1509 *
1510 * find_get_entries() will search for and return a group of up to
1511 * @nr_entries entries in the mapping. The entries are placed at
1512 * @entries. find_get_entries() takes a reference against any actual
1513 * pages it returns.
1514 *
1515 * The search returns a group of mapping-contiguous page cache entries
1516 * with ascending indexes. There may be holes in the indices due to
1517 * not-present pages.
1518 *
Johannes Weiner139b6a62014-05-06 12:50:05 -07001519 * Any shadow entries of evicted pages, or swap entries from
1520 * shmem/tmpfs, are included in the returned array.
Johannes Weiner0cd61442014-04-03 14:47:46 -07001521 *
1522 * find_get_entries() returns the number of pages and shadow entries
1523 * which were found.
1524 */
1525unsigned find_get_entries(struct address_space *mapping,
1526 pgoff_t start, unsigned int nr_entries,
1527 struct page **entries, pgoff_t *indices)
1528{
1529 void **slot;
1530 unsigned int ret = 0;
1531 struct radix_tree_iter iter;
1532
1533 if (!nr_entries)
1534 return 0;
1535
1536 rcu_read_lock();
Johannes Weiner0cd61442014-04-03 14:47:46 -07001537 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001538 struct page *head, *page;
Johannes Weiner0cd61442014-04-03 14:47:46 -07001539repeat:
1540 page = radix_tree_deref_slot(slot);
1541 if (unlikely(!page))
1542 continue;
1543 if (radix_tree_exception(page)) {
Matthew Wilcox2cf938a2016-03-17 14:22:03 -07001544 if (radix_tree_deref_retry(page)) {
1545 slot = radix_tree_iter_retry(&iter);
1546 continue;
1547 }
Johannes Weiner0cd61442014-04-03 14:47:46 -07001548 /*
Ross Zwislerf9fe48b2016-01-22 15:10:40 -08001549 * A shadow entry of a recently evicted page, a swap
1550 * entry from shmem/tmpfs or a DAX entry. Return it
1551 * without attempting to raise page count.
Johannes Weiner0cd61442014-04-03 14:47:46 -07001552 */
1553 goto export;
1554 }
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001555
1556 head = compound_head(page);
1557 if (!page_cache_get_speculative(head))
Johannes Weiner0cd61442014-04-03 14:47:46 -07001558 goto repeat;
1559
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001560 /* The page was split under us? */
1561 if (compound_head(page) != head) {
1562 put_page(head);
1563 goto repeat;
1564 }
1565
Johannes Weiner0cd61442014-04-03 14:47:46 -07001566 /* Has the page moved? */
1567 if (unlikely(page != *slot)) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001568 put_page(head);
Johannes Weiner0cd61442014-04-03 14:47:46 -07001569 goto repeat;
1570 }
1571export:
1572 indices[ret] = iter.index;
1573 entries[ret] = page;
1574 if (++ret == nr_entries)
1575 break;
1576 }
1577 rcu_read_unlock();
1578 return ret;
1579}
1580
1581/**
Jan Karab947cee2017-09-06 16:21:21 -07001582 * find_get_pages_range - gang pagecache lookup
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 * @mapping: The address_space to search
1584 * @start: The starting page index
Jan Karab947cee2017-09-06 16:21:21 -07001585 * @end: The final page index (inclusive)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586 * @nr_pages: The maximum number of pages
1587 * @pages: Where the resulting pages are placed
1588 *
Jan Karab947cee2017-09-06 16:21:21 -07001589 * find_get_pages_range() will search for and return a group of up to @nr_pages
1590 * pages in the mapping starting at index @start and up to index @end
1591 * (inclusive). The pages are placed at @pages. find_get_pages_range() takes
1592 * a reference against the returned pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001593 *
1594 * The search returns a group of mapping-contiguous pages with ascending
1595 * indexes. There may be holes in the indices due to not-present pages.
Jan Karad72dc8a2017-09-06 16:21:18 -07001596 * We also update @start to index the next page for the traversal.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001597 *
Jan Karab947cee2017-09-06 16:21:21 -07001598 * find_get_pages_range() returns the number of pages which were found. If this
1599 * number is smaller than @nr_pages, the end of specified range has been
1600 * reached.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601 */
Jan Karab947cee2017-09-06 16:21:21 -07001602unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
1603 pgoff_t end, unsigned int nr_pages,
1604 struct page **pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001605{
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001606 struct radix_tree_iter iter;
1607 void **slot;
1608 unsigned ret = 0;
1609
1610 if (unlikely(!nr_pages))
1611 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001612
Nick Piggina60637c2008-07-25 19:45:31 -07001613 rcu_read_lock();
Jan Karad72dc8a2017-09-06 16:21:18 -07001614 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, *start) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001615 struct page *head, *page;
Jan Karab947cee2017-09-06 16:21:21 -07001616
1617 if (iter.index > end)
1618 break;
Nick Piggina60637c2008-07-25 19:45:31 -07001619repeat:
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001620 page = radix_tree_deref_slot(slot);
Nick Piggina60637c2008-07-25 19:45:31 -07001621 if (unlikely(!page))
1622 continue;
Hugh Dickins9d8aa4e2011-03-22 16:33:06 -07001623
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001624 if (radix_tree_exception(page)) {
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001625 if (radix_tree_deref_retry(page)) {
Matthew Wilcox2cf938a2016-03-17 14:22:03 -07001626 slot = radix_tree_iter_retry(&iter);
1627 continue;
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001628 }
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001629 /*
Johannes Weiner139b6a62014-05-06 12:50:05 -07001630 * A shadow entry of a recently evicted page,
1631 * or a swap entry from shmem/tmpfs. Skip
1632 * over it.
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001633 */
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001634 continue;
Nick Piggin27d20fd2010-11-11 14:05:19 -08001635 }
Nick Piggina60637c2008-07-25 19:45:31 -07001636
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001637 head = compound_head(page);
1638 if (!page_cache_get_speculative(head))
Nick Piggina60637c2008-07-25 19:45:31 -07001639 goto repeat;
1640
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001641 /* The page was split under us? */
1642 if (compound_head(page) != head) {
1643 put_page(head);
1644 goto repeat;
1645 }
1646
Nick Piggina60637c2008-07-25 19:45:31 -07001647 /* Has the page moved? */
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001648 if (unlikely(page != *slot)) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001649 put_page(head);
Nick Piggina60637c2008-07-25 19:45:31 -07001650 goto repeat;
1651 }
1652
1653 pages[ret] = page;
Jan Karab947cee2017-09-06 16:21:21 -07001654 if (++ret == nr_pages) {
1655 *start = pages[ret - 1]->index + 1;
1656 goto out;
1657 }
Nick Piggina60637c2008-07-25 19:45:31 -07001658 }
Hugh Dickins5b280c02011-03-22 16:33:07 -07001659
Jan Karab947cee2017-09-06 16:21:21 -07001660 /*
1661 * We come here when there is no page beyond @end. We take care to not
1662 * overflow the index @start as it confuses some of the callers. This
1663 * breaks the iteration when there is page at index -1 but that is
1664 * already broken anyway.
1665 */
1666 if (end == (pgoff_t)-1)
1667 *start = (pgoff_t)-1;
1668 else
1669 *start = end + 1;
1670out:
Nick Piggina60637c2008-07-25 19:45:31 -07001671 rcu_read_unlock();
Jan Karad72dc8a2017-09-06 16:21:18 -07001672
Linus Torvalds1da177e2005-04-16 15:20:36 -07001673 return ret;
1674}
1675
Jens Axboeebf43502006-04-27 08:46:01 +02001676/**
1677 * find_get_pages_contig - gang contiguous pagecache lookup
1678 * @mapping: The address_space to search
1679 * @index: The starting page index
1680 * @nr_pages: The maximum number of pages
1681 * @pages: Where the resulting pages are placed
1682 *
1683 * find_get_pages_contig() works exactly like find_get_pages(), except
1684 * that the returned number of pages are guaranteed to be contiguous.
1685 *
1686 * find_get_pages_contig() returns the number of pages which were found.
1687 */
1688unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
1689 unsigned int nr_pages, struct page **pages)
1690{
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001691 struct radix_tree_iter iter;
1692 void **slot;
1693 unsigned int ret = 0;
1694
1695 if (unlikely(!nr_pages))
1696 return 0;
Jens Axboeebf43502006-04-27 08:46:01 +02001697
Nick Piggina60637c2008-07-25 19:45:31 -07001698 rcu_read_lock();
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001699 radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001700 struct page *head, *page;
Nick Piggina60637c2008-07-25 19:45:31 -07001701repeat:
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001702 page = radix_tree_deref_slot(slot);
1703 /* The hole, there no reason to continue */
Nick Piggina60637c2008-07-25 19:45:31 -07001704 if (unlikely(!page))
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001705 break;
Hugh Dickins9d8aa4e2011-03-22 16:33:06 -07001706
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001707 if (radix_tree_exception(page)) {
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001708 if (radix_tree_deref_retry(page)) {
Matthew Wilcox2cf938a2016-03-17 14:22:03 -07001709 slot = radix_tree_iter_retry(&iter);
1710 continue;
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001711 }
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001712 /*
Johannes Weiner139b6a62014-05-06 12:50:05 -07001713 * A shadow entry of a recently evicted page,
1714 * or a swap entry from shmem/tmpfs. Stop
1715 * looking for contiguous pages.
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001716 */
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001717 break;
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001718 }
Nick Piggina60637c2008-07-25 19:45:31 -07001719
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001720 head = compound_head(page);
1721 if (!page_cache_get_speculative(head))
Nick Piggina60637c2008-07-25 19:45:31 -07001722 goto repeat;
1723
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001724 /* The page was split under us? */
1725 if (compound_head(page) != head) {
1726 put_page(head);
1727 goto repeat;
1728 }
1729
Nick Piggina60637c2008-07-25 19:45:31 -07001730 /* Has the page moved? */
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001731 if (unlikely(page != *slot)) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001732 put_page(head);
Nick Piggina60637c2008-07-25 19:45:31 -07001733 goto repeat;
1734 }
1735
Nick Piggin9cbb4cb2011-01-13 15:45:51 -08001736 /*
1737 * must check mapping and index after taking the ref.
1738 * otherwise we can get both false positives and false
1739 * negatives, which is just confusing to the caller.
1740 */
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001741 if (page->mapping == NULL || page_to_pgoff(page) != iter.index) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001742 put_page(page);
Nick Piggin9cbb4cb2011-01-13 15:45:51 -08001743 break;
1744 }
1745
Nick Piggina60637c2008-07-25 19:45:31 -07001746 pages[ret] = page;
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001747 if (++ret == nr_pages)
1748 break;
Jens Axboeebf43502006-04-27 08:46:01 +02001749 }
Nick Piggina60637c2008-07-25 19:45:31 -07001750 rcu_read_unlock();
1751 return ret;
Jens Axboeebf43502006-04-27 08:46:01 +02001752}
David Howellsef71c152007-05-09 02:33:44 -07001753EXPORT_SYMBOL(find_get_pages_contig);
Jens Axboeebf43502006-04-27 08:46:01 +02001754
Randy Dunlap485bb992006-06-23 02:03:49 -07001755/**
1756 * find_get_pages_tag - find and return pages that match @tag
1757 * @mapping: the address_space to search
1758 * @index: the starting page index
1759 * @tag: the tag index
1760 * @nr_pages: the maximum number of pages
1761 * @pages: where the resulting pages are placed
1762 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001763 * Like find_get_pages, except we only return pages which are tagged with
Randy Dunlap485bb992006-06-23 02:03:49 -07001764 * @tag. We update @index to index the next page for the traversal.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001765 */
1766unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
1767 int tag, unsigned int nr_pages, struct page **pages)
1768{
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001769 struct radix_tree_iter iter;
1770 void **slot;
1771 unsigned ret = 0;
1772
1773 if (unlikely(!nr_pages))
1774 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001775
Nick Piggina60637c2008-07-25 19:45:31 -07001776 rcu_read_lock();
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001777 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1778 &iter, *index, tag) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001779 struct page *head, *page;
Nick Piggina60637c2008-07-25 19:45:31 -07001780repeat:
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001781 page = radix_tree_deref_slot(slot);
Nick Piggina60637c2008-07-25 19:45:31 -07001782 if (unlikely(!page))
1783 continue;
Hugh Dickins9d8aa4e2011-03-22 16:33:06 -07001784
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001785 if (radix_tree_exception(page)) {
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001786 if (radix_tree_deref_retry(page)) {
Matthew Wilcox2cf938a2016-03-17 14:22:03 -07001787 slot = radix_tree_iter_retry(&iter);
1788 continue;
Hugh Dickins8079b1c2011-08-03 16:21:28 -07001789 }
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001790 /*
Johannes Weiner139b6a62014-05-06 12:50:05 -07001791 * A shadow entry of a recently evicted page.
1792 *
1793 * Those entries should never be tagged, but
1794 * this tree walk is lockless and the tags are
1795 * looked up in bulk, one radix tree node at a
1796 * time, so there is a sizable window for page
1797 * reclaim to evict a page we saw tagged.
1798 *
1799 * Skip over it.
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001800 */
Johannes Weiner139b6a62014-05-06 12:50:05 -07001801 continue;
Hugh Dickinsa2c16d62011-08-03 16:21:19 -07001802 }
Nick Piggina60637c2008-07-25 19:45:31 -07001803
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001804 head = compound_head(page);
1805 if (!page_cache_get_speculative(head))
Nick Piggina60637c2008-07-25 19:45:31 -07001806 goto repeat;
1807
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001808 /* The page was split under us? */
1809 if (compound_head(page) != head) {
1810 put_page(head);
1811 goto repeat;
1812 }
1813
Nick Piggina60637c2008-07-25 19:45:31 -07001814 /* Has the page moved? */
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001815 if (unlikely(page != *slot)) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001816 put_page(head);
Nick Piggina60637c2008-07-25 19:45:31 -07001817 goto repeat;
1818 }
1819
1820 pages[ret] = page;
Konstantin Khlebnikov0fc9d102012-03-28 14:42:54 -07001821 if (++ret == nr_pages)
1822 break;
Nick Piggina60637c2008-07-25 19:45:31 -07001823 }
Hugh Dickins5b280c02011-03-22 16:33:07 -07001824
Nick Piggina60637c2008-07-25 19:45:31 -07001825 rcu_read_unlock();
1826
Linus Torvalds1da177e2005-04-16 15:20:36 -07001827 if (ret)
1828 *index = pages[ret - 1]->index + 1;
Nick Piggina60637c2008-07-25 19:45:31 -07001829
Linus Torvalds1da177e2005-04-16 15:20:36 -07001830 return ret;
1831}
David Howellsef71c152007-05-09 02:33:44 -07001832EXPORT_SYMBOL(find_get_pages_tag);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833
Ross Zwisler7e7f7742016-01-22 15:10:44 -08001834/**
1835 * find_get_entries_tag - find and return entries that match @tag
1836 * @mapping: the address_space to search
1837 * @start: the starting page cache index
1838 * @tag: the tag index
1839 * @nr_entries: the maximum number of entries
1840 * @entries: where the resulting entries are placed
1841 * @indices: the cache indices corresponding to the entries in @entries
1842 *
1843 * Like find_get_entries, except we only return entries which are tagged with
1844 * @tag.
1845 */
1846unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
1847 int tag, unsigned int nr_entries,
1848 struct page **entries, pgoff_t *indices)
1849{
1850 void **slot;
1851 unsigned int ret = 0;
1852 struct radix_tree_iter iter;
1853
1854 if (!nr_entries)
1855 return 0;
1856
1857 rcu_read_lock();
Ross Zwisler7e7f7742016-01-22 15:10:44 -08001858 radix_tree_for_each_tagged(slot, &mapping->page_tree,
1859 &iter, start, tag) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001860 struct page *head, *page;
Ross Zwisler7e7f7742016-01-22 15:10:44 -08001861repeat:
1862 page = radix_tree_deref_slot(slot);
1863 if (unlikely(!page))
1864 continue;
1865 if (radix_tree_exception(page)) {
1866 if (radix_tree_deref_retry(page)) {
Matthew Wilcox2cf938a2016-03-17 14:22:03 -07001867 slot = radix_tree_iter_retry(&iter);
1868 continue;
Ross Zwisler7e7f7742016-01-22 15:10:44 -08001869 }
1870
1871 /*
1872 * A shadow entry of a recently evicted page, a swap
1873 * entry from shmem/tmpfs or a DAX entry. Return it
1874 * without attempting to raise page count.
1875 */
1876 goto export;
1877 }
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001878
1879 head = compound_head(page);
1880 if (!page_cache_get_speculative(head))
Ross Zwisler7e7f7742016-01-22 15:10:44 -08001881 goto repeat;
1882
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001883 /* The page was split under us? */
1884 if (compound_head(page) != head) {
1885 put_page(head);
1886 goto repeat;
1887 }
1888
Ross Zwisler7e7f7742016-01-22 15:10:44 -08001889 /* Has the page moved? */
1890 if (unlikely(page != *slot)) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07001891 put_page(head);
Ross Zwisler7e7f7742016-01-22 15:10:44 -08001892 goto repeat;
1893 }
1894export:
1895 indices[ret] = iter.index;
1896 entries[ret] = page;
1897 if (++ret == nr_entries)
1898 break;
1899 }
1900 rcu_read_unlock();
1901 return ret;
1902}
1903EXPORT_SYMBOL(find_get_entries_tag);
1904
Wu Fengguang76d42bd2006-06-25 05:48:43 -07001905/*
1906 * CD/DVDs are error prone. When a medium error occurs, the driver may fail
1907 * a _large_ part of the i/o request. Imagine the worst scenario:
1908 *
1909 * ---R__________________________________________B__________
1910 * ^ reading here ^ bad block(assume 4k)
1911 *
1912 * read(R) => miss => readahead(R...B) => media error => frustrating retries
1913 * => failing the whole request => read(R) => read(R+1) =>
1914 * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
1915 * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
1916 * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
1917 *
1918 * It is going insane. Fix it by quickly scaling down the readahead size.
1919 */
1920static void shrink_readahead_size_eio(struct file *filp,
1921 struct file_ra_state *ra)
1922{
Wu Fengguang76d42bd2006-06-25 05:48:43 -07001923 ra->ra_pages /= 4;
Wu Fengguang76d42bd2006-06-25 05:48:43 -07001924}
1925
Randy Dunlap485bb992006-06-23 02:03:49 -07001926/**
Christoph Hellwig47c27bc2017-08-29 16:13:18 +02001927 * generic_file_buffered_read - generic file read routine
1928 * @iocb: the iocb to read
Al Viro6e58e792014-02-03 17:07:03 -05001929 * @iter: data destination
1930 * @written: already copied
Randy Dunlap485bb992006-06-23 02:03:49 -07001931 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07001932 * This is a generic file read routine, and uses the
Randy Dunlap485bb992006-06-23 02:03:49 -07001933 * mapping->a_ops->readpage() function for the actual low-level stuff.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001934 *
1935 * This is really ugly. But the goto's actually try to clarify some
1936 * of the logic when it comes to error handling etc.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001937 */
Christoph Hellwig47c27bc2017-08-29 16:13:18 +02001938static ssize_t generic_file_buffered_read(struct kiocb *iocb,
Al Viro6e58e792014-02-03 17:07:03 -05001939 struct iov_iter *iter, ssize_t written)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001940{
Christoph Hellwig47c27bc2017-08-29 16:13:18 +02001941 struct file *filp = iocb->ki_filp;
Christoph Hellwig36e78912008-02-08 04:21:24 -08001942 struct address_space *mapping = filp->f_mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001943 struct inode *inode = mapping->host;
Christoph Hellwig36e78912008-02-08 04:21:24 -08001944 struct file_ra_state *ra = &filp->f_ra;
Christoph Hellwig47c27bc2017-08-29 16:13:18 +02001945 loff_t *ppos = &iocb->ki_pos;
Fengguang Wu57f6b962007-10-16 01:24:37 -07001946 pgoff_t index;
1947 pgoff_t last_index;
1948 pgoff_t prev_index;
1949 unsigned long offset; /* offset into pagecache page */
Jan Karaec0f1632007-05-06 14:49:25 -07001950 unsigned int prev_offset;
Al Viro6e58e792014-02-03 17:07:03 -05001951 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001952
Wei Fangc2a97372016-10-07 17:01:52 -07001953 if (unlikely(*ppos >= inode->i_sb->s_maxbytes))
Linus Torvaldsd05c5f72016-12-14 12:45:25 -08001954 return 0;
Wei Fangc2a97372016-10-07 17:01:52 -07001955 iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
1956
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001957 index = *ppos >> PAGE_SHIFT;
1958 prev_index = ra->prev_pos >> PAGE_SHIFT;
1959 prev_offset = ra->prev_pos & (PAGE_SIZE-1);
1960 last_index = (*ppos + iter->count + PAGE_SIZE-1) >> PAGE_SHIFT;
1961 offset = *ppos & ~PAGE_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001962
Linus Torvalds1da177e2005-04-16 15:20:36 -07001963 for (;;) {
1964 struct page *page;
Fengguang Wu57f6b962007-10-16 01:24:37 -07001965 pgoff_t end_index;
NeilBrowna32ea1e2007-07-17 04:03:04 -07001966 loff_t isize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001967 unsigned long nr, ret;
1968
Linus Torvalds1da177e2005-04-16 15:20:36 -07001969 cond_resched();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001970find_page:
Michal Hocko5abf1862017-02-03 13:13:29 -08001971 if (fatal_signal_pending(current)) {
1972 error = -EINTR;
1973 goto out;
1974 }
1975
Linus Torvalds1da177e2005-04-16 15:20:36 -07001976 page = find_get_page(mapping, index);
Fengguang Wu3ea89ee2007-07-19 01:48:02 -07001977 if (!page) {
Milosz Tanski3239d832017-08-29 16:13:19 +02001978 if (iocb->ki_flags & IOCB_NOWAIT)
1979 goto would_block;
Rusty Russellcf914a72007-07-19 01:48:08 -07001980 page_cache_sync_readahead(mapping,
Fengguang Wu7ff81072007-10-16 01:24:35 -07001981 ra, filp,
Fengguang Wu3ea89ee2007-07-19 01:48:02 -07001982 index, last_index - index);
1983 page = find_get_page(mapping, index);
1984 if (unlikely(page == NULL))
1985 goto no_cached_page;
1986 }
1987 if (PageReadahead(page)) {
Rusty Russellcf914a72007-07-19 01:48:08 -07001988 page_cache_async_readahead(mapping,
Fengguang Wu7ff81072007-10-16 01:24:35 -07001989 ra, filp, page,
Fengguang Wu3ea89ee2007-07-19 01:48:02 -07001990 index, last_index - index);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001991 }
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07001992 if (!PageUptodate(page)) {
Milosz Tanski3239d832017-08-29 16:13:19 +02001993 if (iocb->ki_flags & IOCB_NOWAIT) {
1994 put_page(page);
1995 goto would_block;
1996 }
1997
Mel Gormanebded022016-03-15 14:55:39 -07001998 /*
1999 * See comment in do_read_cache_page on why
2000 * wait_on_page_locked is used to avoid unnecessarily
2001 * serialisations and why it's safe.
2002 */
Bart Van Asschec4b209a2016-10-07 16:58:33 -07002003 error = wait_on_page_locked_killable(page);
2004 if (unlikely(error))
2005 goto readpage_error;
Mel Gormanebded022016-03-15 14:55:39 -07002006 if (PageUptodate(page))
2007 goto page_ok;
2008
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002009 if (inode->i_blkbits == PAGE_SHIFT ||
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07002010 !mapping->a_ops->is_partially_uptodate)
2011 goto page_not_up_to_date;
Eryu Guan6d6d36b2016-11-01 15:43:07 +08002012 /* pipes can't handle partially uptodate pages */
2013 if (unlikely(iter->type & ITER_PIPE))
2014 goto page_not_up_to_date;
Nick Piggin529ae9a2008-08-02 12:01:03 +02002015 if (!trylock_page(page))
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07002016 goto page_not_up_to_date;
Dave Hansen8d056cb2010-11-11 14:05:15 -08002017 /* Did it get truncated before we got the lock? */
2018 if (!page->mapping)
2019 goto page_not_up_to_date_locked;
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07002020 if (!mapping->a_ops->is_partially_uptodate(page,
Al Viro6e58e792014-02-03 17:07:03 -05002021 offset, iter->count))
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07002022 goto page_not_up_to_date_locked;
2023 unlock_page(page);
2024 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002025page_ok:
NeilBrowna32ea1e2007-07-17 04:03:04 -07002026 /*
2027 * i_size must be checked after we know the page is Uptodate.
2028 *
2029 * Checking i_size after the check allows us to calculate
2030 * the correct value for "nr", which means the zero-filled
2031 * part of the page is not copied back to userspace (unless
2032 * another truncate extends the file - this is desired though).
2033 */
2034
2035 isize = i_size_read(inode);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002036 end_index = (isize - 1) >> PAGE_SHIFT;
NeilBrowna32ea1e2007-07-17 04:03:04 -07002037 if (unlikely(!isize || index > end_index)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002038 put_page(page);
NeilBrowna32ea1e2007-07-17 04:03:04 -07002039 goto out;
2040 }
2041
2042 /* nr is the maximum number of bytes to copy from this page */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002043 nr = PAGE_SIZE;
NeilBrowna32ea1e2007-07-17 04:03:04 -07002044 if (index == end_index) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002045 nr = ((isize - 1) & ~PAGE_MASK) + 1;
NeilBrowna32ea1e2007-07-17 04:03:04 -07002046 if (nr <= offset) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002047 put_page(page);
NeilBrowna32ea1e2007-07-17 04:03:04 -07002048 goto out;
2049 }
2050 }
2051 nr = nr - offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002052
2053 /* If users can be writing to this page using arbitrary
2054 * virtual addresses, take care about potential aliasing
2055 * before reading the page on the kernel side.
2056 */
2057 if (mapping_writably_mapped(mapping))
2058 flush_dcache_page(page);
2059
2060 /*
Jan Karaec0f1632007-05-06 14:49:25 -07002061 * When a sequential read accesses a page several times,
2062 * only mark it as accessed the first time.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002063 */
Jan Karaec0f1632007-05-06 14:49:25 -07002064 if (prev_index != index || offset != prev_offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002065 mark_page_accessed(page);
2066 prev_index = index;
2067
2068 /*
2069 * Ok, we have the page, and it's up-to-date, so
2070 * now we can copy it to user space...
Linus Torvalds1da177e2005-04-16 15:20:36 -07002071 */
Al Viro6e58e792014-02-03 17:07:03 -05002072
2073 ret = copy_page_to_iter(page, offset, nr, iter);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002074 offset += ret;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002075 index += offset >> PAGE_SHIFT;
2076 offset &= ~PAGE_MASK;
Jan Kara6ce745e2007-05-06 14:49:26 -07002077 prev_offset = offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002078
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002079 put_page(page);
Al Viro6e58e792014-02-03 17:07:03 -05002080 written += ret;
2081 if (!iov_iter_count(iter))
2082 goto out;
2083 if (ret < nr) {
2084 error = -EFAULT;
2085 goto out;
2086 }
2087 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002088
2089page_not_up_to_date:
2090 /* Get exclusive access to the page ... */
Oleg Nesterov85462322008-06-08 21:20:43 +04002091 error = lock_page_killable(page);
2092 if (unlikely(error))
2093 goto readpage_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094
Hisashi Hifumi8ab22b92008-07-28 15:46:36 -07002095page_not_up_to_date_locked:
Nick Pigginda6052f2006-09-25 23:31:35 -07002096 /* Did it get truncated before we got the lock? */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002097 if (!page->mapping) {
2098 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002099 put_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002100 continue;
2101 }
2102
2103 /* Did somebody else fill it already? */
2104 if (PageUptodate(page)) {
2105 unlock_page(page);
2106 goto page_ok;
2107 }
2108
2109readpage:
Jeff Moyer91803b42010-05-26 11:49:40 -04002110 /*
2111 * A previous I/O error may have been due to temporary
2112 * failures, eg. multipath errors.
2113 * PG_error will be set again if readpage fails.
2114 */
2115 ClearPageError(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002116 /* Start the actual read. The read will unlock the page. */
2117 error = mapping->a_ops->readpage(filp, page);
2118
Zach Brown994fc28c2005-12-15 14:28:17 -08002119 if (unlikely(error)) {
2120 if (error == AOP_TRUNCATED_PAGE) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002121 put_page(page);
Al Viro6e58e792014-02-03 17:07:03 -05002122 error = 0;
Zach Brown994fc28c2005-12-15 14:28:17 -08002123 goto find_page;
2124 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002125 goto readpage_error;
Zach Brown994fc28c2005-12-15 14:28:17 -08002126 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002127
2128 if (!PageUptodate(page)) {
Oleg Nesterov85462322008-06-08 21:20:43 +04002129 error = lock_page_killable(page);
2130 if (unlikely(error))
2131 goto readpage_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002132 if (!PageUptodate(page)) {
2133 if (page->mapping == NULL) {
2134 /*
Christoph Hellwig2ecdc822010-01-26 17:27:20 +01002135 * invalidate_mapping_pages got it
Linus Torvalds1da177e2005-04-16 15:20:36 -07002136 */
2137 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002138 put_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002139 goto find_page;
2140 }
2141 unlock_page(page);
Fengguang Wu7ff81072007-10-16 01:24:35 -07002142 shrink_readahead_size_eio(filp, ra);
Oleg Nesterov85462322008-06-08 21:20:43 +04002143 error = -EIO;
2144 goto readpage_error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145 }
2146 unlock_page(page);
2147 }
2148
Linus Torvalds1da177e2005-04-16 15:20:36 -07002149 goto page_ok;
2150
2151readpage_error:
2152 /* UHHUH! A synchronous read error occurred. Report it */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002153 put_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002154 goto out;
2155
2156no_cached_page:
2157 /*
2158 * Ok, it wasn't cached, so we need to create a new
2159 * page..
2160 */
Nick Piggineb2be182007-10-16 01:24:57 -07002161 page = page_cache_alloc_cold(mapping);
2162 if (!page) {
Al Viro6e58e792014-02-03 17:07:03 -05002163 error = -ENOMEM;
Nick Piggineb2be182007-10-16 01:24:57 -07002164 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002165 }
Michal Hocko6afdb852015-06-24 16:58:06 -07002166 error = add_to_page_cache_lru(page, mapping, index,
Michal Hockoc62d2552015-11-06 16:28:49 -08002167 mapping_gfp_constraint(mapping, GFP_KERNEL));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002168 if (error) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002169 put_page(page);
Al Viro6e58e792014-02-03 17:07:03 -05002170 if (error == -EEXIST) {
2171 error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002172 goto find_page;
Al Viro6e58e792014-02-03 17:07:03 -05002173 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002174 goto out;
2175 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002176 goto readpage;
2177 }
2178
Milosz Tanski3239d832017-08-29 16:13:19 +02002179would_block:
2180 error = -EAGAIN;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002181out:
Fengguang Wu7ff81072007-10-16 01:24:35 -07002182 ra->prev_pos = prev_index;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002183 ra->prev_pos <<= PAGE_SHIFT;
Fengguang Wu7ff81072007-10-16 01:24:35 -07002184 ra->prev_pos |= prev_offset;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002185
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002186 *ppos = ((loff_t)index << PAGE_SHIFT) + offset;
Krishna Kumar0c6aa262008-10-15 22:01:13 -07002187 file_accessed(filp);
Al Viro6e58e792014-02-03 17:07:03 -05002188 return written ? written : error;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002189}
2190
Randy Dunlap485bb992006-06-23 02:03:49 -07002191/**
Al Viro6abd2322014-04-04 14:20:57 -04002192 * generic_file_read_iter - generic filesystem read routine
Randy Dunlap485bb992006-06-23 02:03:49 -07002193 * @iocb: kernel I/O control block
Al Viro6abd2322014-04-04 14:20:57 -04002194 * @iter: destination for the data read
Randy Dunlap485bb992006-06-23 02:03:49 -07002195 *
Al Viro6abd2322014-04-04 14:20:57 -04002196 * This is the "read_iter()" routine for all filesystems
Linus Torvalds1da177e2005-04-16 15:20:36 -07002197 * that can use the page cache directly.
2198 */
2199ssize_t
Al Viroed978a82014-03-05 22:53:04 -05002200generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002201{
Nicolai Stangee7080a42016-03-25 14:22:14 -07002202 size_t count = iov_iter_count(iter);
Christoph Hellwig47c27bc2017-08-29 16:13:18 +02002203 ssize_t retval = 0;
Nicolai Stangee7080a42016-03-25 14:22:14 -07002204
2205 if (!count)
2206 goto out; /* skip atime */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002207
Al Viro2ba48ce2015-04-09 13:52:01 -04002208 if (iocb->ki_flags & IOCB_DIRECT) {
Christoph Hellwig47c27bc2017-08-29 16:13:18 +02002209 struct file *file = iocb->ki_filp;
Al Viroed978a82014-03-05 22:53:04 -05002210 struct address_space *mapping = file->f_mapping;
2211 struct inode *inode = mapping->host;
Badari Pulavarty543ade12006-09-30 23:28:48 -07002212 loff_t size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002213
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214 size = i_size_read(inode);
Goldwyn Rodrigues6be96d32017-06-20 07:05:44 -05002215 if (iocb->ki_flags & IOCB_NOWAIT) {
2216 if (filemap_range_has_page(mapping, iocb->ki_pos,
2217 iocb->ki_pos + count - 1))
2218 return -EAGAIN;
2219 } else {
2220 retval = filemap_write_and_wait_range(mapping,
2221 iocb->ki_pos,
2222 iocb->ki_pos + count - 1);
2223 if (retval < 0)
2224 goto out;
2225 }
Al Viroed978a82014-03-05 22:53:04 -05002226
Christoph Hellwig0d5b0cf2016-10-03 09:48:08 +11002227 file_accessed(file);
2228
Al Viro5ecda132017-04-13 14:13:36 -04002229 retval = mapping->a_ops->direct_IO(iocb, iter);
Al Viroc3a69022016-10-10 13:26:27 -04002230 if (retval >= 0) {
Christoph Hellwigc64fb5c2016-04-07 08:51:55 -07002231 iocb->ki_pos += retval;
Al Viro5ecda132017-04-13 14:13:36 -04002232 count -= retval;
Steven Whitehouse9fe55ee2014-01-24 14:42:22 +00002233 }
Al Viro5b47d592017-05-08 13:54:47 -04002234 iov_iter_revert(iter, count - iov_iter_count(iter));
Josef Bacik66f998f2010-05-23 11:00:54 -04002235
Steven Whitehouse9fe55ee2014-01-24 14:42:22 +00002236 /*
2237 * Btrfs can have a short DIO read if we encounter
2238 * compressed extents, so if there was an error, or if
2239 * we've already read everything we wanted to, or if
2240 * there was a short read because we hit EOF, go ahead
2241 * and return. Otherwise fallthrough to buffered io for
Matthew Wilcoxfbbbad42015-02-16 15:58:53 -08002242 * the rest of the read. Buffered reads will not work for
2243 * DAX files, so don't bother trying.
Steven Whitehouse9fe55ee2014-01-24 14:42:22 +00002244 */
Al Viro5ecda132017-04-13 14:13:36 -04002245 if (retval < 0 || !count || iocb->ki_pos >= size ||
Christoph Hellwig0d5b0cf2016-10-03 09:48:08 +11002246 IS_DAX(inode))
Steven Whitehouse9fe55ee2014-01-24 14:42:22 +00002247 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002248 }
2249
Christoph Hellwig47c27bc2017-08-29 16:13:18 +02002250 retval = generic_file_buffered_read(iocb, iter, retval);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002251out:
2252 return retval;
2253}
Al Viroed978a82014-03-05 22:53:04 -05002254EXPORT_SYMBOL(generic_file_read_iter);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002255
Linus Torvalds1da177e2005-04-16 15:20:36 -07002256#ifdef CONFIG_MMU
Randy Dunlap485bb992006-06-23 02:03:49 -07002257/**
2258 * page_cache_read - adds requested page to the page cache if not already there
2259 * @file: file to read
2260 * @offset: page index
Randy Dunlap62eb3202016-02-11 16:12:58 -08002261 * @gfp_mask: memory allocation flags
Randy Dunlap485bb992006-06-23 02:03:49 -07002262 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07002263 * This adds the requested page to the page cache if it isn't already there,
2264 * and schedules an I/O to read in its contents from disk.
2265 */
Michal Hockoc20cd452016-01-14 15:20:12 -08002266static int page_cache_read(struct file *file, pgoff_t offset, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002267{
2268 struct address_space *mapping = file->f_mapping;
Paul McQuade99dadfd2014-10-09 15:29:03 -07002269 struct page *page;
Zach Brown994fc28c2005-12-15 14:28:17 -08002270 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002271
Zach Brown994fc28c2005-12-15 14:28:17 -08002272 do {
Michal Hockoc20cd452016-01-14 15:20:12 -08002273 page = __page_cache_alloc(gfp_mask|__GFP_COLD);
Zach Brown994fc28c2005-12-15 14:28:17 -08002274 if (!page)
2275 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002276
Matthew Wilcox90a32d12018-04-20 14:56:20 -07002277 ret = add_to_page_cache_lru(page, mapping, offset, gfp_mask);
Zach Brown994fc28c2005-12-15 14:28:17 -08002278 if (ret == 0)
2279 ret = mapping->a_ops->readpage(file, page);
2280 else if (ret == -EEXIST)
2281 ret = 0; /* losing race to add is OK */
2282
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002283 put_page(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002284
Zach Brown994fc28c2005-12-15 14:28:17 -08002285 } while (ret == AOP_TRUNCATED_PAGE);
Paul McQuade99dadfd2014-10-09 15:29:03 -07002286
Zach Brown994fc28c2005-12-15 14:28:17 -08002287 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002288}
2289
2290#define MMAP_LOTSAMISS (100)
2291
Linus Torvaldsef00e082009-06-16 15:31:25 -07002292/*
2293 * Synchronous readahead happens when we don't even find
2294 * a page in the page cache at all.
2295 */
2296static void do_sync_mmap_readahead(struct vm_area_struct *vma,
2297 struct file_ra_state *ra,
2298 struct file *file,
2299 pgoff_t offset)
2300{
Linus Torvaldsef00e082009-06-16 15:31:25 -07002301 struct address_space *mapping = file->f_mapping;
2302
2303 /* If we don't want any read-ahead, don't bother */
Joe Perches64363aa2013-07-08 16:00:18 -07002304 if (vma->vm_flags & VM_RAND_READ)
Linus Torvaldsef00e082009-06-16 15:31:25 -07002305 return;
Wu Fengguang275b12b2011-05-24 17:12:28 -07002306 if (!ra->ra_pages)
2307 return;
Linus Torvaldsef00e082009-06-16 15:31:25 -07002308
Joe Perches64363aa2013-07-08 16:00:18 -07002309 if (vma->vm_flags & VM_SEQ_READ) {
Wu Fengguang7ffc59b2009-06-16 15:31:38 -07002310 page_cache_sync_readahead(mapping, ra, file, offset,
2311 ra->ra_pages);
Linus Torvaldsef00e082009-06-16 15:31:25 -07002312 return;
2313 }
2314
Andi Kleen207d04b2011-05-24 17:12:29 -07002315 /* Avoid banging the cache line if not needed */
2316 if (ra->mmap_miss < MMAP_LOTSAMISS * 10)
Linus Torvaldsef00e082009-06-16 15:31:25 -07002317 ra->mmap_miss++;
2318
2319 /*
2320 * Do we miss much more than hit in this file? If so,
2321 * stop bothering with read-ahead. It will only hurt.
2322 */
2323 if (ra->mmap_miss > MMAP_LOTSAMISS)
2324 return;
2325
Wu Fengguangd30a1102009-06-16 15:31:30 -07002326 /*
2327 * mmap read-around
2328 */
Roman Gushchin600e19a2015-11-05 18:47:08 -08002329 ra->start = max_t(long, 0, offset - ra->ra_pages / 2);
2330 ra->size = ra->ra_pages;
2331 ra->async_size = ra->ra_pages / 4;
Wu Fengguang275b12b2011-05-24 17:12:28 -07002332 ra_submit(ra, mapping, file);
Linus Torvaldsef00e082009-06-16 15:31:25 -07002333}
2334
2335/*
2336 * Asynchronous readahead happens when we find the page and PG_readahead,
2337 * so we want to possibly extend the readahead further..
2338 */
2339static void do_async_mmap_readahead(struct vm_area_struct *vma,
2340 struct file_ra_state *ra,
2341 struct file *file,
2342 struct page *page,
2343 pgoff_t offset)
2344{
2345 struct address_space *mapping = file->f_mapping;
2346
2347 /* If we don't want any read-ahead, don't bother */
Joe Perches64363aa2013-07-08 16:00:18 -07002348 if (vma->vm_flags & VM_RAND_READ)
Linus Torvaldsef00e082009-06-16 15:31:25 -07002349 return;
2350 if (ra->mmap_miss > 0)
2351 ra->mmap_miss--;
2352 if (PageReadahead(page))
Wu Fengguang2fad6f52009-06-16 15:31:29 -07002353 page_cache_async_readahead(mapping, ra, file,
2354 page, offset, ra->ra_pages);
Linus Torvaldsef00e082009-06-16 15:31:25 -07002355}
2356
Randy Dunlap485bb992006-06-23 02:03:49 -07002357/**
Nick Piggin54cb8822007-07-19 01:46:59 -07002358 * filemap_fault - read in file data for page fault handling
Nick Piggind0217ac2007-07-19 01:47:03 -07002359 * @vmf: struct vm_fault containing details of the fault
Randy Dunlap485bb992006-06-23 02:03:49 -07002360 *
Nick Piggin54cb8822007-07-19 01:46:59 -07002361 * filemap_fault() is invoked via the vma operations vector for a
Linus Torvalds1da177e2005-04-16 15:20:36 -07002362 * mapped memory region to read in file data during a page fault.
2363 *
2364 * The goto's are kind of ugly, but this streamlines the normal case of having
2365 * it in the page cache, and handles the special cases reasonably without
2366 * having a lot of duplicated code.
Paul Cassella9a95f3c2014-08-06 16:07:24 -07002367 *
2368 * vma->vm_mm->mmap_sem must be held on entry.
2369 *
2370 * If our return value has VM_FAULT_RETRY set, it's because
2371 * lock_page_or_retry() returned 0.
2372 * The mmap_sem has usually been released in this case.
2373 * See __lock_page_or_retry() for the exception.
2374 *
2375 * If our return value does not have VM_FAULT_RETRY set, the mmap_sem
2376 * has not been released.
2377 *
2378 * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002379 */
Dave Jiang11bac802017-02-24 14:56:41 -08002380int filemap_fault(struct vm_fault *vmf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002381{
2382 int error;
Dave Jiang11bac802017-02-24 14:56:41 -08002383 struct file *file = vmf->vma->vm_file;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002384 struct address_space *mapping = file->f_mapping;
2385 struct file_ra_state *ra = &file->f_ra;
2386 struct inode *inode = mapping->host;
Linus Torvaldsef00e082009-06-16 15:31:25 -07002387 pgoff_t offset = vmf->pgoff;
Matthew Wilcox9ab25942017-05-03 14:53:29 -07002388 pgoff_t max_off;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002389 struct page *page;
Nick Piggin83c54072007-07-19 01:47:05 -07002390 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002391
Matthew Wilcox9ab25942017-05-03 14:53:29 -07002392 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2393 if (unlikely(offset >= max_off))
Linus Torvalds5307cc12007-10-31 09:19:46 -07002394 return VM_FAULT_SIGBUS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002395
Linus Torvalds1da177e2005-04-16 15:20:36 -07002396 /*
Johannes Weiner49426422013-10-16 13:46:59 -07002397 * Do we have something in the page cache already?
Linus Torvalds1da177e2005-04-16 15:20:36 -07002398 */
Linus Torvaldsef00e082009-06-16 15:31:25 -07002399 page = find_get_page(mapping, offset);
Shaohua Li45cac652012-10-08 16:32:19 -07002400 if (likely(page) && !(vmf->flags & FAULT_FLAG_TRIED)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002401 /*
Linus Torvaldsef00e082009-06-16 15:31:25 -07002402 * We found the page, so try async readahead before
2403 * waiting for the lock.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002404 */
Dave Jiang11bac802017-02-24 14:56:41 -08002405 do_async_mmap_readahead(vmf->vma, ra, file, page, offset);
Shaohua Li45cac652012-10-08 16:32:19 -07002406 } else if (!page) {
Linus Torvaldsef00e082009-06-16 15:31:25 -07002407 /* No page in the page cache at all */
Dave Jiang11bac802017-02-24 14:56:41 -08002408 do_sync_mmap_readahead(vmf->vma, ra, file, offset);
Linus Torvaldsef00e082009-06-16 15:31:25 -07002409 count_vm_event(PGMAJFAULT);
Roman Gushchin22621852017-07-06 15:40:25 -07002410 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
Linus Torvaldsef00e082009-06-16 15:31:25 -07002411 ret = VM_FAULT_MAJOR;
2412retry_find:
Michel Lespinasseb522c942010-10-26 14:21:56 -07002413 page = find_get_page(mapping, offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002414 if (!page)
2415 goto no_cached_page;
2416 }
2417
Dave Jiang11bac802017-02-24 14:56:41 -08002418 if (!lock_page_or_retry(page, vmf->vma->vm_mm, vmf->flags)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002419 put_page(page);
Michel Lespinassed065bd82010-10-26 14:21:57 -07002420 return ret | VM_FAULT_RETRY;
Michel Lespinassed88c0922010-11-02 13:05:18 -07002421 }
Michel Lespinasseb522c942010-10-26 14:21:56 -07002422
2423 /* Did it get truncated? */
2424 if (unlikely(page->mapping != mapping)) {
2425 unlock_page(page);
2426 put_page(page);
2427 goto retry_find;
2428 }
Sasha Levin309381fea2014-01-23 15:52:54 -08002429 VM_BUG_ON_PAGE(page->index != offset, page);
Michel Lespinasseb522c942010-10-26 14:21:56 -07002430
Linus Torvalds1da177e2005-04-16 15:20:36 -07002431 /*
Nick Piggind00806b2007-07-19 01:46:57 -07002432 * We have a locked page in the page cache, now we need to check
2433 * that it's up-to-date. If not, it is going to be due to an error.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002434 */
Nick Piggind00806b2007-07-19 01:46:57 -07002435 if (unlikely(!PageUptodate(page)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002436 goto page_not_uptodate;
2437
Linus Torvaldsef00e082009-06-16 15:31:25 -07002438 /*
2439 * Found the page and have a reference on it.
2440 * We must recheck i_size under page lock.
2441 */
Matthew Wilcox9ab25942017-05-03 14:53:29 -07002442 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2443 if (unlikely(offset >= max_off)) {
Nick Piggind00806b2007-07-19 01:46:57 -07002444 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002445 put_page(page);
Linus Torvalds5307cc12007-10-31 09:19:46 -07002446 return VM_FAULT_SIGBUS;
Nick Piggind00806b2007-07-19 01:46:57 -07002447 }
2448
Nick Piggind0217ac2007-07-19 01:47:03 -07002449 vmf->page = page;
Nick Piggin83c54072007-07-19 01:47:05 -07002450 return ret | VM_FAULT_LOCKED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002451
Linus Torvalds1da177e2005-04-16 15:20:36 -07002452no_cached_page:
2453 /*
2454 * We're only likely to ever get here if MADV_RANDOM is in
2455 * effect.
2456 */
Michal Hockoc20cd452016-01-14 15:20:12 -08002457 error = page_cache_read(file, offset, vmf->gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002458
2459 /*
2460 * The page we want has now been added to the page cache.
2461 * In the unlikely event that someone removed it in the
2462 * meantime, we'll just come back here and read it again.
2463 */
2464 if (error >= 0)
2465 goto retry_find;
2466
2467 /*
2468 * An error return from page_cache_read can result if the
2469 * system is low on memory, or a problem occurs while trying
2470 * to schedule I/O.
2471 */
2472 if (error == -ENOMEM)
Nick Piggind0217ac2007-07-19 01:47:03 -07002473 return VM_FAULT_OOM;
2474 return VM_FAULT_SIGBUS;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002475
2476page_not_uptodate:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002477 /*
2478 * Umm, take care of errors if the page isn't up-to-date.
2479 * Try to re-read it _once_. We do this synchronously,
2480 * because there really aren't any performance issues here
2481 * and we need to check for errors.
2482 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002483 ClearPageError(page);
Zach Brown994fc28c2005-12-15 14:28:17 -08002484 error = mapping->a_ops->readpage(file, page);
Miklos Szeredi3ef0f722008-05-14 16:05:37 -07002485 if (!error) {
2486 wait_on_page_locked(page);
2487 if (!PageUptodate(page))
2488 error = -EIO;
2489 }
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002490 put_page(page);
Nick Piggind00806b2007-07-19 01:46:57 -07002491
2492 if (!error || error == AOP_TRUNCATED_PAGE)
2493 goto retry_find;
2494
2495 /* Things didn't work out. Return zero to tell the mm layer so. */
2496 shrink_readahead_size_eio(file, ra);
Nick Piggind0217ac2007-07-19 01:47:03 -07002497 return VM_FAULT_SIGBUS;
Nick Piggin54cb8822007-07-19 01:46:59 -07002498}
2499EXPORT_SYMBOL(filemap_fault);
2500
Jan Kara82b0f8c2016-12-14 15:06:58 -08002501void filemap_map_pages(struct vm_fault *vmf,
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002502 pgoff_t start_pgoff, pgoff_t end_pgoff)
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002503{
2504 struct radix_tree_iter iter;
2505 void **slot;
Jan Kara82b0f8c2016-12-14 15:06:58 -08002506 struct file *file = vmf->vma->vm_file;
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002507 struct address_space *mapping = file->f_mapping;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002508 pgoff_t last_pgoff = start_pgoff;
Matthew Wilcox9ab25942017-05-03 14:53:29 -07002509 unsigned long max_idx;
Kirill A. Shutemov83929372016-07-26 15:26:04 -07002510 struct page *head, *page;
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002511
2512 rcu_read_lock();
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002513 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter,
2514 start_pgoff) {
2515 if (iter.index > end_pgoff)
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002516 break;
2517repeat:
2518 page = radix_tree_deref_slot(slot);
2519 if (unlikely(!page))
2520 goto next;
2521 if (radix_tree_exception(page)) {
Matthew Wilcox2cf938a2016-03-17 14:22:03 -07002522 if (radix_tree_deref_retry(page)) {
2523 slot = radix_tree_iter_retry(&iter);
2524 continue;
2525 }
2526 goto next;
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002527 }
2528
Kirill A. Shutemov83929372016-07-26 15:26:04 -07002529 head = compound_head(page);
2530 if (!page_cache_get_speculative(head))
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002531 goto repeat;
2532
Kirill A. Shutemov83929372016-07-26 15:26:04 -07002533 /* The page was split under us? */
2534 if (compound_head(page) != head) {
2535 put_page(head);
2536 goto repeat;
2537 }
2538
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002539 /* Has the page moved? */
2540 if (unlikely(page != *slot)) {
Kirill A. Shutemov83929372016-07-26 15:26:04 -07002541 put_page(head);
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002542 goto repeat;
2543 }
2544
2545 if (!PageUptodate(page) ||
2546 PageReadahead(page) ||
2547 PageHWPoison(page))
2548 goto skip;
2549 if (!trylock_page(page))
2550 goto skip;
2551
2552 if (page->mapping != mapping || !PageUptodate(page))
2553 goto unlock;
2554
Matthew Wilcox9ab25942017-05-03 14:53:29 -07002555 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
2556 if (page->index >= max_idx)
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002557 goto unlock;
2558
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002559 if (file->f_ra.mmap_miss > 0)
2560 file->f_ra.mmap_miss--;
Kirill A. Shutemov7267ec002016-07-26 15:25:23 -07002561
Jan Kara82b0f8c2016-12-14 15:06:58 -08002562 vmf->address += (iter.index - last_pgoff) << PAGE_SHIFT;
2563 if (vmf->pte)
2564 vmf->pte += iter.index - last_pgoff;
Kirill A. Shutemov7267ec002016-07-26 15:25:23 -07002565 last_pgoff = iter.index;
Jan Kara82b0f8c2016-12-14 15:06:58 -08002566 if (alloc_set_pte(vmf, NULL, page))
Kirill A. Shutemov7267ec002016-07-26 15:25:23 -07002567 goto unlock;
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002568 unlock_page(page);
2569 goto next;
2570unlock:
2571 unlock_page(page);
2572skip:
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002573 put_page(page);
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002574next:
Kirill A. Shutemov7267ec002016-07-26 15:25:23 -07002575 /* Huge page is mapped? No need to proceed. */
Jan Kara82b0f8c2016-12-14 15:06:58 -08002576 if (pmd_trans_huge(*vmf->pmd))
Kirill A. Shutemov7267ec002016-07-26 15:25:23 -07002577 break;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002578 if (iter.index == end_pgoff)
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002579 break;
2580 }
2581 rcu_read_unlock();
2582}
2583EXPORT_SYMBOL(filemap_map_pages);
2584
Dave Jiang11bac802017-02-24 14:56:41 -08002585int filemap_page_mkwrite(struct vm_fault *vmf)
Jan Kara4fcf1c62012-06-12 16:20:29 +02002586{
2587 struct page *page = vmf->page;
Dave Jiang11bac802017-02-24 14:56:41 -08002588 struct inode *inode = file_inode(vmf->vma->vm_file);
Jan Kara4fcf1c62012-06-12 16:20:29 +02002589 int ret = VM_FAULT_LOCKED;
2590
Jan Kara14da9202012-06-12 16:20:37 +02002591 sb_start_pagefault(inode->i_sb);
Dave Jiang11bac802017-02-24 14:56:41 -08002592 file_update_time(vmf->vma->vm_file);
Jan Kara4fcf1c62012-06-12 16:20:29 +02002593 lock_page(page);
2594 if (page->mapping != inode->i_mapping) {
2595 unlock_page(page);
2596 ret = VM_FAULT_NOPAGE;
2597 goto out;
2598 }
Jan Kara14da9202012-06-12 16:20:37 +02002599 /*
2600 * We mark the page dirty already here so that when freeze is in
2601 * progress, we are guaranteed that writeback during freezing will
2602 * see the dirty page and writeprotect it again.
2603 */
2604 set_page_dirty(page);
Darrick J. Wong1d1d1a72013-02-21 16:42:51 -08002605 wait_for_stable_page(page);
Jan Kara4fcf1c62012-06-12 16:20:29 +02002606out:
Jan Kara14da9202012-06-12 16:20:37 +02002607 sb_end_pagefault(inode->i_sb);
Jan Kara4fcf1c62012-06-12 16:20:29 +02002608 return ret;
2609}
2610EXPORT_SYMBOL(filemap_page_mkwrite);
2611
Alexey Dobriyanf0f37e22009-09-27 22:29:37 +04002612const struct vm_operations_struct generic_file_vm_ops = {
Nick Piggin54cb8822007-07-19 01:46:59 -07002613 .fault = filemap_fault,
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07002614 .map_pages = filemap_map_pages,
Jan Kara4fcf1c62012-06-12 16:20:29 +02002615 .page_mkwrite = filemap_page_mkwrite,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002616};
2617
2618/* This is used for a general mmap of a disk file */
2619
2620int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2621{
2622 struct address_space *mapping = file->f_mapping;
2623
2624 if (!mapping->a_ops->readpage)
2625 return -ENOEXEC;
2626 file_accessed(file);
2627 vma->vm_ops = &generic_file_vm_ops;
2628 return 0;
2629}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630
2631/*
2632 * This is for filesystems which do not implement ->writepage.
2633 */
2634int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
2635{
2636 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2637 return -EINVAL;
2638 return generic_file_mmap(file, vma);
2639}
2640#else
2641int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
2642{
2643 return -ENOSYS;
2644}
2645int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
2646{
2647 return -ENOSYS;
2648}
2649#endif /* CONFIG_MMU */
2650
2651EXPORT_SYMBOL(generic_file_mmap);
2652EXPORT_SYMBOL(generic_file_readonly_mmap);
2653
Sasha Levin67f9fd92014-04-03 14:48:18 -07002654static struct page *wait_on_page_read(struct page *page)
2655{
2656 if (!IS_ERR(page)) {
2657 wait_on_page_locked(page);
2658 if (!PageUptodate(page)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002659 put_page(page);
Sasha Levin67f9fd92014-04-03 14:48:18 -07002660 page = ERR_PTR(-EIO);
2661 }
2662 }
2663 return page;
2664}
2665
Mel Gorman32b63522016-03-15 14:55:36 -07002666static struct page *do_read_cache_page(struct address_space *mapping,
Fengguang Wu57f6b962007-10-16 01:24:37 -07002667 pgoff_t index,
Hugh Dickins5e5358e2011-07-25 17:12:23 -07002668 int (*filler)(void *, struct page *),
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002669 void *data,
2670 gfp_t gfp)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002671{
Nick Piggineb2be182007-10-16 01:24:57 -07002672 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002673 int err;
2674repeat:
2675 page = find_get_page(mapping, index);
2676 if (!page) {
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002677 page = __page_cache_alloc(gfp | __GFP_COLD);
Nick Piggineb2be182007-10-16 01:24:57 -07002678 if (!page)
2679 return ERR_PTR(-ENOMEM);
Dave Kleikampe6f67b82011-12-21 11:05:48 -06002680 err = add_to_page_cache_lru(page, mapping, index, gfp);
Nick Piggineb2be182007-10-16 01:24:57 -07002681 if (unlikely(err)) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002682 put_page(page);
Nick Piggineb2be182007-10-16 01:24:57 -07002683 if (err == -EEXIST)
2684 goto repeat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002685 /* Presumably ENOMEM for radix tree node */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002686 return ERR_PTR(err);
2687 }
Mel Gorman32b63522016-03-15 14:55:36 -07002688
2689filler:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002690 err = filler(data, page);
2691 if (err < 0) {
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002692 put_page(page);
Mel Gorman32b63522016-03-15 14:55:36 -07002693 return ERR_PTR(err);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002694 }
Mel Gorman32b63522016-03-15 14:55:36 -07002695
2696 page = wait_on_page_read(page);
2697 if (IS_ERR(page))
2698 return page;
2699 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002700 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002701 if (PageUptodate(page))
2702 goto out;
2703
Mel Gormanebded022016-03-15 14:55:39 -07002704 /*
2705 * Page is not up to date and may be locked due one of the following
2706 * case a: Page is being filled and the page lock is held
2707 * case b: Read/write error clearing the page uptodate status
2708 * case c: Truncation in progress (page locked)
2709 * case d: Reclaim in progress
2710 *
2711 * Case a, the page will be up to date when the page is unlocked.
2712 * There is no need to serialise on the page lock here as the page
2713 * is pinned so the lock gives no additional protection. Even if the
2714 * the page is truncated, the data is still valid if PageUptodate as
2715 * it's a race vs truncate race.
2716 * Case b, the page will not be up to date
2717 * Case c, the page may be truncated but in itself, the data may still
2718 * be valid after IO completes as it's a read vs truncate race. The
2719 * operation must restart if the page is not uptodate on unlock but
2720 * otherwise serialising on page lock to stabilise the mapping gives
2721 * no additional guarantees to the caller as the page lock is
2722 * released before return.
2723 * Case d, similar to truncation. If reclaim holds the page lock, it
2724 * will be a race with remove_mapping that determines if the mapping
2725 * is valid on unlock but otherwise the data is valid and there is
2726 * no need to serialise with page lock.
2727 *
2728 * As the page lock gives no additional guarantee, we optimistically
2729 * wait on the page to be unlocked and check if it's up to date and
2730 * use the page if it is. Otherwise, the page lock is required to
2731 * distinguish between the different cases. The motivation is that we
2732 * avoid spurious serialisations and wakeups when multiple processes
2733 * wait on the same page for IO to complete.
2734 */
2735 wait_on_page_locked(page);
2736 if (PageUptodate(page))
2737 goto out;
2738
2739 /* Distinguish between all the cases under the safety of the lock */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002740 lock_page(page);
Mel Gormanebded022016-03-15 14:55:39 -07002741
2742 /* Case c or d, restart the operation */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002743 if (!page->mapping) {
2744 unlock_page(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002745 put_page(page);
Mel Gorman32b63522016-03-15 14:55:36 -07002746 goto repeat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002747 }
Mel Gormanebded022016-03-15 14:55:39 -07002748
2749 /* Someone else locked and filled the page in a very small window */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002750 if (PageUptodate(page)) {
2751 unlock_page(page);
2752 goto out;
2753 }
Mel Gorman32b63522016-03-15 14:55:36 -07002754 goto filler;
2755
David Howellsc855ff32007-05-09 13:42:20 +01002756out:
Nick Piggin6fe69002007-05-06 14:49:04 -07002757 mark_page_accessed(page);
2758 return page;
2759}
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002760
2761/**
Sasha Levin67f9fd92014-04-03 14:48:18 -07002762 * read_cache_page - read into page cache, fill it if needed
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002763 * @mapping: the page's address_space
2764 * @index: the page index
2765 * @filler: function to perform the read
Hugh Dickins5e5358e2011-07-25 17:12:23 -07002766 * @data: first arg to filler(data, page) function, often left as NULL
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002767 *
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002768 * Read into the page cache. If a page already exists, and PageUptodate() is
Sasha Levin67f9fd92014-04-03 14:48:18 -07002769 * not set, try to fill the page and wait for it to become unlocked.
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002770 *
2771 * If the page does not get brought uptodate, return -EIO.
2772 */
Sasha Levin67f9fd92014-04-03 14:48:18 -07002773struct page *read_cache_page(struct address_space *mapping,
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002774 pgoff_t index,
Hugh Dickins5e5358e2011-07-25 17:12:23 -07002775 int (*filler)(void *, struct page *),
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002776 void *data)
2777{
2778 return do_read_cache_page(mapping, index, filler, data, mapping_gfp_mask(mapping));
2779}
Sasha Levin67f9fd92014-04-03 14:48:18 -07002780EXPORT_SYMBOL(read_cache_page);
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002781
2782/**
2783 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
2784 * @mapping: the page's address_space
2785 * @index: the page index
2786 * @gfp: the page allocator flags to use if allocating
2787 *
2788 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
Dave Kleikampe6f67b82011-12-21 11:05:48 -06002789 * any new page allocations done using the specified allocation flags.
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002790 *
2791 * If the page does not get brought uptodate, return -EIO.
2792 */
2793struct page *read_cache_page_gfp(struct address_space *mapping,
2794 pgoff_t index,
2795 gfp_t gfp)
2796{
2797 filler_t *filler = (filler_t *)mapping->a_ops->readpage;
2798
Sasha Levin67f9fd92014-04-03 14:48:18 -07002799 return do_read_cache_page(mapping, index, filler, NULL, gfp);
Linus Torvalds0531b2a2010-01-27 09:20:03 -08002800}
2801EXPORT_SYMBOL(read_cache_page_gfp);
2802
Nick Piggin2f718ff2007-10-16 01:24:59 -07002803/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002804 * Performs necessary checks before doing a write
2805 *
Randy Dunlap485bb992006-06-23 02:03:49 -07002806 * Can adjust writing position or amount of bytes to write.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002807 * Returns appropriate error code that caller should return or
2808 * zero in case that write should be allowed.
2809 */
Al Viro3309dd02015-04-09 12:55:47 -04002810inline ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002811{
Al Viro3309dd02015-04-09 12:55:47 -04002812 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002813 struct inode *inode = file->f_mapping->host;
Jiri Slaby59e99e52010-03-05 13:41:44 -08002814 unsigned long limit = rlimit(RLIMIT_FSIZE);
Al Viro3309dd02015-04-09 12:55:47 -04002815 loff_t pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002816
Al Viro3309dd02015-04-09 12:55:47 -04002817 if (!iov_iter_count(from))
2818 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002819
Al Viro0fa6b002015-04-04 04:05:48 -04002820 /* FIXME: this is for backwards compatibility with 2.4 */
Al Viro2ba48ce2015-04-09 13:52:01 -04002821 if (iocb->ki_flags & IOCB_APPEND)
Al Viro3309dd02015-04-09 12:55:47 -04002822 iocb->ki_pos = i_size_read(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002823
Al Viro3309dd02015-04-09 12:55:47 -04002824 pos = iocb->ki_pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002825
Goldwyn Rodrigues6be96d32017-06-20 07:05:44 -05002826 if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT))
2827 return -EINVAL;
2828
Al Viro0fa6b002015-04-04 04:05:48 -04002829 if (limit != RLIM_INFINITY) {
Al Viro3309dd02015-04-09 12:55:47 -04002830 if (iocb->ki_pos >= limit) {
Al Viro0fa6b002015-04-04 04:05:48 -04002831 send_sig(SIGXFSZ, current, 0);
2832 return -EFBIG;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002833 }
Al Viro3309dd02015-04-09 12:55:47 -04002834 iov_iter_truncate(from, limit - (unsigned long)pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002835 }
2836
2837 /*
2838 * LFS rule
2839 */
Al Viro3309dd02015-04-09 12:55:47 -04002840 if (unlikely(pos + iov_iter_count(from) > MAX_NON_LFS &&
Linus Torvalds1da177e2005-04-16 15:20:36 -07002841 !(file->f_flags & O_LARGEFILE))) {
Al Viro3309dd02015-04-09 12:55:47 -04002842 if (pos >= MAX_NON_LFS)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002843 return -EFBIG;
Al Viro3309dd02015-04-09 12:55:47 -04002844 iov_iter_truncate(from, MAX_NON_LFS - (unsigned long)pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002845 }
2846
2847 /*
2848 * Are we about to exceed the fs block limit ?
2849 *
2850 * If we have written data it becomes a short write. If we have
2851 * exceeded without writing data we send a signal and return EFBIG.
2852 * Linus frestrict idea will clean these up nicely..
2853 */
Al Viro3309dd02015-04-09 12:55:47 -04002854 if (unlikely(pos >= inode->i_sb->s_maxbytes))
2855 return -EFBIG;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002856
Al Viro3309dd02015-04-09 12:55:47 -04002857 iov_iter_truncate(from, inode->i_sb->s_maxbytes - pos);
2858 return iov_iter_count(from);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002859}
2860EXPORT_SYMBOL(generic_write_checks);
2861
Nick Pigginafddba42007-10-16 01:25:01 -07002862int pagecache_write_begin(struct file *file, struct address_space *mapping,
2863 loff_t pos, unsigned len, unsigned flags,
2864 struct page **pagep, void **fsdata)
2865{
2866 const struct address_space_operations *aops = mapping->a_ops;
2867
Nick Piggin4e02ed42008-10-29 14:00:55 -07002868 return aops->write_begin(file, mapping, pos, len, flags,
Nick Pigginafddba42007-10-16 01:25:01 -07002869 pagep, fsdata);
Nick Pigginafddba42007-10-16 01:25:01 -07002870}
2871EXPORT_SYMBOL(pagecache_write_begin);
2872
2873int pagecache_write_end(struct file *file, struct address_space *mapping,
2874 loff_t pos, unsigned len, unsigned copied,
2875 struct page *page, void *fsdata)
2876{
2877 const struct address_space_operations *aops = mapping->a_ops;
Nick Pigginafddba42007-10-16 01:25:01 -07002878
Nick Piggin4e02ed42008-10-29 14:00:55 -07002879 return aops->write_end(file, mapping, pos, len, copied, page, fsdata);
Nick Pigginafddba42007-10-16 01:25:01 -07002880}
2881EXPORT_SYMBOL(pagecache_write_end);
2882
Linus Torvalds1da177e2005-04-16 15:20:36 -07002883ssize_t
Christoph Hellwig1af5bb42016-04-07 08:51:56 -07002884generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002885{
2886 struct file *file = iocb->ki_filp;
2887 struct address_space *mapping = file->f_mapping;
2888 struct inode *inode = mapping->host;
Christoph Hellwig1af5bb42016-04-07 08:51:56 -07002889 loff_t pos = iocb->ki_pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002890 ssize_t written;
Christoph Hellwiga969e902008-07-23 21:27:04 -07002891 size_t write_len;
2892 pgoff_t end;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002893
Al Viro0c949332014-03-22 06:51:37 -04002894 write_len = iov_iter_count(from);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002895 end = (pos + write_len - 1) >> PAGE_SHIFT;
Christoph Hellwiga969e902008-07-23 21:27:04 -07002896
Goldwyn Rodrigues6be96d32017-06-20 07:05:44 -05002897 if (iocb->ki_flags & IOCB_NOWAIT) {
2898 /* If there are pages to writeback, return */
2899 if (filemap_range_has_page(inode->i_mapping, pos,
2900 pos + iov_iter_count(from)))
2901 return -EAGAIN;
2902 } else {
2903 written = filemap_write_and_wait_range(mapping, pos,
2904 pos + write_len - 1);
2905 if (written)
2906 goto out;
2907 }
Christoph Hellwiga969e902008-07-23 21:27:04 -07002908
2909 /*
2910 * After a write we want buffered reads to be sure to go to disk to get
2911 * the new data. We invalidate clean cached page from the region we're
2912 * about to write. We do this *before* the write so that we can return
Hisashi Hifumi6ccfa802008-09-02 14:35:40 -07002913 * without clobbering -EIOCBQUEUED from ->direct_IO().
Christoph Hellwiga969e902008-07-23 21:27:04 -07002914 */
Andrey Ryabinin55635ba2017-05-03 14:55:59 -07002915 written = invalidate_inode_pages2_range(mapping,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002916 pos >> PAGE_SHIFT, end);
Andrey Ryabinin55635ba2017-05-03 14:55:59 -07002917 /*
2918 * If a page can not be invalidated, return 0 to fall back
2919 * to buffered write.
2920 */
2921 if (written) {
2922 if (written == -EBUSY)
2923 return 0;
2924 goto out;
Christoph Hellwiga969e902008-07-23 21:27:04 -07002925 }
2926
Al Viro639a93a2017-04-13 14:10:15 -04002927 written = mapping->a_ops->direct_IO(iocb, from);
Christoph Hellwiga969e902008-07-23 21:27:04 -07002928
2929 /*
2930 * Finally, try again to invalidate clean pages which might have been
2931 * cached by non-direct readahead, or faulted in by get_user_pages()
2932 * if the source of the write was an mmap'ed region of the file
2933 * we're writing. Either one is a pretty crazy thing to do,
2934 * so we don't support it 100%. If this invalidation
2935 * fails, tough, the write still worked...
Lukas Czerner332391a2017-09-21 08:16:29 -06002936 *
2937 * Most of the time we do not need this since dio_complete() will do
2938 * the invalidation for us. However there are some file systems that
2939 * do not end up with dio_complete() being called, so let's not break
2940 * them by removing it completely
Christoph Hellwiga969e902008-07-23 21:27:04 -07002941 */
Lukas Czerner332391a2017-09-21 08:16:29 -06002942 if (mapping->nrpages)
2943 invalidate_inode_pages2_range(mapping,
2944 pos >> PAGE_SHIFT, end);
Christoph Hellwiga969e902008-07-23 21:27:04 -07002945
Linus Torvalds1da177e2005-04-16 15:20:36 -07002946 if (written > 0) {
Namhyung Kim01166512010-10-26 14:21:58 -07002947 pos += written;
Al Viro639a93a2017-04-13 14:10:15 -04002948 write_len -= written;
Namhyung Kim01166512010-10-26 14:21:58 -07002949 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
2950 i_size_write(inode, pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002951 mark_inode_dirty(inode);
2952 }
Al Viro5cb6c6c2014-02-11 20:58:20 -05002953 iocb->ki_pos = pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002954 }
Al Viro639a93a2017-04-13 14:10:15 -04002955 iov_iter_revert(from, write_len - iov_iter_count(from));
Christoph Hellwiga969e902008-07-23 21:27:04 -07002956out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002957 return written;
2958}
2959EXPORT_SYMBOL(generic_file_direct_write);
2960
Nick Piggineb2be182007-10-16 01:24:57 -07002961/*
2962 * Find or create a page at the given pagecache position. Return the locked
2963 * page. This function is specifically for buffered writes.
2964 */
Nick Piggin54566b22009-01-04 12:00:53 -08002965struct page *grab_cache_page_write_begin(struct address_space *mapping,
2966 pgoff_t index, unsigned flags)
Nick Piggineb2be182007-10-16 01:24:57 -07002967{
Nick Piggineb2be182007-10-16 01:24:57 -07002968 struct page *page;
Johannes Weinerbbddabe2016-05-20 16:56:28 -07002969 int fgp_flags = FGP_LOCK|FGP_WRITE|FGP_CREAT;
Johannes Weiner0faa70c2012-01-10 15:07:53 -08002970
Nick Piggin54566b22009-01-04 12:00:53 -08002971 if (flags & AOP_FLAG_NOFS)
Mel Gorman2457aec2014-06-04 16:10:31 -07002972 fgp_flags |= FGP_NOFS;
Nick Piggineb2be182007-10-16 01:24:57 -07002973
Mel Gorman2457aec2014-06-04 16:10:31 -07002974 page = pagecache_get_page(mapping, index, fgp_flags,
Michal Hocko45f87de2014-12-29 20:30:35 +01002975 mapping_gfp_mask(mapping));
Mel Gorman2457aec2014-06-04 16:10:31 -07002976 if (page)
2977 wait_for_stable_page(page);
2978
Nick Piggineb2be182007-10-16 01:24:57 -07002979 return page;
2980}
Nick Piggin54566b22009-01-04 12:00:53 -08002981EXPORT_SYMBOL(grab_cache_page_write_begin);
Nick Piggineb2be182007-10-16 01:24:57 -07002982
Al Viro3b93f912014-02-11 21:34:08 -05002983ssize_t generic_perform_write(struct file *file,
Nick Pigginafddba42007-10-16 01:25:01 -07002984 struct iov_iter *i, loff_t pos)
2985{
2986 struct address_space *mapping = file->f_mapping;
2987 const struct address_space_operations *a_ops = mapping->a_ops;
2988 long status = 0;
2989 ssize_t written = 0;
Nick Piggin674b8922007-10-16 01:25:03 -07002990 unsigned int flags = 0;
2991
Nick Pigginafddba42007-10-16 01:25:01 -07002992 do {
2993 struct page *page;
Nick Pigginafddba42007-10-16 01:25:01 -07002994 unsigned long offset; /* Offset into pagecache page */
2995 unsigned long bytes; /* Bytes to write to page */
2996 size_t copied; /* Bytes copied from user */
2997 void *fsdata;
2998
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002999 offset = (pos & (PAGE_SIZE - 1));
3000 bytes = min_t(unsigned long, PAGE_SIZE - offset,
Nick Pigginafddba42007-10-16 01:25:01 -07003001 iov_iter_count(i));
3002
3003again:
Linus Torvalds00a3d662015-10-07 08:32:38 +01003004 /*
3005 * Bring in the user page that we will copy from _first_.
3006 * Otherwise there's a nasty deadlock on copying from the
3007 * same page as we're writing to, without it being marked
3008 * up-to-date.
3009 *
3010 * Not only is this an optimisation, but it is also required
3011 * to check that the address is actually valid, when atomic
3012 * usercopies are used, below.
3013 */
3014 if (unlikely(iov_iter_fault_in_readable(i, bytes))) {
3015 status = -EFAULT;
3016 break;
3017 }
3018
Jan Kara296291c2015-10-22 13:32:21 -07003019 if (fatal_signal_pending(current)) {
3020 status = -EINTR;
3021 break;
3022 }
3023
Nick Piggin674b8922007-10-16 01:25:03 -07003024 status = a_ops->write_begin(file, mapping, pos, bytes, flags,
Nick Pigginafddba42007-10-16 01:25:01 -07003025 &page, &fsdata);
Mel Gorman2457aec2014-06-04 16:10:31 -07003026 if (unlikely(status < 0))
Nick Pigginafddba42007-10-16 01:25:01 -07003027 break;
3028
anfei zhou931e80e2010-02-02 13:44:02 -08003029 if (mapping_writably_mapped(mapping))
3030 flush_dcache_page(page);
Linus Torvalds00a3d662015-10-07 08:32:38 +01003031
Nick Pigginafddba42007-10-16 01:25:01 -07003032 copied = iov_iter_copy_from_user_atomic(page, i, offset, bytes);
Nick Pigginafddba42007-10-16 01:25:01 -07003033 flush_dcache_page(page);
3034
3035 status = a_ops->write_end(file, mapping, pos, bytes, copied,
3036 page, fsdata);
3037 if (unlikely(status < 0))
3038 break;
3039 copied = status;
3040
3041 cond_resched();
3042
Nick Piggin124d3b72008-02-02 15:01:17 +01003043 iov_iter_advance(i, copied);
Nick Pigginafddba42007-10-16 01:25:01 -07003044 if (unlikely(copied == 0)) {
3045 /*
3046 * If we were unable to copy any data at all, we must
3047 * fall back to a single segment length write.
3048 *
3049 * If we didn't fallback here, we could livelock
3050 * because not all segments in the iov can be copied at
3051 * once without a pagefault.
3052 */
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003053 bytes = min_t(unsigned long, PAGE_SIZE - offset,
Nick Pigginafddba42007-10-16 01:25:01 -07003054 iov_iter_single_seg_count(i));
3055 goto again;
3056 }
Nick Pigginafddba42007-10-16 01:25:01 -07003057 pos += copied;
3058 written += copied;
3059
3060 balance_dirty_pages_ratelimited(mapping);
Nick Pigginafddba42007-10-16 01:25:01 -07003061 } while (iov_iter_count(i));
3062
3063 return written ? written : status;
3064}
Al Viro3b93f912014-02-11 21:34:08 -05003065EXPORT_SYMBOL(generic_perform_write);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003066
Jan Karae4dd9de2009-08-17 18:10:06 +02003067/**
Al Viro81742022014-04-03 03:17:43 -04003068 * __generic_file_write_iter - write data to a file
Jan Karae4dd9de2009-08-17 18:10:06 +02003069 * @iocb: IO state structure (file, offset, etc.)
Al Viro81742022014-04-03 03:17:43 -04003070 * @from: iov_iter with data to write
Jan Karae4dd9de2009-08-17 18:10:06 +02003071 *
3072 * This function does all the work needed for actually writing data to a
3073 * file. It does all basic checks, removes SUID from the file, updates
3074 * modification times and calls proper subroutines depending on whether we
3075 * do direct IO or a standard buffered write.
3076 *
3077 * It expects i_mutex to be grabbed unless we work on a block device or similar
3078 * object which does not need locking at all.
3079 *
3080 * This function does *not* take care of syncing data in case of O_SYNC write.
3081 * A caller has to handle it. This is mainly due to the fact that we want to
3082 * avoid syncing under i_mutex.
3083 */
Al Viro81742022014-04-03 03:17:43 -04003084ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003085{
3086 struct file *file = iocb->ki_filp;
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003087 struct address_space * mapping = file->f_mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003088 struct inode *inode = mapping->host;
Al Viro3b93f912014-02-11 21:34:08 -05003089 ssize_t written = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003090 ssize_t err;
Al Viro3b93f912014-02-11 21:34:08 -05003091 ssize_t status;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003092
Linus Torvalds1da177e2005-04-16 15:20:36 -07003093 /* We can write back this queue in page reclaim */
Christoph Hellwigde1414a2015-01-14 10:42:36 +01003094 current->backing_dev_info = inode_to_bdi(inode);
Jan Kara5fa8e0a2015-05-21 16:05:53 +02003095 err = file_remove_privs(file);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003096 if (err)
3097 goto out;
3098
Josef Bacikc3b2da32012-03-26 09:59:21 -04003099 err = file_update_time(file);
3100 if (err)
3101 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003102
Al Viro2ba48ce2015-04-09 13:52:01 -04003103 if (iocb->ki_flags & IOCB_DIRECT) {
Al Viro0b8def92015-04-07 10:22:53 -04003104 loff_t pos, endbyte;
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003105
Christoph Hellwig1af5bb42016-04-07 08:51:56 -07003106 written = generic_file_direct_write(iocb, from);
Matthew Wilcoxfbbbad42015-02-16 15:58:53 -08003107 /*
3108 * If the write stopped short of completing, fall back to
3109 * buffered writes. Some filesystems do this for writes to
3110 * holes, for example. For DAX files, a buffered write will
3111 * not succeed (even if it did, DAX does not handle dirty
3112 * page-cache pages correctly).
3113 */
Al Viro0b8def92015-04-07 10:22:53 -04003114 if (written < 0 || !iov_iter_count(from) || IS_DAX(inode))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003115 goto out;
Al Viro3b93f912014-02-11 21:34:08 -05003116
Al Viro0b8def92015-04-07 10:22:53 -04003117 status = generic_perform_write(file, from, pos = iocb->ki_pos);
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003118 /*
Al Viro3b93f912014-02-11 21:34:08 -05003119 * If generic_perform_write() returned a synchronous error
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003120 * then we want to return the number of bytes which were
3121 * direct-written, or the error code if that was zero. Note
3122 * that this differs from normal direct-io semantics, which
3123 * will return -EFOO even if some bytes were written.
3124 */
Al Viro60bb4522014-08-08 12:39:16 -04003125 if (unlikely(status < 0)) {
Al Viro3b93f912014-02-11 21:34:08 -05003126 err = status;
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003127 goto out;
3128 }
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003129 /*
3130 * We need to ensure that the page cache pages are written to
3131 * disk and invalidated to preserve the expected O_DIRECT
3132 * semantics.
3133 */
Al Viro3b93f912014-02-11 21:34:08 -05003134 endbyte = pos + status - 1;
Al Viro0b8def92015-04-07 10:22:53 -04003135 err = filemap_write_and_wait_range(mapping, pos, endbyte);
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003136 if (err == 0) {
Al Viro0b8def92015-04-07 10:22:53 -04003137 iocb->ki_pos = endbyte + 1;
Al Viro3b93f912014-02-11 21:34:08 -05003138 written += status;
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003139 invalidate_mapping_pages(mapping,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03003140 pos >> PAGE_SHIFT,
3141 endbyte >> PAGE_SHIFT);
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003142 } else {
3143 /*
3144 * We don't know how much we wrote, so just return
3145 * the number of bytes which were direct-written
3146 */
3147 }
3148 } else {
Al Viro0b8def92015-04-07 10:22:53 -04003149 written = generic_perform_write(file, from, iocb->ki_pos);
3150 if (likely(written > 0))
3151 iocb->ki_pos += written;
Jeff Moyerfb5527e2006-10-19 23:28:13 -07003152 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003153out:
3154 current->backing_dev_info = NULL;
3155 return written ? written : err;
3156}
Al Viro81742022014-04-03 03:17:43 -04003157EXPORT_SYMBOL(__generic_file_write_iter);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003158
Jan Karae4dd9de2009-08-17 18:10:06 +02003159/**
Al Viro81742022014-04-03 03:17:43 -04003160 * generic_file_write_iter - write data to a file
Jan Karae4dd9de2009-08-17 18:10:06 +02003161 * @iocb: IO state structure
Al Viro81742022014-04-03 03:17:43 -04003162 * @from: iov_iter with data to write
Jan Karae4dd9de2009-08-17 18:10:06 +02003163 *
Al Viro81742022014-04-03 03:17:43 -04003164 * This is a wrapper around __generic_file_write_iter() to be used by most
Jan Karae4dd9de2009-08-17 18:10:06 +02003165 * filesystems. It takes care of syncing the file in case of O_SYNC file
3166 * and acquires i_mutex as needed.
3167 */
Al Viro81742022014-04-03 03:17:43 -04003168ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003169{
3170 struct file *file = iocb->ki_filp;
Jan Kara148f9482009-08-17 19:52:36 +02003171 struct inode *inode = file->f_mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003172 ssize_t ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003173
Al Viro59551022016-01-22 15:40:57 -05003174 inode_lock(inode);
Al Viro3309dd02015-04-09 12:55:47 -04003175 ret = generic_write_checks(iocb, from);
3176 if (ret > 0)
Al Viro5f380c72015-04-07 11:28:12 -04003177 ret = __generic_file_write_iter(iocb, from);
Al Viro59551022016-01-22 15:40:57 -05003178 inode_unlock(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003179
Christoph Hellwige2592212016-04-07 08:52:01 -07003180 if (ret > 0)
3181 ret = generic_write_sync(iocb, ret);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003182 return ret;
3183}
Al Viro81742022014-04-03 03:17:43 -04003184EXPORT_SYMBOL(generic_file_write_iter);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003185
David Howellscf9a2ae2006-08-29 19:05:54 +01003186/**
3187 * try_to_release_page() - release old fs-specific metadata on a page
3188 *
3189 * @page: the page which the kernel is trying to free
3190 * @gfp_mask: memory allocation flags (and I/O mode)
3191 *
3192 * The address_space is to try to release any data against the page
mchehab@s-opensource.com0e056eb2017-03-30 17:11:36 -03003193 * (presumably at page->private). If the release was successful, return '1'.
David Howellscf9a2ae2006-08-29 19:05:54 +01003194 * Otherwise return zero.
3195 *
David Howells266cf652009-04-03 16:42:36 +01003196 * This may also be called if PG_fscache is set on a page, indicating that the
3197 * page is known to the local caching routines.
3198 *
David Howellscf9a2ae2006-08-29 19:05:54 +01003199 * The @gfp_mask argument specifies whether I/O may be performed to release
Mel Gorman71baba42015-11-06 16:28:28 -08003200 * this page (__GFP_IO), and whether the call may block (__GFP_RECLAIM & __GFP_FS).
David Howellscf9a2ae2006-08-29 19:05:54 +01003201 *
David Howellscf9a2ae2006-08-29 19:05:54 +01003202 */
3203int try_to_release_page(struct page *page, gfp_t gfp_mask)
3204{
3205 struct address_space * const mapping = page->mapping;
3206
3207 BUG_ON(!PageLocked(page));
3208 if (PageWriteback(page))
3209 return 0;
3210
3211 if (mapping && mapping->a_ops->releasepage)
3212 return mapping->a_ops->releasepage(page, gfp_mask);
3213 return try_to_free_buffers(page);
3214}
3215
3216EXPORT_SYMBOL(try_to_release_page);