1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * mm/truncate.c - code for taking down pages from address_spaces
4  *
5  * Copyright (C) 2002, Linus Torvalds
6  *
7  * 10Sep2002	Andrew Morton
8  *		Initial version.
9  */
10 
11 #include <linux/kernel.h>
12 #include <linux/backing-dev.h>
13 #include <linux/dax.h>
14 #include <linux/gfp.h>
15 #include <linux/mm.h>
16 #include <linux/swap.h>
17 #include <linux/export.h>
18 #include <linux/pagemap.h>
19 #include <linux/highmem.h>
20 #include <linux/pagevec.h>
21 #include <linux/task_io_accounting_ops.h>
22 #include <linux/buffer_head.h>	/* grr. try_to_release_page,
23 				   do_invalidatepage */
24 #include <linux/shmem_fs.h>
25 #include <linux/cleancache.h>
26 #include <linux/rmap.h>
27 #include "internal.h"
28 
29 /*
30  * Regular page slots are stabilized by the page lock even without the tree
31  * itself locked.  These unlocked entries need verification under the tree
32  * lock.
33  */
__clear_shadow_entry(struct address_space * mapping,pgoff_t index,void * entry)34 static inline void __clear_shadow_entry(struct address_space *mapping,
35 				pgoff_t index, void *entry)
36 {
37 	XA_STATE(xas, &mapping->i_pages, index);
38 
39 	xas_set_update(&xas, workingset_update_node);
40 	if (xas_load(&xas) != entry)
41 		return;
42 	xas_store(&xas, NULL);
43 }
44 
clear_shadow_entry(struct address_space * mapping,pgoff_t index,void * entry)45 static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
46 			       void *entry)
47 {
48 	spin_lock(&mapping->host->i_lock);
49 	xa_lock_irq(&mapping->i_pages);
50 	__clear_shadow_entry(mapping, index, entry);
51 	xa_unlock_irq(&mapping->i_pages);
52 	if (mapping_shrinkable(mapping))
53 		inode_add_lru(mapping->host);
54 	spin_unlock(&mapping->host->i_lock);
55 }
56 
57 /*
58  * Unconditionally remove exceptional entries. Usually called from truncate
59  * path. Note that the pagevec may be altered by this function by removing
60  * exceptional entries similar to what pagevec_remove_exceptionals does.
61  */
truncate_exceptional_pvec_entries(struct address_space * mapping,struct pagevec * pvec,pgoff_t * indices)62 static void truncate_exceptional_pvec_entries(struct address_space *mapping,
63 				struct pagevec *pvec, pgoff_t *indices)
64 {
65 	int i, j;
66 	bool dax;
67 
68 	/* Handled by shmem itself */
69 	if (shmem_mapping(mapping))
70 		return;
71 
72 	for (j = 0; j < pagevec_count(pvec); j++)
73 		if (xa_is_value(pvec->pages[j]))
74 			break;
75 
76 	if (j == pagevec_count(pvec))
77 		return;
78 
79 	dax = dax_mapping(mapping);
80 	if (!dax) {
81 		spin_lock(&mapping->host->i_lock);
82 		xa_lock_irq(&mapping->i_pages);
83 	}
84 
85 	for (i = j; i < pagevec_count(pvec); i++) {
86 		struct page *page = pvec->pages[i];
87 		pgoff_t index = indices[i];
88 
89 		if (!xa_is_value(page)) {
90 			pvec->pages[j++] = page;
91 			continue;
92 		}
93 
94 		if (unlikely(dax)) {
95 			dax_delete_mapping_entry(mapping, index);
96 			continue;
97 		}
98 
99 		__clear_shadow_entry(mapping, index, page);
100 	}
101 
102 	if (!dax) {
103 		xa_unlock_irq(&mapping->i_pages);
104 		if (mapping_shrinkable(mapping))
105 			inode_add_lru(mapping->host);
106 		spin_unlock(&mapping->host->i_lock);
107 	}
108 	pvec->nr = j;
109 }
110 
111 /*
112  * Invalidate exceptional entry if easily possible. This handles exceptional
113  * entries for invalidate_inode_pages().
114  */
invalidate_exceptional_entry(struct address_space * mapping,pgoff_t index,void * entry)115 static int invalidate_exceptional_entry(struct address_space *mapping,
116 					pgoff_t index, void *entry)
117 {
118 	/* Handled by shmem itself, or for DAX we do nothing. */
119 	if (shmem_mapping(mapping) || dax_mapping(mapping))
120 		return 1;
121 	clear_shadow_entry(mapping, index, entry);
122 	return 1;
123 }
124 
125 /*
126  * Invalidate exceptional entry if clean. This handles exceptional entries for
127  * invalidate_inode_pages2() so for DAX it evicts only clean entries.
128  */
invalidate_exceptional_entry2(struct address_space * mapping,pgoff_t index,void * entry)129 static int invalidate_exceptional_entry2(struct address_space *mapping,
130 					 pgoff_t index, void *entry)
131 {
132 	/* Handled by shmem itself */
133 	if (shmem_mapping(mapping))
134 		return 1;
135 	if (dax_mapping(mapping))
136 		return dax_invalidate_mapping_entry_sync(mapping, index);
137 	clear_shadow_entry(mapping, index, entry);
138 	return 1;
139 }
140 
141 /**
142  * do_invalidatepage - invalidate part or all of a page
143  * @page: the page which is affected
144  * @offset: start of the range to invalidate
145  * @length: length of the range to invalidate
146  *
147  * do_invalidatepage() is called when all or part of the page has become
148  * invalidated by a truncate operation.
149  *
150  * do_invalidatepage() does not have to release all buffers, but it must
151  * ensure that no dirty buffer is left outside @offset and that no I/O
152  * is underway against any of the blocks which are outside the truncation
153  * point.  Because the caller is about to free (and possibly reuse) those
154  * blocks on-disk.
155  */
do_invalidatepage(struct page * page,unsigned int offset,unsigned int length)156 void do_invalidatepage(struct page *page, unsigned int offset,
157 		       unsigned int length)
158 {
159 	void (*invalidatepage)(struct page *, unsigned int, unsigned int);
160 
161 	invalidatepage = page->mapping->a_ops->invalidatepage;
162 #ifdef CONFIG_BLOCK
163 	if (!invalidatepage)
164 		invalidatepage = block_invalidatepage;
165 #endif
166 	if (invalidatepage)
167 		(*invalidatepage)(page, offset, length);
168 }
169 
170 /*
171  * If truncate cannot remove the fs-private metadata from the page, the page
172  * becomes orphaned.  It will be left on the LRU and may even be mapped into
173  * user pagetables if we're racing with filemap_fault().
174  *
175  * We need to bail out if page->mapping is no longer equal to the original
176  * mapping.  This happens a) when the VM reclaimed the page while we waited on
177  * its lock, b) when a concurrent invalidate_mapping_pages got there first and
178  * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
179  */
truncate_cleanup_page(struct page * page)180 static void truncate_cleanup_page(struct page *page)
181 {
182 	if (page_mapped(page))
183 		unmap_mapping_page(page);
184 
185 	if (page_has_private(page))
186 		do_invalidatepage(page, 0, thp_size(page));
187 
188 	/*
189 	 * Some filesystems seem to re-dirty the page even after
190 	 * the VM has canceled the dirty bit (eg ext3 journaling).
191 	 * Hence dirty accounting check is placed after invalidation.
192 	 */
193 	cancel_dirty_page(page);
194 	ClearPageMappedToDisk(page);
195 }
196 
197 /*
198  * This is for invalidate_mapping_pages().  That function can be called at
199  * any time, and is not supposed to throw away dirty pages.  But pages can
200  * be marked dirty at any time too, so use remove_mapping which safely
201  * discards clean, unused pages.
202  *
203  * Returns non-zero if the page was successfully invalidated.
204  */
205 static int
invalidate_complete_page(struct address_space * mapping,struct page * page)206 invalidate_complete_page(struct address_space *mapping, struct page *page)
207 {
208 	int ret;
209 
210 	if (page->mapping != mapping)
211 		return 0;
212 
213 	if (page_has_private(page) && !try_to_release_page(page, 0))
214 		return 0;
215 
216 	ret = remove_mapping(mapping, page);
217 
218 	return ret;
219 }
220 
truncate_inode_page(struct address_space * mapping,struct page * page)221 int truncate_inode_page(struct address_space *mapping, struct page *page)
222 {
223 	VM_BUG_ON_PAGE(PageTail(page), page);
224 
225 	if (page->mapping != mapping)
226 		return -EIO;
227 
228 	truncate_cleanup_page(page);
229 	delete_from_page_cache(page);
230 	return 0;
231 }
232 
233 /*
234  * Used to get rid of pages on hardware memory corruption.
235  */
generic_error_remove_page(struct address_space * mapping,struct page * page)236 int generic_error_remove_page(struct address_space *mapping, struct page *page)
237 {
238 	if (!mapping)
239 		return -EINVAL;
240 	/*
241 	 * Only punch for normal data pages for now.
242 	 * Handling other types like directories would need more auditing.
243 	 */
244 	if (!S_ISREG(mapping->host->i_mode))
245 		return -EIO;
246 	return truncate_inode_page(mapping, page);
247 }
248 EXPORT_SYMBOL(generic_error_remove_page);
249 
250 /*
251  * Safely invalidate one page from its pagecache mapping.
252  * It only drops clean, unused pages. The page must be locked.
253  *
254  * Returns 1 if the page is successfully invalidated, otherwise 0.
255  */
invalidate_inode_page(struct page * page)256 int invalidate_inode_page(struct page *page)
257 {
258 	struct address_space *mapping = page_mapping(page);
259 	if (!mapping)
260 		return 0;
261 	if (PageDirty(page) || PageWriteback(page))
262 		return 0;
263 	if (page_mapped(page))
264 		return 0;
265 	return invalidate_complete_page(mapping, page);
266 }
267 
268 /**
269  * truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
270  * @mapping: mapping to truncate
271  * @lstart: offset from which to truncate
272  * @lend: offset to which to truncate (inclusive)
273  *
274  * Truncate the page cache, removing the pages that are between
275  * specified offsets (and zeroing out partial pages
276  * if lstart or lend + 1 is not page aligned).
277  *
278  * Truncate takes two passes - the first pass is nonblocking.  It will not
279  * block on page locks and it will not block on writeback.  The second pass
280  * will wait.  This is to prevent as much IO as possible in the affected region.
281  * The first pass will remove most pages, so the search cost of the second pass
282  * is low.
283  *
284  * We pass down the cache-hot hint to the page freeing code.  Even if the
285  * mapping is large, it is probably the case that the final pages are the most
286  * recently touched, and freeing happens in ascending file offset order.
287  *
288  * Note that since ->invalidatepage() accepts range to invalidate
289  * truncate_inode_pages_range is able to handle cases where lend + 1 is not
290  * page aligned properly.
291  */
truncate_inode_pages_range(struct address_space * mapping,loff_t lstart,loff_t lend)292 void truncate_inode_pages_range(struct address_space *mapping,
293 				loff_t lstart, loff_t lend)
294 {
295 	pgoff_t		start;		/* inclusive */
296 	pgoff_t		end;		/* exclusive */
297 	unsigned int	partial_start;	/* inclusive */
298 	unsigned int	partial_end;	/* exclusive */
299 	struct pagevec	pvec;
300 	pgoff_t		indices[PAGEVEC_SIZE];
301 	pgoff_t		index;
302 	int		i;
303 
304 	if (mapping_empty(mapping))
305 		goto out;
306 
307 	/* Offsets within partial pages */
308 	partial_start = lstart & (PAGE_SIZE - 1);
309 	partial_end = (lend + 1) & (PAGE_SIZE - 1);
310 
311 	/*
312 	 * 'start' and 'end' always covers the range of pages to be fully
313 	 * truncated. Partial pages are covered with 'partial_start' at the
314 	 * start of the range and 'partial_end' at the end of the range.
315 	 * Note that 'end' is exclusive while 'lend' is inclusive.
316 	 */
317 	start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
318 	if (lend == -1)
319 		/*
320 		 * lend == -1 indicates end-of-file so we have to set 'end'
321 		 * to the highest possible pgoff_t and since the type is
322 		 * unsigned we're using -1.
323 		 */
324 		end = -1;
325 	else
326 		end = (lend + 1) >> PAGE_SHIFT;
327 
328 	pagevec_init(&pvec);
329 	index = start;
330 	while (index < end && find_lock_entries(mapping, index, end - 1,
331 			&pvec, indices)) {
332 		index = indices[pagevec_count(&pvec) - 1] + 1;
333 		truncate_exceptional_pvec_entries(mapping, &pvec, indices);
334 		for (i = 0; i < pagevec_count(&pvec); i++)
335 			truncate_cleanup_page(pvec.pages[i]);
336 		delete_from_page_cache_batch(mapping, &pvec);
337 		for (i = 0; i < pagevec_count(&pvec); i++)
338 			unlock_page(pvec.pages[i]);
339 		pagevec_release(&pvec);
340 		cond_resched();
341 	}
342 
343 	if (partial_start) {
344 		struct page *page = find_lock_page(mapping, start - 1);
345 		if (page) {
346 			unsigned int top = PAGE_SIZE;
347 			if (start > end) {
348 				/* Truncation within a single page */
349 				top = partial_end;
350 				partial_end = 0;
351 			}
352 			wait_on_page_writeback(page);
353 			zero_user_segment(page, partial_start, top);
354 			cleancache_invalidate_page(mapping, page);
355 			if (page_has_private(page))
356 				do_invalidatepage(page, partial_start,
357 						  top - partial_start);
358 			unlock_page(page);
359 			put_page(page);
360 		}
361 	}
362 	if (partial_end) {
363 		struct page *page = find_lock_page(mapping, end);
364 		if (page) {
365 			wait_on_page_writeback(page);
366 			zero_user_segment(page, 0, partial_end);
367 			cleancache_invalidate_page(mapping, page);
368 			if (page_has_private(page))
369 				do_invalidatepage(page, 0,
370 						  partial_end);
371 			unlock_page(page);
372 			put_page(page);
373 		}
374 	}
375 	/*
376 	 * If the truncation happened within a single page no pages
377 	 * will be released, just zeroed, so we can bail out now.
378 	 */
379 	if (start >= end)
380 		goto out;
381 
382 	index = start;
383 	for ( ; ; ) {
384 		cond_resched();
385 		if (!find_get_entries(mapping, index, end - 1, &pvec,
386 				indices)) {
387 			/* If all gone from start onwards, we're done */
388 			if (index == start)
389 				break;
390 			/* Otherwise restart to make sure all gone */
391 			index = start;
392 			continue;
393 		}
394 
395 		for (i = 0; i < pagevec_count(&pvec); i++) {
396 			struct page *page = pvec.pages[i];
397 
398 			/* We rely upon deletion not changing page->index */
399 			index = indices[i];
400 
401 			if (xa_is_value(page))
402 				continue;
403 
404 			lock_page(page);
405 			WARN_ON(page_to_index(page) != index);
406 			wait_on_page_writeback(page);
407 			truncate_inode_page(mapping, page);
408 			unlock_page(page);
409 		}
410 		truncate_exceptional_pvec_entries(mapping, &pvec, indices);
411 		pagevec_release(&pvec);
412 		index++;
413 	}
414 
415 out:
416 	cleancache_invalidate_inode(mapping);
417 }
418 EXPORT_SYMBOL(truncate_inode_pages_range);
419 
420 /**
421  * truncate_inode_pages - truncate *all* the pages from an offset
422  * @mapping: mapping to truncate
423  * @lstart: offset from which to truncate
424  *
425  * Called under (and serialised by) inode->i_rwsem and
426  * mapping->invalidate_lock.
427  *
428  * Note: When this function returns, there can be a page in the process of
429  * deletion (inside __delete_from_page_cache()) in the specified range.  Thus
430  * mapping->nrpages can be non-zero when this function returns even after
431  * truncation of the whole mapping.
432  */
truncate_inode_pages(struct address_space * mapping,loff_t lstart)433 void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
434 {
435 	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
436 }
437 EXPORT_SYMBOL(truncate_inode_pages);
438 
439 /**
440  * truncate_inode_pages_final - truncate *all* pages before inode dies
441  * @mapping: mapping to truncate
442  *
443  * Called under (and serialized by) inode->i_rwsem.
444  *
445  * Filesystems have to use this in the .evict_inode path to inform the
446  * VM that this is the final truncate and the inode is going away.
447  */
truncate_inode_pages_final(struct address_space * mapping)448 void truncate_inode_pages_final(struct address_space *mapping)
449 {
450 	/*
451 	 * Page reclaim can not participate in regular inode lifetime
452 	 * management (can't call iput()) and thus can race with the
453 	 * inode teardown.  Tell it when the address space is exiting,
454 	 * so that it does not install eviction information after the
455 	 * final truncate has begun.
456 	 */
457 	mapping_set_exiting(mapping);
458 
459 	if (!mapping_empty(mapping)) {
460 		/*
461 		 * As truncation uses a lockless tree lookup, cycle
462 		 * the tree lock to make sure any ongoing tree
463 		 * modification that does not see AS_EXITING is
464 		 * completed before starting the final truncate.
465 		 */
466 		xa_lock_irq(&mapping->i_pages);
467 		xa_unlock_irq(&mapping->i_pages);
468 	}
469 
470 	/*
471 	 * Cleancache needs notification even if there are no pages or shadow
472 	 * entries.
473 	 */
474 	truncate_inode_pages(mapping, 0);
475 }
476 EXPORT_SYMBOL(truncate_inode_pages_final);
477 
__invalidate_mapping_pages(struct address_space * mapping,pgoff_t start,pgoff_t end,unsigned long * nr_pagevec)478 static unsigned long __invalidate_mapping_pages(struct address_space *mapping,
479 		pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
480 {
481 	pgoff_t indices[PAGEVEC_SIZE];
482 	struct pagevec pvec;
483 	pgoff_t index = start;
484 	unsigned long ret;
485 	unsigned long count = 0;
486 	int i;
487 
488 	pagevec_init(&pvec);
489 	while (find_lock_entries(mapping, index, end, &pvec, indices)) {
490 		for (i = 0; i < pagevec_count(&pvec); i++) {
491 			struct page *page = pvec.pages[i];
492 
493 			/* We rely upon deletion not changing page->index */
494 			index = indices[i];
495 
496 			if (xa_is_value(page)) {
497 				count += invalidate_exceptional_entry(mapping,
498 								      index,
499 								      page);
500 				continue;
501 			}
502 			index += thp_nr_pages(page) - 1;
503 
504 			ret = invalidate_inode_page(page);
505 			unlock_page(page);
506 			/*
507 			 * Invalidation is a hint that the page is no longer
508 			 * of interest and try to speed up its reclaim.
509 			 */
510 			if (!ret) {
511 				deactivate_file_page(page);
512 				/* It is likely on the pagevec of a remote CPU */
513 				if (nr_pagevec)
514 					(*nr_pagevec)++;
515 			}
516 			count += ret;
517 		}
518 		pagevec_remove_exceptionals(&pvec);
519 		pagevec_release(&pvec);
520 		cond_resched();
521 		index++;
522 	}
523 	return count;
524 }
525 
526 /**
527  * invalidate_mapping_pages - Invalidate all clean, unlocked cache of one inode
528  * @mapping: the address_space which holds the cache to invalidate
529  * @start: the offset 'from' which to invalidate
530  * @end: the offset 'to' which to invalidate (inclusive)
531  *
532  * This function removes pages that are clean, unmapped and unlocked,
533  * as well as shadow entries. It will not block on IO activity.
534  *
535  * If you want to remove all the pages of one inode, regardless of
536  * their use and writeback state, use truncate_inode_pages().
537  *
538  * Return: the number of the cache entries that were invalidated
539  */
invalidate_mapping_pages(struct address_space * mapping,pgoff_t start,pgoff_t end)540 unsigned long invalidate_mapping_pages(struct address_space *mapping,
541 		pgoff_t start, pgoff_t end)
542 {
543 	return __invalidate_mapping_pages(mapping, start, end, NULL);
544 }
545 EXPORT_SYMBOL(invalidate_mapping_pages);
546 
547 /**
548  * invalidate_mapping_pagevec - Invalidate all the unlocked pages of one inode
549  * @mapping: the address_space which holds the pages to invalidate
550  * @start: the offset 'from' which to invalidate
551  * @end: the offset 'to' which to invalidate (inclusive)
552  * @nr_pagevec: invalidate failed page number for caller
553  *
554  * This helper is similar to invalidate_mapping_pages(), except that it accounts
555  * for pages that are likely on a pagevec and counts them in @nr_pagevec, which
556  * will be used by the caller.
557  */
invalidate_mapping_pagevec(struct address_space * mapping,pgoff_t start,pgoff_t end,unsigned long * nr_pagevec)558 void invalidate_mapping_pagevec(struct address_space *mapping,
559 		pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
560 {
561 	__invalidate_mapping_pages(mapping, start, end, nr_pagevec);
562 }
563 
564 /*
565  * This is like invalidate_complete_page(), except it ignores the page's
566  * refcount.  We do this because invalidate_inode_pages2() needs stronger
567  * invalidation guarantees, and cannot afford to leave pages behind because
568  * shrink_page_list() has a temp ref on them, or because they're transiently
569  * sitting in the lru_cache_add() pagevecs.
570  */
571 static int
invalidate_complete_page2(struct address_space * mapping,struct page * page)572 invalidate_complete_page2(struct address_space *mapping, struct page *page)
573 {
574 	if (page->mapping != mapping)
575 		return 0;
576 
577 	if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
578 		return 0;
579 
580 	spin_lock(&mapping->host->i_lock);
581 	xa_lock_irq(&mapping->i_pages);
582 	if (PageDirty(page))
583 		goto failed;
584 
585 	BUG_ON(page_has_private(page));
586 	__delete_from_page_cache(page, NULL);
587 	xa_unlock_irq(&mapping->i_pages);
588 	if (mapping_shrinkable(mapping))
589 		inode_add_lru(mapping->host);
590 	spin_unlock(&mapping->host->i_lock);
591 
592 	if (mapping->a_ops->freepage)
593 		mapping->a_ops->freepage(page);
594 
595 	put_page(page);	/* pagecache ref */
596 	return 1;
597 failed:
598 	xa_unlock_irq(&mapping->i_pages);
599 	spin_unlock(&mapping->host->i_lock);
600 	return 0;
601 }
602 
do_launder_page(struct address_space * mapping,struct page * page)603 static int do_launder_page(struct address_space *mapping, struct page *page)
604 {
605 	if (!PageDirty(page))
606 		return 0;
607 	if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
608 		return 0;
609 	return mapping->a_ops->launder_page(page);
610 }
611 
612 /**
613  * invalidate_inode_pages2_range - remove range of pages from an address_space
614  * @mapping: the address_space
615  * @start: the page offset 'from' which to invalidate
616  * @end: the page offset 'to' which to invalidate (inclusive)
617  *
618  * Any pages which are found to be mapped into pagetables are unmapped prior to
619  * invalidation.
620  *
621  * Return: -EBUSY if any pages could not be invalidated.
622  */
invalidate_inode_pages2_range(struct address_space * mapping,pgoff_t start,pgoff_t end)623 int invalidate_inode_pages2_range(struct address_space *mapping,
624 				  pgoff_t start, pgoff_t end)
625 {
626 	pgoff_t indices[PAGEVEC_SIZE];
627 	struct pagevec pvec;
628 	pgoff_t index;
629 	int i;
630 	int ret = 0;
631 	int ret2 = 0;
632 	int did_range_unmap = 0;
633 
634 	if (mapping_empty(mapping))
635 		goto out;
636 
637 	pagevec_init(&pvec);
638 	index = start;
639 	while (find_get_entries(mapping, index, end, &pvec, indices)) {
640 		for (i = 0; i < pagevec_count(&pvec); i++) {
641 			struct page *page = pvec.pages[i];
642 
643 			/* We rely upon deletion not changing page->index */
644 			index = indices[i];
645 
646 			if (xa_is_value(page)) {
647 				if (!invalidate_exceptional_entry2(mapping,
648 								   index, page))
649 					ret = -EBUSY;
650 				continue;
651 			}
652 
653 			if (!did_range_unmap && page_mapped(page)) {
654 				/*
655 				 * If page is mapped, before taking its lock,
656 				 * zap the rest of the file in one hit.
657 				 */
658 				unmap_mapping_pages(mapping, index,
659 						(1 + end - index), false);
660 				did_range_unmap = 1;
661 			}
662 
663 			lock_page(page);
664 			WARN_ON(page_to_index(page) != index);
665 			if (page->mapping != mapping) {
666 				unlock_page(page);
667 				continue;
668 			}
669 			wait_on_page_writeback(page);
670 
671 			if (page_mapped(page))
672 				unmap_mapping_page(page);
673 			BUG_ON(page_mapped(page));
674 
675 			ret2 = do_launder_page(mapping, page);
676 			if (ret2 == 0) {
677 				if (!invalidate_complete_page2(mapping, page))
678 					ret2 = -EBUSY;
679 			}
680 			if (ret2 < 0)
681 				ret = ret2;
682 			unlock_page(page);
683 		}
684 		pagevec_remove_exceptionals(&pvec);
685 		pagevec_release(&pvec);
686 		cond_resched();
687 		index++;
688 	}
689 	/*
690 	 * For DAX we invalidate page tables after invalidating page cache.  We
691 	 * could invalidate page tables while invalidating each entry however
692 	 * that would be expensive. And doing range unmapping before doesn't
693 	 * work as we have no cheap way to find whether page cache entry didn't
694 	 * get remapped later.
695 	 */
696 	if (dax_mapping(mapping)) {
697 		unmap_mapping_pages(mapping, start, end - start + 1, false);
698 	}
699 out:
700 	cleancache_invalidate_inode(mapping);
701 	return ret;
702 }
703 EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
704 
705 /**
706  * invalidate_inode_pages2 - remove all pages from an address_space
707  * @mapping: the address_space
708  *
709  * Any pages which are found to be mapped into pagetables are unmapped prior to
710  * invalidation.
711  *
712  * Return: -EBUSY if any pages could not be invalidated.
713  */
invalidate_inode_pages2(struct address_space * mapping)714 int invalidate_inode_pages2(struct address_space *mapping)
715 {
716 	return invalidate_inode_pages2_range(mapping, 0, -1);
717 }
718 EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
719 
720 /**
721  * truncate_pagecache - unmap and remove pagecache that has been truncated
722  * @inode: inode
723  * @newsize: new file size
724  *
725  * inode's new i_size must already be written before truncate_pagecache
726  * is called.
727  *
728  * This function should typically be called before the filesystem
729  * releases resources associated with the freed range (eg. deallocates
730  * blocks). This way, pagecache will always stay logically coherent
731  * with on-disk format, and the filesystem would not have to deal with
732  * situations such as writepage being called for a page that has already
733  * had its underlying blocks deallocated.
734  */
truncate_pagecache(struct inode * inode,loff_t newsize)735 void truncate_pagecache(struct inode *inode, loff_t newsize)
736 {
737 	struct address_space *mapping = inode->i_mapping;
738 	loff_t holebegin = round_up(newsize, PAGE_SIZE);
739 
740 	/*
741 	 * unmap_mapping_range is called twice, first simply for
742 	 * efficiency so that truncate_inode_pages does fewer
743 	 * single-page unmaps.  However after this first call, and
744 	 * before truncate_inode_pages finishes, it is possible for
745 	 * private pages to be COWed, which remain after
746 	 * truncate_inode_pages finishes, hence the second
747 	 * unmap_mapping_range call must be made for correctness.
748 	 */
749 	unmap_mapping_range(mapping, holebegin, 0, 1);
750 	truncate_inode_pages(mapping, newsize);
751 	unmap_mapping_range(mapping, holebegin, 0, 1);
752 }
753 EXPORT_SYMBOL(truncate_pagecache);
754 
755 /**
756  * truncate_setsize - update inode and pagecache for a new file size
757  * @inode: inode
758  * @newsize: new file size
759  *
760  * truncate_setsize updates i_size and performs pagecache truncation (if
761  * necessary) to @newsize. It will be typically be called from the filesystem's
762  * setattr function when ATTR_SIZE is passed in.
763  *
764  * Must be called with a lock serializing truncates and writes (generally
765  * i_rwsem but e.g. xfs uses a different lock) and before all filesystem
766  * specific block truncation has been performed.
767  */
truncate_setsize(struct inode * inode,loff_t newsize)768 void truncate_setsize(struct inode *inode, loff_t newsize)
769 {
770 	loff_t oldsize = inode->i_size;
771 
772 	i_size_write(inode, newsize);
773 	if (newsize > oldsize)
774 		pagecache_isize_extended(inode, oldsize, newsize);
775 	truncate_pagecache(inode, newsize);
776 }
777 EXPORT_SYMBOL(truncate_setsize);
778 
779 /**
780  * pagecache_isize_extended - update pagecache after extension of i_size
781  * @inode:	inode for which i_size was extended
782  * @from:	original inode size
783  * @to:		new inode size
784  *
785  * Handle extension of inode size either caused by extending truncate or by
786  * write starting after current i_size. We mark the page straddling current
787  * i_size RO so that page_mkwrite() is called on the nearest write access to
788  * the page.  This way filesystem can be sure that page_mkwrite() is called on
789  * the page before user writes to the page via mmap after the i_size has been
790  * changed.
791  *
792  * The function must be called after i_size is updated so that page fault
793  * coming after we unlock the page will already see the new i_size.
794  * The function must be called while we still hold i_rwsem - this not only
795  * makes sure i_size is stable but also that userspace cannot observe new
796  * i_size value before we are prepared to store mmap writes at new inode size.
797  */
pagecache_isize_extended(struct inode * inode,loff_t from,loff_t to)798 void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to)
799 {
800 	int bsize = i_blocksize(inode);
801 	loff_t rounded_from;
802 	struct page *page;
803 	pgoff_t index;
804 
805 	WARN_ON(to > inode->i_size);
806 
807 	if (from >= to || bsize == PAGE_SIZE)
808 		return;
809 	/* Page straddling @from will not have any hole block created? */
810 	rounded_from = round_up(from, bsize);
811 	if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))
812 		return;
813 
814 	index = from >> PAGE_SHIFT;
815 	page = find_lock_page(inode->i_mapping, index);
816 	/* Page not cached? Nothing to do */
817 	if (!page)
818 		return;
819 	/*
820 	 * See clear_page_dirty_for_io() for details why set_page_dirty()
821 	 * is needed.
822 	 */
823 	if (page_mkclean(page))
824 		set_page_dirty(page);
825 	unlock_page(page);
826 	put_page(page);
827 }
828 EXPORT_SYMBOL(pagecache_isize_extended);
829 
830 /**
831  * truncate_pagecache_range - unmap and remove pagecache that is hole-punched
832  * @inode: inode
833  * @lstart: offset of beginning of hole
834  * @lend: offset of last byte of hole
835  *
836  * This function should typically be called before the filesystem
837  * releases resources associated with the freed range (eg. deallocates
838  * blocks). This way, pagecache will always stay logically coherent
839  * with on-disk format, and the filesystem would not have to deal with
840  * situations such as writepage being called for a page that has already
841  * had its underlying blocks deallocated.
842  */
truncate_pagecache_range(struct inode * inode,loff_t lstart,loff_t lend)843 void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend)
844 {
845 	struct address_space *mapping = inode->i_mapping;
846 	loff_t unmap_start = round_up(lstart, PAGE_SIZE);
847 	loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;
848 	/*
849 	 * This rounding is currently just for example: unmap_mapping_range
850 	 * expands its hole outwards, whereas we want it to contract the hole
851 	 * inwards.  However, existing callers of truncate_pagecache_range are
852 	 * doing their own page rounding first.  Note that unmap_mapping_range
853 	 * allows holelen 0 for all, and we allow lend -1 for end of file.
854 	 */
855 
856 	/*
857 	 * Unlike in truncate_pagecache, unmap_mapping_range is called only
858 	 * once (before truncating pagecache), and without "even_cows" flag:
859 	 * hole-punching should not remove private COWed pages from the hole.
860 	 */
861 	if ((u64)unmap_end > (u64)unmap_start)
862 		unmap_mapping_range(mapping, unmap_start,
863 				    1 + unmap_end - unmap_start, 0);
864 	truncate_inode_pages_range(mapping, lstart, lend);
865 }
866 EXPORT_SYMBOL(truncate_pagecache_range);
867