1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* internal.h: mm/ internal definitions
3  *
4  * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
5  * Written by David Howells (dhowells@redhat.com)
6  */
7 #ifndef __MM_INTERNAL_H
8 #define __MM_INTERNAL_H
9 
10 #include <linux/fs.h>
11 #include <linux/mm.h>
12 #include <linux/pagemap.h>
13 #include <linux/tracepoint-defs.h>
14 
15 /*
16  * The set of flags that only affect watermark checking and reclaim
17  * behaviour. This is used by the MM to obey the caller constraints
18  * about IO, FS and watermark checking while ignoring placement
19  * hints such as HIGHMEM usage.
20  */
21 #define GFP_RECLAIM_MASK (__GFP_RECLAIM|__GFP_HIGH|__GFP_IO|__GFP_FS|\
22 			__GFP_NOWARN|__GFP_RETRY_MAYFAIL|__GFP_NOFAIL|\
23 			__GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|\
24 			__GFP_ATOMIC)
25 
26 /* The GFP flags allowed during early boot */
27 #define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_RECLAIM|__GFP_IO|__GFP_FS))
28 
29 /* Control allocation cpuset and node placement constraints */
30 #define GFP_CONSTRAINT_MASK (__GFP_HARDWALL|__GFP_THISNODE)
31 
32 /* Do not use these with a slab allocator */
33 #define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK)
34 
35 void page_writeback_init(void);
36 
folio_raw_mapping(struct folio * folio)37 static inline void *folio_raw_mapping(struct folio *folio)
38 {
39 	unsigned long mapping = (unsigned long)folio->mapping;
40 
41 	return (void *)(mapping & ~PAGE_MAPPING_FLAGS);
42 }
43 
44 void __acct_reclaim_writeback(pg_data_t *pgdat, struct folio *folio,
45 						int nr_throttled);
acct_reclaim_writeback(struct folio * folio)46 static inline void acct_reclaim_writeback(struct folio *folio)
47 {
48 	pg_data_t *pgdat = folio_pgdat(folio);
49 	int nr_throttled = atomic_read(&pgdat->nr_writeback_throttled);
50 
51 	if (nr_throttled)
52 		__acct_reclaim_writeback(pgdat, folio, nr_throttled);
53 }
54 
wake_throttle_isolated(pg_data_t * pgdat)55 static inline void wake_throttle_isolated(pg_data_t *pgdat)
56 {
57 	wait_queue_head_t *wqh;
58 
59 	wqh = &pgdat->reclaim_wait[VMSCAN_THROTTLE_ISOLATED];
60 	if (waitqueue_active(wqh))
61 		wake_up(wqh);
62 }
63 
64 vm_fault_t do_swap_page(struct vm_fault *vmf);
65 void folio_rotate_reclaimable(struct folio *folio);
66 bool __folio_end_writeback(struct folio *folio);
67 
68 void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
69 		unsigned long floor, unsigned long ceiling);
70 void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte);
71 
can_madv_lru_vma(struct vm_area_struct * vma)72 static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
73 {
74 	return !(vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP));
75 }
76 
77 void unmap_page_range(struct mmu_gather *tlb,
78 			     struct vm_area_struct *vma,
79 			     unsigned long addr, unsigned long end,
80 			     struct zap_details *details);
81 
82 void do_page_cache_ra(struct readahead_control *, unsigned long nr_to_read,
83 		unsigned long lookahead_size);
84 void force_page_cache_ra(struct readahead_control *, unsigned long nr);
force_page_cache_readahead(struct address_space * mapping,struct file * file,pgoff_t index,unsigned long nr_to_read)85 static inline void force_page_cache_readahead(struct address_space *mapping,
86 		struct file *file, pgoff_t index, unsigned long nr_to_read)
87 {
88 	DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, index);
89 	force_page_cache_ra(&ractl, nr_to_read);
90 }
91 
92 unsigned find_lock_entries(struct address_space *mapping, pgoff_t start,
93 		pgoff_t end, struct pagevec *pvec, pgoff_t *indices);
94 
95 /**
96  * folio_evictable - Test whether a folio is evictable.
97  * @folio: The folio to test.
98  *
99  * Test whether @folio is evictable -- i.e., should be placed on
100  * active/inactive lists vs unevictable list.
101  *
102  * Reasons folio might not be evictable:
103  * 1. folio's mapping marked unevictable
104  * 2. One of the pages in the folio is part of an mlocked VMA
105  */
folio_evictable(struct folio * folio)106 static inline bool folio_evictable(struct folio *folio)
107 {
108 	bool ret;
109 
110 	/* Prevent address_space of inode and swap cache from being freed */
111 	rcu_read_lock();
112 	ret = !mapping_unevictable(folio_mapping(folio)) &&
113 			!folio_test_mlocked(folio);
114 	rcu_read_unlock();
115 	return ret;
116 }
117 
page_evictable(struct page * page)118 static inline bool page_evictable(struct page *page)
119 {
120 	bool ret;
121 
122 	/* Prevent address_space of inode and swap cache from being freed */
123 	rcu_read_lock();
124 	ret = !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
125 	rcu_read_unlock();
126 	return ret;
127 }
128 
129 /*
130  * Turn a non-refcounted page (->_refcount == 0) into refcounted with
131  * a count of one.
132  */
set_page_refcounted(struct page * page)133 static inline void set_page_refcounted(struct page *page)
134 {
135 	VM_BUG_ON_PAGE(PageTail(page), page);
136 	VM_BUG_ON_PAGE(page_ref_count(page), page);
137 	set_page_count(page, 1);
138 }
139 
140 extern unsigned long highest_memmap_pfn;
141 
142 /*
143  * Maximum number of reclaim retries without progress before the OOM
144  * killer is consider the only way forward.
145  */
146 #define MAX_RECLAIM_RETRIES 16
147 
148 /*
149  * in mm/vmscan.c:
150  */
151 extern int isolate_lru_page(struct page *page);
152 extern void putback_lru_page(struct page *page);
153 extern void reclaim_throttle(pg_data_t *pgdat, enum vmscan_throttle_state reason);
154 
155 /*
156  * in mm/rmap.c:
157  */
158 extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);
159 
160 /*
161  * in mm/memcontrol.c:
162  */
163 extern bool cgroup_memory_nokmem;
164 
165 /*
166  * in mm/page_alloc.c
167  */
168 
169 /*
170  * Structure for holding the mostly immutable allocation parameters passed
171  * between functions involved in allocations, including the alloc_pages*
172  * family of functions.
173  *
174  * nodemask, migratetype and highest_zoneidx are initialized only once in
175  * __alloc_pages() and then never change.
176  *
177  * zonelist, preferred_zone and highest_zoneidx are set first in
178  * __alloc_pages() for the fast path, and might be later changed
179  * in __alloc_pages_slowpath(). All other functions pass the whole structure
180  * by a const pointer.
181  */
182 struct alloc_context {
183 	struct zonelist *zonelist;
184 	nodemask_t *nodemask;
185 	struct zoneref *preferred_zoneref;
186 	int migratetype;
187 
188 	/*
189 	 * highest_zoneidx represents highest usable zone index of
190 	 * the allocation request. Due to the nature of the zone,
191 	 * memory on lower zone than the highest_zoneidx will be
192 	 * protected by lowmem_reserve[highest_zoneidx].
193 	 *
194 	 * highest_zoneidx is also used by reclaim/compaction to limit
195 	 * the target zone since higher zone than this index cannot be
196 	 * usable for this allocation request.
197 	 */
198 	enum zone_type highest_zoneidx;
199 	bool spread_dirty_pages;
200 };
201 
202 /*
203  * Locate the struct page for both the matching buddy in our
204  * pair (buddy1) and the combined O(n+1) page they form (page).
205  *
206  * 1) Any buddy B1 will have an order O twin B2 which satisfies
207  * the following equation:
208  *     B2 = B1 ^ (1 << O)
209  * For example, if the starting buddy (buddy2) is #8 its order
210  * 1 buddy is #10:
211  *     B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
212  *
213  * 2) Any buddy B will have an order O+1 parent P which
214  * satisfies the following equation:
215  *     P = B & ~(1 << O)
216  *
217  * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
218  */
219 static inline unsigned long
__find_buddy_pfn(unsigned long page_pfn,unsigned int order)220 __find_buddy_pfn(unsigned long page_pfn, unsigned int order)
221 {
222 	return page_pfn ^ (1 << order);
223 }
224 
225 extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
226 				unsigned long end_pfn, struct zone *zone);
227 
pageblock_pfn_to_page(unsigned long start_pfn,unsigned long end_pfn,struct zone * zone)228 static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
229 				unsigned long end_pfn, struct zone *zone)
230 {
231 	if (zone->contiguous)
232 		return pfn_to_page(start_pfn);
233 
234 	return __pageblock_pfn_to_page(start_pfn, end_pfn, zone);
235 }
236 
237 extern int __isolate_free_page(struct page *page, unsigned int order);
238 extern void __putback_isolated_page(struct page *page, unsigned int order,
239 				    int mt);
240 extern void memblock_free_pages(struct page *page, unsigned long pfn,
241 					unsigned int order);
242 extern void __free_pages_core(struct page *page, unsigned int order);
243 extern void prep_compound_page(struct page *page, unsigned int order);
244 extern void post_alloc_hook(struct page *page, unsigned int order,
245 					gfp_t gfp_flags);
246 extern int user_min_free_kbytes;
247 
248 extern void free_unref_page(struct page *page, unsigned int order);
249 extern void free_unref_page_list(struct list_head *list);
250 
251 extern void zone_pcp_update(struct zone *zone, int cpu_online);
252 extern void zone_pcp_reset(struct zone *zone);
253 extern void zone_pcp_disable(struct zone *zone);
254 extern void zone_pcp_enable(struct zone *zone);
255 
256 extern void *memmap_alloc(phys_addr_t size, phys_addr_t align,
257 			  phys_addr_t min_addr,
258 			  int nid, bool exact_nid);
259 
260 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
261 
262 /*
263  * in mm/compaction.c
264  */
265 /*
266  * compact_control is used to track pages being migrated and the free pages
267  * they are being migrated to during memory compaction. The free_pfn starts
268  * at the end of a zone and migrate_pfn begins at the start. Movable pages
269  * are moved to the end of a zone during a compaction run and the run
270  * completes when free_pfn <= migrate_pfn
271  */
272 struct compact_control {
273 	struct list_head freepages;	/* List of free pages to migrate to */
274 	struct list_head migratepages;	/* List of pages being migrated */
275 	unsigned int nr_freepages;	/* Number of isolated free pages */
276 	unsigned int nr_migratepages;	/* Number of pages to migrate */
277 	unsigned long free_pfn;		/* isolate_freepages search base */
278 	/*
279 	 * Acts as an in/out parameter to page isolation for migration.
280 	 * isolate_migratepages uses it as a search base.
281 	 * isolate_migratepages_block will update the value to the next pfn
282 	 * after the last isolated one.
283 	 */
284 	unsigned long migrate_pfn;
285 	unsigned long fast_start_pfn;	/* a pfn to start linear scan from */
286 	struct zone *zone;
287 	unsigned long total_migrate_scanned;
288 	unsigned long total_free_scanned;
289 	unsigned short fast_search_fail;/* failures to use free list searches */
290 	short search_order;		/* order to start a fast search at */
291 	const gfp_t gfp_mask;		/* gfp mask of a direct compactor */
292 	int order;			/* order a direct compactor needs */
293 	int migratetype;		/* migratetype of direct compactor */
294 	const unsigned int alloc_flags;	/* alloc flags of a direct compactor */
295 	const int highest_zoneidx;	/* zone index of a direct compactor */
296 	enum migrate_mode mode;		/* Async or sync migration mode */
297 	bool ignore_skip_hint;		/* Scan blocks even if marked skip */
298 	bool no_set_skip_hint;		/* Don't mark blocks for skipping */
299 	bool ignore_block_suitable;	/* Scan blocks considered unsuitable */
300 	bool direct_compaction;		/* False from kcompactd or /proc/... */
301 	bool proactive_compaction;	/* kcompactd proactive compaction */
302 	bool whole_zone;		/* Whole zone should/has been scanned */
303 	bool contended;			/* Signal lock or sched contention */
304 	bool rescan;			/* Rescanning the same pageblock */
305 	bool alloc_contig;		/* alloc_contig_range allocation */
306 };
307 
308 /*
309  * Used in direct compaction when a page should be taken from the freelists
310  * immediately when one is created during the free path.
311  */
312 struct capture_control {
313 	struct compact_control *cc;
314 	struct page *page;
315 };
316 
317 unsigned long
318 isolate_freepages_range(struct compact_control *cc,
319 			unsigned long start_pfn, unsigned long end_pfn);
320 int
321 isolate_migratepages_range(struct compact_control *cc,
322 			   unsigned long low_pfn, unsigned long end_pfn);
323 #endif
324 int find_suitable_fallback(struct free_area *area, unsigned int order,
325 			int migratetype, bool only_stealable, bool *can_steal);
326 
327 /*
328  * This function returns the order of a free page in the buddy system. In
329  * general, page_zone(page)->lock must be held by the caller to prevent the
330  * page from being allocated in parallel and returning garbage as the order.
331  * If a caller does not hold page_zone(page)->lock, it must guarantee that the
332  * page cannot be allocated or merged in parallel. Alternatively, it must
333  * handle invalid values gracefully, and use buddy_order_unsafe() below.
334  */
buddy_order(struct page * page)335 static inline unsigned int buddy_order(struct page *page)
336 {
337 	/* PageBuddy() must be checked by the caller */
338 	return page_private(page);
339 }
340 
341 /*
342  * Like buddy_order(), but for callers who cannot afford to hold the zone lock.
343  * PageBuddy() should be checked first by the caller to minimize race window,
344  * and invalid values must be handled gracefully.
345  *
346  * READ_ONCE is used so that if the caller assigns the result into a local
347  * variable and e.g. tests it for valid range before using, the compiler cannot
348  * decide to remove the variable and inline the page_private(page) multiple
349  * times, potentially observing different values in the tests and the actual
350  * use of the result.
351  */
352 #define buddy_order_unsafe(page)	READ_ONCE(page_private(page))
353 
354 /*
355  * These three helpers classifies VMAs for virtual memory accounting.
356  */
357 
358 /*
359  * Executable code area - executable, not writable, not stack
360  */
is_exec_mapping(vm_flags_t flags)361 static inline bool is_exec_mapping(vm_flags_t flags)
362 {
363 	return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
364 }
365 
366 /*
367  * Stack area - automatically grows in one direction
368  *
369  * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
370  * do_mmap() forbids all other combinations.
371  */
is_stack_mapping(vm_flags_t flags)372 static inline bool is_stack_mapping(vm_flags_t flags)
373 {
374 	return (flags & VM_STACK) == VM_STACK;
375 }
376 
377 /*
378  * Data area - private, writable, not stack
379  */
is_data_mapping(vm_flags_t flags)380 static inline bool is_data_mapping(vm_flags_t flags)
381 {
382 	return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
383 }
384 
385 /* mm/util.c */
386 void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
387 		struct vm_area_struct *prev);
388 void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma);
389 
390 #ifdef CONFIG_MMU
391 extern long populate_vma_page_range(struct vm_area_struct *vma,
392 		unsigned long start, unsigned long end, int *locked);
393 extern long faultin_vma_page_range(struct vm_area_struct *vma,
394 				   unsigned long start, unsigned long end,
395 				   bool write, int *locked);
396 extern void munlock_vma_pages_range(struct vm_area_struct *vma,
397 			unsigned long start, unsigned long end);
munlock_vma_pages_all(struct vm_area_struct * vma)398 static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
399 {
400 	munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
401 }
402 
403 /*
404  * must be called with vma's mmap_lock held for read or write, and page locked.
405  */
406 extern void mlock_vma_page(struct page *page);
407 extern unsigned int munlock_vma_page(struct page *page);
408 
409 extern int mlock_future_check(struct mm_struct *mm, unsigned long flags,
410 			      unsigned long len);
411 
412 /*
413  * Clear the page's PageMlocked().  This can be useful in a situation where
414  * we want to unconditionally remove a page from the pagecache -- e.g.,
415  * on truncation or freeing.
416  *
417  * It is legal to call this function for any page, mlocked or not.
418  * If called for a page that is still mapped by mlocked vmas, all we do
419  * is revert to lazy LRU behaviour -- semantics are not broken.
420  */
421 extern void clear_page_mlock(struct page *page);
422 
423 extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
424 
425 /*
426  * At what user virtual address is page expected in vma?
427  * Returns -EFAULT if all of the page is outside the range of vma.
428  * If page is a compound head, the entire compound page is considered.
429  */
430 static inline unsigned long
vma_address(struct page * page,struct vm_area_struct * vma)431 vma_address(struct page *page, struct vm_area_struct *vma)
432 {
433 	pgoff_t pgoff;
434 	unsigned long address;
435 
436 	VM_BUG_ON_PAGE(PageKsm(page), page);	/* KSM page->index unusable */
437 	pgoff = page_to_pgoff(page);
438 	if (pgoff >= vma->vm_pgoff) {
439 		address = vma->vm_start +
440 			((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
441 		/* Check for address beyond vma (or wrapped through 0?) */
442 		if (address < vma->vm_start || address >= vma->vm_end)
443 			address = -EFAULT;
444 	} else if (PageHead(page) &&
445 		   pgoff + compound_nr(page) - 1 >= vma->vm_pgoff) {
446 		/* Test above avoids possibility of wrap to 0 on 32-bit */
447 		address = vma->vm_start;
448 	} else {
449 		address = -EFAULT;
450 	}
451 	return address;
452 }
453 
454 /*
455  * Then at what user virtual address will none of the page be found in vma?
456  * Assumes that vma_address() already returned a good starting address.
457  * If page is a compound head, the entire compound page is considered.
458  */
459 static inline unsigned long
vma_address_end(struct page * page,struct vm_area_struct * vma)460 vma_address_end(struct page *page, struct vm_area_struct *vma)
461 {
462 	pgoff_t pgoff;
463 	unsigned long address;
464 
465 	VM_BUG_ON_PAGE(PageKsm(page), page);	/* KSM page->index unusable */
466 	pgoff = page_to_pgoff(page) + compound_nr(page);
467 	address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
468 	/* Check for address beyond vma (or wrapped through 0?) */
469 	if (address < vma->vm_start || address > vma->vm_end)
470 		address = vma->vm_end;
471 	return address;
472 }
473 
maybe_unlock_mmap_for_io(struct vm_fault * vmf,struct file * fpin)474 static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf,
475 						    struct file *fpin)
476 {
477 	int flags = vmf->flags;
478 
479 	if (fpin)
480 		return fpin;
481 
482 	/*
483 	 * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or
484 	 * anything, so we only pin the file and drop the mmap_lock if only
485 	 * FAULT_FLAG_ALLOW_RETRY is set, while this is the first attempt.
486 	 */
487 	if (fault_flag_allow_retry_first(flags) &&
488 	    !(flags & FAULT_FLAG_RETRY_NOWAIT)) {
489 		fpin = get_file(vmf->vma->vm_file);
490 		mmap_read_unlock(vmf->vma->vm_mm);
491 	}
492 	return fpin;
493 }
494 
495 #else /* !CONFIG_MMU */
clear_page_mlock(struct page * page)496 static inline void clear_page_mlock(struct page *page) { }
mlock_vma_page(struct page * page)497 static inline void mlock_vma_page(struct page *page) { }
vunmap_range_noflush(unsigned long start,unsigned long end)498 static inline void vunmap_range_noflush(unsigned long start, unsigned long end)
499 {
500 }
501 #endif /* !CONFIG_MMU */
502 
503 /*
504  * Return the mem_map entry representing the 'offset' subpage within
505  * the maximally aligned gigantic page 'base'.  Handle any discontiguity
506  * in the mem_map at MAX_ORDER_NR_PAGES boundaries.
507  */
mem_map_offset(struct page * base,int offset)508 static inline struct page *mem_map_offset(struct page *base, int offset)
509 {
510 	if (unlikely(offset >= MAX_ORDER_NR_PAGES))
511 		return nth_page(base, offset);
512 	return base + offset;
513 }
514 
515 /*
516  * Iterator over all subpages within the maximally aligned gigantic
517  * page 'base'.  Handle any discontiguity in the mem_map.
518  */
mem_map_next(struct page * iter,struct page * base,int offset)519 static inline struct page *mem_map_next(struct page *iter,
520 						struct page *base, int offset)
521 {
522 	if (unlikely((offset & (MAX_ORDER_NR_PAGES - 1)) == 0)) {
523 		unsigned long pfn = page_to_pfn(base) + offset;
524 		if (!pfn_valid(pfn))
525 			return NULL;
526 		return pfn_to_page(pfn);
527 	}
528 	return iter + 1;
529 }
530 
531 /* Memory initialisation debug and verification */
532 enum mminit_level {
533 	MMINIT_WARNING,
534 	MMINIT_VERIFY,
535 	MMINIT_TRACE
536 };
537 
538 #ifdef CONFIG_DEBUG_MEMORY_INIT
539 
540 extern int mminit_loglevel;
541 
542 #define mminit_dprintk(level, prefix, fmt, arg...) \
543 do { \
544 	if (level < mminit_loglevel) { \
545 		if (level <= MMINIT_WARNING) \
546 			pr_warn("mminit::" prefix " " fmt, ##arg);	\
547 		else \
548 			printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \
549 	} \
550 } while (0)
551 
552 extern void mminit_verify_pageflags_layout(void);
553 extern void mminit_verify_zonelist(void);
554 #else
555 
mminit_dprintk(enum mminit_level level,const char * prefix,const char * fmt,...)556 static inline void mminit_dprintk(enum mminit_level level,
557 				const char *prefix, const char *fmt, ...)
558 {
559 }
560 
mminit_verify_pageflags_layout(void)561 static inline void mminit_verify_pageflags_layout(void)
562 {
563 }
564 
mminit_verify_zonelist(void)565 static inline void mminit_verify_zonelist(void)
566 {
567 }
568 #endif /* CONFIG_DEBUG_MEMORY_INIT */
569 
570 /* mminit_validate_memmodel_limits is independent of CONFIG_DEBUG_MEMORY_INIT */
571 #if defined(CONFIG_SPARSEMEM)
572 extern void mminit_validate_memmodel_limits(unsigned long *start_pfn,
573 				unsigned long *end_pfn);
574 #else
mminit_validate_memmodel_limits(unsigned long * start_pfn,unsigned long * end_pfn)575 static inline void mminit_validate_memmodel_limits(unsigned long *start_pfn,
576 				unsigned long *end_pfn)
577 {
578 }
579 #endif /* CONFIG_SPARSEMEM */
580 
581 #define NODE_RECLAIM_NOSCAN	-2
582 #define NODE_RECLAIM_FULL	-1
583 #define NODE_RECLAIM_SOME	0
584 #define NODE_RECLAIM_SUCCESS	1
585 
586 #ifdef CONFIG_NUMA
587 extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int);
588 extern int find_next_best_node(int node, nodemask_t *used_node_mask);
589 #else
node_reclaim(struct pglist_data * pgdat,gfp_t mask,unsigned int order)590 static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask,
591 				unsigned int order)
592 {
593 	return NODE_RECLAIM_NOSCAN;
594 }
find_next_best_node(int node,nodemask_t * used_node_mask)595 static inline int find_next_best_node(int node, nodemask_t *used_node_mask)
596 {
597 	return NUMA_NO_NODE;
598 }
599 #endif
600 
601 extern int hwpoison_filter(struct page *p);
602 
603 extern u32 hwpoison_filter_dev_major;
604 extern u32 hwpoison_filter_dev_minor;
605 extern u64 hwpoison_filter_flags_mask;
606 extern u64 hwpoison_filter_flags_value;
607 extern u64 hwpoison_filter_memcg;
608 extern u32 hwpoison_filter_enable;
609 
610 extern unsigned long  __must_check vm_mmap_pgoff(struct file *, unsigned long,
611         unsigned long, unsigned long,
612         unsigned long, unsigned long);
613 
614 extern void set_pageblock_order(void);
615 unsigned int reclaim_clean_pages_from_list(struct zone *zone,
616 					    struct list_head *page_list);
617 /* The ALLOC_WMARK bits are used as an index to zone->watermark */
618 #define ALLOC_WMARK_MIN		WMARK_MIN
619 #define ALLOC_WMARK_LOW		WMARK_LOW
620 #define ALLOC_WMARK_HIGH	WMARK_HIGH
621 #define ALLOC_NO_WATERMARKS	0x04 /* don't check watermarks at all */
622 
623 /* Mask to get the watermark bits */
624 #define ALLOC_WMARK_MASK	(ALLOC_NO_WATERMARKS-1)
625 
626 /*
627  * Only MMU archs have async oom victim reclaim - aka oom_reaper so we
628  * cannot assume a reduced access to memory reserves is sufficient for
629  * !MMU
630  */
631 #ifdef CONFIG_MMU
632 #define ALLOC_OOM		0x08
633 #else
634 #define ALLOC_OOM		ALLOC_NO_WATERMARKS
635 #endif
636 
637 #define ALLOC_HARDER		 0x10 /* try to alloc harder */
638 #define ALLOC_HIGH		 0x20 /* __GFP_HIGH set */
639 #define ALLOC_CPUSET		 0x40 /* check for correct cpuset */
640 #define ALLOC_CMA		 0x80 /* allow allocations from CMA areas */
641 #ifdef CONFIG_ZONE_DMA32
642 #define ALLOC_NOFRAGMENT	0x100 /* avoid mixing pageblock types */
643 #else
644 #define ALLOC_NOFRAGMENT	  0x0
645 #endif
646 #define ALLOC_KSWAPD		0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */
647 
648 enum ttu_flags;
649 struct tlbflush_unmap_batch;
650 
651 
652 /*
653  * only for MM internal work items which do not depend on
654  * any allocations or locks which might depend on allocations
655  */
656 extern struct workqueue_struct *mm_percpu_wq;
657 
658 #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
659 void try_to_unmap_flush(void);
660 void try_to_unmap_flush_dirty(void);
661 void flush_tlb_batched_pending(struct mm_struct *mm);
662 #else
try_to_unmap_flush(void)663 static inline void try_to_unmap_flush(void)
664 {
665 }
try_to_unmap_flush_dirty(void)666 static inline void try_to_unmap_flush_dirty(void)
667 {
668 }
flush_tlb_batched_pending(struct mm_struct * mm)669 static inline void flush_tlb_batched_pending(struct mm_struct *mm)
670 {
671 }
672 #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
673 
674 extern const struct trace_print_flags pageflag_names[];
675 extern const struct trace_print_flags vmaflag_names[];
676 extern const struct trace_print_flags gfpflag_names[];
677 
is_migrate_highatomic(enum migratetype migratetype)678 static inline bool is_migrate_highatomic(enum migratetype migratetype)
679 {
680 	return migratetype == MIGRATE_HIGHATOMIC;
681 }
682 
is_migrate_highatomic_page(struct page * page)683 static inline bool is_migrate_highatomic_page(struct page *page)
684 {
685 	return get_pageblock_migratetype(page) == MIGRATE_HIGHATOMIC;
686 }
687 
688 void setup_zone_pageset(struct zone *zone);
689 
690 struct migration_target_control {
691 	int nid;		/* preferred node id */
692 	nodemask_t *nmask;
693 	gfp_t gfp_mask;
694 };
695 
696 /*
697  * mm/vmalloc.c
698  */
699 #ifdef CONFIG_MMU
700 int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
701                 pgprot_t prot, struct page **pages, unsigned int page_shift);
702 #else
703 static inline
vmap_pages_range_noflush(unsigned long addr,unsigned long end,pgprot_t prot,struct page ** pages,unsigned int page_shift)704 int vmap_pages_range_noflush(unsigned long addr, unsigned long end,
705                 pgprot_t prot, struct page **pages, unsigned int page_shift)
706 {
707 	return -EINVAL;
708 }
709 #endif
710 
711 void vunmap_range_noflush(unsigned long start, unsigned long end);
712 
713 int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
714 		      unsigned long addr, int page_nid, int *flags);
715 
716 #endif	/* __MM_INTERNAL_H */
717