1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/mm.h>
3 #include <linux/slab.h>
4 #include <linux/string.h>
5 #include <linux/compiler.h>
6 #include <linux/export.h>
7 #include <linux/err.h>
8 #include <linux/sched.h>
9 #include <linux/sched/mm.h>
10 #include <linux/sched/signal.h>
11 #include <linux/sched/task_stack.h>
12 #include <linux/security.h>
13 #include <linux/swap.h>
14 #include <linux/swapops.h>
15 #include <linux/mman.h>
16 #include <linux/hugetlb.h>
17 #include <linux/vmalloc.h>
18 #include <linux/userfaultfd_k.h>
19 #include <linux/elf.h>
20 #include <linux/elf-randomize.h>
21 #include <linux/personality.h>
22 #include <linux/random.h>
23 #include <linux/processor.h>
24 #include <linux/sizes.h>
25 #include <linux/compat.h>
26 
27 #include <linux/uaccess.h>
28 
29 #include "internal.h"
30 
31 /**
32  * kfree_const - conditionally free memory
33  * @x: pointer to the memory
34  *
35  * Function calls kfree only if @x is not in .rodata section.
36  */
kfree_const(const void * x)37 void kfree_const(const void *x)
38 {
39 	if (!is_kernel_rodata((unsigned long)x))
40 		kfree(x);
41 }
42 EXPORT_SYMBOL(kfree_const);
43 
44 /**
45  * kstrdup - allocate space for and copy an existing string
46  * @s: the string to duplicate
47  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
48  *
49  * Return: newly allocated copy of @s or %NULL in case of error
50  */
kstrdup(const char * s,gfp_t gfp)51 char *kstrdup(const char *s, gfp_t gfp)
52 {
53 	size_t len;
54 	char *buf;
55 
56 	if (!s)
57 		return NULL;
58 
59 	len = strlen(s) + 1;
60 	buf = kmalloc_track_caller(len, gfp);
61 	if (buf)
62 		memcpy(buf, s, len);
63 	return buf;
64 }
65 EXPORT_SYMBOL(kstrdup);
66 
67 /**
68  * kstrdup_const - conditionally duplicate an existing const string
69  * @s: the string to duplicate
70  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
71  *
72  * Note: Strings allocated by kstrdup_const should be freed by kfree_const and
73  * must not be passed to krealloc().
74  *
75  * Return: source string if it is in .rodata section otherwise
76  * fallback to kstrdup.
77  */
kstrdup_const(const char * s,gfp_t gfp)78 const char *kstrdup_const(const char *s, gfp_t gfp)
79 {
80 	if (is_kernel_rodata((unsigned long)s))
81 		return s;
82 
83 	return kstrdup(s, gfp);
84 }
85 EXPORT_SYMBOL(kstrdup_const);
86 
87 /**
88  * kstrndup - allocate space for and copy an existing string
89  * @s: the string to duplicate
90  * @max: read at most @max chars from @s
91  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
92  *
93  * Note: Use kmemdup_nul() instead if the size is known exactly.
94  *
95  * Return: newly allocated copy of @s or %NULL in case of error
96  */
kstrndup(const char * s,size_t max,gfp_t gfp)97 char *kstrndup(const char *s, size_t max, gfp_t gfp)
98 {
99 	size_t len;
100 	char *buf;
101 
102 	if (!s)
103 		return NULL;
104 
105 	len = strnlen(s, max);
106 	buf = kmalloc_track_caller(len+1, gfp);
107 	if (buf) {
108 		memcpy(buf, s, len);
109 		buf[len] = '\0';
110 	}
111 	return buf;
112 }
113 EXPORT_SYMBOL(kstrndup);
114 
115 /**
116  * kmemdup - duplicate region of memory
117  *
118  * @src: memory region to duplicate
119  * @len: memory region length
120  * @gfp: GFP mask to use
121  *
122  * Return: newly allocated copy of @src or %NULL in case of error
123  */
kmemdup(const void * src,size_t len,gfp_t gfp)124 void *kmemdup(const void *src, size_t len, gfp_t gfp)
125 {
126 	void *p;
127 
128 	p = kmalloc_track_caller(len, gfp);
129 	if (p)
130 		memcpy(p, src, len);
131 	return p;
132 }
133 EXPORT_SYMBOL(kmemdup);
134 
135 /**
136  * kmemdup_nul - Create a NUL-terminated string from unterminated data
137  * @s: The data to stringify
138  * @len: The size of the data
139  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
140  *
141  * Return: newly allocated copy of @s with NUL-termination or %NULL in
142  * case of error
143  */
kmemdup_nul(const char * s,size_t len,gfp_t gfp)144 char *kmemdup_nul(const char *s, size_t len, gfp_t gfp)
145 {
146 	char *buf;
147 
148 	if (!s)
149 		return NULL;
150 
151 	buf = kmalloc_track_caller(len + 1, gfp);
152 	if (buf) {
153 		memcpy(buf, s, len);
154 		buf[len] = '\0';
155 	}
156 	return buf;
157 }
158 EXPORT_SYMBOL(kmemdup_nul);
159 
160 /**
161  * memdup_user - duplicate memory region from user space
162  *
163  * @src: source address in user space
164  * @len: number of bytes to copy
165  *
166  * Return: an ERR_PTR() on failure.  Result is physically
167  * contiguous, to be freed by kfree().
168  */
memdup_user(const void __user * src,size_t len)169 void *memdup_user(const void __user *src, size_t len)
170 {
171 	void *p;
172 
173 	p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN);
174 	if (!p)
175 		return ERR_PTR(-ENOMEM);
176 
177 	if (copy_from_user(p, src, len)) {
178 		kfree(p);
179 		return ERR_PTR(-EFAULT);
180 	}
181 
182 	return p;
183 }
184 EXPORT_SYMBOL(memdup_user);
185 
186 /**
187  * vmemdup_user - duplicate memory region from user space
188  *
189  * @src: source address in user space
190  * @len: number of bytes to copy
191  *
192  * Return: an ERR_PTR() on failure.  Result may be not
193  * physically contiguous.  Use kvfree() to free.
194  */
vmemdup_user(const void __user * src,size_t len)195 void *vmemdup_user(const void __user *src, size_t len)
196 {
197 	void *p;
198 
199 	p = kvmalloc(len, GFP_USER);
200 	if (!p)
201 		return ERR_PTR(-ENOMEM);
202 
203 	if (copy_from_user(p, src, len)) {
204 		kvfree(p);
205 		return ERR_PTR(-EFAULT);
206 	}
207 
208 	return p;
209 }
210 EXPORT_SYMBOL(vmemdup_user);
211 
212 /**
213  * strndup_user - duplicate an existing string from user space
214  * @s: The string to duplicate
215  * @n: Maximum number of bytes to copy, including the trailing NUL.
216  *
217  * Return: newly allocated copy of @s or an ERR_PTR() in case of error
218  */
strndup_user(const char __user * s,long n)219 char *strndup_user(const char __user *s, long n)
220 {
221 	char *p;
222 	long length;
223 
224 	length = strnlen_user(s, n);
225 
226 	if (!length)
227 		return ERR_PTR(-EFAULT);
228 
229 	if (length > n)
230 		return ERR_PTR(-EINVAL);
231 
232 	p = memdup_user(s, length);
233 
234 	if (IS_ERR(p))
235 		return p;
236 
237 	p[length - 1] = '\0';
238 
239 	return p;
240 }
241 EXPORT_SYMBOL(strndup_user);
242 
243 /**
244  * memdup_user_nul - duplicate memory region from user space and NUL-terminate
245  *
246  * @src: source address in user space
247  * @len: number of bytes to copy
248  *
249  * Return: an ERR_PTR() on failure.
250  */
memdup_user_nul(const void __user * src,size_t len)251 void *memdup_user_nul(const void __user *src, size_t len)
252 {
253 	char *p;
254 
255 	/*
256 	 * Always use GFP_KERNEL, since copy_from_user() can sleep and
257 	 * cause pagefault, which makes it pointless to use GFP_NOFS
258 	 * or GFP_ATOMIC.
259 	 */
260 	p = kmalloc_track_caller(len + 1, GFP_KERNEL);
261 	if (!p)
262 		return ERR_PTR(-ENOMEM);
263 
264 	if (copy_from_user(p, src, len)) {
265 		kfree(p);
266 		return ERR_PTR(-EFAULT);
267 	}
268 	p[len] = '\0';
269 
270 	return p;
271 }
272 EXPORT_SYMBOL(memdup_user_nul);
273 
__vma_link_list(struct mm_struct * mm,struct vm_area_struct * vma,struct vm_area_struct * prev)274 void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
275 		struct vm_area_struct *prev)
276 {
277 	struct vm_area_struct *next;
278 
279 	vma->vm_prev = prev;
280 	if (prev) {
281 		next = prev->vm_next;
282 		prev->vm_next = vma;
283 	} else {
284 		next = mm->mmap;
285 		mm->mmap = vma;
286 	}
287 	vma->vm_next = next;
288 	if (next)
289 		next->vm_prev = vma;
290 }
291 
__vma_unlink_list(struct mm_struct * mm,struct vm_area_struct * vma)292 void __vma_unlink_list(struct mm_struct *mm, struct vm_area_struct *vma)
293 {
294 	struct vm_area_struct *prev, *next;
295 
296 	next = vma->vm_next;
297 	prev = vma->vm_prev;
298 	if (prev)
299 		prev->vm_next = next;
300 	else
301 		mm->mmap = next;
302 	if (next)
303 		next->vm_prev = prev;
304 }
305 
306 /* Check if the vma is being used as a stack by this task */
vma_is_stack_for_current(struct vm_area_struct * vma)307 int vma_is_stack_for_current(struct vm_area_struct *vma)
308 {
309 	struct task_struct * __maybe_unused t = current;
310 
311 	return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));
312 }
313 
314 /*
315  * Change backing file, only valid to use during initial VMA setup.
316  */
vma_set_file(struct vm_area_struct * vma,struct file * file)317 void vma_set_file(struct vm_area_struct *vma, struct file *file)
318 {
319 	/* Changing an anonymous vma with this is illegal */
320 	get_file(file);
321 	swap(vma->vm_file, file);
322 	fput(file);
323 }
324 EXPORT_SYMBOL(vma_set_file);
325 
326 #ifndef STACK_RND_MASK
327 #define STACK_RND_MASK (0x7ff >> (PAGE_SHIFT - 12))     /* 8MB of VA */
328 #endif
329 
randomize_stack_top(unsigned long stack_top)330 unsigned long randomize_stack_top(unsigned long stack_top)
331 {
332 	unsigned long random_variable = 0;
333 
334 	if (current->flags & PF_RANDOMIZE) {
335 		random_variable = get_random_long();
336 		random_variable &= STACK_RND_MASK;
337 		random_variable <<= PAGE_SHIFT;
338 	}
339 #ifdef CONFIG_STACK_GROWSUP
340 	return PAGE_ALIGN(stack_top) + random_variable;
341 #else
342 	return PAGE_ALIGN(stack_top) - random_variable;
343 #endif
344 }
345 
346 #ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
arch_randomize_brk(struct mm_struct * mm)347 unsigned long arch_randomize_brk(struct mm_struct *mm)
348 {
349 	/* Is the current task 32bit ? */
350 	if (!IS_ENABLED(CONFIG_64BIT) || is_compat_task())
351 		return randomize_page(mm->brk, SZ_32M);
352 
353 	return randomize_page(mm->brk, SZ_1G);
354 }
355 
arch_mmap_rnd(void)356 unsigned long arch_mmap_rnd(void)
357 {
358 	unsigned long rnd;
359 
360 #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
361 	if (is_compat_task())
362 		rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1);
363 	else
364 #endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */
365 		rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1);
366 
367 	return rnd << PAGE_SHIFT;
368 }
369 
mmap_is_legacy(struct rlimit * rlim_stack)370 static int mmap_is_legacy(struct rlimit *rlim_stack)
371 {
372 	if (current->personality & ADDR_COMPAT_LAYOUT)
373 		return 1;
374 
375 	if (rlim_stack->rlim_cur == RLIM_INFINITY)
376 		return 1;
377 
378 	return sysctl_legacy_va_layout;
379 }
380 
381 /*
382  * Leave enough space between the mmap area and the stack to honour ulimit in
383  * the face of randomisation.
384  */
385 #define MIN_GAP		(SZ_128M)
386 #define MAX_GAP		(STACK_TOP / 6 * 5)
387 
mmap_base(unsigned long rnd,struct rlimit * rlim_stack)388 static unsigned long mmap_base(unsigned long rnd, struct rlimit *rlim_stack)
389 {
390 	unsigned long gap = rlim_stack->rlim_cur;
391 	unsigned long pad = stack_guard_gap;
392 
393 	/* Account for stack randomization if necessary */
394 	if (current->flags & PF_RANDOMIZE)
395 		pad += (STACK_RND_MASK << PAGE_SHIFT);
396 
397 	/* Values close to RLIM_INFINITY can overflow. */
398 	if (gap + pad > gap)
399 		gap += pad;
400 
401 	if (gap < MIN_GAP)
402 		gap = MIN_GAP;
403 	else if (gap > MAX_GAP)
404 		gap = MAX_GAP;
405 
406 	return PAGE_ALIGN(STACK_TOP - gap - rnd);
407 }
408 
arch_pick_mmap_layout(struct mm_struct * mm,struct rlimit * rlim_stack)409 void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack)
410 {
411 	unsigned long random_factor = 0UL;
412 
413 	if (current->flags & PF_RANDOMIZE)
414 		random_factor = arch_mmap_rnd();
415 
416 	if (mmap_is_legacy(rlim_stack)) {
417 		mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;
418 		mm->get_unmapped_area = arch_get_unmapped_area;
419 	} else {
420 		mm->mmap_base = mmap_base(random_factor, rlim_stack);
421 		mm->get_unmapped_area = arch_get_unmapped_area_topdown;
422 	}
423 }
424 #elif defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
arch_pick_mmap_layout(struct mm_struct * mm,struct rlimit * rlim_stack)425 void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack)
426 {
427 	mm->mmap_base = TASK_UNMAPPED_BASE;
428 	mm->get_unmapped_area = arch_get_unmapped_area;
429 }
430 #endif
431 
432 /**
433  * __account_locked_vm - account locked pages to an mm's locked_vm
434  * @mm:          mm to account against
435  * @pages:       number of pages to account
436  * @inc:         %true if @pages should be considered positive, %false if not
437  * @task:        task used to check RLIMIT_MEMLOCK
438  * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped
439  *
440  * Assumes @task and @mm are valid (i.e. at least one reference on each), and
441  * that mmap_lock is held as writer.
442  *
443  * Return:
444  * * 0       on success
445  * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
446  */
__account_locked_vm(struct mm_struct * mm,unsigned long pages,bool inc,struct task_struct * task,bool bypass_rlim)447 int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
448 			struct task_struct *task, bool bypass_rlim)
449 {
450 	unsigned long locked_vm, limit;
451 	int ret = 0;
452 
453 	mmap_assert_write_locked(mm);
454 
455 	locked_vm = mm->locked_vm;
456 	if (inc) {
457 		if (!bypass_rlim) {
458 			limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
459 			if (locked_vm + pages > limit)
460 				ret = -ENOMEM;
461 		}
462 		if (!ret)
463 			mm->locked_vm = locked_vm + pages;
464 	} else {
465 		WARN_ON_ONCE(pages > locked_vm);
466 		mm->locked_vm = locked_vm - pages;
467 	}
468 
469 	pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid,
470 		 (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT,
471 		 locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK),
472 		 ret ? " - exceeded" : "");
473 
474 	return ret;
475 }
476 EXPORT_SYMBOL_GPL(__account_locked_vm);
477 
478 /**
479  * account_locked_vm - account locked pages to an mm's locked_vm
480  * @mm:          mm to account against, may be NULL
481  * @pages:       number of pages to account
482  * @inc:         %true if @pages should be considered positive, %false if not
483  *
484  * Assumes a non-NULL @mm is valid (i.e. at least one reference on it).
485  *
486  * Return:
487  * * 0       on success, or if mm is NULL
488  * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
489  */
account_locked_vm(struct mm_struct * mm,unsigned long pages,bool inc)490 int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc)
491 {
492 	int ret;
493 
494 	if (pages == 0 || !mm)
495 		return 0;
496 
497 	mmap_write_lock(mm);
498 	ret = __account_locked_vm(mm, pages, inc, current,
499 				  capable(CAP_IPC_LOCK));
500 	mmap_write_unlock(mm);
501 
502 	return ret;
503 }
504 EXPORT_SYMBOL_GPL(account_locked_vm);
505 
vm_mmap_pgoff(struct file * file,unsigned long addr,unsigned long len,unsigned long prot,unsigned long flag,unsigned long pgoff)506 unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
507 	unsigned long len, unsigned long prot,
508 	unsigned long flag, unsigned long pgoff)
509 {
510 	unsigned long ret;
511 	struct mm_struct *mm = current->mm;
512 	unsigned long populate;
513 	LIST_HEAD(uf);
514 
515 	ret = security_mmap_file(file, prot, flag);
516 	if (!ret) {
517 		if (mmap_write_lock_killable(mm))
518 			return -EINTR;
519 		ret = do_mmap(file, addr, len, prot, flag, pgoff, &populate,
520 			      &uf);
521 		mmap_write_unlock(mm);
522 		userfaultfd_unmap_complete(mm, &uf);
523 		if (populate)
524 			mm_populate(ret, populate);
525 	}
526 	return ret;
527 }
528 
vm_mmap(struct file * file,unsigned long addr,unsigned long len,unsigned long prot,unsigned long flag,unsigned long offset)529 unsigned long vm_mmap(struct file *file, unsigned long addr,
530 	unsigned long len, unsigned long prot,
531 	unsigned long flag, unsigned long offset)
532 {
533 	if (unlikely(offset + PAGE_ALIGN(len) < offset))
534 		return -EINVAL;
535 	if (unlikely(offset_in_page(offset)))
536 		return -EINVAL;
537 
538 	return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
539 }
540 EXPORT_SYMBOL(vm_mmap);
541 
542 /**
543  * kvmalloc_node - attempt to allocate physically contiguous memory, but upon
544  * failure, fall back to non-contiguous (vmalloc) allocation.
545  * @size: size of the request.
546  * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
547  * @node: numa node to allocate from
548  *
549  * Uses kmalloc to get the memory but if the allocation fails then falls back
550  * to the vmalloc allocator. Use kvfree for freeing the memory.
551  *
552  * Reclaim modifiers - __GFP_NORETRY and __GFP_NOFAIL are not supported.
553  * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
554  * preferable to the vmalloc fallback, due to visible performance drawbacks.
555  *
556  * Please note that any use of gfp flags outside of GFP_KERNEL is careful to not
557  * fall back to vmalloc.
558  *
559  * Return: pointer to the allocated memory of %NULL in case of failure
560  */
kvmalloc_node(size_t size,gfp_t flags,int node)561 void *kvmalloc_node(size_t size, gfp_t flags, int node)
562 {
563 	gfp_t kmalloc_flags = flags;
564 	void *ret;
565 
566 	/*
567 	 * vmalloc uses GFP_KERNEL for some internal allocations (e.g page tables)
568 	 * so the given set of flags has to be compatible.
569 	 */
570 	if ((flags & GFP_KERNEL) != GFP_KERNEL)
571 		return kmalloc_node(size, flags, node);
572 
573 	/*
574 	 * We want to attempt a large physically contiguous block first because
575 	 * it is less likely to fragment multiple larger blocks and therefore
576 	 * contribute to a long term fragmentation less than vmalloc fallback.
577 	 * However make sure that larger requests are not too disruptive - no
578 	 * OOM killer and no allocation failure warnings as we have a fallback.
579 	 */
580 	if (size > PAGE_SIZE) {
581 		kmalloc_flags |= __GFP_NOWARN;
582 
583 		if (!(kmalloc_flags & __GFP_RETRY_MAYFAIL))
584 			kmalloc_flags |= __GFP_NORETRY;
585 	}
586 
587 	ret = kmalloc_node(size, kmalloc_flags, node);
588 
589 	/*
590 	 * It doesn't really make sense to fallback to vmalloc for sub page
591 	 * requests
592 	 */
593 	if (ret || size <= PAGE_SIZE)
594 		return ret;
595 
596 	/* Don't even allow crazy sizes */
597 	if (WARN_ON_ONCE(size > INT_MAX))
598 		return NULL;
599 
600 	return __vmalloc_node(size, 1, flags, node,
601 			__builtin_return_address(0));
602 }
603 EXPORT_SYMBOL(kvmalloc_node);
604 
605 /**
606  * kvfree() - Free memory.
607  * @addr: Pointer to allocated memory.
608  *
609  * kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc().
610  * It is slightly more efficient to use kfree() or vfree() if you are certain
611  * that you know which one to use.
612  *
613  * Context: Either preemptible task context or not-NMI interrupt.
614  */
kvfree(const void * addr)615 void kvfree(const void *addr)
616 {
617 	if (is_vmalloc_addr(addr))
618 		vfree(addr);
619 	else
620 		kfree(addr);
621 }
622 EXPORT_SYMBOL(kvfree);
623 
624 /**
625  * kvfree_sensitive - Free a data object containing sensitive information.
626  * @addr: address of the data object to be freed.
627  * @len: length of the data object.
628  *
629  * Use the special memzero_explicit() function to clear the content of a
630  * kvmalloc'ed object containing sensitive data to make sure that the
631  * compiler won't optimize out the data clearing.
632  */
kvfree_sensitive(const void * addr,size_t len)633 void kvfree_sensitive(const void *addr, size_t len)
634 {
635 	if (likely(!ZERO_OR_NULL_PTR(addr))) {
636 		memzero_explicit((void *)addr, len);
637 		kvfree(addr);
638 	}
639 }
640 EXPORT_SYMBOL(kvfree_sensitive);
641 
kvrealloc(const void * p,size_t oldsize,size_t newsize,gfp_t flags)642 void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
643 {
644 	void *newp;
645 
646 	if (oldsize >= newsize)
647 		return (void *)p;
648 	newp = kvmalloc(newsize, flags);
649 	if (!newp)
650 		return NULL;
651 	memcpy(newp, p, oldsize);
652 	kvfree(p);
653 	return newp;
654 }
655 EXPORT_SYMBOL(kvrealloc);
656 
657 /* Neutral page->mapping pointer to address_space or anon_vma or other */
page_rmapping(struct page * page)658 void *page_rmapping(struct page *page)
659 {
660 	return folio_raw_mapping(page_folio(page));
661 }
662 
663 /**
664  * folio_mapped - Is this folio mapped into userspace?
665  * @folio: The folio.
666  *
667  * Return: True if any page in this folio is referenced by user page tables.
668  */
folio_mapped(struct folio * folio)669 bool folio_mapped(struct folio *folio)
670 {
671 	long i, nr;
672 
673 	if (!folio_test_large(folio))
674 		return atomic_read(&folio->_mapcount) >= 0;
675 	if (atomic_read(folio_mapcount_ptr(folio)) >= 0)
676 		return true;
677 	if (folio_test_hugetlb(folio))
678 		return false;
679 
680 	nr = folio_nr_pages(folio);
681 	for (i = 0; i < nr; i++) {
682 		if (atomic_read(&folio_page(folio, i)->_mapcount) >= 0)
683 			return true;
684 	}
685 	return false;
686 }
687 EXPORT_SYMBOL(folio_mapped);
688 
page_anon_vma(struct page * page)689 struct anon_vma *page_anon_vma(struct page *page)
690 {
691 	struct folio *folio = page_folio(page);
692 	unsigned long mapping = (unsigned long)folio->mapping;
693 
694 	if ((mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
695 		return NULL;
696 	return (void *)(mapping - PAGE_MAPPING_ANON);
697 }
698 
699 /**
700  * folio_mapping - Find the mapping where this folio is stored.
701  * @folio: The folio.
702  *
703  * For folios which are in the page cache, return the mapping that this
704  * page belongs to.  Folios in the swap cache return the swap mapping
705  * this page is stored in (which is different from the mapping for the
706  * swap file or swap device where the data is stored).
707  *
708  * You can call this for folios which aren't in the swap cache or page
709  * cache and it will return NULL.
710  */
folio_mapping(struct folio * folio)711 struct address_space *folio_mapping(struct folio *folio)
712 {
713 	struct address_space *mapping;
714 
715 	/* This happens if someone calls flush_dcache_page on slab page */
716 	if (unlikely(folio_test_slab(folio)))
717 		return NULL;
718 
719 	if (unlikely(folio_test_swapcache(folio)))
720 		return swap_address_space(folio_swap_entry(folio));
721 
722 	mapping = folio->mapping;
723 	if ((unsigned long)mapping & PAGE_MAPPING_ANON)
724 		return NULL;
725 
726 	return (void *)((unsigned long)mapping & ~PAGE_MAPPING_FLAGS);
727 }
728 EXPORT_SYMBOL(folio_mapping);
729 
730 /* Slow path of page_mapcount() for compound pages */
__page_mapcount(struct page * page)731 int __page_mapcount(struct page *page)
732 {
733 	int ret;
734 
735 	ret = atomic_read(&page->_mapcount) + 1;
736 	/*
737 	 * For file THP page->_mapcount contains total number of mapping
738 	 * of the page: no need to look into compound_mapcount.
739 	 */
740 	if (!PageAnon(page) && !PageHuge(page))
741 		return ret;
742 	page = compound_head(page);
743 	ret += atomic_read(compound_mapcount_ptr(page)) + 1;
744 	if (PageDoubleMap(page))
745 		ret--;
746 	return ret;
747 }
748 EXPORT_SYMBOL_GPL(__page_mapcount);
749 
750 /**
751  * folio_copy - Copy the contents of one folio to another.
752  * @dst: Folio to copy to.
753  * @src: Folio to copy from.
754  *
755  * The bytes in the folio represented by @src are copied to @dst.
756  * Assumes the caller has validated that @dst is at least as large as @src.
757  * Can be called in atomic context for order-0 folios, but if the folio is
758  * larger, it may sleep.
759  */
folio_copy(struct folio * dst,struct folio * src)760 void folio_copy(struct folio *dst, struct folio *src)
761 {
762 	long i = 0;
763 	long nr = folio_nr_pages(src);
764 
765 	for (;;) {
766 		copy_highpage(folio_page(dst, i), folio_page(src, i));
767 		if (++i == nr)
768 			break;
769 		cond_resched();
770 	}
771 }
772 
773 int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;
774 int sysctl_overcommit_ratio __read_mostly = 50;
775 unsigned long sysctl_overcommit_kbytes __read_mostly;
776 int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
777 unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
778 unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
779 
overcommit_ratio_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)780 int overcommit_ratio_handler(struct ctl_table *table, int write, void *buffer,
781 		size_t *lenp, loff_t *ppos)
782 {
783 	int ret;
784 
785 	ret = proc_dointvec(table, write, buffer, lenp, ppos);
786 	if (ret == 0 && write)
787 		sysctl_overcommit_kbytes = 0;
788 	return ret;
789 }
790 
sync_overcommit_as(struct work_struct * dummy)791 static void sync_overcommit_as(struct work_struct *dummy)
792 {
793 	percpu_counter_sync(&vm_committed_as);
794 }
795 
overcommit_policy_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)796 int overcommit_policy_handler(struct ctl_table *table, int write, void *buffer,
797 		size_t *lenp, loff_t *ppos)
798 {
799 	struct ctl_table t;
800 	int new_policy = -1;
801 	int ret;
802 
803 	/*
804 	 * The deviation of sync_overcommit_as could be big with loose policy
805 	 * like OVERCOMMIT_ALWAYS/OVERCOMMIT_GUESS. When changing policy to
806 	 * strict OVERCOMMIT_NEVER, we need to reduce the deviation to comply
807 	 * with the strict "NEVER", and to avoid possible race condition (even
808 	 * though user usually won't too frequently do the switching to policy
809 	 * OVERCOMMIT_NEVER), the switch is done in the following order:
810 	 *	1. changing the batch
811 	 *	2. sync percpu count on each CPU
812 	 *	3. switch the policy
813 	 */
814 	if (write) {
815 		t = *table;
816 		t.data = &new_policy;
817 		ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
818 		if (ret || new_policy == -1)
819 			return ret;
820 
821 		mm_compute_batch(new_policy);
822 		if (new_policy == OVERCOMMIT_NEVER)
823 			schedule_on_each_cpu(sync_overcommit_as);
824 		sysctl_overcommit_memory = new_policy;
825 	} else {
826 		ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
827 	}
828 
829 	return ret;
830 }
831 
overcommit_kbytes_handler(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)832 int overcommit_kbytes_handler(struct ctl_table *table, int write, void *buffer,
833 		size_t *lenp, loff_t *ppos)
834 {
835 	int ret;
836 
837 	ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
838 	if (ret == 0 && write)
839 		sysctl_overcommit_ratio = 0;
840 	return ret;
841 }
842 
843 /*
844  * Committed memory limit enforced when OVERCOMMIT_NEVER policy is used
845  */
vm_commit_limit(void)846 unsigned long vm_commit_limit(void)
847 {
848 	unsigned long allowed;
849 
850 	if (sysctl_overcommit_kbytes)
851 		allowed = sysctl_overcommit_kbytes >> (PAGE_SHIFT - 10);
852 	else
853 		allowed = ((totalram_pages() - hugetlb_total_pages())
854 			   * sysctl_overcommit_ratio / 100);
855 	allowed += total_swap_pages;
856 
857 	return allowed;
858 }
859 
860 /*
861  * Make sure vm_committed_as in one cacheline and not cacheline shared with
862  * other variables. It can be updated by several CPUs frequently.
863  */
864 struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
865 
866 /*
867  * The global memory commitment made in the system can be a metric
868  * that can be used to drive ballooning decisions when Linux is hosted
869  * as a guest. On Hyper-V, the host implements a policy engine for dynamically
870  * balancing memory across competing virtual machines that are hosted.
871  * Several metrics drive this policy engine including the guest reported
872  * memory commitment.
873  *
874  * The time cost of this is very low for small platforms, and for big
875  * platform like a 2S/36C/72T Skylake server, in worst case where
876  * vm_committed_as's spinlock is under severe contention, the time cost
877  * could be about 30~40 microseconds.
878  */
vm_memory_committed(void)879 unsigned long vm_memory_committed(void)
880 {
881 	return percpu_counter_sum_positive(&vm_committed_as);
882 }
883 EXPORT_SYMBOL_GPL(vm_memory_committed);
884 
885 /*
886  * Check that a process has enough memory to allocate a new virtual
887  * mapping. 0 means there is enough memory for the allocation to
888  * succeed and -ENOMEM implies there is not.
889  *
890  * We currently support three overcommit policies, which are set via the
891  * vm.overcommit_memory sysctl.  See Documentation/vm/overcommit-accounting.rst
892  *
893  * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
894  * Additional code 2002 Jul 20 by Robert Love.
895  *
896  * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
897  *
898  * Note this is a helper function intended to be used by LSMs which
899  * wish to use this logic.
900  */
__vm_enough_memory(struct mm_struct * mm,long pages,int cap_sys_admin)901 int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
902 {
903 	long allowed;
904 
905 	vm_acct_memory(pages);
906 
907 	/*
908 	 * Sometimes we want to use more memory than we have
909 	 */
910 	if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
911 		return 0;
912 
913 	if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
914 		if (pages > totalram_pages() + total_swap_pages)
915 			goto error;
916 		return 0;
917 	}
918 
919 	allowed = vm_commit_limit();
920 	/*
921 	 * Reserve some for root
922 	 */
923 	if (!cap_sys_admin)
924 		allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
925 
926 	/*
927 	 * Don't let a single process grow so big a user can't recover
928 	 */
929 	if (mm) {
930 		long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
931 
932 		allowed -= min_t(long, mm->total_vm / 32, reserve);
933 	}
934 
935 	if (percpu_counter_read_positive(&vm_committed_as) < allowed)
936 		return 0;
937 error:
938 	vm_unacct_memory(pages);
939 
940 	return -ENOMEM;
941 }
942 
943 /**
944  * get_cmdline() - copy the cmdline value to a buffer.
945  * @task:     the task whose cmdline value to copy.
946  * @buffer:   the buffer to copy to.
947  * @buflen:   the length of the buffer. Larger cmdline values are truncated
948  *            to this length.
949  *
950  * Return: the size of the cmdline field copied. Note that the copy does
951  * not guarantee an ending NULL byte.
952  */
get_cmdline(struct task_struct * task,char * buffer,int buflen)953 int get_cmdline(struct task_struct *task, char *buffer, int buflen)
954 {
955 	int res = 0;
956 	unsigned int len;
957 	struct mm_struct *mm = get_task_mm(task);
958 	unsigned long arg_start, arg_end, env_start, env_end;
959 	if (!mm)
960 		goto out;
961 	if (!mm->arg_end)
962 		goto out_mm;	/* Shh! No looking before we're done */
963 
964 	spin_lock(&mm->arg_lock);
965 	arg_start = mm->arg_start;
966 	arg_end = mm->arg_end;
967 	env_start = mm->env_start;
968 	env_end = mm->env_end;
969 	spin_unlock(&mm->arg_lock);
970 
971 	len = arg_end - arg_start;
972 
973 	if (len > buflen)
974 		len = buflen;
975 
976 	res = access_process_vm(task, arg_start, buffer, len, FOLL_FORCE);
977 
978 	/*
979 	 * If the nul at the end of args has been overwritten, then
980 	 * assume application is using setproctitle(3).
981 	 */
982 	if (res > 0 && buffer[res-1] != '\0' && len < buflen) {
983 		len = strnlen(buffer, res);
984 		if (len < res) {
985 			res = len;
986 		} else {
987 			len = env_end - env_start;
988 			if (len > buflen - res)
989 				len = buflen - res;
990 			res += access_process_vm(task, env_start,
991 						 buffer+res, len,
992 						 FOLL_FORCE);
993 			res = strnlen(buffer, res);
994 		}
995 	}
996 out_mm:
997 	mmput(mm);
998 out:
999 	return res;
1000 }
1001 
memcmp_pages(struct page * page1,struct page * page2)1002 int __weak memcmp_pages(struct page *page1, struct page *page2)
1003 {
1004 	char *addr1, *addr2;
1005 	int ret;
1006 
1007 	addr1 = kmap_atomic(page1);
1008 	addr2 = kmap_atomic(page2);
1009 	ret = memcmp(addr1, addr2, PAGE_SIZE);
1010 	kunmap_atomic(addr2);
1011 	kunmap_atomic(addr1);
1012 	return ret;
1013 }
1014 
1015 #ifdef CONFIG_PRINTK
1016 /**
1017  * mem_dump_obj - Print available provenance information
1018  * @object: object for which to find provenance information.
1019  *
1020  * This function uses pr_cont(), so that the caller is expected to have
1021  * printed out whatever preamble is appropriate.  The provenance information
1022  * depends on the type of object and on how much debugging is enabled.
1023  * For example, for a slab-cache object, the slab name is printed, and,
1024  * if available, the return address and stack trace from the allocation
1025  * and last free path of that object.
1026  */
mem_dump_obj(void * object)1027 void mem_dump_obj(void *object)
1028 {
1029 	const char *type;
1030 
1031 	if (kmem_valid_obj(object)) {
1032 		kmem_dump_obj(object);
1033 		return;
1034 	}
1035 
1036 	if (vmalloc_dump_obj(object))
1037 		return;
1038 
1039 	if (virt_addr_valid(object))
1040 		type = "non-slab/vmalloc memory";
1041 	else if (object == NULL)
1042 		type = "NULL pointer";
1043 	else if (object == ZERO_SIZE_PTR)
1044 		type = "zero-size pointer";
1045 	else
1046 		type = "non-paged memory";
1047 
1048 	pr_cont(" %s\n", type);
1049 }
1050 EXPORT_SYMBOL_GPL(mem_dump_obj);
1051 #endif
1052 
1053 /*
1054  * A driver might set a page logically offline -- PageOffline() -- and
1055  * turn the page inaccessible in the hypervisor; after that, access to page
1056  * content can be fatal.
1057  *
1058  * Some special PFN walkers -- i.e., /proc/kcore -- read content of random
1059  * pages after checking PageOffline(); however, these PFN walkers can race
1060  * with drivers that set PageOffline().
1061  *
1062  * page_offline_freeze()/page_offline_thaw() allows for a subsystem to
1063  * synchronize with such drivers, achieving that a page cannot be set
1064  * PageOffline() while frozen.
1065  *
1066  * page_offline_begin()/page_offline_end() is used by drivers that care about
1067  * such races when setting a page PageOffline().
1068  */
1069 static DECLARE_RWSEM(page_offline_rwsem);
1070 
page_offline_freeze(void)1071 void page_offline_freeze(void)
1072 {
1073 	down_read(&page_offline_rwsem);
1074 }
1075 
page_offline_thaw(void)1076 void page_offline_thaw(void)
1077 {
1078 	up_read(&page_offline_rwsem);
1079 }
1080 
page_offline_begin(void)1081 void page_offline_begin(void)
1082 {
1083 	down_write(&page_offline_rwsem);
1084 }
1085 EXPORT_SYMBOL(page_offline_begin);
1086 
page_offline_end(void)1087 void page_offline_end(void)
1088 {
1089 	up_write(&page_offline_rwsem);
1090 }
1091 EXPORT_SYMBOL(page_offline_end);
1092 
1093 #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_FOLIO
flush_dcache_folio(struct folio * folio)1094 void flush_dcache_folio(struct folio *folio)
1095 {
1096 	long i, nr = folio_nr_pages(folio);
1097 
1098 	for (i = 0; i < nr; i++)
1099 		flush_dcache_page(folio_page(folio, i));
1100 }
1101 EXPORT_SYMBOL(flush_dcache_folio);
1102 #endif
1103