Lines Matching refs:gup_flags
855 unsigned int gup_flags, struct vm_area_struct **vma, in get_gate_page() argument
866 if (gup_flags & FOLL_WRITE) in get_gate_page()
892 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte))) in get_gate_page()
896 if (unlikely(!try_grab_page(*page, gup_flags))) { in get_gate_page()
968 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) in check_vma_flags() argument
971 int write = (gup_flags & FOLL_WRITE); in check_vma_flags()
972 int foreign = (gup_flags & FOLL_REMOTE); in check_vma_flags()
977 if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma)) in check_vma_flags()
980 if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma)) in check_vma_flags()
988 if (!(gup_flags & FOLL_FORCE)) in check_vma_flags()
1003 if (!(gup_flags & FOLL_FORCE)) in check_vma_flags()
1083 unsigned int gup_flags, struct page **pages, in __get_user_pages() argument
1095 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN))); in __get_user_pages()
1102 if (!(gup_flags & FOLL_FORCE)) in __get_user_pages()
1103 gup_flags |= FOLL_NUMA; in __get_user_pages()
1107 unsigned int foll_flags = gup_flags; in __get_user_pages()
1115 gup_flags, &vma, in __get_user_pages()
1127 ret = check_vma_flags(vma, gup_flags); in __get_user_pages()
1134 gup_flags, locked); in __get_user_pages()
1141 BUG_ON(gup_flags & FOLL_NOWAIT); in __get_user_pages()
1462 int gup_flags; in populate_vma_page_range() local
1470 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK; in populate_vma_page_range()
1472 gup_flags &= ~FOLL_POPULATE; in populate_vma_page_range()
1479 gup_flags |= FOLL_WRITE; in populate_vma_page_range()
1486 gup_flags |= FOLL_FORCE; in populate_vma_page_range()
1492 return __get_user_pages(mm, start, nr_pages, gup_flags, in populate_vma_page_range()
1524 int gup_flags; in faultin_vma_page_range() local
1542 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK | FOLL_HWPOISON; in faultin_vma_page_range()
1544 gup_flags |= FOLL_WRITE; in faultin_vma_page_range()
1550 if (check_vma_flags(vma, gup_flags)) in faultin_vma_page_range()
1553 return __get_user_pages(mm, start, nr_pages, gup_flags, in faultin_vma_page_range()
1837 unsigned int gup_flags) in check_and_migrate_movable_pages() argument
1890 if (gup_flags & FOLL_PIN) { in check_and_migrate_movable_pages()
1909 unsigned int gup_flags) in check_and_migrate_movable_pages() argument
1924 unsigned int gup_flags) in __gup_longterm_locked() argument
1929 if (!(gup_flags & FOLL_LONGTERM)) in __gup_longterm_locked()
1931 NULL, gup_flags); in __gup_longterm_locked()
1935 NULL, gup_flags); in __gup_longterm_locked()
1938 rc = check_and_migrate_movable_pages(rc, pages, gup_flags); in __gup_longterm_locked()
1945 static bool is_valid_gup_flags(unsigned int gup_flags) in is_valid_gup_flags() argument
1951 if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) in is_valid_gup_flags()
1958 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) in is_valid_gup_flags()
1967 unsigned int gup_flags, struct page **pages, in __get_user_pages_remote() argument
1977 if (gup_flags & FOLL_LONGTERM) { in __get_user_pages_remote()
1985 vmas, gup_flags | FOLL_TOUCH | in __get_user_pages_remote()
1991 gup_flags | FOLL_TOUCH | FOLL_REMOTE); in __get_user_pages_remote()
2056 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
2059 if (!is_valid_gup_flags(gup_flags)) in get_user_pages_remote()
2062 return __get_user_pages_remote(mm, start, nr_pages, gup_flags, in get_user_pages_remote()
2070 unsigned int gup_flags, struct page **pages, in get_user_pages_remote() argument
2078 unsigned int gup_flags, struct page **pages, in __get_user_pages_remote() argument
2102 unsigned int gup_flags, struct page **pages, in get_user_pages() argument
2105 if (!is_valid_gup_flags(gup_flags)) in get_user_pages()
2109 pages, vmas, gup_flags | FOLL_TOUCH); in get_user_pages()
2148 unsigned int gup_flags, struct page **pages, in get_user_pages_locked() argument
2157 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) in get_user_pages_locked()
2163 if (WARN_ON_ONCE(gup_flags & FOLL_PIN)) in get_user_pages_locked()
2168 gup_flags | FOLL_TOUCH); in get_user_pages_locked()
2188 struct page **pages, unsigned int gup_flags) in get_user_pages_unlocked() argument
2200 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) in get_user_pages_unlocked()
2205 &locked, gup_flags | FOLL_TOUCH); in get_user_pages_unlocked()
2765 unsigned int gup_flags, struct page **pages) in __gup_longterm_unlocked() argument
2773 if (gup_flags & FOLL_LONGTERM) { in __gup_longterm_unlocked()
2777 pages, NULL, gup_flags); in __gup_longterm_unlocked()
2781 pages, gup_flags); in __gup_longterm_unlocked()
2789 unsigned int gup_flags, in lockless_pages_from_mm() argument
2800 if (gup_flags & FOLL_PIN) { in lockless_pages_from_mm()
2818 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned); in lockless_pages_from_mm()
2825 if (gup_flags & FOLL_PIN) { in lockless_pages_from_mm()
2836 unsigned int gup_flags, in internal_get_user_pages_fast() argument
2843 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM | in internal_get_user_pages_fast()
2848 if (gup_flags & FOLL_PIN) in internal_get_user_pages_fast()
2851 if (!(gup_flags & FOLL_FAST_ONLY)) in internal_get_user_pages_fast()
2861 nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages); in internal_get_user_pages_fast()
2862 if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY) in internal_get_user_pages_fast()
2868 ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags, in internal_get_user_pages_fast()
2903 unsigned int gup_flags, struct page **pages) in get_user_pages_fast_only() argument
2913 gup_flags |= FOLL_GET | FOLL_FAST_ONLY; in get_user_pages_fast_only()
2915 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags, in get_user_pages_fast_only()
2948 unsigned int gup_flags, struct page **pages) in get_user_pages_fast() argument
2950 if (!is_valid_gup_flags(gup_flags)) in get_user_pages_fast()
2959 gup_flags |= FOLL_GET; in get_user_pages_fast()
2960 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in get_user_pages_fast()
2981 unsigned int gup_flags, struct page **pages) in pin_user_pages_fast() argument
2984 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_fast()
2987 gup_flags |= FOLL_PIN; in pin_user_pages_fast()
2988 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages); in pin_user_pages_fast()
2999 unsigned int gup_flags, struct page **pages) in pin_user_pages_fast_only() argument
3007 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_fast_only()
3013 gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY); in pin_user_pages_fast_only()
3014 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags, in pin_user_pages_fast_only()
3053 unsigned int gup_flags, struct page **pages, in pin_user_pages_remote() argument
3057 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_remote()
3060 gup_flags |= FOLL_PIN; in pin_user_pages_remote()
3061 return __get_user_pages_remote(mm, start, nr_pages, gup_flags, in pin_user_pages_remote()
3085 unsigned int gup_flags, struct page **pages, in pin_user_pages() argument
3089 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages()
3092 gup_flags |= FOLL_PIN; in pin_user_pages()
3094 pages, vmas, gup_flags); in pin_user_pages()
3104 struct page **pages, unsigned int gup_flags) in pin_user_pages_unlocked() argument
3107 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_unlocked()
3110 gup_flags |= FOLL_PIN; in pin_user_pages_unlocked()
3111 return get_user_pages_unlocked(start, nr_pages, pages, gup_flags); in pin_user_pages_unlocked()
3121 unsigned int gup_flags, struct page **pages, in pin_user_pages_locked() argument
3130 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM)) in pin_user_pages_locked()
3134 if (WARN_ON_ONCE(gup_flags & FOLL_GET)) in pin_user_pages_locked()
3137 gup_flags |= FOLL_PIN; in pin_user_pages_locked()
3140 gup_flags | FOLL_TOUCH); in pin_user_pages_locked()