Lines Matching refs:dst_vma
30 struct vm_area_struct *dst_vma; in find_dst_vma() local
32 dst_vma = find_vma(dst_mm, dst_start); in find_dst_vma()
33 if (!dst_vma) in find_dst_vma()
36 if (dst_start < dst_vma->vm_start || in find_dst_vma()
37 dst_start + len > dst_vma->vm_end) in find_dst_vma()
45 if (!dst_vma->vm_userfaultfd_ctx.ctx) in find_dst_vma()
48 return dst_vma; in find_dst_vma()
58 struct vm_area_struct *dst_vma, in mfill_atomic_install_pte() argument
64 bool writable = dst_vma->vm_flags & VM_WRITE; in mfill_atomic_install_pte()
65 bool vm_shared = dst_vma->vm_flags & VM_SHARED; in mfill_atomic_install_pte()
71 _dst_pte = mk_pte(page, dst_vma->vm_page_prot); in mfill_atomic_install_pte()
84 if (vma_is_shmem(dst_vma)) { in mfill_atomic_install_pte()
86 inode = dst_vma->vm_file->f_inode; in mfill_atomic_install_pte()
87 offset = linear_page_index(dst_vma, dst_addr); in mfill_atomic_install_pte()
101 page_add_new_anon_rmap(page, dst_vma, dst_addr, false); in mfill_atomic_install_pte()
110 lru_cache_add_inactive_or_unevictable(page, dst_vma); in mfill_atomic_install_pte()
115 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_install_pte()
124 struct vm_area_struct *dst_vma, in mcopy_atomic_pte() argument
136 page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, dst_vma, dst_addr); in mcopy_atomic_pte()
169 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, in mcopy_atomic_pte()
182 struct vm_area_struct *dst_vma, in mfill_zeropage_pte() argument
192 dst_vma->vm_page_prot)); in mfill_zeropage_pte()
194 if (dst_vma->vm_file) { in mfill_zeropage_pte()
196 inode = dst_vma->vm_file->f_inode; in mfill_zeropage_pte()
197 offset = linear_page_index(dst_vma, dst_addr); in mfill_zeropage_pte()
208 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_zeropage_pte()
218 struct vm_area_struct *dst_vma, in mcontinue_atomic_pte() argument
222 struct inode *inode = file_inode(dst_vma->vm_file); in mcontinue_atomic_pte()
223 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); in mcontinue_atomic_pte()
235 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, in mcontinue_atomic_pte()
277 struct vm_area_struct *dst_vma, in __mcopy_atomic_hugetlb() argument
283 int vm_shared = dst_vma->vm_flags & VM_SHARED; in __mcopy_atomic_hugetlb()
309 vma_hpagesize = vma_kernel_pagesize(dst_vma); in __mcopy_atomic_hugetlb()
323 if (!dst_vma) { in __mcopy_atomic_hugetlb()
325 dst_vma = find_dst_vma(dst_mm, dst_start, len); in __mcopy_atomic_hugetlb()
326 if (!dst_vma || !is_vm_hugetlb_page(dst_vma)) in __mcopy_atomic_hugetlb()
330 if (vma_hpagesize != vma_kernel_pagesize(dst_vma)) in __mcopy_atomic_hugetlb()
333 vm_shared = dst_vma->vm_flags & VM_SHARED; in __mcopy_atomic_hugetlb()
341 if (unlikely(anon_vma_prepare(dst_vma))) in __mcopy_atomic_hugetlb()
354 mapping = dst_vma->vm_file->f_mapping; in __mcopy_atomic_hugetlb()
356 idx = linear_page_index(dst_vma, dst_addr); in __mcopy_atomic_hugetlb()
361 dst_pte = huge_pte_alloc(dst_mm, dst_vma, dst_addr, vma_hpagesize); in __mcopy_atomic_hugetlb()
376 err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma, in __mcopy_atomic_hugetlb()
398 dst_vma = NULL; in __mcopy_atomic_hugetlb()
428 struct vm_area_struct *dst_vma,
437 struct vm_area_struct *dst_vma, in mfill_atomic_pte() argument
447 return mcontinue_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, in mfill_atomic_pte()
461 if (!(dst_vma->vm_flags & VM_SHARED)) { in mfill_atomic_pte()
463 err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma, in mfill_atomic_pte()
468 dst_vma, dst_addr); in mfill_atomic_pte()
471 err = shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, in mfill_atomic_pte()
488 struct vm_area_struct *dst_vma; in __mcopy_atomic() local
527 dst_vma = find_dst_vma(dst_mm, dst_start, len); in __mcopy_atomic()
528 if (!dst_vma) in __mcopy_atomic()
536 if (WARN_ON_ONCE(vma_is_anonymous(dst_vma) && in __mcopy_atomic()
537 dst_vma->vm_flags & VM_SHARED)) in __mcopy_atomic()
545 if (wp_copy && !(dst_vma->vm_flags & VM_UFFD_WP)) in __mcopy_atomic()
551 if (is_vm_hugetlb_page(dst_vma)) in __mcopy_atomic()
552 return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start, in __mcopy_atomic()
555 if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma)) in __mcopy_atomic()
557 if (!vma_is_shmem(dst_vma) && mcopy_mode == MCOPY_ATOMIC_CONTINUE) in __mcopy_atomic()
566 if (!(dst_vma->vm_flags & VM_SHARED) && in __mcopy_atomic()
567 unlikely(anon_vma_prepare(dst_vma))) in __mcopy_atomic()
604 err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr, in __mcopy_atomic()
676 struct vm_area_struct *dst_vma; in mwriteprotect_range() local
701 dst_vma = find_dst_vma(dst_mm, start, len); in mwriteprotect_range()
706 if (!dst_vma || (dst_vma->vm_flags & VM_SHARED)) in mwriteprotect_range()
708 if (!userfaultfd_wp(dst_vma)) in mwriteprotect_range()
710 if (!vma_is_anonymous(dst_vma)) in mwriteprotect_range()
714 newprot = vm_get_page_prot(dst_vma->vm_flags & ~(VM_WRITE)); in mwriteprotect_range()
716 newprot = vm_get_page_prot(dst_vma->vm_flags); in mwriteprotect_range()
718 change_protection(dst_vma, start, start + len, newprot, in mwriteprotect_range()