Lines Matching refs:vma
70 static pud_t *alloc_new_pud(struct mm_struct *mm, struct vm_area_struct *vma, in alloc_new_pud() argument
84 static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma, in alloc_new_pmd() argument
90 pud = alloc_new_pud(mm, vma, addr); in alloc_new_pmd()
103 static void take_rmap_locks(struct vm_area_struct *vma) in take_rmap_locks() argument
105 if (vma->vm_file) in take_rmap_locks()
106 i_mmap_lock_write(vma->vm_file->f_mapping); in take_rmap_locks()
107 if (vma->anon_vma) in take_rmap_locks()
108 anon_vma_lock_write(vma->anon_vma); in take_rmap_locks()
111 static void drop_rmap_locks(struct vm_area_struct *vma) in drop_rmap_locks() argument
113 if (vma->anon_vma) in drop_rmap_locks()
114 anon_vma_unlock_write(vma->anon_vma); in drop_rmap_locks()
115 if (vma->vm_file) in drop_rmap_locks()
116 i_mmap_unlock_write(vma->vm_file->f_mapping); in drop_rmap_locks()
134 static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd, in move_ptes() argument
139 struct mm_struct *mm = vma->vm_mm; in move_ptes()
164 take_rmap_locks(vma); in move_ptes()
175 flush_tlb_batched_pending(vma->vm_mm); in move_ptes()
204 flush_tlb_range(vma, old_end - len, old_end); in move_ptes()
210 drop_rmap_locks(vma); in move_ptes()
223 static bool move_normal_pmd(struct vm_area_struct *vma, unsigned long old_addr, in move_normal_pmd() argument
227 struct mm_struct *mm = vma->vm_mm; in move_normal_pmd()
262 old_ptl = pmd_lock(vma->vm_mm, old_pmd); in move_normal_pmd()
274 flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE); in move_normal_pmd()
282 static inline bool move_normal_pmd(struct vm_area_struct *vma, in move_normal_pmd() argument
291 static bool move_normal_pud(struct vm_area_struct *vma, unsigned long old_addr, in move_normal_pud() argument
295 struct mm_struct *mm = vma->vm_mm; in move_normal_pud()
311 old_ptl = pud_lock(vma->vm_mm, old_pud); in move_normal_pud()
323 flush_tlb_range(vma, old_addr, old_addr + PUD_SIZE); in move_normal_pud()
331 static inline bool move_normal_pud(struct vm_area_struct *vma, in move_normal_pud() argument
340 static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr, in move_huge_pud() argument
344 struct mm_struct *mm = vma->vm_mm; in move_huge_pud()
358 old_ptl = pud_lock(vma->vm_mm, old_pud); in move_huge_pud()
372 flush_pud_tlb_range(vma, old_addr, old_addr + HPAGE_PUD_SIZE); in move_huge_pud()
380 static bool move_huge_pud(struct vm_area_struct *vma, unsigned long old_addr, in move_huge_pud() argument
438 static bool move_pgt_entry(enum pgt_entry entry, struct vm_area_struct *vma, in move_pgt_entry() argument
446 take_rmap_locks(vma); in move_pgt_entry()
450 moved = move_normal_pmd(vma, old_addr, new_addr, old_entry, in move_pgt_entry()
454 moved = move_normal_pud(vma, old_addr, new_addr, old_entry, in move_pgt_entry()
459 move_huge_pmd(vma, old_addr, new_addr, old_entry, in move_pgt_entry()
464 move_huge_pud(vma, old_addr, new_addr, old_entry, in move_pgt_entry()
474 drop_rmap_locks(vma); in move_pgt_entry()
479 unsigned long move_page_tables(struct vm_area_struct *vma, in move_page_tables() argument
490 flush_cache_range(vma, old_addr, old_end); in move_page_tables()
492 if (is_vm_hugetlb_page(vma)) in move_page_tables()
493 return move_hugetlb_page_tables(vma, new_vma, old_addr, in move_page_tables()
496 mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, vma->vm_mm, in move_page_tables()
508 old_pud = get_old_pud(vma->vm_mm, old_addr); in move_page_tables()
511 new_pud = alloc_new_pud(vma->vm_mm, vma, new_addr); in move_page_tables()
516 move_pgt_entry(HPAGE_PUD, vma, old_addr, new_addr, in move_page_tables()
523 if (move_pgt_entry(NORMAL_PUD, vma, old_addr, new_addr, in move_page_tables()
529 old_pmd = get_old_pmd(vma->vm_mm, old_addr); in move_page_tables()
532 new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr); in move_page_tables()
538 move_pgt_entry(HPAGE_PMD, vma, old_addr, new_addr, in move_page_tables()
541 split_huge_pmd(vma, old_pmd, old_addr); in move_page_tables()
550 if (move_pgt_entry(NORMAL_PMD, vma, old_addr, new_addr, in move_page_tables()
557 move_ptes(vma, old_pmd, old_addr, old_addr + extent, new_vma, in move_page_tables()
566 static unsigned long move_vma(struct vm_area_struct *vma, in move_vma() argument
573 struct mm_struct *mm = vma->vm_mm; in move_vma()
575 unsigned long vm_flags = vma->vm_flags; in move_vma()
594 if (vma->vm_ops && vma->vm_ops->may_split) { in move_vma()
595 if (vma->vm_start != old_addr) in move_vma()
596 err = vma->vm_ops->may_split(vma, old_addr); in move_vma()
597 if (!err && vma->vm_end != old_addr + old_len) in move_vma()
598 err = vma->vm_ops->may_split(vma, old_addr + old_len); in move_vma()
610 err = ksm_madvise(vma, old_addr, old_addr + old_len, in move_vma()
620 new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT); in move_vma()
621 new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff, in move_vma()
629 moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len, in move_vma()
633 } else if (vma->vm_ops && vma->vm_ops->mremap) { in move_vma()
634 err = vma->vm_ops->mremap(new_vma); in move_vma()
643 move_page_tables(new_vma, new_addr, vma, old_addr, moved_len, in move_vma()
645 vma = new_vma; in move_vma()
653 if (is_vm_hugetlb_page(vma)) { in move_vma()
654 clear_vma_resv_huge_pages(vma); in move_vma()
659 vma->vm_flags &= ~VM_ACCOUNT; in move_vma()
660 excess = vma->vm_end - vma->vm_start - old_len; in move_vma()
661 if (old_addr > vma->vm_start && in move_vma()
662 old_addr + old_len < vma->vm_end) in move_vma()
676 vm_stat_account(mm, vma->vm_flags, new_len >> PAGE_SHIFT); in move_vma()
679 if (unlikely(vma->vm_flags & VM_PFNMAP)) in move_vma()
680 untrack_pfn_moved(vma); in move_vma()
684 vma->vm_flags &= VM_LOCKED_CLEAR_MASK; in move_vma()
690 if (new_vma != vma && vma->vm_start == old_addr && in move_vma()
691 vma->vm_end == (old_addr + old_len)) in move_vma()
692 unlink_anon_vmas(vma); in move_vma()
714 vma->vm_flags |= VM_ACCOUNT; in move_vma()
716 vma->vm_next->vm_flags |= VM_ACCOUNT; in move_vma()
726 struct vm_area_struct *vma; in vma_to_resize() local
729 vma = vma_lookup(mm, addr); in vma_to_resize()
730 if (!vma) in vma_to_resize()
741 if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) { in vma_to_resize()
747 (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))) in vma_to_resize()
751 if (old_len > vma->vm_end - addr) in vma_to_resize()
755 return vma; in vma_to_resize()
758 pgoff = (addr - vma->vm_start) >> PAGE_SHIFT; in vma_to_resize()
759 pgoff += vma->vm_pgoff; in vma_to_resize()
763 if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)) in vma_to_resize()
766 if (vma->vm_flags & VM_LOCKED) { in vma_to_resize()
775 if (!may_expand_vm(mm, vma->vm_flags, in vma_to_resize()
779 return vma; in vma_to_resize()
789 struct vm_area_struct *vma; in mremap_to() local
833 vma = vma_to_resize(addr, old_len, new_len, flags); in mremap_to()
834 if (IS_ERR(vma)) { in mremap_to()
835 ret = PTR_ERR(vma); in mremap_to()
841 !may_expand_vm(mm, vma->vm_flags, old_len >> PAGE_SHIFT)) { in mremap_to()
849 if (vma->vm_flags & VM_MAYSHARE) in mremap_to()
852 ret = get_unmapped_area(vma->vm_file, new_addr, new_len, vma->vm_pgoff + in mremap_to()
853 ((addr - vma->vm_start) >> PAGE_SHIFT), in mremap_to()
862 ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf, in mremap_to()
869 static int vma_expandable(struct vm_area_struct *vma, unsigned long delta) in vma_expandable() argument
871 unsigned long end = vma->vm_end + delta; in vma_expandable()
872 if (end < vma->vm_end) /* overflow */ in vma_expandable()
874 if (vma->vm_next && vma->vm_next->vm_start < end) /* intersection */ in vma_expandable()
876 if (get_unmapped_area(NULL, vma->vm_start, end - vma->vm_start, in vma_expandable()
894 struct vm_area_struct *vma; in SYSCALL_DEFINE5() local
945 vma = find_vma(mm, addr); in SYSCALL_DEFINE5()
946 if (!vma || vma->vm_start > addr) { in SYSCALL_DEFINE5()
951 if (is_vm_hugetlb_page(vma)) { in SYSCALL_DEFINE5()
952 struct hstate *h __maybe_unused = hstate_vma(vma); in SYSCALL_DEFINE5()
1002 vma = vma_to_resize(addr, old_len, new_len, flags); in SYSCALL_DEFINE5()
1003 if (IS_ERR(vma)) { in SYSCALL_DEFINE5()
1004 ret = PTR_ERR(vma); in SYSCALL_DEFINE5()
1010 if (old_len == vma->vm_end - addr) { in SYSCALL_DEFINE5()
1012 if (vma_expandable(vma, new_len - old_len)) { in SYSCALL_DEFINE5()
1015 if (vma->vm_flags & VM_ACCOUNT) { in SYSCALL_DEFINE5()
1022 if (vma_adjust(vma, vma->vm_start, addr + new_len, in SYSCALL_DEFINE5()
1023 vma->vm_pgoff, NULL)) { in SYSCALL_DEFINE5()
1029 vm_stat_account(mm, vma->vm_flags, pages); in SYSCALL_DEFINE5()
1030 if (vma->vm_flags & VM_LOCKED) { in SYSCALL_DEFINE5()
1047 if (vma->vm_flags & VM_MAYSHARE) in SYSCALL_DEFINE5()
1050 new_addr = get_unmapped_area(vma->vm_file, 0, new_len, in SYSCALL_DEFINE5()
1051 vma->vm_pgoff + in SYSCALL_DEFINE5()
1052 ((addr - vma->vm_start) >> PAGE_SHIFT), in SYSCALL_DEFINE5()
1059 ret = move_vma(vma, addr, old_len, new_len, new_addr, in SYSCALL_DEFINE5()