Lines Matching refs:hpage
862 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) in khugepaged_prealloc_page() argument
864 if (IS_ERR(*hpage)) { in khugepaged_prealloc_page()
869 *hpage = NULL; in khugepaged_prealloc_page()
871 } else if (*hpage) { in khugepaged_prealloc_page()
872 put_page(*hpage); in khugepaged_prealloc_page()
873 *hpage = NULL; in khugepaged_prealloc_page()
880 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) in khugepaged_alloc_page() argument
882 VM_BUG_ON_PAGE(*hpage, *hpage); in khugepaged_alloc_page()
884 *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER); in khugepaged_alloc_page()
885 if (unlikely(!*hpage)) { in khugepaged_alloc_page()
887 *hpage = ERR_PTR(-ENOMEM); in khugepaged_alloc_page()
891 prep_transhuge_page(*hpage); in khugepaged_alloc_page()
893 return *hpage; in khugepaged_alloc_page()
914 struct page *hpage; in khugepaged_alloc_hugepage() local
917 hpage = alloc_khugepaged_hugepage(); in khugepaged_alloc_hugepage()
918 if (!hpage) { in khugepaged_alloc_hugepage()
927 } while (unlikely(!hpage) && likely(khugepaged_enabled())); in khugepaged_alloc_hugepage()
929 return hpage; in khugepaged_alloc_hugepage()
932 static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) in khugepaged_prealloc_page() argument
941 if (*hpage && page_count(*hpage) > 1) { in khugepaged_prealloc_page()
942 put_page(*hpage); in khugepaged_prealloc_page()
943 *hpage = NULL; in khugepaged_prealloc_page()
946 if (!*hpage) in khugepaged_prealloc_page()
947 *hpage = khugepaged_alloc_hugepage(wait); in khugepaged_prealloc_page()
949 if (unlikely(!*hpage)) in khugepaged_prealloc_page()
956 khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) in khugepaged_alloc_page() argument
958 VM_BUG_ON(!*hpage); in khugepaged_alloc_page()
960 return *hpage; in khugepaged_alloc_page()
1061 struct page **hpage, in collapse_huge_page() argument
1087 new_page = khugepaged_alloc_page(hpage, gfp, node); in collapse_huge_page()
1209 *hpage = NULL; in collapse_huge_page()
1216 if (!IS_ERR_OR_NULL(*hpage)) in collapse_huge_page()
1217 mem_cgroup_uncharge(page_folio(*hpage)); in collapse_huge_page()
1225 struct page **hpage) in khugepaged_scan_pmd() argument
1370 collapse_huge_page(mm, address, hpage, node, in khugepaged_scan_pmd()
1437 struct page *hpage; in collapse_pte_mapped_thp() local
1457 hpage = find_lock_page(vma->vm_file->f_mapping, in collapse_pte_mapped_thp()
1459 if (!hpage) in collapse_pte_mapped_thp()
1462 if (!PageHead(hpage)) in collapse_pte_mapped_thp()
1490 if (hpage + i != page) in collapse_pte_mapped_thp()
1510 page_ref_sub(hpage, count); in collapse_pte_mapped_thp()
1511 add_mm_counter(vma->vm_mm, mm_counter_file(hpage), -count); in collapse_pte_mapped_thp()
1522 unlock_page(hpage); in collapse_pte_mapped_thp()
1523 put_page(hpage); in collapse_pte_mapped_thp()
1640 struct page **hpage, int node) in collapse_file() argument
1658 new_page = khugepaged_alloc_page(hpage, gfp, node); in collapse_file()
1938 *hpage = NULL; in collapse_file()
1985 if (!IS_ERR_OR_NULL(*hpage)) in collapse_file()
1986 mem_cgroup_uncharge(page_folio(*hpage)); in collapse_file()
1991 struct file *file, pgoff_t start, struct page **hpage) in khugepaged_scan_file() argument
2059 collapse_file(mm, file, start, hpage, node); in khugepaged_scan_file()
2067 struct file *file, pgoff_t start, struct page **hpage) in khugepaged_scan_file() argument
2078 struct page **hpage) in khugepaged_scan_mm_slot() argument
2154 khugepaged_scan_file(mm, file, pgoff, hpage); in khugepaged_scan_mm_slot()
2159 hpage); in khugepaged_scan_mm_slot()
2217 struct page *hpage = NULL; in khugepaged_do_scan() local
2225 if (!khugepaged_prealloc_page(&hpage, &wait)) in khugepaged_do_scan()
2239 &hpage); in khugepaged_do_scan()
2245 if (!IS_ERR_OR_NULL(hpage)) in khugepaged_do_scan()
2246 put_page(hpage); in khugepaged_do_scan()