Lines Matching refs:cpa
275 static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx) in __cpa_addr() argument
277 if (cpa->flags & CPA_PAGES_ARRAY) { in __cpa_addr()
278 struct page *page = cpa->pages[idx]; in __cpa_addr()
286 if (cpa->flags & CPA_ARRAY) in __cpa_addr()
287 return cpa->vaddr[idx]; in __cpa_addr()
289 return *cpa->vaddr + idx * PAGE_SIZE; in __cpa_addr()
356 struct cpa_data *cpa = data; in __cpa_flush_tlb() local
359 for (i = 0; i < cpa->numpages; i++) in __cpa_flush_tlb()
360 flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i))); in __cpa_flush_tlb()
365 struct cpa_data *cpa = data; in cpa_flush() local
375 if (cpa->force_flush_all || cpa->numpages > tlb_single_page_flush_ceiling) in cpa_flush()
378 on_each_cpu(__cpa_flush_tlb, cpa, 1); in cpa_flush()
384 for (i = 0; i < cpa->numpages; i++) { in cpa_flush()
385 unsigned long addr = __cpa_addr(cpa, i); in cpa_flush()
652 static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address, in _lookup_address_cpa() argument
655 if (cpa->pgd) in _lookup_address_cpa()
656 return lookup_address_in_pgd(cpa->pgd + pgd_index(address), in _lookup_address_cpa()
777 struct cpa_data *cpa) in __should_split_large_page() argument
788 tmp = _lookup_address_cpa(cpa, address, &level); in __should_split_large_page()
816 if (numpages < cpa->numpages) in __should_split_large_page()
817 cpa->numpages = numpages; in __should_split_large_page()
828 pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr); in __should_split_large_page()
829 pgprot_val(req_prot) |= pgprot_val(cpa->mask_set); in __should_split_large_page()
846 cpa->pfn = pfn; in __should_split_large_page()
868 cpa->force_static_prot = 1; in __should_split_large_page()
889 if (address != lpaddr || cpa->numpages != numpages) in __should_split_large_page()
914 cpa->flags |= CPA_FLUSHTLB; in __should_split_large_page()
920 struct cpa_data *cpa) in should_split_large_page() argument
924 if (cpa->force_split) in should_split_large_page()
928 do_split = __should_split_large_page(kpte, address, cpa); in should_split_large_page()
934 static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn, in split_set_pte() argument
945 if (!cpa->force_static_prot) in split_set_pte()
971 __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, in __split_large_page() argument
985 tmp = _lookup_address_cpa(cpa, address, &level); in __split_large_page()
1033 split_set_pte(cpa, pbase + i, pfn, ref_prot, lpaddr, lpinc); in __split_large_page()
1075 static int split_large_page(struct cpa_data *cpa, pte_t *kpte, in split_large_page() argument
1088 if (__split_large_page(cpa, kpte, address, base)) in split_large_page()
1251 static void populate_pte(struct cpa_data *cpa, in populate_pte() argument
1262 set_pte(pte, pfn_pte(cpa->pfn, pgprot)); in populate_pte()
1265 cpa->pfn++; in populate_pte()
1270 static long populate_pmd(struct cpa_data *cpa, in populate_pmd() argument
1297 populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot); in populate_pmd()
1321 set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn, in populate_pmd()
1325 cpa->pfn += PMD_SIZE >> PAGE_SHIFT; in populate_pmd()
1338 populate_pte(cpa, start, end, num_pages - cur_pages, in populate_pmd()
1344 static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d, in populate_pud() argument
1352 end = start + (cpa->numpages << PAGE_SHIFT); in populate_pud()
1364 cur_pages = min_t(int, (int)cpa->numpages, cur_pages); in populate_pud()
1375 cur_pages = populate_pmd(cpa, start, pre_end, cur_pages, in populate_pud()
1384 if (cpa->numpages == cur_pages) in populate_pud()
1394 set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn, in populate_pud()
1398 cpa->pfn += PUD_SIZE >> PAGE_SHIFT; in populate_pud()
1412 tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages, in populate_pud()
1426 static int populate_pgd(struct cpa_data *cpa, unsigned long addr) in populate_pgd() argument
1434 pgd_entry = cpa->pgd + pgd_index(addr); in populate_pgd()
1456 pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr); in populate_pgd()
1457 pgprot_val(pgprot) |= pgprot_val(cpa->mask_set); in populate_pgd()
1459 ret = populate_pud(cpa, addr, p4d, pgprot); in populate_pgd()
1467 addr + (cpa->numpages << PAGE_SHIFT)); in populate_pgd()
1471 cpa->numpages = ret; in populate_pgd()
1475 static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr, in __cpa_process_fault() argument
1478 if (cpa->pgd) { in __cpa_process_fault()
1484 return populate_pgd(cpa, vaddr); in __cpa_process_fault()
1491 cpa->numpages = 1; in __cpa_process_fault()
1504 cpa->numpages = 1; in __cpa_process_fault()
1505 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT; in __cpa_process_fault()
1508 } else if (__cpa_pfn_in_highmap(cpa->pfn)) { in __cpa_process_fault()
1514 *cpa->vaddr); in __cpa_process_fault()
1520 static int __change_page_attr(struct cpa_data *cpa, int primary) in __change_page_attr() argument
1527 address = __cpa_addr(cpa, cpa->curpage); in __change_page_attr()
1529 kpte = _lookup_address_cpa(cpa, address, &level); in __change_page_attr()
1531 return __cpa_process_fault(cpa, address, primary); in __change_page_attr()
1535 return __cpa_process_fault(cpa, address, primary); in __change_page_attr()
1542 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); in __change_page_attr()
1543 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set); in __change_page_attr()
1558 cpa->pfn = pfn; in __change_page_attr()
1564 cpa->flags |= CPA_FLUSHTLB; in __change_page_attr()
1566 cpa->numpages = 1; in __change_page_attr()
1574 do_split = should_split_large_page(kpte, address, cpa); in __change_page_attr()
1586 err = split_large_page(cpa, kpte, address); in __change_page_attr()
1593 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias);
1595 static int cpa_process_alias(struct cpa_data *cpa) in cpa_process_alias() argument
1598 unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT); in cpa_process_alias()
1602 if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1)) in cpa_process_alias()
1609 vaddr = __cpa_addr(cpa, cpa->curpage); in cpa_process_alias()
1613 alias_cpa = *cpa; in cpa_process_alias()
1618 cpa->force_flush_all = 1; in cpa_process_alias()
1632 __cpa_pfn_in_highmap(cpa->pfn)) { in cpa_process_alias()
1633 unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + in cpa_process_alias()
1635 alias_cpa = *cpa; in cpa_process_alias()
1640 cpa->force_flush_all = 1; in cpa_process_alias()
1652 static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias) in __change_page_attr_set_clr() argument
1654 unsigned long numpages = cpa->numpages; in __change_page_attr_set_clr()
1663 cpa->numpages = rempages; in __change_page_attr_set_clr()
1665 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) in __change_page_attr_set_clr()
1666 cpa->numpages = 1; in __change_page_attr_set_clr()
1670 ret = __change_page_attr(cpa, checkalias); in __change_page_attr_set_clr()
1677 ret = cpa_process_alias(cpa); in __change_page_attr_set_clr()
1687 BUG_ON(cpa->numpages > rempages || !cpa->numpages); in __change_page_attr_set_clr()
1688 rempages -= cpa->numpages; in __change_page_attr_set_clr()
1689 cpa->curpage += cpa->numpages; in __change_page_attr_set_clr()
1694 cpa->numpages = numpages; in __change_page_attr_set_clr()
1703 struct cpa_data cpa; in change_page_attr_set_clr() local
1706 memset(&cpa, 0, sizeof(cpa)); in change_page_attr_set_clr()
1745 cpa.vaddr = addr; in change_page_attr_set_clr()
1746 cpa.pages = pages; in change_page_attr_set_clr()
1747 cpa.numpages = numpages; in change_page_attr_set_clr()
1748 cpa.mask_set = mask_set; in change_page_attr_set_clr()
1749 cpa.mask_clr = mask_clr; in change_page_attr_set_clr()
1750 cpa.flags = 0; in change_page_attr_set_clr()
1751 cpa.curpage = 0; in change_page_attr_set_clr()
1752 cpa.force_split = force_split; in change_page_attr_set_clr()
1755 cpa.flags |= in_flag; in change_page_attr_set_clr()
1763 ret = __change_page_attr_set_clr(&cpa, checkalias); in change_page_attr_set_clr()
1768 if (!(cpa.flags & CPA_FLUSHTLB)) in change_page_attr_set_clr()
1785 cpa_flush(&cpa, cache); in change_page_attr_set_clr()
1992 struct cpa_data cpa; in __set_memory_enc_pgtable() local
1999 memset(&cpa, 0, sizeof(cpa)); in __set_memory_enc_pgtable()
2000 cpa.vaddr = &addr; in __set_memory_enc_pgtable()
2001 cpa.numpages = numpages; in __set_memory_enc_pgtable()
2002 cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0); in __set_memory_enc_pgtable()
2003 cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC); in __set_memory_enc_pgtable()
2004 cpa.pgd = init_mm.pgd; in __set_memory_enc_pgtable()
2013 cpa_flush(&cpa, !this_cpu_has(X86_FEATURE_SME_COHERENT)); in __set_memory_enc_pgtable()
2015 ret = __change_page_attr_set_clr(&cpa, 1); in __set_memory_enc_pgtable()
2024 cpa_flush(&cpa, 0); in __set_memory_enc_pgtable()
2180 struct cpa_data cpa = { .vaddr = &tempaddr, in __set_pages_p() local
2193 return __change_page_attr_set_clr(&cpa, 0); in __set_pages_p()
2199 struct cpa_data cpa = { .vaddr = &tempaddr, in __set_pages_np() local
2212 return __change_page_attr_set_clr(&cpa, 0); in __set_pages_np()
2276 struct cpa_data cpa = { in kernel_map_pages_in_pgd() local
2292 cpa.mask_clr = pgprot_encrypted(cpa.mask_clr); in kernel_map_pages_in_pgd()
2294 cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags); in kernel_map_pages_in_pgd()
2296 retval = __change_page_attr_set_clr(&cpa, 0); in kernel_map_pages_in_pgd()
2319 struct cpa_data cpa = { in kernel_unmap_pages_in_pgd() local
2331 retval = __change_page_attr_set_clr(&cpa, 0); in kernel_unmap_pages_in_pgd()