Lines Matching refs:pte
50 static u64 *first_pte_l7(u64 *pte, unsigned long *page_size, in first_pte_l7() argument
56 pg_size = PTE_PAGE_SIZE(*pte); in first_pte_l7()
59 fpte = (u64 *)(((unsigned long)pte) & pte_mask); in first_pte_l7()
183 u64 *pte; in increase_address_space() local
185 pte = (void *)get_zeroed_page(gfp); in increase_address_space()
186 if (!pte) in increase_address_space()
198 *pte = PM_LEVEL_PDE(domain->iop.mode, iommu_virt_to_phys(domain->iop.root)); in increase_address_space()
200 domain->iop.root = pte; in increase_address_space()
209 amd_iommu_domain_set_pgtable(domain, pte, domain->iop.mode); in increase_address_space()
211 pte = NULL; in increase_address_space()
216 free_page((unsigned long)pte); in increase_address_space()
229 u64 *pte, *page; in alloc_pte() local
244 pte = &domain->iop.root[PM_LEVEL_INDEX(level, address)]; in alloc_pte()
252 __pte = *pte; in alloc_pte()
264 lpte = first_pte_l7(pte, NULL, &count); in alloc_pte()
287 if (cmpxchg64(pte, __pte, __npte) != __pte) in alloc_pte()
301 pte = IOMMU_PTE_PAGE(__pte); in alloc_pte()
304 *pte_page = pte; in alloc_pte()
306 pte = &pte[PM_LEVEL_INDEX(level, address)]; in alloc_pte()
309 return pte; in alloc_pte()
321 u64 *pte; in fetch_pte() local
329 pte = &pgtable->root[PM_LEVEL_INDEX(level, address)]; in fetch_pte()
335 if (!IOMMU_PTE_PRESENT(*pte)) in fetch_pte()
339 if (PM_PTE_LEVEL(*pte) == 7 || in fetch_pte()
340 PM_PTE_LEVEL(*pte) == 0) in fetch_pte()
344 if (PM_PTE_LEVEL(*pte) != level) in fetch_pte()
350 pte = IOMMU_PTE_PAGE(*pte); in fetch_pte()
351 pte = &pte[PM_LEVEL_INDEX(level, address)]; in fetch_pte()
359 if (PM_PTE_LEVEL(*pte) == PAGE_MODE_7_LEVEL) in fetch_pte()
360 pte = first_pte_l7(pte, page_size, NULL); in fetch_pte()
362 return pte; in fetch_pte()
365 static struct page *free_clear_pte(u64 *pte, u64 pteval, struct page *freelist) in free_clear_pte() argument
370 while (cmpxchg64(pte, pteval, 0) != pteval) { in free_clear_pte()
372 pteval = *pte; in free_clear_pte()
397 u64 __pte, *pte; in iommu_v1_map_page() local
408 pte = alloc_pte(dom, iova, size, NULL, gfp, &updated); in iommu_v1_map_page()
411 if (!pte) in iommu_v1_map_page()
415 freelist = free_clear_pte(&pte[i], pte[i], freelist); in iommu_v1_map_page()
432 pte[i] = __pte; in iommu_v1_map_page()
465 u64 *pte; in iommu_v1_unmap_page() local
472 pte = fetch_pte(pgtable, iova, &unmap_size); in iommu_v1_unmap_page()
473 if (pte) { in iommu_v1_unmap_page()
478 pte[i] = 0ULL; in iommu_v1_unmap_page()
494 u64 *pte, __pte; in iommu_v1_iova_to_phys() local
496 pte = fetch_pte(pgtable, iova, &pte_pgsize); in iommu_v1_iova_to_phys()
498 if (!pte || !IOMMU_PTE_PRESENT(*pte)) in iommu_v1_iova_to_phys()
502 __pte = __sme_clr(*pte & PM_ADDR_MASK); in iommu_v1_iova_to_phys()