Lines Matching refs:lvl
51 #define _ARM_V7S_LVL_BITS(lvl, cfg) ((lvl) == 1 ? ((cfg)->ias - 20) : 8) argument
52 #define ARM_V7S_LVL_SHIFT(lvl) ((lvl) == 1 ? 20 : 12) argument
55 #define ARM_V7S_PTES_PER_LVL(lvl, cfg) (1 << _ARM_V7S_LVL_BITS(lvl, cfg)) argument
56 #define ARM_V7S_TABLE_SIZE(lvl, cfg) \ argument
57 (ARM_V7S_PTES_PER_LVL(lvl, cfg) * sizeof(arm_v7s_iopte))
59 #define ARM_V7S_BLOCK_SIZE(lvl) (1UL << ARM_V7S_LVL_SHIFT(lvl)) argument
60 #define ARM_V7S_LVL_MASK(lvl) ((u32)(~0U << ARM_V7S_LVL_SHIFT(lvl))) argument
62 #define _ARM_V7S_IDX_MASK(lvl, cfg) (ARM_V7S_PTES_PER_LVL(lvl, cfg) - 1) argument
63 #define ARM_V7S_LVL_IDX(addr, lvl, cfg) ({ \ argument
64 int _l = lvl; \
84 #define ARM_V7S_PTE_IS_TABLE(pte, lvl) \ argument
85 ((lvl) == 1 && (((pte) & 0x3) == ARM_V7S_PTE_TYPE_TABLE))
88 #define ARM_V7S_ATTR_XN(lvl) BIT(4 * (2 - (lvl))) argument
102 #define ARM_V7S_ATTR_SHIFT(lvl) (16 - (lvl) * 6) argument
172 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl);
185 static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl, in paddr_to_iopte() argument
188 arm_v7s_iopte pte = paddr & ARM_V7S_LVL_MASK(lvl); in paddr_to_iopte()
202 static phys_addr_t iopte_to_paddr(arm_v7s_iopte pte, int lvl, in iopte_to_paddr() argument
208 if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) in iopte_to_paddr()
210 else if (arm_v7s_pte_is_cont(pte, lvl)) in iopte_to_paddr()
211 mask = ARM_V7S_LVL_MASK(lvl) * ARM_V7S_CONT_PAGES; in iopte_to_paddr()
213 mask = ARM_V7S_LVL_MASK(lvl); in iopte_to_paddr()
228 static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl, in iopte_deref() argument
231 return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg)); in iopte_deref()
234 static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, in __arm_v7s_alloc_table() argument
241 size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg); in __arm_v7s_alloc_table()
244 if (lvl == 1) in __arm_v7s_alloc_table()
247 else if (lvl == 2) in __arm_v7s_alloc_table()
267 if (lvl == 2) in __arm_v7s_alloc_table()
275 if (lvl == 1) in __arm_v7s_alloc_table()
282 static void __arm_v7s_free_table(void *table, int lvl, in __arm_v7s_free_table() argument
287 size_t size = ARM_V7S_TABLE_SIZE(lvl, cfg); in __arm_v7s_free_table()
292 if (lvl == 1) in __arm_v7s_free_table()
318 static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl, in arm_v7s_prot_to_pte() argument
333 pte <<= ARM_V7S_ATTR_SHIFT(lvl); in arm_v7s_prot_to_pte()
336 pte |= ARM_V7S_ATTR_XN(lvl); in arm_v7s_prot_to_pte()
343 if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)) in arm_v7s_prot_to_pte()
349 static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl) in arm_v7s_pte_to_prot() argument
352 arm_v7s_iopte attr = pte >> ARM_V7S_ATTR_SHIFT(lvl); in arm_v7s_pte_to_prot()
362 if (pte & ARM_V7S_ATTR_XN(lvl)) in arm_v7s_pte_to_prot()
368 static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl) in arm_v7s_pte_to_cont() argument
370 if (lvl == 1) { in arm_v7s_pte_to_cont()
372 } else if (lvl == 2) { in arm_v7s_pte_to_cont()
373 arm_v7s_iopte xn = pte & ARM_V7S_ATTR_XN(lvl); in arm_v7s_pte_to_cont()
384 static arm_v7s_iopte arm_v7s_cont_to_pte(arm_v7s_iopte pte, int lvl) in arm_v7s_cont_to_pte() argument
386 if (lvl == 1) { in arm_v7s_cont_to_pte()
388 } else if (lvl == 2) { in arm_v7s_cont_to_pte()
401 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl) in arm_v7s_pte_is_cont() argument
403 if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl)) in arm_v7s_pte_is_cont()
405 else if (lvl == 2) in arm_v7s_pte_is_cont()
416 int lvl, int num_entries, arm_v7s_iopte *ptep) in arm_v7s_init_pte() argument
423 if (ARM_V7S_PTE_IS_TABLE(ptep[i], lvl)) { in arm_v7s_init_pte()
429 size_t sz = ARM_V7S_BLOCK_SIZE(lvl); in arm_v7s_init_pte()
431 tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl, cfg); in arm_v7s_init_pte()
433 sz, lvl, tblp) != sz)) in arm_v7s_init_pte()
441 pte = arm_v7s_prot_to_pte(prot, lvl, cfg); in arm_v7s_init_pte()
443 pte = arm_v7s_pte_to_cont(pte, lvl); in arm_v7s_init_pte()
445 pte |= paddr_to_iopte(paddr, lvl, cfg); in arm_v7s_init_pte()
477 int lvl, arm_v7s_iopte *ptep, gfp_t gfp) in __arm_v7s_map() argument
481 int num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); in __arm_v7s_map()
484 ptep += ARM_V7S_LVL_IDX(iova, lvl, cfg); in __arm_v7s_map()
489 lvl, num_entries, ptep); in __arm_v7s_map()
492 if (WARN_ON(lvl == 2)) in __arm_v7s_map()
498 cptep = __arm_v7s_alloc_table(lvl + 1, gfp, data); in __arm_v7s_map()
504 __arm_v7s_free_table(cptep, lvl + 1, data); in __arm_v7s_map()
510 if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) { in __arm_v7s_map()
511 cptep = iopte_deref(pte, lvl, data); in __arm_v7s_map()
519 return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp); in __arm_v7s_map()
581 unsigned long iova, int idx, int lvl, in arm_v7s_split_cont() argument
586 size_t size = ARM_V7S_BLOCK_SIZE(lvl); in arm_v7s_split_cont()
591 if (!arm_v7s_pte_is_cont(pte, lvl)) in arm_v7s_split_cont()
595 pte = arm_v7s_cont_to_pte(pte, lvl); in arm_v7s_split_cont()
653 unsigned long iova, size_t size, int lvl, in __arm_v7s_unmap() argument
658 int idx, i = 0, num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); in __arm_v7s_unmap()
661 if (WARN_ON(lvl > 2)) in __arm_v7s_unmap()
664 idx = ARM_V7S_LVL_IDX(iova, lvl, &iop->cfg); in __arm_v7s_unmap()
682 if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) { in __arm_v7s_unmap()
686 pte[0] = arm_v7s_split_cont(data, iova, idx, lvl, ptep); in __arm_v7s_unmap()
692 size_t blk_size = ARM_V7S_BLOCK_SIZE(lvl); in __arm_v7s_unmap()
697 if (ARM_V7S_PTE_IS_TABLE(pte[i], lvl)) { in __arm_v7s_unmap()
700 ARM_V7S_BLOCK_SIZE(lvl + 1)); in __arm_v7s_unmap()
701 ptep = iopte_deref(pte[i], lvl, data); in __arm_v7s_unmap()
702 __arm_v7s_free_table(ptep, lvl + 1, data); in __arm_v7s_unmap()
709 } else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) { in __arm_v7s_unmap()
719 ptep = iopte_deref(pte[0], lvl, data); in __arm_v7s_unmap()
720 return __arm_v7s_unmap(data, gather, iova, size, lvl + 1, ptep); in __arm_v7s_unmap()
756 int lvl = 0; in arm_v7s_iova_to_phys() local
760 ptep += ARM_V7S_LVL_IDX(iova, ++lvl, &data->iop.cfg); in arm_v7s_iova_to_phys()
762 ptep = iopte_deref(pte, lvl, data); in arm_v7s_iova_to_phys()
763 } while (ARM_V7S_PTE_IS_TABLE(pte, lvl)); in arm_v7s_iova_to_phys()
768 mask = ARM_V7S_LVL_MASK(lvl); in arm_v7s_iova_to_phys()
769 if (arm_v7s_pte_is_cont(pte, lvl)) in arm_v7s_iova_to_phys()
771 return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask); in arm_v7s_iova_to_phys()