Lines Matching refs:it
98 nvkm_vmm_trace(struct nvkm_vmm_iter *it, char *buf) in nvkm_vmm_trace() argument
101 for (lvl = it->max; lvl >= 0; lvl--) { in nvkm_vmm_trace()
102 if (lvl >= it->lvl) in nvkm_vmm_trace()
103 buf += sprintf(buf, "%05x:", it->pte[lvl]); in nvkm_vmm_trace()
120 nvkm_vmm_flush_mark(struct nvkm_vmm_iter *it) in nvkm_vmm_flush_mark() argument
122 it->flush = min(it->flush, it->max - it->lvl); in nvkm_vmm_flush_mark()
126 nvkm_vmm_flush(struct nvkm_vmm_iter *it) in nvkm_vmm_flush() argument
128 if (it->flush != NVKM_VMM_LEVELS_MAX) { in nvkm_vmm_flush()
129 if (it->vmm->func->flush) { in nvkm_vmm_flush()
130 TRA(it, "flush: %d", it->flush); in nvkm_vmm_flush()
131 it->vmm->func->flush(it->vmm, it->flush); in nvkm_vmm_flush()
133 it->flush = NVKM_VMM_LEVELS_MAX; in nvkm_vmm_flush()
138 nvkm_vmm_unref_pdes(struct nvkm_vmm_iter *it) in nvkm_vmm_unref_pdes() argument
140 const struct nvkm_vmm_desc *desc = it->desc; in nvkm_vmm_unref_pdes()
141 const int type = desc[it->lvl].type == SPT; in nvkm_vmm_unref_pdes()
142 struct nvkm_vmm_pt *pgd = it->pt[it->lvl + 1]; in nvkm_vmm_unref_pdes()
143 struct nvkm_vmm_pt *pgt = it->pt[it->lvl]; in nvkm_vmm_unref_pdes()
145 struct nvkm_vmm *vmm = it->vmm; in nvkm_vmm_unref_pdes()
146 u32 pdei = it->pte[it->lvl + 1]; in nvkm_vmm_unref_pdes()
149 it->lvl++; in nvkm_vmm_unref_pdes()
151 const struct nvkm_vmm_desc_func *func = desc[it->lvl].func; in nvkm_vmm_unref_pdes()
153 TRA(it, "PDE unmap %s", nvkm_vmm_desc_type(&desc[it->lvl - 1])); in nvkm_vmm_unref_pdes()
181 nvkm_vmm_flush_mark(it); in nvkm_vmm_unref_pdes()
182 nvkm_vmm_flush(it); in nvkm_vmm_unref_pdes()
185 nvkm_vmm_unref_pdes(it); in nvkm_vmm_unref_pdes()
189 TRA(it, "PDE free %s", nvkm_vmm_desc_type(&desc[it->lvl - 1])); in nvkm_vmm_unref_pdes()
193 it->lvl--; in nvkm_vmm_unref_pdes()
197 nvkm_vmm_unref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, in nvkm_vmm_unref_sptes() argument
200 const struct nvkm_vmm_desc *pair = it->page[-1].desc; in nvkm_vmm_unref_sptes()
203 struct nvkm_vmm *vmm = it->vmm; in nvkm_vmm_unref_sptes()
243 TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes); in nvkm_vmm_unref_sptes()
251 TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes); in nvkm_vmm_unref_sptes()
258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) in nvkm_vmm_unref_ptes() argument
260 const struct nvkm_vmm_desc *desc = it->desc; in nvkm_vmm_unref_ptes()
262 struct nvkm_vmm_pt *pgt = it->pt[0]; in nvkm_vmm_unref_ptes()
267 dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes); in nvkm_vmm_unref_ptes()
270 nvkm_vmm_flush_mark(it); in nvkm_vmm_unref_ptes()
271 nvkm_vmm_flush(it); in nvkm_vmm_unref_ptes()
272 desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes); in nvkm_vmm_unref_ptes()
281 nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes); in nvkm_vmm_unref_ptes()
285 it->lvl++; in nvkm_vmm_unref_ptes()
286 TRA(it, "%s empty", nvkm_vmm_desc_type(desc)); in nvkm_vmm_unref_ptes()
287 it->lvl--; in nvkm_vmm_unref_ptes()
288 nvkm_vmm_unref_pdes(it); in nvkm_vmm_unref_ptes()
296 nvkm_vmm_ref_sptes(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgt, in nvkm_vmm_ref_sptes() argument
299 const struct nvkm_vmm_desc *pair = it->page[-1].desc; in nvkm_vmm_ref_sptes()
302 struct nvkm_vmm *vmm = it->vmm; in nvkm_vmm_ref_sptes()
347 TRA(it, "SPTE %05x: U -> S %d PTEs", spti, sptc); in nvkm_vmm_ref_sptes()
350 TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes); in nvkm_vmm_ref_sptes()
357 TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes); in nvkm_vmm_ref_sptes()
364 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) in nvkm_vmm_ref_ptes() argument
366 const struct nvkm_vmm_desc *desc = it->desc; in nvkm_vmm_ref_ptes()
368 struct nvkm_vmm_pt *pgt = it->pt[0]; in nvkm_vmm_ref_ptes()
375 nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes); in nvkm_vmm_ref_ptes()
394 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) in nvkm_vmm_sparse_unref_ptes() argument
396 struct nvkm_vmm_pt *pt = it->pt[0]; in nvkm_vmm_sparse_unref_ptes()
397 if (it->desc->type == PGD) in nvkm_vmm_sparse_unref_ptes()
400 if (it->desc->type == LPT) in nvkm_vmm_sparse_unref_ptes()
402 return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes); in nvkm_vmm_sparse_unref_ptes()
406 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) in nvkm_vmm_sparse_ref_ptes() argument
408 nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes); in nvkm_vmm_sparse_ref_ptes()
409 return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes); in nvkm_vmm_sparse_ref_ptes()
413 nvkm_vmm_ref_hwpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) in nvkm_vmm_ref_hwpt() argument
415 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1]; in nvkm_vmm_ref_hwpt()
419 struct nvkm_vmm *vmm = it->vmm; in nvkm_vmm_ref_hwpt()
430 it->lvl--; in nvkm_vmm_ref_hwpt()
431 nvkm_vmm_unref_pdes(it); in nvkm_vmm_ref_hwpt()
477 TRA(it, "PDE write %s", nvkm_vmm_desc_type(desc)); in nvkm_vmm_ref_hwpt()
478 it->desc[it->lvl].func->pde(it->vmm, pgd, pdei); in nvkm_vmm_ref_hwpt()
479 nvkm_vmm_flush_mark(it); in nvkm_vmm_ref_hwpt()
484 nvkm_vmm_ref_swpt(struct nvkm_vmm_iter *it, struct nvkm_vmm_pt *pgd, u32 pdei) in nvkm_vmm_ref_swpt() argument
486 const struct nvkm_vmm_desc *desc = &it->desc[it->lvl - 1]; in nvkm_vmm_ref_swpt()
489 pgt = nvkm_vmm_pt_new(desc, NVKM_VMM_PDE_SPARSED(pgt), it->page); in nvkm_vmm_ref_swpt()
492 nvkm_vmm_unref_pdes(it); in nvkm_vmm_ref_swpt()
508 struct nvkm_vmm_iter it; in nvkm_vmm_iter() local
511 it.page = page; in nvkm_vmm_iter()
512 it.desc = desc; in nvkm_vmm_iter()
513 it.vmm = vmm; in nvkm_vmm_iter()
514 it.cnt = size >> page->shift; in nvkm_vmm_iter()
515 it.flush = NVKM_VMM_LEVELS_MAX; in nvkm_vmm_iter()
518 for (it.lvl = 0; desc[it.lvl].bits; it.lvl++) { in nvkm_vmm_iter()
519 it.pte[it.lvl] = bits & ((1 << desc[it.lvl].bits) - 1); in nvkm_vmm_iter()
520 bits >>= desc[it.lvl].bits; in nvkm_vmm_iter()
522 it.max = --it.lvl; in nvkm_vmm_iter()
523 it.pt[it.max] = vmm->pd; in nvkm_vmm_iter()
525 it.lvl = 0; in nvkm_vmm_iter()
526 TRA(&it, "%s: %016llx %016llx %d %lld PTEs", name, in nvkm_vmm_iter()
527 addr, size, page->shift, it.cnt); in nvkm_vmm_iter()
528 it.lvl = it.max; in nvkm_vmm_iter()
531 while (it.cnt) { in nvkm_vmm_iter()
532 struct nvkm_vmm_pt *pgt = it.pt[it.lvl]; in nvkm_vmm_iter()
535 const u32 ptei = it.pte[0]; in nvkm_vmm_iter()
536 const u32 ptes = min_t(u64, it.cnt, pten - ptei); in nvkm_vmm_iter()
539 for (; it.lvl; it.lvl--) { in nvkm_vmm_iter()
540 const u32 pdei = it.pte[it.lvl]; in nvkm_vmm_iter()
545 if (!nvkm_vmm_ref_swpt(&it, pgd, pdei)) in nvkm_vmm_iter()
548 it.pt[it.lvl - 1] = pgt = pgd->pde[pdei]; in nvkm_vmm_iter()
556 if (ref && !pgt->refs[desc[it.lvl - 1].type == SPT]) { in nvkm_vmm_iter()
557 if (!nvkm_vmm_ref_hwpt(&it, pgd, pdei)) in nvkm_vmm_iter()
563 if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) { in nvkm_vmm_iter()
570 nvkm_vmm_flush_mark(&it); in nvkm_vmm_iter()
575 it.pte[it.lvl] += ptes; in nvkm_vmm_iter()
576 it.cnt -= ptes; in nvkm_vmm_iter()
577 if (it.cnt) { in nvkm_vmm_iter()
578 while (it.pte[it.lvl] == (1 << desc[it.lvl].bits)) { in nvkm_vmm_iter()
579 it.pte[it.lvl++] = 0; in nvkm_vmm_iter()
580 it.pte[it.lvl]++; in nvkm_vmm_iter()
585 nvkm_vmm_flush(&it); in nvkm_vmm_iter()
592 addr = it.pte[it.max--]; in nvkm_vmm_iter()
594 addr = addr << desc[it.max].bits; in nvkm_vmm_iter()
595 addr |= it.pte[it.max]; in nvkm_vmm_iter()
596 } while (it.max--); in nvkm_vmm_iter()
1801 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes) in nvkm_vmm_boot_ptes() argument
1803 const struct nvkm_vmm_desc *desc = it->desc; in nvkm_vmm_boot_ptes()
1805 nvkm_memory_boot(it->pt[0]->pt[type]->memory, it->vmm); in nvkm_vmm_boot_ptes()