Lines Matching refs:vaddr
48 struct vm_area_struct *gru_find_vma(unsigned long vaddr) in gru_find_vma() argument
52 vma = vma_lookup(current->mm, vaddr); in gru_find_vma()
66 static struct gru_thread_state *gru_find_lock_gts(unsigned long vaddr) in gru_find_lock_gts() argument
73 vma = gru_find_vma(vaddr); in gru_find_lock_gts()
75 gts = gru_find_thread_state(vma, TSID(vaddr, vma)); in gru_find_lock_gts()
83 static struct gru_thread_state *gru_alloc_locked_gts(unsigned long vaddr) in gru_alloc_locked_gts() argument
90 vma = gru_find_vma(vaddr); in gru_alloc_locked_gts()
94 gts = gru_alloc_thread_state(vma, TSID(vaddr, vma)); in gru_alloc_locked_gts()
178 unsigned long vaddr, int write, in non_atomic_pte_lookup() argument
188 if (get_user_pages(vaddr, 1, write ? FOLL_WRITE : 0, &page, NULL) <= 0) in non_atomic_pte_lookup()
205 static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, in atomic_pte_lookup() argument
214 pgdp = pgd_offset(vma->vm_mm, vaddr); in atomic_pte_lookup()
218 p4dp = p4d_offset(pgdp, vaddr); in atomic_pte_lookup()
222 pudp = pud_offset(p4dp, vaddr); in atomic_pte_lookup()
226 pmdp = pmd_offset(pudp, vaddr); in atomic_pte_lookup()
234 pte = *pte_offset_kernel(pmdp, vaddr); in atomic_pte_lookup()
252 static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr, in gru_vtop() argument
260 vma = find_vma(mm, vaddr); in gru_vtop()
269 ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps); in gru_vtop()
273 if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps)) in gru_vtop()
316 unsigned long vaddr = 0, gpa; in gru_preload_tlb() local
323 vaddr = fault_vaddr + GRU_CACHE_LINE_BYTES * cbe->cbe_src_cl - 1; in gru_preload_tlb()
325 vaddr = fault_vaddr + (1 << cbe->xtypecpy) * cbe->cbe_nelemcur - 1; in gru_preload_tlb()
328 vaddr &= PAGE_MASK; in gru_preload_tlb()
329 vaddr = min(vaddr, fault_vaddr + tlb_preload_count * PAGE_SIZE); in gru_preload_tlb()
331 while (vaddr > fault_vaddr) { in gru_preload_tlb()
332 ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); in gru_preload_tlb()
333 if (ret || tfh_write_only(tfh, gpa, GAA_RAM, vaddr, asid, write, in gru_preload_tlb()
339 vaddr, asid, write, pageshift, gpa); in gru_preload_tlb()
340 vaddr -= PAGE_SIZE; in gru_preload_tlb()
363 unsigned long gpa = 0, vaddr = 0; in gru_try_dropin() local
398 vaddr = tfh->missvaddr; in gru_try_dropin()
413 ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); in gru_try_dropin()
428 gru_preload_tlb(gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe); in gru_try_dropin()
434 tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write, in gru_try_dropin()
439 atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, vaddr, asid, in gru_try_dropin()
447 gru_dbg(grudev, "FAILED no_asid tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr); in gru_try_dropin()
460 gru_dbg(grudev, "FAILED upm tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr); in gru_try_dropin()
497 gru_dbg(grudev, "FAILED inval tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr); in gru_try_dropin()
509 tfh, vaddr); in gru_try_dropin()
802 req.vaddr, req.len); in gru_user_flush_tlb()
810 gru_flush_tlb_range(gms, req.vaddr, req.len); in gru_user_flush_tlb()