Lines Matching refs:umem
26 static void xdp_umem_unpin_pages(struct xdp_umem *umem) in xdp_umem_unpin_pages() argument
28 unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true); in xdp_umem_unpin_pages()
30 kvfree(umem->pgs); in xdp_umem_unpin_pages()
31 umem->pgs = NULL; in xdp_umem_unpin_pages()
34 static void xdp_umem_unaccount_pages(struct xdp_umem *umem) in xdp_umem_unaccount_pages() argument
36 if (umem->user) { in xdp_umem_unaccount_pages()
37 atomic_long_sub(umem->npgs, &umem->user->locked_vm); in xdp_umem_unaccount_pages()
38 free_uid(umem->user); in xdp_umem_unaccount_pages()
42 static void xdp_umem_addr_unmap(struct xdp_umem *umem) in xdp_umem_addr_unmap() argument
44 vunmap(umem->addrs); in xdp_umem_addr_unmap()
45 umem->addrs = NULL; in xdp_umem_addr_unmap()
48 static int xdp_umem_addr_map(struct xdp_umem *umem, struct page **pages, in xdp_umem_addr_map() argument
51 umem->addrs = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL); in xdp_umem_addr_map()
52 if (!umem->addrs) in xdp_umem_addr_map()
57 static void xdp_umem_release(struct xdp_umem *umem) in xdp_umem_release() argument
59 umem->zc = false; in xdp_umem_release()
60 ida_simple_remove(&umem_ida, umem->id); in xdp_umem_release()
62 xdp_umem_addr_unmap(umem); in xdp_umem_release()
63 xdp_umem_unpin_pages(umem); in xdp_umem_release()
65 xdp_umem_unaccount_pages(umem); in xdp_umem_release()
66 kfree(umem); in xdp_umem_release()
71 struct xdp_umem *umem = container_of(work, struct xdp_umem, work); in xdp_umem_release_deferred() local
73 xdp_umem_release(umem); in xdp_umem_release_deferred()
76 void xdp_get_umem(struct xdp_umem *umem) in xdp_get_umem() argument
78 refcount_inc(&umem->users); in xdp_get_umem()
81 void xdp_put_umem(struct xdp_umem *umem, bool defer_cleanup) in xdp_put_umem() argument
83 if (!umem) in xdp_put_umem()
86 if (refcount_dec_and_test(&umem->users)) { in xdp_put_umem()
88 INIT_WORK(&umem->work, xdp_umem_release_deferred); in xdp_put_umem()
89 schedule_work(&umem->work); in xdp_put_umem()
91 xdp_umem_release(umem); in xdp_put_umem()
96 static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address) in xdp_umem_pin_pages() argument
102 umem->pgs = kvcalloc(umem->npgs, sizeof(*umem->pgs), GFP_KERNEL | __GFP_NOWARN); in xdp_umem_pin_pages()
103 if (!umem->pgs) in xdp_umem_pin_pages()
107 npgs = pin_user_pages(address, umem->npgs, in xdp_umem_pin_pages()
108 gup_flags | FOLL_LONGTERM, &umem->pgs[0], NULL); in xdp_umem_pin_pages()
111 if (npgs != umem->npgs) { in xdp_umem_pin_pages()
113 umem->npgs = npgs; in xdp_umem_pin_pages()
123 xdp_umem_unpin_pages(umem); in xdp_umem_pin_pages()
125 kvfree(umem->pgs); in xdp_umem_pin_pages()
126 umem->pgs = NULL; in xdp_umem_pin_pages()
130 static int xdp_umem_account_pages(struct xdp_umem *umem) in xdp_umem_account_pages() argument
138 umem->user = get_uid(current_user()); in xdp_umem_account_pages()
141 old_npgs = atomic_long_read(&umem->user->locked_vm); in xdp_umem_account_pages()
142 new_npgs = old_npgs + umem->npgs; in xdp_umem_account_pages()
144 free_uid(umem->user); in xdp_umem_account_pages()
145 umem->user = NULL; in xdp_umem_account_pages()
148 } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs, in xdp_umem_account_pages()
153 static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) in xdp_umem_reg() argument
203 umem->size = size; in xdp_umem_reg()
204 umem->headroom = headroom; in xdp_umem_reg()
205 umem->chunk_size = chunk_size; in xdp_umem_reg()
206 umem->chunks = chunks; in xdp_umem_reg()
207 umem->npgs = (u32)npgs; in xdp_umem_reg()
208 umem->pgs = NULL; in xdp_umem_reg()
209 umem->user = NULL; in xdp_umem_reg()
210 umem->flags = mr->flags; in xdp_umem_reg()
212 INIT_LIST_HEAD(&umem->xsk_dma_list); in xdp_umem_reg()
213 refcount_set(&umem->users, 1); in xdp_umem_reg()
215 err = xdp_umem_account_pages(umem); in xdp_umem_reg()
219 err = xdp_umem_pin_pages(umem, (unsigned long)addr); in xdp_umem_reg()
223 err = xdp_umem_addr_map(umem, umem->pgs, umem->npgs); in xdp_umem_reg()
230 xdp_umem_unpin_pages(umem); in xdp_umem_reg()
232 xdp_umem_unaccount_pages(umem); in xdp_umem_reg()
238 struct xdp_umem *umem; in xdp_umem_create() local
241 umem = kzalloc(sizeof(*umem), GFP_KERNEL); in xdp_umem_create()
242 if (!umem) in xdp_umem_create()
247 kfree(umem); in xdp_umem_create()
250 umem->id = err; in xdp_umem_create()
252 err = xdp_umem_reg(umem, mr); in xdp_umem_create()
254 ida_simple_remove(&umem_ida, umem->id); in xdp_umem_create()
255 kfree(umem); in xdp_umem_create()
259 return umem; in xdp_umem_create()