Lines Matching refs:srq
14 static void *get_wqe(struct mlx5_ib_srq *srq, int n) in get_wqe() argument
16 return mlx5_frag_buf_get_wqe(&srq->fbc, n); in get_wqe()
19 static void mlx5_ib_srq_event(struct mlx5_core_srq *srq, enum mlx5_event type) in mlx5_ib_srq_event() argument
22 struct ib_srq *ibsrq = &to_mibsrq(srq)->ibsrq; in mlx5_ib_srq_event()
26 event.element.srq = ibsrq; in mlx5_ib_srq_event()
36 type, srq->srqn); in mlx5_ib_srq_event()
44 static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, in create_srq_user() argument
77 srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE); in create_srq_user()
79 srq->umem = ib_umem_get(pd->device, ucmd.buf_addr, buf_size, 0); in create_srq_user()
80 if (IS_ERR(srq->umem)) { in create_srq_user()
82 err = PTR_ERR(srq->umem); in create_srq_user()
85 in->umem = srq->umem; in create_srq_user()
87 err = mlx5_ib_db_map_user(ucontext, ucmd.db_addr, &srq->db); in create_srq_user()
101 ib_umem_release(srq->umem); in create_srq_user()
106 static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq, in create_srq_kernel() argument
113 err = mlx5_db_alloc(dev->mdev, &srq->db); in create_srq_kernel()
119 if (mlx5_frag_buf_alloc_node(dev->mdev, buf_size, &srq->buf, in create_srq_kernel()
126 mlx5_init_fbc(srq->buf.frags, srq->msrq.wqe_shift, ilog2(srq->msrq.max), in create_srq_kernel()
127 &srq->fbc); in create_srq_kernel()
129 srq->head = 0; in create_srq_kernel()
130 srq->tail = srq->msrq.max - 1; in create_srq_kernel()
131 srq->wqe_ctr = 0; in create_srq_kernel()
133 for (i = 0; i < srq->msrq.max; i++) { in create_srq_kernel()
134 next = get_wqe(srq, i); in create_srq_kernel()
136 cpu_to_be16((i + 1) & (srq->msrq.max - 1)); in create_srq_kernel()
139 mlx5_ib_dbg(dev, "srq->buf.page_shift = %d\n", srq->buf.page_shift); in create_srq_kernel()
140 in->pas = kvcalloc(srq->buf.npages, sizeof(*in->pas), GFP_KERNEL); in create_srq_kernel()
145 mlx5_fill_page_frag_array(&srq->buf, in->pas); in create_srq_kernel()
147 srq->wrid = kvmalloc_array(srq->msrq.max, sizeof(u64), GFP_KERNEL); in create_srq_kernel()
148 if (!srq->wrid) { in create_srq_kernel()
152 srq->wq_sig = 0; in create_srq_kernel()
154 in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT; in create_srq_kernel()
165 mlx5_frag_buf_free(dev->mdev, &srq->buf); in create_srq_kernel()
168 mlx5_db_free(dev->mdev, &srq->db); in create_srq_kernel()
172 static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq, in destroy_srq_user() argument
180 &srq->db); in destroy_srq_user()
181 ib_umem_release(srq->umem); in destroy_srq_user()
185 static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq) in destroy_srq_kernel() argument
187 kvfree(srq->wrid); in destroy_srq_kernel()
188 mlx5_frag_buf_free(dev->mdev, &srq->buf); in destroy_srq_kernel()
189 mlx5_db_free(dev->mdev, &srq->db); in destroy_srq_kernel()
197 struct mlx5_ib_srq *srq = to_msrq(ib_srq); in mlx5_ib_create_srq() local
217 mutex_init(&srq->mutex); in mlx5_ib_create_srq()
218 spin_lock_init(&srq->lock); in mlx5_ib_create_srq()
219 srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1); in mlx5_ib_create_srq()
220 srq->msrq.max_gs = init_attr->attr.max_sge; in mlx5_ib_create_srq()
223 srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg); in mlx5_ib_create_srq()
224 if (desc_size == 0 || srq->msrq.max_gs > desc_size) in mlx5_ib_create_srq()
232 srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) / in mlx5_ib_create_srq()
234 srq->msrq.wqe_shift = ilog2(desc_size); in mlx5_ib_create_srq()
235 buf_size = srq->msrq.max * desc_size; in mlx5_ib_create_srq()
242 err = create_srq_user(ib_srq->pd, srq, &in, udata, buf_size); in mlx5_ib_create_srq()
244 err = create_srq_kernel(dev, srq, &in, buf_size); in mlx5_ib_create_srq()
252 in.log_size = ilog2(srq->msrq.max); in mlx5_ib_create_srq()
253 in.wqe_shift = srq->msrq.wqe_shift - 4; in mlx5_ib_create_srq()
254 if (srq->wq_sig) in mlx5_ib_create_srq()
280 in.db_record = srq->db.dma; in mlx5_ib_create_srq()
281 err = mlx5_cmd_create_srq(dev, &srq->msrq, &in); in mlx5_ib_create_srq()
288 mlx5_ib_dbg(dev, "create SRQ with srqn 0x%x\n", srq->msrq.srqn); in mlx5_ib_create_srq()
290 srq->msrq.event = mlx5_ib_srq_event; in mlx5_ib_create_srq()
291 srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn; in mlx5_ib_create_srq()
295 .srqn = srq->msrq.srqn, in mlx5_ib_create_srq()
306 init_attr->attr.max_wr = srq->msrq.max - 1; in mlx5_ib_create_srq()
311 mlx5_cmd_destroy_srq(dev, &srq->msrq); in mlx5_ib_create_srq()
315 destroy_srq_user(ib_srq->pd, srq, udata); in mlx5_ib_create_srq()
317 destroy_srq_kernel(dev, srq); in mlx5_ib_create_srq()
326 struct mlx5_ib_srq *srq = to_msrq(ibsrq); in mlx5_ib_modify_srq() local
334 if (attr->srq_limit >= srq->msrq.max) in mlx5_ib_modify_srq()
337 mutex_lock(&srq->mutex); in mlx5_ib_modify_srq()
338 ret = mlx5_cmd_arm_srq(dev, &srq->msrq, attr->srq_limit, 1); in mlx5_ib_modify_srq()
339 mutex_unlock(&srq->mutex); in mlx5_ib_modify_srq()
351 struct mlx5_ib_srq *srq = to_msrq(ibsrq); in mlx5_ib_query_srq() local
359 ret = mlx5_cmd_query_srq(dev, &srq->msrq, out); in mlx5_ib_query_srq()
364 srq_attr->max_wr = srq->msrq.max - 1; in mlx5_ib_query_srq()
365 srq_attr->max_sge = srq->msrq.max_gs; in mlx5_ib_query_srq()
372 int mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) in mlx5_ib_destroy_srq() argument
374 struct mlx5_ib_dev *dev = to_mdev(srq->device); in mlx5_ib_destroy_srq()
375 struct mlx5_ib_srq *msrq = to_msrq(srq); in mlx5_ib_destroy_srq()
383 destroy_srq_user(srq->pd, msrq, udata); in mlx5_ib_destroy_srq()
389 void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index) in mlx5_ib_free_srq_wqe() argument
394 spin_lock(&srq->lock); in mlx5_ib_free_srq_wqe()
396 next = get_wqe(srq, srq->tail); in mlx5_ib_free_srq_wqe()
398 srq->tail = wqe_index; in mlx5_ib_free_srq_wqe()
400 spin_unlock(&srq->lock); in mlx5_ib_free_srq_wqe()
406 struct mlx5_ib_srq *srq = to_msrq(ibsrq); in mlx5_ib_post_srq_recv() local
416 spin_lock_irqsave(&srq->lock, flags); in mlx5_ib_post_srq_recv()
425 if (unlikely(wr->num_sge > srq->msrq.max_gs)) { in mlx5_ib_post_srq_recv()
431 if (unlikely(srq->head == srq->tail)) { in mlx5_ib_post_srq_recv()
437 srq->wrid[srq->head] = wr->wr_id; in mlx5_ib_post_srq_recv()
439 next = get_wqe(srq, srq->head); in mlx5_ib_post_srq_recv()
440 srq->head = be16_to_cpu(next->next_wqe_index); in mlx5_ib_post_srq_recv()
449 if (i < srq->msrq.max_avail_gather) { in mlx5_ib_post_srq_recv()
457 srq->wqe_ctr += nreq; in mlx5_ib_post_srq_recv()
464 *srq->db.db = cpu_to_be32(srq->wqe_ctr); in mlx5_ib_post_srq_recv()
467 spin_unlock_irqrestore(&srq->lock, flags); in mlx5_ib_post_srq_recv()