Lines Matching refs:hr_cq
59 static int alloc_cqn(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) in alloc_cqn() argument
77 hr_cq->cqn = (id << CQ_BANKID_SHIFT) | bankid; in alloc_cqn()
104 static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) in alloc_cqc() argument
113 ret = hns_roce_mtr_find(hr_dev, &hr_cq->mtr, 0, mtts, ARRAY_SIZE(mtts), in alloc_cqc()
121 ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn); in alloc_cqc()
124 hr_cq->cqn, ret); in alloc_cqc()
128 ret = xa_err(xa_store(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL)); in alloc_cqc()
141 hr_dev->hw->write_cqc(hr_dev, hr_cq, mailbox->buf, mtts, dma_handle); in alloc_cqc()
144 ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_cq->cqn, 0, in alloc_cqc()
150 hr_cq->cqn, ret); in alloc_cqc()
154 hr_cq->cons_index = 0; in alloc_cqc()
155 hr_cq->arm_sn = 1; in alloc_cqc()
157 refcount_set(&hr_cq->refcount, 1); in alloc_cqc()
158 init_completion(&hr_cq->free); in alloc_cqc()
163 xa_erase(&cq_table->array, hr_cq->cqn); in alloc_cqc()
166 hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); in alloc_cqc()
172 static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) in free_cqc() argument
178 ret = hns_roce_cmd_mbox(hr_dev, 0, 0, hr_cq->cqn, 1, in free_cqc()
183 hr_cq->cqn); in free_cqc()
185 xa_erase(&cq_table->array, hr_cq->cqn); in free_cqc()
188 synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq); in free_cqc()
191 if (refcount_dec_and_test(&hr_cq->refcount)) in free_cqc()
192 complete(&hr_cq->free); in free_cqc()
193 wait_for_completion(&hr_cq->free); in free_cqc()
195 hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); in free_cqc()
198 static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, in alloc_cq_buf() argument
206 buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size; in alloc_cq_buf()
210 ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr, in alloc_cq_buf()
219 static void free_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) in free_cq_buf() argument
221 hns_roce_mtr_destroy(hr_dev, &hr_cq->mtr); in free_cq_buf()
224 static int alloc_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, in alloc_cq_db() argument
237 err = hns_roce_db_map_user(uctx, addr, &hr_cq->db); in alloc_cq_db()
240 hr_cq->flags |= HNS_ROCE_CQ_FLAG_RECORD_DB; in alloc_cq_db()
245 err = hns_roce_alloc_db(hr_dev, &hr_cq->db, 1); in alloc_cq_db()
248 hr_cq->set_ci_db = hr_cq->db.db_record; in alloc_cq_db()
249 *hr_cq->set_ci_db = 0; in alloc_cq_db()
250 hr_cq->flags |= HNS_ROCE_CQ_FLAG_RECORD_DB; in alloc_cq_db()
252 hr_cq->db_reg = hr_dev->reg_base + hr_dev->odb_offset + in alloc_cq_db()
259 static void free_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq, in free_cq_db() argument
264 if (!(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB)) in free_cq_db()
267 hr_cq->flags &= ~HNS_ROCE_CQ_FLAG_RECORD_DB; in free_cq_db()
272 hns_roce_db_unmap_user(uctx, &hr_cq->db); in free_cq_db()
274 hns_roce_free_db(hr_dev, &hr_cq->db); in free_cq_db()
298 static int get_cq_ucmd(struct hns_roce_cq *hr_cq, struct ib_udata *udata, in get_cq_ucmd() argument
301 struct ib_device *ibdev = hr_cq->ib_cq.device; in get_cq_ucmd()
313 static void set_cq_param(struct hns_roce_cq *hr_cq, u32 cq_entries, int vector, in set_cq_param() argument
316 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); in set_cq_param()
320 hr_cq->ib_cq.cqe = cq_entries - 1; /* used as cqe index */ in set_cq_param()
321 hr_cq->cq_depth = cq_entries; in set_cq_param()
322 hr_cq->vector = vector; in set_cq_param()
324 spin_lock_init(&hr_cq->lock); in set_cq_param()
325 INIT_LIST_HEAD(&hr_cq->sq_list); in set_cq_param()
326 INIT_LIST_HEAD(&hr_cq->rq_list); in set_cq_param()
329 static int set_cqe_size(struct hns_roce_cq *hr_cq, struct ib_udata *udata, in set_cqe_size() argument
332 struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device); in set_cqe_size()
335 hr_cq->cqe_size = hr_dev->caps.cqe_sz; in set_cqe_size()
347 hr_cq->cqe_size = ucmd->cqe_size; in set_cqe_size()
349 hr_cq->cqe_size = HNS_ROCE_V2_CQE_SIZE; in set_cqe_size()
360 struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); in hns_roce_create_cq() local
373 ret = get_cq_ucmd(hr_cq, udata, &ucmd); in hns_roce_create_cq()
378 set_cq_param(hr_cq, attr->cqe, attr->comp_vector, &ucmd); in hns_roce_create_cq()
380 ret = set_cqe_size(hr_cq, udata, &ucmd); in hns_roce_create_cq()
384 ret = alloc_cq_buf(hr_dev, hr_cq, udata, ucmd.buf_addr); in hns_roce_create_cq()
390 ret = alloc_cq_db(hr_dev, hr_cq, udata, ucmd.db_addr, &resp); in hns_roce_create_cq()
396 ret = alloc_cqn(hr_dev, hr_cq); in hns_roce_create_cq()
402 ret = alloc_cqc(hr_dev, hr_cq); in hns_roce_create_cq()
415 if (!udata && hr_cq->tptr_addr) in hns_roce_create_cq()
416 *hr_cq->tptr_addr = 0; in hns_roce_create_cq()
419 resp.cqn = hr_cq->cqn; in hns_roce_create_cq()
429 free_cqc(hr_dev, hr_cq); in hns_roce_create_cq()
431 free_cqn(hr_dev, hr_cq->cqn); in hns_roce_create_cq()
433 free_cq_db(hr_dev, hr_cq, udata); in hns_roce_create_cq()
435 free_cq_buf(hr_dev, hr_cq); in hns_roce_create_cq()
442 struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); in hns_roce_destroy_cq() local
447 free_cqc(hr_dev, hr_cq); in hns_roce_destroy_cq()
448 free_cqn(hr_dev, hr_cq->cqn); in hns_roce_destroy_cq()
449 free_cq_db(hr_dev, hr_cq, udata); in hns_roce_destroy_cq()
450 free_cq_buf(hr_dev, hr_cq); in hns_roce_destroy_cq()
457 struct hns_roce_cq *hr_cq; in hns_roce_cq_completion() local
460 hr_cq = xa_load(&hr_dev->cq_table.array, in hns_roce_cq_completion()
462 if (!hr_cq) { in hns_roce_cq_completion()
468 ++hr_cq->arm_sn; in hns_roce_cq_completion()
469 ibcq = &hr_cq->ib_cq; in hns_roce_cq_completion()
477 struct hns_roce_cq *hr_cq; in hns_roce_cq_event() local
481 hr_cq = xa_load(&hr_dev->cq_table.array, in hns_roce_cq_event()
483 if (!hr_cq) { in hns_roce_cq_event()
496 refcount_inc(&hr_cq->refcount); in hns_roce_cq_event()
498 ibcq = &hr_cq->ib_cq; in hns_roce_cq_event()
506 if (refcount_dec_and_test(&hr_cq->refcount)) in hns_roce_cq_event()
507 complete(&hr_cq->free); in hns_roce_cq_event()