Lines Matching refs:wqe
42 static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, in restart_sge() argument
47 len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu; in restart_sge()
48 return rvt_restart_sge(ss, wqe, len); in restart_sge()
222 struct rvt_swqe *wqe; in qib_make_rc_req() local
252 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in qib_make_rc_req()
253 rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ? in qib_make_rc_req()
276 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in qib_make_rc_req()
297 if ((wqe->wr.send_flags & IB_SEND_FENCE) && in qib_make_rc_req()
303 qp->s_psn = wqe->psn; in qib_make_rc_req()
310 len = wqe->length; in qib_make_rc_req()
313 switch (wqe->wr.opcode) { in qib_make_rc_req()
317 if (!rvt_rc_credit_avail(qp, wqe)) in qib_make_rc_req()
324 if (wqe->wr.opcode == IB_WR_SEND) in qib_make_rc_req()
329 ohdr->u.imm_data = wqe->wr.ex.imm_data; in qib_make_rc_req()
332 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in qib_make_rc_req()
345 if (!rvt_rc_credit_avail(qp, wqe)) in qib_make_rc_req()
349 cpu_to_be64(wqe->rdma_wr.remote_addr); in qib_make_rc_req()
351 cpu_to_be32(wqe->rdma_wr.rkey); in qib_make_rc_req()
359 if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE) in qib_make_rc_req()
365 wqe->rdma_wr.wr.ex.imm_data; in qib_make_rc_req()
367 if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED) in qib_make_rc_req()
392 cpu_to_be64(wqe->rdma_wr.remote_addr); in qib_make_rc_req()
394 cpu_to_be32(wqe->rdma_wr.rkey); in qib_make_rc_req()
421 if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { in qib_make_rc_req()
423 put_ib_ateth_swap(wqe->atomic_wr.swap, in qib_make_rc_req()
425 put_ib_ateth_compare(wqe->atomic_wr.compare_add, in qib_make_rc_req()
429 put_ib_ateth_swap(wqe->atomic_wr.compare_add, in qib_make_rc_req()
433 put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr, in qib_make_rc_req()
436 wqe->atomic_wr.rkey); in qib_make_rc_req()
448 qp->s_sge.sge = wqe->sg_list[0]; in qib_make_rc_req()
449 qp->s_sge.sg_list = wqe->sg_list + 1; in qib_make_rc_req()
450 qp->s_sge.num_sge = wqe->wr.num_sge; in qib_make_rc_req()
451 qp->s_sge.total_len = wqe->length; in qib_make_rc_req()
452 qp->s_len = wqe->length; in qib_make_rc_req()
458 if (wqe->wr.opcode == IB_WR_RDMA_READ) in qib_make_rc_req()
459 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
474 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
487 if (wqe->wr.opcode == IB_WR_SEND) in qib_make_rc_req()
492 ohdr->u.imm_data = wqe->wr.ex.imm_data; in qib_make_rc_req()
495 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in qib_make_rc_req()
513 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
526 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) in qib_make_rc_req()
531 ohdr->u.imm_data = wqe->wr.ex.imm_data; in qib_make_rc_req()
533 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in qib_make_rc_req()
552 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; in qib_make_rc_req()
554 cpu_to_be64(wqe->rdma_wr.remote_addr + len); in qib_make_rc_req()
556 cpu_to_be32(wqe->rdma_wr.rkey); in qib_make_rc_req()
557 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); in qib_make_rc_req()
561 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
570 delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8; in qib_make_rc_req()
739 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); in reset_psn() local
748 if (qib_cmp24(psn, wqe->psn) <= 0) { in reset_psn()
754 opcode = wqe->wr.opcode; in reset_psn()
762 wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
763 diff = qib_cmp24(psn, wqe->psn); in reset_psn()
775 opcode = wqe->wr.opcode; in reset_psn()
823 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_restart_rc() local
831 rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); in qib_restart_rc()
840 if (wqe->wr.opcode == IB_WR_RDMA_READ) in qib_restart_rc()
859 struct rvt_swqe *wqe; in reset_sending_psn() local
864 wqe = rvt_get_swqe_ptr(qp, n); in reset_sending_psn()
865 if (qib_cmp24(psn, wqe->lpsn) <= 0) { in reset_sending_psn()
866 if (wqe->wr.opcode == IB_WR_RDMA_READ) in reset_sending_psn()
867 qp->s_sending_psn = wqe->lpsn + 1; in reset_sending_psn()
885 struct rvt_swqe *wqe; in qib_rc_send_complete() local
919 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in qib_rc_send_complete()
920 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && in qib_rc_send_complete()
924 wqe, in qib_rc_send_complete()
925 ib_qib_wc_opcode[wqe->wr.opcode], in qib_rc_send_complete()
952 struct rvt_swqe *wqe, in do_rc_completion() argument
960 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || in do_rc_completion()
963 wqe, in do_rc_completion()
964 ib_qib_wc_opcode[wqe->wr.opcode], in do_rc_completion()
970 update_last_psn(qp, wqe->lpsn); in do_rc_completion()
981 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in do_rc_completion()
984 qp->s_psn = wqe->psn; in do_rc_completion()
991 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_completion()
993 return wqe; in do_rc_completion()
1012 struct rvt_swqe *wqe; in do_rc_ack() local
1026 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_ack()
1033 while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) { in do_rc_ack()
1040 if (wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
1055 if ((wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
1057 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
1058 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && in do_rc_ack()
1077 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
1078 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { in do_rc_ack()
1079 u64 *vaddr = wqe->sg_list[0].vaddr; in do_rc_ack()
1083 (wqe->wr.opcode == IB_WR_RDMA_READ || in do_rc_ack()
1084 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
1085 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { in do_rc_ack()
1099 wqe = do_rc_completion(qp, wqe, ibp); in do_rc_ack()
1196 rvt_send_complete(qp, wqe, status); in do_rc_ack()
1227 struct rvt_swqe *wqe; in rdma_seq_err() local
1232 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rdma_seq_err()
1234 while (qib_cmp24(psn, wqe->lpsn) > 0) { in rdma_seq_err()
1235 if (wqe->wr.opcode == IB_WR_RDMA_READ || in rdma_seq_err()
1236 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in rdma_seq_err()
1237 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) in rdma_seq_err()
1239 wqe = do_rc_completion(qp, wqe, ibp); in rdma_seq_err()
1277 struct rvt_swqe *wqe; in qib_rc_rcv_resp() local
1341 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1357 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1358 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in qib_rc_rcv_resp()
1366 wqe, psn, pmtu); in qib_rc_rcv_resp()
1373 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in qib_rc_rcv_resp()
1423 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1425 wqe, psn, pmtu); in qib_rc_rcv_resp()
1432 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in qib_rc_rcv_resp()
1468 rvt_send_complete(qp, wqe, status); in qib_rc_rcv_resp()