1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2020 Hewlett Packard Enterprise, Inc. All rights reserved.
4 */
5
6 #include "rxe.h"
7
rxe_alloc_mw(struct ib_mw * ibmw,struct ib_udata * udata)8 int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
9 {
10 struct rxe_mw *mw = to_rmw(ibmw);
11 struct rxe_pd *pd = to_rpd(ibmw->pd);
12 struct rxe_dev *rxe = to_rdev(ibmw->device);
13 int ret;
14
15 rxe_add_ref(pd);
16
17 ret = rxe_add_to_pool(&rxe->mw_pool, mw);
18 if (ret) {
19 rxe_drop_ref(pd);
20 return ret;
21 }
22
23 rxe_add_index(mw);
24 mw->rkey = ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);
25 mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?
26 RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;
27 spin_lock_init(&mw->lock);
28
29 return 0;
30 }
31
rxe_do_dealloc_mw(struct rxe_mw * mw)32 static void rxe_do_dealloc_mw(struct rxe_mw *mw)
33 {
34 if (mw->mr) {
35 struct rxe_mr *mr = mw->mr;
36
37 mw->mr = NULL;
38 atomic_dec(&mr->num_mw);
39 rxe_drop_ref(mr);
40 }
41
42 if (mw->qp) {
43 struct rxe_qp *qp = mw->qp;
44
45 mw->qp = NULL;
46 rxe_drop_ref(qp);
47 }
48
49 mw->access = 0;
50 mw->addr = 0;
51 mw->length = 0;
52 mw->state = RXE_MW_STATE_INVALID;
53 }
54
rxe_dealloc_mw(struct ib_mw * ibmw)55 int rxe_dealloc_mw(struct ib_mw *ibmw)
56 {
57 struct rxe_mw *mw = to_rmw(ibmw);
58 struct rxe_pd *pd = to_rpd(ibmw->pd);
59 unsigned long flags;
60
61 spin_lock_irqsave(&mw->lock, flags);
62 rxe_do_dealloc_mw(mw);
63 spin_unlock_irqrestore(&mw->lock, flags);
64
65 rxe_drop_ref(mw);
66 rxe_drop_ref(pd);
67
68 return 0;
69 }
70
rxe_check_bind_mw(struct rxe_qp * qp,struct rxe_send_wqe * wqe,struct rxe_mw * mw,struct rxe_mr * mr)71 static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
72 struct rxe_mw *mw, struct rxe_mr *mr)
73 {
74 u32 key = wqe->wr.wr.mw.rkey & 0xff;
75
76 if (mw->ibmw.type == IB_MW_TYPE_1) {
77 if (unlikely(mw->state != RXE_MW_STATE_VALID)) {
78 pr_err_once(
79 "attempt to bind a type 1 MW not in the valid state\n");
80 return -EINVAL;
81 }
82
83 /* o10-36.2.2 */
84 if (unlikely((mw->access & IB_ZERO_BASED))) {
85 pr_err_once("attempt to bind a zero based type 1 MW\n");
86 return -EINVAL;
87 }
88 }
89
90 if (mw->ibmw.type == IB_MW_TYPE_2) {
91 /* o10-37.2.30 */
92 if (unlikely(mw->state != RXE_MW_STATE_FREE)) {
93 pr_err_once(
94 "attempt to bind a type 2 MW not in the free state\n");
95 return -EINVAL;
96 }
97
98 /* C10-72 */
99 if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) {
100 pr_err_once(
101 "attempt to bind type 2 MW with qp with different PD\n");
102 return -EINVAL;
103 }
104
105 /* o10-37.2.40 */
106 if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) {
107 pr_err_once(
108 "attempt to invalidate type 2 MW by binding with NULL or zero length MR\n");
109 return -EINVAL;
110 }
111 }
112
113 if (unlikely(key == (mw->rkey & 0xff))) {
114 pr_err_once("attempt to bind MW with same key\n");
115 return -EINVAL;
116 }
117
118 /* remaining checks only apply to a nonzero MR */
119 if (!mr)
120 return 0;
121
122 if (unlikely(mr->access & IB_ZERO_BASED)) {
123 pr_err_once("attempt to bind MW to zero based MR\n");
124 return -EINVAL;
125 }
126
127 /* C10-73 */
128 if (unlikely(!(mr->access & IB_ACCESS_MW_BIND))) {
129 pr_err_once(
130 "attempt to bind an MW to an MR without bind access\n");
131 return -EINVAL;
132 }
133
134 /* C10-74 */
135 if (unlikely((mw->access &
136 (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC)) &&
137 !(mr->access & IB_ACCESS_LOCAL_WRITE))) {
138 pr_err_once(
139 "attempt to bind an writeable MW to an MR without local write access\n");
140 return -EINVAL;
141 }
142
143 /* C10-75 */
144 if (mw->access & IB_ZERO_BASED) {
145 if (unlikely(wqe->wr.wr.mw.length > mr->cur_map_set->length)) {
146 pr_err_once(
147 "attempt to bind a ZB MW outside of the MR\n");
148 return -EINVAL;
149 }
150 } else {
151 if (unlikely((wqe->wr.wr.mw.addr < mr->cur_map_set->iova) ||
152 ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) >
153 (mr->cur_map_set->iova + mr->cur_map_set->length)))) {
154 pr_err_once(
155 "attempt to bind a VA MW outside of the MR\n");
156 return -EINVAL;
157 }
158 }
159
160 return 0;
161 }
162
rxe_do_bind_mw(struct rxe_qp * qp,struct rxe_send_wqe * wqe,struct rxe_mw * mw,struct rxe_mr * mr)163 static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
164 struct rxe_mw *mw, struct rxe_mr *mr)
165 {
166 u32 key = wqe->wr.wr.mw.rkey & 0xff;
167
168 mw->rkey = (mw->rkey & ~0xff) | key;
169 mw->access = wqe->wr.wr.mw.access;
170 mw->state = RXE_MW_STATE_VALID;
171 mw->addr = wqe->wr.wr.mw.addr;
172 mw->length = wqe->wr.wr.mw.length;
173
174 if (mw->mr) {
175 rxe_drop_ref(mw->mr);
176 atomic_dec(&mw->mr->num_mw);
177 mw->mr = NULL;
178 }
179
180 if (mw->length) {
181 mw->mr = mr;
182 atomic_inc(&mr->num_mw);
183 rxe_add_ref(mr);
184 }
185
186 if (mw->ibmw.type == IB_MW_TYPE_2) {
187 rxe_add_ref(qp);
188 mw->qp = qp;
189 }
190 }
191
rxe_bind_mw(struct rxe_qp * qp,struct rxe_send_wqe * wqe)192 int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
193 {
194 int ret;
195 struct rxe_mw *mw;
196 struct rxe_mr *mr;
197 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
198 u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;
199 u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;
200 unsigned long flags;
201
202 mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8);
203 if (unlikely(!mw)) {
204 ret = -EINVAL;
205 goto err;
206 }
207
208 if (unlikely(mw->rkey != mw_rkey)) {
209 ret = -EINVAL;
210 goto err_drop_mw;
211 }
212
213 if (likely(wqe->wr.wr.mw.length)) {
214 mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8);
215 if (unlikely(!mr)) {
216 ret = -EINVAL;
217 goto err_drop_mw;
218 }
219
220 if (unlikely(mr->lkey != mr_lkey)) {
221 ret = -EINVAL;
222 goto err_drop_mr;
223 }
224 } else {
225 mr = NULL;
226 }
227
228 spin_lock_irqsave(&mw->lock, flags);
229
230 ret = rxe_check_bind_mw(qp, wqe, mw, mr);
231 if (ret)
232 goto err_unlock;
233
234 rxe_do_bind_mw(qp, wqe, mw, mr);
235 err_unlock:
236 spin_unlock_irqrestore(&mw->lock, flags);
237 err_drop_mr:
238 if (mr)
239 rxe_drop_ref(mr);
240 err_drop_mw:
241 rxe_drop_ref(mw);
242 err:
243 return ret;
244 }
245
rxe_check_invalidate_mw(struct rxe_qp * qp,struct rxe_mw * mw)246 static int rxe_check_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw)
247 {
248 if (unlikely(mw->state == RXE_MW_STATE_INVALID))
249 return -EINVAL;
250
251 /* o10-37.2.26 */
252 if (unlikely(mw->ibmw.type == IB_MW_TYPE_1))
253 return -EINVAL;
254
255 return 0;
256 }
257
rxe_do_invalidate_mw(struct rxe_mw * mw)258 static void rxe_do_invalidate_mw(struct rxe_mw *mw)
259 {
260 struct rxe_qp *qp;
261 struct rxe_mr *mr;
262
263 /* valid type 2 MW will always have a QP pointer */
264 qp = mw->qp;
265 mw->qp = NULL;
266 rxe_drop_ref(qp);
267
268 /* valid type 2 MW will always have an MR pointer */
269 mr = mw->mr;
270 mw->mr = NULL;
271 atomic_dec(&mr->num_mw);
272 rxe_drop_ref(mr);
273
274 mw->access = 0;
275 mw->addr = 0;
276 mw->length = 0;
277 mw->state = RXE_MW_STATE_FREE;
278 }
279
rxe_invalidate_mw(struct rxe_qp * qp,u32 rkey)280 int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey)
281 {
282 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
283 unsigned long flags;
284 struct rxe_mw *mw;
285 int ret;
286
287 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8);
288 if (!mw) {
289 ret = -EINVAL;
290 goto err;
291 }
292
293 if (rkey != mw->rkey) {
294 ret = -EINVAL;
295 goto err_drop_ref;
296 }
297
298 spin_lock_irqsave(&mw->lock, flags);
299
300 ret = rxe_check_invalidate_mw(qp, mw);
301 if (ret)
302 goto err_unlock;
303
304 rxe_do_invalidate_mw(mw);
305 err_unlock:
306 spin_unlock_irqrestore(&mw->lock, flags);
307 err_drop_ref:
308 rxe_drop_ref(mw);
309 err:
310 return ret;
311 }
312
rxe_lookup_mw(struct rxe_qp * qp,int access,u32 rkey)313 struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey)
314 {
315 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
316 struct rxe_pd *pd = to_rpd(qp->ibqp.pd);
317 struct rxe_mw *mw;
318 int index = rkey >> 8;
319
320 mw = rxe_pool_get_index(&rxe->mw_pool, index);
321 if (!mw)
322 return NULL;
323
324 if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd ||
325 (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) ||
326 (mw->length == 0) ||
327 (access && !(access & mw->access)) ||
328 mw->state != RXE_MW_STATE_VALID)) {
329 rxe_drop_ref(mw);
330 return NULL;
331 }
332
333 return mw;
334 }
335
rxe_mw_cleanup(struct rxe_pool_entry * elem)336 void rxe_mw_cleanup(struct rxe_pool_entry *elem)
337 {
338 struct rxe_mw *mw = container_of(elem, typeof(*mw), pelem);
339
340 rxe_drop_index(mw);
341 }
342