1 /*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/highmem.h>
34 #include <linux/kernel.h>
35 #include <linux/module.h>
36 #include <linux/delay.h>
37 #include <linux/mlx5/driver.h>
38 #include <linux/xarray.h>
39 #include "mlx5_core.h"
40 #include "lib/eq.h"
41 #include "lib/tout.h"
42
43 enum {
44 MLX5_PAGES_CANT_GIVE = 0,
45 MLX5_PAGES_GIVE = 1,
46 MLX5_PAGES_TAKE = 2
47 };
48
49 struct mlx5_pages_req {
50 struct mlx5_core_dev *dev;
51 u16 func_id;
52 u8 ec_function;
53 s32 npages;
54 struct work_struct work;
55 u8 release_all;
56 };
57
58 struct fw_page {
59 struct rb_node rb_node;
60 u64 addr;
61 struct page *page;
62 u32 function;
63 unsigned long bitmask;
64 struct list_head list;
65 unsigned int free_count;
66 };
67
68 enum {
69 MLX5_MAX_RECLAIM_TIME_MILI = 5000,
70 MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE,
71 };
72
get_function(u16 func_id,bool ec_function)73 static u32 get_function(u16 func_id, bool ec_function)
74 {
75 return (u32)func_id | (ec_function << 16);
76 }
77
page_root_per_function(struct mlx5_core_dev * dev,u32 function)78 static struct rb_root *page_root_per_function(struct mlx5_core_dev *dev, u32 function)
79 {
80 struct rb_root *root;
81 int err;
82
83 root = xa_load(&dev->priv.page_root_xa, function);
84 if (root)
85 return root;
86
87 root = kzalloc(sizeof(*root), GFP_KERNEL);
88 if (!root)
89 return ERR_PTR(-ENOMEM);
90
91 err = xa_insert(&dev->priv.page_root_xa, function, root, GFP_KERNEL);
92 if (err) {
93 kfree(root);
94 return ERR_PTR(err);
95 }
96
97 *root = RB_ROOT;
98
99 return root;
100 }
101
insert_page(struct mlx5_core_dev * dev,u64 addr,struct page * page,u32 function)102 static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u32 function)
103 {
104 struct rb_node *parent = NULL;
105 struct rb_root *root;
106 struct rb_node **new;
107 struct fw_page *nfp;
108 struct fw_page *tfp;
109 int i;
110
111 root = page_root_per_function(dev, function);
112 if (IS_ERR(root))
113 return PTR_ERR(root);
114
115 new = &root->rb_node;
116
117 while (*new) {
118 parent = *new;
119 tfp = rb_entry(parent, struct fw_page, rb_node);
120 if (tfp->addr < addr)
121 new = &parent->rb_left;
122 else if (tfp->addr > addr)
123 new = &parent->rb_right;
124 else
125 return -EEXIST;
126 }
127
128 nfp = kzalloc(sizeof(*nfp), GFP_KERNEL);
129 if (!nfp)
130 return -ENOMEM;
131
132 nfp->addr = addr;
133 nfp->page = page;
134 nfp->function = function;
135 nfp->free_count = MLX5_NUM_4K_IN_PAGE;
136 for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++)
137 set_bit(i, &nfp->bitmask);
138
139 rb_link_node(&nfp->rb_node, parent, new);
140 rb_insert_color(&nfp->rb_node, root);
141 list_add(&nfp->list, &dev->priv.free_list);
142
143 return 0;
144 }
145
find_fw_page(struct mlx5_core_dev * dev,u64 addr,u32 function)146 static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr,
147 u32 function)
148 {
149 struct fw_page *result = NULL;
150 struct rb_root *root;
151 struct rb_node *tmp;
152 struct fw_page *tfp;
153
154 root = xa_load(&dev->priv.page_root_xa, function);
155 if (WARN_ON_ONCE(!root))
156 return NULL;
157
158 tmp = root->rb_node;
159
160 while (tmp) {
161 tfp = rb_entry(tmp, struct fw_page, rb_node);
162 if (tfp->addr < addr) {
163 tmp = tmp->rb_left;
164 } else if (tfp->addr > addr) {
165 tmp = tmp->rb_right;
166 } else {
167 result = tfp;
168 break;
169 }
170 }
171
172 return result;
173 }
174
mlx5_cmd_query_pages(struct mlx5_core_dev * dev,u16 * func_id,s32 * npages,int boot)175 static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id,
176 s32 *npages, int boot)
177 {
178 u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {};
179 u32 in[MLX5_ST_SZ_DW(query_pages_in)] = {};
180 int err;
181
182 MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES);
183 MLX5_SET(query_pages_in, in, op_mod, boot ?
184 MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES :
185 MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES);
186 MLX5_SET(query_pages_in, in, embedded_cpu_function, mlx5_core_is_ecpf(dev));
187
188 err = mlx5_cmd_exec_inout(dev, query_pages, in, out);
189 if (err)
190 return err;
191
192 *npages = MLX5_GET(query_pages_out, out, num_pages);
193 *func_id = MLX5_GET(query_pages_out, out, function_id);
194
195 return err;
196 }
197
alloc_4k(struct mlx5_core_dev * dev,u64 * addr,u32 function)198 static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u32 function)
199 {
200 struct fw_page *fp = NULL;
201 struct fw_page *iter;
202 unsigned n;
203
204 list_for_each_entry(iter, &dev->priv.free_list, list) {
205 if (iter->function != function)
206 continue;
207 fp = iter;
208 }
209
210 if (list_empty(&dev->priv.free_list) || !fp)
211 return -ENOMEM;
212
213 n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask));
214 if (n >= MLX5_NUM_4K_IN_PAGE) {
215 mlx5_core_warn(dev, "alloc 4k bug\n");
216 return -ENOENT;
217 }
218 clear_bit(n, &fp->bitmask);
219 fp->free_count--;
220 if (!fp->free_count)
221 list_del(&fp->list);
222
223 *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE;
224
225 return 0;
226 }
227
228 #define MLX5_U64_4K_PAGE_MASK ((~(u64)0U) << PAGE_SHIFT)
229
free_fwp(struct mlx5_core_dev * dev,struct fw_page * fwp,bool in_free_list)230 static void free_fwp(struct mlx5_core_dev *dev, struct fw_page *fwp,
231 bool in_free_list)
232 {
233 struct rb_root *root;
234
235 root = xa_load(&dev->priv.page_root_xa, fwp->function);
236 if (WARN_ON_ONCE(!root))
237 return;
238
239 rb_erase(&fwp->rb_node, root);
240 if (in_free_list)
241 list_del(&fwp->list);
242 dma_unmap_page(mlx5_core_dma_dev(dev), fwp->addr & MLX5_U64_4K_PAGE_MASK,
243 PAGE_SIZE, DMA_BIDIRECTIONAL);
244 __free_page(fwp->page);
245 kfree(fwp);
246 }
247
free_4k(struct mlx5_core_dev * dev,u64 addr,u32 function)248 static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 function)
249 {
250 struct fw_page *fwp;
251 int n;
252
253 fwp = find_fw_page(dev, addr & MLX5_U64_4K_PAGE_MASK, function);
254 if (!fwp) {
255 mlx5_core_warn_rl(dev, "page not found\n");
256 return;
257 }
258 n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT;
259 fwp->free_count++;
260 set_bit(n, &fwp->bitmask);
261 if (fwp->free_count == MLX5_NUM_4K_IN_PAGE)
262 free_fwp(dev, fwp, fwp->free_count != 1);
263 else if (fwp->free_count == 1)
264 list_add(&fwp->list, &dev->priv.free_list);
265 }
266
alloc_system_page(struct mlx5_core_dev * dev,u32 function)267 static int alloc_system_page(struct mlx5_core_dev *dev, u32 function)
268 {
269 struct device *device = mlx5_core_dma_dev(dev);
270 int nid = dev_to_node(device);
271 struct page *page;
272 u64 zero_addr = 1;
273 u64 addr;
274 int err;
275
276 page = alloc_pages_node(nid, GFP_HIGHUSER, 0);
277 if (!page) {
278 mlx5_core_warn(dev, "failed to allocate page\n");
279 return -ENOMEM;
280 }
281 map:
282 addr = dma_map_page(device, page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
283 if (dma_mapping_error(device, addr)) {
284 mlx5_core_warn(dev, "failed dma mapping page\n");
285 err = -ENOMEM;
286 goto err_mapping;
287 }
288
289 /* Firmware doesn't support page with physical address 0 */
290 if (addr == 0) {
291 zero_addr = addr;
292 goto map;
293 }
294
295 err = insert_page(dev, addr, page, function);
296 if (err) {
297 mlx5_core_err(dev, "failed to track allocated page\n");
298 dma_unmap_page(device, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
299 }
300
301 err_mapping:
302 if (err)
303 __free_page(page);
304
305 if (zero_addr == 0)
306 dma_unmap_page(device, zero_addr, PAGE_SIZE,
307 DMA_BIDIRECTIONAL);
308
309 return err;
310 }
311
page_notify_fail(struct mlx5_core_dev * dev,u16 func_id,bool ec_function)312 static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id,
313 bool ec_function)
314 {
315 u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {};
316 int err;
317
318 MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
319 MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_CANT_GIVE);
320 MLX5_SET(manage_pages_in, in, function_id, func_id);
321 MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function);
322
323 err = mlx5_cmd_exec_in(dev, manage_pages, in);
324 if (err)
325 mlx5_core_warn(dev, "page notify failed func_id(%d) err(%d)\n",
326 func_id, err);
327 }
328
give_pages(struct mlx5_core_dev * dev,u16 func_id,int npages,int notify_fail,bool ec_function)329 static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
330 int notify_fail, bool ec_function)
331 {
332 u32 function = get_function(func_id, ec_function);
333 u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0};
334 int inlen = MLX5_ST_SZ_BYTES(manage_pages_in);
335 u64 addr;
336 int err;
337 u32 *in;
338 int i;
339
340 inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]);
341 in = kvzalloc(inlen, GFP_KERNEL);
342 if (!in) {
343 err = -ENOMEM;
344 mlx5_core_warn(dev, "vzalloc failed %d\n", inlen);
345 goto out_free;
346 }
347
348 for (i = 0; i < npages; i++) {
349 retry:
350 err = alloc_4k(dev, &addr, function);
351 if (err) {
352 if (err == -ENOMEM)
353 err = alloc_system_page(dev, function);
354 if (err)
355 goto out_4k;
356
357 goto retry;
358 }
359 MLX5_ARRAY_SET64(manage_pages_in, in, pas, i, addr);
360 }
361
362 MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
363 MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE);
364 MLX5_SET(manage_pages_in, in, function_id, func_id);
365 MLX5_SET(manage_pages_in, in, input_num_entries, npages);
366 MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function);
367
368 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
369 if (err) {
370 mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n",
371 func_id, npages, err);
372 goto out_4k;
373 }
374
375 dev->priv.fw_pages += npages;
376 if (func_id)
377 dev->priv.vfs_pages += npages;
378 else if (mlx5_core_is_ecpf(dev) && !ec_function)
379 dev->priv.host_pf_pages += npages;
380
381 mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x, err %d\n",
382 npages, ec_function, func_id, err);
383
384 kvfree(in);
385 return 0;
386
387 out_4k:
388 for (i--; i >= 0; i--)
389 free_4k(dev, MLX5_GET64(manage_pages_in, in, pas[i]), function);
390 out_free:
391 kvfree(in);
392 if (notify_fail)
393 page_notify_fail(dev, func_id, ec_function);
394 return err;
395 }
396
release_all_pages(struct mlx5_core_dev * dev,u16 func_id,bool ec_function)397 static void release_all_pages(struct mlx5_core_dev *dev, u16 func_id,
398 bool ec_function)
399 {
400 u32 function = get_function(func_id, ec_function);
401 struct rb_root *root;
402 struct rb_node *p;
403 int npages = 0;
404
405 root = xa_load(&dev->priv.page_root_xa, function);
406 if (WARN_ON_ONCE(!root))
407 return;
408
409 p = rb_first(root);
410 while (p) {
411 struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node);
412
413 p = rb_next(p);
414 npages += (MLX5_NUM_4K_IN_PAGE - fwp->free_count);
415 free_fwp(dev, fwp, fwp->free_count);
416 }
417
418 dev->priv.fw_pages -= npages;
419 if (func_id)
420 dev->priv.vfs_pages -= npages;
421 else if (mlx5_core_is_ecpf(dev) && !ec_function)
422 dev->priv.host_pf_pages -= npages;
423
424 mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x\n",
425 npages, ec_function, func_id);
426 }
427
fwp_fill_manage_pages_out(struct fw_page * fwp,u32 * out,u32 index,u32 npages)428 static u32 fwp_fill_manage_pages_out(struct fw_page *fwp, u32 *out, u32 index,
429 u32 npages)
430 {
431 u32 pages_set = 0;
432 unsigned int n;
433
434 for_each_clear_bit(n, &fwp->bitmask, MLX5_NUM_4K_IN_PAGE) {
435 MLX5_ARRAY_SET64(manage_pages_out, out, pas, index + pages_set,
436 fwp->addr + (n * MLX5_ADAPTER_PAGE_SIZE));
437 pages_set++;
438
439 if (!--npages)
440 break;
441 }
442
443 return pages_set;
444 }
445
reclaim_pages_cmd(struct mlx5_core_dev * dev,u32 * in,int in_size,u32 * out,int out_size)446 static int reclaim_pages_cmd(struct mlx5_core_dev *dev,
447 u32 *in, int in_size, u32 *out, int out_size)
448 {
449 struct rb_root *root;
450 struct fw_page *fwp;
451 struct rb_node *p;
452 bool ec_function;
453 u32 func_id;
454 u32 npages;
455 u32 i = 0;
456
457 if (!mlx5_cmd_is_down(dev))
458 return mlx5_cmd_exec(dev, in, in_size, out, out_size);
459
460 /* No hard feelings, we want our pages back! */
461 npages = MLX5_GET(manage_pages_in, in, input_num_entries);
462 func_id = MLX5_GET(manage_pages_in, in, function_id);
463 ec_function = MLX5_GET(manage_pages_in, in, embedded_cpu_function);
464
465 root = xa_load(&dev->priv.page_root_xa, get_function(func_id, ec_function));
466 if (WARN_ON_ONCE(!root))
467 return -EEXIST;
468
469 p = rb_first(root);
470 while (p && i < npages) {
471 fwp = rb_entry(p, struct fw_page, rb_node);
472 p = rb_next(p);
473
474 i += fwp_fill_manage_pages_out(fwp, out, i, npages - i);
475 }
476
477 MLX5_SET(manage_pages_out, out, output_num_entries, i);
478 return 0;
479 }
480
reclaim_pages(struct mlx5_core_dev * dev,u16 func_id,int npages,int * nclaimed,bool ec_function)481 static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
482 int *nclaimed, bool ec_function)
483 {
484 u32 function = get_function(func_id, ec_function);
485 int outlen = MLX5_ST_SZ_BYTES(manage_pages_out);
486 u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {};
487 int num_claimed;
488 u32 *out;
489 int err;
490 int i;
491
492 if (nclaimed)
493 *nclaimed = 0;
494
495 outlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]);
496 out = kvzalloc(outlen, GFP_KERNEL);
497 if (!out)
498 return -ENOMEM;
499
500 MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
501 MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE);
502 MLX5_SET(manage_pages_in, in, function_id, func_id);
503 MLX5_SET(manage_pages_in, in, input_num_entries, npages);
504 MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function);
505
506 mlx5_core_dbg(dev, "func 0x%x, npages %d, outlen %d\n",
507 func_id, npages, outlen);
508 err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen);
509 if (err) {
510 mlx5_core_err(dev, "failed reclaiming pages: err %d\n", err);
511 goto out_free;
512 }
513
514 num_claimed = MLX5_GET(manage_pages_out, out, output_num_entries);
515 if (num_claimed > npages) {
516 mlx5_core_warn(dev, "fw returned %d, driver asked %d => corruption\n",
517 num_claimed, npages);
518 err = -EINVAL;
519 goto out_free;
520 }
521
522 for (i = 0; i < num_claimed; i++)
523 free_4k(dev, MLX5_GET64(manage_pages_out, out, pas[i]), function);
524
525 if (nclaimed)
526 *nclaimed = num_claimed;
527
528 dev->priv.fw_pages -= num_claimed;
529 if (func_id)
530 dev->priv.vfs_pages -= num_claimed;
531 else if (mlx5_core_is_ecpf(dev) && !ec_function)
532 dev->priv.host_pf_pages -= num_claimed;
533
534 out_free:
535 kvfree(out);
536 return err;
537 }
538
pages_work_handler(struct work_struct * work)539 static void pages_work_handler(struct work_struct *work)
540 {
541 struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work);
542 struct mlx5_core_dev *dev = req->dev;
543 int err = 0;
544
545 if (req->release_all)
546 release_all_pages(dev, req->func_id, req->ec_function);
547 else if (req->npages < 0)
548 err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL,
549 req->ec_function);
550 else if (req->npages > 0)
551 err = give_pages(dev, req->func_id, req->npages, 1, req->ec_function);
552
553 if (err)
554 mlx5_core_warn(dev, "%s fail %d\n",
555 req->npages < 0 ? "reclaim" : "give", err);
556
557 kfree(req);
558 }
559
560 enum {
561 EC_FUNCTION_MASK = 0x8000,
562 RELEASE_ALL_PAGES_MASK = 0x4000,
563 };
564
req_pages_handler(struct notifier_block * nb,unsigned long type,void * data)565 static int req_pages_handler(struct notifier_block *nb,
566 unsigned long type, void *data)
567 {
568 struct mlx5_pages_req *req;
569 struct mlx5_core_dev *dev;
570 struct mlx5_priv *priv;
571 struct mlx5_eqe *eqe;
572 bool ec_function;
573 bool release_all;
574 u16 func_id;
575 s32 npages;
576
577 priv = mlx5_nb_cof(nb, struct mlx5_priv, pg_nb);
578 dev = container_of(priv, struct mlx5_core_dev, priv);
579 eqe = data;
580
581 func_id = be16_to_cpu(eqe->data.req_pages.func_id);
582 npages = be32_to_cpu(eqe->data.req_pages.num_pages);
583 ec_function = be16_to_cpu(eqe->data.req_pages.ec_function) & EC_FUNCTION_MASK;
584 release_all = be16_to_cpu(eqe->data.req_pages.ec_function) &
585 RELEASE_ALL_PAGES_MASK;
586 mlx5_core_dbg(dev, "page request for func 0x%x, npages %d, release_all %d\n",
587 func_id, npages, release_all);
588 req = kzalloc(sizeof(*req), GFP_ATOMIC);
589 if (!req) {
590 mlx5_core_warn(dev, "failed to allocate pages request\n");
591 return NOTIFY_DONE;
592 }
593
594 req->dev = dev;
595 req->func_id = func_id;
596 req->npages = npages;
597 req->ec_function = ec_function;
598 req->release_all = release_all;
599 INIT_WORK(&req->work, pages_work_handler);
600 queue_work(dev->priv.pg_wq, &req->work);
601 return NOTIFY_OK;
602 }
603
mlx5_satisfy_startup_pages(struct mlx5_core_dev * dev,int boot)604 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot)
605 {
606 u16 func_id;
607 s32 npages;
608 int err;
609
610 err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot);
611 if (err)
612 return err;
613
614 mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n",
615 npages, boot ? "boot" : "init", func_id);
616
617 return give_pages(dev, func_id, npages, 0, mlx5_core_is_ecpf(dev));
618 }
619
620 enum {
621 MLX5_BLKS_FOR_RECLAIM_PAGES = 12
622 };
623
optimal_reclaimed_pages(void)624 static int optimal_reclaimed_pages(void)
625 {
626 struct mlx5_cmd_prot_block *block;
627 struct mlx5_cmd_layout *lay;
628 int ret;
629
630 ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) -
631 MLX5_ST_SZ_BYTES(manage_pages_out)) /
632 MLX5_FLD_SZ_BYTES(manage_pages_out, pas[0]);
633
634 return ret;
635 }
636
mlx5_reclaim_root_pages(struct mlx5_core_dev * dev,struct rb_root * root,u16 func_id)637 static int mlx5_reclaim_root_pages(struct mlx5_core_dev *dev,
638 struct rb_root *root, u16 func_id)
639 {
640 u64 recl_pages_to_jiffies = msecs_to_jiffies(mlx5_tout_ms(dev, RECLAIM_PAGES));
641 unsigned long end = jiffies + recl_pages_to_jiffies;
642
643 while (!RB_EMPTY_ROOT(root)) {
644 int nclaimed;
645 int err;
646
647 err = reclaim_pages(dev, func_id, optimal_reclaimed_pages(),
648 &nclaimed, mlx5_core_is_ecpf(dev));
649 if (err) {
650 mlx5_core_warn(dev, "failed reclaiming pages (%d) for func id 0x%x\n",
651 err, func_id);
652 return err;
653 }
654
655 if (nclaimed)
656 end = jiffies + recl_pages_to_jiffies;
657
658 if (time_after(jiffies, end)) {
659 mlx5_core_warn(dev, "FW did not return all pages. giving up...\n");
660 break;
661 }
662 }
663
664 return 0;
665 }
666
mlx5_reclaim_startup_pages(struct mlx5_core_dev * dev)667 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)
668 {
669 struct rb_root *root;
670 unsigned long id;
671 void *entry;
672
673 xa_for_each(&dev->priv.page_root_xa, id, entry) {
674 root = entry;
675 mlx5_reclaim_root_pages(dev, root, id);
676 xa_erase(&dev->priv.page_root_xa, id);
677 kfree(root);
678 }
679
680 WARN_ON(!xa_empty(&dev->priv.page_root_xa));
681
682 WARN(dev->priv.fw_pages,
683 "FW pages counter is %d after reclaiming all pages\n",
684 dev->priv.fw_pages);
685 WARN(dev->priv.vfs_pages,
686 "VFs FW pages counter is %d after reclaiming all pages\n",
687 dev->priv.vfs_pages);
688 WARN(dev->priv.host_pf_pages,
689 "External host PF FW pages counter is %d after reclaiming all pages\n",
690 dev->priv.host_pf_pages);
691
692 return 0;
693 }
694
mlx5_pagealloc_init(struct mlx5_core_dev * dev)695 int mlx5_pagealloc_init(struct mlx5_core_dev *dev)
696 {
697 INIT_LIST_HEAD(&dev->priv.free_list);
698 dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator");
699 if (!dev->priv.pg_wq)
700 return -ENOMEM;
701
702 xa_init(&dev->priv.page_root_xa);
703
704 return 0;
705 }
706
mlx5_pagealloc_cleanup(struct mlx5_core_dev * dev)707 void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev)
708 {
709 xa_destroy(&dev->priv.page_root_xa);
710 destroy_workqueue(dev->priv.pg_wq);
711 }
712
mlx5_pagealloc_start(struct mlx5_core_dev * dev)713 void mlx5_pagealloc_start(struct mlx5_core_dev *dev)
714 {
715 MLX5_NB_INIT(&dev->priv.pg_nb, req_pages_handler, PAGE_REQUEST);
716 mlx5_eq_notifier_register(dev, &dev->priv.pg_nb);
717 }
718
mlx5_pagealloc_stop(struct mlx5_core_dev * dev)719 void mlx5_pagealloc_stop(struct mlx5_core_dev *dev)
720 {
721 mlx5_eq_notifier_unregister(dev, &dev->priv.pg_nb);
722 flush_workqueue(dev->priv.pg_wq);
723 }
724
mlx5_wait_for_pages(struct mlx5_core_dev * dev,int * pages)725 int mlx5_wait_for_pages(struct mlx5_core_dev *dev, int *pages)
726 {
727 u64 recl_vf_pages_to_jiffies = msecs_to_jiffies(mlx5_tout_ms(dev, RECLAIM_VFS_PAGES));
728 unsigned long end = jiffies + recl_vf_pages_to_jiffies;
729 int prev_pages = *pages;
730
731 /* In case of internal error we will free the pages manually later */
732 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
733 mlx5_core_warn(dev, "Skipping wait for vf pages stage");
734 return 0;
735 }
736
737 mlx5_core_dbg(dev, "Waiting for %d pages\n", prev_pages);
738 while (*pages) {
739 if (time_after(jiffies, end)) {
740 mlx5_core_warn(dev, "aborting while there are %d pending pages\n", *pages);
741 return -ETIMEDOUT;
742 }
743 if (*pages < prev_pages) {
744 end = jiffies + recl_vf_pages_to_jiffies;
745 prev_pages = *pages;
746 }
747 msleep(50);
748 }
749
750 mlx5_core_dbg(dev, "All pages received\n");
751 return 0;
752 }
753