1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3
4 #include <linux/interrupt.h>
5 #include <linux/notifier.h>
6 #include <linux/module.h>
7 #include <linux/mlx5/driver.h>
8 #include "mlx5_core.h"
9 #include "mlx5_irq.h"
10 #include "lib/sf.h"
11 #ifdef CONFIG_RFS_ACCEL
12 #include <linux/cpu_rmap.h>
13 #endif
14
15 #define MLX5_MAX_IRQ_NAME (32)
16 /* max irq_index is 2047, so four chars */
17 #define MLX5_MAX_IRQ_IDX_CHARS (4)
18
19 #define MLX5_SFS_PER_CTRL_IRQ 64
20 #define MLX5_IRQ_CTRL_SF_MAX 8
21 /* min num of vectors for SFs to be enabled */
22 #define MLX5_IRQ_VEC_COMP_BASE_SF 2
23
24 #define MLX5_EQ_SHARE_IRQ_MAX_COMP (8)
25 #define MLX5_EQ_SHARE_IRQ_MAX_CTRL (UINT_MAX)
26 #define MLX5_EQ_SHARE_IRQ_MIN_COMP (1)
27 #define MLX5_EQ_SHARE_IRQ_MIN_CTRL (4)
28 #define MLX5_EQ_REFS_PER_IRQ (2)
29
30 struct mlx5_irq {
31 struct atomic_notifier_head nh;
32 cpumask_var_t mask;
33 char name[MLX5_MAX_IRQ_NAME];
34 struct mlx5_irq_pool *pool;
35 int refcount;
36 u32 index;
37 int irqn;
38 };
39
40 struct mlx5_irq_pool {
41 char name[MLX5_MAX_IRQ_NAME - MLX5_MAX_IRQ_IDX_CHARS];
42 struct xa_limit xa_num_irqs;
43 struct mutex lock; /* sync IRQs creations */
44 struct xarray irqs;
45 u32 max_threshold;
46 u32 min_threshold;
47 struct mlx5_core_dev *dev;
48 };
49
50 struct mlx5_irq_table {
51 struct mlx5_irq_pool *pf_pool;
52 struct mlx5_irq_pool *sf_ctrl_pool;
53 struct mlx5_irq_pool *sf_comp_pool;
54 };
55
56 /**
57 * mlx5_get_default_msix_vec_count - Get the default number of MSI-X vectors
58 * to be ssigned to each VF.
59 * @dev: PF to work on
60 * @num_vfs: Number of enabled VFs
61 */
mlx5_get_default_msix_vec_count(struct mlx5_core_dev * dev,int num_vfs)62 int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs)
63 {
64 int num_vf_msix, min_msix, max_msix;
65
66 num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix);
67 if (!num_vf_msix)
68 return 0;
69
70 min_msix = MLX5_CAP_GEN(dev, min_dynamic_vf_msix_table_size);
71 max_msix = MLX5_CAP_GEN(dev, max_dynamic_vf_msix_table_size);
72
73 /* Limit maximum number of MSI-X vectors so the default configuration
74 * has some available in the pool. This will allow the user to increase
75 * the number of vectors in a VF without having to first size-down other
76 * VFs.
77 */
78 return max(min(num_vf_msix / num_vfs, max_msix / 2), min_msix);
79 }
80
81 /**
82 * mlx5_set_msix_vec_count - Set dynamically allocated MSI-X on the VF
83 * @dev: PF to work on
84 * @function_id: Internal PCI VF function IDd
85 * @msix_vec_count: Number of MSI-X vectors to set
86 */
mlx5_set_msix_vec_count(struct mlx5_core_dev * dev,int function_id,int msix_vec_count)87 int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int function_id,
88 int msix_vec_count)
89 {
90 int query_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
91 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
92 void *hca_cap = NULL, *query_cap = NULL, *cap;
93 int num_vf_msix, min_msix, max_msix;
94 int ret;
95
96 num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix);
97 if (!num_vf_msix)
98 return 0;
99
100 if (!MLX5_CAP_GEN(dev, vport_group_manager) || !mlx5_core_is_pf(dev))
101 return -EOPNOTSUPP;
102
103 min_msix = MLX5_CAP_GEN(dev, min_dynamic_vf_msix_table_size);
104 max_msix = MLX5_CAP_GEN(dev, max_dynamic_vf_msix_table_size);
105
106 if (msix_vec_count < min_msix)
107 return -EINVAL;
108
109 if (msix_vec_count > max_msix)
110 return -EOVERFLOW;
111
112 query_cap = kzalloc(query_sz, GFP_KERNEL);
113 hca_cap = kzalloc(set_sz, GFP_KERNEL);
114 if (!hca_cap || !query_cap) {
115 ret = -ENOMEM;
116 goto out;
117 }
118
119 ret = mlx5_vport_get_other_func_cap(dev, function_id, query_cap);
120 if (ret)
121 goto out;
122
123 cap = MLX5_ADDR_OF(set_hca_cap_in, hca_cap, capability);
124 memcpy(cap, MLX5_ADDR_OF(query_hca_cap_out, query_cap, capability),
125 MLX5_UN_SZ_BYTES(hca_cap_union));
126 MLX5_SET(cmd_hca_cap, cap, dynamic_msix_table_size, msix_vec_count);
127
128 MLX5_SET(set_hca_cap_in, hca_cap, opcode, MLX5_CMD_OP_SET_HCA_CAP);
129 MLX5_SET(set_hca_cap_in, hca_cap, other_function, 1);
130 MLX5_SET(set_hca_cap_in, hca_cap, function_id, function_id);
131
132 MLX5_SET(set_hca_cap_in, hca_cap, op_mod,
133 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1);
134 ret = mlx5_cmd_exec_in(dev, set_hca_cap, hca_cap);
135 out:
136 kfree(hca_cap);
137 kfree(query_cap);
138 return ret;
139 }
140
irq_release(struct mlx5_irq * irq)141 static void irq_release(struct mlx5_irq *irq)
142 {
143 struct mlx5_irq_pool *pool = irq->pool;
144
145 xa_erase(&pool->irqs, irq->index);
146 /* free_irq requires that affinity and rmap will be cleared
147 * before calling it. This is why there is asymmetry with set_rmap
148 * which should be called after alloc_irq but before request_irq.
149 */
150 irq_set_affinity_hint(irq->irqn, NULL);
151 free_cpumask_var(irq->mask);
152 free_irq(irq->irqn, &irq->nh);
153 kfree(irq);
154 }
155
irq_put(struct mlx5_irq * irq)156 static void irq_put(struct mlx5_irq *irq)
157 {
158 struct mlx5_irq_pool *pool = irq->pool;
159
160 mutex_lock(&pool->lock);
161 irq->refcount--;
162 if (!irq->refcount)
163 irq_release(irq);
164 mutex_unlock(&pool->lock);
165 }
166
irq_get_locked(struct mlx5_irq * irq)167 static int irq_get_locked(struct mlx5_irq *irq)
168 {
169 lockdep_assert_held(&irq->pool->lock);
170 if (WARN_ON_ONCE(!irq->refcount))
171 return 0;
172 irq->refcount++;
173 return 1;
174 }
175
irq_get(struct mlx5_irq * irq)176 static int irq_get(struct mlx5_irq *irq)
177 {
178 int err;
179
180 mutex_lock(&irq->pool->lock);
181 err = irq_get_locked(irq);
182 mutex_unlock(&irq->pool->lock);
183 return err;
184 }
185
irq_int_handler(int irq,void * nh)186 static irqreturn_t irq_int_handler(int irq, void *nh)
187 {
188 atomic_notifier_call_chain(nh, 0, NULL);
189 return IRQ_HANDLED;
190 }
191
irq_sf_set_name(struct mlx5_irq_pool * pool,char * name,int vecidx)192 static void irq_sf_set_name(struct mlx5_irq_pool *pool, char *name, int vecidx)
193 {
194 snprintf(name, MLX5_MAX_IRQ_NAME, "%s%d", pool->name, vecidx);
195 }
196
irq_set_name(struct mlx5_irq_pool * pool,char * name,int vecidx)197 static void irq_set_name(struct mlx5_irq_pool *pool, char *name, int vecidx)
198 {
199 if (!pool->xa_num_irqs.max) {
200 /* in case we only have a single irq for the device */
201 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_combined%d", vecidx);
202 return;
203 }
204
205 if (vecidx == pool->xa_num_irqs.max) {
206 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async%d", vecidx);
207 return;
208 }
209
210 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", vecidx);
211 }
212
irq_pool_is_sf_pool(struct mlx5_irq_pool * pool)213 static bool irq_pool_is_sf_pool(struct mlx5_irq_pool *pool)
214 {
215 return !strncmp("mlx5_sf", pool->name, strlen("mlx5_sf"));
216 }
217
irq_request(struct mlx5_irq_pool * pool,int i)218 static struct mlx5_irq *irq_request(struct mlx5_irq_pool *pool, int i)
219 {
220 struct mlx5_core_dev *dev = pool->dev;
221 char name[MLX5_MAX_IRQ_NAME];
222 struct mlx5_irq *irq;
223 int err;
224
225 irq = kzalloc(sizeof(*irq), GFP_KERNEL);
226 if (!irq)
227 return ERR_PTR(-ENOMEM);
228 irq->irqn = pci_irq_vector(dev->pdev, i);
229 if (!irq_pool_is_sf_pool(pool))
230 irq_set_name(pool, name, i);
231 else
232 irq_sf_set_name(pool, name, i);
233 ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh);
234 snprintf(irq->name, MLX5_MAX_IRQ_NAME,
235 "%s@pci:%s", name, pci_name(dev->pdev));
236 err = request_irq(irq->irqn, irq_int_handler, 0, irq->name,
237 &irq->nh);
238 if (err) {
239 mlx5_core_err(dev, "Failed to request irq. err = %d\n", err);
240 goto err_req_irq;
241 }
242 if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) {
243 mlx5_core_warn(dev, "zalloc_cpumask_var failed\n");
244 err = -ENOMEM;
245 goto err_cpumask;
246 }
247 irq->pool = pool;
248 irq->refcount = 1;
249 irq->index = i;
250 err = xa_err(xa_store(&pool->irqs, irq->index, irq, GFP_KERNEL));
251 if (err) {
252 mlx5_core_err(dev, "Failed to alloc xa entry for irq(%u). err = %d\n",
253 irq->index, err);
254 goto err_xa;
255 }
256 return irq;
257 err_xa:
258 free_cpumask_var(irq->mask);
259 err_cpumask:
260 free_irq(irq->irqn, &irq->nh);
261 err_req_irq:
262 kfree(irq);
263 return ERR_PTR(err);
264 }
265
mlx5_irq_attach_nb(struct mlx5_irq * irq,struct notifier_block * nb)266 int mlx5_irq_attach_nb(struct mlx5_irq *irq, struct notifier_block *nb)
267 {
268 int ret;
269
270 ret = irq_get(irq);
271 if (!ret)
272 /* Something very bad happens here, we are enabling EQ
273 * on non-existing IRQ.
274 */
275 return -ENOENT;
276 ret = atomic_notifier_chain_register(&irq->nh, nb);
277 if (ret)
278 irq_put(irq);
279 return ret;
280 }
281
mlx5_irq_detach_nb(struct mlx5_irq * irq,struct notifier_block * nb)282 int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb)
283 {
284 int err = 0;
285
286 err = atomic_notifier_chain_unregister(&irq->nh, nb);
287 irq_put(irq);
288 return err;
289 }
290
mlx5_irq_get_affinity_mask(struct mlx5_irq * irq)291 struct cpumask *mlx5_irq_get_affinity_mask(struct mlx5_irq *irq)
292 {
293 return irq->mask;
294 }
295
mlx5_irq_get_index(struct mlx5_irq * irq)296 int mlx5_irq_get_index(struct mlx5_irq *irq)
297 {
298 return irq->index;
299 }
300
301 /* irq_pool API */
302
303 /* creating an irq from irq_pool */
irq_pool_create_irq(struct mlx5_irq_pool * pool,struct cpumask * affinity)304 static struct mlx5_irq *irq_pool_create_irq(struct mlx5_irq_pool *pool,
305 struct cpumask *affinity)
306 {
307 struct mlx5_irq *irq;
308 u32 irq_index;
309 int err;
310
311 err = xa_alloc(&pool->irqs, &irq_index, NULL, pool->xa_num_irqs,
312 GFP_KERNEL);
313 if (err)
314 return ERR_PTR(err);
315 irq = irq_request(pool, irq_index);
316 if (IS_ERR(irq))
317 return irq;
318 cpumask_copy(irq->mask, affinity);
319 irq_set_affinity_hint(irq->irqn, irq->mask);
320 return irq;
321 }
322
323 /* looking for the irq with the smallest refcount and the same affinity */
irq_pool_find_least_loaded(struct mlx5_irq_pool * pool,struct cpumask * affinity)324 static struct mlx5_irq *irq_pool_find_least_loaded(struct mlx5_irq_pool *pool,
325 struct cpumask *affinity)
326 {
327 int start = pool->xa_num_irqs.min;
328 int end = pool->xa_num_irqs.max;
329 struct mlx5_irq *irq = NULL;
330 struct mlx5_irq *iter;
331 unsigned long index;
332
333 lockdep_assert_held(&pool->lock);
334 xa_for_each_range(&pool->irqs, index, iter, start, end) {
335 if (!cpumask_equal(iter->mask, affinity))
336 continue;
337 if (iter->refcount < pool->min_threshold)
338 return iter;
339 if (!irq || iter->refcount < irq->refcount)
340 irq = iter;
341 }
342 return irq;
343 }
344
345 /* requesting an irq from a given pool according to given affinity */
irq_pool_request_affinity(struct mlx5_irq_pool * pool,struct cpumask * affinity)346 static struct mlx5_irq *irq_pool_request_affinity(struct mlx5_irq_pool *pool,
347 struct cpumask *affinity)
348 {
349 struct mlx5_irq *least_loaded_irq, *new_irq;
350
351 mutex_lock(&pool->lock);
352 least_loaded_irq = irq_pool_find_least_loaded(pool, affinity);
353 if (least_loaded_irq &&
354 least_loaded_irq->refcount < pool->min_threshold)
355 goto out;
356 new_irq = irq_pool_create_irq(pool, affinity);
357 if (IS_ERR(new_irq)) {
358 if (!least_loaded_irq) {
359 mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %ld\n",
360 PTR_ERR(new_irq));
361 mutex_unlock(&pool->lock);
362 return new_irq;
363 }
364 /* We failed to create a new IRQ for the requested affinity,
365 * sharing existing IRQ.
366 */
367 goto out;
368 }
369 least_loaded_irq = new_irq;
370 goto unlock;
371 out:
372 irq_get_locked(least_loaded_irq);
373 if (least_loaded_irq->refcount > pool->max_threshold)
374 mlx5_core_dbg(pool->dev, "IRQ %u overloaded, pool_name: %s, %u EQs on this irq\n",
375 least_loaded_irq->irqn, pool->name,
376 least_loaded_irq->refcount / MLX5_EQ_REFS_PER_IRQ);
377 unlock:
378 mutex_unlock(&pool->lock);
379 return least_loaded_irq;
380 }
381
382 /* requesting an irq from a given pool according to given index */
383 static struct mlx5_irq *
irq_pool_request_vector(struct mlx5_irq_pool * pool,int vecidx,struct cpumask * affinity)384 irq_pool_request_vector(struct mlx5_irq_pool *pool, int vecidx,
385 struct cpumask *affinity)
386 {
387 struct mlx5_irq *irq;
388
389 mutex_lock(&pool->lock);
390 irq = xa_load(&pool->irqs, vecidx);
391 if (irq) {
392 irq_get_locked(irq);
393 goto unlock;
394 }
395 irq = irq_request(pool, vecidx);
396 if (IS_ERR(irq) || !affinity)
397 goto unlock;
398 cpumask_copy(irq->mask, affinity);
399 if (!irq_pool_is_sf_pool(pool) && !pool->xa_num_irqs.max &&
400 cpumask_empty(irq->mask))
401 cpumask_set_cpu(cpumask_first(cpu_online_mask), irq->mask);
402 irq_set_affinity_hint(irq->irqn, irq->mask);
403 unlock:
404 mutex_unlock(&pool->lock);
405 return irq;
406 }
407
find_sf_irq_pool(struct mlx5_irq_table * irq_table,int i,struct cpumask * affinity)408 static struct mlx5_irq_pool *find_sf_irq_pool(struct mlx5_irq_table *irq_table,
409 int i, struct cpumask *affinity)
410 {
411 if (cpumask_empty(affinity) && i == MLX5_IRQ_EQ_CTRL)
412 return irq_table->sf_ctrl_pool;
413 return irq_table->sf_comp_pool;
414 }
415
416 /**
417 * mlx5_irq_release - release an IRQ back to the system.
418 * @irq: irq to be released.
419 */
mlx5_irq_release(struct mlx5_irq * irq)420 void mlx5_irq_release(struct mlx5_irq *irq)
421 {
422 synchronize_irq(irq->irqn);
423 irq_put(irq);
424 }
425
426 /**
427 * mlx5_irq_request - request an IRQ for mlx5 device.
428 * @dev: mlx5 device that requesting the IRQ.
429 * @vecidx: vector index of the IRQ. This argument is ignore if affinity is
430 * provided.
431 * @affinity: cpumask requested for this IRQ.
432 *
433 * This function returns a pointer to IRQ, or ERR_PTR in case of error.
434 */
mlx5_irq_request(struct mlx5_core_dev * dev,u16 vecidx,struct cpumask * affinity)435 struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, u16 vecidx,
436 struct cpumask *affinity)
437 {
438 struct mlx5_irq_table *irq_table = mlx5_irq_table_get(dev);
439 struct mlx5_irq_pool *pool;
440 struct mlx5_irq *irq;
441
442 if (mlx5_core_is_sf(dev)) {
443 pool = find_sf_irq_pool(irq_table, vecidx, affinity);
444 if (!pool)
445 /* we don't have IRQs for SFs, using the PF IRQs */
446 goto pf_irq;
447 if (cpumask_empty(affinity) && !strcmp(pool->name, "mlx5_sf_comp"))
448 /* In case an SF user request IRQ with vecidx */
449 irq = irq_pool_request_vector(pool, vecidx, NULL);
450 else
451 irq = irq_pool_request_affinity(pool, affinity);
452 goto out;
453 }
454 pf_irq:
455 pool = irq_table->pf_pool;
456 vecidx = (vecidx == MLX5_IRQ_EQ_CTRL) ? pool->xa_num_irqs.max : vecidx;
457 irq = irq_pool_request_vector(pool, vecidx, affinity);
458 out:
459 if (IS_ERR(irq))
460 return irq;
461 mlx5_core_dbg(dev, "irq %u mapped to cpu %*pbl, %u EQs on this irq\n",
462 irq->irqn, cpumask_pr_args(affinity),
463 irq->refcount / MLX5_EQ_REFS_PER_IRQ);
464 return irq;
465 }
466
467 static struct mlx5_irq_pool *
irq_pool_alloc(struct mlx5_core_dev * dev,int start,int size,char * name,u32 min_threshold,u32 max_threshold)468 irq_pool_alloc(struct mlx5_core_dev *dev, int start, int size, char *name,
469 u32 min_threshold, u32 max_threshold)
470 {
471 struct mlx5_irq_pool *pool = kvzalloc(sizeof(*pool), GFP_KERNEL);
472
473 if (!pool)
474 return ERR_PTR(-ENOMEM);
475 pool->dev = dev;
476 mutex_init(&pool->lock);
477 xa_init_flags(&pool->irqs, XA_FLAGS_ALLOC);
478 pool->xa_num_irqs.min = start;
479 pool->xa_num_irqs.max = start + size - 1;
480 if (name)
481 snprintf(pool->name, MLX5_MAX_IRQ_NAME - MLX5_MAX_IRQ_IDX_CHARS,
482 name);
483 pool->min_threshold = min_threshold * MLX5_EQ_REFS_PER_IRQ;
484 pool->max_threshold = max_threshold * MLX5_EQ_REFS_PER_IRQ;
485 mlx5_core_dbg(dev, "pool->name = %s, pool->size = %d, pool->start = %d",
486 name, size, start);
487 return pool;
488 }
489
irq_pool_free(struct mlx5_irq_pool * pool)490 static void irq_pool_free(struct mlx5_irq_pool *pool)
491 {
492 struct mlx5_irq *irq;
493 unsigned long index;
494
495 /* There are cases in which we are destrying the irq_table before
496 * freeing all the IRQs, fast teardown for example. Hence, free the irqs
497 * which might not have been freed.
498 */
499 xa_for_each(&pool->irqs, index, irq)
500 irq_release(irq);
501 xa_destroy(&pool->irqs);
502 mutex_destroy(&pool->lock);
503 kvfree(pool);
504 }
505
irq_pools_init(struct mlx5_core_dev * dev,int sf_vec,int pf_vec)506 static int irq_pools_init(struct mlx5_core_dev *dev, int sf_vec, int pf_vec)
507 {
508 struct mlx5_irq_table *table = dev->priv.irq_table;
509 int num_sf_ctrl_by_msix;
510 int num_sf_ctrl_by_sfs;
511 int num_sf_ctrl;
512 int err;
513
514 /* init pf_pool */
515 table->pf_pool = irq_pool_alloc(dev, 0, pf_vec, NULL,
516 MLX5_EQ_SHARE_IRQ_MIN_COMP,
517 MLX5_EQ_SHARE_IRQ_MAX_COMP);
518 if (IS_ERR(table->pf_pool))
519 return PTR_ERR(table->pf_pool);
520 if (!mlx5_sf_max_functions(dev))
521 return 0;
522 if (sf_vec < MLX5_IRQ_VEC_COMP_BASE_SF) {
523 mlx5_core_dbg(dev, "Not enught IRQs for SFs. SF may run at lower performance\n");
524 return 0;
525 }
526
527 /* init sf_ctrl_pool */
528 num_sf_ctrl_by_msix = DIV_ROUND_UP(sf_vec, MLX5_COMP_EQS_PER_SF);
529 num_sf_ctrl_by_sfs = DIV_ROUND_UP(mlx5_sf_max_functions(dev),
530 MLX5_SFS_PER_CTRL_IRQ);
531 num_sf_ctrl = min_t(int, num_sf_ctrl_by_msix, num_sf_ctrl_by_sfs);
532 num_sf_ctrl = min_t(int, MLX5_IRQ_CTRL_SF_MAX, num_sf_ctrl);
533 table->sf_ctrl_pool = irq_pool_alloc(dev, pf_vec, num_sf_ctrl,
534 "mlx5_sf_ctrl",
535 MLX5_EQ_SHARE_IRQ_MIN_CTRL,
536 MLX5_EQ_SHARE_IRQ_MAX_CTRL);
537 if (IS_ERR(table->sf_ctrl_pool)) {
538 err = PTR_ERR(table->sf_ctrl_pool);
539 goto err_pf;
540 }
541 /* init sf_comp_pool */
542 table->sf_comp_pool = irq_pool_alloc(dev, pf_vec + num_sf_ctrl,
543 sf_vec - num_sf_ctrl, "mlx5_sf_comp",
544 MLX5_EQ_SHARE_IRQ_MIN_COMP,
545 MLX5_EQ_SHARE_IRQ_MAX_COMP);
546 if (IS_ERR(table->sf_comp_pool)) {
547 err = PTR_ERR(table->sf_comp_pool);
548 goto err_sf_ctrl;
549 }
550 return 0;
551 err_sf_ctrl:
552 irq_pool_free(table->sf_ctrl_pool);
553 err_pf:
554 irq_pool_free(table->pf_pool);
555 return err;
556 }
557
irq_pools_destroy(struct mlx5_irq_table * table)558 static void irq_pools_destroy(struct mlx5_irq_table *table)
559 {
560 if (table->sf_ctrl_pool) {
561 irq_pool_free(table->sf_comp_pool);
562 irq_pool_free(table->sf_ctrl_pool);
563 }
564 irq_pool_free(table->pf_pool);
565 }
566
567 /* irq_table API */
568
mlx5_irq_table_init(struct mlx5_core_dev * dev)569 int mlx5_irq_table_init(struct mlx5_core_dev *dev)
570 {
571 struct mlx5_irq_table *irq_table;
572
573 if (mlx5_core_is_sf(dev))
574 return 0;
575
576 irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL);
577 if (!irq_table)
578 return -ENOMEM;
579
580 dev->priv.irq_table = irq_table;
581 return 0;
582 }
583
mlx5_irq_table_cleanup(struct mlx5_core_dev * dev)584 void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev)
585 {
586 if (mlx5_core_is_sf(dev))
587 return;
588
589 kvfree(dev->priv.irq_table);
590 }
591
mlx5_irq_table_get_num_comp(struct mlx5_irq_table * table)592 int mlx5_irq_table_get_num_comp(struct mlx5_irq_table *table)
593 {
594 if (!table->pf_pool->xa_num_irqs.max)
595 return 1;
596 return table->pf_pool->xa_num_irqs.max - table->pf_pool->xa_num_irqs.min;
597 }
598
mlx5_irq_table_create(struct mlx5_core_dev * dev)599 int mlx5_irq_table_create(struct mlx5_core_dev *dev)
600 {
601 int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
602 MLX5_CAP_GEN(dev, max_num_eqs) :
603 1 << MLX5_CAP_GEN(dev, log_max_eq);
604 int total_vec;
605 int pf_vec;
606 int err;
607
608 if (mlx5_core_is_sf(dev))
609 return 0;
610
611 pf_vec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 1;
612 pf_vec = min_t(int, pf_vec, num_eqs);
613
614 total_vec = pf_vec;
615 if (mlx5_sf_max_functions(dev))
616 total_vec += MLX5_IRQ_CTRL_SF_MAX +
617 MLX5_COMP_EQS_PER_SF * mlx5_sf_max_functions(dev);
618
619 total_vec = pci_alloc_irq_vectors(dev->pdev, 1, total_vec, PCI_IRQ_MSIX);
620 if (total_vec < 0)
621 return total_vec;
622 pf_vec = min(pf_vec, total_vec);
623
624 err = irq_pools_init(dev, total_vec - pf_vec, pf_vec);
625 if (err)
626 pci_free_irq_vectors(dev->pdev);
627
628 return err;
629 }
630
mlx5_irq_table_destroy(struct mlx5_core_dev * dev)631 void mlx5_irq_table_destroy(struct mlx5_core_dev *dev)
632 {
633 struct mlx5_irq_table *table = dev->priv.irq_table;
634
635 if (mlx5_core_is_sf(dev))
636 return;
637
638 /* There are cases where IRQs still will be in used when we reaching
639 * to here. Hence, making sure all the irqs are released.
640 */
641 irq_pools_destroy(table);
642 pci_free_irq_vectors(dev->pdev);
643 }
644
mlx5_irq_table_get_sfs_vec(struct mlx5_irq_table * table)645 int mlx5_irq_table_get_sfs_vec(struct mlx5_irq_table *table)
646 {
647 if (table->sf_comp_pool)
648 return min_t(int, num_online_cpus(),
649 table->sf_comp_pool->xa_num_irqs.max -
650 table->sf_comp_pool->xa_num_irqs.min + 1);
651 else
652 return mlx5_irq_table_get_num_comp(table);
653 }
654
mlx5_irq_table_get(struct mlx5_core_dev * dev)655 struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev)
656 {
657 #ifdef CONFIG_MLX5_SF
658 if (mlx5_core_is_sf(dev))
659 return dev->priv.parent_mdev->priv.irq_table;
660 #endif
661 return dev->priv.irq_table;
662 }
663