1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Marvell. */
3 
4 #include "otx2_cpt_common.h"
5 #include "otx2_cptlf.h"
6 #include "rvu_reg.h"
7 
8 #define CPT_TIMER_HOLD 0x03F
9 #define CPT_COUNT_HOLD 32
10 
cptlf_do_set_done_time_wait(struct otx2_cptlf_info * lf,int time_wait)11 static void cptlf_do_set_done_time_wait(struct otx2_cptlf_info *lf,
12 					int time_wait)
13 {
14 	union otx2_cptx_lf_done_wait done_wait;
15 
16 	done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
17 				      OTX2_CPT_LF_DONE_WAIT);
18 	done_wait.s.time_wait = time_wait;
19 	otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
20 			 OTX2_CPT_LF_DONE_WAIT, done_wait.u);
21 }
22 
cptlf_do_set_done_num_wait(struct otx2_cptlf_info * lf,int num_wait)23 static void cptlf_do_set_done_num_wait(struct otx2_cptlf_info *lf, int num_wait)
24 {
25 	union otx2_cptx_lf_done_wait done_wait;
26 
27 	done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
28 				      OTX2_CPT_LF_DONE_WAIT);
29 	done_wait.s.num_wait = num_wait;
30 	otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
31 			 OTX2_CPT_LF_DONE_WAIT, done_wait.u);
32 }
33 
cptlf_set_done_time_wait(struct otx2_cptlfs_info * lfs,int time_wait)34 static void cptlf_set_done_time_wait(struct otx2_cptlfs_info *lfs,
35 				     int time_wait)
36 {
37 	int slot;
38 
39 	for (slot = 0; slot < lfs->lfs_num; slot++)
40 		cptlf_do_set_done_time_wait(&lfs->lf[slot], time_wait);
41 }
42 
cptlf_set_done_num_wait(struct otx2_cptlfs_info * lfs,int num_wait)43 static void cptlf_set_done_num_wait(struct otx2_cptlfs_info *lfs, int num_wait)
44 {
45 	int slot;
46 
47 	for (slot = 0; slot < lfs->lfs_num; slot++)
48 		cptlf_do_set_done_num_wait(&lfs->lf[slot], num_wait);
49 }
50 
cptlf_set_pri(struct otx2_cptlf_info * lf,int pri)51 static int cptlf_set_pri(struct otx2_cptlf_info *lf, int pri)
52 {
53 	struct otx2_cptlfs_info *lfs = lf->lfs;
54 	union otx2_cptx_af_lf_ctrl lf_ctrl;
55 	int ret;
56 
57 	ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev,
58 				   CPT_AF_LFX_CTL(lf->slot),
59 				   &lf_ctrl.u, lfs->blkaddr);
60 	if (ret)
61 		return ret;
62 
63 	lf_ctrl.s.pri = pri ? 1 : 0;
64 
65 	ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev,
66 				    CPT_AF_LFX_CTL(lf->slot),
67 				    lf_ctrl.u, lfs->blkaddr);
68 	return ret;
69 }
70 
cptlf_set_eng_grps_mask(struct otx2_cptlf_info * lf,int eng_grps_mask)71 static int cptlf_set_eng_grps_mask(struct otx2_cptlf_info *lf,
72 				   int eng_grps_mask)
73 {
74 	struct otx2_cptlfs_info *lfs = lf->lfs;
75 	union otx2_cptx_af_lf_ctrl lf_ctrl;
76 	int ret;
77 
78 	ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev,
79 				   CPT_AF_LFX_CTL(lf->slot),
80 				   &lf_ctrl.u, lfs->blkaddr);
81 	if (ret)
82 		return ret;
83 
84 	lf_ctrl.s.grp = eng_grps_mask;
85 
86 	ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev,
87 				    CPT_AF_LFX_CTL(lf->slot),
88 				    lf_ctrl.u, lfs->blkaddr);
89 	return ret;
90 }
91 
cptlf_set_grp_and_pri(struct otx2_cptlfs_info * lfs,int eng_grp_mask,int pri)92 static int cptlf_set_grp_and_pri(struct otx2_cptlfs_info *lfs,
93 				 int eng_grp_mask, int pri)
94 {
95 	int slot, ret = 0;
96 
97 	for (slot = 0; slot < lfs->lfs_num; slot++) {
98 		ret = cptlf_set_pri(&lfs->lf[slot], pri);
99 		if (ret)
100 			return ret;
101 
102 		ret = cptlf_set_eng_grps_mask(&lfs->lf[slot], eng_grp_mask);
103 		if (ret)
104 			return ret;
105 	}
106 	return ret;
107 }
108 
cptlf_hw_init(struct otx2_cptlfs_info * lfs)109 static void cptlf_hw_init(struct otx2_cptlfs_info *lfs)
110 {
111 	/* Disable instruction queues */
112 	otx2_cptlf_disable_iqueues(lfs);
113 
114 	/* Set instruction queues base addresses */
115 	otx2_cptlf_set_iqueues_base_addr(lfs);
116 
117 	/* Set instruction queues sizes */
118 	otx2_cptlf_set_iqueues_size(lfs);
119 
120 	/* Set done interrupts time wait */
121 	cptlf_set_done_time_wait(lfs, CPT_TIMER_HOLD);
122 
123 	/* Set done interrupts num wait */
124 	cptlf_set_done_num_wait(lfs, CPT_COUNT_HOLD);
125 
126 	/* Enable instruction queues */
127 	otx2_cptlf_enable_iqueues(lfs);
128 }
129 
cptlf_hw_cleanup(struct otx2_cptlfs_info * lfs)130 static void cptlf_hw_cleanup(struct otx2_cptlfs_info *lfs)
131 {
132 	/* Disable instruction queues */
133 	otx2_cptlf_disable_iqueues(lfs);
134 }
135 
cptlf_set_misc_intrs(struct otx2_cptlfs_info * lfs,u8 enable)136 static void cptlf_set_misc_intrs(struct otx2_cptlfs_info *lfs, u8 enable)
137 {
138 	union otx2_cptx_lf_misc_int_ena_w1s irq_misc = { .u = 0x0 };
139 	u64 reg = enable ? OTX2_CPT_LF_MISC_INT_ENA_W1S :
140 			   OTX2_CPT_LF_MISC_INT_ENA_W1C;
141 	int slot;
142 
143 	irq_misc.s.fault = 0x1;
144 	irq_misc.s.hwerr = 0x1;
145 	irq_misc.s.irde = 0x1;
146 	irq_misc.s.nqerr = 0x1;
147 	irq_misc.s.nwrp = 0x1;
148 
149 	for (slot = 0; slot < lfs->lfs_num; slot++)
150 		otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot, reg,
151 				 irq_misc.u);
152 }
153 
cptlf_enable_intrs(struct otx2_cptlfs_info * lfs)154 static void cptlf_enable_intrs(struct otx2_cptlfs_info *lfs)
155 {
156 	int slot;
157 
158 	/* Enable done interrupts */
159 	for (slot = 0; slot < lfs->lfs_num; slot++)
160 		otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot,
161 				 OTX2_CPT_LF_DONE_INT_ENA_W1S, 0x1);
162 	/* Enable Misc interrupts */
163 	cptlf_set_misc_intrs(lfs, true);
164 }
165 
cptlf_disable_intrs(struct otx2_cptlfs_info * lfs)166 static void cptlf_disable_intrs(struct otx2_cptlfs_info *lfs)
167 {
168 	int slot;
169 
170 	for (slot = 0; slot < lfs->lfs_num; slot++)
171 		otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot,
172 				 OTX2_CPT_LF_DONE_INT_ENA_W1C, 0x1);
173 	cptlf_set_misc_intrs(lfs, false);
174 }
175 
cptlf_read_done_cnt(struct otx2_cptlf_info * lf)176 static inline int cptlf_read_done_cnt(struct otx2_cptlf_info *lf)
177 {
178 	union otx2_cptx_lf_done irq_cnt;
179 
180 	irq_cnt.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
181 				    OTX2_CPT_LF_DONE);
182 	return irq_cnt.s.done;
183 }
184 
cptlf_misc_intr_handler(int __always_unused irq,void * arg)185 static irqreturn_t cptlf_misc_intr_handler(int __always_unused irq, void *arg)
186 {
187 	union otx2_cptx_lf_misc_int irq_misc, irq_misc_ack;
188 	struct otx2_cptlf_info *lf = arg;
189 	struct device *dev;
190 
191 	dev = &lf->lfs->pdev->dev;
192 	irq_misc.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
193 				     OTX2_CPT_LF_MISC_INT);
194 	irq_misc_ack.u = 0x0;
195 
196 	if (irq_misc.s.fault) {
197 		dev_err(dev, "Memory error detected while executing CPT_INST_S, LF %d.\n",
198 			lf->slot);
199 		irq_misc_ack.s.fault = 0x1;
200 
201 	} else if (irq_misc.s.hwerr) {
202 		dev_err(dev, "HW error from an engine executing CPT_INST_S, LF %d.",
203 			lf->slot);
204 		irq_misc_ack.s.hwerr = 0x1;
205 
206 	} else if (irq_misc.s.nwrp) {
207 		dev_err(dev, "SMMU fault while writing CPT_RES_S to CPT_INST_S[RES_ADDR], LF %d.\n",
208 			lf->slot);
209 		irq_misc_ack.s.nwrp = 0x1;
210 
211 	} else if (irq_misc.s.irde) {
212 		dev_err(dev, "Memory error when accessing instruction memory queue CPT_LF_Q_BASE[ADDR].\n");
213 		irq_misc_ack.s.irde = 0x1;
214 
215 	} else if (irq_misc.s.nqerr) {
216 		dev_err(dev, "Error enqueuing an instruction received at CPT_LF_NQ.\n");
217 		irq_misc_ack.s.nqerr = 0x1;
218 
219 	} else {
220 		dev_err(dev, "Unhandled interrupt in CPT LF %d\n", lf->slot);
221 		return IRQ_NONE;
222 	}
223 
224 	/* Acknowledge interrupts */
225 	otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
226 			 OTX2_CPT_LF_MISC_INT, irq_misc_ack.u);
227 
228 	return IRQ_HANDLED;
229 }
230 
cptlf_done_intr_handler(int irq,void * arg)231 static irqreturn_t cptlf_done_intr_handler(int irq, void *arg)
232 {
233 	union otx2_cptx_lf_done_wait done_wait;
234 	struct otx2_cptlf_info *lf = arg;
235 	int irq_cnt;
236 
237 	/* Read the number of completed requests */
238 	irq_cnt = cptlf_read_done_cnt(lf);
239 	if (irq_cnt) {
240 		done_wait.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0,
241 					      lf->slot, OTX2_CPT_LF_DONE_WAIT);
242 		/* Acknowledge the number of completed requests */
243 		otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
244 				 OTX2_CPT_LF_DONE_ACK, irq_cnt);
245 
246 		otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
247 				 OTX2_CPT_LF_DONE_WAIT, done_wait.u);
248 		if (unlikely(!lf->wqe)) {
249 			dev_err(&lf->lfs->pdev->dev, "No work for LF %d\n",
250 				lf->slot);
251 			return IRQ_NONE;
252 		}
253 
254 		/* Schedule processing of completed requests */
255 		tasklet_hi_schedule(&lf->wqe->work);
256 	}
257 	return IRQ_HANDLED;
258 }
259 
otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info * lfs)260 void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs)
261 {
262 	int i, offs, vector;
263 
264 	for (i = 0; i < lfs->lfs_num; i++) {
265 		for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) {
266 			if (!lfs->lf[i].is_irq_reg[offs])
267 				continue;
268 
269 			vector = pci_irq_vector(lfs->pdev,
270 						lfs->lf[i].msix_offset + offs);
271 			free_irq(vector, &lfs->lf[i]);
272 			lfs->lf[i].is_irq_reg[offs] = false;
273 		}
274 	}
275 	cptlf_disable_intrs(lfs);
276 }
277 
cptlf_do_register_interrrupts(struct otx2_cptlfs_info * lfs,int lf_num,int irq_offset,irq_handler_t handler)278 static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs,
279 					 int lf_num, int irq_offset,
280 					 irq_handler_t handler)
281 {
282 	int ret, vector;
283 
284 	vector = pci_irq_vector(lfs->pdev, lfs->lf[lf_num].msix_offset +
285 				irq_offset);
286 	ret = request_irq(vector, handler, 0,
287 			  lfs->lf[lf_num].irq_name[irq_offset],
288 			  &lfs->lf[lf_num]);
289 	if (ret)
290 		return ret;
291 
292 	lfs->lf[lf_num].is_irq_reg[irq_offset] = true;
293 
294 	return ret;
295 }
296 
otx2_cptlf_register_interrupts(struct otx2_cptlfs_info * lfs)297 int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs)
298 {
299 	int irq_offs, ret, i;
300 
301 	for (i = 0; i < lfs->lfs_num; i++) {
302 		irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC;
303 		snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPTLF Misc%d", i);
304 		ret = cptlf_do_register_interrrupts(lfs, i, irq_offs,
305 						    cptlf_misc_intr_handler);
306 		if (ret)
307 			goto free_irq;
308 
309 		irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE;
310 		snprintf(lfs->lf[i].irq_name[irq_offs], 32, "OTX2_CPTLF Done%d",
311 			 i);
312 		ret = cptlf_do_register_interrrupts(lfs, i, irq_offs,
313 						    cptlf_done_intr_handler);
314 		if (ret)
315 			goto free_irq;
316 	}
317 	cptlf_enable_intrs(lfs);
318 	return 0;
319 
320 free_irq:
321 	otx2_cptlf_unregister_interrupts(lfs);
322 	return ret;
323 }
324 
otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info * lfs)325 void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs)
326 {
327 	int slot, offs;
328 
329 	for (slot = 0; slot < lfs->lfs_num; slot++) {
330 		for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++)
331 			irq_set_affinity_hint(pci_irq_vector(lfs->pdev,
332 					      lfs->lf[slot].msix_offset +
333 					      offs), NULL);
334 		free_cpumask_var(lfs->lf[slot].affinity_mask);
335 	}
336 }
337 
otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info * lfs)338 int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs)
339 {
340 	struct otx2_cptlf_info *lf = lfs->lf;
341 	int slot, offs, ret;
342 
343 	for (slot = 0; slot < lfs->lfs_num; slot++) {
344 		if (!zalloc_cpumask_var(&lf[slot].affinity_mask, GFP_KERNEL)) {
345 			dev_err(&lfs->pdev->dev,
346 				"cpumask allocation failed for LF %d", slot);
347 			ret = -ENOMEM;
348 			goto free_affinity_mask;
349 		}
350 
351 		cpumask_set_cpu(cpumask_local_spread(slot,
352 				dev_to_node(&lfs->pdev->dev)),
353 				lf[slot].affinity_mask);
354 
355 		for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) {
356 			ret = irq_set_affinity_hint(pci_irq_vector(lfs->pdev,
357 						lf[slot].msix_offset + offs),
358 						lf[slot].affinity_mask);
359 			if (ret)
360 				goto free_affinity_mask;
361 		}
362 	}
363 	return 0;
364 
365 free_affinity_mask:
366 	otx2_cptlf_free_irqs_affinity(lfs);
367 	return ret;
368 }
369 
otx2_cptlf_init(struct otx2_cptlfs_info * lfs,u8 eng_grp_mask,int pri,int lfs_num)370 int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri,
371 		    int lfs_num)
372 {
373 	int slot, ret;
374 
375 	if (!lfs->pdev || !lfs->reg_base)
376 		return -EINVAL;
377 
378 	lfs->lfs_num = lfs_num;
379 	for (slot = 0; slot < lfs->lfs_num; slot++) {
380 		lfs->lf[slot].lfs = lfs;
381 		lfs->lf[slot].slot = slot;
382 		if (lfs->lmt_base)
383 			lfs->lf[slot].lmtline = lfs->lmt_base +
384 						(slot * LMTLINE_SIZE);
385 		else
386 			lfs->lf[slot].lmtline = lfs->reg_base +
387 				OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot,
388 						 OTX2_CPT_LMT_LF_LMTLINEX(0));
389 
390 		lfs->lf[slot].ioreg = lfs->reg_base +
391 			OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_CPT0, slot,
392 						 OTX2_CPT_LF_NQX(0));
393 	}
394 	/* Send request to attach LFs */
395 	ret = otx2_cpt_attach_rscrs_msg(lfs);
396 	if (ret)
397 		goto clear_lfs_num;
398 
399 	ret = otx2_cpt_alloc_instruction_queues(lfs);
400 	if (ret) {
401 		dev_err(&lfs->pdev->dev,
402 			"Allocating instruction queues failed\n");
403 		goto detach_rsrcs;
404 	}
405 	cptlf_hw_init(lfs);
406 	/*
407 	 * Allow each LF to execute requests destined to any of 8 engine
408 	 * groups and set queue priority of each LF to high
409 	 */
410 	ret = cptlf_set_grp_and_pri(lfs, eng_grp_mask, pri);
411 	if (ret)
412 		goto free_iq;
413 
414 	return 0;
415 
416 free_iq:
417 	otx2_cpt_free_instruction_queues(lfs);
418 	cptlf_hw_cleanup(lfs);
419 detach_rsrcs:
420 	otx2_cpt_detach_rsrcs_msg(lfs);
421 clear_lfs_num:
422 	lfs->lfs_num = 0;
423 	return ret;
424 }
425 
otx2_cptlf_shutdown(struct otx2_cptlfs_info * lfs)426 void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs)
427 {
428 	lfs->lfs_num = 0;
429 	/* Cleanup LFs hardware side */
430 	cptlf_hw_cleanup(lfs);
431 	/* Send request to detach LFs */
432 	otx2_cpt_detach_rsrcs_msg(lfs);
433 }
434