1 /* SPDX-License-Identifier: GPL-2.0-only
2  * Copyright (C) 2020 Marvell.
3  */
4 #ifndef __OTX2_CPTLF_H
5 #define __OTX2_CPTLF_H
6 
7 #include <linux/soc/marvell/octeontx2/asm.h>
8 #include <mbox.h>
9 #include <rvu.h>
10 #include "otx2_cpt_common.h"
11 #include "otx2_cpt_reqmgr.h"
12 
13 /*
14  * CPT instruction and pending queues user requested length in CPT_INST_S msgs
15  */
16 #define OTX2_CPT_USER_REQUESTED_QLEN_MSGS 8200
17 
18 /*
19  * CPT instruction queue size passed to HW is in units of 40*CPT_INST_S
20  * messages.
21  */
22 #define OTX2_CPT_SIZE_DIV40 (OTX2_CPT_USER_REQUESTED_QLEN_MSGS/40)
23 
24 /*
25  * CPT instruction and pending queues length in CPT_INST_S messages
26  */
27 #define OTX2_CPT_INST_QLEN_MSGS	((OTX2_CPT_SIZE_DIV40 - 1) * 40)
28 
29 /* CPT instruction queue length in bytes */
30 #define OTX2_CPT_INST_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 40 * \
31 				  OTX2_CPT_INST_SIZE)
32 
33 /* CPT instruction group queue length in bytes */
34 #define OTX2_CPT_INST_GRP_QLEN_BYTES (OTX2_CPT_SIZE_DIV40 * 16)
35 
36 /* CPT FC length in bytes */
37 #define OTX2_CPT_Q_FC_LEN 128
38 
39 /* CPT instruction queue alignment */
40 #define OTX2_CPT_INST_Q_ALIGNMENT  128
41 
42 /* Mask which selects all engine groups */
43 #define OTX2_CPT_ALL_ENG_GRPS_MASK 0xFF
44 
45 /* Maximum LFs supported in OcteonTX2 for CPT */
46 #define OTX2_CPT_MAX_LFS_NUM    64
47 
48 /* Queue priority */
49 #define OTX2_CPT_QUEUE_HI_PRIO  0x1
50 #define OTX2_CPT_QUEUE_LOW_PRIO 0x0
51 
52 enum otx2_cptlf_state {
53 	OTX2_CPTLF_IN_RESET,
54 	OTX2_CPTLF_STARTED,
55 };
56 
57 struct otx2_cpt_inst_queue {
58 	u8 *vaddr;
59 	u8 *real_vaddr;
60 	dma_addr_t dma_addr;
61 	dma_addr_t real_dma_addr;
62 	u32 size;
63 };
64 
65 struct otx2_cptlfs_info;
66 struct otx2_cptlf_wqe {
67 	struct tasklet_struct work;
68 	struct otx2_cptlfs_info *lfs;
69 	u8 lf_num;
70 };
71 
72 struct otx2_cptlf_info {
73 	struct otx2_cptlfs_info *lfs;           /* Ptr to cptlfs_info struct */
74 	void __iomem *lmtline;                  /* Address of LMTLINE */
75 	void __iomem *ioreg;                    /* LMTLINE send register */
76 	int msix_offset;                        /* MSI-X interrupts offset */
77 	cpumask_var_t affinity_mask;            /* IRQs affinity mask */
78 	u8 irq_name[OTX2_CPT_LF_MSIX_VECTORS][32];/* Interrupts name */
79 	u8 is_irq_reg[OTX2_CPT_LF_MSIX_VECTORS];  /* Is interrupt registered */
80 	u8 slot;                                /* Slot number of this LF */
81 
82 	struct otx2_cpt_inst_queue iqueue;/* Instruction queue */
83 	struct otx2_cpt_pending_queue pqueue; /* Pending queue */
84 	struct otx2_cptlf_wqe *wqe;       /* Tasklet work info */
85 };
86 
87 struct cpt_hw_ops {
88 	void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num,
89 			 struct otx2_cptlf_info *lf);
90 	u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result);
91 	u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result);
92 };
93 
94 struct otx2_cptlfs_info {
95 	/* Registers start address of VF/PF LFs are attached to */
96 	void __iomem *reg_base;
97 #define LMTLINE_SIZE  128
98 	void __iomem *lmt_base;
99 	struct pci_dev *pdev;   /* Device LFs are attached to */
100 	struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM];
101 	struct otx2_mbox *mbox;
102 	struct cpt_hw_ops *ops;
103 	u8 are_lfs_attached;	/* Whether CPT LFs are attached */
104 	u8 lfs_num;		/* Number of CPT LFs */
105 	u8 kcrypto_eng_grp_num;	/* Kernel crypto engine group number */
106 	u8 kvf_limits;          /* Kernel crypto limits */
107 	atomic_t state;         /* LF's state. started/reset */
108 	int blkaddr;            /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */
109 };
110 
otx2_cpt_free_instruction_queues(struct otx2_cptlfs_info * lfs)111 static inline void otx2_cpt_free_instruction_queues(
112 					struct otx2_cptlfs_info *lfs)
113 {
114 	struct otx2_cpt_inst_queue *iq;
115 	int i;
116 
117 	for (i = 0; i < lfs->lfs_num; i++) {
118 		iq = &lfs->lf[i].iqueue;
119 		if (iq->real_vaddr)
120 			dma_free_coherent(&lfs->pdev->dev,
121 					  iq->size,
122 					  iq->real_vaddr,
123 					  iq->real_dma_addr);
124 		iq->real_vaddr = NULL;
125 		iq->vaddr = NULL;
126 	}
127 }
128 
otx2_cpt_alloc_instruction_queues(struct otx2_cptlfs_info * lfs)129 static inline int otx2_cpt_alloc_instruction_queues(
130 					struct otx2_cptlfs_info *lfs)
131 {
132 	struct otx2_cpt_inst_queue *iq;
133 	int ret = 0, i;
134 
135 	if (!lfs->lfs_num)
136 		return -EINVAL;
137 
138 	for (i = 0; i < lfs->lfs_num; i++) {
139 		iq = &lfs->lf[i].iqueue;
140 		iq->size = OTX2_CPT_INST_QLEN_BYTES +
141 			   OTX2_CPT_Q_FC_LEN +
142 			   OTX2_CPT_INST_GRP_QLEN_BYTES +
143 			   OTX2_CPT_INST_Q_ALIGNMENT;
144 		iq->real_vaddr = dma_alloc_coherent(&lfs->pdev->dev, iq->size,
145 					&iq->real_dma_addr, GFP_KERNEL);
146 		if (!iq->real_vaddr) {
147 			ret = -ENOMEM;
148 			goto error;
149 		}
150 		iq->vaddr = iq->real_vaddr + OTX2_CPT_INST_GRP_QLEN_BYTES;
151 		iq->dma_addr = iq->real_dma_addr + OTX2_CPT_INST_GRP_QLEN_BYTES;
152 
153 		/* Align pointers */
154 		iq->vaddr = PTR_ALIGN(iq->vaddr, OTX2_CPT_INST_Q_ALIGNMENT);
155 		iq->dma_addr = PTR_ALIGN(iq->dma_addr,
156 					 OTX2_CPT_INST_Q_ALIGNMENT);
157 	}
158 	return 0;
159 
160 error:
161 	otx2_cpt_free_instruction_queues(lfs);
162 	return ret;
163 }
164 
otx2_cptlf_set_iqueues_base_addr(struct otx2_cptlfs_info * lfs)165 static inline void otx2_cptlf_set_iqueues_base_addr(
166 					struct otx2_cptlfs_info *lfs)
167 {
168 	union otx2_cptx_lf_q_base lf_q_base;
169 	int slot;
170 
171 	for (slot = 0; slot < lfs->lfs_num; slot++) {
172 		lf_q_base.u = lfs->lf[slot].iqueue.dma_addr;
173 		otx2_cpt_write64(lfs->reg_base, BLKADDR_CPT0, slot,
174 				 OTX2_CPT_LF_Q_BASE, lf_q_base.u);
175 	}
176 }
177 
otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info * lf)178 static inline void otx2_cptlf_do_set_iqueue_size(struct otx2_cptlf_info *lf)
179 {
180 	union otx2_cptx_lf_q_size lf_q_size = { .u = 0x0 };
181 
182 	lf_q_size.s.size_div40 = OTX2_CPT_SIZE_DIV40;
183 	otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
184 			 OTX2_CPT_LF_Q_SIZE, lf_q_size.u);
185 }
186 
otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info * lfs)187 static inline void otx2_cptlf_set_iqueues_size(struct otx2_cptlfs_info *lfs)
188 {
189 	int slot;
190 
191 	for (slot = 0; slot < lfs->lfs_num; slot++)
192 		otx2_cptlf_do_set_iqueue_size(&lfs->lf[slot]);
193 }
194 
otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info * lf)195 static inline void otx2_cptlf_do_disable_iqueue(struct otx2_cptlf_info *lf)
196 {
197 	union otx2_cptx_lf_ctl lf_ctl = { .u = 0x0 };
198 	union otx2_cptx_lf_inprog lf_inprog;
199 	int timeout = 20;
200 
201 	/* Disable instructions enqueuing */
202 	otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
203 			 OTX2_CPT_LF_CTL, lf_ctl.u);
204 
205 	/* Wait for instruction queue to become empty */
206 	do {
207 		lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0,
208 					      lf->slot, OTX2_CPT_LF_INPROG);
209 		if (!lf_inprog.s.inflight)
210 			break;
211 
212 		usleep_range(10000, 20000);
213 		if (timeout-- < 0) {
214 			dev_err(&lf->lfs->pdev->dev,
215 				"Error LF %d is still busy.\n", lf->slot);
216 			break;
217 		}
218 
219 	} while (1);
220 
221 	/*
222 	 * Disable executions in the LF's queue,
223 	 * the queue should be empty at this point
224 	 */
225 	lf_inprog.s.eena = 0x0;
226 	otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
227 			 OTX2_CPT_LF_INPROG, lf_inprog.u);
228 }
229 
otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info * lfs)230 static inline void otx2_cptlf_disable_iqueues(struct otx2_cptlfs_info *lfs)
231 {
232 	int slot;
233 
234 	for (slot = 0; slot < lfs->lfs_num; slot++)
235 		otx2_cptlf_do_disable_iqueue(&lfs->lf[slot]);
236 }
237 
otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info * lf,bool enable)238 static inline void otx2_cptlf_set_iqueue_enq(struct otx2_cptlf_info *lf,
239 					     bool enable)
240 {
241 	union otx2_cptx_lf_ctl lf_ctl;
242 
243 	lf_ctl.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
244 				   OTX2_CPT_LF_CTL);
245 
246 	/* Set iqueue's enqueuing */
247 	lf_ctl.s.ena = enable ? 0x1 : 0x0;
248 	otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
249 			 OTX2_CPT_LF_CTL, lf_ctl.u);
250 }
251 
otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info * lf)252 static inline void otx2_cptlf_enable_iqueue_enq(struct otx2_cptlf_info *lf)
253 {
254 	otx2_cptlf_set_iqueue_enq(lf, true);
255 }
256 
otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info * lf,bool enable)257 static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf,
258 					      bool enable)
259 {
260 	union otx2_cptx_lf_inprog lf_inprog;
261 
262 	lf_inprog.u = otx2_cpt_read64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
263 				      OTX2_CPT_LF_INPROG);
264 
265 	/* Set iqueue's execution */
266 	lf_inprog.s.eena = enable ? 0x1 : 0x0;
267 	otx2_cpt_write64(lf->lfs->reg_base, BLKADDR_CPT0, lf->slot,
268 			 OTX2_CPT_LF_INPROG, lf_inprog.u);
269 }
270 
otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info * lf)271 static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf)
272 {
273 	otx2_cptlf_set_iqueue_exec(lf, true);
274 }
275 
otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info * lf)276 static inline void otx2_cptlf_disable_iqueue_exec(struct otx2_cptlf_info *lf)
277 {
278 	otx2_cptlf_set_iqueue_exec(lf, false);
279 }
280 
otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info * lfs)281 static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs)
282 {
283 	int slot;
284 
285 	for (slot = 0; slot < lfs->lfs_num; slot++) {
286 		otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]);
287 		otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]);
288 	}
289 }
290 
otx2_cpt_fill_inst(union otx2_cpt_inst_s * cptinst,struct otx2_cpt_iq_command * iq_cmd,u64 comp_baddr)291 static inline void otx2_cpt_fill_inst(union otx2_cpt_inst_s *cptinst,
292 				      struct otx2_cpt_iq_command *iq_cmd,
293 				      u64 comp_baddr)
294 {
295 	cptinst->u[0] = 0x0;
296 	cptinst->s.doneint = true;
297 	cptinst->s.res_addr = comp_baddr;
298 	cptinst->u[2] = 0x0;
299 	cptinst->u[3] = 0x0;
300 	cptinst->s.ei0 = iq_cmd->cmd.u;
301 	cptinst->s.ei1 = iq_cmd->dptr;
302 	cptinst->s.ei2 = iq_cmd->rptr;
303 	cptinst->s.ei3 = iq_cmd->cptr.u;
304 }
305 
306 /*
307  * On OcteonTX2 platform the parameter insts_num is used as a count of
308  * instructions to be enqueued. The valid values for insts_num are:
309  * 1 - 1 CPT instruction will be enqueued during LMTST operation
310  * 2 - 2 CPT instructions will be enqueued during LMTST operation
311  */
otx2_cpt_send_cmd(union otx2_cpt_inst_s * cptinst,u32 insts_num,struct otx2_cptlf_info * lf)312 static inline void otx2_cpt_send_cmd(union otx2_cpt_inst_s *cptinst,
313 				     u32 insts_num, struct otx2_cptlf_info *lf)
314 {
315 	void __iomem *lmtline = lf->lmtline;
316 	long ret;
317 
318 	/*
319 	 * Make sure memory areas pointed in CPT_INST_S
320 	 * are flushed before the instruction is sent to CPT
321 	 */
322 	dma_wmb();
323 
324 	do {
325 		/* Copy CPT command to LMTLINE */
326 		memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE);
327 
328 		/*
329 		 * LDEOR initiates atomic transfer to I/O device
330 		 * The following will cause the LMTST to fail (the LDEOR
331 		 * returns zero):
332 		 * - No stores have been performed to the LMTLINE since it was
333 		 * last invalidated.
334 		 * - The bytes which have been stored to LMTLINE since it was
335 		 * last invalidated form a pattern that is non-contiguous, does
336 		 * not start at byte 0, or does not end on a 8-byte boundary.
337 		 * (i.e.comprises a formation of other than 1–16 8-byte
338 		 * words.)
339 		 *
340 		 * These rules are designed such that an operating system
341 		 * context switch or hypervisor guest switch need have no
342 		 * knowledge of the LMTST operations; the switch code does not
343 		 * need to store to LMTCANCEL. Also note as LMTLINE data cannot
344 		 * be read, there is no information leakage between processes.
345 		 */
346 		ret = otx2_lmt_flush(lf->ioreg);
347 
348 	} while (!ret);
349 }
350 
otx2_cptlf_started(struct otx2_cptlfs_info * lfs)351 static inline bool otx2_cptlf_started(struct otx2_cptlfs_info *lfs)
352 {
353 	return atomic_read(&lfs->state) == OTX2_CPTLF_STARTED;
354 }
355 
356 int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri,
357 		    int lfs_num);
358 void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs);
359 int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs);
360 void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs);
361 void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs);
362 int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs);
363 
364 #endif /* __OTX2_CPTLF_H */
365