1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
3 
4 #include <linux/kernel.h>
5 #include <linux/nospec.h>
6 #include "cc_driver.h"
7 #include "cc_buffer_mgr.h"
8 #include "cc_request_mgr.h"
9 #include "cc_pm.h"
10 
11 #define CC_MAX_POLL_ITER	10
12 /* The highest descriptor count in used */
13 #define CC_MAX_DESC_SEQ_LEN	23
14 
15 struct cc_req_mgr_handle {
16 	/* Request manager resources */
17 	unsigned int hw_queue_size; /* HW capability */
18 	unsigned int min_free_hw_slots;
19 	unsigned int max_used_sw_slots;
20 	struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];
21 	u32 req_queue_head;
22 	u32 req_queue_tail;
23 	u32 axi_completed;
24 	u32 q_free_slots;
25 	/* This lock protects access to HW register
26 	 * that must be single request at a time
27 	 */
28 	spinlock_t hw_lock;
29 	struct cc_hw_desc compl_desc;
30 	u8 *dummy_comp_buff;
31 	dma_addr_t dummy_comp_buff_dma;
32 
33 	/* backlog queue */
34 	struct list_head backlog;
35 	unsigned int bl_len;
36 	spinlock_t bl_lock; /* protect backlog queue */
37 
38 #ifdef COMP_IN_WQ
39 	struct workqueue_struct *workq;
40 	struct delayed_work compwork;
41 #else
42 	struct tasklet_struct comptask;
43 #endif
44 };
45 
46 struct cc_bl_item {
47 	struct cc_crypto_req creq;
48 	struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN];
49 	unsigned int len;
50 	struct list_head list;
51 	bool notif;
52 };
53 
54 static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = {
55 	{ BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT),
56 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT),
57 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT),
58 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT),
59 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT),
60 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT),
61 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT),
62 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) },
63 	{ BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT),
64 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT),
65 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT),
66 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT),
67 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT),
68 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT),
69 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT),
70 	  BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) }
71 };
72 
73 static void comp_handler(unsigned long devarg);
74 #ifdef COMP_IN_WQ
75 static void comp_work_handler(struct work_struct *work);
76 #endif
77 
cc_cpp_int_mask(enum cc_cpp_alg alg,int slot)78 static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot)
79 {
80 	alg = array_index_nospec(alg, CC_CPP_NUM_ALGS);
81 	slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS);
82 
83 	return cc_cpp_int_masks[alg][slot];
84 }
85 
cc_req_mgr_fini(struct cc_drvdata * drvdata)86 void cc_req_mgr_fini(struct cc_drvdata *drvdata)
87 {
88 	struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
89 	struct device *dev = drvdata_to_dev(drvdata);
90 
91 	if (!req_mgr_h)
92 		return; /* Not allocated */
93 
94 	if (req_mgr_h->dummy_comp_buff_dma) {
95 		dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff,
96 				  req_mgr_h->dummy_comp_buff_dma);
97 	}
98 
99 	dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -
100 						req_mgr_h->min_free_hw_slots));
101 	dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
102 
103 #ifdef COMP_IN_WQ
104 	flush_workqueue(req_mgr_h->workq);
105 	destroy_workqueue(req_mgr_h->workq);
106 #else
107 	/* Kill tasklet */
108 	tasklet_kill(&req_mgr_h->comptask);
109 #endif
110 	kfree_sensitive(req_mgr_h);
111 	drvdata->request_mgr_handle = NULL;
112 }
113 
cc_req_mgr_init(struct cc_drvdata * drvdata)114 int cc_req_mgr_init(struct cc_drvdata *drvdata)
115 {
116 	struct cc_req_mgr_handle *req_mgr_h;
117 	struct device *dev = drvdata_to_dev(drvdata);
118 	int rc = 0;
119 
120 	req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL);
121 	if (!req_mgr_h) {
122 		rc = -ENOMEM;
123 		goto req_mgr_init_err;
124 	}
125 
126 	drvdata->request_mgr_handle = req_mgr_h;
127 
128 	spin_lock_init(&req_mgr_h->hw_lock);
129 	spin_lock_init(&req_mgr_h->bl_lock);
130 	INIT_LIST_HEAD(&req_mgr_h->backlog);
131 
132 #ifdef COMP_IN_WQ
133 	dev_dbg(dev, "Initializing completion workqueue\n");
134 	req_mgr_h->workq = create_singlethread_workqueue("ccree");
135 	if (!req_mgr_h->workq) {
136 		dev_err(dev, "Failed creating work queue\n");
137 		rc = -ENOMEM;
138 		goto req_mgr_init_err;
139 	}
140 	INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);
141 #else
142 	dev_dbg(dev, "Initializing completion tasklet\n");
143 	tasklet_init(&req_mgr_h->comptask, comp_handler,
144 		     (unsigned long)drvdata);
145 #endif
146 	req_mgr_h->hw_queue_size = cc_ioread(drvdata,
147 					     CC_REG(DSCRPTR_QUEUE_SRAM_SIZE));
148 	dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
149 	if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
150 		dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n",
151 			req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
152 		rc = -ENOMEM;
153 		goto req_mgr_init_err;
154 	}
155 	req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;
156 	req_mgr_h->max_used_sw_slots = 0;
157 
158 	/* Allocate DMA word for "dummy" completion descriptor use */
159 	req_mgr_h->dummy_comp_buff =
160 		dma_alloc_coherent(dev, sizeof(u32),
161 				   &req_mgr_h->dummy_comp_buff_dma,
162 				   GFP_KERNEL);
163 	if (!req_mgr_h->dummy_comp_buff) {
164 		dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n",
165 			sizeof(u32));
166 		rc = -ENOMEM;
167 		goto req_mgr_init_err;
168 	}
169 
170 	/* Init. "dummy" completion descriptor */
171 	hw_desc_init(&req_mgr_h->compl_desc);
172 	set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32));
173 	set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma,
174 		      sizeof(u32), NS_BIT, 1);
175 	set_flow_mode(&req_mgr_h->compl_desc, BYPASS);
176 	set_queue_last_ind(drvdata, &req_mgr_h->compl_desc);
177 
178 	return 0;
179 
180 req_mgr_init_err:
181 	cc_req_mgr_fini(drvdata);
182 	return rc;
183 }
184 
enqueue_seq(struct cc_drvdata * drvdata,struct cc_hw_desc seq[],unsigned int seq_len)185 static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[],
186 			unsigned int seq_len)
187 {
188 	int i, w;
189 	void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0);
190 	struct device *dev = drvdata_to_dev(drvdata);
191 
192 	/*
193 	 * We do indeed write all 6 command words to the same
194 	 * register. The HW supports this.
195 	 */
196 
197 	for (i = 0; i < seq_len; i++) {
198 		for (w = 0; w <= 5; w++)
199 			writel_relaxed(seq[i].word[w], reg);
200 
201 		if (cc_dump_desc)
202 			dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
203 				i, seq[i].word[0], seq[i].word[1],
204 				seq[i].word[2], seq[i].word[3],
205 				seq[i].word[4], seq[i].word[5]);
206 	}
207 }
208 
209 /**
210  * request_mgr_complete() - Completion will take place if and only if user
211  * requested completion by cc_send_sync_request().
212  *
213  * @dev: Device pointer
214  * @dx_compl_h: The completion event to signal
215  * @dummy: unused error code
216  */
request_mgr_complete(struct device * dev,void * dx_compl_h,int dummy)217 static void request_mgr_complete(struct device *dev, void *dx_compl_h,
218 				 int dummy)
219 {
220 	struct completion *this_compl = dx_compl_h;
221 
222 	complete(this_compl);
223 }
224 
cc_queues_status(struct cc_drvdata * drvdata,struct cc_req_mgr_handle * req_mgr_h,unsigned int total_seq_len)225 static int cc_queues_status(struct cc_drvdata *drvdata,
226 			    struct cc_req_mgr_handle *req_mgr_h,
227 			    unsigned int total_seq_len)
228 {
229 	unsigned long poll_queue;
230 	struct device *dev = drvdata_to_dev(drvdata);
231 
232 	/* SW queue is checked only once as it will not
233 	 * be changed during the poll because the spinlock_bh
234 	 * is held by the thread
235 	 */
236 	if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) ==
237 	    req_mgr_h->req_queue_tail) {
238 		dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
239 			req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
240 		return -ENOSPC;
241 	}
242 
243 	if (req_mgr_h->q_free_slots >= total_seq_len)
244 		return 0;
245 
246 	/* Wait for space in HW queue. Poll constant num of iterations. */
247 	for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) {
248 		req_mgr_h->q_free_slots =
249 			cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
250 		if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)
251 			req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;
252 
253 		if (req_mgr_h->q_free_slots >= total_seq_len) {
254 			/* If there is enough place return */
255 			return 0;
256 		}
257 
258 		dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
259 			req_mgr_h->q_free_slots, total_seq_len);
260 	}
261 	/* No room in the HW queue try again later */
262 	dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",
263 		req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE,
264 		req_mgr_h->q_free_slots, total_seq_len);
265 	return -ENOSPC;
266 }
267 
268 /**
269  * cc_do_send_request() - Enqueue caller request to crypto hardware.
270  * Need to be called with HW lock held and PM running
271  *
272  * @drvdata: Associated device driver context
273  * @cc_req: The request to enqueue
274  * @desc: The crypto sequence
275  * @len: The crypto sequence length
276  * @add_comp: If "true": add an artificial dout DMA to mark completion
277  *
278  */
cc_do_send_request(struct cc_drvdata * drvdata,struct cc_crypto_req * cc_req,struct cc_hw_desc * desc,unsigned int len,bool add_comp)279 static void cc_do_send_request(struct cc_drvdata *drvdata,
280 			       struct cc_crypto_req *cc_req,
281 			       struct cc_hw_desc *desc, unsigned int len,
282 			       bool add_comp)
283 {
284 	struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
285 	unsigned int used_sw_slots;
286 	unsigned int total_seq_len = len; /*initial sequence length*/
287 	struct device *dev = drvdata_to_dev(drvdata);
288 
289 	used_sw_slots = ((req_mgr_h->req_queue_head -
290 			  req_mgr_h->req_queue_tail) &
291 			 (MAX_REQUEST_QUEUE_SIZE - 1));
292 	if (used_sw_slots > req_mgr_h->max_used_sw_slots)
293 		req_mgr_h->max_used_sw_slots = used_sw_slots;
294 
295 	/* Enqueue request - must be locked with HW lock*/
296 	req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req;
297 	req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) &
298 				    (MAX_REQUEST_QUEUE_SIZE - 1);
299 
300 	dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head);
301 
302 	/*
303 	 * We are about to push command to the HW via the command registers
304 	 * that may reference host memory. We need to issue a memory barrier
305 	 * to make sure there are no outstanding memory writes
306 	 */
307 	wmb();
308 
309 	/* STAT_PHASE_4: Push sequence */
310 
311 	enqueue_seq(drvdata, desc, len);
312 
313 	if (add_comp) {
314 		enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1);
315 		total_seq_len++;
316 	}
317 
318 	if (req_mgr_h->q_free_slots < total_seq_len) {
319 		/* This situation should never occur. Maybe indicating problem
320 		 * with resuming power. Set the free slot count to 0 and hope
321 		 * for the best.
322 		 */
323 		dev_err(dev, "HW free slot count mismatch.");
324 		req_mgr_h->q_free_slots = 0;
325 	} else {
326 		/* Update the free slots in HW queue */
327 		req_mgr_h->q_free_slots -= total_seq_len;
328 	}
329 }
330 
cc_enqueue_backlog(struct cc_drvdata * drvdata,struct cc_bl_item * bli)331 static void cc_enqueue_backlog(struct cc_drvdata *drvdata,
332 			       struct cc_bl_item *bli)
333 {
334 	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
335 	struct device *dev = drvdata_to_dev(drvdata);
336 
337 	spin_lock_bh(&mgr->bl_lock);
338 	list_add_tail(&bli->list, &mgr->backlog);
339 	++mgr->bl_len;
340 	dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len);
341 	spin_unlock_bh(&mgr->bl_lock);
342 	tasklet_schedule(&mgr->comptask);
343 }
344 
cc_proc_backlog(struct cc_drvdata * drvdata)345 static void cc_proc_backlog(struct cc_drvdata *drvdata)
346 {
347 	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
348 	struct cc_bl_item *bli;
349 	struct cc_crypto_req *creq;
350 	void *req;
351 	struct device *dev = drvdata_to_dev(drvdata);
352 	int rc;
353 
354 	spin_lock(&mgr->bl_lock);
355 
356 	while (mgr->bl_len) {
357 		bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list);
358 		dev_dbg(dev, "---bl len: %d\n", mgr->bl_len);
359 
360 		spin_unlock(&mgr->bl_lock);
361 
362 
363 		creq = &bli->creq;
364 		req = creq->user_arg;
365 
366 		/*
367 		 * Notify the request we're moving out of the backlog
368 		 * but only if we haven't done so already.
369 		 */
370 		if (!bli->notif) {
371 			creq->user_cb(dev, req, -EINPROGRESS);
372 			bli->notif = true;
373 		}
374 
375 		spin_lock(&mgr->hw_lock);
376 
377 		rc = cc_queues_status(drvdata, mgr, bli->len);
378 		if (rc) {
379 			/*
380 			 * There is still no room in the FIFO for
381 			 * this request. Bail out. We'll return here
382 			 * on the next completion irq.
383 			 */
384 			spin_unlock(&mgr->hw_lock);
385 			return;
386 		}
387 
388 		cc_do_send_request(drvdata, &bli->creq, bli->desc, bli->len,
389 				   false);
390 		spin_unlock(&mgr->hw_lock);
391 
392 		/* Remove ourselves from the backlog list */
393 		spin_lock(&mgr->bl_lock);
394 		list_del(&bli->list);
395 		--mgr->bl_len;
396 		kfree(bli);
397 	}
398 
399 	spin_unlock(&mgr->bl_lock);
400 }
401 
cc_send_request(struct cc_drvdata * drvdata,struct cc_crypto_req * cc_req,struct cc_hw_desc * desc,unsigned int len,struct crypto_async_request * req)402 int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
403 		    struct cc_hw_desc *desc, unsigned int len,
404 		    struct crypto_async_request *req)
405 {
406 	int rc;
407 	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
408 	struct device *dev = drvdata_to_dev(drvdata);
409 	bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
410 	gfp_t flags = cc_gfp_flags(req);
411 	struct cc_bl_item *bli;
412 
413 	rc = cc_pm_get(dev);
414 	if (rc) {
415 		dev_err(dev, "cc_pm_get returned %x\n", rc);
416 		return rc;
417 	}
418 
419 	spin_lock_bh(&mgr->hw_lock);
420 	rc = cc_queues_status(drvdata, mgr, len);
421 
422 #ifdef CC_DEBUG_FORCE_BACKLOG
423 	if (backlog_ok)
424 		rc = -ENOSPC;
425 #endif /* CC_DEBUG_FORCE_BACKLOG */
426 
427 	if (rc == -ENOSPC && backlog_ok) {
428 		spin_unlock_bh(&mgr->hw_lock);
429 
430 		bli = kmalloc(sizeof(*bli), flags);
431 		if (!bli) {
432 			cc_pm_put_suspend(dev);
433 			return -ENOMEM;
434 		}
435 
436 		memcpy(&bli->creq, cc_req, sizeof(*cc_req));
437 		memcpy(&bli->desc, desc, len * sizeof(*desc));
438 		bli->len = len;
439 		bli->notif = false;
440 		cc_enqueue_backlog(drvdata, bli);
441 		return -EBUSY;
442 	}
443 
444 	if (!rc) {
445 		cc_do_send_request(drvdata, cc_req, desc, len, false);
446 		rc = -EINPROGRESS;
447 	}
448 
449 	spin_unlock_bh(&mgr->hw_lock);
450 	return rc;
451 }
452 
cc_send_sync_request(struct cc_drvdata * drvdata,struct cc_crypto_req * cc_req,struct cc_hw_desc * desc,unsigned int len)453 int cc_send_sync_request(struct cc_drvdata *drvdata,
454 			 struct cc_crypto_req *cc_req, struct cc_hw_desc *desc,
455 			 unsigned int len)
456 {
457 	int rc;
458 	struct device *dev = drvdata_to_dev(drvdata);
459 	struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
460 
461 	init_completion(&cc_req->seq_compl);
462 	cc_req->user_cb = request_mgr_complete;
463 	cc_req->user_arg = &cc_req->seq_compl;
464 
465 	rc = cc_pm_get(dev);
466 	if (rc) {
467 		dev_err(dev, "cc_pm_get returned %x\n", rc);
468 		return rc;
469 	}
470 
471 	while (true) {
472 		spin_lock_bh(&mgr->hw_lock);
473 		rc = cc_queues_status(drvdata, mgr, len + 1);
474 
475 		if (!rc)
476 			break;
477 
478 		spin_unlock_bh(&mgr->hw_lock);
479 		wait_for_completion_interruptible(&drvdata->hw_queue_avail);
480 		reinit_completion(&drvdata->hw_queue_avail);
481 	}
482 
483 	cc_do_send_request(drvdata, cc_req, desc, len, true);
484 	spin_unlock_bh(&mgr->hw_lock);
485 	wait_for_completion(&cc_req->seq_compl);
486 	return 0;
487 }
488 
489 /**
490  * send_request_init() - Enqueue caller request to crypto hardware during init
491  * process.
492  * Assume this function is not called in the middle of a flow,
493  * since we set QUEUE_LAST_IND flag in the last descriptor.
494  *
495  * @drvdata: Associated device driver context
496  * @desc: The crypto sequence
497  * @len: The crypto sequence length
498  *
499  * Return:
500  * Returns "0" upon success
501  */
send_request_init(struct cc_drvdata * drvdata,struct cc_hw_desc * desc,unsigned int len)502 int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc,
503 		      unsigned int len)
504 {
505 	struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
506 	unsigned int total_seq_len = len; /*initial sequence length*/
507 	int rc = 0;
508 
509 	/* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT.
510 	 */
511 	rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len);
512 	if (rc)
513 		return rc;
514 
515 	set_queue_last_ind(drvdata, &desc[(len - 1)]);
516 
517 	/*
518 	 * We are about to push command to the HW via the command registers
519 	 * that may reference host memory. We need to issue a memory barrier
520 	 * to make sure there are no outstanding memory writes
521 	 */
522 	wmb();
523 	enqueue_seq(drvdata, desc, len);
524 
525 	/* Update the free slots in HW queue */
526 	req_mgr_h->q_free_slots =
527 		cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
528 
529 	return 0;
530 }
531 
complete_request(struct cc_drvdata * drvdata)532 void complete_request(struct cc_drvdata *drvdata)
533 {
534 	struct cc_req_mgr_handle *request_mgr_handle =
535 						drvdata->request_mgr_handle;
536 
537 	complete(&drvdata->hw_queue_avail);
538 #ifdef COMP_IN_WQ
539 	queue_delayed_work(request_mgr_handle->workq,
540 			   &request_mgr_handle->compwork, 0);
541 #else
542 	tasklet_schedule(&request_mgr_handle->comptask);
543 #endif
544 }
545 
546 #ifdef COMP_IN_WQ
comp_work_handler(struct work_struct * work)547 static void comp_work_handler(struct work_struct *work)
548 {
549 	struct cc_drvdata *drvdata =
550 		container_of(work, struct cc_drvdata, compwork.work);
551 
552 	comp_handler((unsigned long)drvdata);
553 }
554 #endif
555 
proc_completions(struct cc_drvdata * drvdata)556 static void proc_completions(struct cc_drvdata *drvdata)
557 {
558 	struct cc_crypto_req *cc_req;
559 	struct device *dev = drvdata_to_dev(drvdata);
560 	struct cc_req_mgr_handle *request_mgr_handle =
561 						drvdata->request_mgr_handle;
562 	unsigned int *tail = &request_mgr_handle->req_queue_tail;
563 	unsigned int *head = &request_mgr_handle->req_queue_head;
564 	int rc;
565 	u32 mask;
566 
567 	while (request_mgr_handle->axi_completed) {
568 		request_mgr_handle->axi_completed--;
569 
570 		/* Dequeue request */
571 		if (*head == *tail) {
572 			/* We are supposed to handle a completion but our
573 			 * queue is empty. This is not normal. Return and
574 			 * hope for the best.
575 			 */
576 			dev_err(dev, "Request queue is empty head == tail %u\n",
577 				*head);
578 			break;
579 		}
580 
581 		cc_req = &request_mgr_handle->req_queue[*tail];
582 
583 		if (cc_req->cpp.is_cpp) {
584 
585 			dev_dbg(dev, "CPP request completion slot: %d alg:%d\n",
586 				cc_req->cpp.slot, cc_req->cpp.alg);
587 			mask = cc_cpp_int_mask(cc_req->cpp.alg,
588 					       cc_req->cpp.slot);
589 			rc = (drvdata->irq & mask ? -EPERM : 0);
590 			dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask,
591 				drvdata->irq, rc);
592 		} else {
593 			dev_dbg(dev, "None CPP request completion\n");
594 			rc = 0;
595 		}
596 
597 		if (cc_req->user_cb)
598 			cc_req->user_cb(dev, cc_req->user_arg, rc);
599 		*tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
600 		dev_dbg(dev, "Dequeue request tail=%u\n", *tail);
601 		dev_dbg(dev, "Request completed. axi_completed=%d\n",
602 			request_mgr_handle->axi_completed);
603 		cc_pm_put_suspend(dev);
604 	}
605 }
606 
cc_axi_comp_count(struct cc_drvdata * drvdata)607 static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata)
608 {
609 	return FIELD_GET(AXIM_MON_COMP_VALUE,
610 			 cc_ioread(drvdata, drvdata->axim_mon_offset));
611 }
612 
613 /* Deferred service handler, run as interrupt-fired tasklet */
comp_handler(unsigned long devarg)614 static void comp_handler(unsigned long devarg)
615 {
616 	struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg;
617 	struct cc_req_mgr_handle *request_mgr_handle =
618 						drvdata->request_mgr_handle;
619 	struct device *dev = drvdata_to_dev(drvdata);
620 	u32 irq;
621 
622 	dev_dbg(dev, "Completion handler called!\n");
623 	irq = (drvdata->irq & drvdata->comp_mask);
624 
625 	/* To avoid the interrupt from firing as we unmask it,
626 	 * we clear it now
627 	 */
628 	cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
629 
630 	/* Avoid race with above clear: Test completion counter once more */
631 
632 	request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
633 
634 	dev_dbg(dev, "AXI completion after updated: %d\n",
635 		request_mgr_handle->axi_completed);
636 
637 	while (request_mgr_handle->axi_completed) {
638 		do {
639 			drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR));
640 			irq = (drvdata->irq & drvdata->comp_mask);
641 			proc_completions(drvdata);
642 
643 			/* At this point (after proc_completions()),
644 			 * request_mgr_handle->axi_completed is 0.
645 			 */
646 			request_mgr_handle->axi_completed +=
647 						cc_axi_comp_count(drvdata);
648 		} while (request_mgr_handle->axi_completed > 0);
649 
650 		cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
651 
652 		request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
653 	}
654 
655 	/* after verifying that there is nothing to do,
656 	 * unmask AXI completion interrupt
657 	 */
658 	cc_iowrite(drvdata, CC_REG(HOST_IMR),
659 		   cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask);
660 
661 	cc_proc_backlog(drvdata);
662 	dev_dbg(dev, "Comp. handler done.\n");
663 }
664