1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Qualcomm Technologies HIDMA DMA engine low level code
4 *
5 * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
6 */
7
8 #include <linux/dmaengine.h>
9 #include <linux/slab.h>
10 #include <linux/interrupt.h>
11 #include <linux/mm.h>
12 #include <linux/highmem.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/delay.h>
15 #include <linux/atomic.h>
16 #include <linux/iopoll.h>
17 #include <linux/kfifo.h>
18 #include <linux/bitops.h>
19
20 #include "hidma.h"
21
22 #define HIDMA_EVRE_SIZE 16 /* each EVRE is 16 bytes */
23
24 #define HIDMA_TRCA_CTRLSTS_REG 0x000
25 #define HIDMA_TRCA_RING_LOW_REG 0x008
26 #define HIDMA_TRCA_RING_HIGH_REG 0x00C
27 #define HIDMA_TRCA_RING_LEN_REG 0x010
28 #define HIDMA_TRCA_DOORBELL_REG 0x400
29
30 #define HIDMA_EVCA_CTRLSTS_REG 0x000
31 #define HIDMA_EVCA_INTCTRL_REG 0x004
32 #define HIDMA_EVCA_RING_LOW_REG 0x008
33 #define HIDMA_EVCA_RING_HIGH_REG 0x00C
34 #define HIDMA_EVCA_RING_LEN_REG 0x010
35 #define HIDMA_EVCA_WRITE_PTR_REG 0x020
36 #define HIDMA_EVCA_DOORBELL_REG 0x400
37
38 #define HIDMA_EVCA_IRQ_STAT_REG 0x100
39 #define HIDMA_EVCA_IRQ_CLR_REG 0x108
40 #define HIDMA_EVCA_IRQ_EN_REG 0x110
41
42 #define HIDMA_EVRE_CFG_IDX 0
43
44 #define HIDMA_EVRE_ERRINFO_BIT_POS 24
45 #define HIDMA_EVRE_CODE_BIT_POS 28
46
47 #define HIDMA_EVRE_ERRINFO_MASK GENMASK(3, 0)
48 #define HIDMA_EVRE_CODE_MASK GENMASK(3, 0)
49
50 #define HIDMA_CH_CONTROL_MASK GENMASK(7, 0)
51 #define HIDMA_CH_STATE_MASK GENMASK(7, 0)
52 #define HIDMA_CH_STATE_BIT_POS 0x8
53
54 #define HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS 0
55 #define HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS 1
56 #define HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS 9
57 #define HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS 10
58 #define HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS 11
59 #define HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS 14
60
61 #define ENABLE_IRQS (BIT(HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS) | \
62 BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \
63 BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \
64 BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \
65 BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS) | \
66 BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS))
67
68 #define HIDMA_INCREMENT_ITERATOR(iter, size, ring_size) \
69 do { \
70 iter += size; \
71 if (iter >= ring_size) \
72 iter -= ring_size; \
73 } while (0)
74
75 #define HIDMA_CH_STATE(val) \
76 ((val >> HIDMA_CH_STATE_BIT_POS) & HIDMA_CH_STATE_MASK)
77
78 #define HIDMA_ERR_INT_MASK \
79 (BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS) | \
80 BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \
81 BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \
82 BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \
83 BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS))
84
85 enum ch_command {
86 HIDMA_CH_DISABLE = 0,
87 HIDMA_CH_ENABLE = 1,
88 HIDMA_CH_SUSPEND = 2,
89 HIDMA_CH_RESET = 9,
90 };
91
92 enum ch_state {
93 HIDMA_CH_DISABLED = 0,
94 HIDMA_CH_ENABLED = 1,
95 HIDMA_CH_RUNNING = 2,
96 HIDMA_CH_SUSPENDED = 3,
97 HIDMA_CH_STOPPED = 4,
98 };
99
100 enum err_code {
101 HIDMA_EVRE_STATUS_COMPLETE = 1,
102 HIDMA_EVRE_STATUS_ERROR = 4,
103 };
104
hidma_is_chan_enabled(int state)105 static int hidma_is_chan_enabled(int state)
106 {
107 switch (state) {
108 case HIDMA_CH_ENABLED:
109 case HIDMA_CH_RUNNING:
110 return true;
111 default:
112 return false;
113 }
114 }
115
hidma_ll_free(struct hidma_lldev * lldev,u32 tre_ch)116 void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch)
117 {
118 struct hidma_tre *tre;
119
120 if (tre_ch >= lldev->nr_tres) {
121 dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch);
122 return;
123 }
124
125 tre = &lldev->trepool[tre_ch];
126 if (atomic_read(&tre->allocated) != true) {
127 dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch);
128 return;
129 }
130
131 atomic_set(&tre->allocated, 0);
132 }
133
hidma_ll_request(struct hidma_lldev * lldev,u32 sig,const char * dev_name,void (* callback)(void * data),void * data,u32 * tre_ch)134 int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name,
135 void (*callback)(void *data), void *data, u32 *tre_ch)
136 {
137 unsigned int i;
138 struct hidma_tre *tre;
139 u32 *tre_local;
140
141 if (!tre_ch || !lldev)
142 return -EINVAL;
143
144 /* need to have at least one empty spot in the queue */
145 for (i = 0; i < lldev->nr_tres - 1; i++) {
146 if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1))
147 break;
148 }
149
150 if (i == (lldev->nr_tres - 1))
151 return -ENOMEM;
152
153 tre = &lldev->trepool[i];
154 tre->dma_sig = sig;
155 tre->dev_name = dev_name;
156 tre->callback = callback;
157 tre->data = data;
158 tre->idx = i;
159 tre->status = 0;
160 tre->queued = 0;
161 tre->err_code = 0;
162 tre->err_info = 0;
163 tre->lldev = lldev;
164 tre_local = &tre->tre_local[0];
165 tre_local[HIDMA_TRE_CFG_IDX] = (lldev->chidx & 0xFF) << 8;
166 tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16); /* set IEOB */
167 *tre_ch = i;
168 if (callback)
169 callback(data);
170 return 0;
171 }
172
173 /*
174 * Multiple TREs may be queued and waiting in the pending queue.
175 */
hidma_ll_tre_complete(struct tasklet_struct * t)176 static void hidma_ll_tre_complete(struct tasklet_struct *t)
177 {
178 struct hidma_lldev *lldev = from_tasklet(lldev, t, task);
179 struct hidma_tre *tre;
180
181 while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) {
182 /* call the user if it has been read by the hardware */
183 if (tre->callback)
184 tre->callback(tre->data);
185 }
186 }
187
hidma_post_completed(struct hidma_lldev * lldev,u8 err_info,u8 err_code)188 static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info,
189 u8 err_code)
190 {
191 struct hidma_tre *tre;
192 unsigned long flags;
193 u32 tre_iterator;
194
195 spin_lock_irqsave(&lldev->lock, flags);
196
197 tre_iterator = lldev->tre_processed_off;
198 tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE];
199 if (!tre) {
200 spin_unlock_irqrestore(&lldev->lock, flags);
201 dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n",
202 tre_iterator / HIDMA_TRE_SIZE);
203 return -EINVAL;
204 }
205 lldev->pending_tre_list[tre->tre_index] = NULL;
206
207 /*
208 * Keep track of pending TREs that SW is expecting to receive
209 * from HW. We got one now. Decrement our counter.
210 */
211 if (atomic_dec_return(&lldev->pending_tre_count) < 0) {
212 dev_warn(lldev->dev, "tre count mismatch on completion");
213 atomic_set(&lldev->pending_tre_count, 0);
214 }
215
216 HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE,
217 lldev->tre_ring_size);
218 lldev->tre_processed_off = tre_iterator;
219 spin_unlock_irqrestore(&lldev->lock, flags);
220
221 tre->err_info = err_info;
222 tre->err_code = err_code;
223 tre->queued = 0;
224
225 kfifo_put(&lldev->handoff_fifo, tre);
226 tasklet_schedule(&lldev->task);
227
228 return 0;
229 }
230
231 /*
232 * Called to handle the interrupt for the channel.
233 * Return a positive number if TRE or EVRE were consumed on this run.
234 * Return a positive number if there are pending TREs or EVREs.
235 * Return 0 if there is nothing to consume or no pending TREs/EVREs found.
236 */
hidma_handle_tre_completion(struct hidma_lldev * lldev)237 static int hidma_handle_tre_completion(struct hidma_lldev *lldev)
238 {
239 u32 evre_ring_size = lldev->evre_ring_size;
240 u32 err_info, err_code, evre_write_off;
241 u32 evre_iterator;
242 u32 num_completed = 0;
243
244 evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
245 evre_iterator = lldev->evre_processed_off;
246
247 if ((evre_write_off > evre_ring_size) ||
248 (evre_write_off % HIDMA_EVRE_SIZE)) {
249 dev_err(lldev->dev, "HW reports invalid EVRE write offset\n");
250 return 0;
251 }
252
253 /*
254 * By the time control reaches here the number of EVREs and TREs
255 * may not match. Only consume the ones that hardware told us.
256 */
257 while ((evre_iterator != evre_write_off)) {
258 u32 *current_evre = lldev->evre_ring + evre_iterator;
259 u32 cfg;
260
261 cfg = current_evre[HIDMA_EVRE_CFG_IDX];
262 err_info = cfg >> HIDMA_EVRE_ERRINFO_BIT_POS;
263 err_info &= HIDMA_EVRE_ERRINFO_MASK;
264 err_code =
265 (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK;
266
267 if (hidma_post_completed(lldev, err_info, err_code))
268 break;
269
270 HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE,
271 evre_ring_size);
272
273 /*
274 * Read the new event descriptor written by the HW.
275 * As we are processing the delivered events, other events
276 * get queued to the SW for processing.
277 */
278 evre_write_off =
279 readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
280 num_completed++;
281
282 /*
283 * An error interrupt might have arrived while we are processing
284 * the completed interrupt.
285 */
286 if (!hidma_ll_isenabled(lldev))
287 break;
288 }
289
290 if (num_completed) {
291 u32 evre_read_off = (lldev->evre_processed_off +
292 HIDMA_EVRE_SIZE * num_completed);
293 evre_read_off = evre_read_off % evre_ring_size;
294 writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG);
295
296 /* record the last processed tre offset */
297 lldev->evre_processed_off = evre_read_off;
298 }
299
300 return num_completed;
301 }
302
hidma_cleanup_pending_tre(struct hidma_lldev * lldev,u8 err_info,u8 err_code)303 void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info,
304 u8 err_code)
305 {
306 while (atomic_read(&lldev->pending_tre_count)) {
307 if (hidma_post_completed(lldev, err_info, err_code))
308 break;
309 }
310 }
311
hidma_ll_reset(struct hidma_lldev * lldev)312 static int hidma_ll_reset(struct hidma_lldev *lldev)
313 {
314 u32 val;
315 int ret;
316
317 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
318 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
319 val |= HIDMA_CH_RESET << 16;
320 writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
321
322 /*
323 * Delay 10ms after reset to allow DMA logic to quiesce.
324 * Do a polled read up to 1ms and 10ms maximum.
325 */
326 ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
327 HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED,
328 1000, 10000);
329 if (ret) {
330 dev_err(lldev->dev, "transfer channel did not reset\n");
331 return ret;
332 }
333
334 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
335 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
336 val |= HIDMA_CH_RESET << 16;
337 writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
338
339 /*
340 * Delay 10ms after reset to allow DMA logic to quiesce.
341 * Do a polled read up to 1ms and 10ms maximum.
342 */
343 ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
344 HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED,
345 1000, 10000);
346 if (ret)
347 return ret;
348
349 lldev->trch_state = HIDMA_CH_DISABLED;
350 lldev->evch_state = HIDMA_CH_DISABLED;
351 return 0;
352 }
353
354 /*
355 * The interrupt handler for HIDMA will try to consume as many pending
356 * EVRE from the event queue as possible. Each EVRE has an associated
357 * TRE that holds the user interface parameters. EVRE reports the
358 * result of the transaction. Hardware guarantees ordering between EVREs
359 * and TREs. We use last processed offset to figure out which TRE is
360 * associated with which EVRE. If two TREs are consumed by HW, the EVREs
361 * are in order in the event ring.
362 *
363 * This handler will do a one pass for consuming EVREs. Other EVREs may
364 * be delivered while we are working. It will try to consume incoming
365 * EVREs one more time and return.
366 *
367 * For unprocessed EVREs, hardware will trigger another interrupt until
368 * all the interrupt bits are cleared.
369 *
370 * Hardware guarantees that by the time interrupt is observed, all data
371 * transactions in flight are delivered to their respective places and
372 * are visible to the CPU.
373 *
374 * On demand paging for IOMMU is only supported for PCIe via PRI
375 * (Page Request Interface) not for HIDMA. All other hardware instances
376 * including HIDMA work on pinned DMA addresses.
377 *
378 * HIDMA is not aware of IOMMU presence since it follows the DMA API. All
379 * IOMMU latency will be built into the data movement time. By the time
380 * interrupt happens, IOMMU lookups + data movement has already taken place.
381 *
382 * While the first read in a typical PCI endpoint ISR flushes all outstanding
383 * requests traditionally to the destination, this concept does not apply
384 * here for this HW.
385 */
hidma_ll_int_handler_internal(struct hidma_lldev * lldev,int cause)386 static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)
387 {
388 unsigned long irqflags;
389
390 if (cause & HIDMA_ERR_INT_MASK) {
391 dev_err(lldev->dev, "error 0x%x, disabling...\n",
392 cause);
393
394 /* Clear out pending interrupts */
395 writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
396
397 /* No further submissions. */
398 hidma_ll_disable(lldev);
399
400 /* Driver completes the txn and intimates the client.*/
401 hidma_cleanup_pending_tre(lldev, 0xFF,
402 HIDMA_EVRE_STATUS_ERROR);
403
404 return;
405 }
406
407 spin_lock_irqsave(&lldev->lock, irqflags);
408 writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
409 spin_unlock_irqrestore(&lldev->lock, irqflags);
410
411 /*
412 * Fine tuned for this HW...
413 *
414 * This ISR has been designed for this particular hardware. Relaxed
415 * read and write accessors are used for performance reasons due to
416 * interrupt delivery guarantees. Do not copy this code blindly and
417 * expect that to work.
418 *
419 * Try to consume as many EVREs as possible.
420 */
421 hidma_handle_tre_completion(lldev);
422 }
423
hidma_ll_inthandler(int chirq,void * arg)424 irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
425 {
426 struct hidma_lldev *lldev = arg;
427 u32 status;
428 u32 enable;
429 u32 cause;
430
431 status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
432 enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
433 cause = status & enable;
434
435 while (cause) {
436 hidma_ll_int_handler_internal(lldev, cause);
437
438 /*
439 * Another interrupt might have arrived while we are
440 * processing this one. Read the new cause.
441 */
442 status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
443 enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
444 cause = status & enable;
445 }
446
447 return IRQ_HANDLED;
448 }
449
hidma_ll_inthandler_msi(int chirq,void * arg,int cause)450 irqreturn_t hidma_ll_inthandler_msi(int chirq, void *arg, int cause)
451 {
452 struct hidma_lldev *lldev = arg;
453
454 hidma_ll_int_handler_internal(lldev, cause);
455 return IRQ_HANDLED;
456 }
457
hidma_ll_enable(struct hidma_lldev * lldev)458 int hidma_ll_enable(struct hidma_lldev *lldev)
459 {
460 u32 val;
461 int ret;
462
463 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
464 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
465 val |= HIDMA_CH_ENABLE << 16;
466 writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
467
468 ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
469 hidma_is_chan_enabled(HIDMA_CH_STATE(val)),
470 1000, 10000);
471 if (ret) {
472 dev_err(lldev->dev, "event channel did not get enabled\n");
473 return ret;
474 }
475
476 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
477 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
478 val |= HIDMA_CH_ENABLE << 16;
479 writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
480
481 ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
482 hidma_is_chan_enabled(HIDMA_CH_STATE(val)),
483 1000, 10000);
484 if (ret) {
485 dev_err(lldev->dev, "transfer channel did not get enabled\n");
486 return ret;
487 }
488
489 lldev->trch_state = HIDMA_CH_ENABLED;
490 lldev->evch_state = HIDMA_CH_ENABLED;
491
492 /* enable irqs */
493 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
494
495 return 0;
496 }
497
hidma_ll_start(struct hidma_lldev * lldev)498 void hidma_ll_start(struct hidma_lldev *lldev)
499 {
500 unsigned long irqflags;
501
502 spin_lock_irqsave(&lldev->lock, irqflags);
503 writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG);
504 spin_unlock_irqrestore(&lldev->lock, irqflags);
505 }
506
hidma_ll_isenabled(struct hidma_lldev * lldev)507 bool hidma_ll_isenabled(struct hidma_lldev *lldev)
508 {
509 u32 val;
510
511 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
512 lldev->trch_state = HIDMA_CH_STATE(val);
513 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
514 lldev->evch_state = HIDMA_CH_STATE(val);
515
516 /* both channels have to be enabled before calling this function */
517 if (hidma_is_chan_enabled(lldev->trch_state) &&
518 hidma_is_chan_enabled(lldev->evch_state))
519 return true;
520
521 return false;
522 }
523
hidma_ll_queue_request(struct hidma_lldev * lldev,u32 tre_ch)524 void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch)
525 {
526 struct hidma_tre *tre;
527 unsigned long flags;
528
529 tre = &lldev->trepool[tre_ch];
530
531 /* copy the TRE into its location in the TRE ring */
532 spin_lock_irqsave(&lldev->lock, flags);
533 tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE;
534 lldev->pending_tre_list[tre->tre_index] = tre;
535 memcpy(lldev->tre_ring + lldev->tre_write_offset,
536 &tre->tre_local[0], HIDMA_TRE_SIZE);
537 tre->err_code = 0;
538 tre->err_info = 0;
539 tre->queued = 1;
540 atomic_inc(&lldev->pending_tre_count);
541 lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE)
542 % lldev->tre_ring_size;
543 spin_unlock_irqrestore(&lldev->lock, flags);
544 }
545
546 /*
547 * Note that even though we stop this channel if there is a pending transaction
548 * in flight it will complete and follow the callback. This request will
549 * prevent further requests to be made.
550 */
hidma_ll_disable(struct hidma_lldev * lldev)551 int hidma_ll_disable(struct hidma_lldev *lldev)
552 {
553 u32 val;
554 int ret;
555
556 /* The channel needs to be in working state */
557 if (!hidma_ll_isenabled(lldev))
558 return 0;
559
560 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
561 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
562 val |= HIDMA_CH_SUSPEND << 16;
563 writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
564
565 /*
566 * Start the wait right after the suspend is confirmed.
567 * Do a polled read up to 1ms and 10ms maximum.
568 */
569 ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
570 HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED,
571 1000, 10000);
572 if (ret)
573 return ret;
574
575 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
576 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
577 val |= HIDMA_CH_SUSPEND << 16;
578 writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
579
580 /*
581 * Start the wait right after the suspend is confirmed
582 * Delay up to 10ms after reset to allow DMA logic to quiesce.
583 */
584 ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
585 HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED,
586 1000, 10000);
587 if (ret)
588 return ret;
589
590 lldev->trch_state = HIDMA_CH_SUSPENDED;
591 lldev->evch_state = HIDMA_CH_SUSPENDED;
592
593 /* disable interrupts */
594 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
595 return 0;
596 }
597
hidma_ll_set_transfer_params(struct hidma_lldev * lldev,u32 tre_ch,dma_addr_t src,dma_addr_t dest,u32 len,u32 flags,u32 txntype)598 void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch,
599 dma_addr_t src, dma_addr_t dest, u32 len,
600 u32 flags, u32 txntype)
601 {
602 struct hidma_tre *tre;
603 u32 *tre_local;
604
605 if (tre_ch >= lldev->nr_tres) {
606 dev_err(lldev->dev, "invalid TRE number in transfer params:%d",
607 tre_ch);
608 return;
609 }
610
611 tre = &lldev->trepool[tre_ch];
612 if (atomic_read(&tre->allocated) != true) {
613 dev_err(lldev->dev, "trying to set params on an unused TRE:%d",
614 tre_ch);
615 return;
616 }
617
618 tre_local = &tre->tre_local[0];
619 tre_local[HIDMA_TRE_CFG_IDX] &= ~GENMASK(7, 0);
620 tre_local[HIDMA_TRE_CFG_IDX] |= txntype;
621 tre_local[HIDMA_TRE_LEN_IDX] = len;
622 tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src);
623 tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src);
624 tre_local[HIDMA_TRE_DEST_LOW_IDX] = lower_32_bits(dest);
625 tre_local[HIDMA_TRE_DEST_HI_IDX] = upper_32_bits(dest);
626 tre->int_flags = flags;
627 }
628
629 /*
630 * Called during initialization and after an error condition
631 * to restore hardware state.
632 */
hidma_ll_setup(struct hidma_lldev * lldev)633 int hidma_ll_setup(struct hidma_lldev *lldev)
634 {
635 int rc;
636 u64 addr;
637 u32 val;
638 u32 nr_tres = lldev->nr_tres;
639
640 atomic_set(&lldev->pending_tre_count, 0);
641 lldev->tre_processed_off = 0;
642 lldev->evre_processed_off = 0;
643 lldev->tre_write_offset = 0;
644
645 /* disable interrupts */
646 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
647
648 /* clear all pending interrupts */
649 val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
650 writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
651
652 rc = hidma_ll_reset(lldev);
653 if (rc)
654 return rc;
655
656 /*
657 * Clear all pending interrupts again.
658 * Otherwise, we observe reset complete interrupts.
659 */
660 val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
661 writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
662
663 /* disable interrupts again after reset */
664 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
665
666 addr = lldev->tre_dma;
667 writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG);
668 writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG);
669 writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG);
670
671 addr = lldev->evre_dma;
672 writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG);
673 writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG);
674 writel(HIDMA_EVRE_SIZE * nr_tres,
675 lldev->evca + HIDMA_EVCA_RING_LEN_REG);
676
677 /* configure interrupts */
678 hidma_ll_setup_irq(lldev, lldev->msi_support);
679
680 rc = hidma_ll_enable(lldev);
681 if (rc)
682 return rc;
683
684 return rc;
685 }
686
hidma_ll_setup_irq(struct hidma_lldev * lldev,bool msi)687 void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi)
688 {
689 u32 val;
690
691 lldev->msi_support = msi;
692
693 /* disable interrupts again after reset */
694 writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
695 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
696
697 /* support IRQ by default */
698 val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG);
699 val &= ~0xF;
700 if (!lldev->msi_support)
701 val = val | 0x1;
702 writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG);
703
704 /* clear all pending interrupts and enable them */
705 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
706 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
707 }
708
hidma_ll_init(struct device * dev,u32 nr_tres,void __iomem * trca,void __iomem * evca,u8 chidx)709 struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres,
710 void __iomem *trca, void __iomem *evca,
711 u8 chidx)
712 {
713 u32 required_bytes;
714 struct hidma_lldev *lldev;
715 int rc;
716 size_t sz;
717
718 if (!trca || !evca || !dev || !nr_tres)
719 return NULL;
720
721 /* need at least four TREs */
722 if (nr_tres < 4)
723 return NULL;
724
725 /* need an extra space */
726 nr_tres += 1;
727
728 lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL);
729 if (!lldev)
730 return NULL;
731
732 lldev->evca = evca;
733 lldev->trca = trca;
734 lldev->dev = dev;
735 sz = sizeof(struct hidma_tre);
736 lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL);
737 if (!lldev->trepool)
738 return NULL;
739
740 required_bytes = sizeof(lldev->pending_tre_list[0]);
741 lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes,
742 GFP_KERNEL);
743 if (!lldev->pending_tre_list)
744 return NULL;
745
746 sz = (HIDMA_TRE_SIZE + 1) * nr_tres;
747 lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma,
748 GFP_KERNEL);
749 if (!lldev->tre_ring)
750 return NULL;
751
752 lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres;
753 lldev->nr_tres = nr_tres;
754
755 /* the TRE ring has to be TRE_SIZE aligned */
756 if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) {
757 u8 tre_ring_shift;
758
759 tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE;
760 tre_ring_shift = HIDMA_TRE_SIZE - tre_ring_shift;
761 lldev->tre_dma += tre_ring_shift;
762 lldev->tre_ring += tre_ring_shift;
763 }
764
765 sz = (HIDMA_EVRE_SIZE + 1) * nr_tres;
766 lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma,
767 GFP_KERNEL);
768 if (!lldev->evre_ring)
769 return NULL;
770
771 lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres;
772
773 /* the EVRE ring has to be EVRE_SIZE aligned */
774 if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) {
775 u8 evre_ring_shift;
776
777 evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE;
778 evre_ring_shift = HIDMA_EVRE_SIZE - evre_ring_shift;
779 lldev->evre_dma += evre_ring_shift;
780 lldev->evre_ring += evre_ring_shift;
781 }
782 lldev->nr_tres = nr_tres;
783 lldev->chidx = chidx;
784
785 sz = nr_tres * sizeof(struct hidma_tre *);
786 rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL);
787 if (rc)
788 return NULL;
789
790 rc = hidma_ll_setup(lldev);
791 if (rc)
792 return NULL;
793
794 spin_lock_init(&lldev->lock);
795 tasklet_setup(&lldev->task, hidma_ll_tre_complete);
796 lldev->initialized = 1;
797 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
798 return lldev;
799 }
800
hidma_ll_uninit(struct hidma_lldev * lldev)801 int hidma_ll_uninit(struct hidma_lldev *lldev)
802 {
803 u32 required_bytes;
804 int rc = 0;
805 u32 val;
806
807 if (!lldev)
808 return -ENODEV;
809
810 if (!lldev->initialized)
811 return 0;
812
813 lldev->initialized = 0;
814
815 required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres;
816 tasklet_kill(&lldev->task);
817 memset(lldev->trepool, 0, required_bytes);
818 lldev->trepool = NULL;
819 atomic_set(&lldev->pending_tre_count, 0);
820 lldev->tre_write_offset = 0;
821
822 rc = hidma_ll_reset(lldev);
823
824 /*
825 * Clear all pending interrupts again.
826 * Otherwise, we observe reset complete interrupts.
827 */
828 val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
829 writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
830 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
831 return rc;
832 }
833
hidma_ll_status(struct hidma_lldev * lldev,u32 tre_ch)834 enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch)
835 {
836 enum dma_status ret = DMA_ERROR;
837 struct hidma_tre *tre;
838 unsigned long flags;
839 u8 err_code;
840
841 spin_lock_irqsave(&lldev->lock, flags);
842
843 tre = &lldev->trepool[tre_ch];
844 err_code = tre->err_code;
845
846 if (err_code & HIDMA_EVRE_STATUS_COMPLETE)
847 ret = DMA_COMPLETE;
848 else if (err_code & HIDMA_EVRE_STATUS_ERROR)
849 ret = DMA_ERROR;
850 else
851 ret = DMA_IN_PROGRESS;
852 spin_unlock_irqrestore(&lldev->lock, flags);
853
854 return ret;
855 }
856