Lines Matching refs:chan
140 #define ZYNQMP_DMA_DESC_SIZE(chan) (chan->desc_size) argument
142 #define to_chan(chan) container_of(chan, struct zynqmp_dma_chan, \ argument
250 struct zynqmp_dma_chan *chan; member
255 static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg, in zynqmp_dma_writeq() argument
258 lo_hi_writeq(value, chan->regs + reg); in zynqmp_dma_writeq()
266 static void zynqmp_dma_update_desc_to_ctrlr(struct zynqmp_dma_chan *chan, in zynqmp_dma_update_desc_to_ctrlr() argument
272 zynqmp_dma_writeq(chan, ZYNQMP_DMA_SRC_START_LSB, addr); in zynqmp_dma_update_desc_to_ctrlr()
274 zynqmp_dma_writeq(chan, ZYNQMP_DMA_DST_START_LSB, addr); in zynqmp_dma_update_desc_to_ctrlr()
282 static void zynqmp_dma_desc_config_eod(struct zynqmp_dma_chan *chan, in zynqmp_dma_desc_config_eod() argument
301 static void zynqmp_dma_config_sg_ll_desc(struct zynqmp_dma_chan *chan, in zynqmp_dma_config_sg_ll_desc() argument
313 if (chan->is_dmacoherent) { in zynqmp_dma_config_sg_ll_desc()
319 dma_addr_t addr = chan->desc_pool_p + in zynqmp_dma_config_sg_ll_desc()
320 ((uintptr_t)sdesc - (uintptr_t)chan->desc_pool_v); in zynqmp_dma_config_sg_ll_desc()
323 ddesc->nxtdscraddr = addr + ZYNQMP_DMA_DESC_SIZE(chan); in zynqmp_dma_config_sg_ll_desc()
331 static void zynqmp_dma_init(struct zynqmp_dma_chan *chan) in zynqmp_dma_init() argument
335 writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS); in zynqmp_dma_init()
336 val = readl(chan->regs + ZYNQMP_DMA_ISR); in zynqmp_dma_init()
337 writel(val, chan->regs + ZYNQMP_DMA_ISR); in zynqmp_dma_init()
339 if (chan->is_dmacoherent) { in zynqmp_dma_init()
343 writel(val, chan->regs + ZYNQMP_DMA_DSCR_ATTR); in zynqmp_dma_init()
346 val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR); in zynqmp_dma_init()
347 if (chan->is_dmacoherent) { in zynqmp_dma_init()
353 writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR); in zynqmp_dma_init()
356 val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT); in zynqmp_dma_init()
357 val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); in zynqmp_dma_init()
359 chan->idle = true; in zynqmp_dma_init()
370 struct zynqmp_dma_chan *chan = to_chan(tx->chan); in zynqmp_dma_tx_submit() local
376 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_tx_submit()
379 if (!list_empty(&chan->pending_list)) { in zynqmp_dma_tx_submit()
380 desc = list_last_entry(&chan->pending_list, in zynqmp_dma_tx_submit()
391 list_add_tail(&new->node, &chan->pending_list); in zynqmp_dma_tx_submit()
392 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_tx_submit()
404 zynqmp_dma_get_descriptor(struct zynqmp_dma_chan *chan) in zynqmp_dma_get_descriptor() argument
409 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_get_descriptor()
410 desc = list_first_entry(&chan->free_list, in zynqmp_dma_get_descriptor()
413 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_get_descriptor()
417 memset((void *)desc->src_v, 0, ZYNQMP_DMA_DESC_SIZE(chan)); in zynqmp_dma_get_descriptor()
418 memset((void *)desc->dst_v, 0, ZYNQMP_DMA_DESC_SIZE(chan)); in zynqmp_dma_get_descriptor()
428 static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan, in zynqmp_dma_free_descriptor() argument
433 chan->desc_free_cnt++; in zynqmp_dma_free_descriptor()
434 list_move_tail(&sdesc->node, &chan->free_list); in zynqmp_dma_free_descriptor()
436 chan->desc_free_cnt++; in zynqmp_dma_free_descriptor()
437 list_move_tail(&child->node, &chan->free_list); in zynqmp_dma_free_descriptor()
446 static void zynqmp_dma_free_desc_list(struct zynqmp_dma_chan *chan, in zynqmp_dma_free_desc_list() argument
452 zynqmp_dma_free_descriptor(chan, desc); in zynqmp_dma_free_desc_list()
463 struct zynqmp_dma_chan *chan = to_chan(dchan); in zynqmp_dma_alloc_chan_resources() local
467 ret = pm_runtime_resume_and_get(chan->dev); in zynqmp_dma_alloc_chan_resources()
471 chan->sw_desc_pool = kcalloc(ZYNQMP_DMA_NUM_DESCS, sizeof(*desc), in zynqmp_dma_alloc_chan_resources()
473 if (!chan->sw_desc_pool) in zynqmp_dma_alloc_chan_resources()
476 chan->idle = true; in zynqmp_dma_alloc_chan_resources()
477 chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS; in zynqmp_dma_alloc_chan_resources()
479 INIT_LIST_HEAD(&chan->free_list); in zynqmp_dma_alloc_chan_resources()
482 desc = chan->sw_desc_pool + i; in zynqmp_dma_alloc_chan_resources()
483 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); in zynqmp_dma_alloc_chan_resources()
485 list_add_tail(&desc->node, &chan->free_list); in zynqmp_dma_alloc_chan_resources()
488 chan->desc_pool_v = dma_alloc_coherent(chan->dev, in zynqmp_dma_alloc_chan_resources()
489 (2 * chan->desc_size * ZYNQMP_DMA_NUM_DESCS), in zynqmp_dma_alloc_chan_resources()
490 &chan->desc_pool_p, GFP_KERNEL); in zynqmp_dma_alloc_chan_resources()
491 if (!chan->desc_pool_v) in zynqmp_dma_alloc_chan_resources()
495 desc = chan->sw_desc_pool + i; in zynqmp_dma_alloc_chan_resources()
496 desc->src_v = (struct zynqmp_dma_desc_ll *) (chan->desc_pool_v + in zynqmp_dma_alloc_chan_resources()
497 (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2)); in zynqmp_dma_alloc_chan_resources()
499 desc->src_p = chan->desc_pool_p + in zynqmp_dma_alloc_chan_resources()
500 (i * ZYNQMP_DMA_DESC_SIZE(chan) * 2); in zynqmp_dma_alloc_chan_resources()
501 desc->dst_p = desc->src_p + ZYNQMP_DMA_DESC_SIZE(chan); in zynqmp_dma_alloc_chan_resources()
511 static void zynqmp_dma_start(struct zynqmp_dma_chan *chan) in zynqmp_dma_start() argument
513 writel(ZYNQMP_DMA_INT_EN_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IER); in zynqmp_dma_start()
514 writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE); in zynqmp_dma_start()
515 chan->idle = false; in zynqmp_dma_start()
516 writel(ZYNQMP_DMA_ENABLE, chan->regs + ZYNQMP_DMA_CTRL2); in zynqmp_dma_start()
524 static void zynqmp_dma_handle_ovfl_int(struct zynqmp_dma_chan *chan, u32 status) in zynqmp_dma_handle_ovfl_int() argument
527 writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE); in zynqmp_dma_handle_ovfl_int()
529 readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); in zynqmp_dma_handle_ovfl_int()
531 readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT); in zynqmp_dma_handle_ovfl_int()
534 static void zynqmp_dma_config(struct zynqmp_dma_chan *chan) in zynqmp_dma_config() argument
538 val = readl(chan->regs + ZYNQMP_DMA_CTRL0); in zynqmp_dma_config()
540 writel(val, chan->regs + ZYNQMP_DMA_CTRL0); in zynqmp_dma_config()
542 val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR); in zynqmp_dma_config()
543 burst_val = __ilog2_u32(chan->src_burst_len); in zynqmp_dma_config()
546 burst_val = __ilog2_u32(chan->dst_burst_len); in zynqmp_dma_config()
549 writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR); in zynqmp_dma_config()
562 struct zynqmp_dma_chan *chan = to_chan(dchan); in zynqmp_dma_device_config() local
564 chan->src_burst_len = clamp(config->src_maxburst, 1U, in zynqmp_dma_device_config()
566 chan->dst_burst_len = clamp(config->dst_maxburst, 1U, in zynqmp_dma_device_config()
576 static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan) in zynqmp_dma_start_transfer() argument
580 if (!chan->idle) in zynqmp_dma_start_transfer()
583 zynqmp_dma_config(chan); in zynqmp_dma_start_transfer()
585 desc = list_first_entry_or_null(&chan->pending_list, in zynqmp_dma_start_transfer()
590 list_splice_tail_init(&chan->pending_list, &chan->active_list); in zynqmp_dma_start_transfer()
591 zynqmp_dma_update_desc_to_ctrlr(chan, desc); in zynqmp_dma_start_transfer()
592 zynqmp_dma_start(chan); in zynqmp_dma_start_transfer()
600 static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan) in zynqmp_dma_chan_desc_cleanup() argument
605 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_chan_desc_cleanup()
607 list_for_each_entry_safe(desc, next, &chan->done_list, node) { in zynqmp_dma_chan_desc_cleanup()
612 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_chan_desc_cleanup()
614 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_chan_desc_cleanup()
618 zynqmp_dma_free_descriptor(chan, desc); in zynqmp_dma_chan_desc_cleanup()
621 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_chan_desc_cleanup()
628 static void zynqmp_dma_complete_descriptor(struct zynqmp_dma_chan *chan) in zynqmp_dma_complete_descriptor() argument
632 desc = list_first_entry_or_null(&chan->active_list, in zynqmp_dma_complete_descriptor()
638 list_add_tail(&desc->node, &chan->done_list); in zynqmp_dma_complete_descriptor()
647 struct zynqmp_dma_chan *chan = to_chan(dchan); in zynqmp_dma_issue_pending() local
650 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_issue_pending()
651 zynqmp_dma_start_transfer(chan); in zynqmp_dma_issue_pending()
652 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_issue_pending()
659 static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan) in zynqmp_dma_free_descriptors() argument
663 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_free_descriptors()
664 zynqmp_dma_free_desc_list(chan, &chan->active_list); in zynqmp_dma_free_descriptors()
665 zynqmp_dma_free_desc_list(chan, &chan->pending_list); in zynqmp_dma_free_descriptors()
666 zynqmp_dma_free_desc_list(chan, &chan->done_list); in zynqmp_dma_free_descriptors()
667 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_free_descriptors()
676 struct zynqmp_dma_chan *chan = to_chan(dchan); in zynqmp_dma_free_chan_resources() local
678 zynqmp_dma_free_descriptors(chan); in zynqmp_dma_free_chan_resources()
679 dma_free_coherent(chan->dev, in zynqmp_dma_free_chan_resources()
680 (2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS), in zynqmp_dma_free_chan_resources()
681 chan->desc_pool_v, chan->desc_pool_p); in zynqmp_dma_free_chan_resources()
682 kfree(chan->sw_desc_pool); in zynqmp_dma_free_chan_resources()
683 pm_runtime_mark_last_busy(chan->dev); in zynqmp_dma_free_chan_resources()
684 pm_runtime_put_autosuspend(chan->dev); in zynqmp_dma_free_chan_resources()
691 static void zynqmp_dma_reset(struct zynqmp_dma_chan *chan) in zynqmp_dma_reset() argument
695 writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS); in zynqmp_dma_reset()
697 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_reset()
698 zynqmp_dma_complete_descriptor(chan); in zynqmp_dma_reset()
699 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_reset()
700 zynqmp_dma_chan_desc_cleanup(chan); in zynqmp_dma_reset()
701 zynqmp_dma_free_descriptors(chan); in zynqmp_dma_reset()
703 zynqmp_dma_init(chan); in zynqmp_dma_reset()
715 struct zynqmp_dma_chan *chan = (struct zynqmp_dma_chan *)data; in zynqmp_dma_irq_handler() local
719 isr = readl(chan->regs + ZYNQMP_DMA_ISR); in zynqmp_dma_irq_handler()
720 imr = readl(chan->regs + ZYNQMP_DMA_IMR); in zynqmp_dma_irq_handler()
723 writel(isr, chan->regs + ZYNQMP_DMA_ISR); in zynqmp_dma_irq_handler()
725 tasklet_schedule(&chan->tasklet); in zynqmp_dma_irq_handler()
730 chan->idle = true; in zynqmp_dma_irq_handler()
733 chan->err = true; in zynqmp_dma_irq_handler()
734 tasklet_schedule(&chan->tasklet); in zynqmp_dma_irq_handler()
735 dev_err(chan->dev, "Channel %p has errors\n", chan); in zynqmp_dma_irq_handler()
740 zynqmp_dma_handle_ovfl_int(chan, status); in zynqmp_dma_irq_handler()
741 dev_dbg(chan->dev, "Channel %p overflow interrupt\n", chan); in zynqmp_dma_irq_handler()
754 struct zynqmp_dma_chan *chan = from_tasklet(chan, t, tasklet); in zynqmp_dma_do_tasklet() local
758 if (chan->err) { in zynqmp_dma_do_tasklet()
759 zynqmp_dma_reset(chan); in zynqmp_dma_do_tasklet()
760 chan->err = false; in zynqmp_dma_do_tasklet()
764 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_do_tasklet()
765 count = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT); in zynqmp_dma_do_tasklet()
767 zynqmp_dma_complete_descriptor(chan); in zynqmp_dma_do_tasklet()
770 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_do_tasklet()
772 zynqmp_dma_chan_desc_cleanup(chan); in zynqmp_dma_do_tasklet()
774 if (chan->idle) { in zynqmp_dma_do_tasklet()
775 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_do_tasklet()
776 zynqmp_dma_start_transfer(chan); in zynqmp_dma_do_tasklet()
777 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_do_tasklet()
789 struct zynqmp_dma_chan *chan = to_chan(dchan); in zynqmp_dma_device_terminate_all() local
791 writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS); in zynqmp_dma_device_terminate_all()
792 zynqmp_dma_free_descriptors(chan); in zynqmp_dma_device_terminate_all()
811 struct zynqmp_dma_chan *chan; in zynqmp_dma_prep_memcpy() local
818 chan = to_chan(dchan); in zynqmp_dma_prep_memcpy()
822 spin_lock_irqsave(&chan->lock, irqflags); in zynqmp_dma_prep_memcpy()
823 if (desc_cnt > chan->desc_free_cnt) { in zynqmp_dma_prep_memcpy()
824 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_prep_memcpy()
825 dev_dbg(chan->dev, "chan %p descs are not available\n", chan); in zynqmp_dma_prep_memcpy()
828 chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt; in zynqmp_dma_prep_memcpy()
829 spin_unlock_irqrestore(&chan->lock, irqflags); in zynqmp_dma_prep_memcpy()
833 new = zynqmp_dma_get_descriptor(chan); in zynqmp_dma_prep_memcpy()
837 zynqmp_dma_config_sg_ll_desc(chan, desc, dma_src, in zynqmp_dma_prep_memcpy()
849 zynqmp_dma_desc_config_eod(chan, desc); in zynqmp_dma_prep_memcpy()
859 static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan) in zynqmp_dma_chan_remove() argument
861 if (!chan) in zynqmp_dma_chan_remove()
864 if (chan->irq) in zynqmp_dma_chan_remove()
865 devm_free_irq(chan->zdev->dev, chan->irq, chan); in zynqmp_dma_chan_remove()
866 tasklet_kill(&chan->tasklet); in zynqmp_dma_chan_remove()
867 list_del(&chan->common.device_node); in zynqmp_dma_chan_remove()
880 struct zynqmp_dma_chan *chan; in zynqmp_dma_chan_probe() local
885 chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL); in zynqmp_dma_chan_probe()
886 if (!chan) in zynqmp_dma_chan_probe()
888 chan->dev = zdev->dev; in zynqmp_dma_chan_probe()
889 chan->zdev = zdev; in zynqmp_dma_chan_probe()
892 chan->regs = devm_ioremap_resource(&pdev->dev, res); in zynqmp_dma_chan_probe()
893 if (IS_ERR(chan->regs)) in zynqmp_dma_chan_probe()
894 return PTR_ERR(chan->regs); in zynqmp_dma_chan_probe()
896 chan->bus_width = ZYNQMP_DMA_BUS_WIDTH_64; in zynqmp_dma_chan_probe()
897 chan->dst_burst_len = ZYNQMP_DMA_MAX_DST_BURST_LEN; in zynqmp_dma_chan_probe()
898 chan->src_burst_len = ZYNQMP_DMA_MAX_SRC_BURST_LEN; in zynqmp_dma_chan_probe()
899 err = of_property_read_u32(node, "xlnx,bus-width", &chan->bus_width); in zynqmp_dma_chan_probe()
905 if (chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_64 && in zynqmp_dma_chan_probe()
906 chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_128) { in zynqmp_dma_chan_probe()
911 chan->is_dmacoherent = of_property_read_bool(node, "dma-coherent"); in zynqmp_dma_chan_probe()
912 zdev->chan = chan; in zynqmp_dma_chan_probe()
913 tasklet_setup(&chan->tasklet, zynqmp_dma_do_tasklet); in zynqmp_dma_chan_probe()
914 spin_lock_init(&chan->lock); in zynqmp_dma_chan_probe()
915 INIT_LIST_HEAD(&chan->active_list); in zynqmp_dma_chan_probe()
916 INIT_LIST_HEAD(&chan->pending_list); in zynqmp_dma_chan_probe()
917 INIT_LIST_HEAD(&chan->done_list); in zynqmp_dma_chan_probe()
918 INIT_LIST_HEAD(&chan->free_list); in zynqmp_dma_chan_probe()
920 dma_cookie_init(&chan->common); in zynqmp_dma_chan_probe()
921 chan->common.device = &zdev->common; in zynqmp_dma_chan_probe()
922 list_add_tail(&chan->common.device_node, &zdev->common.channels); in zynqmp_dma_chan_probe()
924 zynqmp_dma_init(chan); in zynqmp_dma_chan_probe()
925 chan->irq = platform_get_irq(pdev, 0); in zynqmp_dma_chan_probe()
926 if (chan->irq < 0) in zynqmp_dma_chan_probe()
928 err = devm_request_irq(&pdev->dev, chan->irq, zynqmp_dma_irq_handler, 0, in zynqmp_dma_chan_probe()
929 "zynqmp-dma", chan); in zynqmp_dma_chan_probe()
933 chan->desc_size = sizeof(struct zynqmp_dma_desc_ll); in zynqmp_dma_chan_probe()
934 chan->idle = true; in zynqmp_dma_chan_probe()
950 return dma_get_slave_channel(&zdev->chan->common); in of_zynqmp_dma_xlate()
1093 p->dst_addr_widths = BIT(zdev->chan->bus_width / 8); in zynqmp_dma_probe()
1094 p->src_addr_widths = BIT(zdev->chan->bus_width / 8); in zynqmp_dma_probe()
1112 zynqmp_dma_chan_remove(zdev->chan); in zynqmp_dma_probe()
1133 zynqmp_dma_chan_remove(zdev->chan); in zynqmp_dma_remove()