Lines Matching refs:ring
115 int (*push_tail)(struct k3_ring *ring, void *elm);
116 int (*push_head)(struct k3_ring *ring, void *elm);
117 int (*pop_tail)(struct k3_ring *ring, void *elm);
118 int (*pop_head)(struct k3_ring *ring, void *elm);
240 static int k3_ringacc_ring_read_occ(struct k3_ring *ring) in k3_ringacc_ring_read_occ() argument
242 return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK; in k3_ringacc_ring_read_occ()
245 static void k3_ringacc_ring_update_occ(struct k3_ring *ring) in k3_ringacc_ring_update_occ() argument
249 val = readl(&ring->rt->occ); in k3_ringacc_ring_update_occ()
251 ring->state.occ = val & K3_RINGACC_RT_OCC_MASK; in k3_ringacc_ring_update_occ()
252 ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE); in k3_ringacc_ring_update_occ()
255 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) in k3_ringacc_ring_get_fifo_pos() argument
258 (4 << ring->elm_size); in k3_ringacc_ring_get_fifo_pos()
261 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) in k3_ringacc_get_elm_addr() argument
263 return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); in k3_ringacc_get_elm_addr()
266 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
267 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
268 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem);
269 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem);
286 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
287 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
288 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
289 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
298 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
299 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
300 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
301 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
310 static void k3_ringacc_ring_dump(struct k3_ring *ring) in k3_ringacc_ring_dump() argument
312 struct device *dev = ring->parent->dev; in k3_ringacc_ring_dump()
314 dev_dbg(dev, "dump ring: %d\n", ring->ring_id); in k3_ringacc_ring_dump()
315 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, in k3_ringacc_ring_dump()
316 &ring->ring_mem_dma); in k3_ringacc_ring_dump()
318 ring->elm_size, ring->size, ring->mode, ring->proxy_id); in k3_ringacc_ring_dump()
319 dev_dbg(dev, "dump flags %08X\n", ring->flags); in k3_ringacc_ring_dump()
321 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); in k3_ringacc_ring_dump()
322 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); in k3_ringacc_ring_dump()
323 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); in k3_ringacc_ring_dump()
324 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); in k3_ringacc_ring_dump()
325 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); in k3_ringacc_ring_dump()
327 if (ring->ring_mem_virt) in k3_ringacc_ring_dump()
329 16, 1, ring->ring_mem_virt, 16 * 8, false); in k3_ringacc_ring_dump()
450 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) in k3_ringacc_ring_reset_sci() argument
453 struct k3_ringacc *ringacc = ring->parent; in k3_ringacc_ring_reset_sci()
457 ring_cfg.index = ring->ring_id; in k3_ringacc_ring_reset_sci()
459 ring_cfg.count = ring->size; in k3_ringacc_ring_reset_sci()
464 ret, ring->ring_id); in k3_ringacc_ring_reset_sci()
467 void k3_ringacc_ring_reset(struct k3_ring *ring) in k3_ringacc_ring_reset() argument
469 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_reset()
472 memset(&ring->state, 0, sizeof(ring->state)); in k3_ringacc_ring_reset()
474 k3_ringacc_ring_reset_sci(ring); in k3_ringacc_ring_reset()
478 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, in k3_ringacc_ring_reconfig_qmode_sci() argument
482 struct k3_ringacc *ringacc = ring->parent; in k3_ringacc_ring_reconfig_qmode_sci()
486 ring_cfg.index = ring->ring_id; in k3_ringacc_ring_reconfig_qmode_sci()
493 ret, ring->ring_id); in k3_ringacc_ring_reconfig_qmode_sci()
496 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) in k3_ringacc_ring_reset_dma() argument
498 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_reset_dma()
501 if (!ring->parent->dma_ring_reset_quirk) in k3_ringacc_ring_reset_dma()
505 occ = k3_ringacc_ring_read_occ(ring); in k3_ringacc_ring_reset_dma()
510 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, in k3_ringacc_ring_reset_dma()
511 ring->ring_id, occ); in k3_ringacc_ring_reset_dma()
513 k3_ringacc_ring_reset_sci(ring); in k3_ringacc_ring_reset_dma()
519 if (ring->mode != K3_RINGACC_RING_MODE_RING) in k3_ringacc_ring_reset_dma()
521 ring, K3_RINGACC_RING_MODE_RING); in k3_ringacc_ring_reset_dma()
540 writel(db_ring_cnt_cur, &ring->rt->db); in k3_ringacc_ring_reset_dma()
545 if (ring->mode != K3_RINGACC_RING_MODE_RING) in k3_ringacc_ring_reset_dma()
546 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); in k3_ringacc_ring_reset_dma()
551 k3_ringacc_ring_reset(ring); in k3_ringacc_ring_reset_dma()
555 static void k3_ringacc_ring_free_sci(struct k3_ring *ring) in k3_ringacc_ring_free_sci() argument
558 struct k3_ringacc *ringacc = ring->parent; in k3_ringacc_ring_free_sci()
562 ring_cfg.index = ring->ring_id; in k3_ringacc_ring_free_sci()
568 ret, ring->ring_id); in k3_ringacc_ring_free_sci()
571 int k3_ringacc_ring_free(struct k3_ring *ring) in k3_ringacc_ring_free() argument
575 if (!ring) in k3_ringacc_ring_free()
578 ringacc = ring->parent; in k3_ringacc_ring_free()
584 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) in k3_ringacc_ring_free()
587 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); in k3_ringacc_ring_free()
589 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) in k3_ringacc_ring_free()
594 if (--ring->use_count) in k3_ringacc_ring_free()
597 if (!(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_free()
600 k3_ringacc_ring_free_sci(ring); in k3_ringacc_ring_free()
602 dma_free_coherent(ring->dma_dev, in k3_ringacc_ring_free()
603 ring->size * (4 << ring->elm_size), in k3_ringacc_ring_free()
604 ring->ring_mem_virt, ring->ring_mem_dma); in k3_ringacc_ring_free()
605 ring->flags = 0; in k3_ringacc_ring_free()
606 ring->ops = NULL; in k3_ringacc_ring_free()
607 ring->dma_dev = NULL; in k3_ringacc_ring_free()
608 ring->asel = 0; in k3_ringacc_ring_free()
610 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { in k3_ringacc_ring_free()
611 clear_bit(ring->proxy_id, ringacc->proxy_inuse); in k3_ringacc_ring_free()
612 ring->proxy = NULL; in k3_ringacc_ring_free()
613 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; in k3_ringacc_ring_free()
617 clear_bit(ring->ring_id, ringacc->rings_inuse); in k3_ringacc_ring_free()
625 u32 k3_ringacc_get_ring_id(struct k3_ring *ring) in k3_ringacc_get_ring_id() argument
627 if (!ring) in k3_ringacc_get_ring_id()
630 return ring->ring_id; in k3_ringacc_get_ring_id()
634 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) in k3_ringacc_get_tisci_dev_id() argument
636 if (!ring) in k3_ringacc_get_tisci_dev_id()
639 return ring->parent->tisci_dev_id; in k3_ringacc_get_tisci_dev_id()
643 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) in k3_ringacc_get_ring_irq_num() argument
647 if (!ring) in k3_ringacc_get_ring_irq_num()
650 irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); in k3_ringacc_get_ring_irq_num()
657 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) in k3_ringacc_ring_cfg_sci() argument
660 struct k3_ringacc *ringacc = ring->parent; in k3_ringacc_ring_cfg_sci()
667 ring_cfg.index = ring->ring_id; in k3_ringacc_ring_cfg_sci()
669 ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma); in k3_ringacc_ring_cfg_sci()
670 ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma); in k3_ringacc_ring_cfg_sci()
671 ring_cfg.count = ring->size; in k3_ringacc_ring_cfg_sci()
672 ring_cfg.mode = ring->mode; in k3_ringacc_ring_cfg_sci()
673 ring_cfg.size = ring->elm_size; in k3_ringacc_ring_cfg_sci()
674 ring_cfg.asel = ring->asel; in k3_ringacc_ring_cfg_sci()
679 ret, ring->ring_id); in k3_ringacc_ring_cfg_sci()
684 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) in k3_dmaring_cfg() argument
695 ringacc = ring->parent; in k3_dmaring_cfg()
701 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) in k3_dmaring_cfg()
704 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) in k3_dmaring_cfg()
707 ring->size = cfg->size; in k3_dmaring_cfg()
708 ring->elm_size = cfg->elm_size; in k3_dmaring_cfg()
709 ring->mode = cfg->mode; in k3_dmaring_cfg()
710 ring->asel = cfg->asel; in k3_dmaring_cfg()
711 ring->dma_dev = cfg->dma_dev; in k3_dmaring_cfg()
712 if (!ring->dma_dev) { in k3_dmaring_cfg()
714 ring->ring_id); in k3_dmaring_cfg()
715 ring->dma_dev = ringacc->dev; in k3_dmaring_cfg()
718 memset(&ring->state, 0, sizeof(ring->state)); in k3_dmaring_cfg()
720 ring->ops = &k3_dmaring_fwd_ops; in k3_dmaring_cfg()
722 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, in k3_dmaring_cfg()
723 ring->size * (4 << ring->elm_size), in k3_dmaring_cfg()
724 &ring->ring_mem_dma, GFP_KERNEL); in k3_dmaring_cfg()
725 if (!ring->ring_mem_virt) { in k3_dmaring_cfg()
731 ret = k3_ringacc_ring_cfg_sci(ring); in k3_dmaring_cfg()
735 ring->flags |= K3_RING_FLAG_BUSY; in k3_dmaring_cfg()
737 k3_ringacc_ring_dump(ring); in k3_dmaring_cfg()
740 reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings]; in k3_dmaring_cfg()
748 reverse_ring->ring_mem_virt = ring->ring_mem_virt; in k3_dmaring_cfg()
749 reverse_ring->ring_mem_dma = ring->ring_mem_dma; in k3_dmaring_cfg()
756 dma_free_coherent(ring->dma_dev, in k3_dmaring_cfg()
757 ring->size * (4 << ring->elm_size), in k3_dmaring_cfg()
758 ring->ring_mem_virt, in k3_dmaring_cfg()
759 ring->ring_mem_dma); in k3_dmaring_cfg()
761 ring->ops = NULL; in k3_dmaring_cfg()
762 ring->proxy = NULL; in k3_dmaring_cfg()
763 ring->dma_dev = NULL; in k3_dmaring_cfg()
764 ring->asel = 0; in k3_dmaring_cfg()
768 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) in k3_ringacc_ring_cfg() argument
773 if (!ring || !cfg) in k3_ringacc_ring_cfg()
776 ringacc = ring->parent; in k3_ringacc_ring_cfg()
779 return k3_dmaring_cfg(ring, cfg); in k3_ringacc_ring_cfg()
784 !test_bit(ring->ring_id, ringacc->rings_inuse)) in k3_ringacc_ring_cfg()
788 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && in k3_ringacc_ring_cfg()
792 4 << ring->elm_size); in k3_ringacc_ring_cfg()
804 if (ring->use_count != 1) in k3_ringacc_ring_cfg()
807 ring->size = cfg->size; in k3_ringacc_ring_cfg()
808 ring->elm_size = cfg->elm_size; in k3_ringacc_ring_cfg()
809 ring->mode = cfg->mode; in k3_ringacc_ring_cfg()
810 memset(&ring->state, 0, sizeof(ring->state)); in k3_ringacc_ring_cfg()
812 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) in k3_ringacc_ring_cfg()
813 ring->proxy = ringacc->proxy_target_base + in k3_ringacc_ring_cfg()
814 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; in k3_ringacc_ring_cfg()
816 switch (ring->mode) { in k3_ringacc_ring_cfg()
818 ring->ops = &k3_ring_mode_ring_ops; in k3_ringacc_ring_cfg()
819 ring->dma_dev = cfg->dma_dev; in k3_ringacc_ring_cfg()
820 if (!ring->dma_dev) in k3_ringacc_ring_cfg()
821 ring->dma_dev = ringacc->dev; in k3_ringacc_ring_cfg()
824 ring->dma_dev = ringacc->dev; in k3_ringacc_ring_cfg()
825 if (ring->proxy) in k3_ringacc_ring_cfg()
826 ring->ops = &k3_ring_mode_proxy_ops; in k3_ringacc_ring_cfg()
828 ring->ops = &k3_ring_mode_msg_ops; in k3_ringacc_ring_cfg()
831 ring->ops = NULL; in k3_ringacc_ring_cfg()
836 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, in k3_ringacc_ring_cfg()
837 ring->size * (4 << ring->elm_size), in k3_ringacc_ring_cfg()
838 &ring->ring_mem_dma, GFP_KERNEL); in k3_ringacc_ring_cfg()
839 if (!ring->ring_mem_virt) { in k3_ringacc_ring_cfg()
845 ret = k3_ringacc_ring_cfg_sci(ring); in k3_ringacc_ring_cfg()
850 ring->flags |= K3_RING_FLAG_BUSY; in k3_ringacc_ring_cfg()
851 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? in k3_ringacc_ring_cfg()
854 k3_ringacc_ring_dump(ring); in k3_ringacc_ring_cfg()
859 dma_free_coherent(ring->dma_dev, in k3_ringacc_ring_cfg()
860 ring->size * (4 << ring->elm_size), in k3_ringacc_ring_cfg()
861 ring->ring_mem_virt, in k3_ringacc_ring_cfg()
862 ring->ring_mem_dma); in k3_ringacc_ring_cfg()
864 ring->ops = NULL; in k3_ringacc_ring_cfg()
865 ring->dma_dev = NULL; in k3_ringacc_ring_cfg()
867 ring->proxy = NULL; in k3_ringacc_ring_cfg()
872 u32 k3_ringacc_ring_get_size(struct k3_ring *ring) in k3_ringacc_ring_get_size() argument
874 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_get_size()
877 return ring->size; in k3_ringacc_ring_get_size()
881 u32 k3_ringacc_ring_get_free(struct k3_ring *ring) in k3_ringacc_ring_get_free() argument
883 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_get_free()
886 if (!ring->state.free) in k3_ringacc_ring_get_free()
887 ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring); in k3_ringacc_ring_get_free()
889 return ring->state.free; in k3_ringacc_ring_get_free()
893 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) in k3_ringacc_ring_get_occ() argument
895 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_get_occ()
898 return k3_ringacc_ring_read_occ(ring); in k3_ringacc_ring_get_occ()
902 u32 k3_ringacc_ring_is_full(struct k3_ring *ring) in k3_ringacc_ring_is_full() argument
904 return !k3_ringacc_ring_get_free(ring); in k3_ringacc_ring_is_full()
919 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, in k3_ringacc_ring_cfg_proxy() argument
924 val = ring->ring_id; in k3_ringacc_ring_cfg_proxy()
926 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); in k3_ringacc_ring_cfg_proxy()
927 writel(val, &ring->proxy->control); in k3_ringacc_ring_cfg_proxy()
931 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, in k3_ringacc_ring_access_proxy() argument
936 ptr = (void __iomem *)&ring->proxy->data; in k3_ringacc_ring_access_proxy()
941 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); in k3_ringacc_ring_access_proxy()
945 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); in k3_ringacc_ring_access_proxy()
951 ptr += k3_ringacc_ring_get_fifo_pos(ring); in k3_ringacc_ring_access_proxy()
956 dev_dbg(ring->parent->dev, in k3_ringacc_ring_access_proxy()
959 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); in k3_ringacc_ring_access_proxy()
960 ring->state.occ--; in k3_ringacc_ring_access_proxy()
964 dev_dbg(ring->parent->dev, in k3_ringacc_ring_access_proxy()
967 memcpy_toio(ptr, elem, (4 << ring->elm_size)); in k3_ringacc_ring_access_proxy()
968 ring->state.free--; in k3_ringacc_ring_access_proxy()
974 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, in k3_ringacc_ring_access_proxy()
975 ring->state.occ); in k3_ringacc_ring_access_proxy()
979 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_head_proxy() argument
981 return k3_ringacc_ring_access_proxy(ring, elem, in k3_ringacc_ring_push_head_proxy()
985 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_tail_proxy() argument
987 return k3_ringacc_ring_access_proxy(ring, elem, in k3_ringacc_ring_push_tail_proxy()
991 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_head_proxy() argument
993 return k3_ringacc_ring_access_proxy(ring, elem, in k3_ringacc_ring_pop_head_proxy()
997 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_tail_proxy() argument
999 return k3_ringacc_ring_access_proxy(ring, elem, in k3_ringacc_ring_pop_tail_proxy()
1003 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, in k3_ringacc_ring_access_io() argument
1011 ptr = (void __iomem *)&ring->fifos->head_data; in k3_ringacc_ring_access_io()
1015 ptr = (void __iomem *)&ring->fifos->tail_data; in k3_ringacc_ring_access_io()
1021 ptr += k3_ringacc_ring_get_fifo_pos(ring); in k3_ringacc_ring_access_io()
1026 dev_dbg(ring->parent->dev, in k3_ringacc_ring_access_io()
1029 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); in k3_ringacc_ring_access_io()
1030 ring->state.occ--; in k3_ringacc_ring_access_io()
1034 dev_dbg(ring->parent->dev, in k3_ringacc_ring_access_io()
1037 memcpy_toio(ptr, elem, (4 << ring->elm_size)); in k3_ringacc_ring_access_io()
1038 ring->state.free--; in k3_ringacc_ring_access_io()
1044 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", in k3_ringacc_ring_access_io()
1045 ring->state.free, ring->state.windex, ring->state.occ, in k3_ringacc_ring_access_io()
1046 ring->state.rindex); in k3_ringacc_ring_access_io()
1050 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_head_io() argument
1052 return k3_ringacc_ring_access_io(ring, elem, in k3_ringacc_ring_push_head_io()
1056 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_io() argument
1058 return k3_ringacc_ring_access_io(ring, elem, in k3_ringacc_ring_push_io()
1062 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_io() argument
1064 return k3_ringacc_ring_access_io(ring, elem, in k3_ringacc_ring_pop_io()
1068 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_tail_io() argument
1070 return k3_ringacc_ring_access_io(ring, elem, in k3_ringacc_ring_pop_tail_io()
1084 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem) in k3_dmaring_fwd_pop() argument
1095 ring->state.occ = k3_ringacc_ring_read_occ(ring); in k3_dmaring_fwd_pop()
1096 if (ring->state.windex >= ring->state.occ) in k3_dmaring_fwd_pop()
1097 elem_idx = ring->state.windex - ring->state.occ; in k3_dmaring_fwd_pop()
1099 elem_idx = ring->size - (ring->state.occ - ring->state.windex); in k3_dmaring_fwd_pop()
1101 elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx); in k3_dmaring_fwd_pop()
1102 memcpy(elem, elem_ptr, (4 << ring->elm_size)); in k3_dmaring_fwd_pop()
1105 ring->state.occ--; in k3_dmaring_fwd_pop()
1106 writel(-1, &ring->rt->db); in k3_dmaring_fwd_pop()
1108 dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n", in k3_dmaring_fwd_pop()
1109 __func__, ring->state.occ, ring->state.windex, elem_idx, in k3_dmaring_fwd_pop()
1114 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem) in k3_dmaring_reverse_pop() argument
1118 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); in k3_dmaring_reverse_pop()
1120 if (ring->state.occ) { in k3_dmaring_reverse_pop()
1121 memcpy(elem, elem_ptr, (4 << ring->elm_size)); in k3_dmaring_reverse_pop()
1124 ring->state.rindex = (ring->state.rindex + 1) % ring->size; in k3_dmaring_reverse_pop()
1125 ring->state.occ--; in k3_dmaring_reverse_pop()
1126 writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db); in k3_dmaring_reverse_pop()
1127 } else if (ring->state.tdown_complete) { in k3_dmaring_reverse_pop()
1131 writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db); in k3_dmaring_reverse_pop()
1132 ring->state.tdown_complete = false; in k3_dmaring_reverse_pop()
1135 dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n", in k3_dmaring_reverse_pop()
1136 __func__, ring->state.occ, ring->state.rindex, elem_ptr); in k3_dmaring_reverse_pop()
1140 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_mem() argument
1144 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); in k3_ringacc_ring_push_mem()
1146 memcpy(elem_ptr, elem, (4 << ring->elm_size)); in k3_ringacc_ring_push_mem()
1147 if (ring->parent->dma_rings) { in k3_ringacc_ring_push_mem()
1150 *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT); in k3_ringacc_ring_push_mem()
1153 ring->state.windex = (ring->state.windex + 1) % ring->size; in k3_ringacc_ring_push_mem()
1154 ring->state.free--; in k3_ringacc_ring_push_mem()
1155 writel(1, &ring->rt->db); in k3_ringacc_ring_push_mem()
1157 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", in k3_ringacc_ring_push_mem()
1158 ring->state.free, ring->state.windex); in k3_ringacc_ring_push_mem()
1163 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_mem() argument
1167 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); in k3_ringacc_ring_pop_mem()
1169 memcpy(elem, elem_ptr, (4 << ring->elm_size)); in k3_ringacc_ring_pop_mem()
1171 ring->state.rindex = (ring->state.rindex + 1) % ring->size; in k3_ringacc_ring_pop_mem()
1172 ring->state.occ--; in k3_ringacc_ring_pop_mem()
1173 writel(-1, &ring->rt->db); in k3_ringacc_ring_pop_mem()
1175 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", in k3_ringacc_ring_pop_mem()
1176 ring->state.occ, ring->state.rindex, elem_ptr); in k3_ringacc_ring_pop_mem()
1180 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push() argument
1184 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_push()
1187 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", in k3_ringacc_ring_push()
1188 ring->state.free, ring->state.windex); in k3_ringacc_ring_push()
1190 if (k3_ringacc_ring_is_full(ring)) in k3_ringacc_ring_push()
1193 if (ring->ops && ring->ops->push_tail) in k3_ringacc_ring_push()
1194 ret = ring->ops->push_tail(ring, elem); in k3_ringacc_ring_push()
1200 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) in k3_ringacc_ring_push_head() argument
1204 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_push_head()
1207 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", in k3_ringacc_ring_push_head()
1208 ring->state.free, ring->state.windex); in k3_ringacc_ring_push_head()
1210 if (k3_ringacc_ring_is_full(ring)) in k3_ringacc_ring_push_head()
1213 if (ring->ops && ring->ops->push_head) in k3_ringacc_ring_push_head()
1214 ret = ring->ops->push_head(ring, elem); in k3_ringacc_ring_push_head()
1220 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop() argument
1224 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_pop()
1227 if (!ring->state.occ) in k3_ringacc_ring_pop()
1228 k3_ringacc_ring_update_occ(ring); in k3_ringacc_ring_pop()
1230 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, in k3_ringacc_ring_pop()
1231 ring->state.rindex); in k3_ringacc_ring_pop()
1233 if (!ring->state.occ && !ring->state.tdown_complete) in k3_ringacc_ring_pop()
1236 if (ring->ops && ring->ops->pop_head) in k3_ringacc_ring_pop()
1237 ret = ring->ops->pop_head(ring, elem); in k3_ringacc_ring_pop()
1243 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) in k3_ringacc_ring_pop_tail() argument
1247 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) in k3_ringacc_ring_pop_tail()
1250 if (!ring->state.occ) in k3_ringacc_ring_pop_tail()
1251 k3_ringacc_ring_update_occ(ring); in k3_ringacc_ring_pop_tail()
1253 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", in k3_ringacc_ring_pop_tail()
1254 ring->state.occ, ring->state.rindex); in k3_ringacc_ring_pop_tail()
1256 if (!ring->state.occ) in k3_ringacc_ring_pop_tail()
1259 if (ring->ops && ring->ops->pop_tail) in k3_ringacc_ring_pop_tail()
1260 ret = ring->ops->pop_tail(ring, elem); in k3_ringacc_ring_pop_tail()
1494 struct k3_ring *ring = &ringacc->rings[i]; in k3_ringacc_dmarings_init() local
1496 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i; in k3_ringacc_dmarings_init()
1497 ring->parent = ringacc; in k3_ringacc_dmarings_init()
1498 ring->ring_id = i; in k3_ringacc_dmarings_init()
1499 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; in k3_ringacc_dmarings_init()
1501 ring = &ringacc->rings[ringacc->num_rings + i]; in k3_ringacc_dmarings_init()
1502 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i + in k3_ringacc_dmarings_init()
1504 ring->parent = ringacc; in k3_ringacc_dmarings_init()
1505 ring->ring_id = i; in k3_ringacc_dmarings_init()
1506 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; in k3_ringacc_dmarings_init()
1507 ring->flags = K3_RING_FLAG_REVERSE; in k3_ringacc_dmarings_init()