Lines Matching refs:split
142 } split; member
400 struct vring_desc_extra *extra = vq->split.desc_extra; in vring_unmap_one_split()
458 struct vring_desc_extra *extra = vring->split.desc_extra; in virtqueue_add_desc_split()
514 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect); in virtqueue_add_split()
525 desc = vq->split.vring.desc; in virtqueue_add_split()
579 vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &= in virtqueue_add_split()
590 virtqueue_add_desc_split(_vq, vq->split.vring.desc, in virtqueue_add_split()
602 vq->free_head = vq->split.desc_extra[head].next; in virtqueue_add_split()
607 vq->split.desc_state[head].data = data; in virtqueue_add_split()
609 vq->split.desc_state[head].indir_desc = desc; in virtqueue_add_split()
611 vq->split.desc_state[head].indir_desc = ctx; in virtqueue_add_split()
615 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1); in virtqueue_add_split()
616 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head); in virtqueue_add_split()
621 vq->split.avail_idx_shadow++; in virtqueue_add_split()
622 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
623 vq->split.avail_idx_shadow); in virtqueue_add_split()
672 old = vq->split.avail_idx_shadow - vq->num_added; in virtqueue_kick_prepare_split()
673 new = vq->split.avail_idx_shadow; in virtqueue_kick_prepare_split()
681 vring_avail_event(&vq->split.vring)), in virtqueue_kick_prepare_split()
684 needs_kick = !(vq->split.vring.used->flags & in virtqueue_kick_prepare_split()
699 vq->split.desc_state[head].data = NULL; in detach_buf_split()
704 while (vq->split.vring.desc[i].flags & nextflag) { in detach_buf_split()
706 i = vq->split.desc_extra[i].next; in detach_buf_split()
711 vq->split.desc_extra[i].next = vq->free_head; in detach_buf_split()
719 vq->split.desc_state[head].indir_desc; in detach_buf_split()
726 len = vq->split.desc_extra[head].len; in detach_buf_split()
728 BUG_ON(!(vq->split.desc_extra[head].flags & in detach_buf_split()
736 vq->split.desc_state[head].indir_desc = NULL; in detach_buf_split()
738 *ctx = vq->split.desc_state[head].indir_desc; in detach_buf_split()
745 vq->split.vring.used->idx); in more_used_split()
773 last_used = (vq->last_used_idx & (vq->split.vring.num - 1)); in virtqueue_get_buf_ctx_split()
775 vq->split.vring.used->ring[last_used].id); in virtqueue_get_buf_ctx_split()
777 vq->split.vring.used->ring[last_used].len); in virtqueue_get_buf_ctx_split()
779 if (unlikely(i >= vq->split.vring.num)) { in virtqueue_get_buf_ctx_split()
783 if (unlikely(!vq->split.desc_state[i].data)) { in virtqueue_get_buf_ctx_split()
789 ret = vq->split.desc_state[i].data; in virtqueue_get_buf_ctx_split()
795 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) in virtqueue_get_buf_ctx_split()
797 &vring_used_event(&vq->split.vring), in virtqueue_get_buf_ctx_split()
810 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { in virtqueue_disable_cb_split()
811 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_disable_cb_split()
814 vring_used_event(&vq->split.vring) = 0x0; in virtqueue_disable_cb_split()
816 vq->split.vring.avail->flags = in virtqueue_disable_cb_split()
818 vq->split.avail_flags_shadow); in virtqueue_disable_cb_split()
834 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_prepare_split()
835 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_prepare_split()
837 vq->split.vring.avail->flags = in virtqueue_enable_cb_prepare_split()
839 vq->split.avail_flags_shadow); in virtqueue_enable_cb_prepare_split()
841 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev, in virtqueue_enable_cb_prepare_split()
852 vq->split.vring.used->idx); in virtqueue_poll_split()
867 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_delayed_split()
868 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_delayed_split()
870 vq->split.vring.avail->flags = in virtqueue_enable_cb_delayed_split()
872 vq->split.avail_flags_shadow); in virtqueue_enable_cb_delayed_split()
875 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4; in virtqueue_enable_cb_delayed_split()
878 &vring_used_event(&vq->split.vring), in virtqueue_enable_cb_delayed_split()
881 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx) in virtqueue_enable_cb_delayed_split()
899 for (i = 0; i < vq->split.vring.num; i++) { in virtqueue_detach_unused_buf_split()
900 if (!vq->split.desc_state[i].data) in virtqueue_detach_unused_buf_split()
903 buf = vq->split.desc_state[i].data; in virtqueue_detach_unused_buf_split()
905 vq->split.avail_idx_shadow--; in virtqueue_detach_unused_buf_split()
906 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_detach_unused_buf_split()
907 vq->split.avail_idx_shadow); in virtqueue_detach_unused_buf_split()
912 BUG_ON(vq->vq.num_free != vq->split.vring.num); in virtqueue_detach_unused_buf_split()
975 to_vvq(vq)->split.queue_dma_addr = dma_addr; in vring_create_virtqueue_split()
976 to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes; in vring_create_virtqueue_split()
2214 vq->split.queue_dma_addr = 0; in __vring_new_virtqueue()
2215 vq->split.queue_size_in_bytes = 0; in __vring_new_virtqueue()
2217 vq->split.vring = vring; in __vring_new_virtqueue()
2218 vq->split.avail_flags_shadow = 0; in __vring_new_virtqueue()
2219 vq->split.avail_idx_shadow = 0; in __vring_new_virtqueue()
2223 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; in __vring_new_virtqueue()
2225 vq->split.vring.avail->flags = cpu_to_virtio16(vdev, in __vring_new_virtqueue()
2226 vq->split.avail_flags_shadow); in __vring_new_virtqueue()
2229 vq->split.desc_state = kmalloc_array(vring.num, in __vring_new_virtqueue()
2231 if (!vq->split.desc_state) in __vring_new_virtqueue()
2234 vq->split.desc_extra = vring_alloc_desc_extra(vq, vring.num); in __vring_new_virtqueue()
2235 if (!vq->split.desc_extra) in __vring_new_virtqueue()
2240 memset(vq->split.desc_state, 0, vring.num * in __vring_new_virtqueue()
2249 kfree(vq->split.desc_state); in __vring_new_virtqueue()
2332 vq->split.queue_size_in_bytes, in vring_del_virtqueue()
2333 vq->split.vring.desc, in vring_del_virtqueue()
2334 vq->split.queue_dma_addr); in vring_del_virtqueue()
2338 kfree(vq->split.desc_state); in vring_del_virtqueue()
2339 kfree(vq->split.desc_extra); in vring_del_virtqueue()
2384 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; in virtqueue_get_vring_size()
2424 return vq->split.queue_dma_addr; in virtqueue_get_desc_addr()
2437 return vq->split.queue_dma_addr + in virtqueue_get_avail_addr()
2438 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc); in virtqueue_get_avail_addr()
2451 return vq->split.queue_dma_addr + in virtqueue_get_used_addr()
2452 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc); in virtqueue_get_used_addr()
2459 return &to_vvq(vq)->split.vring; in virtqueue_get_vring()