| /linux/net/core/ |
| A D | xdp.c | 377 __xdp_return(xdpf->data, &xdpf->mem, false, NULL); in xdp_return_frame()
383 __xdp_return(xdpf->data, &xdpf->mem, true, NULL); in xdp_return_frame_rx_napi()
418 __xdp_return(xdpf->data, &xdpf->mem, false, NULL); in xdp_return_frame_bulk()
491 xdpf = addr; in xdp_convert_zc_to_xdp_frame()
492 memset(xdpf, 0, sizeof(*xdpf)); in xdp_convert_zc_to_xdp_frame()
494 addr += sizeof(*xdpf); in xdp_convert_zc_to_xdp_frame()
500 xdpf->headroom = 0; in xdp_convert_zc_to_xdp_frame()
506 return xdpf; in xdp_convert_zc_to_xdp_frame()
536 headroom = sizeof(*xdpf) + xdpf->headroom; in __xdp_build_skb_from_frame()
550 if (xdpf->metasize) in __xdp_build_skb_from_frame()
[all …]
|
| /linux/kernel/bpf/ |
| A D | cpumap.c | 152 struct xdp_frame *xdpf; in __cpu_map_ring_cleanup() local
155 if (WARN_ON_ONCE(xdpf)) in __cpu_map_ring_cleanup()
156 xdp_return_frame(xdpf); in __cpu_map_ring_cleanup()
228 rxq.dev = xdpf->dev_rx; in cpu_map_bpf_prog_run_xdp()
229 rxq.mem = xdpf->mem; in cpu_map_bpf_prog_run_xdp()
260 xdp_return_frame(xdpf); in cpu_map_bpf_prog_run_xdp()
378 xdpf->dev_rx); in cpu_map_kthread_run()
752 struct xdp_frame *xdpf; in cpu_map_enqueue() local
755 if (unlikely(!xdpf)) in cpu_map_enqueue()
759 xdpf->dev_rx = dev_rx; in cpu_map_enqueue()
[all …]
|
| A D | devmap.c | 346 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run()
348 frames[nframes++] = xdpf; in dev_map_bpf_prog_run()
357 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run()
378 prefetch(xdpf); in bq_xmit_all()
467 bq->q[bq->count++] = xdpf; in bq_enqueue()
474 struct xdp_frame *xdpf; in __xdp_enqueue() local
485 if (unlikely(!xdpf)) in __xdp_enqueue()
551 struct xdp_frame *xdpf) in dev_map_enqueue_clone() argument
555 nxdpf = xdpf_clone(xdpf); in dev_map_enqueue_clone()
596 struct xdp_frame *xdpf; in dev_map_enqueue_multi() local
[all …]
|
| /linux/drivers/net/ethernet/mellanox/mlx5/core/en/ |
| A D | xdp.c | 64 struct xdp_frame *xdpf; in mlx5e_xmit_xdp_buff() local
67 xdpf = xdp_convert_buff_to_frame(xdp); in mlx5e_xmit_xdp_buff()
68 if (unlikely(!xdpf)) in mlx5e_xmit_xdp_buff()
71 xdptxd.data = xdpf->data; in mlx5e_xmit_xdp_buff()
72 xdptxd.len = xdpf->len; in mlx5e_xmit_xdp_buff()
92 xdp_return_frame(xdpf); in mlx5e_xmit_xdp_buff()
97 xdpi.frame.xdpf = xdpf; in mlx5e_xmit_xdp_buff()
108 dma_addr = di->addr + (xdpf->data - (void *)xdpf); in mlx5e_xmit_xdp_buff()
527 xdptxd.data = xdpf->data; in mlx5e_xdp_xmit()
528 xdptxd.len = xdpf->len; in mlx5e_xdp_xmit()
[all …]
|
| /linux/include/net/ |
| A D | xdp.h | 167 struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
170 struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf,
173 struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf);
233 void xdp_return_frame(struct xdp_frame *xdpf);
234 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
237 void xdp_return_frame_bulk(struct xdp_frame *xdpf,
246 static inline void xdp_release_frame(struct xdp_frame *xdpf) in xdp_release_frame() argument
248 struct xdp_mem_info *mem = &xdpf->mem; in xdp_release_frame()
252 __xdp_release_frame(xdpf->data, mem); in xdp_release_frame()
|
| /linux/drivers/net/ethernet/intel/ixgbe/ |
| A D | ixgbe_xsk.c | 104 struct xdp_frame *xdpf; in ixgbe_run_xdp_zc() local
121 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc()
122 if (unlikely(!xdpf)) in ixgbe_run_xdp_zc()
127 result = ixgbe_xmit_xdp_ring(ring, xdpf); in ixgbe_run_xdp_zc()
407 tx_bi->xdpf = NULL; in ixgbe_xmit_zc()
438 xdp_return_frame(tx_bi->xdpf); in ixgbe_clean_xdp_tx_buffer()
465 if (tx_bi->xdpf) in ixgbe_clean_xdp_tx_irq()
470 tx_bi->xdpf = NULL; in ixgbe_clean_xdp_tx_irq()
544 if (tx_bi->xdpf) in ixgbe_xsk_clean_tx_ring()
549 tx_bi->xdpf = NULL; in ixgbe_xsk_clean_tx_ring()
|
| A D | ixgbe_txrx_common.h | 16 struct xdp_frame *xdpf);
|
| /linux/drivers/net/ethernet/ti/ |
| A D | cpsw_priv.c | 54 struct xdp_frame *xdpf; in cpsw_tx_handler() local
61 xdpf = cpsw_handle_to_xdpf(token); in cpsw_tx_handler()
62 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET; in cpsw_tx_handler()
65 xdp_return_frame(xdpf); in cpsw_tx_handler()
1306 dma, xdpf->len, port); in cpsw_xdp_tx_frame()
1308 if (sizeof(*xmeta) > xdpf->headroom) in cpsw_xdp_tx_frame()
1312 xdpf->data, xdpf->len, port); in cpsw_xdp_tx_frame()
1327 struct xdp_frame *xdpf; in cpsw_run_xdp() local
1344 xdpf = xdp_convert_buff_to_frame(xdp); in cpsw_run_xdp()
1345 if (unlikely(!xdpf)) in cpsw_run_xdp()
[all …]
|
| A D | cpsw_priv.h | 419 static inline void *cpsw_xdpf_to_handle(struct xdp_frame *xdpf) in cpsw_xdpf_to_handle() argument
421 return (void *)((unsigned long)xdpf | BIT(0)); in cpsw_xdpf_to_handle()
440 int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf,
|
| /linux/drivers/net/ethernet/freescale/dpaa/ |
| A D | dpaa_eth.c | 2213 memmove(aligned_data, xdpf->data, xdpf->len); in dpaa_a050385_wa_xdpf()
2240 memcpy(new_buff + headroom, xdpf->data, xdpf->len); in dpaa_a050385_wa_xdpf()
2247 new_xdpf->len = xdpf->len; in dpaa_a050385_wa_xdpf()
2508 buff_start = xdpf->data - xdpf->headroom; in dpaa_xdp_xmit_frame()
2515 swbp->xdpf = xdpf; in dpaa_xdp_xmit_frame()
2520 qm_fd_set_contig(&fd, xdpf->headroom, xdpf->len); in dpaa_xdp_xmit_frame()
2523 xdpf->headroom + xdpf->len, in dpaa_xdp_xmit_frame()
2556 struct xdp_frame *xdpf; in dpaa_run_xdp() local
2605 if (unlikely(!xdpf)) { in dpaa_run_xdp()
3077 struct xdp_frame *xdpf; in dpaa_xdp_xmit() local
[all …]
|
| A D | dpaa_eth.h | 155 struct xdp_frame *xdpf; member
|
| /linux/drivers/net/ethernet/socionext/ |
| A D | netsec.c | 272 struct xdp_frame *xdpf; member
675 bytes += desc->xdpf->len; in netsec_clean_tx_dring()
677 xdp_return_frame_rx_napi(desc->xdpf); in netsec_clean_tx_dring()
821 dring->desc[idx].xdpf = buf; in netsec_set_tx_de()
829 struct xdp_frame *xdpf, bool is_ndo) in netsec_xdp_queue_one() argument
851 dma_handle = dma_map_single(priv->dev, xdpf->data, xdpf->len, in netsec_xdp_queue_one()
866 sizeof(*xdpf); in netsec_xdp_queue_one()
873 tx_desc.addr = xdpf->data; in netsec_xdp_queue_one()
874 tx_desc.len = xdpf->len; in netsec_xdp_queue_one()
888 if (unlikely(!xdpf)) in netsec_xdp_xmit_back()
[all …]
|
| /linux/drivers/net/ethernet/broadcom/bnxt/ |
| A D | bnxt_xdp.c | 60 struct xdp_frame *xdpf) in __bnxt_xmit_xdp_redirect() argument
66 tx_buf->xdpf = xdpf; in __bnxt_xmit_xdp_redirect()
91 xdp_return_frame(tx_buf->xdpf); in bnxt_tx_int_xdp()
93 tx_buf->xdpf = NULL; in bnxt_tx_int_xdp()
|
| /linux/drivers/net/ethernet/intel/ice/ |
| A D | ice_txrx_lib.c | 331 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); in ice_xmit_xdp_buff() local
333 if (unlikely(!xdpf)) in ice_xmit_xdp_buff()
336 return ice_xmit_xdp_ring(xdpf->data, xdpf->len, xdp_ring); in ice_xmit_xdp_buff()
|
| /linux/drivers/net/ethernet/sfc/ |
| A D | tx.c | 426 struct xdp_frame *xdpf; in efx_xdp_tx_buffers() local
465 xdpf = xdpfs[i]; in efx_xdp_tx_buffers()
473 len = xdpf->len; in efx_xdp_tx_buffers()
477 xdpf->data, len, in efx_xdp_tx_buffers()
484 tx_buffer->xdpf = xdpf; in efx_xdp_tx_buffers()
|
| A D | rx.c | 257 struct xdp_frame *xdpf; in efx_do_xdp() local
312 xdpf = xdp_convert_buff_to_frame(&xdp); in efx_do_xdp()
313 err = efx_xdp_tx_buffers(efx, 1, &xdpf, true); in efx_do_xdp()
|
| /linux/drivers/net/ethernet/qlogic/qede/ |
| A D | qede_fp.c | 333 xdp->xdpf = xdpf; in qede_xdp_xmit()
346 struct xdp_frame *xdpf; in qede_xdp_transmit() local
363 xdpf = frames[i]; in qede_xdp_transmit()
365 mapping = dma_map_single(dmadev, xdpf->data, xdpf->len, in qede_xdp_transmit()
371 NULL, xdpf))) in qede_xdp_transmit()
405 struct xdp_frame *xdpf; in qede_xdp_tx_int() local
413 xdpf = xdp_info->xdpf; in qede_xdp_tx_int()
415 if (xdpf) { in qede_xdp_tx_int()
416 dma_unmap_single(dev, xdp_info->mapping, xdpf->len, in qede_xdp_tx_int()
418 xdp_return_frame(xdpf); in qede_xdp_tx_int()
[all …]
|
| /linux/drivers/net/ethernet/intel/i40e/ |
| A D | i40e_xsk.c | 504 xdp_return_frame(tx_bi->xdpf); in i40e_clean_xdp_tx_buffer()
544 if (tx_bi->xdpf) { in i40e_clean_xdp_tx_irq()
546 tx_bi->xdpf = NULL; in i40e_clean_xdp_tx_irq()
643 if (tx_bi->xdpf) in i40e_xsk_clean_tx_ring()
648 tx_bi->xdpf = NULL; in i40e_xsk_clean_tx_ring()
|
| A D | i40e_txrx.c | 763 xdp_return_frame(tx_buffer->xdpf); in i40e_unmap_and_free_tx_resource()
971 xdp_return_frame(tx_buf->xdpf); in i40e_clean_tx_irq()
2276 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
2283 if (unlikely(!xdpf)) in i40e_xmit_xdp_tx_ring()
2286 return i40e_xmit_xdp_ring(xdpf, xdp_ring); in i40e_xmit_xdp_tx_ring()
3688 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, in i40e_xmit_xdp_ring() argument
3694 void *data = xdpf->data; in i40e_xmit_xdp_ring()
3695 u32 size = xdpf->len; in i40e_xmit_xdp_ring()
3709 tx_bi->xdpf = xdpf; in i40e_xmit_xdp_ring()
3914 struct xdp_frame *xdpf = frames[i]; in i40e_xdp_xmit() local
[all …]
|
| /linux/drivers/net/ |
| A D | virtio_net.c | 504 struct xdp_frame *xdpf) in __virtnet_xdp_xmit_one() argument
513 xdpf->data -= vi->hdr_len; in __virtnet_xdp_xmit_one()
515 hdr = xdpf->data; in __virtnet_xdp_xmit_one()
517 xdpf->len += vi->hdr_len; in __virtnet_xdp_xmit_one()
519 sg_init_one(sq->sg, xdpf->data, xdpf->len); in __virtnet_xdp_xmit_one()
748 struct xdp_frame *xdpf; in receive_small() local
794 if (unlikely(!xdpf)) in receive_small()
798 xdp_return_frame_rx_napi(xdpf); in receive_small()
914 struct xdp_frame *xdpf; in receive_mergeable() local
995 if (unlikely(!xdpf)) in receive_mergeable()
[all …]
|
| A D | xen-netfront.c | 609 struct xdp_frame *xdpf) in xennet_xdp_xmit_one() argument
616 .page = virt_to_page(xdpf->data), in xennet_xdp_xmit_one()
621 offset_in_page(xdpf->data), in xennet_xdp_xmit_one()
622 xdpf->len); in xennet_xdp_xmit_one()
631 tx_stats->bytes += xdpf->len; in xennet_xdp_xmit_one()
659 struct xdp_frame *xdpf = frames[i]; in xennet_xdp_xmit() local
661 if (!xdpf) in xennet_xdp_xmit()
663 if (xennet_xdp_xmit_one(dev, queue, xdpf)) in xennet_xdp_xmit()
914 struct xdp_frame *xdpf; in xennet_run_xdp() local
928 xdpf = xdp_convert_buff_to_frame(xdp); in xennet_run_xdp()
[all …]
|
| /linux/drivers/net/ethernet/amazon/ena/ |
| A D | ena_netdev.c | 241 struct xdp_frame *xdpf, in ena_xdp_tx_map_frame() argument
251 tx_info->xdpf = xdpf; in ena_xdp_tx_map_frame()
321 xdpf->len); in ena_xdp_xmit_frame()
335 tx_info->xdpf = NULL; in ena_xdp_xmit_frame()
384 struct xdp_frame *xdpf; in ena_xdp_execute() local
397 if (unlikely(!xdpf)) { in ena_xdp_execute()
412 xdp_return_frame(xdpf); in ena_xdp_execute()
1920 xdpf = tx_info->xdpf; in ena_clean_xdp_irq()
1922 tx_info->xdpf = NULL; in ena_clean_xdp_irq()
1928 xdpf); in ena_clean_xdp_irq()
[all …]
|
| /linux/drivers/net/ethernet/stmicro/stmmac/ |
| A D | stmmac_main.c | 1541 if (tx_q->xdpf[i] && in stmmac_free_tx_buffer()
1545 tx_q->xdpf[i] = NULL; in stmmac_free_tx_buffer()
2511 xdpf = tx_q->xdpf[entry]; in stmmac_tx_clean()
2514 xdpf = NULL; in stmmac_tx_clean()
2517 xdpf = NULL; in stmmac_tx_clean()
2577 if (xdpf && in stmmac_tx_clean()
2583 if (xdpf && in stmmac_tx_clean()
4627 xdpf->headroom; in stmmac_xdp_xmit_xdpf()
4640 tx_q->xdpf[entry] = xdpf; in stmmac_xdp_xmit_xdpf()
4646 xdpf->len); in stmmac_xdp_xmit_xdpf()
[all …]
|
| /linux/drivers/net/ethernet/marvell/ |
| A D | mvneta.c | 618 struct xdp_frame *xdpf; member
1862 xdp_return_frame_rx_napi(buf->xdpf); in mvneta_txq_bufs_free()
2064 xdpf->len, DMA_TO_DEVICE); in mvneta_xdp_submit_frame()
2074 sizeof(*xdpf) + xdpf->headroom; in mvneta_xdp_submit_frame()
2076 xdpf->len, DMA_BIDIRECTIONAL); in mvneta_xdp_submit_frame()
2079 buf->xdpf = xdpf; in mvneta_xdp_submit_frame()
2083 tx_desc->data_size = xdpf->len; in mvneta_xdp_submit_frame()
2098 struct xdp_frame *xdpf; in mvneta_xdp_xmit_back() local
2102 xdpf = xdp_convert_buff_to_frame(xdp); in mvneta_xdp_xmit_back()
2103 if (unlikely(!xdpf)) in mvneta_xdp_xmit_back()
[all …]
|
| /linux/drivers/net/ethernet/intel/igc/ |
| A D | igc_main.c | 209 xdp_return_frame(tx_buffer->xdpf); in igc_clean_tx_ring()
2120 struct xdp_frame *xdpf, in igc_xdp_init_tx_buffer() argument
2125 dma = dma_map_single(ring->dev, xdpf->data, xdpf->len, DMA_TO_DEVICE); in igc_xdp_init_tx_buffer()
2132 buffer->xdpf = xdpf; in igc_xdp_init_tx_buffer()
2134 buffer->bytecount = xdpf->len; in igc_xdp_init_tx_buffer()
2137 dma_unmap_len_set(buffer, len, xdpf->len); in igc_xdp_init_tx_buffer()
2144 struct xdp_frame *xdpf) in igc_xdp_init_tx_descriptor() argument
2202 if (unlikely(!xdpf)) in igc_xdp_xmit_back()
2701 xdp_return_frame(tx_buffer->xdpf); in igc_clean_tx_irq()
6040 struct xdp_frame *xdpf = frames[i]; in igc_xdp_xmit() local
[all …]
|