1 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
2
3 /* Packet receive logic for Mellanox Gigabit Ethernet driver
4 *
5 * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
6 */
7
8 #include <linux/etherdevice.h>
9 #include <linux/skbuff.h>
10
11 #include "mlxbf_gige.h"
12 #include "mlxbf_gige_regs.h"
13
mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige * priv,unsigned int index,u64 dmac)14 void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv,
15 unsigned int index, u64 dmac)
16 {
17 void __iomem *base = priv->base;
18 u64 control;
19
20 /* Write destination MAC to specified MAC RX filter */
21 writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER +
22 (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
23
24 /* Enable MAC receive filter mask for specified index */
25 control = readq(base + MLXBF_GIGE_CONTROL);
26 control |= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);
27 writeq(control, base + MLXBF_GIGE_CONTROL);
28 }
29
mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige * priv,unsigned int index,u64 * dmac)30 void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv,
31 unsigned int index, u64 *dmac)
32 {
33 void __iomem *base = priv->base;
34
35 /* Read destination MAC from specified MAC RX filter */
36 *dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER +
37 (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
38 }
39
mlxbf_gige_enable_promisc(struct mlxbf_gige * priv)40 void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv)
41 {
42 void __iomem *base = priv->base;
43 u64 control;
44 u64 end_mac;
45
46 /* Enable MAC_ID_RANGE match functionality */
47 control = readq(base + MLXBF_GIGE_CONTROL);
48 control |= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
49 writeq(control, base + MLXBF_GIGE_CONTROL);
50
51 /* Set start of destination MAC range check to 0 */
52 writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START);
53
54 /* Set end of destination MAC range check to all FFs */
55 end_mac = BCAST_MAC_ADDR;
56 writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END);
57 }
58
mlxbf_gige_disable_promisc(struct mlxbf_gige * priv)59 void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv)
60 {
61 void __iomem *base = priv->base;
62 u64 control;
63
64 /* Disable MAC_ID_RANGE match functionality */
65 control = readq(base + MLXBF_GIGE_CONTROL);
66 control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
67 writeq(control, base + MLXBF_GIGE_CONTROL);
68
69 /* NOTE: no need to change DMAC_RANGE_START or END;
70 * those values are ignored since MAC_ID_RANGE_EN=0
71 */
72 }
73
74 /* Receive Initialization
75 * 1) Configures RX MAC filters via MMIO registers
76 * 2) Allocates RX WQE array using coherent DMA mapping
77 * 3) Initializes each element of RX WQE array with a receive
78 * buffer pointer (also using coherent DMA mapping)
79 * 4) Allocates RX CQE array using coherent DMA mapping
80 * 5) Completes other misc receive initialization
81 */
mlxbf_gige_rx_init(struct mlxbf_gige * priv)82 int mlxbf_gige_rx_init(struct mlxbf_gige *priv)
83 {
84 size_t wq_size, cq_size;
85 dma_addr_t *rx_wqe_ptr;
86 dma_addr_t rx_buf_dma;
87 u64 data;
88 int i, j;
89
90 /* Configure MAC RX filter #0 to allow RX of broadcast pkts */
91 mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX,
92 BCAST_MAC_ADDR);
93
94 wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
95 priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size,
96 &priv->rx_wqe_base_dma,
97 GFP_KERNEL);
98 if (!priv->rx_wqe_base)
99 return -ENOMEM;
100
101 /* Initialize 'rx_wqe_ptr' to point to first RX WQE in array
102 * Each RX WQE is simply a receive buffer pointer, so walk
103 * the entire array, allocating a 2KB buffer for each element
104 */
105 rx_wqe_ptr = priv->rx_wqe_base;
106
107 for (i = 0; i < priv->rx_q_entries; i++) {
108 priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
109 &rx_buf_dma, DMA_FROM_DEVICE);
110 if (!priv->rx_skb[i])
111 goto free_wqe_and_skb;
112 *rx_wqe_ptr++ = rx_buf_dma;
113 }
114
115 /* Write RX WQE base address into MMIO reg */
116 writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE);
117
118 cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
119 priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size,
120 &priv->rx_cqe_base_dma,
121 GFP_KERNEL);
122 if (!priv->rx_cqe_base)
123 goto free_wqe_and_skb;
124
125 for (i = 0; i < priv->rx_q_entries; i++)
126 priv->rx_cqe_base[i] |= MLXBF_GIGE_RX_CQE_VALID_MASK;
127
128 /* Write RX CQE base address into MMIO reg */
129 writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE);
130
131 /* Write RX_WQE_PI with current number of replenished buffers */
132 writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI);
133
134 /* Enable removal of CRC during RX */
135 data = readq(priv->base + MLXBF_GIGE_RX);
136 data |= MLXBF_GIGE_RX_STRIP_CRC_EN;
137 writeq(data, priv->base + MLXBF_GIGE_RX);
138
139 /* Enable RX MAC filter pass and discard counters */
140 writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN,
141 priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC);
142 writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN,
143 priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS);
144
145 /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
146 * indicate readiness to receive interrupts
147 */
148 data = readq(priv->base + MLXBF_GIGE_INT_MASK);
149 data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
150 writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
151
152 /* Enable RX DMA to write new packets to memory */
153 data = readq(priv->base + MLXBF_GIGE_RX_DMA);
154 data |= MLXBF_GIGE_RX_DMA_EN;
155 writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
156
157 writeq(ilog2(priv->rx_q_entries),
158 priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2);
159
160 return 0;
161
162 free_wqe_and_skb:
163 rx_wqe_ptr = priv->rx_wqe_base;
164 for (j = 0; j < i; j++) {
165 dma_unmap_single(priv->dev, *rx_wqe_ptr,
166 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
167 dev_kfree_skb(priv->rx_skb[j]);
168 rx_wqe_ptr++;
169 }
170 dma_free_coherent(priv->dev, wq_size,
171 priv->rx_wqe_base, priv->rx_wqe_base_dma);
172 return -ENOMEM;
173 }
174
175 /* Receive Deinitialization
176 * This routine will free allocations done by mlxbf_gige_rx_init(),
177 * namely the RX WQE and RX CQE arrays, as well as all RX buffers
178 */
mlxbf_gige_rx_deinit(struct mlxbf_gige * priv)179 void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv)
180 {
181 dma_addr_t *rx_wqe_ptr;
182 size_t size;
183 u64 data;
184 int i;
185
186 /* Disable RX DMA to prevent packet transfers to memory */
187 data = readq(priv->base + MLXBF_GIGE_RX_DMA);
188 data &= ~MLXBF_GIGE_RX_DMA_EN;
189 writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
190
191 rx_wqe_ptr = priv->rx_wqe_base;
192
193 for (i = 0; i < priv->rx_q_entries; i++) {
194 dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ,
195 DMA_FROM_DEVICE);
196 dev_kfree_skb(priv->rx_skb[i]);
197 rx_wqe_ptr++;
198 }
199
200 size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
201 dma_free_coherent(priv->dev, size,
202 priv->rx_wqe_base, priv->rx_wqe_base_dma);
203
204 size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
205 dma_free_coherent(priv->dev, size,
206 priv->rx_cqe_base, priv->rx_cqe_base_dma);
207
208 priv->rx_wqe_base = NULL;
209 priv->rx_wqe_base_dma = 0;
210 priv->rx_cqe_base = NULL;
211 priv->rx_cqe_base_dma = 0;
212 writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE);
213 writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE);
214 }
215
mlxbf_gige_rx_packet(struct mlxbf_gige * priv,int * rx_pkts)216 static bool mlxbf_gige_rx_packet(struct mlxbf_gige *priv, int *rx_pkts)
217 {
218 struct net_device *netdev = priv->netdev;
219 struct sk_buff *skb = NULL, *rx_skb;
220 u16 rx_pi_rem, rx_ci_rem;
221 dma_addr_t *rx_wqe_addr;
222 dma_addr_t rx_buf_dma;
223 u64 *rx_cqe_addr;
224 u64 datalen;
225 u64 rx_cqe;
226 u16 rx_ci;
227 u16 rx_pi;
228
229 /* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */
230 rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI);
231 rx_pi_rem = rx_pi % priv->rx_q_entries;
232
233 rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem;
234 rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem;
235 rx_cqe = *rx_cqe_addr;
236
237 if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity)
238 return false;
239
240 if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) {
241 /* Packet is OK, increment stats */
242 datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK;
243 netdev->stats.rx_packets++;
244 netdev->stats.rx_bytes += datalen;
245
246 skb = priv->rx_skb[rx_pi_rem];
247
248 skb_put(skb, datalen);
249
250 skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */
251
252 skb->protocol = eth_type_trans(skb, netdev);
253
254 /* Alloc another RX SKB for this same index */
255 rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
256 &rx_buf_dma, DMA_FROM_DEVICE);
257 if (!rx_skb)
258 return false;
259 priv->rx_skb[rx_pi_rem] = rx_skb;
260 dma_unmap_single(priv->dev, *rx_wqe_addr,
261 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
262 *rx_wqe_addr = rx_buf_dma;
263 } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) {
264 priv->stats.rx_mac_errors++;
265 } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) {
266 priv->stats.rx_truncate_errors++;
267 }
268
269 /* Let hardware know we've replenished one buffer */
270 rx_pi++;
271
272 /* Ensure completion of all writes before notifying HW of replenish */
273 wmb();
274 writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI);
275
276 (*rx_pkts)++;
277
278 rx_pi_rem = rx_pi % priv->rx_q_entries;
279 if (rx_pi_rem == 0)
280 priv->valid_polarity ^= 1;
281 rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI);
282 rx_ci_rem = rx_ci % priv->rx_q_entries;
283
284 if (skb)
285 netif_receive_skb(skb);
286
287 return rx_pi_rem != rx_ci_rem;
288 }
289
290 /* Driver poll() function called by NAPI infrastructure */
mlxbf_gige_poll(struct napi_struct * napi,int budget)291 int mlxbf_gige_poll(struct napi_struct *napi, int budget)
292 {
293 struct mlxbf_gige *priv;
294 bool remaining_pkts;
295 int work_done = 0;
296 u64 data;
297
298 priv = container_of(napi, struct mlxbf_gige, napi);
299
300 mlxbf_gige_handle_tx_complete(priv);
301
302 do {
303 remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done);
304 } while (remaining_pkts && work_done < budget);
305
306 /* If amount of work done < budget, turn off NAPI polling
307 * via napi_complete_done(napi, work_done) and then
308 * re-enable interrupts.
309 */
310 if (work_done < budget && napi_complete_done(napi, work_done)) {
311 /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
312 * indicate receive readiness
313 */
314 data = readq(priv->base + MLXBF_GIGE_INT_MASK);
315 data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
316 writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
317 }
318
319 return work_done;
320 }
321