1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * net/dst.h	Protocol independent destination cache definitions.
4  *
5  * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6  *
7  */
8 
9 #ifndef _NET_DST_H
10 #define _NET_DST_H
11 
12 #include <net/dst_ops.h>
13 #include <linux/netdevice.h>
14 #include <linux/rtnetlink.h>
15 #include <linux/rcupdate.h>
16 #include <linux/bug.h>
17 #include <linux/jiffies.h>
18 #include <linux/refcount.h>
19 #include <net/neighbour.h>
20 #include <asm/processor.h>
21 #include <linux/indirect_call_wrapper.h>
22 
23 struct sk_buff;
24 
25 struct dst_entry {
26 	struct net_device       *dev;
27 	struct  dst_ops	        *ops;
28 	unsigned long		_metrics;
29 	unsigned long           expires;
30 #ifdef CONFIG_XFRM
31 	struct xfrm_state	*xfrm;
32 #else
33 	void			*__pad1;
34 #endif
35 	int			(*input)(struct sk_buff *);
36 	int			(*output)(struct net *net, struct sock *sk, struct sk_buff *skb);
37 
38 	unsigned short		flags;
39 #define DST_NOXFRM		0x0002
40 #define DST_NOPOLICY		0x0004
41 #define DST_NOCOUNT		0x0008
42 #define DST_FAKE_RTABLE		0x0010
43 #define DST_XFRM_TUNNEL		0x0020
44 #define DST_XFRM_QUEUE		0x0040
45 #define DST_METADATA		0x0080
46 
47 	/* A non-zero value of dst->obsolete forces by-hand validation
48 	 * of the route entry.  Positive values are set by the generic
49 	 * dst layer to indicate that the entry has been forcefully
50 	 * destroyed.
51 	 *
52 	 * Negative values are used by the implementation layer code to
53 	 * force invocation of the dst_ops->check() method.
54 	 */
55 	short			obsolete;
56 #define DST_OBSOLETE_NONE	0
57 #define DST_OBSOLETE_DEAD	2
58 #define DST_OBSOLETE_FORCE_CHK	-1
59 #define DST_OBSOLETE_KILL	-2
60 	unsigned short		header_len;	/* more space at head required */
61 	unsigned short		trailer_len;	/* space to reserve at tail */
62 
63 	/*
64 	 * __refcnt wants to be on a different cache line from
65 	 * input/output/ops or performance tanks badly
66 	 */
67 #ifdef CONFIG_64BIT
68 	atomic_t		__refcnt;	/* 64-bit offset 64 */
69 #endif
70 	int			__use;
71 	unsigned long		lastuse;
72 	struct lwtunnel_state   *lwtstate;
73 	struct rcu_head		rcu_head;
74 	short			error;
75 	short			__pad;
76 	__u32			tclassid;
77 #ifndef CONFIG_64BIT
78 	atomic_t		__refcnt;	/* 32-bit offset 64 */
79 #endif
80 };
81 
82 struct dst_metrics {
83 	u32		metrics[RTAX_MAX];
84 	refcount_t	refcnt;
85 } __aligned(4);		/* Low pointer bits contain DST_METRICS_FLAGS */
86 extern const struct dst_metrics dst_default_metrics;
87 
88 u32 *dst_cow_metrics_generic(struct dst_entry *dst, unsigned long old);
89 
90 #define DST_METRICS_READ_ONLY		0x1UL
91 #define DST_METRICS_REFCOUNTED		0x2UL
92 #define DST_METRICS_FLAGS		0x3UL
93 #define __DST_METRICS_PTR(Y)	\
94 	((u32 *)((Y) & ~DST_METRICS_FLAGS))
95 #define DST_METRICS_PTR(X)	__DST_METRICS_PTR((X)->_metrics)
96 
dst_metrics_read_only(const struct dst_entry * dst)97 static inline bool dst_metrics_read_only(const struct dst_entry *dst)
98 {
99 	return dst->_metrics & DST_METRICS_READ_ONLY;
100 }
101 
102 void __dst_destroy_metrics_generic(struct dst_entry *dst, unsigned long old);
103 
dst_destroy_metrics_generic(struct dst_entry * dst)104 static inline void dst_destroy_metrics_generic(struct dst_entry *dst)
105 {
106 	unsigned long val = dst->_metrics;
107 	if (!(val & DST_METRICS_READ_ONLY))
108 		__dst_destroy_metrics_generic(dst, val);
109 }
110 
dst_metrics_write_ptr(struct dst_entry * dst)111 static inline u32 *dst_metrics_write_ptr(struct dst_entry *dst)
112 {
113 	unsigned long p = dst->_metrics;
114 
115 	BUG_ON(!p);
116 
117 	if (p & DST_METRICS_READ_ONLY)
118 		return dst->ops->cow_metrics(dst, p);
119 	return __DST_METRICS_PTR(p);
120 }
121 
122 /* This may only be invoked before the entry has reached global
123  * visibility.
124  */
dst_init_metrics(struct dst_entry * dst,const u32 * src_metrics,bool read_only)125 static inline void dst_init_metrics(struct dst_entry *dst,
126 				    const u32 *src_metrics,
127 				    bool read_only)
128 {
129 	dst->_metrics = ((unsigned long) src_metrics) |
130 		(read_only ? DST_METRICS_READ_ONLY : 0);
131 }
132 
dst_copy_metrics(struct dst_entry * dest,const struct dst_entry * src)133 static inline void dst_copy_metrics(struct dst_entry *dest, const struct dst_entry *src)
134 {
135 	u32 *dst_metrics = dst_metrics_write_ptr(dest);
136 
137 	if (dst_metrics) {
138 		u32 *src_metrics = DST_METRICS_PTR(src);
139 
140 		memcpy(dst_metrics, src_metrics, RTAX_MAX * sizeof(u32));
141 	}
142 }
143 
dst_metrics_ptr(struct dst_entry * dst)144 static inline u32 *dst_metrics_ptr(struct dst_entry *dst)
145 {
146 	return DST_METRICS_PTR(dst);
147 }
148 
149 static inline u32
dst_metric_raw(const struct dst_entry * dst,const int metric)150 dst_metric_raw(const struct dst_entry *dst, const int metric)
151 {
152 	u32 *p = DST_METRICS_PTR(dst);
153 
154 	return p[metric-1];
155 }
156 
157 static inline u32
dst_metric(const struct dst_entry * dst,const int metric)158 dst_metric(const struct dst_entry *dst, const int metric)
159 {
160 	WARN_ON_ONCE(metric == RTAX_HOPLIMIT ||
161 		     metric == RTAX_ADVMSS ||
162 		     metric == RTAX_MTU);
163 	return dst_metric_raw(dst, metric);
164 }
165 
166 static inline u32
dst_metric_advmss(const struct dst_entry * dst)167 dst_metric_advmss(const struct dst_entry *dst)
168 {
169 	u32 advmss = dst_metric_raw(dst, RTAX_ADVMSS);
170 
171 	if (!advmss)
172 		advmss = dst->ops->default_advmss(dst);
173 
174 	return advmss;
175 }
176 
dst_metric_set(struct dst_entry * dst,int metric,u32 val)177 static inline void dst_metric_set(struct dst_entry *dst, int metric, u32 val)
178 {
179 	u32 *p = dst_metrics_write_ptr(dst);
180 
181 	if (p)
182 		p[metric-1] = val;
183 }
184 
185 /* Kernel-internal feature bits that are unallocated in user space. */
186 #define DST_FEATURE_ECN_CA	(1U << 31)
187 
188 #define DST_FEATURE_MASK	(DST_FEATURE_ECN_CA)
189 #define DST_FEATURE_ECN_MASK	(DST_FEATURE_ECN_CA | RTAX_FEATURE_ECN)
190 
191 static inline u32
dst_feature(const struct dst_entry * dst,u32 feature)192 dst_feature(const struct dst_entry *dst, u32 feature)
193 {
194 	return dst_metric(dst, RTAX_FEATURES) & feature;
195 }
196 
197 INDIRECT_CALLABLE_DECLARE(unsigned int ip6_mtu(const struct dst_entry *));
198 INDIRECT_CALLABLE_DECLARE(unsigned int ipv4_mtu(const struct dst_entry *));
dst_mtu(const struct dst_entry * dst)199 static inline u32 dst_mtu(const struct dst_entry *dst)
200 {
201 	return INDIRECT_CALL_INET(dst->ops->mtu, ip6_mtu, ipv4_mtu, dst);
202 }
203 
204 /* RTT metrics are stored in milliseconds for user ABI, but used as jiffies */
dst_metric_rtt(const struct dst_entry * dst,int metric)205 static inline unsigned long dst_metric_rtt(const struct dst_entry *dst, int metric)
206 {
207 	return msecs_to_jiffies(dst_metric(dst, metric));
208 }
209 
210 static inline u32
dst_allfrag(const struct dst_entry * dst)211 dst_allfrag(const struct dst_entry *dst)
212 {
213 	int ret = dst_feature(dst,  RTAX_FEATURE_ALLFRAG);
214 	return ret;
215 }
216 
217 static inline int
dst_metric_locked(const struct dst_entry * dst,int metric)218 dst_metric_locked(const struct dst_entry *dst, int metric)
219 {
220 	return dst_metric(dst, RTAX_LOCK) & (1 << metric);
221 }
222 
dst_hold(struct dst_entry * dst)223 static inline void dst_hold(struct dst_entry *dst)
224 {
225 	/*
226 	 * If your kernel compilation stops here, please check
227 	 * the placement of __refcnt in struct dst_entry
228 	 */
229 	BUILD_BUG_ON(offsetof(struct dst_entry, __refcnt) & 63);
230 	WARN_ON(atomic_inc_not_zero(&dst->__refcnt) == 0);
231 }
232 
dst_use_noref(struct dst_entry * dst,unsigned long time)233 static inline void dst_use_noref(struct dst_entry *dst, unsigned long time)
234 {
235 	if (unlikely(time != dst->lastuse)) {
236 		dst->__use++;
237 		dst->lastuse = time;
238 	}
239 }
240 
dst_hold_and_use(struct dst_entry * dst,unsigned long time)241 static inline void dst_hold_and_use(struct dst_entry *dst, unsigned long time)
242 {
243 	dst_hold(dst);
244 	dst_use_noref(dst, time);
245 }
246 
dst_clone(struct dst_entry * dst)247 static inline struct dst_entry *dst_clone(struct dst_entry *dst)
248 {
249 	if (dst)
250 		dst_hold(dst);
251 	return dst;
252 }
253 
254 void dst_release(struct dst_entry *dst);
255 
256 void dst_release_immediate(struct dst_entry *dst);
257 
refdst_drop(unsigned long refdst)258 static inline void refdst_drop(unsigned long refdst)
259 {
260 	if (!(refdst & SKB_DST_NOREF))
261 		dst_release((struct dst_entry *)(refdst & SKB_DST_PTRMASK));
262 }
263 
264 /**
265  * skb_dst_drop - drops skb dst
266  * @skb: buffer
267  *
268  * Drops dst reference count if a reference was taken.
269  */
skb_dst_drop(struct sk_buff * skb)270 static inline void skb_dst_drop(struct sk_buff *skb)
271 {
272 	if (skb->_skb_refdst) {
273 		refdst_drop(skb->_skb_refdst);
274 		skb->_skb_refdst = 0UL;
275 	}
276 }
277 
__skb_dst_copy(struct sk_buff * nskb,unsigned long refdst)278 static inline void __skb_dst_copy(struct sk_buff *nskb, unsigned long refdst)
279 {
280 	nskb->slow_gro |= !!refdst;
281 	nskb->_skb_refdst = refdst;
282 	if (!(nskb->_skb_refdst & SKB_DST_NOREF))
283 		dst_clone(skb_dst(nskb));
284 }
285 
skb_dst_copy(struct sk_buff * nskb,const struct sk_buff * oskb)286 static inline void skb_dst_copy(struct sk_buff *nskb, const struct sk_buff *oskb)
287 {
288 	__skb_dst_copy(nskb, oskb->_skb_refdst);
289 }
290 
291 /**
292  * dst_hold_safe - Take a reference on a dst if possible
293  * @dst: pointer to dst entry
294  *
295  * This helper returns false if it could not safely
296  * take a reference on a dst.
297  */
dst_hold_safe(struct dst_entry * dst)298 static inline bool dst_hold_safe(struct dst_entry *dst)
299 {
300 	return atomic_inc_not_zero(&dst->__refcnt);
301 }
302 
303 /**
304  * skb_dst_force - makes sure skb dst is refcounted
305  * @skb: buffer
306  *
307  * If dst is not yet refcounted and not destroyed, grab a ref on it.
308  * Returns true if dst is refcounted.
309  */
skb_dst_force(struct sk_buff * skb)310 static inline bool skb_dst_force(struct sk_buff *skb)
311 {
312 	if (skb_dst_is_noref(skb)) {
313 		struct dst_entry *dst = skb_dst(skb);
314 
315 		WARN_ON(!rcu_read_lock_held());
316 		if (!dst_hold_safe(dst))
317 			dst = NULL;
318 
319 		skb->_skb_refdst = (unsigned long)dst;
320 		skb->slow_gro |= !!dst;
321 	}
322 
323 	return skb->_skb_refdst != 0UL;
324 }
325 
326 
327 /**
328  *	__skb_tunnel_rx - prepare skb for rx reinsert
329  *	@skb: buffer
330  *	@dev: tunnel device
331  *	@net: netns for packet i/o
332  *
333  *	After decapsulation, packet is going to re-enter (netif_rx()) our stack,
334  *	so make some cleanups. (no accounting done)
335  */
__skb_tunnel_rx(struct sk_buff * skb,struct net_device * dev,struct net * net)336 static inline void __skb_tunnel_rx(struct sk_buff *skb, struct net_device *dev,
337 				   struct net *net)
338 {
339 	skb->dev = dev;
340 
341 	/*
342 	 * Clear hash so that we can recalulate the hash for the
343 	 * encapsulated packet, unless we have already determine the hash
344 	 * over the L4 4-tuple.
345 	 */
346 	skb_clear_hash_if_not_l4(skb);
347 	skb_set_queue_mapping(skb, 0);
348 	skb_scrub_packet(skb, !net_eq(net, dev_net(dev)));
349 }
350 
351 /**
352  *	skb_tunnel_rx - prepare skb for rx reinsert
353  *	@skb: buffer
354  *	@dev: tunnel device
355  *	@net: netns for packet i/o
356  *
357  *	After decapsulation, packet is going to re-enter (netif_rx()) our stack,
358  *	so make some cleanups, and perform accounting.
359  *	Note: this accounting is not SMP safe.
360  */
skb_tunnel_rx(struct sk_buff * skb,struct net_device * dev,struct net * net)361 static inline void skb_tunnel_rx(struct sk_buff *skb, struct net_device *dev,
362 				 struct net *net)
363 {
364 	/* TODO : stats should be SMP safe */
365 	dev->stats.rx_packets++;
366 	dev->stats.rx_bytes += skb->len;
367 	__skb_tunnel_rx(skb, dev, net);
368 }
369 
dst_tclassid(const struct sk_buff * skb)370 static inline u32 dst_tclassid(const struct sk_buff *skb)
371 {
372 #ifdef CONFIG_IP_ROUTE_CLASSID
373 	const struct dst_entry *dst;
374 
375 	dst = skb_dst(skb);
376 	if (dst)
377 		return dst->tclassid;
378 #endif
379 	return 0;
380 }
381 
382 int dst_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
dst_discard(struct sk_buff * skb)383 static inline int dst_discard(struct sk_buff *skb)
384 {
385 	return dst_discard_out(&init_net, skb->sk, skb);
386 }
387 void *dst_alloc(struct dst_ops *ops, struct net_device *dev, int initial_ref,
388 		int initial_obsolete, unsigned short flags);
389 void dst_init(struct dst_entry *dst, struct dst_ops *ops,
390 	      struct net_device *dev, int initial_ref, int initial_obsolete,
391 	      unsigned short flags);
392 struct dst_entry *dst_destroy(struct dst_entry *dst);
393 void dst_dev_put(struct dst_entry *dst);
394 
dst_confirm(struct dst_entry * dst)395 static inline void dst_confirm(struct dst_entry *dst)
396 {
397 }
398 
dst_neigh_lookup(const struct dst_entry * dst,const void * daddr)399 static inline struct neighbour *dst_neigh_lookup(const struct dst_entry *dst, const void *daddr)
400 {
401 	struct neighbour *n = dst->ops->neigh_lookup(dst, NULL, daddr);
402 	return IS_ERR(n) ? NULL : n;
403 }
404 
dst_neigh_lookup_skb(const struct dst_entry * dst,struct sk_buff * skb)405 static inline struct neighbour *dst_neigh_lookup_skb(const struct dst_entry *dst,
406 						     struct sk_buff *skb)
407 {
408 	struct neighbour *n;
409 
410 	if (WARN_ON_ONCE(!dst->ops->neigh_lookup))
411 		return NULL;
412 
413 	n = dst->ops->neigh_lookup(dst, skb, NULL);
414 
415 	return IS_ERR(n) ? NULL : n;
416 }
417 
dst_confirm_neigh(const struct dst_entry * dst,const void * daddr)418 static inline void dst_confirm_neigh(const struct dst_entry *dst,
419 				     const void *daddr)
420 {
421 	if (dst->ops->confirm_neigh)
422 		dst->ops->confirm_neigh(dst, daddr);
423 }
424 
dst_link_failure(struct sk_buff * skb)425 static inline void dst_link_failure(struct sk_buff *skb)
426 {
427 	struct dst_entry *dst = skb_dst(skb);
428 	if (dst && dst->ops && dst->ops->link_failure)
429 		dst->ops->link_failure(skb);
430 }
431 
dst_set_expires(struct dst_entry * dst,int timeout)432 static inline void dst_set_expires(struct dst_entry *dst, int timeout)
433 {
434 	unsigned long expires = jiffies + timeout;
435 
436 	if (expires == 0)
437 		expires = 1;
438 
439 	if (dst->expires == 0 || time_before(expires, dst->expires))
440 		dst->expires = expires;
441 }
442 
443 INDIRECT_CALLABLE_DECLARE(int ip6_output(struct net *, struct sock *,
444 					 struct sk_buff *));
445 INDIRECT_CALLABLE_DECLARE(int ip_output(struct net *, struct sock *,
446 					 struct sk_buff *));
447 /* Output packet to network from transport.  */
dst_output(struct net * net,struct sock * sk,struct sk_buff * skb)448 static inline int dst_output(struct net *net, struct sock *sk, struct sk_buff *skb)
449 {
450 	return INDIRECT_CALL_INET(skb_dst(skb)->output,
451 				  ip6_output, ip_output,
452 				  net, sk, skb);
453 }
454 
455 INDIRECT_CALLABLE_DECLARE(int ip6_input(struct sk_buff *));
456 INDIRECT_CALLABLE_DECLARE(int ip_local_deliver(struct sk_buff *));
457 /* Input packet from network to transport.  */
dst_input(struct sk_buff * skb)458 static inline int dst_input(struct sk_buff *skb)
459 {
460 	return INDIRECT_CALL_INET(skb_dst(skb)->input,
461 				  ip6_input, ip_local_deliver, skb);
462 }
463 
464 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *,
465 							  u32));
466 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
467 							   u32));
dst_check(struct dst_entry * dst,u32 cookie)468 static inline struct dst_entry *dst_check(struct dst_entry *dst, u32 cookie)
469 {
470 	if (dst->obsolete)
471 		dst = INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check,
472 					 ipv4_dst_check, dst, cookie);
473 	return dst;
474 }
475 
476 /* Flags for xfrm_lookup flags argument. */
477 enum {
478 	XFRM_LOOKUP_ICMP = 1 << 0,
479 	XFRM_LOOKUP_QUEUE = 1 << 1,
480 	XFRM_LOOKUP_KEEP_DST_REF = 1 << 2,
481 };
482 
483 struct flowi;
484 #ifndef CONFIG_XFRM
xfrm_lookup(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags)485 static inline struct dst_entry *xfrm_lookup(struct net *net,
486 					    struct dst_entry *dst_orig,
487 					    const struct flowi *fl,
488 					    const struct sock *sk,
489 					    int flags)
490 {
491 	return dst_orig;
492 }
493 
494 static inline struct dst_entry *
xfrm_lookup_with_ifid(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags,u32 if_id)495 xfrm_lookup_with_ifid(struct net *net, struct dst_entry *dst_orig,
496 		      const struct flowi *fl, const struct sock *sk,
497 		      int flags, u32 if_id)
498 {
499 	return dst_orig;
500 }
501 
xfrm_lookup_route(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags)502 static inline struct dst_entry *xfrm_lookup_route(struct net *net,
503 						  struct dst_entry *dst_orig,
504 						  const struct flowi *fl,
505 						  const struct sock *sk,
506 						  int flags)
507 {
508 	return dst_orig;
509 }
510 
dst_xfrm(const struct dst_entry * dst)511 static inline struct xfrm_state *dst_xfrm(const struct dst_entry *dst)
512 {
513 	return NULL;
514 }
515 
516 #else
517 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
518 			      const struct flowi *fl, const struct sock *sk,
519 			      int flags);
520 
521 struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
522 					struct dst_entry *dst_orig,
523 					const struct flowi *fl,
524 					const struct sock *sk, int flags,
525 					u32 if_id);
526 
527 struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
528 				    const struct flowi *fl, const struct sock *sk,
529 				    int flags);
530 
531 /* skb attached with this dst needs transformation if dst->xfrm is valid */
dst_xfrm(const struct dst_entry * dst)532 static inline struct xfrm_state *dst_xfrm(const struct dst_entry *dst)
533 {
534 	return dst->xfrm;
535 }
536 #endif
537 
skb_dst_update_pmtu(struct sk_buff * skb,u32 mtu)538 static inline void skb_dst_update_pmtu(struct sk_buff *skb, u32 mtu)
539 {
540 	struct dst_entry *dst = skb_dst(skb);
541 
542 	if (dst && dst->ops->update_pmtu)
543 		dst->ops->update_pmtu(dst, NULL, skb, mtu, true);
544 }
545 
546 /* update dst pmtu but not do neighbor confirm */
skb_dst_update_pmtu_no_confirm(struct sk_buff * skb,u32 mtu)547 static inline void skb_dst_update_pmtu_no_confirm(struct sk_buff *skb, u32 mtu)
548 {
549 	struct dst_entry *dst = skb_dst(skb);
550 
551 	if (dst && dst->ops->update_pmtu)
552 		dst->ops->update_pmtu(dst, NULL, skb, mtu, false);
553 }
554 
555 struct dst_entry *dst_blackhole_check(struct dst_entry *dst, u32 cookie);
556 void dst_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
557 			       struct sk_buff *skb, u32 mtu, bool confirm_neigh);
558 void dst_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
559 			    struct sk_buff *skb);
560 u32 *dst_blackhole_cow_metrics(struct dst_entry *dst, unsigned long old);
561 struct neighbour *dst_blackhole_neigh_lookup(const struct dst_entry *dst,
562 					     struct sk_buff *skb,
563 					     const void *daddr);
564 unsigned int dst_blackhole_mtu(const struct dst_entry *dst);
565 
566 #endif /* _NET_DST_H */
567