1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef LINUX_HARDIRQ_H
3 #define LINUX_HARDIRQ_H
4 
5 #include <linux/context_tracking_state.h>
6 #include <linux/preempt.h>
7 #include <linux/lockdep.h>
8 #include <linux/ftrace_irq.h>
9 #include <linux/sched.h>
10 #include <linux/vtime.h>
11 #include <asm/hardirq.h>
12 
13 extern void synchronize_irq(unsigned int irq);
14 extern bool synchronize_hardirq(unsigned int irq);
15 
16 #ifdef CONFIG_NO_HZ_FULL
17 void __rcu_irq_enter_check_tick(void);
18 #else
__rcu_irq_enter_check_tick(void)19 static inline void __rcu_irq_enter_check_tick(void) { }
20 #endif
21 
rcu_irq_enter_check_tick(void)22 static __always_inline void rcu_irq_enter_check_tick(void)
23 {
24 	if (context_tracking_enabled())
25 		__rcu_irq_enter_check_tick();
26 }
27 
28 /*
29  * It is safe to do non-atomic ops on ->hardirq_context,
30  * because NMI handlers may not preempt and the ops are
31  * always balanced, so the interrupted value of ->hardirq_context
32  * will always be restored.
33  */
34 #define __irq_enter()					\
35 	do {						\
36 		preempt_count_add(HARDIRQ_OFFSET);	\
37 		lockdep_hardirq_enter();		\
38 		account_hardirq_enter(current);		\
39 	} while (0)
40 
41 /*
42  * Like __irq_enter() without time accounting for fast
43  * interrupts, e.g. reschedule IPI where time accounting
44  * is more expensive than the actual interrupt.
45  */
46 #define __irq_enter_raw()				\
47 	do {						\
48 		preempt_count_add(HARDIRQ_OFFSET);	\
49 		lockdep_hardirq_enter();		\
50 	} while (0)
51 
52 /*
53  * Enter irq context (on NO_HZ, update jiffies):
54  */
55 void irq_enter(void);
56 /*
57  * Like irq_enter(), but RCU is already watching.
58  */
59 void irq_enter_rcu(void);
60 
61 /*
62  * Exit irq context without processing softirqs:
63  */
64 #define __irq_exit()					\
65 	do {						\
66 		account_hardirq_exit(current);		\
67 		lockdep_hardirq_exit();			\
68 		preempt_count_sub(HARDIRQ_OFFSET);	\
69 	} while (0)
70 
71 /*
72  * Like __irq_exit() without time accounting
73  */
74 #define __irq_exit_raw()				\
75 	do {						\
76 		lockdep_hardirq_exit();			\
77 		preempt_count_sub(HARDIRQ_OFFSET);	\
78 	} while (0)
79 
80 /*
81  * Exit irq context and process softirqs if needed:
82  */
83 void irq_exit(void);
84 
85 /*
86  * Like irq_exit(), but return with RCU watching.
87  */
88 void irq_exit_rcu(void);
89 
90 #ifndef arch_nmi_enter
91 #define arch_nmi_enter()	do { } while (0)
92 #define arch_nmi_exit()		do { } while (0)
93 #endif
94 
95 #ifdef CONFIG_TINY_RCU
rcu_nmi_enter(void)96 static inline void rcu_nmi_enter(void) { }
rcu_nmi_exit(void)97 static inline void rcu_nmi_exit(void) { }
98 #else
99 extern void rcu_nmi_enter(void);
100 extern void rcu_nmi_exit(void);
101 #endif
102 
103 /*
104  * NMI vs Tracing
105  * --------------
106  *
107  * We must not land in a tracer until (or after) we've changed preempt_count
108  * such that in_nmi() becomes true. To that effect all NMI C entry points must
109  * be marked 'notrace' and call nmi_enter() as soon as possible.
110  */
111 
112 /*
113  * nmi_enter() can nest up to 15 times; see NMI_BITS.
114  */
115 #define __nmi_enter()						\
116 	do {							\
117 		lockdep_off();					\
118 		arch_nmi_enter();				\
119 		BUG_ON(in_nmi() == NMI_MASK);			\
120 		__preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);	\
121 	} while (0)
122 
123 #define nmi_enter()						\
124 	do {							\
125 		__nmi_enter();					\
126 		lockdep_hardirq_enter();			\
127 		rcu_nmi_enter();				\
128 		instrumentation_begin();			\
129 		ftrace_nmi_enter();				\
130 		instrumentation_end();				\
131 	} while (0)
132 
133 #define __nmi_exit()						\
134 	do {							\
135 		BUG_ON(!in_nmi());				\
136 		__preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);	\
137 		arch_nmi_exit();				\
138 		lockdep_on();					\
139 	} while (0)
140 
141 #define nmi_exit()						\
142 	do {							\
143 		instrumentation_begin();			\
144 		ftrace_nmi_exit();				\
145 		instrumentation_end();				\
146 		rcu_nmi_exit();					\
147 		lockdep_hardirq_exit();				\
148 		__nmi_exit();					\
149 	} while (0)
150 
151 #endif /* LINUX_HARDIRQ_H */
152