1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Generic barrier definitions. 4 * 5 * It should be possible to use these on really simple architectures, 6 * but it serves more as a starting point for new ports. 7 * 8 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. 9 * Written by David Howells (dhowells@redhat.com) 10 */ 11 #ifndef __ASM_GENERIC_BARRIER_H 12 #define __ASM_GENERIC_BARRIER_H 13 14 #ifndef __ASSEMBLY__ 15 16 #include <linux/compiler.h> 17 #include <asm/rwonce.h> 18 19 #ifndef nop 20 #define nop() asm volatile ("nop") 21 #endif 22 23 /* 24 * Force strict CPU ordering. And yes, this is required on UP too when we're 25 * talking to devices. 26 * 27 * Fall back to compiler barriers if nothing better is provided. 28 */ 29 30 #ifndef mb 31 #define mb() barrier() 32 #endif 33 34 #ifndef rmb 35 #define rmb() mb() 36 #endif 37 38 #ifndef wmb 39 #define wmb() mb() 40 #endif 41 42 #ifndef dma_rmb 43 #define dma_rmb() rmb() 44 #endif 45 46 #ifndef dma_wmb 47 #define dma_wmb() wmb() 48 #endif 49 50 #ifndef __smp_mb 51 #define __smp_mb() mb() 52 #endif 53 54 #ifndef __smp_rmb 55 #define __smp_rmb() rmb() 56 #endif 57 58 #ifndef __smp_wmb 59 #define __smp_wmb() wmb() 60 #endif 61 62 #ifdef CONFIG_SMP 63 64 #ifndef smp_mb 65 #define smp_mb() __smp_mb() 66 #endif 67 68 #ifndef smp_rmb 69 #define smp_rmb() __smp_rmb() 70 #endif 71 72 #ifndef smp_wmb 73 #define smp_wmb() __smp_wmb() 74 #endif 75 76 #else /* !CONFIG_SMP */ 77 78 #ifndef smp_mb 79 #define smp_mb() barrier() 80 #endif 81 82 #ifndef smp_rmb 83 #define smp_rmb() barrier() 84 #endif 85 86 #ifndef smp_wmb 87 #define smp_wmb() barrier() 88 #endif 89 90 #endif /* CONFIG_SMP */ 91 92 #ifndef __smp_store_mb 93 #define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0) 94 #endif 95 96 #ifndef __smp_mb__before_atomic 97 #define __smp_mb__before_atomic() __smp_mb() 98 #endif 99 100 #ifndef __smp_mb__after_atomic 101 #define __smp_mb__after_atomic() __smp_mb() 102 #endif 103 104 #ifndef __smp_store_release 105 #define __smp_store_release(p, v) \ 106 do { \ 107 compiletime_assert_atomic_type(*p); \ 108 __smp_mb(); \ 109 WRITE_ONCE(*p, v); \ 110 } while (0) 111 #endif 112 113 #ifndef __smp_load_acquire 114 #define __smp_load_acquire(p) \ 115 ({ \ 116 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ 117 compiletime_assert_atomic_type(*p); \ 118 __smp_mb(); \ 119 (typeof(*p))___p1; \ 120 }) 121 #endif 122 123 #ifdef CONFIG_SMP 124 125 #ifndef smp_store_mb 126 #define smp_store_mb(var, value) __smp_store_mb(var, value) 127 #endif 128 129 #ifndef smp_mb__before_atomic 130 #define smp_mb__before_atomic() __smp_mb__before_atomic() 131 #endif 132 133 #ifndef smp_mb__after_atomic 134 #define smp_mb__after_atomic() __smp_mb__after_atomic() 135 #endif 136 137 #ifndef smp_store_release 138 #define smp_store_release(p, v) __smp_store_release(p, v) 139 #endif 140 141 #ifndef smp_load_acquire 142 #define smp_load_acquire(p) __smp_load_acquire(p) 143 #endif 144 145 #else /* !CONFIG_SMP */ 146 147 #ifndef smp_store_mb 148 #define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0) 149 #endif 150 151 #ifndef smp_mb__before_atomic 152 #define smp_mb__before_atomic() barrier() 153 #endif 154 155 #ifndef smp_mb__after_atomic 156 #define smp_mb__after_atomic() barrier() 157 #endif 158 159 #ifndef smp_store_release 160 #define smp_store_release(p, v) \ 161 do { \ 162 compiletime_assert_atomic_type(*p); \ 163 barrier(); \ 164 WRITE_ONCE(*p, v); \ 165 } while (0) 166 #endif 167 168 #ifndef smp_load_acquire 169 #define smp_load_acquire(p) \ 170 ({ \ 171 __unqual_scalar_typeof(*p) ___p1 = READ_ONCE(*p); \ 172 compiletime_assert_atomic_type(*p); \ 173 barrier(); \ 174 (typeof(*p))___p1; \ 175 }) 176 #endif 177 178 #endif /* CONFIG_SMP */ 179 180 /* Barriers for virtual machine guests when talking to an SMP host */ 181 #define virt_mb() __smp_mb() 182 #define virt_rmb() __smp_rmb() 183 #define virt_wmb() __smp_wmb() 184 #define virt_store_mb(var, value) __smp_store_mb(var, value) 185 #define virt_mb__before_atomic() __smp_mb__before_atomic() 186 #define virt_mb__after_atomic() __smp_mb__after_atomic() 187 #define virt_store_release(p, v) __smp_store_release(p, v) 188 #define virt_load_acquire(p) __smp_load_acquire(p) 189 190 /** 191 * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency 192 * 193 * A control dependency provides a LOAD->STORE order, the additional RMB 194 * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order, 195 * aka. (load)-ACQUIRE. 196 * 197 * Architectures that do not do load speculation can have this be barrier(). 198 */ 199 #ifndef smp_acquire__after_ctrl_dep 200 #define smp_acquire__after_ctrl_dep() smp_rmb() 201 #endif 202 203 /** 204 * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees 205 * @ptr: pointer to the variable to wait on 206 * @cond: boolean expression to wait for 207 * 208 * Equivalent to using READ_ONCE() on the condition variable. 209 * 210 * Due to C lacking lambda expressions we load the value of *ptr into a 211 * pre-named variable @VAL to be used in @cond. 212 */ 213 #ifndef smp_cond_load_relaxed 214 #define smp_cond_load_relaxed(ptr, cond_expr) ({ \ 215 typeof(ptr) __PTR = (ptr); \ 216 __unqual_scalar_typeof(*ptr) VAL; \ 217 for (;;) { \ 218 VAL = READ_ONCE(*__PTR); \ 219 if (cond_expr) \ 220 break; \ 221 cpu_relax(); \ 222 } \ 223 (typeof(*ptr))VAL; \ 224 }) 225 #endif 226 227 /** 228 * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering 229 * @ptr: pointer to the variable to wait on 230 * @cond: boolean expression to wait for 231 * 232 * Equivalent to using smp_load_acquire() on the condition variable but employs 233 * the control dependency of the wait to reduce the barrier on many platforms. 234 */ 235 #ifndef smp_cond_load_acquire 236 #define smp_cond_load_acquire(ptr, cond_expr) ({ \ 237 __unqual_scalar_typeof(*ptr) _val; \ 238 _val = smp_cond_load_relaxed(ptr, cond_expr); \ 239 smp_acquire__after_ctrl_dep(); \ 240 (typeof(*ptr))_val; \ 241 }) 242 #endif 243 244 /* 245 * pmem_wmb() ensures that all stores for which the modification 246 * are written to persistent storage by preceding instructions have 247 * updated persistent storage before any data access or data transfer 248 * caused by subsequent instructions is initiated. 249 */ 250 #ifndef pmem_wmb 251 #define pmem_wmb() wmb() 252 #endif 253 254 #endif /* !__ASSEMBLY__ */ 255 #endif /* __ASM_GENERIC_BARRIER_H */ 256