1 #ifndef __RWLOCK_H__
2 #define __RWLOCK_H__
3
4 #include <xen/percpu.h>
5 #include <xen/preempt.h>
6 #include <xen/smp.h>
7 #include <xen/spinlock.h>
8
9 #include <asm/atomic.h>
10 #include <asm/system.h>
11
12 typedef struct {
13 atomic_t cnts;
14 spinlock_t lock;
15 } rwlock_t;
16
17 #define RW_LOCK_UNLOCKED { \
18 .cnts = ATOMIC_INIT(0), \
19 .lock = SPIN_LOCK_UNLOCKED \
20 }
21
22 #define DEFINE_RWLOCK(l) rwlock_t l = RW_LOCK_UNLOCKED
23 #define rwlock_init(l) (*(l) = (rwlock_t)RW_LOCK_UNLOCKED)
24
25 /* Writer states & reader shift and bias. */
26 #define _QW_CPUMASK 0xfffU /* Writer CPU mask */
27 #define _QW_SHIFT 12 /* Writer flags shift */
28 #define _QW_WAITING (1U << _QW_SHIFT) /* A writer is waiting */
29 #define _QW_LOCKED (3U << _QW_SHIFT) /* A writer holds the lock */
30 #define _QW_WMASK (3U << _QW_SHIFT) /* Writer mask */
31 #define _QR_SHIFT 14 /* Reader count shift */
32 #define _QR_BIAS (1U << _QR_SHIFT)
33
34 void queue_read_lock_slowpath(rwlock_t *lock);
35 void queue_write_lock_slowpath(rwlock_t *lock);
36
_is_write_locked_by_me(unsigned int cnts)37 static inline bool _is_write_locked_by_me(unsigned int cnts)
38 {
39 BUILD_BUG_ON(_QW_CPUMASK < NR_CPUS);
40 return (cnts & _QW_WMASK) == _QW_LOCKED &&
41 (cnts & _QW_CPUMASK) == smp_processor_id();
42 }
43
_can_read_lock(unsigned int cnts)44 static inline bool _can_read_lock(unsigned int cnts)
45 {
46 return !(cnts & _QW_WMASK) || _is_write_locked_by_me(cnts);
47 }
48
49 /*
50 * _read_trylock - try to acquire read lock of a queue rwlock.
51 * @lock : Pointer to queue rwlock structure.
52 * Return: 1 if lock acquired, 0 if failed.
53 */
_read_trylock(rwlock_t * lock)54 static inline int _read_trylock(rwlock_t *lock)
55 {
56 u32 cnts;
57
58 preempt_disable();
59 cnts = atomic_read(&lock->cnts);
60 if ( likely(_can_read_lock(cnts)) )
61 {
62 cnts = (u32)atomic_add_return(_QR_BIAS, &lock->cnts);
63 /*
64 * atomic_add_return() is a full barrier so no need for an
65 * arch_lock_acquire_barrier().
66 */
67 if ( likely(_can_read_lock(cnts)) )
68 return 1;
69 atomic_sub(_QR_BIAS, &lock->cnts);
70 }
71 preempt_enable();
72 return 0;
73 }
74
75 /*
76 * _read_lock - acquire read lock of a queue rwlock.
77 * @lock: Pointer to queue rwlock structure.
78 */
_read_lock(rwlock_t * lock)79 static inline void _read_lock(rwlock_t *lock)
80 {
81 u32 cnts;
82
83 preempt_disable();
84 cnts = atomic_add_return(_QR_BIAS, &lock->cnts);
85 /*
86 * atomic_add_return() is a full barrier so no need for an
87 * arch_lock_acquire_barrier().
88 */
89 if ( likely(_can_read_lock(cnts)) )
90 return;
91
92 /* The slowpath will decrement the reader count, if necessary. */
93 queue_read_lock_slowpath(lock);
94 /*
95 * queue_read_lock_slowpath() is using spinlock and therefore is a
96 * full barrier. So no need for an arch_lock_acquire_barrier().
97 */
98 }
99
_read_lock_irq(rwlock_t * lock)100 static inline void _read_lock_irq(rwlock_t *lock)
101 {
102 ASSERT(local_irq_is_enabled());
103 local_irq_disable();
104 _read_lock(lock);
105 }
106
_read_lock_irqsave(rwlock_t * lock)107 static inline unsigned long _read_lock_irqsave(rwlock_t *lock)
108 {
109 unsigned long flags;
110 local_irq_save(flags);
111 _read_lock(lock);
112 return flags;
113 }
114
115 /*
116 * _read_unlock - release read lock of a queue rwlock.
117 * @lock : Pointer to queue rwlock structure.
118 */
_read_unlock(rwlock_t * lock)119 static inline void _read_unlock(rwlock_t *lock)
120 {
121 arch_lock_release_barrier();
122 /*
123 * Atomically decrement the reader count
124 */
125 atomic_sub(_QR_BIAS, &lock->cnts);
126 preempt_enable();
127 }
128
_read_unlock_irq(rwlock_t * lock)129 static inline void _read_unlock_irq(rwlock_t *lock)
130 {
131 _read_unlock(lock);
132 local_irq_enable();
133 }
134
_read_unlock_irqrestore(rwlock_t * lock,unsigned long flags)135 static inline void _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
136 {
137 _read_unlock(lock);
138 local_irq_restore(flags);
139 }
140
_rw_is_locked(rwlock_t * lock)141 static inline int _rw_is_locked(rwlock_t *lock)
142 {
143 return atomic_read(&lock->cnts);
144 }
145
_write_lock_val(void)146 static inline unsigned int _write_lock_val(void)
147 {
148 return _QW_LOCKED | smp_processor_id();
149 }
150
151 /*
152 * queue_write_lock - acquire write lock of a queue rwlock.
153 * @lock : Pointer to queue rwlock structure.
154 */
_write_lock(rwlock_t * lock)155 static inline void _write_lock(rwlock_t *lock)
156 {
157 preempt_disable();
158 /*
159 * Optimize for the unfair lock case where the fair flag is 0.
160 *
161 * atomic_cmpxchg() is a full barrier so no need for an
162 * arch_lock_acquire_barrier().
163 */
164 if ( atomic_cmpxchg(&lock->cnts, 0, _write_lock_val()) == 0 )
165 return;
166
167 queue_write_lock_slowpath(lock);
168 /*
169 * queue_write_lock_slowpath() is using spinlock and therefore is a
170 * full barrier. So no need for an arch_lock_acquire_barrier().
171 */
172 }
173
_write_lock_irq(rwlock_t * lock)174 static inline void _write_lock_irq(rwlock_t *lock)
175 {
176 ASSERT(local_irq_is_enabled());
177 local_irq_disable();
178 _write_lock(lock);
179 }
180
_write_lock_irqsave(rwlock_t * lock)181 static inline unsigned long _write_lock_irqsave(rwlock_t *lock)
182 {
183 unsigned long flags;
184
185 local_irq_save(flags);
186 _write_lock(lock);
187 return flags;
188 }
189
190 /*
191 * queue_write_trylock - try to acquire write lock of a queue rwlock.
192 * @lock : Pointer to queue rwlock structure.
193 * Return: 1 if lock acquired, 0 if failed.
194 */
_write_trylock(rwlock_t * lock)195 static inline int _write_trylock(rwlock_t *lock)
196 {
197 u32 cnts;
198
199 preempt_disable();
200 cnts = atomic_read(&lock->cnts);
201 if ( unlikely(cnts) ||
202 unlikely(atomic_cmpxchg(&lock->cnts, 0, _write_lock_val()) != 0) )
203 {
204 preempt_enable();
205 return 0;
206 }
207
208 /*
209 * atomic_cmpxchg() is a full barrier so no need for an
210 * arch_lock_acquire_barrier().
211 */
212 return 1;
213 }
214
_write_unlock(rwlock_t * lock)215 static inline void _write_unlock(rwlock_t *lock)
216 {
217 ASSERT(_is_write_locked_by_me(atomic_read(&lock->cnts)));
218 arch_lock_release_barrier();
219 atomic_and(~(_QW_CPUMASK | _QW_WMASK), &lock->cnts);
220 preempt_enable();
221 }
222
_write_unlock_irq(rwlock_t * lock)223 static inline void _write_unlock_irq(rwlock_t *lock)
224 {
225 _write_unlock(lock);
226 local_irq_enable();
227 }
228
_write_unlock_irqrestore(rwlock_t * lock,unsigned long flags)229 static inline void _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
230 {
231 _write_unlock(lock);
232 local_irq_restore(flags);
233 }
234
_rw_is_write_locked(rwlock_t * lock)235 static inline int _rw_is_write_locked(rwlock_t *lock)
236 {
237 return (atomic_read(&lock->cnts) & _QW_WMASK) == _QW_LOCKED;
238 }
239
240 #define read_lock(l) _read_lock(l)
241 #define read_lock_irq(l) _read_lock_irq(l)
242 #define read_lock_irqsave(l, f) \
243 ({ \
244 BUILD_BUG_ON(sizeof(f) != sizeof(unsigned long)); \
245 ((f) = _read_lock_irqsave(l)); \
246 })
247
248 #define read_unlock(l) _read_unlock(l)
249 #define read_unlock_irq(l) _read_unlock_irq(l)
250 #define read_unlock_irqrestore(l, f) _read_unlock_irqrestore(l, f)
251 #define read_trylock(l) _read_trylock(l)
252
253 #define write_lock(l) _write_lock(l)
254 #define write_lock_irq(l) _write_lock_irq(l)
255 #define write_lock_irqsave(l, f) \
256 ({ \
257 BUILD_BUG_ON(sizeof(f) != sizeof(unsigned long)); \
258 ((f) = _write_lock_irqsave(l)); \
259 })
260 #define write_trylock(l) _write_trylock(l)
261
262 #define write_unlock(l) _write_unlock(l)
263 #define write_unlock_irq(l) _write_unlock_irq(l)
264 #define write_unlock_irqrestore(l, f) _write_unlock_irqrestore(l, f)
265
266 #define rw_is_locked(l) _rw_is_locked(l)
267 #define rw_is_write_locked(l) _rw_is_write_locked(l)
268
269
270 typedef struct percpu_rwlock percpu_rwlock_t;
271
272 struct percpu_rwlock {
273 rwlock_t rwlock;
274 bool_t writer_activating;
275 #ifndef NDEBUG
276 percpu_rwlock_t **percpu_owner;
277 #endif
278 };
279
280 #ifndef NDEBUG
281 #define PERCPU_RW_LOCK_UNLOCKED(owner) { RW_LOCK_UNLOCKED, 0, owner }
_percpu_rwlock_owner_check(percpu_rwlock_t ** per_cpudata,percpu_rwlock_t * percpu_rwlock)282 static inline void _percpu_rwlock_owner_check(percpu_rwlock_t **per_cpudata,
283 percpu_rwlock_t *percpu_rwlock)
284 {
285 ASSERT(per_cpudata == percpu_rwlock->percpu_owner);
286 }
287 #else
288 #define PERCPU_RW_LOCK_UNLOCKED(owner) { RW_LOCK_UNLOCKED, 0 }
289 #define _percpu_rwlock_owner_check(data, lock) ((void)0)
290 #endif
291
292 #define DEFINE_PERCPU_RWLOCK_RESOURCE(l, owner) \
293 percpu_rwlock_t l = PERCPU_RW_LOCK_UNLOCKED(&get_per_cpu_var(owner))
294 #define percpu_rwlock_resource_init(l, owner) \
295 (*(l) = (percpu_rwlock_t)PERCPU_RW_LOCK_UNLOCKED(&get_per_cpu_var(owner)))
296
_percpu_read_lock(percpu_rwlock_t ** per_cpudata,percpu_rwlock_t * percpu_rwlock)297 static inline void _percpu_read_lock(percpu_rwlock_t **per_cpudata,
298 percpu_rwlock_t *percpu_rwlock)
299 {
300 /* Validate the correct per_cpudata variable has been provided. */
301 _percpu_rwlock_owner_check(per_cpudata, percpu_rwlock);
302
303 /* We cannot support recursion on the same lock. */
304 ASSERT(this_cpu_ptr(per_cpudata) != percpu_rwlock);
305 /*
306 * Detect using a second percpu_rwlock_t simulatenously and fallback
307 * to standard read_lock.
308 */
309 if ( unlikely(this_cpu_ptr(per_cpudata) != NULL ) )
310 {
311 read_lock(&percpu_rwlock->rwlock);
312 return;
313 }
314
315 /* Indicate this cpu is reading. */
316 preempt_disable();
317 this_cpu_ptr(per_cpudata) = percpu_rwlock;
318 smp_mb();
319 /* Check if a writer is waiting. */
320 if ( unlikely(percpu_rwlock->writer_activating) )
321 {
322 /* Let the waiting writer know we aren't holding the lock. */
323 this_cpu_ptr(per_cpudata) = NULL;
324 /* Wait using the read lock to keep the lock fair. */
325 read_lock(&percpu_rwlock->rwlock);
326 /* Set the per CPU data again and continue. */
327 this_cpu_ptr(per_cpudata) = percpu_rwlock;
328 /* Drop the read lock because we don't need it anymore. */
329 read_unlock(&percpu_rwlock->rwlock);
330 }
331 }
332
_percpu_read_unlock(percpu_rwlock_t ** per_cpudata,percpu_rwlock_t * percpu_rwlock)333 static inline void _percpu_read_unlock(percpu_rwlock_t **per_cpudata,
334 percpu_rwlock_t *percpu_rwlock)
335 {
336 /* Validate the correct per_cpudata variable has been provided. */
337 _percpu_rwlock_owner_check(per_cpudata, percpu_rwlock);
338
339 /* Verify the read lock was taken for this lock */
340 ASSERT(this_cpu_ptr(per_cpudata) != NULL);
341 /*
342 * Detect using a second percpu_rwlock_t simulatenously and fallback
343 * to standard read_unlock.
344 */
345 if ( unlikely(this_cpu_ptr(per_cpudata) != percpu_rwlock ) )
346 {
347 read_unlock(&percpu_rwlock->rwlock);
348 return;
349 }
350 this_cpu_ptr(per_cpudata) = NULL;
351 smp_wmb();
352 preempt_enable();
353 }
354
355 /* Don't inline percpu write lock as it's a complex function. */
356 void _percpu_write_lock(percpu_rwlock_t **per_cpudata,
357 percpu_rwlock_t *percpu_rwlock);
358
_percpu_write_unlock(percpu_rwlock_t ** per_cpudata,percpu_rwlock_t * percpu_rwlock)359 static inline void _percpu_write_unlock(percpu_rwlock_t **per_cpudata,
360 percpu_rwlock_t *percpu_rwlock)
361 {
362 /* Validate the correct per_cpudata variable has been provided. */
363 _percpu_rwlock_owner_check(per_cpudata, percpu_rwlock);
364
365 ASSERT(percpu_rwlock->writer_activating);
366 percpu_rwlock->writer_activating = 0;
367 write_unlock(&percpu_rwlock->rwlock);
368 }
369
370 #define percpu_rw_is_write_locked(l) _rw_is_write_locked(&((l)->rwlock))
371
372 #define percpu_read_lock(percpu, lock) \
373 _percpu_read_lock(&get_per_cpu_var(percpu), lock)
374 #define percpu_read_unlock(percpu, lock) \
375 _percpu_read_unlock(&get_per_cpu_var(percpu), lock)
376 #define percpu_write_lock(percpu, lock) \
377 _percpu_write_lock(&get_per_cpu_var(percpu), lock)
378 #define percpu_write_unlock(percpu, lock) \
379 _percpu_write_unlock(&get_per_cpu_var(percpu), lock)
380
381 #define DEFINE_PERCPU_RWLOCK_GLOBAL(name) DEFINE_PER_CPU(percpu_rwlock_t *, \
382 name)
383 #define DECLARE_PERCPU_RWLOCK_GLOBAL(name) DECLARE_PER_CPU(percpu_rwlock_t *, \
384 name)
385
386 #endif /* __RWLOCK_H__ */
387