1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Queued read/write locks
4 *
5 * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
6 *
7 * Authors: Waiman Long <waiman.long@hp.com>
8 */
9 #include <linux/smp.h>
10 #include <linux/bug.h>
11 #include <linux/cpumask.h>
12 #include <linux/percpu.h>
13 #include <linux/hardirq.h>
14 #include <linux/spinlock.h>
15
16 /**
17 * queued_read_lock_slowpath - acquire read lock of a queue rwlock
18 * @lock: Pointer to queue rwlock structure
19 */
queued_read_lock_slowpath(struct qrwlock * lock)20 void queued_read_lock_slowpath(struct qrwlock *lock)
21 {
22 /*
23 * Readers come here when they cannot get the lock without waiting
24 */
25 if (unlikely(in_interrupt())) {
26 /*
27 * Readers in interrupt context will get the lock immediately
28 * if the writer is just waiting (not holding the lock yet),
29 * so spin with ACQUIRE semantics until the lock is available
30 * without waiting in the queue.
31 */
32 atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
33 return;
34 }
35 atomic_sub(_QR_BIAS, &lock->cnts);
36
37 /*
38 * Put the reader into the wait queue
39 */
40 arch_spin_lock(&lock->wait_lock);
41 atomic_add(_QR_BIAS, &lock->cnts);
42
43 /*
44 * The ACQUIRE semantics of the following spinning code ensure
45 * that accesses can't leak upwards out of our subsequent critical
46 * section in the case that the lock is currently held for write.
47 */
48 atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
49
50 /*
51 * Signal the next one in queue to become queue head
52 */
53 arch_spin_unlock(&lock->wait_lock);
54 }
55 EXPORT_SYMBOL(queued_read_lock_slowpath);
56
57 /**
58 * queued_write_lock_slowpath - acquire write lock of a queue rwlock
59 * @lock : Pointer to queue rwlock structure
60 */
queued_write_lock_slowpath(struct qrwlock * lock)61 void queued_write_lock_slowpath(struct qrwlock *lock)
62 {
63 int cnts;
64
65 /* Put the writer into the wait queue */
66 arch_spin_lock(&lock->wait_lock);
67
68 /* Try to acquire the lock directly if no reader is present */
69 if (!(cnts = atomic_read(&lock->cnts)) &&
70 atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))
71 goto unlock;
72
73 /* Set the waiting flag to notify readers that a writer is pending */
74 atomic_or(_QW_WAITING, &lock->cnts);
75
76 /* When no more readers or writers, set the locked flag */
77 do {
78 cnts = atomic_cond_read_relaxed(&lock->cnts, VAL == _QW_WAITING);
79 } while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));
80 unlock:
81 arch_spin_unlock(&lock->wait_lock);
82 }
83 EXPORT_SYMBOL(queued_write_lock_slowpath);
84