1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Queued read/write locks
4  *
5  * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
6  *
7  * Authors: Waiman Long <waiman.long@hp.com>
8  */
9 #include <linux/smp.h>
10 #include <linux/bug.h>
11 #include <linux/cpumask.h>
12 #include <linux/percpu.h>
13 #include <linux/hardirq.h>
14 #include <linux/spinlock.h>
15 
16 /**
17  * queued_read_lock_slowpath - acquire read lock of a queue rwlock
18  * @lock: Pointer to queue rwlock structure
19  */
queued_read_lock_slowpath(struct qrwlock * lock)20 void queued_read_lock_slowpath(struct qrwlock *lock)
21 {
22 	/*
23 	 * Readers come here when they cannot get the lock without waiting
24 	 */
25 	if (unlikely(in_interrupt())) {
26 		/*
27 		 * Readers in interrupt context will get the lock immediately
28 		 * if the writer is just waiting (not holding the lock yet),
29 		 * so spin with ACQUIRE semantics until the lock is available
30 		 * without waiting in the queue.
31 		 */
32 		atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
33 		return;
34 	}
35 	atomic_sub(_QR_BIAS, &lock->cnts);
36 
37 	/*
38 	 * Put the reader into the wait queue
39 	 */
40 	arch_spin_lock(&lock->wait_lock);
41 	atomic_add(_QR_BIAS, &lock->cnts);
42 
43 	/*
44 	 * The ACQUIRE semantics of the following spinning code ensure
45 	 * that accesses can't leak upwards out of our subsequent critical
46 	 * section in the case that the lock is currently held for write.
47 	 */
48 	atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
49 
50 	/*
51 	 * Signal the next one in queue to become queue head
52 	 */
53 	arch_spin_unlock(&lock->wait_lock);
54 }
55 EXPORT_SYMBOL(queued_read_lock_slowpath);
56 
57 /**
58  * queued_write_lock_slowpath - acquire write lock of a queue rwlock
59  * @lock : Pointer to queue rwlock structure
60  */
queued_write_lock_slowpath(struct qrwlock * lock)61 void queued_write_lock_slowpath(struct qrwlock *lock)
62 {
63 	int cnts;
64 
65 	/* Put the writer into the wait queue */
66 	arch_spin_lock(&lock->wait_lock);
67 
68 	/* Try to acquire the lock directly if no reader is present */
69 	if (!(cnts = atomic_read(&lock->cnts)) &&
70 	    atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))
71 		goto unlock;
72 
73 	/* Set the waiting flag to notify readers that a writer is pending */
74 	atomic_or(_QW_WAITING, &lock->cnts);
75 
76 	/* When no more readers or writers, set the locked flag */
77 	do {
78 		cnts = atomic_cond_read_relaxed(&lock->cnts, VAL == _QW_WAITING);
79 	} while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));
80 unlock:
81 	arch_spin_unlock(&lock->wait_lock);
82 }
83 EXPORT_SYMBOL(queued_write_lock_slowpath);
84