Lines Matching refs:sem
12 int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, in __percpu_init_rwsem() argument
15 sem->read_count = alloc_percpu(int); in __percpu_init_rwsem()
16 if (unlikely(!sem->read_count)) in __percpu_init_rwsem()
19 rcu_sync_init(&sem->rss); in __percpu_init_rwsem()
20 rcuwait_init(&sem->writer); in __percpu_init_rwsem()
21 init_waitqueue_head(&sem->waiters); in __percpu_init_rwsem()
22 atomic_set(&sem->block, 0); in __percpu_init_rwsem()
24 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); in __percpu_init_rwsem()
25 lockdep_init_map(&sem->dep_map, name, key, 0); in __percpu_init_rwsem()
31 void percpu_free_rwsem(struct percpu_rw_semaphore *sem) in percpu_free_rwsem() argument
37 if (!sem->read_count) in percpu_free_rwsem()
40 rcu_sync_dtor(&sem->rss); in percpu_free_rwsem()
41 free_percpu(sem->read_count); in percpu_free_rwsem()
42 sem->read_count = NULL; /* catch use after free bugs */ in percpu_free_rwsem()
46 static bool __percpu_down_read_trylock(struct percpu_rw_semaphore *sem) in __percpu_down_read_trylock() argument
48 this_cpu_inc(*sem->read_count); in __percpu_down_read_trylock()
71 if (likely(!atomic_read_acquire(&sem->block))) in __percpu_down_read_trylock()
74 this_cpu_dec(*sem->read_count); in __percpu_down_read_trylock()
77 rcuwait_wake_up(&sem->writer); in __percpu_down_read_trylock()
82 static inline bool __percpu_down_write_trylock(struct percpu_rw_semaphore *sem) in __percpu_down_write_trylock() argument
84 if (atomic_read(&sem->block)) in __percpu_down_write_trylock()
87 return atomic_xchg(&sem->block, 1) == 0; in __percpu_down_write_trylock()
90 static bool __percpu_rwsem_trylock(struct percpu_rw_semaphore *sem, bool reader) in __percpu_rwsem_trylock() argument
96 ret = __percpu_down_read_trylock(sem); in __percpu_rwsem_trylock()
101 return __percpu_down_write_trylock(sem); in __percpu_rwsem_trylock()
122 struct percpu_rw_semaphore *sem = key; in percpu_rwsem_wake_function() local
126 if (!__percpu_rwsem_trylock(sem, reader)) in percpu_rwsem_wake_function()
139 static void percpu_rwsem_wait(struct percpu_rw_semaphore *sem, bool reader) in percpu_rwsem_wait() argument
144 spin_lock_irq(&sem->waiters.lock); in percpu_rwsem_wait()
149 wait = !__percpu_rwsem_trylock(sem, reader); in percpu_rwsem_wait()
152 __add_wait_queue_entry_tail(&sem->waiters, &wq_entry); in percpu_rwsem_wait()
154 spin_unlock_irq(&sem->waiters.lock); in percpu_rwsem_wait()
165 bool __percpu_down_read(struct percpu_rw_semaphore *sem, bool try) in __percpu_down_read() argument
167 if (__percpu_down_read_trylock(sem)) in __percpu_down_read()
174 percpu_rwsem_wait(sem, /* .reader = */ true); in __percpu_down_read()
199 static bool readers_active_check(struct percpu_rw_semaphore *sem) in readers_active_check() argument
201 if (per_cpu_sum(*sem->read_count) != 0) in readers_active_check()
214 void percpu_down_write(struct percpu_rw_semaphore *sem) in percpu_down_write() argument
217 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); in percpu_down_write()
220 rcu_sync_enter(&sem->rss); in percpu_down_write()
226 if (!__percpu_down_write_trylock(sem)) in percpu_down_write()
227 percpu_rwsem_wait(sem, /* .reader = */ false); in percpu_down_write()
238 rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE); in percpu_down_write()
242 void percpu_up_write(struct percpu_rw_semaphore *sem) in percpu_up_write() argument
244 rwsem_release(&sem->dep_map, _RET_IP_); in percpu_up_write()
256 atomic_set_release(&sem->block, 0); in percpu_up_write()
261 __wake_up(&sem->waiters, TASK_NORMAL, 1, sem); in percpu_up_write()
268 rcu_sync_exit(&sem->rss); in percpu_up_write()