Lines Matching refs:sem
57 void __init_ldsem(struct ld_semaphore *sem, const char *name, in __init_ldsem() argument
64 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); in __init_ldsem()
65 lockdep_init_map(&sem->dep_map, name, key, 0); in __init_ldsem()
67 atomic_long_set(&sem->count, LDSEM_UNLOCKED); in __init_ldsem()
68 sem->wait_readers = 0; in __init_ldsem()
69 raw_spin_lock_init(&sem->wait_lock); in __init_ldsem()
70 INIT_LIST_HEAD(&sem->read_wait); in __init_ldsem()
71 INIT_LIST_HEAD(&sem->write_wait); in __init_ldsem()
74 static void __ldsem_wake_readers(struct ld_semaphore *sem) in __ldsem_wake_readers() argument
85 adjust = sem->wait_readers * (LDSEM_ACTIVE_BIAS - LDSEM_WAIT_BIAS); in __ldsem_wake_readers()
86 count = atomic_long_add_return(adjust, &sem->count); in __ldsem_wake_readers()
90 if (atomic_long_try_cmpxchg(&sem->count, &count, count - adjust)) in __ldsem_wake_readers()
94 list_for_each_entry_safe(waiter, next, &sem->read_wait, list) { in __ldsem_wake_readers()
100 INIT_LIST_HEAD(&sem->read_wait); in __ldsem_wake_readers()
101 sem->wait_readers = 0; in __ldsem_wake_readers()
104 static inline int writer_trylock(struct ld_semaphore *sem) in writer_trylock() argument
110 long count = atomic_long_add_return(LDSEM_ACTIVE_BIAS, &sem->count); in writer_trylock()
114 if (atomic_long_try_cmpxchg(&sem->count, &count, count - LDSEM_ACTIVE_BIAS)) in writer_trylock()
119 static void __ldsem_wake_writer(struct ld_semaphore *sem) in __ldsem_wake_writer() argument
123 waiter = list_entry(sem->write_wait.next, struct ldsem_waiter, list); in __ldsem_wake_writer()
135 static void __ldsem_wake(struct ld_semaphore *sem) in __ldsem_wake() argument
137 if (!list_empty(&sem->write_wait)) in __ldsem_wake()
138 __ldsem_wake_writer(sem); in __ldsem_wake()
139 else if (!list_empty(&sem->read_wait)) in __ldsem_wake()
140 __ldsem_wake_readers(sem); in __ldsem_wake()
143 static void ldsem_wake(struct ld_semaphore *sem) in ldsem_wake() argument
147 raw_spin_lock_irqsave(&sem->wait_lock, flags); in ldsem_wake()
148 __ldsem_wake(sem); in ldsem_wake()
149 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); in ldsem_wake()
156 down_read_failed(struct ld_semaphore *sem, long count, long timeout) in down_read_failed() argument
162 raw_spin_lock_irq(&sem->wait_lock); in down_read_failed()
170 if (atomic_long_try_cmpxchg(&sem->count, &count, count + adjust)) { in down_read_failed()
175 raw_spin_unlock_irq(&sem->wait_lock); in down_read_failed()
176 return sem; in down_read_failed()
180 list_add_tail(&waiter.list, &sem->read_wait); in down_read_failed()
181 sem->wait_readers++; in down_read_failed()
188 __ldsem_wake(sem); in down_read_failed()
190 raw_spin_unlock_irq(&sem->wait_lock); in down_read_failed()
211 raw_spin_lock_irq(&sem->wait_lock); in down_read_failed()
213 atomic_long_add_return(-LDSEM_WAIT_BIAS, &sem->count); in down_read_failed()
214 sem->wait_readers--; in down_read_failed()
216 raw_spin_unlock_irq(&sem->wait_lock); in down_read_failed()
220 raw_spin_unlock_irq(&sem->wait_lock); in down_read_failed()
223 return sem; in down_read_failed()
230 down_write_failed(struct ld_semaphore *sem, long count, long timeout) in down_write_failed() argument
237 raw_spin_lock_irq(&sem->wait_lock); in down_write_failed()
245 if (atomic_long_try_cmpxchg(&sem->count, &count, count + adjust)) in down_write_failed()
248 raw_spin_unlock_irq(&sem->wait_lock); in down_write_failed()
249 return sem; in down_write_failed()
253 list_add_tail(&waiter.list, &sem->write_wait); in down_write_failed()
261 raw_spin_unlock_irq(&sem->wait_lock); in down_write_failed()
263 raw_spin_lock_irq(&sem->wait_lock); in down_write_failed()
265 locked = writer_trylock(sem); in down_write_failed()
271 atomic_long_add_return(-LDSEM_WAIT_BIAS, &sem->count); in down_write_failed()
280 if (!locked && list_empty(&sem->write_wait)) in down_write_failed()
281 __ldsem_wake_readers(sem); in down_write_failed()
283 raw_spin_unlock_irq(&sem->wait_lock); in down_write_failed()
290 return sem; in down_write_failed()
295 static int __ldsem_down_read_nested(struct ld_semaphore *sem, in __ldsem_down_read_nested() argument
300 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); in __ldsem_down_read_nested()
302 count = atomic_long_add_return(LDSEM_READ_BIAS, &sem->count); in __ldsem_down_read_nested()
304 lock_contended(&sem->dep_map, _RET_IP_); in __ldsem_down_read_nested()
305 if (!down_read_failed(sem, count, timeout)) { in __ldsem_down_read_nested()
306 rwsem_release(&sem->dep_map, _RET_IP_); in __ldsem_down_read_nested()
310 lock_acquired(&sem->dep_map, _RET_IP_); in __ldsem_down_read_nested()
314 static int __ldsem_down_write_nested(struct ld_semaphore *sem, in __ldsem_down_write_nested() argument
319 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); in __ldsem_down_write_nested()
321 count = atomic_long_add_return(LDSEM_WRITE_BIAS, &sem->count); in __ldsem_down_write_nested()
323 lock_contended(&sem->dep_map, _RET_IP_); in __ldsem_down_write_nested()
324 if (!down_write_failed(sem, count, timeout)) { in __ldsem_down_write_nested()
325 rwsem_release(&sem->dep_map, _RET_IP_); in __ldsem_down_write_nested()
329 lock_acquired(&sem->dep_map, _RET_IP_); in __ldsem_down_write_nested()
337 int __sched ldsem_down_read(struct ld_semaphore *sem, long timeout) in ldsem_down_read() argument
340 return __ldsem_down_read_nested(sem, 0, timeout); in ldsem_down_read()
346 int ldsem_down_read_trylock(struct ld_semaphore *sem) in ldsem_down_read_trylock() argument
348 long count = atomic_long_read(&sem->count); in ldsem_down_read_trylock()
351 if (atomic_long_try_cmpxchg(&sem->count, &count, count + LDSEM_READ_BIAS)) { in ldsem_down_read_trylock()
352 rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); in ldsem_down_read_trylock()
353 lock_acquired(&sem->dep_map, _RET_IP_); in ldsem_down_read_trylock()
363 int __sched ldsem_down_write(struct ld_semaphore *sem, long timeout) in ldsem_down_write() argument
366 return __ldsem_down_write_nested(sem, 0, timeout); in ldsem_down_write()
372 int ldsem_down_write_trylock(struct ld_semaphore *sem) in ldsem_down_write_trylock() argument
374 long count = atomic_long_read(&sem->count); in ldsem_down_write_trylock()
377 if (atomic_long_try_cmpxchg(&sem->count, &count, count + LDSEM_WRITE_BIAS)) { in ldsem_down_write_trylock()
378 rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_); in ldsem_down_write_trylock()
379 lock_acquired(&sem->dep_map, _RET_IP_); in ldsem_down_write_trylock()
389 void ldsem_up_read(struct ld_semaphore *sem) in ldsem_up_read() argument
393 rwsem_release(&sem->dep_map, _RET_IP_); in ldsem_up_read()
395 count = atomic_long_add_return(-LDSEM_READ_BIAS, &sem->count); in ldsem_up_read()
397 ldsem_wake(sem); in ldsem_up_read()
403 void ldsem_up_write(struct ld_semaphore *sem) in ldsem_up_write() argument
407 rwsem_release(&sem->dep_map, _RET_IP_); in ldsem_up_write()
409 count = atomic_long_add_return(-LDSEM_WRITE_BIAS, &sem->count); in ldsem_up_write()
411 ldsem_wake(sem); in ldsem_up_write()
417 int ldsem_down_read_nested(struct ld_semaphore *sem, int subclass, long timeout) in ldsem_down_read_nested() argument
420 return __ldsem_down_read_nested(sem, subclass, timeout); in ldsem_down_read_nested()
423 int ldsem_down_write_nested(struct ld_semaphore *sem, int subclass, in ldsem_down_write_nested() argument
427 return __ldsem_down_write_nested(sem, subclass, timeout); in ldsem_down_write_nested()