Lines Matching refs:sem
67 # define DEBUG_RWSEMS_WARN_ON(c, sem) do { \ argument
70 #c, atomic_long_read(&(sem)->count), \
71 (unsigned long) sem->magic, \
72 atomic_long_read(&(sem)->owner), (long)current, \
73 list_empty(&(sem)->wait_list) ? "" : "not ")) \
77 # define DEBUG_RWSEMS_WARN_ON(c, sem) argument
136 static inline void rwsem_set_owner(struct rw_semaphore *sem) in rwsem_set_owner() argument
138 atomic_long_set(&sem->owner, (long)current); in rwsem_set_owner()
141 static inline void rwsem_clear_owner(struct rw_semaphore *sem) in rwsem_clear_owner() argument
143 atomic_long_set(&sem->owner, 0); in rwsem_clear_owner()
149 static inline bool rwsem_test_oflags(struct rw_semaphore *sem, long flags) in rwsem_test_oflags() argument
151 return atomic_long_read(&sem->owner) & flags; in rwsem_test_oflags()
164 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, in __rwsem_set_reader_owned() argument
168 (atomic_long_read(&sem->owner) & RWSEM_NONSPINNABLE); in __rwsem_set_reader_owned()
170 atomic_long_set(&sem->owner, val); in __rwsem_set_reader_owned()
173 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) in rwsem_set_reader_owned() argument
175 __rwsem_set_reader_owned(sem, current); in rwsem_set_reader_owned()
181 static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem) in is_rwsem_reader_owned() argument
187 long count = atomic_long_read(&sem->count); in is_rwsem_reader_owned()
192 return rwsem_test_oflags(sem, RWSEM_READER_OWNED); in is_rwsem_reader_owned()
202 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) in rwsem_clear_reader_owned() argument
204 unsigned long val = atomic_long_read(&sem->owner); in rwsem_clear_reader_owned()
207 if (atomic_long_try_cmpxchg(&sem->owner, &val, in rwsem_clear_reader_owned()
213 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) in rwsem_clear_reader_owned() argument
222 static inline void rwsem_set_nonspinnable(struct rw_semaphore *sem) in rwsem_set_nonspinnable() argument
224 unsigned long owner = atomic_long_read(&sem->owner); in rwsem_set_nonspinnable()
231 } while (!atomic_long_try_cmpxchg(&sem->owner, &owner, in rwsem_set_nonspinnable()
235 static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) in rwsem_read_trylock() argument
237 *cntp = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count); in rwsem_read_trylock()
240 rwsem_set_nonspinnable(sem); in rwsem_read_trylock()
243 rwsem_set_reader_owned(sem); in rwsem_read_trylock()
250 static inline bool rwsem_write_trylock(struct rw_semaphore *sem) in rwsem_write_trylock() argument
254 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) { in rwsem_write_trylock()
255 rwsem_set_owner(sem); in rwsem_write_trylock()
265 static inline struct task_struct *rwsem_owner(struct rw_semaphore *sem) in rwsem_owner() argument
268 (atomic_long_read(&sem->owner) & ~RWSEM_OWNER_FLAGS_MASK); in rwsem_owner()
276 rwsem_owner_flags(struct rw_semaphore *sem, unsigned long *pflags) in rwsem_owner_flags() argument
278 unsigned long owner = atomic_long_read(&sem->owner); in rwsem_owner_flags()
304 void __init_rwsem(struct rw_semaphore *sem, const char *name, in __init_rwsem() argument
311 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); in __init_rwsem()
312 lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP); in __init_rwsem()
315 sem->magic = sem; in __init_rwsem()
317 atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE); in __init_rwsem()
318 raw_spin_lock_init(&sem->wait_lock); in __init_rwsem()
319 INIT_LIST_HEAD(&sem->wait_list); in __init_rwsem()
320 atomic_long_set(&sem->owner, 0L); in __init_rwsem()
322 osq_lock_init(&sem->osq); in __init_rwsem()
341 #define rwsem_first_waiter(sem) \ argument
342 list_first_entry(&sem->wait_list, struct rwsem_waiter, list)
366 rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) in rwsem_add_waiter() argument
368 lockdep_assert_held(&sem->wait_lock); in rwsem_add_waiter()
369 list_add_tail(&waiter->list, &sem->wait_list); in rwsem_add_waiter()
380 rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) in rwsem_del_waiter() argument
382 lockdep_assert_held(&sem->wait_lock); in rwsem_del_waiter()
384 if (likely(!list_empty(&sem->wait_list))) in rwsem_del_waiter()
387 atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count); in rwsem_del_waiter()
404 static void rwsem_mark_wake(struct rw_semaphore *sem, in rwsem_mark_wake() argument
412 lockdep_assert_held(&sem->wait_lock); in rwsem_mark_wake()
418 waiter = rwsem_first_waiter(sem); in rwsem_mark_wake()
439 if (unlikely(atomic_long_read(&sem->count) < 0)) in rwsem_mark_wake()
451 oldcount = atomic_long_fetch_add(adjustment, &sem->count); in rwsem_mark_wake()
464 atomic_long_add(-adjustment, &sem->count); in rwsem_mark_wake()
474 __rwsem_set_reader_owned(sem, owner); in rwsem_mark_wake()
501 list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) { in rwsem_mark_wake()
518 oldcount = atomic_long_read(&sem->count); in rwsem_mark_wake()
519 if (list_empty(&sem->wait_list)) { in rwsem_mark_wake()
537 atomic_long_add(adjustment, &sem->count); in rwsem_mark_wake()
568 static inline bool rwsem_try_write_lock(struct rw_semaphore *sem, in rwsem_try_write_lock() argument
571 bool first = rwsem_first_waiter(sem) == waiter; in rwsem_try_write_lock()
574 lockdep_assert_held(&sem->wait_lock); in rwsem_try_write_lock()
576 count = atomic_long_read(&sem->count); in rwsem_try_write_lock()
600 if (list_is_singular(&sem->wait_list)) in rwsem_try_write_lock()
603 } while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new)); in rwsem_try_write_lock()
620 rwsem_set_owner(sem); in rwsem_try_write_lock()
646 static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) in rwsem_try_write_lock_unqueued() argument
648 long count = atomic_long_read(&sem->count); in rwsem_try_write_lock_unqueued()
651 if (atomic_long_try_cmpxchg_acquire(&sem->count, &count, in rwsem_try_write_lock_unqueued()
653 rwsem_set_owner(sem); in rwsem_try_write_lock_unqueued()
670 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) in rwsem_can_spin_on_owner() argument
686 owner = rwsem_owner_flags(sem, &flags); in rwsem_can_spin_on_owner()
714 rwsem_spin_on_owner(struct rw_semaphore *sem) in rwsem_spin_on_owner() argument
722 owner = rwsem_owner_flags(sem, &flags); in rwsem_spin_on_owner()
734 new = rwsem_owner_flags(sem, &new_flags); in rwsem_spin_on_owner()
773 static inline u64 rwsem_rspin_threshold(struct rw_semaphore *sem) in rwsem_rspin_threshold() argument
775 long count = atomic_long_read(&sem->count); in rwsem_rspin_threshold()
786 static bool rwsem_optimistic_spin(struct rw_semaphore *sem) in rwsem_optimistic_spin() argument
796 if (!osq_lock(&sem->osq)) in rwsem_optimistic_spin()
808 owner_state = rwsem_spin_on_owner(sem); in rwsem_optimistic_spin()
815 taken = rwsem_try_write_lock_unqueued(sem); in rwsem_optimistic_spin()
832 if (rwsem_test_oflags(sem, RWSEM_NONSPINNABLE)) in rwsem_optimistic_spin()
834 rspin_threshold = rwsem_rspin_threshold(sem); in rwsem_optimistic_spin()
846 rwsem_set_nonspinnable(sem); in rwsem_optimistic_spin()
900 osq_unlock(&sem->osq); in rwsem_optimistic_spin()
911 static inline void clear_nonspinnable(struct rw_semaphore *sem) in clear_nonspinnable() argument
913 if (rwsem_test_oflags(sem, RWSEM_NONSPINNABLE)) in clear_nonspinnable()
914 atomic_long_andnot(RWSEM_NONSPINNABLE, &sem->owner); in clear_nonspinnable()
918 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) in rwsem_can_spin_on_owner() argument
923 static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem) in rwsem_optimistic_spin() argument
928 static inline void clear_nonspinnable(struct rw_semaphore *sem) { } in clear_nonspinnable() argument
931 rwsem_spin_on_owner(struct rw_semaphore *sem) in rwsem_spin_on_owner() argument
941 rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int state) in rwsem_down_read_slowpath() argument
954 if ((atomic_long_read(&sem->owner) & RWSEM_READER_OWNED) && in rwsem_down_read_slowpath()
962 rwsem_set_reader_owned(sem); in rwsem_down_read_slowpath()
970 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
971 if (!list_empty(&sem->wait_list)) in rwsem_down_read_slowpath()
972 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, in rwsem_down_read_slowpath()
974 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
977 return sem; in rwsem_down_read_slowpath()
985 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
986 if (list_empty(&sem->wait_list)) { in rwsem_down_read_slowpath()
993 if (!(atomic_long_read(&sem->count) & in rwsem_down_read_slowpath()
997 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
998 rwsem_set_reader_owned(sem); in rwsem_down_read_slowpath()
1000 return sem; in rwsem_down_read_slowpath()
1004 rwsem_add_waiter(sem, &waiter); in rwsem_down_read_slowpath()
1007 count = atomic_long_add_return(adjustment, &sem->count); in rwsem_down_read_slowpath()
1016 clear_nonspinnable(sem); in rwsem_down_read_slowpath()
1021 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); in rwsem_down_read_slowpath()
1023 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1034 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1037 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1047 return sem; in rwsem_down_read_slowpath()
1050 rwsem_del_waiter(sem, &waiter); in rwsem_down_read_slowpath()
1051 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1061 rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) in rwsem_down_write_slowpath() argument
1068 if (rwsem_can_spin_on_owner(sem) && rwsem_optimistic_spin(sem)) { in rwsem_down_write_slowpath()
1070 return sem; in rwsem_down_write_slowpath()
1082 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1083 rwsem_add_waiter(sem, &waiter); in rwsem_down_write_slowpath()
1086 if (rwsem_first_waiter(sem) != &waiter) { in rwsem_down_write_slowpath()
1087 count = atomic_long_read(&sem->count); in rwsem_down_write_slowpath()
1100 rwsem_mark_wake(sem, (count & RWSEM_READER_MASK) in rwsem_down_write_slowpath()
1109 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1112 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1115 atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count); in rwsem_down_write_slowpath()
1122 if (rwsem_try_write_lock(sem, &waiter)) { in rwsem_down_write_slowpath()
1127 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1144 owner_state = rwsem_spin_on_owner(sem); in rwsem_down_write_slowpath()
1155 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1158 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1160 return sem; in rwsem_down_write_slowpath()
1164 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1165 rwsem_del_waiter(sem, &waiter); in rwsem_down_write_slowpath()
1166 if (!list_empty(&sem->wait_list)) in rwsem_down_write_slowpath()
1167 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); in rwsem_down_write_slowpath()
1168 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1178 static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem) in rwsem_wake() argument
1183 raw_spin_lock_irqsave(&sem->wait_lock, flags); in rwsem_wake()
1185 if (!list_empty(&sem->wait_list)) in rwsem_wake()
1186 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); in rwsem_wake()
1188 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); in rwsem_wake()
1191 return sem; in rwsem_wake()
1199 static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem) in rwsem_downgrade_wake() argument
1204 raw_spin_lock_irqsave(&sem->wait_lock, flags); in rwsem_downgrade_wake()
1206 if (!list_empty(&sem->wait_list)) in rwsem_downgrade_wake()
1207 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q); in rwsem_downgrade_wake()
1209 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); in rwsem_downgrade_wake()
1212 return sem; in rwsem_downgrade_wake()
1218 static inline int __down_read_common(struct rw_semaphore *sem, int state) in __down_read_common() argument
1222 if (!rwsem_read_trylock(sem, &count)) { in __down_read_common()
1223 if (IS_ERR(rwsem_down_read_slowpath(sem, count, state))) in __down_read_common()
1225 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in __down_read_common()
1230 static inline void __down_read(struct rw_semaphore *sem) in __down_read() argument
1232 __down_read_common(sem, TASK_UNINTERRUPTIBLE); in __down_read()
1235 static inline int __down_read_interruptible(struct rw_semaphore *sem) in __down_read_interruptible() argument
1237 return __down_read_common(sem, TASK_INTERRUPTIBLE); in __down_read_interruptible()
1240 static inline int __down_read_killable(struct rw_semaphore *sem) in __down_read_killable() argument
1242 return __down_read_common(sem, TASK_KILLABLE); in __down_read_killable()
1245 static inline int __down_read_trylock(struct rw_semaphore *sem) in __down_read_trylock() argument
1249 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __down_read_trylock()
1251 tmp = atomic_long_read(&sem->count); in __down_read_trylock()
1253 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, in __down_read_trylock()
1255 rwsem_set_reader_owned(sem); in __down_read_trylock()
1265 static inline int __down_write_common(struct rw_semaphore *sem, int state) in __down_write_common() argument
1267 if (unlikely(!rwsem_write_trylock(sem))) { in __down_write_common()
1268 if (IS_ERR(rwsem_down_write_slowpath(sem, state))) in __down_write_common()
1275 static inline void __down_write(struct rw_semaphore *sem) in __down_write() argument
1277 __down_write_common(sem, TASK_UNINTERRUPTIBLE); in __down_write()
1280 static inline int __down_write_killable(struct rw_semaphore *sem) in __down_write_killable() argument
1282 return __down_write_common(sem, TASK_KILLABLE); in __down_write_killable()
1285 static inline int __down_write_trylock(struct rw_semaphore *sem) in __down_write_trylock() argument
1287 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __down_write_trylock()
1288 return rwsem_write_trylock(sem); in __down_write_trylock()
1294 static inline void __up_read(struct rw_semaphore *sem) in __up_read() argument
1298 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __up_read()
1299 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in __up_read()
1301 rwsem_clear_reader_owned(sem); in __up_read()
1302 tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count); in __up_read()
1303 DEBUG_RWSEMS_WARN_ON(tmp < 0, sem); in __up_read()
1306 clear_nonspinnable(sem); in __up_read()
1307 rwsem_wake(sem); in __up_read()
1314 static inline void __up_write(struct rw_semaphore *sem) in __up_write() argument
1318 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __up_write()
1323 DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) && in __up_write()
1324 !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem); in __up_write()
1326 rwsem_clear_owner(sem); in __up_write()
1327 tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); in __up_write()
1329 rwsem_wake(sem); in __up_write()
1335 static inline void __downgrade_write(struct rw_semaphore *sem) in __downgrade_write() argument
1346 DEBUG_RWSEMS_WARN_ON(rwsem_owner(sem) != current, sem); in __downgrade_write()
1348 -RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count); in __downgrade_write()
1349 rwsem_set_reader_owned(sem); in __downgrade_write()
1351 rwsem_downgrade_wake(sem); in __downgrade_write()
1385 void __init_rwsem(struct rw_semaphore *sem, const char *name, in __init_rwsem() argument
1388 init_rwbase_rt(&(sem)->rwbase); in __init_rwsem()
1391 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); in __init_rwsem()
1392 lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP); in __init_rwsem()
1397 static inline void __down_read(struct rw_semaphore *sem) in __down_read() argument
1399 rwbase_read_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE); in __down_read()
1402 static inline int __down_read_interruptible(struct rw_semaphore *sem) in __down_read_interruptible() argument
1404 return rwbase_read_lock(&sem->rwbase, TASK_INTERRUPTIBLE); in __down_read_interruptible()
1407 static inline int __down_read_killable(struct rw_semaphore *sem) in __down_read_killable() argument
1409 return rwbase_read_lock(&sem->rwbase, TASK_KILLABLE); in __down_read_killable()
1412 static inline int __down_read_trylock(struct rw_semaphore *sem) in __down_read_trylock() argument
1414 return rwbase_read_trylock(&sem->rwbase); in __down_read_trylock()
1417 static inline void __up_read(struct rw_semaphore *sem) in __up_read() argument
1419 rwbase_read_unlock(&sem->rwbase, TASK_NORMAL); in __up_read()
1422 static inline void __sched __down_write(struct rw_semaphore *sem) in __down_write() argument
1424 rwbase_write_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE); in __down_write()
1427 static inline int __sched __down_write_killable(struct rw_semaphore *sem) in __down_write_killable() argument
1429 return rwbase_write_lock(&sem->rwbase, TASK_KILLABLE); in __down_write_killable()
1432 static inline int __down_write_trylock(struct rw_semaphore *sem) in __down_write_trylock() argument
1434 return rwbase_write_trylock(&sem->rwbase); in __down_write_trylock()
1437 static inline void __up_write(struct rw_semaphore *sem) in __up_write() argument
1439 rwbase_write_unlock(&sem->rwbase); in __up_write()
1442 static inline void __downgrade_write(struct rw_semaphore *sem) in __downgrade_write() argument
1444 rwbase_write_downgrade(&sem->rwbase); in __downgrade_write()
1448 #define DEBUG_RWSEMS_WARN_ON(c, sem) argument
1450 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, in __rwsem_set_reader_owned() argument
1455 static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem) in is_rwsem_reader_owned() argument
1457 int count = atomic_read(&sem->rwbase.readers); in is_rwsem_reader_owned()
1467 void __sched down_read(struct rw_semaphore *sem) in down_read() argument
1470 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); in down_read()
1472 LOCK_CONTENDED(sem, __down_read_trylock, __down_read); in down_read()
1476 int __sched down_read_interruptible(struct rw_semaphore *sem) in down_read_interruptible() argument
1479 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); in down_read_interruptible()
1481 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_interruptible)) { in down_read_interruptible()
1482 rwsem_release(&sem->dep_map, _RET_IP_); in down_read_interruptible()
1490 int __sched down_read_killable(struct rw_semaphore *sem) in down_read_killable() argument
1493 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); in down_read_killable()
1495 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { in down_read_killable()
1496 rwsem_release(&sem->dep_map, _RET_IP_); in down_read_killable()
1507 int down_read_trylock(struct rw_semaphore *sem) in down_read_trylock() argument
1509 int ret = __down_read_trylock(sem); in down_read_trylock()
1512 rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); in down_read_trylock()
1520 void __sched down_write(struct rw_semaphore *sem) in down_write() argument
1523 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); in down_write()
1524 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); in down_write()
1531 int __sched down_write_killable(struct rw_semaphore *sem) in down_write_killable() argument
1534 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); in down_write_killable()
1536 if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, in down_write_killable()
1538 rwsem_release(&sem->dep_map, _RET_IP_); in down_write_killable()
1549 int down_write_trylock(struct rw_semaphore *sem) in down_write_trylock() argument
1551 int ret = __down_write_trylock(sem); in down_write_trylock()
1554 rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_); in down_write_trylock()
1563 void up_read(struct rw_semaphore *sem) in up_read() argument
1565 rwsem_release(&sem->dep_map, _RET_IP_); in up_read()
1566 __up_read(sem); in up_read()
1573 void up_write(struct rw_semaphore *sem) in up_write() argument
1575 rwsem_release(&sem->dep_map, _RET_IP_); in up_write()
1576 __up_write(sem); in up_write()
1583 void downgrade_write(struct rw_semaphore *sem) in downgrade_write() argument
1585 lock_downgrade(&sem->dep_map, _RET_IP_); in downgrade_write()
1586 __downgrade_write(sem); in downgrade_write()
1592 void down_read_nested(struct rw_semaphore *sem, int subclass) in down_read_nested() argument
1595 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); in down_read_nested()
1596 LOCK_CONTENDED(sem, __down_read_trylock, __down_read); in down_read_nested()
1600 int down_read_killable_nested(struct rw_semaphore *sem, int subclass) in down_read_killable_nested() argument
1603 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); in down_read_killable_nested()
1605 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { in down_read_killable_nested()
1606 rwsem_release(&sem->dep_map, _RET_IP_); in down_read_killable_nested()
1614 void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest) in _down_write_nest_lock() argument
1617 rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_); in _down_write_nest_lock()
1618 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); in _down_write_nest_lock()
1622 void down_read_non_owner(struct rw_semaphore *sem) in down_read_non_owner() argument
1625 __down_read(sem); in down_read_non_owner()
1626 __rwsem_set_reader_owned(sem, NULL); in down_read_non_owner()
1630 void down_write_nested(struct rw_semaphore *sem, int subclass) in down_write_nested() argument
1633 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); in down_write_nested()
1634 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); in down_write_nested()
1638 int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass) in down_write_killable_nested() argument
1641 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); in down_write_killable_nested()
1643 if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, in down_write_killable_nested()
1645 rwsem_release(&sem->dep_map, _RET_IP_); in down_write_killable_nested()
1653 void up_read_non_owner(struct rw_semaphore *sem) in up_read_non_owner() argument
1655 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in up_read_non_owner()
1656 __up_read(sem); in up_read_non_owner()