Lines Matching refs:rwlock
219 __pthread_rwlock_get_private (pthread_rwlock_t *rwlock) in __pthread_rwlock_get_private() argument
221 return rwlock->__data.__shared != 0 ? FUTEX_SHARED : FUTEX_PRIVATE; in __pthread_rwlock_get_private()
225 __pthread_rwlock_rdunlock (pthread_rwlock_t *rwlock) in __pthread_rwlock_rdunlock() argument
227 int private = __pthread_rwlock_get_private (rwlock); in __pthread_rwlock_rdunlock()
232 unsigned int r = atomic_load_relaxed (&rwlock->__data.__readers); in __pthread_rwlock_rdunlock()
254 if (atomic_compare_exchange_weak_release (&rwlock->__data.__readers, in __pthread_rwlock_rdunlock()
270 if ((atomic_exchange_relaxed (&rwlock->__data.__wrphase_futex, 1) in __pthread_rwlock_rdunlock()
272 futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private); in __pthread_rwlock_rdunlock()
276 futex_wake (&rwlock->__data.__readers, INT_MAX, private); in __pthread_rwlock_rdunlock()
281 __pthread_rwlock_rdlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, in __pthread_rwlock_rdlock_full64() argument
298 if (__glibc_unlikely (atomic_load_relaxed (&rwlock->__data.__cur_writer) in __pthread_rwlock_rdlock_full64()
310 if (rwlock->__data.__flags == PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP) in __pthread_rwlock_rdlock_full64()
312 r = atomic_load_relaxed (&rwlock->__data.__readers); in __pthread_rwlock_rdlock_full64()
322 (&rwlock->__data.__readers, &r, r | PTHREAD_RWLOCK_RWAITING)) in __pthread_rwlock_rdlock_full64()
328 while (((r = atomic_load_relaxed (&rwlock->__data.__readers)) in __pthread_rwlock_rdlock_full64()
331 int private = __pthread_rwlock_get_private (rwlock); in __pthread_rwlock_rdlock_full64()
332 int err = __futex_abstimed_wait64 (&rwlock->__data.__readers, in __pthread_rwlock_rdlock_full64()
353 r = (atomic_fetch_add_acquire (&rwlock->__data.__readers, in __pthread_rwlock_rdlock_full64()
376 (&rwlock->__data.__readers, in __pthread_rwlock_rdlock_full64()
402 if (atomic_compare_exchange_weak_acquire (&rwlock->__data.__readers, &r, in __pthread_rwlock_rdlock_full64()
413 if ((atomic_exchange_relaxed (&rwlock->__data.__wrphase_futex, 0) in __pthread_rwlock_rdlock_full64()
416 int private = __pthread_rwlock_get_private (rwlock); in __pthread_rwlock_rdlock_full64()
417 futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private); in __pthread_rwlock_rdlock_full64()
451 while (((wpf = atomic_load_relaxed (&rwlock->__data.__wrphase_futex)) in __pthread_rwlock_rdlock_full64()
454 int private = __pthread_rwlock_get_private (rwlock); in __pthread_rwlock_rdlock_full64()
457 (&rwlock->__data.__wrphase_futex, in __pthread_rwlock_rdlock_full64()
460 int err = __futex_abstimed_wait64 (&rwlock->__data.__wrphase_futex, in __pthread_rwlock_rdlock_full64()
473 r = atomic_load_relaxed (&rwlock->__data.__readers); in __pthread_rwlock_rdlock_full64()
480 (&rwlock->__data.__readers, &r, in __pthread_rwlock_rdlock_full64()
494 while ((atomic_load_relaxed (&rwlock->__data.__wrphase_futex) in __pthread_rwlock_rdlock_full64()
512 if ((atomic_load_acquire (&rwlock->__data.__readers) in __pthread_rwlock_rdlock_full64()
527 __pthread_rwlock_wrunlock (pthread_rwlock_t *rwlock) in __pthread_rwlock_wrunlock() argument
529 int private = __pthread_rwlock_get_private (rwlock); in __pthread_rwlock_wrunlock()
531 atomic_store_relaxed (&rwlock->__data.__cur_writer, 0); in __pthread_rwlock_wrunlock()
535 = ((atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0) in __pthread_rwlock_wrunlock()
538 if (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP) in __pthread_rwlock_wrunlock()
541 unsigned int w = atomic_load_relaxed (&rwlock->__data.__writers); in __pthread_rwlock_wrunlock()
549 (&rwlock->__data.__writers, &w, w | PTHREAD_RWLOCK_WRHANDOVER)) in __pthread_rwlock_wrunlock()
559 unsigned int r = atomic_load_relaxed (&rwlock->__data.__readers); in __pthread_rwlock_wrunlock()
562 (&rwlock->__data.__readers, &r, in __pthread_rwlock_wrunlock()
575 if ((atomic_exchange_relaxed (&rwlock->__data.__wrphase_futex, 0) in __pthread_rwlock_wrunlock()
577 futex_wake (&rwlock->__data.__wrphase_futex, INT_MAX, private); in __pthread_rwlock_wrunlock()
583 futex_wake (&rwlock->__data.__writers_futex, 1, private); in __pthread_rwlock_wrunlock()
588 __pthread_rwlock_wrlock_full64 (pthread_rwlock_t *rwlock, clockid_t clockid, in __pthread_rwlock_wrlock_full64() argument
603 if (__glibc_unlikely (atomic_load_relaxed (&rwlock->__data.__cur_writer) in __pthread_rwlock_wrlock_full64()
622 unsigned int r = atomic_fetch_or_acquire (&rwlock->__data.__readers, in __pthread_rwlock_wrlock_full64()
628 = (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP); in __pthread_rwlock_wrlock_full64()
635 atomic_fetch_add_relaxed (&rwlock->__data.__writers, 1); in __pthread_rwlock_wrlock_full64()
647 (&rwlock->__data.__readers, &r, r | PTHREAD_RWLOCK_WRLOCKED)) in __pthread_rwlock_wrlock_full64()
662 atomic_fetch_add_relaxed (&rwlock->__data.__writers, -1); in __pthread_rwlock_wrlock_full64()
674 unsigned int w = atomic_load_relaxed (&rwlock->__data.__writers); in __pthread_rwlock_wrlock_full64()
686 (&rwlock->__data.__writers, in __pthread_rwlock_wrlock_full64()
691 r = atomic_load_relaxed (&rwlock->__data.__readers); in __pthread_rwlock_wrlock_full64()
703 int private = __pthread_rwlock_get_private (rwlock); in __pthread_rwlock_wrlock_full64()
705 = atomic_load_relaxed (&rwlock->__data.__writers_futex); in __pthread_rwlock_wrlock_full64()
709 (&rwlock->__data.__writers_futex, &wf, in __pthread_rwlock_wrlock_full64()
718 r = atomic_load_relaxed (&rwlock->__data.__readers); in __pthread_rwlock_wrlock_full64()
730 int err = __futex_abstimed_wait64 (&rwlock->__data.__writers_futex, in __pthread_rwlock_wrlock_full64()
745 = atomic_load_relaxed (&rwlock->__data.__writers); in __pthread_rwlock_wrlock_full64()
747 (&rwlock->__data.__writers, &w, in __pthread_rwlock_wrlock_full64()
755 r = atomic_load_relaxed (&rwlock->__data.__readers); in __pthread_rwlock_wrlock_full64()
765 r = atomic_load_relaxed (&rwlock->__data.__readers); in __pthread_rwlock_wrlock_full64()
778 atomic_store_relaxed (&rwlock->__data.__writers_futex, in __pthread_rwlock_wrlock_full64()
795 if (atomic_compare_exchange_weak_acquire (&rwlock->__data.__readers, in __pthread_rwlock_wrlock_full64()
802 atomic_store_relaxed (&rwlock->__data.__wrphase_futex, 1); in __pthread_rwlock_wrlock_full64()
820 while (((wpf = atomic_load_relaxed (&rwlock->__data.__wrphase_futex)) in __pthread_rwlock_wrlock_full64()
823 int private = __pthread_rwlock_get_private (rwlock); in __pthread_rwlock_wrlock_full64()
826 (&rwlock->__data.__wrphase_futex, &wpf, in __pthread_rwlock_wrlock_full64()
829 int err = __futex_abstimed_wait64 (&rwlock->__data.__wrphase_futex, in __pthread_rwlock_wrlock_full64()
834 if (rwlock->__data.__flags != PTHREAD_RWLOCK_PREFER_READER_NP) in __pthread_rwlock_wrlock_full64()
838 = atomic_load_relaxed (&rwlock->__data.__writers); in __pthread_rwlock_wrlock_full64()
853 = atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0); in __pthread_rwlock_wrlock_full64()
857 (&rwlock->__data.__writers, &w, in __pthread_rwlock_wrlock_full64()
862 futex_wake (&rwlock->__data.__writers_futex, in __pthread_rwlock_wrlock_full64()
873 atomic_store_relaxed (&rwlock->__data.__writers_futex, wf); in __pthread_rwlock_wrlock_full64()
879 r = atomic_load_relaxed (&rwlock->__data.__readers); in __pthread_rwlock_wrlock_full64()
886 = atomic_exchange_relaxed (&rwlock->__data.__writers_futex, 0); in __pthread_rwlock_wrlock_full64()
898 (&rwlock->__data.__readers, &r, in __pthread_rwlock_wrlock_full64()
904 futex_wake (&rwlock->__data.__writers_futex, in __pthread_rwlock_wrlock_full64()
908 futex_wake (&rwlock->__data.__readers, in __pthread_rwlock_wrlock_full64()
917 atomic_store_relaxed (&rwlock->__data.__writers_futex, wf); in __pthread_rwlock_wrlock_full64()
926 while ((atomic_load_relaxed (&rwlock->__data.__wrphase_futex) in __pthread_rwlock_wrlock_full64()
941 if ((atomic_load_acquire (&rwlock->__data.__readers) in __pthread_rwlock_wrlock_full64()
947 atomic_store_relaxed (&rwlock->__data.__cur_writer, in __pthread_rwlock_wrlock_full64()