Lines Matching refs:mutex
35 lll_mutex_lock_optimized (pthread_mutex_t *mutex) in lll_mutex_lock_optimized() argument
44 int private = PTHREAD_MUTEX_PSHARED (mutex); in lll_mutex_lock_optimized()
45 if (private == LLL_PRIVATE && SINGLE_THREAD_P && mutex->__data.__lock == 0) in lll_mutex_lock_optimized()
46 mutex->__data.__lock = 1; in lll_mutex_lock_optimized()
48 lll_lock (mutex->__data.__lock, private); in lll_mutex_lock_optimized()
51 # define LLL_MUTEX_LOCK(mutex) \ argument
52 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
53 # define LLL_MUTEX_LOCK_OPTIMIZED(mutex) lll_mutex_lock_optimized (mutex) argument
54 # define LLL_MUTEX_TRYLOCK(mutex) \ argument
55 lll_trylock ((mutex)->__data.__lock)
57 # define LLL_MUTEX_LOCK_ELISION(mutex) \ argument
58 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
59 PTHREAD_MUTEX_PSHARED (mutex))
60 # define LLL_MUTEX_TRYLOCK_ELISION(mutex) \ argument
61 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
62 PTHREAD_MUTEX_PSHARED (mutex))
68 # define LLL_MUTEX_READ_LOCK(mutex) \ argument
69 atomic_load_relaxed (&(mutex)->__data.__lock)
72 static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
76 PTHREAD_MUTEX_LOCK (pthread_mutex_t *mutex) in PTHREAD_MUTEX_LOCK() argument
80 unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex); in PTHREAD_MUTEX_LOCK()
82 LIBC_PROBE (mutex_entry, 1, mutex); in PTHREAD_MUTEX_LOCK()
86 return __pthread_mutex_lock_full (mutex); in PTHREAD_MUTEX_LOCK()
90 FORCE_ELISION (mutex, goto elision); in PTHREAD_MUTEX_LOCK()
93 LLL_MUTEX_LOCK_OPTIMIZED (mutex); in PTHREAD_MUTEX_LOCK()
94 assert (mutex->__data.__owner == 0); in PTHREAD_MUTEX_LOCK()
105 return LLL_MUTEX_LOCK_ELISION (mutex); in PTHREAD_MUTEX_LOCK()
108 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) in PTHREAD_MUTEX_LOCK()
115 if (mutex->__data.__owner == id) in PTHREAD_MUTEX_LOCK()
118 if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) in PTHREAD_MUTEX_LOCK()
122 ++mutex->__data.__count; in PTHREAD_MUTEX_LOCK()
128 LLL_MUTEX_LOCK_OPTIMIZED (mutex); in PTHREAD_MUTEX_LOCK()
130 assert (mutex->__data.__owner == 0); in PTHREAD_MUTEX_LOCK()
131 mutex->__data.__count = 1; in PTHREAD_MUTEX_LOCK()
133 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex) in PTHREAD_MUTEX_LOCK()
136 if (LLL_MUTEX_TRYLOCK (mutex) != 0) in PTHREAD_MUTEX_LOCK()
140 mutex->__data.__spins * 2 + 10); in PTHREAD_MUTEX_LOCK()
145 LLL_MUTEX_LOCK (mutex); in PTHREAD_MUTEX_LOCK()
149 if (LLL_MUTEX_READ_LOCK (mutex) != 0) in PTHREAD_MUTEX_LOCK()
152 while (LLL_MUTEX_TRYLOCK (mutex) != 0); in PTHREAD_MUTEX_LOCK()
154 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8; in PTHREAD_MUTEX_LOCK()
156 assert (mutex->__data.__owner == 0); in PTHREAD_MUTEX_LOCK()
161 assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP); in PTHREAD_MUTEX_LOCK()
163 if (__glibc_unlikely (mutex->__data.__owner == id)) in PTHREAD_MUTEX_LOCK()
171 mutex->__data.__owner = id; in PTHREAD_MUTEX_LOCK()
173 ++mutex->__data.__nusers; in PTHREAD_MUTEX_LOCK()
176 LIBC_PROBE (mutex_acquired, 1, mutex); in PTHREAD_MUTEX_LOCK()
182 __pthread_mutex_lock_full (pthread_mutex_t *mutex) in __pthread_mutex_lock_full() argument
187 switch (PTHREAD_MUTEX_TYPE (mutex)) in __pthread_mutex_lock_full()
194 &mutex->__data.__list.__next); in __pthread_mutex_lock_full()
199 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
215 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
234 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
244 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
246 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; in __pthread_mutex_lock_full()
251 ENQUEUE_MUTEX (mutex); in __pthread_mutex_lock_full()
263 --mutex->__data.__nusers; in __pthread_mutex_lock_full()
272 int kind = PTHREAD_MUTEX_TYPE (mutex); in __pthread_mutex_lock_full()
290 if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) in __pthread_mutex_lock_full()
294 ++mutex->__data.__count; in __pthread_mutex_lock_full()
308 (&mutex->__data.__lock, oldval | FUTEX_WAITERS, oldval); in __pthread_mutex_lock_full()
324 futex_wait ((unsigned int *) &mutex->__data.__lock, oldval, in __pthread_mutex_lock_full()
325 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); in __pthread_mutex_lock_full()
326 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
330 if (__builtin_expect (mutex->__data.__owner in __pthread_mutex_lock_full()
334 mutex->__data.__count = 0; in __pthread_mutex_lock_full()
335 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex); in __pthread_mutex_lock_full()
336 lll_unlock (mutex->__data.__lock, private); in __pthread_mutex_lock_full()
343 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
347 ENQUEUE_MUTEX (mutex); in __pthread_mutex_lock_full()
370 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind)); in __pthread_mutex_lock_full()
379 (void *) (((uintptr_t) &mutex->__data.__list.__next) in __pthread_mutex_lock_full()
386 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
406 if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) in __pthread_mutex_lock_full()
410 ++mutex->__data.__count; in __pthread_mutex_lock_full()
420 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
428 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex) in __pthread_mutex_lock_full()
429 : PTHREAD_MUTEX_PSHARED (mutex)); in __pthread_mutex_lock_full()
430 int e = __futex_lock_pi64 (&mutex->__data.__lock, 0 /* ununsed */, in __pthread_mutex_lock_full()
447 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
454 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED); in __pthread_mutex_lock_full()
457 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
459 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; in __pthread_mutex_lock_full()
464 ENQUEUE_MUTEX_PI (mutex); in __pthread_mutex_lock_full()
475 --mutex->__data.__nusers; in __pthread_mutex_lock_full()
482 && __builtin_expect (mutex->__data.__owner in __pthread_mutex_lock_full()
486 mutex->__data.__count = 0; in __pthread_mutex_lock_full()
488 futex_unlock_pi ((unsigned int *) &mutex->__data.__lock, in __pthread_mutex_lock_full()
489 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); in __pthread_mutex_lock_full()
497 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
503 ENQUEUE_MUTEX_PI (mutex); in __pthread_mutex_lock_full()
519 int kind = atomic_load_relaxed (&(mutex->__data.__kind)) in __pthread_mutex_lock_full()
522 oldval = mutex->__data.__lock; in __pthread_mutex_lock_full()
525 if (mutex->__data.__owner == id) in __pthread_mutex_lock_full()
533 if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) in __pthread_mutex_lock_full()
537 ++mutex->__data.__count; in __pthread_mutex_lock_full()
564 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
578 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
586 futex_wait ((unsigned int * ) &mutex->__data.__lock, in __pthread_mutex_lock_full()
588 PTHREAD_MUTEX_PSHARED (mutex)); in __pthread_mutex_lock_full()
590 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, in __pthread_mutex_lock_full()
596 assert (mutex->__data.__owner == 0); in __pthread_mutex_lock_full()
597 mutex->__data.__count = 1; in __pthread_mutex_lock_full()
607 mutex->__data.__owner = id; in __pthread_mutex_lock_full()
609 ++mutex->__data.__nusers; in __pthread_mutex_lock_full()
612 LIBC_PROBE (mutex_acquired, 1, mutex); in __pthread_mutex_lock_full()
634 __pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex) in __pthread_mutex_cond_lock_adjust() argument
638 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind)); in __pthread_mutex_cond_lock_adjust()
645 mutex->__data.__owner = id; in __pthread_mutex_cond_lock_adjust()
648 ++mutex->__data.__count; in __pthread_mutex_cond_lock_adjust()