Lines Matching refs:mutex
148 static inline __always_inline int PIMutexTryLock(PIMutex& mutex) { in PIMutexTryLock() argument
152 if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex.owner_tid, in PIMutexTryLock()
160 if (mutex.type == PTHREAD_MUTEX_NORMAL) { in PIMutexTryLock()
163 if (mutex.type == PTHREAD_MUTEX_ERRORCHECK) { in PIMutexTryLock()
166 if (mutex.counter == 0xffff) { in PIMutexTryLock()
169 mutex.counter++; in PIMutexTryLock()
177 static int __attribute__((noinline)) PIMutexTimedLock(PIMutex& mutex, in PIMutexTimedLock() argument
180 int ret = PIMutexTryLock(mutex); in PIMutexTimedLock()
186 ret = -__futex_pi_lock_ex(&mutex.owner_tid, mutex.shared, use_realtime_clock, abs_timeout); in PIMutexTimedLock()
191 static int PIMutexUnlock(PIMutex& mutex) { in PIMutexUnlock() argument
195 if (__predict_true(mutex.type == PTHREAD_MUTEX_NORMAL)) { in PIMutexUnlock()
196 if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex.owner_tid, in PIMutexUnlock()
203 old_owner = atomic_load_explicit(&mutex.owner_tid, memory_order_relaxed); in PIMutexUnlock()
210 if (mutex.type == PTHREAD_MUTEX_RECURSIVE) { in PIMutexUnlock()
211 if (mutex.counter != 0u) { in PIMutexUnlock()
212 --mutex.counter; in PIMutexUnlock()
218 if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex.owner_tid, in PIMutexUnlock()
225 return -__futex_pi_unlock(&mutex.owner_tid, mutex.shared); in PIMutexUnlock()
228 static int PIMutexDestroy(PIMutex& mutex) { in PIMutexDestroy() argument
232 if (atomic_compare_exchange_strong_explicit(&mutex.owner_tid, &old_owner, 0xffffffff, in PIMutexDestroy()
261 PIMutex mutex; member
278 return IdToNode(id).mutex; in IdToPIMutex()
502 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); in pthread_mutex_init() local
504 memset(mutex, 0, sizeof(pthread_mutex_internal_t)); in pthread_mutex_init()
507 atomic_init(&mutex->state, MUTEX_TYPE_BITS_NORMAL); in pthread_mutex_init()
539 mutex->pi_mutex_id = id; in pthread_mutex_init()
541 atomic_init(&mutex->state, PI_MUTEX_STATE); in pthread_mutex_init()
542 PIMutex& pi_mutex = mutex->ToPIMutex(); in pthread_mutex_init()
546 atomic_init(&mutex->state, state); in pthread_mutex_init()
547 atomic_init(&mutex->owner_tid, 0); in pthread_mutex_init()
555 static inline __always_inline int NormalMutexTryLock(pthread_mutex_internal_t* mutex, in NormalMutexTryLock() argument
561 if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, in NormalMutexTryLock()
580 static inline __always_inline int NormalMutexLock(pthread_mutex_internal_t* mutex, in NormalMutexLock() argument
584 if (__predict_true(NormalMutexTryLock(mutex, shared) == 0)) { in NormalMutexLock()
605 while (atomic_exchange_explicit(&mutex->state, locked_contended, in NormalMutexLock()
607 if (__futex_wait_ex(&mutex->state, shared, locked_contended, use_realtime_clock, in NormalMutexLock()
619 static inline __always_inline void NormalMutexUnlock(pthread_mutex_internal_t* mutex, in NormalMutexUnlock() argument
629 if (atomic_exchange_explicit(&mutex->state, unlocked, in NormalMutexUnlock()
658 __futex_wake_ex(&mutex->state, shared, 1); in NormalMutexUnlock()
668 static inline __always_inline int RecursiveIncrement(pthread_mutex_internal_t* mutex, in RecursiveIncrement() argument
680 atomic_fetch_add_explicit(&mutex->state, MUTEX_COUNTER_BITS_ONE, memory_order_relaxed); in RecursiveIncrement()
685 static inline __always_inline int RecursiveOrErrorcheckMutexWait(pthread_mutex_internal_t* mutex, in RecursiveOrErrorcheckMutexWait() argument
696 return __futex_wait_ex(&mutex->state, shared, old_state, use_realtime_clock, abs_timeout); in RecursiveOrErrorcheckMutexWait()
704 uint32_t owner_tid = atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed); in RecursiveOrErrorcheckMutexWait()
705 return __futex_wait_ex(&mutex->state, shared, (owner_tid << 16) | old_state, in RecursiveOrErrorcheckMutexWait()
711 static int MutexLockWithTimeout(pthread_mutex_internal_t* mutex, bool use_realtime_clock, in MutexLockWithTimeout() argument
713 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); in MutexLockWithTimeout()
719 return NormalMutexLock(mutex, shared, use_realtime_clock, abs_timeout_or_null); in MutexLockWithTimeout()
724 if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { in MutexLockWithTimeout()
728 return RecursiveIncrement(mutex, old_state); in MutexLockWithTimeout()
740 if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, in MutexLockWithTimeout()
742 atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); in MutexLockWithTimeout()
757 if (__predict_true(atomic_compare_exchange_weak_explicit(&mutex->state, in MutexLockWithTimeout()
761 atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); in MutexLockWithTimeout()
770 if (__predict_false(!atomic_compare_exchange_weak_explicit(&mutex->state, in MutexLockWithTimeout()
784 if (RecursiveOrErrorcheckMutexWait(mutex, shared, old_state, use_realtime_clock, in MutexLockWithTimeout()
788 old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); in MutexLockWithTimeout()
800 static int __attribute__((noinline)) HandleUsingDestroyedMutex(pthread_mutex_t* mutex, in HandleUsingDestroyedMutex() argument
803 __fortify_fatal("%s called on a destroyed mutex (%p)", function_name, mutex); in HandleUsingDestroyedMutex()
818 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); in pthread_mutex_lock() local
819 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); in pthread_mutex_lock()
824 if (__predict_true(NonPI::NormalMutexTryLock(mutex, shared) == 0)) { in pthread_mutex_lock()
829 PIMutex& m = mutex->ToPIMutex(); in pthread_mutex_lock()
834 return PIMutexTimedLock(mutex->ToPIMutex(), false, nullptr); in pthread_mutex_lock()
839 return NonPI::MutexLockWithTimeout(mutex, false, nullptr); in pthread_mutex_lock()
852 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); in pthread_mutex_unlock() local
853 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); in pthread_mutex_unlock()
859 NonPI::NormalMutexUnlock(mutex, shared); in pthread_mutex_unlock()
863 return PIMutexUnlock(mutex->ToPIMutex()); in pthread_mutex_unlock()
871 if ( tid != atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed) ) { in pthread_mutex_unlock()
880 atomic_fetch_sub_explicit(&mutex->state, MUTEX_COUNTER_BITS_ONE, memory_order_relaxed); in pthread_mutex_unlock()
890 atomic_store_explicit(&mutex->owner_tid, 0, memory_order_relaxed); in pthread_mutex_unlock()
892 old_state = atomic_exchange_explicit(&mutex->state, unlocked, memory_order_release); in pthread_mutex_unlock()
894 __futex_wake_ex(&mutex->state, shared, 1); in pthread_mutex_unlock()
901 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); in pthread_mutex_trylock() local
903 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); in pthread_mutex_trylock()
909 return NonPI::NormalMutexTryLock(mutex, shared); in pthread_mutex_trylock()
912 return PIMutexTryLock(mutex->ToPIMutex()); in pthread_mutex_trylock()
920 if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { in pthread_mutex_trylock()
924 return NonPI::RecursiveIncrement(mutex, old_state); in pthread_mutex_trylock()
937 if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, in pthread_mutex_trylock()
941 atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); in pthread_mutex_trylock()
964 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); in __pthread_mutex_timedlock() local
965 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); in __pthread_mutex_timedlock()
970 if (__predict_true(NonPI::NormalMutexTryLock(mutex, shared) == 0)) { in __pthread_mutex_timedlock()
975 return PIMutexTimedLock(mutex->ToPIMutex(), use_realtime_clock, abs_timeout); in __pthread_mutex_timedlock()
980 return NonPI::MutexLockWithTimeout(mutex, use_realtime_clock, abs_timeout); in __pthread_mutex_timedlock()
1000 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); in pthread_mutex_clocklock() local
1001 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); in pthread_mutex_clocklock()
1011 pthread_mutex_internal_t* mutex = __get_internal_mutex(mutex_interface); in pthread_mutex_destroy() local
1012 uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); in pthread_mutex_destroy()
1017 int result = PIMutexDestroy(mutex->ToPIMutex()); in pthread_mutex_destroy()
1019 mutex->FreePIMutex(); in pthread_mutex_destroy()
1020 atomic_store(&mutex->state, 0xffff); in pthread_mutex_destroy()
1027 atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, 0xffff, in pthread_mutex_destroy()