17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 52be60c5eSraf * Common Development and Distribution License (the "License"). 62be60c5eSraf * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21e8031f0aSraf 227c478bd9Sstevel@tonic-gate /* 2341efec22Sraf * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 307c478bd9Sstevel@tonic-gate 317c478bd9Sstevel@tonic-gate #include "lint.h" 327c478bd9Sstevel@tonic-gate #include "thr_uberdata.h" 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate /* 357c478bd9Sstevel@tonic-gate * This mutex is initialized to be held by lwp#1. 367c478bd9Sstevel@tonic-gate * It is used to block a thread that has returned from a mutex_lock() 37*883492d5Sraf * of a LOCK_PRIO_INHERIT mutex with an unrecoverable error. 387c478bd9Sstevel@tonic-gate */ 397c478bd9Sstevel@tonic-gate mutex_t stall_mutex = DEFAULTMUTEX; 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate static int shared_mutex_held(mutex_t *); 42*883492d5Sraf static int mutex_unlock_internal(mutex_t *, int); 43*883492d5Sraf static int mutex_queuelock_adaptive(mutex_t *); 44*883492d5Sraf static void mutex_wakeup_all(mutex_t *); 457c478bd9Sstevel@tonic-gate 467c478bd9Sstevel@tonic-gate /* 477c478bd9Sstevel@tonic-gate * Lock statistics support functions. 487c478bd9Sstevel@tonic-gate */ 497c478bd9Sstevel@tonic-gate void 507c478bd9Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp) 517c478bd9Sstevel@tonic-gate { 527c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_lock); 537c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = gethrtime(); 547c478bd9Sstevel@tonic-gate } 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate hrtime_t 577c478bd9Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp) 587c478bd9Sstevel@tonic-gate { 597c478bd9Sstevel@tonic-gate hrtime_t now = gethrtime(); 607c478bd9Sstevel@tonic-gate 617c478bd9Sstevel@tonic-gate if (msp->mutex_begin_hold) 627c478bd9Sstevel@tonic-gate msp->mutex_hold_time += now - msp->mutex_begin_hold; 637c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = 0; 647c478bd9Sstevel@tonic-gate return (now); 657c478bd9Sstevel@tonic-gate } 667c478bd9Sstevel@tonic-gate 677c478bd9Sstevel@tonic-gate /* 687c478bd9Sstevel@tonic-gate * Called once at library initialization. 697c478bd9Sstevel@tonic-gate */ 707c478bd9Sstevel@tonic-gate void 717c478bd9Sstevel@tonic-gate mutex_setup(void) 727c478bd9Sstevel@tonic-gate { 737c478bd9Sstevel@tonic-gate if (set_lock_byte(&stall_mutex.mutex_lockw)) 747c478bd9Sstevel@tonic-gate thr_panic("mutex_setup() cannot acquire stall_mutex"); 757c478bd9Sstevel@tonic-gate stall_mutex.mutex_owner = (uintptr_t)curthread; 767c478bd9Sstevel@tonic-gate } 777c478bd9Sstevel@tonic-gate 787c478bd9Sstevel@tonic-gate /* 797c478bd9Sstevel@tonic-gate * The default spin counts of 1000 and 500 are experimentally determined. 807c478bd9Sstevel@tonic-gate * On sun4u machines with any number of processors they could be raised 817c478bd9Sstevel@tonic-gate * to 10,000 but that (experimentally) makes almost no difference. 827c478bd9Sstevel@tonic-gate * The environment variables: 837c478bd9Sstevel@tonic-gate * _THREAD_ADAPTIVE_SPIN=count 847c478bd9Sstevel@tonic-gate * _THREAD_RELEASE_SPIN=count 857c478bd9Sstevel@tonic-gate * can be used to override and set the counts in the range [0 .. 1,000,000]. 867c478bd9Sstevel@tonic-gate */ 877c478bd9Sstevel@tonic-gate int thread_adaptive_spin = 1000; 887c478bd9Sstevel@tonic-gate uint_t thread_max_spinners = 100; 897c478bd9Sstevel@tonic-gate int thread_release_spin = 500; 907c478bd9Sstevel@tonic-gate int thread_queue_verify = 0; 917c478bd9Sstevel@tonic-gate static int ncpus; 927c478bd9Sstevel@tonic-gate 937c478bd9Sstevel@tonic-gate /* 947c478bd9Sstevel@tonic-gate * Distinguish spinning for queue locks from spinning for regular locks. 957c478bd9Sstevel@tonic-gate * The environment variable: 967c478bd9Sstevel@tonic-gate * _THREAD_QUEUE_SPIN=count 977c478bd9Sstevel@tonic-gate * can be used to override and set the count in the range [0 .. 1,000,000]. 987c478bd9Sstevel@tonic-gate * There is no release spin concept for queue locks. 997c478bd9Sstevel@tonic-gate */ 1007c478bd9Sstevel@tonic-gate int thread_queue_spin = 1000; 1017c478bd9Sstevel@tonic-gate 1027c478bd9Sstevel@tonic-gate /* 1037c478bd9Sstevel@tonic-gate * Use the otherwise-unused 'mutex_ownerpid' field of a USYNC_THREAD 1047c478bd9Sstevel@tonic-gate * mutex to be a count of adaptive spins in progress. 1057c478bd9Sstevel@tonic-gate */ 1067c478bd9Sstevel@tonic-gate #define mutex_spinners mutex_ownerpid 1077c478bd9Sstevel@tonic-gate 108*883492d5Sraf #define ALL_ATTRIBUTES \ 109*883492d5Sraf (LOCK_RECURSIVE | LOCK_ERRORCHECK | \ 110*883492d5Sraf LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT | \ 111*883492d5Sraf LOCK_ROBUST) 1127c478bd9Sstevel@tonic-gate 1137c478bd9Sstevel@tonic-gate /* 114*883492d5Sraf * 'type' can be one of USYNC_THREAD, USYNC_PROCESS, or USYNC_PROCESS_ROBUST, 115*883492d5Sraf * augmented by zero or more the flags: 116*883492d5Sraf * LOCK_RECURSIVE 117*883492d5Sraf * LOCK_ERRORCHECK 118*883492d5Sraf * LOCK_PRIO_INHERIT 119*883492d5Sraf * LOCK_PRIO_PROTECT 120*883492d5Sraf * LOCK_ROBUST 1217c478bd9Sstevel@tonic-gate */ 1227c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_init = __mutex_init 1237c478bd9Sstevel@tonic-gate #pragma weak mutex_init = __mutex_init 1247c478bd9Sstevel@tonic-gate #pragma weak _mutex_init = __mutex_init 1257c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 1267c478bd9Sstevel@tonic-gate int 1277c478bd9Sstevel@tonic-gate __mutex_init(mutex_t *mp, int type, void *arg) 1287c478bd9Sstevel@tonic-gate { 129*883492d5Sraf int basetype = (type & ~ALL_ATTRIBUTES); 130*883492d5Sraf int error = 0; 131*883492d5Sraf 132*883492d5Sraf if (basetype == USYNC_PROCESS_ROBUST) { 133*883492d5Sraf /* 134*883492d5Sraf * USYNC_PROCESS_ROBUST is a deprecated historical type. 135*883492d5Sraf * We change it into (USYNC_PROCESS | LOCK_ROBUST) but 136*883492d5Sraf * retain the USYNC_PROCESS_ROBUST flag so we can return 137*883492d5Sraf * ELOCKUNMAPPED when necessary (only USYNC_PROCESS_ROBUST 138*883492d5Sraf * mutexes will ever draw ELOCKUNMAPPED). 139*883492d5Sraf */ 140*883492d5Sraf type |= (USYNC_PROCESS | LOCK_ROBUST); 141*883492d5Sraf basetype = USYNC_PROCESS; 142*883492d5Sraf } 1437c478bd9Sstevel@tonic-gate 144*883492d5Sraf if (!(basetype == USYNC_THREAD || basetype == USYNC_PROCESS) || 145*883492d5Sraf (type & (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) 146*883492d5Sraf == (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) { 147*883492d5Sraf error = EINVAL; 148*883492d5Sraf } else if (type & LOCK_ROBUST) { 149*883492d5Sraf /* 150*883492d5Sraf * Callers of mutex_init() with the LOCK_ROBUST attribute 151*883492d5Sraf * are required to pass an initially all-zero mutex. 152*883492d5Sraf * Multiple calls to mutex_init() are allowed; all but 153*883492d5Sraf * the first return EBUSY. A call to mutex_init() is 154*883492d5Sraf * allowed to make an inconsistent robust lock consistent 155*883492d5Sraf * (for historical usage, even though the proper interface 156*883492d5Sraf * for this is mutex_consistent()). Note that we use 157*883492d5Sraf * atomic_or_16() to set the LOCK_INITED flag so as 158*883492d5Sraf * not to disturb surrounding bits (LOCK_OWNERDEAD, etc). 159*883492d5Sraf */ 160*883492d5Sraf extern void _atomic_or_16(volatile uint16_t *, uint16_t); 161*883492d5Sraf if (!(mp->mutex_flag & LOCK_INITED)) { 162*883492d5Sraf mp->mutex_type = (uint8_t)type; 163*883492d5Sraf _atomic_or_16(&mp->mutex_flag, LOCK_INITED); 164*883492d5Sraf mp->mutex_magic = MUTEX_MAGIC; 165*883492d5Sraf } else if (type != mp->mutex_type || 166*883492d5Sraf ((type & LOCK_PRIO_PROTECT) && 167*883492d5Sraf mp->mutex_ceiling != (*(int *)arg))) { 168*883492d5Sraf error = EINVAL; 169*883492d5Sraf } else if (__mutex_consistent(mp) != 0) { 170*883492d5Sraf error = EBUSY; 171*883492d5Sraf } 172*883492d5Sraf /* register a process robust mutex with the kernel */ 173*883492d5Sraf if (basetype == USYNC_PROCESS) 174*883492d5Sraf register_lock(mp); 175*883492d5Sraf } else { 1767c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 1777c478bd9Sstevel@tonic-gate mp->mutex_type = (uint8_t)type; 1787c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 1797c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 180*883492d5Sraf } 181*883492d5Sraf 182*883492d5Sraf if (error == 0 && (type & LOCK_PRIO_PROTECT)) 183*883492d5Sraf mp->mutex_ceiling = (uint8_t)(*(int *)arg); 184*883492d5Sraf 1857c478bd9Sstevel@tonic-gate return (error); 1867c478bd9Sstevel@tonic-gate } 1877c478bd9Sstevel@tonic-gate 1887c478bd9Sstevel@tonic-gate /* 1897c478bd9Sstevel@tonic-gate * Delete mp from list of ceil mutexes owned by curthread. 1907c478bd9Sstevel@tonic-gate * Return 1 if the head of the chain was updated. 1917c478bd9Sstevel@tonic-gate */ 1927c478bd9Sstevel@tonic-gate int 1937c478bd9Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp) 1947c478bd9Sstevel@tonic-gate { 1957c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1967c478bd9Sstevel@tonic-gate mxchain_t **mcpp; 1977c478bd9Sstevel@tonic-gate mxchain_t *mcp; 1987c478bd9Sstevel@tonic-gate 1997c478bd9Sstevel@tonic-gate mcpp = &self->ul_mxchain; 2007c478bd9Sstevel@tonic-gate while ((*mcpp)->mxchain_mx != mp) 2017c478bd9Sstevel@tonic-gate mcpp = &(*mcpp)->mxchain_next; 2027c478bd9Sstevel@tonic-gate mcp = *mcpp; 2037c478bd9Sstevel@tonic-gate *mcpp = mcp->mxchain_next; 2047c478bd9Sstevel@tonic-gate lfree(mcp, sizeof (*mcp)); 2057c478bd9Sstevel@tonic-gate return (mcpp == &self->ul_mxchain); 2067c478bd9Sstevel@tonic-gate } 2077c478bd9Sstevel@tonic-gate 2087c478bd9Sstevel@tonic-gate /* 2097c478bd9Sstevel@tonic-gate * Add mp to head of list of ceil mutexes owned by curthread. 2107c478bd9Sstevel@tonic-gate * Return ENOMEM if no memory could be allocated. 2117c478bd9Sstevel@tonic-gate */ 2127c478bd9Sstevel@tonic-gate int 2137c478bd9Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp) 2147c478bd9Sstevel@tonic-gate { 2157c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2167c478bd9Sstevel@tonic-gate mxchain_t *mcp; 2177c478bd9Sstevel@tonic-gate 2187c478bd9Sstevel@tonic-gate if ((mcp = lmalloc(sizeof (*mcp))) == NULL) 2197c478bd9Sstevel@tonic-gate return (ENOMEM); 2207c478bd9Sstevel@tonic-gate mcp->mxchain_mx = mp; 2217c478bd9Sstevel@tonic-gate mcp->mxchain_next = self->ul_mxchain; 2227c478bd9Sstevel@tonic-gate self->ul_mxchain = mcp; 2237c478bd9Sstevel@tonic-gate return (0); 2247c478bd9Sstevel@tonic-gate } 2257c478bd9Sstevel@tonic-gate 2267c478bd9Sstevel@tonic-gate /* 2277c478bd9Sstevel@tonic-gate * Inherit priority from ceiling. The inheritance impacts the effective 2287c478bd9Sstevel@tonic-gate * priority, not the assigned priority. See _thread_setschedparam_main(). 2297c478bd9Sstevel@tonic-gate */ 2307c478bd9Sstevel@tonic-gate void 2317c478bd9Sstevel@tonic-gate _ceil_prio_inherit(int ceil) 2327c478bd9Sstevel@tonic-gate { 2337c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2347c478bd9Sstevel@tonic-gate struct sched_param param; 2357c478bd9Sstevel@tonic-gate 2367c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 2377c478bd9Sstevel@tonic-gate param.sched_priority = ceil; 2387c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2397c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) { 2407c478bd9Sstevel@tonic-gate /* 2417c478bd9Sstevel@tonic-gate * Panic since unclear what error code to return. 2427c478bd9Sstevel@tonic-gate * If we do return the error codes returned by above 2437c478bd9Sstevel@tonic-gate * called routine, update the man page... 2447c478bd9Sstevel@tonic-gate */ 2457c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2467c478bd9Sstevel@tonic-gate } 2477c478bd9Sstevel@tonic-gate } 2487c478bd9Sstevel@tonic-gate 2497c478bd9Sstevel@tonic-gate /* 2507c478bd9Sstevel@tonic-gate * Waive inherited ceiling priority. Inherit from head of owned ceiling locks 2517c478bd9Sstevel@tonic-gate * if holding at least one ceiling lock. If no ceiling locks are held at this 2527c478bd9Sstevel@tonic-gate * point, disinherit completely, reverting back to assigned priority. 2537c478bd9Sstevel@tonic-gate */ 2547c478bd9Sstevel@tonic-gate void 2557c478bd9Sstevel@tonic-gate _ceil_prio_waive(void) 2567c478bd9Sstevel@tonic-gate { 2577c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2587c478bd9Sstevel@tonic-gate struct sched_param param; 2597c478bd9Sstevel@tonic-gate 2607c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 2617c478bd9Sstevel@tonic-gate if (self->ul_mxchain == NULL) { 2627c478bd9Sstevel@tonic-gate /* 2637c478bd9Sstevel@tonic-gate * No ceil locks held. Zero the epri, revert back to ul_pri. 2647c478bd9Sstevel@tonic-gate * Since thread's hash lock is not held, one cannot just 2657c478bd9Sstevel@tonic-gate * read ul_pri here...do it in the called routine... 2667c478bd9Sstevel@tonic-gate */ 2677c478bd9Sstevel@tonic-gate param.sched_priority = self->ul_pri; /* ignored */ 2687c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2697c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_DISINHERIT)) 2707c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2717c478bd9Sstevel@tonic-gate } else { 2727c478bd9Sstevel@tonic-gate /* 2737c478bd9Sstevel@tonic-gate * Set priority to that of the mutex at the head 2747c478bd9Sstevel@tonic-gate * of the ceilmutex chain. 2757c478bd9Sstevel@tonic-gate */ 2767c478bd9Sstevel@tonic-gate param.sched_priority = 2777c478bd9Sstevel@tonic-gate self->ul_mxchain->mxchain_mx->mutex_ceiling; 2787c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2797c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) 2807c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2817c478bd9Sstevel@tonic-gate } 2827c478bd9Sstevel@tonic-gate } 2837c478bd9Sstevel@tonic-gate 2847c478bd9Sstevel@tonic-gate /* 2857c478bd9Sstevel@tonic-gate * Non-preemptive spin locks. Used by queue_lock(). 2867c478bd9Sstevel@tonic-gate * No lock statistics are gathered for these locks. 2877c478bd9Sstevel@tonic-gate */ 2887c478bd9Sstevel@tonic-gate void 2897c478bd9Sstevel@tonic-gate spin_lock_set(mutex_t *mp) 2907c478bd9Sstevel@tonic-gate { 2917c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2927c478bd9Sstevel@tonic-gate 2937c478bd9Sstevel@tonic-gate no_preempt(self); 2947c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 2957c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2967c478bd9Sstevel@tonic-gate return; 2977c478bd9Sstevel@tonic-gate } 2987c478bd9Sstevel@tonic-gate /* 2997c478bd9Sstevel@tonic-gate * Spin for a while, attempting to acquire the lock. 3007c478bd9Sstevel@tonic-gate */ 3017c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin != UINT_MAX) 3027c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin++; 3037c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 3047c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 3057c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 3067c478bd9Sstevel@tonic-gate return; 3077c478bd9Sstevel@tonic-gate } 3087c478bd9Sstevel@tonic-gate /* 3097c478bd9Sstevel@tonic-gate * Try harder if we were previously at a no premption level. 3107c478bd9Sstevel@tonic-gate */ 3117c478bd9Sstevel@tonic-gate if (self->ul_preempt > 1) { 3127c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin2 != UINT_MAX) 3137c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin2++; 3147c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 3157c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 3167c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 3177c478bd9Sstevel@tonic-gate return; 3187c478bd9Sstevel@tonic-gate } 3197c478bd9Sstevel@tonic-gate } 3207c478bd9Sstevel@tonic-gate /* 3217c478bd9Sstevel@tonic-gate * Give up and block in the kernel for the mutex. 3227c478bd9Sstevel@tonic-gate */ 3237c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_sleep != UINT_MAX) 3247c478bd9Sstevel@tonic-gate self->ul_spin_lock_sleep++; 3257c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_timedlock(mp, NULL); 3267c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 3277c478bd9Sstevel@tonic-gate } 3287c478bd9Sstevel@tonic-gate 3297c478bd9Sstevel@tonic-gate void 3307c478bd9Sstevel@tonic-gate spin_lock_clear(mutex_t *mp) 3317c478bd9Sstevel@tonic-gate { 3327c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3337c478bd9Sstevel@tonic-gate 3347c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 33541efec22Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) { 336*883492d5Sraf (void) ___lwp_mutex_wakeup(mp, 0); 3377c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_wakeup != UINT_MAX) 3387c478bd9Sstevel@tonic-gate self->ul_spin_lock_wakeup++; 3397c478bd9Sstevel@tonic-gate } 3407c478bd9Sstevel@tonic-gate preempt(self); 3417c478bd9Sstevel@tonic-gate } 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate /* 3447c478bd9Sstevel@tonic-gate * Allocate the sleep queue hash table. 3457c478bd9Sstevel@tonic-gate */ 3467c478bd9Sstevel@tonic-gate void 3477c478bd9Sstevel@tonic-gate queue_alloc(void) 3487c478bd9Sstevel@tonic-gate { 3497c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3507c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 351*883492d5Sraf mutex_t *mp; 3527c478bd9Sstevel@tonic-gate void *data; 3537c478bd9Sstevel@tonic-gate int i; 3547c478bd9Sstevel@tonic-gate 3557c478bd9Sstevel@tonic-gate /* 3567c478bd9Sstevel@tonic-gate * No locks are needed; we call here only when single-threaded. 3577c478bd9Sstevel@tonic-gate */ 3587c478bd9Sstevel@tonic-gate ASSERT(self == udp->ulwp_one); 3597c478bd9Sstevel@tonic-gate ASSERT(!udp->uberflags.uf_mt); 3607c478bd9Sstevel@tonic-gate if ((data = _private_mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t), 3617c478bd9Sstevel@tonic-gate PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0)) 3627c478bd9Sstevel@tonic-gate == MAP_FAILED) 3637c478bd9Sstevel@tonic-gate thr_panic("cannot allocate thread queue_head table"); 3647c478bd9Sstevel@tonic-gate udp->queue_head = (queue_head_t *)data; 365*883492d5Sraf for (i = 0; i < 2 * QHASHSIZE; i++) { 366*883492d5Sraf mp = &udp->queue_head[i].qh_lock; 367*883492d5Sraf mp->mutex_flag = LOCK_INITED; 368*883492d5Sraf mp->mutex_magic = MUTEX_MAGIC; 369*883492d5Sraf } 3707c478bd9Sstevel@tonic-gate } 3717c478bd9Sstevel@tonic-gate 3727c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 3737c478bd9Sstevel@tonic-gate 3747c478bd9Sstevel@tonic-gate /* 3757c478bd9Sstevel@tonic-gate * Debugging: verify correctness of a sleep queue. 3767c478bd9Sstevel@tonic-gate */ 3777c478bd9Sstevel@tonic-gate void 3787c478bd9Sstevel@tonic-gate QVERIFY(queue_head_t *qp) 3797c478bd9Sstevel@tonic-gate { 3807c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3817c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 3827c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 3837c478bd9Sstevel@tonic-gate ulwp_t *prev; 3847c478bd9Sstevel@tonic-gate uint_t index; 3857c478bd9Sstevel@tonic-gate uint32_t cnt = 0; 3867c478bd9Sstevel@tonic-gate char qtype; 3877c478bd9Sstevel@tonic-gate void *wchan; 3887c478bd9Sstevel@tonic-gate 3897c478bd9Sstevel@tonic-gate ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE); 3907c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 3917c478bd9Sstevel@tonic-gate ASSERT((qp->qh_head != NULL && qp->qh_tail != NULL) || 3927c478bd9Sstevel@tonic-gate (qp->qh_head == NULL && qp->qh_tail == NULL)); 3937c478bd9Sstevel@tonic-gate if (!thread_queue_verify) 3947c478bd9Sstevel@tonic-gate return; 3957c478bd9Sstevel@tonic-gate /* real expensive stuff, only for _THREAD_QUEUE_VERIFY */ 3967c478bd9Sstevel@tonic-gate qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV; 3977c478bd9Sstevel@tonic-gate for (prev = NULL, ulwp = qp->qh_head; ulwp != NULL; 3987c478bd9Sstevel@tonic-gate prev = ulwp, ulwp = ulwp->ul_link, cnt++) { 3997c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_qtype == qtype); 4007c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_wchan != NULL); 4017c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq == qp); 4027c478bd9Sstevel@tonic-gate wchan = ulwp->ul_wchan; 4037c478bd9Sstevel@tonic-gate index = QUEUE_HASH(wchan, qtype); 4047c478bd9Sstevel@tonic-gate ASSERT(&udp->queue_head[index] == qp); 4057c478bd9Sstevel@tonic-gate } 4067c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == prev); 4077c478bd9Sstevel@tonic-gate ASSERT(qp->qh_qlen == cnt); 4087c478bd9Sstevel@tonic-gate } 4097c478bd9Sstevel@tonic-gate 4107c478bd9Sstevel@tonic-gate #else /* THREAD_DEBUG */ 4117c478bd9Sstevel@tonic-gate 4127c478bd9Sstevel@tonic-gate #define QVERIFY(qp) 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate #endif /* THREAD_DEBUG */ 4157c478bd9Sstevel@tonic-gate 4167c478bd9Sstevel@tonic-gate /* 4177c478bd9Sstevel@tonic-gate * Acquire a queue head. 4187c478bd9Sstevel@tonic-gate */ 4197c478bd9Sstevel@tonic-gate queue_head_t * 4207c478bd9Sstevel@tonic-gate queue_lock(void *wchan, int qtype) 4217c478bd9Sstevel@tonic-gate { 4227c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 4237c478bd9Sstevel@tonic-gate queue_head_t *qp; 4247c478bd9Sstevel@tonic-gate 4257c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate /* 4287c478bd9Sstevel@tonic-gate * It is possible that we could be called while still single-threaded. 4297c478bd9Sstevel@tonic-gate * If so, we call queue_alloc() to allocate the queue_head[] array. 4307c478bd9Sstevel@tonic-gate */ 4317c478bd9Sstevel@tonic-gate if ((qp = udp->queue_head) == NULL) { 4327c478bd9Sstevel@tonic-gate queue_alloc(); 4337c478bd9Sstevel@tonic-gate qp = udp->queue_head; 4347c478bd9Sstevel@tonic-gate } 4357c478bd9Sstevel@tonic-gate qp += QUEUE_HASH(wchan, qtype); 4367c478bd9Sstevel@tonic-gate spin_lock_set(&qp->qh_lock); 4377c478bd9Sstevel@tonic-gate /* 4387c478bd9Sstevel@tonic-gate * At once per nanosecond, qh_lockcount will wrap after 512 years. 4397c478bd9Sstevel@tonic-gate * Were we to care about this, we could peg the value at UINT64_MAX. 4407c478bd9Sstevel@tonic-gate */ 4417c478bd9Sstevel@tonic-gate qp->qh_lockcount++; 4427c478bd9Sstevel@tonic-gate QVERIFY(qp); 4437c478bd9Sstevel@tonic-gate return (qp); 4447c478bd9Sstevel@tonic-gate } 4457c478bd9Sstevel@tonic-gate 4467c478bd9Sstevel@tonic-gate /* 4477c478bd9Sstevel@tonic-gate * Release a queue head. 4487c478bd9Sstevel@tonic-gate */ 4497c478bd9Sstevel@tonic-gate void 4507c478bd9Sstevel@tonic-gate queue_unlock(queue_head_t *qp) 4517c478bd9Sstevel@tonic-gate { 4527c478bd9Sstevel@tonic-gate QVERIFY(qp); 4537c478bd9Sstevel@tonic-gate spin_lock_clear(&qp->qh_lock); 4547c478bd9Sstevel@tonic-gate } 4557c478bd9Sstevel@tonic-gate 4567c478bd9Sstevel@tonic-gate /* 4577c478bd9Sstevel@tonic-gate * For rwlock queueing, we must queue writers ahead of readers of the 4587c478bd9Sstevel@tonic-gate * same priority. We do this by making writers appear to have a half 4597c478bd9Sstevel@tonic-gate * point higher priority for purposes of priority comparisons below. 4607c478bd9Sstevel@tonic-gate */ 4617c478bd9Sstevel@tonic-gate #define CMP_PRIO(ulwp) ((real_priority(ulwp) << 1) + (ulwp)->ul_writer) 4627c478bd9Sstevel@tonic-gate 4637c478bd9Sstevel@tonic-gate void 4647c478bd9Sstevel@tonic-gate enqueue(queue_head_t *qp, ulwp_t *ulwp, void *wchan, int qtype) 4657c478bd9Sstevel@tonic-gate { 4667c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 4677c478bd9Sstevel@tonic-gate ulwp_t *next; 4687c478bd9Sstevel@tonic-gate int pri = CMP_PRIO(ulwp); 4697c478bd9Sstevel@tonic-gate int force_fifo = (qtype & FIFOQ); 4707c478bd9Sstevel@tonic-gate int do_fifo; 4717c478bd9Sstevel@tonic-gate 4727c478bd9Sstevel@tonic-gate qtype &= ~FIFOQ; 4737c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 4747c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 4757c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq != qp); 4767c478bd9Sstevel@tonic-gate 4777c478bd9Sstevel@tonic-gate /* 4787c478bd9Sstevel@tonic-gate * LIFO queue ordering is unfair and can lead to starvation, 4797c478bd9Sstevel@tonic-gate * but it gives better performance for heavily contended locks. 4807c478bd9Sstevel@tonic-gate * We use thread_queue_fifo (range is 0..8) to determine 4817c478bd9Sstevel@tonic-gate * the frequency of FIFO vs LIFO queuing: 4827c478bd9Sstevel@tonic-gate * 0 : every 256th time (almost always LIFO) 4837c478bd9Sstevel@tonic-gate * 1 : every 128th time 4847c478bd9Sstevel@tonic-gate * 2 : every 64th time 4857c478bd9Sstevel@tonic-gate * 3 : every 32nd time 4867c478bd9Sstevel@tonic-gate * 4 : every 16th time (the default value, mostly LIFO) 4877c478bd9Sstevel@tonic-gate * 5 : every 8th time 4887c478bd9Sstevel@tonic-gate * 6 : every 4th time 4897c478bd9Sstevel@tonic-gate * 7 : every 2nd time 4907c478bd9Sstevel@tonic-gate * 8 : every time (never LIFO, always FIFO) 4917c478bd9Sstevel@tonic-gate * Note that there is always some degree of FIFO ordering. 4927c478bd9Sstevel@tonic-gate * This breaks live lock conditions that occur in applications 4937c478bd9Sstevel@tonic-gate * that are written assuming (incorrectly) that threads acquire 4947c478bd9Sstevel@tonic-gate * locks fairly, that is, in roughly round-robin order. 4957c478bd9Sstevel@tonic-gate * In any event, the queue is maintained in priority order. 4967c478bd9Sstevel@tonic-gate * 4977c478bd9Sstevel@tonic-gate * If we are given the FIFOQ flag in qtype, fifo queueing is forced. 4987c478bd9Sstevel@tonic-gate * SUSV3 requires this for semaphores. 4997c478bd9Sstevel@tonic-gate */ 5007c478bd9Sstevel@tonic-gate do_fifo = (force_fifo || 5017c478bd9Sstevel@tonic-gate ((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0); 5027c478bd9Sstevel@tonic-gate 5037c478bd9Sstevel@tonic-gate if (qp->qh_head == NULL) { 5047c478bd9Sstevel@tonic-gate /* 5057c478bd9Sstevel@tonic-gate * The queue is empty. LIFO/FIFO doesn't matter. 5067c478bd9Sstevel@tonic-gate */ 5077c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == NULL); 5087c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 5097c478bd9Sstevel@tonic-gate } else if (do_fifo) { 5107c478bd9Sstevel@tonic-gate /* 5117c478bd9Sstevel@tonic-gate * Enqueue after the last thread whose priority is greater 5127c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 5137c478bd9Sstevel@tonic-gate * Attempt first to go directly onto the tail of the queue. 5147c478bd9Sstevel@tonic-gate */ 5157c478bd9Sstevel@tonic-gate if (pri <= CMP_PRIO(qp->qh_tail)) 5167c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_tail->ul_link; 5177c478bd9Sstevel@tonic-gate else { 5187c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 5197c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 5207c478bd9Sstevel@tonic-gate if (pri > CMP_PRIO(next)) 5217c478bd9Sstevel@tonic-gate break; 5227c478bd9Sstevel@tonic-gate } 5237c478bd9Sstevel@tonic-gate } else { 5247c478bd9Sstevel@tonic-gate /* 5257c478bd9Sstevel@tonic-gate * Enqueue before the first thread whose priority is less 5267c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 5277c478bd9Sstevel@tonic-gate * Hopefully we can go directly onto the head of the queue. 5287c478bd9Sstevel@tonic-gate */ 5297c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 5307c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 5317c478bd9Sstevel@tonic-gate if (pri >= CMP_PRIO(next)) 5327c478bd9Sstevel@tonic-gate break; 5337c478bd9Sstevel@tonic-gate } 5347c478bd9Sstevel@tonic-gate if ((ulwp->ul_link = *ulwpp) == NULL) 5357c478bd9Sstevel@tonic-gate qp->qh_tail = ulwp; 5367c478bd9Sstevel@tonic-gate *ulwpp = ulwp; 5377c478bd9Sstevel@tonic-gate 5387c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = qp; 5397c478bd9Sstevel@tonic-gate ulwp->ul_wchan = wchan; 5407c478bd9Sstevel@tonic-gate ulwp->ul_qtype = qtype; 5417c478bd9Sstevel@tonic-gate if (qp->qh_qmax < ++qp->qh_qlen) 5427c478bd9Sstevel@tonic-gate qp->qh_qmax = qp->qh_qlen; 5437c478bd9Sstevel@tonic-gate } 5447c478bd9Sstevel@tonic-gate 5457c478bd9Sstevel@tonic-gate /* 5467c478bd9Sstevel@tonic-gate * Return a pointer to the queue slot of the 5477c478bd9Sstevel@tonic-gate * highest priority thread on the queue. 5487c478bd9Sstevel@tonic-gate * On return, prevp, if not NULL, will contain a pointer 5497c478bd9Sstevel@tonic-gate * to the thread's predecessor on the queue 5507c478bd9Sstevel@tonic-gate */ 5517c478bd9Sstevel@tonic-gate static ulwp_t ** 5527c478bd9Sstevel@tonic-gate queue_slot(queue_head_t *qp, void *wchan, int *more, ulwp_t **prevp) 5537c478bd9Sstevel@tonic-gate { 5547c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 5557c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 5567c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 5577c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 5587c478bd9Sstevel@tonic-gate ulwp_t *susprev; 5597c478bd9Sstevel@tonic-gate 5607c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 5617c478bd9Sstevel@tonic-gate 5627c478bd9Sstevel@tonic-gate /* 5637c478bd9Sstevel@tonic-gate * Find a waiter on the sleep queue. 5647c478bd9Sstevel@tonic-gate */ 5657c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 5667c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 5677c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 5687c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 5697c478bd9Sstevel@tonic-gate break; 5707c478bd9Sstevel@tonic-gate /* 5717c478bd9Sstevel@tonic-gate * Try not to return a suspended thread. 5727c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 5737c478bd9Sstevel@tonic-gate */ 5747c478bd9Sstevel@tonic-gate if (suspp == NULL) { 5757c478bd9Sstevel@tonic-gate suspp = ulwpp; 5767c478bd9Sstevel@tonic-gate susprev = prev; 5777c478bd9Sstevel@tonic-gate } 5787c478bd9Sstevel@tonic-gate } 5797c478bd9Sstevel@tonic-gate } 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 5827c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 5837c478bd9Sstevel@tonic-gate prev = susprev; 5847c478bd9Sstevel@tonic-gate suspp = NULL; 5857c478bd9Sstevel@tonic-gate } 5867c478bd9Sstevel@tonic-gate if (ulwp == NULL) { 5877c478bd9Sstevel@tonic-gate if (more != NULL) 5887c478bd9Sstevel@tonic-gate *more = 0; 5897c478bd9Sstevel@tonic-gate return (NULL); 5907c478bd9Sstevel@tonic-gate } 5917c478bd9Sstevel@tonic-gate 5927c478bd9Sstevel@tonic-gate if (prevp != NULL) 5937c478bd9Sstevel@tonic-gate *prevp = prev; 5947c478bd9Sstevel@tonic-gate if (more == NULL) 5957c478bd9Sstevel@tonic-gate return (ulwpp); 5967c478bd9Sstevel@tonic-gate 5977c478bd9Sstevel@tonic-gate /* 5987c478bd9Sstevel@tonic-gate * Scan the remainder of the queue for another waiter. 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate if (suspp != NULL) { 6017c478bd9Sstevel@tonic-gate *more = 1; 6027c478bd9Sstevel@tonic-gate return (ulwpp); 6037c478bd9Sstevel@tonic-gate } 6047c478bd9Sstevel@tonic-gate for (ulwp = ulwp->ul_link; ulwp != NULL; ulwp = ulwp->ul_link) { 6057c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 6067c478bd9Sstevel@tonic-gate *more = 1; 6077c478bd9Sstevel@tonic-gate return (ulwpp); 6087c478bd9Sstevel@tonic-gate } 6097c478bd9Sstevel@tonic-gate } 6107c478bd9Sstevel@tonic-gate 6117c478bd9Sstevel@tonic-gate *more = 0; 6127c478bd9Sstevel@tonic-gate return (ulwpp); 6137c478bd9Sstevel@tonic-gate } 6147c478bd9Sstevel@tonic-gate 6157c478bd9Sstevel@tonic-gate ulwp_t * 61641efec22Sraf queue_unlink(queue_head_t *qp, ulwp_t **ulwpp, ulwp_t *prev) 6177c478bd9Sstevel@tonic-gate { 6187c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 6197c478bd9Sstevel@tonic-gate 6207c478bd9Sstevel@tonic-gate ulwp = *ulwpp; 6217c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 6227c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 6237c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 6247c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 6257c478bd9Sstevel@tonic-gate qp->qh_qlen--; 6267c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 6277c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 6287c478bd9Sstevel@tonic-gate 6297c478bd9Sstevel@tonic-gate return (ulwp); 6307c478bd9Sstevel@tonic-gate } 6317c478bd9Sstevel@tonic-gate 63241efec22Sraf ulwp_t * 63341efec22Sraf dequeue(queue_head_t *qp, void *wchan, int *more) 63441efec22Sraf { 63541efec22Sraf ulwp_t **ulwpp; 63641efec22Sraf ulwp_t *prev; 63741efec22Sraf 63841efec22Sraf if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL) 63941efec22Sraf return (NULL); 64041efec22Sraf return (queue_unlink(qp, ulwpp, prev)); 64141efec22Sraf } 64241efec22Sraf 6437c478bd9Sstevel@tonic-gate /* 6447c478bd9Sstevel@tonic-gate * Return a pointer to the highest priority thread sleeping on wchan. 6457c478bd9Sstevel@tonic-gate */ 6467c478bd9Sstevel@tonic-gate ulwp_t * 6477c478bd9Sstevel@tonic-gate queue_waiter(queue_head_t *qp, void *wchan) 6487c478bd9Sstevel@tonic-gate { 6497c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 6507c478bd9Sstevel@tonic-gate 6517c478bd9Sstevel@tonic-gate if ((ulwpp = queue_slot(qp, wchan, NULL, NULL)) == NULL) 6527c478bd9Sstevel@tonic-gate return (NULL); 6537c478bd9Sstevel@tonic-gate return (*ulwpp); 6547c478bd9Sstevel@tonic-gate } 6557c478bd9Sstevel@tonic-gate 6567c478bd9Sstevel@tonic-gate uint8_t 6577c478bd9Sstevel@tonic-gate dequeue_self(queue_head_t *qp, void *wchan) 6587c478bd9Sstevel@tonic-gate { 6597c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 6607c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 6617c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 6627c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 6637c478bd9Sstevel@tonic-gate int found = 0; 6647c478bd9Sstevel@tonic-gate int more = 0; 6657c478bd9Sstevel@tonic-gate 6667c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 6677c478bd9Sstevel@tonic-gate 6687c478bd9Sstevel@tonic-gate /* find self on the sleep queue */ 6697c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 6707c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 6717c478bd9Sstevel@tonic-gate if (ulwp == self) { 6727c478bd9Sstevel@tonic-gate /* dequeue ourself */ 6737c478bd9Sstevel@tonic-gate ASSERT(self->ul_wchan == wchan); 67441efec22Sraf (void) queue_unlink(qp, ulwpp, prev); 6757c478bd9Sstevel@tonic-gate self->ul_cvmutex = NULL; 6767c478bd9Sstevel@tonic-gate self->ul_cv_wake = 0; 6777c478bd9Sstevel@tonic-gate found = 1; 6787c478bd9Sstevel@tonic-gate break; 6797c478bd9Sstevel@tonic-gate } 6807c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 6817c478bd9Sstevel@tonic-gate more = 1; 6827c478bd9Sstevel@tonic-gate } 6837c478bd9Sstevel@tonic-gate 6847c478bd9Sstevel@tonic-gate if (!found) 6857c478bd9Sstevel@tonic-gate thr_panic("dequeue_self(): curthread not found on queue"); 6867c478bd9Sstevel@tonic-gate 6877c478bd9Sstevel@tonic-gate if (more) 6887c478bd9Sstevel@tonic-gate return (1); 6897c478bd9Sstevel@tonic-gate 6907c478bd9Sstevel@tonic-gate /* scan the remainder of the queue for another waiter */ 6917c478bd9Sstevel@tonic-gate for (ulwp = *ulwpp; ulwp != NULL; ulwp = ulwp->ul_link) { 6927c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 6937c478bd9Sstevel@tonic-gate return (1); 6947c478bd9Sstevel@tonic-gate } 6957c478bd9Sstevel@tonic-gate 6967c478bd9Sstevel@tonic-gate return (0); 6977c478bd9Sstevel@tonic-gate } 6987c478bd9Sstevel@tonic-gate 6997c478bd9Sstevel@tonic-gate /* 7007c478bd9Sstevel@tonic-gate * Called from call_user_handler() and _thrp_suspend() to take 7017c478bd9Sstevel@tonic-gate * ourself off of our sleep queue so we can grab locks. 7027c478bd9Sstevel@tonic-gate */ 7037c478bd9Sstevel@tonic-gate void 7047c478bd9Sstevel@tonic-gate unsleep_self(void) 7057c478bd9Sstevel@tonic-gate { 7067c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 7077c478bd9Sstevel@tonic-gate queue_head_t *qp; 7087c478bd9Sstevel@tonic-gate 7097c478bd9Sstevel@tonic-gate /* 7107c478bd9Sstevel@tonic-gate * Calling enter_critical()/exit_critical() here would lead 7117c478bd9Sstevel@tonic-gate * to recursion. Just manipulate self->ul_critical directly. 7127c478bd9Sstevel@tonic-gate */ 7137c478bd9Sstevel@tonic-gate self->ul_critical++; 7147c478bd9Sstevel@tonic-gate while (self->ul_sleepq != NULL) { 7157c478bd9Sstevel@tonic-gate qp = queue_lock(self->ul_wchan, self->ul_qtype); 7167c478bd9Sstevel@tonic-gate /* 7177c478bd9Sstevel@tonic-gate * We may have been moved from a CV queue to a 7187c478bd9Sstevel@tonic-gate * mutex queue while we were attempting queue_lock(). 7197c478bd9Sstevel@tonic-gate * If so, just loop around and try again. 7207c478bd9Sstevel@tonic-gate * dequeue_self() clears self->ul_sleepq. 7217c478bd9Sstevel@tonic-gate */ 72241efec22Sraf if (qp == self->ul_sleepq) { 7237c478bd9Sstevel@tonic-gate (void) dequeue_self(qp, self->ul_wchan); 72441efec22Sraf self->ul_writer = 0; 72541efec22Sraf } 7267c478bd9Sstevel@tonic-gate queue_unlock(qp); 7277c478bd9Sstevel@tonic-gate } 7287c478bd9Sstevel@tonic-gate self->ul_critical--; 7297c478bd9Sstevel@tonic-gate } 7307c478bd9Sstevel@tonic-gate 7317c478bd9Sstevel@tonic-gate /* 7327c478bd9Sstevel@tonic-gate * Common code for calling the the ___lwp_mutex_timedlock() system call. 7337c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 7347c478bd9Sstevel@tonic-gate */ 735*883492d5Sraf static int 7367c478bd9Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp) 7377c478bd9Sstevel@tonic-gate { 7387c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 7397c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 740*883492d5Sraf int mtype = mp->mutex_type; 7417c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 742*883492d5Sraf int acquired; 7437c478bd9Sstevel@tonic-gate int error; 7447c478bd9Sstevel@tonic-gate 7457c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 7467c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 7477c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 7487c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 7497c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 7507c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 7517c478bd9Sstevel@tonic-gate } 7527c478bd9Sstevel@tonic-gate if (msp) { 7537c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 7547c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 7557c478bd9Sstevel@tonic-gate } 7567c478bd9Sstevel@tonic-gate 7577c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 7587c478bd9Sstevel@tonic-gate 7597c478bd9Sstevel@tonic-gate for (;;) { 760*883492d5Sraf /* 761*883492d5Sraf * A return value of EOWNERDEAD or ELOCKUNMAPPED 762*883492d5Sraf * means we successfully acquired the lock. 763*883492d5Sraf */ 764*883492d5Sraf if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0 && 765*883492d5Sraf error != EOWNERDEAD && error != ELOCKUNMAPPED) { 766*883492d5Sraf acquired = 0; 7677c478bd9Sstevel@tonic-gate break; 7687c478bd9Sstevel@tonic-gate } 7697c478bd9Sstevel@tonic-gate 770*883492d5Sraf if (mtype & USYNC_PROCESS) { 7717c478bd9Sstevel@tonic-gate /* 7727c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 7737c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 7747c478bd9Sstevel@tonic-gate */ 7757c478bd9Sstevel@tonic-gate enter_critical(self); 7767c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 7777c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 7787c478bd9Sstevel@tonic-gate exit_critical(self); 779*883492d5Sraf acquired = 1; 7807c478bd9Sstevel@tonic-gate break; 7817c478bd9Sstevel@tonic-gate } 7827c478bd9Sstevel@tonic-gate exit_critical(self); 7837c478bd9Sstevel@tonic-gate } else { 7847c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 785*883492d5Sraf acquired = 1; 7867c478bd9Sstevel@tonic-gate break; 7877c478bd9Sstevel@tonic-gate } 7887c478bd9Sstevel@tonic-gate } 7897c478bd9Sstevel@tonic-gate if (msp) 7907c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 7917c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 7927c478bd9Sstevel@tonic-gate self->ul_sp = 0; 7937c478bd9Sstevel@tonic-gate 794*883492d5Sraf if (acquired) { 795*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 796*883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 797*883492d5Sraf } else { 798*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 799*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 800*883492d5Sraf } 801*883492d5Sraf 8027c478bd9Sstevel@tonic-gate return (error); 8037c478bd9Sstevel@tonic-gate } 8047c478bd9Sstevel@tonic-gate 8057c478bd9Sstevel@tonic-gate /* 8067c478bd9Sstevel@tonic-gate * Common code for calling the ___lwp_mutex_trylock() system call. 8077c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 8087c478bd9Sstevel@tonic-gate */ 8097c478bd9Sstevel@tonic-gate int 8107c478bd9Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp) 8117c478bd9Sstevel@tonic-gate { 8127c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 8137c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 814*883492d5Sraf int mtype = mp->mutex_type; 8157c478bd9Sstevel@tonic-gate int error; 816*883492d5Sraf int acquired; 8177c478bd9Sstevel@tonic-gate 8187c478bd9Sstevel@tonic-gate for (;;) { 819*883492d5Sraf /* 820*883492d5Sraf * A return value of EOWNERDEAD or ELOCKUNMAPPED 821*883492d5Sraf * means we successfully acquired the lock. 822*883492d5Sraf */ 823*883492d5Sraf if ((error = ___lwp_mutex_trylock(mp)) != 0 && 824*883492d5Sraf error != EOWNERDEAD && error != ELOCKUNMAPPED) { 825*883492d5Sraf acquired = 0; 8267c478bd9Sstevel@tonic-gate break; 8277c478bd9Sstevel@tonic-gate } 8287c478bd9Sstevel@tonic-gate 829*883492d5Sraf if (mtype & USYNC_PROCESS) { 8307c478bd9Sstevel@tonic-gate /* 8317c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 8327c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 8337c478bd9Sstevel@tonic-gate */ 8347c478bd9Sstevel@tonic-gate enter_critical(self); 8357c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 8367c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 8377c478bd9Sstevel@tonic-gate exit_critical(self); 838*883492d5Sraf acquired = 1; 8397c478bd9Sstevel@tonic-gate break; 8407c478bd9Sstevel@tonic-gate } 8417c478bd9Sstevel@tonic-gate exit_critical(self); 8427c478bd9Sstevel@tonic-gate } else { 8437c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 844*883492d5Sraf acquired = 1; 8457c478bd9Sstevel@tonic-gate break; 8467c478bd9Sstevel@tonic-gate } 8477c478bd9Sstevel@tonic-gate } 8487c478bd9Sstevel@tonic-gate 849*883492d5Sraf if (acquired) { 850*883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 851*883492d5Sraf } else if (error != EBUSY) { 852*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 853*883492d5Sraf } 854*883492d5Sraf 8557c478bd9Sstevel@tonic-gate return (error); 8567c478bd9Sstevel@tonic-gate } 8577c478bd9Sstevel@tonic-gate 8587c478bd9Sstevel@tonic-gate volatile sc_shared_t * 8597c478bd9Sstevel@tonic-gate setup_schedctl(void) 8607c478bd9Sstevel@tonic-gate { 8617c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 8627c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 8637c478bd9Sstevel@tonic-gate sc_shared_t *tmp; 8647c478bd9Sstevel@tonic-gate 8657c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */ 8667c478bd9Sstevel@tonic-gate !self->ul_vfork && /* not a child of vfork() */ 8677c478bd9Sstevel@tonic-gate !self->ul_schedctl_called) { /* haven't been called before */ 8687c478bd9Sstevel@tonic-gate enter_critical(self); 8697c478bd9Sstevel@tonic-gate self->ul_schedctl_called = &self->ul_uberdata->uberflags; 8707c478bd9Sstevel@tonic-gate if ((tmp = __schedctl()) != (sc_shared_t *)(-1)) 8717c478bd9Sstevel@tonic-gate self->ul_schedctl = scp = tmp; 8727c478bd9Sstevel@tonic-gate exit_critical(self); 8737c478bd9Sstevel@tonic-gate } 8747c478bd9Sstevel@tonic-gate /* 8757c478bd9Sstevel@tonic-gate * Unless the call to setup_schedctl() is surrounded 8767c478bd9Sstevel@tonic-gate * by enter_critical()/exit_critical(), the address 8777c478bd9Sstevel@tonic-gate * we are returning could be invalid due to a forkall() 8787c478bd9Sstevel@tonic-gate * having occurred in another thread. 8797c478bd9Sstevel@tonic-gate */ 8807c478bd9Sstevel@tonic-gate return (scp); 8817c478bd9Sstevel@tonic-gate } 8827c478bd9Sstevel@tonic-gate 8837c478bd9Sstevel@tonic-gate /* 8847c478bd9Sstevel@tonic-gate * Interfaces from libsched, incorporated into libc. 8857c478bd9Sstevel@tonic-gate * libsched.so.1 is now a filter library onto libc. 8867c478bd9Sstevel@tonic-gate */ 8877c478bd9Sstevel@tonic-gate #pragma weak schedctl_lookup = _schedctl_init 8887c478bd9Sstevel@tonic-gate #pragma weak _schedctl_lookup = _schedctl_init 8897c478bd9Sstevel@tonic-gate #pragma weak schedctl_init = _schedctl_init 8907c478bd9Sstevel@tonic-gate schedctl_t * 8917c478bd9Sstevel@tonic-gate _schedctl_init(void) 8927c478bd9Sstevel@tonic-gate { 8937c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = setup_schedctl(); 8947c478bd9Sstevel@tonic-gate return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl); 8957c478bd9Sstevel@tonic-gate } 8967c478bd9Sstevel@tonic-gate 8977c478bd9Sstevel@tonic-gate #pragma weak schedctl_exit = _schedctl_exit 8987c478bd9Sstevel@tonic-gate void 8997c478bd9Sstevel@tonic-gate _schedctl_exit(void) 9007c478bd9Sstevel@tonic-gate { 9017c478bd9Sstevel@tonic-gate } 9027c478bd9Sstevel@tonic-gate 9037c478bd9Sstevel@tonic-gate /* 9047c478bd9Sstevel@tonic-gate * Contract private interface for java. 9057c478bd9Sstevel@tonic-gate * Set up the schedctl data if it doesn't exist yet. 9067c478bd9Sstevel@tonic-gate * Return a pointer to the pointer to the schedctl data. 9077c478bd9Sstevel@tonic-gate */ 9087c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile * 9097c478bd9Sstevel@tonic-gate _thr_schedctl(void) 9107c478bd9Sstevel@tonic-gate { 9117c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 9127c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile *ptr; 9137c478bd9Sstevel@tonic-gate 9147c478bd9Sstevel@tonic-gate if (self->ul_vfork) 9157c478bd9Sstevel@tonic-gate return (NULL); 9167c478bd9Sstevel@tonic-gate if (*(ptr = &self->ul_schedctl) == NULL) 9177c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 9187c478bd9Sstevel@tonic-gate return (ptr); 9197c478bd9Sstevel@tonic-gate } 9207c478bd9Sstevel@tonic-gate 9217c478bd9Sstevel@tonic-gate /* 9227c478bd9Sstevel@tonic-gate * Block signals and attempt to block preemption. 9237c478bd9Sstevel@tonic-gate * no_preempt()/preempt() must be used in pairs but can be nested. 9247c478bd9Sstevel@tonic-gate */ 9257c478bd9Sstevel@tonic-gate void 9267c478bd9Sstevel@tonic-gate no_preempt(ulwp_t *self) 9277c478bd9Sstevel@tonic-gate { 9287c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 9297c478bd9Sstevel@tonic-gate 9307c478bd9Sstevel@tonic-gate if (self->ul_preempt++ == 0) { 9317c478bd9Sstevel@tonic-gate enter_critical(self); 9327c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL || 9337c478bd9Sstevel@tonic-gate (scp = setup_schedctl()) != NULL) { 9347c478bd9Sstevel@tonic-gate /* 9357c478bd9Sstevel@tonic-gate * Save the pre-existing preempt value. 9367c478bd9Sstevel@tonic-gate */ 9377c478bd9Sstevel@tonic-gate self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt; 9387c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = 1; 9397c478bd9Sstevel@tonic-gate } 9407c478bd9Sstevel@tonic-gate } 9417c478bd9Sstevel@tonic-gate } 9427c478bd9Sstevel@tonic-gate 9437c478bd9Sstevel@tonic-gate /* 9447c478bd9Sstevel@tonic-gate * Undo the effects of no_preempt(). 9457c478bd9Sstevel@tonic-gate */ 9467c478bd9Sstevel@tonic-gate void 9477c478bd9Sstevel@tonic-gate preempt(ulwp_t *self) 9487c478bd9Sstevel@tonic-gate { 9497c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 9507c478bd9Sstevel@tonic-gate 9517c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt > 0); 9527c478bd9Sstevel@tonic-gate if (--self->ul_preempt == 0) { 9537c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL) { 9547c478bd9Sstevel@tonic-gate /* 9557c478bd9Sstevel@tonic-gate * Restore the pre-existing preempt value. 9567c478bd9Sstevel@tonic-gate */ 9577c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt; 9587c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield && 9597c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt == 0) { 9607c478bd9Sstevel@tonic-gate lwp_yield(); 9617c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield) { 9627c478bd9Sstevel@tonic-gate /* 9637c478bd9Sstevel@tonic-gate * Shouldn't happen. This is either 9647c478bd9Sstevel@tonic-gate * a race condition or the thread 9657c478bd9Sstevel@tonic-gate * just entered the real-time class. 9667c478bd9Sstevel@tonic-gate */ 9677c478bd9Sstevel@tonic-gate lwp_yield(); 9687c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_yield = 0; 9697c478bd9Sstevel@tonic-gate } 9707c478bd9Sstevel@tonic-gate } 9717c478bd9Sstevel@tonic-gate } 9727c478bd9Sstevel@tonic-gate exit_critical(self); 9737c478bd9Sstevel@tonic-gate } 9747c478bd9Sstevel@tonic-gate } 9757c478bd9Sstevel@tonic-gate 9767c478bd9Sstevel@tonic-gate /* 9777c478bd9Sstevel@tonic-gate * If a call to preempt() would cause the current thread to yield or to 9787c478bd9Sstevel@tonic-gate * take deferred actions in exit_critical(), then unpark the specified 9797c478bd9Sstevel@tonic-gate * lwp so it can run while we delay. Return the original lwpid if the 9807c478bd9Sstevel@tonic-gate * unpark was not performed, else return zero. The tests are a repeat 9817c478bd9Sstevel@tonic-gate * of some of the tests in preempt(), above. This is a statistical 9827c478bd9Sstevel@tonic-gate * optimization solely for cond_sleep_queue(), below. 9837c478bd9Sstevel@tonic-gate */ 9847c478bd9Sstevel@tonic-gate static lwpid_t 9857c478bd9Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid) 9867c478bd9Sstevel@tonic-gate { 9877c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = self->ul_schedctl; 9887c478bd9Sstevel@tonic-gate 9897c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt == 1 && self->ul_critical > 0); 9907c478bd9Sstevel@tonic-gate if ((scp != NULL && scp->sc_preemptctl.sc_yield) || 9917c478bd9Sstevel@tonic-gate (self->ul_curplease && self->ul_critical == 1)) { 9927c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 9937c478bd9Sstevel@tonic-gate lwpid = 0; 9947c478bd9Sstevel@tonic-gate } 9957c478bd9Sstevel@tonic-gate return (lwpid); 9967c478bd9Sstevel@tonic-gate } 9977c478bd9Sstevel@tonic-gate 9987c478bd9Sstevel@tonic-gate /* 999*883492d5Sraf * Spin for a while, trying to grab the lock. 10007c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 10017c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread. 10027c478bd9Sstevel@tonic-gate */ 1003*883492d5Sraf static int 10047c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mutex_t *mp) 10057c478bd9Sstevel@tonic-gate { 10067c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1007*883492d5Sraf int error = EBUSY; 10087c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 10097c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 10107c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 10117c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 1012*883492d5Sraf int count; 1013*883492d5Sraf int max; 10147c478bd9Sstevel@tonic-gate 1015*883492d5Sraf ASSERT(!(mp->mutex_type & USYNC_PROCESS)); 10167c478bd9Sstevel@tonic-gate 1017*883492d5Sraf if (MUTEX_OWNER(mp) == self) 10187c478bd9Sstevel@tonic-gate return (EBUSY); 10197c478bd9Sstevel@tonic-gate 1020*883492d5Sraf /* short-cut, not definitive (see below) */ 1021*883492d5Sraf if (mp->mutex_flag & LOCK_NOTRECOVERABLE) { 1022*883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1023*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, ENOTRECOVERABLE); 1024*883492d5Sraf return (ENOTRECOVERABLE); 1025*883492d5Sraf } 1026*883492d5Sraf 1027*883492d5Sraf if ((max = self->ul_adaptive_spin) == 0 || 1028*883492d5Sraf mp->mutex_spinners >= self->ul_max_spinners) 1029*883492d5Sraf max = 1; /* try at least once */ 10307c478bd9Sstevel@tonic-gate 10317c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__spin, mp); 10327c478bd9Sstevel@tonic-gate 1033*883492d5Sraf lockp = (volatile uint8_t *)&mp->mutex_lockw; 1034*883492d5Sraf ownerp = (volatile uint64_t *)&mp->mutex_owner; 10357c478bd9Sstevel@tonic-gate /* 10367c478bd9Sstevel@tonic-gate * This spin loop is unfair to lwps that have already dropped into 10377c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a highly-contended mutex. 10387c478bd9Sstevel@tonic-gate * This is just too bad. The adaptive spin algorithm is intended 10397c478bd9Sstevel@tonic-gate * to allow programs with highly-contended locks (that is, broken 10407c478bd9Sstevel@tonic-gate * programs) to execute with reasonable speed despite their contention. 10417c478bd9Sstevel@tonic-gate * Being fair would reduce the speed of such programs and well-written 10427c478bd9Sstevel@tonic-gate * programs will not suffer in any case. 10437c478bd9Sstevel@tonic-gate */ 10447c478bd9Sstevel@tonic-gate enter_critical(self); /* protects ul_schedctl */ 104541efec22Sraf atomic_inc_32(&mp->mutex_spinners); 1046*883492d5Sraf for (count = 1; count <= max; count++) { 10477c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) { 10487c478bd9Sstevel@tonic-gate *ownerp = (uintptr_t)self; 1049*883492d5Sraf error = 0; 1050*883492d5Sraf break; 10517c478bd9Sstevel@tonic-gate } 10527c478bd9Sstevel@tonic-gate SMT_PAUSE(); 10537c478bd9Sstevel@tonic-gate /* 10547c478bd9Sstevel@tonic-gate * Stop spinning if the mutex owner is not running on 10557c478bd9Sstevel@tonic-gate * a processor; it will not drop the lock any time soon 10567c478bd9Sstevel@tonic-gate * and we would just be wasting time to keep spinning. 10577c478bd9Sstevel@tonic-gate * 10587c478bd9Sstevel@tonic-gate * Note that we are looking at another thread (ulwp_t) 10597c478bd9Sstevel@tonic-gate * without ensuring that the other thread does not exit. 10607c478bd9Sstevel@tonic-gate * The scheme relies on ulwp_t structures never being 10617c478bd9Sstevel@tonic-gate * deallocated by the library (the library employs a free 10627c478bd9Sstevel@tonic-gate * list of ulwp_t structs that are reused when new threads 10637c478bd9Sstevel@tonic-gate * are created) and on schedctl shared memory never being 10647c478bd9Sstevel@tonic-gate * deallocated once created via __schedctl(). 10657c478bd9Sstevel@tonic-gate * 10667c478bd9Sstevel@tonic-gate * Thus, the worst that can happen when the spinning thread 10677c478bd9Sstevel@tonic-gate * looks at the owner's schedctl data is that it is looking 10687c478bd9Sstevel@tonic-gate * at some other thread's schedctl data. This almost never 10697c478bd9Sstevel@tonic-gate * happens and is benign when it does. 10707c478bd9Sstevel@tonic-gate */ 10717c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 10727c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 10737c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 10747c478bd9Sstevel@tonic-gate break; 10757c478bd9Sstevel@tonic-gate } 107641efec22Sraf atomic_dec_32(&mp->mutex_spinners); 10777c478bd9Sstevel@tonic-gate exit_critical(self); 10787c478bd9Sstevel@tonic-gate 1079*883492d5Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) { 1080*883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1081*883492d5Sraf /* 1082*883492d5Sraf * We shouldn't own the mutex; clear the lock. 1083*883492d5Sraf */ 1084*883492d5Sraf mp->mutex_owner = 0; 1085*883492d5Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) 1086*883492d5Sraf mutex_wakeup_all(mp); 1087*883492d5Sraf error = ENOTRECOVERABLE; 1088*883492d5Sraf } 10897c478bd9Sstevel@tonic-gate 1090*883492d5Sraf if (error) { 1091*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__spun, 0, count); 1092*883492d5Sraf if (error != EBUSY) { 1093*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1094*883492d5Sraf } 1095*883492d5Sraf } else { 1096*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 1097*883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 1098*883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) { 1099*883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1100*883492d5Sraf error = EOWNERDEAD; 1101*883492d5Sraf } 1102*883492d5Sraf } 1103*883492d5Sraf 1104*883492d5Sraf return (error); 11057c478bd9Sstevel@tonic-gate } 11067c478bd9Sstevel@tonic-gate 11077c478bd9Sstevel@tonic-gate /* 11087c478bd9Sstevel@tonic-gate * Same as mutex_trylock_adaptive(), except specifically for queue locks. 11097c478bd9Sstevel@tonic-gate * The owner field is not set here; the caller (spin_lock_set()) sets it. 11107c478bd9Sstevel@tonic-gate */ 1111*883492d5Sraf static int 11127c478bd9Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp) 11137c478bd9Sstevel@tonic-gate { 11147c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 11157c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 11167c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 11177c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 11187c478bd9Sstevel@tonic-gate int count = curthread->ul_queue_spin; 11197c478bd9Sstevel@tonic-gate 11207c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 11217c478bd9Sstevel@tonic-gate 11227c478bd9Sstevel@tonic-gate if (count == 0) 11237c478bd9Sstevel@tonic-gate return (EBUSY); 11247c478bd9Sstevel@tonic-gate 11257c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 11267c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 11277c478bd9Sstevel@tonic-gate while (--count >= 0) { 11287c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) 11297c478bd9Sstevel@tonic-gate return (0); 11307c478bd9Sstevel@tonic-gate SMT_PAUSE(); 11317c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 11327c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 11337c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 11347c478bd9Sstevel@tonic-gate break; 11357c478bd9Sstevel@tonic-gate } 11367c478bd9Sstevel@tonic-gate 11377c478bd9Sstevel@tonic-gate return (EBUSY); 11387c478bd9Sstevel@tonic-gate } 11397c478bd9Sstevel@tonic-gate 11407c478bd9Sstevel@tonic-gate /* 11417c478bd9Sstevel@tonic-gate * Like mutex_trylock_adaptive(), but for process-shared mutexes. 1142*883492d5Sraf * Spin for a while, trying to grab the lock. 11437c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 11447c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread 11457c478bd9Sstevel@tonic-gate * and mutex_ownerpid set to the current pid. 11467c478bd9Sstevel@tonic-gate */ 1147*883492d5Sraf static int 11487c478bd9Sstevel@tonic-gate mutex_trylock_process(mutex_t *mp) 11497c478bd9Sstevel@tonic-gate { 11507c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1151*883492d5Sraf int error = EBUSY; 11527c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 1153*883492d5Sraf int count; 1154*883492d5Sraf int max; 11557c478bd9Sstevel@tonic-gate 1156*883492d5Sraf ASSERT(mp->mutex_type & USYNC_PROCESS); 11577c478bd9Sstevel@tonic-gate 1158*883492d5Sraf if (shared_mutex_held(mp)) 11597c478bd9Sstevel@tonic-gate return (EBUSY); 11607c478bd9Sstevel@tonic-gate 1161*883492d5Sraf /* short-cut, not definitive (see below) */ 1162*883492d5Sraf if (mp->mutex_flag & LOCK_NOTRECOVERABLE) { 1163*883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1164*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, ENOTRECOVERABLE); 1165*883492d5Sraf return (ENOTRECOVERABLE); 1166*883492d5Sraf } 1167*883492d5Sraf 1168*883492d5Sraf if (ncpus == 0) 1169*883492d5Sraf ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 1170*883492d5Sraf max = (ncpus > 1)? self->ul_adaptive_spin : 1; 1171*883492d5Sraf if (max == 0) 1172*883492d5Sraf max = 1; /* try at least once */ 1173*883492d5Sraf 1174*883492d5Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp); 1175*883492d5Sraf 11767c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 11777c478bd9Sstevel@tonic-gate /* 11787c478bd9Sstevel@tonic-gate * This is a process-shared mutex. 11797c478bd9Sstevel@tonic-gate * We cannot know if the owner is running on a processor. 11807c478bd9Sstevel@tonic-gate * We just spin and hope that it is on a processor. 11817c478bd9Sstevel@tonic-gate */ 1182*883492d5Sraf enter_critical(self); 1183*883492d5Sraf for (count = 1; count <= max; count++) { 1184*883492d5Sraf if (*lockp == 0 && set_lock_byte(lockp) == 0) { 1185*883492d5Sraf mp->mutex_owner = (uintptr_t)self; 1186*883492d5Sraf mp->mutex_ownerpid = self->ul_uberdata->pid; 1187*883492d5Sraf error = 0; 1188*883492d5Sraf break; 11897c478bd9Sstevel@tonic-gate } 1190*883492d5Sraf SMT_PAUSE(); 1191*883492d5Sraf } 1192*883492d5Sraf exit_critical(self); 1193*883492d5Sraf 1194*883492d5Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) { 1195*883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 11967c478bd9Sstevel@tonic-gate /* 1197*883492d5Sraf * We shouldn't own the mutex; clear the lock. 11987c478bd9Sstevel@tonic-gate */ 1199*883492d5Sraf mp->mutex_owner = 0; 1200*883492d5Sraf mp->mutex_ownerpid = 0; 1201*883492d5Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) { 1202*883492d5Sraf no_preempt(self); 1203*883492d5Sraf (void) ___lwp_mutex_wakeup(mp, 1); 1204*883492d5Sraf preempt(self); 1205*883492d5Sraf } 1206*883492d5Sraf error = ENOTRECOVERABLE; 12077c478bd9Sstevel@tonic-gate } 12087c478bd9Sstevel@tonic-gate 1209*883492d5Sraf if (error) { 1210*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__spun, 0, count); 1211*883492d5Sraf if (error != EBUSY) { 1212*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1213*883492d5Sraf } 1214*883492d5Sraf } else { 1215*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 1216*883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 1217*883492d5Sraf if (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) { 1218*883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1219*883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) 1220*883492d5Sraf error = EOWNERDEAD; 1221*883492d5Sraf else if (mp->mutex_type & USYNC_PROCESS_ROBUST) 1222*883492d5Sraf error = ELOCKUNMAPPED; 1223*883492d5Sraf else 1224*883492d5Sraf error = EOWNERDEAD; 1225*883492d5Sraf } 1226*883492d5Sraf } 1227*883492d5Sraf 1228*883492d5Sraf return (error); 12297c478bd9Sstevel@tonic-gate } 12307c478bd9Sstevel@tonic-gate 12317c478bd9Sstevel@tonic-gate /* 12327c478bd9Sstevel@tonic-gate * Mutex wakeup code for releasing a USYNC_THREAD mutex. 12337c478bd9Sstevel@tonic-gate * Returns the lwpid of the thread that was dequeued, if any. 12347c478bd9Sstevel@tonic-gate * The caller of mutex_wakeup() must call __lwp_unpark(lwpid) 12357c478bd9Sstevel@tonic-gate * to wake up the specified lwp. 12367c478bd9Sstevel@tonic-gate */ 1237*883492d5Sraf static lwpid_t 12387c478bd9Sstevel@tonic-gate mutex_wakeup(mutex_t *mp) 12397c478bd9Sstevel@tonic-gate { 12407c478bd9Sstevel@tonic-gate lwpid_t lwpid = 0; 12417c478bd9Sstevel@tonic-gate queue_head_t *qp; 12427c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 12437c478bd9Sstevel@tonic-gate int more; 12447c478bd9Sstevel@tonic-gate 12457c478bd9Sstevel@tonic-gate /* 12467c478bd9Sstevel@tonic-gate * Dequeue a waiter from the sleep queue. Don't touch the mutex 12477c478bd9Sstevel@tonic-gate * waiters bit if no one was found on the queue because the mutex 12487c478bd9Sstevel@tonic-gate * might have been deallocated or reallocated for another purpose. 12497c478bd9Sstevel@tonic-gate */ 12507c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 12517c478bd9Sstevel@tonic-gate if ((ulwp = dequeue(qp, mp, &more)) != NULL) { 12527c478bd9Sstevel@tonic-gate lwpid = ulwp->ul_lwpid; 12537c478bd9Sstevel@tonic-gate mp->mutex_waiters = (more? 1 : 0); 12547c478bd9Sstevel@tonic-gate } 12557c478bd9Sstevel@tonic-gate queue_unlock(qp); 12567c478bd9Sstevel@tonic-gate return (lwpid); 12577c478bd9Sstevel@tonic-gate } 12587c478bd9Sstevel@tonic-gate 1259*883492d5Sraf /* 1260*883492d5Sraf * Mutex wakeup code for releasing all waiters on a USYNC_THREAD mutex. 1261*883492d5Sraf */ 1262*883492d5Sraf static void 1263*883492d5Sraf mutex_wakeup_all(mutex_t *mp) 1264*883492d5Sraf { 1265*883492d5Sraf queue_head_t *qp; 1266*883492d5Sraf int nlwpid = 0; 1267*883492d5Sraf int maxlwps = MAXLWPS; 1268*883492d5Sraf ulwp_t **ulwpp; 1269*883492d5Sraf ulwp_t *ulwp; 1270*883492d5Sraf ulwp_t *prev = NULL; 1271*883492d5Sraf lwpid_t buffer[MAXLWPS]; 1272*883492d5Sraf lwpid_t *lwpid = buffer; 1273*883492d5Sraf 1274*883492d5Sraf /* 1275*883492d5Sraf * Walk the list of waiters and prepare to wake up all of them. 1276*883492d5Sraf * The waiters flag has already been cleared from the mutex. 1277*883492d5Sraf * 1278*883492d5Sraf * We keep track of lwpids that are to be unparked in lwpid[]. 1279*883492d5Sraf * __lwp_unpark_all() is called to unpark all of them after 1280*883492d5Sraf * they have been removed from the sleep queue and the sleep 1281*883492d5Sraf * queue lock has been dropped. If we run out of space in our 1282*883492d5Sraf * on-stack buffer, we need to allocate more but we can't call 1283*883492d5Sraf * lmalloc() because we are holding a queue lock when the overflow 1284*883492d5Sraf * occurs and lmalloc() acquires a lock. We can't use alloca() 1285*883492d5Sraf * either because the application may have allocated a small 1286*883492d5Sraf * stack and we don't want to overrun the stack. So we call 1287*883492d5Sraf * alloc_lwpids() to allocate a bigger buffer using the mmap() 1288*883492d5Sraf * system call directly since that path acquires no locks. 1289*883492d5Sraf */ 1290*883492d5Sraf qp = queue_lock(mp, MX); 1291*883492d5Sraf ulwpp = &qp->qh_head; 1292*883492d5Sraf while ((ulwp = *ulwpp) != NULL) { 1293*883492d5Sraf if (ulwp->ul_wchan != mp) { 1294*883492d5Sraf prev = ulwp; 1295*883492d5Sraf ulwpp = &ulwp->ul_link; 1296*883492d5Sraf } else { 1297*883492d5Sraf if (nlwpid == maxlwps) 1298*883492d5Sraf lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps); 1299*883492d5Sraf (void) queue_unlink(qp, ulwpp, prev); 1300*883492d5Sraf lwpid[nlwpid++] = ulwp->ul_lwpid; 1301*883492d5Sraf } 1302*883492d5Sraf } 1303*883492d5Sraf mp->mutex_waiters = 0; 1304*883492d5Sraf 1305*883492d5Sraf if (nlwpid == 0) { 1306*883492d5Sraf queue_unlock(qp); 1307*883492d5Sraf } else { 1308*883492d5Sraf no_preempt(curthread); 1309*883492d5Sraf queue_unlock(qp); 1310*883492d5Sraf if (nlwpid == 1) 1311*883492d5Sraf (void) __lwp_unpark(lwpid[0]); 1312*883492d5Sraf else 1313*883492d5Sraf (void) __lwp_unpark_all(lwpid, nlwpid); 1314*883492d5Sraf preempt(curthread); 1315*883492d5Sraf } 1316*883492d5Sraf 1317*883492d5Sraf if (lwpid != buffer) 1318*883492d5Sraf (void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t)); 1319*883492d5Sraf } 1320*883492d5Sraf 13217c478bd9Sstevel@tonic-gate /* 13227c478bd9Sstevel@tonic-gate * Spin for a while, testing to see if the lock has been grabbed. 13237c478bd9Sstevel@tonic-gate * If this fails, call mutex_wakeup() to release a waiter. 13247c478bd9Sstevel@tonic-gate */ 1325*883492d5Sraf static lwpid_t 1326*883492d5Sraf mutex_unlock_queue(mutex_t *mp, int release_all) 13277c478bd9Sstevel@tonic-gate { 13287c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 13297c478bd9Sstevel@tonic-gate uint32_t *lockw = &mp->mutex_lockword; 13307c478bd9Sstevel@tonic-gate lwpid_t lwpid; 13317c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 13327c478bd9Sstevel@tonic-gate volatile uint32_t *spinp; 13337c478bd9Sstevel@tonic-gate int count; 13347c478bd9Sstevel@tonic-gate 13357c478bd9Sstevel@tonic-gate /* 13367c478bd9Sstevel@tonic-gate * We use the swap primitive to clear the lock, but we must 13377c478bd9Sstevel@tonic-gate * atomically retain the waiters bit for the remainder of this 13387c478bd9Sstevel@tonic-gate * code to work. We first check to see if the waiters bit is 13397c478bd9Sstevel@tonic-gate * set and if so clear the lock by swapping in a word containing 13407c478bd9Sstevel@tonic-gate * only the waiters bit. This could produce a false positive test 13417c478bd9Sstevel@tonic-gate * for whether there are waiters that need to be waked up, but 13427c478bd9Sstevel@tonic-gate * this just causes an extra call to mutex_wakeup() to do nothing. 13437c478bd9Sstevel@tonic-gate * The opposite case is more delicate: If there are no waiters, 13447c478bd9Sstevel@tonic-gate * we swap in a zero lock byte and a zero waiters bit. The result 13457c478bd9Sstevel@tonic-gate * of the swap could indicate that there really was a waiter so in 13467c478bd9Sstevel@tonic-gate * this case we go directly to mutex_wakeup() without performing 13477c478bd9Sstevel@tonic-gate * any of the adaptive code because the waiter bit has been cleared 13487c478bd9Sstevel@tonic-gate * and the adaptive code is unreliable in this case. 13497c478bd9Sstevel@tonic-gate */ 1350*883492d5Sraf if (release_all || !(*lockw & WAITERMASK)) { 13517c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 13527c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 135341efec22Sraf if (!(atomic_swap_32(lockw, 0) & WAITERMASK)) 1354*883492d5Sraf return (0); /* no waiters */ 13557c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 13567c478bd9Sstevel@tonic-gate } else { 13577c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 13587c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 13597c478bd9Sstevel@tonic-gate spinp = (volatile uint32_t *)&mp->mutex_spinners; 13607c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 13617c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 136241efec22Sraf /* clear lock, retain waiter */ 136341efec22Sraf (void) atomic_swap_32(lockw, WAITER); 13647c478bd9Sstevel@tonic-gate 13657c478bd9Sstevel@tonic-gate /* 13667c478bd9Sstevel@tonic-gate * We spin here fewer times than mutex_trylock_adaptive(). 13677c478bd9Sstevel@tonic-gate * We are trying to balance two conflicting goals: 13687c478bd9Sstevel@tonic-gate * 1. Avoid waking up anyone if a spinning thread 13697c478bd9Sstevel@tonic-gate * grabs the lock. 13707c478bd9Sstevel@tonic-gate * 2. Wake up a sleeping thread promptly to get on 13717c478bd9Sstevel@tonic-gate * with useful work. 13727c478bd9Sstevel@tonic-gate * We don't spin at all if there is no acquiring spinner; 13737c478bd9Sstevel@tonic-gate * (mp->mutex_spinners is non-zero if there are spinners). 13747c478bd9Sstevel@tonic-gate */ 13757c478bd9Sstevel@tonic-gate for (count = self->ul_release_spin; 13767c478bd9Sstevel@tonic-gate *spinp && count > 0; count--) { 13777c478bd9Sstevel@tonic-gate /* 13787c478bd9Sstevel@tonic-gate * There is a waiter that we will have to wake 13797c478bd9Sstevel@tonic-gate * up unless someone else grabs the lock while 13807c478bd9Sstevel@tonic-gate * we are busy spinning. Like the spin loop in 13817c478bd9Sstevel@tonic-gate * mutex_trylock_adaptive(), this spin loop is 13827c478bd9Sstevel@tonic-gate * unfair to lwps that have already dropped into 13837c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a 13847c478bd9Sstevel@tonic-gate * highly-contended mutex. Too bad. 13857c478bd9Sstevel@tonic-gate */ 13867c478bd9Sstevel@tonic-gate if (*lockp != 0) { /* somebody grabbed the lock */ 13877c478bd9Sstevel@tonic-gate preempt(self); 13887c478bd9Sstevel@tonic-gate return (0); 13897c478bd9Sstevel@tonic-gate } 13907c478bd9Sstevel@tonic-gate SMT_PAUSE(); 13917c478bd9Sstevel@tonic-gate } 13927c478bd9Sstevel@tonic-gate 13937c478bd9Sstevel@tonic-gate /* 13947c478bd9Sstevel@tonic-gate * No one grabbed the lock. 13957c478bd9Sstevel@tonic-gate * Wake up some lwp that is waiting for it. 13967c478bd9Sstevel@tonic-gate */ 13977c478bd9Sstevel@tonic-gate mp->mutex_waiters = 0; 13987c478bd9Sstevel@tonic-gate } 13997c478bd9Sstevel@tonic-gate 1400*883492d5Sraf if (release_all) { 1401*883492d5Sraf mutex_wakeup_all(mp); 1402*883492d5Sraf lwpid = 0; 1403*883492d5Sraf } else { 1404*883492d5Sraf lwpid = mutex_wakeup(mp); 1405*883492d5Sraf } 14067c478bd9Sstevel@tonic-gate if (lwpid == 0) 14077c478bd9Sstevel@tonic-gate preempt(self); 14087c478bd9Sstevel@tonic-gate return (lwpid); 14097c478bd9Sstevel@tonic-gate } 14107c478bd9Sstevel@tonic-gate 14117c478bd9Sstevel@tonic-gate /* 14127c478bd9Sstevel@tonic-gate * Like mutex_unlock_queue(), but for process-shared mutexes. 14137c478bd9Sstevel@tonic-gate * We tested the waiters field before calling here and it was non-zero. 14147c478bd9Sstevel@tonic-gate */ 1415*883492d5Sraf static void 1416*883492d5Sraf mutex_unlock_process(mutex_t *mp, int release_all) 14177c478bd9Sstevel@tonic-gate { 14187c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 14197c478bd9Sstevel@tonic-gate int count; 14207c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 14217c478bd9Sstevel@tonic-gate 14227c478bd9Sstevel@tonic-gate /* 14237c478bd9Sstevel@tonic-gate * See the comments in mutex_unlock_queue(), above. 14247c478bd9Sstevel@tonic-gate */ 1425*883492d5Sraf if (ncpus == 0) 1426*883492d5Sraf ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 1427*883492d5Sraf count = (ncpus > 1)? self->ul_release_spin : 0; 14287c478bd9Sstevel@tonic-gate no_preempt(self); 14297c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 14307c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 14317c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 1432*883492d5Sraf if (release_all || count == 0) { 14337c478bd9Sstevel@tonic-gate /* clear lock, test waiter */ 143441efec22Sraf if (!(atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK)) { 14357c478bd9Sstevel@tonic-gate /* no waiters now */ 14367c478bd9Sstevel@tonic-gate preempt(self); 14377c478bd9Sstevel@tonic-gate return; 14387c478bd9Sstevel@tonic-gate } 14397c478bd9Sstevel@tonic-gate } else { 14407c478bd9Sstevel@tonic-gate /* clear lock, retain waiter */ 144141efec22Sraf (void) atomic_swap_32(&mp->mutex_lockword, WAITER); 14427c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 14437c478bd9Sstevel@tonic-gate while (--count >= 0) { 14447c478bd9Sstevel@tonic-gate if (*lockp != 0) { 14457c478bd9Sstevel@tonic-gate /* somebody grabbed the lock */ 14467c478bd9Sstevel@tonic-gate preempt(self); 14477c478bd9Sstevel@tonic-gate return; 14487c478bd9Sstevel@tonic-gate } 14497c478bd9Sstevel@tonic-gate SMT_PAUSE(); 14507c478bd9Sstevel@tonic-gate } 14517c478bd9Sstevel@tonic-gate /* 14527c478bd9Sstevel@tonic-gate * We must clear the waiters field before going 14537c478bd9Sstevel@tonic-gate * to the kernel, else it could remain set forever. 14547c478bd9Sstevel@tonic-gate */ 14557c478bd9Sstevel@tonic-gate mp->mutex_waiters = 0; 14567c478bd9Sstevel@tonic-gate } 1457*883492d5Sraf (void) ___lwp_mutex_wakeup(mp, release_all); 14587c478bd9Sstevel@tonic-gate preempt(self); 14597c478bd9Sstevel@tonic-gate } 14607c478bd9Sstevel@tonic-gate 14617c478bd9Sstevel@tonic-gate /* 14627c478bd9Sstevel@tonic-gate * Return the real priority of a thread. 14637c478bd9Sstevel@tonic-gate */ 14647c478bd9Sstevel@tonic-gate int 14657c478bd9Sstevel@tonic-gate real_priority(ulwp_t *ulwp) 14667c478bd9Sstevel@tonic-gate { 14677c478bd9Sstevel@tonic-gate if (ulwp->ul_epri == 0) 14687c478bd9Sstevel@tonic-gate return (ulwp->ul_mappedpri? ulwp->ul_mappedpri : ulwp->ul_pri); 14697c478bd9Sstevel@tonic-gate return (ulwp->ul_emappedpri? ulwp->ul_emappedpri : ulwp->ul_epri); 14707c478bd9Sstevel@tonic-gate } 14717c478bd9Sstevel@tonic-gate 14727c478bd9Sstevel@tonic-gate void 14737c478bd9Sstevel@tonic-gate stall(void) 14747c478bd9Sstevel@tonic-gate { 14757c478bd9Sstevel@tonic-gate for (;;) 14767c478bd9Sstevel@tonic-gate (void) mutex_lock_kernel(&stall_mutex, NULL, NULL); 14777c478bd9Sstevel@tonic-gate } 14787c478bd9Sstevel@tonic-gate 14797c478bd9Sstevel@tonic-gate /* 14807c478bd9Sstevel@tonic-gate * Acquire a USYNC_THREAD mutex via user-level sleep queues. 14817c478bd9Sstevel@tonic-gate * We failed set_lock_byte(&mp->mutex_lockw) before coming here. 1482*883492d5Sraf * If successful, returns with mutex_owner set correctly. 14837c478bd9Sstevel@tonic-gate */ 14847c478bd9Sstevel@tonic-gate int 14857c478bd9Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp, 14867c478bd9Sstevel@tonic-gate timespec_t *tsp) 14877c478bd9Sstevel@tonic-gate { 14887c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 14897c478bd9Sstevel@tonic-gate queue_head_t *qp; 14907c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 14917c478bd9Sstevel@tonic-gate int error = 0; 14927c478bd9Sstevel@tonic-gate 14937c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 14947c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 14957c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 14967c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 14977c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 14987c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 14997c478bd9Sstevel@tonic-gate } 15007c478bd9Sstevel@tonic-gate if (msp) { 15017c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 15027c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 15037c478bd9Sstevel@tonic-gate } 15047c478bd9Sstevel@tonic-gate 15057c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 15067c478bd9Sstevel@tonic-gate 15077c478bd9Sstevel@tonic-gate /* 15087c478bd9Sstevel@tonic-gate * Put ourself on the sleep queue, and while we are 15097c478bd9Sstevel@tonic-gate * unable to grab the lock, go park in the kernel. 15107c478bd9Sstevel@tonic-gate * Take ourself off the sleep queue after we acquire the lock. 15117c478bd9Sstevel@tonic-gate * The waiter bit can be set/cleared only while holding the queue lock. 15127c478bd9Sstevel@tonic-gate */ 15137c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 15147c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 15157c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 15167c478bd9Sstevel@tonic-gate for (;;) { 15177c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 15187c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15197c478bd9Sstevel@tonic-gate mp->mutex_waiters = dequeue_self(qp, mp); 15207c478bd9Sstevel@tonic-gate break; 15217c478bd9Sstevel@tonic-gate } 15227c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 15237c478bd9Sstevel@tonic-gate queue_unlock(qp); 15247c478bd9Sstevel@tonic-gate /* 15257c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 15267c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 15277c478bd9Sstevel@tonic-gate */ 15287c478bd9Sstevel@tonic-gate if ((error = __lwp_park(tsp, 0)) == EINTR) 15297c478bd9Sstevel@tonic-gate error = 0; 15307c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 15317c478bd9Sstevel@tonic-gate /* 15327c478bd9Sstevel@tonic-gate * We could have taken a signal or suspended ourself. 15337c478bd9Sstevel@tonic-gate * If we did, then we removed ourself from the queue. 15347c478bd9Sstevel@tonic-gate * Someone else may have removed us from the queue 15357c478bd9Sstevel@tonic-gate * as a consequence of mutex_unlock(). We may have 15367c478bd9Sstevel@tonic-gate * gotten a timeout from __lwp_park(). Or we may still 15377c478bd9Sstevel@tonic-gate * be on the queue and this is just a spurious wakeup. 15387c478bd9Sstevel@tonic-gate */ 15397c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 15407c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) { 1541*883492d5Sraf if (error) 15427c478bd9Sstevel@tonic-gate break; 15437c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 15447c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15457c478bd9Sstevel@tonic-gate break; 15467c478bd9Sstevel@tonic-gate } 15477c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 15487c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 15497c478bd9Sstevel@tonic-gate } 15507c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == qp && 15517c478bd9Sstevel@tonic-gate self->ul_qtype == MX && 15527c478bd9Sstevel@tonic-gate self->ul_wchan == mp); 15537c478bd9Sstevel@tonic-gate if (error) { 15547c478bd9Sstevel@tonic-gate mp->mutex_waiters = dequeue_self(qp, mp); 15557c478bd9Sstevel@tonic-gate break; 15567c478bd9Sstevel@tonic-gate } 15577c478bd9Sstevel@tonic-gate } 15587c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 15597c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 15607c478bd9Sstevel@tonic-gate self->ul_sp = 0; 15617c478bd9Sstevel@tonic-gate queue_unlock(qp); 1562*883492d5Sraf 15637c478bd9Sstevel@tonic-gate if (msp) 15647c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 15657c478bd9Sstevel@tonic-gate 15667c478bd9Sstevel@tonic-gate ASSERT(error == 0 || error == EINVAL || error == ETIME); 1567*883492d5Sraf 1568*883492d5Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) { 1569*883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1570*883492d5Sraf /* 1571*883492d5Sraf * We shouldn't own the mutex; clear the lock. 1572*883492d5Sraf */ 1573*883492d5Sraf mp->mutex_owner = 0; 1574*883492d5Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) 1575*883492d5Sraf mutex_wakeup_all(mp); 1576*883492d5Sraf error = ENOTRECOVERABLE; 1577*883492d5Sraf } 1578*883492d5Sraf 1579*883492d5Sraf if (error) { 1580*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 1581*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1582*883492d5Sraf } else { 1583*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 1584*883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1585*883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) { 1586*883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1587*883492d5Sraf error = EOWNERDEAD; 1588*883492d5Sraf } 1589*883492d5Sraf } 1590*883492d5Sraf 15917c478bd9Sstevel@tonic-gate return (error); 15927c478bd9Sstevel@tonic-gate } 15937c478bd9Sstevel@tonic-gate 1594*883492d5Sraf static int 1595*883492d5Sraf mutex_recursion(mutex_t *mp, int mtype, int try) 1596*883492d5Sraf { 1597*883492d5Sraf ASSERT(mutex_is_held(mp)); 1598*883492d5Sraf ASSERT(mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)); 1599*883492d5Sraf ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK); 1600*883492d5Sraf 1601*883492d5Sraf if (mtype & LOCK_RECURSIVE) { 1602*883492d5Sraf if (mp->mutex_rcount == RECURSION_MAX) { 1603*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EAGAIN); 1604*883492d5Sraf return (EAGAIN); 1605*883492d5Sraf } 1606*883492d5Sraf mp->mutex_rcount++; 1607*883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0); 1608*883492d5Sraf return (0); 1609*883492d5Sraf } 1610*883492d5Sraf if (try == MUTEX_LOCK) { 1611*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 1612*883492d5Sraf return (EDEADLK); 1613*883492d5Sraf } 1614*883492d5Sraf return (EBUSY); 1615*883492d5Sraf } 1616*883492d5Sraf 1617*883492d5Sraf /* 1618*883492d5Sraf * Register this USYNC_PROCESS|LOCK_ROBUST mutex with the kernel so 1619*883492d5Sraf * it can apply LOCK_OWNERDEAD|LOCK_UNMAPPED if it becomes necessary. 1620*883492d5Sraf * We use tdb_hash_lock here and in the synch object tracking code in 1621*883492d5Sraf * the tdb_agent.c file. There is no conflict between these two usages. 1622*883492d5Sraf */ 1623*883492d5Sraf void 1624*883492d5Sraf register_lock(mutex_t *mp) 1625*883492d5Sraf { 1626*883492d5Sraf uberdata_t *udp = curthread->ul_uberdata; 1627*883492d5Sraf uint_t hash = LOCK_HASH(mp); 1628*883492d5Sraf robust_t *rlp; 1629*883492d5Sraf robust_t **rlpp; 1630*883492d5Sraf robust_t **table; 1631*883492d5Sraf 1632*883492d5Sraf if ((table = udp->robustlocks) == NULL) { 1633*883492d5Sraf lmutex_lock(&udp->tdb_hash_lock); 1634*883492d5Sraf if ((table = udp->robustlocks) == NULL) { 1635*883492d5Sraf table = lmalloc(LOCKHASHSZ * sizeof (robust_t *)); 1636*883492d5Sraf _membar_producer(); 1637*883492d5Sraf udp->robustlocks = table; 1638*883492d5Sraf } 1639*883492d5Sraf lmutex_unlock(&udp->tdb_hash_lock); 1640*883492d5Sraf } 1641*883492d5Sraf _membar_consumer(); 1642*883492d5Sraf 1643*883492d5Sraf /* 1644*883492d5Sraf * First search the registered table with no locks held. 1645*883492d5Sraf * This is safe because the table never shrinks 1646*883492d5Sraf * and we can only get a false negative. 1647*883492d5Sraf */ 1648*883492d5Sraf for (rlp = table[hash]; rlp != NULL; rlp = rlp->robust_next) { 1649*883492d5Sraf if (rlp->robust_lock == mp) /* already registered */ 1650*883492d5Sraf return; 1651*883492d5Sraf } 1652*883492d5Sraf 1653*883492d5Sraf /* 1654*883492d5Sraf * The lock was not found. 1655*883492d5Sraf * Repeat the operation with tdb_hash_lock held. 1656*883492d5Sraf */ 1657*883492d5Sraf lmutex_lock(&udp->tdb_hash_lock); 1658*883492d5Sraf 1659*883492d5Sraf for (rlpp = &table[hash]; 1660*883492d5Sraf (rlp = *rlpp) != NULL; 1661*883492d5Sraf rlpp = &rlp->robust_next) { 1662*883492d5Sraf if (rlp->robust_lock == mp) { /* already registered */ 1663*883492d5Sraf lmutex_unlock(&udp->tdb_hash_lock); 1664*883492d5Sraf return; 1665*883492d5Sraf } 1666*883492d5Sraf } 1667*883492d5Sraf 1668*883492d5Sraf /* 1669*883492d5Sraf * The lock has never been registered. 1670*883492d5Sraf * Register it now and add it to the table. 1671*883492d5Sraf */ 1672*883492d5Sraf (void) ___lwp_mutex_register(mp); 1673*883492d5Sraf rlp = lmalloc(sizeof (*rlp)); 1674*883492d5Sraf rlp->robust_lock = mp; 1675*883492d5Sraf _membar_producer(); 1676*883492d5Sraf *rlpp = rlp; 1677*883492d5Sraf 1678*883492d5Sraf lmutex_unlock(&udp->tdb_hash_lock); 1679*883492d5Sraf } 1680*883492d5Sraf 1681*883492d5Sraf /* 1682*883492d5Sraf * This is called in the child of fork()/forkall() to start over 1683*883492d5Sraf * with a clean slate. (Each process must register its own locks.) 1684*883492d5Sraf * No locks are needed because all other threads are suspended or gone. 1685*883492d5Sraf */ 1686*883492d5Sraf void 1687*883492d5Sraf unregister_locks(void) 1688*883492d5Sraf { 1689*883492d5Sraf uberdata_t *udp = curthread->ul_uberdata; 1690*883492d5Sraf uint_t hash; 1691*883492d5Sraf robust_t **table; 1692*883492d5Sraf robust_t *rlp; 1693*883492d5Sraf robust_t *next; 1694*883492d5Sraf 1695*883492d5Sraf if ((table = udp->robustlocks) != NULL) { 1696*883492d5Sraf for (hash = 0; hash < LOCKHASHSZ; hash++) { 1697*883492d5Sraf rlp = table[hash]; 1698*883492d5Sraf while (rlp != NULL) { 1699*883492d5Sraf next = rlp->robust_next; 1700*883492d5Sraf lfree(rlp, sizeof (*rlp)); 1701*883492d5Sraf rlp = next; 1702*883492d5Sraf } 1703*883492d5Sraf } 1704*883492d5Sraf lfree(table, LOCKHASHSZ * sizeof (robust_t *)); 1705*883492d5Sraf udp->robustlocks = NULL; 1706*883492d5Sraf } 1707*883492d5Sraf } 1708*883492d5Sraf 17097c478bd9Sstevel@tonic-gate /* 17107c478bd9Sstevel@tonic-gate * Returns with mutex_owner set correctly. 17117c478bd9Sstevel@tonic-gate */ 1712*883492d5Sraf static int 17137c478bd9Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try) 17147c478bd9Sstevel@tonic-gate { 17157c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 17167c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 17177c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 17187c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 17197c478bd9Sstevel@tonic-gate int error = 0; 1720*883492d5Sraf uint8_t ceil; 1721*883492d5Sraf int myprio; 17227c478bd9Sstevel@tonic-gate 17237c478bd9Sstevel@tonic-gate ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK); 17247c478bd9Sstevel@tonic-gate 17257c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 17267c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 17277c478bd9Sstevel@tonic-gate 17287c478bd9Sstevel@tonic-gate if (msp && try == MUTEX_TRY) 17297c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try); 17307c478bd9Sstevel@tonic-gate 1731*883492d5Sraf if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) 1732*883492d5Sraf return (mutex_recursion(mp, mtype, try)); 17337c478bd9Sstevel@tonic-gate 17347c478bd9Sstevel@tonic-gate if (self->ul_error_detection && try == MUTEX_LOCK && 17357c478bd9Sstevel@tonic-gate tsp == NULL && mutex_is_held(mp)) 17367c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_lock", NULL, NULL); 17377c478bd9Sstevel@tonic-gate 1738*883492d5Sraf if (mtype & LOCK_PRIO_PROTECT) { 1739*883492d5Sraf ceil = mp->mutex_ceiling; 1740*883492d5Sraf ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0); 1741*883492d5Sraf myprio = real_priority(self); 1742*883492d5Sraf if (myprio > ceil) { 1743*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EINVAL); 1744*883492d5Sraf return (EINVAL); 17457c478bd9Sstevel@tonic-gate } 1746*883492d5Sraf if ((error = _ceil_mylist_add(mp)) != 0) { 1747*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1748*883492d5Sraf return (error); 17497c478bd9Sstevel@tonic-gate } 1750*883492d5Sraf if (myprio < ceil) 1751*883492d5Sraf _ceil_prio_inherit(ceil); 1752*883492d5Sraf } 17537c478bd9Sstevel@tonic-gate 1754*883492d5Sraf if ((mtype & (USYNC_PROCESS | LOCK_ROBUST)) 1755*883492d5Sraf == (USYNC_PROCESS | LOCK_ROBUST)) 1756*883492d5Sraf register_lock(mp); 1757*883492d5Sraf 1758*883492d5Sraf if (mtype & LOCK_PRIO_INHERIT) { 1759*883492d5Sraf /* go straight to the kernel */ 1760*883492d5Sraf if (try == MUTEX_TRY) 1761*883492d5Sraf error = mutex_trylock_kernel(mp); 1762*883492d5Sraf else /* MUTEX_LOCK */ 1763*883492d5Sraf error = mutex_lock_kernel(mp, tsp, msp); 17647c478bd9Sstevel@tonic-gate /* 1765*883492d5Sraf * The kernel never sets or clears the lock byte 1766*883492d5Sraf * for LOCK_PRIO_INHERIT mutexes. 1767*883492d5Sraf * Set it here for consistency. 17687c478bd9Sstevel@tonic-gate */ 1769*883492d5Sraf switch (error) { 1770*883492d5Sraf case 0: 1771*883492d5Sraf mp->mutex_lockw = LOCKSET; 1772*883492d5Sraf break; 1773*883492d5Sraf case EOWNERDEAD: 1774*883492d5Sraf case ELOCKUNMAPPED: 1775*883492d5Sraf mp->mutex_lockw = LOCKSET; 1776*883492d5Sraf /* FALLTHROUGH */ 1777*883492d5Sraf case ENOTRECOVERABLE: 1778*883492d5Sraf ASSERT(mtype & LOCK_ROBUST); 1779*883492d5Sraf break; 1780*883492d5Sraf case EDEADLK: 1781*883492d5Sraf if (try == MUTEX_LOCK) 1782*883492d5Sraf stall(); 1783*883492d5Sraf error = EBUSY; 1784*883492d5Sraf break; 17857c478bd9Sstevel@tonic-gate } 1786*883492d5Sraf } else if (mtype & USYNC_PROCESS) { 1787*883492d5Sraf error = mutex_trylock_process(mp); 1788*883492d5Sraf if (error == EBUSY && try == MUTEX_LOCK) 17897c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 17907c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 1791*883492d5Sraf error = mutex_trylock_adaptive(mp); 1792*883492d5Sraf if (error == EBUSY && try == MUTEX_LOCK) 1793*883492d5Sraf error = mutex_lock_queue(self, msp, mp, tsp); 17947c478bd9Sstevel@tonic-gate } 17957c478bd9Sstevel@tonic-gate 17967c478bd9Sstevel@tonic-gate switch (error) { 1797*883492d5Sraf case 0: 17987c478bd9Sstevel@tonic-gate case EOWNERDEAD: 17997c478bd9Sstevel@tonic-gate case ELOCKUNMAPPED: 1800*883492d5Sraf if (mtype & LOCK_ROBUST) 1801*883492d5Sraf remember_lock(mp); 18027c478bd9Sstevel@tonic-gate if (msp) 18037c478bd9Sstevel@tonic-gate record_begin_hold(msp); 18047c478bd9Sstevel@tonic-gate break; 18057c478bd9Sstevel@tonic-gate default: 1806*883492d5Sraf if (mtype & LOCK_PRIO_PROTECT) { 1807*883492d5Sraf (void) _ceil_mylist_del(mp); 1808*883492d5Sraf if (myprio < ceil) 1809*883492d5Sraf _ceil_prio_waive(); 1810*883492d5Sraf } 18117c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) { 18127c478bd9Sstevel@tonic-gate if (msp) 18137c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try_fail); 18147c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 18157c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 18167c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 18177c478bd9Sstevel@tonic-gate } 18187c478bd9Sstevel@tonic-gate } 18197c478bd9Sstevel@tonic-gate break; 18207c478bd9Sstevel@tonic-gate } 18217c478bd9Sstevel@tonic-gate 18227c478bd9Sstevel@tonic-gate return (error); 18237c478bd9Sstevel@tonic-gate } 18247c478bd9Sstevel@tonic-gate 18257c478bd9Sstevel@tonic-gate int 18267c478bd9Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try) 18277c478bd9Sstevel@tonic-gate { 18287c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 18297c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 18307c478bd9Sstevel@tonic-gate 18317c478bd9Sstevel@tonic-gate /* 18327c478bd9Sstevel@tonic-gate * We know that USYNC_PROCESS is set in mtype and that 18337c478bd9Sstevel@tonic-gate * zero, one, or both of the flags LOCK_RECURSIVE and 18347c478bd9Sstevel@tonic-gate * LOCK_ERRORCHECK are set, and that no other flags are set. 18357c478bd9Sstevel@tonic-gate */ 1836*883492d5Sraf ASSERT((mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0); 18377c478bd9Sstevel@tonic-gate enter_critical(self); 18387c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 18397c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 18407c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 18417c478bd9Sstevel@tonic-gate exit_critical(self); 18427c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 18437c478bd9Sstevel@tonic-gate return (0); 18447c478bd9Sstevel@tonic-gate } 18457c478bd9Sstevel@tonic-gate exit_critical(self); 18467c478bd9Sstevel@tonic-gate 1847*883492d5Sraf if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && shared_mutex_held(mp)) 1848*883492d5Sraf return (mutex_recursion(mp, mtype, try)); 18497c478bd9Sstevel@tonic-gate 1850*883492d5Sraf /* try a little harder */ 1851*883492d5Sraf if (mutex_trylock_process(mp) == 0) 18527c478bd9Sstevel@tonic-gate return (0); 18537c478bd9Sstevel@tonic-gate 18547c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) 18557c478bd9Sstevel@tonic-gate return (mutex_lock_kernel(mp, tsp, NULL)); 18567c478bd9Sstevel@tonic-gate 18577c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 18587c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 18597c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 18607c478bd9Sstevel@tonic-gate } 18617c478bd9Sstevel@tonic-gate return (EBUSY); 18627c478bd9Sstevel@tonic-gate } 18637c478bd9Sstevel@tonic-gate 18647c478bd9Sstevel@tonic-gate static int 18657c478bd9Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp) 18667c478bd9Sstevel@tonic-gate { 18677c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 18687c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 18697c478bd9Sstevel@tonic-gate uberflags_t *gflags; 18707c478bd9Sstevel@tonic-gate int mtype; 18717c478bd9Sstevel@tonic-gate 18727c478bd9Sstevel@tonic-gate /* 18737c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 18747c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 18757c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 18767c478bd9Sstevel@tonic-gate * and the process has only a single thread. 18777c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 18787c478bd9Sstevel@tonic-gate */ 18797c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 18807c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 18817c478bd9Sstevel@tonic-gate /* 18827c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 18837c478bd9Sstevel@tonic-gate */ 18847c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 18857c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 18867c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 18877c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 18887c478bd9Sstevel@tonic-gate return (0); 18897c478bd9Sstevel@tonic-gate } 1890*883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 1891*883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_LOCK)); 18927c478bd9Sstevel@tonic-gate /* 18937c478bd9Sstevel@tonic-gate * We have reached a deadlock, probably because the 18947c478bd9Sstevel@tonic-gate * process is executing non-async-signal-safe code in 18957c478bd9Sstevel@tonic-gate * a signal handler and is attempting to acquire a lock 18967c478bd9Sstevel@tonic-gate * that it already owns. This is not surprising, given 18977c478bd9Sstevel@tonic-gate * bad programming practices over the years that has 18987c478bd9Sstevel@tonic-gate * resulted in applications calling printf() and such 18997c478bd9Sstevel@tonic-gate * in their signal handlers. Unless the user has told 19007c478bd9Sstevel@tonic-gate * us that the signal handlers are safe by setting: 19017c478bd9Sstevel@tonic-gate * export _THREAD_ASYNC_SAFE=1 19027c478bd9Sstevel@tonic-gate * we return EDEADLK rather than actually deadlocking. 19037c478bd9Sstevel@tonic-gate */ 19047c478bd9Sstevel@tonic-gate if (tsp == NULL && 19057c478bd9Sstevel@tonic-gate MUTEX_OWNER(mp) == self && !self->ul_async_safe) { 19067c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 19077c478bd9Sstevel@tonic-gate return (EDEADLK); 19087c478bd9Sstevel@tonic-gate } 19097c478bd9Sstevel@tonic-gate } 19107c478bd9Sstevel@tonic-gate 19117c478bd9Sstevel@tonic-gate /* 19127c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 19137c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 19147c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 19157c478bd9Sstevel@tonic-gate */ 19167c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 19177c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 19187c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 19197c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 19207c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK)); 19217c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 19227c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 19237c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 19247c478bd9Sstevel@tonic-gate return (0); 19257c478bd9Sstevel@tonic-gate } 1926*883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 1927*883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_LOCK)); 1928*883492d5Sraf if (mutex_trylock_adaptive(mp) != 0) 1929*883492d5Sraf return (mutex_lock_queue(self, NULL, mp, tsp)); 1930*883492d5Sraf return (0); 19317c478bd9Sstevel@tonic-gate } 19327c478bd9Sstevel@tonic-gate 19337c478bd9Sstevel@tonic-gate /* else do it the long way */ 19347c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, tsp, MUTEX_LOCK)); 19357c478bd9Sstevel@tonic-gate } 19367c478bd9Sstevel@tonic-gate 19377c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_lock = __mutex_lock 19387c478bd9Sstevel@tonic-gate #pragma weak mutex_lock = __mutex_lock 19397c478bd9Sstevel@tonic-gate #pragma weak _mutex_lock = __mutex_lock 19407c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_lock = __mutex_lock 19417c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_lock = __mutex_lock 19427c478bd9Sstevel@tonic-gate int 19437c478bd9Sstevel@tonic-gate __mutex_lock(mutex_t *mp) 19447c478bd9Sstevel@tonic-gate { 19457c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 19467c478bd9Sstevel@tonic-gate return (mutex_lock_impl(mp, NULL)); 19477c478bd9Sstevel@tonic-gate } 19487c478bd9Sstevel@tonic-gate 19497c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_timedlock = _pthread_mutex_timedlock 19507c478bd9Sstevel@tonic-gate int 19517c478bd9Sstevel@tonic-gate _pthread_mutex_timedlock(mutex_t *mp, const timespec_t *abstime) 19527c478bd9Sstevel@tonic-gate { 19537c478bd9Sstevel@tonic-gate timespec_t tslocal; 19547c478bd9Sstevel@tonic-gate int error; 19557c478bd9Sstevel@tonic-gate 19567c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 19577c478bd9Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 19587c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 19597c478bd9Sstevel@tonic-gate if (error == ETIME) 19607c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 19617c478bd9Sstevel@tonic-gate return (error); 19627c478bd9Sstevel@tonic-gate } 19637c478bd9Sstevel@tonic-gate 19647c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_reltimedlock_np = _pthread_mutex_reltimedlock_np 19657c478bd9Sstevel@tonic-gate int 19667c478bd9Sstevel@tonic-gate _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime) 19677c478bd9Sstevel@tonic-gate { 19687c478bd9Sstevel@tonic-gate timespec_t tslocal; 19697c478bd9Sstevel@tonic-gate int error; 19707c478bd9Sstevel@tonic-gate 19717c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 19727c478bd9Sstevel@tonic-gate tslocal = *reltime; 19737c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 19747c478bd9Sstevel@tonic-gate if (error == ETIME) 19757c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 19767c478bd9Sstevel@tonic-gate return (error); 19777c478bd9Sstevel@tonic-gate } 19787c478bd9Sstevel@tonic-gate 19797c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_trylock = __mutex_trylock 19807c478bd9Sstevel@tonic-gate #pragma weak mutex_trylock = __mutex_trylock 19817c478bd9Sstevel@tonic-gate #pragma weak _mutex_trylock = __mutex_trylock 19827c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_trylock = __mutex_trylock 19837c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_trylock = __mutex_trylock 19847c478bd9Sstevel@tonic-gate int 19857c478bd9Sstevel@tonic-gate __mutex_trylock(mutex_t *mp) 19867c478bd9Sstevel@tonic-gate { 19877c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 19887c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 19897c478bd9Sstevel@tonic-gate uberflags_t *gflags; 19907c478bd9Sstevel@tonic-gate int mtype; 19917c478bd9Sstevel@tonic-gate 19927c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 19937c478bd9Sstevel@tonic-gate /* 19947c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 19957c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 19967c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 19977c478bd9Sstevel@tonic-gate * and the process has only a single thread. 19987c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 19997c478bd9Sstevel@tonic-gate */ 20007c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 20017c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 20027c478bd9Sstevel@tonic-gate /* 20037c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 20047c478bd9Sstevel@tonic-gate */ 20057c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 20067c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 20077c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 20087c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 20097c478bd9Sstevel@tonic-gate return (0); 20107c478bd9Sstevel@tonic-gate } 2011*883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 2012*883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_TRY)); 20137c478bd9Sstevel@tonic-gate return (EBUSY); 20147c478bd9Sstevel@tonic-gate } 20157c478bd9Sstevel@tonic-gate 20167c478bd9Sstevel@tonic-gate /* 20177c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 20187c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 20197c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 20207c478bd9Sstevel@tonic-gate */ 20217c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 20227c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 20237c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 20247c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 20257c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY)); 20267c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 20277c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 20287c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 20297c478bd9Sstevel@tonic-gate return (0); 20307c478bd9Sstevel@tonic-gate } 2031*883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 2032*883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_TRY)); 2033*883492d5Sraf if (mutex_trylock_adaptive(mp) != 0) { 2034*883492d5Sraf if (__td_event_report(self, TD_LOCK_TRY, udp)) { 2035*883492d5Sraf self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 2036*883492d5Sraf tdb_event(TD_LOCK_TRY, udp); 20377c478bd9Sstevel@tonic-gate } 2038*883492d5Sraf return (EBUSY); 20397c478bd9Sstevel@tonic-gate } 2040*883492d5Sraf return (0); 20417c478bd9Sstevel@tonic-gate } 20427c478bd9Sstevel@tonic-gate 20437c478bd9Sstevel@tonic-gate /* else do it the long way */ 20447c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, NULL, MUTEX_TRY)); 20457c478bd9Sstevel@tonic-gate } 20467c478bd9Sstevel@tonic-gate 20477c478bd9Sstevel@tonic-gate int 2048*883492d5Sraf mutex_unlock_internal(mutex_t *mp, int retain_robust_flags) 20497c478bd9Sstevel@tonic-gate { 20507c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 20517c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 20527c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 20537c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp; 2054*883492d5Sraf int error = 0; 2055*883492d5Sraf int release_all; 20567c478bd9Sstevel@tonic-gate lwpid_t lwpid; 20577c478bd9Sstevel@tonic-gate 20587c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp)) 20597c478bd9Sstevel@tonic-gate return (EPERM); 20607c478bd9Sstevel@tonic-gate 20617c478bd9Sstevel@tonic-gate if (self->ul_error_detection && !mutex_is_held(mp)) 20627c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_unlock", NULL, NULL); 20637c478bd9Sstevel@tonic-gate 20647c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 20657c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 20667c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 20677c478bd9Sstevel@tonic-gate return (0); 20687c478bd9Sstevel@tonic-gate } 20697c478bd9Sstevel@tonic-gate 20707c478bd9Sstevel@tonic-gate if ((msp = MUTEX_STATS(mp, udp)) != NULL) 20717c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 20727c478bd9Sstevel@tonic-gate 2073*883492d5Sraf if (!retain_robust_flags && !(mtype & LOCK_PRIO_INHERIT) && 2074*883492d5Sraf (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) { 2075*883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 2076*883492d5Sraf mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED); 2077*883492d5Sraf mp->mutex_flag |= LOCK_NOTRECOVERABLE; 2078*883492d5Sraf } 2079*883492d5Sraf release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0); 2080*883492d5Sraf 2081*883492d5Sraf if (mtype & LOCK_PRIO_INHERIT) { 20827c478bd9Sstevel@tonic-gate no_preempt(self); 20837c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 20847c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 20857c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 2086*883492d5Sraf mp->mutex_lockw = LOCKCLEAR; 2087*883492d5Sraf error = ___lwp_mutex_unlock(mp); 20887c478bd9Sstevel@tonic-gate preempt(self); 20897c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) { 2090*883492d5Sraf if (mp->mutex_lockword & WAITERMASK) { 2091*883492d5Sraf mutex_unlock_process(mp, release_all); 2092*883492d5Sraf } else { 20937c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 20947c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 20957c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 209641efec22Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & 2097*883492d5Sraf WAITERMASK) { /* a waiter suddenly appeared */ 20987c478bd9Sstevel@tonic-gate no_preempt(self); 2099*883492d5Sraf (void) ___lwp_mutex_wakeup(mp, release_all); 21007c478bd9Sstevel@tonic-gate preempt(self); 21017c478bd9Sstevel@tonic-gate } 21027c478bd9Sstevel@tonic-gate } 21037c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 2104*883492d5Sraf if ((lwpid = mutex_unlock_queue(mp, release_all)) != 0) { 21057c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 21067c478bd9Sstevel@tonic-gate preempt(self); 21077c478bd9Sstevel@tonic-gate } 21087c478bd9Sstevel@tonic-gate } 21097c478bd9Sstevel@tonic-gate 2110*883492d5Sraf if (mtype & LOCK_ROBUST) 2111*883492d5Sraf forget_lock(mp); 2112*883492d5Sraf 2113*883492d5Sraf if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp)) 2114*883492d5Sraf _ceil_prio_waive(); 2115*883492d5Sraf 21167c478bd9Sstevel@tonic-gate return (error); 21177c478bd9Sstevel@tonic-gate } 21187c478bd9Sstevel@tonic-gate 21197c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_unlock = __mutex_unlock 21207c478bd9Sstevel@tonic-gate #pragma weak mutex_unlock = __mutex_unlock 21217c478bd9Sstevel@tonic-gate #pragma weak _mutex_unlock = __mutex_unlock 21227c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_unlock = __mutex_unlock 21237c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_unlock = __mutex_unlock 21247c478bd9Sstevel@tonic-gate int 21257c478bd9Sstevel@tonic-gate __mutex_unlock(mutex_t *mp) 21267c478bd9Sstevel@tonic-gate { 21277c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 21287c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 21297c478bd9Sstevel@tonic-gate uberflags_t *gflags; 21307c478bd9Sstevel@tonic-gate lwpid_t lwpid; 21317c478bd9Sstevel@tonic-gate int mtype; 21327c478bd9Sstevel@tonic-gate short el; 21337c478bd9Sstevel@tonic-gate 21347c478bd9Sstevel@tonic-gate /* 21357c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 21367c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 21377c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 21387c478bd9Sstevel@tonic-gate * and the process has only a single thread. 21397c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 21407c478bd9Sstevel@tonic-gate */ 21417c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 21427c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 21437c478bd9Sstevel@tonic-gate if (mtype) { 21447c478bd9Sstevel@tonic-gate /* 21457c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 21467c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 21477c478bd9Sstevel@tonic-gate */ 21487c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 21497c478bd9Sstevel@tonic-gate return (EPERM); 21507c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 21517c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 21527c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 21537c478bd9Sstevel@tonic-gate return (0); 21547c478bd9Sstevel@tonic-gate } 21557c478bd9Sstevel@tonic-gate } 21567c478bd9Sstevel@tonic-gate /* 21577c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 21587c478bd9Sstevel@tonic-gate * Also, there can be no waiters. 21597c478bd9Sstevel@tonic-gate */ 21607c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 21617c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 21627c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 21637c478bd9Sstevel@tonic-gate return (0); 21647c478bd9Sstevel@tonic-gate } 21657c478bd9Sstevel@tonic-gate 21667c478bd9Sstevel@tonic-gate /* 21677c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 21687c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 21697c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 21707c478bd9Sstevel@tonic-gate */ 21717c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL) { 21727c478bd9Sstevel@tonic-gate if (((el = gflags->uf_trs_ted) | mtype) == 0) { 21737c478bd9Sstevel@tonic-gate fast_unlock: 21747c478bd9Sstevel@tonic-gate if (!(mp->mutex_lockword & WAITERMASK)) { 21757c478bd9Sstevel@tonic-gate /* no waiter exists right now */ 21767c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 21777c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 217841efec22Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & 21797c478bd9Sstevel@tonic-gate WAITERMASK) { 21807c478bd9Sstevel@tonic-gate /* a waiter suddenly appeared */ 21817c478bd9Sstevel@tonic-gate no_preempt(self); 21827c478bd9Sstevel@tonic-gate if ((lwpid = mutex_wakeup(mp)) != 0) 21837c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 21847c478bd9Sstevel@tonic-gate preempt(self); 21857c478bd9Sstevel@tonic-gate } 2186*883492d5Sraf } else if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) { 21877c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 21887c478bd9Sstevel@tonic-gate preempt(self); 21897c478bd9Sstevel@tonic-gate } 21907c478bd9Sstevel@tonic-gate return (0); 21917c478bd9Sstevel@tonic-gate } 21927c478bd9Sstevel@tonic-gate if (el) /* error detection or lock statistics */ 21937c478bd9Sstevel@tonic-gate goto slow_unlock; 21947c478bd9Sstevel@tonic-gate if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 21957c478bd9Sstevel@tonic-gate /* 21967c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 21977c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 21987c478bd9Sstevel@tonic-gate */ 21997c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 22007c478bd9Sstevel@tonic-gate return (EPERM); 22017c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 22027c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 22037c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 22047c478bd9Sstevel@tonic-gate return (0); 22057c478bd9Sstevel@tonic-gate } 22067c478bd9Sstevel@tonic-gate goto fast_unlock; 22077c478bd9Sstevel@tonic-gate } 22087c478bd9Sstevel@tonic-gate if ((mtype & 22097c478bd9Sstevel@tonic-gate ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 22107c478bd9Sstevel@tonic-gate /* 22117c478bd9Sstevel@tonic-gate * At this point we know that zero, one, or both of the 22127c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and 22137c478bd9Sstevel@tonic-gate * that the USYNC_PROCESS flag is set. 22147c478bd9Sstevel@tonic-gate */ 22157c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp)) 22167c478bd9Sstevel@tonic-gate return (EPERM); 22177c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 22187c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 22197c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 22207c478bd9Sstevel@tonic-gate return (0); 22217c478bd9Sstevel@tonic-gate } 2222*883492d5Sraf if (mp->mutex_lockword & WAITERMASK) { 2223*883492d5Sraf mutex_unlock_process(mp, 0); 2224*883492d5Sraf } else { 22257c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 22267c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 22277c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 222841efec22Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & 22297c478bd9Sstevel@tonic-gate WAITERMASK) { 22307c478bd9Sstevel@tonic-gate no_preempt(self); 2231*883492d5Sraf (void) ___lwp_mutex_wakeup(mp, 0); 22327c478bd9Sstevel@tonic-gate preempt(self); 22337c478bd9Sstevel@tonic-gate } 22347c478bd9Sstevel@tonic-gate } 22357c478bd9Sstevel@tonic-gate return (0); 22367c478bd9Sstevel@tonic-gate } 22377c478bd9Sstevel@tonic-gate } 22387c478bd9Sstevel@tonic-gate 22397c478bd9Sstevel@tonic-gate /* else do it the long way */ 22407c478bd9Sstevel@tonic-gate slow_unlock: 2241*883492d5Sraf return (mutex_unlock_internal(mp, 0)); 22427c478bd9Sstevel@tonic-gate } 22437c478bd9Sstevel@tonic-gate 22447c478bd9Sstevel@tonic-gate /* 22457c478bd9Sstevel@tonic-gate * Internally to the library, almost all mutex lock/unlock actions 22467c478bd9Sstevel@tonic-gate * go through these lmutex_ functions, to protect critical regions. 22477c478bd9Sstevel@tonic-gate * We replicate a bit of code from __mutex_lock() and __mutex_unlock() 22487c478bd9Sstevel@tonic-gate * to make these functions faster since we know that the mutex type 22497c478bd9Sstevel@tonic-gate * of all internal locks is USYNC_THREAD. We also know that internal 22507c478bd9Sstevel@tonic-gate * locking can never fail, so we panic if it does. 22517c478bd9Sstevel@tonic-gate */ 22527c478bd9Sstevel@tonic-gate void 22537c478bd9Sstevel@tonic-gate lmutex_lock(mutex_t *mp) 22547c478bd9Sstevel@tonic-gate { 22557c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 22567c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 22577c478bd9Sstevel@tonic-gate 22587c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 22597c478bd9Sstevel@tonic-gate 22607c478bd9Sstevel@tonic-gate enter_critical(self); 22617c478bd9Sstevel@tonic-gate /* 22627c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 22637c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 22647c478bd9Sstevel@tonic-gate */ 22657c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 22667c478bd9Sstevel@tonic-gate /* 22677c478bd9Sstevel@tonic-gate * Only one thread exists; the mutex must be free. 22687c478bd9Sstevel@tonic-gate */ 22697c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_lockw == 0); 22707c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 22717c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 22727c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 22737c478bd9Sstevel@tonic-gate } else { 22747c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 22757c478bd9Sstevel@tonic-gate 22767c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 22777c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 22787c478bd9Sstevel@tonic-gate 22797c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 22807c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 22817c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 22827c478bd9Sstevel@tonic-gate } else if (mutex_trylock_adaptive(mp) != 0) { 22837c478bd9Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL); 22847c478bd9Sstevel@tonic-gate } 22857c478bd9Sstevel@tonic-gate 22867c478bd9Sstevel@tonic-gate if (msp) 22877c478bd9Sstevel@tonic-gate record_begin_hold(msp); 22887c478bd9Sstevel@tonic-gate } 22897c478bd9Sstevel@tonic-gate } 22907c478bd9Sstevel@tonic-gate 22917c478bd9Sstevel@tonic-gate void 22927c478bd9Sstevel@tonic-gate lmutex_unlock(mutex_t *mp) 22937c478bd9Sstevel@tonic-gate { 22947c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 22957c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 22967c478bd9Sstevel@tonic-gate 22977c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 22987c478bd9Sstevel@tonic-gate 22997c478bd9Sstevel@tonic-gate /* 23007c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 23017c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 23027c478bd9Sstevel@tonic-gate */ 23037c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 23047c478bd9Sstevel@tonic-gate /* 23057c478bd9Sstevel@tonic-gate * Only one thread exists so there can be no waiters. 23067c478bd9Sstevel@tonic-gate */ 23077c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 23087c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 23097c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 23107c478bd9Sstevel@tonic-gate } else { 23117c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 23127c478bd9Sstevel@tonic-gate lwpid_t lwpid; 23137c478bd9Sstevel@tonic-gate 23147c478bd9Sstevel@tonic-gate if (msp) 23157c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 2316*883492d5Sraf if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) { 23177c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 23187c478bd9Sstevel@tonic-gate preempt(self); 23197c478bd9Sstevel@tonic-gate } 23207c478bd9Sstevel@tonic-gate } 23217c478bd9Sstevel@tonic-gate exit_critical(self); 23227c478bd9Sstevel@tonic-gate } 23237c478bd9Sstevel@tonic-gate 2324f841f6adSraf /* 2325f841f6adSraf * For specialized code in libc, like the asynchronous i/o code, 2326f841f6adSraf * the following sig_*() locking primitives are used in order 2327f841f6adSraf * to make the code asynchronous signal safe. Signals are 2328f841f6adSraf * deferred while locks acquired by these functions are held. 2329f841f6adSraf */ 2330f841f6adSraf void 2331f841f6adSraf sig_mutex_lock(mutex_t *mp) 2332f841f6adSraf { 2333f841f6adSraf sigoff(curthread); 2334f841f6adSraf (void) _private_mutex_lock(mp); 2335f841f6adSraf } 2336f841f6adSraf 2337f841f6adSraf void 2338f841f6adSraf sig_mutex_unlock(mutex_t *mp) 2339f841f6adSraf { 2340f841f6adSraf (void) _private_mutex_unlock(mp); 2341f841f6adSraf sigon(curthread); 2342f841f6adSraf } 2343f841f6adSraf 2344f841f6adSraf int 2345f841f6adSraf sig_mutex_trylock(mutex_t *mp) 2346f841f6adSraf { 2347f841f6adSraf int error; 2348f841f6adSraf 2349f841f6adSraf sigoff(curthread); 2350f841f6adSraf if ((error = _private_mutex_trylock(mp)) != 0) 2351f841f6adSraf sigon(curthread); 2352f841f6adSraf return (error); 2353f841f6adSraf } 2354f841f6adSraf 2355f841f6adSraf /* 2356f841f6adSraf * sig_cond_wait() is a cancellation point. 2357f841f6adSraf */ 2358f841f6adSraf int 2359f841f6adSraf sig_cond_wait(cond_t *cv, mutex_t *mp) 2360f841f6adSraf { 2361f841f6adSraf int error; 2362f841f6adSraf 2363f841f6adSraf ASSERT(curthread->ul_sigdefer != 0); 2364f841f6adSraf _private_testcancel(); 2365f841f6adSraf error = _cond_wait(cv, mp); 2366f841f6adSraf if (error == EINTR && curthread->ul_cursig) { 2367f841f6adSraf sig_mutex_unlock(mp); 2368f841f6adSraf /* take the deferred signal here */ 2369f841f6adSraf sig_mutex_lock(mp); 2370f841f6adSraf } 2371f841f6adSraf _private_testcancel(); 2372f841f6adSraf return (error); 2373f841f6adSraf } 2374f841f6adSraf 2375f841f6adSraf /* 2376f841f6adSraf * sig_cond_reltimedwait() is a cancellation point. 2377f841f6adSraf */ 2378f841f6adSraf int 2379f841f6adSraf sig_cond_reltimedwait(cond_t *cv, mutex_t *mp, const timespec_t *ts) 2380f841f6adSraf { 2381f841f6adSraf int error; 2382f841f6adSraf 2383f841f6adSraf ASSERT(curthread->ul_sigdefer != 0); 2384f841f6adSraf _private_testcancel(); 2385f841f6adSraf error = _cond_reltimedwait(cv, mp, ts); 2386f841f6adSraf if (error == EINTR && curthread->ul_cursig) { 2387f841f6adSraf sig_mutex_unlock(mp); 2388f841f6adSraf /* take the deferred signal here */ 2389f841f6adSraf sig_mutex_lock(mp); 2390f841f6adSraf } 2391f841f6adSraf _private_testcancel(); 2392f841f6adSraf return (error); 2393f841f6adSraf } 2394f841f6adSraf 23957c478bd9Sstevel@tonic-gate static int 23967c478bd9Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg) 23977c478bd9Sstevel@tonic-gate { 23987c478bd9Sstevel@tonic-gate /* 2399*883492d5Sraf * The 'volatile' is necessary to make sure the compiler doesn't 2400*883492d5Sraf * reorder the tests of the various components of the mutex. 2401*883492d5Sraf * They must be tested in this order: 2402*883492d5Sraf * mutex_lockw 2403*883492d5Sraf * mutex_owner 2404*883492d5Sraf * mutex_ownerpid 2405*883492d5Sraf * This relies on the fact that everywhere mutex_lockw is cleared, 2406*883492d5Sraf * mutex_owner and mutex_ownerpid are cleared before mutex_lockw 2407*883492d5Sraf * is cleared, and that everywhere mutex_lockw is set, mutex_owner 2408*883492d5Sraf * and mutex_ownerpid are set after mutex_lockw is set, and that 2409*883492d5Sraf * mutex_lockw is set or cleared with a memory barrier. 24107c478bd9Sstevel@tonic-gate */ 24117c478bd9Sstevel@tonic-gate volatile mutex_t *mp = (volatile mutex_t *)mparg; 24127c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 24137c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 24147c478bd9Sstevel@tonic-gate 2415*883492d5Sraf return (MUTEX_OWNED(mp, self) && mp->mutex_ownerpid == udp->pid); 24167c478bd9Sstevel@tonic-gate } 24177c478bd9Sstevel@tonic-gate 24187c478bd9Sstevel@tonic-gate /* 24197c478bd9Sstevel@tonic-gate * Some crufty old programs define their own version of _mutex_held() 24207c478bd9Sstevel@tonic-gate * to be simply return(1). This breaks internal libc logic, so we 24217c478bd9Sstevel@tonic-gate * define a private version for exclusive use by libc, mutex_is_held(), 24227c478bd9Sstevel@tonic-gate * and also a new public function, __mutex_held(), to be used in new 24237c478bd9Sstevel@tonic-gate * code to circumvent these crufty old programs. 24247c478bd9Sstevel@tonic-gate */ 24257c478bd9Sstevel@tonic-gate #pragma weak mutex_held = mutex_is_held 24267c478bd9Sstevel@tonic-gate #pragma weak _mutex_held = mutex_is_held 24277c478bd9Sstevel@tonic-gate #pragma weak __mutex_held = mutex_is_held 24287c478bd9Sstevel@tonic-gate int 2429*883492d5Sraf mutex_is_held(mutex_t *mparg) 24307c478bd9Sstevel@tonic-gate { 2431*883492d5Sraf volatile mutex_t *mp = (volatile mutex_t *)mparg; 2432*883492d5Sraf 2433*883492d5Sraf if (mparg->mutex_type & USYNC_PROCESS) 2434*883492d5Sraf return (shared_mutex_held(mparg)); 24357c478bd9Sstevel@tonic-gate return (MUTEX_OWNED(mp, curthread)); 24367c478bd9Sstevel@tonic-gate } 24377c478bd9Sstevel@tonic-gate 24387c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_destroy = __mutex_destroy 24397c478bd9Sstevel@tonic-gate #pragma weak mutex_destroy = __mutex_destroy 24407c478bd9Sstevel@tonic-gate #pragma weak _mutex_destroy = __mutex_destroy 24417c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_destroy = __mutex_destroy 24427c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_destroy = __mutex_destroy 24437c478bd9Sstevel@tonic-gate int 24447c478bd9Sstevel@tonic-gate __mutex_destroy(mutex_t *mp) 24457c478bd9Sstevel@tonic-gate { 2446*883492d5Sraf if (mp->mutex_type & USYNC_PROCESS) 2447*883492d5Sraf forget_lock(mp); 2448*883492d5Sraf (void) _memset(mp, 0, sizeof (*mp)); 24497c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(mp); 24507c478bd9Sstevel@tonic-gate return (0); 24517c478bd9Sstevel@tonic-gate } 24527c478bd9Sstevel@tonic-gate 2453*883492d5Sraf #pragma weak mutex_consistent = __mutex_consistent 2454*883492d5Sraf #pragma weak _mutex_consistent = __mutex_consistent 2455*883492d5Sraf #pragma weak pthread_mutex_consistent_np = __mutex_consistent 2456*883492d5Sraf #pragma weak _pthread_mutex_consistent_np = __mutex_consistent 2457*883492d5Sraf int 2458*883492d5Sraf __mutex_consistent(mutex_t *mp) 2459*883492d5Sraf { 2460*883492d5Sraf /* 2461*883492d5Sraf * Do this only for an inconsistent, initialized robust lock 2462*883492d5Sraf * that we hold. For all other cases, return EINVAL. 2463*883492d5Sraf */ 2464*883492d5Sraf if (mutex_is_held(mp) && 2465*883492d5Sraf (mp->mutex_type & LOCK_ROBUST) && 2466*883492d5Sraf (mp->mutex_flag & LOCK_INITED) && 2467*883492d5Sraf (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) { 2468*883492d5Sraf mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED); 2469*883492d5Sraf mp->mutex_rcount = 0; 2470*883492d5Sraf return (0); 2471*883492d5Sraf } 2472*883492d5Sraf return (EINVAL); 2473*883492d5Sraf } 2474*883492d5Sraf 24757c478bd9Sstevel@tonic-gate /* 24767c478bd9Sstevel@tonic-gate * Spin locks are separate from ordinary mutexes, 24777c478bd9Sstevel@tonic-gate * but we use the same data structure for them. 24787c478bd9Sstevel@tonic-gate */ 24797c478bd9Sstevel@tonic-gate 24807c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_init = _pthread_spin_init 24817c478bd9Sstevel@tonic-gate int 24827c478bd9Sstevel@tonic-gate _pthread_spin_init(pthread_spinlock_t *lock, int pshared) 24837c478bd9Sstevel@tonic-gate { 24847c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 24857c478bd9Sstevel@tonic-gate 24867c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 24877c478bd9Sstevel@tonic-gate if (pshared == PTHREAD_PROCESS_SHARED) 24887c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_PROCESS; 24897c478bd9Sstevel@tonic-gate else 24907c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_THREAD; 24917c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 24927c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 24937c478bd9Sstevel@tonic-gate return (0); 24947c478bd9Sstevel@tonic-gate } 24957c478bd9Sstevel@tonic-gate 24967c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_destroy = _pthread_spin_destroy 24977c478bd9Sstevel@tonic-gate int 24987c478bd9Sstevel@tonic-gate _pthread_spin_destroy(pthread_spinlock_t *lock) 24997c478bd9Sstevel@tonic-gate { 25007c478bd9Sstevel@tonic-gate (void) _memset(lock, 0, sizeof (*lock)); 25017c478bd9Sstevel@tonic-gate return (0); 25027c478bd9Sstevel@tonic-gate } 25037c478bd9Sstevel@tonic-gate 25047c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_trylock = _pthread_spin_trylock 25057c478bd9Sstevel@tonic-gate int 25067c478bd9Sstevel@tonic-gate _pthread_spin_trylock(pthread_spinlock_t *lock) 25077c478bd9Sstevel@tonic-gate { 25087c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 25097c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 25107c478bd9Sstevel@tonic-gate int error = 0; 25117c478bd9Sstevel@tonic-gate 25127c478bd9Sstevel@tonic-gate no_preempt(self); 25137c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) != 0) 25147c478bd9Sstevel@tonic-gate error = EBUSY; 25157c478bd9Sstevel@tonic-gate else { 25167c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 25177c478bd9Sstevel@tonic-gate if (mp->mutex_type == USYNC_PROCESS) 25187c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = self->ul_uberdata->pid; 25197c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 25207c478bd9Sstevel@tonic-gate } 25217c478bd9Sstevel@tonic-gate preempt(self); 25227c478bd9Sstevel@tonic-gate return (error); 25237c478bd9Sstevel@tonic-gate } 25247c478bd9Sstevel@tonic-gate 25257c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_lock = _pthread_spin_lock 25267c478bd9Sstevel@tonic-gate int 25277c478bd9Sstevel@tonic-gate _pthread_spin_lock(pthread_spinlock_t *lock) 25287c478bd9Sstevel@tonic-gate { 2529*883492d5Sraf mutex_t *mp = (mutex_t *)lock; 2530*883492d5Sraf ulwp_t *self = curthread; 2531*883492d5Sraf volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw; 2532*883492d5Sraf int count = 0; 2533*883492d5Sraf 2534*883492d5Sraf ASSERT(!self->ul_critical || self->ul_bindflags); 2535*883492d5Sraf 2536*883492d5Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp); 25377c478bd9Sstevel@tonic-gate 25387c478bd9Sstevel@tonic-gate /* 25397c478bd9Sstevel@tonic-gate * We don't care whether the owner is running on a processor. 25407c478bd9Sstevel@tonic-gate * We just spin because that's what this interface requires. 25417c478bd9Sstevel@tonic-gate */ 25427c478bd9Sstevel@tonic-gate for (;;) { 2543*883492d5Sraf if (count < INT_MAX) 2544*883492d5Sraf count++; 25457c478bd9Sstevel@tonic-gate if (*lockp == 0) { /* lock byte appears to be clear */ 2546*883492d5Sraf no_preempt(self); 2547*883492d5Sraf if (set_lock_byte(lockp) == 0) 2548*883492d5Sraf break; 2549*883492d5Sraf preempt(self); 25507c478bd9Sstevel@tonic-gate } 25517c478bd9Sstevel@tonic-gate SMT_PAUSE(); 25527c478bd9Sstevel@tonic-gate } 2553*883492d5Sraf mp->mutex_owner = (uintptr_t)self; 2554*883492d5Sraf if (mp->mutex_type == USYNC_PROCESS) 2555*883492d5Sraf mp->mutex_ownerpid = self->ul_uberdata->pid; 2556*883492d5Sraf preempt(self); 2557*883492d5Sraf DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 2558*883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 2559*883492d5Sraf return (0); 25607c478bd9Sstevel@tonic-gate } 25617c478bd9Sstevel@tonic-gate 25627c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_unlock = _pthread_spin_unlock 25637c478bd9Sstevel@tonic-gate int 25647c478bd9Sstevel@tonic-gate _pthread_spin_unlock(pthread_spinlock_t *lock) 25657c478bd9Sstevel@tonic-gate { 25667c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 25677c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 25687c478bd9Sstevel@tonic-gate 25697c478bd9Sstevel@tonic-gate no_preempt(self); 25707c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 25717c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 25727c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 257341efec22Sraf (void) atomic_swap_32(&mp->mutex_lockword, 0); 25747c478bd9Sstevel@tonic-gate preempt(self); 25757c478bd9Sstevel@tonic-gate return (0); 25767c478bd9Sstevel@tonic-gate } 25777c478bd9Sstevel@tonic-gate 2578*883492d5Sraf #define INITIAL_LOCKS 8 /* initialial size of ul_heldlocks.array */ 2579*883492d5Sraf 2580*883492d5Sraf /* 2581*883492d5Sraf * Find/allocate an entry for 'lock' in our array of held locks. 2582*883492d5Sraf */ 2583*883492d5Sraf static mutex_t ** 2584*883492d5Sraf find_lock_entry(mutex_t *lock) 2585*883492d5Sraf { 2586*883492d5Sraf ulwp_t *self = curthread; 2587*883492d5Sraf mutex_t **remembered = NULL; 2588*883492d5Sraf mutex_t **lockptr; 2589*883492d5Sraf uint_t nlocks; 2590*883492d5Sraf 2591*883492d5Sraf if ((nlocks = self->ul_heldlockcnt) != 0) 2592*883492d5Sraf lockptr = self->ul_heldlocks.array; 2593*883492d5Sraf else { 2594*883492d5Sraf nlocks = 1; 2595*883492d5Sraf lockptr = &self->ul_heldlocks.single; 2596*883492d5Sraf } 2597*883492d5Sraf 2598*883492d5Sraf for (; nlocks; nlocks--, lockptr++) { 2599*883492d5Sraf if (*lockptr == lock) 2600*883492d5Sraf return (lockptr); 2601*883492d5Sraf if (*lockptr == NULL && remembered == NULL) 2602*883492d5Sraf remembered = lockptr; 2603*883492d5Sraf } 2604*883492d5Sraf if (remembered != NULL) { 2605*883492d5Sraf *remembered = lock; 2606*883492d5Sraf return (remembered); 2607*883492d5Sraf } 2608*883492d5Sraf 2609*883492d5Sraf /* 2610*883492d5Sraf * No entry available. Allocate more space, converting 2611*883492d5Sraf * the single entry into an array of entries if necessary. 2612*883492d5Sraf */ 2613*883492d5Sraf if ((nlocks = self->ul_heldlockcnt) == 0) { 2614*883492d5Sraf /* 2615*883492d5Sraf * Initial allocation of the array. 2616*883492d5Sraf * Convert the single entry into an array. 2617*883492d5Sraf */ 2618*883492d5Sraf self->ul_heldlockcnt = nlocks = INITIAL_LOCKS; 2619*883492d5Sraf lockptr = lmalloc(nlocks * sizeof (mutex_t *)); 2620*883492d5Sraf /* 2621*883492d5Sraf * The single entry becomes the first entry in the array. 2622*883492d5Sraf */ 2623*883492d5Sraf *lockptr = self->ul_heldlocks.single; 2624*883492d5Sraf self->ul_heldlocks.array = lockptr; 2625*883492d5Sraf /* 2626*883492d5Sraf * Return the next available entry in the array. 2627*883492d5Sraf */ 2628*883492d5Sraf *++lockptr = lock; 2629*883492d5Sraf return (lockptr); 2630*883492d5Sraf } 2631*883492d5Sraf /* 2632*883492d5Sraf * Reallocate the array, double the size each time. 2633*883492d5Sraf */ 2634*883492d5Sraf lockptr = lmalloc(nlocks * 2 * sizeof (mutex_t *)); 2635*883492d5Sraf (void) _memcpy(lockptr, self->ul_heldlocks.array, 2636*883492d5Sraf nlocks * sizeof (mutex_t *)); 2637*883492d5Sraf lfree(self->ul_heldlocks.array, nlocks * sizeof (mutex_t *)); 2638*883492d5Sraf self->ul_heldlocks.array = lockptr; 2639*883492d5Sraf self->ul_heldlockcnt *= 2; 2640*883492d5Sraf /* 2641*883492d5Sraf * Return the next available entry in the newly allocated array. 2642*883492d5Sraf */ 2643*883492d5Sraf *(lockptr += nlocks) = lock; 2644*883492d5Sraf return (lockptr); 2645*883492d5Sraf } 2646*883492d5Sraf 2647*883492d5Sraf /* 2648*883492d5Sraf * Insert 'lock' into our list of held locks. 2649*883492d5Sraf * Currently only used for LOCK_ROBUST mutexes. 2650*883492d5Sraf */ 2651*883492d5Sraf void 2652*883492d5Sraf remember_lock(mutex_t *lock) 2653*883492d5Sraf { 2654*883492d5Sraf (void) find_lock_entry(lock); 2655*883492d5Sraf } 2656*883492d5Sraf 2657*883492d5Sraf /* 2658*883492d5Sraf * Remove 'lock' from our list of held locks. 2659*883492d5Sraf * Currently only used for LOCK_ROBUST mutexes. 2660*883492d5Sraf */ 2661*883492d5Sraf void 2662*883492d5Sraf forget_lock(mutex_t *lock) 2663*883492d5Sraf { 2664*883492d5Sraf *find_lock_entry(lock) = NULL; 2665*883492d5Sraf } 2666*883492d5Sraf 2667*883492d5Sraf /* 2668*883492d5Sraf * Free the array of held locks. 2669*883492d5Sraf */ 2670*883492d5Sraf void 2671*883492d5Sraf heldlock_free(ulwp_t *ulwp) 2672*883492d5Sraf { 2673*883492d5Sraf uint_t nlocks; 2674*883492d5Sraf 2675*883492d5Sraf if ((nlocks = ulwp->ul_heldlockcnt) != 0) 2676*883492d5Sraf lfree(ulwp->ul_heldlocks.array, nlocks * sizeof (mutex_t *)); 2677*883492d5Sraf ulwp->ul_heldlockcnt = 0; 2678*883492d5Sraf ulwp->ul_heldlocks.array = NULL; 2679*883492d5Sraf } 2680*883492d5Sraf 2681*883492d5Sraf /* 2682*883492d5Sraf * Mark all held LOCK_ROBUST mutexes LOCK_OWNERDEAD. 2683*883492d5Sraf * Called from _thrp_exit() to deal with abandoned locks. 2684*883492d5Sraf */ 2685*883492d5Sraf void 2686*883492d5Sraf heldlock_exit(void) 2687*883492d5Sraf { 2688*883492d5Sraf ulwp_t *self = curthread; 2689*883492d5Sraf mutex_t **lockptr; 2690*883492d5Sraf uint_t nlocks; 2691*883492d5Sraf mutex_t *mp; 2692*883492d5Sraf 2693*883492d5Sraf if ((nlocks = self->ul_heldlockcnt) != 0) 2694*883492d5Sraf lockptr = self->ul_heldlocks.array; 2695*883492d5Sraf else { 2696*883492d5Sraf nlocks = 1; 2697*883492d5Sraf lockptr = &self->ul_heldlocks.single; 2698*883492d5Sraf } 2699*883492d5Sraf 2700*883492d5Sraf for (; nlocks; nlocks--, lockptr++) { 2701*883492d5Sraf /* 2702*883492d5Sraf * The kernel takes care of transitioning held 2703*883492d5Sraf * LOCK_PRIO_INHERIT mutexes to LOCK_OWNERDEAD. 2704*883492d5Sraf * We avoid that case here. 2705*883492d5Sraf */ 2706*883492d5Sraf if ((mp = *lockptr) != NULL && 2707*883492d5Sraf mutex_is_held(mp) && 2708*883492d5Sraf (mp->mutex_type & (LOCK_ROBUST | LOCK_PRIO_INHERIT)) == 2709*883492d5Sraf LOCK_ROBUST) { 2710*883492d5Sraf mp->mutex_rcount = 0; 2711*883492d5Sraf if (!(mp->mutex_flag & LOCK_UNMAPPED)) 2712*883492d5Sraf mp->mutex_flag |= LOCK_OWNERDEAD; 2713*883492d5Sraf (void) mutex_unlock_internal(mp, 1); 2714*883492d5Sraf } 2715*883492d5Sraf } 2716*883492d5Sraf 2717*883492d5Sraf heldlock_free(self); 2718*883492d5Sraf } 2719*883492d5Sraf 27207c478bd9Sstevel@tonic-gate #pragma weak cond_init = _cond_init 27217c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 27227c478bd9Sstevel@tonic-gate int 27237c478bd9Sstevel@tonic-gate _cond_init(cond_t *cvp, int type, void *arg) 27247c478bd9Sstevel@tonic-gate { 27257c478bd9Sstevel@tonic-gate if (type != USYNC_THREAD && type != USYNC_PROCESS) 27267c478bd9Sstevel@tonic-gate return (EINVAL); 27277c478bd9Sstevel@tonic-gate (void) _memset(cvp, 0, sizeof (*cvp)); 27287c478bd9Sstevel@tonic-gate cvp->cond_type = (uint16_t)type; 27297c478bd9Sstevel@tonic-gate cvp->cond_magic = COND_MAGIC; 27307c478bd9Sstevel@tonic-gate return (0); 27317c478bd9Sstevel@tonic-gate } 27327c478bd9Sstevel@tonic-gate 27337c478bd9Sstevel@tonic-gate /* 27347c478bd9Sstevel@tonic-gate * cond_sleep_queue(): utility function for cond_wait_queue(). 27357c478bd9Sstevel@tonic-gate * 27367c478bd9Sstevel@tonic-gate * Go to sleep on a condvar sleep queue, expect to be waked up 27377c478bd9Sstevel@tonic-gate * by someone calling cond_signal() or cond_broadcast() or due 27387c478bd9Sstevel@tonic-gate * to receiving a UNIX signal or being cancelled, or just simply 27397c478bd9Sstevel@tonic-gate * due to a spurious wakeup (like someome calling forkall()). 27407c478bd9Sstevel@tonic-gate * 27417c478bd9Sstevel@tonic-gate * The associated mutex is *not* reacquired before returning. 27427c478bd9Sstevel@tonic-gate * That must be done by the caller of cond_sleep_queue(). 27437c478bd9Sstevel@tonic-gate */ 2744*883492d5Sraf static int 27457c478bd9Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 27467c478bd9Sstevel@tonic-gate { 27477c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 27487c478bd9Sstevel@tonic-gate queue_head_t *qp; 27497c478bd9Sstevel@tonic-gate queue_head_t *mqp; 27507c478bd9Sstevel@tonic-gate lwpid_t lwpid; 27517c478bd9Sstevel@tonic-gate int signalled; 27527c478bd9Sstevel@tonic-gate int error; 2753*883492d5Sraf int release_all; 27547c478bd9Sstevel@tonic-gate 27557c478bd9Sstevel@tonic-gate /* 27567c478bd9Sstevel@tonic-gate * Put ourself on the CV sleep queue, unlock the mutex, then 27577c478bd9Sstevel@tonic-gate * park ourself and unpark a candidate lwp to grab the mutex. 27587c478bd9Sstevel@tonic-gate * We must go onto the CV sleep queue before dropping the 27597c478bd9Sstevel@tonic-gate * mutex in order to guarantee atomicity of the operation. 27607c478bd9Sstevel@tonic-gate */ 27617c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 27627c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 27637c478bd9Sstevel@tonic-gate enqueue(qp, self, cvp, CV); 27647c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 1; 27657c478bd9Sstevel@tonic-gate self->ul_cvmutex = mp; 27667c478bd9Sstevel@tonic-gate self->ul_cv_wake = (tsp != NULL); 27677c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 2768*883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) { 2769*883492d5Sraf mp->mutex_flag &= ~LOCK_OWNERDEAD; 2770*883492d5Sraf mp->mutex_flag |= LOCK_NOTRECOVERABLE; 2771*883492d5Sraf } 2772*883492d5Sraf release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0); 2773*883492d5Sraf lwpid = mutex_unlock_queue(mp, release_all); 27747c478bd9Sstevel@tonic-gate for (;;) { 27757c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 27767c478bd9Sstevel@tonic-gate queue_unlock(qp); 27777c478bd9Sstevel@tonic-gate if (lwpid != 0) { 27787c478bd9Sstevel@tonic-gate lwpid = preempt_unpark(self, lwpid); 27797c478bd9Sstevel@tonic-gate preempt(self); 27807c478bd9Sstevel@tonic-gate } 27817c478bd9Sstevel@tonic-gate /* 27827c478bd9Sstevel@tonic-gate * We may have a deferred signal present, 27837c478bd9Sstevel@tonic-gate * in which case we should return EINTR. 27847c478bd9Sstevel@tonic-gate * Also, we may have received a SIGCANCEL; if so 27857c478bd9Sstevel@tonic-gate * and we are cancelable we should return EINTR. 27867c478bd9Sstevel@tonic-gate * We force an immediate EINTR return from 27877c478bd9Sstevel@tonic-gate * __lwp_park() by turning our parking flag off. 27887c478bd9Sstevel@tonic-gate */ 27897c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 27907c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 27917c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 27927c478bd9Sstevel@tonic-gate /* 27937c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 27947c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 27957c478bd9Sstevel@tonic-gate */ 27967c478bd9Sstevel@tonic-gate error = __lwp_park(tsp, lwpid); 27977c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 27987c478bd9Sstevel@tonic-gate lwpid = 0; /* unpark the other lwp only once */ 27997c478bd9Sstevel@tonic-gate /* 28007c478bd9Sstevel@tonic-gate * We were waked up by cond_signal(), cond_broadcast(), 28017c478bd9Sstevel@tonic-gate * by an interrupt or timeout (EINTR or ETIME), 28027c478bd9Sstevel@tonic-gate * or we may just have gotten a spurious wakeup. 28037c478bd9Sstevel@tonic-gate */ 28047c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 28057c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 28067c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) 28077c478bd9Sstevel@tonic-gate break; 28087c478bd9Sstevel@tonic-gate /* 28097c478bd9Sstevel@tonic-gate * We are on either the condvar sleep queue or the 28102be60c5eSraf * mutex sleep queue. Break out of the sleep if we 28112be60c5eSraf * were interrupted or we timed out (EINTR or ETIME). 28127c478bd9Sstevel@tonic-gate * Else this is a spurious wakeup; continue the loop. 28137c478bd9Sstevel@tonic-gate */ 28142be60c5eSraf if (self->ul_sleepq == mqp) { /* mutex queue */ 28152be60c5eSraf if (error) { 28162be60c5eSraf mp->mutex_waiters = dequeue_self(mqp, mp); 28172be60c5eSraf break; 28182be60c5eSraf } 28192be60c5eSraf tsp = NULL; /* no more timeout */ 28202be60c5eSraf } else if (self->ul_sleepq == qp) { /* condvar queue */ 28217c478bd9Sstevel@tonic-gate if (error) { 28227c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = dequeue_self(qp, cvp); 28237c478bd9Sstevel@tonic-gate break; 28247c478bd9Sstevel@tonic-gate } 28257c478bd9Sstevel@tonic-gate /* 28267c478bd9Sstevel@tonic-gate * Else a spurious wakeup on the condvar queue. 28277c478bd9Sstevel@tonic-gate * __lwp_park() has already adjusted the timeout. 28287c478bd9Sstevel@tonic-gate */ 28297c478bd9Sstevel@tonic-gate } else { 28307c478bd9Sstevel@tonic-gate thr_panic("cond_sleep_queue(): thread not on queue"); 28317c478bd9Sstevel@tonic-gate } 28327c478bd9Sstevel@tonic-gate queue_unlock(mqp); 28337c478bd9Sstevel@tonic-gate } 28347c478bd9Sstevel@tonic-gate 28357c478bd9Sstevel@tonic-gate self->ul_sp = 0; 28367c478bd9Sstevel@tonic-gate ASSERT(self->ul_cvmutex == NULL && self->ul_cv_wake == 0); 28377c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 28387c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 28397c478bd9Sstevel@tonic-gate 28407c478bd9Sstevel@tonic-gate signalled = self->ul_signalled; 28417c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 28427c478bd9Sstevel@tonic-gate queue_unlock(qp); 28437c478bd9Sstevel@tonic-gate queue_unlock(mqp); 28447c478bd9Sstevel@tonic-gate 28457c478bd9Sstevel@tonic-gate /* 28467c478bd9Sstevel@tonic-gate * If we were concurrently cond_signal()d and any of: 28477c478bd9Sstevel@tonic-gate * received a UNIX signal, were cancelled, or got a timeout, 28487c478bd9Sstevel@tonic-gate * then perform another cond_signal() to avoid consuming it. 28497c478bd9Sstevel@tonic-gate */ 28507c478bd9Sstevel@tonic-gate if (error && signalled) 28517c478bd9Sstevel@tonic-gate (void) cond_signal_internal(cvp); 28527c478bd9Sstevel@tonic-gate 28537c478bd9Sstevel@tonic-gate return (error); 28547c478bd9Sstevel@tonic-gate } 28557c478bd9Sstevel@tonic-gate 28567c478bd9Sstevel@tonic-gate int 28577c478bd9Sstevel@tonic-gate cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp, 28587c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp) 28597c478bd9Sstevel@tonic-gate { 28607c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 28617c478bd9Sstevel@tonic-gate int error; 2862*883492d5Sraf int merror; 28637c478bd9Sstevel@tonic-gate 28647c478bd9Sstevel@tonic-gate /* 28657c478bd9Sstevel@tonic-gate * The old thread library was programmed to defer signals 28667c478bd9Sstevel@tonic-gate * while in cond_wait() so that the associated mutex would 28677c478bd9Sstevel@tonic-gate * be guaranteed to be held when the application signal 28687c478bd9Sstevel@tonic-gate * handler was invoked. 28697c478bd9Sstevel@tonic-gate * 28707c478bd9Sstevel@tonic-gate * We do not behave this way by default; the state of the 28717c478bd9Sstevel@tonic-gate * associated mutex in the signal handler is undefined. 28727c478bd9Sstevel@tonic-gate * 28737c478bd9Sstevel@tonic-gate * To accommodate applications that depend on the old 28747c478bd9Sstevel@tonic-gate * behavior, the _THREAD_COND_WAIT_DEFER environment 28757c478bd9Sstevel@tonic-gate * variable can be set to 1 and we will behave in the 28767c478bd9Sstevel@tonic-gate * old way with respect to cond_wait(). 28777c478bd9Sstevel@tonic-gate */ 28787c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 28797c478bd9Sstevel@tonic-gate sigoff(self); 28807c478bd9Sstevel@tonic-gate 28817c478bd9Sstevel@tonic-gate error = cond_sleep_queue(cvp, mp, tsp); 28827c478bd9Sstevel@tonic-gate 28837c478bd9Sstevel@tonic-gate /* 28847c478bd9Sstevel@tonic-gate * Reacquire the mutex. 28857c478bd9Sstevel@tonic-gate */ 2886*883492d5Sraf if ((merror = mutex_trylock_adaptive(mp)) == EBUSY) 2887*883492d5Sraf merror = mutex_lock_queue(self, msp, mp, NULL); 2888*883492d5Sraf if (merror) 2889*883492d5Sraf error = merror; 2890*883492d5Sraf if (msp && (merror == 0 || merror == EOWNERDEAD)) 28917c478bd9Sstevel@tonic-gate record_begin_hold(msp); 28927c478bd9Sstevel@tonic-gate 28937c478bd9Sstevel@tonic-gate /* 28947c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 28957c478bd9Sstevel@tonic-gate */ 28967c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 28977c478bd9Sstevel@tonic-gate sigon(self); 28987c478bd9Sstevel@tonic-gate 28997c478bd9Sstevel@tonic-gate return (error); 29007c478bd9Sstevel@tonic-gate } 29017c478bd9Sstevel@tonic-gate 29027c478bd9Sstevel@tonic-gate /* 29037c478bd9Sstevel@tonic-gate * cond_sleep_kernel(): utility function for cond_wait_kernel(). 29047c478bd9Sstevel@tonic-gate * See the comment ahead of cond_sleep_queue(), above. 29057c478bd9Sstevel@tonic-gate */ 2906*883492d5Sraf static int 29077c478bd9Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 29087c478bd9Sstevel@tonic-gate { 29097c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 29107c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 29117c478bd9Sstevel@tonic-gate int error; 29127c478bd9Sstevel@tonic-gate 2913*883492d5Sraf if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp)) 2914*883492d5Sraf _ceil_prio_waive(); 29157c478bd9Sstevel@tonic-gate 29167c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 29177c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 29187c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 29197c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 2920*883492d5Sraf if (mtype & LOCK_PRIO_INHERIT) 29217c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 29227c478bd9Sstevel@tonic-gate /* 29237c478bd9Sstevel@tonic-gate * ___lwp_cond_wait() returns immediately with EINTR if 29247c478bd9Sstevel@tonic-gate * set_parking_flag(self,0) is called on this lwp before it 29257c478bd9Sstevel@tonic-gate * goes to sleep in the kernel. sigacthandler() calls this 29267c478bd9Sstevel@tonic-gate * when a deferred signal is noted. This assures that we don't 29277c478bd9Sstevel@tonic-gate * get stuck in ___lwp_cond_wait() with all signals blocked 29287c478bd9Sstevel@tonic-gate * due to taking a deferred signal before going to sleep. 29297c478bd9Sstevel@tonic-gate */ 29307c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 29317c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 29327c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 29337c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 29347c478bd9Sstevel@tonic-gate error = ___lwp_cond_wait(cvp, mp, tsp, 1); 29357c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 29367c478bd9Sstevel@tonic-gate self->ul_sp = 0; 29377c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 29387c478bd9Sstevel@tonic-gate return (error); 29397c478bd9Sstevel@tonic-gate } 29407c478bd9Sstevel@tonic-gate 29417c478bd9Sstevel@tonic-gate int 29427c478bd9Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 29437c478bd9Sstevel@tonic-gate { 29447c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 29457c478bd9Sstevel@tonic-gate int error; 29467c478bd9Sstevel@tonic-gate int merror; 29477c478bd9Sstevel@tonic-gate 29487c478bd9Sstevel@tonic-gate /* 29497c478bd9Sstevel@tonic-gate * See the large comment in cond_wait_queue(), above. 29507c478bd9Sstevel@tonic-gate */ 29517c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 29527c478bd9Sstevel@tonic-gate sigoff(self); 29537c478bd9Sstevel@tonic-gate 29547c478bd9Sstevel@tonic-gate error = cond_sleep_kernel(cvp, mp, tsp); 29557c478bd9Sstevel@tonic-gate 29567c478bd9Sstevel@tonic-gate /* 29577c478bd9Sstevel@tonic-gate * Override the return code from ___lwp_cond_wait() 29587c478bd9Sstevel@tonic-gate * with any non-zero return code from mutex_lock(). 29597c478bd9Sstevel@tonic-gate * This addresses robust lock failures in particular; 29607c478bd9Sstevel@tonic-gate * the caller must see the EOWNERDEAD or ENOTRECOVERABLE 29617c478bd9Sstevel@tonic-gate * errors in order to take corrective action. 29627c478bd9Sstevel@tonic-gate */ 29637c478bd9Sstevel@tonic-gate if ((merror = _private_mutex_lock(mp)) != 0) 29647c478bd9Sstevel@tonic-gate error = merror; 29657c478bd9Sstevel@tonic-gate 29667c478bd9Sstevel@tonic-gate /* 29677c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 29687c478bd9Sstevel@tonic-gate */ 29697c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 29707c478bd9Sstevel@tonic-gate sigon(self); 29717c478bd9Sstevel@tonic-gate 29727c478bd9Sstevel@tonic-gate return (error); 29737c478bd9Sstevel@tonic-gate } 29747c478bd9Sstevel@tonic-gate 29757c478bd9Sstevel@tonic-gate /* 29767c478bd9Sstevel@tonic-gate * Common code for _cond_wait() and _cond_timedwait() 29777c478bd9Sstevel@tonic-gate */ 29787c478bd9Sstevel@tonic-gate int 29797c478bd9Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 29807c478bd9Sstevel@tonic-gate { 29817c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 29827c478bd9Sstevel@tonic-gate hrtime_t begin_sleep = 0; 29837c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 29847c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 29857c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 29867c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 29877c478bd9Sstevel@tonic-gate uint8_t rcount; 29887c478bd9Sstevel@tonic-gate int error = 0; 29897c478bd9Sstevel@tonic-gate 29907c478bd9Sstevel@tonic-gate /* 29917c478bd9Sstevel@tonic-gate * The SUSV3 Posix spec for pthread_cond_timedwait() states: 29927c478bd9Sstevel@tonic-gate * Except in the case of [ETIMEDOUT], all these error checks 29937c478bd9Sstevel@tonic-gate * shall act as if they were performed immediately at the 29947c478bd9Sstevel@tonic-gate * beginning of processing for the function and shall cause 29957c478bd9Sstevel@tonic-gate * an error return, in effect, prior to modifying the state 29967c478bd9Sstevel@tonic-gate * of the mutex specified by mutex or the condition variable 29977c478bd9Sstevel@tonic-gate * specified by cond. 29987c478bd9Sstevel@tonic-gate * Therefore, we must return EINVAL now if the timout is invalid. 29997c478bd9Sstevel@tonic-gate */ 30007c478bd9Sstevel@tonic-gate if (tsp != NULL && 30017c478bd9Sstevel@tonic-gate (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC)) 30027c478bd9Sstevel@tonic-gate return (EINVAL); 30037c478bd9Sstevel@tonic-gate 30047c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 30057c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 30067c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 30077c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 30087c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = cvp; 30097c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 30107c478bd9Sstevel@tonic-gate self->ul_sp = 0; 30117c478bd9Sstevel@tonic-gate } 30127c478bd9Sstevel@tonic-gate if (csp) { 30137c478bd9Sstevel@tonic-gate if (tsp) 30147c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait); 30157c478bd9Sstevel@tonic-gate else 30167c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_wait); 30177c478bd9Sstevel@tonic-gate } 30187c478bd9Sstevel@tonic-gate if (msp) 30197c478bd9Sstevel@tonic-gate begin_sleep = record_hold_time(msp); 30207c478bd9Sstevel@tonic-gate else if (csp) 30217c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 30227c478bd9Sstevel@tonic-gate 30237c478bd9Sstevel@tonic-gate if (self->ul_error_detection) { 30247c478bd9Sstevel@tonic-gate if (!mutex_is_held(mp)) 30257c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, NULL); 30267c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) 30277c478bd9Sstevel@tonic-gate lock_error(mp, "recursive mutex in cond_wait", 30287c478bd9Sstevel@tonic-gate cvp, NULL); 30297c478bd9Sstevel@tonic-gate if (cvp->cond_type & USYNC_PROCESS) { 3030*883492d5Sraf if (!(mtype & USYNC_PROCESS)) 30317c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 30327c478bd9Sstevel@tonic-gate "condvar process-shared, " 30337c478bd9Sstevel@tonic-gate "mutex process-private"); 30347c478bd9Sstevel@tonic-gate } else { 3035*883492d5Sraf if (mtype & USYNC_PROCESS) 30367c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 30377c478bd9Sstevel@tonic-gate "condvar process-private, " 30387c478bd9Sstevel@tonic-gate "mutex process-shared"); 30397c478bd9Sstevel@tonic-gate } 30407c478bd9Sstevel@tonic-gate } 30417c478bd9Sstevel@tonic-gate 30427c478bd9Sstevel@tonic-gate /* 30437c478bd9Sstevel@tonic-gate * We deal with recursive mutexes by completely 30447c478bd9Sstevel@tonic-gate * dropping the lock and restoring the recursion 30457c478bd9Sstevel@tonic-gate * count after waking up. This is arguably wrong, 30467c478bd9Sstevel@tonic-gate * but it obeys the principle of least astonishment. 30477c478bd9Sstevel@tonic-gate */ 30487c478bd9Sstevel@tonic-gate rcount = mp->mutex_rcount; 30497c478bd9Sstevel@tonic-gate mp->mutex_rcount = 0; 3050*883492d5Sraf if ((mtype & 3051*883492d5Sraf (USYNC_PROCESS | LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) | 30527c478bd9Sstevel@tonic-gate (cvp->cond_type & USYNC_PROCESS)) 30537c478bd9Sstevel@tonic-gate error = cond_wait_kernel(cvp, mp, tsp); 30547c478bd9Sstevel@tonic-gate else 30557c478bd9Sstevel@tonic-gate error = cond_wait_queue(cvp, mp, tsp, msp); 30567c478bd9Sstevel@tonic-gate mp->mutex_rcount = rcount; 30577c478bd9Sstevel@tonic-gate 30587c478bd9Sstevel@tonic-gate if (csp) { 30597c478bd9Sstevel@tonic-gate hrtime_t lapse = gethrtime() - begin_sleep; 30607c478bd9Sstevel@tonic-gate if (tsp == NULL) 30617c478bd9Sstevel@tonic-gate csp->cond_wait_sleep_time += lapse; 30627c478bd9Sstevel@tonic-gate else { 30637c478bd9Sstevel@tonic-gate csp->cond_timedwait_sleep_time += lapse; 30647c478bd9Sstevel@tonic-gate if (error == ETIME) 30657c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait_timeout); 30667c478bd9Sstevel@tonic-gate } 30677c478bd9Sstevel@tonic-gate } 30687c478bd9Sstevel@tonic-gate return (error); 30697c478bd9Sstevel@tonic-gate } 30707c478bd9Sstevel@tonic-gate 30717c478bd9Sstevel@tonic-gate /* 30727c478bd9Sstevel@tonic-gate * cond_wait() is a cancellation point but _cond_wait() is not. 30737c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 30747c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 30757c478bd9Sstevel@tonic-gate */ 30767c478bd9Sstevel@tonic-gate int 30777c478bd9Sstevel@tonic-gate _cond_wait(cond_t *cvp, mutex_t *mp) 30787c478bd9Sstevel@tonic-gate { 30797c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 30807c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 30817c478bd9Sstevel@tonic-gate uberflags_t *gflags; 30827c478bd9Sstevel@tonic-gate 30837c478bd9Sstevel@tonic-gate /* 30847c478bd9Sstevel@tonic-gate * Optimize the common case of USYNC_THREAD plus 30857c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, and no event tracing. 30867c478bd9Sstevel@tonic-gate */ 30877c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 30887c478bd9Sstevel@tonic-gate (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted | 30897c478bd9Sstevel@tonic-gate self->ul_td_events_enable | 30907c478bd9Sstevel@tonic-gate udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0) 30917c478bd9Sstevel@tonic-gate return (cond_wait_queue(cvp, mp, NULL, NULL)); 30927c478bd9Sstevel@tonic-gate 30937c478bd9Sstevel@tonic-gate /* 30947c478bd9Sstevel@tonic-gate * Else do it the long way. 30957c478bd9Sstevel@tonic-gate */ 30967c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, NULL)); 30977c478bd9Sstevel@tonic-gate } 30987c478bd9Sstevel@tonic-gate 30997c478bd9Sstevel@tonic-gate int 31007c478bd9Sstevel@tonic-gate cond_wait(cond_t *cvp, mutex_t *mp) 31017c478bd9Sstevel@tonic-gate { 31027c478bd9Sstevel@tonic-gate int error; 31037c478bd9Sstevel@tonic-gate 31047c478bd9Sstevel@tonic-gate _cancelon(); 31057c478bd9Sstevel@tonic-gate error = _cond_wait(cvp, mp); 31067c478bd9Sstevel@tonic-gate if (error == EINTR) 31077c478bd9Sstevel@tonic-gate _canceloff(); 31087c478bd9Sstevel@tonic-gate else 31097c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 31107c478bd9Sstevel@tonic-gate return (error); 31117c478bd9Sstevel@tonic-gate } 31127c478bd9Sstevel@tonic-gate 31137c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_wait = _pthread_cond_wait 31147c478bd9Sstevel@tonic-gate int 31157c478bd9Sstevel@tonic-gate _pthread_cond_wait(cond_t *cvp, mutex_t *mp) 31167c478bd9Sstevel@tonic-gate { 31177c478bd9Sstevel@tonic-gate int error; 31187c478bd9Sstevel@tonic-gate 31197c478bd9Sstevel@tonic-gate error = cond_wait(cvp, mp); 31207c478bd9Sstevel@tonic-gate return ((error == EINTR)? 0 : error); 31217c478bd9Sstevel@tonic-gate } 31227c478bd9Sstevel@tonic-gate 31237c478bd9Sstevel@tonic-gate /* 31247c478bd9Sstevel@tonic-gate * cond_timedwait() is a cancellation point but _cond_timedwait() is not. 31257c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 31267c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 31277c478bd9Sstevel@tonic-gate */ 31287c478bd9Sstevel@tonic-gate int 31297c478bd9Sstevel@tonic-gate _cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 31307c478bd9Sstevel@tonic-gate { 31317c478bd9Sstevel@tonic-gate clockid_t clock_id = cvp->cond_clockid; 31327c478bd9Sstevel@tonic-gate timespec_t reltime; 31337c478bd9Sstevel@tonic-gate int error; 31347c478bd9Sstevel@tonic-gate 31357c478bd9Sstevel@tonic-gate if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES) 31367c478bd9Sstevel@tonic-gate clock_id = CLOCK_REALTIME; 31377c478bd9Sstevel@tonic-gate abstime_to_reltime(clock_id, abstime, &reltime); 31387c478bd9Sstevel@tonic-gate error = cond_wait_common(cvp, mp, &reltime); 31397c478bd9Sstevel@tonic-gate if (error == ETIME && clock_id == CLOCK_HIGHRES) { 31407c478bd9Sstevel@tonic-gate /* 31417c478bd9Sstevel@tonic-gate * Don't return ETIME if we didn't really get a timeout. 31427c478bd9Sstevel@tonic-gate * This can happen if we return because someone resets 31437c478bd9Sstevel@tonic-gate * the system clock. Just return zero in this case, 31447c478bd9Sstevel@tonic-gate * giving a spurious wakeup but not a timeout. 31457c478bd9Sstevel@tonic-gate */ 31467c478bd9Sstevel@tonic-gate if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC + 31477c478bd9Sstevel@tonic-gate abstime->tv_nsec > gethrtime()) 31487c478bd9Sstevel@tonic-gate error = 0; 31497c478bd9Sstevel@tonic-gate } 31507c478bd9Sstevel@tonic-gate return (error); 31517c478bd9Sstevel@tonic-gate } 31527c478bd9Sstevel@tonic-gate 31537c478bd9Sstevel@tonic-gate int 31547c478bd9Sstevel@tonic-gate cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 31557c478bd9Sstevel@tonic-gate { 31567c478bd9Sstevel@tonic-gate int error; 31577c478bd9Sstevel@tonic-gate 31587c478bd9Sstevel@tonic-gate _cancelon(); 31597c478bd9Sstevel@tonic-gate error = _cond_timedwait(cvp, mp, abstime); 31607c478bd9Sstevel@tonic-gate if (error == EINTR) 31617c478bd9Sstevel@tonic-gate _canceloff(); 31627c478bd9Sstevel@tonic-gate else 31637c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 31647c478bd9Sstevel@tonic-gate return (error); 31657c478bd9Sstevel@tonic-gate } 31667c478bd9Sstevel@tonic-gate 31677c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_timedwait = _pthread_cond_timedwait 31687c478bd9Sstevel@tonic-gate int 31697c478bd9Sstevel@tonic-gate _pthread_cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 31707c478bd9Sstevel@tonic-gate { 31717c478bd9Sstevel@tonic-gate int error; 31727c478bd9Sstevel@tonic-gate 31737c478bd9Sstevel@tonic-gate error = cond_timedwait(cvp, mp, abstime); 31747c478bd9Sstevel@tonic-gate if (error == ETIME) 31757c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 31767c478bd9Sstevel@tonic-gate else if (error == EINTR) 31777c478bd9Sstevel@tonic-gate error = 0; 31787c478bd9Sstevel@tonic-gate return (error); 31797c478bd9Sstevel@tonic-gate } 31807c478bd9Sstevel@tonic-gate 31817c478bd9Sstevel@tonic-gate /* 31827c478bd9Sstevel@tonic-gate * cond_reltimedwait() is a cancellation point but _cond_reltimedwait() 31837c478bd9Sstevel@tonic-gate * is not. System libraries call the non-cancellation version. 31847c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 31857c478bd9Sstevel@tonic-gate */ 31867c478bd9Sstevel@tonic-gate int 31877c478bd9Sstevel@tonic-gate _cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 31887c478bd9Sstevel@tonic-gate { 31897c478bd9Sstevel@tonic-gate timespec_t tslocal = *reltime; 31907c478bd9Sstevel@tonic-gate 31917c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, &tslocal)); 31927c478bd9Sstevel@tonic-gate } 31937c478bd9Sstevel@tonic-gate 31947c478bd9Sstevel@tonic-gate #pragma weak cond_reltimedwait = _cond_reltimedwait_cancel 31957c478bd9Sstevel@tonic-gate int 31967c478bd9Sstevel@tonic-gate _cond_reltimedwait_cancel(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 31977c478bd9Sstevel@tonic-gate { 31987c478bd9Sstevel@tonic-gate int error; 31997c478bd9Sstevel@tonic-gate 32007c478bd9Sstevel@tonic-gate _cancelon(); 32017c478bd9Sstevel@tonic-gate error = _cond_reltimedwait(cvp, mp, reltime); 32027c478bd9Sstevel@tonic-gate if (error == EINTR) 32037c478bd9Sstevel@tonic-gate _canceloff(); 32047c478bd9Sstevel@tonic-gate else 32057c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 32067c478bd9Sstevel@tonic-gate return (error); 32077c478bd9Sstevel@tonic-gate } 32087c478bd9Sstevel@tonic-gate 32097c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_reltimedwait_np = _pthread_cond_reltimedwait_np 32107c478bd9Sstevel@tonic-gate int 32117c478bd9Sstevel@tonic-gate _pthread_cond_reltimedwait_np(cond_t *cvp, mutex_t *mp, 32127c478bd9Sstevel@tonic-gate const timespec_t *reltime) 32137c478bd9Sstevel@tonic-gate { 32147c478bd9Sstevel@tonic-gate int error; 32157c478bd9Sstevel@tonic-gate 32167c478bd9Sstevel@tonic-gate error = _cond_reltimedwait_cancel(cvp, mp, reltime); 32177c478bd9Sstevel@tonic-gate if (error == ETIME) 32187c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 32197c478bd9Sstevel@tonic-gate else if (error == EINTR) 32207c478bd9Sstevel@tonic-gate error = 0; 32217c478bd9Sstevel@tonic-gate return (error); 32227c478bd9Sstevel@tonic-gate } 32237c478bd9Sstevel@tonic-gate 32247c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_signal = cond_signal_internal 32257c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_signal = cond_signal_internal 32267c478bd9Sstevel@tonic-gate #pragma weak cond_signal = cond_signal_internal 32277c478bd9Sstevel@tonic-gate #pragma weak _cond_signal = cond_signal_internal 32287c478bd9Sstevel@tonic-gate int 32297c478bd9Sstevel@tonic-gate cond_signal_internal(cond_t *cvp) 32307c478bd9Sstevel@tonic-gate { 32317c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 32327c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 32337c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 32347c478bd9Sstevel@tonic-gate int error = 0; 32357c478bd9Sstevel@tonic-gate queue_head_t *qp; 32367c478bd9Sstevel@tonic-gate mutex_t *mp; 32377c478bd9Sstevel@tonic-gate queue_head_t *mqp; 32387c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 32397c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 32407c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 32417c478bd9Sstevel@tonic-gate ulwp_t *next; 32427c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 32437c478bd9Sstevel@tonic-gate ulwp_t *susprev; 32447c478bd9Sstevel@tonic-gate 32457c478bd9Sstevel@tonic-gate if (csp) 32467c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_signal); 32477c478bd9Sstevel@tonic-gate 32487c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 32497c478bd9Sstevel@tonic-gate error = __lwp_cond_signal(cvp); 32507c478bd9Sstevel@tonic-gate 32517c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 32527c478bd9Sstevel@tonic-gate return (error); 32537c478bd9Sstevel@tonic-gate 32547c478bd9Sstevel@tonic-gate /* 32557c478bd9Sstevel@tonic-gate * Move someone from the condvar sleep queue to the mutex sleep 32567c478bd9Sstevel@tonic-gate * queue for the mutex that he will acquire on being waked up. 32577c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex he will acquire. 32587c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if his ul_cv_wake flag 32597c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark him. 32607c478bd9Sstevel@tonic-gate */ 32617c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 32627c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 32637c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 32647c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == cvp) { 32657c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 32667c478bd9Sstevel@tonic-gate break; 32677c478bd9Sstevel@tonic-gate /* 32687c478bd9Sstevel@tonic-gate * Try not to dequeue a suspended thread. 32697c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 32707c478bd9Sstevel@tonic-gate */ 32717c478bd9Sstevel@tonic-gate if (suspp == NULL) { 32727c478bd9Sstevel@tonic-gate suspp = ulwpp; 32737c478bd9Sstevel@tonic-gate susprev = prev; 32747c478bd9Sstevel@tonic-gate } 32757c478bd9Sstevel@tonic-gate } 32767c478bd9Sstevel@tonic-gate } 32777c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 32787c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 32797c478bd9Sstevel@tonic-gate prev = susprev; 32807c478bd9Sstevel@tonic-gate suspp = NULL; 32817c478bd9Sstevel@tonic-gate } 32827c478bd9Sstevel@tonic-gate if (ulwp == NULL) { /* no one on the sleep queue */ 32837c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 32847c478bd9Sstevel@tonic-gate queue_unlock(qp); 32857c478bd9Sstevel@tonic-gate return (error); 32867c478bd9Sstevel@tonic-gate } 32877c478bd9Sstevel@tonic-gate /* 32887c478bd9Sstevel@tonic-gate * Scan the remainder of the CV queue for another waiter. 32897c478bd9Sstevel@tonic-gate */ 32907c478bd9Sstevel@tonic-gate if (suspp != NULL) { 32917c478bd9Sstevel@tonic-gate next = *suspp; 32927c478bd9Sstevel@tonic-gate } else { 32937c478bd9Sstevel@tonic-gate for (next = ulwp->ul_link; next != NULL; next = next->ul_link) 32947c478bd9Sstevel@tonic-gate if (next->ul_wchan == cvp) 32957c478bd9Sstevel@tonic-gate break; 32967c478bd9Sstevel@tonic-gate } 32977c478bd9Sstevel@tonic-gate if (next == NULL) 32987c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 32997c478bd9Sstevel@tonic-gate 33007c478bd9Sstevel@tonic-gate /* 33017c478bd9Sstevel@tonic-gate * Inform the thread that he was the recipient of a cond_signal(). 33027c478bd9Sstevel@tonic-gate * This lets him deal with cond_signal() and, concurrently, 33037c478bd9Sstevel@tonic-gate * one or more of a cancellation, a UNIX signal, or a timeout. 33047c478bd9Sstevel@tonic-gate * These latter conditions must not consume a cond_signal(). 33057c478bd9Sstevel@tonic-gate */ 33067c478bd9Sstevel@tonic-gate ulwp->ul_signalled = 1; 33077c478bd9Sstevel@tonic-gate 33087c478bd9Sstevel@tonic-gate /* 33097c478bd9Sstevel@tonic-gate * Dequeue the waiter but leave his ul_sleepq non-NULL 33107c478bd9Sstevel@tonic-gate * while we move him to the mutex queue so that he can 33117c478bd9Sstevel@tonic-gate * deal properly with spurious wakeups. 33127c478bd9Sstevel@tonic-gate */ 33137c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 3314*883492d5Sraf ulwp->ul_link = NULL; 33157c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 33167c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 33177c478bd9Sstevel@tonic-gate qp->qh_qlen--; 33187c478bd9Sstevel@tonic-gate 33197c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* the mutex he will acquire */ 33207c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 33217c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 33227c478bd9Sstevel@tonic-gate 33237c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 33247c478bd9Sstevel@tonic-gate lwpid_t lwpid = ulwp->ul_lwpid; 33257c478bd9Sstevel@tonic-gate 33267c478bd9Sstevel@tonic-gate no_preempt(self); 33277c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 33287c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 33297c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 33307c478bd9Sstevel@tonic-gate queue_unlock(qp); 33317c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 33327c478bd9Sstevel@tonic-gate preempt(self); 33337c478bd9Sstevel@tonic-gate } else { 33347c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 33357c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 33367c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 33377c478bd9Sstevel@tonic-gate queue_unlock(mqp); 33387c478bd9Sstevel@tonic-gate queue_unlock(qp); 33397c478bd9Sstevel@tonic-gate } 33407c478bd9Sstevel@tonic-gate 33417c478bd9Sstevel@tonic-gate return (error); 33427c478bd9Sstevel@tonic-gate } 33437c478bd9Sstevel@tonic-gate 334441efec22Sraf /* 3345*883492d5Sraf * Utility function called by mutex_wakeup_all(), cond_broadcast(), 3346*883492d5Sraf * and rw_queue_release() to (re)allocate a big buffer to hold the 3347*883492d5Sraf * lwpids of all the threads to be set running after they are removed 3348*883492d5Sraf * from their sleep queues. Since we are holding a queue lock, we 3349*883492d5Sraf * cannot call any function that might acquire a lock. mmap(), munmap(), 3350*883492d5Sraf * lwp_unpark_all() are simple system calls and are safe in this regard. 335141efec22Sraf */ 335241efec22Sraf lwpid_t * 335341efec22Sraf alloc_lwpids(lwpid_t *lwpid, int *nlwpid_ptr, int *maxlwps_ptr) 335441efec22Sraf { 335541efec22Sraf /* 335641efec22Sraf * Allocate NEWLWPS ids on the first overflow. 335741efec22Sraf * Double the allocation each time after that. 335841efec22Sraf */ 335941efec22Sraf int nlwpid = *nlwpid_ptr; 336041efec22Sraf int maxlwps = *maxlwps_ptr; 336141efec22Sraf int first_allocation; 336241efec22Sraf int newlwps; 336341efec22Sraf void *vaddr; 336441efec22Sraf 336541efec22Sraf ASSERT(nlwpid == maxlwps); 336641efec22Sraf 336741efec22Sraf first_allocation = (maxlwps == MAXLWPS); 336841efec22Sraf newlwps = first_allocation? NEWLWPS : 2 * maxlwps; 336941efec22Sraf vaddr = _private_mmap(NULL, newlwps * sizeof (lwpid_t), 337041efec22Sraf PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0); 337141efec22Sraf 337241efec22Sraf if (vaddr == MAP_FAILED) { 337341efec22Sraf /* 337441efec22Sraf * Let's hope this never happens. 337541efec22Sraf * If it does, then we have a terrible 337641efec22Sraf * thundering herd on our hands. 337741efec22Sraf */ 337841efec22Sraf (void) __lwp_unpark_all(lwpid, nlwpid); 337941efec22Sraf *nlwpid_ptr = 0; 338041efec22Sraf } else { 338141efec22Sraf (void) _memcpy(vaddr, lwpid, maxlwps * sizeof (lwpid_t)); 338241efec22Sraf if (!first_allocation) 338341efec22Sraf (void) _private_munmap(lwpid, 338441efec22Sraf maxlwps * sizeof (lwpid_t)); 338541efec22Sraf lwpid = vaddr; 338641efec22Sraf *maxlwps_ptr = newlwps; 338741efec22Sraf } 338841efec22Sraf 338941efec22Sraf return (lwpid); 339041efec22Sraf } 33917c478bd9Sstevel@tonic-gate 33927c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_broadcast = cond_broadcast_internal 33937c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_broadcast = cond_broadcast_internal 33947c478bd9Sstevel@tonic-gate #pragma weak cond_broadcast = cond_broadcast_internal 33957c478bd9Sstevel@tonic-gate #pragma weak _cond_broadcast = cond_broadcast_internal 33967c478bd9Sstevel@tonic-gate int 33977c478bd9Sstevel@tonic-gate cond_broadcast_internal(cond_t *cvp) 33987c478bd9Sstevel@tonic-gate { 33997c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 34007c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 34017c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 34027c478bd9Sstevel@tonic-gate int error = 0; 34037c478bd9Sstevel@tonic-gate queue_head_t *qp; 34047c478bd9Sstevel@tonic-gate mutex_t *mp; 34057c478bd9Sstevel@tonic-gate mutex_t *mp_cache = NULL; 340641efec22Sraf queue_head_t *mqp = NULL; 34077c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 34087c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 34097c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 34107c478bd9Sstevel@tonic-gate int nlwpid = 0; 34117c478bd9Sstevel@tonic-gate int maxlwps = MAXLWPS; 341241efec22Sraf lwpid_t buffer[MAXLWPS]; 341341efec22Sraf lwpid_t *lwpid = buffer; 34147c478bd9Sstevel@tonic-gate 34157c478bd9Sstevel@tonic-gate if (csp) 34167c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_broadcast); 34177c478bd9Sstevel@tonic-gate 34187c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 34197c478bd9Sstevel@tonic-gate error = __lwp_cond_broadcast(cvp); 34207c478bd9Sstevel@tonic-gate 34217c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 34227c478bd9Sstevel@tonic-gate return (error); 34237c478bd9Sstevel@tonic-gate 34247c478bd9Sstevel@tonic-gate /* 34257c478bd9Sstevel@tonic-gate * Move everyone from the condvar sleep queue to the mutex sleep 34267c478bd9Sstevel@tonic-gate * queue for the mutex that they will acquire on being waked up. 34277c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex they will acquire. 34287c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if their ul_cv_wake flag 34297c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark them. 34307c478bd9Sstevel@tonic-gate * 34317c478bd9Sstevel@tonic-gate * We keep track of lwpids that are to be unparked in lwpid[]. 34327c478bd9Sstevel@tonic-gate * __lwp_unpark_all() is called to unpark all of them after 34337c478bd9Sstevel@tonic-gate * they have been removed from the sleep queue and the sleep 34347c478bd9Sstevel@tonic-gate * queue lock has been dropped. If we run out of space in our 34357c478bd9Sstevel@tonic-gate * on-stack buffer, we need to allocate more but we can't call 34367c478bd9Sstevel@tonic-gate * lmalloc() because we are holding a queue lock when the overflow 34377c478bd9Sstevel@tonic-gate * occurs and lmalloc() acquires a lock. We can't use alloca() 343841efec22Sraf * either because the application may have allocated a small 343941efec22Sraf * stack and we don't want to overrun the stack. So we call 344041efec22Sraf * alloc_lwpids() to allocate a bigger buffer using the mmap() 34417c478bd9Sstevel@tonic-gate * system call directly since that path acquires no locks. 34427c478bd9Sstevel@tonic-gate */ 34437c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 34447c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 34457c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 34467c478bd9Sstevel@tonic-gate while ((ulwp = *ulwpp) != NULL) { 34477c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan != cvp) { 34487c478bd9Sstevel@tonic-gate prev = ulwp; 34497c478bd9Sstevel@tonic-gate ulwpp = &ulwp->ul_link; 34507c478bd9Sstevel@tonic-gate continue; 34517c478bd9Sstevel@tonic-gate } 34527c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 3453*883492d5Sraf ulwp->ul_link = NULL; 34547c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 34557c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 34567c478bd9Sstevel@tonic-gate qp->qh_qlen--; 34577c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* his mutex */ 34587c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 34597c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 34607c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 34617c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 34627c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 34637c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 346441efec22Sraf if (nlwpid == maxlwps) 346541efec22Sraf lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps); 34667c478bd9Sstevel@tonic-gate lwpid[nlwpid++] = ulwp->ul_lwpid; 34677c478bd9Sstevel@tonic-gate } else { 34687c478bd9Sstevel@tonic-gate if (mp != mp_cache) { 34697c478bd9Sstevel@tonic-gate mp_cache = mp; 347041efec22Sraf if (mqp != NULL) 347141efec22Sraf queue_unlock(mqp); 347241efec22Sraf mqp = queue_lock(mp, MX); 34737c478bd9Sstevel@tonic-gate } 34747c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 34757c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 34767c478bd9Sstevel@tonic-gate } 34777c478bd9Sstevel@tonic-gate } 347841efec22Sraf if (mqp != NULL) 347941efec22Sraf queue_unlock(mqp); 348041efec22Sraf if (nlwpid == 0) { 348141efec22Sraf queue_unlock(qp); 348241efec22Sraf } else { 348341efec22Sraf no_preempt(self); 348441efec22Sraf queue_unlock(qp); 34857c478bd9Sstevel@tonic-gate if (nlwpid == 1) 34867c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid[0]); 34877c478bd9Sstevel@tonic-gate else 34887c478bd9Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid); 348941efec22Sraf preempt(self); 34907c478bd9Sstevel@tonic-gate } 34917c478bd9Sstevel@tonic-gate if (lwpid != buffer) 34927c478bd9Sstevel@tonic-gate (void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t)); 34937c478bd9Sstevel@tonic-gate return (error); 34947c478bd9Sstevel@tonic-gate } 34957c478bd9Sstevel@tonic-gate 34967c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_destroy = _cond_destroy 34977c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_destroy = _cond_destroy 34987c478bd9Sstevel@tonic-gate #pragma weak cond_destroy = _cond_destroy 34997c478bd9Sstevel@tonic-gate int 35007c478bd9Sstevel@tonic-gate _cond_destroy(cond_t *cvp) 35017c478bd9Sstevel@tonic-gate { 35027c478bd9Sstevel@tonic-gate cvp->cond_magic = 0; 35037c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(cvp); 35047c478bd9Sstevel@tonic-gate return (0); 35057c478bd9Sstevel@tonic-gate } 35067c478bd9Sstevel@tonic-gate 35077c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 35087c478bd9Sstevel@tonic-gate void 35097c478bd9Sstevel@tonic-gate assert_no_libc_locks_held(void) 35107c478bd9Sstevel@tonic-gate { 35117c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 35127c478bd9Sstevel@tonic-gate } 35137c478bd9Sstevel@tonic-gate #endif 35147c478bd9Sstevel@tonic-gate 35157c478bd9Sstevel@tonic-gate /* protected by link_lock */ 35167c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin; 35177c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin2; 35187c478bd9Sstevel@tonic-gate uint64_t spin_lock_sleep; 35197c478bd9Sstevel@tonic-gate uint64_t spin_lock_wakeup; 35207c478bd9Sstevel@tonic-gate 35217c478bd9Sstevel@tonic-gate /* 35227c478bd9Sstevel@tonic-gate * Record spin lock statistics. 35237c478bd9Sstevel@tonic-gate * Called by a thread exiting itself in thrp_exit(). 35247c478bd9Sstevel@tonic-gate * Also called via atexit() from the thread calling 35257c478bd9Sstevel@tonic-gate * exit() to do all the other threads as well. 35267c478bd9Sstevel@tonic-gate */ 35277c478bd9Sstevel@tonic-gate void 35287c478bd9Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp) 35297c478bd9Sstevel@tonic-gate { 35307c478bd9Sstevel@tonic-gate spin_lock_spin += ulwp->ul_spin_lock_spin; 35317c478bd9Sstevel@tonic-gate spin_lock_spin2 += ulwp->ul_spin_lock_spin2; 35327c478bd9Sstevel@tonic-gate spin_lock_sleep += ulwp->ul_spin_lock_sleep; 35337c478bd9Sstevel@tonic-gate spin_lock_wakeup += ulwp->ul_spin_lock_wakeup; 35347c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin = 0; 35357c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin2 = 0; 35367c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_sleep = 0; 35377c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_wakeup = 0; 35387c478bd9Sstevel@tonic-gate } 35397c478bd9Sstevel@tonic-gate 35407c478bd9Sstevel@tonic-gate /* 35417c478bd9Sstevel@tonic-gate * atexit function: dump the queue statistics to stderr. 35427c478bd9Sstevel@tonic-gate */ 3543e8031f0aSraf #if !defined(__lint) 3544e8031f0aSraf #define fprintf _fprintf 3545e8031f0aSraf #endif 35467c478bd9Sstevel@tonic-gate #include <stdio.h> 35477c478bd9Sstevel@tonic-gate void 35487c478bd9Sstevel@tonic-gate dump_queue_statistics(void) 35497c478bd9Sstevel@tonic-gate { 35507c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 35517c478bd9Sstevel@tonic-gate queue_head_t *qp; 35527c478bd9Sstevel@tonic-gate int qn; 35537c478bd9Sstevel@tonic-gate uint64_t spin_lock_total = 0; 35547c478bd9Sstevel@tonic-gate 35557c478bd9Sstevel@tonic-gate if (udp->queue_head == NULL || thread_queue_dump == 0) 35567c478bd9Sstevel@tonic-gate return; 35577c478bd9Sstevel@tonic-gate 35587c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 || 35597c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 35607c478bd9Sstevel@tonic-gate return; 35617c478bd9Sstevel@tonic-gate for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) { 35627c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 35637c478bd9Sstevel@tonic-gate continue; 35647c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 35657c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 35667c478bd9Sstevel@tonic-gate (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 35677c478bd9Sstevel@tonic-gate return; 35687c478bd9Sstevel@tonic-gate } 35697c478bd9Sstevel@tonic-gate 35707c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 || 35717c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 35727c478bd9Sstevel@tonic-gate return; 35737c478bd9Sstevel@tonic-gate for (qn = 0; qn < QHASHSIZE; qn++, qp++) { 35747c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 35757c478bd9Sstevel@tonic-gate continue; 35767c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 35777c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 35787c478bd9Sstevel@tonic-gate (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 35797c478bd9Sstevel@tonic-gate return; 35807c478bd9Sstevel@tonic-gate } 35817c478bd9Sstevel@tonic-gate 35827c478bd9Sstevel@tonic-gate (void) fprintf(stderr, "\n spin_lock_total = %10llu\n", 35837c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_total); 35847c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin = %10llu\n", 35857c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_spin); 35867c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin2 = %10llu\n", 35877c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_spin2); 35887c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_sleep = %10llu\n", 35897c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_sleep); 35907c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_wakeup = %10llu\n", 35917c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_wakeup); 35927c478bd9Sstevel@tonic-gate } 3593