1*7c478bd9Sstevel@tonic-gate /* 2*7c478bd9Sstevel@tonic-gate * CDDL HEADER START 3*7c478bd9Sstevel@tonic-gate * 4*7c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*7c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 6*7c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 7*7c478bd9Sstevel@tonic-gate * with the License. 8*7c478bd9Sstevel@tonic-gate * 9*7c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10*7c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 11*7c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 12*7c478bd9Sstevel@tonic-gate * and limitations under the License. 13*7c478bd9Sstevel@tonic-gate * 14*7c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 15*7c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16*7c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 17*7c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 18*7c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 19*7c478bd9Sstevel@tonic-gate * 20*7c478bd9Sstevel@tonic-gate * CDDL HEADER END 21*7c478bd9Sstevel@tonic-gate */ 22*7c478bd9Sstevel@tonic-gate /* 23*7c478bd9Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24*7c478bd9Sstevel@tonic-gate * Use is subject to license terms. 25*7c478bd9Sstevel@tonic-gate */ 26*7c478bd9Sstevel@tonic-gate 27*7c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 28*7c478bd9Sstevel@tonic-gate 29*7c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 30*7c478bd9Sstevel@tonic-gate 31*7c478bd9Sstevel@tonic-gate #include "lint.h" 32*7c478bd9Sstevel@tonic-gate #include "thr_uberdata.h" 33*7c478bd9Sstevel@tonic-gate 34*7c478bd9Sstevel@tonic-gate /* 35*7c478bd9Sstevel@tonic-gate * This mutex is initialized to be held by lwp#1. 36*7c478bd9Sstevel@tonic-gate * It is used to block a thread that has returned from a mutex_lock() 37*7c478bd9Sstevel@tonic-gate * of a PTHREAD_PRIO_INHERIT mutex with an unrecoverable error. 38*7c478bd9Sstevel@tonic-gate */ 39*7c478bd9Sstevel@tonic-gate mutex_t stall_mutex = DEFAULTMUTEX; 40*7c478bd9Sstevel@tonic-gate 41*7c478bd9Sstevel@tonic-gate static int shared_mutex_held(mutex_t *); 42*7c478bd9Sstevel@tonic-gate 43*7c478bd9Sstevel@tonic-gate /* 44*7c478bd9Sstevel@tonic-gate * Lock statistics support functions. 45*7c478bd9Sstevel@tonic-gate */ 46*7c478bd9Sstevel@tonic-gate void 47*7c478bd9Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp) 48*7c478bd9Sstevel@tonic-gate { 49*7c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_lock); 50*7c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = gethrtime(); 51*7c478bd9Sstevel@tonic-gate } 52*7c478bd9Sstevel@tonic-gate 53*7c478bd9Sstevel@tonic-gate hrtime_t 54*7c478bd9Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp) 55*7c478bd9Sstevel@tonic-gate { 56*7c478bd9Sstevel@tonic-gate hrtime_t now = gethrtime(); 57*7c478bd9Sstevel@tonic-gate 58*7c478bd9Sstevel@tonic-gate if (msp->mutex_begin_hold) 59*7c478bd9Sstevel@tonic-gate msp->mutex_hold_time += now - msp->mutex_begin_hold; 60*7c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = 0; 61*7c478bd9Sstevel@tonic-gate return (now); 62*7c478bd9Sstevel@tonic-gate } 63*7c478bd9Sstevel@tonic-gate 64*7c478bd9Sstevel@tonic-gate /* 65*7c478bd9Sstevel@tonic-gate * Called once at library initialization. 66*7c478bd9Sstevel@tonic-gate */ 67*7c478bd9Sstevel@tonic-gate void 68*7c478bd9Sstevel@tonic-gate mutex_setup(void) 69*7c478bd9Sstevel@tonic-gate { 70*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&stall_mutex.mutex_lockw)) 71*7c478bd9Sstevel@tonic-gate thr_panic("mutex_setup() cannot acquire stall_mutex"); 72*7c478bd9Sstevel@tonic-gate stall_mutex.mutex_owner = (uintptr_t)curthread; 73*7c478bd9Sstevel@tonic-gate } 74*7c478bd9Sstevel@tonic-gate 75*7c478bd9Sstevel@tonic-gate /* 76*7c478bd9Sstevel@tonic-gate * The default spin counts of 1000 and 500 are experimentally determined. 77*7c478bd9Sstevel@tonic-gate * On sun4u machines with any number of processors they could be raised 78*7c478bd9Sstevel@tonic-gate * to 10,000 but that (experimentally) makes almost no difference. 79*7c478bd9Sstevel@tonic-gate * The environment variables: 80*7c478bd9Sstevel@tonic-gate * _THREAD_ADAPTIVE_SPIN=count 81*7c478bd9Sstevel@tonic-gate * _THREAD_RELEASE_SPIN=count 82*7c478bd9Sstevel@tonic-gate * can be used to override and set the counts in the range [0 .. 1,000,000]. 83*7c478bd9Sstevel@tonic-gate */ 84*7c478bd9Sstevel@tonic-gate int thread_adaptive_spin = 1000; 85*7c478bd9Sstevel@tonic-gate uint_t thread_max_spinners = 100; 86*7c478bd9Sstevel@tonic-gate int thread_release_spin = 500; 87*7c478bd9Sstevel@tonic-gate int thread_queue_verify = 0; 88*7c478bd9Sstevel@tonic-gate static int ncpus; 89*7c478bd9Sstevel@tonic-gate 90*7c478bd9Sstevel@tonic-gate /* 91*7c478bd9Sstevel@tonic-gate * Distinguish spinning for queue locks from spinning for regular locks. 92*7c478bd9Sstevel@tonic-gate * The environment variable: 93*7c478bd9Sstevel@tonic-gate * _THREAD_QUEUE_SPIN=count 94*7c478bd9Sstevel@tonic-gate * can be used to override and set the count in the range [0 .. 1,000,000]. 95*7c478bd9Sstevel@tonic-gate * There is no release spin concept for queue locks. 96*7c478bd9Sstevel@tonic-gate */ 97*7c478bd9Sstevel@tonic-gate int thread_queue_spin = 1000; 98*7c478bd9Sstevel@tonic-gate 99*7c478bd9Sstevel@tonic-gate /* 100*7c478bd9Sstevel@tonic-gate * Use the otherwise-unused 'mutex_ownerpid' field of a USYNC_THREAD 101*7c478bd9Sstevel@tonic-gate * mutex to be a count of adaptive spins in progress. 102*7c478bd9Sstevel@tonic-gate */ 103*7c478bd9Sstevel@tonic-gate #define mutex_spinners mutex_ownerpid 104*7c478bd9Sstevel@tonic-gate 105*7c478bd9Sstevel@tonic-gate void 106*7c478bd9Sstevel@tonic-gate _mutex_set_typeattr(mutex_t *mp, int attr) 107*7c478bd9Sstevel@tonic-gate { 108*7c478bd9Sstevel@tonic-gate mp->mutex_type |= (uint8_t)attr; 109*7c478bd9Sstevel@tonic-gate } 110*7c478bd9Sstevel@tonic-gate 111*7c478bd9Sstevel@tonic-gate /* 112*7c478bd9Sstevel@tonic-gate * 'type' can be one of USYNC_THREAD or USYNC_PROCESS, possibly 113*7c478bd9Sstevel@tonic-gate * augmented by the flags LOCK_RECURSIVE and/or LOCK_ERRORCHECK, 114*7c478bd9Sstevel@tonic-gate * or it can be USYNC_PROCESS_ROBUST with no extra flags. 115*7c478bd9Sstevel@tonic-gate */ 116*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_init = __mutex_init 117*7c478bd9Sstevel@tonic-gate #pragma weak mutex_init = __mutex_init 118*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_init = __mutex_init 119*7c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 120*7c478bd9Sstevel@tonic-gate int 121*7c478bd9Sstevel@tonic-gate __mutex_init(mutex_t *mp, int type, void *arg) 122*7c478bd9Sstevel@tonic-gate { 123*7c478bd9Sstevel@tonic-gate int error; 124*7c478bd9Sstevel@tonic-gate 125*7c478bd9Sstevel@tonic-gate switch (type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) { 126*7c478bd9Sstevel@tonic-gate case USYNC_THREAD: 127*7c478bd9Sstevel@tonic-gate case USYNC_PROCESS: 128*7c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 129*7c478bd9Sstevel@tonic-gate mp->mutex_type = (uint8_t)type; 130*7c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 131*7c478bd9Sstevel@tonic-gate error = 0; 132*7c478bd9Sstevel@tonic-gate break; 133*7c478bd9Sstevel@tonic-gate case USYNC_PROCESS_ROBUST: 134*7c478bd9Sstevel@tonic-gate if (type & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) 135*7c478bd9Sstevel@tonic-gate error = EINVAL; 136*7c478bd9Sstevel@tonic-gate else 137*7c478bd9Sstevel@tonic-gate error = ___lwp_mutex_init(mp, type); 138*7c478bd9Sstevel@tonic-gate break; 139*7c478bd9Sstevel@tonic-gate default: 140*7c478bd9Sstevel@tonic-gate error = EINVAL; 141*7c478bd9Sstevel@tonic-gate break; 142*7c478bd9Sstevel@tonic-gate } 143*7c478bd9Sstevel@tonic-gate if (error == 0) 144*7c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 145*7c478bd9Sstevel@tonic-gate return (error); 146*7c478bd9Sstevel@tonic-gate } 147*7c478bd9Sstevel@tonic-gate 148*7c478bd9Sstevel@tonic-gate /* 149*7c478bd9Sstevel@tonic-gate * Delete mp from list of ceil mutexes owned by curthread. 150*7c478bd9Sstevel@tonic-gate * Return 1 if the head of the chain was updated. 151*7c478bd9Sstevel@tonic-gate */ 152*7c478bd9Sstevel@tonic-gate int 153*7c478bd9Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp) 154*7c478bd9Sstevel@tonic-gate { 155*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 156*7c478bd9Sstevel@tonic-gate mxchain_t **mcpp; 157*7c478bd9Sstevel@tonic-gate mxchain_t *mcp; 158*7c478bd9Sstevel@tonic-gate 159*7c478bd9Sstevel@tonic-gate mcpp = &self->ul_mxchain; 160*7c478bd9Sstevel@tonic-gate while ((*mcpp)->mxchain_mx != mp) 161*7c478bd9Sstevel@tonic-gate mcpp = &(*mcpp)->mxchain_next; 162*7c478bd9Sstevel@tonic-gate mcp = *mcpp; 163*7c478bd9Sstevel@tonic-gate *mcpp = mcp->mxchain_next; 164*7c478bd9Sstevel@tonic-gate lfree(mcp, sizeof (*mcp)); 165*7c478bd9Sstevel@tonic-gate return (mcpp == &self->ul_mxchain); 166*7c478bd9Sstevel@tonic-gate } 167*7c478bd9Sstevel@tonic-gate 168*7c478bd9Sstevel@tonic-gate /* 169*7c478bd9Sstevel@tonic-gate * Add mp to head of list of ceil mutexes owned by curthread. 170*7c478bd9Sstevel@tonic-gate * Return ENOMEM if no memory could be allocated. 171*7c478bd9Sstevel@tonic-gate */ 172*7c478bd9Sstevel@tonic-gate int 173*7c478bd9Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp) 174*7c478bd9Sstevel@tonic-gate { 175*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 176*7c478bd9Sstevel@tonic-gate mxchain_t *mcp; 177*7c478bd9Sstevel@tonic-gate 178*7c478bd9Sstevel@tonic-gate if ((mcp = lmalloc(sizeof (*mcp))) == NULL) 179*7c478bd9Sstevel@tonic-gate return (ENOMEM); 180*7c478bd9Sstevel@tonic-gate mcp->mxchain_mx = mp; 181*7c478bd9Sstevel@tonic-gate mcp->mxchain_next = self->ul_mxchain; 182*7c478bd9Sstevel@tonic-gate self->ul_mxchain = mcp; 183*7c478bd9Sstevel@tonic-gate return (0); 184*7c478bd9Sstevel@tonic-gate } 185*7c478bd9Sstevel@tonic-gate 186*7c478bd9Sstevel@tonic-gate /* 187*7c478bd9Sstevel@tonic-gate * Inherit priority from ceiling. The inheritance impacts the effective 188*7c478bd9Sstevel@tonic-gate * priority, not the assigned priority. See _thread_setschedparam_main(). 189*7c478bd9Sstevel@tonic-gate */ 190*7c478bd9Sstevel@tonic-gate void 191*7c478bd9Sstevel@tonic-gate _ceil_prio_inherit(int ceil) 192*7c478bd9Sstevel@tonic-gate { 193*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 194*7c478bd9Sstevel@tonic-gate struct sched_param param; 195*7c478bd9Sstevel@tonic-gate 196*7c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 197*7c478bd9Sstevel@tonic-gate param.sched_priority = ceil; 198*7c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 199*7c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) { 200*7c478bd9Sstevel@tonic-gate /* 201*7c478bd9Sstevel@tonic-gate * Panic since unclear what error code to return. 202*7c478bd9Sstevel@tonic-gate * If we do return the error codes returned by above 203*7c478bd9Sstevel@tonic-gate * called routine, update the man page... 204*7c478bd9Sstevel@tonic-gate */ 205*7c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 206*7c478bd9Sstevel@tonic-gate } 207*7c478bd9Sstevel@tonic-gate } 208*7c478bd9Sstevel@tonic-gate 209*7c478bd9Sstevel@tonic-gate /* 210*7c478bd9Sstevel@tonic-gate * Waive inherited ceiling priority. Inherit from head of owned ceiling locks 211*7c478bd9Sstevel@tonic-gate * if holding at least one ceiling lock. If no ceiling locks are held at this 212*7c478bd9Sstevel@tonic-gate * point, disinherit completely, reverting back to assigned priority. 213*7c478bd9Sstevel@tonic-gate */ 214*7c478bd9Sstevel@tonic-gate void 215*7c478bd9Sstevel@tonic-gate _ceil_prio_waive(void) 216*7c478bd9Sstevel@tonic-gate { 217*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 218*7c478bd9Sstevel@tonic-gate struct sched_param param; 219*7c478bd9Sstevel@tonic-gate 220*7c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 221*7c478bd9Sstevel@tonic-gate if (self->ul_mxchain == NULL) { 222*7c478bd9Sstevel@tonic-gate /* 223*7c478bd9Sstevel@tonic-gate * No ceil locks held. Zero the epri, revert back to ul_pri. 224*7c478bd9Sstevel@tonic-gate * Since thread's hash lock is not held, one cannot just 225*7c478bd9Sstevel@tonic-gate * read ul_pri here...do it in the called routine... 226*7c478bd9Sstevel@tonic-gate */ 227*7c478bd9Sstevel@tonic-gate param.sched_priority = self->ul_pri; /* ignored */ 228*7c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 229*7c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_DISINHERIT)) 230*7c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 231*7c478bd9Sstevel@tonic-gate } else { 232*7c478bd9Sstevel@tonic-gate /* 233*7c478bd9Sstevel@tonic-gate * Set priority to that of the mutex at the head 234*7c478bd9Sstevel@tonic-gate * of the ceilmutex chain. 235*7c478bd9Sstevel@tonic-gate */ 236*7c478bd9Sstevel@tonic-gate param.sched_priority = 237*7c478bd9Sstevel@tonic-gate self->ul_mxchain->mxchain_mx->mutex_ceiling; 238*7c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 239*7c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) 240*7c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 241*7c478bd9Sstevel@tonic-gate } 242*7c478bd9Sstevel@tonic-gate } 243*7c478bd9Sstevel@tonic-gate 244*7c478bd9Sstevel@tonic-gate /* 245*7c478bd9Sstevel@tonic-gate * Non-preemptive spin locks. Used by queue_lock(). 246*7c478bd9Sstevel@tonic-gate * No lock statistics are gathered for these locks. 247*7c478bd9Sstevel@tonic-gate */ 248*7c478bd9Sstevel@tonic-gate void 249*7c478bd9Sstevel@tonic-gate spin_lock_set(mutex_t *mp) 250*7c478bd9Sstevel@tonic-gate { 251*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 252*7c478bd9Sstevel@tonic-gate 253*7c478bd9Sstevel@tonic-gate no_preempt(self); 254*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 255*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 256*7c478bd9Sstevel@tonic-gate return; 257*7c478bd9Sstevel@tonic-gate } 258*7c478bd9Sstevel@tonic-gate /* 259*7c478bd9Sstevel@tonic-gate * Spin for a while, attempting to acquire the lock. 260*7c478bd9Sstevel@tonic-gate */ 261*7c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin != UINT_MAX) 262*7c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin++; 263*7c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 264*7c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 265*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 266*7c478bd9Sstevel@tonic-gate return; 267*7c478bd9Sstevel@tonic-gate } 268*7c478bd9Sstevel@tonic-gate /* 269*7c478bd9Sstevel@tonic-gate * Try harder if we were previously at a no premption level. 270*7c478bd9Sstevel@tonic-gate */ 271*7c478bd9Sstevel@tonic-gate if (self->ul_preempt > 1) { 272*7c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin2 != UINT_MAX) 273*7c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin2++; 274*7c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 275*7c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 276*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 277*7c478bd9Sstevel@tonic-gate return; 278*7c478bd9Sstevel@tonic-gate } 279*7c478bd9Sstevel@tonic-gate } 280*7c478bd9Sstevel@tonic-gate /* 281*7c478bd9Sstevel@tonic-gate * Give up and block in the kernel for the mutex. 282*7c478bd9Sstevel@tonic-gate */ 283*7c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_sleep != UINT_MAX) 284*7c478bd9Sstevel@tonic-gate self->ul_spin_lock_sleep++; 285*7c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_timedlock(mp, NULL); 286*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 287*7c478bd9Sstevel@tonic-gate } 288*7c478bd9Sstevel@tonic-gate 289*7c478bd9Sstevel@tonic-gate void 290*7c478bd9Sstevel@tonic-gate spin_lock_clear(mutex_t *mp) 291*7c478bd9Sstevel@tonic-gate { 292*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 293*7c478bd9Sstevel@tonic-gate 294*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 295*7c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) { 296*7c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 297*7c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_wakeup != UINT_MAX) 298*7c478bd9Sstevel@tonic-gate self->ul_spin_lock_wakeup++; 299*7c478bd9Sstevel@tonic-gate } 300*7c478bd9Sstevel@tonic-gate preempt(self); 301*7c478bd9Sstevel@tonic-gate } 302*7c478bd9Sstevel@tonic-gate 303*7c478bd9Sstevel@tonic-gate /* 304*7c478bd9Sstevel@tonic-gate * Allocate the sleep queue hash table. 305*7c478bd9Sstevel@tonic-gate */ 306*7c478bd9Sstevel@tonic-gate void 307*7c478bd9Sstevel@tonic-gate queue_alloc(void) 308*7c478bd9Sstevel@tonic-gate { 309*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 310*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 311*7c478bd9Sstevel@tonic-gate void *data; 312*7c478bd9Sstevel@tonic-gate int i; 313*7c478bd9Sstevel@tonic-gate 314*7c478bd9Sstevel@tonic-gate /* 315*7c478bd9Sstevel@tonic-gate * No locks are needed; we call here only when single-threaded. 316*7c478bd9Sstevel@tonic-gate */ 317*7c478bd9Sstevel@tonic-gate ASSERT(self == udp->ulwp_one); 318*7c478bd9Sstevel@tonic-gate ASSERT(!udp->uberflags.uf_mt); 319*7c478bd9Sstevel@tonic-gate if ((data = _private_mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t), 320*7c478bd9Sstevel@tonic-gate PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0)) 321*7c478bd9Sstevel@tonic-gate == MAP_FAILED) 322*7c478bd9Sstevel@tonic-gate thr_panic("cannot allocate thread queue_head table"); 323*7c478bd9Sstevel@tonic-gate udp->queue_head = (queue_head_t *)data; 324*7c478bd9Sstevel@tonic-gate for (i = 0; i < 2 * QHASHSIZE; i++) 325*7c478bd9Sstevel@tonic-gate udp->queue_head[i].qh_lock.mutex_magic = MUTEX_MAGIC; 326*7c478bd9Sstevel@tonic-gate } 327*7c478bd9Sstevel@tonic-gate 328*7c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 329*7c478bd9Sstevel@tonic-gate 330*7c478bd9Sstevel@tonic-gate /* 331*7c478bd9Sstevel@tonic-gate * Debugging: verify correctness of a sleep queue. 332*7c478bd9Sstevel@tonic-gate */ 333*7c478bd9Sstevel@tonic-gate void 334*7c478bd9Sstevel@tonic-gate QVERIFY(queue_head_t *qp) 335*7c478bd9Sstevel@tonic-gate { 336*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 337*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 338*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 339*7c478bd9Sstevel@tonic-gate ulwp_t *prev; 340*7c478bd9Sstevel@tonic-gate uint_t index; 341*7c478bd9Sstevel@tonic-gate uint32_t cnt = 0; 342*7c478bd9Sstevel@tonic-gate char qtype; 343*7c478bd9Sstevel@tonic-gate void *wchan; 344*7c478bd9Sstevel@tonic-gate 345*7c478bd9Sstevel@tonic-gate ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE); 346*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 347*7c478bd9Sstevel@tonic-gate ASSERT((qp->qh_head != NULL && qp->qh_tail != NULL) || 348*7c478bd9Sstevel@tonic-gate (qp->qh_head == NULL && qp->qh_tail == NULL)); 349*7c478bd9Sstevel@tonic-gate if (!thread_queue_verify) 350*7c478bd9Sstevel@tonic-gate return; 351*7c478bd9Sstevel@tonic-gate /* real expensive stuff, only for _THREAD_QUEUE_VERIFY */ 352*7c478bd9Sstevel@tonic-gate qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV; 353*7c478bd9Sstevel@tonic-gate for (prev = NULL, ulwp = qp->qh_head; ulwp != NULL; 354*7c478bd9Sstevel@tonic-gate prev = ulwp, ulwp = ulwp->ul_link, cnt++) { 355*7c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_qtype == qtype); 356*7c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_wchan != NULL); 357*7c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq == qp); 358*7c478bd9Sstevel@tonic-gate wchan = ulwp->ul_wchan; 359*7c478bd9Sstevel@tonic-gate index = QUEUE_HASH(wchan, qtype); 360*7c478bd9Sstevel@tonic-gate ASSERT(&udp->queue_head[index] == qp); 361*7c478bd9Sstevel@tonic-gate } 362*7c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == prev); 363*7c478bd9Sstevel@tonic-gate ASSERT(qp->qh_qlen == cnt); 364*7c478bd9Sstevel@tonic-gate } 365*7c478bd9Sstevel@tonic-gate 366*7c478bd9Sstevel@tonic-gate #else /* THREAD_DEBUG */ 367*7c478bd9Sstevel@tonic-gate 368*7c478bd9Sstevel@tonic-gate #define QVERIFY(qp) 369*7c478bd9Sstevel@tonic-gate 370*7c478bd9Sstevel@tonic-gate #endif /* THREAD_DEBUG */ 371*7c478bd9Sstevel@tonic-gate 372*7c478bd9Sstevel@tonic-gate /* 373*7c478bd9Sstevel@tonic-gate * Acquire a queue head. 374*7c478bd9Sstevel@tonic-gate */ 375*7c478bd9Sstevel@tonic-gate queue_head_t * 376*7c478bd9Sstevel@tonic-gate queue_lock(void *wchan, int qtype) 377*7c478bd9Sstevel@tonic-gate { 378*7c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 379*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 380*7c478bd9Sstevel@tonic-gate 381*7c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 382*7c478bd9Sstevel@tonic-gate 383*7c478bd9Sstevel@tonic-gate /* 384*7c478bd9Sstevel@tonic-gate * It is possible that we could be called while still single-threaded. 385*7c478bd9Sstevel@tonic-gate * If so, we call queue_alloc() to allocate the queue_head[] array. 386*7c478bd9Sstevel@tonic-gate */ 387*7c478bd9Sstevel@tonic-gate if ((qp = udp->queue_head) == NULL) { 388*7c478bd9Sstevel@tonic-gate queue_alloc(); 389*7c478bd9Sstevel@tonic-gate qp = udp->queue_head; 390*7c478bd9Sstevel@tonic-gate } 391*7c478bd9Sstevel@tonic-gate qp += QUEUE_HASH(wchan, qtype); 392*7c478bd9Sstevel@tonic-gate spin_lock_set(&qp->qh_lock); 393*7c478bd9Sstevel@tonic-gate /* 394*7c478bd9Sstevel@tonic-gate * At once per nanosecond, qh_lockcount will wrap after 512 years. 395*7c478bd9Sstevel@tonic-gate * Were we to care about this, we could peg the value at UINT64_MAX. 396*7c478bd9Sstevel@tonic-gate */ 397*7c478bd9Sstevel@tonic-gate qp->qh_lockcount++; 398*7c478bd9Sstevel@tonic-gate QVERIFY(qp); 399*7c478bd9Sstevel@tonic-gate return (qp); 400*7c478bd9Sstevel@tonic-gate } 401*7c478bd9Sstevel@tonic-gate 402*7c478bd9Sstevel@tonic-gate /* 403*7c478bd9Sstevel@tonic-gate * Release a queue head. 404*7c478bd9Sstevel@tonic-gate */ 405*7c478bd9Sstevel@tonic-gate void 406*7c478bd9Sstevel@tonic-gate queue_unlock(queue_head_t *qp) 407*7c478bd9Sstevel@tonic-gate { 408*7c478bd9Sstevel@tonic-gate QVERIFY(qp); 409*7c478bd9Sstevel@tonic-gate spin_lock_clear(&qp->qh_lock); 410*7c478bd9Sstevel@tonic-gate } 411*7c478bd9Sstevel@tonic-gate 412*7c478bd9Sstevel@tonic-gate /* 413*7c478bd9Sstevel@tonic-gate * For rwlock queueing, we must queue writers ahead of readers of the 414*7c478bd9Sstevel@tonic-gate * same priority. We do this by making writers appear to have a half 415*7c478bd9Sstevel@tonic-gate * point higher priority for purposes of priority comparisons below. 416*7c478bd9Sstevel@tonic-gate */ 417*7c478bd9Sstevel@tonic-gate #define CMP_PRIO(ulwp) ((real_priority(ulwp) << 1) + (ulwp)->ul_writer) 418*7c478bd9Sstevel@tonic-gate 419*7c478bd9Sstevel@tonic-gate void 420*7c478bd9Sstevel@tonic-gate enqueue(queue_head_t *qp, ulwp_t *ulwp, void *wchan, int qtype) 421*7c478bd9Sstevel@tonic-gate { 422*7c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 423*7c478bd9Sstevel@tonic-gate ulwp_t *next; 424*7c478bd9Sstevel@tonic-gate int pri = CMP_PRIO(ulwp); 425*7c478bd9Sstevel@tonic-gate int force_fifo = (qtype & FIFOQ); 426*7c478bd9Sstevel@tonic-gate int do_fifo; 427*7c478bd9Sstevel@tonic-gate 428*7c478bd9Sstevel@tonic-gate qtype &= ~FIFOQ; 429*7c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 430*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 431*7c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq != qp); 432*7c478bd9Sstevel@tonic-gate 433*7c478bd9Sstevel@tonic-gate /* 434*7c478bd9Sstevel@tonic-gate * LIFO queue ordering is unfair and can lead to starvation, 435*7c478bd9Sstevel@tonic-gate * but it gives better performance for heavily contended locks. 436*7c478bd9Sstevel@tonic-gate * We use thread_queue_fifo (range is 0..8) to determine 437*7c478bd9Sstevel@tonic-gate * the frequency of FIFO vs LIFO queuing: 438*7c478bd9Sstevel@tonic-gate * 0 : every 256th time (almost always LIFO) 439*7c478bd9Sstevel@tonic-gate * 1 : every 128th time 440*7c478bd9Sstevel@tonic-gate * 2 : every 64th time 441*7c478bd9Sstevel@tonic-gate * 3 : every 32nd time 442*7c478bd9Sstevel@tonic-gate * 4 : every 16th time (the default value, mostly LIFO) 443*7c478bd9Sstevel@tonic-gate * 5 : every 8th time 444*7c478bd9Sstevel@tonic-gate * 6 : every 4th time 445*7c478bd9Sstevel@tonic-gate * 7 : every 2nd time 446*7c478bd9Sstevel@tonic-gate * 8 : every time (never LIFO, always FIFO) 447*7c478bd9Sstevel@tonic-gate * Note that there is always some degree of FIFO ordering. 448*7c478bd9Sstevel@tonic-gate * This breaks live lock conditions that occur in applications 449*7c478bd9Sstevel@tonic-gate * that are written assuming (incorrectly) that threads acquire 450*7c478bd9Sstevel@tonic-gate * locks fairly, that is, in roughly round-robin order. 451*7c478bd9Sstevel@tonic-gate * In any event, the queue is maintained in priority order. 452*7c478bd9Sstevel@tonic-gate * 453*7c478bd9Sstevel@tonic-gate * If we are given the FIFOQ flag in qtype, fifo queueing is forced. 454*7c478bd9Sstevel@tonic-gate * SUSV3 requires this for semaphores. 455*7c478bd9Sstevel@tonic-gate */ 456*7c478bd9Sstevel@tonic-gate do_fifo = (force_fifo || 457*7c478bd9Sstevel@tonic-gate ((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0); 458*7c478bd9Sstevel@tonic-gate 459*7c478bd9Sstevel@tonic-gate if (qp->qh_head == NULL) { 460*7c478bd9Sstevel@tonic-gate /* 461*7c478bd9Sstevel@tonic-gate * The queue is empty. LIFO/FIFO doesn't matter. 462*7c478bd9Sstevel@tonic-gate */ 463*7c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == NULL); 464*7c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 465*7c478bd9Sstevel@tonic-gate } else if (do_fifo) { 466*7c478bd9Sstevel@tonic-gate /* 467*7c478bd9Sstevel@tonic-gate * Enqueue after the last thread whose priority is greater 468*7c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 469*7c478bd9Sstevel@tonic-gate * Attempt first to go directly onto the tail of the queue. 470*7c478bd9Sstevel@tonic-gate */ 471*7c478bd9Sstevel@tonic-gate if (pri <= CMP_PRIO(qp->qh_tail)) 472*7c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_tail->ul_link; 473*7c478bd9Sstevel@tonic-gate else { 474*7c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 475*7c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 476*7c478bd9Sstevel@tonic-gate if (pri > CMP_PRIO(next)) 477*7c478bd9Sstevel@tonic-gate break; 478*7c478bd9Sstevel@tonic-gate } 479*7c478bd9Sstevel@tonic-gate } else { 480*7c478bd9Sstevel@tonic-gate /* 481*7c478bd9Sstevel@tonic-gate * Enqueue before the first thread whose priority is less 482*7c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 483*7c478bd9Sstevel@tonic-gate * Hopefully we can go directly onto the head of the queue. 484*7c478bd9Sstevel@tonic-gate */ 485*7c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 486*7c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 487*7c478bd9Sstevel@tonic-gate if (pri >= CMP_PRIO(next)) 488*7c478bd9Sstevel@tonic-gate break; 489*7c478bd9Sstevel@tonic-gate } 490*7c478bd9Sstevel@tonic-gate if ((ulwp->ul_link = *ulwpp) == NULL) 491*7c478bd9Sstevel@tonic-gate qp->qh_tail = ulwp; 492*7c478bd9Sstevel@tonic-gate *ulwpp = ulwp; 493*7c478bd9Sstevel@tonic-gate 494*7c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = qp; 495*7c478bd9Sstevel@tonic-gate ulwp->ul_wchan = wchan; 496*7c478bd9Sstevel@tonic-gate ulwp->ul_qtype = qtype; 497*7c478bd9Sstevel@tonic-gate if (qp->qh_qmax < ++qp->qh_qlen) 498*7c478bd9Sstevel@tonic-gate qp->qh_qmax = qp->qh_qlen; 499*7c478bd9Sstevel@tonic-gate } 500*7c478bd9Sstevel@tonic-gate 501*7c478bd9Sstevel@tonic-gate /* 502*7c478bd9Sstevel@tonic-gate * Return a pointer to the queue slot of the 503*7c478bd9Sstevel@tonic-gate * highest priority thread on the queue. 504*7c478bd9Sstevel@tonic-gate * On return, prevp, if not NULL, will contain a pointer 505*7c478bd9Sstevel@tonic-gate * to the thread's predecessor on the queue 506*7c478bd9Sstevel@tonic-gate */ 507*7c478bd9Sstevel@tonic-gate static ulwp_t ** 508*7c478bd9Sstevel@tonic-gate queue_slot(queue_head_t *qp, void *wchan, int *more, ulwp_t **prevp) 509*7c478bd9Sstevel@tonic-gate { 510*7c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 511*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 512*7c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 513*7c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 514*7c478bd9Sstevel@tonic-gate ulwp_t *susprev; 515*7c478bd9Sstevel@tonic-gate 516*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 517*7c478bd9Sstevel@tonic-gate 518*7c478bd9Sstevel@tonic-gate /* 519*7c478bd9Sstevel@tonic-gate * Find a waiter on the sleep queue. 520*7c478bd9Sstevel@tonic-gate */ 521*7c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 522*7c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 523*7c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 524*7c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 525*7c478bd9Sstevel@tonic-gate break; 526*7c478bd9Sstevel@tonic-gate /* 527*7c478bd9Sstevel@tonic-gate * Try not to return a suspended thread. 528*7c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 529*7c478bd9Sstevel@tonic-gate */ 530*7c478bd9Sstevel@tonic-gate if (suspp == NULL) { 531*7c478bd9Sstevel@tonic-gate suspp = ulwpp; 532*7c478bd9Sstevel@tonic-gate susprev = prev; 533*7c478bd9Sstevel@tonic-gate } 534*7c478bd9Sstevel@tonic-gate } 535*7c478bd9Sstevel@tonic-gate } 536*7c478bd9Sstevel@tonic-gate 537*7c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 538*7c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 539*7c478bd9Sstevel@tonic-gate prev = susprev; 540*7c478bd9Sstevel@tonic-gate suspp = NULL; 541*7c478bd9Sstevel@tonic-gate } 542*7c478bd9Sstevel@tonic-gate if (ulwp == NULL) { 543*7c478bd9Sstevel@tonic-gate if (more != NULL) 544*7c478bd9Sstevel@tonic-gate *more = 0; 545*7c478bd9Sstevel@tonic-gate return (NULL); 546*7c478bd9Sstevel@tonic-gate } 547*7c478bd9Sstevel@tonic-gate 548*7c478bd9Sstevel@tonic-gate if (prevp != NULL) 549*7c478bd9Sstevel@tonic-gate *prevp = prev; 550*7c478bd9Sstevel@tonic-gate if (more == NULL) 551*7c478bd9Sstevel@tonic-gate return (ulwpp); 552*7c478bd9Sstevel@tonic-gate 553*7c478bd9Sstevel@tonic-gate /* 554*7c478bd9Sstevel@tonic-gate * Scan the remainder of the queue for another waiter. 555*7c478bd9Sstevel@tonic-gate */ 556*7c478bd9Sstevel@tonic-gate if (suspp != NULL) { 557*7c478bd9Sstevel@tonic-gate *more = 1; 558*7c478bd9Sstevel@tonic-gate return (ulwpp); 559*7c478bd9Sstevel@tonic-gate } 560*7c478bd9Sstevel@tonic-gate for (ulwp = ulwp->ul_link; ulwp != NULL; ulwp = ulwp->ul_link) { 561*7c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 562*7c478bd9Sstevel@tonic-gate *more = 1; 563*7c478bd9Sstevel@tonic-gate return (ulwpp); 564*7c478bd9Sstevel@tonic-gate } 565*7c478bd9Sstevel@tonic-gate } 566*7c478bd9Sstevel@tonic-gate 567*7c478bd9Sstevel@tonic-gate *more = 0; 568*7c478bd9Sstevel@tonic-gate return (ulwpp); 569*7c478bd9Sstevel@tonic-gate } 570*7c478bd9Sstevel@tonic-gate 571*7c478bd9Sstevel@tonic-gate ulwp_t * 572*7c478bd9Sstevel@tonic-gate dequeue(queue_head_t *qp, void *wchan, int *more) 573*7c478bd9Sstevel@tonic-gate { 574*7c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 575*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 576*7c478bd9Sstevel@tonic-gate ulwp_t *prev; 577*7c478bd9Sstevel@tonic-gate 578*7c478bd9Sstevel@tonic-gate if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL) 579*7c478bd9Sstevel@tonic-gate return (NULL); 580*7c478bd9Sstevel@tonic-gate 581*7c478bd9Sstevel@tonic-gate /* 582*7c478bd9Sstevel@tonic-gate * Dequeue the waiter. 583*7c478bd9Sstevel@tonic-gate */ 584*7c478bd9Sstevel@tonic-gate ulwp = *ulwpp; 585*7c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 586*7c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 587*7c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 588*7c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 589*7c478bd9Sstevel@tonic-gate qp->qh_qlen--; 590*7c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 591*7c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 592*7c478bd9Sstevel@tonic-gate 593*7c478bd9Sstevel@tonic-gate return (ulwp); 594*7c478bd9Sstevel@tonic-gate } 595*7c478bd9Sstevel@tonic-gate 596*7c478bd9Sstevel@tonic-gate /* 597*7c478bd9Sstevel@tonic-gate * Return a pointer to the highest priority thread sleeping on wchan. 598*7c478bd9Sstevel@tonic-gate */ 599*7c478bd9Sstevel@tonic-gate ulwp_t * 600*7c478bd9Sstevel@tonic-gate queue_waiter(queue_head_t *qp, void *wchan) 601*7c478bd9Sstevel@tonic-gate { 602*7c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 603*7c478bd9Sstevel@tonic-gate 604*7c478bd9Sstevel@tonic-gate if ((ulwpp = queue_slot(qp, wchan, NULL, NULL)) == NULL) 605*7c478bd9Sstevel@tonic-gate return (NULL); 606*7c478bd9Sstevel@tonic-gate return (*ulwpp); 607*7c478bd9Sstevel@tonic-gate } 608*7c478bd9Sstevel@tonic-gate 609*7c478bd9Sstevel@tonic-gate uint8_t 610*7c478bd9Sstevel@tonic-gate dequeue_self(queue_head_t *qp, void *wchan) 611*7c478bd9Sstevel@tonic-gate { 612*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 613*7c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 614*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 615*7c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 616*7c478bd9Sstevel@tonic-gate int found = 0; 617*7c478bd9Sstevel@tonic-gate int more = 0; 618*7c478bd9Sstevel@tonic-gate 619*7c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 620*7c478bd9Sstevel@tonic-gate 621*7c478bd9Sstevel@tonic-gate /* find self on the sleep queue */ 622*7c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 623*7c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 624*7c478bd9Sstevel@tonic-gate if (ulwp == self) { 625*7c478bd9Sstevel@tonic-gate /* dequeue ourself */ 626*7c478bd9Sstevel@tonic-gate *ulwpp = self->ul_link; 627*7c478bd9Sstevel@tonic-gate if (qp->qh_tail == self) 628*7c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 629*7c478bd9Sstevel@tonic-gate qp->qh_qlen--; 630*7c478bd9Sstevel@tonic-gate ASSERT(self->ul_wchan == wchan); 631*7c478bd9Sstevel@tonic-gate self->ul_cvmutex = NULL; 632*7c478bd9Sstevel@tonic-gate self->ul_sleepq = NULL; 633*7c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 634*7c478bd9Sstevel@tonic-gate self->ul_cv_wake = 0; 635*7c478bd9Sstevel@tonic-gate self->ul_link = NULL; 636*7c478bd9Sstevel@tonic-gate found = 1; 637*7c478bd9Sstevel@tonic-gate break; 638*7c478bd9Sstevel@tonic-gate } 639*7c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 640*7c478bd9Sstevel@tonic-gate more = 1; 641*7c478bd9Sstevel@tonic-gate } 642*7c478bd9Sstevel@tonic-gate 643*7c478bd9Sstevel@tonic-gate if (!found) 644*7c478bd9Sstevel@tonic-gate thr_panic("dequeue_self(): curthread not found on queue"); 645*7c478bd9Sstevel@tonic-gate 646*7c478bd9Sstevel@tonic-gate if (more) 647*7c478bd9Sstevel@tonic-gate return (1); 648*7c478bd9Sstevel@tonic-gate 649*7c478bd9Sstevel@tonic-gate /* scan the remainder of the queue for another waiter */ 650*7c478bd9Sstevel@tonic-gate for (ulwp = *ulwpp; ulwp != NULL; ulwp = ulwp->ul_link) { 651*7c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 652*7c478bd9Sstevel@tonic-gate return (1); 653*7c478bd9Sstevel@tonic-gate } 654*7c478bd9Sstevel@tonic-gate 655*7c478bd9Sstevel@tonic-gate return (0); 656*7c478bd9Sstevel@tonic-gate } 657*7c478bd9Sstevel@tonic-gate 658*7c478bd9Sstevel@tonic-gate /* 659*7c478bd9Sstevel@tonic-gate * Called from call_user_handler() and _thrp_suspend() to take 660*7c478bd9Sstevel@tonic-gate * ourself off of our sleep queue so we can grab locks. 661*7c478bd9Sstevel@tonic-gate */ 662*7c478bd9Sstevel@tonic-gate void 663*7c478bd9Sstevel@tonic-gate unsleep_self(void) 664*7c478bd9Sstevel@tonic-gate { 665*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 666*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 667*7c478bd9Sstevel@tonic-gate 668*7c478bd9Sstevel@tonic-gate /* 669*7c478bd9Sstevel@tonic-gate * Calling enter_critical()/exit_critical() here would lead 670*7c478bd9Sstevel@tonic-gate * to recursion. Just manipulate self->ul_critical directly. 671*7c478bd9Sstevel@tonic-gate */ 672*7c478bd9Sstevel@tonic-gate self->ul_critical++; 673*7c478bd9Sstevel@tonic-gate self->ul_writer = 0; 674*7c478bd9Sstevel@tonic-gate while (self->ul_sleepq != NULL) { 675*7c478bd9Sstevel@tonic-gate qp = queue_lock(self->ul_wchan, self->ul_qtype); 676*7c478bd9Sstevel@tonic-gate /* 677*7c478bd9Sstevel@tonic-gate * We may have been moved from a CV queue to a 678*7c478bd9Sstevel@tonic-gate * mutex queue while we were attempting queue_lock(). 679*7c478bd9Sstevel@tonic-gate * If so, just loop around and try again. 680*7c478bd9Sstevel@tonic-gate * dequeue_self() clears self->ul_sleepq. 681*7c478bd9Sstevel@tonic-gate */ 682*7c478bd9Sstevel@tonic-gate if (qp == self->ul_sleepq) 683*7c478bd9Sstevel@tonic-gate (void) dequeue_self(qp, self->ul_wchan); 684*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 685*7c478bd9Sstevel@tonic-gate } 686*7c478bd9Sstevel@tonic-gate self->ul_critical--; 687*7c478bd9Sstevel@tonic-gate } 688*7c478bd9Sstevel@tonic-gate 689*7c478bd9Sstevel@tonic-gate /* 690*7c478bd9Sstevel@tonic-gate * Common code for calling the the ___lwp_mutex_timedlock() system call. 691*7c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 692*7c478bd9Sstevel@tonic-gate */ 693*7c478bd9Sstevel@tonic-gate int 694*7c478bd9Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp) 695*7c478bd9Sstevel@tonic-gate { 696*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 697*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 698*7c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 699*7c478bd9Sstevel@tonic-gate int error; 700*7c478bd9Sstevel@tonic-gate 701*7c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 702*7c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 703*7c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 704*7c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 705*7c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 706*7c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 707*7c478bd9Sstevel@tonic-gate } 708*7c478bd9Sstevel@tonic-gate if (msp) { 709*7c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 710*7c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 711*7c478bd9Sstevel@tonic-gate } 712*7c478bd9Sstevel@tonic-gate 713*7c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 714*7c478bd9Sstevel@tonic-gate 715*7c478bd9Sstevel@tonic-gate for (;;) { 716*7c478bd9Sstevel@tonic-gate if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0) { 717*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 718*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, error); 719*7c478bd9Sstevel@tonic-gate break; 720*7c478bd9Sstevel@tonic-gate } 721*7c478bd9Sstevel@tonic-gate 722*7c478bd9Sstevel@tonic-gate if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) { 723*7c478bd9Sstevel@tonic-gate /* 724*7c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 725*7c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 726*7c478bd9Sstevel@tonic-gate */ 727*7c478bd9Sstevel@tonic-gate enter_critical(self); 728*7c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 729*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 730*7c478bd9Sstevel@tonic-gate exit_critical(self); 731*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 732*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 733*7c478bd9Sstevel@tonic-gate 0, 0); 734*7c478bd9Sstevel@tonic-gate break; 735*7c478bd9Sstevel@tonic-gate } 736*7c478bd9Sstevel@tonic-gate exit_critical(self); 737*7c478bd9Sstevel@tonic-gate } else { 738*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 739*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 740*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 741*7c478bd9Sstevel@tonic-gate break; 742*7c478bd9Sstevel@tonic-gate } 743*7c478bd9Sstevel@tonic-gate } 744*7c478bd9Sstevel@tonic-gate if (msp) 745*7c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 746*7c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 747*7c478bd9Sstevel@tonic-gate self->ul_sp = 0; 748*7c478bd9Sstevel@tonic-gate 749*7c478bd9Sstevel@tonic-gate return (error); 750*7c478bd9Sstevel@tonic-gate } 751*7c478bd9Sstevel@tonic-gate 752*7c478bd9Sstevel@tonic-gate /* 753*7c478bd9Sstevel@tonic-gate * Common code for calling the ___lwp_mutex_trylock() system call. 754*7c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 755*7c478bd9Sstevel@tonic-gate */ 756*7c478bd9Sstevel@tonic-gate int 757*7c478bd9Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp) 758*7c478bd9Sstevel@tonic-gate { 759*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 760*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 761*7c478bd9Sstevel@tonic-gate int error; 762*7c478bd9Sstevel@tonic-gate 763*7c478bd9Sstevel@tonic-gate for (;;) { 764*7c478bd9Sstevel@tonic-gate if ((error = ___lwp_mutex_trylock(mp)) != 0) { 765*7c478bd9Sstevel@tonic-gate if (error != EBUSY) { 766*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 767*7c478bd9Sstevel@tonic-gate error); 768*7c478bd9Sstevel@tonic-gate } 769*7c478bd9Sstevel@tonic-gate break; 770*7c478bd9Sstevel@tonic-gate } 771*7c478bd9Sstevel@tonic-gate 772*7c478bd9Sstevel@tonic-gate if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) { 773*7c478bd9Sstevel@tonic-gate /* 774*7c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 775*7c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 776*7c478bd9Sstevel@tonic-gate */ 777*7c478bd9Sstevel@tonic-gate enter_critical(self); 778*7c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 779*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 780*7c478bd9Sstevel@tonic-gate exit_critical(self); 781*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 782*7c478bd9Sstevel@tonic-gate 0, 0); 783*7c478bd9Sstevel@tonic-gate break; 784*7c478bd9Sstevel@tonic-gate } 785*7c478bd9Sstevel@tonic-gate exit_critical(self); 786*7c478bd9Sstevel@tonic-gate } else { 787*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 788*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 789*7c478bd9Sstevel@tonic-gate break; 790*7c478bd9Sstevel@tonic-gate } 791*7c478bd9Sstevel@tonic-gate } 792*7c478bd9Sstevel@tonic-gate 793*7c478bd9Sstevel@tonic-gate return (error); 794*7c478bd9Sstevel@tonic-gate } 795*7c478bd9Sstevel@tonic-gate 796*7c478bd9Sstevel@tonic-gate volatile sc_shared_t * 797*7c478bd9Sstevel@tonic-gate setup_schedctl(void) 798*7c478bd9Sstevel@tonic-gate { 799*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 800*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 801*7c478bd9Sstevel@tonic-gate sc_shared_t *tmp; 802*7c478bd9Sstevel@tonic-gate 803*7c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */ 804*7c478bd9Sstevel@tonic-gate !self->ul_vfork && /* not a child of vfork() */ 805*7c478bd9Sstevel@tonic-gate !self->ul_schedctl_called) { /* haven't been called before */ 806*7c478bd9Sstevel@tonic-gate enter_critical(self); 807*7c478bd9Sstevel@tonic-gate self->ul_schedctl_called = &self->ul_uberdata->uberflags; 808*7c478bd9Sstevel@tonic-gate if ((tmp = __schedctl()) != (sc_shared_t *)(-1)) 809*7c478bd9Sstevel@tonic-gate self->ul_schedctl = scp = tmp; 810*7c478bd9Sstevel@tonic-gate exit_critical(self); 811*7c478bd9Sstevel@tonic-gate } 812*7c478bd9Sstevel@tonic-gate /* 813*7c478bd9Sstevel@tonic-gate * Unless the call to setup_schedctl() is surrounded 814*7c478bd9Sstevel@tonic-gate * by enter_critical()/exit_critical(), the address 815*7c478bd9Sstevel@tonic-gate * we are returning could be invalid due to a forkall() 816*7c478bd9Sstevel@tonic-gate * having occurred in another thread. 817*7c478bd9Sstevel@tonic-gate */ 818*7c478bd9Sstevel@tonic-gate return (scp); 819*7c478bd9Sstevel@tonic-gate } 820*7c478bd9Sstevel@tonic-gate 821*7c478bd9Sstevel@tonic-gate /* 822*7c478bd9Sstevel@tonic-gate * Interfaces from libsched, incorporated into libc. 823*7c478bd9Sstevel@tonic-gate * libsched.so.1 is now a filter library onto libc. 824*7c478bd9Sstevel@tonic-gate */ 825*7c478bd9Sstevel@tonic-gate #pragma weak schedctl_lookup = _schedctl_init 826*7c478bd9Sstevel@tonic-gate #pragma weak _schedctl_lookup = _schedctl_init 827*7c478bd9Sstevel@tonic-gate #pragma weak schedctl_init = _schedctl_init 828*7c478bd9Sstevel@tonic-gate schedctl_t * 829*7c478bd9Sstevel@tonic-gate _schedctl_init(void) 830*7c478bd9Sstevel@tonic-gate { 831*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = setup_schedctl(); 832*7c478bd9Sstevel@tonic-gate return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl); 833*7c478bd9Sstevel@tonic-gate } 834*7c478bd9Sstevel@tonic-gate 835*7c478bd9Sstevel@tonic-gate #pragma weak schedctl_exit = _schedctl_exit 836*7c478bd9Sstevel@tonic-gate void 837*7c478bd9Sstevel@tonic-gate _schedctl_exit(void) 838*7c478bd9Sstevel@tonic-gate { 839*7c478bd9Sstevel@tonic-gate } 840*7c478bd9Sstevel@tonic-gate 841*7c478bd9Sstevel@tonic-gate /* 842*7c478bd9Sstevel@tonic-gate * Contract private interface for java. 843*7c478bd9Sstevel@tonic-gate * Set up the schedctl data if it doesn't exist yet. 844*7c478bd9Sstevel@tonic-gate * Return a pointer to the pointer to the schedctl data. 845*7c478bd9Sstevel@tonic-gate */ 846*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile * 847*7c478bd9Sstevel@tonic-gate _thr_schedctl(void) 848*7c478bd9Sstevel@tonic-gate { 849*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 850*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile *ptr; 851*7c478bd9Sstevel@tonic-gate 852*7c478bd9Sstevel@tonic-gate if (self->ul_vfork) 853*7c478bd9Sstevel@tonic-gate return (NULL); 854*7c478bd9Sstevel@tonic-gate if (*(ptr = &self->ul_schedctl) == NULL) 855*7c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 856*7c478bd9Sstevel@tonic-gate return (ptr); 857*7c478bd9Sstevel@tonic-gate } 858*7c478bd9Sstevel@tonic-gate 859*7c478bd9Sstevel@tonic-gate /* 860*7c478bd9Sstevel@tonic-gate * Block signals and attempt to block preemption. 861*7c478bd9Sstevel@tonic-gate * no_preempt()/preempt() must be used in pairs but can be nested. 862*7c478bd9Sstevel@tonic-gate */ 863*7c478bd9Sstevel@tonic-gate void 864*7c478bd9Sstevel@tonic-gate no_preempt(ulwp_t *self) 865*7c478bd9Sstevel@tonic-gate { 866*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 867*7c478bd9Sstevel@tonic-gate 868*7c478bd9Sstevel@tonic-gate if (self->ul_preempt++ == 0) { 869*7c478bd9Sstevel@tonic-gate enter_critical(self); 870*7c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL || 871*7c478bd9Sstevel@tonic-gate (scp = setup_schedctl()) != NULL) { 872*7c478bd9Sstevel@tonic-gate /* 873*7c478bd9Sstevel@tonic-gate * Save the pre-existing preempt value. 874*7c478bd9Sstevel@tonic-gate */ 875*7c478bd9Sstevel@tonic-gate self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt; 876*7c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = 1; 877*7c478bd9Sstevel@tonic-gate } 878*7c478bd9Sstevel@tonic-gate } 879*7c478bd9Sstevel@tonic-gate } 880*7c478bd9Sstevel@tonic-gate 881*7c478bd9Sstevel@tonic-gate /* 882*7c478bd9Sstevel@tonic-gate * Undo the effects of no_preempt(). 883*7c478bd9Sstevel@tonic-gate */ 884*7c478bd9Sstevel@tonic-gate void 885*7c478bd9Sstevel@tonic-gate preempt(ulwp_t *self) 886*7c478bd9Sstevel@tonic-gate { 887*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 888*7c478bd9Sstevel@tonic-gate 889*7c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt > 0); 890*7c478bd9Sstevel@tonic-gate if (--self->ul_preempt == 0) { 891*7c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL) { 892*7c478bd9Sstevel@tonic-gate /* 893*7c478bd9Sstevel@tonic-gate * Restore the pre-existing preempt value. 894*7c478bd9Sstevel@tonic-gate */ 895*7c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt; 896*7c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield && 897*7c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt == 0) { 898*7c478bd9Sstevel@tonic-gate lwp_yield(); 899*7c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield) { 900*7c478bd9Sstevel@tonic-gate /* 901*7c478bd9Sstevel@tonic-gate * Shouldn't happen. This is either 902*7c478bd9Sstevel@tonic-gate * a race condition or the thread 903*7c478bd9Sstevel@tonic-gate * just entered the real-time class. 904*7c478bd9Sstevel@tonic-gate */ 905*7c478bd9Sstevel@tonic-gate lwp_yield(); 906*7c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_yield = 0; 907*7c478bd9Sstevel@tonic-gate } 908*7c478bd9Sstevel@tonic-gate } 909*7c478bd9Sstevel@tonic-gate } 910*7c478bd9Sstevel@tonic-gate exit_critical(self); 911*7c478bd9Sstevel@tonic-gate } 912*7c478bd9Sstevel@tonic-gate } 913*7c478bd9Sstevel@tonic-gate 914*7c478bd9Sstevel@tonic-gate /* 915*7c478bd9Sstevel@tonic-gate * If a call to preempt() would cause the current thread to yield or to 916*7c478bd9Sstevel@tonic-gate * take deferred actions in exit_critical(), then unpark the specified 917*7c478bd9Sstevel@tonic-gate * lwp so it can run while we delay. Return the original lwpid if the 918*7c478bd9Sstevel@tonic-gate * unpark was not performed, else return zero. The tests are a repeat 919*7c478bd9Sstevel@tonic-gate * of some of the tests in preempt(), above. This is a statistical 920*7c478bd9Sstevel@tonic-gate * optimization solely for cond_sleep_queue(), below. 921*7c478bd9Sstevel@tonic-gate */ 922*7c478bd9Sstevel@tonic-gate static lwpid_t 923*7c478bd9Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid) 924*7c478bd9Sstevel@tonic-gate { 925*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = self->ul_schedctl; 926*7c478bd9Sstevel@tonic-gate 927*7c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt == 1 && self->ul_critical > 0); 928*7c478bd9Sstevel@tonic-gate if ((scp != NULL && scp->sc_preemptctl.sc_yield) || 929*7c478bd9Sstevel@tonic-gate (self->ul_curplease && self->ul_critical == 1)) { 930*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 931*7c478bd9Sstevel@tonic-gate lwpid = 0; 932*7c478bd9Sstevel@tonic-gate } 933*7c478bd9Sstevel@tonic-gate return (lwpid); 934*7c478bd9Sstevel@tonic-gate } 935*7c478bd9Sstevel@tonic-gate 936*7c478bd9Sstevel@tonic-gate /* 937*7c478bd9Sstevel@tonic-gate * Spin for a while, trying to grab the lock. We know that we 938*7c478bd9Sstevel@tonic-gate * failed set_lock_byte(&mp->mutex_lockw) once before coming here. 939*7c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 940*7c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread. 941*7c478bd9Sstevel@tonic-gate */ 942*7c478bd9Sstevel@tonic-gate int 943*7c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mutex_t *mp) 944*7c478bd9Sstevel@tonic-gate { 945*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 946*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 947*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 948*7c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 949*7c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 950*7c478bd9Sstevel@tonic-gate int count, max = self->ul_adaptive_spin; 951*7c478bd9Sstevel@tonic-gate 952*7c478bd9Sstevel@tonic-gate ASSERT(!(mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))); 953*7c478bd9Sstevel@tonic-gate 954*7c478bd9Sstevel@tonic-gate if (max == 0 || (mp->mutex_spinners >= self->ul_max_spinners)) 955*7c478bd9Sstevel@tonic-gate return (EBUSY); 956*7c478bd9Sstevel@tonic-gate 957*7c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 958*7c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 959*7c478bd9Sstevel@tonic-gate 960*7c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__spin, mp); 961*7c478bd9Sstevel@tonic-gate 962*7c478bd9Sstevel@tonic-gate /* 963*7c478bd9Sstevel@tonic-gate * This spin loop is unfair to lwps that have already dropped into 964*7c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a highly-contended mutex. 965*7c478bd9Sstevel@tonic-gate * This is just too bad. The adaptive spin algorithm is intended 966*7c478bd9Sstevel@tonic-gate * to allow programs with highly-contended locks (that is, broken 967*7c478bd9Sstevel@tonic-gate * programs) to execute with reasonable speed despite their contention. 968*7c478bd9Sstevel@tonic-gate * Being fair would reduce the speed of such programs and well-written 969*7c478bd9Sstevel@tonic-gate * programs will not suffer in any case. 970*7c478bd9Sstevel@tonic-gate */ 971*7c478bd9Sstevel@tonic-gate enter_critical(self); /* protects ul_schedctl */ 972*7c478bd9Sstevel@tonic-gate incr32(&mp->mutex_spinners); 973*7c478bd9Sstevel@tonic-gate for (count = 0; count < max; count++) { 974*7c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) { 975*7c478bd9Sstevel@tonic-gate *ownerp = (uintptr_t)self; 976*7c478bd9Sstevel@tonic-gate decr32(&mp->mutex_spinners); 977*7c478bd9Sstevel@tonic-gate exit_critical(self); 978*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 979*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 980*7c478bd9Sstevel@tonic-gate return (0); 981*7c478bd9Sstevel@tonic-gate } 982*7c478bd9Sstevel@tonic-gate SMT_PAUSE(); 983*7c478bd9Sstevel@tonic-gate /* 984*7c478bd9Sstevel@tonic-gate * Stop spinning if the mutex owner is not running on 985*7c478bd9Sstevel@tonic-gate * a processor; it will not drop the lock any time soon 986*7c478bd9Sstevel@tonic-gate * and we would just be wasting time to keep spinning. 987*7c478bd9Sstevel@tonic-gate * 988*7c478bd9Sstevel@tonic-gate * Note that we are looking at another thread (ulwp_t) 989*7c478bd9Sstevel@tonic-gate * without ensuring that the other thread does not exit. 990*7c478bd9Sstevel@tonic-gate * The scheme relies on ulwp_t structures never being 991*7c478bd9Sstevel@tonic-gate * deallocated by the library (the library employs a free 992*7c478bd9Sstevel@tonic-gate * list of ulwp_t structs that are reused when new threads 993*7c478bd9Sstevel@tonic-gate * are created) and on schedctl shared memory never being 994*7c478bd9Sstevel@tonic-gate * deallocated once created via __schedctl(). 995*7c478bd9Sstevel@tonic-gate * 996*7c478bd9Sstevel@tonic-gate * Thus, the worst that can happen when the spinning thread 997*7c478bd9Sstevel@tonic-gate * looks at the owner's schedctl data is that it is looking 998*7c478bd9Sstevel@tonic-gate * at some other thread's schedctl data. This almost never 999*7c478bd9Sstevel@tonic-gate * happens and is benign when it does. 1000*7c478bd9Sstevel@tonic-gate */ 1001*7c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 1002*7c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 1003*7c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 1004*7c478bd9Sstevel@tonic-gate break; 1005*7c478bd9Sstevel@tonic-gate } 1006*7c478bd9Sstevel@tonic-gate decr32(&mp->mutex_spinners); 1007*7c478bd9Sstevel@tonic-gate exit_critical(self); 1008*7c478bd9Sstevel@tonic-gate 1009*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__spun, 0, count); 1010*7c478bd9Sstevel@tonic-gate 1011*7c478bd9Sstevel@tonic-gate return (EBUSY); 1012*7c478bd9Sstevel@tonic-gate } 1013*7c478bd9Sstevel@tonic-gate 1014*7c478bd9Sstevel@tonic-gate /* 1015*7c478bd9Sstevel@tonic-gate * Same as mutex_trylock_adaptive(), except specifically for queue locks. 1016*7c478bd9Sstevel@tonic-gate * The owner field is not set here; the caller (spin_lock_set()) sets it. 1017*7c478bd9Sstevel@tonic-gate */ 1018*7c478bd9Sstevel@tonic-gate int 1019*7c478bd9Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp) 1020*7c478bd9Sstevel@tonic-gate { 1021*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 1022*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 1023*7c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 1024*7c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 1025*7c478bd9Sstevel@tonic-gate int count = curthread->ul_queue_spin; 1026*7c478bd9Sstevel@tonic-gate 1027*7c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 1028*7c478bd9Sstevel@tonic-gate 1029*7c478bd9Sstevel@tonic-gate if (count == 0) 1030*7c478bd9Sstevel@tonic-gate return (EBUSY); 1031*7c478bd9Sstevel@tonic-gate 1032*7c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 1033*7c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 1034*7c478bd9Sstevel@tonic-gate while (--count >= 0) { 1035*7c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) 1036*7c478bd9Sstevel@tonic-gate return (0); 1037*7c478bd9Sstevel@tonic-gate SMT_PAUSE(); 1038*7c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 1039*7c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 1040*7c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 1041*7c478bd9Sstevel@tonic-gate break; 1042*7c478bd9Sstevel@tonic-gate } 1043*7c478bd9Sstevel@tonic-gate 1044*7c478bd9Sstevel@tonic-gate return (EBUSY); 1045*7c478bd9Sstevel@tonic-gate } 1046*7c478bd9Sstevel@tonic-gate 1047*7c478bd9Sstevel@tonic-gate /* 1048*7c478bd9Sstevel@tonic-gate * Like mutex_trylock_adaptive(), but for process-shared mutexes. 1049*7c478bd9Sstevel@tonic-gate * Spin for a while, trying to grab the lock. We know that we 1050*7c478bd9Sstevel@tonic-gate * failed set_lock_byte(&mp->mutex_lockw) once before coming here. 1051*7c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 1052*7c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread 1053*7c478bd9Sstevel@tonic-gate * and mutex_ownerpid set to the current pid. 1054*7c478bd9Sstevel@tonic-gate */ 1055*7c478bd9Sstevel@tonic-gate int 1056*7c478bd9Sstevel@tonic-gate mutex_trylock_process(mutex_t *mp) 1057*7c478bd9Sstevel@tonic-gate { 1058*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1059*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 1060*7c478bd9Sstevel@tonic-gate int count; 1061*7c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 1062*7c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 1063*7c478bd9Sstevel@tonic-gate volatile int32_t *pidp; 1064*7c478bd9Sstevel@tonic-gate pid_t pid, newpid; 1065*7c478bd9Sstevel@tonic-gate uint64_t owner, newowner; 1066*7c478bd9Sstevel@tonic-gate 1067*7c478bd9Sstevel@tonic-gate if ((count = ncpus) == 0) 1068*7c478bd9Sstevel@tonic-gate count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 1069*7c478bd9Sstevel@tonic-gate count = (count > 1)? self->ul_adaptive_spin : 0; 1070*7c478bd9Sstevel@tonic-gate 1071*7c478bd9Sstevel@tonic-gate ASSERT((mp->mutex_type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 1072*7c478bd9Sstevel@tonic-gate USYNC_PROCESS); 1073*7c478bd9Sstevel@tonic-gate 1074*7c478bd9Sstevel@tonic-gate if (count == 0) 1075*7c478bd9Sstevel@tonic-gate return (EBUSY); 1076*7c478bd9Sstevel@tonic-gate 1077*7c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 1078*7c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 1079*7c478bd9Sstevel@tonic-gate pidp = (volatile int32_t *)&mp->mutex_ownerpid; 1080*7c478bd9Sstevel@tonic-gate owner = *ownerp; 1081*7c478bd9Sstevel@tonic-gate pid = *pidp; 1082*7c478bd9Sstevel@tonic-gate /* 1083*7c478bd9Sstevel@tonic-gate * This is a process-shared mutex. 1084*7c478bd9Sstevel@tonic-gate * We cannot know if the owner is running on a processor. 1085*7c478bd9Sstevel@tonic-gate * We just spin and hope that it is on a processor. 1086*7c478bd9Sstevel@tonic-gate */ 1087*7c478bd9Sstevel@tonic-gate while (--count >= 0) { 1088*7c478bd9Sstevel@tonic-gate if (*lockp == 0) { 1089*7c478bd9Sstevel@tonic-gate enter_critical(self); 1090*7c478bd9Sstevel@tonic-gate if (set_lock_byte(lockp) == 0) { 1091*7c478bd9Sstevel@tonic-gate *ownerp = (uintptr_t)self; 1092*7c478bd9Sstevel@tonic-gate *pidp = udp->pid; 1093*7c478bd9Sstevel@tonic-gate exit_critical(self); 1094*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1095*7c478bd9Sstevel@tonic-gate 0, 0); 1096*7c478bd9Sstevel@tonic-gate return (0); 1097*7c478bd9Sstevel@tonic-gate } 1098*7c478bd9Sstevel@tonic-gate exit_critical(self); 1099*7c478bd9Sstevel@tonic-gate } else if ((newowner = *ownerp) == owner && 1100*7c478bd9Sstevel@tonic-gate (newpid = *pidp) == pid) { 1101*7c478bd9Sstevel@tonic-gate SMT_PAUSE(); 1102*7c478bd9Sstevel@tonic-gate continue; 1103*7c478bd9Sstevel@tonic-gate } 1104*7c478bd9Sstevel@tonic-gate /* 1105*7c478bd9Sstevel@tonic-gate * The owner of the lock changed; start the count over again. 1106*7c478bd9Sstevel@tonic-gate * This may be too aggressive; it needs testing. 1107*7c478bd9Sstevel@tonic-gate */ 1108*7c478bd9Sstevel@tonic-gate owner = newowner; 1109*7c478bd9Sstevel@tonic-gate pid = newpid; 1110*7c478bd9Sstevel@tonic-gate count = self->ul_adaptive_spin; 1111*7c478bd9Sstevel@tonic-gate } 1112*7c478bd9Sstevel@tonic-gate 1113*7c478bd9Sstevel@tonic-gate return (EBUSY); 1114*7c478bd9Sstevel@tonic-gate } 1115*7c478bd9Sstevel@tonic-gate 1116*7c478bd9Sstevel@tonic-gate /* 1117*7c478bd9Sstevel@tonic-gate * Mutex wakeup code for releasing a USYNC_THREAD mutex. 1118*7c478bd9Sstevel@tonic-gate * Returns the lwpid of the thread that was dequeued, if any. 1119*7c478bd9Sstevel@tonic-gate * The caller of mutex_wakeup() must call __lwp_unpark(lwpid) 1120*7c478bd9Sstevel@tonic-gate * to wake up the specified lwp. 1121*7c478bd9Sstevel@tonic-gate */ 1122*7c478bd9Sstevel@tonic-gate lwpid_t 1123*7c478bd9Sstevel@tonic-gate mutex_wakeup(mutex_t *mp) 1124*7c478bd9Sstevel@tonic-gate { 1125*7c478bd9Sstevel@tonic-gate lwpid_t lwpid = 0; 1126*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 1127*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 1128*7c478bd9Sstevel@tonic-gate int more; 1129*7c478bd9Sstevel@tonic-gate 1130*7c478bd9Sstevel@tonic-gate /* 1131*7c478bd9Sstevel@tonic-gate * Dequeue a waiter from the sleep queue. Don't touch the mutex 1132*7c478bd9Sstevel@tonic-gate * waiters bit if no one was found on the queue because the mutex 1133*7c478bd9Sstevel@tonic-gate * might have been deallocated or reallocated for another purpose. 1134*7c478bd9Sstevel@tonic-gate */ 1135*7c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 1136*7c478bd9Sstevel@tonic-gate if ((ulwp = dequeue(qp, mp, &more)) != NULL) { 1137*7c478bd9Sstevel@tonic-gate lwpid = ulwp->ul_lwpid; 1138*7c478bd9Sstevel@tonic-gate mp->mutex_waiters = (more? 1 : 0); 1139*7c478bd9Sstevel@tonic-gate } 1140*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 1141*7c478bd9Sstevel@tonic-gate return (lwpid); 1142*7c478bd9Sstevel@tonic-gate } 1143*7c478bd9Sstevel@tonic-gate 1144*7c478bd9Sstevel@tonic-gate /* 1145*7c478bd9Sstevel@tonic-gate * Spin for a while, testing to see if the lock has been grabbed. 1146*7c478bd9Sstevel@tonic-gate * If this fails, call mutex_wakeup() to release a waiter. 1147*7c478bd9Sstevel@tonic-gate */ 1148*7c478bd9Sstevel@tonic-gate lwpid_t 1149*7c478bd9Sstevel@tonic-gate mutex_unlock_queue(mutex_t *mp) 1150*7c478bd9Sstevel@tonic-gate { 1151*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1152*7c478bd9Sstevel@tonic-gate uint32_t *lockw = &mp->mutex_lockword; 1153*7c478bd9Sstevel@tonic-gate lwpid_t lwpid; 1154*7c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 1155*7c478bd9Sstevel@tonic-gate volatile uint32_t *spinp; 1156*7c478bd9Sstevel@tonic-gate int count; 1157*7c478bd9Sstevel@tonic-gate 1158*7c478bd9Sstevel@tonic-gate /* 1159*7c478bd9Sstevel@tonic-gate * We use the swap primitive to clear the lock, but we must 1160*7c478bd9Sstevel@tonic-gate * atomically retain the waiters bit for the remainder of this 1161*7c478bd9Sstevel@tonic-gate * code to work. We first check to see if the waiters bit is 1162*7c478bd9Sstevel@tonic-gate * set and if so clear the lock by swapping in a word containing 1163*7c478bd9Sstevel@tonic-gate * only the waiters bit. This could produce a false positive test 1164*7c478bd9Sstevel@tonic-gate * for whether there are waiters that need to be waked up, but 1165*7c478bd9Sstevel@tonic-gate * this just causes an extra call to mutex_wakeup() to do nothing. 1166*7c478bd9Sstevel@tonic-gate * The opposite case is more delicate: If there are no waiters, 1167*7c478bd9Sstevel@tonic-gate * we swap in a zero lock byte and a zero waiters bit. The result 1168*7c478bd9Sstevel@tonic-gate * of the swap could indicate that there really was a waiter so in 1169*7c478bd9Sstevel@tonic-gate * this case we go directly to mutex_wakeup() without performing 1170*7c478bd9Sstevel@tonic-gate * any of the adaptive code because the waiter bit has been cleared 1171*7c478bd9Sstevel@tonic-gate * and the adaptive code is unreliable in this case. 1172*7c478bd9Sstevel@tonic-gate */ 1173*7c478bd9Sstevel@tonic-gate if (!(*lockw & WAITERMASK)) { /* no waiter exists right now */ 1174*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 1175*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 1176*7c478bd9Sstevel@tonic-gate if (!(swap32(lockw, 0) & WAITERMASK)) /* still no waiters */ 1177*7c478bd9Sstevel@tonic-gate return (0); 1178*7c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 1179*7c478bd9Sstevel@tonic-gate lwpid = mutex_wakeup(mp); 1180*7c478bd9Sstevel@tonic-gate } else { 1181*7c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 1182*7c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 1183*7c478bd9Sstevel@tonic-gate spinp = (volatile uint32_t *)&mp->mutex_spinners; 1184*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 1185*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 1186*7c478bd9Sstevel@tonic-gate (void) swap32(lockw, WAITER); /* clear lock, retain waiter */ 1187*7c478bd9Sstevel@tonic-gate 1188*7c478bd9Sstevel@tonic-gate /* 1189*7c478bd9Sstevel@tonic-gate * We spin here fewer times than mutex_trylock_adaptive(). 1190*7c478bd9Sstevel@tonic-gate * We are trying to balance two conflicting goals: 1191*7c478bd9Sstevel@tonic-gate * 1. Avoid waking up anyone if a spinning thread 1192*7c478bd9Sstevel@tonic-gate * grabs the lock. 1193*7c478bd9Sstevel@tonic-gate * 2. Wake up a sleeping thread promptly to get on 1194*7c478bd9Sstevel@tonic-gate * with useful work. 1195*7c478bd9Sstevel@tonic-gate * We don't spin at all if there is no acquiring spinner; 1196*7c478bd9Sstevel@tonic-gate * (mp->mutex_spinners is non-zero if there are spinners). 1197*7c478bd9Sstevel@tonic-gate */ 1198*7c478bd9Sstevel@tonic-gate for (count = self->ul_release_spin; 1199*7c478bd9Sstevel@tonic-gate *spinp && count > 0; count--) { 1200*7c478bd9Sstevel@tonic-gate /* 1201*7c478bd9Sstevel@tonic-gate * There is a waiter that we will have to wake 1202*7c478bd9Sstevel@tonic-gate * up unless someone else grabs the lock while 1203*7c478bd9Sstevel@tonic-gate * we are busy spinning. Like the spin loop in 1204*7c478bd9Sstevel@tonic-gate * mutex_trylock_adaptive(), this spin loop is 1205*7c478bd9Sstevel@tonic-gate * unfair to lwps that have already dropped into 1206*7c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a 1207*7c478bd9Sstevel@tonic-gate * highly-contended mutex. Too bad. 1208*7c478bd9Sstevel@tonic-gate */ 1209*7c478bd9Sstevel@tonic-gate if (*lockp != 0) { /* somebody grabbed the lock */ 1210*7c478bd9Sstevel@tonic-gate preempt(self); 1211*7c478bd9Sstevel@tonic-gate return (0); 1212*7c478bd9Sstevel@tonic-gate } 1213*7c478bd9Sstevel@tonic-gate SMT_PAUSE(); 1214*7c478bd9Sstevel@tonic-gate } 1215*7c478bd9Sstevel@tonic-gate 1216*7c478bd9Sstevel@tonic-gate /* 1217*7c478bd9Sstevel@tonic-gate * No one grabbed the lock. 1218*7c478bd9Sstevel@tonic-gate * Wake up some lwp that is waiting for it. 1219*7c478bd9Sstevel@tonic-gate */ 1220*7c478bd9Sstevel@tonic-gate mp->mutex_waiters = 0; 1221*7c478bd9Sstevel@tonic-gate lwpid = mutex_wakeup(mp); 1222*7c478bd9Sstevel@tonic-gate } 1223*7c478bd9Sstevel@tonic-gate 1224*7c478bd9Sstevel@tonic-gate if (lwpid == 0) 1225*7c478bd9Sstevel@tonic-gate preempt(self); 1226*7c478bd9Sstevel@tonic-gate return (lwpid); 1227*7c478bd9Sstevel@tonic-gate } 1228*7c478bd9Sstevel@tonic-gate 1229*7c478bd9Sstevel@tonic-gate /* 1230*7c478bd9Sstevel@tonic-gate * Like mutex_unlock_queue(), but for process-shared mutexes. 1231*7c478bd9Sstevel@tonic-gate * We tested the waiters field before calling here and it was non-zero. 1232*7c478bd9Sstevel@tonic-gate */ 1233*7c478bd9Sstevel@tonic-gate void 1234*7c478bd9Sstevel@tonic-gate mutex_unlock_process(mutex_t *mp) 1235*7c478bd9Sstevel@tonic-gate { 1236*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1237*7c478bd9Sstevel@tonic-gate int count; 1238*7c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 1239*7c478bd9Sstevel@tonic-gate 1240*7c478bd9Sstevel@tonic-gate /* 1241*7c478bd9Sstevel@tonic-gate * See the comments in mutex_unlock_queue(), above. 1242*7c478bd9Sstevel@tonic-gate */ 1243*7c478bd9Sstevel@tonic-gate if ((count = ncpus) == 0) 1244*7c478bd9Sstevel@tonic-gate count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 1245*7c478bd9Sstevel@tonic-gate count = (count > 1)? self->ul_release_spin : 0; 1246*7c478bd9Sstevel@tonic-gate no_preempt(self); 1247*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 1248*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 1249*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 1250*7c478bd9Sstevel@tonic-gate if (count == 0) { 1251*7c478bd9Sstevel@tonic-gate /* clear lock, test waiter */ 1252*7c478bd9Sstevel@tonic-gate if (!(swap32(&mp->mutex_lockword, 0) & WAITERMASK)) { 1253*7c478bd9Sstevel@tonic-gate /* no waiters now */ 1254*7c478bd9Sstevel@tonic-gate preempt(self); 1255*7c478bd9Sstevel@tonic-gate return; 1256*7c478bd9Sstevel@tonic-gate } 1257*7c478bd9Sstevel@tonic-gate } else { 1258*7c478bd9Sstevel@tonic-gate /* clear lock, retain waiter */ 1259*7c478bd9Sstevel@tonic-gate (void) swap32(&mp->mutex_lockword, WAITER); 1260*7c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 1261*7c478bd9Sstevel@tonic-gate while (--count >= 0) { 1262*7c478bd9Sstevel@tonic-gate if (*lockp != 0) { 1263*7c478bd9Sstevel@tonic-gate /* somebody grabbed the lock */ 1264*7c478bd9Sstevel@tonic-gate preempt(self); 1265*7c478bd9Sstevel@tonic-gate return; 1266*7c478bd9Sstevel@tonic-gate } 1267*7c478bd9Sstevel@tonic-gate SMT_PAUSE(); 1268*7c478bd9Sstevel@tonic-gate } 1269*7c478bd9Sstevel@tonic-gate /* 1270*7c478bd9Sstevel@tonic-gate * We must clear the waiters field before going 1271*7c478bd9Sstevel@tonic-gate * to the kernel, else it could remain set forever. 1272*7c478bd9Sstevel@tonic-gate */ 1273*7c478bd9Sstevel@tonic-gate mp->mutex_waiters = 0; 1274*7c478bd9Sstevel@tonic-gate } 1275*7c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 1276*7c478bd9Sstevel@tonic-gate preempt(self); 1277*7c478bd9Sstevel@tonic-gate } 1278*7c478bd9Sstevel@tonic-gate 1279*7c478bd9Sstevel@tonic-gate /* 1280*7c478bd9Sstevel@tonic-gate * Return the real priority of a thread. 1281*7c478bd9Sstevel@tonic-gate */ 1282*7c478bd9Sstevel@tonic-gate int 1283*7c478bd9Sstevel@tonic-gate real_priority(ulwp_t *ulwp) 1284*7c478bd9Sstevel@tonic-gate { 1285*7c478bd9Sstevel@tonic-gate if (ulwp->ul_epri == 0) 1286*7c478bd9Sstevel@tonic-gate return (ulwp->ul_mappedpri? ulwp->ul_mappedpri : ulwp->ul_pri); 1287*7c478bd9Sstevel@tonic-gate return (ulwp->ul_emappedpri? ulwp->ul_emappedpri : ulwp->ul_epri); 1288*7c478bd9Sstevel@tonic-gate } 1289*7c478bd9Sstevel@tonic-gate 1290*7c478bd9Sstevel@tonic-gate void 1291*7c478bd9Sstevel@tonic-gate stall(void) 1292*7c478bd9Sstevel@tonic-gate { 1293*7c478bd9Sstevel@tonic-gate for (;;) 1294*7c478bd9Sstevel@tonic-gate (void) mutex_lock_kernel(&stall_mutex, NULL, NULL); 1295*7c478bd9Sstevel@tonic-gate } 1296*7c478bd9Sstevel@tonic-gate 1297*7c478bd9Sstevel@tonic-gate /* 1298*7c478bd9Sstevel@tonic-gate * Acquire a USYNC_THREAD mutex via user-level sleep queues. 1299*7c478bd9Sstevel@tonic-gate * We failed set_lock_byte(&mp->mutex_lockw) before coming here. 1300*7c478bd9Sstevel@tonic-gate * Returns with mutex_owner set correctly. 1301*7c478bd9Sstevel@tonic-gate */ 1302*7c478bd9Sstevel@tonic-gate int 1303*7c478bd9Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp, 1304*7c478bd9Sstevel@tonic-gate timespec_t *tsp) 1305*7c478bd9Sstevel@tonic-gate { 1306*7c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 1307*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 1308*7c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 1309*7c478bd9Sstevel@tonic-gate int error = 0; 1310*7c478bd9Sstevel@tonic-gate 1311*7c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 1312*7c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 1313*7c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 1314*7c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 1315*7c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 1316*7c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 1317*7c478bd9Sstevel@tonic-gate } 1318*7c478bd9Sstevel@tonic-gate if (msp) { 1319*7c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 1320*7c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 1321*7c478bd9Sstevel@tonic-gate } 1322*7c478bd9Sstevel@tonic-gate 1323*7c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 1324*7c478bd9Sstevel@tonic-gate 1325*7c478bd9Sstevel@tonic-gate /* 1326*7c478bd9Sstevel@tonic-gate * Put ourself on the sleep queue, and while we are 1327*7c478bd9Sstevel@tonic-gate * unable to grab the lock, go park in the kernel. 1328*7c478bd9Sstevel@tonic-gate * Take ourself off the sleep queue after we acquire the lock. 1329*7c478bd9Sstevel@tonic-gate * The waiter bit can be set/cleared only while holding the queue lock. 1330*7c478bd9Sstevel@tonic-gate */ 1331*7c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 1332*7c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 1333*7c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 1334*7c478bd9Sstevel@tonic-gate for (;;) { 1335*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 1336*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1337*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 1338*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1339*7c478bd9Sstevel@tonic-gate mp->mutex_waiters = dequeue_self(qp, mp); 1340*7c478bd9Sstevel@tonic-gate break; 1341*7c478bd9Sstevel@tonic-gate } 1342*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 1343*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 1344*7c478bd9Sstevel@tonic-gate /* 1345*7c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 1346*7c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 1347*7c478bd9Sstevel@tonic-gate */ 1348*7c478bd9Sstevel@tonic-gate if ((error = __lwp_park(tsp, 0)) == EINTR) 1349*7c478bd9Sstevel@tonic-gate error = 0; 1350*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 1351*7c478bd9Sstevel@tonic-gate /* 1352*7c478bd9Sstevel@tonic-gate * We could have taken a signal or suspended ourself. 1353*7c478bd9Sstevel@tonic-gate * If we did, then we removed ourself from the queue. 1354*7c478bd9Sstevel@tonic-gate * Someone else may have removed us from the queue 1355*7c478bd9Sstevel@tonic-gate * as a consequence of mutex_unlock(). We may have 1356*7c478bd9Sstevel@tonic-gate * gotten a timeout from __lwp_park(). Or we may still 1357*7c478bd9Sstevel@tonic-gate * be on the queue and this is just a spurious wakeup. 1358*7c478bd9Sstevel@tonic-gate */ 1359*7c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 1360*7c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) { 1361*7c478bd9Sstevel@tonic-gate if (error) { 1362*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 1363*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 1364*7c478bd9Sstevel@tonic-gate error); 1365*7c478bd9Sstevel@tonic-gate break; 1366*7c478bd9Sstevel@tonic-gate } 1367*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 1368*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1369*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 1370*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1371*7c478bd9Sstevel@tonic-gate 0, 0); 1372*7c478bd9Sstevel@tonic-gate break; 1373*7c478bd9Sstevel@tonic-gate } 1374*7c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 1375*7c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 1376*7c478bd9Sstevel@tonic-gate } 1377*7c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == qp && 1378*7c478bd9Sstevel@tonic-gate self->ul_qtype == MX && 1379*7c478bd9Sstevel@tonic-gate self->ul_wchan == mp); 1380*7c478bd9Sstevel@tonic-gate if (error) { 1381*7c478bd9Sstevel@tonic-gate mp->mutex_waiters = dequeue_self(qp, mp); 1382*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 1383*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1384*7c478bd9Sstevel@tonic-gate break; 1385*7c478bd9Sstevel@tonic-gate } 1386*7c478bd9Sstevel@tonic-gate } 1387*7c478bd9Sstevel@tonic-gate 1388*7c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 1389*7c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 1390*7c478bd9Sstevel@tonic-gate self->ul_sp = 0; 1391*7c478bd9Sstevel@tonic-gate 1392*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 1393*7c478bd9Sstevel@tonic-gate if (msp) 1394*7c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 1395*7c478bd9Sstevel@tonic-gate 1396*7c478bd9Sstevel@tonic-gate ASSERT(error == 0 || error == EINVAL || error == ETIME); 1397*7c478bd9Sstevel@tonic-gate return (error); 1398*7c478bd9Sstevel@tonic-gate } 1399*7c478bd9Sstevel@tonic-gate 1400*7c478bd9Sstevel@tonic-gate /* 1401*7c478bd9Sstevel@tonic-gate * Returns with mutex_owner set correctly. 1402*7c478bd9Sstevel@tonic-gate */ 1403*7c478bd9Sstevel@tonic-gate int 1404*7c478bd9Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try) 1405*7c478bd9Sstevel@tonic-gate { 1406*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1407*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 1408*7c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 1409*7c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 1410*7c478bd9Sstevel@tonic-gate int error = 0; 1411*7c478bd9Sstevel@tonic-gate 1412*7c478bd9Sstevel@tonic-gate ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK); 1413*7c478bd9Sstevel@tonic-gate 1414*7c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 1415*7c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 1416*7c478bd9Sstevel@tonic-gate 1417*7c478bd9Sstevel@tonic-gate if (msp && try == MUTEX_TRY) 1418*7c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try); 1419*7c478bd9Sstevel@tonic-gate 1420*7c478bd9Sstevel@tonic-gate if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) { 1421*7c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 1422*7c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) { 1423*7c478bd9Sstevel@tonic-gate error = EAGAIN; 1424*7c478bd9Sstevel@tonic-gate } else { 1425*7c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 1426*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1427*7c478bd9Sstevel@tonic-gate 1, 0); 1428*7c478bd9Sstevel@tonic-gate return (0); 1429*7c478bd9Sstevel@tonic-gate } 1430*7c478bd9Sstevel@tonic-gate } else if (try == MUTEX_TRY) { 1431*7c478bd9Sstevel@tonic-gate return (EBUSY); 1432*7c478bd9Sstevel@tonic-gate } else { 1433*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 1434*7c478bd9Sstevel@tonic-gate return (EDEADLK); 1435*7c478bd9Sstevel@tonic-gate } 1436*7c478bd9Sstevel@tonic-gate } 1437*7c478bd9Sstevel@tonic-gate 1438*7c478bd9Sstevel@tonic-gate if (self->ul_error_detection && try == MUTEX_LOCK && 1439*7c478bd9Sstevel@tonic-gate tsp == NULL && mutex_is_held(mp)) 1440*7c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_lock", NULL, NULL); 1441*7c478bd9Sstevel@tonic-gate 1442*7c478bd9Sstevel@tonic-gate if (mtype & 1443*7c478bd9Sstevel@tonic-gate (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) { 1444*7c478bd9Sstevel@tonic-gate uint8_t ceil; 1445*7c478bd9Sstevel@tonic-gate int myprio; 1446*7c478bd9Sstevel@tonic-gate 1447*7c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_PROTECT) { 1448*7c478bd9Sstevel@tonic-gate ceil = mp->mutex_ceiling; 1449*7c478bd9Sstevel@tonic-gate ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0); 1450*7c478bd9Sstevel@tonic-gate myprio = real_priority(self); 1451*7c478bd9Sstevel@tonic-gate if (myprio > ceil) { 1452*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 1453*7c478bd9Sstevel@tonic-gate EINVAL); 1454*7c478bd9Sstevel@tonic-gate return (EINVAL); 1455*7c478bd9Sstevel@tonic-gate } 1456*7c478bd9Sstevel@tonic-gate if ((error = _ceil_mylist_add(mp)) != 0) { 1457*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 1458*7c478bd9Sstevel@tonic-gate error); 1459*7c478bd9Sstevel@tonic-gate return (error); 1460*7c478bd9Sstevel@tonic-gate } 1461*7c478bd9Sstevel@tonic-gate if (myprio < ceil) 1462*7c478bd9Sstevel@tonic-gate _ceil_prio_inherit(ceil); 1463*7c478bd9Sstevel@tonic-gate } 1464*7c478bd9Sstevel@tonic-gate 1465*7c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) { 1466*7c478bd9Sstevel@tonic-gate /* go straight to the kernel */ 1467*7c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) 1468*7c478bd9Sstevel@tonic-gate error = mutex_trylock_kernel(mp); 1469*7c478bd9Sstevel@tonic-gate else /* MUTEX_LOCK */ 1470*7c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 1471*7c478bd9Sstevel@tonic-gate /* 1472*7c478bd9Sstevel@tonic-gate * The kernel never sets or clears the lock byte 1473*7c478bd9Sstevel@tonic-gate * for PTHREAD_PRIO_INHERIT mutexes. 1474*7c478bd9Sstevel@tonic-gate * Set it here for debugging consistency. 1475*7c478bd9Sstevel@tonic-gate */ 1476*7c478bd9Sstevel@tonic-gate switch (error) { 1477*7c478bd9Sstevel@tonic-gate case 0: 1478*7c478bd9Sstevel@tonic-gate case EOWNERDEAD: 1479*7c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 1480*7c478bd9Sstevel@tonic-gate break; 1481*7c478bd9Sstevel@tonic-gate } 1482*7c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS_ROBUST) { 1483*7c478bd9Sstevel@tonic-gate /* go straight to the kernel */ 1484*7c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) 1485*7c478bd9Sstevel@tonic-gate error = mutex_trylock_kernel(mp); 1486*7c478bd9Sstevel@tonic-gate else /* MUTEX_LOCK */ 1487*7c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 1488*7c478bd9Sstevel@tonic-gate } else { /* PTHREAD_PRIO_PROTECT */ 1489*7c478bd9Sstevel@tonic-gate /* 1490*7c478bd9Sstevel@tonic-gate * Try once at user level before going to the kernel. 1491*7c478bd9Sstevel@tonic-gate * If this is a process shared mutex then protect 1492*7c478bd9Sstevel@tonic-gate * against forkall() while setting mp->mutex_ownerpid. 1493*7c478bd9Sstevel@tonic-gate */ 1494*7c478bd9Sstevel@tonic-gate if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) { 1495*7c478bd9Sstevel@tonic-gate enter_critical(self); 1496*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 1497*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1498*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 1499*7c478bd9Sstevel@tonic-gate exit_critical(self); 1500*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, 1501*7c478bd9Sstevel@tonic-gate mutex__acquire, mp, 0, 0); 1502*7c478bd9Sstevel@tonic-gate } else { 1503*7c478bd9Sstevel@tonic-gate exit_critical(self); 1504*7c478bd9Sstevel@tonic-gate error = EBUSY; 1505*7c478bd9Sstevel@tonic-gate } 1506*7c478bd9Sstevel@tonic-gate } else { 1507*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 1508*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1509*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, 1510*7c478bd9Sstevel@tonic-gate mutex__acquire, mp, 0, 0); 1511*7c478bd9Sstevel@tonic-gate } else { 1512*7c478bd9Sstevel@tonic-gate error = EBUSY; 1513*7c478bd9Sstevel@tonic-gate } 1514*7c478bd9Sstevel@tonic-gate } 1515*7c478bd9Sstevel@tonic-gate if (error && try == MUTEX_LOCK) 1516*7c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 1517*7c478bd9Sstevel@tonic-gate } 1518*7c478bd9Sstevel@tonic-gate 1519*7c478bd9Sstevel@tonic-gate if (error) { 1520*7c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) { 1521*7c478bd9Sstevel@tonic-gate switch (error) { 1522*7c478bd9Sstevel@tonic-gate case EOWNERDEAD: 1523*7c478bd9Sstevel@tonic-gate case ENOTRECOVERABLE: 1524*7c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_MUTEX_ROBUST_NP) 1525*7c478bd9Sstevel@tonic-gate break; 1526*7c478bd9Sstevel@tonic-gate if (error == EOWNERDEAD) { 1527*7c478bd9Sstevel@tonic-gate /* 1528*7c478bd9Sstevel@tonic-gate * We own the mutex; unlock it. 1529*7c478bd9Sstevel@tonic-gate * It becomes ENOTRECOVERABLE. 1530*7c478bd9Sstevel@tonic-gate * All waiters are waked up. 1531*7c478bd9Sstevel@tonic-gate */ 1532*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 1533*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 1534*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, 1535*7c478bd9Sstevel@tonic-gate mutex__release, mp, 0); 1536*7c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 1537*7c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_unlock(mp); 1538*7c478bd9Sstevel@tonic-gate } 1539*7c478bd9Sstevel@tonic-gate /* FALLTHROUGH */ 1540*7c478bd9Sstevel@tonic-gate case EDEADLK: 1541*7c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) 1542*7c478bd9Sstevel@tonic-gate stall(); 1543*7c478bd9Sstevel@tonic-gate error = EBUSY; 1544*7c478bd9Sstevel@tonic-gate break; 1545*7c478bd9Sstevel@tonic-gate } 1546*7c478bd9Sstevel@tonic-gate } 1547*7c478bd9Sstevel@tonic-gate if ((mtype & PTHREAD_PRIO_PROTECT) && 1548*7c478bd9Sstevel@tonic-gate error != EOWNERDEAD) { 1549*7c478bd9Sstevel@tonic-gate (void) _ceil_mylist_del(mp); 1550*7c478bd9Sstevel@tonic-gate if (myprio < ceil) 1551*7c478bd9Sstevel@tonic-gate _ceil_prio_waive(); 1552*7c478bd9Sstevel@tonic-gate } 1553*7c478bd9Sstevel@tonic-gate } 1554*7c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) { 1555*7c478bd9Sstevel@tonic-gate /* 1556*7c478bd9Sstevel@tonic-gate * This is a process shared mutex. Protect against 1557*7c478bd9Sstevel@tonic-gate * forkall() while setting mp->mutex_ownerpid. 1558*7c478bd9Sstevel@tonic-gate */ 1559*7c478bd9Sstevel@tonic-gate enter_critical(self); 1560*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 1561*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1562*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 1563*7c478bd9Sstevel@tonic-gate exit_critical(self); 1564*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1565*7c478bd9Sstevel@tonic-gate } else { 1566*7c478bd9Sstevel@tonic-gate /* try a little harder */ 1567*7c478bd9Sstevel@tonic-gate exit_critical(self); 1568*7c478bd9Sstevel@tonic-gate error = mutex_trylock_process(mp); 1569*7c478bd9Sstevel@tonic-gate } 1570*7c478bd9Sstevel@tonic-gate if (error && try == MUTEX_LOCK) 1571*7c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 1572*7c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 1573*7c478bd9Sstevel@tonic-gate /* try once */ 1574*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 1575*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1576*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1577*7c478bd9Sstevel@tonic-gate } else { 1578*7c478bd9Sstevel@tonic-gate /* try a little harder if we don't own the mutex */ 1579*7c478bd9Sstevel@tonic-gate error = EBUSY; 1580*7c478bd9Sstevel@tonic-gate if (MUTEX_OWNER(mp) != self) 1581*7c478bd9Sstevel@tonic-gate error = mutex_trylock_adaptive(mp); 1582*7c478bd9Sstevel@tonic-gate if (error && try == MUTEX_LOCK) /* go park */ 1583*7c478bd9Sstevel@tonic-gate error = mutex_lock_queue(self, msp, mp, tsp); 1584*7c478bd9Sstevel@tonic-gate } 1585*7c478bd9Sstevel@tonic-gate } 1586*7c478bd9Sstevel@tonic-gate 1587*7c478bd9Sstevel@tonic-gate switch (error) { 1588*7c478bd9Sstevel@tonic-gate case EOWNERDEAD: 1589*7c478bd9Sstevel@tonic-gate case ELOCKUNMAPPED: 1590*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1591*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1592*7c478bd9Sstevel@tonic-gate /* FALLTHROUGH */ 1593*7c478bd9Sstevel@tonic-gate case 0: 1594*7c478bd9Sstevel@tonic-gate if (msp) 1595*7c478bd9Sstevel@tonic-gate record_begin_hold(msp); 1596*7c478bd9Sstevel@tonic-gate break; 1597*7c478bd9Sstevel@tonic-gate default: 1598*7c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) { 1599*7c478bd9Sstevel@tonic-gate if (msp) 1600*7c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try_fail); 1601*7c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 1602*7c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 1603*7c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 1604*7c478bd9Sstevel@tonic-gate } 1605*7c478bd9Sstevel@tonic-gate } 1606*7c478bd9Sstevel@tonic-gate break; 1607*7c478bd9Sstevel@tonic-gate } 1608*7c478bd9Sstevel@tonic-gate 1609*7c478bd9Sstevel@tonic-gate return (error); 1610*7c478bd9Sstevel@tonic-gate } 1611*7c478bd9Sstevel@tonic-gate 1612*7c478bd9Sstevel@tonic-gate int 1613*7c478bd9Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try) 1614*7c478bd9Sstevel@tonic-gate { 1615*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1616*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 1617*7c478bd9Sstevel@tonic-gate 1618*7c478bd9Sstevel@tonic-gate /* 1619*7c478bd9Sstevel@tonic-gate * We know that USYNC_PROCESS is set in mtype and that 1620*7c478bd9Sstevel@tonic-gate * zero, one, or both of the flags LOCK_RECURSIVE and 1621*7c478bd9Sstevel@tonic-gate * LOCK_ERRORCHECK are set, and that no other flags are set. 1622*7c478bd9Sstevel@tonic-gate */ 1623*7c478bd9Sstevel@tonic-gate enter_critical(self); 1624*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 1625*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1626*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 1627*7c478bd9Sstevel@tonic-gate exit_critical(self); 1628*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1629*7c478bd9Sstevel@tonic-gate return (0); 1630*7c478bd9Sstevel@tonic-gate } 1631*7c478bd9Sstevel@tonic-gate exit_critical(self); 1632*7c478bd9Sstevel@tonic-gate 1633*7c478bd9Sstevel@tonic-gate if ((mtype & ~USYNC_PROCESS) && shared_mutex_held(mp)) { 1634*7c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 1635*7c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 1636*7c478bd9Sstevel@tonic-gate return (EAGAIN); 1637*7c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 1638*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0); 1639*7c478bd9Sstevel@tonic-gate return (0); 1640*7c478bd9Sstevel@tonic-gate } 1641*7c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) { 1642*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 1643*7c478bd9Sstevel@tonic-gate return (EDEADLK); 1644*7c478bd9Sstevel@tonic-gate } 1645*7c478bd9Sstevel@tonic-gate return (EBUSY); 1646*7c478bd9Sstevel@tonic-gate } 1647*7c478bd9Sstevel@tonic-gate 1648*7c478bd9Sstevel@tonic-gate /* try a little harder if we don't own the mutex */ 1649*7c478bd9Sstevel@tonic-gate if (!shared_mutex_held(mp) && mutex_trylock_process(mp) == 0) 1650*7c478bd9Sstevel@tonic-gate return (0); 1651*7c478bd9Sstevel@tonic-gate 1652*7c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) 1653*7c478bd9Sstevel@tonic-gate return (mutex_lock_kernel(mp, tsp, NULL)); 1654*7c478bd9Sstevel@tonic-gate 1655*7c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 1656*7c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 1657*7c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 1658*7c478bd9Sstevel@tonic-gate } 1659*7c478bd9Sstevel@tonic-gate return (EBUSY); 1660*7c478bd9Sstevel@tonic-gate } 1661*7c478bd9Sstevel@tonic-gate 1662*7c478bd9Sstevel@tonic-gate static int 1663*7c478bd9Sstevel@tonic-gate slow_lock(ulwp_t *self, mutex_t *mp, timespec_t *tsp) 1664*7c478bd9Sstevel@tonic-gate { 1665*7c478bd9Sstevel@tonic-gate int error = 0; 1666*7c478bd9Sstevel@tonic-gate 1667*7c478bd9Sstevel@tonic-gate if (MUTEX_OWNER(mp) == self || mutex_trylock_adaptive(mp) != 0) 1668*7c478bd9Sstevel@tonic-gate error = mutex_lock_queue(self, NULL, mp, tsp); 1669*7c478bd9Sstevel@tonic-gate return (error); 1670*7c478bd9Sstevel@tonic-gate } 1671*7c478bd9Sstevel@tonic-gate 1672*7c478bd9Sstevel@tonic-gate int 1673*7c478bd9Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp) 1674*7c478bd9Sstevel@tonic-gate { 1675*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1676*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 1677*7c478bd9Sstevel@tonic-gate uberflags_t *gflags; 1678*7c478bd9Sstevel@tonic-gate int mtype; 1679*7c478bd9Sstevel@tonic-gate 1680*7c478bd9Sstevel@tonic-gate /* 1681*7c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 1682*7c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 1683*7c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 1684*7c478bd9Sstevel@tonic-gate * and the process has only a single thread. 1685*7c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 1686*7c478bd9Sstevel@tonic-gate */ 1687*7c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 1688*7c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 1689*7c478bd9Sstevel@tonic-gate /* 1690*7c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 1691*7c478bd9Sstevel@tonic-gate */ 1692*7c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 1693*7c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 1694*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1695*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1696*7c478bd9Sstevel@tonic-gate return (0); 1697*7c478bd9Sstevel@tonic-gate } 1698*7c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 1699*7c478bd9Sstevel@tonic-gate /* 1700*7c478bd9Sstevel@tonic-gate * LOCK_RECURSIVE, LOCK_ERRORCHECK, or both. 1701*7c478bd9Sstevel@tonic-gate */ 1702*7c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 1703*7c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 1704*7c478bd9Sstevel@tonic-gate return (EAGAIN); 1705*7c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 1706*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1707*7c478bd9Sstevel@tonic-gate 1, 0); 1708*7c478bd9Sstevel@tonic-gate return (0); 1709*7c478bd9Sstevel@tonic-gate } 1710*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 1711*7c478bd9Sstevel@tonic-gate return (EDEADLK); /* LOCK_ERRORCHECK */ 1712*7c478bd9Sstevel@tonic-gate } 1713*7c478bd9Sstevel@tonic-gate /* 1714*7c478bd9Sstevel@tonic-gate * We have reached a deadlock, probably because the 1715*7c478bd9Sstevel@tonic-gate * process is executing non-async-signal-safe code in 1716*7c478bd9Sstevel@tonic-gate * a signal handler and is attempting to acquire a lock 1717*7c478bd9Sstevel@tonic-gate * that it already owns. This is not surprising, given 1718*7c478bd9Sstevel@tonic-gate * bad programming practices over the years that has 1719*7c478bd9Sstevel@tonic-gate * resulted in applications calling printf() and such 1720*7c478bd9Sstevel@tonic-gate * in their signal handlers. Unless the user has told 1721*7c478bd9Sstevel@tonic-gate * us that the signal handlers are safe by setting: 1722*7c478bd9Sstevel@tonic-gate * export _THREAD_ASYNC_SAFE=1 1723*7c478bd9Sstevel@tonic-gate * we return EDEADLK rather than actually deadlocking. 1724*7c478bd9Sstevel@tonic-gate */ 1725*7c478bd9Sstevel@tonic-gate if (tsp == NULL && 1726*7c478bd9Sstevel@tonic-gate MUTEX_OWNER(mp) == self && !self->ul_async_safe) { 1727*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 1728*7c478bd9Sstevel@tonic-gate return (EDEADLK); 1729*7c478bd9Sstevel@tonic-gate } 1730*7c478bd9Sstevel@tonic-gate } 1731*7c478bd9Sstevel@tonic-gate 1732*7c478bd9Sstevel@tonic-gate /* 1733*7c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 1734*7c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 1735*7c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 1736*7c478bd9Sstevel@tonic-gate */ 1737*7c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 1738*7c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 1739*7c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 1740*7c478bd9Sstevel@tonic-gate 1741*7c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 1742*7c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK)); 1743*7c478bd9Sstevel@tonic-gate 1744*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 1745*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1746*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1747*7c478bd9Sstevel@tonic-gate return (0); 1748*7c478bd9Sstevel@tonic-gate } 1749*7c478bd9Sstevel@tonic-gate 1750*7c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 1751*7c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 1752*7c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 1753*7c478bd9Sstevel@tonic-gate return (EAGAIN); 1754*7c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 1755*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1756*7c478bd9Sstevel@tonic-gate 1, 0); 1757*7c478bd9Sstevel@tonic-gate return (0); 1758*7c478bd9Sstevel@tonic-gate } 1759*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 1760*7c478bd9Sstevel@tonic-gate return (EDEADLK); /* LOCK_ERRORCHECK */ 1761*7c478bd9Sstevel@tonic-gate } 1762*7c478bd9Sstevel@tonic-gate 1763*7c478bd9Sstevel@tonic-gate return (slow_lock(self, mp, tsp)); 1764*7c478bd9Sstevel@tonic-gate } 1765*7c478bd9Sstevel@tonic-gate 1766*7c478bd9Sstevel@tonic-gate /* else do it the long way */ 1767*7c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, tsp, MUTEX_LOCK)); 1768*7c478bd9Sstevel@tonic-gate } 1769*7c478bd9Sstevel@tonic-gate 1770*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_lock = __mutex_lock 1771*7c478bd9Sstevel@tonic-gate #pragma weak mutex_lock = __mutex_lock 1772*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_lock = __mutex_lock 1773*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_lock = __mutex_lock 1774*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_lock = __mutex_lock 1775*7c478bd9Sstevel@tonic-gate int 1776*7c478bd9Sstevel@tonic-gate __mutex_lock(mutex_t *mp) 1777*7c478bd9Sstevel@tonic-gate { 1778*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 1779*7c478bd9Sstevel@tonic-gate return (mutex_lock_impl(mp, NULL)); 1780*7c478bd9Sstevel@tonic-gate } 1781*7c478bd9Sstevel@tonic-gate 1782*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_timedlock = _pthread_mutex_timedlock 1783*7c478bd9Sstevel@tonic-gate int 1784*7c478bd9Sstevel@tonic-gate _pthread_mutex_timedlock(mutex_t *mp, const timespec_t *abstime) 1785*7c478bd9Sstevel@tonic-gate { 1786*7c478bd9Sstevel@tonic-gate timespec_t tslocal; 1787*7c478bd9Sstevel@tonic-gate int error; 1788*7c478bd9Sstevel@tonic-gate 1789*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 1790*7c478bd9Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 1791*7c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 1792*7c478bd9Sstevel@tonic-gate if (error == ETIME) 1793*7c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 1794*7c478bd9Sstevel@tonic-gate return (error); 1795*7c478bd9Sstevel@tonic-gate } 1796*7c478bd9Sstevel@tonic-gate 1797*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_reltimedlock_np = _pthread_mutex_reltimedlock_np 1798*7c478bd9Sstevel@tonic-gate int 1799*7c478bd9Sstevel@tonic-gate _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime) 1800*7c478bd9Sstevel@tonic-gate { 1801*7c478bd9Sstevel@tonic-gate timespec_t tslocal; 1802*7c478bd9Sstevel@tonic-gate int error; 1803*7c478bd9Sstevel@tonic-gate 1804*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 1805*7c478bd9Sstevel@tonic-gate tslocal = *reltime; 1806*7c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 1807*7c478bd9Sstevel@tonic-gate if (error == ETIME) 1808*7c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 1809*7c478bd9Sstevel@tonic-gate return (error); 1810*7c478bd9Sstevel@tonic-gate } 1811*7c478bd9Sstevel@tonic-gate 1812*7c478bd9Sstevel@tonic-gate static int 1813*7c478bd9Sstevel@tonic-gate slow_trylock(mutex_t *mp, ulwp_t *self) 1814*7c478bd9Sstevel@tonic-gate { 1815*7c478bd9Sstevel@tonic-gate if (MUTEX_OWNER(mp) == self || 1816*7c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mp) != 0) { 1817*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 1818*7c478bd9Sstevel@tonic-gate 1819*7c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 1820*7c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 1821*7c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 1822*7c478bd9Sstevel@tonic-gate } 1823*7c478bd9Sstevel@tonic-gate return (EBUSY); 1824*7c478bd9Sstevel@tonic-gate } 1825*7c478bd9Sstevel@tonic-gate return (0); 1826*7c478bd9Sstevel@tonic-gate } 1827*7c478bd9Sstevel@tonic-gate 1828*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_trylock = __mutex_trylock 1829*7c478bd9Sstevel@tonic-gate #pragma weak mutex_trylock = __mutex_trylock 1830*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_trylock = __mutex_trylock 1831*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_trylock = __mutex_trylock 1832*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_trylock = __mutex_trylock 1833*7c478bd9Sstevel@tonic-gate int 1834*7c478bd9Sstevel@tonic-gate __mutex_trylock(mutex_t *mp) 1835*7c478bd9Sstevel@tonic-gate { 1836*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1837*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 1838*7c478bd9Sstevel@tonic-gate uberflags_t *gflags; 1839*7c478bd9Sstevel@tonic-gate int mtype; 1840*7c478bd9Sstevel@tonic-gate 1841*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 1842*7c478bd9Sstevel@tonic-gate /* 1843*7c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 1844*7c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 1845*7c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 1846*7c478bd9Sstevel@tonic-gate * and the process has only a single thread. 1847*7c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 1848*7c478bd9Sstevel@tonic-gate */ 1849*7c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 1850*7c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 1851*7c478bd9Sstevel@tonic-gate /* 1852*7c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 1853*7c478bd9Sstevel@tonic-gate */ 1854*7c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 1855*7c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 1856*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1857*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1858*7c478bd9Sstevel@tonic-gate return (0); 1859*7c478bd9Sstevel@tonic-gate } 1860*7c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 1861*7c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 1862*7c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 1863*7c478bd9Sstevel@tonic-gate return (EAGAIN); 1864*7c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 1865*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1866*7c478bd9Sstevel@tonic-gate 1, 0); 1867*7c478bd9Sstevel@tonic-gate return (0); 1868*7c478bd9Sstevel@tonic-gate } 1869*7c478bd9Sstevel@tonic-gate return (EDEADLK); /* LOCK_ERRORCHECK */ 1870*7c478bd9Sstevel@tonic-gate } 1871*7c478bd9Sstevel@tonic-gate return (EBUSY); 1872*7c478bd9Sstevel@tonic-gate } 1873*7c478bd9Sstevel@tonic-gate 1874*7c478bd9Sstevel@tonic-gate /* 1875*7c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 1876*7c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 1877*7c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 1878*7c478bd9Sstevel@tonic-gate */ 1879*7c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 1880*7c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 1881*7c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 1882*7c478bd9Sstevel@tonic-gate 1883*7c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 1884*7c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY)); 1885*7c478bd9Sstevel@tonic-gate 1886*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 1887*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1888*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1889*7c478bd9Sstevel@tonic-gate return (0); 1890*7c478bd9Sstevel@tonic-gate } 1891*7c478bd9Sstevel@tonic-gate 1892*7c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 1893*7c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 1894*7c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 1895*7c478bd9Sstevel@tonic-gate return (EAGAIN); 1896*7c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 1897*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1898*7c478bd9Sstevel@tonic-gate 1, 0); 1899*7c478bd9Sstevel@tonic-gate return (0); 1900*7c478bd9Sstevel@tonic-gate } 1901*7c478bd9Sstevel@tonic-gate return (EBUSY); /* LOCK_ERRORCHECK */ 1902*7c478bd9Sstevel@tonic-gate } 1903*7c478bd9Sstevel@tonic-gate 1904*7c478bd9Sstevel@tonic-gate return (slow_trylock(mp, self)); 1905*7c478bd9Sstevel@tonic-gate } 1906*7c478bd9Sstevel@tonic-gate 1907*7c478bd9Sstevel@tonic-gate /* else do it the long way */ 1908*7c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, NULL, MUTEX_TRY)); 1909*7c478bd9Sstevel@tonic-gate } 1910*7c478bd9Sstevel@tonic-gate 1911*7c478bd9Sstevel@tonic-gate int 1912*7c478bd9Sstevel@tonic-gate mutex_unlock_internal(mutex_t *mp) 1913*7c478bd9Sstevel@tonic-gate { 1914*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1915*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 1916*7c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 1917*7c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp; 1918*7c478bd9Sstevel@tonic-gate int error; 1919*7c478bd9Sstevel@tonic-gate lwpid_t lwpid; 1920*7c478bd9Sstevel@tonic-gate 1921*7c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp)) 1922*7c478bd9Sstevel@tonic-gate return (EPERM); 1923*7c478bd9Sstevel@tonic-gate 1924*7c478bd9Sstevel@tonic-gate if (self->ul_error_detection && !mutex_is_held(mp)) 1925*7c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_unlock", NULL, NULL); 1926*7c478bd9Sstevel@tonic-gate 1927*7c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 1928*7c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 1929*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 1930*7c478bd9Sstevel@tonic-gate return (0); 1931*7c478bd9Sstevel@tonic-gate } 1932*7c478bd9Sstevel@tonic-gate 1933*7c478bd9Sstevel@tonic-gate if ((msp = MUTEX_STATS(mp, udp)) != NULL) 1934*7c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 1935*7c478bd9Sstevel@tonic-gate 1936*7c478bd9Sstevel@tonic-gate if (mtype & 1937*7c478bd9Sstevel@tonic-gate (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) { 1938*7c478bd9Sstevel@tonic-gate no_preempt(self); 1939*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 1940*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 1941*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 1942*7c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) { 1943*7c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 1944*7c478bd9Sstevel@tonic-gate error = ___lwp_mutex_unlock(mp); 1945*7c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS_ROBUST) { 1946*7c478bd9Sstevel@tonic-gate error = ___lwp_mutex_unlock(mp); 1947*7c478bd9Sstevel@tonic-gate } else { 1948*7c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) 1949*7c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 1950*7c478bd9Sstevel@tonic-gate error = 0; 1951*7c478bd9Sstevel@tonic-gate } 1952*7c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_PROTECT) { 1953*7c478bd9Sstevel@tonic-gate if (_ceil_mylist_del(mp)) 1954*7c478bd9Sstevel@tonic-gate _ceil_prio_waive(); 1955*7c478bd9Sstevel@tonic-gate } 1956*7c478bd9Sstevel@tonic-gate preempt(self); 1957*7c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) { 1958*7c478bd9Sstevel@tonic-gate if (mp->mutex_lockword & WAITERMASK) 1959*7c478bd9Sstevel@tonic-gate mutex_unlock_process(mp); 1960*7c478bd9Sstevel@tonic-gate else { 1961*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 1962*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 1963*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 1964*7c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) { 1965*7c478bd9Sstevel@tonic-gate no_preempt(self); 1966*7c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 1967*7c478bd9Sstevel@tonic-gate preempt(self); 1968*7c478bd9Sstevel@tonic-gate } 1969*7c478bd9Sstevel@tonic-gate } 1970*7c478bd9Sstevel@tonic-gate error = 0; 1971*7c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 1972*7c478bd9Sstevel@tonic-gate if ((lwpid = mutex_unlock_queue(mp)) != 0) { 1973*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 1974*7c478bd9Sstevel@tonic-gate preempt(self); 1975*7c478bd9Sstevel@tonic-gate } 1976*7c478bd9Sstevel@tonic-gate error = 0; 1977*7c478bd9Sstevel@tonic-gate } 1978*7c478bd9Sstevel@tonic-gate 1979*7c478bd9Sstevel@tonic-gate return (error); 1980*7c478bd9Sstevel@tonic-gate } 1981*7c478bd9Sstevel@tonic-gate 1982*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_unlock = __mutex_unlock 1983*7c478bd9Sstevel@tonic-gate #pragma weak mutex_unlock = __mutex_unlock 1984*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_unlock = __mutex_unlock 1985*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_unlock = __mutex_unlock 1986*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_unlock = __mutex_unlock 1987*7c478bd9Sstevel@tonic-gate int 1988*7c478bd9Sstevel@tonic-gate __mutex_unlock(mutex_t *mp) 1989*7c478bd9Sstevel@tonic-gate { 1990*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1991*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 1992*7c478bd9Sstevel@tonic-gate uberflags_t *gflags; 1993*7c478bd9Sstevel@tonic-gate lwpid_t lwpid; 1994*7c478bd9Sstevel@tonic-gate int mtype; 1995*7c478bd9Sstevel@tonic-gate short el; 1996*7c478bd9Sstevel@tonic-gate 1997*7c478bd9Sstevel@tonic-gate /* 1998*7c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 1999*7c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 2000*7c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 2001*7c478bd9Sstevel@tonic-gate * and the process has only a single thread. 2002*7c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 2003*7c478bd9Sstevel@tonic-gate */ 2004*7c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 2005*7c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 2006*7c478bd9Sstevel@tonic-gate if (mtype) { 2007*7c478bd9Sstevel@tonic-gate /* 2008*7c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 2009*7c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 2010*7c478bd9Sstevel@tonic-gate */ 2011*7c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 2012*7c478bd9Sstevel@tonic-gate return (EPERM); 2013*7c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 2014*7c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 2015*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 2016*7c478bd9Sstevel@tonic-gate return (0); 2017*7c478bd9Sstevel@tonic-gate } 2018*7c478bd9Sstevel@tonic-gate } 2019*7c478bd9Sstevel@tonic-gate /* 2020*7c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 2021*7c478bd9Sstevel@tonic-gate * Also, there can be no waiters. 2022*7c478bd9Sstevel@tonic-gate */ 2023*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 2024*7c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 2025*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 2026*7c478bd9Sstevel@tonic-gate return (0); 2027*7c478bd9Sstevel@tonic-gate } 2028*7c478bd9Sstevel@tonic-gate 2029*7c478bd9Sstevel@tonic-gate /* 2030*7c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 2031*7c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 2032*7c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 2033*7c478bd9Sstevel@tonic-gate */ 2034*7c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL) { 2035*7c478bd9Sstevel@tonic-gate if (((el = gflags->uf_trs_ted) | mtype) == 0) { 2036*7c478bd9Sstevel@tonic-gate fast_unlock: 2037*7c478bd9Sstevel@tonic-gate if (!(mp->mutex_lockword & WAITERMASK)) { 2038*7c478bd9Sstevel@tonic-gate /* no waiter exists right now */ 2039*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 2040*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 2041*7c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & 2042*7c478bd9Sstevel@tonic-gate WAITERMASK) { 2043*7c478bd9Sstevel@tonic-gate /* a waiter suddenly appeared */ 2044*7c478bd9Sstevel@tonic-gate no_preempt(self); 2045*7c478bd9Sstevel@tonic-gate if ((lwpid = mutex_wakeup(mp)) != 0) 2046*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 2047*7c478bd9Sstevel@tonic-gate preempt(self); 2048*7c478bd9Sstevel@tonic-gate } 2049*7c478bd9Sstevel@tonic-gate } else if ((lwpid = mutex_unlock_queue(mp)) != 0) { 2050*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 2051*7c478bd9Sstevel@tonic-gate preempt(self); 2052*7c478bd9Sstevel@tonic-gate } 2053*7c478bd9Sstevel@tonic-gate return (0); 2054*7c478bd9Sstevel@tonic-gate } 2055*7c478bd9Sstevel@tonic-gate if (el) /* error detection or lock statistics */ 2056*7c478bd9Sstevel@tonic-gate goto slow_unlock; 2057*7c478bd9Sstevel@tonic-gate if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 2058*7c478bd9Sstevel@tonic-gate /* 2059*7c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 2060*7c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 2061*7c478bd9Sstevel@tonic-gate */ 2062*7c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 2063*7c478bd9Sstevel@tonic-gate return (EPERM); 2064*7c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 2065*7c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 2066*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 2067*7c478bd9Sstevel@tonic-gate return (0); 2068*7c478bd9Sstevel@tonic-gate } 2069*7c478bd9Sstevel@tonic-gate goto fast_unlock; 2070*7c478bd9Sstevel@tonic-gate } 2071*7c478bd9Sstevel@tonic-gate if ((mtype & 2072*7c478bd9Sstevel@tonic-gate ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 2073*7c478bd9Sstevel@tonic-gate /* 2074*7c478bd9Sstevel@tonic-gate * At this point we know that zero, one, or both of the 2075*7c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and 2076*7c478bd9Sstevel@tonic-gate * that the USYNC_PROCESS flag is set. 2077*7c478bd9Sstevel@tonic-gate */ 2078*7c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp)) 2079*7c478bd9Sstevel@tonic-gate return (EPERM); 2080*7c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 2081*7c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 2082*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 2083*7c478bd9Sstevel@tonic-gate return (0); 2084*7c478bd9Sstevel@tonic-gate } 2085*7c478bd9Sstevel@tonic-gate if (mp->mutex_lockword & WAITERMASK) 2086*7c478bd9Sstevel@tonic-gate mutex_unlock_process(mp); 2087*7c478bd9Sstevel@tonic-gate else { 2088*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 2089*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 2090*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 2091*7c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & 2092*7c478bd9Sstevel@tonic-gate WAITERMASK) { 2093*7c478bd9Sstevel@tonic-gate no_preempt(self); 2094*7c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 2095*7c478bd9Sstevel@tonic-gate preempt(self); 2096*7c478bd9Sstevel@tonic-gate } 2097*7c478bd9Sstevel@tonic-gate } 2098*7c478bd9Sstevel@tonic-gate return (0); 2099*7c478bd9Sstevel@tonic-gate } 2100*7c478bd9Sstevel@tonic-gate } 2101*7c478bd9Sstevel@tonic-gate 2102*7c478bd9Sstevel@tonic-gate /* else do it the long way */ 2103*7c478bd9Sstevel@tonic-gate slow_unlock: 2104*7c478bd9Sstevel@tonic-gate return (mutex_unlock_internal(mp)); 2105*7c478bd9Sstevel@tonic-gate } 2106*7c478bd9Sstevel@tonic-gate 2107*7c478bd9Sstevel@tonic-gate /* 2108*7c478bd9Sstevel@tonic-gate * Internally to the library, almost all mutex lock/unlock actions 2109*7c478bd9Sstevel@tonic-gate * go through these lmutex_ functions, to protect critical regions. 2110*7c478bd9Sstevel@tonic-gate * We replicate a bit of code from __mutex_lock() and __mutex_unlock() 2111*7c478bd9Sstevel@tonic-gate * to make these functions faster since we know that the mutex type 2112*7c478bd9Sstevel@tonic-gate * of all internal locks is USYNC_THREAD. We also know that internal 2113*7c478bd9Sstevel@tonic-gate * locking can never fail, so we panic if it does. 2114*7c478bd9Sstevel@tonic-gate */ 2115*7c478bd9Sstevel@tonic-gate void 2116*7c478bd9Sstevel@tonic-gate lmutex_lock(mutex_t *mp) 2117*7c478bd9Sstevel@tonic-gate { 2118*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2119*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 2120*7c478bd9Sstevel@tonic-gate 2121*7c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 2122*7c478bd9Sstevel@tonic-gate 2123*7c478bd9Sstevel@tonic-gate enter_critical(self); 2124*7c478bd9Sstevel@tonic-gate /* 2125*7c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 2126*7c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 2127*7c478bd9Sstevel@tonic-gate */ 2128*7c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 2129*7c478bd9Sstevel@tonic-gate /* 2130*7c478bd9Sstevel@tonic-gate * Only one thread exists; the mutex must be free. 2131*7c478bd9Sstevel@tonic-gate */ 2132*7c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_lockw == 0); 2133*7c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 2134*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2135*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 2136*7c478bd9Sstevel@tonic-gate } else { 2137*7c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 2138*7c478bd9Sstevel@tonic-gate 2139*7c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 2140*7c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 2141*7c478bd9Sstevel@tonic-gate 2142*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 2143*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2144*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 2145*7c478bd9Sstevel@tonic-gate } else if (mutex_trylock_adaptive(mp) != 0) { 2146*7c478bd9Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL); 2147*7c478bd9Sstevel@tonic-gate } 2148*7c478bd9Sstevel@tonic-gate 2149*7c478bd9Sstevel@tonic-gate if (msp) 2150*7c478bd9Sstevel@tonic-gate record_begin_hold(msp); 2151*7c478bd9Sstevel@tonic-gate } 2152*7c478bd9Sstevel@tonic-gate } 2153*7c478bd9Sstevel@tonic-gate 2154*7c478bd9Sstevel@tonic-gate void 2155*7c478bd9Sstevel@tonic-gate lmutex_unlock(mutex_t *mp) 2156*7c478bd9Sstevel@tonic-gate { 2157*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2158*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 2159*7c478bd9Sstevel@tonic-gate 2160*7c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 2161*7c478bd9Sstevel@tonic-gate 2162*7c478bd9Sstevel@tonic-gate /* 2163*7c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 2164*7c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 2165*7c478bd9Sstevel@tonic-gate */ 2166*7c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 2167*7c478bd9Sstevel@tonic-gate /* 2168*7c478bd9Sstevel@tonic-gate * Only one thread exists so there can be no waiters. 2169*7c478bd9Sstevel@tonic-gate */ 2170*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 2171*7c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 2172*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 2173*7c478bd9Sstevel@tonic-gate } else { 2174*7c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 2175*7c478bd9Sstevel@tonic-gate lwpid_t lwpid; 2176*7c478bd9Sstevel@tonic-gate 2177*7c478bd9Sstevel@tonic-gate if (msp) 2178*7c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 2179*7c478bd9Sstevel@tonic-gate if ((lwpid = mutex_unlock_queue(mp)) != 0) { 2180*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 2181*7c478bd9Sstevel@tonic-gate preempt(self); 2182*7c478bd9Sstevel@tonic-gate } 2183*7c478bd9Sstevel@tonic-gate } 2184*7c478bd9Sstevel@tonic-gate exit_critical(self); 2185*7c478bd9Sstevel@tonic-gate } 2186*7c478bd9Sstevel@tonic-gate 2187*7c478bd9Sstevel@tonic-gate static int 2188*7c478bd9Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg) 2189*7c478bd9Sstevel@tonic-gate { 2190*7c478bd9Sstevel@tonic-gate /* 2191*7c478bd9Sstevel@tonic-gate * There is an inherent data race in the current ownership design. 2192*7c478bd9Sstevel@tonic-gate * The mutex_owner and mutex_ownerpid fields cannot be set or tested 2193*7c478bd9Sstevel@tonic-gate * atomically as a pair. The original implementation tested each 2194*7c478bd9Sstevel@tonic-gate * field just once. This was exposed to trivial false positives in 2195*7c478bd9Sstevel@tonic-gate * the case of multiple multithreaded processes with thread addresses 2196*7c478bd9Sstevel@tonic-gate * in common. To close the window to an acceptable level we now use a 2197*7c478bd9Sstevel@tonic-gate * sequence of five tests: pid-thr-pid-thr-pid. This ensures that any 2198*7c478bd9Sstevel@tonic-gate * single interruption will still leave one uninterrupted sequence of 2199*7c478bd9Sstevel@tonic-gate * pid-thr-pid tests intact. 2200*7c478bd9Sstevel@tonic-gate * 2201*7c478bd9Sstevel@tonic-gate * It is assumed that all updates are always ordered thr-pid and that 2202*7c478bd9Sstevel@tonic-gate * we have TSO hardware. 2203*7c478bd9Sstevel@tonic-gate */ 2204*7c478bd9Sstevel@tonic-gate volatile mutex_t *mp = (volatile mutex_t *)mparg; 2205*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2206*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 2207*7c478bd9Sstevel@tonic-gate 2208*7c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid != udp->pid) 2209*7c478bd9Sstevel@tonic-gate return (0); 2210*7c478bd9Sstevel@tonic-gate 2211*7c478bd9Sstevel@tonic-gate if (!MUTEX_OWNED(mp, self)) 2212*7c478bd9Sstevel@tonic-gate return (0); 2213*7c478bd9Sstevel@tonic-gate 2214*7c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid != udp->pid) 2215*7c478bd9Sstevel@tonic-gate return (0); 2216*7c478bd9Sstevel@tonic-gate 2217*7c478bd9Sstevel@tonic-gate if (!MUTEX_OWNED(mp, self)) 2218*7c478bd9Sstevel@tonic-gate return (0); 2219*7c478bd9Sstevel@tonic-gate 2220*7c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid != udp->pid) 2221*7c478bd9Sstevel@tonic-gate return (0); 2222*7c478bd9Sstevel@tonic-gate 2223*7c478bd9Sstevel@tonic-gate return (1); 2224*7c478bd9Sstevel@tonic-gate } 2225*7c478bd9Sstevel@tonic-gate 2226*7c478bd9Sstevel@tonic-gate /* 2227*7c478bd9Sstevel@tonic-gate * Some crufty old programs define their own version of _mutex_held() 2228*7c478bd9Sstevel@tonic-gate * to be simply return(1). This breaks internal libc logic, so we 2229*7c478bd9Sstevel@tonic-gate * define a private version for exclusive use by libc, mutex_is_held(), 2230*7c478bd9Sstevel@tonic-gate * and also a new public function, __mutex_held(), to be used in new 2231*7c478bd9Sstevel@tonic-gate * code to circumvent these crufty old programs. 2232*7c478bd9Sstevel@tonic-gate */ 2233*7c478bd9Sstevel@tonic-gate #pragma weak mutex_held = mutex_is_held 2234*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_held = mutex_is_held 2235*7c478bd9Sstevel@tonic-gate #pragma weak __mutex_held = mutex_is_held 2236*7c478bd9Sstevel@tonic-gate int 2237*7c478bd9Sstevel@tonic-gate mutex_is_held(mutex_t *mp) 2238*7c478bd9Sstevel@tonic-gate { 2239*7c478bd9Sstevel@tonic-gate if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) 2240*7c478bd9Sstevel@tonic-gate return (shared_mutex_held(mp)); 2241*7c478bd9Sstevel@tonic-gate return (MUTEX_OWNED(mp, curthread)); 2242*7c478bd9Sstevel@tonic-gate } 2243*7c478bd9Sstevel@tonic-gate 2244*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_destroy = __mutex_destroy 2245*7c478bd9Sstevel@tonic-gate #pragma weak mutex_destroy = __mutex_destroy 2246*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_destroy = __mutex_destroy 2247*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_destroy = __mutex_destroy 2248*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_destroy = __mutex_destroy 2249*7c478bd9Sstevel@tonic-gate int 2250*7c478bd9Sstevel@tonic-gate __mutex_destroy(mutex_t *mp) 2251*7c478bd9Sstevel@tonic-gate { 2252*7c478bd9Sstevel@tonic-gate mp->mutex_magic = 0; 2253*7c478bd9Sstevel@tonic-gate mp->mutex_flag &= ~LOCK_INITED; 2254*7c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(mp); 2255*7c478bd9Sstevel@tonic-gate return (0); 2256*7c478bd9Sstevel@tonic-gate } 2257*7c478bd9Sstevel@tonic-gate 2258*7c478bd9Sstevel@tonic-gate /* 2259*7c478bd9Sstevel@tonic-gate * Spin locks are separate from ordinary mutexes, 2260*7c478bd9Sstevel@tonic-gate * but we use the same data structure for them. 2261*7c478bd9Sstevel@tonic-gate */ 2262*7c478bd9Sstevel@tonic-gate 2263*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_init = _pthread_spin_init 2264*7c478bd9Sstevel@tonic-gate int 2265*7c478bd9Sstevel@tonic-gate _pthread_spin_init(pthread_spinlock_t *lock, int pshared) 2266*7c478bd9Sstevel@tonic-gate { 2267*7c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 2268*7c478bd9Sstevel@tonic-gate 2269*7c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 2270*7c478bd9Sstevel@tonic-gate if (pshared == PTHREAD_PROCESS_SHARED) 2271*7c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_PROCESS; 2272*7c478bd9Sstevel@tonic-gate else 2273*7c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_THREAD; 2274*7c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 2275*7c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 2276*7c478bd9Sstevel@tonic-gate return (0); 2277*7c478bd9Sstevel@tonic-gate } 2278*7c478bd9Sstevel@tonic-gate 2279*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_destroy = _pthread_spin_destroy 2280*7c478bd9Sstevel@tonic-gate int 2281*7c478bd9Sstevel@tonic-gate _pthread_spin_destroy(pthread_spinlock_t *lock) 2282*7c478bd9Sstevel@tonic-gate { 2283*7c478bd9Sstevel@tonic-gate (void) _memset(lock, 0, sizeof (*lock)); 2284*7c478bd9Sstevel@tonic-gate return (0); 2285*7c478bd9Sstevel@tonic-gate } 2286*7c478bd9Sstevel@tonic-gate 2287*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_trylock = _pthread_spin_trylock 2288*7c478bd9Sstevel@tonic-gate int 2289*7c478bd9Sstevel@tonic-gate _pthread_spin_trylock(pthread_spinlock_t *lock) 2290*7c478bd9Sstevel@tonic-gate { 2291*7c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 2292*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2293*7c478bd9Sstevel@tonic-gate int error = 0; 2294*7c478bd9Sstevel@tonic-gate 2295*7c478bd9Sstevel@tonic-gate no_preempt(self); 2296*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) != 0) 2297*7c478bd9Sstevel@tonic-gate error = EBUSY; 2298*7c478bd9Sstevel@tonic-gate else { 2299*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2300*7c478bd9Sstevel@tonic-gate if (mp->mutex_type == USYNC_PROCESS) 2301*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = self->ul_uberdata->pid; 2302*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 2303*7c478bd9Sstevel@tonic-gate } 2304*7c478bd9Sstevel@tonic-gate preempt(self); 2305*7c478bd9Sstevel@tonic-gate return (error); 2306*7c478bd9Sstevel@tonic-gate } 2307*7c478bd9Sstevel@tonic-gate 2308*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_lock = _pthread_spin_lock 2309*7c478bd9Sstevel@tonic-gate int 2310*7c478bd9Sstevel@tonic-gate _pthread_spin_lock(pthread_spinlock_t *lock) 2311*7c478bd9Sstevel@tonic-gate { 2312*7c478bd9Sstevel@tonic-gate volatile uint8_t *lockp = 2313*7c478bd9Sstevel@tonic-gate (volatile uint8_t *)&((mutex_t *)lock)->mutex_lockw; 2314*7c478bd9Sstevel@tonic-gate 2315*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 2316*7c478bd9Sstevel@tonic-gate /* 2317*7c478bd9Sstevel@tonic-gate * We don't care whether the owner is running on a processor. 2318*7c478bd9Sstevel@tonic-gate * We just spin because that's what this interface requires. 2319*7c478bd9Sstevel@tonic-gate */ 2320*7c478bd9Sstevel@tonic-gate for (;;) { 2321*7c478bd9Sstevel@tonic-gate if (*lockp == 0) { /* lock byte appears to be clear */ 2322*7c478bd9Sstevel@tonic-gate if (_pthread_spin_trylock(lock) == 0) 2323*7c478bd9Sstevel@tonic-gate return (0); 2324*7c478bd9Sstevel@tonic-gate } 2325*7c478bd9Sstevel@tonic-gate SMT_PAUSE(); 2326*7c478bd9Sstevel@tonic-gate } 2327*7c478bd9Sstevel@tonic-gate } 2328*7c478bd9Sstevel@tonic-gate 2329*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_unlock = _pthread_spin_unlock 2330*7c478bd9Sstevel@tonic-gate int 2331*7c478bd9Sstevel@tonic-gate _pthread_spin_unlock(pthread_spinlock_t *lock) 2332*7c478bd9Sstevel@tonic-gate { 2333*7c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 2334*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2335*7c478bd9Sstevel@tonic-gate 2336*7c478bd9Sstevel@tonic-gate no_preempt(self); 2337*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 2338*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 2339*7c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 2340*7c478bd9Sstevel@tonic-gate (void) swap32(&mp->mutex_lockword, 0); 2341*7c478bd9Sstevel@tonic-gate preempt(self); 2342*7c478bd9Sstevel@tonic-gate return (0); 2343*7c478bd9Sstevel@tonic-gate } 2344*7c478bd9Sstevel@tonic-gate 2345*7c478bd9Sstevel@tonic-gate #pragma weak cond_init = _cond_init 2346*7c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 2347*7c478bd9Sstevel@tonic-gate int 2348*7c478bd9Sstevel@tonic-gate _cond_init(cond_t *cvp, int type, void *arg) 2349*7c478bd9Sstevel@tonic-gate { 2350*7c478bd9Sstevel@tonic-gate if (type != USYNC_THREAD && type != USYNC_PROCESS) 2351*7c478bd9Sstevel@tonic-gate return (EINVAL); 2352*7c478bd9Sstevel@tonic-gate (void) _memset(cvp, 0, sizeof (*cvp)); 2353*7c478bd9Sstevel@tonic-gate cvp->cond_type = (uint16_t)type; 2354*7c478bd9Sstevel@tonic-gate cvp->cond_magic = COND_MAGIC; 2355*7c478bd9Sstevel@tonic-gate return (0); 2356*7c478bd9Sstevel@tonic-gate } 2357*7c478bd9Sstevel@tonic-gate 2358*7c478bd9Sstevel@tonic-gate /* 2359*7c478bd9Sstevel@tonic-gate * cond_sleep_queue(): utility function for cond_wait_queue(). 2360*7c478bd9Sstevel@tonic-gate * 2361*7c478bd9Sstevel@tonic-gate * Go to sleep on a condvar sleep queue, expect to be waked up 2362*7c478bd9Sstevel@tonic-gate * by someone calling cond_signal() or cond_broadcast() or due 2363*7c478bd9Sstevel@tonic-gate * to receiving a UNIX signal or being cancelled, or just simply 2364*7c478bd9Sstevel@tonic-gate * due to a spurious wakeup (like someome calling forkall()). 2365*7c478bd9Sstevel@tonic-gate * 2366*7c478bd9Sstevel@tonic-gate * The associated mutex is *not* reacquired before returning. 2367*7c478bd9Sstevel@tonic-gate * That must be done by the caller of cond_sleep_queue(). 2368*7c478bd9Sstevel@tonic-gate */ 2369*7c478bd9Sstevel@tonic-gate int 2370*7c478bd9Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 2371*7c478bd9Sstevel@tonic-gate { 2372*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2373*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 2374*7c478bd9Sstevel@tonic-gate queue_head_t *mqp; 2375*7c478bd9Sstevel@tonic-gate lwpid_t lwpid; 2376*7c478bd9Sstevel@tonic-gate int signalled; 2377*7c478bd9Sstevel@tonic-gate int error; 2378*7c478bd9Sstevel@tonic-gate 2379*7c478bd9Sstevel@tonic-gate /* 2380*7c478bd9Sstevel@tonic-gate * Put ourself on the CV sleep queue, unlock the mutex, then 2381*7c478bd9Sstevel@tonic-gate * park ourself and unpark a candidate lwp to grab the mutex. 2382*7c478bd9Sstevel@tonic-gate * We must go onto the CV sleep queue before dropping the 2383*7c478bd9Sstevel@tonic-gate * mutex in order to guarantee atomicity of the operation. 2384*7c478bd9Sstevel@tonic-gate */ 2385*7c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 2386*7c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 2387*7c478bd9Sstevel@tonic-gate enqueue(qp, self, cvp, CV); 2388*7c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 1; 2389*7c478bd9Sstevel@tonic-gate self->ul_cvmutex = mp; 2390*7c478bd9Sstevel@tonic-gate self->ul_cv_wake = (tsp != NULL); 2391*7c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 2392*7c478bd9Sstevel@tonic-gate lwpid = mutex_unlock_queue(mp); 2393*7c478bd9Sstevel@tonic-gate for (;;) { 2394*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 2395*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 2396*7c478bd9Sstevel@tonic-gate if (lwpid != 0) { 2397*7c478bd9Sstevel@tonic-gate lwpid = preempt_unpark(self, lwpid); 2398*7c478bd9Sstevel@tonic-gate preempt(self); 2399*7c478bd9Sstevel@tonic-gate } 2400*7c478bd9Sstevel@tonic-gate /* 2401*7c478bd9Sstevel@tonic-gate * We may have a deferred signal present, 2402*7c478bd9Sstevel@tonic-gate * in which case we should return EINTR. 2403*7c478bd9Sstevel@tonic-gate * Also, we may have received a SIGCANCEL; if so 2404*7c478bd9Sstevel@tonic-gate * and we are cancelable we should return EINTR. 2405*7c478bd9Sstevel@tonic-gate * We force an immediate EINTR return from 2406*7c478bd9Sstevel@tonic-gate * __lwp_park() by turning our parking flag off. 2407*7c478bd9Sstevel@tonic-gate */ 2408*7c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 2409*7c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 2410*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 2411*7c478bd9Sstevel@tonic-gate /* 2412*7c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 2413*7c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 2414*7c478bd9Sstevel@tonic-gate */ 2415*7c478bd9Sstevel@tonic-gate error = __lwp_park(tsp, lwpid); 2416*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 2417*7c478bd9Sstevel@tonic-gate lwpid = 0; /* unpark the other lwp only once */ 2418*7c478bd9Sstevel@tonic-gate /* 2419*7c478bd9Sstevel@tonic-gate * We were waked up by cond_signal(), cond_broadcast(), 2420*7c478bd9Sstevel@tonic-gate * by an interrupt or timeout (EINTR or ETIME), 2421*7c478bd9Sstevel@tonic-gate * or we may just have gotten a spurious wakeup. 2422*7c478bd9Sstevel@tonic-gate */ 2423*7c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 2424*7c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 2425*7c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) 2426*7c478bd9Sstevel@tonic-gate break; 2427*7c478bd9Sstevel@tonic-gate /* 2428*7c478bd9Sstevel@tonic-gate * We are on either the condvar sleep queue or the 2429*7c478bd9Sstevel@tonic-gate * mutex sleep queue. If we are on the mutex sleep 2430*7c478bd9Sstevel@tonic-gate * queue, continue sleeping. If we are on the condvar 2431*7c478bd9Sstevel@tonic-gate * sleep queue, break out of the sleep if we were 2432*7c478bd9Sstevel@tonic-gate * interrupted or we timed out (EINTR or ETIME). 2433*7c478bd9Sstevel@tonic-gate * Else this is a spurious wakeup; continue the loop. 2434*7c478bd9Sstevel@tonic-gate */ 2435*7c478bd9Sstevel@tonic-gate if (self->ul_sleepq == mqp) /* mutex queue */ 2436*7c478bd9Sstevel@tonic-gate tsp = NULL; 2437*7c478bd9Sstevel@tonic-gate else if (self->ul_sleepq == qp) { /* condvar queue */ 2438*7c478bd9Sstevel@tonic-gate if (error) { 2439*7c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = dequeue_self(qp, cvp); 2440*7c478bd9Sstevel@tonic-gate break; 2441*7c478bd9Sstevel@tonic-gate } 2442*7c478bd9Sstevel@tonic-gate /* 2443*7c478bd9Sstevel@tonic-gate * Else a spurious wakeup on the condvar queue. 2444*7c478bd9Sstevel@tonic-gate * __lwp_park() has already adjusted the timeout. 2445*7c478bd9Sstevel@tonic-gate */ 2446*7c478bd9Sstevel@tonic-gate } else { 2447*7c478bd9Sstevel@tonic-gate thr_panic("cond_sleep_queue(): thread not on queue"); 2448*7c478bd9Sstevel@tonic-gate } 2449*7c478bd9Sstevel@tonic-gate queue_unlock(mqp); 2450*7c478bd9Sstevel@tonic-gate } 2451*7c478bd9Sstevel@tonic-gate 2452*7c478bd9Sstevel@tonic-gate self->ul_sp = 0; 2453*7c478bd9Sstevel@tonic-gate ASSERT(self->ul_cvmutex == NULL && self->ul_cv_wake == 0); 2454*7c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 2455*7c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 2456*7c478bd9Sstevel@tonic-gate 2457*7c478bd9Sstevel@tonic-gate signalled = self->ul_signalled; 2458*7c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 2459*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 2460*7c478bd9Sstevel@tonic-gate queue_unlock(mqp); 2461*7c478bd9Sstevel@tonic-gate 2462*7c478bd9Sstevel@tonic-gate /* 2463*7c478bd9Sstevel@tonic-gate * If we were concurrently cond_signal()d and any of: 2464*7c478bd9Sstevel@tonic-gate * received a UNIX signal, were cancelled, or got a timeout, 2465*7c478bd9Sstevel@tonic-gate * then perform another cond_signal() to avoid consuming it. 2466*7c478bd9Sstevel@tonic-gate */ 2467*7c478bd9Sstevel@tonic-gate if (error && signalled) 2468*7c478bd9Sstevel@tonic-gate (void) cond_signal_internal(cvp); 2469*7c478bd9Sstevel@tonic-gate 2470*7c478bd9Sstevel@tonic-gate return (error); 2471*7c478bd9Sstevel@tonic-gate } 2472*7c478bd9Sstevel@tonic-gate 2473*7c478bd9Sstevel@tonic-gate int 2474*7c478bd9Sstevel@tonic-gate cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp, 2475*7c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp) 2476*7c478bd9Sstevel@tonic-gate { 2477*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2478*7c478bd9Sstevel@tonic-gate int error; 2479*7c478bd9Sstevel@tonic-gate 2480*7c478bd9Sstevel@tonic-gate /* 2481*7c478bd9Sstevel@tonic-gate * The old thread library was programmed to defer signals 2482*7c478bd9Sstevel@tonic-gate * while in cond_wait() so that the associated mutex would 2483*7c478bd9Sstevel@tonic-gate * be guaranteed to be held when the application signal 2484*7c478bd9Sstevel@tonic-gate * handler was invoked. 2485*7c478bd9Sstevel@tonic-gate * 2486*7c478bd9Sstevel@tonic-gate * We do not behave this way by default; the state of the 2487*7c478bd9Sstevel@tonic-gate * associated mutex in the signal handler is undefined. 2488*7c478bd9Sstevel@tonic-gate * 2489*7c478bd9Sstevel@tonic-gate * To accommodate applications that depend on the old 2490*7c478bd9Sstevel@tonic-gate * behavior, the _THREAD_COND_WAIT_DEFER environment 2491*7c478bd9Sstevel@tonic-gate * variable can be set to 1 and we will behave in the 2492*7c478bd9Sstevel@tonic-gate * old way with respect to cond_wait(). 2493*7c478bd9Sstevel@tonic-gate */ 2494*7c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 2495*7c478bd9Sstevel@tonic-gate sigoff(self); 2496*7c478bd9Sstevel@tonic-gate 2497*7c478bd9Sstevel@tonic-gate error = cond_sleep_queue(cvp, mp, tsp); 2498*7c478bd9Sstevel@tonic-gate 2499*7c478bd9Sstevel@tonic-gate /* 2500*7c478bd9Sstevel@tonic-gate * Reacquire the mutex. 2501*7c478bd9Sstevel@tonic-gate */ 2502*7c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 2503*7c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2504*7c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 2505*7c478bd9Sstevel@tonic-gate } else if (mutex_trylock_adaptive(mp) != 0) { 2506*7c478bd9Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL); 2507*7c478bd9Sstevel@tonic-gate } 2508*7c478bd9Sstevel@tonic-gate 2509*7c478bd9Sstevel@tonic-gate if (msp) 2510*7c478bd9Sstevel@tonic-gate record_begin_hold(msp); 2511*7c478bd9Sstevel@tonic-gate 2512*7c478bd9Sstevel@tonic-gate /* 2513*7c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 2514*7c478bd9Sstevel@tonic-gate */ 2515*7c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 2516*7c478bd9Sstevel@tonic-gate sigon(self); 2517*7c478bd9Sstevel@tonic-gate 2518*7c478bd9Sstevel@tonic-gate return (error); 2519*7c478bd9Sstevel@tonic-gate } 2520*7c478bd9Sstevel@tonic-gate 2521*7c478bd9Sstevel@tonic-gate /* 2522*7c478bd9Sstevel@tonic-gate * cond_sleep_kernel(): utility function for cond_wait_kernel(). 2523*7c478bd9Sstevel@tonic-gate * See the comment ahead of cond_sleep_queue(), above. 2524*7c478bd9Sstevel@tonic-gate */ 2525*7c478bd9Sstevel@tonic-gate int 2526*7c478bd9Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 2527*7c478bd9Sstevel@tonic-gate { 2528*7c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 2529*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2530*7c478bd9Sstevel@tonic-gate int error; 2531*7c478bd9Sstevel@tonic-gate 2532*7c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_PROTECT) { 2533*7c478bd9Sstevel@tonic-gate if (_ceil_mylist_del(mp)) 2534*7c478bd9Sstevel@tonic-gate _ceil_prio_waive(); 2535*7c478bd9Sstevel@tonic-gate } 2536*7c478bd9Sstevel@tonic-gate 2537*7c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 2538*7c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 2539*7c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 2540*7c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 2541*7c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) 2542*7c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 2543*7c478bd9Sstevel@tonic-gate /* 2544*7c478bd9Sstevel@tonic-gate * ___lwp_cond_wait() returns immediately with EINTR if 2545*7c478bd9Sstevel@tonic-gate * set_parking_flag(self,0) is called on this lwp before it 2546*7c478bd9Sstevel@tonic-gate * goes to sleep in the kernel. sigacthandler() calls this 2547*7c478bd9Sstevel@tonic-gate * when a deferred signal is noted. This assures that we don't 2548*7c478bd9Sstevel@tonic-gate * get stuck in ___lwp_cond_wait() with all signals blocked 2549*7c478bd9Sstevel@tonic-gate * due to taking a deferred signal before going to sleep. 2550*7c478bd9Sstevel@tonic-gate */ 2551*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 2552*7c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 2553*7c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 2554*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 2555*7c478bd9Sstevel@tonic-gate error = ___lwp_cond_wait(cvp, mp, tsp, 1); 2556*7c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 2557*7c478bd9Sstevel@tonic-gate self->ul_sp = 0; 2558*7c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 2559*7c478bd9Sstevel@tonic-gate return (error); 2560*7c478bd9Sstevel@tonic-gate } 2561*7c478bd9Sstevel@tonic-gate 2562*7c478bd9Sstevel@tonic-gate int 2563*7c478bd9Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 2564*7c478bd9Sstevel@tonic-gate { 2565*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2566*7c478bd9Sstevel@tonic-gate int error; 2567*7c478bd9Sstevel@tonic-gate int merror; 2568*7c478bd9Sstevel@tonic-gate 2569*7c478bd9Sstevel@tonic-gate /* 2570*7c478bd9Sstevel@tonic-gate * See the large comment in cond_wait_queue(), above. 2571*7c478bd9Sstevel@tonic-gate */ 2572*7c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 2573*7c478bd9Sstevel@tonic-gate sigoff(self); 2574*7c478bd9Sstevel@tonic-gate 2575*7c478bd9Sstevel@tonic-gate error = cond_sleep_kernel(cvp, mp, tsp); 2576*7c478bd9Sstevel@tonic-gate 2577*7c478bd9Sstevel@tonic-gate /* 2578*7c478bd9Sstevel@tonic-gate * Override the return code from ___lwp_cond_wait() 2579*7c478bd9Sstevel@tonic-gate * with any non-zero return code from mutex_lock(). 2580*7c478bd9Sstevel@tonic-gate * This addresses robust lock failures in particular; 2581*7c478bd9Sstevel@tonic-gate * the caller must see the EOWNERDEAD or ENOTRECOVERABLE 2582*7c478bd9Sstevel@tonic-gate * errors in order to take corrective action. 2583*7c478bd9Sstevel@tonic-gate */ 2584*7c478bd9Sstevel@tonic-gate if ((merror = _private_mutex_lock(mp)) != 0) 2585*7c478bd9Sstevel@tonic-gate error = merror; 2586*7c478bd9Sstevel@tonic-gate 2587*7c478bd9Sstevel@tonic-gate /* 2588*7c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 2589*7c478bd9Sstevel@tonic-gate */ 2590*7c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 2591*7c478bd9Sstevel@tonic-gate sigon(self); 2592*7c478bd9Sstevel@tonic-gate 2593*7c478bd9Sstevel@tonic-gate return (error); 2594*7c478bd9Sstevel@tonic-gate } 2595*7c478bd9Sstevel@tonic-gate 2596*7c478bd9Sstevel@tonic-gate /* 2597*7c478bd9Sstevel@tonic-gate * Common code for _cond_wait() and _cond_timedwait() 2598*7c478bd9Sstevel@tonic-gate */ 2599*7c478bd9Sstevel@tonic-gate int 2600*7c478bd9Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 2601*7c478bd9Sstevel@tonic-gate { 2602*7c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 2603*7c478bd9Sstevel@tonic-gate hrtime_t begin_sleep = 0; 2604*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2605*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 2606*7c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 2607*7c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 2608*7c478bd9Sstevel@tonic-gate uint8_t rcount; 2609*7c478bd9Sstevel@tonic-gate int error = 0; 2610*7c478bd9Sstevel@tonic-gate 2611*7c478bd9Sstevel@tonic-gate /* 2612*7c478bd9Sstevel@tonic-gate * The SUSV3 Posix spec for pthread_cond_timedwait() states: 2613*7c478bd9Sstevel@tonic-gate * Except in the case of [ETIMEDOUT], all these error checks 2614*7c478bd9Sstevel@tonic-gate * shall act as if they were performed immediately at the 2615*7c478bd9Sstevel@tonic-gate * beginning of processing for the function and shall cause 2616*7c478bd9Sstevel@tonic-gate * an error return, in effect, prior to modifying the state 2617*7c478bd9Sstevel@tonic-gate * of the mutex specified by mutex or the condition variable 2618*7c478bd9Sstevel@tonic-gate * specified by cond. 2619*7c478bd9Sstevel@tonic-gate * Therefore, we must return EINVAL now if the timout is invalid. 2620*7c478bd9Sstevel@tonic-gate */ 2621*7c478bd9Sstevel@tonic-gate if (tsp != NULL && 2622*7c478bd9Sstevel@tonic-gate (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC)) 2623*7c478bd9Sstevel@tonic-gate return (EINVAL); 2624*7c478bd9Sstevel@tonic-gate 2625*7c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 2626*7c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 2627*7c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 2628*7c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 2629*7c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = cvp; 2630*7c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 2631*7c478bd9Sstevel@tonic-gate self->ul_sp = 0; 2632*7c478bd9Sstevel@tonic-gate } 2633*7c478bd9Sstevel@tonic-gate if (csp) { 2634*7c478bd9Sstevel@tonic-gate if (tsp) 2635*7c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait); 2636*7c478bd9Sstevel@tonic-gate else 2637*7c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_wait); 2638*7c478bd9Sstevel@tonic-gate } 2639*7c478bd9Sstevel@tonic-gate if (msp) 2640*7c478bd9Sstevel@tonic-gate begin_sleep = record_hold_time(msp); 2641*7c478bd9Sstevel@tonic-gate else if (csp) 2642*7c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 2643*7c478bd9Sstevel@tonic-gate 2644*7c478bd9Sstevel@tonic-gate if (self->ul_error_detection) { 2645*7c478bd9Sstevel@tonic-gate if (!mutex_is_held(mp)) 2646*7c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, NULL); 2647*7c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) 2648*7c478bd9Sstevel@tonic-gate lock_error(mp, "recursive mutex in cond_wait", 2649*7c478bd9Sstevel@tonic-gate cvp, NULL); 2650*7c478bd9Sstevel@tonic-gate if (cvp->cond_type & USYNC_PROCESS) { 2651*7c478bd9Sstevel@tonic-gate if (!(mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))) 2652*7c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 2653*7c478bd9Sstevel@tonic-gate "condvar process-shared, " 2654*7c478bd9Sstevel@tonic-gate "mutex process-private"); 2655*7c478bd9Sstevel@tonic-gate } else { 2656*7c478bd9Sstevel@tonic-gate if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) 2657*7c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 2658*7c478bd9Sstevel@tonic-gate "condvar process-private, " 2659*7c478bd9Sstevel@tonic-gate "mutex process-shared"); 2660*7c478bd9Sstevel@tonic-gate } 2661*7c478bd9Sstevel@tonic-gate } 2662*7c478bd9Sstevel@tonic-gate 2663*7c478bd9Sstevel@tonic-gate /* 2664*7c478bd9Sstevel@tonic-gate * We deal with recursive mutexes by completely 2665*7c478bd9Sstevel@tonic-gate * dropping the lock and restoring the recursion 2666*7c478bd9Sstevel@tonic-gate * count after waking up. This is arguably wrong, 2667*7c478bd9Sstevel@tonic-gate * but it obeys the principle of least astonishment. 2668*7c478bd9Sstevel@tonic-gate */ 2669*7c478bd9Sstevel@tonic-gate rcount = mp->mutex_rcount; 2670*7c478bd9Sstevel@tonic-gate mp->mutex_rcount = 0; 2671*7c478bd9Sstevel@tonic-gate if ((mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST | 2672*7c478bd9Sstevel@tonic-gate PTHREAD_PRIO_INHERIT | PTHREAD_PRIO_PROTECT)) | 2673*7c478bd9Sstevel@tonic-gate (cvp->cond_type & USYNC_PROCESS)) 2674*7c478bd9Sstevel@tonic-gate error = cond_wait_kernel(cvp, mp, tsp); 2675*7c478bd9Sstevel@tonic-gate else 2676*7c478bd9Sstevel@tonic-gate error = cond_wait_queue(cvp, mp, tsp, msp); 2677*7c478bd9Sstevel@tonic-gate mp->mutex_rcount = rcount; 2678*7c478bd9Sstevel@tonic-gate 2679*7c478bd9Sstevel@tonic-gate if (csp) { 2680*7c478bd9Sstevel@tonic-gate hrtime_t lapse = gethrtime() - begin_sleep; 2681*7c478bd9Sstevel@tonic-gate if (tsp == NULL) 2682*7c478bd9Sstevel@tonic-gate csp->cond_wait_sleep_time += lapse; 2683*7c478bd9Sstevel@tonic-gate else { 2684*7c478bd9Sstevel@tonic-gate csp->cond_timedwait_sleep_time += lapse; 2685*7c478bd9Sstevel@tonic-gate if (error == ETIME) 2686*7c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait_timeout); 2687*7c478bd9Sstevel@tonic-gate } 2688*7c478bd9Sstevel@tonic-gate } 2689*7c478bd9Sstevel@tonic-gate return (error); 2690*7c478bd9Sstevel@tonic-gate } 2691*7c478bd9Sstevel@tonic-gate 2692*7c478bd9Sstevel@tonic-gate /* 2693*7c478bd9Sstevel@tonic-gate * cond_wait() is a cancellation point but _cond_wait() is not. 2694*7c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 2695*7c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 2696*7c478bd9Sstevel@tonic-gate */ 2697*7c478bd9Sstevel@tonic-gate int 2698*7c478bd9Sstevel@tonic-gate _cond_wait(cond_t *cvp, mutex_t *mp) 2699*7c478bd9Sstevel@tonic-gate { 2700*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2701*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 2702*7c478bd9Sstevel@tonic-gate uberflags_t *gflags; 2703*7c478bd9Sstevel@tonic-gate 2704*7c478bd9Sstevel@tonic-gate /* 2705*7c478bd9Sstevel@tonic-gate * Optimize the common case of USYNC_THREAD plus 2706*7c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, and no event tracing. 2707*7c478bd9Sstevel@tonic-gate */ 2708*7c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 2709*7c478bd9Sstevel@tonic-gate (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted | 2710*7c478bd9Sstevel@tonic-gate self->ul_td_events_enable | 2711*7c478bd9Sstevel@tonic-gate udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0) 2712*7c478bd9Sstevel@tonic-gate return (cond_wait_queue(cvp, mp, NULL, NULL)); 2713*7c478bd9Sstevel@tonic-gate 2714*7c478bd9Sstevel@tonic-gate /* 2715*7c478bd9Sstevel@tonic-gate * Else do it the long way. 2716*7c478bd9Sstevel@tonic-gate */ 2717*7c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, NULL)); 2718*7c478bd9Sstevel@tonic-gate } 2719*7c478bd9Sstevel@tonic-gate 2720*7c478bd9Sstevel@tonic-gate int 2721*7c478bd9Sstevel@tonic-gate cond_wait(cond_t *cvp, mutex_t *mp) 2722*7c478bd9Sstevel@tonic-gate { 2723*7c478bd9Sstevel@tonic-gate int error; 2724*7c478bd9Sstevel@tonic-gate 2725*7c478bd9Sstevel@tonic-gate _cancelon(); 2726*7c478bd9Sstevel@tonic-gate error = _cond_wait(cvp, mp); 2727*7c478bd9Sstevel@tonic-gate if (error == EINTR) 2728*7c478bd9Sstevel@tonic-gate _canceloff(); 2729*7c478bd9Sstevel@tonic-gate else 2730*7c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 2731*7c478bd9Sstevel@tonic-gate return (error); 2732*7c478bd9Sstevel@tonic-gate } 2733*7c478bd9Sstevel@tonic-gate 2734*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_wait = _pthread_cond_wait 2735*7c478bd9Sstevel@tonic-gate int 2736*7c478bd9Sstevel@tonic-gate _pthread_cond_wait(cond_t *cvp, mutex_t *mp) 2737*7c478bd9Sstevel@tonic-gate { 2738*7c478bd9Sstevel@tonic-gate int error; 2739*7c478bd9Sstevel@tonic-gate 2740*7c478bd9Sstevel@tonic-gate error = cond_wait(cvp, mp); 2741*7c478bd9Sstevel@tonic-gate return ((error == EINTR)? 0 : error); 2742*7c478bd9Sstevel@tonic-gate } 2743*7c478bd9Sstevel@tonic-gate 2744*7c478bd9Sstevel@tonic-gate /* 2745*7c478bd9Sstevel@tonic-gate * cond_timedwait() is a cancellation point but _cond_timedwait() is not. 2746*7c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 2747*7c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 2748*7c478bd9Sstevel@tonic-gate */ 2749*7c478bd9Sstevel@tonic-gate int 2750*7c478bd9Sstevel@tonic-gate _cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 2751*7c478bd9Sstevel@tonic-gate { 2752*7c478bd9Sstevel@tonic-gate clockid_t clock_id = cvp->cond_clockid; 2753*7c478bd9Sstevel@tonic-gate timespec_t reltime; 2754*7c478bd9Sstevel@tonic-gate int error; 2755*7c478bd9Sstevel@tonic-gate 2756*7c478bd9Sstevel@tonic-gate if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES) 2757*7c478bd9Sstevel@tonic-gate clock_id = CLOCK_REALTIME; 2758*7c478bd9Sstevel@tonic-gate abstime_to_reltime(clock_id, abstime, &reltime); 2759*7c478bd9Sstevel@tonic-gate error = cond_wait_common(cvp, mp, &reltime); 2760*7c478bd9Sstevel@tonic-gate if (error == ETIME && clock_id == CLOCK_HIGHRES) { 2761*7c478bd9Sstevel@tonic-gate /* 2762*7c478bd9Sstevel@tonic-gate * Don't return ETIME if we didn't really get a timeout. 2763*7c478bd9Sstevel@tonic-gate * This can happen if we return because someone resets 2764*7c478bd9Sstevel@tonic-gate * the system clock. Just return zero in this case, 2765*7c478bd9Sstevel@tonic-gate * giving a spurious wakeup but not a timeout. 2766*7c478bd9Sstevel@tonic-gate */ 2767*7c478bd9Sstevel@tonic-gate if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC + 2768*7c478bd9Sstevel@tonic-gate abstime->tv_nsec > gethrtime()) 2769*7c478bd9Sstevel@tonic-gate error = 0; 2770*7c478bd9Sstevel@tonic-gate } 2771*7c478bd9Sstevel@tonic-gate return (error); 2772*7c478bd9Sstevel@tonic-gate } 2773*7c478bd9Sstevel@tonic-gate 2774*7c478bd9Sstevel@tonic-gate int 2775*7c478bd9Sstevel@tonic-gate cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 2776*7c478bd9Sstevel@tonic-gate { 2777*7c478bd9Sstevel@tonic-gate int error; 2778*7c478bd9Sstevel@tonic-gate 2779*7c478bd9Sstevel@tonic-gate _cancelon(); 2780*7c478bd9Sstevel@tonic-gate error = _cond_timedwait(cvp, mp, abstime); 2781*7c478bd9Sstevel@tonic-gate if (error == EINTR) 2782*7c478bd9Sstevel@tonic-gate _canceloff(); 2783*7c478bd9Sstevel@tonic-gate else 2784*7c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 2785*7c478bd9Sstevel@tonic-gate return (error); 2786*7c478bd9Sstevel@tonic-gate } 2787*7c478bd9Sstevel@tonic-gate 2788*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_timedwait = _pthread_cond_timedwait 2789*7c478bd9Sstevel@tonic-gate int 2790*7c478bd9Sstevel@tonic-gate _pthread_cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 2791*7c478bd9Sstevel@tonic-gate { 2792*7c478bd9Sstevel@tonic-gate int error; 2793*7c478bd9Sstevel@tonic-gate 2794*7c478bd9Sstevel@tonic-gate error = cond_timedwait(cvp, mp, abstime); 2795*7c478bd9Sstevel@tonic-gate if (error == ETIME) 2796*7c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 2797*7c478bd9Sstevel@tonic-gate else if (error == EINTR) 2798*7c478bd9Sstevel@tonic-gate error = 0; 2799*7c478bd9Sstevel@tonic-gate return (error); 2800*7c478bd9Sstevel@tonic-gate } 2801*7c478bd9Sstevel@tonic-gate 2802*7c478bd9Sstevel@tonic-gate /* 2803*7c478bd9Sstevel@tonic-gate * cond_reltimedwait() is a cancellation point but _cond_reltimedwait() 2804*7c478bd9Sstevel@tonic-gate * is not. System libraries call the non-cancellation version. 2805*7c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 2806*7c478bd9Sstevel@tonic-gate */ 2807*7c478bd9Sstevel@tonic-gate int 2808*7c478bd9Sstevel@tonic-gate _cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 2809*7c478bd9Sstevel@tonic-gate { 2810*7c478bd9Sstevel@tonic-gate timespec_t tslocal = *reltime; 2811*7c478bd9Sstevel@tonic-gate 2812*7c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, &tslocal)); 2813*7c478bd9Sstevel@tonic-gate } 2814*7c478bd9Sstevel@tonic-gate 2815*7c478bd9Sstevel@tonic-gate #pragma weak cond_reltimedwait = _cond_reltimedwait_cancel 2816*7c478bd9Sstevel@tonic-gate int 2817*7c478bd9Sstevel@tonic-gate _cond_reltimedwait_cancel(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 2818*7c478bd9Sstevel@tonic-gate { 2819*7c478bd9Sstevel@tonic-gate int error; 2820*7c478bd9Sstevel@tonic-gate 2821*7c478bd9Sstevel@tonic-gate _cancelon(); 2822*7c478bd9Sstevel@tonic-gate error = _cond_reltimedwait(cvp, mp, reltime); 2823*7c478bd9Sstevel@tonic-gate if (error == EINTR) 2824*7c478bd9Sstevel@tonic-gate _canceloff(); 2825*7c478bd9Sstevel@tonic-gate else 2826*7c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 2827*7c478bd9Sstevel@tonic-gate return (error); 2828*7c478bd9Sstevel@tonic-gate } 2829*7c478bd9Sstevel@tonic-gate 2830*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_reltimedwait_np = _pthread_cond_reltimedwait_np 2831*7c478bd9Sstevel@tonic-gate int 2832*7c478bd9Sstevel@tonic-gate _pthread_cond_reltimedwait_np(cond_t *cvp, mutex_t *mp, 2833*7c478bd9Sstevel@tonic-gate const timespec_t *reltime) 2834*7c478bd9Sstevel@tonic-gate { 2835*7c478bd9Sstevel@tonic-gate int error; 2836*7c478bd9Sstevel@tonic-gate 2837*7c478bd9Sstevel@tonic-gate error = _cond_reltimedwait_cancel(cvp, mp, reltime); 2838*7c478bd9Sstevel@tonic-gate if (error == ETIME) 2839*7c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 2840*7c478bd9Sstevel@tonic-gate else if (error == EINTR) 2841*7c478bd9Sstevel@tonic-gate error = 0; 2842*7c478bd9Sstevel@tonic-gate return (error); 2843*7c478bd9Sstevel@tonic-gate } 2844*7c478bd9Sstevel@tonic-gate 2845*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_signal = cond_signal_internal 2846*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_signal = cond_signal_internal 2847*7c478bd9Sstevel@tonic-gate #pragma weak cond_signal = cond_signal_internal 2848*7c478bd9Sstevel@tonic-gate #pragma weak _cond_signal = cond_signal_internal 2849*7c478bd9Sstevel@tonic-gate int 2850*7c478bd9Sstevel@tonic-gate cond_signal_internal(cond_t *cvp) 2851*7c478bd9Sstevel@tonic-gate { 2852*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2853*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 2854*7c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 2855*7c478bd9Sstevel@tonic-gate int error = 0; 2856*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 2857*7c478bd9Sstevel@tonic-gate mutex_t *mp; 2858*7c478bd9Sstevel@tonic-gate queue_head_t *mqp; 2859*7c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 2860*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 2861*7c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 2862*7c478bd9Sstevel@tonic-gate ulwp_t *next; 2863*7c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 2864*7c478bd9Sstevel@tonic-gate ulwp_t *susprev; 2865*7c478bd9Sstevel@tonic-gate 2866*7c478bd9Sstevel@tonic-gate if (csp) 2867*7c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_signal); 2868*7c478bd9Sstevel@tonic-gate 2869*7c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 2870*7c478bd9Sstevel@tonic-gate error = __lwp_cond_signal(cvp); 2871*7c478bd9Sstevel@tonic-gate 2872*7c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 2873*7c478bd9Sstevel@tonic-gate return (error); 2874*7c478bd9Sstevel@tonic-gate 2875*7c478bd9Sstevel@tonic-gate /* 2876*7c478bd9Sstevel@tonic-gate * Move someone from the condvar sleep queue to the mutex sleep 2877*7c478bd9Sstevel@tonic-gate * queue for the mutex that he will acquire on being waked up. 2878*7c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex he will acquire. 2879*7c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if his ul_cv_wake flag 2880*7c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark him. 2881*7c478bd9Sstevel@tonic-gate */ 2882*7c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 2883*7c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 2884*7c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 2885*7c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == cvp) { 2886*7c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 2887*7c478bd9Sstevel@tonic-gate break; 2888*7c478bd9Sstevel@tonic-gate /* 2889*7c478bd9Sstevel@tonic-gate * Try not to dequeue a suspended thread. 2890*7c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 2891*7c478bd9Sstevel@tonic-gate */ 2892*7c478bd9Sstevel@tonic-gate if (suspp == NULL) { 2893*7c478bd9Sstevel@tonic-gate suspp = ulwpp; 2894*7c478bd9Sstevel@tonic-gate susprev = prev; 2895*7c478bd9Sstevel@tonic-gate } 2896*7c478bd9Sstevel@tonic-gate } 2897*7c478bd9Sstevel@tonic-gate } 2898*7c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 2899*7c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 2900*7c478bd9Sstevel@tonic-gate prev = susprev; 2901*7c478bd9Sstevel@tonic-gate suspp = NULL; 2902*7c478bd9Sstevel@tonic-gate } 2903*7c478bd9Sstevel@tonic-gate if (ulwp == NULL) { /* no one on the sleep queue */ 2904*7c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 2905*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 2906*7c478bd9Sstevel@tonic-gate return (error); 2907*7c478bd9Sstevel@tonic-gate } 2908*7c478bd9Sstevel@tonic-gate /* 2909*7c478bd9Sstevel@tonic-gate * Scan the remainder of the CV queue for another waiter. 2910*7c478bd9Sstevel@tonic-gate */ 2911*7c478bd9Sstevel@tonic-gate if (suspp != NULL) { 2912*7c478bd9Sstevel@tonic-gate next = *suspp; 2913*7c478bd9Sstevel@tonic-gate } else { 2914*7c478bd9Sstevel@tonic-gate for (next = ulwp->ul_link; next != NULL; next = next->ul_link) 2915*7c478bd9Sstevel@tonic-gate if (next->ul_wchan == cvp) 2916*7c478bd9Sstevel@tonic-gate break; 2917*7c478bd9Sstevel@tonic-gate } 2918*7c478bd9Sstevel@tonic-gate if (next == NULL) 2919*7c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 2920*7c478bd9Sstevel@tonic-gate 2921*7c478bd9Sstevel@tonic-gate /* 2922*7c478bd9Sstevel@tonic-gate * Inform the thread that he was the recipient of a cond_signal(). 2923*7c478bd9Sstevel@tonic-gate * This lets him deal with cond_signal() and, concurrently, 2924*7c478bd9Sstevel@tonic-gate * one or more of a cancellation, a UNIX signal, or a timeout. 2925*7c478bd9Sstevel@tonic-gate * These latter conditions must not consume a cond_signal(). 2926*7c478bd9Sstevel@tonic-gate */ 2927*7c478bd9Sstevel@tonic-gate ulwp->ul_signalled = 1; 2928*7c478bd9Sstevel@tonic-gate 2929*7c478bd9Sstevel@tonic-gate /* 2930*7c478bd9Sstevel@tonic-gate * Dequeue the waiter but leave his ul_sleepq non-NULL 2931*7c478bd9Sstevel@tonic-gate * while we move him to the mutex queue so that he can 2932*7c478bd9Sstevel@tonic-gate * deal properly with spurious wakeups. 2933*7c478bd9Sstevel@tonic-gate */ 2934*7c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 2935*7c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 2936*7c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 2937*7c478bd9Sstevel@tonic-gate qp->qh_qlen--; 2938*7c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 2939*7c478bd9Sstevel@tonic-gate 2940*7c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* the mutex he will acquire */ 2941*7c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 2942*7c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 2943*7c478bd9Sstevel@tonic-gate 2944*7c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 2945*7c478bd9Sstevel@tonic-gate lwpid_t lwpid = ulwp->ul_lwpid; 2946*7c478bd9Sstevel@tonic-gate 2947*7c478bd9Sstevel@tonic-gate no_preempt(self); 2948*7c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 2949*7c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 2950*7c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 2951*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 2952*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 2953*7c478bd9Sstevel@tonic-gate preempt(self); 2954*7c478bd9Sstevel@tonic-gate } else { 2955*7c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 2956*7c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 2957*7c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 2958*7c478bd9Sstevel@tonic-gate queue_unlock(mqp); 2959*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 2960*7c478bd9Sstevel@tonic-gate } 2961*7c478bd9Sstevel@tonic-gate 2962*7c478bd9Sstevel@tonic-gate return (error); 2963*7c478bd9Sstevel@tonic-gate } 2964*7c478bd9Sstevel@tonic-gate 2965*7c478bd9Sstevel@tonic-gate #define MAXLWPS 128 /* max remembered lwpids before overflow */ 2966*7c478bd9Sstevel@tonic-gate #define NEWLWPS 2048 /* max remembered lwpids at first overflow */ 2967*7c478bd9Sstevel@tonic-gate 2968*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_broadcast = cond_broadcast_internal 2969*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_broadcast = cond_broadcast_internal 2970*7c478bd9Sstevel@tonic-gate #pragma weak cond_broadcast = cond_broadcast_internal 2971*7c478bd9Sstevel@tonic-gate #pragma weak _cond_broadcast = cond_broadcast_internal 2972*7c478bd9Sstevel@tonic-gate int 2973*7c478bd9Sstevel@tonic-gate cond_broadcast_internal(cond_t *cvp) 2974*7c478bd9Sstevel@tonic-gate { 2975*7c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2976*7c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 2977*7c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 2978*7c478bd9Sstevel@tonic-gate int error = 0; 2979*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 2980*7c478bd9Sstevel@tonic-gate mutex_t *mp; 2981*7c478bd9Sstevel@tonic-gate queue_head_t *mqp; 2982*7c478bd9Sstevel@tonic-gate mutex_t *mp_cache = NULL; 2983*7c478bd9Sstevel@tonic-gate queue_head_t *mqp_cache = NULL; 2984*7c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 2985*7c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 2986*7c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 2987*7c478bd9Sstevel@tonic-gate lwpid_t buffer[MAXLWPS]; 2988*7c478bd9Sstevel@tonic-gate lwpid_t *lwpid = buffer; 2989*7c478bd9Sstevel@tonic-gate int nlwpid = 0; 2990*7c478bd9Sstevel@tonic-gate int maxlwps = MAXLWPS; 2991*7c478bd9Sstevel@tonic-gate 2992*7c478bd9Sstevel@tonic-gate if (csp) 2993*7c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_broadcast); 2994*7c478bd9Sstevel@tonic-gate 2995*7c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 2996*7c478bd9Sstevel@tonic-gate error = __lwp_cond_broadcast(cvp); 2997*7c478bd9Sstevel@tonic-gate 2998*7c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 2999*7c478bd9Sstevel@tonic-gate return (error); 3000*7c478bd9Sstevel@tonic-gate 3001*7c478bd9Sstevel@tonic-gate /* 3002*7c478bd9Sstevel@tonic-gate * Move everyone from the condvar sleep queue to the mutex sleep 3003*7c478bd9Sstevel@tonic-gate * queue for the mutex that they will acquire on being waked up. 3004*7c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex they will acquire. 3005*7c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if their ul_cv_wake flag 3006*7c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark them. 3007*7c478bd9Sstevel@tonic-gate * 3008*7c478bd9Sstevel@tonic-gate * We keep track of lwpids that are to be unparked in lwpid[]. 3009*7c478bd9Sstevel@tonic-gate * __lwp_unpark_all() is called to unpark all of them after 3010*7c478bd9Sstevel@tonic-gate * they have been removed from the sleep queue and the sleep 3011*7c478bd9Sstevel@tonic-gate * queue lock has been dropped. If we run out of space in our 3012*7c478bd9Sstevel@tonic-gate * on-stack buffer, we need to allocate more but we can't call 3013*7c478bd9Sstevel@tonic-gate * lmalloc() because we are holding a queue lock when the overflow 3014*7c478bd9Sstevel@tonic-gate * occurs and lmalloc() acquires a lock. We can't use alloca() 3015*7c478bd9Sstevel@tonic-gate * either because the application may have allocated a small stack 3016*7c478bd9Sstevel@tonic-gate * and we don't want to overrun the stack. So we use the mmap() 3017*7c478bd9Sstevel@tonic-gate * system call directly since that path acquires no locks. 3018*7c478bd9Sstevel@tonic-gate */ 3019*7c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 3020*7c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 3021*7c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 3022*7c478bd9Sstevel@tonic-gate while ((ulwp = *ulwpp) != NULL) { 3023*7c478bd9Sstevel@tonic-gate 3024*7c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan != cvp) { 3025*7c478bd9Sstevel@tonic-gate prev = ulwp; 3026*7c478bd9Sstevel@tonic-gate ulwpp = &ulwp->ul_link; 3027*7c478bd9Sstevel@tonic-gate continue; 3028*7c478bd9Sstevel@tonic-gate } 3029*7c478bd9Sstevel@tonic-gate 3030*7c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 3031*7c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 3032*7c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 3033*7c478bd9Sstevel@tonic-gate qp->qh_qlen--; 3034*7c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 3035*7c478bd9Sstevel@tonic-gate 3036*7c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* his mutex */ 3037*7c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 3038*7c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 3039*7c478bd9Sstevel@tonic-gate 3040*7c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 3041*7c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 3042*7c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 3043*7c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 3044*7c478bd9Sstevel@tonic-gate if (nlwpid == maxlwps) { 3045*7c478bd9Sstevel@tonic-gate /* 3046*7c478bd9Sstevel@tonic-gate * Allocate NEWLWPS ids on the first overflow. 3047*7c478bd9Sstevel@tonic-gate * Double the allocation each time after that. 3048*7c478bd9Sstevel@tonic-gate */ 3049*7c478bd9Sstevel@tonic-gate int newlwps = (lwpid == buffer)? NEWLWPS : 3050*7c478bd9Sstevel@tonic-gate 2 * maxlwps; 3051*7c478bd9Sstevel@tonic-gate void *vaddr = _private_mmap(NULL, 3052*7c478bd9Sstevel@tonic-gate newlwps * sizeof (lwpid_t), 3053*7c478bd9Sstevel@tonic-gate PROT_READ|PROT_WRITE, 3054*7c478bd9Sstevel@tonic-gate MAP_PRIVATE|MAP_ANON, -1, (off_t)0); 3055*7c478bd9Sstevel@tonic-gate if (vaddr == MAP_FAILED) { 3056*7c478bd9Sstevel@tonic-gate /* 3057*7c478bd9Sstevel@tonic-gate * Let's hope this never happens. 3058*7c478bd9Sstevel@tonic-gate * If it does, then we have a terrible 3059*7c478bd9Sstevel@tonic-gate * thundering herd on our hands. 3060*7c478bd9Sstevel@tonic-gate */ 3061*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid); 3062*7c478bd9Sstevel@tonic-gate nlwpid = 0; 3063*7c478bd9Sstevel@tonic-gate } else { 3064*7c478bd9Sstevel@tonic-gate (void) _memcpy(vaddr, lwpid, 3065*7c478bd9Sstevel@tonic-gate maxlwps * sizeof (lwpid_t)); 3066*7c478bd9Sstevel@tonic-gate if (lwpid != buffer) 3067*7c478bd9Sstevel@tonic-gate (void) _private_munmap(lwpid, 3068*7c478bd9Sstevel@tonic-gate maxlwps * sizeof (lwpid_t)); 3069*7c478bd9Sstevel@tonic-gate lwpid = vaddr; 3070*7c478bd9Sstevel@tonic-gate maxlwps = newlwps; 3071*7c478bd9Sstevel@tonic-gate } 3072*7c478bd9Sstevel@tonic-gate } 3073*7c478bd9Sstevel@tonic-gate lwpid[nlwpid++] = ulwp->ul_lwpid; 3074*7c478bd9Sstevel@tonic-gate } else { 3075*7c478bd9Sstevel@tonic-gate if (mp != mp_cache) { 3076*7c478bd9Sstevel@tonic-gate if (mqp_cache != NULL) 3077*7c478bd9Sstevel@tonic-gate queue_unlock(mqp_cache); 3078*7c478bd9Sstevel@tonic-gate mqp_cache = queue_lock(mp, MX); 3079*7c478bd9Sstevel@tonic-gate mp_cache = mp; 3080*7c478bd9Sstevel@tonic-gate } 3081*7c478bd9Sstevel@tonic-gate mqp = mqp_cache; 3082*7c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 3083*7c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 3084*7c478bd9Sstevel@tonic-gate } 3085*7c478bd9Sstevel@tonic-gate } 3086*7c478bd9Sstevel@tonic-gate if (mqp_cache != NULL) 3087*7c478bd9Sstevel@tonic-gate queue_unlock(mqp_cache); 3088*7c478bd9Sstevel@tonic-gate queue_unlock(qp); 3089*7c478bd9Sstevel@tonic-gate if (nlwpid) { 3090*7c478bd9Sstevel@tonic-gate if (nlwpid == 1) 3091*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid[0]); 3092*7c478bd9Sstevel@tonic-gate else 3093*7c478bd9Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid); 3094*7c478bd9Sstevel@tonic-gate } 3095*7c478bd9Sstevel@tonic-gate if (lwpid != buffer) 3096*7c478bd9Sstevel@tonic-gate (void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t)); 3097*7c478bd9Sstevel@tonic-gate 3098*7c478bd9Sstevel@tonic-gate return (error); 3099*7c478bd9Sstevel@tonic-gate } 3100*7c478bd9Sstevel@tonic-gate 3101*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_destroy = _cond_destroy 3102*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_destroy = _cond_destroy 3103*7c478bd9Sstevel@tonic-gate #pragma weak cond_destroy = _cond_destroy 3104*7c478bd9Sstevel@tonic-gate int 3105*7c478bd9Sstevel@tonic-gate _cond_destroy(cond_t *cvp) 3106*7c478bd9Sstevel@tonic-gate { 3107*7c478bd9Sstevel@tonic-gate cvp->cond_magic = 0; 3108*7c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(cvp); 3109*7c478bd9Sstevel@tonic-gate return (0); 3110*7c478bd9Sstevel@tonic-gate } 3111*7c478bd9Sstevel@tonic-gate 3112*7c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 3113*7c478bd9Sstevel@tonic-gate void 3114*7c478bd9Sstevel@tonic-gate assert_no_libc_locks_held(void) 3115*7c478bd9Sstevel@tonic-gate { 3116*7c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 3117*7c478bd9Sstevel@tonic-gate } 3118*7c478bd9Sstevel@tonic-gate #endif 3119*7c478bd9Sstevel@tonic-gate 3120*7c478bd9Sstevel@tonic-gate /* protected by link_lock */ 3121*7c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin; 3122*7c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin2; 3123*7c478bd9Sstevel@tonic-gate uint64_t spin_lock_sleep; 3124*7c478bd9Sstevel@tonic-gate uint64_t spin_lock_wakeup; 3125*7c478bd9Sstevel@tonic-gate 3126*7c478bd9Sstevel@tonic-gate /* 3127*7c478bd9Sstevel@tonic-gate * Record spin lock statistics. 3128*7c478bd9Sstevel@tonic-gate * Called by a thread exiting itself in thrp_exit(). 3129*7c478bd9Sstevel@tonic-gate * Also called via atexit() from the thread calling 3130*7c478bd9Sstevel@tonic-gate * exit() to do all the other threads as well. 3131*7c478bd9Sstevel@tonic-gate */ 3132*7c478bd9Sstevel@tonic-gate void 3133*7c478bd9Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp) 3134*7c478bd9Sstevel@tonic-gate { 3135*7c478bd9Sstevel@tonic-gate spin_lock_spin += ulwp->ul_spin_lock_spin; 3136*7c478bd9Sstevel@tonic-gate spin_lock_spin2 += ulwp->ul_spin_lock_spin2; 3137*7c478bd9Sstevel@tonic-gate spin_lock_sleep += ulwp->ul_spin_lock_sleep; 3138*7c478bd9Sstevel@tonic-gate spin_lock_wakeup += ulwp->ul_spin_lock_wakeup; 3139*7c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin = 0; 3140*7c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin2 = 0; 3141*7c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_sleep = 0; 3142*7c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_wakeup = 0; 3143*7c478bd9Sstevel@tonic-gate } 3144*7c478bd9Sstevel@tonic-gate 3145*7c478bd9Sstevel@tonic-gate /* 3146*7c478bd9Sstevel@tonic-gate * atexit function: dump the queue statistics to stderr. 3147*7c478bd9Sstevel@tonic-gate */ 3148*7c478bd9Sstevel@tonic-gate #include <stdio.h> 3149*7c478bd9Sstevel@tonic-gate void 3150*7c478bd9Sstevel@tonic-gate dump_queue_statistics(void) 3151*7c478bd9Sstevel@tonic-gate { 3152*7c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 3153*7c478bd9Sstevel@tonic-gate queue_head_t *qp; 3154*7c478bd9Sstevel@tonic-gate int qn; 3155*7c478bd9Sstevel@tonic-gate uint64_t spin_lock_total = 0; 3156*7c478bd9Sstevel@tonic-gate 3157*7c478bd9Sstevel@tonic-gate if (udp->queue_head == NULL || thread_queue_dump == 0) 3158*7c478bd9Sstevel@tonic-gate return; 3159*7c478bd9Sstevel@tonic-gate 3160*7c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 || 3161*7c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 3162*7c478bd9Sstevel@tonic-gate return; 3163*7c478bd9Sstevel@tonic-gate for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) { 3164*7c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 3165*7c478bd9Sstevel@tonic-gate continue; 3166*7c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 3167*7c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 3168*7c478bd9Sstevel@tonic-gate (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 3169*7c478bd9Sstevel@tonic-gate return; 3170*7c478bd9Sstevel@tonic-gate } 3171*7c478bd9Sstevel@tonic-gate 3172*7c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 || 3173*7c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 3174*7c478bd9Sstevel@tonic-gate return; 3175*7c478bd9Sstevel@tonic-gate for (qn = 0; qn < QHASHSIZE; qn++, qp++) { 3176*7c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 3177*7c478bd9Sstevel@tonic-gate continue; 3178*7c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 3179*7c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 3180*7c478bd9Sstevel@tonic-gate (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 3181*7c478bd9Sstevel@tonic-gate return; 3182*7c478bd9Sstevel@tonic-gate } 3183*7c478bd9Sstevel@tonic-gate 3184*7c478bd9Sstevel@tonic-gate (void) fprintf(stderr, "\n spin_lock_total = %10llu\n", 3185*7c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_total); 3186*7c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin = %10llu\n", 3187*7c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_spin); 3188*7c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin2 = %10llu\n", 3189*7c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_spin2); 3190*7c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_sleep = %10llu\n", 3191*7c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_sleep); 3192*7c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_wakeup = %10llu\n", 3193*7c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_wakeup); 3194*7c478bd9Sstevel@tonic-gate } 3195