17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 52be60c5eSraf * Common Development and Distribution License (the "License"). 62be60c5eSraf * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21e8031f0aSraf 227c478bd9Sstevel@tonic-gate /* 2341efec22Sraf * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 307c478bd9Sstevel@tonic-gate 317c478bd9Sstevel@tonic-gate #include "lint.h" 327c478bd9Sstevel@tonic-gate #include "thr_uberdata.h" 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate /* 357c478bd9Sstevel@tonic-gate * This mutex is initialized to be held by lwp#1. 367c478bd9Sstevel@tonic-gate * It is used to block a thread that has returned from a mutex_lock() 37883492d5Sraf * of a LOCK_PRIO_INHERIT mutex with an unrecoverable error. 387c478bd9Sstevel@tonic-gate */ 397c478bd9Sstevel@tonic-gate mutex_t stall_mutex = DEFAULTMUTEX; 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate static int shared_mutex_held(mutex_t *); 42883492d5Sraf static int mutex_unlock_internal(mutex_t *, int); 43883492d5Sraf static int mutex_queuelock_adaptive(mutex_t *); 44883492d5Sraf static void mutex_wakeup_all(mutex_t *); 457c478bd9Sstevel@tonic-gate 467c478bd9Sstevel@tonic-gate /* 477c478bd9Sstevel@tonic-gate * Lock statistics support functions. 487c478bd9Sstevel@tonic-gate */ 497c478bd9Sstevel@tonic-gate void 507c478bd9Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp) 517c478bd9Sstevel@tonic-gate { 527c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_lock); 537c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = gethrtime(); 547c478bd9Sstevel@tonic-gate } 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate hrtime_t 577c478bd9Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp) 587c478bd9Sstevel@tonic-gate { 597c478bd9Sstevel@tonic-gate hrtime_t now = gethrtime(); 607c478bd9Sstevel@tonic-gate 617c478bd9Sstevel@tonic-gate if (msp->mutex_begin_hold) 627c478bd9Sstevel@tonic-gate msp->mutex_hold_time += now - msp->mutex_begin_hold; 637c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = 0; 647c478bd9Sstevel@tonic-gate return (now); 657c478bd9Sstevel@tonic-gate } 667c478bd9Sstevel@tonic-gate 677c478bd9Sstevel@tonic-gate /* 687c478bd9Sstevel@tonic-gate * Called once at library initialization. 697c478bd9Sstevel@tonic-gate */ 707c478bd9Sstevel@tonic-gate void 717c478bd9Sstevel@tonic-gate mutex_setup(void) 727c478bd9Sstevel@tonic-gate { 737c478bd9Sstevel@tonic-gate if (set_lock_byte(&stall_mutex.mutex_lockw)) 747c478bd9Sstevel@tonic-gate thr_panic("mutex_setup() cannot acquire stall_mutex"); 757c478bd9Sstevel@tonic-gate stall_mutex.mutex_owner = (uintptr_t)curthread; 767c478bd9Sstevel@tonic-gate } 777c478bd9Sstevel@tonic-gate 787c478bd9Sstevel@tonic-gate /* 79*5d1dd9a9Sraf * The default spin count of 1000 is experimentally determined. 80*5d1dd9a9Sraf * On sun4u machines with any number of processors it could be raised 817c478bd9Sstevel@tonic-gate * to 10,000 but that (experimentally) makes almost no difference. 82*5d1dd9a9Sraf * The environment variable: 837c478bd9Sstevel@tonic-gate * _THREAD_ADAPTIVE_SPIN=count 84*5d1dd9a9Sraf * can be used to override and set the count in the range [0 .. 1,000,000]. 857c478bd9Sstevel@tonic-gate */ 867c478bd9Sstevel@tonic-gate int thread_adaptive_spin = 1000; 877c478bd9Sstevel@tonic-gate uint_t thread_max_spinners = 100; 887c478bd9Sstevel@tonic-gate int thread_queue_verify = 0; 897c478bd9Sstevel@tonic-gate static int ncpus; 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate /* 927c478bd9Sstevel@tonic-gate * Distinguish spinning for queue locks from spinning for regular locks. 93*5d1dd9a9Sraf * We try harder to acquire queue locks by spinning. 947c478bd9Sstevel@tonic-gate * The environment variable: 957c478bd9Sstevel@tonic-gate * _THREAD_QUEUE_SPIN=count 967c478bd9Sstevel@tonic-gate * can be used to override and set the count in the range [0 .. 1,000,000]. 977c478bd9Sstevel@tonic-gate */ 98*5d1dd9a9Sraf int thread_queue_spin = 10000; 997c478bd9Sstevel@tonic-gate 100883492d5Sraf #define ALL_ATTRIBUTES \ 101883492d5Sraf (LOCK_RECURSIVE | LOCK_ERRORCHECK | \ 102883492d5Sraf LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT | \ 103883492d5Sraf LOCK_ROBUST) 1047c478bd9Sstevel@tonic-gate 1057c478bd9Sstevel@tonic-gate /* 106883492d5Sraf * 'type' can be one of USYNC_THREAD, USYNC_PROCESS, or USYNC_PROCESS_ROBUST, 107883492d5Sraf * augmented by zero or more the flags: 108883492d5Sraf * LOCK_RECURSIVE 109883492d5Sraf * LOCK_ERRORCHECK 110883492d5Sraf * LOCK_PRIO_INHERIT 111883492d5Sraf * LOCK_PRIO_PROTECT 112883492d5Sraf * LOCK_ROBUST 1137c478bd9Sstevel@tonic-gate */ 1147c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_init = __mutex_init 1157c478bd9Sstevel@tonic-gate #pragma weak mutex_init = __mutex_init 1167c478bd9Sstevel@tonic-gate #pragma weak _mutex_init = __mutex_init 1177c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 1187c478bd9Sstevel@tonic-gate int 1197c478bd9Sstevel@tonic-gate __mutex_init(mutex_t *mp, int type, void *arg) 1207c478bd9Sstevel@tonic-gate { 121883492d5Sraf int basetype = (type & ~ALL_ATTRIBUTES); 122883492d5Sraf int error = 0; 123883492d5Sraf 124883492d5Sraf if (basetype == USYNC_PROCESS_ROBUST) { 125883492d5Sraf /* 126883492d5Sraf * USYNC_PROCESS_ROBUST is a deprecated historical type. 127883492d5Sraf * We change it into (USYNC_PROCESS | LOCK_ROBUST) but 128883492d5Sraf * retain the USYNC_PROCESS_ROBUST flag so we can return 129883492d5Sraf * ELOCKUNMAPPED when necessary (only USYNC_PROCESS_ROBUST 130883492d5Sraf * mutexes will ever draw ELOCKUNMAPPED). 131883492d5Sraf */ 132883492d5Sraf type |= (USYNC_PROCESS | LOCK_ROBUST); 133883492d5Sraf basetype = USYNC_PROCESS; 134883492d5Sraf } 1357c478bd9Sstevel@tonic-gate 136883492d5Sraf if (!(basetype == USYNC_THREAD || basetype == USYNC_PROCESS) || 137883492d5Sraf (type & (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) 138883492d5Sraf == (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) { 139883492d5Sraf error = EINVAL; 140883492d5Sraf } else if (type & LOCK_ROBUST) { 141883492d5Sraf /* 142883492d5Sraf * Callers of mutex_init() with the LOCK_ROBUST attribute 143883492d5Sraf * are required to pass an initially all-zero mutex. 144883492d5Sraf * Multiple calls to mutex_init() are allowed; all but 145883492d5Sraf * the first return EBUSY. A call to mutex_init() is 146883492d5Sraf * allowed to make an inconsistent robust lock consistent 147883492d5Sraf * (for historical usage, even though the proper interface 148883492d5Sraf * for this is mutex_consistent()). Note that we use 149883492d5Sraf * atomic_or_16() to set the LOCK_INITED flag so as 150883492d5Sraf * not to disturb surrounding bits (LOCK_OWNERDEAD, etc). 151883492d5Sraf */ 152883492d5Sraf extern void _atomic_or_16(volatile uint16_t *, uint16_t); 153883492d5Sraf if (!(mp->mutex_flag & LOCK_INITED)) { 154883492d5Sraf mp->mutex_type = (uint8_t)type; 155883492d5Sraf _atomic_or_16(&mp->mutex_flag, LOCK_INITED); 156883492d5Sraf mp->mutex_magic = MUTEX_MAGIC; 157883492d5Sraf } else if (type != mp->mutex_type || 158883492d5Sraf ((type & LOCK_PRIO_PROTECT) && 159883492d5Sraf mp->mutex_ceiling != (*(int *)arg))) { 160883492d5Sraf error = EINVAL; 161883492d5Sraf } else if (__mutex_consistent(mp) != 0) { 162883492d5Sraf error = EBUSY; 163883492d5Sraf } 164883492d5Sraf /* register a process robust mutex with the kernel */ 165883492d5Sraf if (basetype == USYNC_PROCESS) 166883492d5Sraf register_lock(mp); 167883492d5Sraf } else { 1687c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 1697c478bd9Sstevel@tonic-gate mp->mutex_type = (uint8_t)type; 1707c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 1717c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 172883492d5Sraf } 173883492d5Sraf 174883492d5Sraf if (error == 0 && (type & LOCK_PRIO_PROTECT)) 175883492d5Sraf mp->mutex_ceiling = (uint8_t)(*(int *)arg); 176883492d5Sraf 1777c478bd9Sstevel@tonic-gate return (error); 1787c478bd9Sstevel@tonic-gate } 1797c478bd9Sstevel@tonic-gate 1807c478bd9Sstevel@tonic-gate /* 1817c478bd9Sstevel@tonic-gate * Delete mp from list of ceil mutexes owned by curthread. 1827c478bd9Sstevel@tonic-gate * Return 1 if the head of the chain was updated. 1837c478bd9Sstevel@tonic-gate */ 1847c478bd9Sstevel@tonic-gate int 1857c478bd9Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp) 1867c478bd9Sstevel@tonic-gate { 1877c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1887c478bd9Sstevel@tonic-gate mxchain_t **mcpp; 1897c478bd9Sstevel@tonic-gate mxchain_t *mcp; 1907c478bd9Sstevel@tonic-gate 1917c478bd9Sstevel@tonic-gate mcpp = &self->ul_mxchain; 1927c478bd9Sstevel@tonic-gate while ((*mcpp)->mxchain_mx != mp) 1937c478bd9Sstevel@tonic-gate mcpp = &(*mcpp)->mxchain_next; 1947c478bd9Sstevel@tonic-gate mcp = *mcpp; 1957c478bd9Sstevel@tonic-gate *mcpp = mcp->mxchain_next; 1967c478bd9Sstevel@tonic-gate lfree(mcp, sizeof (*mcp)); 1977c478bd9Sstevel@tonic-gate return (mcpp == &self->ul_mxchain); 1987c478bd9Sstevel@tonic-gate } 1997c478bd9Sstevel@tonic-gate 2007c478bd9Sstevel@tonic-gate /* 2017c478bd9Sstevel@tonic-gate * Add mp to head of list of ceil mutexes owned by curthread. 2027c478bd9Sstevel@tonic-gate * Return ENOMEM if no memory could be allocated. 2037c478bd9Sstevel@tonic-gate */ 2047c478bd9Sstevel@tonic-gate int 2057c478bd9Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp) 2067c478bd9Sstevel@tonic-gate { 2077c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2087c478bd9Sstevel@tonic-gate mxchain_t *mcp; 2097c478bd9Sstevel@tonic-gate 2107c478bd9Sstevel@tonic-gate if ((mcp = lmalloc(sizeof (*mcp))) == NULL) 2117c478bd9Sstevel@tonic-gate return (ENOMEM); 2127c478bd9Sstevel@tonic-gate mcp->mxchain_mx = mp; 2137c478bd9Sstevel@tonic-gate mcp->mxchain_next = self->ul_mxchain; 2147c478bd9Sstevel@tonic-gate self->ul_mxchain = mcp; 2157c478bd9Sstevel@tonic-gate return (0); 2167c478bd9Sstevel@tonic-gate } 2177c478bd9Sstevel@tonic-gate 2187c478bd9Sstevel@tonic-gate /* 2197c478bd9Sstevel@tonic-gate * Inherit priority from ceiling. The inheritance impacts the effective 2207c478bd9Sstevel@tonic-gate * priority, not the assigned priority. See _thread_setschedparam_main(). 2217c478bd9Sstevel@tonic-gate */ 2227c478bd9Sstevel@tonic-gate void 2237c478bd9Sstevel@tonic-gate _ceil_prio_inherit(int ceil) 2247c478bd9Sstevel@tonic-gate { 2257c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2267c478bd9Sstevel@tonic-gate struct sched_param param; 2277c478bd9Sstevel@tonic-gate 2287c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 2297c478bd9Sstevel@tonic-gate param.sched_priority = ceil; 2307c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2317c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) { 2327c478bd9Sstevel@tonic-gate /* 2337c478bd9Sstevel@tonic-gate * Panic since unclear what error code to return. 2347c478bd9Sstevel@tonic-gate * If we do return the error codes returned by above 2357c478bd9Sstevel@tonic-gate * called routine, update the man page... 2367c478bd9Sstevel@tonic-gate */ 2377c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2387c478bd9Sstevel@tonic-gate } 2397c478bd9Sstevel@tonic-gate } 2407c478bd9Sstevel@tonic-gate 2417c478bd9Sstevel@tonic-gate /* 2427c478bd9Sstevel@tonic-gate * Waive inherited ceiling priority. Inherit from head of owned ceiling locks 2437c478bd9Sstevel@tonic-gate * if holding at least one ceiling lock. If no ceiling locks are held at this 2447c478bd9Sstevel@tonic-gate * point, disinherit completely, reverting back to assigned priority. 2457c478bd9Sstevel@tonic-gate */ 2467c478bd9Sstevel@tonic-gate void 2477c478bd9Sstevel@tonic-gate _ceil_prio_waive(void) 2487c478bd9Sstevel@tonic-gate { 2497c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2507c478bd9Sstevel@tonic-gate struct sched_param param; 2517c478bd9Sstevel@tonic-gate 2527c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 2537c478bd9Sstevel@tonic-gate if (self->ul_mxchain == NULL) { 2547c478bd9Sstevel@tonic-gate /* 2557c478bd9Sstevel@tonic-gate * No ceil locks held. Zero the epri, revert back to ul_pri. 2567c478bd9Sstevel@tonic-gate * Since thread's hash lock is not held, one cannot just 2577c478bd9Sstevel@tonic-gate * read ul_pri here...do it in the called routine... 2587c478bd9Sstevel@tonic-gate */ 2597c478bd9Sstevel@tonic-gate param.sched_priority = self->ul_pri; /* ignored */ 2607c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2617c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_DISINHERIT)) 2627c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2637c478bd9Sstevel@tonic-gate } else { 2647c478bd9Sstevel@tonic-gate /* 2657c478bd9Sstevel@tonic-gate * Set priority to that of the mutex at the head 2667c478bd9Sstevel@tonic-gate * of the ceilmutex chain. 2677c478bd9Sstevel@tonic-gate */ 2687c478bd9Sstevel@tonic-gate param.sched_priority = 2697c478bd9Sstevel@tonic-gate self->ul_mxchain->mxchain_mx->mutex_ceiling; 2707c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2717c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) 2727c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2737c478bd9Sstevel@tonic-gate } 2747c478bd9Sstevel@tonic-gate } 2757c478bd9Sstevel@tonic-gate 276*5d1dd9a9Sraf /* 277*5d1dd9a9Sraf * Clear the lock byte. Retain the waiters byte and the spinners byte. 278*5d1dd9a9Sraf * Return the old value of the lock word. 279*5d1dd9a9Sraf */ 280*5d1dd9a9Sraf static uint32_t 281*5d1dd9a9Sraf clear_lockbyte(volatile uint32_t *lockword) 282*5d1dd9a9Sraf { 283*5d1dd9a9Sraf uint32_t old; 284*5d1dd9a9Sraf uint32_t new; 285*5d1dd9a9Sraf 286*5d1dd9a9Sraf do { 287*5d1dd9a9Sraf old = *lockword; 288*5d1dd9a9Sraf new = old & ~LOCKMASK; 289*5d1dd9a9Sraf } while (atomic_cas_32(lockword, old, new) != old); 290*5d1dd9a9Sraf 291*5d1dd9a9Sraf return (old); 292*5d1dd9a9Sraf } 293*5d1dd9a9Sraf 294*5d1dd9a9Sraf /* 295*5d1dd9a9Sraf * Increment the spinners count in the mutex lock word. 296*5d1dd9a9Sraf * Return 0 on success. Return -1 if the count would overflow. 297*5d1dd9a9Sraf */ 298*5d1dd9a9Sraf static int 299*5d1dd9a9Sraf spinners_incr(volatile uint32_t *lockword, uint8_t max_spinners) 300*5d1dd9a9Sraf { 301*5d1dd9a9Sraf uint32_t old; 302*5d1dd9a9Sraf uint32_t new; 303*5d1dd9a9Sraf 304*5d1dd9a9Sraf do { 305*5d1dd9a9Sraf old = *lockword; 306*5d1dd9a9Sraf if (((old & SPINNERMASK) >> SPINNERSHIFT) >= max_spinners) 307*5d1dd9a9Sraf return (-1); 308*5d1dd9a9Sraf new = old + (1 << SPINNERSHIFT); 309*5d1dd9a9Sraf } while (atomic_cas_32(lockword, old, new) != old); 310*5d1dd9a9Sraf 311*5d1dd9a9Sraf return (0); 312*5d1dd9a9Sraf } 313*5d1dd9a9Sraf 314*5d1dd9a9Sraf /* 315*5d1dd9a9Sraf * Decrement the spinners count in the mutex lock word. 316*5d1dd9a9Sraf * Return the new value of the lock word. 317*5d1dd9a9Sraf */ 318*5d1dd9a9Sraf static uint32_t 319*5d1dd9a9Sraf spinners_decr(volatile uint32_t *lockword) 320*5d1dd9a9Sraf { 321*5d1dd9a9Sraf uint32_t old; 322*5d1dd9a9Sraf uint32_t new; 323*5d1dd9a9Sraf 324*5d1dd9a9Sraf do { 325*5d1dd9a9Sraf new = old = *lockword; 326*5d1dd9a9Sraf if (new & SPINNERMASK) 327*5d1dd9a9Sraf new -= (1 << SPINNERSHIFT); 328*5d1dd9a9Sraf } while (atomic_cas_32(lockword, old, new) != old); 329*5d1dd9a9Sraf 330*5d1dd9a9Sraf return (new); 331*5d1dd9a9Sraf } 332*5d1dd9a9Sraf 3337c478bd9Sstevel@tonic-gate /* 3347c478bd9Sstevel@tonic-gate * Non-preemptive spin locks. Used by queue_lock(). 3357c478bd9Sstevel@tonic-gate * No lock statistics are gathered for these locks. 336*5d1dd9a9Sraf * No DTrace probes are provided for these locks. 3377c478bd9Sstevel@tonic-gate */ 3387c478bd9Sstevel@tonic-gate void 3397c478bd9Sstevel@tonic-gate spin_lock_set(mutex_t *mp) 3407c478bd9Sstevel@tonic-gate { 3417c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3427c478bd9Sstevel@tonic-gate 3437c478bd9Sstevel@tonic-gate no_preempt(self); 3447c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 3457c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 3467c478bd9Sstevel@tonic-gate return; 3477c478bd9Sstevel@tonic-gate } 3487c478bd9Sstevel@tonic-gate /* 3497c478bd9Sstevel@tonic-gate * Spin for a while, attempting to acquire the lock. 3507c478bd9Sstevel@tonic-gate */ 3517c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin != UINT_MAX) 3527c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin++; 3537c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 3547c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 3557c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 3567c478bd9Sstevel@tonic-gate return; 3577c478bd9Sstevel@tonic-gate } 3587c478bd9Sstevel@tonic-gate /* 3597c478bd9Sstevel@tonic-gate * Try harder if we were previously at a no premption level. 3607c478bd9Sstevel@tonic-gate */ 3617c478bd9Sstevel@tonic-gate if (self->ul_preempt > 1) { 3627c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin2 != UINT_MAX) 3637c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin2++; 3647c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 3657c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 3667c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 3677c478bd9Sstevel@tonic-gate return; 3687c478bd9Sstevel@tonic-gate } 3697c478bd9Sstevel@tonic-gate } 3707c478bd9Sstevel@tonic-gate /* 3717c478bd9Sstevel@tonic-gate * Give up and block in the kernel for the mutex. 3727c478bd9Sstevel@tonic-gate */ 3737c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_sleep != UINT_MAX) 3747c478bd9Sstevel@tonic-gate self->ul_spin_lock_sleep++; 3757c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_timedlock(mp, NULL); 3767c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 3777c478bd9Sstevel@tonic-gate } 3787c478bd9Sstevel@tonic-gate 3797c478bd9Sstevel@tonic-gate void 3807c478bd9Sstevel@tonic-gate spin_lock_clear(mutex_t *mp) 3817c478bd9Sstevel@tonic-gate { 3827c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3837c478bd9Sstevel@tonic-gate 3847c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 38541efec22Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) { 386883492d5Sraf (void) ___lwp_mutex_wakeup(mp, 0); 3877c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_wakeup != UINT_MAX) 3887c478bd9Sstevel@tonic-gate self->ul_spin_lock_wakeup++; 3897c478bd9Sstevel@tonic-gate } 3907c478bd9Sstevel@tonic-gate preempt(self); 3917c478bd9Sstevel@tonic-gate } 3927c478bd9Sstevel@tonic-gate 3937c478bd9Sstevel@tonic-gate /* 3947c478bd9Sstevel@tonic-gate * Allocate the sleep queue hash table. 3957c478bd9Sstevel@tonic-gate */ 3967c478bd9Sstevel@tonic-gate void 3977c478bd9Sstevel@tonic-gate queue_alloc(void) 3987c478bd9Sstevel@tonic-gate { 3997c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 4007c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 401883492d5Sraf mutex_t *mp; 4027c478bd9Sstevel@tonic-gate void *data; 4037c478bd9Sstevel@tonic-gate int i; 4047c478bd9Sstevel@tonic-gate 4057c478bd9Sstevel@tonic-gate /* 4067c478bd9Sstevel@tonic-gate * No locks are needed; we call here only when single-threaded. 4077c478bd9Sstevel@tonic-gate */ 4087c478bd9Sstevel@tonic-gate ASSERT(self == udp->ulwp_one); 4097c478bd9Sstevel@tonic-gate ASSERT(!udp->uberflags.uf_mt); 4107c478bd9Sstevel@tonic-gate if ((data = _private_mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t), 4117c478bd9Sstevel@tonic-gate PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0)) 4127c478bd9Sstevel@tonic-gate == MAP_FAILED) 4137c478bd9Sstevel@tonic-gate thr_panic("cannot allocate thread queue_head table"); 4147c478bd9Sstevel@tonic-gate udp->queue_head = (queue_head_t *)data; 415883492d5Sraf for (i = 0; i < 2 * QHASHSIZE; i++) { 416883492d5Sraf mp = &udp->queue_head[i].qh_lock; 417883492d5Sraf mp->mutex_flag = LOCK_INITED; 418883492d5Sraf mp->mutex_magic = MUTEX_MAGIC; 419883492d5Sraf } 4207c478bd9Sstevel@tonic-gate } 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 4237c478bd9Sstevel@tonic-gate 4247c478bd9Sstevel@tonic-gate /* 4257c478bd9Sstevel@tonic-gate * Debugging: verify correctness of a sleep queue. 4267c478bd9Sstevel@tonic-gate */ 4277c478bd9Sstevel@tonic-gate void 4287c478bd9Sstevel@tonic-gate QVERIFY(queue_head_t *qp) 4297c478bd9Sstevel@tonic-gate { 4307c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 4317c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 4327c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 4337c478bd9Sstevel@tonic-gate ulwp_t *prev; 4347c478bd9Sstevel@tonic-gate uint_t index; 4357c478bd9Sstevel@tonic-gate uint32_t cnt = 0; 4367c478bd9Sstevel@tonic-gate char qtype; 4377c478bd9Sstevel@tonic-gate void *wchan; 4387c478bd9Sstevel@tonic-gate 4397c478bd9Sstevel@tonic-gate ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE); 4407c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 4417c478bd9Sstevel@tonic-gate ASSERT((qp->qh_head != NULL && qp->qh_tail != NULL) || 442*5d1dd9a9Sraf (qp->qh_head == NULL && qp->qh_tail == NULL)); 4437c478bd9Sstevel@tonic-gate if (!thread_queue_verify) 4447c478bd9Sstevel@tonic-gate return; 4457c478bd9Sstevel@tonic-gate /* real expensive stuff, only for _THREAD_QUEUE_VERIFY */ 4467c478bd9Sstevel@tonic-gate qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV; 4477c478bd9Sstevel@tonic-gate for (prev = NULL, ulwp = qp->qh_head; ulwp != NULL; 4487c478bd9Sstevel@tonic-gate prev = ulwp, ulwp = ulwp->ul_link, cnt++) { 4497c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_qtype == qtype); 4507c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_wchan != NULL); 4517c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq == qp); 4527c478bd9Sstevel@tonic-gate wchan = ulwp->ul_wchan; 4537c478bd9Sstevel@tonic-gate index = QUEUE_HASH(wchan, qtype); 4547c478bd9Sstevel@tonic-gate ASSERT(&udp->queue_head[index] == qp); 4557c478bd9Sstevel@tonic-gate } 4567c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == prev); 4577c478bd9Sstevel@tonic-gate ASSERT(qp->qh_qlen == cnt); 4587c478bd9Sstevel@tonic-gate } 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate #else /* THREAD_DEBUG */ 4617c478bd9Sstevel@tonic-gate 4627c478bd9Sstevel@tonic-gate #define QVERIFY(qp) 4637c478bd9Sstevel@tonic-gate 4647c478bd9Sstevel@tonic-gate #endif /* THREAD_DEBUG */ 4657c478bd9Sstevel@tonic-gate 4667c478bd9Sstevel@tonic-gate /* 4677c478bd9Sstevel@tonic-gate * Acquire a queue head. 4687c478bd9Sstevel@tonic-gate */ 4697c478bd9Sstevel@tonic-gate queue_head_t * 4707c478bd9Sstevel@tonic-gate queue_lock(void *wchan, int qtype) 4717c478bd9Sstevel@tonic-gate { 4727c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 4737c478bd9Sstevel@tonic-gate queue_head_t *qp; 4747c478bd9Sstevel@tonic-gate 4757c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 4767c478bd9Sstevel@tonic-gate 4777c478bd9Sstevel@tonic-gate /* 4787c478bd9Sstevel@tonic-gate * It is possible that we could be called while still single-threaded. 4797c478bd9Sstevel@tonic-gate * If so, we call queue_alloc() to allocate the queue_head[] array. 4807c478bd9Sstevel@tonic-gate */ 4817c478bd9Sstevel@tonic-gate if ((qp = udp->queue_head) == NULL) { 4827c478bd9Sstevel@tonic-gate queue_alloc(); 4837c478bd9Sstevel@tonic-gate qp = udp->queue_head; 4847c478bd9Sstevel@tonic-gate } 4857c478bd9Sstevel@tonic-gate qp += QUEUE_HASH(wchan, qtype); 4867c478bd9Sstevel@tonic-gate spin_lock_set(&qp->qh_lock); 4877c478bd9Sstevel@tonic-gate /* 4887c478bd9Sstevel@tonic-gate * At once per nanosecond, qh_lockcount will wrap after 512 years. 4897c478bd9Sstevel@tonic-gate * Were we to care about this, we could peg the value at UINT64_MAX. 4907c478bd9Sstevel@tonic-gate */ 4917c478bd9Sstevel@tonic-gate qp->qh_lockcount++; 4927c478bd9Sstevel@tonic-gate QVERIFY(qp); 4937c478bd9Sstevel@tonic-gate return (qp); 4947c478bd9Sstevel@tonic-gate } 4957c478bd9Sstevel@tonic-gate 4967c478bd9Sstevel@tonic-gate /* 4977c478bd9Sstevel@tonic-gate * Release a queue head. 4987c478bd9Sstevel@tonic-gate */ 4997c478bd9Sstevel@tonic-gate void 5007c478bd9Sstevel@tonic-gate queue_unlock(queue_head_t *qp) 5017c478bd9Sstevel@tonic-gate { 5027c478bd9Sstevel@tonic-gate QVERIFY(qp); 5037c478bd9Sstevel@tonic-gate spin_lock_clear(&qp->qh_lock); 5047c478bd9Sstevel@tonic-gate } 5057c478bd9Sstevel@tonic-gate 5067c478bd9Sstevel@tonic-gate /* 5077c478bd9Sstevel@tonic-gate * For rwlock queueing, we must queue writers ahead of readers of the 5087c478bd9Sstevel@tonic-gate * same priority. We do this by making writers appear to have a half 5097c478bd9Sstevel@tonic-gate * point higher priority for purposes of priority comparisons below. 5107c478bd9Sstevel@tonic-gate */ 5117c478bd9Sstevel@tonic-gate #define CMP_PRIO(ulwp) ((real_priority(ulwp) << 1) + (ulwp)->ul_writer) 5127c478bd9Sstevel@tonic-gate 5137c478bd9Sstevel@tonic-gate void 5147c478bd9Sstevel@tonic-gate enqueue(queue_head_t *qp, ulwp_t *ulwp, void *wchan, int qtype) 5157c478bd9Sstevel@tonic-gate { 5167c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 5177c478bd9Sstevel@tonic-gate ulwp_t *next; 5187c478bd9Sstevel@tonic-gate int pri = CMP_PRIO(ulwp); 5197c478bd9Sstevel@tonic-gate int force_fifo = (qtype & FIFOQ); 5207c478bd9Sstevel@tonic-gate int do_fifo; 5217c478bd9Sstevel@tonic-gate 5227c478bd9Sstevel@tonic-gate qtype &= ~FIFOQ; 5237c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 5247c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 5257c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq != qp); 5267c478bd9Sstevel@tonic-gate 5277c478bd9Sstevel@tonic-gate /* 5287c478bd9Sstevel@tonic-gate * LIFO queue ordering is unfair and can lead to starvation, 5297c478bd9Sstevel@tonic-gate * but it gives better performance for heavily contended locks. 5307c478bd9Sstevel@tonic-gate * We use thread_queue_fifo (range is 0..8) to determine 5317c478bd9Sstevel@tonic-gate * the frequency of FIFO vs LIFO queuing: 5327c478bd9Sstevel@tonic-gate * 0 : every 256th time (almost always LIFO) 5337c478bd9Sstevel@tonic-gate * 1 : every 128th time 5347c478bd9Sstevel@tonic-gate * 2 : every 64th time 5357c478bd9Sstevel@tonic-gate * 3 : every 32nd time 5367c478bd9Sstevel@tonic-gate * 4 : every 16th time (the default value, mostly LIFO) 5377c478bd9Sstevel@tonic-gate * 5 : every 8th time 5387c478bd9Sstevel@tonic-gate * 6 : every 4th time 5397c478bd9Sstevel@tonic-gate * 7 : every 2nd time 5407c478bd9Sstevel@tonic-gate * 8 : every time (never LIFO, always FIFO) 5417c478bd9Sstevel@tonic-gate * Note that there is always some degree of FIFO ordering. 5427c478bd9Sstevel@tonic-gate * This breaks live lock conditions that occur in applications 5437c478bd9Sstevel@tonic-gate * that are written assuming (incorrectly) that threads acquire 5447c478bd9Sstevel@tonic-gate * locks fairly, that is, in roughly round-robin order. 5457c478bd9Sstevel@tonic-gate * In any event, the queue is maintained in priority order. 5467c478bd9Sstevel@tonic-gate * 5477c478bd9Sstevel@tonic-gate * If we are given the FIFOQ flag in qtype, fifo queueing is forced. 5487c478bd9Sstevel@tonic-gate * SUSV3 requires this for semaphores. 5497c478bd9Sstevel@tonic-gate */ 5507c478bd9Sstevel@tonic-gate do_fifo = (force_fifo || 551*5d1dd9a9Sraf ((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0); 5527c478bd9Sstevel@tonic-gate 5537c478bd9Sstevel@tonic-gate if (qp->qh_head == NULL) { 5547c478bd9Sstevel@tonic-gate /* 5557c478bd9Sstevel@tonic-gate * The queue is empty. LIFO/FIFO doesn't matter. 5567c478bd9Sstevel@tonic-gate */ 5577c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == NULL); 5587c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 5597c478bd9Sstevel@tonic-gate } else if (do_fifo) { 5607c478bd9Sstevel@tonic-gate /* 5617c478bd9Sstevel@tonic-gate * Enqueue after the last thread whose priority is greater 5627c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 5637c478bd9Sstevel@tonic-gate * Attempt first to go directly onto the tail of the queue. 5647c478bd9Sstevel@tonic-gate */ 5657c478bd9Sstevel@tonic-gate if (pri <= CMP_PRIO(qp->qh_tail)) 5667c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_tail->ul_link; 5677c478bd9Sstevel@tonic-gate else { 5687c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 5697c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 5707c478bd9Sstevel@tonic-gate if (pri > CMP_PRIO(next)) 5717c478bd9Sstevel@tonic-gate break; 5727c478bd9Sstevel@tonic-gate } 5737c478bd9Sstevel@tonic-gate } else { 5747c478bd9Sstevel@tonic-gate /* 5757c478bd9Sstevel@tonic-gate * Enqueue before the first thread whose priority is less 5767c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 5777c478bd9Sstevel@tonic-gate * Hopefully we can go directly onto the head of the queue. 5787c478bd9Sstevel@tonic-gate */ 5797c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 5807c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 5817c478bd9Sstevel@tonic-gate if (pri >= CMP_PRIO(next)) 5827c478bd9Sstevel@tonic-gate break; 5837c478bd9Sstevel@tonic-gate } 5847c478bd9Sstevel@tonic-gate if ((ulwp->ul_link = *ulwpp) == NULL) 5857c478bd9Sstevel@tonic-gate qp->qh_tail = ulwp; 5867c478bd9Sstevel@tonic-gate *ulwpp = ulwp; 5877c478bd9Sstevel@tonic-gate 5887c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = qp; 5897c478bd9Sstevel@tonic-gate ulwp->ul_wchan = wchan; 5907c478bd9Sstevel@tonic-gate ulwp->ul_qtype = qtype; 5917c478bd9Sstevel@tonic-gate if (qp->qh_qmax < ++qp->qh_qlen) 5927c478bd9Sstevel@tonic-gate qp->qh_qmax = qp->qh_qlen; 5937c478bd9Sstevel@tonic-gate } 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate /* 5967c478bd9Sstevel@tonic-gate * Return a pointer to the queue slot of the 5977c478bd9Sstevel@tonic-gate * highest priority thread on the queue. 5987c478bd9Sstevel@tonic-gate * On return, prevp, if not NULL, will contain a pointer 5997c478bd9Sstevel@tonic-gate * to the thread's predecessor on the queue 6007c478bd9Sstevel@tonic-gate */ 6017c478bd9Sstevel@tonic-gate static ulwp_t ** 6027c478bd9Sstevel@tonic-gate queue_slot(queue_head_t *qp, void *wchan, int *more, ulwp_t **prevp) 6037c478bd9Sstevel@tonic-gate { 6047c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 6057c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 6067c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 6077c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 6087c478bd9Sstevel@tonic-gate ulwp_t *susprev; 6097c478bd9Sstevel@tonic-gate 6107c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 6117c478bd9Sstevel@tonic-gate 6127c478bd9Sstevel@tonic-gate /* 6137c478bd9Sstevel@tonic-gate * Find a waiter on the sleep queue. 6147c478bd9Sstevel@tonic-gate */ 6157c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 6167c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 6177c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 6187c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 6197c478bd9Sstevel@tonic-gate break; 6207c478bd9Sstevel@tonic-gate /* 6217c478bd9Sstevel@tonic-gate * Try not to return a suspended thread. 6227c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 6237c478bd9Sstevel@tonic-gate */ 6247c478bd9Sstevel@tonic-gate if (suspp == NULL) { 6257c478bd9Sstevel@tonic-gate suspp = ulwpp; 6267c478bd9Sstevel@tonic-gate susprev = prev; 6277c478bd9Sstevel@tonic-gate } 6287c478bd9Sstevel@tonic-gate } 6297c478bd9Sstevel@tonic-gate } 6307c478bd9Sstevel@tonic-gate 6317c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 6327c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 6337c478bd9Sstevel@tonic-gate prev = susprev; 6347c478bd9Sstevel@tonic-gate suspp = NULL; 6357c478bd9Sstevel@tonic-gate } 6367c478bd9Sstevel@tonic-gate if (ulwp == NULL) { 6377c478bd9Sstevel@tonic-gate if (more != NULL) 6387c478bd9Sstevel@tonic-gate *more = 0; 6397c478bd9Sstevel@tonic-gate return (NULL); 6407c478bd9Sstevel@tonic-gate } 6417c478bd9Sstevel@tonic-gate 6427c478bd9Sstevel@tonic-gate if (prevp != NULL) 6437c478bd9Sstevel@tonic-gate *prevp = prev; 6447c478bd9Sstevel@tonic-gate if (more == NULL) 6457c478bd9Sstevel@tonic-gate return (ulwpp); 6467c478bd9Sstevel@tonic-gate 6477c478bd9Sstevel@tonic-gate /* 6487c478bd9Sstevel@tonic-gate * Scan the remainder of the queue for another waiter. 6497c478bd9Sstevel@tonic-gate */ 6507c478bd9Sstevel@tonic-gate if (suspp != NULL) { 6517c478bd9Sstevel@tonic-gate *more = 1; 6527c478bd9Sstevel@tonic-gate return (ulwpp); 6537c478bd9Sstevel@tonic-gate } 6547c478bd9Sstevel@tonic-gate for (ulwp = ulwp->ul_link; ulwp != NULL; ulwp = ulwp->ul_link) { 6557c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 6567c478bd9Sstevel@tonic-gate *more = 1; 6577c478bd9Sstevel@tonic-gate return (ulwpp); 6587c478bd9Sstevel@tonic-gate } 6597c478bd9Sstevel@tonic-gate } 6607c478bd9Sstevel@tonic-gate 6617c478bd9Sstevel@tonic-gate *more = 0; 6627c478bd9Sstevel@tonic-gate return (ulwpp); 6637c478bd9Sstevel@tonic-gate } 6647c478bd9Sstevel@tonic-gate 6657c478bd9Sstevel@tonic-gate ulwp_t * 66641efec22Sraf queue_unlink(queue_head_t *qp, ulwp_t **ulwpp, ulwp_t *prev) 6677c478bd9Sstevel@tonic-gate { 6687c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 6697c478bd9Sstevel@tonic-gate 6707c478bd9Sstevel@tonic-gate ulwp = *ulwpp; 6717c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 6727c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 6737c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 6747c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 6757c478bd9Sstevel@tonic-gate qp->qh_qlen--; 6767c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 6777c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 6787c478bd9Sstevel@tonic-gate 6797c478bd9Sstevel@tonic-gate return (ulwp); 6807c478bd9Sstevel@tonic-gate } 6817c478bd9Sstevel@tonic-gate 68241efec22Sraf ulwp_t * 68341efec22Sraf dequeue(queue_head_t *qp, void *wchan, int *more) 68441efec22Sraf { 68541efec22Sraf ulwp_t **ulwpp; 68641efec22Sraf ulwp_t *prev; 68741efec22Sraf 68841efec22Sraf if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL) 68941efec22Sraf return (NULL); 69041efec22Sraf return (queue_unlink(qp, ulwpp, prev)); 69141efec22Sraf } 69241efec22Sraf 6937c478bd9Sstevel@tonic-gate /* 6947c478bd9Sstevel@tonic-gate * Return a pointer to the highest priority thread sleeping on wchan. 6957c478bd9Sstevel@tonic-gate */ 6967c478bd9Sstevel@tonic-gate ulwp_t * 6977c478bd9Sstevel@tonic-gate queue_waiter(queue_head_t *qp, void *wchan) 6987c478bd9Sstevel@tonic-gate { 6997c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate if ((ulwpp = queue_slot(qp, wchan, NULL, NULL)) == NULL) 7027c478bd9Sstevel@tonic-gate return (NULL); 7037c478bd9Sstevel@tonic-gate return (*ulwpp); 7047c478bd9Sstevel@tonic-gate } 7057c478bd9Sstevel@tonic-gate 7067c478bd9Sstevel@tonic-gate uint8_t 7077c478bd9Sstevel@tonic-gate dequeue_self(queue_head_t *qp, void *wchan) 7087c478bd9Sstevel@tonic-gate { 7097c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 7107c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 7117c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 7127c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 7137c478bd9Sstevel@tonic-gate int found = 0; 7147c478bd9Sstevel@tonic-gate int more = 0; 7157c478bd9Sstevel@tonic-gate 7167c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 7177c478bd9Sstevel@tonic-gate 7187c478bd9Sstevel@tonic-gate /* find self on the sleep queue */ 7197c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 7207c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 7217c478bd9Sstevel@tonic-gate if (ulwp == self) { 7227c478bd9Sstevel@tonic-gate /* dequeue ourself */ 7237c478bd9Sstevel@tonic-gate ASSERT(self->ul_wchan == wchan); 72441efec22Sraf (void) queue_unlink(qp, ulwpp, prev); 7257c478bd9Sstevel@tonic-gate self->ul_cvmutex = NULL; 7267c478bd9Sstevel@tonic-gate self->ul_cv_wake = 0; 7277c478bd9Sstevel@tonic-gate found = 1; 7287c478bd9Sstevel@tonic-gate break; 7297c478bd9Sstevel@tonic-gate } 7307c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 7317c478bd9Sstevel@tonic-gate more = 1; 7327c478bd9Sstevel@tonic-gate } 7337c478bd9Sstevel@tonic-gate 7347c478bd9Sstevel@tonic-gate if (!found) 7357c478bd9Sstevel@tonic-gate thr_panic("dequeue_self(): curthread not found on queue"); 7367c478bd9Sstevel@tonic-gate 7377c478bd9Sstevel@tonic-gate if (more) 7387c478bd9Sstevel@tonic-gate return (1); 7397c478bd9Sstevel@tonic-gate 7407c478bd9Sstevel@tonic-gate /* scan the remainder of the queue for another waiter */ 7417c478bd9Sstevel@tonic-gate for (ulwp = *ulwpp; ulwp != NULL; ulwp = ulwp->ul_link) { 7427c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 7437c478bd9Sstevel@tonic-gate return (1); 7447c478bd9Sstevel@tonic-gate } 7457c478bd9Sstevel@tonic-gate 7467c478bd9Sstevel@tonic-gate return (0); 7477c478bd9Sstevel@tonic-gate } 7487c478bd9Sstevel@tonic-gate 7497c478bd9Sstevel@tonic-gate /* 7507c478bd9Sstevel@tonic-gate * Called from call_user_handler() and _thrp_suspend() to take 7517c478bd9Sstevel@tonic-gate * ourself off of our sleep queue so we can grab locks. 7527c478bd9Sstevel@tonic-gate */ 7537c478bd9Sstevel@tonic-gate void 7547c478bd9Sstevel@tonic-gate unsleep_self(void) 7557c478bd9Sstevel@tonic-gate { 7567c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 7577c478bd9Sstevel@tonic-gate queue_head_t *qp; 7587c478bd9Sstevel@tonic-gate 7597c478bd9Sstevel@tonic-gate /* 7607c478bd9Sstevel@tonic-gate * Calling enter_critical()/exit_critical() here would lead 7617c478bd9Sstevel@tonic-gate * to recursion. Just manipulate self->ul_critical directly. 7627c478bd9Sstevel@tonic-gate */ 7637c478bd9Sstevel@tonic-gate self->ul_critical++; 7647c478bd9Sstevel@tonic-gate while (self->ul_sleepq != NULL) { 7657c478bd9Sstevel@tonic-gate qp = queue_lock(self->ul_wchan, self->ul_qtype); 7667c478bd9Sstevel@tonic-gate /* 7677c478bd9Sstevel@tonic-gate * We may have been moved from a CV queue to a 7687c478bd9Sstevel@tonic-gate * mutex queue while we were attempting queue_lock(). 7697c478bd9Sstevel@tonic-gate * If so, just loop around and try again. 7707c478bd9Sstevel@tonic-gate * dequeue_self() clears self->ul_sleepq. 7717c478bd9Sstevel@tonic-gate */ 77241efec22Sraf if (qp == self->ul_sleepq) { 7737c478bd9Sstevel@tonic-gate (void) dequeue_self(qp, self->ul_wchan); 77441efec22Sraf self->ul_writer = 0; 77541efec22Sraf } 7767c478bd9Sstevel@tonic-gate queue_unlock(qp); 7777c478bd9Sstevel@tonic-gate } 7787c478bd9Sstevel@tonic-gate self->ul_critical--; 7797c478bd9Sstevel@tonic-gate } 7807c478bd9Sstevel@tonic-gate 7817c478bd9Sstevel@tonic-gate /* 7827c478bd9Sstevel@tonic-gate * Common code for calling the the ___lwp_mutex_timedlock() system call. 7837c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 7847c478bd9Sstevel@tonic-gate */ 785883492d5Sraf static int 7867c478bd9Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp) 7877c478bd9Sstevel@tonic-gate { 7887c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 7897c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 790883492d5Sraf int mtype = mp->mutex_type; 7917c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 792883492d5Sraf int acquired; 7937c478bd9Sstevel@tonic-gate int error; 7947c478bd9Sstevel@tonic-gate 7957c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 7967c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 7977c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 7987c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 7997c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 8007c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 8017c478bd9Sstevel@tonic-gate } 8027c478bd9Sstevel@tonic-gate if (msp) { 8037c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 8047c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 8057c478bd9Sstevel@tonic-gate } 8067c478bd9Sstevel@tonic-gate 8077c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 8087c478bd9Sstevel@tonic-gate 8097c478bd9Sstevel@tonic-gate for (;;) { 810883492d5Sraf /* 811883492d5Sraf * A return value of EOWNERDEAD or ELOCKUNMAPPED 812883492d5Sraf * means we successfully acquired the lock. 813883492d5Sraf */ 814883492d5Sraf if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0 && 815883492d5Sraf error != EOWNERDEAD && error != ELOCKUNMAPPED) { 816883492d5Sraf acquired = 0; 8177c478bd9Sstevel@tonic-gate break; 8187c478bd9Sstevel@tonic-gate } 8197c478bd9Sstevel@tonic-gate 820883492d5Sraf if (mtype & USYNC_PROCESS) { 8217c478bd9Sstevel@tonic-gate /* 8227c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 8237c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 8247c478bd9Sstevel@tonic-gate */ 8257c478bd9Sstevel@tonic-gate enter_critical(self); 8267c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 8277c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 8287c478bd9Sstevel@tonic-gate exit_critical(self); 829883492d5Sraf acquired = 1; 8307c478bd9Sstevel@tonic-gate break; 8317c478bd9Sstevel@tonic-gate } 8327c478bd9Sstevel@tonic-gate exit_critical(self); 8337c478bd9Sstevel@tonic-gate } else { 8347c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 835883492d5Sraf acquired = 1; 8367c478bd9Sstevel@tonic-gate break; 8377c478bd9Sstevel@tonic-gate } 8387c478bd9Sstevel@tonic-gate } 8397c478bd9Sstevel@tonic-gate if (msp) 8407c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 8417c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 8427c478bd9Sstevel@tonic-gate self->ul_sp = 0; 8437c478bd9Sstevel@tonic-gate 844883492d5Sraf if (acquired) { 845883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 846883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 847883492d5Sraf } else { 848883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 849883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 850883492d5Sraf } 851883492d5Sraf 8527c478bd9Sstevel@tonic-gate return (error); 8537c478bd9Sstevel@tonic-gate } 8547c478bd9Sstevel@tonic-gate 8557c478bd9Sstevel@tonic-gate /* 8567c478bd9Sstevel@tonic-gate * Common code for calling the ___lwp_mutex_trylock() system call. 8577c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 8587c478bd9Sstevel@tonic-gate */ 8597c478bd9Sstevel@tonic-gate int 8607c478bd9Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp) 8617c478bd9Sstevel@tonic-gate { 8627c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 8637c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 864883492d5Sraf int mtype = mp->mutex_type; 8657c478bd9Sstevel@tonic-gate int error; 866883492d5Sraf int acquired; 8677c478bd9Sstevel@tonic-gate 8687c478bd9Sstevel@tonic-gate for (;;) { 869883492d5Sraf /* 870883492d5Sraf * A return value of EOWNERDEAD or ELOCKUNMAPPED 871883492d5Sraf * means we successfully acquired the lock. 872883492d5Sraf */ 873883492d5Sraf if ((error = ___lwp_mutex_trylock(mp)) != 0 && 874883492d5Sraf error != EOWNERDEAD && error != ELOCKUNMAPPED) { 875883492d5Sraf acquired = 0; 8767c478bd9Sstevel@tonic-gate break; 8777c478bd9Sstevel@tonic-gate } 8787c478bd9Sstevel@tonic-gate 879883492d5Sraf if (mtype & USYNC_PROCESS) { 8807c478bd9Sstevel@tonic-gate /* 8817c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 8827c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 8837c478bd9Sstevel@tonic-gate */ 8847c478bd9Sstevel@tonic-gate enter_critical(self); 8857c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 8867c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 8877c478bd9Sstevel@tonic-gate exit_critical(self); 888883492d5Sraf acquired = 1; 8897c478bd9Sstevel@tonic-gate break; 8907c478bd9Sstevel@tonic-gate } 8917c478bd9Sstevel@tonic-gate exit_critical(self); 8927c478bd9Sstevel@tonic-gate } else { 8937c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 894883492d5Sraf acquired = 1; 8957c478bd9Sstevel@tonic-gate break; 8967c478bd9Sstevel@tonic-gate } 8977c478bd9Sstevel@tonic-gate } 8987c478bd9Sstevel@tonic-gate 899883492d5Sraf if (acquired) { 900883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 901883492d5Sraf } else if (error != EBUSY) { 902883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 903883492d5Sraf } 904883492d5Sraf 9057c478bd9Sstevel@tonic-gate return (error); 9067c478bd9Sstevel@tonic-gate } 9077c478bd9Sstevel@tonic-gate 9087c478bd9Sstevel@tonic-gate volatile sc_shared_t * 9097c478bd9Sstevel@tonic-gate setup_schedctl(void) 9107c478bd9Sstevel@tonic-gate { 9117c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 9127c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 9137c478bd9Sstevel@tonic-gate sc_shared_t *tmp; 9147c478bd9Sstevel@tonic-gate 9157c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */ 9167c478bd9Sstevel@tonic-gate !self->ul_vfork && /* not a child of vfork() */ 9177c478bd9Sstevel@tonic-gate !self->ul_schedctl_called) { /* haven't been called before */ 9187c478bd9Sstevel@tonic-gate enter_critical(self); 9197c478bd9Sstevel@tonic-gate self->ul_schedctl_called = &self->ul_uberdata->uberflags; 9207c478bd9Sstevel@tonic-gate if ((tmp = __schedctl()) != (sc_shared_t *)(-1)) 9217c478bd9Sstevel@tonic-gate self->ul_schedctl = scp = tmp; 9227c478bd9Sstevel@tonic-gate exit_critical(self); 9237c478bd9Sstevel@tonic-gate } 9247c478bd9Sstevel@tonic-gate /* 9257c478bd9Sstevel@tonic-gate * Unless the call to setup_schedctl() is surrounded 9267c478bd9Sstevel@tonic-gate * by enter_critical()/exit_critical(), the address 9277c478bd9Sstevel@tonic-gate * we are returning could be invalid due to a forkall() 9287c478bd9Sstevel@tonic-gate * having occurred in another thread. 9297c478bd9Sstevel@tonic-gate */ 9307c478bd9Sstevel@tonic-gate return (scp); 9317c478bd9Sstevel@tonic-gate } 9327c478bd9Sstevel@tonic-gate 9337c478bd9Sstevel@tonic-gate /* 9347c478bd9Sstevel@tonic-gate * Interfaces from libsched, incorporated into libc. 9357c478bd9Sstevel@tonic-gate * libsched.so.1 is now a filter library onto libc. 9367c478bd9Sstevel@tonic-gate */ 9377c478bd9Sstevel@tonic-gate #pragma weak schedctl_lookup = _schedctl_init 9387c478bd9Sstevel@tonic-gate #pragma weak _schedctl_lookup = _schedctl_init 9397c478bd9Sstevel@tonic-gate #pragma weak schedctl_init = _schedctl_init 9407c478bd9Sstevel@tonic-gate schedctl_t * 9417c478bd9Sstevel@tonic-gate _schedctl_init(void) 9427c478bd9Sstevel@tonic-gate { 9437c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = setup_schedctl(); 9447c478bd9Sstevel@tonic-gate return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl); 9457c478bd9Sstevel@tonic-gate } 9467c478bd9Sstevel@tonic-gate 9477c478bd9Sstevel@tonic-gate #pragma weak schedctl_exit = _schedctl_exit 9487c478bd9Sstevel@tonic-gate void 9497c478bd9Sstevel@tonic-gate _schedctl_exit(void) 9507c478bd9Sstevel@tonic-gate { 9517c478bd9Sstevel@tonic-gate } 9527c478bd9Sstevel@tonic-gate 9537c478bd9Sstevel@tonic-gate /* 9547c478bd9Sstevel@tonic-gate * Contract private interface for java. 9557c478bd9Sstevel@tonic-gate * Set up the schedctl data if it doesn't exist yet. 9567c478bd9Sstevel@tonic-gate * Return a pointer to the pointer to the schedctl data. 9577c478bd9Sstevel@tonic-gate */ 9587c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile * 9597c478bd9Sstevel@tonic-gate _thr_schedctl(void) 9607c478bd9Sstevel@tonic-gate { 9617c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 9627c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile *ptr; 9637c478bd9Sstevel@tonic-gate 9647c478bd9Sstevel@tonic-gate if (self->ul_vfork) 9657c478bd9Sstevel@tonic-gate return (NULL); 9667c478bd9Sstevel@tonic-gate if (*(ptr = &self->ul_schedctl) == NULL) 9677c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 9687c478bd9Sstevel@tonic-gate return (ptr); 9697c478bd9Sstevel@tonic-gate } 9707c478bd9Sstevel@tonic-gate 9717c478bd9Sstevel@tonic-gate /* 9727c478bd9Sstevel@tonic-gate * Block signals and attempt to block preemption. 9737c478bd9Sstevel@tonic-gate * no_preempt()/preempt() must be used in pairs but can be nested. 9747c478bd9Sstevel@tonic-gate */ 9757c478bd9Sstevel@tonic-gate void 9767c478bd9Sstevel@tonic-gate no_preempt(ulwp_t *self) 9777c478bd9Sstevel@tonic-gate { 9787c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 9797c478bd9Sstevel@tonic-gate 9807c478bd9Sstevel@tonic-gate if (self->ul_preempt++ == 0) { 9817c478bd9Sstevel@tonic-gate enter_critical(self); 9827c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL || 9837c478bd9Sstevel@tonic-gate (scp = setup_schedctl()) != NULL) { 9847c478bd9Sstevel@tonic-gate /* 9857c478bd9Sstevel@tonic-gate * Save the pre-existing preempt value. 9867c478bd9Sstevel@tonic-gate */ 9877c478bd9Sstevel@tonic-gate self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt; 9887c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = 1; 9897c478bd9Sstevel@tonic-gate } 9907c478bd9Sstevel@tonic-gate } 9917c478bd9Sstevel@tonic-gate } 9927c478bd9Sstevel@tonic-gate 9937c478bd9Sstevel@tonic-gate /* 9947c478bd9Sstevel@tonic-gate * Undo the effects of no_preempt(). 9957c478bd9Sstevel@tonic-gate */ 9967c478bd9Sstevel@tonic-gate void 9977c478bd9Sstevel@tonic-gate preempt(ulwp_t *self) 9987c478bd9Sstevel@tonic-gate { 9997c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 10007c478bd9Sstevel@tonic-gate 10017c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt > 0); 10027c478bd9Sstevel@tonic-gate if (--self->ul_preempt == 0) { 10037c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL) { 10047c478bd9Sstevel@tonic-gate /* 10057c478bd9Sstevel@tonic-gate * Restore the pre-existing preempt value. 10067c478bd9Sstevel@tonic-gate */ 10077c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt; 10087c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield && 10097c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt == 0) { 10107c478bd9Sstevel@tonic-gate lwp_yield(); 10117c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield) { 10127c478bd9Sstevel@tonic-gate /* 10137c478bd9Sstevel@tonic-gate * Shouldn't happen. This is either 10147c478bd9Sstevel@tonic-gate * a race condition or the thread 10157c478bd9Sstevel@tonic-gate * just entered the real-time class. 10167c478bd9Sstevel@tonic-gate */ 10177c478bd9Sstevel@tonic-gate lwp_yield(); 10187c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_yield = 0; 10197c478bd9Sstevel@tonic-gate } 10207c478bd9Sstevel@tonic-gate } 10217c478bd9Sstevel@tonic-gate } 10227c478bd9Sstevel@tonic-gate exit_critical(self); 10237c478bd9Sstevel@tonic-gate } 10247c478bd9Sstevel@tonic-gate } 10257c478bd9Sstevel@tonic-gate 10267c478bd9Sstevel@tonic-gate /* 10277c478bd9Sstevel@tonic-gate * If a call to preempt() would cause the current thread to yield or to 10287c478bd9Sstevel@tonic-gate * take deferred actions in exit_critical(), then unpark the specified 10297c478bd9Sstevel@tonic-gate * lwp so it can run while we delay. Return the original lwpid if the 10307c478bd9Sstevel@tonic-gate * unpark was not performed, else return zero. The tests are a repeat 10317c478bd9Sstevel@tonic-gate * of some of the tests in preempt(), above. This is a statistical 10327c478bd9Sstevel@tonic-gate * optimization solely for cond_sleep_queue(), below. 10337c478bd9Sstevel@tonic-gate */ 10347c478bd9Sstevel@tonic-gate static lwpid_t 10357c478bd9Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid) 10367c478bd9Sstevel@tonic-gate { 10377c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = self->ul_schedctl; 10387c478bd9Sstevel@tonic-gate 10397c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt == 1 && self->ul_critical > 0); 10407c478bd9Sstevel@tonic-gate if ((scp != NULL && scp->sc_preemptctl.sc_yield) || 10417c478bd9Sstevel@tonic-gate (self->ul_curplease && self->ul_critical == 1)) { 10427c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 10437c478bd9Sstevel@tonic-gate lwpid = 0; 10447c478bd9Sstevel@tonic-gate } 10457c478bd9Sstevel@tonic-gate return (lwpid); 10467c478bd9Sstevel@tonic-gate } 10477c478bd9Sstevel@tonic-gate 10487c478bd9Sstevel@tonic-gate /* 104916b01779Sraf * Spin for a while (if 'tryhard' is true), trying to grab the lock. 10507c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 10517c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread. 10527c478bd9Sstevel@tonic-gate */ 1053883492d5Sraf static int 105416b01779Sraf mutex_trylock_adaptive(mutex_t *mp, int tryhard) 10557c478bd9Sstevel@tonic-gate { 10567c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1057883492d5Sraf int error = EBUSY; 10587c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 10597c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 1060*5d1dd9a9Sraf volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw; 1061*5d1dd9a9Sraf volatile uint64_t *ownerp = (volatile uint64_t *)&mp->mutex_owner; 1062*5d1dd9a9Sraf uint32_t new_lockword; 1063*5d1dd9a9Sraf int count = 0; 1064*5d1dd9a9Sraf int max_count; 1065*5d1dd9a9Sraf uint8_t max_spinners; 10667c478bd9Sstevel@tonic-gate 1067883492d5Sraf ASSERT(!(mp->mutex_type & USYNC_PROCESS)); 10687c478bd9Sstevel@tonic-gate 1069883492d5Sraf if (MUTEX_OWNER(mp) == self) 10707c478bd9Sstevel@tonic-gate return (EBUSY); 10717c478bd9Sstevel@tonic-gate 1072883492d5Sraf /* short-cut, not definitive (see below) */ 1073883492d5Sraf if (mp->mutex_flag & LOCK_NOTRECOVERABLE) { 1074883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1075*5d1dd9a9Sraf error = ENOTRECOVERABLE; 1076*5d1dd9a9Sraf goto done; 1077883492d5Sraf } 1078883492d5Sraf 1079*5d1dd9a9Sraf /* 1080*5d1dd9a9Sraf * Make one attempt to acquire the lock before 1081*5d1dd9a9Sraf * incurring the overhead of the spin loop. 1082*5d1dd9a9Sraf */ 1083*5d1dd9a9Sraf if (set_lock_byte(lockp) == 0) { 1084*5d1dd9a9Sraf *ownerp = (uintptr_t)self; 1085*5d1dd9a9Sraf error = 0; 1086*5d1dd9a9Sraf goto done; 1087*5d1dd9a9Sraf } 1088*5d1dd9a9Sraf if (!tryhard) 1089*5d1dd9a9Sraf goto done; 1090*5d1dd9a9Sraf if (ncpus == 0) 1091*5d1dd9a9Sraf ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 1092*5d1dd9a9Sraf if ((max_spinners = self->ul_max_spinners) >= ncpus) 1093*5d1dd9a9Sraf max_spinners = ncpus - 1; 1094*5d1dd9a9Sraf max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0; 1095*5d1dd9a9Sraf if (max_count == 0) 1096*5d1dd9a9Sraf goto done; 10977c478bd9Sstevel@tonic-gate 10987c478bd9Sstevel@tonic-gate /* 10997c478bd9Sstevel@tonic-gate * This spin loop is unfair to lwps that have already dropped into 11007c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a highly-contended mutex. 11017c478bd9Sstevel@tonic-gate * This is just too bad. The adaptive spin algorithm is intended 11027c478bd9Sstevel@tonic-gate * to allow programs with highly-contended locks (that is, broken 11037c478bd9Sstevel@tonic-gate * programs) to execute with reasonable speed despite their contention. 11047c478bd9Sstevel@tonic-gate * Being fair would reduce the speed of such programs and well-written 11057c478bd9Sstevel@tonic-gate * programs will not suffer in any case. 11067c478bd9Sstevel@tonic-gate */ 1107*5d1dd9a9Sraf enter_critical(self); 1108*5d1dd9a9Sraf if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1) { 1109*5d1dd9a9Sraf exit_critical(self); 1110*5d1dd9a9Sraf goto done; 1111*5d1dd9a9Sraf } 1112*5d1dd9a9Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp); 1113*5d1dd9a9Sraf for (count = 1; ; count++) { 11147c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) { 11157c478bd9Sstevel@tonic-gate *ownerp = (uintptr_t)self; 1116883492d5Sraf error = 0; 1117883492d5Sraf break; 11187c478bd9Sstevel@tonic-gate } 1119*5d1dd9a9Sraf if (count == max_count) 1120*5d1dd9a9Sraf break; 11217c478bd9Sstevel@tonic-gate SMT_PAUSE(); 11227c478bd9Sstevel@tonic-gate /* 11237c478bd9Sstevel@tonic-gate * Stop spinning if the mutex owner is not running on 11247c478bd9Sstevel@tonic-gate * a processor; it will not drop the lock any time soon 11257c478bd9Sstevel@tonic-gate * and we would just be wasting time to keep spinning. 11267c478bd9Sstevel@tonic-gate * 11277c478bd9Sstevel@tonic-gate * Note that we are looking at another thread (ulwp_t) 11287c478bd9Sstevel@tonic-gate * without ensuring that the other thread does not exit. 11297c478bd9Sstevel@tonic-gate * The scheme relies on ulwp_t structures never being 11307c478bd9Sstevel@tonic-gate * deallocated by the library (the library employs a free 11317c478bd9Sstevel@tonic-gate * list of ulwp_t structs that are reused when new threads 11327c478bd9Sstevel@tonic-gate * are created) and on schedctl shared memory never being 11337c478bd9Sstevel@tonic-gate * deallocated once created via __schedctl(). 11347c478bd9Sstevel@tonic-gate * 11357c478bd9Sstevel@tonic-gate * Thus, the worst that can happen when the spinning thread 11367c478bd9Sstevel@tonic-gate * looks at the owner's schedctl data is that it is looking 11377c478bd9Sstevel@tonic-gate * at some other thread's schedctl data. This almost never 11387c478bd9Sstevel@tonic-gate * happens and is benign when it does. 11397c478bd9Sstevel@tonic-gate */ 11407c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 11417c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 11427c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 11437c478bd9Sstevel@tonic-gate break; 11447c478bd9Sstevel@tonic-gate } 1145*5d1dd9a9Sraf new_lockword = spinners_decr(&mp->mutex_lockword); 1146*5d1dd9a9Sraf if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) { 1147*5d1dd9a9Sraf /* 1148*5d1dd9a9Sraf * We haven't yet acquired the lock, the lock 1149*5d1dd9a9Sraf * is free, and there are no other spinners. 1150*5d1dd9a9Sraf * Make one final attempt to acquire the lock. 1151*5d1dd9a9Sraf * 1152*5d1dd9a9Sraf * This isn't strictly necessary since mutex_lock_queue() 1153*5d1dd9a9Sraf * (the next action this thread will take if it doesn't 1154*5d1dd9a9Sraf * acquire the lock here) makes one attempt to acquire 1155*5d1dd9a9Sraf * the lock before putting the thread to sleep. 1156*5d1dd9a9Sraf * 1157*5d1dd9a9Sraf * If the next action for this thread (on failure here) 1158*5d1dd9a9Sraf * were not to call mutex_lock_queue(), this would be 1159*5d1dd9a9Sraf * necessary for correctness, to avoid ending up with an 1160*5d1dd9a9Sraf * unheld mutex with waiters but no one to wake them up. 1161*5d1dd9a9Sraf */ 1162*5d1dd9a9Sraf if (set_lock_byte(lockp) == 0) { 1163*5d1dd9a9Sraf *ownerp = (uintptr_t)self; 1164*5d1dd9a9Sraf error = 0; 1165*5d1dd9a9Sraf } 1166*5d1dd9a9Sraf count++; 1167*5d1dd9a9Sraf } 11687c478bd9Sstevel@tonic-gate exit_critical(self); 11697c478bd9Sstevel@tonic-gate 1170*5d1dd9a9Sraf done: 1171883492d5Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) { 1172883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1173883492d5Sraf /* 1174883492d5Sraf * We shouldn't own the mutex; clear the lock. 1175883492d5Sraf */ 1176883492d5Sraf mp->mutex_owner = 0; 1177*5d1dd9a9Sraf if (clear_lockbyte(&mp->mutex_lockword) & WAITERMASK) 1178883492d5Sraf mutex_wakeup_all(mp); 1179883492d5Sraf error = ENOTRECOVERABLE; 1180883492d5Sraf } 11817c478bd9Sstevel@tonic-gate 1182883492d5Sraf if (error) { 1183*5d1dd9a9Sraf if (count) { 1184*5d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 0, count); 1185*5d1dd9a9Sraf } 1186883492d5Sraf if (error != EBUSY) { 1187883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1188883492d5Sraf } 1189883492d5Sraf } else { 1190*5d1dd9a9Sraf if (count) { 1191*5d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 1192*5d1dd9a9Sraf } 1193883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 1194883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) { 1195883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1196883492d5Sraf error = EOWNERDEAD; 1197883492d5Sraf } 1198883492d5Sraf } 1199883492d5Sraf 1200883492d5Sraf return (error); 12017c478bd9Sstevel@tonic-gate } 12027c478bd9Sstevel@tonic-gate 12037c478bd9Sstevel@tonic-gate /* 12047c478bd9Sstevel@tonic-gate * Same as mutex_trylock_adaptive(), except specifically for queue locks. 12057c478bd9Sstevel@tonic-gate * The owner field is not set here; the caller (spin_lock_set()) sets it. 12067c478bd9Sstevel@tonic-gate */ 1207883492d5Sraf static int 12087c478bd9Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp) 12097c478bd9Sstevel@tonic-gate { 12107c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 12117c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 12127c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 12137c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 12147c478bd9Sstevel@tonic-gate int count = curthread->ul_queue_spin; 12157c478bd9Sstevel@tonic-gate 12167c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 12177c478bd9Sstevel@tonic-gate 12187c478bd9Sstevel@tonic-gate if (count == 0) 12197c478bd9Sstevel@tonic-gate return (EBUSY); 12207c478bd9Sstevel@tonic-gate 12217c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 12227c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 12237c478bd9Sstevel@tonic-gate while (--count >= 0) { 12247c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) 12257c478bd9Sstevel@tonic-gate return (0); 12267c478bd9Sstevel@tonic-gate SMT_PAUSE(); 12277c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 12287c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 12297c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 12307c478bd9Sstevel@tonic-gate break; 12317c478bd9Sstevel@tonic-gate } 12327c478bd9Sstevel@tonic-gate 12337c478bd9Sstevel@tonic-gate return (EBUSY); 12347c478bd9Sstevel@tonic-gate } 12357c478bd9Sstevel@tonic-gate 12367c478bd9Sstevel@tonic-gate /* 12377c478bd9Sstevel@tonic-gate * Like mutex_trylock_adaptive(), but for process-shared mutexes. 123816b01779Sraf * Spin for a while (if 'tryhard' is true), trying to grab the lock. 12397c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 12407c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread 12417c478bd9Sstevel@tonic-gate * and mutex_ownerpid set to the current pid. 12427c478bd9Sstevel@tonic-gate */ 1243883492d5Sraf static int 124416b01779Sraf mutex_trylock_process(mutex_t *mp, int tryhard) 12457c478bd9Sstevel@tonic-gate { 12467c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1247*5d1dd9a9Sraf uberdata_t *udp = self->ul_uberdata; 1248883492d5Sraf int error = EBUSY; 1249*5d1dd9a9Sraf volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw; 1250*5d1dd9a9Sraf uint32_t new_lockword; 1251*5d1dd9a9Sraf int count = 0; 1252*5d1dd9a9Sraf int max_count; 1253*5d1dd9a9Sraf uint8_t max_spinners; 12547c478bd9Sstevel@tonic-gate 1255883492d5Sraf ASSERT(mp->mutex_type & USYNC_PROCESS); 12567c478bd9Sstevel@tonic-gate 1257883492d5Sraf if (shared_mutex_held(mp)) 12587c478bd9Sstevel@tonic-gate return (EBUSY); 12597c478bd9Sstevel@tonic-gate 1260883492d5Sraf /* short-cut, not definitive (see below) */ 1261883492d5Sraf if (mp->mutex_flag & LOCK_NOTRECOVERABLE) { 1262883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1263*5d1dd9a9Sraf error = ENOTRECOVERABLE; 1264*5d1dd9a9Sraf goto done; 1265883492d5Sraf } 1266883492d5Sraf 1267*5d1dd9a9Sraf /* 1268*5d1dd9a9Sraf * Make one attempt to acquire the lock before 1269*5d1dd9a9Sraf * incurring the overhead of the spin loop. 1270*5d1dd9a9Sraf */ 1271*5d1dd9a9Sraf enter_critical(self); 1272*5d1dd9a9Sraf if (set_lock_byte(lockp) == 0) { 1273*5d1dd9a9Sraf mp->mutex_owner = (uintptr_t)self; 1274*5d1dd9a9Sraf mp->mutex_ownerpid = udp->pid; 1275*5d1dd9a9Sraf exit_critical(self); 1276*5d1dd9a9Sraf error = 0; 1277*5d1dd9a9Sraf goto done; 1278*5d1dd9a9Sraf } 1279*5d1dd9a9Sraf exit_critical(self); 1280*5d1dd9a9Sraf if (!tryhard) 1281*5d1dd9a9Sraf goto done; 1282883492d5Sraf if (ncpus == 0) 1283883492d5Sraf ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 1284*5d1dd9a9Sraf if ((max_spinners = self->ul_max_spinners) >= ncpus) 1285*5d1dd9a9Sraf max_spinners = ncpus - 1; 1286*5d1dd9a9Sraf max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0; 1287*5d1dd9a9Sraf if (max_count == 0) 1288*5d1dd9a9Sraf goto done; 1289883492d5Sraf 12907c478bd9Sstevel@tonic-gate /* 12917c478bd9Sstevel@tonic-gate * This is a process-shared mutex. 12927c478bd9Sstevel@tonic-gate * We cannot know if the owner is running on a processor. 12937c478bd9Sstevel@tonic-gate * We just spin and hope that it is on a processor. 12947c478bd9Sstevel@tonic-gate */ 1295883492d5Sraf enter_critical(self); 1296*5d1dd9a9Sraf if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1) { 1297*5d1dd9a9Sraf exit_critical(self); 1298*5d1dd9a9Sraf goto done; 1299*5d1dd9a9Sraf } 1300*5d1dd9a9Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp); 1301*5d1dd9a9Sraf for (count = 1; ; count++) { 1302883492d5Sraf if (*lockp == 0 && set_lock_byte(lockp) == 0) { 1303883492d5Sraf mp->mutex_owner = (uintptr_t)self; 1304*5d1dd9a9Sraf mp->mutex_ownerpid = udp->pid; 1305883492d5Sraf error = 0; 1306883492d5Sraf break; 13077c478bd9Sstevel@tonic-gate } 1308*5d1dd9a9Sraf if (count == max_count) 1309*5d1dd9a9Sraf break; 1310883492d5Sraf SMT_PAUSE(); 1311883492d5Sraf } 1312*5d1dd9a9Sraf new_lockword = spinners_decr(&mp->mutex_lockword); 1313*5d1dd9a9Sraf if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) { 1314*5d1dd9a9Sraf /* 1315*5d1dd9a9Sraf * We haven't yet acquired the lock, the lock 1316*5d1dd9a9Sraf * is free, and there are no other spinners. 1317*5d1dd9a9Sraf * Make one final attempt to acquire the lock. 1318*5d1dd9a9Sraf * 1319*5d1dd9a9Sraf * This isn't strictly necessary since mutex_lock_kernel() 1320*5d1dd9a9Sraf * (the next action this thread will take if it doesn't 1321*5d1dd9a9Sraf * acquire the lock here) makes one attempt to acquire 1322*5d1dd9a9Sraf * the lock before putting the thread to sleep. 1323*5d1dd9a9Sraf * 1324*5d1dd9a9Sraf * If the next action for this thread (on failure here) 1325*5d1dd9a9Sraf * were not to call mutex_lock_kernel(), this would be 1326*5d1dd9a9Sraf * necessary for correctness, to avoid ending up with an 1327*5d1dd9a9Sraf * unheld mutex with waiters but no one to wake them up. 1328*5d1dd9a9Sraf */ 1329*5d1dd9a9Sraf if (set_lock_byte(lockp) == 0) { 1330*5d1dd9a9Sraf mp->mutex_owner = (uintptr_t)self; 1331*5d1dd9a9Sraf mp->mutex_ownerpid = udp->pid; 1332*5d1dd9a9Sraf error = 0; 1333*5d1dd9a9Sraf } 1334*5d1dd9a9Sraf count++; 1335*5d1dd9a9Sraf } 1336883492d5Sraf exit_critical(self); 1337883492d5Sraf 1338*5d1dd9a9Sraf done: 1339883492d5Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) { 1340883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 13417c478bd9Sstevel@tonic-gate /* 1342883492d5Sraf * We shouldn't own the mutex; clear the lock. 13437c478bd9Sstevel@tonic-gate */ 1344883492d5Sraf mp->mutex_owner = 0; 1345883492d5Sraf mp->mutex_ownerpid = 0; 1346*5d1dd9a9Sraf if (clear_lockbyte(&mp->mutex_lockword) & WAITERMASK) { 1347883492d5Sraf no_preempt(self); 1348883492d5Sraf (void) ___lwp_mutex_wakeup(mp, 1); 1349883492d5Sraf preempt(self); 1350883492d5Sraf } 1351883492d5Sraf error = ENOTRECOVERABLE; 13527c478bd9Sstevel@tonic-gate } 13537c478bd9Sstevel@tonic-gate 1354883492d5Sraf if (error) { 1355*5d1dd9a9Sraf if (count) { 1356*5d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 0, count); 1357*5d1dd9a9Sraf } 1358883492d5Sraf if (error != EBUSY) { 1359883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1360883492d5Sraf } 1361883492d5Sraf } else { 1362*5d1dd9a9Sraf if (count) { 1363*5d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 1364*5d1dd9a9Sraf } 1365883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 1366883492d5Sraf if (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) { 1367883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1368883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) 1369883492d5Sraf error = EOWNERDEAD; 1370883492d5Sraf else if (mp->mutex_type & USYNC_PROCESS_ROBUST) 1371883492d5Sraf error = ELOCKUNMAPPED; 1372883492d5Sraf else 1373883492d5Sraf error = EOWNERDEAD; 1374883492d5Sraf } 1375883492d5Sraf } 1376883492d5Sraf 1377883492d5Sraf return (error); 13787c478bd9Sstevel@tonic-gate } 13797c478bd9Sstevel@tonic-gate 13807c478bd9Sstevel@tonic-gate /* 13817c478bd9Sstevel@tonic-gate * Mutex wakeup code for releasing a USYNC_THREAD mutex. 13827c478bd9Sstevel@tonic-gate * Returns the lwpid of the thread that was dequeued, if any. 13837c478bd9Sstevel@tonic-gate * The caller of mutex_wakeup() must call __lwp_unpark(lwpid) 13847c478bd9Sstevel@tonic-gate * to wake up the specified lwp. 13857c478bd9Sstevel@tonic-gate */ 1386883492d5Sraf static lwpid_t 13877c478bd9Sstevel@tonic-gate mutex_wakeup(mutex_t *mp) 13887c478bd9Sstevel@tonic-gate { 13897c478bd9Sstevel@tonic-gate lwpid_t lwpid = 0; 13907c478bd9Sstevel@tonic-gate queue_head_t *qp; 13917c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 13927c478bd9Sstevel@tonic-gate int more; 13937c478bd9Sstevel@tonic-gate 13947c478bd9Sstevel@tonic-gate /* 13957c478bd9Sstevel@tonic-gate * Dequeue a waiter from the sleep queue. Don't touch the mutex 13967c478bd9Sstevel@tonic-gate * waiters bit if no one was found on the queue because the mutex 13977c478bd9Sstevel@tonic-gate * might have been deallocated or reallocated for another purpose. 13987c478bd9Sstevel@tonic-gate */ 13997c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 14007c478bd9Sstevel@tonic-gate if ((ulwp = dequeue(qp, mp, &more)) != NULL) { 14017c478bd9Sstevel@tonic-gate lwpid = ulwp->ul_lwpid; 14027c478bd9Sstevel@tonic-gate mp->mutex_waiters = (more? 1 : 0); 14037c478bd9Sstevel@tonic-gate } 14047c478bd9Sstevel@tonic-gate queue_unlock(qp); 14057c478bd9Sstevel@tonic-gate return (lwpid); 14067c478bd9Sstevel@tonic-gate } 14077c478bd9Sstevel@tonic-gate 1408883492d5Sraf /* 1409883492d5Sraf * Mutex wakeup code for releasing all waiters on a USYNC_THREAD mutex. 1410883492d5Sraf */ 1411883492d5Sraf static void 1412883492d5Sraf mutex_wakeup_all(mutex_t *mp) 1413883492d5Sraf { 1414883492d5Sraf queue_head_t *qp; 1415883492d5Sraf int nlwpid = 0; 1416883492d5Sraf int maxlwps = MAXLWPS; 1417883492d5Sraf ulwp_t **ulwpp; 1418883492d5Sraf ulwp_t *ulwp; 1419883492d5Sraf ulwp_t *prev = NULL; 1420883492d5Sraf lwpid_t buffer[MAXLWPS]; 1421883492d5Sraf lwpid_t *lwpid = buffer; 1422883492d5Sraf 1423883492d5Sraf /* 1424883492d5Sraf * Walk the list of waiters and prepare to wake up all of them. 1425883492d5Sraf * The waiters flag has already been cleared from the mutex. 1426883492d5Sraf * 1427883492d5Sraf * We keep track of lwpids that are to be unparked in lwpid[]. 1428883492d5Sraf * __lwp_unpark_all() is called to unpark all of them after 1429883492d5Sraf * they have been removed from the sleep queue and the sleep 1430883492d5Sraf * queue lock has been dropped. If we run out of space in our 1431883492d5Sraf * on-stack buffer, we need to allocate more but we can't call 1432883492d5Sraf * lmalloc() because we are holding a queue lock when the overflow 1433883492d5Sraf * occurs and lmalloc() acquires a lock. We can't use alloca() 1434883492d5Sraf * either because the application may have allocated a small 1435883492d5Sraf * stack and we don't want to overrun the stack. So we call 1436883492d5Sraf * alloc_lwpids() to allocate a bigger buffer using the mmap() 1437883492d5Sraf * system call directly since that path acquires no locks. 1438883492d5Sraf */ 1439883492d5Sraf qp = queue_lock(mp, MX); 1440883492d5Sraf ulwpp = &qp->qh_head; 1441883492d5Sraf while ((ulwp = *ulwpp) != NULL) { 1442883492d5Sraf if (ulwp->ul_wchan != mp) { 1443883492d5Sraf prev = ulwp; 1444883492d5Sraf ulwpp = &ulwp->ul_link; 1445883492d5Sraf } else { 1446883492d5Sraf if (nlwpid == maxlwps) 1447883492d5Sraf lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps); 1448883492d5Sraf (void) queue_unlink(qp, ulwpp, prev); 1449883492d5Sraf lwpid[nlwpid++] = ulwp->ul_lwpid; 1450883492d5Sraf } 1451883492d5Sraf } 1452883492d5Sraf 1453883492d5Sraf if (nlwpid == 0) { 1454883492d5Sraf queue_unlock(qp); 1455883492d5Sraf } else { 1456*5d1dd9a9Sraf mp->mutex_waiters = 0; 1457883492d5Sraf no_preempt(curthread); 1458883492d5Sraf queue_unlock(qp); 1459883492d5Sraf if (nlwpid == 1) 1460883492d5Sraf (void) __lwp_unpark(lwpid[0]); 1461883492d5Sraf else 1462883492d5Sraf (void) __lwp_unpark_all(lwpid, nlwpid); 1463883492d5Sraf preempt(curthread); 1464883492d5Sraf } 1465883492d5Sraf 1466883492d5Sraf if (lwpid != buffer) 1467883492d5Sraf (void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t)); 1468883492d5Sraf } 1469883492d5Sraf 14707c478bd9Sstevel@tonic-gate /* 1471*5d1dd9a9Sraf * Release a process-private mutex. 1472*5d1dd9a9Sraf * As an optimization, if there are waiters but there are also spinners 1473*5d1dd9a9Sraf * attempting to acquire the mutex, then don't bother waking up a waiter; 1474*5d1dd9a9Sraf * one of the spinners will acquire the mutex soon and it would be a waste 1475*5d1dd9a9Sraf * of resources to wake up some thread just to have it spin for a while 1476*5d1dd9a9Sraf * and then possibly go back to sleep. See mutex_trylock_adaptive(). 14777c478bd9Sstevel@tonic-gate */ 1478883492d5Sraf static lwpid_t 1479883492d5Sraf mutex_unlock_queue(mutex_t *mp, int release_all) 14807c478bd9Sstevel@tonic-gate { 1481*5d1dd9a9Sraf lwpid_t lwpid = 0; 1482*5d1dd9a9Sraf uint32_t old_lockword; 14837c478bd9Sstevel@tonic-gate 1484*5d1dd9a9Sraf mp->mutex_owner = 0; 1485*5d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 1486*5d1dd9a9Sraf old_lockword = clear_lockbyte(&mp->mutex_lockword); 1487*5d1dd9a9Sraf if ((old_lockword & WAITERMASK) && 1488*5d1dd9a9Sraf (release_all || (old_lockword & SPINNERMASK) == 0)) { 1489*5d1dd9a9Sraf ulwp_t *self = curthread; 14907c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 1491*5d1dd9a9Sraf if (release_all) 1492*5d1dd9a9Sraf mutex_wakeup_all(mp); 1493*5d1dd9a9Sraf else 1494*5d1dd9a9Sraf lwpid = mutex_wakeup(mp); 1495*5d1dd9a9Sraf if (lwpid == 0) 1496*5d1dd9a9Sraf preempt(self); 1497883492d5Sraf } 14987c478bd9Sstevel@tonic-gate return (lwpid); 14997c478bd9Sstevel@tonic-gate } 15007c478bd9Sstevel@tonic-gate 15017c478bd9Sstevel@tonic-gate /* 15027c478bd9Sstevel@tonic-gate * Like mutex_unlock_queue(), but for process-shared mutexes. 15037c478bd9Sstevel@tonic-gate */ 1504883492d5Sraf static void 1505883492d5Sraf mutex_unlock_process(mutex_t *mp, int release_all) 15067c478bd9Sstevel@tonic-gate { 1507*5d1dd9a9Sraf uint32_t old_lockword; 15087c478bd9Sstevel@tonic-gate 15097c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 15107c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 15117c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 1512*5d1dd9a9Sraf old_lockword = clear_lockbyte(&mp->mutex_lockword); 1513*5d1dd9a9Sraf if ((old_lockword & WAITERMASK) && 1514*5d1dd9a9Sraf (release_all || (old_lockword & SPINNERMASK) == 0)) { 1515*5d1dd9a9Sraf ulwp_t *self = curthread; 1516*5d1dd9a9Sraf no_preempt(self); /* ensure a prompt wakeup */ 1517*5d1dd9a9Sraf (void) ___lwp_mutex_wakeup(mp, release_all); 1518*5d1dd9a9Sraf preempt(self); 15197c478bd9Sstevel@tonic-gate } 15207c478bd9Sstevel@tonic-gate } 15217c478bd9Sstevel@tonic-gate 15227c478bd9Sstevel@tonic-gate /* 15237c478bd9Sstevel@tonic-gate * Return the real priority of a thread. 15247c478bd9Sstevel@tonic-gate */ 15257c478bd9Sstevel@tonic-gate int 15267c478bd9Sstevel@tonic-gate real_priority(ulwp_t *ulwp) 15277c478bd9Sstevel@tonic-gate { 15287c478bd9Sstevel@tonic-gate if (ulwp->ul_epri == 0) 15297c478bd9Sstevel@tonic-gate return (ulwp->ul_mappedpri? ulwp->ul_mappedpri : ulwp->ul_pri); 15307c478bd9Sstevel@tonic-gate return (ulwp->ul_emappedpri? ulwp->ul_emappedpri : ulwp->ul_epri); 15317c478bd9Sstevel@tonic-gate } 15327c478bd9Sstevel@tonic-gate 15337c478bd9Sstevel@tonic-gate void 15347c478bd9Sstevel@tonic-gate stall(void) 15357c478bd9Sstevel@tonic-gate { 15367c478bd9Sstevel@tonic-gate for (;;) 15377c478bd9Sstevel@tonic-gate (void) mutex_lock_kernel(&stall_mutex, NULL, NULL); 15387c478bd9Sstevel@tonic-gate } 15397c478bd9Sstevel@tonic-gate 15407c478bd9Sstevel@tonic-gate /* 15417c478bd9Sstevel@tonic-gate * Acquire a USYNC_THREAD mutex via user-level sleep queues. 15427c478bd9Sstevel@tonic-gate * We failed set_lock_byte(&mp->mutex_lockw) before coming here. 1543883492d5Sraf * If successful, returns with mutex_owner set correctly. 15447c478bd9Sstevel@tonic-gate */ 15457c478bd9Sstevel@tonic-gate int 15467c478bd9Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp, 15477c478bd9Sstevel@tonic-gate timespec_t *tsp) 15487c478bd9Sstevel@tonic-gate { 15497c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 15507c478bd9Sstevel@tonic-gate queue_head_t *qp; 15517c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 15527c478bd9Sstevel@tonic-gate int error = 0; 15537c478bd9Sstevel@tonic-gate 15547c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 15557c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 15567c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 15577c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 15587c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 15597c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 15607c478bd9Sstevel@tonic-gate } 15617c478bd9Sstevel@tonic-gate if (msp) { 15627c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 15637c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 15647c478bd9Sstevel@tonic-gate } 15657c478bd9Sstevel@tonic-gate 15667c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 15677c478bd9Sstevel@tonic-gate 15687c478bd9Sstevel@tonic-gate /* 15697c478bd9Sstevel@tonic-gate * Put ourself on the sleep queue, and while we are 15707c478bd9Sstevel@tonic-gate * unable to grab the lock, go park in the kernel. 15717c478bd9Sstevel@tonic-gate * Take ourself off the sleep queue after we acquire the lock. 15727c478bd9Sstevel@tonic-gate * The waiter bit can be set/cleared only while holding the queue lock. 15737c478bd9Sstevel@tonic-gate */ 15747c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 15757c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 15767c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 15777c478bd9Sstevel@tonic-gate for (;;) { 15787c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 15797c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15807c478bd9Sstevel@tonic-gate mp->mutex_waiters = dequeue_self(qp, mp); 15817c478bd9Sstevel@tonic-gate break; 15827c478bd9Sstevel@tonic-gate } 15837c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 15847c478bd9Sstevel@tonic-gate queue_unlock(qp); 15857c478bd9Sstevel@tonic-gate /* 15867c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 15877c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 15887c478bd9Sstevel@tonic-gate */ 1589*5d1dd9a9Sraf error = __lwp_park(tsp, 0); 15907c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 15917c478bd9Sstevel@tonic-gate /* 15927c478bd9Sstevel@tonic-gate * We could have taken a signal or suspended ourself. 15937c478bd9Sstevel@tonic-gate * If we did, then we removed ourself from the queue. 15947c478bd9Sstevel@tonic-gate * Someone else may have removed us from the queue 15957c478bd9Sstevel@tonic-gate * as a consequence of mutex_unlock(). We may have 15967c478bd9Sstevel@tonic-gate * gotten a timeout from __lwp_park(). Or we may still 15977c478bd9Sstevel@tonic-gate * be on the queue and this is just a spurious wakeup. 15987c478bd9Sstevel@tonic-gate */ 15997c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 16007c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) { 1601*5d1dd9a9Sraf if (error) { 1602*5d1dd9a9Sraf mp->mutex_waiters = queue_waiter(qp, mp)? 1 : 0; 1603*5d1dd9a9Sraf if (error != EINTR) 1604*5d1dd9a9Sraf break; 1605*5d1dd9a9Sraf error = 0; 1606*5d1dd9a9Sraf } 16077c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 16087c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 16097c478bd9Sstevel@tonic-gate break; 16107c478bd9Sstevel@tonic-gate } 16117c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 16127c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 16137c478bd9Sstevel@tonic-gate } 16147c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == qp && 16157c478bd9Sstevel@tonic-gate self->ul_qtype == MX && 16167c478bd9Sstevel@tonic-gate self->ul_wchan == mp); 16177c478bd9Sstevel@tonic-gate if (error) { 1618*5d1dd9a9Sraf if (error != EINTR) { 1619*5d1dd9a9Sraf mp->mutex_waiters = dequeue_self(qp, mp); 1620*5d1dd9a9Sraf break; 1621*5d1dd9a9Sraf } 1622*5d1dd9a9Sraf error = 0; 16237c478bd9Sstevel@tonic-gate } 16247c478bd9Sstevel@tonic-gate } 16257c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 16267c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 16277c478bd9Sstevel@tonic-gate self->ul_sp = 0; 16287c478bd9Sstevel@tonic-gate queue_unlock(qp); 1629883492d5Sraf 16307c478bd9Sstevel@tonic-gate if (msp) 16317c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 16327c478bd9Sstevel@tonic-gate 16337c478bd9Sstevel@tonic-gate ASSERT(error == 0 || error == EINVAL || error == ETIME); 1634883492d5Sraf 1635883492d5Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) { 1636883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1637883492d5Sraf /* 1638883492d5Sraf * We shouldn't own the mutex; clear the lock. 1639883492d5Sraf */ 1640883492d5Sraf mp->mutex_owner = 0; 1641*5d1dd9a9Sraf if (clear_lockbyte(&mp->mutex_lockword) & WAITERMASK) 1642883492d5Sraf mutex_wakeup_all(mp); 1643883492d5Sraf error = ENOTRECOVERABLE; 1644883492d5Sraf } 1645883492d5Sraf 1646883492d5Sraf if (error) { 1647883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 1648883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1649883492d5Sraf } else { 1650883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 1651883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1652883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) { 1653883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1654883492d5Sraf error = EOWNERDEAD; 1655883492d5Sraf } 1656883492d5Sraf } 1657883492d5Sraf 16587c478bd9Sstevel@tonic-gate return (error); 16597c478bd9Sstevel@tonic-gate } 16607c478bd9Sstevel@tonic-gate 1661883492d5Sraf static int 1662883492d5Sraf mutex_recursion(mutex_t *mp, int mtype, int try) 1663883492d5Sraf { 1664883492d5Sraf ASSERT(mutex_is_held(mp)); 1665883492d5Sraf ASSERT(mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)); 1666883492d5Sraf ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK); 1667883492d5Sraf 1668883492d5Sraf if (mtype & LOCK_RECURSIVE) { 1669883492d5Sraf if (mp->mutex_rcount == RECURSION_MAX) { 1670883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EAGAIN); 1671883492d5Sraf return (EAGAIN); 1672883492d5Sraf } 1673883492d5Sraf mp->mutex_rcount++; 1674883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0); 1675883492d5Sraf return (0); 1676883492d5Sraf } 1677883492d5Sraf if (try == MUTEX_LOCK) { 1678883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 1679883492d5Sraf return (EDEADLK); 1680883492d5Sraf } 1681883492d5Sraf return (EBUSY); 1682883492d5Sraf } 1683883492d5Sraf 1684883492d5Sraf /* 1685883492d5Sraf * Register this USYNC_PROCESS|LOCK_ROBUST mutex with the kernel so 1686883492d5Sraf * it can apply LOCK_OWNERDEAD|LOCK_UNMAPPED if it becomes necessary. 1687883492d5Sraf * We use tdb_hash_lock here and in the synch object tracking code in 1688883492d5Sraf * the tdb_agent.c file. There is no conflict between these two usages. 1689883492d5Sraf */ 1690883492d5Sraf void 1691883492d5Sraf register_lock(mutex_t *mp) 1692883492d5Sraf { 1693883492d5Sraf uberdata_t *udp = curthread->ul_uberdata; 1694883492d5Sraf uint_t hash = LOCK_HASH(mp); 1695883492d5Sraf robust_t *rlp; 1696883492d5Sraf robust_t **rlpp; 1697883492d5Sraf robust_t **table; 1698883492d5Sraf 1699883492d5Sraf if ((table = udp->robustlocks) == NULL) { 1700883492d5Sraf lmutex_lock(&udp->tdb_hash_lock); 1701883492d5Sraf if ((table = udp->robustlocks) == NULL) { 1702883492d5Sraf table = lmalloc(LOCKHASHSZ * sizeof (robust_t *)); 1703883492d5Sraf _membar_producer(); 1704883492d5Sraf udp->robustlocks = table; 1705883492d5Sraf } 1706883492d5Sraf lmutex_unlock(&udp->tdb_hash_lock); 1707883492d5Sraf } 1708883492d5Sraf _membar_consumer(); 1709883492d5Sraf 1710883492d5Sraf /* 1711883492d5Sraf * First search the registered table with no locks held. 1712883492d5Sraf * This is safe because the table never shrinks 1713883492d5Sraf * and we can only get a false negative. 1714883492d5Sraf */ 1715883492d5Sraf for (rlp = table[hash]; rlp != NULL; rlp = rlp->robust_next) { 1716883492d5Sraf if (rlp->robust_lock == mp) /* already registered */ 1717883492d5Sraf return; 1718883492d5Sraf } 1719883492d5Sraf 1720883492d5Sraf /* 1721883492d5Sraf * The lock was not found. 1722883492d5Sraf * Repeat the operation with tdb_hash_lock held. 1723883492d5Sraf */ 1724883492d5Sraf lmutex_lock(&udp->tdb_hash_lock); 1725883492d5Sraf 1726883492d5Sraf for (rlpp = &table[hash]; 1727883492d5Sraf (rlp = *rlpp) != NULL; 1728883492d5Sraf rlpp = &rlp->robust_next) { 1729883492d5Sraf if (rlp->robust_lock == mp) { /* already registered */ 1730883492d5Sraf lmutex_unlock(&udp->tdb_hash_lock); 1731883492d5Sraf return; 1732883492d5Sraf } 1733883492d5Sraf } 1734883492d5Sraf 1735883492d5Sraf /* 1736883492d5Sraf * The lock has never been registered. 1737883492d5Sraf * Register it now and add it to the table. 1738883492d5Sraf */ 1739883492d5Sraf (void) ___lwp_mutex_register(mp); 1740883492d5Sraf rlp = lmalloc(sizeof (*rlp)); 1741883492d5Sraf rlp->robust_lock = mp; 1742883492d5Sraf _membar_producer(); 1743883492d5Sraf *rlpp = rlp; 1744883492d5Sraf 1745883492d5Sraf lmutex_unlock(&udp->tdb_hash_lock); 1746883492d5Sraf } 1747883492d5Sraf 1748883492d5Sraf /* 1749883492d5Sraf * This is called in the child of fork()/forkall() to start over 1750883492d5Sraf * with a clean slate. (Each process must register its own locks.) 1751883492d5Sraf * No locks are needed because all other threads are suspended or gone. 1752883492d5Sraf */ 1753883492d5Sraf void 1754883492d5Sraf unregister_locks(void) 1755883492d5Sraf { 1756883492d5Sraf uberdata_t *udp = curthread->ul_uberdata; 1757883492d5Sraf uint_t hash; 1758883492d5Sraf robust_t **table; 1759883492d5Sraf robust_t *rlp; 1760883492d5Sraf robust_t *next; 1761883492d5Sraf 1762883492d5Sraf if ((table = udp->robustlocks) != NULL) { 1763883492d5Sraf for (hash = 0; hash < LOCKHASHSZ; hash++) { 1764883492d5Sraf rlp = table[hash]; 1765883492d5Sraf while (rlp != NULL) { 1766883492d5Sraf next = rlp->robust_next; 1767883492d5Sraf lfree(rlp, sizeof (*rlp)); 1768883492d5Sraf rlp = next; 1769883492d5Sraf } 1770883492d5Sraf } 1771883492d5Sraf lfree(table, LOCKHASHSZ * sizeof (robust_t *)); 1772883492d5Sraf udp->robustlocks = NULL; 1773883492d5Sraf } 1774883492d5Sraf } 1775883492d5Sraf 17767c478bd9Sstevel@tonic-gate /* 17777c478bd9Sstevel@tonic-gate * Returns with mutex_owner set correctly. 17787c478bd9Sstevel@tonic-gate */ 1779883492d5Sraf static int 17807c478bd9Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try) 17817c478bd9Sstevel@tonic-gate { 17827c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 17837c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 17847c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 17857c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 17867c478bd9Sstevel@tonic-gate int error = 0; 1787883492d5Sraf uint8_t ceil; 1788883492d5Sraf int myprio; 17897c478bd9Sstevel@tonic-gate 17907c478bd9Sstevel@tonic-gate ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK); 17917c478bd9Sstevel@tonic-gate 17927c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 17937c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 17947c478bd9Sstevel@tonic-gate 17957c478bd9Sstevel@tonic-gate if (msp && try == MUTEX_TRY) 17967c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try); 17977c478bd9Sstevel@tonic-gate 1798883492d5Sraf if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) 1799883492d5Sraf return (mutex_recursion(mp, mtype, try)); 18007c478bd9Sstevel@tonic-gate 18017c478bd9Sstevel@tonic-gate if (self->ul_error_detection && try == MUTEX_LOCK && 18027c478bd9Sstevel@tonic-gate tsp == NULL && mutex_is_held(mp)) 18037c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_lock", NULL, NULL); 18047c478bd9Sstevel@tonic-gate 1805883492d5Sraf if (mtype & LOCK_PRIO_PROTECT) { 1806883492d5Sraf ceil = mp->mutex_ceiling; 1807883492d5Sraf ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0); 1808883492d5Sraf myprio = real_priority(self); 1809883492d5Sraf if (myprio > ceil) { 1810883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EINVAL); 1811883492d5Sraf return (EINVAL); 18127c478bd9Sstevel@tonic-gate } 1813883492d5Sraf if ((error = _ceil_mylist_add(mp)) != 0) { 1814883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1815883492d5Sraf return (error); 18167c478bd9Sstevel@tonic-gate } 1817883492d5Sraf if (myprio < ceil) 1818883492d5Sraf _ceil_prio_inherit(ceil); 1819883492d5Sraf } 18207c478bd9Sstevel@tonic-gate 1821883492d5Sraf if ((mtype & (USYNC_PROCESS | LOCK_ROBUST)) 1822883492d5Sraf == (USYNC_PROCESS | LOCK_ROBUST)) 1823883492d5Sraf register_lock(mp); 1824883492d5Sraf 1825883492d5Sraf if (mtype & LOCK_PRIO_INHERIT) { 1826883492d5Sraf /* go straight to the kernel */ 1827883492d5Sraf if (try == MUTEX_TRY) 1828883492d5Sraf error = mutex_trylock_kernel(mp); 1829883492d5Sraf else /* MUTEX_LOCK */ 1830883492d5Sraf error = mutex_lock_kernel(mp, tsp, msp); 18317c478bd9Sstevel@tonic-gate /* 1832883492d5Sraf * The kernel never sets or clears the lock byte 1833883492d5Sraf * for LOCK_PRIO_INHERIT mutexes. 1834883492d5Sraf * Set it here for consistency. 18357c478bd9Sstevel@tonic-gate */ 1836883492d5Sraf switch (error) { 1837883492d5Sraf case 0: 1838883492d5Sraf mp->mutex_lockw = LOCKSET; 1839883492d5Sraf break; 1840883492d5Sraf case EOWNERDEAD: 1841883492d5Sraf case ELOCKUNMAPPED: 1842883492d5Sraf mp->mutex_lockw = LOCKSET; 1843883492d5Sraf /* FALLTHROUGH */ 1844883492d5Sraf case ENOTRECOVERABLE: 1845883492d5Sraf ASSERT(mtype & LOCK_ROBUST); 1846883492d5Sraf break; 1847883492d5Sraf case EDEADLK: 1848883492d5Sraf if (try == MUTEX_LOCK) 1849883492d5Sraf stall(); 1850883492d5Sraf error = EBUSY; 1851883492d5Sraf break; 18527c478bd9Sstevel@tonic-gate } 1853883492d5Sraf } else if (mtype & USYNC_PROCESS) { 185416b01779Sraf error = mutex_trylock_process(mp, try == MUTEX_LOCK); 1855883492d5Sraf if (error == EBUSY && try == MUTEX_LOCK) 18567c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 1857*5d1dd9a9Sraf } else { /* USYNC_THREAD */ 185816b01779Sraf error = mutex_trylock_adaptive(mp, try == MUTEX_LOCK); 1859883492d5Sraf if (error == EBUSY && try == MUTEX_LOCK) 1860883492d5Sraf error = mutex_lock_queue(self, msp, mp, tsp); 18617c478bd9Sstevel@tonic-gate } 18627c478bd9Sstevel@tonic-gate 18637c478bd9Sstevel@tonic-gate switch (error) { 1864883492d5Sraf case 0: 18657c478bd9Sstevel@tonic-gate case EOWNERDEAD: 18667c478bd9Sstevel@tonic-gate case ELOCKUNMAPPED: 1867883492d5Sraf if (mtype & LOCK_ROBUST) 1868883492d5Sraf remember_lock(mp); 18697c478bd9Sstevel@tonic-gate if (msp) 18707c478bd9Sstevel@tonic-gate record_begin_hold(msp); 18717c478bd9Sstevel@tonic-gate break; 18727c478bd9Sstevel@tonic-gate default: 1873883492d5Sraf if (mtype & LOCK_PRIO_PROTECT) { 1874883492d5Sraf (void) _ceil_mylist_del(mp); 1875883492d5Sraf if (myprio < ceil) 1876883492d5Sraf _ceil_prio_waive(); 1877883492d5Sraf } 18787c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) { 18797c478bd9Sstevel@tonic-gate if (msp) 18807c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try_fail); 18817c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 18827c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 18837c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 18847c478bd9Sstevel@tonic-gate } 18857c478bd9Sstevel@tonic-gate } 18867c478bd9Sstevel@tonic-gate break; 18877c478bd9Sstevel@tonic-gate } 18887c478bd9Sstevel@tonic-gate 18897c478bd9Sstevel@tonic-gate return (error); 18907c478bd9Sstevel@tonic-gate } 18917c478bd9Sstevel@tonic-gate 18927c478bd9Sstevel@tonic-gate int 18937c478bd9Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try) 18947c478bd9Sstevel@tonic-gate { 18957c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 18967c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 18977c478bd9Sstevel@tonic-gate 18987c478bd9Sstevel@tonic-gate /* 18997c478bd9Sstevel@tonic-gate * We know that USYNC_PROCESS is set in mtype and that 19007c478bd9Sstevel@tonic-gate * zero, one, or both of the flags LOCK_RECURSIVE and 19017c478bd9Sstevel@tonic-gate * LOCK_ERRORCHECK are set, and that no other flags are set. 19027c478bd9Sstevel@tonic-gate */ 1903883492d5Sraf ASSERT((mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0); 19047c478bd9Sstevel@tonic-gate enter_critical(self); 19057c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 19067c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 19077c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 19087c478bd9Sstevel@tonic-gate exit_critical(self); 19097c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 19107c478bd9Sstevel@tonic-gate return (0); 19117c478bd9Sstevel@tonic-gate } 19127c478bd9Sstevel@tonic-gate exit_critical(self); 19137c478bd9Sstevel@tonic-gate 1914883492d5Sraf if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && shared_mutex_held(mp)) 1915883492d5Sraf return (mutex_recursion(mp, mtype, try)); 19167c478bd9Sstevel@tonic-gate 191716b01779Sraf if (try == MUTEX_LOCK) { 191816b01779Sraf if (mutex_trylock_process(mp, 1) == 0) 191916b01779Sraf return (0); 19207c478bd9Sstevel@tonic-gate return (mutex_lock_kernel(mp, tsp, NULL)); 192116b01779Sraf } 19227c478bd9Sstevel@tonic-gate 19237c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 19247c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 19257c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 19267c478bd9Sstevel@tonic-gate } 19277c478bd9Sstevel@tonic-gate return (EBUSY); 19287c478bd9Sstevel@tonic-gate } 19297c478bd9Sstevel@tonic-gate 19307c478bd9Sstevel@tonic-gate static int 19317c478bd9Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp) 19327c478bd9Sstevel@tonic-gate { 19337c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 19347c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 19357c478bd9Sstevel@tonic-gate uberflags_t *gflags; 19367c478bd9Sstevel@tonic-gate int mtype; 19377c478bd9Sstevel@tonic-gate 19387c478bd9Sstevel@tonic-gate /* 19397c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 19407c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 19417c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 19427c478bd9Sstevel@tonic-gate * and the process has only a single thread. 19437c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 19447c478bd9Sstevel@tonic-gate */ 19457c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 19467c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 19477c478bd9Sstevel@tonic-gate /* 19487c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 19497c478bd9Sstevel@tonic-gate */ 19507c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 19517c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 19527c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 19537c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 19547c478bd9Sstevel@tonic-gate return (0); 19557c478bd9Sstevel@tonic-gate } 1956883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 1957883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_LOCK)); 19587c478bd9Sstevel@tonic-gate /* 19597c478bd9Sstevel@tonic-gate * We have reached a deadlock, probably because the 19607c478bd9Sstevel@tonic-gate * process is executing non-async-signal-safe code in 19617c478bd9Sstevel@tonic-gate * a signal handler and is attempting to acquire a lock 19627c478bd9Sstevel@tonic-gate * that it already owns. This is not surprising, given 19637c478bd9Sstevel@tonic-gate * bad programming practices over the years that has 19647c478bd9Sstevel@tonic-gate * resulted in applications calling printf() and such 19657c478bd9Sstevel@tonic-gate * in their signal handlers. Unless the user has told 19667c478bd9Sstevel@tonic-gate * us that the signal handlers are safe by setting: 19677c478bd9Sstevel@tonic-gate * export _THREAD_ASYNC_SAFE=1 19687c478bd9Sstevel@tonic-gate * we return EDEADLK rather than actually deadlocking. 19697c478bd9Sstevel@tonic-gate */ 19707c478bd9Sstevel@tonic-gate if (tsp == NULL && 19717c478bd9Sstevel@tonic-gate MUTEX_OWNER(mp) == self && !self->ul_async_safe) { 19727c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 19737c478bd9Sstevel@tonic-gate return (EDEADLK); 19747c478bd9Sstevel@tonic-gate } 19757c478bd9Sstevel@tonic-gate } 19767c478bd9Sstevel@tonic-gate 19777c478bd9Sstevel@tonic-gate /* 19787c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 19797c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 19807c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 19817c478bd9Sstevel@tonic-gate */ 19827c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 19837c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 19847c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 19857c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 19867c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK)); 19877c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 19887c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 19897c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 19907c478bd9Sstevel@tonic-gate return (0); 19917c478bd9Sstevel@tonic-gate } 1992883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 1993883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_LOCK)); 199416b01779Sraf if (mutex_trylock_adaptive(mp, 1) != 0) 1995883492d5Sraf return (mutex_lock_queue(self, NULL, mp, tsp)); 1996883492d5Sraf return (0); 19977c478bd9Sstevel@tonic-gate } 19987c478bd9Sstevel@tonic-gate 19997c478bd9Sstevel@tonic-gate /* else do it the long way */ 20007c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, tsp, MUTEX_LOCK)); 20017c478bd9Sstevel@tonic-gate } 20027c478bd9Sstevel@tonic-gate 20037c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_lock = __mutex_lock 20047c478bd9Sstevel@tonic-gate #pragma weak mutex_lock = __mutex_lock 20057c478bd9Sstevel@tonic-gate #pragma weak _mutex_lock = __mutex_lock 20067c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_lock = __mutex_lock 20077c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_lock = __mutex_lock 20087c478bd9Sstevel@tonic-gate int 20097c478bd9Sstevel@tonic-gate __mutex_lock(mutex_t *mp) 20107c478bd9Sstevel@tonic-gate { 20117c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 20127c478bd9Sstevel@tonic-gate return (mutex_lock_impl(mp, NULL)); 20137c478bd9Sstevel@tonic-gate } 20147c478bd9Sstevel@tonic-gate 20157c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_timedlock = _pthread_mutex_timedlock 20167c478bd9Sstevel@tonic-gate int 20177c478bd9Sstevel@tonic-gate _pthread_mutex_timedlock(mutex_t *mp, const timespec_t *abstime) 20187c478bd9Sstevel@tonic-gate { 20197c478bd9Sstevel@tonic-gate timespec_t tslocal; 20207c478bd9Sstevel@tonic-gate int error; 20217c478bd9Sstevel@tonic-gate 20227c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 20237c478bd9Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 20247c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 20257c478bd9Sstevel@tonic-gate if (error == ETIME) 20267c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 20277c478bd9Sstevel@tonic-gate return (error); 20287c478bd9Sstevel@tonic-gate } 20297c478bd9Sstevel@tonic-gate 20307c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_reltimedlock_np = _pthread_mutex_reltimedlock_np 20317c478bd9Sstevel@tonic-gate int 20327c478bd9Sstevel@tonic-gate _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime) 20337c478bd9Sstevel@tonic-gate { 20347c478bd9Sstevel@tonic-gate timespec_t tslocal; 20357c478bd9Sstevel@tonic-gate int error; 20367c478bd9Sstevel@tonic-gate 20377c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 20387c478bd9Sstevel@tonic-gate tslocal = *reltime; 20397c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 20407c478bd9Sstevel@tonic-gate if (error == ETIME) 20417c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 20427c478bd9Sstevel@tonic-gate return (error); 20437c478bd9Sstevel@tonic-gate } 20447c478bd9Sstevel@tonic-gate 20457c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_trylock = __mutex_trylock 20467c478bd9Sstevel@tonic-gate #pragma weak mutex_trylock = __mutex_trylock 20477c478bd9Sstevel@tonic-gate #pragma weak _mutex_trylock = __mutex_trylock 20487c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_trylock = __mutex_trylock 20497c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_trylock = __mutex_trylock 20507c478bd9Sstevel@tonic-gate int 20517c478bd9Sstevel@tonic-gate __mutex_trylock(mutex_t *mp) 20527c478bd9Sstevel@tonic-gate { 20537c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 20547c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 20557c478bd9Sstevel@tonic-gate uberflags_t *gflags; 20567c478bd9Sstevel@tonic-gate int mtype; 20577c478bd9Sstevel@tonic-gate 20587c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 20597c478bd9Sstevel@tonic-gate /* 20607c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 20617c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 20627c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 20637c478bd9Sstevel@tonic-gate * and the process has only a single thread. 20647c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 20657c478bd9Sstevel@tonic-gate */ 20667c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 20677c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 20687c478bd9Sstevel@tonic-gate /* 20697c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 20707c478bd9Sstevel@tonic-gate */ 20717c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 20727c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 20737c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 20747c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 20757c478bd9Sstevel@tonic-gate return (0); 20767c478bd9Sstevel@tonic-gate } 2077883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 2078883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_TRY)); 20797c478bd9Sstevel@tonic-gate return (EBUSY); 20807c478bd9Sstevel@tonic-gate } 20817c478bd9Sstevel@tonic-gate 20827c478bd9Sstevel@tonic-gate /* 20837c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 20847c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 20857c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 20867c478bd9Sstevel@tonic-gate */ 20877c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 20887c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 20897c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 20907c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 20917c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY)); 20927c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 20937c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 20947c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 20957c478bd9Sstevel@tonic-gate return (0); 20967c478bd9Sstevel@tonic-gate } 2097883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 2098883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_TRY)); 209916b01779Sraf if (__td_event_report(self, TD_LOCK_TRY, udp)) { 210016b01779Sraf self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 210116b01779Sraf tdb_event(TD_LOCK_TRY, udp); 21027c478bd9Sstevel@tonic-gate } 210316b01779Sraf return (EBUSY); 21047c478bd9Sstevel@tonic-gate } 21057c478bd9Sstevel@tonic-gate 21067c478bd9Sstevel@tonic-gate /* else do it the long way */ 21077c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, NULL, MUTEX_TRY)); 21087c478bd9Sstevel@tonic-gate } 21097c478bd9Sstevel@tonic-gate 21107c478bd9Sstevel@tonic-gate int 2111883492d5Sraf mutex_unlock_internal(mutex_t *mp, int retain_robust_flags) 21127c478bd9Sstevel@tonic-gate { 21137c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 21147c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 21157c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 21167c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp; 2117883492d5Sraf int error = 0; 2118883492d5Sraf int release_all; 21197c478bd9Sstevel@tonic-gate lwpid_t lwpid; 21207c478bd9Sstevel@tonic-gate 21217c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp)) 21227c478bd9Sstevel@tonic-gate return (EPERM); 21237c478bd9Sstevel@tonic-gate 21247c478bd9Sstevel@tonic-gate if (self->ul_error_detection && !mutex_is_held(mp)) 21257c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_unlock", NULL, NULL); 21267c478bd9Sstevel@tonic-gate 21277c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 21287c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 21297c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 21307c478bd9Sstevel@tonic-gate return (0); 21317c478bd9Sstevel@tonic-gate } 21327c478bd9Sstevel@tonic-gate 21337c478bd9Sstevel@tonic-gate if ((msp = MUTEX_STATS(mp, udp)) != NULL) 21347c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 21357c478bd9Sstevel@tonic-gate 2136883492d5Sraf if (!retain_robust_flags && !(mtype & LOCK_PRIO_INHERIT) && 2137883492d5Sraf (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) { 2138883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 2139883492d5Sraf mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED); 2140883492d5Sraf mp->mutex_flag |= LOCK_NOTRECOVERABLE; 2141883492d5Sraf } 2142883492d5Sraf release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0); 2143883492d5Sraf 2144883492d5Sraf if (mtype & LOCK_PRIO_INHERIT) { 21457c478bd9Sstevel@tonic-gate no_preempt(self); 21467c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 21477c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 21487c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 2149883492d5Sraf mp->mutex_lockw = LOCKCLEAR; 2150883492d5Sraf error = ___lwp_mutex_unlock(mp); 21517c478bd9Sstevel@tonic-gate preempt(self); 21527c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) { 2153*5d1dd9a9Sraf mutex_unlock_process(mp, release_all); 21547c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 2155883492d5Sraf if ((lwpid = mutex_unlock_queue(mp, release_all)) != 0) { 21567c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 21577c478bd9Sstevel@tonic-gate preempt(self); 21587c478bd9Sstevel@tonic-gate } 21597c478bd9Sstevel@tonic-gate } 21607c478bd9Sstevel@tonic-gate 2161883492d5Sraf if (mtype & LOCK_ROBUST) 2162883492d5Sraf forget_lock(mp); 2163883492d5Sraf 2164883492d5Sraf if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp)) 2165883492d5Sraf _ceil_prio_waive(); 2166883492d5Sraf 21677c478bd9Sstevel@tonic-gate return (error); 21687c478bd9Sstevel@tonic-gate } 21697c478bd9Sstevel@tonic-gate 21707c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_unlock = __mutex_unlock 21717c478bd9Sstevel@tonic-gate #pragma weak mutex_unlock = __mutex_unlock 21727c478bd9Sstevel@tonic-gate #pragma weak _mutex_unlock = __mutex_unlock 21737c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_unlock = __mutex_unlock 21747c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_unlock = __mutex_unlock 21757c478bd9Sstevel@tonic-gate int 21767c478bd9Sstevel@tonic-gate __mutex_unlock(mutex_t *mp) 21777c478bd9Sstevel@tonic-gate { 21787c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 21797c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 21807c478bd9Sstevel@tonic-gate uberflags_t *gflags; 21817c478bd9Sstevel@tonic-gate lwpid_t lwpid; 21827c478bd9Sstevel@tonic-gate int mtype; 21837c478bd9Sstevel@tonic-gate short el; 21847c478bd9Sstevel@tonic-gate 21857c478bd9Sstevel@tonic-gate /* 21867c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 21877c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 21887c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 21897c478bd9Sstevel@tonic-gate * and the process has only a single thread. 21907c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 21917c478bd9Sstevel@tonic-gate */ 21927c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 21937c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 21947c478bd9Sstevel@tonic-gate if (mtype) { 21957c478bd9Sstevel@tonic-gate /* 21967c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 21977c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 21987c478bd9Sstevel@tonic-gate */ 21997c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 22007c478bd9Sstevel@tonic-gate return (EPERM); 22017c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 22027c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 22037c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 22047c478bd9Sstevel@tonic-gate return (0); 22057c478bd9Sstevel@tonic-gate } 22067c478bd9Sstevel@tonic-gate } 22077c478bd9Sstevel@tonic-gate /* 22087c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 22097c478bd9Sstevel@tonic-gate * Also, there can be no waiters. 22107c478bd9Sstevel@tonic-gate */ 22117c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 22127c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 22137c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 22147c478bd9Sstevel@tonic-gate return (0); 22157c478bd9Sstevel@tonic-gate } 22167c478bd9Sstevel@tonic-gate 22177c478bd9Sstevel@tonic-gate /* 22187c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 22197c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 22207c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 22217c478bd9Sstevel@tonic-gate */ 22227c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL) { 22237c478bd9Sstevel@tonic-gate if (((el = gflags->uf_trs_ted) | mtype) == 0) { 22247c478bd9Sstevel@tonic-gate fast_unlock: 2225*5d1dd9a9Sraf if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) { 22267c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 22277c478bd9Sstevel@tonic-gate preempt(self); 22287c478bd9Sstevel@tonic-gate } 22297c478bd9Sstevel@tonic-gate return (0); 22307c478bd9Sstevel@tonic-gate } 22317c478bd9Sstevel@tonic-gate if (el) /* error detection or lock statistics */ 22327c478bd9Sstevel@tonic-gate goto slow_unlock; 22337c478bd9Sstevel@tonic-gate if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 22347c478bd9Sstevel@tonic-gate /* 22357c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 22367c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 22377c478bd9Sstevel@tonic-gate */ 22387c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 22397c478bd9Sstevel@tonic-gate return (EPERM); 22407c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 22417c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 22427c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 22437c478bd9Sstevel@tonic-gate return (0); 22447c478bd9Sstevel@tonic-gate } 22457c478bd9Sstevel@tonic-gate goto fast_unlock; 22467c478bd9Sstevel@tonic-gate } 22477c478bd9Sstevel@tonic-gate if ((mtype & 22487c478bd9Sstevel@tonic-gate ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 22497c478bd9Sstevel@tonic-gate /* 22507c478bd9Sstevel@tonic-gate * At this point we know that zero, one, or both of the 22517c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and 22527c478bd9Sstevel@tonic-gate * that the USYNC_PROCESS flag is set. 22537c478bd9Sstevel@tonic-gate */ 22547c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp)) 22557c478bd9Sstevel@tonic-gate return (EPERM); 22567c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 22577c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 22587c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 22597c478bd9Sstevel@tonic-gate return (0); 22607c478bd9Sstevel@tonic-gate } 2261*5d1dd9a9Sraf mutex_unlock_process(mp, 0); 22627c478bd9Sstevel@tonic-gate return (0); 22637c478bd9Sstevel@tonic-gate } 22647c478bd9Sstevel@tonic-gate } 22657c478bd9Sstevel@tonic-gate 22667c478bd9Sstevel@tonic-gate /* else do it the long way */ 22677c478bd9Sstevel@tonic-gate slow_unlock: 2268883492d5Sraf return (mutex_unlock_internal(mp, 0)); 22697c478bd9Sstevel@tonic-gate } 22707c478bd9Sstevel@tonic-gate 22717c478bd9Sstevel@tonic-gate /* 22727c478bd9Sstevel@tonic-gate * Internally to the library, almost all mutex lock/unlock actions 22737c478bd9Sstevel@tonic-gate * go through these lmutex_ functions, to protect critical regions. 22747c478bd9Sstevel@tonic-gate * We replicate a bit of code from __mutex_lock() and __mutex_unlock() 22757c478bd9Sstevel@tonic-gate * to make these functions faster since we know that the mutex type 22767c478bd9Sstevel@tonic-gate * of all internal locks is USYNC_THREAD. We also know that internal 22777c478bd9Sstevel@tonic-gate * locking can never fail, so we panic if it does. 22787c478bd9Sstevel@tonic-gate */ 22797c478bd9Sstevel@tonic-gate void 22807c478bd9Sstevel@tonic-gate lmutex_lock(mutex_t *mp) 22817c478bd9Sstevel@tonic-gate { 22827c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 22837c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 22847c478bd9Sstevel@tonic-gate 22857c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 22867c478bd9Sstevel@tonic-gate 22877c478bd9Sstevel@tonic-gate enter_critical(self); 22887c478bd9Sstevel@tonic-gate /* 22897c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 22907c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 22917c478bd9Sstevel@tonic-gate */ 22927c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 22937c478bd9Sstevel@tonic-gate /* 22947c478bd9Sstevel@tonic-gate * Only one thread exists; the mutex must be free. 22957c478bd9Sstevel@tonic-gate */ 22967c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_lockw == 0); 22977c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 22987c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 22997c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 23007c478bd9Sstevel@tonic-gate } else { 23017c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 23027c478bd9Sstevel@tonic-gate 23037c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 23047c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 23057c478bd9Sstevel@tonic-gate 23067c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 23077c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 23087c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 230916b01779Sraf } else if (mutex_trylock_adaptive(mp, 1) != 0) { 23107c478bd9Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL); 23117c478bd9Sstevel@tonic-gate } 23127c478bd9Sstevel@tonic-gate 23137c478bd9Sstevel@tonic-gate if (msp) 23147c478bd9Sstevel@tonic-gate record_begin_hold(msp); 23157c478bd9Sstevel@tonic-gate } 23167c478bd9Sstevel@tonic-gate } 23177c478bd9Sstevel@tonic-gate 23187c478bd9Sstevel@tonic-gate void 23197c478bd9Sstevel@tonic-gate lmutex_unlock(mutex_t *mp) 23207c478bd9Sstevel@tonic-gate { 23217c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 23227c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 23237c478bd9Sstevel@tonic-gate 23247c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 23257c478bd9Sstevel@tonic-gate 23267c478bd9Sstevel@tonic-gate /* 23277c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 23287c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 23297c478bd9Sstevel@tonic-gate */ 23307c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 23317c478bd9Sstevel@tonic-gate /* 23327c478bd9Sstevel@tonic-gate * Only one thread exists so there can be no waiters. 23337c478bd9Sstevel@tonic-gate */ 23347c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 23357c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 23367c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 23377c478bd9Sstevel@tonic-gate } else { 23387c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 23397c478bd9Sstevel@tonic-gate lwpid_t lwpid; 23407c478bd9Sstevel@tonic-gate 23417c478bd9Sstevel@tonic-gate if (msp) 23427c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 2343883492d5Sraf if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) { 23447c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 23457c478bd9Sstevel@tonic-gate preempt(self); 23467c478bd9Sstevel@tonic-gate } 23477c478bd9Sstevel@tonic-gate } 23487c478bd9Sstevel@tonic-gate exit_critical(self); 23497c478bd9Sstevel@tonic-gate } 23507c478bd9Sstevel@tonic-gate 2351f841f6adSraf /* 2352f841f6adSraf * For specialized code in libc, like the asynchronous i/o code, 2353f841f6adSraf * the following sig_*() locking primitives are used in order 2354f841f6adSraf * to make the code asynchronous signal safe. Signals are 2355f841f6adSraf * deferred while locks acquired by these functions are held. 2356f841f6adSraf */ 2357f841f6adSraf void 2358f841f6adSraf sig_mutex_lock(mutex_t *mp) 2359f841f6adSraf { 2360f841f6adSraf sigoff(curthread); 2361f841f6adSraf (void) _private_mutex_lock(mp); 2362f841f6adSraf } 2363f841f6adSraf 2364f841f6adSraf void 2365f841f6adSraf sig_mutex_unlock(mutex_t *mp) 2366f841f6adSraf { 2367f841f6adSraf (void) _private_mutex_unlock(mp); 2368f841f6adSraf sigon(curthread); 2369f841f6adSraf } 2370f841f6adSraf 2371f841f6adSraf int 2372f841f6adSraf sig_mutex_trylock(mutex_t *mp) 2373f841f6adSraf { 2374f841f6adSraf int error; 2375f841f6adSraf 2376f841f6adSraf sigoff(curthread); 2377f841f6adSraf if ((error = _private_mutex_trylock(mp)) != 0) 2378f841f6adSraf sigon(curthread); 2379f841f6adSraf return (error); 2380f841f6adSraf } 2381f841f6adSraf 2382f841f6adSraf /* 2383f841f6adSraf * sig_cond_wait() is a cancellation point. 2384f841f6adSraf */ 2385f841f6adSraf int 2386f841f6adSraf sig_cond_wait(cond_t *cv, mutex_t *mp) 2387f841f6adSraf { 2388f841f6adSraf int error; 2389f841f6adSraf 2390f841f6adSraf ASSERT(curthread->ul_sigdefer != 0); 2391f841f6adSraf _private_testcancel(); 2392f841f6adSraf error = _cond_wait(cv, mp); 2393f841f6adSraf if (error == EINTR && curthread->ul_cursig) { 2394f841f6adSraf sig_mutex_unlock(mp); 2395f841f6adSraf /* take the deferred signal here */ 2396f841f6adSraf sig_mutex_lock(mp); 2397f841f6adSraf } 2398f841f6adSraf _private_testcancel(); 2399f841f6adSraf return (error); 2400f841f6adSraf } 2401f841f6adSraf 2402f841f6adSraf /* 2403f841f6adSraf * sig_cond_reltimedwait() is a cancellation point. 2404f841f6adSraf */ 2405f841f6adSraf int 2406f841f6adSraf sig_cond_reltimedwait(cond_t *cv, mutex_t *mp, const timespec_t *ts) 2407f841f6adSraf { 2408f841f6adSraf int error; 2409f841f6adSraf 2410f841f6adSraf ASSERT(curthread->ul_sigdefer != 0); 2411f841f6adSraf _private_testcancel(); 2412f841f6adSraf error = _cond_reltimedwait(cv, mp, ts); 2413f841f6adSraf if (error == EINTR && curthread->ul_cursig) { 2414f841f6adSraf sig_mutex_unlock(mp); 2415f841f6adSraf /* take the deferred signal here */ 2416f841f6adSraf sig_mutex_lock(mp); 2417f841f6adSraf } 2418f841f6adSraf _private_testcancel(); 2419f841f6adSraf return (error); 2420f841f6adSraf } 2421f841f6adSraf 24227c478bd9Sstevel@tonic-gate static int 24237c478bd9Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg) 24247c478bd9Sstevel@tonic-gate { 24257c478bd9Sstevel@tonic-gate /* 2426883492d5Sraf * The 'volatile' is necessary to make sure the compiler doesn't 2427883492d5Sraf * reorder the tests of the various components of the mutex. 2428883492d5Sraf * They must be tested in this order: 2429883492d5Sraf * mutex_lockw 2430883492d5Sraf * mutex_owner 2431883492d5Sraf * mutex_ownerpid 2432883492d5Sraf * This relies on the fact that everywhere mutex_lockw is cleared, 2433883492d5Sraf * mutex_owner and mutex_ownerpid are cleared before mutex_lockw 2434883492d5Sraf * is cleared, and that everywhere mutex_lockw is set, mutex_owner 2435883492d5Sraf * and mutex_ownerpid are set after mutex_lockw is set, and that 2436883492d5Sraf * mutex_lockw is set or cleared with a memory barrier. 24377c478bd9Sstevel@tonic-gate */ 24387c478bd9Sstevel@tonic-gate volatile mutex_t *mp = (volatile mutex_t *)mparg; 24397c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 24407c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 24417c478bd9Sstevel@tonic-gate 2442883492d5Sraf return (MUTEX_OWNED(mp, self) && mp->mutex_ownerpid == udp->pid); 24437c478bd9Sstevel@tonic-gate } 24447c478bd9Sstevel@tonic-gate 24457c478bd9Sstevel@tonic-gate /* 24467c478bd9Sstevel@tonic-gate * Some crufty old programs define their own version of _mutex_held() 24477c478bd9Sstevel@tonic-gate * to be simply return(1). This breaks internal libc logic, so we 24487c478bd9Sstevel@tonic-gate * define a private version for exclusive use by libc, mutex_is_held(), 24497c478bd9Sstevel@tonic-gate * and also a new public function, __mutex_held(), to be used in new 24507c478bd9Sstevel@tonic-gate * code to circumvent these crufty old programs. 24517c478bd9Sstevel@tonic-gate */ 24527c478bd9Sstevel@tonic-gate #pragma weak mutex_held = mutex_is_held 24537c478bd9Sstevel@tonic-gate #pragma weak _mutex_held = mutex_is_held 24547c478bd9Sstevel@tonic-gate #pragma weak __mutex_held = mutex_is_held 24557c478bd9Sstevel@tonic-gate int 2456883492d5Sraf mutex_is_held(mutex_t *mparg) 24577c478bd9Sstevel@tonic-gate { 2458883492d5Sraf volatile mutex_t *mp = (volatile mutex_t *)mparg; 2459883492d5Sraf 2460883492d5Sraf if (mparg->mutex_type & USYNC_PROCESS) 2461883492d5Sraf return (shared_mutex_held(mparg)); 24627c478bd9Sstevel@tonic-gate return (MUTEX_OWNED(mp, curthread)); 24637c478bd9Sstevel@tonic-gate } 24647c478bd9Sstevel@tonic-gate 24657c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_destroy = __mutex_destroy 24667c478bd9Sstevel@tonic-gate #pragma weak mutex_destroy = __mutex_destroy 24677c478bd9Sstevel@tonic-gate #pragma weak _mutex_destroy = __mutex_destroy 24687c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_destroy = __mutex_destroy 24697c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_destroy = __mutex_destroy 24707c478bd9Sstevel@tonic-gate int 24717c478bd9Sstevel@tonic-gate __mutex_destroy(mutex_t *mp) 24727c478bd9Sstevel@tonic-gate { 2473883492d5Sraf if (mp->mutex_type & USYNC_PROCESS) 2474883492d5Sraf forget_lock(mp); 2475883492d5Sraf (void) _memset(mp, 0, sizeof (*mp)); 24767c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(mp); 24777c478bd9Sstevel@tonic-gate return (0); 24787c478bd9Sstevel@tonic-gate } 24797c478bd9Sstevel@tonic-gate 2480883492d5Sraf #pragma weak mutex_consistent = __mutex_consistent 2481883492d5Sraf #pragma weak _mutex_consistent = __mutex_consistent 2482883492d5Sraf #pragma weak pthread_mutex_consistent_np = __mutex_consistent 2483883492d5Sraf #pragma weak _pthread_mutex_consistent_np = __mutex_consistent 2484883492d5Sraf int 2485883492d5Sraf __mutex_consistent(mutex_t *mp) 2486883492d5Sraf { 2487883492d5Sraf /* 2488883492d5Sraf * Do this only for an inconsistent, initialized robust lock 2489883492d5Sraf * that we hold. For all other cases, return EINVAL. 2490883492d5Sraf */ 2491883492d5Sraf if (mutex_is_held(mp) && 2492883492d5Sraf (mp->mutex_type & LOCK_ROBUST) && 2493883492d5Sraf (mp->mutex_flag & LOCK_INITED) && 2494883492d5Sraf (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) { 2495883492d5Sraf mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED); 2496883492d5Sraf mp->mutex_rcount = 0; 2497883492d5Sraf return (0); 2498883492d5Sraf } 2499883492d5Sraf return (EINVAL); 2500883492d5Sraf } 2501883492d5Sraf 25027c478bd9Sstevel@tonic-gate /* 25037c478bd9Sstevel@tonic-gate * Spin locks are separate from ordinary mutexes, 25047c478bd9Sstevel@tonic-gate * but we use the same data structure for them. 25057c478bd9Sstevel@tonic-gate */ 25067c478bd9Sstevel@tonic-gate 25077c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_init = _pthread_spin_init 25087c478bd9Sstevel@tonic-gate int 25097c478bd9Sstevel@tonic-gate _pthread_spin_init(pthread_spinlock_t *lock, int pshared) 25107c478bd9Sstevel@tonic-gate { 25117c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 25127c478bd9Sstevel@tonic-gate 25137c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 25147c478bd9Sstevel@tonic-gate if (pshared == PTHREAD_PROCESS_SHARED) 25157c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_PROCESS; 25167c478bd9Sstevel@tonic-gate else 25177c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_THREAD; 25187c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 25197c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 25207c478bd9Sstevel@tonic-gate return (0); 25217c478bd9Sstevel@tonic-gate } 25227c478bd9Sstevel@tonic-gate 25237c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_destroy = _pthread_spin_destroy 25247c478bd9Sstevel@tonic-gate int 25257c478bd9Sstevel@tonic-gate _pthread_spin_destroy(pthread_spinlock_t *lock) 25267c478bd9Sstevel@tonic-gate { 25277c478bd9Sstevel@tonic-gate (void) _memset(lock, 0, sizeof (*lock)); 25287c478bd9Sstevel@tonic-gate return (0); 25297c478bd9Sstevel@tonic-gate } 25307c478bd9Sstevel@tonic-gate 25317c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_trylock = _pthread_spin_trylock 25327c478bd9Sstevel@tonic-gate int 25337c478bd9Sstevel@tonic-gate _pthread_spin_trylock(pthread_spinlock_t *lock) 25347c478bd9Sstevel@tonic-gate { 25357c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 25367c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 25377c478bd9Sstevel@tonic-gate int error = 0; 25387c478bd9Sstevel@tonic-gate 25397c478bd9Sstevel@tonic-gate no_preempt(self); 25407c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) != 0) 25417c478bd9Sstevel@tonic-gate error = EBUSY; 25427c478bd9Sstevel@tonic-gate else { 25437c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 25447c478bd9Sstevel@tonic-gate if (mp->mutex_type == USYNC_PROCESS) 25457c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = self->ul_uberdata->pid; 25467c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 25477c478bd9Sstevel@tonic-gate } 25487c478bd9Sstevel@tonic-gate preempt(self); 25497c478bd9Sstevel@tonic-gate return (error); 25507c478bd9Sstevel@tonic-gate } 25517c478bd9Sstevel@tonic-gate 25527c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_lock = _pthread_spin_lock 25537c478bd9Sstevel@tonic-gate int 25547c478bd9Sstevel@tonic-gate _pthread_spin_lock(pthread_spinlock_t *lock) 25557c478bd9Sstevel@tonic-gate { 2556883492d5Sraf mutex_t *mp = (mutex_t *)lock; 2557883492d5Sraf ulwp_t *self = curthread; 2558883492d5Sraf volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw; 2559883492d5Sraf int count = 0; 2560883492d5Sraf 2561883492d5Sraf ASSERT(!self->ul_critical || self->ul_bindflags); 2562883492d5Sraf 2563883492d5Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp); 25647c478bd9Sstevel@tonic-gate 25657c478bd9Sstevel@tonic-gate /* 25667c478bd9Sstevel@tonic-gate * We don't care whether the owner is running on a processor. 25677c478bd9Sstevel@tonic-gate * We just spin because that's what this interface requires. 25687c478bd9Sstevel@tonic-gate */ 25697c478bd9Sstevel@tonic-gate for (;;) { 25707c478bd9Sstevel@tonic-gate if (*lockp == 0) { /* lock byte appears to be clear */ 2571883492d5Sraf no_preempt(self); 2572883492d5Sraf if (set_lock_byte(lockp) == 0) 2573883492d5Sraf break; 2574883492d5Sraf preempt(self); 25757c478bd9Sstevel@tonic-gate } 2576*5d1dd9a9Sraf if (count < INT_MAX) 2577*5d1dd9a9Sraf count++; 25787c478bd9Sstevel@tonic-gate SMT_PAUSE(); 25797c478bd9Sstevel@tonic-gate } 2580883492d5Sraf mp->mutex_owner = (uintptr_t)self; 2581883492d5Sraf if (mp->mutex_type == USYNC_PROCESS) 2582883492d5Sraf mp->mutex_ownerpid = self->ul_uberdata->pid; 2583883492d5Sraf preempt(self); 2584*5d1dd9a9Sraf if (count) { 2585*5d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 2586*5d1dd9a9Sraf } 2587883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 2588883492d5Sraf return (0); 25897c478bd9Sstevel@tonic-gate } 25907c478bd9Sstevel@tonic-gate 25917c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_unlock = _pthread_spin_unlock 25927c478bd9Sstevel@tonic-gate int 25937c478bd9Sstevel@tonic-gate _pthread_spin_unlock(pthread_spinlock_t *lock) 25947c478bd9Sstevel@tonic-gate { 25957c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 25967c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 25977c478bd9Sstevel@tonic-gate 25987c478bd9Sstevel@tonic-gate no_preempt(self); 25997c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 26007c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 26017c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 260241efec22Sraf (void) atomic_swap_32(&mp->mutex_lockword, 0); 26037c478bd9Sstevel@tonic-gate preempt(self); 26047c478bd9Sstevel@tonic-gate return (0); 26057c478bd9Sstevel@tonic-gate } 26067c478bd9Sstevel@tonic-gate 2607*5d1dd9a9Sraf #define INITIAL_LOCKS 8 /* initial size of ul_heldlocks.array */ 2608883492d5Sraf 2609883492d5Sraf /* 2610883492d5Sraf * Find/allocate an entry for 'lock' in our array of held locks. 2611883492d5Sraf */ 2612883492d5Sraf static mutex_t ** 2613883492d5Sraf find_lock_entry(mutex_t *lock) 2614883492d5Sraf { 2615883492d5Sraf ulwp_t *self = curthread; 2616883492d5Sraf mutex_t **remembered = NULL; 2617883492d5Sraf mutex_t **lockptr; 2618883492d5Sraf uint_t nlocks; 2619883492d5Sraf 2620883492d5Sraf if ((nlocks = self->ul_heldlockcnt) != 0) 2621883492d5Sraf lockptr = self->ul_heldlocks.array; 2622883492d5Sraf else { 2623883492d5Sraf nlocks = 1; 2624883492d5Sraf lockptr = &self->ul_heldlocks.single; 2625883492d5Sraf } 2626883492d5Sraf 2627883492d5Sraf for (; nlocks; nlocks--, lockptr++) { 2628883492d5Sraf if (*lockptr == lock) 2629883492d5Sraf return (lockptr); 2630883492d5Sraf if (*lockptr == NULL && remembered == NULL) 2631883492d5Sraf remembered = lockptr; 2632883492d5Sraf } 2633883492d5Sraf if (remembered != NULL) { 2634883492d5Sraf *remembered = lock; 2635883492d5Sraf return (remembered); 2636883492d5Sraf } 2637883492d5Sraf 2638883492d5Sraf /* 2639883492d5Sraf * No entry available. Allocate more space, converting 2640883492d5Sraf * the single entry into an array of entries if necessary. 2641883492d5Sraf */ 2642883492d5Sraf if ((nlocks = self->ul_heldlockcnt) == 0) { 2643883492d5Sraf /* 2644883492d5Sraf * Initial allocation of the array. 2645883492d5Sraf * Convert the single entry into an array. 2646883492d5Sraf */ 2647883492d5Sraf self->ul_heldlockcnt = nlocks = INITIAL_LOCKS; 2648883492d5Sraf lockptr = lmalloc(nlocks * sizeof (mutex_t *)); 2649883492d5Sraf /* 2650883492d5Sraf * The single entry becomes the first entry in the array. 2651883492d5Sraf */ 2652883492d5Sraf *lockptr = self->ul_heldlocks.single; 2653883492d5Sraf self->ul_heldlocks.array = lockptr; 2654883492d5Sraf /* 2655883492d5Sraf * Return the next available entry in the array. 2656883492d5Sraf */ 2657883492d5Sraf *++lockptr = lock; 2658883492d5Sraf return (lockptr); 2659883492d5Sraf } 2660883492d5Sraf /* 2661883492d5Sraf * Reallocate the array, double the size each time. 2662883492d5Sraf */ 2663883492d5Sraf lockptr = lmalloc(nlocks * 2 * sizeof (mutex_t *)); 2664883492d5Sraf (void) _memcpy(lockptr, self->ul_heldlocks.array, 2665883492d5Sraf nlocks * sizeof (mutex_t *)); 2666883492d5Sraf lfree(self->ul_heldlocks.array, nlocks * sizeof (mutex_t *)); 2667883492d5Sraf self->ul_heldlocks.array = lockptr; 2668883492d5Sraf self->ul_heldlockcnt *= 2; 2669883492d5Sraf /* 2670883492d5Sraf * Return the next available entry in the newly allocated array. 2671883492d5Sraf */ 2672883492d5Sraf *(lockptr += nlocks) = lock; 2673883492d5Sraf return (lockptr); 2674883492d5Sraf } 2675883492d5Sraf 2676883492d5Sraf /* 2677883492d5Sraf * Insert 'lock' into our list of held locks. 2678883492d5Sraf * Currently only used for LOCK_ROBUST mutexes. 2679883492d5Sraf */ 2680883492d5Sraf void 2681883492d5Sraf remember_lock(mutex_t *lock) 2682883492d5Sraf { 2683883492d5Sraf (void) find_lock_entry(lock); 2684883492d5Sraf } 2685883492d5Sraf 2686883492d5Sraf /* 2687883492d5Sraf * Remove 'lock' from our list of held locks. 2688883492d5Sraf * Currently only used for LOCK_ROBUST mutexes. 2689883492d5Sraf */ 2690883492d5Sraf void 2691883492d5Sraf forget_lock(mutex_t *lock) 2692883492d5Sraf { 2693883492d5Sraf *find_lock_entry(lock) = NULL; 2694883492d5Sraf } 2695883492d5Sraf 2696883492d5Sraf /* 2697883492d5Sraf * Free the array of held locks. 2698883492d5Sraf */ 2699883492d5Sraf void 2700883492d5Sraf heldlock_free(ulwp_t *ulwp) 2701883492d5Sraf { 2702883492d5Sraf uint_t nlocks; 2703883492d5Sraf 2704883492d5Sraf if ((nlocks = ulwp->ul_heldlockcnt) != 0) 2705883492d5Sraf lfree(ulwp->ul_heldlocks.array, nlocks * sizeof (mutex_t *)); 2706883492d5Sraf ulwp->ul_heldlockcnt = 0; 2707883492d5Sraf ulwp->ul_heldlocks.array = NULL; 2708883492d5Sraf } 2709883492d5Sraf 2710883492d5Sraf /* 2711883492d5Sraf * Mark all held LOCK_ROBUST mutexes LOCK_OWNERDEAD. 2712883492d5Sraf * Called from _thrp_exit() to deal with abandoned locks. 2713883492d5Sraf */ 2714883492d5Sraf void 2715883492d5Sraf heldlock_exit(void) 2716883492d5Sraf { 2717883492d5Sraf ulwp_t *self = curthread; 2718883492d5Sraf mutex_t **lockptr; 2719883492d5Sraf uint_t nlocks; 2720883492d5Sraf mutex_t *mp; 2721883492d5Sraf 2722883492d5Sraf if ((nlocks = self->ul_heldlockcnt) != 0) 2723883492d5Sraf lockptr = self->ul_heldlocks.array; 2724883492d5Sraf else { 2725883492d5Sraf nlocks = 1; 2726883492d5Sraf lockptr = &self->ul_heldlocks.single; 2727883492d5Sraf } 2728883492d5Sraf 2729883492d5Sraf for (; nlocks; nlocks--, lockptr++) { 2730883492d5Sraf /* 2731883492d5Sraf * The kernel takes care of transitioning held 2732883492d5Sraf * LOCK_PRIO_INHERIT mutexes to LOCK_OWNERDEAD. 2733883492d5Sraf * We avoid that case here. 2734883492d5Sraf */ 2735883492d5Sraf if ((mp = *lockptr) != NULL && 2736883492d5Sraf mutex_is_held(mp) && 2737883492d5Sraf (mp->mutex_type & (LOCK_ROBUST | LOCK_PRIO_INHERIT)) == 2738883492d5Sraf LOCK_ROBUST) { 2739883492d5Sraf mp->mutex_rcount = 0; 2740883492d5Sraf if (!(mp->mutex_flag & LOCK_UNMAPPED)) 2741883492d5Sraf mp->mutex_flag |= LOCK_OWNERDEAD; 2742883492d5Sraf (void) mutex_unlock_internal(mp, 1); 2743883492d5Sraf } 2744883492d5Sraf } 2745883492d5Sraf 2746883492d5Sraf heldlock_free(self); 2747883492d5Sraf } 2748883492d5Sraf 27497c478bd9Sstevel@tonic-gate #pragma weak cond_init = _cond_init 27507c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 27517c478bd9Sstevel@tonic-gate int 27527c478bd9Sstevel@tonic-gate _cond_init(cond_t *cvp, int type, void *arg) 27537c478bd9Sstevel@tonic-gate { 27547c478bd9Sstevel@tonic-gate if (type != USYNC_THREAD && type != USYNC_PROCESS) 27557c478bd9Sstevel@tonic-gate return (EINVAL); 27567c478bd9Sstevel@tonic-gate (void) _memset(cvp, 0, sizeof (*cvp)); 27577c478bd9Sstevel@tonic-gate cvp->cond_type = (uint16_t)type; 27587c478bd9Sstevel@tonic-gate cvp->cond_magic = COND_MAGIC; 27597c478bd9Sstevel@tonic-gate return (0); 27607c478bd9Sstevel@tonic-gate } 27617c478bd9Sstevel@tonic-gate 27627c478bd9Sstevel@tonic-gate /* 27637c478bd9Sstevel@tonic-gate * cond_sleep_queue(): utility function for cond_wait_queue(). 27647c478bd9Sstevel@tonic-gate * 27657c478bd9Sstevel@tonic-gate * Go to sleep on a condvar sleep queue, expect to be waked up 27667c478bd9Sstevel@tonic-gate * by someone calling cond_signal() or cond_broadcast() or due 27677c478bd9Sstevel@tonic-gate * to receiving a UNIX signal or being cancelled, or just simply 27687c478bd9Sstevel@tonic-gate * due to a spurious wakeup (like someome calling forkall()). 27697c478bd9Sstevel@tonic-gate * 27707c478bd9Sstevel@tonic-gate * The associated mutex is *not* reacquired before returning. 27717c478bd9Sstevel@tonic-gate * That must be done by the caller of cond_sleep_queue(). 27727c478bd9Sstevel@tonic-gate */ 2773883492d5Sraf static int 27747c478bd9Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 27757c478bd9Sstevel@tonic-gate { 27767c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 27777c478bd9Sstevel@tonic-gate queue_head_t *qp; 27787c478bd9Sstevel@tonic-gate queue_head_t *mqp; 27797c478bd9Sstevel@tonic-gate lwpid_t lwpid; 27807c478bd9Sstevel@tonic-gate int signalled; 27817c478bd9Sstevel@tonic-gate int error; 2782883492d5Sraf int release_all; 27837c478bd9Sstevel@tonic-gate 27847c478bd9Sstevel@tonic-gate /* 27857c478bd9Sstevel@tonic-gate * Put ourself on the CV sleep queue, unlock the mutex, then 27867c478bd9Sstevel@tonic-gate * park ourself and unpark a candidate lwp to grab the mutex. 27877c478bd9Sstevel@tonic-gate * We must go onto the CV sleep queue before dropping the 27887c478bd9Sstevel@tonic-gate * mutex in order to guarantee atomicity of the operation. 27897c478bd9Sstevel@tonic-gate */ 27907c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 27917c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 27927c478bd9Sstevel@tonic-gate enqueue(qp, self, cvp, CV); 27937c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 1; 27947c478bd9Sstevel@tonic-gate self->ul_cvmutex = mp; 27957c478bd9Sstevel@tonic-gate self->ul_cv_wake = (tsp != NULL); 27967c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 2797883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) { 2798883492d5Sraf mp->mutex_flag &= ~LOCK_OWNERDEAD; 2799883492d5Sraf mp->mutex_flag |= LOCK_NOTRECOVERABLE; 2800883492d5Sraf } 2801883492d5Sraf release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0); 2802883492d5Sraf lwpid = mutex_unlock_queue(mp, release_all); 28037c478bd9Sstevel@tonic-gate for (;;) { 28047c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 28057c478bd9Sstevel@tonic-gate queue_unlock(qp); 28067c478bd9Sstevel@tonic-gate if (lwpid != 0) { 28077c478bd9Sstevel@tonic-gate lwpid = preempt_unpark(self, lwpid); 28087c478bd9Sstevel@tonic-gate preempt(self); 28097c478bd9Sstevel@tonic-gate } 28107c478bd9Sstevel@tonic-gate /* 28117c478bd9Sstevel@tonic-gate * We may have a deferred signal present, 28127c478bd9Sstevel@tonic-gate * in which case we should return EINTR. 28137c478bd9Sstevel@tonic-gate * Also, we may have received a SIGCANCEL; if so 28147c478bd9Sstevel@tonic-gate * and we are cancelable we should return EINTR. 28157c478bd9Sstevel@tonic-gate * We force an immediate EINTR return from 28167c478bd9Sstevel@tonic-gate * __lwp_park() by turning our parking flag off. 28177c478bd9Sstevel@tonic-gate */ 28187c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 28197c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 28207c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 28217c478bd9Sstevel@tonic-gate /* 28227c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 28237c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 28247c478bd9Sstevel@tonic-gate */ 28257c478bd9Sstevel@tonic-gate error = __lwp_park(tsp, lwpid); 28267c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 28277c478bd9Sstevel@tonic-gate lwpid = 0; /* unpark the other lwp only once */ 28287c478bd9Sstevel@tonic-gate /* 28297c478bd9Sstevel@tonic-gate * We were waked up by cond_signal(), cond_broadcast(), 28307c478bd9Sstevel@tonic-gate * by an interrupt or timeout (EINTR or ETIME), 28317c478bd9Sstevel@tonic-gate * or we may just have gotten a spurious wakeup. 28327c478bd9Sstevel@tonic-gate */ 28337c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 28347c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 28357c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) 28367c478bd9Sstevel@tonic-gate break; 28377c478bd9Sstevel@tonic-gate /* 28387c478bd9Sstevel@tonic-gate * We are on either the condvar sleep queue or the 28392be60c5eSraf * mutex sleep queue. Break out of the sleep if we 28402be60c5eSraf * were interrupted or we timed out (EINTR or ETIME). 28417c478bd9Sstevel@tonic-gate * Else this is a spurious wakeup; continue the loop. 28427c478bd9Sstevel@tonic-gate */ 28432be60c5eSraf if (self->ul_sleepq == mqp) { /* mutex queue */ 28442be60c5eSraf if (error) { 28452be60c5eSraf mp->mutex_waiters = dequeue_self(mqp, mp); 28462be60c5eSraf break; 28472be60c5eSraf } 28482be60c5eSraf tsp = NULL; /* no more timeout */ 28492be60c5eSraf } else if (self->ul_sleepq == qp) { /* condvar queue */ 28507c478bd9Sstevel@tonic-gate if (error) { 28517c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = dequeue_self(qp, cvp); 28527c478bd9Sstevel@tonic-gate break; 28537c478bd9Sstevel@tonic-gate } 28547c478bd9Sstevel@tonic-gate /* 28557c478bd9Sstevel@tonic-gate * Else a spurious wakeup on the condvar queue. 28567c478bd9Sstevel@tonic-gate * __lwp_park() has already adjusted the timeout. 28577c478bd9Sstevel@tonic-gate */ 28587c478bd9Sstevel@tonic-gate } else { 28597c478bd9Sstevel@tonic-gate thr_panic("cond_sleep_queue(): thread not on queue"); 28607c478bd9Sstevel@tonic-gate } 28617c478bd9Sstevel@tonic-gate queue_unlock(mqp); 28627c478bd9Sstevel@tonic-gate } 28637c478bd9Sstevel@tonic-gate 28647c478bd9Sstevel@tonic-gate self->ul_sp = 0; 28657c478bd9Sstevel@tonic-gate ASSERT(self->ul_cvmutex == NULL && self->ul_cv_wake == 0); 28667c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 28677c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 28687c478bd9Sstevel@tonic-gate 28697c478bd9Sstevel@tonic-gate signalled = self->ul_signalled; 28707c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 28717c478bd9Sstevel@tonic-gate queue_unlock(qp); 28727c478bd9Sstevel@tonic-gate queue_unlock(mqp); 28737c478bd9Sstevel@tonic-gate 28747c478bd9Sstevel@tonic-gate /* 28757c478bd9Sstevel@tonic-gate * If we were concurrently cond_signal()d and any of: 28767c478bd9Sstevel@tonic-gate * received a UNIX signal, were cancelled, or got a timeout, 28777c478bd9Sstevel@tonic-gate * then perform another cond_signal() to avoid consuming it. 28787c478bd9Sstevel@tonic-gate */ 28797c478bd9Sstevel@tonic-gate if (error && signalled) 28807c478bd9Sstevel@tonic-gate (void) cond_signal_internal(cvp); 28817c478bd9Sstevel@tonic-gate 28827c478bd9Sstevel@tonic-gate return (error); 28837c478bd9Sstevel@tonic-gate } 28847c478bd9Sstevel@tonic-gate 28857c478bd9Sstevel@tonic-gate int 2886*5d1dd9a9Sraf cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 28877c478bd9Sstevel@tonic-gate { 28887c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 28897c478bd9Sstevel@tonic-gate int error; 2890883492d5Sraf int merror; 28917c478bd9Sstevel@tonic-gate 28927c478bd9Sstevel@tonic-gate /* 28937c478bd9Sstevel@tonic-gate * The old thread library was programmed to defer signals 28947c478bd9Sstevel@tonic-gate * while in cond_wait() so that the associated mutex would 28957c478bd9Sstevel@tonic-gate * be guaranteed to be held when the application signal 28967c478bd9Sstevel@tonic-gate * handler was invoked. 28977c478bd9Sstevel@tonic-gate * 28987c478bd9Sstevel@tonic-gate * We do not behave this way by default; the state of the 28997c478bd9Sstevel@tonic-gate * associated mutex in the signal handler is undefined. 29007c478bd9Sstevel@tonic-gate * 29017c478bd9Sstevel@tonic-gate * To accommodate applications that depend on the old 29027c478bd9Sstevel@tonic-gate * behavior, the _THREAD_COND_WAIT_DEFER environment 29037c478bd9Sstevel@tonic-gate * variable can be set to 1 and we will behave in the 29047c478bd9Sstevel@tonic-gate * old way with respect to cond_wait(). 29057c478bd9Sstevel@tonic-gate */ 29067c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 29077c478bd9Sstevel@tonic-gate sigoff(self); 29087c478bd9Sstevel@tonic-gate 29097c478bd9Sstevel@tonic-gate error = cond_sleep_queue(cvp, mp, tsp); 29107c478bd9Sstevel@tonic-gate 29117c478bd9Sstevel@tonic-gate /* 29127c478bd9Sstevel@tonic-gate * Reacquire the mutex. 29137c478bd9Sstevel@tonic-gate */ 2914*5d1dd9a9Sraf if ((merror = mutex_lock_impl(mp, NULL)) != 0) 2915883492d5Sraf error = merror; 29167c478bd9Sstevel@tonic-gate 29177c478bd9Sstevel@tonic-gate /* 29187c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 29197c478bd9Sstevel@tonic-gate */ 29207c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 29217c478bd9Sstevel@tonic-gate sigon(self); 29227c478bd9Sstevel@tonic-gate 29237c478bd9Sstevel@tonic-gate return (error); 29247c478bd9Sstevel@tonic-gate } 29257c478bd9Sstevel@tonic-gate 29267c478bd9Sstevel@tonic-gate /* 29277c478bd9Sstevel@tonic-gate * cond_sleep_kernel(): utility function for cond_wait_kernel(). 29287c478bd9Sstevel@tonic-gate * See the comment ahead of cond_sleep_queue(), above. 29297c478bd9Sstevel@tonic-gate */ 2930883492d5Sraf static int 29317c478bd9Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 29327c478bd9Sstevel@tonic-gate { 29337c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 29347c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 29357c478bd9Sstevel@tonic-gate int error; 29367c478bd9Sstevel@tonic-gate 2937883492d5Sraf if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp)) 2938883492d5Sraf _ceil_prio_waive(); 29397c478bd9Sstevel@tonic-gate 29407c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 29417c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 29427c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 29437c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 2944883492d5Sraf if (mtype & LOCK_PRIO_INHERIT) 29457c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 29467c478bd9Sstevel@tonic-gate /* 29477c478bd9Sstevel@tonic-gate * ___lwp_cond_wait() returns immediately with EINTR if 29487c478bd9Sstevel@tonic-gate * set_parking_flag(self,0) is called on this lwp before it 29497c478bd9Sstevel@tonic-gate * goes to sleep in the kernel. sigacthandler() calls this 29507c478bd9Sstevel@tonic-gate * when a deferred signal is noted. This assures that we don't 29517c478bd9Sstevel@tonic-gate * get stuck in ___lwp_cond_wait() with all signals blocked 29527c478bd9Sstevel@tonic-gate * due to taking a deferred signal before going to sleep. 29537c478bd9Sstevel@tonic-gate */ 29547c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 29557c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 29567c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 29577c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 29587c478bd9Sstevel@tonic-gate error = ___lwp_cond_wait(cvp, mp, tsp, 1); 29597c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 29607c478bd9Sstevel@tonic-gate self->ul_sp = 0; 29617c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 29627c478bd9Sstevel@tonic-gate return (error); 29637c478bd9Sstevel@tonic-gate } 29647c478bd9Sstevel@tonic-gate 29657c478bd9Sstevel@tonic-gate int 29667c478bd9Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 29677c478bd9Sstevel@tonic-gate { 29687c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 29697c478bd9Sstevel@tonic-gate int error; 29707c478bd9Sstevel@tonic-gate int merror; 29717c478bd9Sstevel@tonic-gate 29727c478bd9Sstevel@tonic-gate /* 29737c478bd9Sstevel@tonic-gate * See the large comment in cond_wait_queue(), above. 29747c478bd9Sstevel@tonic-gate */ 29757c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 29767c478bd9Sstevel@tonic-gate sigoff(self); 29777c478bd9Sstevel@tonic-gate 29787c478bd9Sstevel@tonic-gate error = cond_sleep_kernel(cvp, mp, tsp); 29797c478bd9Sstevel@tonic-gate 29807c478bd9Sstevel@tonic-gate /* 29817c478bd9Sstevel@tonic-gate * Override the return code from ___lwp_cond_wait() 29827c478bd9Sstevel@tonic-gate * with any non-zero return code from mutex_lock(). 29837c478bd9Sstevel@tonic-gate * This addresses robust lock failures in particular; 29847c478bd9Sstevel@tonic-gate * the caller must see the EOWNERDEAD or ENOTRECOVERABLE 29857c478bd9Sstevel@tonic-gate * errors in order to take corrective action. 29867c478bd9Sstevel@tonic-gate */ 2987*5d1dd9a9Sraf if ((merror = mutex_lock_impl(mp, NULL)) != 0) 29887c478bd9Sstevel@tonic-gate error = merror; 29897c478bd9Sstevel@tonic-gate 29907c478bd9Sstevel@tonic-gate /* 29917c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 29927c478bd9Sstevel@tonic-gate */ 29937c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 29947c478bd9Sstevel@tonic-gate sigon(self); 29957c478bd9Sstevel@tonic-gate 29967c478bd9Sstevel@tonic-gate return (error); 29977c478bd9Sstevel@tonic-gate } 29987c478bd9Sstevel@tonic-gate 29997c478bd9Sstevel@tonic-gate /* 30007c478bd9Sstevel@tonic-gate * Common code for _cond_wait() and _cond_timedwait() 30017c478bd9Sstevel@tonic-gate */ 30027c478bd9Sstevel@tonic-gate int 30037c478bd9Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 30047c478bd9Sstevel@tonic-gate { 30057c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 30067c478bd9Sstevel@tonic-gate hrtime_t begin_sleep = 0; 30077c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 30087c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 30097c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 30107c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 30117c478bd9Sstevel@tonic-gate uint8_t rcount; 30127c478bd9Sstevel@tonic-gate int error = 0; 30137c478bd9Sstevel@tonic-gate 30147c478bd9Sstevel@tonic-gate /* 30157c478bd9Sstevel@tonic-gate * The SUSV3 Posix spec for pthread_cond_timedwait() states: 30167c478bd9Sstevel@tonic-gate * Except in the case of [ETIMEDOUT], all these error checks 30177c478bd9Sstevel@tonic-gate * shall act as if they were performed immediately at the 30187c478bd9Sstevel@tonic-gate * beginning of processing for the function and shall cause 30197c478bd9Sstevel@tonic-gate * an error return, in effect, prior to modifying the state 30207c478bd9Sstevel@tonic-gate * of the mutex specified by mutex or the condition variable 30217c478bd9Sstevel@tonic-gate * specified by cond. 30227c478bd9Sstevel@tonic-gate * Therefore, we must return EINVAL now if the timout is invalid. 30237c478bd9Sstevel@tonic-gate */ 30247c478bd9Sstevel@tonic-gate if (tsp != NULL && 30257c478bd9Sstevel@tonic-gate (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC)) 30267c478bd9Sstevel@tonic-gate return (EINVAL); 30277c478bd9Sstevel@tonic-gate 30287c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 30297c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 30307c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 30317c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 30327c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = cvp; 30337c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 30347c478bd9Sstevel@tonic-gate self->ul_sp = 0; 30357c478bd9Sstevel@tonic-gate } 30367c478bd9Sstevel@tonic-gate if (csp) { 30377c478bd9Sstevel@tonic-gate if (tsp) 30387c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait); 30397c478bd9Sstevel@tonic-gate else 30407c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_wait); 30417c478bd9Sstevel@tonic-gate } 30427c478bd9Sstevel@tonic-gate if (msp) 30437c478bd9Sstevel@tonic-gate begin_sleep = record_hold_time(msp); 30447c478bd9Sstevel@tonic-gate else if (csp) 30457c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 30467c478bd9Sstevel@tonic-gate 30477c478bd9Sstevel@tonic-gate if (self->ul_error_detection) { 30487c478bd9Sstevel@tonic-gate if (!mutex_is_held(mp)) 30497c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, NULL); 30507c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) 30517c478bd9Sstevel@tonic-gate lock_error(mp, "recursive mutex in cond_wait", 3052*5d1dd9a9Sraf cvp, NULL); 30537c478bd9Sstevel@tonic-gate if (cvp->cond_type & USYNC_PROCESS) { 3054883492d5Sraf if (!(mtype & USYNC_PROCESS)) 30557c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 3056*5d1dd9a9Sraf "condvar process-shared, " 3057*5d1dd9a9Sraf "mutex process-private"); 30587c478bd9Sstevel@tonic-gate } else { 3059883492d5Sraf if (mtype & USYNC_PROCESS) 30607c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 3061*5d1dd9a9Sraf "condvar process-private, " 3062*5d1dd9a9Sraf "mutex process-shared"); 30637c478bd9Sstevel@tonic-gate } 30647c478bd9Sstevel@tonic-gate } 30657c478bd9Sstevel@tonic-gate 30667c478bd9Sstevel@tonic-gate /* 30677c478bd9Sstevel@tonic-gate * We deal with recursive mutexes by completely 30687c478bd9Sstevel@tonic-gate * dropping the lock and restoring the recursion 30697c478bd9Sstevel@tonic-gate * count after waking up. This is arguably wrong, 30707c478bd9Sstevel@tonic-gate * but it obeys the principle of least astonishment. 30717c478bd9Sstevel@tonic-gate */ 30727c478bd9Sstevel@tonic-gate rcount = mp->mutex_rcount; 30737c478bd9Sstevel@tonic-gate mp->mutex_rcount = 0; 3074883492d5Sraf if ((mtype & 3075883492d5Sraf (USYNC_PROCESS | LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) | 30767c478bd9Sstevel@tonic-gate (cvp->cond_type & USYNC_PROCESS)) 30777c478bd9Sstevel@tonic-gate error = cond_wait_kernel(cvp, mp, tsp); 30787c478bd9Sstevel@tonic-gate else 3079*5d1dd9a9Sraf error = cond_wait_queue(cvp, mp, tsp); 30807c478bd9Sstevel@tonic-gate mp->mutex_rcount = rcount; 30817c478bd9Sstevel@tonic-gate 30827c478bd9Sstevel@tonic-gate if (csp) { 30837c478bd9Sstevel@tonic-gate hrtime_t lapse = gethrtime() - begin_sleep; 30847c478bd9Sstevel@tonic-gate if (tsp == NULL) 30857c478bd9Sstevel@tonic-gate csp->cond_wait_sleep_time += lapse; 30867c478bd9Sstevel@tonic-gate else { 30877c478bd9Sstevel@tonic-gate csp->cond_timedwait_sleep_time += lapse; 30887c478bd9Sstevel@tonic-gate if (error == ETIME) 30897c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait_timeout); 30907c478bd9Sstevel@tonic-gate } 30917c478bd9Sstevel@tonic-gate } 30927c478bd9Sstevel@tonic-gate return (error); 30937c478bd9Sstevel@tonic-gate } 30947c478bd9Sstevel@tonic-gate 30957c478bd9Sstevel@tonic-gate /* 30967c478bd9Sstevel@tonic-gate * cond_wait() is a cancellation point but _cond_wait() is not. 30977c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 30987c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 30997c478bd9Sstevel@tonic-gate */ 31007c478bd9Sstevel@tonic-gate int 31017c478bd9Sstevel@tonic-gate _cond_wait(cond_t *cvp, mutex_t *mp) 31027c478bd9Sstevel@tonic-gate { 31037c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 31047c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 31057c478bd9Sstevel@tonic-gate uberflags_t *gflags; 31067c478bd9Sstevel@tonic-gate 31077c478bd9Sstevel@tonic-gate /* 31087c478bd9Sstevel@tonic-gate * Optimize the common case of USYNC_THREAD plus 31097c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, and no event tracing. 31107c478bd9Sstevel@tonic-gate */ 31117c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 31127c478bd9Sstevel@tonic-gate (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted | 31137c478bd9Sstevel@tonic-gate self->ul_td_events_enable | 31147c478bd9Sstevel@tonic-gate udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0) 3115*5d1dd9a9Sraf return (cond_wait_queue(cvp, mp, NULL)); 31167c478bd9Sstevel@tonic-gate 31177c478bd9Sstevel@tonic-gate /* 31187c478bd9Sstevel@tonic-gate * Else do it the long way. 31197c478bd9Sstevel@tonic-gate */ 31207c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, NULL)); 31217c478bd9Sstevel@tonic-gate } 31227c478bd9Sstevel@tonic-gate 31237c478bd9Sstevel@tonic-gate int 31247c478bd9Sstevel@tonic-gate cond_wait(cond_t *cvp, mutex_t *mp) 31257c478bd9Sstevel@tonic-gate { 31267c478bd9Sstevel@tonic-gate int error; 31277c478bd9Sstevel@tonic-gate 31287c478bd9Sstevel@tonic-gate _cancelon(); 31297c478bd9Sstevel@tonic-gate error = _cond_wait(cvp, mp); 31307c478bd9Sstevel@tonic-gate if (error == EINTR) 31317c478bd9Sstevel@tonic-gate _canceloff(); 31327c478bd9Sstevel@tonic-gate else 31337c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 31347c478bd9Sstevel@tonic-gate return (error); 31357c478bd9Sstevel@tonic-gate } 31367c478bd9Sstevel@tonic-gate 31377c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_wait = _pthread_cond_wait 31387c478bd9Sstevel@tonic-gate int 31397c478bd9Sstevel@tonic-gate _pthread_cond_wait(cond_t *cvp, mutex_t *mp) 31407c478bd9Sstevel@tonic-gate { 31417c478bd9Sstevel@tonic-gate int error; 31427c478bd9Sstevel@tonic-gate 31437c478bd9Sstevel@tonic-gate error = cond_wait(cvp, mp); 31447c478bd9Sstevel@tonic-gate return ((error == EINTR)? 0 : error); 31457c478bd9Sstevel@tonic-gate } 31467c478bd9Sstevel@tonic-gate 31477c478bd9Sstevel@tonic-gate /* 31487c478bd9Sstevel@tonic-gate * cond_timedwait() is a cancellation point but _cond_timedwait() is not. 31497c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 31507c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 31517c478bd9Sstevel@tonic-gate */ 31527c478bd9Sstevel@tonic-gate int 31537c478bd9Sstevel@tonic-gate _cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 31547c478bd9Sstevel@tonic-gate { 31557c478bd9Sstevel@tonic-gate clockid_t clock_id = cvp->cond_clockid; 31567c478bd9Sstevel@tonic-gate timespec_t reltime; 31577c478bd9Sstevel@tonic-gate int error; 31587c478bd9Sstevel@tonic-gate 31597c478bd9Sstevel@tonic-gate if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES) 31607c478bd9Sstevel@tonic-gate clock_id = CLOCK_REALTIME; 31617c478bd9Sstevel@tonic-gate abstime_to_reltime(clock_id, abstime, &reltime); 31627c478bd9Sstevel@tonic-gate error = cond_wait_common(cvp, mp, &reltime); 31637c478bd9Sstevel@tonic-gate if (error == ETIME && clock_id == CLOCK_HIGHRES) { 31647c478bd9Sstevel@tonic-gate /* 31657c478bd9Sstevel@tonic-gate * Don't return ETIME if we didn't really get a timeout. 31667c478bd9Sstevel@tonic-gate * This can happen if we return because someone resets 31677c478bd9Sstevel@tonic-gate * the system clock. Just return zero in this case, 31687c478bd9Sstevel@tonic-gate * giving a spurious wakeup but not a timeout. 31697c478bd9Sstevel@tonic-gate */ 31707c478bd9Sstevel@tonic-gate if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC + 31717c478bd9Sstevel@tonic-gate abstime->tv_nsec > gethrtime()) 31727c478bd9Sstevel@tonic-gate error = 0; 31737c478bd9Sstevel@tonic-gate } 31747c478bd9Sstevel@tonic-gate return (error); 31757c478bd9Sstevel@tonic-gate } 31767c478bd9Sstevel@tonic-gate 31777c478bd9Sstevel@tonic-gate int 31787c478bd9Sstevel@tonic-gate cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 31797c478bd9Sstevel@tonic-gate { 31807c478bd9Sstevel@tonic-gate int error; 31817c478bd9Sstevel@tonic-gate 31827c478bd9Sstevel@tonic-gate _cancelon(); 31837c478bd9Sstevel@tonic-gate error = _cond_timedwait(cvp, mp, abstime); 31847c478bd9Sstevel@tonic-gate if (error == EINTR) 31857c478bd9Sstevel@tonic-gate _canceloff(); 31867c478bd9Sstevel@tonic-gate else 31877c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 31887c478bd9Sstevel@tonic-gate return (error); 31897c478bd9Sstevel@tonic-gate } 31907c478bd9Sstevel@tonic-gate 31917c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_timedwait = _pthread_cond_timedwait 31927c478bd9Sstevel@tonic-gate int 31937c478bd9Sstevel@tonic-gate _pthread_cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 31947c478bd9Sstevel@tonic-gate { 31957c478bd9Sstevel@tonic-gate int error; 31967c478bd9Sstevel@tonic-gate 31977c478bd9Sstevel@tonic-gate error = cond_timedwait(cvp, mp, abstime); 31987c478bd9Sstevel@tonic-gate if (error == ETIME) 31997c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 32007c478bd9Sstevel@tonic-gate else if (error == EINTR) 32017c478bd9Sstevel@tonic-gate error = 0; 32027c478bd9Sstevel@tonic-gate return (error); 32037c478bd9Sstevel@tonic-gate } 32047c478bd9Sstevel@tonic-gate 32057c478bd9Sstevel@tonic-gate /* 32067c478bd9Sstevel@tonic-gate * cond_reltimedwait() is a cancellation point but _cond_reltimedwait() 32077c478bd9Sstevel@tonic-gate * is not. System libraries call the non-cancellation version. 32087c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 32097c478bd9Sstevel@tonic-gate */ 32107c478bd9Sstevel@tonic-gate int 32117c478bd9Sstevel@tonic-gate _cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 32127c478bd9Sstevel@tonic-gate { 32137c478bd9Sstevel@tonic-gate timespec_t tslocal = *reltime; 32147c478bd9Sstevel@tonic-gate 32157c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, &tslocal)); 32167c478bd9Sstevel@tonic-gate } 32177c478bd9Sstevel@tonic-gate 32187c478bd9Sstevel@tonic-gate #pragma weak cond_reltimedwait = _cond_reltimedwait_cancel 32197c478bd9Sstevel@tonic-gate int 32207c478bd9Sstevel@tonic-gate _cond_reltimedwait_cancel(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 32217c478bd9Sstevel@tonic-gate { 32227c478bd9Sstevel@tonic-gate int error; 32237c478bd9Sstevel@tonic-gate 32247c478bd9Sstevel@tonic-gate _cancelon(); 32257c478bd9Sstevel@tonic-gate error = _cond_reltimedwait(cvp, mp, reltime); 32267c478bd9Sstevel@tonic-gate if (error == EINTR) 32277c478bd9Sstevel@tonic-gate _canceloff(); 32287c478bd9Sstevel@tonic-gate else 32297c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 32307c478bd9Sstevel@tonic-gate return (error); 32317c478bd9Sstevel@tonic-gate } 32327c478bd9Sstevel@tonic-gate 32337c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_reltimedwait_np = _pthread_cond_reltimedwait_np 32347c478bd9Sstevel@tonic-gate int 32357c478bd9Sstevel@tonic-gate _pthread_cond_reltimedwait_np(cond_t *cvp, mutex_t *mp, 32367c478bd9Sstevel@tonic-gate const timespec_t *reltime) 32377c478bd9Sstevel@tonic-gate { 32387c478bd9Sstevel@tonic-gate int error; 32397c478bd9Sstevel@tonic-gate 32407c478bd9Sstevel@tonic-gate error = _cond_reltimedwait_cancel(cvp, mp, reltime); 32417c478bd9Sstevel@tonic-gate if (error == ETIME) 32427c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 32437c478bd9Sstevel@tonic-gate else if (error == EINTR) 32447c478bd9Sstevel@tonic-gate error = 0; 32457c478bd9Sstevel@tonic-gate return (error); 32467c478bd9Sstevel@tonic-gate } 32477c478bd9Sstevel@tonic-gate 32487c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_signal = cond_signal_internal 32497c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_signal = cond_signal_internal 32507c478bd9Sstevel@tonic-gate #pragma weak cond_signal = cond_signal_internal 32517c478bd9Sstevel@tonic-gate #pragma weak _cond_signal = cond_signal_internal 32527c478bd9Sstevel@tonic-gate int 32537c478bd9Sstevel@tonic-gate cond_signal_internal(cond_t *cvp) 32547c478bd9Sstevel@tonic-gate { 32557c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 32567c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 32577c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 32587c478bd9Sstevel@tonic-gate int error = 0; 32597c478bd9Sstevel@tonic-gate queue_head_t *qp; 32607c478bd9Sstevel@tonic-gate mutex_t *mp; 32617c478bd9Sstevel@tonic-gate queue_head_t *mqp; 32627c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 32637c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 32647c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 32657c478bd9Sstevel@tonic-gate ulwp_t *next; 32667c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 32677c478bd9Sstevel@tonic-gate ulwp_t *susprev; 32687c478bd9Sstevel@tonic-gate 32697c478bd9Sstevel@tonic-gate if (csp) 32707c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_signal); 32717c478bd9Sstevel@tonic-gate 32727c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 32737c478bd9Sstevel@tonic-gate error = __lwp_cond_signal(cvp); 32747c478bd9Sstevel@tonic-gate 32757c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 32767c478bd9Sstevel@tonic-gate return (error); 32777c478bd9Sstevel@tonic-gate 32787c478bd9Sstevel@tonic-gate /* 32797c478bd9Sstevel@tonic-gate * Move someone from the condvar sleep queue to the mutex sleep 32807c478bd9Sstevel@tonic-gate * queue for the mutex that he will acquire on being waked up. 32817c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex he will acquire. 32827c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if his ul_cv_wake flag 32837c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark him. 32847c478bd9Sstevel@tonic-gate */ 32857c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 32867c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 32877c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 32887c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == cvp) { 32897c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 32907c478bd9Sstevel@tonic-gate break; 32917c478bd9Sstevel@tonic-gate /* 32927c478bd9Sstevel@tonic-gate * Try not to dequeue a suspended thread. 32937c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 32947c478bd9Sstevel@tonic-gate */ 32957c478bd9Sstevel@tonic-gate if (suspp == NULL) { 32967c478bd9Sstevel@tonic-gate suspp = ulwpp; 32977c478bd9Sstevel@tonic-gate susprev = prev; 32987c478bd9Sstevel@tonic-gate } 32997c478bd9Sstevel@tonic-gate } 33007c478bd9Sstevel@tonic-gate } 33017c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 33027c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 33037c478bd9Sstevel@tonic-gate prev = susprev; 33047c478bd9Sstevel@tonic-gate suspp = NULL; 33057c478bd9Sstevel@tonic-gate } 33067c478bd9Sstevel@tonic-gate if (ulwp == NULL) { /* no one on the sleep queue */ 33077c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 33087c478bd9Sstevel@tonic-gate queue_unlock(qp); 33097c478bd9Sstevel@tonic-gate return (error); 33107c478bd9Sstevel@tonic-gate } 33117c478bd9Sstevel@tonic-gate /* 33127c478bd9Sstevel@tonic-gate * Scan the remainder of the CV queue for another waiter. 33137c478bd9Sstevel@tonic-gate */ 33147c478bd9Sstevel@tonic-gate if (suspp != NULL) { 33157c478bd9Sstevel@tonic-gate next = *suspp; 33167c478bd9Sstevel@tonic-gate } else { 33177c478bd9Sstevel@tonic-gate for (next = ulwp->ul_link; next != NULL; next = next->ul_link) 33187c478bd9Sstevel@tonic-gate if (next->ul_wchan == cvp) 33197c478bd9Sstevel@tonic-gate break; 33207c478bd9Sstevel@tonic-gate } 33217c478bd9Sstevel@tonic-gate if (next == NULL) 33227c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 33237c478bd9Sstevel@tonic-gate 33247c478bd9Sstevel@tonic-gate /* 33257c478bd9Sstevel@tonic-gate * Inform the thread that he was the recipient of a cond_signal(). 33267c478bd9Sstevel@tonic-gate * This lets him deal with cond_signal() and, concurrently, 33277c478bd9Sstevel@tonic-gate * one or more of a cancellation, a UNIX signal, or a timeout. 33287c478bd9Sstevel@tonic-gate * These latter conditions must not consume a cond_signal(). 33297c478bd9Sstevel@tonic-gate */ 33307c478bd9Sstevel@tonic-gate ulwp->ul_signalled = 1; 33317c478bd9Sstevel@tonic-gate 33327c478bd9Sstevel@tonic-gate /* 33337c478bd9Sstevel@tonic-gate * Dequeue the waiter but leave his ul_sleepq non-NULL 33347c478bd9Sstevel@tonic-gate * while we move him to the mutex queue so that he can 33357c478bd9Sstevel@tonic-gate * deal properly with spurious wakeups. 33367c478bd9Sstevel@tonic-gate */ 33377c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 3338883492d5Sraf ulwp->ul_link = NULL; 33397c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 33407c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 33417c478bd9Sstevel@tonic-gate qp->qh_qlen--; 33427c478bd9Sstevel@tonic-gate 33437c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* the mutex he will acquire */ 33447c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 33457c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 33467c478bd9Sstevel@tonic-gate 33477c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 33487c478bd9Sstevel@tonic-gate lwpid_t lwpid = ulwp->ul_lwpid; 33497c478bd9Sstevel@tonic-gate 33507c478bd9Sstevel@tonic-gate no_preempt(self); 33517c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 33527c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 33537c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 33547c478bd9Sstevel@tonic-gate queue_unlock(qp); 33557c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 33567c478bd9Sstevel@tonic-gate preempt(self); 33577c478bd9Sstevel@tonic-gate } else { 33587c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 33597c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 33607c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 33617c478bd9Sstevel@tonic-gate queue_unlock(mqp); 33627c478bd9Sstevel@tonic-gate queue_unlock(qp); 33637c478bd9Sstevel@tonic-gate } 33647c478bd9Sstevel@tonic-gate 33657c478bd9Sstevel@tonic-gate return (error); 33667c478bd9Sstevel@tonic-gate } 33677c478bd9Sstevel@tonic-gate 336841efec22Sraf /* 3369883492d5Sraf * Utility function called by mutex_wakeup_all(), cond_broadcast(), 3370883492d5Sraf * and rw_queue_release() to (re)allocate a big buffer to hold the 3371883492d5Sraf * lwpids of all the threads to be set running after they are removed 3372883492d5Sraf * from their sleep queues. Since we are holding a queue lock, we 3373883492d5Sraf * cannot call any function that might acquire a lock. mmap(), munmap(), 3374883492d5Sraf * lwp_unpark_all() are simple system calls and are safe in this regard. 337541efec22Sraf */ 337641efec22Sraf lwpid_t * 337741efec22Sraf alloc_lwpids(lwpid_t *lwpid, int *nlwpid_ptr, int *maxlwps_ptr) 337841efec22Sraf { 337941efec22Sraf /* 338041efec22Sraf * Allocate NEWLWPS ids on the first overflow. 338141efec22Sraf * Double the allocation each time after that. 338241efec22Sraf */ 338341efec22Sraf int nlwpid = *nlwpid_ptr; 338441efec22Sraf int maxlwps = *maxlwps_ptr; 338541efec22Sraf int first_allocation; 338641efec22Sraf int newlwps; 338741efec22Sraf void *vaddr; 338841efec22Sraf 338941efec22Sraf ASSERT(nlwpid == maxlwps); 339041efec22Sraf 339141efec22Sraf first_allocation = (maxlwps == MAXLWPS); 339241efec22Sraf newlwps = first_allocation? NEWLWPS : 2 * maxlwps; 339341efec22Sraf vaddr = _private_mmap(NULL, newlwps * sizeof (lwpid_t), 339441efec22Sraf PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0); 339541efec22Sraf 339641efec22Sraf if (vaddr == MAP_FAILED) { 339741efec22Sraf /* 339841efec22Sraf * Let's hope this never happens. 339941efec22Sraf * If it does, then we have a terrible 340041efec22Sraf * thundering herd on our hands. 340141efec22Sraf */ 340241efec22Sraf (void) __lwp_unpark_all(lwpid, nlwpid); 340341efec22Sraf *nlwpid_ptr = 0; 340441efec22Sraf } else { 340541efec22Sraf (void) _memcpy(vaddr, lwpid, maxlwps * sizeof (lwpid_t)); 340641efec22Sraf if (!first_allocation) 340741efec22Sraf (void) _private_munmap(lwpid, 340841efec22Sraf maxlwps * sizeof (lwpid_t)); 340941efec22Sraf lwpid = vaddr; 341041efec22Sraf *maxlwps_ptr = newlwps; 341141efec22Sraf } 341241efec22Sraf 341341efec22Sraf return (lwpid); 341441efec22Sraf } 34157c478bd9Sstevel@tonic-gate 34167c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_broadcast = cond_broadcast_internal 34177c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_broadcast = cond_broadcast_internal 34187c478bd9Sstevel@tonic-gate #pragma weak cond_broadcast = cond_broadcast_internal 34197c478bd9Sstevel@tonic-gate #pragma weak _cond_broadcast = cond_broadcast_internal 34207c478bd9Sstevel@tonic-gate int 34217c478bd9Sstevel@tonic-gate cond_broadcast_internal(cond_t *cvp) 34227c478bd9Sstevel@tonic-gate { 34237c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 34247c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 34257c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 34267c478bd9Sstevel@tonic-gate int error = 0; 34277c478bd9Sstevel@tonic-gate queue_head_t *qp; 34287c478bd9Sstevel@tonic-gate mutex_t *mp; 34297c478bd9Sstevel@tonic-gate mutex_t *mp_cache = NULL; 343041efec22Sraf queue_head_t *mqp = NULL; 34317c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 34327c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 34337c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 34347c478bd9Sstevel@tonic-gate int nlwpid = 0; 34357c478bd9Sstevel@tonic-gate int maxlwps = MAXLWPS; 343641efec22Sraf lwpid_t buffer[MAXLWPS]; 343741efec22Sraf lwpid_t *lwpid = buffer; 34387c478bd9Sstevel@tonic-gate 34397c478bd9Sstevel@tonic-gate if (csp) 34407c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_broadcast); 34417c478bd9Sstevel@tonic-gate 34427c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 34437c478bd9Sstevel@tonic-gate error = __lwp_cond_broadcast(cvp); 34447c478bd9Sstevel@tonic-gate 34457c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 34467c478bd9Sstevel@tonic-gate return (error); 34477c478bd9Sstevel@tonic-gate 34487c478bd9Sstevel@tonic-gate /* 34497c478bd9Sstevel@tonic-gate * Move everyone from the condvar sleep queue to the mutex sleep 34507c478bd9Sstevel@tonic-gate * queue for the mutex that they will acquire on being waked up. 34517c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex they will acquire. 34527c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if their ul_cv_wake flag 34537c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark them. 34547c478bd9Sstevel@tonic-gate * 34557c478bd9Sstevel@tonic-gate * We keep track of lwpids that are to be unparked in lwpid[]. 34567c478bd9Sstevel@tonic-gate * __lwp_unpark_all() is called to unpark all of them after 34577c478bd9Sstevel@tonic-gate * they have been removed from the sleep queue and the sleep 34587c478bd9Sstevel@tonic-gate * queue lock has been dropped. If we run out of space in our 34597c478bd9Sstevel@tonic-gate * on-stack buffer, we need to allocate more but we can't call 34607c478bd9Sstevel@tonic-gate * lmalloc() because we are holding a queue lock when the overflow 34617c478bd9Sstevel@tonic-gate * occurs and lmalloc() acquires a lock. We can't use alloca() 346241efec22Sraf * either because the application may have allocated a small 346341efec22Sraf * stack and we don't want to overrun the stack. So we call 346441efec22Sraf * alloc_lwpids() to allocate a bigger buffer using the mmap() 34657c478bd9Sstevel@tonic-gate * system call directly since that path acquires no locks. 34667c478bd9Sstevel@tonic-gate */ 34677c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 34687c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 34697c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 34707c478bd9Sstevel@tonic-gate while ((ulwp = *ulwpp) != NULL) { 34717c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan != cvp) { 34727c478bd9Sstevel@tonic-gate prev = ulwp; 34737c478bd9Sstevel@tonic-gate ulwpp = &ulwp->ul_link; 34747c478bd9Sstevel@tonic-gate continue; 34757c478bd9Sstevel@tonic-gate } 34767c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 3477883492d5Sraf ulwp->ul_link = NULL; 34787c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 34797c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 34807c478bd9Sstevel@tonic-gate qp->qh_qlen--; 34817c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* his mutex */ 34827c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 34837c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 34847c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 34857c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 34867c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 34877c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 348841efec22Sraf if (nlwpid == maxlwps) 348941efec22Sraf lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps); 34907c478bd9Sstevel@tonic-gate lwpid[nlwpid++] = ulwp->ul_lwpid; 34917c478bd9Sstevel@tonic-gate } else { 34927c478bd9Sstevel@tonic-gate if (mp != mp_cache) { 34937c478bd9Sstevel@tonic-gate mp_cache = mp; 349441efec22Sraf if (mqp != NULL) 349541efec22Sraf queue_unlock(mqp); 349641efec22Sraf mqp = queue_lock(mp, MX); 34977c478bd9Sstevel@tonic-gate } 34987c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 34997c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 35007c478bd9Sstevel@tonic-gate } 35017c478bd9Sstevel@tonic-gate } 350241efec22Sraf if (mqp != NULL) 350341efec22Sraf queue_unlock(mqp); 350441efec22Sraf if (nlwpid == 0) { 350541efec22Sraf queue_unlock(qp); 350641efec22Sraf } else { 350741efec22Sraf no_preempt(self); 350841efec22Sraf queue_unlock(qp); 35097c478bd9Sstevel@tonic-gate if (nlwpid == 1) 35107c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid[0]); 35117c478bd9Sstevel@tonic-gate else 35127c478bd9Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid); 351341efec22Sraf preempt(self); 35147c478bd9Sstevel@tonic-gate } 35157c478bd9Sstevel@tonic-gate if (lwpid != buffer) 35167c478bd9Sstevel@tonic-gate (void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t)); 35177c478bd9Sstevel@tonic-gate return (error); 35187c478bd9Sstevel@tonic-gate } 35197c478bd9Sstevel@tonic-gate 35207c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_destroy = _cond_destroy 35217c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_destroy = _cond_destroy 35227c478bd9Sstevel@tonic-gate #pragma weak cond_destroy = _cond_destroy 35237c478bd9Sstevel@tonic-gate int 35247c478bd9Sstevel@tonic-gate _cond_destroy(cond_t *cvp) 35257c478bd9Sstevel@tonic-gate { 35267c478bd9Sstevel@tonic-gate cvp->cond_magic = 0; 35277c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(cvp); 35287c478bd9Sstevel@tonic-gate return (0); 35297c478bd9Sstevel@tonic-gate } 35307c478bd9Sstevel@tonic-gate 35317c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 35327c478bd9Sstevel@tonic-gate void 35337c478bd9Sstevel@tonic-gate assert_no_libc_locks_held(void) 35347c478bd9Sstevel@tonic-gate { 35357c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 35367c478bd9Sstevel@tonic-gate } 35377c478bd9Sstevel@tonic-gate #endif 35387c478bd9Sstevel@tonic-gate 35397c478bd9Sstevel@tonic-gate /* protected by link_lock */ 35407c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin; 35417c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin2; 35427c478bd9Sstevel@tonic-gate uint64_t spin_lock_sleep; 35437c478bd9Sstevel@tonic-gate uint64_t spin_lock_wakeup; 35447c478bd9Sstevel@tonic-gate 35457c478bd9Sstevel@tonic-gate /* 35467c478bd9Sstevel@tonic-gate * Record spin lock statistics. 35477c478bd9Sstevel@tonic-gate * Called by a thread exiting itself in thrp_exit(). 35487c478bd9Sstevel@tonic-gate * Also called via atexit() from the thread calling 35497c478bd9Sstevel@tonic-gate * exit() to do all the other threads as well. 35507c478bd9Sstevel@tonic-gate */ 35517c478bd9Sstevel@tonic-gate void 35527c478bd9Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp) 35537c478bd9Sstevel@tonic-gate { 35547c478bd9Sstevel@tonic-gate spin_lock_spin += ulwp->ul_spin_lock_spin; 35557c478bd9Sstevel@tonic-gate spin_lock_spin2 += ulwp->ul_spin_lock_spin2; 35567c478bd9Sstevel@tonic-gate spin_lock_sleep += ulwp->ul_spin_lock_sleep; 35577c478bd9Sstevel@tonic-gate spin_lock_wakeup += ulwp->ul_spin_lock_wakeup; 35587c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin = 0; 35597c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin2 = 0; 35607c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_sleep = 0; 35617c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_wakeup = 0; 35627c478bd9Sstevel@tonic-gate } 35637c478bd9Sstevel@tonic-gate 35647c478bd9Sstevel@tonic-gate /* 35657c478bd9Sstevel@tonic-gate * atexit function: dump the queue statistics to stderr. 35667c478bd9Sstevel@tonic-gate */ 3567e8031f0aSraf #if !defined(__lint) 3568e8031f0aSraf #define fprintf _fprintf 3569e8031f0aSraf #endif 35707c478bd9Sstevel@tonic-gate #include <stdio.h> 35717c478bd9Sstevel@tonic-gate void 35727c478bd9Sstevel@tonic-gate dump_queue_statistics(void) 35737c478bd9Sstevel@tonic-gate { 35747c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 35757c478bd9Sstevel@tonic-gate queue_head_t *qp; 35767c478bd9Sstevel@tonic-gate int qn; 35777c478bd9Sstevel@tonic-gate uint64_t spin_lock_total = 0; 35787c478bd9Sstevel@tonic-gate 35797c478bd9Sstevel@tonic-gate if (udp->queue_head == NULL || thread_queue_dump == 0) 35807c478bd9Sstevel@tonic-gate return; 35817c478bd9Sstevel@tonic-gate 35827c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 || 35837c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 35847c478bd9Sstevel@tonic-gate return; 35857c478bd9Sstevel@tonic-gate for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) { 35867c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 35877c478bd9Sstevel@tonic-gate continue; 35887c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 35897c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 3590*5d1dd9a9Sraf (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 3591*5d1dd9a9Sraf return; 35927c478bd9Sstevel@tonic-gate } 35937c478bd9Sstevel@tonic-gate 35947c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 || 35957c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 35967c478bd9Sstevel@tonic-gate return; 35977c478bd9Sstevel@tonic-gate for (qn = 0; qn < QHASHSIZE; qn++, qp++) { 35987c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 35997c478bd9Sstevel@tonic-gate continue; 36007c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 36017c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 3602*5d1dd9a9Sraf (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 3603*5d1dd9a9Sraf return; 36047c478bd9Sstevel@tonic-gate } 36057c478bd9Sstevel@tonic-gate 36067c478bd9Sstevel@tonic-gate (void) fprintf(stderr, "\n spin_lock_total = %10llu\n", 3607*5d1dd9a9Sraf (u_longlong_t)spin_lock_total); 36087c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin = %10llu\n", 3609*5d1dd9a9Sraf (u_longlong_t)spin_lock_spin); 36107c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin2 = %10llu\n", 3611*5d1dd9a9Sraf (u_longlong_t)spin_lock_spin2); 36127c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_sleep = %10llu\n", 3613*5d1dd9a9Sraf (u_longlong_t)spin_lock_sleep); 36147c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_wakeup = %10llu\n", 3615*5d1dd9a9Sraf (u_longlong_t)spin_lock_wakeup); 36167c478bd9Sstevel@tonic-gate } 3617