17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 52be60c5eSraf * Common Development and Distribution License (the "License"). 62be60c5eSraf * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21e8031f0aSraf 227c478bd9Sstevel@tonic-gate /* 23a574db85Sraf * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 2931db3c26Sraf #define atomic_cas_64 _atomic_cas_64 307c478bd9Sstevel@tonic-gate 317c478bd9Sstevel@tonic-gate #include "lint.h" 327c478bd9Sstevel@tonic-gate #include "thr_uberdata.h" 33d4204c85Sraf #include <sys/rtpriocntl.h> 3431db3c26Sraf #include <sys/sdt.h> 3531db3c26Sraf #include <atomic.h> 367c478bd9Sstevel@tonic-gate 37d4204c85Sraf #if defined(THREAD_DEBUG) 38d4204c85Sraf #define INCR32(x) (((x) != UINT32_MAX)? (x)++ : 0) 39d4204c85Sraf #define INCR(x) ((x)++) 40d4204c85Sraf #define DECR(x) ((x)--) 41d4204c85Sraf #define MAXINCR(m, x) ((m < ++x)? (m = x) : 0) 42d4204c85Sraf #else 43d4204c85Sraf #define INCR32(x) 44d4204c85Sraf #define INCR(x) 45d4204c85Sraf #define DECR(x) 46d4204c85Sraf #define MAXINCR(m, x) 47d4204c85Sraf #endif 48d4204c85Sraf 497c478bd9Sstevel@tonic-gate /* 507c478bd9Sstevel@tonic-gate * This mutex is initialized to be held by lwp#1. 517c478bd9Sstevel@tonic-gate * It is used to block a thread that has returned from a mutex_lock() 52883492d5Sraf * of a LOCK_PRIO_INHERIT mutex with an unrecoverable error. 537c478bd9Sstevel@tonic-gate */ 547c478bd9Sstevel@tonic-gate mutex_t stall_mutex = DEFAULTMUTEX; 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate static int shared_mutex_held(mutex_t *); 57883492d5Sraf static int mutex_queuelock_adaptive(mutex_t *); 58883492d5Sraf static void mutex_wakeup_all(mutex_t *); 597c478bd9Sstevel@tonic-gate 607c478bd9Sstevel@tonic-gate /* 617c478bd9Sstevel@tonic-gate * Lock statistics support functions. 627c478bd9Sstevel@tonic-gate */ 637c478bd9Sstevel@tonic-gate void 647c478bd9Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp) 657c478bd9Sstevel@tonic-gate { 667c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_lock); 677c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = gethrtime(); 687c478bd9Sstevel@tonic-gate } 697c478bd9Sstevel@tonic-gate 707c478bd9Sstevel@tonic-gate hrtime_t 717c478bd9Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp) 727c478bd9Sstevel@tonic-gate { 737c478bd9Sstevel@tonic-gate hrtime_t now = gethrtime(); 747c478bd9Sstevel@tonic-gate 757c478bd9Sstevel@tonic-gate if (msp->mutex_begin_hold) 767c478bd9Sstevel@tonic-gate msp->mutex_hold_time += now - msp->mutex_begin_hold; 777c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = 0; 787c478bd9Sstevel@tonic-gate return (now); 797c478bd9Sstevel@tonic-gate } 807c478bd9Sstevel@tonic-gate 817c478bd9Sstevel@tonic-gate /* 827c478bd9Sstevel@tonic-gate * Called once at library initialization. 837c478bd9Sstevel@tonic-gate */ 847c478bd9Sstevel@tonic-gate void 857c478bd9Sstevel@tonic-gate mutex_setup(void) 867c478bd9Sstevel@tonic-gate { 877c478bd9Sstevel@tonic-gate if (set_lock_byte(&stall_mutex.mutex_lockw)) 887c478bd9Sstevel@tonic-gate thr_panic("mutex_setup() cannot acquire stall_mutex"); 897c478bd9Sstevel@tonic-gate stall_mutex.mutex_owner = (uintptr_t)curthread; 907c478bd9Sstevel@tonic-gate } 917c478bd9Sstevel@tonic-gate 927c478bd9Sstevel@tonic-gate /* 935d1dd9a9Sraf * The default spin count of 1000 is experimentally determined. 945d1dd9a9Sraf * On sun4u machines with any number of processors it could be raised 957c478bd9Sstevel@tonic-gate * to 10,000 but that (experimentally) makes almost no difference. 965d1dd9a9Sraf * The environment variable: 977c478bd9Sstevel@tonic-gate * _THREAD_ADAPTIVE_SPIN=count 985d1dd9a9Sraf * can be used to override and set the count in the range [0 .. 1,000,000]. 997c478bd9Sstevel@tonic-gate */ 1007c478bd9Sstevel@tonic-gate int thread_adaptive_spin = 1000; 1017c478bd9Sstevel@tonic-gate uint_t thread_max_spinners = 100; 1027c478bd9Sstevel@tonic-gate int thread_queue_verify = 0; 1037c478bd9Sstevel@tonic-gate static int ncpus; 1047c478bd9Sstevel@tonic-gate 1057c478bd9Sstevel@tonic-gate /* 1067c478bd9Sstevel@tonic-gate * Distinguish spinning for queue locks from spinning for regular locks. 1075d1dd9a9Sraf * We try harder to acquire queue locks by spinning. 1087c478bd9Sstevel@tonic-gate * The environment variable: 1097c478bd9Sstevel@tonic-gate * _THREAD_QUEUE_SPIN=count 1107c478bd9Sstevel@tonic-gate * can be used to override and set the count in the range [0 .. 1,000,000]. 1117c478bd9Sstevel@tonic-gate */ 1125d1dd9a9Sraf int thread_queue_spin = 10000; 1137c478bd9Sstevel@tonic-gate 114883492d5Sraf #define ALL_ATTRIBUTES \ 115883492d5Sraf (LOCK_RECURSIVE | LOCK_ERRORCHECK | \ 116883492d5Sraf LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT | \ 117883492d5Sraf LOCK_ROBUST) 1187c478bd9Sstevel@tonic-gate 1197c478bd9Sstevel@tonic-gate /* 120883492d5Sraf * 'type' can be one of USYNC_THREAD, USYNC_PROCESS, or USYNC_PROCESS_ROBUST, 121883492d5Sraf * augmented by zero or more the flags: 122883492d5Sraf * LOCK_RECURSIVE 123883492d5Sraf * LOCK_ERRORCHECK 124883492d5Sraf * LOCK_PRIO_INHERIT 125883492d5Sraf * LOCK_PRIO_PROTECT 126883492d5Sraf * LOCK_ROBUST 1277c478bd9Sstevel@tonic-gate */ 1287c478bd9Sstevel@tonic-gate #pragma weak mutex_init = __mutex_init 1297c478bd9Sstevel@tonic-gate #pragma weak _mutex_init = __mutex_init 1307c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 1317c478bd9Sstevel@tonic-gate int 1327c478bd9Sstevel@tonic-gate __mutex_init(mutex_t *mp, int type, void *arg) 1337c478bd9Sstevel@tonic-gate { 134883492d5Sraf int basetype = (type & ~ALL_ATTRIBUTES); 135d4204c85Sraf const pcclass_t *pccp; 136883492d5Sraf int error = 0; 137d4204c85Sraf int ceil; 138883492d5Sraf 139883492d5Sraf if (basetype == USYNC_PROCESS_ROBUST) { 140883492d5Sraf /* 141883492d5Sraf * USYNC_PROCESS_ROBUST is a deprecated historical type. 142883492d5Sraf * We change it into (USYNC_PROCESS | LOCK_ROBUST) but 143883492d5Sraf * retain the USYNC_PROCESS_ROBUST flag so we can return 144883492d5Sraf * ELOCKUNMAPPED when necessary (only USYNC_PROCESS_ROBUST 145883492d5Sraf * mutexes will ever draw ELOCKUNMAPPED). 146883492d5Sraf */ 147883492d5Sraf type |= (USYNC_PROCESS | LOCK_ROBUST); 148883492d5Sraf basetype = USYNC_PROCESS; 149883492d5Sraf } 1507c478bd9Sstevel@tonic-gate 151d4204c85Sraf if (type & LOCK_PRIO_PROTECT) 152d4204c85Sraf pccp = get_info_by_policy(SCHED_FIFO); 153d4204c85Sraf if ((basetype != USYNC_THREAD && basetype != USYNC_PROCESS) || 154883492d5Sraf (type & (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) 155d4204c85Sraf == (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT) || 156d4204c85Sraf ((type & LOCK_PRIO_PROTECT) && 157d4204c85Sraf ((ceil = *(int *)arg) < pccp->pcc_primin || 158d4204c85Sraf ceil > pccp->pcc_primax))) { 159883492d5Sraf error = EINVAL; 160883492d5Sraf } else if (type & LOCK_ROBUST) { 161883492d5Sraf /* 162883492d5Sraf * Callers of mutex_init() with the LOCK_ROBUST attribute 163883492d5Sraf * are required to pass an initially all-zero mutex. 164883492d5Sraf * Multiple calls to mutex_init() are allowed; all but 165883492d5Sraf * the first return EBUSY. A call to mutex_init() is 166883492d5Sraf * allowed to make an inconsistent robust lock consistent 167883492d5Sraf * (for historical usage, even though the proper interface 168883492d5Sraf * for this is mutex_consistent()). Note that we use 169883492d5Sraf * atomic_or_16() to set the LOCK_INITED flag so as 170883492d5Sraf * not to disturb surrounding bits (LOCK_OWNERDEAD, etc). 171883492d5Sraf */ 172883492d5Sraf extern void _atomic_or_16(volatile uint16_t *, uint16_t); 173883492d5Sraf if (!(mp->mutex_flag & LOCK_INITED)) { 174883492d5Sraf mp->mutex_type = (uint8_t)type; 175883492d5Sraf _atomic_or_16(&mp->mutex_flag, LOCK_INITED); 176883492d5Sraf mp->mutex_magic = MUTEX_MAGIC; 177883492d5Sraf } else if (type != mp->mutex_type || 178d4204c85Sraf ((type & LOCK_PRIO_PROTECT) && mp->mutex_ceiling != ceil)) { 179883492d5Sraf error = EINVAL; 180883492d5Sraf } else if (__mutex_consistent(mp) != 0) { 181883492d5Sraf error = EBUSY; 182883492d5Sraf } 183883492d5Sraf /* register a process robust mutex with the kernel */ 184883492d5Sraf if (basetype == USYNC_PROCESS) 185883492d5Sraf register_lock(mp); 186883492d5Sraf } else { 187*8cd45542Sraf (void) memset(mp, 0, sizeof (*mp)); 1887c478bd9Sstevel@tonic-gate mp->mutex_type = (uint8_t)type; 1897c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 1907c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 191883492d5Sraf } 192883492d5Sraf 193d4204c85Sraf if (error == 0 && (type & LOCK_PRIO_PROTECT)) { 194d4204c85Sraf mp->mutex_ceiling = ceil; 195d4204c85Sraf } 196883492d5Sraf 1977c478bd9Sstevel@tonic-gate return (error); 1987c478bd9Sstevel@tonic-gate } 1997c478bd9Sstevel@tonic-gate 2007c478bd9Sstevel@tonic-gate /* 201d4204c85Sraf * Delete mp from list of ceiling mutexes owned by curthread. 2027c478bd9Sstevel@tonic-gate * Return 1 if the head of the chain was updated. 2037c478bd9Sstevel@tonic-gate */ 2047c478bd9Sstevel@tonic-gate int 2057c478bd9Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp) 2067c478bd9Sstevel@tonic-gate { 2077c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2087c478bd9Sstevel@tonic-gate mxchain_t **mcpp; 2097c478bd9Sstevel@tonic-gate mxchain_t *mcp; 2107c478bd9Sstevel@tonic-gate 211d4204c85Sraf for (mcpp = &self->ul_mxchain; 212d4204c85Sraf (mcp = *mcpp) != NULL; 213d4204c85Sraf mcpp = &mcp->mxchain_next) { 214d4204c85Sraf if (mcp->mxchain_mx == mp) { 215d4204c85Sraf *mcpp = mcp->mxchain_next; 216d4204c85Sraf lfree(mcp, sizeof (*mcp)); 217d4204c85Sraf return (mcpp == &self->ul_mxchain); 218d4204c85Sraf } 219d4204c85Sraf } 220d4204c85Sraf return (0); 2217c478bd9Sstevel@tonic-gate } 2227c478bd9Sstevel@tonic-gate 2237c478bd9Sstevel@tonic-gate /* 224d4204c85Sraf * Add mp to the list of ceiling mutexes owned by curthread. 2257c478bd9Sstevel@tonic-gate * Return ENOMEM if no memory could be allocated. 2267c478bd9Sstevel@tonic-gate */ 2277c478bd9Sstevel@tonic-gate int 2287c478bd9Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp) 2297c478bd9Sstevel@tonic-gate { 2307c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2317c478bd9Sstevel@tonic-gate mxchain_t *mcp; 2327c478bd9Sstevel@tonic-gate 2337c478bd9Sstevel@tonic-gate if ((mcp = lmalloc(sizeof (*mcp))) == NULL) 2347c478bd9Sstevel@tonic-gate return (ENOMEM); 2357c478bd9Sstevel@tonic-gate mcp->mxchain_mx = mp; 2367c478bd9Sstevel@tonic-gate mcp->mxchain_next = self->ul_mxchain; 2377c478bd9Sstevel@tonic-gate self->ul_mxchain = mcp; 2387c478bd9Sstevel@tonic-gate return (0); 2397c478bd9Sstevel@tonic-gate } 2407c478bd9Sstevel@tonic-gate 2417c478bd9Sstevel@tonic-gate /* 242d4204c85Sraf * Helper function for _ceil_prio_inherit() and _ceil_prio_waive(), below. 243d4204c85Sraf */ 244d4204c85Sraf static void 245d4204c85Sraf set_rt_priority(ulwp_t *self, int prio) 246d4204c85Sraf { 247d4204c85Sraf pcparms_t pcparm; 248d4204c85Sraf 249d4204c85Sraf pcparm.pc_cid = self->ul_rtclassid; 250d4204c85Sraf ((rtparms_t *)pcparm.pc_clparms)->rt_tqnsecs = RT_NOCHANGE; 251d4204c85Sraf ((rtparms_t *)pcparm.pc_clparms)->rt_pri = prio; 252*8cd45542Sraf (void) priocntl(P_LWPID, self->ul_lwpid, PC_SETPARMS, &pcparm); 253d4204c85Sraf } 254d4204c85Sraf 255d4204c85Sraf /* 256d4204c85Sraf * Inherit priority from ceiling. 257d4204c85Sraf * This changes the effective priority, not the assigned priority. 2587c478bd9Sstevel@tonic-gate */ 2597c478bd9Sstevel@tonic-gate void 260d4204c85Sraf _ceil_prio_inherit(int prio) 2617c478bd9Sstevel@tonic-gate { 2627c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2637c478bd9Sstevel@tonic-gate 264d4204c85Sraf self->ul_epri = prio; 265d4204c85Sraf set_rt_priority(self, prio); 2667c478bd9Sstevel@tonic-gate } 2677c478bd9Sstevel@tonic-gate 2687c478bd9Sstevel@tonic-gate /* 2697c478bd9Sstevel@tonic-gate * Waive inherited ceiling priority. Inherit from head of owned ceiling locks 2707c478bd9Sstevel@tonic-gate * if holding at least one ceiling lock. If no ceiling locks are held at this 2717c478bd9Sstevel@tonic-gate * point, disinherit completely, reverting back to assigned priority. 2727c478bd9Sstevel@tonic-gate */ 2737c478bd9Sstevel@tonic-gate void 2747c478bd9Sstevel@tonic-gate _ceil_prio_waive(void) 2757c478bd9Sstevel@tonic-gate { 2767c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 277d4204c85Sraf mxchain_t *mcp = self->ul_mxchain; 278d4204c85Sraf int prio; 2797c478bd9Sstevel@tonic-gate 280d4204c85Sraf if (mcp == NULL) { 281d4204c85Sraf prio = self->ul_pri; 282d4204c85Sraf self->ul_epri = 0; 2837c478bd9Sstevel@tonic-gate } else { 284d4204c85Sraf prio = mcp->mxchain_mx->mutex_ceiling; 285d4204c85Sraf self->ul_epri = prio; 2867c478bd9Sstevel@tonic-gate } 287d4204c85Sraf set_rt_priority(self, prio); 2887c478bd9Sstevel@tonic-gate } 2897c478bd9Sstevel@tonic-gate 2905d1dd9a9Sraf /* 2915d1dd9a9Sraf * Clear the lock byte. Retain the waiters byte and the spinners byte. 2925d1dd9a9Sraf * Return the old value of the lock word. 2935d1dd9a9Sraf */ 2945d1dd9a9Sraf static uint32_t 2955d1dd9a9Sraf clear_lockbyte(volatile uint32_t *lockword) 2965d1dd9a9Sraf { 2975d1dd9a9Sraf uint32_t old; 2985d1dd9a9Sraf uint32_t new; 2995d1dd9a9Sraf 3005d1dd9a9Sraf do { 3015d1dd9a9Sraf old = *lockword; 3025d1dd9a9Sraf new = old & ~LOCKMASK; 3035d1dd9a9Sraf } while (atomic_cas_32(lockword, old, new) != old); 3045d1dd9a9Sraf 3055d1dd9a9Sraf return (old); 3065d1dd9a9Sraf } 3075d1dd9a9Sraf 30831db3c26Sraf /* 30931db3c26Sraf * Same as clear_lockbyte(), but operates on mutex_lockword64. 31031db3c26Sraf * The mutex_ownerpid field is cleared along with the lock byte. 31131db3c26Sraf */ 31231db3c26Sraf static uint64_t 31331db3c26Sraf clear_lockbyte64(volatile uint64_t *lockword64) 31431db3c26Sraf { 31531db3c26Sraf uint64_t old; 31631db3c26Sraf uint64_t new; 31731db3c26Sraf 31831db3c26Sraf do { 31931db3c26Sraf old = *lockword64; 32031db3c26Sraf new = old & ~LOCKMASK64; 32131db3c26Sraf } while (atomic_cas_64(lockword64, old, new) != old); 32231db3c26Sraf 32331db3c26Sraf return (old); 32431db3c26Sraf } 32531db3c26Sraf 32631db3c26Sraf /* 32731db3c26Sraf * Similar to set_lock_byte(), which only tries to set the lock byte. 32831db3c26Sraf * Here, we attempt to set the lock byte AND the mutex_ownerpid, 32931db3c26Sraf * keeping the remaining bytes constant. 33031db3c26Sraf */ 33131db3c26Sraf static int 33231db3c26Sraf set_lock_byte64(volatile uint64_t *lockword64, pid_t ownerpid) 33331db3c26Sraf { 33431db3c26Sraf uint64_t old; 33531db3c26Sraf uint64_t new; 33631db3c26Sraf 33731db3c26Sraf old = *lockword64 & ~LOCKMASK64; 33831db3c26Sraf new = old | ((uint64_t)(uint_t)ownerpid << PIDSHIFT) | LOCKBYTE64; 33931db3c26Sraf if (atomic_cas_64(lockword64, old, new) == old) 34031db3c26Sraf return (LOCKCLEAR); 34131db3c26Sraf 34231db3c26Sraf return (LOCKSET); 34331db3c26Sraf } 34431db3c26Sraf 3455d1dd9a9Sraf /* 3465d1dd9a9Sraf * Increment the spinners count in the mutex lock word. 3475d1dd9a9Sraf * Return 0 on success. Return -1 if the count would overflow. 3485d1dd9a9Sraf */ 3495d1dd9a9Sraf static int 3505d1dd9a9Sraf spinners_incr(volatile uint32_t *lockword, uint8_t max_spinners) 3515d1dd9a9Sraf { 3525d1dd9a9Sraf uint32_t old; 3535d1dd9a9Sraf uint32_t new; 3545d1dd9a9Sraf 3555d1dd9a9Sraf do { 3565d1dd9a9Sraf old = *lockword; 3575d1dd9a9Sraf if (((old & SPINNERMASK) >> SPINNERSHIFT) >= max_spinners) 3585d1dd9a9Sraf return (-1); 3595d1dd9a9Sraf new = old + (1 << SPINNERSHIFT); 3605d1dd9a9Sraf } while (atomic_cas_32(lockword, old, new) != old); 3615d1dd9a9Sraf 3625d1dd9a9Sraf return (0); 3635d1dd9a9Sraf } 3645d1dd9a9Sraf 3655d1dd9a9Sraf /* 3665d1dd9a9Sraf * Decrement the spinners count in the mutex lock word. 3675d1dd9a9Sraf * Return the new value of the lock word. 3685d1dd9a9Sraf */ 3695d1dd9a9Sraf static uint32_t 3705d1dd9a9Sraf spinners_decr(volatile uint32_t *lockword) 3715d1dd9a9Sraf { 3725d1dd9a9Sraf uint32_t old; 3735d1dd9a9Sraf uint32_t new; 3745d1dd9a9Sraf 3755d1dd9a9Sraf do { 3765d1dd9a9Sraf new = old = *lockword; 3775d1dd9a9Sraf if (new & SPINNERMASK) 3785d1dd9a9Sraf new -= (1 << SPINNERSHIFT); 3795d1dd9a9Sraf } while (atomic_cas_32(lockword, old, new) != old); 3805d1dd9a9Sraf 3815d1dd9a9Sraf return (new); 3825d1dd9a9Sraf } 3835d1dd9a9Sraf 3847c478bd9Sstevel@tonic-gate /* 3857c478bd9Sstevel@tonic-gate * Non-preemptive spin locks. Used by queue_lock(). 3867c478bd9Sstevel@tonic-gate * No lock statistics are gathered for these locks. 3875d1dd9a9Sraf * No DTrace probes are provided for these locks. 3887c478bd9Sstevel@tonic-gate */ 3897c478bd9Sstevel@tonic-gate void 3907c478bd9Sstevel@tonic-gate spin_lock_set(mutex_t *mp) 3917c478bd9Sstevel@tonic-gate { 3927c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3937c478bd9Sstevel@tonic-gate 3947c478bd9Sstevel@tonic-gate no_preempt(self); 3957c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 3967c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 3977c478bd9Sstevel@tonic-gate return; 3987c478bd9Sstevel@tonic-gate } 3997c478bd9Sstevel@tonic-gate /* 4007c478bd9Sstevel@tonic-gate * Spin for a while, attempting to acquire the lock. 4017c478bd9Sstevel@tonic-gate */ 402d4204c85Sraf INCR32(self->ul_spin_lock_spin); 4037c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 4047c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 4057c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 4067c478bd9Sstevel@tonic-gate return; 4077c478bd9Sstevel@tonic-gate } 4087c478bd9Sstevel@tonic-gate /* 4097c478bd9Sstevel@tonic-gate * Try harder if we were previously at a no premption level. 4107c478bd9Sstevel@tonic-gate */ 4117c478bd9Sstevel@tonic-gate if (self->ul_preempt > 1) { 412d4204c85Sraf INCR32(self->ul_spin_lock_spin2); 4137c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 4147c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 4157c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 4167c478bd9Sstevel@tonic-gate return; 4177c478bd9Sstevel@tonic-gate } 4187c478bd9Sstevel@tonic-gate } 4197c478bd9Sstevel@tonic-gate /* 4207c478bd9Sstevel@tonic-gate * Give up and block in the kernel for the mutex. 4217c478bd9Sstevel@tonic-gate */ 422d4204c85Sraf INCR32(self->ul_spin_lock_sleep); 4237c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_timedlock(mp, NULL); 4247c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 4257c478bd9Sstevel@tonic-gate } 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate void 4287c478bd9Sstevel@tonic-gate spin_lock_clear(mutex_t *mp) 4297c478bd9Sstevel@tonic-gate { 4307c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 4317c478bd9Sstevel@tonic-gate 4327c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 43341efec22Sraf if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) { 434883492d5Sraf (void) ___lwp_mutex_wakeup(mp, 0); 435d4204c85Sraf INCR32(self->ul_spin_lock_wakeup); 4367c478bd9Sstevel@tonic-gate } 4377c478bd9Sstevel@tonic-gate preempt(self); 4387c478bd9Sstevel@tonic-gate } 4397c478bd9Sstevel@tonic-gate 4407c478bd9Sstevel@tonic-gate /* 4417c478bd9Sstevel@tonic-gate * Allocate the sleep queue hash table. 4427c478bd9Sstevel@tonic-gate */ 4437c478bd9Sstevel@tonic-gate void 4447c478bd9Sstevel@tonic-gate queue_alloc(void) 4457c478bd9Sstevel@tonic-gate { 4467c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 4477c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 448d4204c85Sraf queue_head_t *qp; 4497c478bd9Sstevel@tonic-gate void *data; 4507c478bd9Sstevel@tonic-gate int i; 4517c478bd9Sstevel@tonic-gate 4527c478bd9Sstevel@tonic-gate /* 4537c478bd9Sstevel@tonic-gate * No locks are needed; we call here only when single-threaded. 4547c478bd9Sstevel@tonic-gate */ 4557c478bd9Sstevel@tonic-gate ASSERT(self == udp->ulwp_one); 4567c478bd9Sstevel@tonic-gate ASSERT(!udp->uberflags.uf_mt); 457*8cd45542Sraf if ((data = mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t), 4587c478bd9Sstevel@tonic-gate PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0)) 4597c478bd9Sstevel@tonic-gate == MAP_FAILED) 4607c478bd9Sstevel@tonic-gate thr_panic("cannot allocate thread queue_head table"); 461d4204c85Sraf udp->queue_head = qp = (queue_head_t *)data; 462d4204c85Sraf for (i = 0; i < 2 * QHASHSIZE; qp++, i++) { 463d4204c85Sraf qp->qh_type = (i < QHASHSIZE)? MX : CV; 464d4204c85Sraf qp->qh_lock.mutex_flag = LOCK_INITED; 465d4204c85Sraf qp->qh_lock.mutex_magic = MUTEX_MAGIC; 466d4204c85Sraf qp->qh_hlist = &qp->qh_def_root; 467d4204c85Sraf #if defined(THREAD_DEBUG) 468d4204c85Sraf qp->qh_hlen = 1; 469d4204c85Sraf qp->qh_hmax = 1; 470d4204c85Sraf #endif 471883492d5Sraf } 4727c478bd9Sstevel@tonic-gate } 4737c478bd9Sstevel@tonic-gate 4747c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 4757c478bd9Sstevel@tonic-gate 4767c478bd9Sstevel@tonic-gate /* 4777c478bd9Sstevel@tonic-gate * Debugging: verify correctness of a sleep queue. 4787c478bd9Sstevel@tonic-gate */ 4797c478bd9Sstevel@tonic-gate void 4807c478bd9Sstevel@tonic-gate QVERIFY(queue_head_t *qp) 4817c478bd9Sstevel@tonic-gate { 4827c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 4837c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 484d4204c85Sraf queue_root_t *qrp; 4857c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 4867c478bd9Sstevel@tonic-gate ulwp_t *prev; 4877c478bd9Sstevel@tonic-gate uint_t index; 488d4204c85Sraf uint32_t cnt; 4897c478bd9Sstevel@tonic-gate char qtype; 4907c478bd9Sstevel@tonic-gate void *wchan; 4917c478bd9Sstevel@tonic-gate 4927c478bd9Sstevel@tonic-gate ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE); 4937c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 494d4204c85Sraf for (cnt = 0, qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next) { 495d4204c85Sraf cnt++; 496d4204c85Sraf ASSERT((qrp->qr_head != NULL && qrp->qr_tail != NULL) || 497d4204c85Sraf (qrp->qr_head == NULL && qrp->qr_tail == NULL)); 498d4204c85Sraf } 499d4204c85Sraf ASSERT(qp->qh_hlen == cnt && qp->qh_hmax >= cnt); 500d4204c85Sraf qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV; 501d4204c85Sraf ASSERT(qp->qh_type == qtype); 5027c478bd9Sstevel@tonic-gate if (!thread_queue_verify) 5037c478bd9Sstevel@tonic-gate return; 5047c478bd9Sstevel@tonic-gate /* real expensive stuff, only for _THREAD_QUEUE_VERIFY */ 505d4204c85Sraf for (cnt = 0, qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next) { 506d4204c85Sraf for (prev = NULL, ulwp = qrp->qr_head; ulwp != NULL; 507d4204c85Sraf prev = ulwp, ulwp = ulwp->ul_link) { 508d4204c85Sraf cnt++; 509d4204c85Sraf if (ulwp->ul_writer) 510d4204c85Sraf ASSERT(prev == NULL || prev->ul_writer); 511d4204c85Sraf ASSERT(ulwp->ul_qtype == qtype); 512d4204c85Sraf ASSERT(ulwp->ul_wchan != NULL); 513d4204c85Sraf ASSERT(ulwp->ul_sleepq == qp); 514d4204c85Sraf wchan = ulwp->ul_wchan; 515d4204c85Sraf ASSERT(qrp->qr_wchan == wchan); 516d4204c85Sraf index = QUEUE_HASH(wchan, qtype); 517d4204c85Sraf ASSERT(&udp->queue_head[index] == qp); 518d4204c85Sraf } 519d4204c85Sraf ASSERT(qrp->qr_tail == prev); 520d4204c85Sraf } 5217c478bd9Sstevel@tonic-gate ASSERT(qp->qh_qlen == cnt); 5227c478bd9Sstevel@tonic-gate } 5237c478bd9Sstevel@tonic-gate 5247c478bd9Sstevel@tonic-gate #else /* THREAD_DEBUG */ 5257c478bd9Sstevel@tonic-gate 5267c478bd9Sstevel@tonic-gate #define QVERIFY(qp) 5277c478bd9Sstevel@tonic-gate 5287c478bd9Sstevel@tonic-gate #endif /* THREAD_DEBUG */ 5297c478bd9Sstevel@tonic-gate 5307c478bd9Sstevel@tonic-gate /* 5317c478bd9Sstevel@tonic-gate * Acquire a queue head. 5327c478bd9Sstevel@tonic-gate */ 5337c478bd9Sstevel@tonic-gate queue_head_t * 5347c478bd9Sstevel@tonic-gate queue_lock(void *wchan, int qtype) 5357c478bd9Sstevel@tonic-gate { 5367c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 5377c478bd9Sstevel@tonic-gate queue_head_t *qp; 538d4204c85Sraf queue_root_t *qrp; 5397c478bd9Sstevel@tonic-gate 5407c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 5417c478bd9Sstevel@tonic-gate 5427c478bd9Sstevel@tonic-gate /* 5437c478bd9Sstevel@tonic-gate * It is possible that we could be called while still single-threaded. 5447c478bd9Sstevel@tonic-gate * If so, we call queue_alloc() to allocate the queue_head[] array. 5457c478bd9Sstevel@tonic-gate */ 5467c478bd9Sstevel@tonic-gate if ((qp = udp->queue_head) == NULL) { 5477c478bd9Sstevel@tonic-gate queue_alloc(); 5487c478bd9Sstevel@tonic-gate qp = udp->queue_head; 5497c478bd9Sstevel@tonic-gate } 5507c478bd9Sstevel@tonic-gate qp += QUEUE_HASH(wchan, qtype); 5517c478bd9Sstevel@tonic-gate spin_lock_set(&qp->qh_lock); 552d4204c85Sraf for (qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next) 553d4204c85Sraf if (qrp->qr_wchan == wchan) 554d4204c85Sraf break; 555d4204c85Sraf if (qrp == NULL && qp->qh_def_root.qr_head == NULL) { 556d4204c85Sraf /* the default queue root is available; use it */ 557d4204c85Sraf qrp = &qp->qh_def_root; 558d4204c85Sraf qrp->qr_wchan = wchan; 559d4204c85Sraf ASSERT(qrp->qr_next == NULL); 560d4204c85Sraf ASSERT(qrp->qr_tail == NULL && 561d4204c85Sraf qrp->qr_rtcount == 0 && qrp->qr_qlen == 0); 562d4204c85Sraf } 563d4204c85Sraf qp->qh_wchan = wchan; /* valid until queue_unlock() is called */ 564d4204c85Sraf qp->qh_root = qrp; /* valid until queue_unlock() is called */ 565d4204c85Sraf INCR32(qp->qh_lockcount); 5667c478bd9Sstevel@tonic-gate QVERIFY(qp); 5677c478bd9Sstevel@tonic-gate return (qp); 5687c478bd9Sstevel@tonic-gate } 5697c478bd9Sstevel@tonic-gate 5707c478bd9Sstevel@tonic-gate /* 5717c478bd9Sstevel@tonic-gate * Release a queue head. 5727c478bd9Sstevel@tonic-gate */ 5737c478bd9Sstevel@tonic-gate void 5747c478bd9Sstevel@tonic-gate queue_unlock(queue_head_t *qp) 5757c478bd9Sstevel@tonic-gate { 5767c478bd9Sstevel@tonic-gate QVERIFY(qp); 5777c478bd9Sstevel@tonic-gate spin_lock_clear(&qp->qh_lock); 5787c478bd9Sstevel@tonic-gate } 5797c478bd9Sstevel@tonic-gate 5807c478bd9Sstevel@tonic-gate /* 5817c478bd9Sstevel@tonic-gate * For rwlock queueing, we must queue writers ahead of readers of the 5827c478bd9Sstevel@tonic-gate * same priority. We do this by making writers appear to have a half 5837c478bd9Sstevel@tonic-gate * point higher priority for purposes of priority comparisons below. 5847c478bd9Sstevel@tonic-gate */ 5857c478bd9Sstevel@tonic-gate #define CMP_PRIO(ulwp) ((real_priority(ulwp) << 1) + (ulwp)->ul_writer) 5867c478bd9Sstevel@tonic-gate 5877c478bd9Sstevel@tonic-gate void 588d4204c85Sraf enqueue(queue_head_t *qp, ulwp_t *ulwp, int force_fifo) 5897c478bd9Sstevel@tonic-gate { 590d4204c85Sraf queue_root_t *qrp; 5917c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 5927c478bd9Sstevel@tonic-gate ulwp_t *next; 5937c478bd9Sstevel@tonic-gate int pri = CMP_PRIO(ulwp); 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 5967c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq != qp); 5977c478bd9Sstevel@tonic-gate 598d4204c85Sraf if ((qrp = qp->qh_root) == NULL) { 599d4204c85Sraf /* use the thread's queue root for the linkage */ 600d4204c85Sraf qrp = &ulwp->ul_queue_root; 601d4204c85Sraf qrp->qr_next = qp->qh_hlist; 602d4204c85Sraf qrp->qr_prev = NULL; 603d4204c85Sraf qrp->qr_head = NULL; 604d4204c85Sraf qrp->qr_tail = NULL; 605d4204c85Sraf qrp->qr_wchan = qp->qh_wchan; 606d4204c85Sraf qrp->qr_rtcount = 0; 607d4204c85Sraf qrp->qr_qlen = 0; 608d4204c85Sraf qrp->qr_qmax = 0; 609d4204c85Sraf qp->qh_hlist->qr_prev = qrp; 610d4204c85Sraf qp->qh_hlist = qrp; 611d4204c85Sraf qp->qh_root = qrp; 612d4204c85Sraf MAXINCR(qp->qh_hmax, qp->qh_hlen); 613d4204c85Sraf } 614d4204c85Sraf 6157c478bd9Sstevel@tonic-gate /* 6167c478bd9Sstevel@tonic-gate * LIFO queue ordering is unfair and can lead to starvation, 6177c478bd9Sstevel@tonic-gate * but it gives better performance for heavily contended locks. 6187c478bd9Sstevel@tonic-gate * We use thread_queue_fifo (range is 0..8) to determine 6197c478bd9Sstevel@tonic-gate * the frequency of FIFO vs LIFO queuing: 6207c478bd9Sstevel@tonic-gate * 0 : every 256th time (almost always LIFO) 6217c478bd9Sstevel@tonic-gate * 1 : every 128th time 6227c478bd9Sstevel@tonic-gate * 2 : every 64th time 6237c478bd9Sstevel@tonic-gate * 3 : every 32nd time 6247c478bd9Sstevel@tonic-gate * 4 : every 16th time (the default value, mostly LIFO) 6257c478bd9Sstevel@tonic-gate * 5 : every 8th time 6267c478bd9Sstevel@tonic-gate * 6 : every 4th time 6277c478bd9Sstevel@tonic-gate * 7 : every 2nd time 6287c478bd9Sstevel@tonic-gate * 8 : every time (never LIFO, always FIFO) 6297c478bd9Sstevel@tonic-gate * Note that there is always some degree of FIFO ordering. 6307c478bd9Sstevel@tonic-gate * This breaks live lock conditions that occur in applications 6317c478bd9Sstevel@tonic-gate * that are written assuming (incorrectly) that threads acquire 6327c478bd9Sstevel@tonic-gate * locks fairly, that is, in roughly round-robin order. 633d4204c85Sraf * In any event, the queue is maintained in kernel priority order. 6347c478bd9Sstevel@tonic-gate * 635d4204c85Sraf * If force_fifo is non-zero, fifo queueing is forced. 6367c478bd9Sstevel@tonic-gate * SUSV3 requires this for semaphores. 6377c478bd9Sstevel@tonic-gate */ 638d4204c85Sraf if (qrp->qr_head == NULL) { 6397c478bd9Sstevel@tonic-gate /* 6407c478bd9Sstevel@tonic-gate * The queue is empty. LIFO/FIFO doesn't matter. 6417c478bd9Sstevel@tonic-gate */ 642d4204c85Sraf ASSERT(qrp->qr_tail == NULL); 643d4204c85Sraf ulwpp = &qrp->qr_head; 644d4204c85Sraf } else if (force_fifo | 645d4204c85Sraf (((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0)) { 6467c478bd9Sstevel@tonic-gate /* 6477c478bd9Sstevel@tonic-gate * Enqueue after the last thread whose priority is greater 6487c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 6497c478bd9Sstevel@tonic-gate * Attempt first to go directly onto the tail of the queue. 6507c478bd9Sstevel@tonic-gate */ 651d4204c85Sraf if (pri <= CMP_PRIO(qrp->qr_tail)) 652d4204c85Sraf ulwpp = &qrp->qr_tail->ul_link; 6537c478bd9Sstevel@tonic-gate else { 654d4204c85Sraf for (ulwpp = &qrp->qr_head; (next = *ulwpp) != NULL; 6557c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 6567c478bd9Sstevel@tonic-gate if (pri > CMP_PRIO(next)) 6577c478bd9Sstevel@tonic-gate break; 6587c478bd9Sstevel@tonic-gate } 6597c478bd9Sstevel@tonic-gate } else { 6607c478bd9Sstevel@tonic-gate /* 6617c478bd9Sstevel@tonic-gate * Enqueue before the first thread whose priority is less 6627c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 6637c478bd9Sstevel@tonic-gate * Hopefully we can go directly onto the head of the queue. 6647c478bd9Sstevel@tonic-gate */ 665d4204c85Sraf for (ulwpp = &qrp->qr_head; (next = *ulwpp) != NULL; 6667c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 6677c478bd9Sstevel@tonic-gate if (pri >= CMP_PRIO(next)) 6687c478bd9Sstevel@tonic-gate break; 6697c478bd9Sstevel@tonic-gate } 6707c478bd9Sstevel@tonic-gate if ((ulwp->ul_link = *ulwpp) == NULL) 671d4204c85Sraf qrp->qr_tail = ulwp; 6727c478bd9Sstevel@tonic-gate *ulwpp = ulwp; 6737c478bd9Sstevel@tonic-gate 6747c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = qp; 675d4204c85Sraf ulwp->ul_wchan = qp->qh_wchan; 676d4204c85Sraf ulwp->ul_qtype = qp->qh_type; 677d4204c85Sraf if ((ulwp->ul_schedctl != NULL && 678d4204c85Sraf ulwp->ul_schedctl->sc_cid == ulwp->ul_rtclassid) | 679d4204c85Sraf ulwp->ul_pilocks) { 680d4204c85Sraf ulwp->ul_rtqueued = 1; 681d4204c85Sraf qrp->qr_rtcount++; 682d4204c85Sraf } 683d4204c85Sraf MAXINCR(qrp->qr_qmax, qrp->qr_qlen); 684d4204c85Sraf MAXINCR(qp->qh_qmax, qp->qh_qlen); 6857c478bd9Sstevel@tonic-gate } 6867c478bd9Sstevel@tonic-gate 6877c478bd9Sstevel@tonic-gate /* 688d4204c85Sraf * Helper function for queue_slot() and queue_slot_rt(). 689d4204c85Sraf * Try to find a non-suspended thread on the queue. 6907c478bd9Sstevel@tonic-gate */ 6917c478bd9Sstevel@tonic-gate static ulwp_t ** 692d4204c85Sraf queue_slot_runnable(ulwp_t **ulwpp, ulwp_t **prevp, int rt) 6937c478bd9Sstevel@tonic-gate { 6947c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 695d4204c85Sraf ulwp_t **foundpp = NULL; 696d4204c85Sraf int priority = -1; 697d4204c85Sraf ulwp_t *prev; 698d4204c85Sraf int tpri; 6997c478bd9Sstevel@tonic-gate 700d4204c85Sraf for (prev = NULL; 701d4204c85Sraf (ulwp = *ulwpp) != NULL; 7027c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 703d4204c85Sraf if (ulwp->ul_stop) /* skip suspended threads */ 704d4204c85Sraf continue; 705d4204c85Sraf tpri = rt? CMP_PRIO(ulwp) : 0; 706d4204c85Sraf if (tpri > priority) { 707d4204c85Sraf foundpp = ulwpp; 708d4204c85Sraf *prevp = prev; 709d4204c85Sraf priority = tpri; 710d4204c85Sraf if (!rt) 7117c478bd9Sstevel@tonic-gate break; 7127c478bd9Sstevel@tonic-gate } 7137c478bd9Sstevel@tonic-gate } 714d4204c85Sraf return (foundpp); 715d4204c85Sraf } 716d4204c85Sraf 717d4204c85Sraf /* 718d4204c85Sraf * For real-time, we search the entire queue because the dispatch 719d4204c85Sraf * (kernel) priorities may have changed since enqueueing. 720d4204c85Sraf */ 721d4204c85Sraf static ulwp_t ** 722d4204c85Sraf queue_slot_rt(ulwp_t **ulwpp_org, ulwp_t **prevp) 723d4204c85Sraf { 724d4204c85Sraf ulwp_t **ulwpp = ulwpp_org; 725d4204c85Sraf ulwp_t *ulwp = *ulwpp; 726d4204c85Sraf ulwp_t **foundpp = ulwpp; 727d4204c85Sraf int priority = CMP_PRIO(ulwp); 728d4204c85Sraf ulwp_t *prev; 729d4204c85Sraf int tpri; 7307c478bd9Sstevel@tonic-gate 731d4204c85Sraf for (prev = ulwp, ulwpp = &ulwp->ul_link; 732d4204c85Sraf (ulwp = *ulwpp) != NULL; 733d4204c85Sraf prev = ulwp, ulwpp = &ulwp->ul_link) { 734d4204c85Sraf tpri = CMP_PRIO(ulwp); 735d4204c85Sraf if (tpri > priority) { 736d4204c85Sraf foundpp = ulwpp; 737d4204c85Sraf *prevp = prev; 738d4204c85Sraf priority = tpri; 739d4204c85Sraf } 7407c478bd9Sstevel@tonic-gate } 741d4204c85Sraf ulwp = *foundpp; 742d4204c85Sraf 743d4204c85Sraf /* 744d4204c85Sraf * Try not to return a suspended thread. 745d4204c85Sraf * This mimics the old libthread's behavior. 746d4204c85Sraf */ 747d4204c85Sraf if (ulwp->ul_stop && 748d4204c85Sraf (ulwpp = queue_slot_runnable(ulwpp_org, prevp, 1)) != NULL) { 749d4204c85Sraf foundpp = ulwpp; 750d4204c85Sraf ulwp = *foundpp; 7517c478bd9Sstevel@tonic-gate } 752d4204c85Sraf ulwp->ul_rt = 1; 753d4204c85Sraf return (foundpp); 754d4204c85Sraf } 7557c478bd9Sstevel@tonic-gate 756d4204c85Sraf ulwp_t ** 757d4204c85Sraf queue_slot(queue_head_t *qp, ulwp_t **prevp, int *more) 758d4204c85Sraf { 759d4204c85Sraf queue_root_t *qrp; 760d4204c85Sraf ulwp_t **ulwpp; 761d4204c85Sraf ulwp_t *ulwp; 762d4204c85Sraf int rt; 7637c478bd9Sstevel@tonic-gate 764d4204c85Sraf ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 765d4204c85Sraf 766d4204c85Sraf if ((qrp = qp->qh_root) == NULL || (ulwp = qrp->qr_head) == NULL) { 767d4204c85Sraf *more = 0; 768d4204c85Sraf return (NULL); /* no lwps on the queue */ 769d4204c85Sraf } 770d4204c85Sraf rt = (qrp->qr_rtcount != 0); 771d4204c85Sraf *prevp = NULL; 772d4204c85Sraf if (ulwp->ul_link == NULL) { /* only one lwp on the queue */ 773d4204c85Sraf *more = 0; 774d4204c85Sraf ulwp->ul_rt = rt; 775d4204c85Sraf return (&qrp->qr_head); 776d4204c85Sraf } 777d4204c85Sraf *more = 1; 778d4204c85Sraf 779d4204c85Sraf if (rt) /* real-time queue */ 780d4204c85Sraf return (queue_slot_rt(&qrp->qr_head, prevp)); 7817c478bd9Sstevel@tonic-gate /* 782d4204c85Sraf * Try not to return a suspended thread. 783d4204c85Sraf * This mimics the old libthread's behavior. 7847c478bd9Sstevel@tonic-gate */ 785d4204c85Sraf if (ulwp->ul_stop && 786d4204c85Sraf (ulwpp = queue_slot_runnable(&qrp->qr_head, prevp, 0)) != NULL) { 787d4204c85Sraf ulwp = *ulwpp; 788d4204c85Sraf ulwp->ul_rt = 0; 7897c478bd9Sstevel@tonic-gate return (ulwpp); 7907c478bd9Sstevel@tonic-gate } 791d4204c85Sraf /* 792d4204c85Sraf * The common case; just pick the first thread on the queue. 793d4204c85Sraf */ 794d4204c85Sraf ulwp->ul_rt = 0; 795d4204c85Sraf return (&qrp->qr_head); 7967c478bd9Sstevel@tonic-gate } 7977c478bd9Sstevel@tonic-gate 798d4204c85Sraf /* 799d4204c85Sraf * Common code for unlinking an lwp from a user-level sleep queue. 800d4204c85Sraf */ 801d4204c85Sraf void 80241efec22Sraf queue_unlink(queue_head_t *qp, ulwp_t **ulwpp, ulwp_t *prev) 8037c478bd9Sstevel@tonic-gate { 804d4204c85Sraf queue_root_t *qrp = qp->qh_root; 805d4204c85Sraf queue_root_t *nqrp; 806d4204c85Sraf ulwp_t *ulwp = *ulwpp; 807d4204c85Sraf ulwp_t *next; 8087c478bd9Sstevel@tonic-gate 809d4204c85Sraf ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 810d4204c85Sraf ASSERT(qp->qh_wchan != NULL && ulwp->ul_wchan == qp->qh_wchan); 8117c478bd9Sstevel@tonic-gate 812d4204c85Sraf DECR(qp->qh_qlen); 813d4204c85Sraf DECR(qrp->qr_qlen); 814d4204c85Sraf if (ulwp->ul_rtqueued) { 815d4204c85Sraf ulwp->ul_rtqueued = 0; 816d4204c85Sraf qrp->qr_rtcount--; 817d4204c85Sraf } 818d4204c85Sraf next = ulwp->ul_link; 819d4204c85Sraf *ulwpp = next; 820d4204c85Sraf ulwp->ul_link = NULL; 821d4204c85Sraf if (qrp->qr_tail == ulwp) 822d4204c85Sraf qrp->qr_tail = prev; 823d4204c85Sraf if (qrp == &ulwp->ul_queue_root) { 824d4204c85Sraf /* 825d4204c85Sraf * We can't continue to use the unlinked thread's 826d4204c85Sraf * queue root for the linkage. 827d4204c85Sraf */ 828d4204c85Sraf queue_root_t *qr_next = qrp->qr_next; 829d4204c85Sraf queue_root_t *qr_prev = qrp->qr_prev; 830d4204c85Sraf 831d4204c85Sraf if (qrp->qr_tail) { 832d4204c85Sraf /* switch to using the last thread's queue root */ 833d4204c85Sraf ASSERT(qrp->qr_qlen != 0); 834d4204c85Sraf nqrp = &qrp->qr_tail->ul_queue_root; 835d4204c85Sraf *nqrp = *qrp; 836d4204c85Sraf if (qr_next) 837d4204c85Sraf qr_next->qr_prev = nqrp; 838d4204c85Sraf if (qr_prev) 839d4204c85Sraf qr_prev->qr_next = nqrp; 840d4204c85Sraf else 841d4204c85Sraf qp->qh_hlist = nqrp; 842d4204c85Sraf qp->qh_root = nqrp; 843d4204c85Sraf } else { 844d4204c85Sraf /* empty queue root; just delete from the hash list */ 845d4204c85Sraf ASSERT(qrp->qr_qlen == 0); 846d4204c85Sraf if (qr_next) 847d4204c85Sraf qr_next->qr_prev = qr_prev; 848d4204c85Sraf if (qr_prev) 849d4204c85Sraf qr_prev->qr_next = qr_next; 850d4204c85Sraf else 851d4204c85Sraf qp->qh_hlist = qr_next; 852d4204c85Sraf qp->qh_root = NULL; 853d4204c85Sraf DECR(qp->qh_hlen); 854d4204c85Sraf } 855d4204c85Sraf } 8567c478bd9Sstevel@tonic-gate } 8577c478bd9Sstevel@tonic-gate 85841efec22Sraf ulwp_t * 859d4204c85Sraf dequeue(queue_head_t *qp, int *more) 86041efec22Sraf { 86141efec22Sraf ulwp_t **ulwpp; 862d4204c85Sraf ulwp_t *ulwp; 86341efec22Sraf ulwp_t *prev; 86441efec22Sraf 865d4204c85Sraf if ((ulwpp = queue_slot(qp, &prev, more)) == NULL) 86641efec22Sraf return (NULL); 867d4204c85Sraf ulwp = *ulwpp; 868d4204c85Sraf queue_unlink(qp, ulwpp, prev); 869d4204c85Sraf ulwp->ul_sleepq = NULL; 870d4204c85Sraf ulwp->ul_wchan = NULL; 871d4204c85Sraf return (ulwp); 87241efec22Sraf } 87341efec22Sraf 8747c478bd9Sstevel@tonic-gate /* 8757c478bd9Sstevel@tonic-gate * Return a pointer to the highest priority thread sleeping on wchan. 8767c478bd9Sstevel@tonic-gate */ 8777c478bd9Sstevel@tonic-gate ulwp_t * 878d4204c85Sraf queue_waiter(queue_head_t *qp) 8797c478bd9Sstevel@tonic-gate { 8807c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 881d4204c85Sraf ulwp_t *prev; 882d4204c85Sraf int more; 8837c478bd9Sstevel@tonic-gate 884d4204c85Sraf if ((ulwpp = queue_slot(qp, &prev, &more)) == NULL) 8857c478bd9Sstevel@tonic-gate return (NULL); 8867c478bd9Sstevel@tonic-gate return (*ulwpp); 8877c478bd9Sstevel@tonic-gate } 8887c478bd9Sstevel@tonic-gate 889d4204c85Sraf int 890d4204c85Sraf dequeue_self(queue_head_t *qp) 8917c478bd9Sstevel@tonic-gate { 8927c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 893d4204c85Sraf queue_root_t *qrp; 8947c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 8957c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 896d4204c85Sraf ulwp_t *prev; 8977c478bd9Sstevel@tonic-gate int found = 0; 8987c478bd9Sstevel@tonic-gate 8997c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 9007c478bd9Sstevel@tonic-gate 9017c478bd9Sstevel@tonic-gate /* find self on the sleep queue */ 902d4204c85Sraf if ((qrp = qp->qh_root) != NULL) { 903d4204c85Sraf for (prev = NULL, ulwpp = &qrp->qr_head; 904d4204c85Sraf (ulwp = *ulwpp) != NULL; 905d4204c85Sraf prev = ulwp, ulwpp = &ulwp->ul_link) { 906d4204c85Sraf if (ulwp == self) { 907d4204c85Sraf queue_unlink(qp, ulwpp, prev); 908d4204c85Sraf self->ul_cvmutex = NULL; 909d4204c85Sraf self->ul_sleepq = NULL; 910d4204c85Sraf self->ul_wchan = NULL; 911d4204c85Sraf found = 1; 912d4204c85Sraf break; 913d4204c85Sraf } 9147c478bd9Sstevel@tonic-gate } 9157c478bd9Sstevel@tonic-gate } 9167c478bd9Sstevel@tonic-gate 9177c478bd9Sstevel@tonic-gate if (!found) 9187c478bd9Sstevel@tonic-gate thr_panic("dequeue_self(): curthread not found on queue"); 9197c478bd9Sstevel@tonic-gate 920d4204c85Sraf return ((qrp = qp->qh_root) != NULL && qrp->qr_head != NULL); 9217c478bd9Sstevel@tonic-gate } 9227c478bd9Sstevel@tonic-gate 9237c478bd9Sstevel@tonic-gate /* 9247c478bd9Sstevel@tonic-gate * Called from call_user_handler() and _thrp_suspend() to take 9257c478bd9Sstevel@tonic-gate * ourself off of our sleep queue so we can grab locks. 9267c478bd9Sstevel@tonic-gate */ 9277c478bd9Sstevel@tonic-gate void 9287c478bd9Sstevel@tonic-gate unsleep_self(void) 9297c478bd9Sstevel@tonic-gate { 9307c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 9317c478bd9Sstevel@tonic-gate queue_head_t *qp; 9327c478bd9Sstevel@tonic-gate 9337c478bd9Sstevel@tonic-gate /* 9347c478bd9Sstevel@tonic-gate * Calling enter_critical()/exit_critical() here would lead 9357c478bd9Sstevel@tonic-gate * to recursion. Just manipulate self->ul_critical directly. 9367c478bd9Sstevel@tonic-gate */ 9377c478bd9Sstevel@tonic-gate self->ul_critical++; 9387c478bd9Sstevel@tonic-gate while (self->ul_sleepq != NULL) { 9397c478bd9Sstevel@tonic-gate qp = queue_lock(self->ul_wchan, self->ul_qtype); 9407c478bd9Sstevel@tonic-gate /* 9417c478bd9Sstevel@tonic-gate * We may have been moved from a CV queue to a 9427c478bd9Sstevel@tonic-gate * mutex queue while we were attempting queue_lock(). 9437c478bd9Sstevel@tonic-gate * If so, just loop around and try again. 9447c478bd9Sstevel@tonic-gate * dequeue_self() clears self->ul_sleepq. 9457c478bd9Sstevel@tonic-gate */ 946d4204c85Sraf if (qp == self->ul_sleepq) 947d4204c85Sraf (void) dequeue_self(qp); 9487c478bd9Sstevel@tonic-gate queue_unlock(qp); 9497c478bd9Sstevel@tonic-gate } 950d4204c85Sraf self->ul_writer = 0; 9517c478bd9Sstevel@tonic-gate self->ul_critical--; 9527c478bd9Sstevel@tonic-gate } 9537c478bd9Sstevel@tonic-gate 9547c478bd9Sstevel@tonic-gate /* 9557c478bd9Sstevel@tonic-gate * Common code for calling the the ___lwp_mutex_timedlock() system call. 9567c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 9577c478bd9Sstevel@tonic-gate */ 958883492d5Sraf static int 9597c478bd9Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp) 9607c478bd9Sstevel@tonic-gate { 9617c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 9627c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 963883492d5Sraf int mtype = mp->mutex_type; 9647c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 965883492d5Sraf int acquired; 9667c478bd9Sstevel@tonic-gate int error; 9677c478bd9Sstevel@tonic-gate 9687c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 9697c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 9707c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 9717c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 9727c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 9737c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 9747c478bd9Sstevel@tonic-gate } 9757c478bd9Sstevel@tonic-gate if (msp) { 9767c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 9777c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 9787c478bd9Sstevel@tonic-gate } 9797c478bd9Sstevel@tonic-gate 9807c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 9817c478bd9Sstevel@tonic-gate 9827c478bd9Sstevel@tonic-gate for (;;) { 983883492d5Sraf /* 984883492d5Sraf * A return value of EOWNERDEAD or ELOCKUNMAPPED 985883492d5Sraf * means we successfully acquired the lock. 986883492d5Sraf */ 987883492d5Sraf if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0 && 988883492d5Sraf error != EOWNERDEAD && error != ELOCKUNMAPPED) { 989883492d5Sraf acquired = 0; 9907c478bd9Sstevel@tonic-gate break; 9917c478bd9Sstevel@tonic-gate } 9927c478bd9Sstevel@tonic-gate 993883492d5Sraf if (mtype & USYNC_PROCESS) { 9947c478bd9Sstevel@tonic-gate /* 9957c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 9967c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 9977c478bd9Sstevel@tonic-gate */ 9987c478bd9Sstevel@tonic-gate enter_critical(self); 9997c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 10007c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 10017c478bd9Sstevel@tonic-gate exit_critical(self); 1002883492d5Sraf acquired = 1; 10037c478bd9Sstevel@tonic-gate break; 10047c478bd9Sstevel@tonic-gate } 10057c478bd9Sstevel@tonic-gate exit_critical(self); 10067c478bd9Sstevel@tonic-gate } else { 10077c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1008883492d5Sraf acquired = 1; 10097c478bd9Sstevel@tonic-gate break; 10107c478bd9Sstevel@tonic-gate } 10117c478bd9Sstevel@tonic-gate } 10127c478bd9Sstevel@tonic-gate if (msp) 10137c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 10147c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 10157c478bd9Sstevel@tonic-gate self->ul_sp = 0; 10167c478bd9Sstevel@tonic-gate 1017883492d5Sraf if (acquired) { 1018883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 1019883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1020883492d5Sraf } else { 1021883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 1022883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1023883492d5Sraf } 1024883492d5Sraf 10257c478bd9Sstevel@tonic-gate return (error); 10267c478bd9Sstevel@tonic-gate } 10277c478bd9Sstevel@tonic-gate 10287c478bd9Sstevel@tonic-gate /* 10297c478bd9Sstevel@tonic-gate * Common code for calling the ___lwp_mutex_trylock() system call. 10307c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 10317c478bd9Sstevel@tonic-gate */ 10327c478bd9Sstevel@tonic-gate int 10337c478bd9Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp) 10347c478bd9Sstevel@tonic-gate { 10357c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 10367c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 1037883492d5Sraf int mtype = mp->mutex_type; 10387c478bd9Sstevel@tonic-gate int error; 1039883492d5Sraf int acquired; 10407c478bd9Sstevel@tonic-gate 10417c478bd9Sstevel@tonic-gate for (;;) { 1042883492d5Sraf /* 1043883492d5Sraf * A return value of EOWNERDEAD or ELOCKUNMAPPED 1044883492d5Sraf * means we successfully acquired the lock. 1045883492d5Sraf */ 1046883492d5Sraf if ((error = ___lwp_mutex_trylock(mp)) != 0 && 1047883492d5Sraf error != EOWNERDEAD && error != ELOCKUNMAPPED) { 1048883492d5Sraf acquired = 0; 10497c478bd9Sstevel@tonic-gate break; 10507c478bd9Sstevel@tonic-gate } 10517c478bd9Sstevel@tonic-gate 1052883492d5Sraf if (mtype & USYNC_PROCESS) { 10537c478bd9Sstevel@tonic-gate /* 10547c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 10557c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 10567c478bd9Sstevel@tonic-gate */ 10577c478bd9Sstevel@tonic-gate enter_critical(self); 10587c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 10597c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 10607c478bd9Sstevel@tonic-gate exit_critical(self); 1061883492d5Sraf acquired = 1; 10627c478bd9Sstevel@tonic-gate break; 10637c478bd9Sstevel@tonic-gate } 10647c478bd9Sstevel@tonic-gate exit_critical(self); 10657c478bd9Sstevel@tonic-gate } else { 10667c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1067883492d5Sraf acquired = 1; 10687c478bd9Sstevel@tonic-gate break; 10697c478bd9Sstevel@tonic-gate } 10707c478bd9Sstevel@tonic-gate } 10717c478bd9Sstevel@tonic-gate 1072883492d5Sraf if (acquired) { 1073883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1074883492d5Sraf } else if (error != EBUSY) { 1075883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1076883492d5Sraf } 1077883492d5Sraf 10787c478bd9Sstevel@tonic-gate return (error); 10797c478bd9Sstevel@tonic-gate } 10807c478bd9Sstevel@tonic-gate 10817c478bd9Sstevel@tonic-gate volatile sc_shared_t * 10827c478bd9Sstevel@tonic-gate setup_schedctl(void) 10837c478bd9Sstevel@tonic-gate { 10847c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 10857c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 10867c478bd9Sstevel@tonic-gate sc_shared_t *tmp; 10877c478bd9Sstevel@tonic-gate 10887c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */ 10897c478bd9Sstevel@tonic-gate !self->ul_vfork && /* not a child of vfork() */ 10907c478bd9Sstevel@tonic-gate !self->ul_schedctl_called) { /* haven't been called before */ 10917c478bd9Sstevel@tonic-gate enter_critical(self); 10927c478bd9Sstevel@tonic-gate self->ul_schedctl_called = &self->ul_uberdata->uberflags; 10937c478bd9Sstevel@tonic-gate if ((tmp = __schedctl()) != (sc_shared_t *)(-1)) 10947c478bd9Sstevel@tonic-gate self->ul_schedctl = scp = tmp; 10957c478bd9Sstevel@tonic-gate exit_critical(self); 10967c478bd9Sstevel@tonic-gate } 10977c478bd9Sstevel@tonic-gate /* 10987c478bd9Sstevel@tonic-gate * Unless the call to setup_schedctl() is surrounded 10997c478bd9Sstevel@tonic-gate * by enter_critical()/exit_critical(), the address 11007c478bd9Sstevel@tonic-gate * we are returning could be invalid due to a forkall() 11017c478bd9Sstevel@tonic-gate * having occurred in another thread. 11027c478bd9Sstevel@tonic-gate */ 11037c478bd9Sstevel@tonic-gate return (scp); 11047c478bd9Sstevel@tonic-gate } 11057c478bd9Sstevel@tonic-gate 11067c478bd9Sstevel@tonic-gate /* 11077c478bd9Sstevel@tonic-gate * Interfaces from libsched, incorporated into libc. 11087c478bd9Sstevel@tonic-gate * libsched.so.1 is now a filter library onto libc. 11097c478bd9Sstevel@tonic-gate */ 11107c478bd9Sstevel@tonic-gate #pragma weak schedctl_lookup = _schedctl_init 11117c478bd9Sstevel@tonic-gate #pragma weak _schedctl_lookup = _schedctl_init 11127c478bd9Sstevel@tonic-gate #pragma weak schedctl_init = _schedctl_init 11137c478bd9Sstevel@tonic-gate schedctl_t * 11147c478bd9Sstevel@tonic-gate _schedctl_init(void) 11157c478bd9Sstevel@tonic-gate { 11167c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = setup_schedctl(); 11177c478bd9Sstevel@tonic-gate return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl); 11187c478bd9Sstevel@tonic-gate } 11197c478bd9Sstevel@tonic-gate 11207c478bd9Sstevel@tonic-gate #pragma weak schedctl_exit = _schedctl_exit 11217c478bd9Sstevel@tonic-gate void 11227c478bd9Sstevel@tonic-gate _schedctl_exit(void) 11237c478bd9Sstevel@tonic-gate { 11247c478bd9Sstevel@tonic-gate } 11257c478bd9Sstevel@tonic-gate 11267c478bd9Sstevel@tonic-gate /* 11277c478bd9Sstevel@tonic-gate * Contract private interface for java. 11287c478bd9Sstevel@tonic-gate * Set up the schedctl data if it doesn't exist yet. 11297c478bd9Sstevel@tonic-gate * Return a pointer to the pointer to the schedctl data. 11307c478bd9Sstevel@tonic-gate */ 11317c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile * 11327c478bd9Sstevel@tonic-gate _thr_schedctl(void) 11337c478bd9Sstevel@tonic-gate { 11347c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 11357c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile *ptr; 11367c478bd9Sstevel@tonic-gate 11377c478bd9Sstevel@tonic-gate if (self->ul_vfork) 11387c478bd9Sstevel@tonic-gate return (NULL); 11397c478bd9Sstevel@tonic-gate if (*(ptr = &self->ul_schedctl) == NULL) 11407c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 11417c478bd9Sstevel@tonic-gate return (ptr); 11427c478bd9Sstevel@tonic-gate } 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate /* 11457c478bd9Sstevel@tonic-gate * Block signals and attempt to block preemption. 11467c478bd9Sstevel@tonic-gate * no_preempt()/preempt() must be used in pairs but can be nested. 11477c478bd9Sstevel@tonic-gate */ 11487c478bd9Sstevel@tonic-gate void 11497c478bd9Sstevel@tonic-gate no_preempt(ulwp_t *self) 11507c478bd9Sstevel@tonic-gate { 11517c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 11527c478bd9Sstevel@tonic-gate 11537c478bd9Sstevel@tonic-gate if (self->ul_preempt++ == 0) { 11547c478bd9Sstevel@tonic-gate enter_critical(self); 11557c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL || 11567c478bd9Sstevel@tonic-gate (scp = setup_schedctl()) != NULL) { 11577c478bd9Sstevel@tonic-gate /* 11587c478bd9Sstevel@tonic-gate * Save the pre-existing preempt value. 11597c478bd9Sstevel@tonic-gate */ 11607c478bd9Sstevel@tonic-gate self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt; 11617c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = 1; 11627c478bd9Sstevel@tonic-gate } 11637c478bd9Sstevel@tonic-gate } 11647c478bd9Sstevel@tonic-gate } 11657c478bd9Sstevel@tonic-gate 11667c478bd9Sstevel@tonic-gate /* 11677c478bd9Sstevel@tonic-gate * Undo the effects of no_preempt(). 11687c478bd9Sstevel@tonic-gate */ 11697c478bd9Sstevel@tonic-gate void 11707c478bd9Sstevel@tonic-gate preempt(ulwp_t *self) 11717c478bd9Sstevel@tonic-gate { 11727c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 11737c478bd9Sstevel@tonic-gate 11747c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt > 0); 11757c478bd9Sstevel@tonic-gate if (--self->ul_preempt == 0) { 11767c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL) { 11777c478bd9Sstevel@tonic-gate /* 11787c478bd9Sstevel@tonic-gate * Restore the pre-existing preempt value. 11797c478bd9Sstevel@tonic-gate */ 11807c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt; 11817c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield && 11827c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt == 0) { 1183*8cd45542Sraf yield(); 11847c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield) { 11857c478bd9Sstevel@tonic-gate /* 11867c478bd9Sstevel@tonic-gate * Shouldn't happen. This is either 11877c478bd9Sstevel@tonic-gate * a race condition or the thread 11887c478bd9Sstevel@tonic-gate * just entered the real-time class. 11897c478bd9Sstevel@tonic-gate */ 1190*8cd45542Sraf yield(); 11917c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_yield = 0; 11927c478bd9Sstevel@tonic-gate } 11937c478bd9Sstevel@tonic-gate } 11947c478bd9Sstevel@tonic-gate } 11957c478bd9Sstevel@tonic-gate exit_critical(self); 11967c478bd9Sstevel@tonic-gate } 11977c478bd9Sstevel@tonic-gate } 11987c478bd9Sstevel@tonic-gate 11997c478bd9Sstevel@tonic-gate /* 12007c478bd9Sstevel@tonic-gate * If a call to preempt() would cause the current thread to yield or to 12017c478bd9Sstevel@tonic-gate * take deferred actions in exit_critical(), then unpark the specified 12027c478bd9Sstevel@tonic-gate * lwp so it can run while we delay. Return the original lwpid if the 12037c478bd9Sstevel@tonic-gate * unpark was not performed, else return zero. The tests are a repeat 12047c478bd9Sstevel@tonic-gate * of some of the tests in preempt(), above. This is a statistical 12057c478bd9Sstevel@tonic-gate * optimization solely for cond_sleep_queue(), below. 12067c478bd9Sstevel@tonic-gate */ 12077c478bd9Sstevel@tonic-gate static lwpid_t 12087c478bd9Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid) 12097c478bd9Sstevel@tonic-gate { 12107c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = self->ul_schedctl; 12117c478bd9Sstevel@tonic-gate 12127c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt == 1 && self->ul_critical > 0); 12137c478bd9Sstevel@tonic-gate if ((scp != NULL && scp->sc_preemptctl.sc_yield) || 12147c478bd9Sstevel@tonic-gate (self->ul_curplease && self->ul_critical == 1)) { 12157c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 12167c478bd9Sstevel@tonic-gate lwpid = 0; 12177c478bd9Sstevel@tonic-gate } 12187c478bd9Sstevel@tonic-gate return (lwpid); 12197c478bd9Sstevel@tonic-gate } 12207c478bd9Sstevel@tonic-gate 12217c478bd9Sstevel@tonic-gate /* 122216b01779Sraf * Spin for a while (if 'tryhard' is true), trying to grab the lock. 12237c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 12247c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread. 12257c478bd9Sstevel@tonic-gate */ 1226883492d5Sraf static int 122716b01779Sraf mutex_trylock_adaptive(mutex_t *mp, int tryhard) 12287c478bd9Sstevel@tonic-gate { 12297c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1230883492d5Sraf int error = EBUSY; 12317c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 12327c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 12335d1dd9a9Sraf volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw; 12345d1dd9a9Sraf volatile uint64_t *ownerp = (volatile uint64_t *)&mp->mutex_owner; 12355d1dd9a9Sraf uint32_t new_lockword; 12365d1dd9a9Sraf int count = 0; 12375d1dd9a9Sraf int max_count; 12385d1dd9a9Sraf uint8_t max_spinners; 12397c478bd9Sstevel@tonic-gate 1240883492d5Sraf ASSERT(!(mp->mutex_type & USYNC_PROCESS)); 12417c478bd9Sstevel@tonic-gate 1242883492d5Sraf if (MUTEX_OWNER(mp) == self) 12437c478bd9Sstevel@tonic-gate return (EBUSY); 12447c478bd9Sstevel@tonic-gate 1245883492d5Sraf /* short-cut, not definitive (see below) */ 1246883492d5Sraf if (mp->mutex_flag & LOCK_NOTRECOVERABLE) { 1247883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 12485d1dd9a9Sraf error = ENOTRECOVERABLE; 12495d1dd9a9Sraf goto done; 1250883492d5Sraf } 1251883492d5Sraf 12525d1dd9a9Sraf /* 12535d1dd9a9Sraf * Make one attempt to acquire the lock before 12545d1dd9a9Sraf * incurring the overhead of the spin loop. 12555d1dd9a9Sraf */ 12565d1dd9a9Sraf if (set_lock_byte(lockp) == 0) { 12575d1dd9a9Sraf *ownerp = (uintptr_t)self; 12585d1dd9a9Sraf error = 0; 12595d1dd9a9Sraf goto done; 12605d1dd9a9Sraf } 12615d1dd9a9Sraf if (!tryhard) 12625d1dd9a9Sraf goto done; 12635d1dd9a9Sraf if (ncpus == 0) 12645d1dd9a9Sraf ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 12655d1dd9a9Sraf if ((max_spinners = self->ul_max_spinners) >= ncpus) 12665d1dd9a9Sraf max_spinners = ncpus - 1; 12675d1dd9a9Sraf max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0; 12685d1dd9a9Sraf if (max_count == 0) 12695d1dd9a9Sraf goto done; 12707c478bd9Sstevel@tonic-gate 12717c478bd9Sstevel@tonic-gate /* 12727c478bd9Sstevel@tonic-gate * This spin loop is unfair to lwps that have already dropped into 12737c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a highly-contended mutex. 12747c478bd9Sstevel@tonic-gate * This is just too bad. The adaptive spin algorithm is intended 12757c478bd9Sstevel@tonic-gate * to allow programs with highly-contended locks (that is, broken 12767c478bd9Sstevel@tonic-gate * programs) to execute with reasonable speed despite their contention. 12777c478bd9Sstevel@tonic-gate * Being fair would reduce the speed of such programs and well-written 12787c478bd9Sstevel@tonic-gate * programs will not suffer in any case. 12797c478bd9Sstevel@tonic-gate */ 12805d1dd9a9Sraf enter_critical(self); 12815d1dd9a9Sraf if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1) { 12825d1dd9a9Sraf exit_critical(self); 12835d1dd9a9Sraf goto done; 12845d1dd9a9Sraf } 12855d1dd9a9Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp); 12865d1dd9a9Sraf for (count = 1; ; count++) { 12877c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) { 12887c478bd9Sstevel@tonic-gate *ownerp = (uintptr_t)self; 1289883492d5Sraf error = 0; 1290883492d5Sraf break; 12917c478bd9Sstevel@tonic-gate } 12925d1dd9a9Sraf if (count == max_count) 12935d1dd9a9Sraf break; 12947c478bd9Sstevel@tonic-gate SMT_PAUSE(); 12957c478bd9Sstevel@tonic-gate /* 12967c478bd9Sstevel@tonic-gate * Stop spinning if the mutex owner is not running on 12977c478bd9Sstevel@tonic-gate * a processor; it will not drop the lock any time soon 12987c478bd9Sstevel@tonic-gate * and we would just be wasting time to keep spinning. 12997c478bd9Sstevel@tonic-gate * 13007c478bd9Sstevel@tonic-gate * Note that we are looking at another thread (ulwp_t) 13017c478bd9Sstevel@tonic-gate * without ensuring that the other thread does not exit. 13027c478bd9Sstevel@tonic-gate * The scheme relies on ulwp_t structures never being 13037c478bd9Sstevel@tonic-gate * deallocated by the library (the library employs a free 13047c478bd9Sstevel@tonic-gate * list of ulwp_t structs that are reused when new threads 13057c478bd9Sstevel@tonic-gate * are created) and on schedctl shared memory never being 13067c478bd9Sstevel@tonic-gate * deallocated once created via __schedctl(). 13077c478bd9Sstevel@tonic-gate * 13087c478bd9Sstevel@tonic-gate * Thus, the worst that can happen when the spinning thread 13097c478bd9Sstevel@tonic-gate * looks at the owner's schedctl data is that it is looking 13107c478bd9Sstevel@tonic-gate * at some other thread's schedctl data. This almost never 13117c478bd9Sstevel@tonic-gate * happens and is benign when it does. 13127c478bd9Sstevel@tonic-gate */ 13137c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 13147c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 13157c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 13167c478bd9Sstevel@tonic-gate break; 13177c478bd9Sstevel@tonic-gate } 13185d1dd9a9Sraf new_lockword = spinners_decr(&mp->mutex_lockword); 13195d1dd9a9Sraf if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) { 13205d1dd9a9Sraf /* 13215d1dd9a9Sraf * We haven't yet acquired the lock, the lock 13225d1dd9a9Sraf * is free, and there are no other spinners. 13235d1dd9a9Sraf * Make one final attempt to acquire the lock. 13245d1dd9a9Sraf * 13255d1dd9a9Sraf * This isn't strictly necessary since mutex_lock_queue() 13265d1dd9a9Sraf * (the next action this thread will take if it doesn't 13275d1dd9a9Sraf * acquire the lock here) makes one attempt to acquire 13285d1dd9a9Sraf * the lock before putting the thread to sleep. 13295d1dd9a9Sraf * 13305d1dd9a9Sraf * If the next action for this thread (on failure here) 13315d1dd9a9Sraf * were not to call mutex_lock_queue(), this would be 13325d1dd9a9Sraf * necessary for correctness, to avoid ending up with an 13335d1dd9a9Sraf * unheld mutex with waiters but no one to wake them up. 13345d1dd9a9Sraf */ 13355d1dd9a9Sraf if (set_lock_byte(lockp) == 0) { 13365d1dd9a9Sraf *ownerp = (uintptr_t)self; 13375d1dd9a9Sraf error = 0; 13385d1dd9a9Sraf } 13395d1dd9a9Sraf count++; 13405d1dd9a9Sraf } 13417c478bd9Sstevel@tonic-gate exit_critical(self); 13427c478bd9Sstevel@tonic-gate 13435d1dd9a9Sraf done: 1344883492d5Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) { 1345883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1346883492d5Sraf /* 134731db3c26Sraf * We shouldn't own the mutex. 134831db3c26Sraf * Just clear the lock; everyone has already been waked up. 1349883492d5Sraf */ 1350883492d5Sraf mp->mutex_owner = 0; 135131db3c26Sraf (void) clear_lockbyte(&mp->mutex_lockword); 1352883492d5Sraf error = ENOTRECOVERABLE; 1353883492d5Sraf } 13547c478bd9Sstevel@tonic-gate 1355883492d5Sraf if (error) { 13565d1dd9a9Sraf if (count) { 13575d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 0, count); 13585d1dd9a9Sraf } 1359883492d5Sraf if (error != EBUSY) { 1360883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1361883492d5Sraf } 1362883492d5Sraf } else { 13635d1dd9a9Sraf if (count) { 13645d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 13655d1dd9a9Sraf } 1366883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 1367883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) { 1368883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1369883492d5Sraf error = EOWNERDEAD; 1370883492d5Sraf } 1371883492d5Sraf } 1372883492d5Sraf 1373883492d5Sraf return (error); 13747c478bd9Sstevel@tonic-gate } 13757c478bd9Sstevel@tonic-gate 13767c478bd9Sstevel@tonic-gate /* 13777c478bd9Sstevel@tonic-gate * Same as mutex_trylock_adaptive(), except specifically for queue locks. 13787c478bd9Sstevel@tonic-gate * The owner field is not set here; the caller (spin_lock_set()) sets it. 13797c478bd9Sstevel@tonic-gate */ 1380883492d5Sraf static int 13817c478bd9Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp) 13827c478bd9Sstevel@tonic-gate { 13837c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 13847c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 13857c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 13867c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 13877c478bd9Sstevel@tonic-gate int count = curthread->ul_queue_spin; 13887c478bd9Sstevel@tonic-gate 13897c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 13907c478bd9Sstevel@tonic-gate 13917c478bd9Sstevel@tonic-gate if (count == 0) 13927c478bd9Sstevel@tonic-gate return (EBUSY); 13937c478bd9Sstevel@tonic-gate 13947c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 13957c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 13967c478bd9Sstevel@tonic-gate while (--count >= 0) { 13977c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) 13987c478bd9Sstevel@tonic-gate return (0); 13997c478bd9Sstevel@tonic-gate SMT_PAUSE(); 14007c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 14017c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 14027c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 14037c478bd9Sstevel@tonic-gate break; 14047c478bd9Sstevel@tonic-gate } 14057c478bd9Sstevel@tonic-gate 14067c478bd9Sstevel@tonic-gate return (EBUSY); 14077c478bd9Sstevel@tonic-gate } 14087c478bd9Sstevel@tonic-gate 14097c478bd9Sstevel@tonic-gate /* 14107c478bd9Sstevel@tonic-gate * Like mutex_trylock_adaptive(), but for process-shared mutexes. 141116b01779Sraf * Spin for a while (if 'tryhard' is true), trying to grab the lock. 14127c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 14137c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread 14147c478bd9Sstevel@tonic-gate * and mutex_ownerpid set to the current pid. 14157c478bd9Sstevel@tonic-gate */ 1416883492d5Sraf static int 141716b01779Sraf mutex_trylock_process(mutex_t *mp, int tryhard) 14187c478bd9Sstevel@tonic-gate { 14197c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 14205d1dd9a9Sraf uberdata_t *udp = self->ul_uberdata; 1421883492d5Sraf int error = EBUSY; 142231db3c26Sraf volatile uint64_t *lockp = (volatile uint64_t *)&mp->mutex_lockword64; 14235d1dd9a9Sraf uint32_t new_lockword; 14245d1dd9a9Sraf int count = 0; 14255d1dd9a9Sraf int max_count; 14265d1dd9a9Sraf uint8_t max_spinners; 14277c478bd9Sstevel@tonic-gate 1428883492d5Sraf ASSERT(mp->mutex_type & USYNC_PROCESS); 14297c478bd9Sstevel@tonic-gate 1430883492d5Sraf if (shared_mutex_held(mp)) 14317c478bd9Sstevel@tonic-gate return (EBUSY); 14327c478bd9Sstevel@tonic-gate 1433883492d5Sraf /* short-cut, not definitive (see below) */ 1434883492d5Sraf if (mp->mutex_flag & LOCK_NOTRECOVERABLE) { 1435883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 14365d1dd9a9Sraf error = ENOTRECOVERABLE; 14375d1dd9a9Sraf goto done; 1438883492d5Sraf } 1439883492d5Sraf 14405d1dd9a9Sraf /* 14415d1dd9a9Sraf * Make one attempt to acquire the lock before 14425d1dd9a9Sraf * incurring the overhead of the spin loop. 14435d1dd9a9Sraf */ 14445d1dd9a9Sraf enter_critical(self); 144531db3c26Sraf if (set_lock_byte64(lockp, udp->pid) == 0) { 14465d1dd9a9Sraf mp->mutex_owner = (uintptr_t)self; 144731db3c26Sraf /* mp->mutex_ownerpid was set by set_lock_byte64() */ 14485d1dd9a9Sraf exit_critical(self); 14495d1dd9a9Sraf error = 0; 14505d1dd9a9Sraf goto done; 14515d1dd9a9Sraf } 14525d1dd9a9Sraf exit_critical(self); 14535d1dd9a9Sraf if (!tryhard) 14545d1dd9a9Sraf goto done; 1455883492d5Sraf if (ncpus == 0) 1456883492d5Sraf ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 14575d1dd9a9Sraf if ((max_spinners = self->ul_max_spinners) >= ncpus) 14585d1dd9a9Sraf max_spinners = ncpus - 1; 14595d1dd9a9Sraf max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0; 14605d1dd9a9Sraf if (max_count == 0) 14615d1dd9a9Sraf goto done; 1462883492d5Sraf 14637c478bd9Sstevel@tonic-gate /* 14647c478bd9Sstevel@tonic-gate * This is a process-shared mutex. 14657c478bd9Sstevel@tonic-gate * We cannot know if the owner is running on a processor. 14667c478bd9Sstevel@tonic-gate * We just spin and hope that it is on a processor. 14677c478bd9Sstevel@tonic-gate */ 1468883492d5Sraf enter_critical(self); 14695d1dd9a9Sraf if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1) { 14705d1dd9a9Sraf exit_critical(self); 14715d1dd9a9Sraf goto done; 14725d1dd9a9Sraf } 14735d1dd9a9Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp); 14745d1dd9a9Sraf for (count = 1; ; count++) { 147531db3c26Sraf if ((*lockp & LOCKMASK64) == 0 && 147631db3c26Sraf set_lock_byte64(lockp, udp->pid) == 0) { 1477883492d5Sraf mp->mutex_owner = (uintptr_t)self; 147831db3c26Sraf /* mp->mutex_ownerpid was set by set_lock_byte64() */ 1479883492d5Sraf error = 0; 1480883492d5Sraf break; 14817c478bd9Sstevel@tonic-gate } 14825d1dd9a9Sraf if (count == max_count) 14835d1dd9a9Sraf break; 1484883492d5Sraf SMT_PAUSE(); 1485883492d5Sraf } 14865d1dd9a9Sraf new_lockword = spinners_decr(&mp->mutex_lockword); 14875d1dd9a9Sraf if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) { 14885d1dd9a9Sraf /* 14895d1dd9a9Sraf * We haven't yet acquired the lock, the lock 14905d1dd9a9Sraf * is free, and there are no other spinners. 14915d1dd9a9Sraf * Make one final attempt to acquire the lock. 14925d1dd9a9Sraf * 14935d1dd9a9Sraf * This isn't strictly necessary since mutex_lock_kernel() 14945d1dd9a9Sraf * (the next action this thread will take if it doesn't 14955d1dd9a9Sraf * acquire the lock here) makes one attempt to acquire 14965d1dd9a9Sraf * the lock before putting the thread to sleep. 14975d1dd9a9Sraf * 14985d1dd9a9Sraf * If the next action for this thread (on failure here) 14995d1dd9a9Sraf * were not to call mutex_lock_kernel(), this would be 15005d1dd9a9Sraf * necessary for correctness, to avoid ending up with an 15015d1dd9a9Sraf * unheld mutex with waiters but no one to wake them up. 15025d1dd9a9Sraf */ 150331db3c26Sraf if (set_lock_byte64(lockp, udp->pid) == 0) { 15045d1dd9a9Sraf mp->mutex_owner = (uintptr_t)self; 150531db3c26Sraf /* mp->mutex_ownerpid was set by set_lock_byte64() */ 15065d1dd9a9Sraf error = 0; 15075d1dd9a9Sraf } 15085d1dd9a9Sraf count++; 15095d1dd9a9Sraf } 1510883492d5Sraf exit_critical(self); 1511883492d5Sraf 15125d1dd9a9Sraf done: 1513883492d5Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) { 1514883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 15157c478bd9Sstevel@tonic-gate /* 151631db3c26Sraf * We shouldn't own the mutex. 151731db3c26Sraf * Just clear the lock; everyone has already been waked up. 15187c478bd9Sstevel@tonic-gate */ 1519883492d5Sraf mp->mutex_owner = 0; 152031db3c26Sraf /* mp->mutex_ownerpid is cleared by clear_lockbyte64() */ 152131db3c26Sraf (void) clear_lockbyte64(&mp->mutex_lockword64); 1522883492d5Sraf error = ENOTRECOVERABLE; 15237c478bd9Sstevel@tonic-gate } 15247c478bd9Sstevel@tonic-gate 1525883492d5Sraf if (error) { 15265d1dd9a9Sraf if (count) { 15275d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 0, count); 15285d1dd9a9Sraf } 1529883492d5Sraf if (error != EBUSY) { 1530883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1531883492d5Sraf } 1532883492d5Sraf } else { 15335d1dd9a9Sraf if (count) { 15345d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 15355d1dd9a9Sraf } 1536883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 1537883492d5Sraf if (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) { 1538883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1539883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) 1540883492d5Sraf error = EOWNERDEAD; 1541883492d5Sraf else if (mp->mutex_type & USYNC_PROCESS_ROBUST) 1542883492d5Sraf error = ELOCKUNMAPPED; 1543883492d5Sraf else 1544883492d5Sraf error = EOWNERDEAD; 1545883492d5Sraf } 1546883492d5Sraf } 1547883492d5Sraf 1548883492d5Sraf return (error); 15497c478bd9Sstevel@tonic-gate } 15507c478bd9Sstevel@tonic-gate 15517c478bd9Sstevel@tonic-gate /* 15527c478bd9Sstevel@tonic-gate * Mutex wakeup code for releasing a USYNC_THREAD mutex. 15537c478bd9Sstevel@tonic-gate * Returns the lwpid of the thread that was dequeued, if any. 15547c478bd9Sstevel@tonic-gate * The caller of mutex_wakeup() must call __lwp_unpark(lwpid) 15557c478bd9Sstevel@tonic-gate * to wake up the specified lwp. 15567c478bd9Sstevel@tonic-gate */ 1557883492d5Sraf static lwpid_t 15587c478bd9Sstevel@tonic-gate mutex_wakeup(mutex_t *mp) 15597c478bd9Sstevel@tonic-gate { 15607c478bd9Sstevel@tonic-gate lwpid_t lwpid = 0; 1561d4204c85Sraf int more; 15627c478bd9Sstevel@tonic-gate queue_head_t *qp; 15637c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 15647c478bd9Sstevel@tonic-gate 15657c478bd9Sstevel@tonic-gate /* 15667c478bd9Sstevel@tonic-gate * Dequeue a waiter from the sleep queue. Don't touch the mutex 15677c478bd9Sstevel@tonic-gate * waiters bit if no one was found on the queue because the mutex 15687c478bd9Sstevel@tonic-gate * might have been deallocated or reallocated for another purpose. 15697c478bd9Sstevel@tonic-gate */ 15707c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 1571d4204c85Sraf if ((ulwp = dequeue(qp, &more)) != NULL) { 15727c478bd9Sstevel@tonic-gate lwpid = ulwp->ul_lwpid; 1573d4204c85Sraf mp->mutex_waiters = more; 15747c478bd9Sstevel@tonic-gate } 15757c478bd9Sstevel@tonic-gate queue_unlock(qp); 15767c478bd9Sstevel@tonic-gate return (lwpid); 15777c478bd9Sstevel@tonic-gate } 15787c478bd9Sstevel@tonic-gate 1579883492d5Sraf /* 1580883492d5Sraf * Mutex wakeup code for releasing all waiters on a USYNC_THREAD mutex. 1581883492d5Sraf */ 1582883492d5Sraf static void 1583883492d5Sraf mutex_wakeup_all(mutex_t *mp) 1584883492d5Sraf { 1585883492d5Sraf queue_head_t *qp; 1586d4204c85Sraf queue_root_t *qrp; 1587883492d5Sraf int nlwpid = 0; 1588883492d5Sraf int maxlwps = MAXLWPS; 1589883492d5Sraf ulwp_t *ulwp; 1590883492d5Sraf lwpid_t buffer[MAXLWPS]; 1591883492d5Sraf lwpid_t *lwpid = buffer; 1592883492d5Sraf 1593883492d5Sraf /* 1594883492d5Sraf * Walk the list of waiters and prepare to wake up all of them. 1595883492d5Sraf * The waiters flag has already been cleared from the mutex. 1596883492d5Sraf * 1597883492d5Sraf * We keep track of lwpids that are to be unparked in lwpid[]. 1598883492d5Sraf * __lwp_unpark_all() is called to unpark all of them after 1599883492d5Sraf * they have been removed from the sleep queue and the sleep 1600883492d5Sraf * queue lock has been dropped. If we run out of space in our 1601883492d5Sraf * on-stack buffer, we need to allocate more but we can't call 1602883492d5Sraf * lmalloc() because we are holding a queue lock when the overflow 1603883492d5Sraf * occurs and lmalloc() acquires a lock. We can't use alloca() 1604883492d5Sraf * either because the application may have allocated a small 1605883492d5Sraf * stack and we don't want to overrun the stack. So we call 1606883492d5Sraf * alloc_lwpids() to allocate a bigger buffer using the mmap() 1607883492d5Sraf * system call directly since that path acquires no locks. 1608883492d5Sraf */ 1609883492d5Sraf qp = queue_lock(mp, MX); 1610d4204c85Sraf for (;;) { 1611d4204c85Sraf if ((qrp = qp->qh_root) == NULL || 1612d4204c85Sraf (ulwp = qrp->qr_head) == NULL) 1613d4204c85Sraf break; 1614d4204c85Sraf ASSERT(ulwp->ul_wchan == mp); 1615d4204c85Sraf queue_unlink(qp, &qrp->qr_head, NULL); 1616d4204c85Sraf ulwp->ul_sleepq = NULL; 1617d4204c85Sraf ulwp->ul_wchan = NULL; 1618d4204c85Sraf if (nlwpid == maxlwps) 1619d4204c85Sraf lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps); 1620d4204c85Sraf lwpid[nlwpid++] = ulwp->ul_lwpid; 1621883492d5Sraf } 1622883492d5Sraf 1623883492d5Sraf if (nlwpid == 0) { 1624883492d5Sraf queue_unlock(qp); 1625883492d5Sraf } else { 16265d1dd9a9Sraf mp->mutex_waiters = 0; 1627883492d5Sraf no_preempt(curthread); 1628883492d5Sraf queue_unlock(qp); 1629883492d5Sraf if (nlwpid == 1) 1630883492d5Sraf (void) __lwp_unpark(lwpid[0]); 1631883492d5Sraf else 1632883492d5Sraf (void) __lwp_unpark_all(lwpid, nlwpid); 1633883492d5Sraf preempt(curthread); 1634883492d5Sraf } 1635883492d5Sraf 1636883492d5Sraf if (lwpid != buffer) 1637*8cd45542Sraf (void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t)); 1638883492d5Sraf } 1639883492d5Sraf 16407c478bd9Sstevel@tonic-gate /* 16415d1dd9a9Sraf * Release a process-private mutex. 16425d1dd9a9Sraf * As an optimization, if there are waiters but there are also spinners 16435d1dd9a9Sraf * attempting to acquire the mutex, then don't bother waking up a waiter; 16445d1dd9a9Sraf * one of the spinners will acquire the mutex soon and it would be a waste 16455d1dd9a9Sraf * of resources to wake up some thread just to have it spin for a while 16465d1dd9a9Sraf * and then possibly go back to sleep. See mutex_trylock_adaptive(). 16477c478bd9Sstevel@tonic-gate */ 1648883492d5Sraf static lwpid_t 1649883492d5Sraf mutex_unlock_queue(mutex_t *mp, int release_all) 16507c478bd9Sstevel@tonic-gate { 16515d1dd9a9Sraf lwpid_t lwpid = 0; 16525d1dd9a9Sraf uint32_t old_lockword; 16537c478bd9Sstevel@tonic-gate 16545d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 165531db3c26Sraf mp->mutex_owner = 0; 16565d1dd9a9Sraf old_lockword = clear_lockbyte(&mp->mutex_lockword); 16575d1dd9a9Sraf if ((old_lockword & WAITERMASK) && 16585d1dd9a9Sraf (release_all || (old_lockword & SPINNERMASK) == 0)) { 16595d1dd9a9Sraf ulwp_t *self = curthread; 16607c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 16615d1dd9a9Sraf if (release_all) 16625d1dd9a9Sraf mutex_wakeup_all(mp); 16635d1dd9a9Sraf else 16645d1dd9a9Sraf lwpid = mutex_wakeup(mp); 16655d1dd9a9Sraf if (lwpid == 0) 16665d1dd9a9Sraf preempt(self); 1667883492d5Sraf } 16687c478bd9Sstevel@tonic-gate return (lwpid); 16697c478bd9Sstevel@tonic-gate } 16707c478bd9Sstevel@tonic-gate 16717c478bd9Sstevel@tonic-gate /* 16727c478bd9Sstevel@tonic-gate * Like mutex_unlock_queue(), but for process-shared mutexes. 16737c478bd9Sstevel@tonic-gate */ 1674883492d5Sraf static void 1675883492d5Sraf mutex_unlock_process(mutex_t *mp, int release_all) 16767c478bd9Sstevel@tonic-gate { 167731db3c26Sraf uint64_t old_lockword64; 16787c478bd9Sstevel@tonic-gate 16797c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 168031db3c26Sraf mp->mutex_owner = 0; 168131db3c26Sraf /* mp->mutex_ownerpid is cleared by clear_lockbyte64() */ 168231db3c26Sraf old_lockword64 = clear_lockbyte64(&mp->mutex_lockword64); 168331db3c26Sraf if ((old_lockword64 & WAITERMASK64) && 168431db3c26Sraf (release_all || (old_lockword64 & SPINNERMASK64) == 0)) { 16855d1dd9a9Sraf ulwp_t *self = curthread; 16865d1dd9a9Sraf no_preempt(self); /* ensure a prompt wakeup */ 16875d1dd9a9Sraf (void) ___lwp_mutex_wakeup(mp, release_all); 16885d1dd9a9Sraf preempt(self); 16897c478bd9Sstevel@tonic-gate } 16907c478bd9Sstevel@tonic-gate } 16917c478bd9Sstevel@tonic-gate 16927c478bd9Sstevel@tonic-gate void 16937c478bd9Sstevel@tonic-gate stall(void) 16947c478bd9Sstevel@tonic-gate { 16957c478bd9Sstevel@tonic-gate for (;;) 16967c478bd9Sstevel@tonic-gate (void) mutex_lock_kernel(&stall_mutex, NULL, NULL); 16977c478bd9Sstevel@tonic-gate } 16987c478bd9Sstevel@tonic-gate 16997c478bd9Sstevel@tonic-gate /* 17007c478bd9Sstevel@tonic-gate * Acquire a USYNC_THREAD mutex via user-level sleep queues. 17017c478bd9Sstevel@tonic-gate * We failed set_lock_byte(&mp->mutex_lockw) before coming here. 1702883492d5Sraf * If successful, returns with mutex_owner set correctly. 17037c478bd9Sstevel@tonic-gate */ 17047c478bd9Sstevel@tonic-gate int 17057c478bd9Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp, 17067c478bd9Sstevel@tonic-gate timespec_t *tsp) 17077c478bd9Sstevel@tonic-gate { 17087c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 17097c478bd9Sstevel@tonic-gate queue_head_t *qp; 17107c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 17117c478bd9Sstevel@tonic-gate int error = 0; 17127c478bd9Sstevel@tonic-gate 17137c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 17147c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 17157c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 17167c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 17177c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 17187c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 17197c478bd9Sstevel@tonic-gate } 17207c478bd9Sstevel@tonic-gate if (msp) { 17217c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 17227c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 17237c478bd9Sstevel@tonic-gate } 17247c478bd9Sstevel@tonic-gate 17257c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 17267c478bd9Sstevel@tonic-gate 17277c478bd9Sstevel@tonic-gate /* 17287c478bd9Sstevel@tonic-gate * Put ourself on the sleep queue, and while we are 17297c478bd9Sstevel@tonic-gate * unable to grab the lock, go park in the kernel. 17307c478bd9Sstevel@tonic-gate * Take ourself off the sleep queue after we acquire the lock. 17317c478bd9Sstevel@tonic-gate * The waiter bit can be set/cleared only while holding the queue lock. 17327c478bd9Sstevel@tonic-gate */ 17337c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 1734d4204c85Sraf enqueue(qp, self, 0); 17357c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 17367c478bd9Sstevel@tonic-gate for (;;) { 17377c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 17387c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 1739d4204c85Sraf mp->mutex_waiters = dequeue_self(qp); 17407c478bd9Sstevel@tonic-gate break; 17417c478bd9Sstevel@tonic-gate } 17427c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 17437c478bd9Sstevel@tonic-gate queue_unlock(qp); 17447c478bd9Sstevel@tonic-gate /* 17457c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 17467c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 17477c478bd9Sstevel@tonic-gate */ 17485d1dd9a9Sraf error = __lwp_park(tsp, 0); 17497c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 17507c478bd9Sstevel@tonic-gate /* 17517c478bd9Sstevel@tonic-gate * We could have taken a signal or suspended ourself. 17527c478bd9Sstevel@tonic-gate * If we did, then we removed ourself from the queue. 17537c478bd9Sstevel@tonic-gate * Someone else may have removed us from the queue 17547c478bd9Sstevel@tonic-gate * as a consequence of mutex_unlock(). We may have 17557c478bd9Sstevel@tonic-gate * gotten a timeout from __lwp_park(). Or we may still 17567c478bd9Sstevel@tonic-gate * be on the queue and this is just a spurious wakeup. 17577c478bd9Sstevel@tonic-gate */ 17587c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 17597c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) { 17605d1dd9a9Sraf if (error) { 1761d4204c85Sraf mp->mutex_waiters = queue_waiter(qp)? 1 : 0; 17625d1dd9a9Sraf if (error != EINTR) 17635d1dd9a9Sraf break; 17645d1dd9a9Sraf error = 0; 17655d1dd9a9Sraf } 17667c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 17677c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 17687c478bd9Sstevel@tonic-gate break; 17697c478bd9Sstevel@tonic-gate } 1770d4204c85Sraf enqueue(qp, self, 0); 17717c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 17727c478bd9Sstevel@tonic-gate } 17737c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == qp && 17747c478bd9Sstevel@tonic-gate self->ul_qtype == MX && 17757c478bd9Sstevel@tonic-gate self->ul_wchan == mp); 17767c478bd9Sstevel@tonic-gate if (error) { 17775d1dd9a9Sraf if (error != EINTR) { 1778d4204c85Sraf mp->mutex_waiters = dequeue_self(qp); 17795d1dd9a9Sraf break; 17805d1dd9a9Sraf } 17815d1dd9a9Sraf error = 0; 17827c478bd9Sstevel@tonic-gate } 17837c478bd9Sstevel@tonic-gate } 17847c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 17857c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 17867c478bd9Sstevel@tonic-gate self->ul_sp = 0; 17877c478bd9Sstevel@tonic-gate queue_unlock(qp); 1788883492d5Sraf 17897c478bd9Sstevel@tonic-gate if (msp) 17907c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 17917c478bd9Sstevel@tonic-gate 17927c478bd9Sstevel@tonic-gate ASSERT(error == 0 || error == EINVAL || error == ETIME); 1793883492d5Sraf 1794883492d5Sraf if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) { 1795883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1796883492d5Sraf /* 179731db3c26Sraf * We shouldn't own the mutex. 179831db3c26Sraf * Just clear the lock; everyone has already been waked up. 1799883492d5Sraf */ 1800883492d5Sraf mp->mutex_owner = 0; 180131db3c26Sraf (void) clear_lockbyte(&mp->mutex_lockword); 1802883492d5Sraf error = ENOTRECOVERABLE; 1803883492d5Sraf } 1804883492d5Sraf 1805883492d5Sraf if (error) { 1806883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 1807883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1808883492d5Sraf } else { 1809883492d5Sraf DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 1810883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 1811883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) { 1812883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 1813883492d5Sraf error = EOWNERDEAD; 1814883492d5Sraf } 1815883492d5Sraf } 1816883492d5Sraf 18177c478bd9Sstevel@tonic-gate return (error); 18187c478bd9Sstevel@tonic-gate } 18197c478bd9Sstevel@tonic-gate 1820883492d5Sraf static int 1821883492d5Sraf mutex_recursion(mutex_t *mp, int mtype, int try) 1822883492d5Sraf { 1823883492d5Sraf ASSERT(mutex_is_held(mp)); 1824883492d5Sraf ASSERT(mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)); 1825883492d5Sraf ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK); 1826883492d5Sraf 1827883492d5Sraf if (mtype & LOCK_RECURSIVE) { 1828883492d5Sraf if (mp->mutex_rcount == RECURSION_MAX) { 1829883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EAGAIN); 1830883492d5Sraf return (EAGAIN); 1831883492d5Sraf } 1832883492d5Sraf mp->mutex_rcount++; 1833883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0); 1834883492d5Sraf return (0); 1835883492d5Sraf } 1836883492d5Sraf if (try == MUTEX_LOCK) { 1837883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 1838883492d5Sraf return (EDEADLK); 1839883492d5Sraf } 1840883492d5Sraf return (EBUSY); 1841883492d5Sraf } 1842883492d5Sraf 1843883492d5Sraf /* 1844883492d5Sraf * Register this USYNC_PROCESS|LOCK_ROBUST mutex with the kernel so 1845883492d5Sraf * it can apply LOCK_OWNERDEAD|LOCK_UNMAPPED if it becomes necessary. 1846883492d5Sraf * We use tdb_hash_lock here and in the synch object tracking code in 1847883492d5Sraf * the tdb_agent.c file. There is no conflict between these two usages. 1848883492d5Sraf */ 1849883492d5Sraf void 1850883492d5Sraf register_lock(mutex_t *mp) 1851883492d5Sraf { 1852883492d5Sraf uberdata_t *udp = curthread->ul_uberdata; 1853883492d5Sraf uint_t hash = LOCK_HASH(mp); 1854883492d5Sraf robust_t *rlp; 1855883492d5Sraf robust_t **rlpp; 1856883492d5Sraf robust_t **table; 1857883492d5Sraf 1858883492d5Sraf if ((table = udp->robustlocks) == NULL) { 1859883492d5Sraf lmutex_lock(&udp->tdb_hash_lock); 1860883492d5Sraf if ((table = udp->robustlocks) == NULL) { 1861883492d5Sraf table = lmalloc(LOCKHASHSZ * sizeof (robust_t *)); 1862883492d5Sraf _membar_producer(); 1863883492d5Sraf udp->robustlocks = table; 1864883492d5Sraf } 1865883492d5Sraf lmutex_unlock(&udp->tdb_hash_lock); 1866883492d5Sraf } 1867883492d5Sraf _membar_consumer(); 1868883492d5Sraf 1869883492d5Sraf /* 1870883492d5Sraf * First search the registered table with no locks held. 1871883492d5Sraf * This is safe because the table never shrinks 1872883492d5Sraf * and we can only get a false negative. 1873883492d5Sraf */ 1874883492d5Sraf for (rlp = table[hash]; rlp != NULL; rlp = rlp->robust_next) { 1875883492d5Sraf if (rlp->robust_lock == mp) /* already registered */ 1876883492d5Sraf return; 1877883492d5Sraf } 1878883492d5Sraf 1879883492d5Sraf /* 1880883492d5Sraf * The lock was not found. 1881883492d5Sraf * Repeat the operation with tdb_hash_lock held. 1882883492d5Sraf */ 1883883492d5Sraf lmutex_lock(&udp->tdb_hash_lock); 1884883492d5Sraf 1885883492d5Sraf for (rlpp = &table[hash]; 1886883492d5Sraf (rlp = *rlpp) != NULL; 1887883492d5Sraf rlpp = &rlp->robust_next) { 1888883492d5Sraf if (rlp->robust_lock == mp) { /* already registered */ 1889883492d5Sraf lmutex_unlock(&udp->tdb_hash_lock); 1890883492d5Sraf return; 1891883492d5Sraf } 1892883492d5Sraf } 1893883492d5Sraf 1894883492d5Sraf /* 1895883492d5Sraf * The lock has never been registered. 1896883492d5Sraf * Register it now and add it to the table. 1897883492d5Sraf */ 1898883492d5Sraf (void) ___lwp_mutex_register(mp); 1899883492d5Sraf rlp = lmalloc(sizeof (*rlp)); 1900883492d5Sraf rlp->robust_lock = mp; 1901883492d5Sraf _membar_producer(); 1902883492d5Sraf *rlpp = rlp; 1903883492d5Sraf 1904883492d5Sraf lmutex_unlock(&udp->tdb_hash_lock); 1905883492d5Sraf } 1906883492d5Sraf 1907883492d5Sraf /* 1908883492d5Sraf * This is called in the child of fork()/forkall() to start over 1909883492d5Sraf * with a clean slate. (Each process must register its own locks.) 1910883492d5Sraf * No locks are needed because all other threads are suspended or gone. 1911883492d5Sraf */ 1912883492d5Sraf void 1913883492d5Sraf unregister_locks(void) 1914883492d5Sraf { 1915883492d5Sraf uberdata_t *udp = curthread->ul_uberdata; 1916883492d5Sraf uint_t hash; 1917883492d5Sraf robust_t **table; 1918883492d5Sraf robust_t *rlp; 1919883492d5Sraf robust_t *next; 1920883492d5Sraf 1921883492d5Sraf if ((table = udp->robustlocks) != NULL) { 1922883492d5Sraf for (hash = 0; hash < LOCKHASHSZ; hash++) { 1923883492d5Sraf rlp = table[hash]; 1924883492d5Sraf while (rlp != NULL) { 1925883492d5Sraf next = rlp->robust_next; 1926883492d5Sraf lfree(rlp, sizeof (*rlp)); 1927883492d5Sraf rlp = next; 1928883492d5Sraf } 1929883492d5Sraf } 1930883492d5Sraf lfree(table, LOCKHASHSZ * sizeof (robust_t *)); 1931883492d5Sraf udp->robustlocks = NULL; 1932883492d5Sraf } 1933883492d5Sraf } 1934883492d5Sraf 19357c478bd9Sstevel@tonic-gate /* 19367c478bd9Sstevel@tonic-gate * Returns with mutex_owner set correctly. 19377c478bd9Sstevel@tonic-gate */ 1938d4204c85Sraf int 19397c478bd9Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try) 19407c478bd9Sstevel@tonic-gate { 19417c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 19427c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 19437c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 19447c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 19457c478bd9Sstevel@tonic-gate int error = 0; 1946d4204c85Sraf int noceil = try & MUTEX_NOCEIL; 1947883492d5Sraf uint8_t ceil; 1948883492d5Sraf int myprio; 19497c478bd9Sstevel@tonic-gate 1950d4204c85Sraf try &= ~MUTEX_NOCEIL; 19517c478bd9Sstevel@tonic-gate ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK); 19527c478bd9Sstevel@tonic-gate 19537c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 19547c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 19557c478bd9Sstevel@tonic-gate 19567c478bd9Sstevel@tonic-gate if (msp && try == MUTEX_TRY) 19577c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try); 19587c478bd9Sstevel@tonic-gate 1959883492d5Sraf if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) 1960883492d5Sraf return (mutex_recursion(mp, mtype, try)); 19617c478bd9Sstevel@tonic-gate 19627c478bd9Sstevel@tonic-gate if (self->ul_error_detection && try == MUTEX_LOCK && 19637c478bd9Sstevel@tonic-gate tsp == NULL && mutex_is_held(mp)) 19647c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_lock", NULL, NULL); 19657c478bd9Sstevel@tonic-gate 1966d4204c85Sraf if ((mtype & LOCK_PRIO_PROTECT) && noceil == 0) { 1967d4204c85Sraf update_sched(self); 1968d4204c85Sraf if (self->ul_cid != self->ul_rtclassid) { 1969d4204c85Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EPERM); 1970d4204c85Sraf return (EPERM); 1971d4204c85Sraf } 1972883492d5Sraf ceil = mp->mutex_ceiling; 1973d4204c85Sraf myprio = self->ul_epri? self->ul_epri : self->ul_pri; 1974883492d5Sraf if (myprio > ceil) { 1975883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, EINVAL); 1976883492d5Sraf return (EINVAL); 19777c478bd9Sstevel@tonic-gate } 1978883492d5Sraf if ((error = _ceil_mylist_add(mp)) != 0) { 1979883492d5Sraf DTRACE_PROBE2(plockstat, mutex__error, mp, error); 1980883492d5Sraf return (error); 19817c478bd9Sstevel@tonic-gate } 1982883492d5Sraf if (myprio < ceil) 1983883492d5Sraf _ceil_prio_inherit(ceil); 1984883492d5Sraf } 19857c478bd9Sstevel@tonic-gate 1986883492d5Sraf if ((mtype & (USYNC_PROCESS | LOCK_ROBUST)) 1987883492d5Sraf == (USYNC_PROCESS | LOCK_ROBUST)) 1988883492d5Sraf register_lock(mp); 1989883492d5Sraf 1990883492d5Sraf if (mtype & LOCK_PRIO_INHERIT) { 1991883492d5Sraf /* go straight to the kernel */ 1992883492d5Sraf if (try == MUTEX_TRY) 1993883492d5Sraf error = mutex_trylock_kernel(mp); 1994883492d5Sraf else /* MUTEX_LOCK */ 1995883492d5Sraf error = mutex_lock_kernel(mp, tsp, msp); 19967c478bd9Sstevel@tonic-gate /* 1997883492d5Sraf * The kernel never sets or clears the lock byte 1998883492d5Sraf * for LOCK_PRIO_INHERIT mutexes. 1999883492d5Sraf * Set it here for consistency. 20007c478bd9Sstevel@tonic-gate */ 2001883492d5Sraf switch (error) { 2002883492d5Sraf case 0: 2003d4204c85Sraf self->ul_pilocks++; 2004883492d5Sraf mp->mutex_lockw = LOCKSET; 2005883492d5Sraf break; 2006883492d5Sraf case EOWNERDEAD: 2007883492d5Sraf case ELOCKUNMAPPED: 2008d4204c85Sraf self->ul_pilocks++; 2009883492d5Sraf mp->mutex_lockw = LOCKSET; 2010883492d5Sraf /* FALLTHROUGH */ 2011883492d5Sraf case ENOTRECOVERABLE: 2012883492d5Sraf ASSERT(mtype & LOCK_ROBUST); 2013883492d5Sraf break; 2014883492d5Sraf case EDEADLK: 2015883492d5Sraf if (try == MUTEX_LOCK) 2016883492d5Sraf stall(); 2017883492d5Sraf error = EBUSY; 2018883492d5Sraf break; 20197c478bd9Sstevel@tonic-gate } 2020883492d5Sraf } else if (mtype & USYNC_PROCESS) { 202116b01779Sraf error = mutex_trylock_process(mp, try == MUTEX_LOCK); 2022883492d5Sraf if (error == EBUSY && try == MUTEX_LOCK) 20237c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 20245d1dd9a9Sraf } else { /* USYNC_THREAD */ 202516b01779Sraf error = mutex_trylock_adaptive(mp, try == MUTEX_LOCK); 2026883492d5Sraf if (error == EBUSY && try == MUTEX_LOCK) 2027883492d5Sraf error = mutex_lock_queue(self, msp, mp, tsp); 20287c478bd9Sstevel@tonic-gate } 20297c478bd9Sstevel@tonic-gate 20307c478bd9Sstevel@tonic-gate switch (error) { 2031883492d5Sraf case 0: 20327c478bd9Sstevel@tonic-gate case EOWNERDEAD: 20337c478bd9Sstevel@tonic-gate case ELOCKUNMAPPED: 2034883492d5Sraf if (mtype & LOCK_ROBUST) 2035883492d5Sraf remember_lock(mp); 20367c478bd9Sstevel@tonic-gate if (msp) 20377c478bd9Sstevel@tonic-gate record_begin_hold(msp); 20387c478bd9Sstevel@tonic-gate break; 20397c478bd9Sstevel@tonic-gate default: 2040d4204c85Sraf if ((mtype & LOCK_PRIO_PROTECT) && noceil == 0) { 2041883492d5Sraf (void) _ceil_mylist_del(mp); 2042883492d5Sraf if (myprio < ceil) 2043883492d5Sraf _ceil_prio_waive(); 2044883492d5Sraf } 20457c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) { 20467c478bd9Sstevel@tonic-gate if (msp) 20477c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try_fail); 20487c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 20497c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 20507c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 20517c478bd9Sstevel@tonic-gate } 20527c478bd9Sstevel@tonic-gate } 20537c478bd9Sstevel@tonic-gate break; 20547c478bd9Sstevel@tonic-gate } 20557c478bd9Sstevel@tonic-gate 20567c478bd9Sstevel@tonic-gate return (error); 20577c478bd9Sstevel@tonic-gate } 20587c478bd9Sstevel@tonic-gate 20597c478bd9Sstevel@tonic-gate int 20607c478bd9Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try) 20617c478bd9Sstevel@tonic-gate { 20627c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 20637c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 20647c478bd9Sstevel@tonic-gate 20657c478bd9Sstevel@tonic-gate /* 20667c478bd9Sstevel@tonic-gate * We know that USYNC_PROCESS is set in mtype and that 20677c478bd9Sstevel@tonic-gate * zero, one, or both of the flags LOCK_RECURSIVE and 20687c478bd9Sstevel@tonic-gate * LOCK_ERRORCHECK are set, and that no other flags are set. 20697c478bd9Sstevel@tonic-gate */ 2070883492d5Sraf ASSERT((mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0); 20717c478bd9Sstevel@tonic-gate enter_critical(self); 207231db3c26Sraf if (set_lock_byte64(&mp->mutex_lockword64, udp->pid) == 0) { 20737c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 207431db3c26Sraf /* mp->mutex_ownerpid was set by set_lock_byte64() */ 20757c478bd9Sstevel@tonic-gate exit_critical(self); 20767c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 20777c478bd9Sstevel@tonic-gate return (0); 20787c478bd9Sstevel@tonic-gate } 20797c478bd9Sstevel@tonic-gate exit_critical(self); 20807c478bd9Sstevel@tonic-gate 2081883492d5Sraf if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && shared_mutex_held(mp)) 2082883492d5Sraf return (mutex_recursion(mp, mtype, try)); 20837c478bd9Sstevel@tonic-gate 208416b01779Sraf if (try == MUTEX_LOCK) { 208516b01779Sraf if (mutex_trylock_process(mp, 1) == 0) 208616b01779Sraf return (0); 20877c478bd9Sstevel@tonic-gate return (mutex_lock_kernel(mp, tsp, NULL)); 208816b01779Sraf } 20897c478bd9Sstevel@tonic-gate 20907c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 20917c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 20927c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 20937c478bd9Sstevel@tonic-gate } 20947c478bd9Sstevel@tonic-gate return (EBUSY); 20957c478bd9Sstevel@tonic-gate } 20967c478bd9Sstevel@tonic-gate 20977c478bd9Sstevel@tonic-gate static int 20987c478bd9Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp) 20997c478bd9Sstevel@tonic-gate { 21007c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2101d4204c85Sraf int mtype = mp->mutex_type; 21027c478bd9Sstevel@tonic-gate uberflags_t *gflags; 21037c478bd9Sstevel@tonic-gate 21047c478bd9Sstevel@tonic-gate /* 21057c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 21067c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 21077c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 21087c478bd9Sstevel@tonic-gate * and the process has only a single thread. 21097c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 21107c478bd9Sstevel@tonic-gate */ 2111d4204c85Sraf if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 2112d4204c85Sraf self->ul_uberdata->uberflags.uf_all) == 0) { 21137c478bd9Sstevel@tonic-gate /* 21147c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 21157c478bd9Sstevel@tonic-gate */ 21167c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 21177c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 21187c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 21197c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 21207c478bd9Sstevel@tonic-gate return (0); 21217c478bd9Sstevel@tonic-gate } 2122883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 2123883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_LOCK)); 21247c478bd9Sstevel@tonic-gate /* 21257c478bd9Sstevel@tonic-gate * We have reached a deadlock, probably because the 21267c478bd9Sstevel@tonic-gate * process is executing non-async-signal-safe code in 21277c478bd9Sstevel@tonic-gate * a signal handler and is attempting to acquire a lock 21287c478bd9Sstevel@tonic-gate * that it already owns. This is not surprising, given 21297c478bd9Sstevel@tonic-gate * bad programming practices over the years that has 21307c478bd9Sstevel@tonic-gate * resulted in applications calling printf() and such 21317c478bd9Sstevel@tonic-gate * in their signal handlers. Unless the user has told 21327c478bd9Sstevel@tonic-gate * us that the signal handlers are safe by setting: 21337c478bd9Sstevel@tonic-gate * export _THREAD_ASYNC_SAFE=1 21347c478bd9Sstevel@tonic-gate * we return EDEADLK rather than actually deadlocking. 21357c478bd9Sstevel@tonic-gate */ 21367c478bd9Sstevel@tonic-gate if (tsp == NULL && 21377c478bd9Sstevel@tonic-gate MUTEX_OWNER(mp) == self && !self->ul_async_safe) { 21387c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 21397c478bd9Sstevel@tonic-gate return (EDEADLK); 21407c478bd9Sstevel@tonic-gate } 21417c478bd9Sstevel@tonic-gate } 21427c478bd9Sstevel@tonic-gate 21437c478bd9Sstevel@tonic-gate /* 21447c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 21457c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 21467c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 21477c478bd9Sstevel@tonic-gate */ 21487c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 21497c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 21507c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 21517c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 21527c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK)); 21537c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 21547c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 21557c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 21567c478bd9Sstevel@tonic-gate return (0); 21577c478bd9Sstevel@tonic-gate } 2158883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 2159883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_LOCK)); 216016b01779Sraf if (mutex_trylock_adaptive(mp, 1) != 0) 2161883492d5Sraf return (mutex_lock_queue(self, NULL, mp, tsp)); 2162883492d5Sraf return (0); 21637c478bd9Sstevel@tonic-gate } 21647c478bd9Sstevel@tonic-gate 21657c478bd9Sstevel@tonic-gate /* else do it the long way */ 21667c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, tsp, MUTEX_LOCK)); 21677c478bd9Sstevel@tonic-gate } 21687c478bd9Sstevel@tonic-gate 21697c478bd9Sstevel@tonic-gate #pragma weak mutex_lock = __mutex_lock 21707c478bd9Sstevel@tonic-gate #pragma weak _mutex_lock = __mutex_lock 21717c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_lock = __mutex_lock 21727c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_lock = __mutex_lock 21737c478bd9Sstevel@tonic-gate int 21747c478bd9Sstevel@tonic-gate __mutex_lock(mutex_t *mp) 21757c478bd9Sstevel@tonic-gate { 21767c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 21777c478bd9Sstevel@tonic-gate return (mutex_lock_impl(mp, NULL)); 21787c478bd9Sstevel@tonic-gate } 21797c478bd9Sstevel@tonic-gate 21807c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_timedlock = _pthread_mutex_timedlock 21817c478bd9Sstevel@tonic-gate int 21827c478bd9Sstevel@tonic-gate _pthread_mutex_timedlock(mutex_t *mp, const timespec_t *abstime) 21837c478bd9Sstevel@tonic-gate { 21847c478bd9Sstevel@tonic-gate timespec_t tslocal; 21857c478bd9Sstevel@tonic-gate int error; 21867c478bd9Sstevel@tonic-gate 21877c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 21887c478bd9Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 21897c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 21907c478bd9Sstevel@tonic-gate if (error == ETIME) 21917c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 21927c478bd9Sstevel@tonic-gate return (error); 21937c478bd9Sstevel@tonic-gate } 21947c478bd9Sstevel@tonic-gate 21957c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_reltimedlock_np = _pthread_mutex_reltimedlock_np 21967c478bd9Sstevel@tonic-gate int 21977c478bd9Sstevel@tonic-gate _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime) 21987c478bd9Sstevel@tonic-gate { 21997c478bd9Sstevel@tonic-gate timespec_t tslocal; 22007c478bd9Sstevel@tonic-gate int error; 22017c478bd9Sstevel@tonic-gate 22027c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 22037c478bd9Sstevel@tonic-gate tslocal = *reltime; 22047c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 22057c478bd9Sstevel@tonic-gate if (error == ETIME) 22067c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 22077c478bd9Sstevel@tonic-gate return (error); 22087c478bd9Sstevel@tonic-gate } 22097c478bd9Sstevel@tonic-gate 22107c478bd9Sstevel@tonic-gate #pragma weak mutex_trylock = __mutex_trylock 22117c478bd9Sstevel@tonic-gate #pragma weak _mutex_trylock = __mutex_trylock 22127c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_trylock = __mutex_trylock 22137c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_trylock = __mutex_trylock 22147c478bd9Sstevel@tonic-gate int 22157c478bd9Sstevel@tonic-gate __mutex_trylock(mutex_t *mp) 22167c478bd9Sstevel@tonic-gate { 22177c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 22187c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 2219d4204c85Sraf int mtype = mp->mutex_type; 22207c478bd9Sstevel@tonic-gate uberflags_t *gflags; 22217c478bd9Sstevel@tonic-gate 22227c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 2223d4204c85Sraf 22247c478bd9Sstevel@tonic-gate /* 22257c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 22267c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 22277c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 22287c478bd9Sstevel@tonic-gate * and the process has only a single thread. 22297c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 22307c478bd9Sstevel@tonic-gate */ 2231d4204c85Sraf if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 22327c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 22337c478bd9Sstevel@tonic-gate /* 22347c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 22357c478bd9Sstevel@tonic-gate */ 22367c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 22377c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 22387c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 22397c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 22407c478bd9Sstevel@tonic-gate return (0); 22417c478bd9Sstevel@tonic-gate } 2242883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 2243883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_TRY)); 22447c478bd9Sstevel@tonic-gate return (EBUSY); 22457c478bd9Sstevel@tonic-gate } 22467c478bd9Sstevel@tonic-gate 22477c478bd9Sstevel@tonic-gate /* 22487c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 22497c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 22507c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 22517c478bd9Sstevel@tonic-gate */ 22527c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 22537c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 22547c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 22557c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 22567c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY)); 22577c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 22587c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 22597c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 22607c478bd9Sstevel@tonic-gate return (0); 22617c478bd9Sstevel@tonic-gate } 2262883492d5Sraf if (mtype && MUTEX_OWNER(mp) == self) 2263883492d5Sraf return (mutex_recursion(mp, mtype, MUTEX_TRY)); 226416b01779Sraf if (__td_event_report(self, TD_LOCK_TRY, udp)) { 226516b01779Sraf self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 226616b01779Sraf tdb_event(TD_LOCK_TRY, udp); 22677c478bd9Sstevel@tonic-gate } 226816b01779Sraf return (EBUSY); 22697c478bd9Sstevel@tonic-gate } 22707c478bd9Sstevel@tonic-gate 22717c478bd9Sstevel@tonic-gate /* else do it the long way */ 22727c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, NULL, MUTEX_TRY)); 22737c478bd9Sstevel@tonic-gate } 22747c478bd9Sstevel@tonic-gate 22757c478bd9Sstevel@tonic-gate int 2276883492d5Sraf mutex_unlock_internal(mutex_t *mp, int retain_robust_flags) 22777c478bd9Sstevel@tonic-gate { 22787c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 22797c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 22807c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 22817c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp; 2282883492d5Sraf int error = 0; 2283883492d5Sraf int release_all; 22847c478bd9Sstevel@tonic-gate lwpid_t lwpid; 22857c478bd9Sstevel@tonic-gate 22867c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp)) 22877c478bd9Sstevel@tonic-gate return (EPERM); 22887c478bd9Sstevel@tonic-gate 22897c478bd9Sstevel@tonic-gate if (self->ul_error_detection && !mutex_is_held(mp)) 22907c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_unlock", NULL, NULL); 22917c478bd9Sstevel@tonic-gate 22927c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 22937c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 22947c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 22957c478bd9Sstevel@tonic-gate return (0); 22967c478bd9Sstevel@tonic-gate } 22977c478bd9Sstevel@tonic-gate 22987c478bd9Sstevel@tonic-gate if ((msp = MUTEX_STATS(mp, udp)) != NULL) 22997c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 23007c478bd9Sstevel@tonic-gate 2301883492d5Sraf if (!retain_robust_flags && !(mtype & LOCK_PRIO_INHERIT) && 2302883492d5Sraf (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) { 2303883492d5Sraf ASSERT(mp->mutex_type & LOCK_ROBUST); 2304883492d5Sraf mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED); 2305883492d5Sraf mp->mutex_flag |= LOCK_NOTRECOVERABLE; 2306883492d5Sraf } 2307883492d5Sraf release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0); 2308883492d5Sraf 2309883492d5Sraf if (mtype & LOCK_PRIO_INHERIT) { 23107c478bd9Sstevel@tonic-gate no_preempt(self); 23117c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 231231db3c26Sraf /* mp->mutex_ownerpid is cleared by ___lwp_mutex_unlock() */ 23137c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 2314883492d5Sraf mp->mutex_lockw = LOCKCLEAR; 2315d4204c85Sraf self->ul_pilocks--; 2316883492d5Sraf error = ___lwp_mutex_unlock(mp); 23177c478bd9Sstevel@tonic-gate preempt(self); 23187c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) { 23195d1dd9a9Sraf mutex_unlock_process(mp, release_all); 23207c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 2321883492d5Sraf if ((lwpid = mutex_unlock_queue(mp, release_all)) != 0) { 23227c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 23237c478bd9Sstevel@tonic-gate preempt(self); 23247c478bd9Sstevel@tonic-gate } 23257c478bd9Sstevel@tonic-gate } 23267c478bd9Sstevel@tonic-gate 2327883492d5Sraf if (mtype & LOCK_ROBUST) 2328883492d5Sraf forget_lock(mp); 2329883492d5Sraf 2330883492d5Sraf if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp)) 2331883492d5Sraf _ceil_prio_waive(); 2332883492d5Sraf 23337c478bd9Sstevel@tonic-gate return (error); 23347c478bd9Sstevel@tonic-gate } 23357c478bd9Sstevel@tonic-gate 23367c478bd9Sstevel@tonic-gate #pragma weak mutex_unlock = __mutex_unlock 23377c478bd9Sstevel@tonic-gate #pragma weak _mutex_unlock = __mutex_unlock 23387c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_unlock = __mutex_unlock 23397c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_unlock = __mutex_unlock 23407c478bd9Sstevel@tonic-gate int 23417c478bd9Sstevel@tonic-gate __mutex_unlock(mutex_t *mp) 23427c478bd9Sstevel@tonic-gate { 23437c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2344d4204c85Sraf int mtype = mp->mutex_type; 23457c478bd9Sstevel@tonic-gate uberflags_t *gflags; 23467c478bd9Sstevel@tonic-gate lwpid_t lwpid; 23477c478bd9Sstevel@tonic-gate short el; 23487c478bd9Sstevel@tonic-gate 23497c478bd9Sstevel@tonic-gate /* 23507c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 23517c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 23527c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 23537c478bd9Sstevel@tonic-gate * and the process has only a single thread. 23547c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 23557c478bd9Sstevel@tonic-gate */ 2356d4204c85Sraf if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 2357d4204c85Sraf self->ul_uberdata->uberflags.uf_all) == 0) { 23587c478bd9Sstevel@tonic-gate if (mtype) { 23597c478bd9Sstevel@tonic-gate /* 23607c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 23617c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 23627c478bd9Sstevel@tonic-gate */ 23637c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 23647c478bd9Sstevel@tonic-gate return (EPERM); 23657c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 23667c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 23677c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 23687c478bd9Sstevel@tonic-gate return (0); 23697c478bd9Sstevel@tonic-gate } 23707c478bd9Sstevel@tonic-gate } 23717c478bd9Sstevel@tonic-gate /* 23727c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 23737c478bd9Sstevel@tonic-gate * Also, there can be no waiters. 23747c478bd9Sstevel@tonic-gate */ 23757c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 23767c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 23777c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 23787c478bd9Sstevel@tonic-gate return (0); 23797c478bd9Sstevel@tonic-gate } 23807c478bd9Sstevel@tonic-gate 23817c478bd9Sstevel@tonic-gate /* 23827c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 23837c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 23847c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 23857c478bd9Sstevel@tonic-gate */ 23867c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL) { 23877c478bd9Sstevel@tonic-gate if (((el = gflags->uf_trs_ted) | mtype) == 0) { 23887c478bd9Sstevel@tonic-gate fast_unlock: 23895d1dd9a9Sraf if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) { 23907c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 23917c478bd9Sstevel@tonic-gate preempt(self); 23927c478bd9Sstevel@tonic-gate } 23937c478bd9Sstevel@tonic-gate return (0); 23947c478bd9Sstevel@tonic-gate } 23957c478bd9Sstevel@tonic-gate if (el) /* error detection or lock statistics */ 23967c478bd9Sstevel@tonic-gate goto slow_unlock; 23977c478bd9Sstevel@tonic-gate if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 23987c478bd9Sstevel@tonic-gate /* 23997c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 24007c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 24017c478bd9Sstevel@tonic-gate */ 24027c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 24037c478bd9Sstevel@tonic-gate return (EPERM); 24047c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 24057c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 24067c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 24077c478bd9Sstevel@tonic-gate return (0); 24087c478bd9Sstevel@tonic-gate } 24097c478bd9Sstevel@tonic-gate goto fast_unlock; 24107c478bd9Sstevel@tonic-gate } 24117c478bd9Sstevel@tonic-gate if ((mtype & 24127c478bd9Sstevel@tonic-gate ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 24137c478bd9Sstevel@tonic-gate /* 24147c478bd9Sstevel@tonic-gate * At this point we know that zero, one, or both of the 24157c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and 24167c478bd9Sstevel@tonic-gate * that the USYNC_PROCESS flag is set. 24177c478bd9Sstevel@tonic-gate */ 24187c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp)) 24197c478bd9Sstevel@tonic-gate return (EPERM); 24207c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 24217c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 24227c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 24237c478bd9Sstevel@tonic-gate return (0); 24247c478bd9Sstevel@tonic-gate } 24255d1dd9a9Sraf mutex_unlock_process(mp, 0); 24267c478bd9Sstevel@tonic-gate return (0); 24277c478bd9Sstevel@tonic-gate } 24287c478bd9Sstevel@tonic-gate } 24297c478bd9Sstevel@tonic-gate 24307c478bd9Sstevel@tonic-gate /* else do it the long way */ 24317c478bd9Sstevel@tonic-gate slow_unlock: 2432883492d5Sraf return (mutex_unlock_internal(mp, 0)); 24337c478bd9Sstevel@tonic-gate } 24347c478bd9Sstevel@tonic-gate 24357c478bd9Sstevel@tonic-gate /* 24367c478bd9Sstevel@tonic-gate * Internally to the library, almost all mutex lock/unlock actions 24377c478bd9Sstevel@tonic-gate * go through these lmutex_ functions, to protect critical regions. 24387c478bd9Sstevel@tonic-gate * We replicate a bit of code from __mutex_lock() and __mutex_unlock() 24397c478bd9Sstevel@tonic-gate * to make these functions faster since we know that the mutex type 24407c478bd9Sstevel@tonic-gate * of all internal locks is USYNC_THREAD. We also know that internal 24417c478bd9Sstevel@tonic-gate * locking can never fail, so we panic if it does. 24427c478bd9Sstevel@tonic-gate */ 24437c478bd9Sstevel@tonic-gate void 24447c478bd9Sstevel@tonic-gate lmutex_lock(mutex_t *mp) 24457c478bd9Sstevel@tonic-gate { 24467c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 24477c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 24487c478bd9Sstevel@tonic-gate 24497c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 24507c478bd9Sstevel@tonic-gate 24517c478bd9Sstevel@tonic-gate enter_critical(self); 24527c478bd9Sstevel@tonic-gate /* 24537c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 24547c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 24557c478bd9Sstevel@tonic-gate */ 24567c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 24577c478bd9Sstevel@tonic-gate /* 24587c478bd9Sstevel@tonic-gate * Only one thread exists; the mutex must be free. 24597c478bd9Sstevel@tonic-gate */ 24607c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_lockw == 0); 24617c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 24627c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 24637c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 24647c478bd9Sstevel@tonic-gate } else { 24657c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 24667c478bd9Sstevel@tonic-gate 24677c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 24687c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 24697c478bd9Sstevel@tonic-gate 24707c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 24717c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 24727c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 247316b01779Sraf } else if (mutex_trylock_adaptive(mp, 1) != 0) { 24747c478bd9Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL); 24757c478bd9Sstevel@tonic-gate } 24767c478bd9Sstevel@tonic-gate 24777c478bd9Sstevel@tonic-gate if (msp) 24787c478bd9Sstevel@tonic-gate record_begin_hold(msp); 24797c478bd9Sstevel@tonic-gate } 24807c478bd9Sstevel@tonic-gate } 24817c478bd9Sstevel@tonic-gate 24827c478bd9Sstevel@tonic-gate void 24837c478bd9Sstevel@tonic-gate lmutex_unlock(mutex_t *mp) 24847c478bd9Sstevel@tonic-gate { 24857c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 24867c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 24877c478bd9Sstevel@tonic-gate 24887c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 24897c478bd9Sstevel@tonic-gate 24907c478bd9Sstevel@tonic-gate /* 24917c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 24927c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 24937c478bd9Sstevel@tonic-gate */ 24947c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 24957c478bd9Sstevel@tonic-gate /* 24967c478bd9Sstevel@tonic-gate * Only one thread exists so there can be no waiters. 24977c478bd9Sstevel@tonic-gate */ 24987c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 24997c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 25007c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 25017c478bd9Sstevel@tonic-gate } else { 25027c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 25037c478bd9Sstevel@tonic-gate lwpid_t lwpid; 25047c478bd9Sstevel@tonic-gate 25057c478bd9Sstevel@tonic-gate if (msp) 25067c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 2507883492d5Sraf if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) { 25087c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 25097c478bd9Sstevel@tonic-gate preempt(self); 25107c478bd9Sstevel@tonic-gate } 25117c478bd9Sstevel@tonic-gate } 25127c478bd9Sstevel@tonic-gate exit_critical(self); 25137c478bd9Sstevel@tonic-gate } 25147c478bd9Sstevel@tonic-gate 2515f841f6adSraf /* 2516f841f6adSraf * For specialized code in libc, like the asynchronous i/o code, 2517f841f6adSraf * the following sig_*() locking primitives are used in order 2518f841f6adSraf * to make the code asynchronous signal safe. Signals are 2519f841f6adSraf * deferred while locks acquired by these functions are held. 2520f841f6adSraf */ 2521f841f6adSraf void 2522f841f6adSraf sig_mutex_lock(mutex_t *mp) 2523f841f6adSraf { 2524f841f6adSraf sigoff(curthread); 2525*8cd45542Sraf (void) mutex_lock(mp); 2526f841f6adSraf } 2527f841f6adSraf 2528f841f6adSraf void 2529f841f6adSraf sig_mutex_unlock(mutex_t *mp) 2530f841f6adSraf { 2531*8cd45542Sraf (void) mutex_unlock(mp); 2532f841f6adSraf sigon(curthread); 2533f841f6adSraf } 2534f841f6adSraf 2535f841f6adSraf int 2536f841f6adSraf sig_mutex_trylock(mutex_t *mp) 2537f841f6adSraf { 2538f841f6adSraf int error; 2539f841f6adSraf 2540f841f6adSraf sigoff(curthread); 2541*8cd45542Sraf if ((error = mutex_trylock(mp)) != 0) 2542f841f6adSraf sigon(curthread); 2543f841f6adSraf return (error); 2544f841f6adSraf } 2545f841f6adSraf 2546f841f6adSraf /* 2547f841f6adSraf * sig_cond_wait() is a cancellation point. 2548f841f6adSraf */ 2549f841f6adSraf int 2550f841f6adSraf sig_cond_wait(cond_t *cv, mutex_t *mp) 2551f841f6adSraf { 2552f841f6adSraf int error; 2553f841f6adSraf 2554f841f6adSraf ASSERT(curthread->ul_sigdefer != 0); 2555*8cd45542Sraf pthread_testcancel(); 2556a574db85Sraf error = __cond_wait(cv, mp); 2557f841f6adSraf if (error == EINTR && curthread->ul_cursig) { 2558f841f6adSraf sig_mutex_unlock(mp); 2559f841f6adSraf /* take the deferred signal here */ 2560f841f6adSraf sig_mutex_lock(mp); 2561f841f6adSraf } 2562*8cd45542Sraf pthread_testcancel(); 2563f841f6adSraf return (error); 2564f841f6adSraf } 2565f841f6adSraf 2566f841f6adSraf /* 2567f841f6adSraf * sig_cond_reltimedwait() is a cancellation point. 2568f841f6adSraf */ 2569f841f6adSraf int 2570f841f6adSraf sig_cond_reltimedwait(cond_t *cv, mutex_t *mp, const timespec_t *ts) 2571f841f6adSraf { 2572f841f6adSraf int error; 2573f841f6adSraf 2574f841f6adSraf ASSERT(curthread->ul_sigdefer != 0); 2575*8cd45542Sraf pthread_testcancel(); 2576a574db85Sraf error = __cond_reltimedwait(cv, mp, ts); 2577f841f6adSraf if (error == EINTR && curthread->ul_cursig) { 2578f841f6adSraf sig_mutex_unlock(mp); 2579f841f6adSraf /* take the deferred signal here */ 2580f841f6adSraf sig_mutex_lock(mp); 2581f841f6adSraf } 2582*8cd45542Sraf pthread_testcancel(); 2583f841f6adSraf return (error); 2584f841f6adSraf } 2585f841f6adSraf 2586a574db85Sraf /* 2587a574db85Sraf * For specialized code in libc, like the stdio code. 2588a574db85Sraf * the following cancel_safe_*() locking primitives are used in 2589a574db85Sraf * order to make the code cancellation-safe. Cancellation is 2590a574db85Sraf * deferred while locks acquired by these functions are held. 2591a574db85Sraf */ 2592a574db85Sraf void 2593a574db85Sraf cancel_safe_mutex_lock(mutex_t *mp) 2594a574db85Sraf { 2595*8cd45542Sraf (void) mutex_lock(mp); 2596a574db85Sraf curthread->ul_libc_locks++; 2597a574db85Sraf } 2598a574db85Sraf 2599a574db85Sraf int 2600a574db85Sraf cancel_safe_mutex_trylock(mutex_t *mp) 2601a574db85Sraf { 2602a574db85Sraf int error; 2603a574db85Sraf 2604*8cd45542Sraf if ((error = mutex_trylock(mp)) == 0) 2605a574db85Sraf curthread->ul_libc_locks++; 2606a574db85Sraf return (error); 2607a574db85Sraf } 2608a574db85Sraf 2609a574db85Sraf void 2610a574db85Sraf cancel_safe_mutex_unlock(mutex_t *mp) 2611a574db85Sraf { 2612a574db85Sraf ulwp_t *self = curthread; 2613a574db85Sraf 2614a574db85Sraf ASSERT(self->ul_libc_locks != 0); 2615a574db85Sraf 2616*8cd45542Sraf (void) mutex_unlock(mp); 2617a574db85Sraf 2618a574db85Sraf /* 2619a574db85Sraf * Decrement the count of locks held by cancel_safe_mutex_lock(). 2620a574db85Sraf * If we are then in a position to terminate cleanly and 2621a574db85Sraf * if there is a pending cancellation and cancellation 2622a574db85Sraf * is not disabled and we received EINTR from a recent 2623a574db85Sraf * system call then perform the cancellation action now. 2624a574db85Sraf */ 2625a574db85Sraf if (--self->ul_libc_locks == 0 && 2626a574db85Sraf !(self->ul_vfork | self->ul_nocancel | 2627a574db85Sraf self->ul_critical | self->ul_sigdefer) && 2628a574db85Sraf cancel_active()) 2629a574db85Sraf _pthread_exit(PTHREAD_CANCELED); 2630a574db85Sraf } 2631a574db85Sraf 26327c478bd9Sstevel@tonic-gate static int 26337c478bd9Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg) 26347c478bd9Sstevel@tonic-gate { 26357c478bd9Sstevel@tonic-gate /* 2636883492d5Sraf * The 'volatile' is necessary to make sure the compiler doesn't 2637883492d5Sraf * reorder the tests of the various components of the mutex. 2638883492d5Sraf * They must be tested in this order: 2639883492d5Sraf * mutex_lockw 2640883492d5Sraf * mutex_owner 2641883492d5Sraf * mutex_ownerpid 2642883492d5Sraf * This relies on the fact that everywhere mutex_lockw is cleared, 2643883492d5Sraf * mutex_owner and mutex_ownerpid are cleared before mutex_lockw 2644883492d5Sraf * is cleared, and that everywhere mutex_lockw is set, mutex_owner 2645883492d5Sraf * and mutex_ownerpid are set after mutex_lockw is set, and that 2646883492d5Sraf * mutex_lockw is set or cleared with a memory barrier. 26477c478bd9Sstevel@tonic-gate */ 26487c478bd9Sstevel@tonic-gate volatile mutex_t *mp = (volatile mutex_t *)mparg; 26497c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 26507c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 26517c478bd9Sstevel@tonic-gate 2652883492d5Sraf return (MUTEX_OWNED(mp, self) && mp->mutex_ownerpid == udp->pid); 26537c478bd9Sstevel@tonic-gate } 26547c478bd9Sstevel@tonic-gate 26557c478bd9Sstevel@tonic-gate /* 26567c478bd9Sstevel@tonic-gate * Some crufty old programs define their own version of _mutex_held() 26577c478bd9Sstevel@tonic-gate * to be simply return(1). This breaks internal libc logic, so we 26587c478bd9Sstevel@tonic-gate * define a private version for exclusive use by libc, mutex_is_held(), 26597c478bd9Sstevel@tonic-gate * and also a new public function, __mutex_held(), to be used in new 26607c478bd9Sstevel@tonic-gate * code to circumvent these crufty old programs. 26617c478bd9Sstevel@tonic-gate */ 26627c478bd9Sstevel@tonic-gate #pragma weak mutex_held = mutex_is_held 26637c478bd9Sstevel@tonic-gate #pragma weak _mutex_held = mutex_is_held 26647c478bd9Sstevel@tonic-gate #pragma weak __mutex_held = mutex_is_held 26657c478bd9Sstevel@tonic-gate int 2666883492d5Sraf mutex_is_held(mutex_t *mparg) 26677c478bd9Sstevel@tonic-gate { 2668883492d5Sraf volatile mutex_t *mp = (volatile mutex_t *)mparg; 2669883492d5Sraf 2670883492d5Sraf if (mparg->mutex_type & USYNC_PROCESS) 2671883492d5Sraf return (shared_mutex_held(mparg)); 26727c478bd9Sstevel@tonic-gate return (MUTEX_OWNED(mp, curthread)); 26737c478bd9Sstevel@tonic-gate } 26747c478bd9Sstevel@tonic-gate 26757c478bd9Sstevel@tonic-gate #pragma weak mutex_destroy = __mutex_destroy 26767c478bd9Sstevel@tonic-gate #pragma weak _mutex_destroy = __mutex_destroy 26777c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_destroy = __mutex_destroy 26787c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_destroy = __mutex_destroy 26797c478bd9Sstevel@tonic-gate int 26807c478bd9Sstevel@tonic-gate __mutex_destroy(mutex_t *mp) 26817c478bd9Sstevel@tonic-gate { 2682883492d5Sraf if (mp->mutex_type & USYNC_PROCESS) 2683883492d5Sraf forget_lock(mp); 2684*8cd45542Sraf (void) memset(mp, 0, sizeof (*mp)); 26857c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(mp); 26867c478bd9Sstevel@tonic-gate return (0); 26877c478bd9Sstevel@tonic-gate } 26887c478bd9Sstevel@tonic-gate 2689883492d5Sraf #pragma weak mutex_consistent = __mutex_consistent 2690883492d5Sraf #pragma weak _mutex_consistent = __mutex_consistent 2691883492d5Sraf #pragma weak pthread_mutex_consistent_np = __mutex_consistent 2692883492d5Sraf #pragma weak _pthread_mutex_consistent_np = __mutex_consistent 2693883492d5Sraf int 2694883492d5Sraf __mutex_consistent(mutex_t *mp) 2695883492d5Sraf { 2696883492d5Sraf /* 2697883492d5Sraf * Do this only for an inconsistent, initialized robust lock 2698883492d5Sraf * that we hold. For all other cases, return EINVAL. 2699883492d5Sraf */ 2700883492d5Sraf if (mutex_is_held(mp) && 2701883492d5Sraf (mp->mutex_type & LOCK_ROBUST) && 2702883492d5Sraf (mp->mutex_flag & LOCK_INITED) && 2703883492d5Sraf (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) { 2704883492d5Sraf mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED); 2705883492d5Sraf mp->mutex_rcount = 0; 2706883492d5Sraf return (0); 2707883492d5Sraf } 2708883492d5Sraf return (EINVAL); 2709883492d5Sraf } 2710883492d5Sraf 27117c478bd9Sstevel@tonic-gate /* 27127c478bd9Sstevel@tonic-gate * Spin locks are separate from ordinary mutexes, 27137c478bd9Sstevel@tonic-gate * but we use the same data structure for them. 27147c478bd9Sstevel@tonic-gate */ 27157c478bd9Sstevel@tonic-gate 27167c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_init = _pthread_spin_init 27177c478bd9Sstevel@tonic-gate int 27187c478bd9Sstevel@tonic-gate _pthread_spin_init(pthread_spinlock_t *lock, int pshared) 27197c478bd9Sstevel@tonic-gate { 27207c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 27217c478bd9Sstevel@tonic-gate 2722*8cd45542Sraf (void) memset(mp, 0, sizeof (*mp)); 27237c478bd9Sstevel@tonic-gate if (pshared == PTHREAD_PROCESS_SHARED) 27247c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_PROCESS; 27257c478bd9Sstevel@tonic-gate else 27267c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_THREAD; 27277c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 27287c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 27297c478bd9Sstevel@tonic-gate return (0); 27307c478bd9Sstevel@tonic-gate } 27317c478bd9Sstevel@tonic-gate 27327c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_destroy = _pthread_spin_destroy 27337c478bd9Sstevel@tonic-gate int 27347c478bd9Sstevel@tonic-gate _pthread_spin_destroy(pthread_spinlock_t *lock) 27357c478bd9Sstevel@tonic-gate { 2736*8cd45542Sraf (void) memset(lock, 0, sizeof (*lock)); 27377c478bd9Sstevel@tonic-gate return (0); 27387c478bd9Sstevel@tonic-gate } 27397c478bd9Sstevel@tonic-gate 27407c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_trylock = _pthread_spin_trylock 27417c478bd9Sstevel@tonic-gate int 27427c478bd9Sstevel@tonic-gate _pthread_spin_trylock(pthread_spinlock_t *lock) 27437c478bd9Sstevel@tonic-gate { 27447c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 27457c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 27467c478bd9Sstevel@tonic-gate int error = 0; 27477c478bd9Sstevel@tonic-gate 27487c478bd9Sstevel@tonic-gate no_preempt(self); 27497c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) != 0) 27507c478bd9Sstevel@tonic-gate error = EBUSY; 27517c478bd9Sstevel@tonic-gate else { 27527c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 27537c478bd9Sstevel@tonic-gate if (mp->mutex_type == USYNC_PROCESS) 27547c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = self->ul_uberdata->pid; 27557c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 27567c478bd9Sstevel@tonic-gate } 27577c478bd9Sstevel@tonic-gate preempt(self); 27587c478bd9Sstevel@tonic-gate return (error); 27597c478bd9Sstevel@tonic-gate } 27607c478bd9Sstevel@tonic-gate 27617c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_lock = _pthread_spin_lock 27627c478bd9Sstevel@tonic-gate int 27637c478bd9Sstevel@tonic-gate _pthread_spin_lock(pthread_spinlock_t *lock) 27647c478bd9Sstevel@tonic-gate { 2765883492d5Sraf mutex_t *mp = (mutex_t *)lock; 2766883492d5Sraf ulwp_t *self = curthread; 2767883492d5Sraf volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw; 2768883492d5Sraf int count = 0; 2769883492d5Sraf 2770883492d5Sraf ASSERT(!self->ul_critical || self->ul_bindflags); 2771883492d5Sraf 2772883492d5Sraf DTRACE_PROBE1(plockstat, mutex__spin, mp); 27737c478bd9Sstevel@tonic-gate 27747c478bd9Sstevel@tonic-gate /* 27757c478bd9Sstevel@tonic-gate * We don't care whether the owner is running on a processor. 27767c478bd9Sstevel@tonic-gate * We just spin because that's what this interface requires. 27777c478bd9Sstevel@tonic-gate */ 27787c478bd9Sstevel@tonic-gate for (;;) { 27797c478bd9Sstevel@tonic-gate if (*lockp == 0) { /* lock byte appears to be clear */ 2780883492d5Sraf no_preempt(self); 2781883492d5Sraf if (set_lock_byte(lockp) == 0) 2782883492d5Sraf break; 2783883492d5Sraf preempt(self); 27847c478bd9Sstevel@tonic-gate } 27855d1dd9a9Sraf if (count < INT_MAX) 27865d1dd9a9Sraf count++; 27877c478bd9Sstevel@tonic-gate SMT_PAUSE(); 27887c478bd9Sstevel@tonic-gate } 2789883492d5Sraf mp->mutex_owner = (uintptr_t)self; 2790883492d5Sraf if (mp->mutex_type == USYNC_PROCESS) 2791883492d5Sraf mp->mutex_ownerpid = self->ul_uberdata->pid; 2792883492d5Sraf preempt(self); 27935d1dd9a9Sraf if (count) { 27945d1dd9a9Sraf DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 27955d1dd9a9Sraf } 2796883492d5Sraf DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 2797883492d5Sraf return (0); 27987c478bd9Sstevel@tonic-gate } 27997c478bd9Sstevel@tonic-gate 28007c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_unlock = _pthread_spin_unlock 28017c478bd9Sstevel@tonic-gate int 28027c478bd9Sstevel@tonic-gate _pthread_spin_unlock(pthread_spinlock_t *lock) 28037c478bd9Sstevel@tonic-gate { 28047c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 28057c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 28067c478bd9Sstevel@tonic-gate 28077c478bd9Sstevel@tonic-gate no_preempt(self); 28087c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 28097c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 28107c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 281141efec22Sraf (void) atomic_swap_32(&mp->mutex_lockword, 0); 28127c478bd9Sstevel@tonic-gate preempt(self); 28137c478bd9Sstevel@tonic-gate return (0); 28147c478bd9Sstevel@tonic-gate } 28157c478bd9Sstevel@tonic-gate 28165d1dd9a9Sraf #define INITIAL_LOCKS 8 /* initial size of ul_heldlocks.array */ 2817883492d5Sraf 2818883492d5Sraf /* 2819883492d5Sraf * Find/allocate an entry for 'lock' in our array of held locks. 2820883492d5Sraf */ 2821883492d5Sraf static mutex_t ** 2822883492d5Sraf find_lock_entry(mutex_t *lock) 2823883492d5Sraf { 2824883492d5Sraf ulwp_t *self = curthread; 2825883492d5Sraf mutex_t **remembered = NULL; 2826883492d5Sraf mutex_t **lockptr; 2827883492d5Sraf uint_t nlocks; 2828883492d5Sraf 2829883492d5Sraf if ((nlocks = self->ul_heldlockcnt) != 0) 2830883492d5Sraf lockptr = self->ul_heldlocks.array; 2831883492d5Sraf else { 2832883492d5Sraf nlocks = 1; 2833883492d5Sraf lockptr = &self->ul_heldlocks.single; 2834883492d5Sraf } 2835883492d5Sraf 2836883492d5Sraf for (; nlocks; nlocks--, lockptr++) { 2837883492d5Sraf if (*lockptr == lock) 2838883492d5Sraf return (lockptr); 2839883492d5Sraf if (*lockptr == NULL && remembered == NULL) 2840883492d5Sraf remembered = lockptr; 2841883492d5Sraf } 2842883492d5Sraf if (remembered != NULL) { 2843883492d5Sraf *remembered = lock; 2844883492d5Sraf return (remembered); 2845883492d5Sraf } 2846883492d5Sraf 2847883492d5Sraf /* 2848883492d5Sraf * No entry available. Allocate more space, converting 2849883492d5Sraf * the single entry into an array of entries if necessary. 2850883492d5Sraf */ 2851883492d5Sraf if ((nlocks = self->ul_heldlockcnt) == 0) { 2852883492d5Sraf /* 2853883492d5Sraf * Initial allocation of the array. 2854883492d5Sraf * Convert the single entry into an array. 2855883492d5Sraf */ 2856883492d5Sraf self->ul_heldlockcnt = nlocks = INITIAL_LOCKS; 2857883492d5Sraf lockptr = lmalloc(nlocks * sizeof (mutex_t *)); 2858883492d5Sraf /* 2859883492d5Sraf * The single entry becomes the first entry in the array. 2860883492d5Sraf */ 2861883492d5Sraf *lockptr = self->ul_heldlocks.single; 2862883492d5Sraf self->ul_heldlocks.array = lockptr; 2863883492d5Sraf /* 2864883492d5Sraf * Return the next available entry in the array. 2865883492d5Sraf */ 2866883492d5Sraf *++lockptr = lock; 2867883492d5Sraf return (lockptr); 2868883492d5Sraf } 2869883492d5Sraf /* 2870883492d5Sraf * Reallocate the array, double the size each time. 2871883492d5Sraf */ 2872883492d5Sraf lockptr = lmalloc(nlocks * 2 * sizeof (mutex_t *)); 2873*8cd45542Sraf (void) memcpy(lockptr, self->ul_heldlocks.array, 2874883492d5Sraf nlocks * sizeof (mutex_t *)); 2875883492d5Sraf lfree(self->ul_heldlocks.array, nlocks * sizeof (mutex_t *)); 2876883492d5Sraf self->ul_heldlocks.array = lockptr; 2877883492d5Sraf self->ul_heldlockcnt *= 2; 2878883492d5Sraf /* 2879883492d5Sraf * Return the next available entry in the newly allocated array. 2880883492d5Sraf */ 2881883492d5Sraf *(lockptr += nlocks) = lock; 2882883492d5Sraf return (lockptr); 2883883492d5Sraf } 2884883492d5Sraf 2885883492d5Sraf /* 2886883492d5Sraf * Insert 'lock' into our list of held locks. 2887883492d5Sraf * Currently only used for LOCK_ROBUST mutexes. 2888883492d5Sraf */ 2889883492d5Sraf void 2890883492d5Sraf remember_lock(mutex_t *lock) 2891883492d5Sraf { 2892883492d5Sraf (void) find_lock_entry(lock); 2893883492d5Sraf } 2894883492d5Sraf 2895883492d5Sraf /* 2896883492d5Sraf * Remove 'lock' from our list of held locks. 2897883492d5Sraf * Currently only used for LOCK_ROBUST mutexes. 2898883492d5Sraf */ 2899883492d5Sraf void 2900883492d5Sraf forget_lock(mutex_t *lock) 2901883492d5Sraf { 2902883492d5Sraf *find_lock_entry(lock) = NULL; 2903883492d5Sraf } 2904883492d5Sraf 2905883492d5Sraf /* 2906883492d5Sraf * Free the array of held locks. 2907883492d5Sraf */ 2908883492d5Sraf void 2909883492d5Sraf heldlock_free(ulwp_t *ulwp) 2910883492d5Sraf { 2911883492d5Sraf uint_t nlocks; 2912883492d5Sraf 2913883492d5Sraf if ((nlocks = ulwp->ul_heldlockcnt) != 0) 2914883492d5Sraf lfree(ulwp->ul_heldlocks.array, nlocks * sizeof (mutex_t *)); 2915883492d5Sraf ulwp->ul_heldlockcnt = 0; 2916883492d5Sraf ulwp->ul_heldlocks.array = NULL; 2917883492d5Sraf } 2918883492d5Sraf 2919883492d5Sraf /* 2920883492d5Sraf * Mark all held LOCK_ROBUST mutexes LOCK_OWNERDEAD. 2921883492d5Sraf * Called from _thrp_exit() to deal with abandoned locks. 2922883492d5Sraf */ 2923883492d5Sraf void 2924883492d5Sraf heldlock_exit(void) 2925883492d5Sraf { 2926883492d5Sraf ulwp_t *self = curthread; 2927883492d5Sraf mutex_t **lockptr; 2928883492d5Sraf uint_t nlocks; 2929883492d5Sraf mutex_t *mp; 2930883492d5Sraf 2931883492d5Sraf if ((nlocks = self->ul_heldlockcnt) != 0) 2932883492d5Sraf lockptr = self->ul_heldlocks.array; 2933883492d5Sraf else { 2934883492d5Sraf nlocks = 1; 2935883492d5Sraf lockptr = &self->ul_heldlocks.single; 2936883492d5Sraf } 2937883492d5Sraf 2938883492d5Sraf for (; nlocks; nlocks--, lockptr++) { 2939883492d5Sraf /* 2940883492d5Sraf * The kernel takes care of transitioning held 2941883492d5Sraf * LOCK_PRIO_INHERIT mutexes to LOCK_OWNERDEAD. 2942883492d5Sraf * We avoid that case here. 2943883492d5Sraf */ 2944883492d5Sraf if ((mp = *lockptr) != NULL && 2945883492d5Sraf mutex_is_held(mp) && 2946883492d5Sraf (mp->mutex_type & (LOCK_ROBUST | LOCK_PRIO_INHERIT)) == 2947883492d5Sraf LOCK_ROBUST) { 2948883492d5Sraf mp->mutex_rcount = 0; 2949883492d5Sraf if (!(mp->mutex_flag & LOCK_UNMAPPED)) 2950883492d5Sraf mp->mutex_flag |= LOCK_OWNERDEAD; 2951883492d5Sraf (void) mutex_unlock_internal(mp, 1); 2952883492d5Sraf } 2953883492d5Sraf } 2954883492d5Sraf 2955883492d5Sraf heldlock_free(self); 2956883492d5Sraf } 2957883492d5Sraf 29587c478bd9Sstevel@tonic-gate #pragma weak cond_init = _cond_init 29597c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 29607c478bd9Sstevel@tonic-gate int 29617c478bd9Sstevel@tonic-gate _cond_init(cond_t *cvp, int type, void *arg) 29627c478bd9Sstevel@tonic-gate { 29637c478bd9Sstevel@tonic-gate if (type != USYNC_THREAD && type != USYNC_PROCESS) 29647c478bd9Sstevel@tonic-gate return (EINVAL); 2965*8cd45542Sraf (void) memset(cvp, 0, sizeof (*cvp)); 29667c478bd9Sstevel@tonic-gate cvp->cond_type = (uint16_t)type; 29677c478bd9Sstevel@tonic-gate cvp->cond_magic = COND_MAGIC; 29687c478bd9Sstevel@tonic-gate return (0); 29697c478bd9Sstevel@tonic-gate } 29707c478bd9Sstevel@tonic-gate 29717c478bd9Sstevel@tonic-gate /* 29727c478bd9Sstevel@tonic-gate * cond_sleep_queue(): utility function for cond_wait_queue(). 29737c478bd9Sstevel@tonic-gate * 29747c478bd9Sstevel@tonic-gate * Go to sleep on a condvar sleep queue, expect to be waked up 29757c478bd9Sstevel@tonic-gate * by someone calling cond_signal() or cond_broadcast() or due 29767c478bd9Sstevel@tonic-gate * to receiving a UNIX signal or being cancelled, or just simply 29777c478bd9Sstevel@tonic-gate * due to a spurious wakeup (like someome calling forkall()). 29787c478bd9Sstevel@tonic-gate * 29797c478bd9Sstevel@tonic-gate * The associated mutex is *not* reacquired before returning. 29807c478bd9Sstevel@tonic-gate * That must be done by the caller of cond_sleep_queue(). 29817c478bd9Sstevel@tonic-gate */ 2982883492d5Sraf static int 29837c478bd9Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 29847c478bd9Sstevel@tonic-gate { 29857c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 29867c478bd9Sstevel@tonic-gate queue_head_t *qp; 29877c478bd9Sstevel@tonic-gate queue_head_t *mqp; 29887c478bd9Sstevel@tonic-gate lwpid_t lwpid; 29897c478bd9Sstevel@tonic-gate int signalled; 29907c478bd9Sstevel@tonic-gate int error; 2991d4204c85Sraf int cv_wake; 2992883492d5Sraf int release_all; 29937c478bd9Sstevel@tonic-gate 29947c478bd9Sstevel@tonic-gate /* 29957c478bd9Sstevel@tonic-gate * Put ourself on the CV sleep queue, unlock the mutex, then 29967c478bd9Sstevel@tonic-gate * park ourself and unpark a candidate lwp to grab the mutex. 29977c478bd9Sstevel@tonic-gate * We must go onto the CV sleep queue before dropping the 29987c478bd9Sstevel@tonic-gate * mutex in order to guarantee atomicity of the operation. 29997c478bd9Sstevel@tonic-gate */ 30007c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 30017c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 3002d4204c85Sraf enqueue(qp, self, 0); 30037c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 1; 30047c478bd9Sstevel@tonic-gate self->ul_cvmutex = mp; 3005d4204c85Sraf self->ul_cv_wake = cv_wake = (tsp != NULL); 30067c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 3007883492d5Sraf if (mp->mutex_flag & LOCK_OWNERDEAD) { 3008883492d5Sraf mp->mutex_flag &= ~LOCK_OWNERDEAD; 3009883492d5Sraf mp->mutex_flag |= LOCK_NOTRECOVERABLE; 3010883492d5Sraf } 3011883492d5Sraf release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0); 3012883492d5Sraf lwpid = mutex_unlock_queue(mp, release_all); 30137c478bd9Sstevel@tonic-gate for (;;) { 30147c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 30157c478bd9Sstevel@tonic-gate queue_unlock(qp); 30167c478bd9Sstevel@tonic-gate if (lwpid != 0) { 30177c478bd9Sstevel@tonic-gate lwpid = preempt_unpark(self, lwpid); 30187c478bd9Sstevel@tonic-gate preempt(self); 30197c478bd9Sstevel@tonic-gate } 30207c478bd9Sstevel@tonic-gate /* 30217c478bd9Sstevel@tonic-gate * We may have a deferred signal present, 30227c478bd9Sstevel@tonic-gate * in which case we should return EINTR. 30237c478bd9Sstevel@tonic-gate * Also, we may have received a SIGCANCEL; if so 30247c478bd9Sstevel@tonic-gate * and we are cancelable we should return EINTR. 30257c478bd9Sstevel@tonic-gate * We force an immediate EINTR return from 30267c478bd9Sstevel@tonic-gate * __lwp_park() by turning our parking flag off. 30277c478bd9Sstevel@tonic-gate */ 30287c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 30297c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 30307c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 30317c478bd9Sstevel@tonic-gate /* 30327c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 30337c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 30347c478bd9Sstevel@tonic-gate */ 30357c478bd9Sstevel@tonic-gate error = __lwp_park(tsp, lwpid); 30367c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 30377c478bd9Sstevel@tonic-gate lwpid = 0; /* unpark the other lwp only once */ 30387c478bd9Sstevel@tonic-gate /* 30397c478bd9Sstevel@tonic-gate * We were waked up by cond_signal(), cond_broadcast(), 30407c478bd9Sstevel@tonic-gate * by an interrupt or timeout (EINTR or ETIME), 30417c478bd9Sstevel@tonic-gate * or we may just have gotten a spurious wakeup. 30427c478bd9Sstevel@tonic-gate */ 30437c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 3044d4204c85Sraf if (!cv_wake) 3045d4204c85Sraf mqp = queue_lock(mp, MX); 30467c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) 30477c478bd9Sstevel@tonic-gate break; 30487c478bd9Sstevel@tonic-gate /* 30497c478bd9Sstevel@tonic-gate * We are on either the condvar sleep queue or the 30502be60c5eSraf * mutex sleep queue. Break out of the sleep if we 30512be60c5eSraf * were interrupted or we timed out (EINTR or ETIME). 30527c478bd9Sstevel@tonic-gate * Else this is a spurious wakeup; continue the loop. 30537c478bd9Sstevel@tonic-gate */ 3054d4204c85Sraf if (!cv_wake && self->ul_sleepq == mqp) { /* mutex queue */ 30552be60c5eSraf if (error) { 3056d4204c85Sraf mp->mutex_waiters = dequeue_self(mqp); 30572be60c5eSraf break; 30582be60c5eSraf } 30592be60c5eSraf tsp = NULL; /* no more timeout */ 30602be60c5eSraf } else if (self->ul_sleepq == qp) { /* condvar queue */ 30617c478bd9Sstevel@tonic-gate if (error) { 3062d4204c85Sraf cvp->cond_waiters_user = dequeue_self(qp); 30637c478bd9Sstevel@tonic-gate break; 30647c478bd9Sstevel@tonic-gate } 30657c478bd9Sstevel@tonic-gate /* 30667c478bd9Sstevel@tonic-gate * Else a spurious wakeup on the condvar queue. 30677c478bd9Sstevel@tonic-gate * __lwp_park() has already adjusted the timeout. 30687c478bd9Sstevel@tonic-gate */ 30697c478bd9Sstevel@tonic-gate } else { 30707c478bd9Sstevel@tonic-gate thr_panic("cond_sleep_queue(): thread not on queue"); 30717c478bd9Sstevel@tonic-gate } 3072d4204c85Sraf if (!cv_wake) 3073d4204c85Sraf queue_unlock(mqp); 30747c478bd9Sstevel@tonic-gate } 30757c478bd9Sstevel@tonic-gate 30767c478bd9Sstevel@tonic-gate self->ul_sp = 0; 3077d4204c85Sraf self->ul_cv_wake = 0; 3078d4204c85Sraf ASSERT(self->ul_cvmutex == NULL); 30797c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 30807c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 30817c478bd9Sstevel@tonic-gate 30827c478bd9Sstevel@tonic-gate signalled = self->ul_signalled; 30837c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 30847c478bd9Sstevel@tonic-gate queue_unlock(qp); 3085d4204c85Sraf if (!cv_wake) 3086d4204c85Sraf queue_unlock(mqp); 30877c478bd9Sstevel@tonic-gate 30887c478bd9Sstevel@tonic-gate /* 30897c478bd9Sstevel@tonic-gate * If we were concurrently cond_signal()d and any of: 30907c478bd9Sstevel@tonic-gate * received a UNIX signal, were cancelled, or got a timeout, 30917c478bd9Sstevel@tonic-gate * then perform another cond_signal() to avoid consuming it. 30927c478bd9Sstevel@tonic-gate */ 30937c478bd9Sstevel@tonic-gate if (error && signalled) 30947c478bd9Sstevel@tonic-gate (void) cond_signal_internal(cvp); 30957c478bd9Sstevel@tonic-gate 30967c478bd9Sstevel@tonic-gate return (error); 30977c478bd9Sstevel@tonic-gate } 30987c478bd9Sstevel@tonic-gate 30997c478bd9Sstevel@tonic-gate int 31005d1dd9a9Sraf cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 31017c478bd9Sstevel@tonic-gate { 31027c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 31037c478bd9Sstevel@tonic-gate int error; 3104883492d5Sraf int merror; 31057c478bd9Sstevel@tonic-gate 31067c478bd9Sstevel@tonic-gate /* 31077c478bd9Sstevel@tonic-gate * The old thread library was programmed to defer signals 31087c478bd9Sstevel@tonic-gate * while in cond_wait() so that the associated mutex would 31097c478bd9Sstevel@tonic-gate * be guaranteed to be held when the application signal 31107c478bd9Sstevel@tonic-gate * handler was invoked. 31117c478bd9Sstevel@tonic-gate * 31127c478bd9Sstevel@tonic-gate * We do not behave this way by default; the state of the 31137c478bd9Sstevel@tonic-gate * associated mutex in the signal handler is undefined. 31147c478bd9Sstevel@tonic-gate * 31157c478bd9Sstevel@tonic-gate * To accommodate applications that depend on the old 31167c478bd9Sstevel@tonic-gate * behavior, the _THREAD_COND_WAIT_DEFER environment 31177c478bd9Sstevel@tonic-gate * variable can be set to 1 and we will behave in the 31187c478bd9Sstevel@tonic-gate * old way with respect to cond_wait(). 31197c478bd9Sstevel@tonic-gate */ 31207c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 31217c478bd9Sstevel@tonic-gate sigoff(self); 31227c478bd9Sstevel@tonic-gate 31237c478bd9Sstevel@tonic-gate error = cond_sleep_queue(cvp, mp, tsp); 31247c478bd9Sstevel@tonic-gate 31257c478bd9Sstevel@tonic-gate /* 31267c478bd9Sstevel@tonic-gate * Reacquire the mutex. 31277c478bd9Sstevel@tonic-gate */ 31285d1dd9a9Sraf if ((merror = mutex_lock_impl(mp, NULL)) != 0) 3129883492d5Sraf error = merror; 31307c478bd9Sstevel@tonic-gate 31317c478bd9Sstevel@tonic-gate /* 31327c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 31337c478bd9Sstevel@tonic-gate */ 31347c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 31357c478bd9Sstevel@tonic-gate sigon(self); 31367c478bd9Sstevel@tonic-gate 31377c478bd9Sstevel@tonic-gate return (error); 31387c478bd9Sstevel@tonic-gate } 31397c478bd9Sstevel@tonic-gate 31407c478bd9Sstevel@tonic-gate /* 31417c478bd9Sstevel@tonic-gate * cond_sleep_kernel(): utility function for cond_wait_kernel(). 31427c478bd9Sstevel@tonic-gate * See the comment ahead of cond_sleep_queue(), above. 31437c478bd9Sstevel@tonic-gate */ 3144883492d5Sraf static int 31457c478bd9Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 31467c478bd9Sstevel@tonic-gate { 31477c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 31487c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 31497c478bd9Sstevel@tonic-gate int error; 31507c478bd9Sstevel@tonic-gate 3151883492d5Sraf if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp)) 3152883492d5Sraf _ceil_prio_waive(); 31537c478bd9Sstevel@tonic-gate 31547c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 31557c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 31567c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 315731db3c26Sraf /* mp->mutex_ownerpid is cleared by ___lwp_cond_wait() */ 3158d4204c85Sraf if (mtype & LOCK_PRIO_INHERIT) { 31597c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 3160d4204c85Sraf self->ul_pilocks--; 3161d4204c85Sraf } 31627c478bd9Sstevel@tonic-gate /* 31637c478bd9Sstevel@tonic-gate * ___lwp_cond_wait() returns immediately with EINTR if 31647c478bd9Sstevel@tonic-gate * set_parking_flag(self,0) is called on this lwp before it 31657c478bd9Sstevel@tonic-gate * goes to sleep in the kernel. sigacthandler() calls this 31667c478bd9Sstevel@tonic-gate * when a deferred signal is noted. This assures that we don't 31677c478bd9Sstevel@tonic-gate * get stuck in ___lwp_cond_wait() with all signals blocked 31687c478bd9Sstevel@tonic-gate * due to taking a deferred signal before going to sleep. 31697c478bd9Sstevel@tonic-gate */ 31707c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 31717c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 31727c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 31737c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 31747c478bd9Sstevel@tonic-gate error = ___lwp_cond_wait(cvp, mp, tsp, 1); 31757c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 31767c478bd9Sstevel@tonic-gate self->ul_sp = 0; 31777c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 31787c478bd9Sstevel@tonic-gate return (error); 31797c478bd9Sstevel@tonic-gate } 31807c478bd9Sstevel@tonic-gate 31817c478bd9Sstevel@tonic-gate int 31827c478bd9Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 31837c478bd9Sstevel@tonic-gate { 31847c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 31857c478bd9Sstevel@tonic-gate int error; 31867c478bd9Sstevel@tonic-gate int merror; 31877c478bd9Sstevel@tonic-gate 31887c478bd9Sstevel@tonic-gate /* 31897c478bd9Sstevel@tonic-gate * See the large comment in cond_wait_queue(), above. 31907c478bd9Sstevel@tonic-gate */ 31917c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 31927c478bd9Sstevel@tonic-gate sigoff(self); 31937c478bd9Sstevel@tonic-gate 31947c478bd9Sstevel@tonic-gate error = cond_sleep_kernel(cvp, mp, tsp); 31957c478bd9Sstevel@tonic-gate 31967c478bd9Sstevel@tonic-gate /* 31977c478bd9Sstevel@tonic-gate * Override the return code from ___lwp_cond_wait() 31987c478bd9Sstevel@tonic-gate * with any non-zero return code from mutex_lock(). 31997c478bd9Sstevel@tonic-gate * This addresses robust lock failures in particular; 32007c478bd9Sstevel@tonic-gate * the caller must see the EOWNERDEAD or ENOTRECOVERABLE 32017c478bd9Sstevel@tonic-gate * errors in order to take corrective action. 32027c478bd9Sstevel@tonic-gate */ 32035d1dd9a9Sraf if ((merror = mutex_lock_impl(mp, NULL)) != 0) 32047c478bd9Sstevel@tonic-gate error = merror; 32057c478bd9Sstevel@tonic-gate 32067c478bd9Sstevel@tonic-gate /* 32077c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 32087c478bd9Sstevel@tonic-gate */ 32097c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 32107c478bd9Sstevel@tonic-gate sigon(self); 32117c478bd9Sstevel@tonic-gate 32127c478bd9Sstevel@tonic-gate return (error); 32137c478bd9Sstevel@tonic-gate } 32147c478bd9Sstevel@tonic-gate 32157c478bd9Sstevel@tonic-gate /* 32167c478bd9Sstevel@tonic-gate * Common code for _cond_wait() and _cond_timedwait() 32177c478bd9Sstevel@tonic-gate */ 32187c478bd9Sstevel@tonic-gate int 32197c478bd9Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 32207c478bd9Sstevel@tonic-gate { 32217c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 32227c478bd9Sstevel@tonic-gate hrtime_t begin_sleep = 0; 32237c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 32247c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 32257c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 32267c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 32277c478bd9Sstevel@tonic-gate uint8_t rcount; 32287c478bd9Sstevel@tonic-gate int error = 0; 32297c478bd9Sstevel@tonic-gate 32307c478bd9Sstevel@tonic-gate /* 32317c478bd9Sstevel@tonic-gate * The SUSV3 Posix spec for pthread_cond_timedwait() states: 32327c478bd9Sstevel@tonic-gate * Except in the case of [ETIMEDOUT], all these error checks 32337c478bd9Sstevel@tonic-gate * shall act as if they were performed immediately at the 32347c478bd9Sstevel@tonic-gate * beginning of processing for the function and shall cause 32357c478bd9Sstevel@tonic-gate * an error return, in effect, prior to modifying the state 32367c478bd9Sstevel@tonic-gate * of the mutex specified by mutex or the condition variable 32377c478bd9Sstevel@tonic-gate * specified by cond. 32387c478bd9Sstevel@tonic-gate * Therefore, we must return EINVAL now if the timout is invalid. 32397c478bd9Sstevel@tonic-gate */ 32407c478bd9Sstevel@tonic-gate if (tsp != NULL && 32417c478bd9Sstevel@tonic-gate (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC)) 32427c478bd9Sstevel@tonic-gate return (EINVAL); 32437c478bd9Sstevel@tonic-gate 32447c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 32457c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 32467c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 32477c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 32487c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = cvp; 32497c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 32507c478bd9Sstevel@tonic-gate self->ul_sp = 0; 32517c478bd9Sstevel@tonic-gate } 32527c478bd9Sstevel@tonic-gate if (csp) { 32537c478bd9Sstevel@tonic-gate if (tsp) 32547c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait); 32557c478bd9Sstevel@tonic-gate else 32567c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_wait); 32577c478bd9Sstevel@tonic-gate } 32587c478bd9Sstevel@tonic-gate if (msp) 32597c478bd9Sstevel@tonic-gate begin_sleep = record_hold_time(msp); 32607c478bd9Sstevel@tonic-gate else if (csp) 32617c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 32627c478bd9Sstevel@tonic-gate 32637c478bd9Sstevel@tonic-gate if (self->ul_error_detection) { 32647c478bd9Sstevel@tonic-gate if (!mutex_is_held(mp)) 32657c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, NULL); 32667c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) 32677c478bd9Sstevel@tonic-gate lock_error(mp, "recursive mutex in cond_wait", 32685d1dd9a9Sraf cvp, NULL); 32697c478bd9Sstevel@tonic-gate if (cvp->cond_type & USYNC_PROCESS) { 3270883492d5Sraf if (!(mtype & USYNC_PROCESS)) 32717c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 32725d1dd9a9Sraf "condvar process-shared, " 32735d1dd9a9Sraf "mutex process-private"); 32747c478bd9Sstevel@tonic-gate } else { 3275883492d5Sraf if (mtype & USYNC_PROCESS) 32767c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 32775d1dd9a9Sraf "condvar process-private, " 32785d1dd9a9Sraf "mutex process-shared"); 32797c478bd9Sstevel@tonic-gate } 32807c478bd9Sstevel@tonic-gate } 32817c478bd9Sstevel@tonic-gate 32827c478bd9Sstevel@tonic-gate /* 32837c478bd9Sstevel@tonic-gate * We deal with recursive mutexes by completely 32847c478bd9Sstevel@tonic-gate * dropping the lock and restoring the recursion 32857c478bd9Sstevel@tonic-gate * count after waking up. This is arguably wrong, 32867c478bd9Sstevel@tonic-gate * but it obeys the principle of least astonishment. 32877c478bd9Sstevel@tonic-gate */ 32887c478bd9Sstevel@tonic-gate rcount = mp->mutex_rcount; 32897c478bd9Sstevel@tonic-gate mp->mutex_rcount = 0; 3290883492d5Sraf if ((mtype & 3291883492d5Sraf (USYNC_PROCESS | LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT)) | 32927c478bd9Sstevel@tonic-gate (cvp->cond_type & USYNC_PROCESS)) 32937c478bd9Sstevel@tonic-gate error = cond_wait_kernel(cvp, mp, tsp); 32947c478bd9Sstevel@tonic-gate else 32955d1dd9a9Sraf error = cond_wait_queue(cvp, mp, tsp); 32967c478bd9Sstevel@tonic-gate mp->mutex_rcount = rcount; 32977c478bd9Sstevel@tonic-gate 32987c478bd9Sstevel@tonic-gate if (csp) { 32997c478bd9Sstevel@tonic-gate hrtime_t lapse = gethrtime() - begin_sleep; 33007c478bd9Sstevel@tonic-gate if (tsp == NULL) 33017c478bd9Sstevel@tonic-gate csp->cond_wait_sleep_time += lapse; 33027c478bd9Sstevel@tonic-gate else { 33037c478bd9Sstevel@tonic-gate csp->cond_timedwait_sleep_time += lapse; 33047c478bd9Sstevel@tonic-gate if (error == ETIME) 33057c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait_timeout); 33067c478bd9Sstevel@tonic-gate } 33077c478bd9Sstevel@tonic-gate } 33087c478bd9Sstevel@tonic-gate return (error); 33097c478bd9Sstevel@tonic-gate } 33107c478bd9Sstevel@tonic-gate 33117c478bd9Sstevel@tonic-gate /* 3312a574db85Sraf * cond_wait() and _cond_wait() are cancellation points but __cond_wait() 3313a574db85Sraf * is not. Internally, libc calls the non-cancellation version. 3314a574db85Sraf * Other libraries need to use pthread_setcancelstate(), as appropriate, 3315a574db85Sraf * since __cond_wait() is not exported from libc. 33167c478bd9Sstevel@tonic-gate */ 33177c478bd9Sstevel@tonic-gate int 3318a574db85Sraf __cond_wait(cond_t *cvp, mutex_t *mp) 33197c478bd9Sstevel@tonic-gate { 33207c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 33217c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 33227c478bd9Sstevel@tonic-gate uberflags_t *gflags; 33237c478bd9Sstevel@tonic-gate 33247c478bd9Sstevel@tonic-gate /* 33257c478bd9Sstevel@tonic-gate * Optimize the common case of USYNC_THREAD plus 33267c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, and no event tracing. 33277c478bd9Sstevel@tonic-gate */ 33287c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 33297c478bd9Sstevel@tonic-gate (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted | 33307c478bd9Sstevel@tonic-gate self->ul_td_events_enable | 33317c478bd9Sstevel@tonic-gate udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0) 33325d1dd9a9Sraf return (cond_wait_queue(cvp, mp, NULL)); 33337c478bd9Sstevel@tonic-gate 33347c478bd9Sstevel@tonic-gate /* 33357c478bd9Sstevel@tonic-gate * Else do it the long way. 33367c478bd9Sstevel@tonic-gate */ 33377c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, NULL)); 33387c478bd9Sstevel@tonic-gate } 33397c478bd9Sstevel@tonic-gate 3340a574db85Sraf #pragma weak cond_wait = _cond_wait 33417c478bd9Sstevel@tonic-gate int 3342a574db85Sraf _cond_wait(cond_t *cvp, mutex_t *mp) 33437c478bd9Sstevel@tonic-gate { 33447c478bd9Sstevel@tonic-gate int error; 33457c478bd9Sstevel@tonic-gate 33467c478bd9Sstevel@tonic-gate _cancelon(); 3347a574db85Sraf error = __cond_wait(cvp, mp); 33487c478bd9Sstevel@tonic-gate if (error == EINTR) 33497c478bd9Sstevel@tonic-gate _canceloff(); 33507c478bd9Sstevel@tonic-gate else 33517c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 33527c478bd9Sstevel@tonic-gate return (error); 33537c478bd9Sstevel@tonic-gate } 33547c478bd9Sstevel@tonic-gate 3355a574db85Sraf /* 3356a574db85Sraf * pthread_cond_wait() is a cancellation point. 3357a574db85Sraf */ 33587c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_wait = _pthread_cond_wait 33597c478bd9Sstevel@tonic-gate int 33607c478bd9Sstevel@tonic-gate _pthread_cond_wait(cond_t *cvp, mutex_t *mp) 33617c478bd9Sstevel@tonic-gate { 33627c478bd9Sstevel@tonic-gate int error; 33637c478bd9Sstevel@tonic-gate 3364a574db85Sraf error = _cond_wait(cvp, mp); 33657c478bd9Sstevel@tonic-gate return ((error == EINTR)? 0 : error); 33667c478bd9Sstevel@tonic-gate } 33677c478bd9Sstevel@tonic-gate 33687c478bd9Sstevel@tonic-gate /* 3369a574db85Sraf * cond_timedwait() and _cond_timedwait() are cancellation points 3370a574db85Sraf * but __cond_timedwait() is not. 33717c478bd9Sstevel@tonic-gate */ 33727c478bd9Sstevel@tonic-gate int 3373a574db85Sraf __cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 33747c478bd9Sstevel@tonic-gate { 33757c478bd9Sstevel@tonic-gate clockid_t clock_id = cvp->cond_clockid; 33767c478bd9Sstevel@tonic-gate timespec_t reltime; 33777c478bd9Sstevel@tonic-gate int error; 33787c478bd9Sstevel@tonic-gate 33797c478bd9Sstevel@tonic-gate if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES) 33807c478bd9Sstevel@tonic-gate clock_id = CLOCK_REALTIME; 33817c478bd9Sstevel@tonic-gate abstime_to_reltime(clock_id, abstime, &reltime); 33827c478bd9Sstevel@tonic-gate error = cond_wait_common(cvp, mp, &reltime); 33837c478bd9Sstevel@tonic-gate if (error == ETIME && clock_id == CLOCK_HIGHRES) { 33847c478bd9Sstevel@tonic-gate /* 33857c478bd9Sstevel@tonic-gate * Don't return ETIME if we didn't really get a timeout. 33867c478bd9Sstevel@tonic-gate * This can happen if we return because someone resets 33877c478bd9Sstevel@tonic-gate * the system clock. Just return zero in this case, 33887c478bd9Sstevel@tonic-gate * giving a spurious wakeup but not a timeout. 33897c478bd9Sstevel@tonic-gate */ 33907c478bd9Sstevel@tonic-gate if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC + 33917c478bd9Sstevel@tonic-gate abstime->tv_nsec > gethrtime()) 33927c478bd9Sstevel@tonic-gate error = 0; 33937c478bd9Sstevel@tonic-gate } 33947c478bd9Sstevel@tonic-gate return (error); 33957c478bd9Sstevel@tonic-gate } 33967c478bd9Sstevel@tonic-gate 3397a574db85Sraf #pragma weak cond_timedwait = _cond_timedwait 33987c478bd9Sstevel@tonic-gate int 3399a574db85Sraf _cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 34007c478bd9Sstevel@tonic-gate { 34017c478bd9Sstevel@tonic-gate int error; 34027c478bd9Sstevel@tonic-gate 34037c478bd9Sstevel@tonic-gate _cancelon(); 3404a574db85Sraf error = __cond_timedwait(cvp, mp, abstime); 34057c478bd9Sstevel@tonic-gate if (error == EINTR) 34067c478bd9Sstevel@tonic-gate _canceloff(); 34077c478bd9Sstevel@tonic-gate else 34087c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 34097c478bd9Sstevel@tonic-gate return (error); 34107c478bd9Sstevel@tonic-gate } 34117c478bd9Sstevel@tonic-gate 3412a574db85Sraf /* 3413a574db85Sraf * pthread_cond_timedwait() is a cancellation point. 3414a574db85Sraf */ 34157c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_timedwait = _pthread_cond_timedwait 34167c478bd9Sstevel@tonic-gate int 34177c478bd9Sstevel@tonic-gate _pthread_cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 34187c478bd9Sstevel@tonic-gate { 34197c478bd9Sstevel@tonic-gate int error; 34207c478bd9Sstevel@tonic-gate 3421a574db85Sraf error = _cond_timedwait(cvp, mp, abstime); 34227c478bd9Sstevel@tonic-gate if (error == ETIME) 34237c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 34247c478bd9Sstevel@tonic-gate else if (error == EINTR) 34257c478bd9Sstevel@tonic-gate error = 0; 34267c478bd9Sstevel@tonic-gate return (error); 34277c478bd9Sstevel@tonic-gate } 34287c478bd9Sstevel@tonic-gate 34297c478bd9Sstevel@tonic-gate /* 3430a574db85Sraf * cond_reltimedwait() and _cond_reltimedwait() are cancellation points 3431a574db85Sraf * but __cond_reltimedwait() is not. 34327c478bd9Sstevel@tonic-gate */ 34337c478bd9Sstevel@tonic-gate int 3434a574db85Sraf __cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 34357c478bd9Sstevel@tonic-gate { 34367c478bd9Sstevel@tonic-gate timespec_t tslocal = *reltime; 34377c478bd9Sstevel@tonic-gate 34387c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, &tslocal)); 34397c478bd9Sstevel@tonic-gate } 34407c478bd9Sstevel@tonic-gate 3441a574db85Sraf #pragma weak cond_reltimedwait = _cond_reltimedwait 34427c478bd9Sstevel@tonic-gate int 3443a574db85Sraf _cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 34447c478bd9Sstevel@tonic-gate { 34457c478bd9Sstevel@tonic-gate int error; 34467c478bd9Sstevel@tonic-gate 34477c478bd9Sstevel@tonic-gate _cancelon(); 3448a574db85Sraf error = __cond_reltimedwait(cvp, mp, reltime); 34497c478bd9Sstevel@tonic-gate if (error == EINTR) 34507c478bd9Sstevel@tonic-gate _canceloff(); 34517c478bd9Sstevel@tonic-gate else 34527c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 34537c478bd9Sstevel@tonic-gate return (error); 34547c478bd9Sstevel@tonic-gate } 34557c478bd9Sstevel@tonic-gate 34567c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_reltimedwait_np = _pthread_cond_reltimedwait_np 34577c478bd9Sstevel@tonic-gate int 34587c478bd9Sstevel@tonic-gate _pthread_cond_reltimedwait_np(cond_t *cvp, mutex_t *mp, 34597c478bd9Sstevel@tonic-gate const timespec_t *reltime) 34607c478bd9Sstevel@tonic-gate { 34617c478bd9Sstevel@tonic-gate int error; 34627c478bd9Sstevel@tonic-gate 3463a574db85Sraf error = _cond_reltimedwait(cvp, mp, reltime); 34647c478bd9Sstevel@tonic-gate if (error == ETIME) 34657c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 34667c478bd9Sstevel@tonic-gate else if (error == EINTR) 34677c478bd9Sstevel@tonic-gate error = 0; 34687c478bd9Sstevel@tonic-gate return (error); 34697c478bd9Sstevel@tonic-gate } 34707c478bd9Sstevel@tonic-gate 34717c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_signal = cond_signal_internal 34727c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_signal = cond_signal_internal 34737c478bd9Sstevel@tonic-gate #pragma weak cond_signal = cond_signal_internal 34747c478bd9Sstevel@tonic-gate #pragma weak _cond_signal = cond_signal_internal 34757c478bd9Sstevel@tonic-gate int 34767c478bd9Sstevel@tonic-gate cond_signal_internal(cond_t *cvp) 34777c478bd9Sstevel@tonic-gate { 34787c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 34797c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 34807c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 34817c478bd9Sstevel@tonic-gate int error = 0; 3482d4204c85Sraf int more; 3483d4204c85Sraf lwpid_t lwpid; 34847c478bd9Sstevel@tonic-gate queue_head_t *qp; 34857c478bd9Sstevel@tonic-gate mutex_t *mp; 34867c478bd9Sstevel@tonic-gate queue_head_t *mqp; 34877c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 34887c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 3489d4204c85Sraf ulwp_t *prev; 34907c478bd9Sstevel@tonic-gate 34917c478bd9Sstevel@tonic-gate if (csp) 34927c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_signal); 34937c478bd9Sstevel@tonic-gate 34947c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 34957c478bd9Sstevel@tonic-gate error = __lwp_cond_signal(cvp); 34967c478bd9Sstevel@tonic-gate 34977c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 34987c478bd9Sstevel@tonic-gate return (error); 34997c478bd9Sstevel@tonic-gate 35007c478bd9Sstevel@tonic-gate /* 35017c478bd9Sstevel@tonic-gate * Move someone from the condvar sleep queue to the mutex sleep 35027c478bd9Sstevel@tonic-gate * queue for the mutex that he will acquire on being waked up. 35037c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex he will acquire. 35047c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if his ul_cv_wake flag 35057c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark him. 35067c478bd9Sstevel@tonic-gate */ 35077c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 3508d4204c85Sraf ulwpp = queue_slot(qp, &prev, &more); 3509d4204c85Sraf cvp->cond_waiters_user = more; 3510d4204c85Sraf if (ulwpp == NULL) { /* no one on the sleep queue */ 35117c478bd9Sstevel@tonic-gate queue_unlock(qp); 35127c478bd9Sstevel@tonic-gate return (error); 35137c478bd9Sstevel@tonic-gate } 3514d4204c85Sraf ulwp = *ulwpp; 35157c478bd9Sstevel@tonic-gate 35167c478bd9Sstevel@tonic-gate /* 35177c478bd9Sstevel@tonic-gate * Inform the thread that he was the recipient of a cond_signal(). 35187c478bd9Sstevel@tonic-gate * This lets him deal with cond_signal() and, concurrently, 35197c478bd9Sstevel@tonic-gate * one or more of a cancellation, a UNIX signal, or a timeout. 35207c478bd9Sstevel@tonic-gate * These latter conditions must not consume a cond_signal(). 35217c478bd9Sstevel@tonic-gate */ 35227c478bd9Sstevel@tonic-gate ulwp->ul_signalled = 1; 35237c478bd9Sstevel@tonic-gate 35247c478bd9Sstevel@tonic-gate /* 35257c478bd9Sstevel@tonic-gate * Dequeue the waiter but leave his ul_sleepq non-NULL 35267c478bd9Sstevel@tonic-gate * while we move him to the mutex queue so that he can 35277c478bd9Sstevel@tonic-gate * deal properly with spurious wakeups. 35287c478bd9Sstevel@tonic-gate */ 3529d4204c85Sraf queue_unlink(qp, ulwpp, prev); 35307c478bd9Sstevel@tonic-gate 35317c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* the mutex he will acquire */ 35327c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 35337c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 35347c478bd9Sstevel@tonic-gate 35357c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 3536d4204c85Sraf /* just wake him up */ 3537d4204c85Sraf lwpid = ulwp->ul_lwpid; 35387c478bd9Sstevel@tonic-gate no_preempt(self); 35397c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 35407c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 35417c478bd9Sstevel@tonic-gate queue_unlock(qp); 35427c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 35437c478bd9Sstevel@tonic-gate preempt(self); 35447c478bd9Sstevel@tonic-gate } else { 3545d4204c85Sraf /* move him to the mutex queue */ 35467c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 3547d4204c85Sraf enqueue(mqp, ulwp, 0); 35487c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 35497c478bd9Sstevel@tonic-gate queue_unlock(mqp); 35507c478bd9Sstevel@tonic-gate queue_unlock(qp); 35517c478bd9Sstevel@tonic-gate } 35527c478bd9Sstevel@tonic-gate 35537c478bd9Sstevel@tonic-gate return (error); 35547c478bd9Sstevel@tonic-gate } 35557c478bd9Sstevel@tonic-gate 355641efec22Sraf /* 3557883492d5Sraf * Utility function called by mutex_wakeup_all(), cond_broadcast(), 3558883492d5Sraf * and rw_queue_release() to (re)allocate a big buffer to hold the 3559883492d5Sraf * lwpids of all the threads to be set running after they are removed 3560883492d5Sraf * from their sleep queues. Since we are holding a queue lock, we 3561883492d5Sraf * cannot call any function that might acquire a lock. mmap(), munmap(), 3562883492d5Sraf * lwp_unpark_all() are simple system calls and are safe in this regard. 356341efec22Sraf */ 356441efec22Sraf lwpid_t * 356541efec22Sraf alloc_lwpids(lwpid_t *lwpid, int *nlwpid_ptr, int *maxlwps_ptr) 356641efec22Sraf { 356741efec22Sraf /* 356841efec22Sraf * Allocate NEWLWPS ids on the first overflow. 356941efec22Sraf * Double the allocation each time after that. 357041efec22Sraf */ 357141efec22Sraf int nlwpid = *nlwpid_ptr; 357241efec22Sraf int maxlwps = *maxlwps_ptr; 357341efec22Sraf int first_allocation; 357441efec22Sraf int newlwps; 357541efec22Sraf void *vaddr; 357641efec22Sraf 357741efec22Sraf ASSERT(nlwpid == maxlwps); 357841efec22Sraf 357941efec22Sraf first_allocation = (maxlwps == MAXLWPS); 358041efec22Sraf newlwps = first_allocation? NEWLWPS : 2 * maxlwps; 3581*8cd45542Sraf vaddr = mmap(NULL, newlwps * sizeof (lwpid_t), 358241efec22Sraf PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, (off_t)0); 358341efec22Sraf 358441efec22Sraf if (vaddr == MAP_FAILED) { 358541efec22Sraf /* 358641efec22Sraf * Let's hope this never happens. 358741efec22Sraf * If it does, then we have a terrible 358841efec22Sraf * thundering herd on our hands. 358941efec22Sraf */ 359041efec22Sraf (void) __lwp_unpark_all(lwpid, nlwpid); 359141efec22Sraf *nlwpid_ptr = 0; 359241efec22Sraf } else { 3593*8cd45542Sraf (void) memcpy(vaddr, lwpid, maxlwps * sizeof (lwpid_t)); 359441efec22Sraf if (!first_allocation) 3595*8cd45542Sraf (void) munmap((caddr_t)lwpid, 359641efec22Sraf maxlwps * sizeof (lwpid_t)); 359741efec22Sraf lwpid = vaddr; 359841efec22Sraf *maxlwps_ptr = newlwps; 359941efec22Sraf } 360041efec22Sraf 360141efec22Sraf return (lwpid); 360241efec22Sraf } 36037c478bd9Sstevel@tonic-gate 36047c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_broadcast = cond_broadcast_internal 36057c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_broadcast = cond_broadcast_internal 36067c478bd9Sstevel@tonic-gate #pragma weak cond_broadcast = cond_broadcast_internal 36077c478bd9Sstevel@tonic-gate #pragma weak _cond_broadcast = cond_broadcast_internal 36087c478bd9Sstevel@tonic-gate int 36097c478bd9Sstevel@tonic-gate cond_broadcast_internal(cond_t *cvp) 36107c478bd9Sstevel@tonic-gate { 36117c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 36127c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 36137c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 36147c478bd9Sstevel@tonic-gate int error = 0; 36157c478bd9Sstevel@tonic-gate queue_head_t *qp; 3616d4204c85Sraf queue_root_t *qrp; 36177c478bd9Sstevel@tonic-gate mutex_t *mp; 36187c478bd9Sstevel@tonic-gate mutex_t *mp_cache = NULL; 361941efec22Sraf queue_head_t *mqp = NULL; 36207c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 36217c478bd9Sstevel@tonic-gate int nlwpid = 0; 36227c478bd9Sstevel@tonic-gate int maxlwps = MAXLWPS; 362341efec22Sraf lwpid_t buffer[MAXLWPS]; 362441efec22Sraf lwpid_t *lwpid = buffer; 36257c478bd9Sstevel@tonic-gate 36267c478bd9Sstevel@tonic-gate if (csp) 36277c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_broadcast); 36287c478bd9Sstevel@tonic-gate 36297c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 36307c478bd9Sstevel@tonic-gate error = __lwp_cond_broadcast(cvp); 36317c478bd9Sstevel@tonic-gate 36327c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 36337c478bd9Sstevel@tonic-gate return (error); 36347c478bd9Sstevel@tonic-gate 36357c478bd9Sstevel@tonic-gate /* 36367c478bd9Sstevel@tonic-gate * Move everyone from the condvar sleep queue to the mutex sleep 36377c478bd9Sstevel@tonic-gate * queue for the mutex that they will acquire on being waked up. 36387c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex they will acquire. 36397c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if their ul_cv_wake flag 36407c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark them. 36417c478bd9Sstevel@tonic-gate * 36427c478bd9Sstevel@tonic-gate * We keep track of lwpids that are to be unparked in lwpid[]. 36437c478bd9Sstevel@tonic-gate * __lwp_unpark_all() is called to unpark all of them after 36447c478bd9Sstevel@tonic-gate * they have been removed from the sleep queue and the sleep 36457c478bd9Sstevel@tonic-gate * queue lock has been dropped. If we run out of space in our 36467c478bd9Sstevel@tonic-gate * on-stack buffer, we need to allocate more but we can't call 36477c478bd9Sstevel@tonic-gate * lmalloc() because we are holding a queue lock when the overflow 36487c478bd9Sstevel@tonic-gate * occurs and lmalloc() acquires a lock. We can't use alloca() 364941efec22Sraf * either because the application may have allocated a small 365041efec22Sraf * stack and we don't want to overrun the stack. So we call 365141efec22Sraf * alloc_lwpids() to allocate a bigger buffer using the mmap() 36527c478bd9Sstevel@tonic-gate * system call directly since that path acquires no locks. 36537c478bd9Sstevel@tonic-gate */ 36547c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 36557c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 3656d4204c85Sraf for (;;) { 3657d4204c85Sraf if ((qrp = qp->qh_root) == NULL || 3658d4204c85Sraf (ulwp = qrp->qr_head) == NULL) 3659d4204c85Sraf break; 3660d4204c85Sraf ASSERT(ulwp->ul_wchan == cvp); 3661d4204c85Sraf queue_unlink(qp, &qrp->qr_head, NULL); 36627c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* his mutex */ 36637c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 36647c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 36657c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 3666d4204c85Sraf /* just wake him up */ 36677c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 36687c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 366941efec22Sraf if (nlwpid == maxlwps) 367041efec22Sraf lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps); 36717c478bd9Sstevel@tonic-gate lwpid[nlwpid++] = ulwp->ul_lwpid; 36727c478bd9Sstevel@tonic-gate } else { 3673d4204c85Sraf /* move him to the mutex queue */ 36747c478bd9Sstevel@tonic-gate if (mp != mp_cache) { 36757c478bd9Sstevel@tonic-gate mp_cache = mp; 367641efec22Sraf if (mqp != NULL) 367741efec22Sraf queue_unlock(mqp); 367841efec22Sraf mqp = queue_lock(mp, MX); 36797c478bd9Sstevel@tonic-gate } 3680d4204c85Sraf enqueue(mqp, ulwp, 0); 36817c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 36827c478bd9Sstevel@tonic-gate } 36837c478bd9Sstevel@tonic-gate } 368441efec22Sraf if (mqp != NULL) 368541efec22Sraf queue_unlock(mqp); 368641efec22Sraf if (nlwpid == 0) { 368741efec22Sraf queue_unlock(qp); 368841efec22Sraf } else { 368941efec22Sraf no_preempt(self); 369041efec22Sraf queue_unlock(qp); 36917c478bd9Sstevel@tonic-gate if (nlwpid == 1) 36927c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid[0]); 36937c478bd9Sstevel@tonic-gate else 36947c478bd9Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid); 369541efec22Sraf preempt(self); 36967c478bd9Sstevel@tonic-gate } 36977c478bd9Sstevel@tonic-gate if (lwpid != buffer) 3698*8cd45542Sraf (void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t)); 36997c478bd9Sstevel@tonic-gate return (error); 37007c478bd9Sstevel@tonic-gate } 37017c478bd9Sstevel@tonic-gate 37027c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_destroy = _cond_destroy 37037c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_destroy = _cond_destroy 37047c478bd9Sstevel@tonic-gate #pragma weak cond_destroy = _cond_destroy 37057c478bd9Sstevel@tonic-gate int 37067c478bd9Sstevel@tonic-gate _cond_destroy(cond_t *cvp) 37077c478bd9Sstevel@tonic-gate { 37087c478bd9Sstevel@tonic-gate cvp->cond_magic = 0; 37097c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(cvp); 37107c478bd9Sstevel@tonic-gate return (0); 37117c478bd9Sstevel@tonic-gate } 37127c478bd9Sstevel@tonic-gate 37137c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 37147c478bd9Sstevel@tonic-gate void 37157c478bd9Sstevel@tonic-gate assert_no_libc_locks_held(void) 37167c478bd9Sstevel@tonic-gate { 37177c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 37187c478bd9Sstevel@tonic-gate } 37197c478bd9Sstevel@tonic-gate 37207c478bd9Sstevel@tonic-gate /* protected by link_lock */ 37217c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin; 37227c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin2; 37237c478bd9Sstevel@tonic-gate uint64_t spin_lock_sleep; 37247c478bd9Sstevel@tonic-gate uint64_t spin_lock_wakeup; 37257c478bd9Sstevel@tonic-gate 37267c478bd9Sstevel@tonic-gate /* 37277c478bd9Sstevel@tonic-gate * Record spin lock statistics. 37287c478bd9Sstevel@tonic-gate * Called by a thread exiting itself in thrp_exit(). 37297c478bd9Sstevel@tonic-gate * Also called via atexit() from the thread calling 37307c478bd9Sstevel@tonic-gate * exit() to do all the other threads as well. 37317c478bd9Sstevel@tonic-gate */ 37327c478bd9Sstevel@tonic-gate void 37337c478bd9Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp) 37347c478bd9Sstevel@tonic-gate { 37357c478bd9Sstevel@tonic-gate spin_lock_spin += ulwp->ul_spin_lock_spin; 37367c478bd9Sstevel@tonic-gate spin_lock_spin2 += ulwp->ul_spin_lock_spin2; 37377c478bd9Sstevel@tonic-gate spin_lock_sleep += ulwp->ul_spin_lock_sleep; 37387c478bd9Sstevel@tonic-gate spin_lock_wakeup += ulwp->ul_spin_lock_wakeup; 37397c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin = 0; 37407c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin2 = 0; 37417c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_sleep = 0; 37427c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_wakeup = 0; 37437c478bd9Sstevel@tonic-gate } 37447c478bd9Sstevel@tonic-gate 37457c478bd9Sstevel@tonic-gate /* 37467c478bd9Sstevel@tonic-gate * atexit function: dump the queue statistics to stderr. 37477c478bd9Sstevel@tonic-gate */ 3748e8031f0aSraf #if !defined(__lint) 3749e8031f0aSraf #define fprintf _fprintf 3750e8031f0aSraf #endif 37517c478bd9Sstevel@tonic-gate #include <stdio.h> 37527c478bd9Sstevel@tonic-gate void 37537c478bd9Sstevel@tonic-gate dump_queue_statistics(void) 37547c478bd9Sstevel@tonic-gate { 37557c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 37567c478bd9Sstevel@tonic-gate queue_head_t *qp; 37577c478bd9Sstevel@tonic-gate int qn; 37587c478bd9Sstevel@tonic-gate uint64_t spin_lock_total = 0; 37597c478bd9Sstevel@tonic-gate 37607c478bd9Sstevel@tonic-gate if (udp->queue_head == NULL || thread_queue_dump == 0) 37617c478bd9Sstevel@tonic-gate return; 37627c478bd9Sstevel@tonic-gate 37637c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 || 3764d4204c85Sraf fprintf(stderr, "queue# lockcount max qlen max hlen\n") < 0) 37657c478bd9Sstevel@tonic-gate return; 37667c478bd9Sstevel@tonic-gate for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) { 37677c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 37687c478bd9Sstevel@tonic-gate continue; 37697c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 3770d4204c85Sraf if (fprintf(stderr, "%5d %12llu%12u%12u\n", qn, 3771d4204c85Sraf (u_longlong_t)qp->qh_lockcount, 3772d4204c85Sraf qp->qh_qmax, qp->qh_hmax) < 0) 37735d1dd9a9Sraf return; 37747c478bd9Sstevel@tonic-gate } 37757c478bd9Sstevel@tonic-gate 37767c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 || 3777d4204c85Sraf fprintf(stderr, "queue# lockcount max qlen max hlen\n") < 0) 37787c478bd9Sstevel@tonic-gate return; 37797c478bd9Sstevel@tonic-gate for (qn = 0; qn < QHASHSIZE; qn++, qp++) { 37807c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 37817c478bd9Sstevel@tonic-gate continue; 37827c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 3783d4204c85Sraf if (fprintf(stderr, "%5d %12llu%12u%12u\n", qn, 3784d4204c85Sraf (u_longlong_t)qp->qh_lockcount, 3785d4204c85Sraf qp->qh_qmax, qp->qh_hmax) < 0) 37865d1dd9a9Sraf return; 37877c478bd9Sstevel@tonic-gate } 37887c478bd9Sstevel@tonic-gate 37897c478bd9Sstevel@tonic-gate (void) fprintf(stderr, "\n spin_lock_total = %10llu\n", 37905d1dd9a9Sraf (u_longlong_t)spin_lock_total); 37917c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin = %10llu\n", 37925d1dd9a9Sraf (u_longlong_t)spin_lock_spin); 37937c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin2 = %10llu\n", 37945d1dd9a9Sraf (u_longlong_t)spin_lock_spin2); 37957c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_sleep = %10llu\n", 37965d1dd9a9Sraf (u_longlong_t)spin_lock_sleep); 37977c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_wakeup = %10llu\n", 37985d1dd9a9Sraf (u_longlong_t)spin_lock_wakeup); 37997c478bd9Sstevel@tonic-gate } 3800d4204c85Sraf #endif 3801