17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 22*e8031f0aSraf 237c478bd9Sstevel@tonic-gate /* 24*e8031f0aSraf * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 257c478bd9Sstevel@tonic-gate * Use is subject to license terms. 267c478bd9Sstevel@tonic-gate */ 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 317c478bd9Sstevel@tonic-gate 327c478bd9Sstevel@tonic-gate #include "lint.h" 337c478bd9Sstevel@tonic-gate #include "thr_uberdata.h" 347c478bd9Sstevel@tonic-gate 357c478bd9Sstevel@tonic-gate /* 367c478bd9Sstevel@tonic-gate * This mutex is initialized to be held by lwp#1. 377c478bd9Sstevel@tonic-gate * It is used to block a thread that has returned from a mutex_lock() 387c478bd9Sstevel@tonic-gate * of a PTHREAD_PRIO_INHERIT mutex with an unrecoverable error. 397c478bd9Sstevel@tonic-gate */ 407c478bd9Sstevel@tonic-gate mutex_t stall_mutex = DEFAULTMUTEX; 417c478bd9Sstevel@tonic-gate 427c478bd9Sstevel@tonic-gate static int shared_mutex_held(mutex_t *); 437c478bd9Sstevel@tonic-gate 447c478bd9Sstevel@tonic-gate /* 457c478bd9Sstevel@tonic-gate * Lock statistics support functions. 467c478bd9Sstevel@tonic-gate */ 477c478bd9Sstevel@tonic-gate void 487c478bd9Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp) 497c478bd9Sstevel@tonic-gate { 507c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_lock); 517c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = gethrtime(); 527c478bd9Sstevel@tonic-gate } 537c478bd9Sstevel@tonic-gate 547c478bd9Sstevel@tonic-gate hrtime_t 557c478bd9Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp) 567c478bd9Sstevel@tonic-gate { 577c478bd9Sstevel@tonic-gate hrtime_t now = gethrtime(); 587c478bd9Sstevel@tonic-gate 597c478bd9Sstevel@tonic-gate if (msp->mutex_begin_hold) 607c478bd9Sstevel@tonic-gate msp->mutex_hold_time += now - msp->mutex_begin_hold; 617c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = 0; 627c478bd9Sstevel@tonic-gate return (now); 637c478bd9Sstevel@tonic-gate } 647c478bd9Sstevel@tonic-gate 657c478bd9Sstevel@tonic-gate /* 667c478bd9Sstevel@tonic-gate * Called once at library initialization. 677c478bd9Sstevel@tonic-gate */ 687c478bd9Sstevel@tonic-gate void 697c478bd9Sstevel@tonic-gate mutex_setup(void) 707c478bd9Sstevel@tonic-gate { 717c478bd9Sstevel@tonic-gate if (set_lock_byte(&stall_mutex.mutex_lockw)) 727c478bd9Sstevel@tonic-gate thr_panic("mutex_setup() cannot acquire stall_mutex"); 737c478bd9Sstevel@tonic-gate stall_mutex.mutex_owner = (uintptr_t)curthread; 747c478bd9Sstevel@tonic-gate } 757c478bd9Sstevel@tonic-gate 767c478bd9Sstevel@tonic-gate /* 777c478bd9Sstevel@tonic-gate * The default spin counts of 1000 and 500 are experimentally determined. 787c478bd9Sstevel@tonic-gate * On sun4u machines with any number of processors they could be raised 797c478bd9Sstevel@tonic-gate * to 10,000 but that (experimentally) makes almost no difference. 807c478bd9Sstevel@tonic-gate * The environment variables: 817c478bd9Sstevel@tonic-gate * _THREAD_ADAPTIVE_SPIN=count 827c478bd9Sstevel@tonic-gate * _THREAD_RELEASE_SPIN=count 837c478bd9Sstevel@tonic-gate * can be used to override and set the counts in the range [0 .. 1,000,000]. 847c478bd9Sstevel@tonic-gate */ 857c478bd9Sstevel@tonic-gate int thread_adaptive_spin = 1000; 867c478bd9Sstevel@tonic-gate uint_t thread_max_spinners = 100; 877c478bd9Sstevel@tonic-gate int thread_release_spin = 500; 887c478bd9Sstevel@tonic-gate int thread_queue_verify = 0; 897c478bd9Sstevel@tonic-gate static int ncpus; 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate /* 927c478bd9Sstevel@tonic-gate * Distinguish spinning for queue locks from spinning for regular locks. 937c478bd9Sstevel@tonic-gate * The environment variable: 947c478bd9Sstevel@tonic-gate * _THREAD_QUEUE_SPIN=count 957c478bd9Sstevel@tonic-gate * can be used to override and set the count in the range [0 .. 1,000,000]. 967c478bd9Sstevel@tonic-gate * There is no release spin concept for queue locks. 977c478bd9Sstevel@tonic-gate */ 987c478bd9Sstevel@tonic-gate int thread_queue_spin = 1000; 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate /* 1017c478bd9Sstevel@tonic-gate * Use the otherwise-unused 'mutex_ownerpid' field of a USYNC_THREAD 1027c478bd9Sstevel@tonic-gate * mutex to be a count of adaptive spins in progress. 1037c478bd9Sstevel@tonic-gate */ 1047c478bd9Sstevel@tonic-gate #define mutex_spinners mutex_ownerpid 1057c478bd9Sstevel@tonic-gate 1067c478bd9Sstevel@tonic-gate void 1077c478bd9Sstevel@tonic-gate _mutex_set_typeattr(mutex_t *mp, int attr) 1087c478bd9Sstevel@tonic-gate { 1097c478bd9Sstevel@tonic-gate mp->mutex_type |= (uint8_t)attr; 1107c478bd9Sstevel@tonic-gate } 1117c478bd9Sstevel@tonic-gate 1127c478bd9Sstevel@tonic-gate /* 1137c478bd9Sstevel@tonic-gate * 'type' can be one of USYNC_THREAD or USYNC_PROCESS, possibly 1147c478bd9Sstevel@tonic-gate * augmented by the flags LOCK_RECURSIVE and/or LOCK_ERRORCHECK, 1157c478bd9Sstevel@tonic-gate * or it can be USYNC_PROCESS_ROBUST with no extra flags. 1167c478bd9Sstevel@tonic-gate */ 1177c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_init = __mutex_init 1187c478bd9Sstevel@tonic-gate #pragma weak mutex_init = __mutex_init 1197c478bd9Sstevel@tonic-gate #pragma weak _mutex_init = __mutex_init 1207c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 1217c478bd9Sstevel@tonic-gate int 1227c478bd9Sstevel@tonic-gate __mutex_init(mutex_t *mp, int type, void *arg) 1237c478bd9Sstevel@tonic-gate { 1247c478bd9Sstevel@tonic-gate int error; 1257c478bd9Sstevel@tonic-gate 1267c478bd9Sstevel@tonic-gate switch (type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) { 1277c478bd9Sstevel@tonic-gate case USYNC_THREAD: 1287c478bd9Sstevel@tonic-gate case USYNC_PROCESS: 1297c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 1307c478bd9Sstevel@tonic-gate mp->mutex_type = (uint8_t)type; 1317c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 1327c478bd9Sstevel@tonic-gate error = 0; 1337c478bd9Sstevel@tonic-gate break; 1347c478bd9Sstevel@tonic-gate case USYNC_PROCESS_ROBUST: 1357c478bd9Sstevel@tonic-gate if (type & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) 1367c478bd9Sstevel@tonic-gate error = EINVAL; 1377c478bd9Sstevel@tonic-gate else 1387c478bd9Sstevel@tonic-gate error = ___lwp_mutex_init(mp, type); 1397c478bd9Sstevel@tonic-gate break; 1407c478bd9Sstevel@tonic-gate default: 1417c478bd9Sstevel@tonic-gate error = EINVAL; 1427c478bd9Sstevel@tonic-gate break; 1437c478bd9Sstevel@tonic-gate } 1447c478bd9Sstevel@tonic-gate if (error == 0) 1457c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 1467c478bd9Sstevel@tonic-gate return (error); 1477c478bd9Sstevel@tonic-gate } 1487c478bd9Sstevel@tonic-gate 1497c478bd9Sstevel@tonic-gate /* 1507c478bd9Sstevel@tonic-gate * Delete mp from list of ceil mutexes owned by curthread. 1517c478bd9Sstevel@tonic-gate * Return 1 if the head of the chain was updated. 1527c478bd9Sstevel@tonic-gate */ 1537c478bd9Sstevel@tonic-gate int 1547c478bd9Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp) 1557c478bd9Sstevel@tonic-gate { 1567c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1577c478bd9Sstevel@tonic-gate mxchain_t **mcpp; 1587c478bd9Sstevel@tonic-gate mxchain_t *mcp; 1597c478bd9Sstevel@tonic-gate 1607c478bd9Sstevel@tonic-gate mcpp = &self->ul_mxchain; 1617c478bd9Sstevel@tonic-gate while ((*mcpp)->mxchain_mx != mp) 1627c478bd9Sstevel@tonic-gate mcpp = &(*mcpp)->mxchain_next; 1637c478bd9Sstevel@tonic-gate mcp = *mcpp; 1647c478bd9Sstevel@tonic-gate *mcpp = mcp->mxchain_next; 1657c478bd9Sstevel@tonic-gate lfree(mcp, sizeof (*mcp)); 1667c478bd9Sstevel@tonic-gate return (mcpp == &self->ul_mxchain); 1677c478bd9Sstevel@tonic-gate } 1687c478bd9Sstevel@tonic-gate 1697c478bd9Sstevel@tonic-gate /* 1707c478bd9Sstevel@tonic-gate * Add mp to head of list of ceil mutexes owned by curthread. 1717c478bd9Sstevel@tonic-gate * Return ENOMEM if no memory could be allocated. 1727c478bd9Sstevel@tonic-gate */ 1737c478bd9Sstevel@tonic-gate int 1747c478bd9Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp) 1757c478bd9Sstevel@tonic-gate { 1767c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1777c478bd9Sstevel@tonic-gate mxchain_t *mcp; 1787c478bd9Sstevel@tonic-gate 1797c478bd9Sstevel@tonic-gate if ((mcp = lmalloc(sizeof (*mcp))) == NULL) 1807c478bd9Sstevel@tonic-gate return (ENOMEM); 1817c478bd9Sstevel@tonic-gate mcp->mxchain_mx = mp; 1827c478bd9Sstevel@tonic-gate mcp->mxchain_next = self->ul_mxchain; 1837c478bd9Sstevel@tonic-gate self->ul_mxchain = mcp; 1847c478bd9Sstevel@tonic-gate return (0); 1857c478bd9Sstevel@tonic-gate } 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate /* 1887c478bd9Sstevel@tonic-gate * Inherit priority from ceiling. The inheritance impacts the effective 1897c478bd9Sstevel@tonic-gate * priority, not the assigned priority. See _thread_setschedparam_main(). 1907c478bd9Sstevel@tonic-gate */ 1917c478bd9Sstevel@tonic-gate void 1927c478bd9Sstevel@tonic-gate _ceil_prio_inherit(int ceil) 1937c478bd9Sstevel@tonic-gate { 1947c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1957c478bd9Sstevel@tonic-gate struct sched_param param; 1967c478bd9Sstevel@tonic-gate 1977c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 1987c478bd9Sstevel@tonic-gate param.sched_priority = ceil; 1997c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2007c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) { 2017c478bd9Sstevel@tonic-gate /* 2027c478bd9Sstevel@tonic-gate * Panic since unclear what error code to return. 2037c478bd9Sstevel@tonic-gate * If we do return the error codes returned by above 2047c478bd9Sstevel@tonic-gate * called routine, update the man page... 2057c478bd9Sstevel@tonic-gate */ 2067c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2077c478bd9Sstevel@tonic-gate } 2087c478bd9Sstevel@tonic-gate } 2097c478bd9Sstevel@tonic-gate 2107c478bd9Sstevel@tonic-gate /* 2117c478bd9Sstevel@tonic-gate * Waive inherited ceiling priority. Inherit from head of owned ceiling locks 2127c478bd9Sstevel@tonic-gate * if holding at least one ceiling lock. If no ceiling locks are held at this 2137c478bd9Sstevel@tonic-gate * point, disinherit completely, reverting back to assigned priority. 2147c478bd9Sstevel@tonic-gate */ 2157c478bd9Sstevel@tonic-gate void 2167c478bd9Sstevel@tonic-gate _ceil_prio_waive(void) 2177c478bd9Sstevel@tonic-gate { 2187c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2197c478bd9Sstevel@tonic-gate struct sched_param param; 2207c478bd9Sstevel@tonic-gate 2217c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 2227c478bd9Sstevel@tonic-gate if (self->ul_mxchain == NULL) { 2237c478bd9Sstevel@tonic-gate /* 2247c478bd9Sstevel@tonic-gate * No ceil locks held. Zero the epri, revert back to ul_pri. 2257c478bd9Sstevel@tonic-gate * Since thread's hash lock is not held, one cannot just 2267c478bd9Sstevel@tonic-gate * read ul_pri here...do it in the called routine... 2277c478bd9Sstevel@tonic-gate */ 2287c478bd9Sstevel@tonic-gate param.sched_priority = self->ul_pri; /* ignored */ 2297c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2307c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_DISINHERIT)) 2317c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2327c478bd9Sstevel@tonic-gate } else { 2337c478bd9Sstevel@tonic-gate /* 2347c478bd9Sstevel@tonic-gate * Set priority to that of the mutex at the head 2357c478bd9Sstevel@tonic-gate * of the ceilmutex chain. 2367c478bd9Sstevel@tonic-gate */ 2377c478bd9Sstevel@tonic-gate param.sched_priority = 2387c478bd9Sstevel@tonic-gate self->ul_mxchain->mxchain_mx->mutex_ceiling; 2397c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2407c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) 2417c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2427c478bd9Sstevel@tonic-gate } 2437c478bd9Sstevel@tonic-gate } 2447c478bd9Sstevel@tonic-gate 2457c478bd9Sstevel@tonic-gate /* 2467c478bd9Sstevel@tonic-gate * Non-preemptive spin locks. Used by queue_lock(). 2477c478bd9Sstevel@tonic-gate * No lock statistics are gathered for these locks. 2487c478bd9Sstevel@tonic-gate */ 2497c478bd9Sstevel@tonic-gate void 2507c478bd9Sstevel@tonic-gate spin_lock_set(mutex_t *mp) 2517c478bd9Sstevel@tonic-gate { 2527c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2537c478bd9Sstevel@tonic-gate 2547c478bd9Sstevel@tonic-gate no_preempt(self); 2557c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 2567c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2577c478bd9Sstevel@tonic-gate return; 2587c478bd9Sstevel@tonic-gate } 2597c478bd9Sstevel@tonic-gate /* 2607c478bd9Sstevel@tonic-gate * Spin for a while, attempting to acquire the lock. 2617c478bd9Sstevel@tonic-gate */ 2627c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin != UINT_MAX) 2637c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin++; 2647c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 2657c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 2667c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2677c478bd9Sstevel@tonic-gate return; 2687c478bd9Sstevel@tonic-gate } 2697c478bd9Sstevel@tonic-gate /* 2707c478bd9Sstevel@tonic-gate * Try harder if we were previously at a no premption level. 2717c478bd9Sstevel@tonic-gate */ 2727c478bd9Sstevel@tonic-gate if (self->ul_preempt > 1) { 2737c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin2 != UINT_MAX) 2747c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin2++; 2757c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 2767c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 2777c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2787c478bd9Sstevel@tonic-gate return; 2797c478bd9Sstevel@tonic-gate } 2807c478bd9Sstevel@tonic-gate } 2817c478bd9Sstevel@tonic-gate /* 2827c478bd9Sstevel@tonic-gate * Give up and block in the kernel for the mutex. 2837c478bd9Sstevel@tonic-gate */ 2847c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_sleep != UINT_MAX) 2857c478bd9Sstevel@tonic-gate self->ul_spin_lock_sleep++; 2867c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_timedlock(mp, NULL); 2877c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2887c478bd9Sstevel@tonic-gate } 2897c478bd9Sstevel@tonic-gate 2907c478bd9Sstevel@tonic-gate void 2917c478bd9Sstevel@tonic-gate spin_lock_clear(mutex_t *mp) 2927c478bd9Sstevel@tonic-gate { 2937c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2947c478bd9Sstevel@tonic-gate 2957c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 2967c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) { 2977c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 2987c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_wakeup != UINT_MAX) 2997c478bd9Sstevel@tonic-gate self->ul_spin_lock_wakeup++; 3007c478bd9Sstevel@tonic-gate } 3017c478bd9Sstevel@tonic-gate preempt(self); 3027c478bd9Sstevel@tonic-gate } 3037c478bd9Sstevel@tonic-gate 3047c478bd9Sstevel@tonic-gate /* 3057c478bd9Sstevel@tonic-gate * Allocate the sleep queue hash table. 3067c478bd9Sstevel@tonic-gate */ 3077c478bd9Sstevel@tonic-gate void 3087c478bd9Sstevel@tonic-gate queue_alloc(void) 3097c478bd9Sstevel@tonic-gate { 3107c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3117c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 3127c478bd9Sstevel@tonic-gate void *data; 3137c478bd9Sstevel@tonic-gate int i; 3147c478bd9Sstevel@tonic-gate 3157c478bd9Sstevel@tonic-gate /* 3167c478bd9Sstevel@tonic-gate * No locks are needed; we call here only when single-threaded. 3177c478bd9Sstevel@tonic-gate */ 3187c478bd9Sstevel@tonic-gate ASSERT(self == udp->ulwp_one); 3197c478bd9Sstevel@tonic-gate ASSERT(!udp->uberflags.uf_mt); 3207c478bd9Sstevel@tonic-gate if ((data = _private_mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t), 3217c478bd9Sstevel@tonic-gate PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0)) 3227c478bd9Sstevel@tonic-gate == MAP_FAILED) 3237c478bd9Sstevel@tonic-gate thr_panic("cannot allocate thread queue_head table"); 3247c478bd9Sstevel@tonic-gate udp->queue_head = (queue_head_t *)data; 3257c478bd9Sstevel@tonic-gate for (i = 0; i < 2 * QHASHSIZE; i++) 3267c478bd9Sstevel@tonic-gate udp->queue_head[i].qh_lock.mutex_magic = MUTEX_MAGIC; 3277c478bd9Sstevel@tonic-gate } 3287c478bd9Sstevel@tonic-gate 3297c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 3307c478bd9Sstevel@tonic-gate 3317c478bd9Sstevel@tonic-gate /* 3327c478bd9Sstevel@tonic-gate * Debugging: verify correctness of a sleep queue. 3337c478bd9Sstevel@tonic-gate */ 3347c478bd9Sstevel@tonic-gate void 3357c478bd9Sstevel@tonic-gate QVERIFY(queue_head_t *qp) 3367c478bd9Sstevel@tonic-gate { 3377c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3387c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 3397c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 3407c478bd9Sstevel@tonic-gate ulwp_t *prev; 3417c478bd9Sstevel@tonic-gate uint_t index; 3427c478bd9Sstevel@tonic-gate uint32_t cnt = 0; 3437c478bd9Sstevel@tonic-gate char qtype; 3447c478bd9Sstevel@tonic-gate void *wchan; 3457c478bd9Sstevel@tonic-gate 3467c478bd9Sstevel@tonic-gate ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE); 3477c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 3487c478bd9Sstevel@tonic-gate ASSERT((qp->qh_head != NULL && qp->qh_tail != NULL) || 3497c478bd9Sstevel@tonic-gate (qp->qh_head == NULL && qp->qh_tail == NULL)); 3507c478bd9Sstevel@tonic-gate if (!thread_queue_verify) 3517c478bd9Sstevel@tonic-gate return; 3527c478bd9Sstevel@tonic-gate /* real expensive stuff, only for _THREAD_QUEUE_VERIFY */ 3537c478bd9Sstevel@tonic-gate qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV; 3547c478bd9Sstevel@tonic-gate for (prev = NULL, ulwp = qp->qh_head; ulwp != NULL; 3557c478bd9Sstevel@tonic-gate prev = ulwp, ulwp = ulwp->ul_link, cnt++) { 3567c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_qtype == qtype); 3577c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_wchan != NULL); 3587c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq == qp); 3597c478bd9Sstevel@tonic-gate wchan = ulwp->ul_wchan; 3607c478bd9Sstevel@tonic-gate index = QUEUE_HASH(wchan, qtype); 3617c478bd9Sstevel@tonic-gate ASSERT(&udp->queue_head[index] == qp); 3627c478bd9Sstevel@tonic-gate } 3637c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == prev); 3647c478bd9Sstevel@tonic-gate ASSERT(qp->qh_qlen == cnt); 3657c478bd9Sstevel@tonic-gate } 3667c478bd9Sstevel@tonic-gate 3677c478bd9Sstevel@tonic-gate #else /* THREAD_DEBUG */ 3687c478bd9Sstevel@tonic-gate 3697c478bd9Sstevel@tonic-gate #define QVERIFY(qp) 3707c478bd9Sstevel@tonic-gate 3717c478bd9Sstevel@tonic-gate #endif /* THREAD_DEBUG */ 3727c478bd9Sstevel@tonic-gate 3737c478bd9Sstevel@tonic-gate /* 3747c478bd9Sstevel@tonic-gate * Acquire a queue head. 3757c478bd9Sstevel@tonic-gate */ 3767c478bd9Sstevel@tonic-gate queue_head_t * 3777c478bd9Sstevel@tonic-gate queue_lock(void *wchan, int qtype) 3787c478bd9Sstevel@tonic-gate { 3797c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 3807c478bd9Sstevel@tonic-gate queue_head_t *qp; 3817c478bd9Sstevel@tonic-gate 3827c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 3837c478bd9Sstevel@tonic-gate 3847c478bd9Sstevel@tonic-gate /* 3857c478bd9Sstevel@tonic-gate * It is possible that we could be called while still single-threaded. 3867c478bd9Sstevel@tonic-gate * If so, we call queue_alloc() to allocate the queue_head[] array. 3877c478bd9Sstevel@tonic-gate */ 3887c478bd9Sstevel@tonic-gate if ((qp = udp->queue_head) == NULL) { 3897c478bd9Sstevel@tonic-gate queue_alloc(); 3907c478bd9Sstevel@tonic-gate qp = udp->queue_head; 3917c478bd9Sstevel@tonic-gate } 3927c478bd9Sstevel@tonic-gate qp += QUEUE_HASH(wchan, qtype); 3937c478bd9Sstevel@tonic-gate spin_lock_set(&qp->qh_lock); 3947c478bd9Sstevel@tonic-gate /* 3957c478bd9Sstevel@tonic-gate * At once per nanosecond, qh_lockcount will wrap after 512 years. 3967c478bd9Sstevel@tonic-gate * Were we to care about this, we could peg the value at UINT64_MAX. 3977c478bd9Sstevel@tonic-gate */ 3987c478bd9Sstevel@tonic-gate qp->qh_lockcount++; 3997c478bd9Sstevel@tonic-gate QVERIFY(qp); 4007c478bd9Sstevel@tonic-gate return (qp); 4017c478bd9Sstevel@tonic-gate } 4027c478bd9Sstevel@tonic-gate 4037c478bd9Sstevel@tonic-gate /* 4047c478bd9Sstevel@tonic-gate * Release a queue head. 4057c478bd9Sstevel@tonic-gate */ 4067c478bd9Sstevel@tonic-gate void 4077c478bd9Sstevel@tonic-gate queue_unlock(queue_head_t *qp) 4087c478bd9Sstevel@tonic-gate { 4097c478bd9Sstevel@tonic-gate QVERIFY(qp); 4107c478bd9Sstevel@tonic-gate spin_lock_clear(&qp->qh_lock); 4117c478bd9Sstevel@tonic-gate } 4127c478bd9Sstevel@tonic-gate 4137c478bd9Sstevel@tonic-gate /* 4147c478bd9Sstevel@tonic-gate * For rwlock queueing, we must queue writers ahead of readers of the 4157c478bd9Sstevel@tonic-gate * same priority. We do this by making writers appear to have a half 4167c478bd9Sstevel@tonic-gate * point higher priority for purposes of priority comparisons below. 4177c478bd9Sstevel@tonic-gate */ 4187c478bd9Sstevel@tonic-gate #define CMP_PRIO(ulwp) ((real_priority(ulwp) << 1) + (ulwp)->ul_writer) 4197c478bd9Sstevel@tonic-gate 4207c478bd9Sstevel@tonic-gate void 4217c478bd9Sstevel@tonic-gate enqueue(queue_head_t *qp, ulwp_t *ulwp, void *wchan, int qtype) 4227c478bd9Sstevel@tonic-gate { 4237c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 4247c478bd9Sstevel@tonic-gate ulwp_t *next; 4257c478bd9Sstevel@tonic-gate int pri = CMP_PRIO(ulwp); 4267c478bd9Sstevel@tonic-gate int force_fifo = (qtype & FIFOQ); 4277c478bd9Sstevel@tonic-gate int do_fifo; 4287c478bd9Sstevel@tonic-gate 4297c478bd9Sstevel@tonic-gate qtype &= ~FIFOQ; 4307c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 4317c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 4327c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq != qp); 4337c478bd9Sstevel@tonic-gate 4347c478bd9Sstevel@tonic-gate /* 4357c478bd9Sstevel@tonic-gate * LIFO queue ordering is unfair and can lead to starvation, 4367c478bd9Sstevel@tonic-gate * but it gives better performance for heavily contended locks. 4377c478bd9Sstevel@tonic-gate * We use thread_queue_fifo (range is 0..8) to determine 4387c478bd9Sstevel@tonic-gate * the frequency of FIFO vs LIFO queuing: 4397c478bd9Sstevel@tonic-gate * 0 : every 256th time (almost always LIFO) 4407c478bd9Sstevel@tonic-gate * 1 : every 128th time 4417c478bd9Sstevel@tonic-gate * 2 : every 64th time 4427c478bd9Sstevel@tonic-gate * 3 : every 32nd time 4437c478bd9Sstevel@tonic-gate * 4 : every 16th time (the default value, mostly LIFO) 4447c478bd9Sstevel@tonic-gate * 5 : every 8th time 4457c478bd9Sstevel@tonic-gate * 6 : every 4th time 4467c478bd9Sstevel@tonic-gate * 7 : every 2nd time 4477c478bd9Sstevel@tonic-gate * 8 : every time (never LIFO, always FIFO) 4487c478bd9Sstevel@tonic-gate * Note that there is always some degree of FIFO ordering. 4497c478bd9Sstevel@tonic-gate * This breaks live lock conditions that occur in applications 4507c478bd9Sstevel@tonic-gate * that are written assuming (incorrectly) that threads acquire 4517c478bd9Sstevel@tonic-gate * locks fairly, that is, in roughly round-robin order. 4527c478bd9Sstevel@tonic-gate * In any event, the queue is maintained in priority order. 4537c478bd9Sstevel@tonic-gate * 4547c478bd9Sstevel@tonic-gate * If we are given the FIFOQ flag in qtype, fifo queueing is forced. 4557c478bd9Sstevel@tonic-gate * SUSV3 requires this for semaphores. 4567c478bd9Sstevel@tonic-gate */ 4577c478bd9Sstevel@tonic-gate do_fifo = (force_fifo || 4587c478bd9Sstevel@tonic-gate ((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0); 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate if (qp->qh_head == NULL) { 4617c478bd9Sstevel@tonic-gate /* 4627c478bd9Sstevel@tonic-gate * The queue is empty. LIFO/FIFO doesn't matter. 4637c478bd9Sstevel@tonic-gate */ 4647c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == NULL); 4657c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 4667c478bd9Sstevel@tonic-gate } else if (do_fifo) { 4677c478bd9Sstevel@tonic-gate /* 4687c478bd9Sstevel@tonic-gate * Enqueue after the last thread whose priority is greater 4697c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 4707c478bd9Sstevel@tonic-gate * Attempt first to go directly onto the tail of the queue. 4717c478bd9Sstevel@tonic-gate */ 4727c478bd9Sstevel@tonic-gate if (pri <= CMP_PRIO(qp->qh_tail)) 4737c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_tail->ul_link; 4747c478bd9Sstevel@tonic-gate else { 4757c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 4767c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 4777c478bd9Sstevel@tonic-gate if (pri > CMP_PRIO(next)) 4787c478bd9Sstevel@tonic-gate break; 4797c478bd9Sstevel@tonic-gate } 4807c478bd9Sstevel@tonic-gate } else { 4817c478bd9Sstevel@tonic-gate /* 4827c478bd9Sstevel@tonic-gate * Enqueue before the first thread whose priority is less 4837c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 4847c478bd9Sstevel@tonic-gate * Hopefully we can go directly onto the head of the queue. 4857c478bd9Sstevel@tonic-gate */ 4867c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 4877c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 4887c478bd9Sstevel@tonic-gate if (pri >= CMP_PRIO(next)) 4897c478bd9Sstevel@tonic-gate break; 4907c478bd9Sstevel@tonic-gate } 4917c478bd9Sstevel@tonic-gate if ((ulwp->ul_link = *ulwpp) == NULL) 4927c478bd9Sstevel@tonic-gate qp->qh_tail = ulwp; 4937c478bd9Sstevel@tonic-gate *ulwpp = ulwp; 4947c478bd9Sstevel@tonic-gate 4957c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = qp; 4967c478bd9Sstevel@tonic-gate ulwp->ul_wchan = wchan; 4977c478bd9Sstevel@tonic-gate ulwp->ul_qtype = qtype; 4987c478bd9Sstevel@tonic-gate if (qp->qh_qmax < ++qp->qh_qlen) 4997c478bd9Sstevel@tonic-gate qp->qh_qmax = qp->qh_qlen; 5007c478bd9Sstevel@tonic-gate } 5017c478bd9Sstevel@tonic-gate 5027c478bd9Sstevel@tonic-gate /* 5037c478bd9Sstevel@tonic-gate * Return a pointer to the queue slot of the 5047c478bd9Sstevel@tonic-gate * highest priority thread on the queue. 5057c478bd9Sstevel@tonic-gate * On return, prevp, if not NULL, will contain a pointer 5067c478bd9Sstevel@tonic-gate * to the thread's predecessor on the queue 5077c478bd9Sstevel@tonic-gate */ 5087c478bd9Sstevel@tonic-gate static ulwp_t ** 5097c478bd9Sstevel@tonic-gate queue_slot(queue_head_t *qp, void *wchan, int *more, ulwp_t **prevp) 5107c478bd9Sstevel@tonic-gate { 5117c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 5127c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 5137c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 5147c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 5157c478bd9Sstevel@tonic-gate ulwp_t *susprev; 5167c478bd9Sstevel@tonic-gate 5177c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 5187c478bd9Sstevel@tonic-gate 5197c478bd9Sstevel@tonic-gate /* 5207c478bd9Sstevel@tonic-gate * Find a waiter on the sleep queue. 5217c478bd9Sstevel@tonic-gate */ 5227c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 5237c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 5247c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 5257c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 5267c478bd9Sstevel@tonic-gate break; 5277c478bd9Sstevel@tonic-gate /* 5287c478bd9Sstevel@tonic-gate * Try not to return a suspended thread. 5297c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 5307c478bd9Sstevel@tonic-gate */ 5317c478bd9Sstevel@tonic-gate if (suspp == NULL) { 5327c478bd9Sstevel@tonic-gate suspp = ulwpp; 5337c478bd9Sstevel@tonic-gate susprev = prev; 5347c478bd9Sstevel@tonic-gate } 5357c478bd9Sstevel@tonic-gate } 5367c478bd9Sstevel@tonic-gate } 5377c478bd9Sstevel@tonic-gate 5387c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 5397c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 5407c478bd9Sstevel@tonic-gate prev = susprev; 5417c478bd9Sstevel@tonic-gate suspp = NULL; 5427c478bd9Sstevel@tonic-gate } 5437c478bd9Sstevel@tonic-gate if (ulwp == NULL) { 5447c478bd9Sstevel@tonic-gate if (more != NULL) 5457c478bd9Sstevel@tonic-gate *more = 0; 5467c478bd9Sstevel@tonic-gate return (NULL); 5477c478bd9Sstevel@tonic-gate } 5487c478bd9Sstevel@tonic-gate 5497c478bd9Sstevel@tonic-gate if (prevp != NULL) 5507c478bd9Sstevel@tonic-gate *prevp = prev; 5517c478bd9Sstevel@tonic-gate if (more == NULL) 5527c478bd9Sstevel@tonic-gate return (ulwpp); 5537c478bd9Sstevel@tonic-gate 5547c478bd9Sstevel@tonic-gate /* 5557c478bd9Sstevel@tonic-gate * Scan the remainder of the queue for another waiter. 5567c478bd9Sstevel@tonic-gate */ 5577c478bd9Sstevel@tonic-gate if (suspp != NULL) { 5587c478bd9Sstevel@tonic-gate *more = 1; 5597c478bd9Sstevel@tonic-gate return (ulwpp); 5607c478bd9Sstevel@tonic-gate } 5617c478bd9Sstevel@tonic-gate for (ulwp = ulwp->ul_link; ulwp != NULL; ulwp = ulwp->ul_link) { 5627c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 5637c478bd9Sstevel@tonic-gate *more = 1; 5647c478bd9Sstevel@tonic-gate return (ulwpp); 5657c478bd9Sstevel@tonic-gate } 5667c478bd9Sstevel@tonic-gate } 5677c478bd9Sstevel@tonic-gate 5687c478bd9Sstevel@tonic-gate *more = 0; 5697c478bd9Sstevel@tonic-gate return (ulwpp); 5707c478bd9Sstevel@tonic-gate } 5717c478bd9Sstevel@tonic-gate 5727c478bd9Sstevel@tonic-gate ulwp_t * 5737c478bd9Sstevel@tonic-gate dequeue(queue_head_t *qp, void *wchan, int *more) 5747c478bd9Sstevel@tonic-gate { 5757c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 5767c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 5777c478bd9Sstevel@tonic-gate ulwp_t *prev; 5787c478bd9Sstevel@tonic-gate 5797c478bd9Sstevel@tonic-gate if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL) 5807c478bd9Sstevel@tonic-gate return (NULL); 5817c478bd9Sstevel@tonic-gate 5827c478bd9Sstevel@tonic-gate /* 5837c478bd9Sstevel@tonic-gate * Dequeue the waiter. 5847c478bd9Sstevel@tonic-gate */ 5857c478bd9Sstevel@tonic-gate ulwp = *ulwpp; 5867c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 5877c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 5887c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 5897c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 5907c478bd9Sstevel@tonic-gate qp->qh_qlen--; 5917c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 5927c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 5937c478bd9Sstevel@tonic-gate 5947c478bd9Sstevel@tonic-gate return (ulwp); 5957c478bd9Sstevel@tonic-gate } 5967c478bd9Sstevel@tonic-gate 5977c478bd9Sstevel@tonic-gate /* 5987c478bd9Sstevel@tonic-gate * Return a pointer to the highest priority thread sleeping on wchan. 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate ulwp_t * 6017c478bd9Sstevel@tonic-gate queue_waiter(queue_head_t *qp, void *wchan) 6027c478bd9Sstevel@tonic-gate { 6037c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 6047c478bd9Sstevel@tonic-gate 6057c478bd9Sstevel@tonic-gate if ((ulwpp = queue_slot(qp, wchan, NULL, NULL)) == NULL) 6067c478bd9Sstevel@tonic-gate return (NULL); 6077c478bd9Sstevel@tonic-gate return (*ulwpp); 6087c478bd9Sstevel@tonic-gate } 6097c478bd9Sstevel@tonic-gate 6107c478bd9Sstevel@tonic-gate uint8_t 6117c478bd9Sstevel@tonic-gate dequeue_self(queue_head_t *qp, void *wchan) 6127c478bd9Sstevel@tonic-gate { 6137c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 6147c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 6157c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 6167c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 6177c478bd9Sstevel@tonic-gate int found = 0; 6187c478bd9Sstevel@tonic-gate int more = 0; 6197c478bd9Sstevel@tonic-gate 6207c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 6217c478bd9Sstevel@tonic-gate 6227c478bd9Sstevel@tonic-gate /* find self on the sleep queue */ 6237c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 6247c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 6257c478bd9Sstevel@tonic-gate if (ulwp == self) { 6267c478bd9Sstevel@tonic-gate /* dequeue ourself */ 6277c478bd9Sstevel@tonic-gate *ulwpp = self->ul_link; 6287c478bd9Sstevel@tonic-gate if (qp->qh_tail == self) 6297c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 6307c478bd9Sstevel@tonic-gate qp->qh_qlen--; 6317c478bd9Sstevel@tonic-gate ASSERT(self->ul_wchan == wchan); 6327c478bd9Sstevel@tonic-gate self->ul_cvmutex = NULL; 6337c478bd9Sstevel@tonic-gate self->ul_sleepq = NULL; 6347c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 6357c478bd9Sstevel@tonic-gate self->ul_cv_wake = 0; 6367c478bd9Sstevel@tonic-gate self->ul_link = NULL; 6377c478bd9Sstevel@tonic-gate found = 1; 6387c478bd9Sstevel@tonic-gate break; 6397c478bd9Sstevel@tonic-gate } 6407c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 6417c478bd9Sstevel@tonic-gate more = 1; 6427c478bd9Sstevel@tonic-gate } 6437c478bd9Sstevel@tonic-gate 6447c478bd9Sstevel@tonic-gate if (!found) 6457c478bd9Sstevel@tonic-gate thr_panic("dequeue_self(): curthread not found on queue"); 6467c478bd9Sstevel@tonic-gate 6477c478bd9Sstevel@tonic-gate if (more) 6487c478bd9Sstevel@tonic-gate return (1); 6497c478bd9Sstevel@tonic-gate 6507c478bd9Sstevel@tonic-gate /* scan the remainder of the queue for another waiter */ 6517c478bd9Sstevel@tonic-gate for (ulwp = *ulwpp; ulwp != NULL; ulwp = ulwp->ul_link) { 6527c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 6537c478bd9Sstevel@tonic-gate return (1); 6547c478bd9Sstevel@tonic-gate } 6557c478bd9Sstevel@tonic-gate 6567c478bd9Sstevel@tonic-gate return (0); 6577c478bd9Sstevel@tonic-gate } 6587c478bd9Sstevel@tonic-gate 6597c478bd9Sstevel@tonic-gate /* 6607c478bd9Sstevel@tonic-gate * Called from call_user_handler() and _thrp_suspend() to take 6617c478bd9Sstevel@tonic-gate * ourself off of our sleep queue so we can grab locks. 6627c478bd9Sstevel@tonic-gate */ 6637c478bd9Sstevel@tonic-gate void 6647c478bd9Sstevel@tonic-gate unsleep_self(void) 6657c478bd9Sstevel@tonic-gate { 6667c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 6677c478bd9Sstevel@tonic-gate queue_head_t *qp; 6687c478bd9Sstevel@tonic-gate 6697c478bd9Sstevel@tonic-gate /* 6707c478bd9Sstevel@tonic-gate * Calling enter_critical()/exit_critical() here would lead 6717c478bd9Sstevel@tonic-gate * to recursion. Just manipulate self->ul_critical directly. 6727c478bd9Sstevel@tonic-gate */ 6737c478bd9Sstevel@tonic-gate self->ul_critical++; 6747c478bd9Sstevel@tonic-gate self->ul_writer = 0; 6757c478bd9Sstevel@tonic-gate while (self->ul_sleepq != NULL) { 6767c478bd9Sstevel@tonic-gate qp = queue_lock(self->ul_wchan, self->ul_qtype); 6777c478bd9Sstevel@tonic-gate /* 6787c478bd9Sstevel@tonic-gate * We may have been moved from a CV queue to a 6797c478bd9Sstevel@tonic-gate * mutex queue while we were attempting queue_lock(). 6807c478bd9Sstevel@tonic-gate * If so, just loop around and try again. 6817c478bd9Sstevel@tonic-gate * dequeue_self() clears self->ul_sleepq. 6827c478bd9Sstevel@tonic-gate */ 6837c478bd9Sstevel@tonic-gate if (qp == self->ul_sleepq) 6847c478bd9Sstevel@tonic-gate (void) dequeue_self(qp, self->ul_wchan); 6857c478bd9Sstevel@tonic-gate queue_unlock(qp); 6867c478bd9Sstevel@tonic-gate } 6877c478bd9Sstevel@tonic-gate self->ul_critical--; 6887c478bd9Sstevel@tonic-gate } 6897c478bd9Sstevel@tonic-gate 6907c478bd9Sstevel@tonic-gate /* 6917c478bd9Sstevel@tonic-gate * Common code for calling the the ___lwp_mutex_timedlock() system call. 6927c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 6937c478bd9Sstevel@tonic-gate */ 6947c478bd9Sstevel@tonic-gate int 6957c478bd9Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp) 6967c478bd9Sstevel@tonic-gate { 6977c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 6987c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 6997c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 7007c478bd9Sstevel@tonic-gate int error; 7017c478bd9Sstevel@tonic-gate 7027c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 7037c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 7047c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 7057c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 7067c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 7077c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 7087c478bd9Sstevel@tonic-gate } 7097c478bd9Sstevel@tonic-gate if (msp) { 7107c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 7117c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 7127c478bd9Sstevel@tonic-gate } 7137c478bd9Sstevel@tonic-gate 7147c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 7157c478bd9Sstevel@tonic-gate 7167c478bd9Sstevel@tonic-gate for (;;) { 7177c478bd9Sstevel@tonic-gate if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0) { 7187c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 7197c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, error); 7207c478bd9Sstevel@tonic-gate break; 7217c478bd9Sstevel@tonic-gate } 7227c478bd9Sstevel@tonic-gate 7237c478bd9Sstevel@tonic-gate if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) { 7247c478bd9Sstevel@tonic-gate /* 7257c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 7267c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 7277c478bd9Sstevel@tonic-gate */ 7287c478bd9Sstevel@tonic-gate enter_critical(self); 7297c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 7307c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 7317c478bd9Sstevel@tonic-gate exit_critical(self); 7327c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 7337c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 7347c478bd9Sstevel@tonic-gate 0, 0); 7357c478bd9Sstevel@tonic-gate break; 7367c478bd9Sstevel@tonic-gate } 7377c478bd9Sstevel@tonic-gate exit_critical(self); 7387c478bd9Sstevel@tonic-gate } else { 7397c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 7407c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 7417c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 7427c478bd9Sstevel@tonic-gate break; 7437c478bd9Sstevel@tonic-gate } 7447c478bd9Sstevel@tonic-gate } 7457c478bd9Sstevel@tonic-gate if (msp) 7467c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 7477c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 7487c478bd9Sstevel@tonic-gate self->ul_sp = 0; 7497c478bd9Sstevel@tonic-gate 7507c478bd9Sstevel@tonic-gate return (error); 7517c478bd9Sstevel@tonic-gate } 7527c478bd9Sstevel@tonic-gate 7537c478bd9Sstevel@tonic-gate /* 7547c478bd9Sstevel@tonic-gate * Common code for calling the ___lwp_mutex_trylock() system call. 7557c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 7567c478bd9Sstevel@tonic-gate */ 7577c478bd9Sstevel@tonic-gate int 7587c478bd9Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp) 7597c478bd9Sstevel@tonic-gate { 7607c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 7617c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 7627c478bd9Sstevel@tonic-gate int error; 7637c478bd9Sstevel@tonic-gate 7647c478bd9Sstevel@tonic-gate for (;;) { 7657c478bd9Sstevel@tonic-gate if ((error = ___lwp_mutex_trylock(mp)) != 0) { 7667c478bd9Sstevel@tonic-gate if (error != EBUSY) { 7677c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 7687c478bd9Sstevel@tonic-gate error); 7697c478bd9Sstevel@tonic-gate } 7707c478bd9Sstevel@tonic-gate break; 7717c478bd9Sstevel@tonic-gate } 7727c478bd9Sstevel@tonic-gate 7737c478bd9Sstevel@tonic-gate if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) { 7747c478bd9Sstevel@tonic-gate /* 7757c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 7767c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 7777c478bd9Sstevel@tonic-gate */ 7787c478bd9Sstevel@tonic-gate enter_critical(self); 7797c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 7807c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 7817c478bd9Sstevel@tonic-gate exit_critical(self); 7827c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 7837c478bd9Sstevel@tonic-gate 0, 0); 7847c478bd9Sstevel@tonic-gate break; 7857c478bd9Sstevel@tonic-gate } 7867c478bd9Sstevel@tonic-gate exit_critical(self); 7877c478bd9Sstevel@tonic-gate } else { 7887c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 7897c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 7907c478bd9Sstevel@tonic-gate break; 7917c478bd9Sstevel@tonic-gate } 7927c478bd9Sstevel@tonic-gate } 7937c478bd9Sstevel@tonic-gate 7947c478bd9Sstevel@tonic-gate return (error); 7957c478bd9Sstevel@tonic-gate } 7967c478bd9Sstevel@tonic-gate 7977c478bd9Sstevel@tonic-gate volatile sc_shared_t * 7987c478bd9Sstevel@tonic-gate setup_schedctl(void) 7997c478bd9Sstevel@tonic-gate { 8007c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 8017c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 8027c478bd9Sstevel@tonic-gate sc_shared_t *tmp; 8037c478bd9Sstevel@tonic-gate 8047c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */ 8057c478bd9Sstevel@tonic-gate !self->ul_vfork && /* not a child of vfork() */ 8067c478bd9Sstevel@tonic-gate !self->ul_schedctl_called) { /* haven't been called before */ 8077c478bd9Sstevel@tonic-gate enter_critical(self); 8087c478bd9Sstevel@tonic-gate self->ul_schedctl_called = &self->ul_uberdata->uberflags; 8097c478bd9Sstevel@tonic-gate if ((tmp = __schedctl()) != (sc_shared_t *)(-1)) 8107c478bd9Sstevel@tonic-gate self->ul_schedctl = scp = tmp; 8117c478bd9Sstevel@tonic-gate exit_critical(self); 8127c478bd9Sstevel@tonic-gate } 8137c478bd9Sstevel@tonic-gate /* 8147c478bd9Sstevel@tonic-gate * Unless the call to setup_schedctl() is surrounded 8157c478bd9Sstevel@tonic-gate * by enter_critical()/exit_critical(), the address 8167c478bd9Sstevel@tonic-gate * we are returning could be invalid due to a forkall() 8177c478bd9Sstevel@tonic-gate * having occurred in another thread. 8187c478bd9Sstevel@tonic-gate */ 8197c478bd9Sstevel@tonic-gate return (scp); 8207c478bd9Sstevel@tonic-gate } 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate /* 8237c478bd9Sstevel@tonic-gate * Interfaces from libsched, incorporated into libc. 8247c478bd9Sstevel@tonic-gate * libsched.so.1 is now a filter library onto libc. 8257c478bd9Sstevel@tonic-gate */ 8267c478bd9Sstevel@tonic-gate #pragma weak schedctl_lookup = _schedctl_init 8277c478bd9Sstevel@tonic-gate #pragma weak _schedctl_lookup = _schedctl_init 8287c478bd9Sstevel@tonic-gate #pragma weak schedctl_init = _schedctl_init 8297c478bd9Sstevel@tonic-gate schedctl_t * 8307c478bd9Sstevel@tonic-gate _schedctl_init(void) 8317c478bd9Sstevel@tonic-gate { 8327c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = setup_schedctl(); 8337c478bd9Sstevel@tonic-gate return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl); 8347c478bd9Sstevel@tonic-gate } 8357c478bd9Sstevel@tonic-gate 8367c478bd9Sstevel@tonic-gate #pragma weak schedctl_exit = _schedctl_exit 8377c478bd9Sstevel@tonic-gate void 8387c478bd9Sstevel@tonic-gate _schedctl_exit(void) 8397c478bd9Sstevel@tonic-gate { 8407c478bd9Sstevel@tonic-gate } 8417c478bd9Sstevel@tonic-gate 8427c478bd9Sstevel@tonic-gate /* 8437c478bd9Sstevel@tonic-gate * Contract private interface for java. 8447c478bd9Sstevel@tonic-gate * Set up the schedctl data if it doesn't exist yet. 8457c478bd9Sstevel@tonic-gate * Return a pointer to the pointer to the schedctl data. 8467c478bd9Sstevel@tonic-gate */ 8477c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile * 8487c478bd9Sstevel@tonic-gate _thr_schedctl(void) 8497c478bd9Sstevel@tonic-gate { 8507c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 8517c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile *ptr; 8527c478bd9Sstevel@tonic-gate 8537c478bd9Sstevel@tonic-gate if (self->ul_vfork) 8547c478bd9Sstevel@tonic-gate return (NULL); 8557c478bd9Sstevel@tonic-gate if (*(ptr = &self->ul_schedctl) == NULL) 8567c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 8577c478bd9Sstevel@tonic-gate return (ptr); 8587c478bd9Sstevel@tonic-gate } 8597c478bd9Sstevel@tonic-gate 8607c478bd9Sstevel@tonic-gate /* 8617c478bd9Sstevel@tonic-gate * Block signals and attempt to block preemption. 8627c478bd9Sstevel@tonic-gate * no_preempt()/preempt() must be used in pairs but can be nested. 8637c478bd9Sstevel@tonic-gate */ 8647c478bd9Sstevel@tonic-gate void 8657c478bd9Sstevel@tonic-gate no_preempt(ulwp_t *self) 8667c478bd9Sstevel@tonic-gate { 8677c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 8687c478bd9Sstevel@tonic-gate 8697c478bd9Sstevel@tonic-gate if (self->ul_preempt++ == 0) { 8707c478bd9Sstevel@tonic-gate enter_critical(self); 8717c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL || 8727c478bd9Sstevel@tonic-gate (scp = setup_schedctl()) != NULL) { 8737c478bd9Sstevel@tonic-gate /* 8747c478bd9Sstevel@tonic-gate * Save the pre-existing preempt value. 8757c478bd9Sstevel@tonic-gate */ 8767c478bd9Sstevel@tonic-gate self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt; 8777c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = 1; 8787c478bd9Sstevel@tonic-gate } 8797c478bd9Sstevel@tonic-gate } 8807c478bd9Sstevel@tonic-gate } 8817c478bd9Sstevel@tonic-gate 8827c478bd9Sstevel@tonic-gate /* 8837c478bd9Sstevel@tonic-gate * Undo the effects of no_preempt(). 8847c478bd9Sstevel@tonic-gate */ 8857c478bd9Sstevel@tonic-gate void 8867c478bd9Sstevel@tonic-gate preempt(ulwp_t *self) 8877c478bd9Sstevel@tonic-gate { 8887c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 8897c478bd9Sstevel@tonic-gate 8907c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt > 0); 8917c478bd9Sstevel@tonic-gate if (--self->ul_preempt == 0) { 8927c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL) { 8937c478bd9Sstevel@tonic-gate /* 8947c478bd9Sstevel@tonic-gate * Restore the pre-existing preempt value. 8957c478bd9Sstevel@tonic-gate */ 8967c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt; 8977c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield && 8987c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt == 0) { 8997c478bd9Sstevel@tonic-gate lwp_yield(); 9007c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield) { 9017c478bd9Sstevel@tonic-gate /* 9027c478bd9Sstevel@tonic-gate * Shouldn't happen. This is either 9037c478bd9Sstevel@tonic-gate * a race condition or the thread 9047c478bd9Sstevel@tonic-gate * just entered the real-time class. 9057c478bd9Sstevel@tonic-gate */ 9067c478bd9Sstevel@tonic-gate lwp_yield(); 9077c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_yield = 0; 9087c478bd9Sstevel@tonic-gate } 9097c478bd9Sstevel@tonic-gate } 9107c478bd9Sstevel@tonic-gate } 9117c478bd9Sstevel@tonic-gate exit_critical(self); 9127c478bd9Sstevel@tonic-gate } 9137c478bd9Sstevel@tonic-gate } 9147c478bd9Sstevel@tonic-gate 9157c478bd9Sstevel@tonic-gate /* 9167c478bd9Sstevel@tonic-gate * If a call to preempt() would cause the current thread to yield or to 9177c478bd9Sstevel@tonic-gate * take deferred actions in exit_critical(), then unpark the specified 9187c478bd9Sstevel@tonic-gate * lwp so it can run while we delay. Return the original lwpid if the 9197c478bd9Sstevel@tonic-gate * unpark was not performed, else return zero. The tests are a repeat 9207c478bd9Sstevel@tonic-gate * of some of the tests in preempt(), above. This is a statistical 9217c478bd9Sstevel@tonic-gate * optimization solely for cond_sleep_queue(), below. 9227c478bd9Sstevel@tonic-gate */ 9237c478bd9Sstevel@tonic-gate static lwpid_t 9247c478bd9Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid) 9257c478bd9Sstevel@tonic-gate { 9267c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = self->ul_schedctl; 9277c478bd9Sstevel@tonic-gate 9287c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt == 1 && self->ul_critical > 0); 9297c478bd9Sstevel@tonic-gate if ((scp != NULL && scp->sc_preemptctl.sc_yield) || 9307c478bd9Sstevel@tonic-gate (self->ul_curplease && self->ul_critical == 1)) { 9317c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 9327c478bd9Sstevel@tonic-gate lwpid = 0; 9337c478bd9Sstevel@tonic-gate } 9347c478bd9Sstevel@tonic-gate return (lwpid); 9357c478bd9Sstevel@tonic-gate } 9367c478bd9Sstevel@tonic-gate 9377c478bd9Sstevel@tonic-gate /* 9387c478bd9Sstevel@tonic-gate * Spin for a while, trying to grab the lock. We know that we 9397c478bd9Sstevel@tonic-gate * failed set_lock_byte(&mp->mutex_lockw) once before coming here. 9407c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 9417c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread. 9427c478bd9Sstevel@tonic-gate */ 9437c478bd9Sstevel@tonic-gate int 9447c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mutex_t *mp) 9457c478bd9Sstevel@tonic-gate { 9467c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 9477c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 9487c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 9497c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 9507c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 9517c478bd9Sstevel@tonic-gate int count, max = self->ul_adaptive_spin; 9527c478bd9Sstevel@tonic-gate 9537c478bd9Sstevel@tonic-gate ASSERT(!(mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))); 9547c478bd9Sstevel@tonic-gate 9557c478bd9Sstevel@tonic-gate if (max == 0 || (mp->mutex_spinners >= self->ul_max_spinners)) 9567c478bd9Sstevel@tonic-gate return (EBUSY); 9577c478bd9Sstevel@tonic-gate 9587c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 9597c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 9607c478bd9Sstevel@tonic-gate 9617c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__spin, mp); 9627c478bd9Sstevel@tonic-gate 9637c478bd9Sstevel@tonic-gate /* 9647c478bd9Sstevel@tonic-gate * This spin loop is unfair to lwps that have already dropped into 9657c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a highly-contended mutex. 9667c478bd9Sstevel@tonic-gate * This is just too bad. The adaptive spin algorithm is intended 9677c478bd9Sstevel@tonic-gate * to allow programs with highly-contended locks (that is, broken 9687c478bd9Sstevel@tonic-gate * programs) to execute with reasonable speed despite their contention. 9697c478bd9Sstevel@tonic-gate * Being fair would reduce the speed of such programs and well-written 9707c478bd9Sstevel@tonic-gate * programs will not suffer in any case. 9717c478bd9Sstevel@tonic-gate */ 9727c478bd9Sstevel@tonic-gate enter_critical(self); /* protects ul_schedctl */ 9737c478bd9Sstevel@tonic-gate incr32(&mp->mutex_spinners); 9747c478bd9Sstevel@tonic-gate for (count = 0; count < max; count++) { 9757c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) { 9767c478bd9Sstevel@tonic-gate *ownerp = (uintptr_t)self; 9777c478bd9Sstevel@tonic-gate decr32(&mp->mutex_spinners); 9787c478bd9Sstevel@tonic-gate exit_critical(self); 9797c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 9807c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 9817c478bd9Sstevel@tonic-gate return (0); 9827c478bd9Sstevel@tonic-gate } 9837c478bd9Sstevel@tonic-gate SMT_PAUSE(); 9847c478bd9Sstevel@tonic-gate /* 9857c478bd9Sstevel@tonic-gate * Stop spinning if the mutex owner is not running on 9867c478bd9Sstevel@tonic-gate * a processor; it will not drop the lock any time soon 9877c478bd9Sstevel@tonic-gate * and we would just be wasting time to keep spinning. 9887c478bd9Sstevel@tonic-gate * 9897c478bd9Sstevel@tonic-gate * Note that we are looking at another thread (ulwp_t) 9907c478bd9Sstevel@tonic-gate * without ensuring that the other thread does not exit. 9917c478bd9Sstevel@tonic-gate * The scheme relies on ulwp_t structures never being 9927c478bd9Sstevel@tonic-gate * deallocated by the library (the library employs a free 9937c478bd9Sstevel@tonic-gate * list of ulwp_t structs that are reused when new threads 9947c478bd9Sstevel@tonic-gate * are created) and on schedctl shared memory never being 9957c478bd9Sstevel@tonic-gate * deallocated once created via __schedctl(). 9967c478bd9Sstevel@tonic-gate * 9977c478bd9Sstevel@tonic-gate * Thus, the worst that can happen when the spinning thread 9987c478bd9Sstevel@tonic-gate * looks at the owner's schedctl data is that it is looking 9997c478bd9Sstevel@tonic-gate * at some other thread's schedctl data. This almost never 10007c478bd9Sstevel@tonic-gate * happens and is benign when it does. 10017c478bd9Sstevel@tonic-gate */ 10027c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 10037c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 10047c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 10057c478bd9Sstevel@tonic-gate break; 10067c478bd9Sstevel@tonic-gate } 10077c478bd9Sstevel@tonic-gate decr32(&mp->mutex_spinners); 10087c478bd9Sstevel@tonic-gate exit_critical(self); 10097c478bd9Sstevel@tonic-gate 10107c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__spun, 0, count); 10117c478bd9Sstevel@tonic-gate 10127c478bd9Sstevel@tonic-gate return (EBUSY); 10137c478bd9Sstevel@tonic-gate } 10147c478bd9Sstevel@tonic-gate 10157c478bd9Sstevel@tonic-gate /* 10167c478bd9Sstevel@tonic-gate * Same as mutex_trylock_adaptive(), except specifically for queue locks. 10177c478bd9Sstevel@tonic-gate * The owner field is not set here; the caller (spin_lock_set()) sets it. 10187c478bd9Sstevel@tonic-gate */ 10197c478bd9Sstevel@tonic-gate int 10207c478bd9Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp) 10217c478bd9Sstevel@tonic-gate { 10227c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 10237c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 10247c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 10257c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 10267c478bd9Sstevel@tonic-gate int count = curthread->ul_queue_spin; 10277c478bd9Sstevel@tonic-gate 10287c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 10297c478bd9Sstevel@tonic-gate 10307c478bd9Sstevel@tonic-gate if (count == 0) 10317c478bd9Sstevel@tonic-gate return (EBUSY); 10327c478bd9Sstevel@tonic-gate 10337c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 10347c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 10357c478bd9Sstevel@tonic-gate while (--count >= 0) { 10367c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) 10377c478bd9Sstevel@tonic-gate return (0); 10387c478bd9Sstevel@tonic-gate SMT_PAUSE(); 10397c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 10407c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 10417c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 10427c478bd9Sstevel@tonic-gate break; 10437c478bd9Sstevel@tonic-gate } 10447c478bd9Sstevel@tonic-gate 10457c478bd9Sstevel@tonic-gate return (EBUSY); 10467c478bd9Sstevel@tonic-gate } 10477c478bd9Sstevel@tonic-gate 10487c478bd9Sstevel@tonic-gate /* 10497c478bd9Sstevel@tonic-gate * Like mutex_trylock_adaptive(), but for process-shared mutexes. 10507c478bd9Sstevel@tonic-gate * Spin for a while, trying to grab the lock. We know that we 10517c478bd9Sstevel@tonic-gate * failed set_lock_byte(&mp->mutex_lockw) once before coming here. 10527c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 10537c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread 10547c478bd9Sstevel@tonic-gate * and mutex_ownerpid set to the current pid. 10557c478bd9Sstevel@tonic-gate */ 10567c478bd9Sstevel@tonic-gate int 10577c478bd9Sstevel@tonic-gate mutex_trylock_process(mutex_t *mp) 10587c478bd9Sstevel@tonic-gate { 10597c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 10607c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 10617c478bd9Sstevel@tonic-gate int count; 10627c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 10637c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 10647c478bd9Sstevel@tonic-gate volatile int32_t *pidp; 10657c478bd9Sstevel@tonic-gate pid_t pid, newpid; 10667c478bd9Sstevel@tonic-gate uint64_t owner, newowner; 10677c478bd9Sstevel@tonic-gate 10687c478bd9Sstevel@tonic-gate if ((count = ncpus) == 0) 10697c478bd9Sstevel@tonic-gate count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 10707c478bd9Sstevel@tonic-gate count = (count > 1)? self->ul_adaptive_spin : 0; 10717c478bd9Sstevel@tonic-gate 10727c478bd9Sstevel@tonic-gate ASSERT((mp->mutex_type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 10737c478bd9Sstevel@tonic-gate USYNC_PROCESS); 10747c478bd9Sstevel@tonic-gate 10757c478bd9Sstevel@tonic-gate if (count == 0) 10767c478bd9Sstevel@tonic-gate return (EBUSY); 10777c478bd9Sstevel@tonic-gate 10787c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 10797c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 10807c478bd9Sstevel@tonic-gate pidp = (volatile int32_t *)&mp->mutex_ownerpid; 10817c478bd9Sstevel@tonic-gate owner = *ownerp; 10827c478bd9Sstevel@tonic-gate pid = *pidp; 10837c478bd9Sstevel@tonic-gate /* 10847c478bd9Sstevel@tonic-gate * This is a process-shared mutex. 10857c478bd9Sstevel@tonic-gate * We cannot know if the owner is running on a processor. 10867c478bd9Sstevel@tonic-gate * We just spin and hope that it is on a processor. 10877c478bd9Sstevel@tonic-gate */ 10887c478bd9Sstevel@tonic-gate while (--count >= 0) { 10897c478bd9Sstevel@tonic-gate if (*lockp == 0) { 10907c478bd9Sstevel@tonic-gate enter_critical(self); 10917c478bd9Sstevel@tonic-gate if (set_lock_byte(lockp) == 0) { 10927c478bd9Sstevel@tonic-gate *ownerp = (uintptr_t)self; 10937c478bd9Sstevel@tonic-gate *pidp = udp->pid; 10947c478bd9Sstevel@tonic-gate exit_critical(self); 10957c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 10967c478bd9Sstevel@tonic-gate 0, 0); 10977c478bd9Sstevel@tonic-gate return (0); 10987c478bd9Sstevel@tonic-gate } 10997c478bd9Sstevel@tonic-gate exit_critical(self); 11007c478bd9Sstevel@tonic-gate } else if ((newowner = *ownerp) == owner && 11017c478bd9Sstevel@tonic-gate (newpid = *pidp) == pid) { 11027c478bd9Sstevel@tonic-gate SMT_PAUSE(); 11037c478bd9Sstevel@tonic-gate continue; 11047c478bd9Sstevel@tonic-gate } 11057c478bd9Sstevel@tonic-gate /* 11067c478bd9Sstevel@tonic-gate * The owner of the lock changed; start the count over again. 11077c478bd9Sstevel@tonic-gate * This may be too aggressive; it needs testing. 11087c478bd9Sstevel@tonic-gate */ 11097c478bd9Sstevel@tonic-gate owner = newowner; 11107c478bd9Sstevel@tonic-gate pid = newpid; 11117c478bd9Sstevel@tonic-gate count = self->ul_adaptive_spin; 11127c478bd9Sstevel@tonic-gate } 11137c478bd9Sstevel@tonic-gate 11147c478bd9Sstevel@tonic-gate return (EBUSY); 11157c478bd9Sstevel@tonic-gate } 11167c478bd9Sstevel@tonic-gate 11177c478bd9Sstevel@tonic-gate /* 11187c478bd9Sstevel@tonic-gate * Mutex wakeup code for releasing a USYNC_THREAD mutex. 11197c478bd9Sstevel@tonic-gate * Returns the lwpid of the thread that was dequeued, if any. 11207c478bd9Sstevel@tonic-gate * The caller of mutex_wakeup() must call __lwp_unpark(lwpid) 11217c478bd9Sstevel@tonic-gate * to wake up the specified lwp. 11227c478bd9Sstevel@tonic-gate */ 11237c478bd9Sstevel@tonic-gate lwpid_t 11247c478bd9Sstevel@tonic-gate mutex_wakeup(mutex_t *mp) 11257c478bd9Sstevel@tonic-gate { 11267c478bd9Sstevel@tonic-gate lwpid_t lwpid = 0; 11277c478bd9Sstevel@tonic-gate queue_head_t *qp; 11287c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 11297c478bd9Sstevel@tonic-gate int more; 11307c478bd9Sstevel@tonic-gate 11317c478bd9Sstevel@tonic-gate /* 11327c478bd9Sstevel@tonic-gate * Dequeue a waiter from the sleep queue. Don't touch the mutex 11337c478bd9Sstevel@tonic-gate * waiters bit if no one was found on the queue because the mutex 11347c478bd9Sstevel@tonic-gate * might have been deallocated or reallocated for another purpose. 11357c478bd9Sstevel@tonic-gate */ 11367c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 11377c478bd9Sstevel@tonic-gate if ((ulwp = dequeue(qp, mp, &more)) != NULL) { 11387c478bd9Sstevel@tonic-gate lwpid = ulwp->ul_lwpid; 11397c478bd9Sstevel@tonic-gate mp->mutex_waiters = (more? 1 : 0); 11407c478bd9Sstevel@tonic-gate } 11417c478bd9Sstevel@tonic-gate queue_unlock(qp); 11427c478bd9Sstevel@tonic-gate return (lwpid); 11437c478bd9Sstevel@tonic-gate } 11447c478bd9Sstevel@tonic-gate 11457c478bd9Sstevel@tonic-gate /* 11467c478bd9Sstevel@tonic-gate * Spin for a while, testing to see if the lock has been grabbed. 11477c478bd9Sstevel@tonic-gate * If this fails, call mutex_wakeup() to release a waiter. 11487c478bd9Sstevel@tonic-gate */ 11497c478bd9Sstevel@tonic-gate lwpid_t 11507c478bd9Sstevel@tonic-gate mutex_unlock_queue(mutex_t *mp) 11517c478bd9Sstevel@tonic-gate { 11527c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 11537c478bd9Sstevel@tonic-gate uint32_t *lockw = &mp->mutex_lockword; 11547c478bd9Sstevel@tonic-gate lwpid_t lwpid; 11557c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 11567c478bd9Sstevel@tonic-gate volatile uint32_t *spinp; 11577c478bd9Sstevel@tonic-gate int count; 11587c478bd9Sstevel@tonic-gate 11597c478bd9Sstevel@tonic-gate /* 11607c478bd9Sstevel@tonic-gate * We use the swap primitive to clear the lock, but we must 11617c478bd9Sstevel@tonic-gate * atomically retain the waiters bit for the remainder of this 11627c478bd9Sstevel@tonic-gate * code to work. We first check to see if the waiters bit is 11637c478bd9Sstevel@tonic-gate * set and if so clear the lock by swapping in a word containing 11647c478bd9Sstevel@tonic-gate * only the waiters bit. This could produce a false positive test 11657c478bd9Sstevel@tonic-gate * for whether there are waiters that need to be waked up, but 11667c478bd9Sstevel@tonic-gate * this just causes an extra call to mutex_wakeup() to do nothing. 11677c478bd9Sstevel@tonic-gate * The opposite case is more delicate: If there are no waiters, 11687c478bd9Sstevel@tonic-gate * we swap in a zero lock byte and a zero waiters bit. The result 11697c478bd9Sstevel@tonic-gate * of the swap could indicate that there really was a waiter so in 11707c478bd9Sstevel@tonic-gate * this case we go directly to mutex_wakeup() without performing 11717c478bd9Sstevel@tonic-gate * any of the adaptive code because the waiter bit has been cleared 11727c478bd9Sstevel@tonic-gate * and the adaptive code is unreliable in this case. 11737c478bd9Sstevel@tonic-gate */ 11747c478bd9Sstevel@tonic-gate if (!(*lockw & WAITERMASK)) { /* no waiter exists right now */ 11757c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 11767c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 11777c478bd9Sstevel@tonic-gate if (!(swap32(lockw, 0) & WAITERMASK)) /* still no waiters */ 11787c478bd9Sstevel@tonic-gate return (0); 11797c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 11807c478bd9Sstevel@tonic-gate lwpid = mutex_wakeup(mp); 11817c478bd9Sstevel@tonic-gate } else { 11827c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 11837c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 11847c478bd9Sstevel@tonic-gate spinp = (volatile uint32_t *)&mp->mutex_spinners; 11857c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 11867c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 11877c478bd9Sstevel@tonic-gate (void) swap32(lockw, WAITER); /* clear lock, retain waiter */ 11887c478bd9Sstevel@tonic-gate 11897c478bd9Sstevel@tonic-gate /* 11907c478bd9Sstevel@tonic-gate * We spin here fewer times than mutex_trylock_adaptive(). 11917c478bd9Sstevel@tonic-gate * We are trying to balance two conflicting goals: 11927c478bd9Sstevel@tonic-gate * 1. Avoid waking up anyone if a spinning thread 11937c478bd9Sstevel@tonic-gate * grabs the lock. 11947c478bd9Sstevel@tonic-gate * 2. Wake up a sleeping thread promptly to get on 11957c478bd9Sstevel@tonic-gate * with useful work. 11967c478bd9Sstevel@tonic-gate * We don't spin at all if there is no acquiring spinner; 11977c478bd9Sstevel@tonic-gate * (mp->mutex_spinners is non-zero if there are spinners). 11987c478bd9Sstevel@tonic-gate */ 11997c478bd9Sstevel@tonic-gate for (count = self->ul_release_spin; 12007c478bd9Sstevel@tonic-gate *spinp && count > 0; count--) { 12017c478bd9Sstevel@tonic-gate /* 12027c478bd9Sstevel@tonic-gate * There is a waiter that we will have to wake 12037c478bd9Sstevel@tonic-gate * up unless someone else grabs the lock while 12047c478bd9Sstevel@tonic-gate * we are busy spinning. Like the spin loop in 12057c478bd9Sstevel@tonic-gate * mutex_trylock_adaptive(), this spin loop is 12067c478bd9Sstevel@tonic-gate * unfair to lwps that have already dropped into 12077c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a 12087c478bd9Sstevel@tonic-gate * highly-contended mutex. Too bad. 12097c478bd9Sstevel@tonic-gate */ 12107c478bd9Sstevel@tonic-gate if (*lockp != 0) { /* somebody grabbed the lock */ 12117c478bd9Sstevel@tonic-gate preempt(self); 12127c478bd9Sstevel@tonic-gate return (0); 12137c478bd9Sstevel@tonic-gate } 12147c478bd9Sstevel@tonic-gate SMT_PAUSE(); 12157c478bd9Sstevel@tonic-gate } 12167c478bd9Sstevel@tonic-gate 12177c478bd9Sstevel@tonic-gate /* 12187c478bd9Sstevel@tonic-gate * No one grabbed the lock. 12197c478bd9Sstevel@tonic-gate * Wake up some lwp that is waiting for it. 12207c478bd9Sstevel@tonic-gate */ 12217c478bd9Sstevel@tonic-gate mp->mutex_waiters = 0; 12227c478bd9Sstevel@tonic-gate lwpid = mutex_wakeup(mp); 12237c478bd9Sstevel@tonic-gate } 12247c478bd9Sstevel@tonic-gate 12257c478bd9Sstevel@tonic-gate if (lwpid == 0) 12267c478bd9Sstevel@tonic-gate preempt(self); 12277c478bd9Sstevel@tonic-gate return (lwpid); 12287c478bd9Sstevel@tonic-gate } 12297c478bd9Sstevel@tonic-gate 12307c478bd9Sstevel@tonic-gate /* 12317c478bd9Sstevel@tonic-gate * Like mutex_unlock_queue(), but for process-shared mutexes. 12327c478bd9Sstevel@tonic-gate * We tested the waiters field before calling here and it was non-zero. 12337c478bd9Sstevel@tonic-gate */ 12347c478bd9Sstevel@tonic-gate void 12357c478bd9Sstevel@tonic-gate mutex_unlock_process(mutex_t *mp) 12367c478bd9Sstevel@tonic-gate { 12377c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 12387c478bd9Sstevel@tonic-gate int count; 12397c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 12407c478bd9Sstevel@tonic-gate 12417c478bd9Sstevel@tonic-gate /* 12427c478bd9Sstevel@tonic-gate * See the comments in mutex_unlock_queue(), above. 12437c478bd9Sstevel@tonic-gate */ 12447c478bd9Sstevel@tonic-gate if ((count = ncpus) == 0) 12457c478bd9Sstevel@tonic-gate count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 12467c478bd9Sstevel@tonic-gate count = (count > 1)? self->ul_release_spin : 0; 12477c478bd9Sstevel@tonic-gate no_preempt(self); 12487c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 12497c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 12507c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 12517c478bd9Sstevel@tonic-gate if (count == 0) { 12527c478bd9Sstevel@tonic-gate /* clear lock, test waiter */ 12537c478bd9Sstevel@tonic-gate if (!(swap32(&mp->mutex_lockword, 0) & WAITERMASK)) { 12547c478bd9Sstevel@tonic-gate /* no waiters now */ 12557c478bd9Sstevel@tonic-gate preempt(self); 12567c478bd9Sstevel@tonic-gate return; 12577c478bd9Sstevel@tonic-gate } 12587c478bd9Sstevel@tonic-gate } else { 12597c478bd9Sstevel@tonic-gate /* clear lock, retain waiter */ 12607c478bd9Sstevel@tonic-gate (void) swap32(&mp->mutex_lockword, WAITER); 12617c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 12627c478bd9Sstevel@tonic-gate while (--count >= 0) { 12637c478bd9Sstevel@tonic-gate if (*lockp != 0) { 12647c478bd9Sstevel@tonic-gate /* somebody grabbed the lock */ 12657c478bd9Sstevel@tonic-gate preempt(self); 12667c478bd9Sstevel@tonic-gate return; 12677c478bd9Sstevel@tonic-gate } 12687c478bd9Sstevel@tonic-gate SMT_PAUSE(); 12697c478bd9Sstevel@tonic-gate } 12707c478bd9Sstevel@tonic-gate /* 12717c478bd9Sstevel@tonic-gate * We must clear the waiters field before going 12727c478bd9Sstevel@tonic-gate * to the kernel, else it could remain set forever. 12737c478bd9Sstevel@tonic-gate */ 12747c478bd9Sstevel@tonic-gate mp->mutex_waiters = 0; 12757c478bd9Sstevel@tonic-gate } 12767c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 12777c478bd9Sstevel@tonic-gate preempt(self); 12787c478bd9Sstevel@tonic-gate } 12797c478bd9Sstevel@tonic-gate 12807c478bd9Sstevel@tonic-gate /* 12817c478bd9Sstevel@tonic-gate * Return the real priority of a thread. 12827c478bd9Sstevel@tonic-gate */ 12837c478bd9Sstevel@tonic-gate int 12847c478bd9Sstevel@tonic-gate real_priority(ulwp_t *ulwp) 12857c478bd9Sstevel@tonic-gate { 12867c478bd9Sstevel@tonic-gate if (ulwp->ul_epri == 0) 12877c478bd9Sstevel@tonic-gate return (ulwp->ul_mappedpri? ulwp->ul_mappedpri : ulwp->ul_pri); 12887c478bd9Sstevel@tonic-gate return (ulwp->ul_emappedpri? ulwp->ul_emappedpri : ulwp->ul_epri); 12897c478bd9Sstevel@tonic-gate } 12907c478bd9Sstevel@tonic-gate 12917c478bd9Sstevel@tonic-gate void 12927c478bd9Sstevel@tonic-gate stall(void) 12937c478bd9Sstevel@tonic-gate { 12947c478bd9Sstevel@tonic-gate for (;;) 12957c478bd9Sstevel@tonic-gate (void) mutex_lock_kernel(&stall_mutex, NULL, NULL); 12967c478bd9Sstevel@tonic-gate } 12977c478bd9Sstevel@tonic-gate 12987c478bd9Sstevel@tonic-gate /* 12997c478bd9Sstevel@tonic-gate * Acquire a USYNC_THREAD mutex via user-level sleep queues. 13007c478bd9Sstevel@tonic-gate * We failed set_lock_byte(&mp->mutex_lockw) before coming here. 13017c478bd9Sstevel@tonic-gate * Returns with mutex_owner set correctly. 13027c478bd9Sstevel@tonic-gate */ 13037c478bd9Sstevel@tonic-gate int 13047c478bd9Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp, 13057c478bd9Sstevel@tonic-gate timespec_t *tsp) 13067c478bd9Sstevel@tonic-gate { 13077c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 13087c478bd9Sstevel@tonic-gate queue_head_t *qp; 13097c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 13107c478bd9Sstevel@tonic-gate int error = 0; 13117c478bd9Sstevel@tonic-gate 13127c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 13137c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 13147c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 13157c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 13167c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 13177c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 13187c478bd9Sstevel@tonic-gate } 13197c478bd9Sstevel@tonic-gate if (msp) { 13207c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 13217c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 13227c478bd9Sstevel@tonic-gate } 13237c478bd9Sstevel@tonic-gate 13247c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 13257c478bd9Sstevel@tonic-gate 13267c478bd9Sstevel@tonic-gate /* 13277c478bd9Sstevel@tonic-gate * Put ourself on the sleep queue, and while we are 13287c478bd9Sstevel@tonic-gate * unable to grab the lock, go park in the kernel. 13297c478bd9Sstevel@tonic-gate * Take ourself off the sleep queue after we acquire the lock. 13307c478bd9Sstevel@tonic-gate * The waiter bit can be set/cleared only while holding the queue lock. 13317c478bd9Sstevel@tonic-gate */ 13327c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 13337c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 13347c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 13357c478bd9Sstevel@tonic-gate for (;;) { 13367c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 13377c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 13387c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 13397c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 13407c478bd9Sstevel@tonic-gate mp->mutex_waiters = dequeue_self(qp, mp); 13417c478bd9Sstevel@tonic-gate break; 13427c478bd9Sstevel@tonic-gate } 13437c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 13447c478bd9Sstevel@tonic-gate queue_unlock(qp); 13457c478bd9Sstevel@tonic-gate /* 13467c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 13477c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 13487c478bd9Sstevel@tonic-gate */ 13497c478bd9Sstevel@tonic-gate if ((error = __lwp_park(tsp, 0)) == EINTR) 13507c478bd9Sstevel@tonic-gate error = 0; 13517c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 13527c478bd9Sstevel@tonic-gate /* 13537c478bd9Sstevel@tonic-gate * We could have taken a signal or suspended ourself. 13547c478bd9Sstevel@tonic-gate * If we did, then we removed ourself from the queue. 13557c478bd9Sstevel@tonic-gate * Someone else may have removed us from the queue 13567c478bd9Sstevel@tonic-gate * as a consequence of mutex_unlock(). We may have 13577c478bd9Sstevel@tonic-gate * gotten a timeout from __lwp_park(). Or we may still 13587c478bd9Sstevel@tonic-gate * be on the queue and this is just a spurious wakeup. 13597c478bd9Sstevel@tonic-gate */ 13607c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 13617c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) { 13627c478bd9Sstevel@tonic-gate if (error) { 13637c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 13647c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 13657c478bd9Sstevel@tonic-gate error); 13667c478bd9Sstevel@tonic-gate break; 13677c478bd9Sstevel@tonic-gate } 13687c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 13697c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 13707c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 13717c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 13727c478bd9Sstevel@tonic-gate 0, 0); 13737c478bd9Sstevel@tonic-gate break; 13747c478bd9Sstevel@tonic-gate } 13757c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 13767c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 13777c478bd9Sstevel@tonic-gate } 13787c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == qp && 13797c478bd9Sstevel@tonic-gate self->ul_qtype == MX && 13807c478bd9Sstevel@tonic-gate self->ul_wchan == mp); 13817c478bd9Sstevel@tonic-gate if (error) { 13827c478bd9Sstevel@tonic-gate mp->mutex_waiters = dequeue_self(qp, mp); 13837c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 13847c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, error); 13857c478bd9Sstevel@tonic-gate break; 13867c478bd9Sstevel@tonic-gate } 13877c478bd9Sstevel@tonic-gate } 13887c478bd9Sstevel@tonic-gate 13897c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 13907c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 13917c478bd9Sstevel@tonic-gate self->ul_sp = 0; 13927c478bd9Sstevel@tonic-gate 13937c478bd9Sstevel@tonic-gate queue_unlock(qp); 13947c478bd9Sstevel@tonic-gate if (msp) 13957c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 13967c478bd9Sstevel@tonic-gate 13977c478bd9Sstevel@tonic-gate ASSERT(error == 0 || error == EINVAL || error == ETIME); 13987c478bd9Sstevel@tonic-gate return (error); 13997c478bd9Sstevel@tonic-gate } 14007c478bd9Sstevel@tonic-gate 14017c478bd9Sstevel@tonic-gate /* 14027c478bd9Sstevel@tonic-gate * Returns with mutex_owner set correctly. 14037c478bd9Sstevel@tonic-gate */ 14047c478bd9Sstevel@tonic-gate int 14057c478bd9Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try) 14067c478bd9Sstevel@tonic-gate { 14077c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 14087c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 14097c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 14107c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 14117c478bd9Sstevel@tonic-gate int error = 0; 14127c478bd9Sstevel@tonic-gate 14137c478bd9Sstevel@tonic-gate ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK); 14147c478bd9Sstevel@tonic-gate 14157c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 14167c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 14177c478bd9Sstevel@tonic-gate 14187c478bd9Sstevel@tonic-gate if (msp && try == MUTEX_TRY) 14197c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try); 14207c478bd9Sstevel@tonic-gate 14217c478bd9Sstevel@tonic-gate if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) { 14227c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 14237c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) { 14247c478bd9Sstevel@tonic-gate error = EAGAIN; 14257c478bd9Sstevel@tonic-gate } else { 14267c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 14277c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 14287c478bd9Sstevel@tonic-gate 1, 0); 14297c478bd9Sstevel@tonic-gate return (0); 14307c478bd9Sstevel@tonic-gate } 14317c478bd9Sstevel@tonic-gate } else if (try == MUTEX_TRY) { 14327c478bd9Sstevel@tonic-gate return (EBUSY); 14337c478bd9Sstevel@tonic-gate } else { 14347c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 14357c478bd9Sstevel@tonic-gate return (EDEADLK); 14367c478bd9Sstevel@tonic-gate } 14377c478bd9Sstevel@tonic-gate } 14387c478bd9Sstevel@tonic-gate 14397c478bd9Sstevel@tonic-gate if (self->ul_error_detection && try == MUTEX_LOCK && 14407c478bd9Sstevel@tonic-gate tsp == NULL && mutex_is_held(mp)) 14417c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_lock", NULL, NULL); 14427c478bd9Sstevel@tonic-gate 14437c478bd9Sstevel@tonic-gate if (mtype & 14447c478bd9Sstevel@tonic-gate (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) { 14457c478bd9Sstevel@tonic-gate uint8_t ceil; 14467c478bd9Sstevel@tonic-gate int myprio; 14477c478bd9Sstevel@tonic-gate 14487c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_PROTECT) { 14497c478bd9Sstevel@tonic-gate ceil = mp->mutex_ceiling; 14507c478bd9Sstevel@tonic-gate ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0); 14517c478bd9Sstevel@tonic-gate myprio = real_priority(self); 14527c478bd9Sstevel@tonic-gate if (myprio > ceil) { 14537c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 14547c478bd9Sstevel@tonic-gate EINVAL); 14557c478bd9Sstevel@tonic-gate return (EINVAL); 14567c478bd9Sstevel@tonic-gate } 14577c478bd9Sstevel@tonic-gate if ((error = _ceil_mylist_add(mp)) != 0) { 14587c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 14597c478bd9Sstevel@tonic-gate error); 14607c478bd9Sstevel@tonic-gate return (error); 14617c478bd9Sstevel@tonic-gate } 14627c478bd9Sstevel@tonic-gate if (myprio < ceil) 14637c478bd9Sstevel@tonic-gate _ceil_prio_inherit(ceil); 14647c478bd9Sstevel@tonic-gate } 14657c478bd9Sstevel@tonic-gate 14667c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) { 14677c478bd9Sstevel@tonic-gate /* go straight to the kernel */ 14687c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) 14697c478bd9Sstevel@tonic-gate error = mutex_trylock_kernel(mp); 14707c478bd9Sstevel@tonic-gate else /* MUTEX_LOCK */ 14717c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 14727c478bd9Sstevel@tonic-gate /* 14737c478bd9Sstevel@tonic-gate * The kernel never sets or clears the lock byte 14747c478bd9Sstevel@tonic-gate * for PTHREAD_PRIO_INHERIT mutexes. 14757c478bd9Sstevel@tonic-gate * Set it here for debugging consistency. 14767c478bd9Sstevel@tonic-gate */ 14777c478bd9Sstevel@tonic-gate switch (error) { 14787c478bd9Sstevel@tonic-gate case 0: 14797c478bd9Sstevel@tonic-gate case EOWNERDEAD: 14807c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 14817c478bd9Sstevel@tonic-gate break; 14827c478bd9Sstevel@tonic-gate } 14837c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS_ROBUST) { 14847c478bd9Sstevel@tonic-gate /* go straight to the kernel */ 14857c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) 14867c478bd9Sstevel@tonic-gate error = mutex_trylock_kernel(mp); 14877c478bd9Sstevel@tonic-gate else /* MUTEX_LOCK */ 14887c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 14897c478bd9Sstevel@tonic-gate } else { /* PTHREAD_PRIO_PROTECT */ 14907c478bd9Sstevel@tonic-gate /* 14917c478bd9Sstevel@tonic-gate * Try once at user level before going to the kernel. 14927c478bd9Sstevel@tonic-gate * If this is a process shared mutex then protect 14937c478bd9Sstevel@tonic-gate * against forkall() while setting mp->mutex_ownerpid. 14947c478bd9Sstevel@tonic-gate */ 14957c478bd9Sstevel@tonic-gate if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) { 14967c478bd9Sstevel@tonic-gate enter_critical(self); 14977c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 14987c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 14997c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 15007c478bd9Sstevel@tonic-gate exit_critical(self); 15017c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, 15027c478bd9Sstevel@tonic-gate mutex__acquire, mp, 0, 0); 15037c478bd9Sstevel@tonic-gate } else { 15047c478bd9Sstevel@tonic-gate exit_critical(self); 15057c478bd9Sstevel@tonic-gate error = EBUSY; 15067c478bd9Sstevel@tonic-gate } 15077c478bd9Sstevel@tonic-gate } else { 15087c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 15097c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15107c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, 15117c478bd9Sstevel@tonic-gate mutex__acquire, mp, 0, 0); 15127c478bd9Sstevel@tonic-gate } else { 15137c478bd9Sstevel@tonic-gate error = EBUSY; 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate } 15167c478bd9Sstevel@tonic-gate if (error && try == MUTEX_LOCK) 15177c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 15187c478bd9Sstevel@tonic-gate } 15197c478bd9Sstevel@tonic-gate 15207c478bd9Sstevel@tonic-gate if (error) { 15217c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) { 15227c478bd9Sstevel@tonic-gate switch (error) { 15237c478bd9Sstevel@tonic-gate case EOWNERDEAD: 15247c478bd9Sstevel@tonic-gate case ENOTRECOVERABLE: 15257c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_MUTEX_ROBUST_NP) 15267c478bd9Sstevel@tonic-gate break; 15277c478bd9Sstevel@tonic-gate if (error == EOWNERDEAD) { 15287c478bd9Sstevel@tonic-gate /* 15297c478bd9Sstevel@tonic-gate * We own the mutex; unlock it. 15307c478bd9Sstevel@tonic-gate * It becomes ENOTRECOVERABLE. 15317c478bd9Sstevel@tonic-gate * All waiters are waked up. 15327c478bd9Sstevel@tonic-gate */ 15337c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 15347c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 15357c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, 15367c478bd9Sstevel@tonic-gate mutex__release, mp, 0); 15377c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 15387c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_unlock(mp); 15397c478bd9Sstevel@tonic-gate } 15407c478bd9Sstevel@tonic-gate /* FALLTHROUGH */ 15417c478bd9Sstevel@tonic-gate case EDEADLK: 15427c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) 15437c478bd9Sstevel@tonic-gate stall(); 15447c478bd9Sstevel@tonic-gate error = EBUSY; 15457c478bd9Sstevel@tonic-gate break; 15467c478bd9Sstevel@tonic-gate } 15477c478bd9Sstevel@tonic-gate } 15487c478bd9Sstevel@tonic-gate if ((mtype & PTHREAD_PRIO_PROTECT) && 15497c478bd9Sstevel@tonic-gate error != EOWNERDEAD) { 15507c478bd9Sstevel@tonic-gate (void) _ceil_mylist_del(mp); 15517c478bd9Sstevel@tonic-gate if (myprio < ceil) 15527c478bd9Sstevel@tonic-gate _ceil_prio_waive(); 15537c478bd9Sstevel@tonic-gate } 15547c478bd9Sstevel@tonic-gate } 15557c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) { 15567c478bd9Sstevel@tonic-gate /* 15577c478bd9Sstevel@tonic-gate * This is a process shared mutex. Protect against 15587c478bd9Sstevel@tonic-gate * forkall() while setting mp->mutex_ownerpid. 15597c478bd9Sstevel@tonic-gate */ 15607c478bd9Sstevel@tonic-gate enter_critical(self); 15617c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 15627c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15637c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 15647c478bd9Sstevel@tonic-gate exit_critical(self); 15657c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 15667c478bd9Sstevel@tonic-gate } else { 15677c478bd9Sstevel@tonic-gate /* try a little harder */ 15687c478bd9Sstevel@tonic-gate exit_critical(self); 15697c478bd9Sstevel@tonic-gate error = mutex_trylock_process(mp); 15707c478bd9Sstevel@tonic-gate } 15717c478bd9Sstevel@tonic-gate if (error && try == MUTEX_LOCK) 15727c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 15737c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 15747c478bd9Sstevel@tonic-gate /* try once */ 15757c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 15767c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15777c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 15787c478bd9Sstevel@tonic-gate } else { 15797c478bd9Sstevel@tonic-gate /* try a little harder if we don't own the mutex */ 15807c478bd9Sstevel@tonic-gate error = EBUSY; 15817c478bd9Sstevel@tonic-gate if (MUTEX_OWNER(mp) != self) 15827c478bd9Sstevel@tonic-gate error = mutex_trylock_adaptive(mp); 15837c478bd9Sstevel@tonic-gate if (error && try == MUTEX_LOCK) /* go park */ 15847c478bd9Sstevel@tonic-gate error = mutex_lock_queue(self, msp, mp, tsp); 15857c478bd9Sstevel@tonic-gate } 15867c478bd9Sstevel@tonic-gate } 15877c478bd9Sstevel@tonic-gate 15887c478bd9Sstevel@tonic-gate switch (error) { 15897c478bd9Sstevel@tonic-gate case EOWNERDEAD: 15907c478bd9Sstevel@tonic-gate case ELOCKUNMAPPED: 15917c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15927c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 15937c478bd9Sstevel@tonic-gate /* FALLTHROUGH */ 15947c478bd9Sstevel@tonic-gate case 0: 15957c478bd9Sstevel@tonic-gate if (msp) 15967c478bd9Sstevel@tonic-gate record_begin_hold(msp); 15977c478bd9Sstevel@tonic-gate break; 15987c478bd9Sstevel@tonic-gate default: 15997c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) { 16007c478bd9Sstevel@tonic-gate if (msp) 16017c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try_fail); 16027c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 16037c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 16047c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 16057c478bd9Sstevel@tonic-gate } 16067c478bd9Sstevel@tonic-gate } 16077c478bd9Sstevel@tonic-gate break; 16087c478bd9Sstevel@tonic-gate } 16097c478bd9Sstevel@tonic-gate 16107c478bd9Sstevel@tonic-gate return (error); 16117c478bd9Sstevel@tonic-gate } 16127c478bd9Sstevel@tonic-gate 16137c478bd9Sstevel@tonic-gate int 16147c478bd9Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try) 16157c478bd9Sstevel@tonic-gate { 16167c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 16177c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 16187c478bd9Sstevel@tonic-gate 16197c478bd9Sstevel@tonic-gate /* 16207c478bd9Sstevel@tonic-gate * We know that USYNC_PROCESS is set in mtype and that 16217c478bd9Sstevel@tonic-gate * zero, one, or both of the flags LOCK_RECURSIVE and 16227c478bd9Sstevel@tonic-gate * LOCK_ERRORCHECK are set, and that no other flags are set. 16237c478bd9Sstevel@tonic-gate */ 16247c478bd9Sstevel@tonic-gate enter_critical(self); 16257c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 16267c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 16277c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 16287c478bd9Sstevel@tonic-gate exit_critical(self); 16297c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 16307c478bd9Sstevel@tonic-gate return (0); 16317c478bd9Sstevel@tonic-gate } 16327c478bd9Sstevel@tonic-gate exit_critical(self); 16337c478bd9Sstevel@tonic-gate 16347c478bd9Sstevel@tonic-gate if ((mtype & ~USYNC_PROCESS) && shared_mutex_held(mp)) { 16357c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 16367c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 16377c478bd9Sstevel@tonic-gate return (EAGAIN); 16387c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 16397c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0); 16407c478bd9Sstevel@tonic-gate return (0); 16417c478bd9Sstevel@tonic-gate } 16427c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) { 16437c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 16447c478bd9Sstevel@tonic-gate return (EDEADLK); 16457c478bd9Sstevel@tonic-gate } 16467c478bd9Sstevel@tonic-gate return (EBUSY); 16477c478bd9Sstevel@tonic-gate } 16487c478bd9Sstevel@tonic-gate 16497c478bd9Sstevel@tonic-gate /* try a little harder if we don't own the mutex */ 16507c478bd9Sstevel@tonic-gate if (!shared_mutex_held(mp) && mutex_trylock_process(mp) == 0) 16517c478bd9Sstevel@tonic-gate return (0); 16527c478bd9Sstevel@tonic-gate 16537c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) 16547c478bd9Sstevel@tonic-gate return (mutex_lock_kernel(mp, tsp, NULL)); 16557c478bd9Sstevel@tonic-gate 16567c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 16577c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 16587c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 16597c478bd9Sstevel@tonic-gate } 16607c478bd9Sstevel@tonic-gate return (EBUSY); 16617c478bd9Sstevel@tonic-gate } 16627c478bd9Sstevel@tonic-gate 16637c478bd9Sstevel@tonic-gate static int 16647c478bd9Sstevel@tonic-gate slow_lock(ulwp_t *self, mutex_t *mp, timespec_t *tsp) 16657c478bd9Sstevel@tonic-gate { 16667c478bd9Sstevel@tonic-gate int error = 0; 16677c478bd9Sstevel@tonic-gate 16687c478bd9Sstevel@tonic-gate if (MUTEX_OWNER(mp) == self || mutex_trylock_adaptive(mp) != 0) 16697c478bd9Sstevel@tonic-gate error = mutex_lock_queue(self, NULL, mp, tsp); 16707c478bd9Sstevel@tonic-gate return (error); 16717c478bd9Sstevel@tonic-gate } 16727c478bd9Sstevel@tonic-gate 16737c478bd9Sstevel@tonic-gate int 16747c478bd9Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp) 16757c478bd9Sstevel@tonic-gate { 16767c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 16777c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 16787c478bd9Sstevel@tonic-gate uberflags_t *gflags; 16797c478bd9Sstevel@tonic-gate int mtype; 16807c478bd9Sstevel@tonic-gate 16817c478bd9Sstevel@tonic-gate /* 16827c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 16837c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 16847c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 16857c478bd9Sstevel@tonic-gate * and the process has only a single thread. 16867c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 16877c478bd9Sstevel@tonic-gate */ 16887c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 16897c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 16907c478bd9Sstevel@tonic-gate /* 16917c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 16927c478bd9Sstevel@tonic-gate */ 16937c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 16947c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 16957c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 16967c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 16977c478bd9Sstevel@tonic-gate return (0); 16987c478bd9Sstevel@tonic-gate } 16997c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 17007c478bd9Sstevel@tonic-gate /* 17017c478bd9Sstevel@tonic-gate * LOCK_RECURSIVE, LOCK_ERRORCHECK, or both. 17027c478bd9Sstevel@tonic-gate */ 17037c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 17047c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 17057c478bd9Sstevel@tonic-gate return (EAGAIN); 17067c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 17077c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 17087c478bd9Sstevel@tonic-gate 1, 0); 17097c478bd9Sstevel@tonic-gate return (0); 17107c478bd9Sstevel@tonic-gate } 17117c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 17127c478bd9Sstevel@tonic-gate return (EDEADLK); /* LOCK_ERRORCHECK */ 17137c478bd9Sstevel@tonic-gate } 17147c478bd9Sstevel@tonic-gate /* 17157c478bd9Sstevel@tonic-gate * We have reached a deadlock, probably because the 17167c478bd9Sstevel@tonic-gate * process is executing non-async-signal-safe code in 17177c478bd9Sstevel@tonic-gate * a signal handler and is attempting to acquire a lock 17187c478bd9Sstevel@tonic-gate * that it already owns. This is not surprising, given 17197c478bd9Sstevel@tonic-gate * bad programming practices over the years that has 17207c478bd9Sstevel@tonic-gate * resulted in applications calling printf() and such 17217c478bd9Sstevel@tonic-gate * in their signal handlers. Unless the user has told 17227c478bd9Sstevel@tonic-gate * us that the signal handlers are safe by setting: 17237c478bd9Sstevel@tonic-gate * export _THREAD_ASYNC_SAFE=1 17247c478bd9Sstevel@tonic-gate * we return EDEADLK rather than actually deadlocking. 17257c478bd9Sstevel@tonic-gate */ 17267c478bd9Sstevel@tonic-gate if (tsp == NULL && 17277c478bd9Sstevel@tonic-gate MUTEX_OWNER(mp) == self && !self->ul_async_safe) { 17287c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 17297c478bd9Sstevel@tonic-gate return (EDEADLK); 17307c478bd9Sstevel@tonic-gate } 17317c478bd9Sstevel@tonic-gate } 17327c478bd9Sstevel@tonic-gate 17337c478bd9Sstevel@tonic-gate /* 17347c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 17357c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 17367c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 17377c478bd9Sstevel@tonic-gate */ 17387c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 17397c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 17407c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 17417c478bd9Sstevel@tonic-gate 17427c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 17437c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK)); 17447c478bd9Sstevel@tonic-gate 17457c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 17467c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 17477c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 17487c478bd9Sstevel@tonic-gate return (0); 17497c478bd9Sstevel@tonic-gate } 17507c478bd9Sstevel@tonic-gate 17517c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 17527c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 17537c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 17547c478bd9Sstevel@tonic-gate return (EAGAIN); 17557c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 17567c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 17577c478bd9Sstevel@tonic-gate 1, 0); 17587c478bd9Sstevel@tonic-gate return (0); 17597c478bd9Sstevel@tonic-gate } 17607c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 17617c478bd9Sstevel@tonic-gate return (EDEADLK); /* LOCK_ERRORCHECK */ 17627c478bd9Sstevel@tonic-gate } 17637c478bd9Sstevel@tonic-gate 17647c478bd9Sstevel@tonic-gate return (slow_lock(self, mp, tsp)); 17657c478bd9Sstevel@tonic-gate } 17667c478bd9Sstevel@tonic-gate 17677c478bd9Sstevel@tonic-gate /* else do it the long way */ 17687c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, tsp, MUTEX_LOCK)); 17697c478bd9Sstevel@tonic-gate } 17707c478bd9Sstevel@tonic-gate 17717c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_lock = __mutex_lock 17727c478bd9Sstevel@tonic-gate #pragma weak mutex_lock = __mutex_lock 17737c478bd9Sstevel@tonic-gate #pragma weak _mutex_lock = __mutex_lock 17747c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_lock = __mutex_lock 17757c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_lock = __mutex_lock 17767c478bd9Sstevel@tonic-gate int 17777c478bd9Sstevel@tonic-gate __mutex_lock(mutex_t *mp) 17787c478bd9Sstevel@tonic-gate { 17797c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 17807c478bd9Sstevel@tonic-gate return (mutex_lock_impl(mp, NULL)); 17817c478bd9Sstevel@tonic-gate } 17827c478bd9Sstevel@tonic-gate 17837c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_timedlock = _pthread_mutex_timedlock 17847c478bd9Sstevel@tonic-gate int 17857c478bd9Sstevel@tonic-gate _pthread_mutex_timedlock(mutex_t *mp, const timespec_t *abstime) 17867c478bd9Sstevel@tonic-gate { 17877c478bd9Sstevel@tonic-gate timespec_t tslocal; 17887c478bd9Sstevel@tonic-gate int error; 17897c478bd9Sstevel@tonic-gate 17907c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 17917c478bd9Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 17927c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 17937c478bd9Sstevel@tonic-gate if (error == ETIME) 17947c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 17957c478bd9Sstevel@tonic-gate return (error); 17967c478bd9Sstevel@tonic-gate } 17977c478bd9Sstevel@tonic-gate 17987c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_reltimedlock_np = _pthread_mutex_reltimedlock_np 17997c478bd9Sstevel@tonic-gate int 18007c478bd9Sstevel@tonic-gate _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime) 18017c478bd9Sstevel@tonic-gate { 18027c478bd9Sstevel@tonic-gate timespec_t tslocal; 18037c478bd9Sstevel@tonic-gate int error; 18047c478bd9Sstevel@tonic-gate 18057c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 18067c478bd9Sstevel@tonic-gate tslocal = *reltime; 18077c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 18087c478bd9Sstevel@tonic-gate if (error == ETIME) 18097c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 18107c478bd9Sstevel@tonic-gate return (error); 18117c478bd9Sstevel@tonic-gate } 18127c478bd9Sstevel@tonic-gate 18137c478bd9Sstevel@tonic-gate static int 18147c478bd9Sstevel@tonic-gate slow_trylock(mutex_t *mp, ulwp_t *self) 18157c478bd9Sstevel@tonic-gate { 18167c478bd9Sstevel@tonic-gate if (MUTEX_OWNER(mp) == self || 18177c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mp) != 0) { 18187c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 18197c478bd9Sstevel@tonic-gate 18207c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 18217c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 18227c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 18237c478bd9Sstevel@tonic-gate } 18247c478bd9Sstevel@tonic-gate return (EBUSY); 18257c478bd9Sstevel@tonic-gate } 18267c478bd9Sstevel@tonic-gate return (0); 18277c478bd9Sstevel@tonic-gate } 18287c478bd9Sstevel@tonic-gate 18297c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_trylock = __mutex_trylock 18307c478bd9Sstevel@tonic-gate #pragma weak mutex_trylock = __mutex_trylock 18317c478bd9Sstevel@tonic-gate #pragma weak _mutex_trylock = __mutex_trylock 18327c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_trylock = __mutex_trylock 18337c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_trylock = __mutex_trylock 18347c478bd9Sstevel@tonic-gate int 18357c478bd9Sstevel@tonic-gate __mutex_trylock(mutex_t *mp) 18367c478bd9Sstevel@tonic-gate { 18377c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 18387c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 18397c478bd9Sstevel@tonic-gate uberflags_t *gflags; 18407c478bd9Sstevel@tonic-gate int mtype; 18417c478bd9Sstevel@tonic-gate 18427c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 18437c478bd9Sstevel@tonic-gate /* 18447c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 18457c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 18467c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 18477c478bd9Sstevel@tonic-gate * and the process has only a single thread. 18487c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 18497c478bd9Sstevel@tonic-gate */ 18507c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 18517c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 18527c478bd9Sstevel@tonic-gate /* 18537c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 18547c478bd9Sstevel@tonic-gate */ 18557c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 18567c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 18577c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 18587c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 18597c478bd9Sstevel@tonic-gate return (0); 18607c478bd9Sstevel@tonic-gate } 18617c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 18627c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 18637c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 18647c478bd9Sstevel@tonic-gate return (EAGAIN); 18657c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 18667c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 18677c478bd9Sstevel@tonic-gate 1, 0); 18687c478bd9Sstevel@tonic-gate return (0); 18697c478bd9Sstevel@tonic-gate } 18707c478bd9Sstevel@tonic-gate return (EDEADLK); /* LOCK_ERRORCHECK */ 18717c478bd9Sstevel@tonic-gate } 18727c478bd9Sstevel@tonic-gate return (EBUSY); 18737c478bd9Sstevel@tonic-gate } 18747c478bd9Sstevel@tonic-gate 18757c478bd9Sstevel@tonic-gate /* 18767c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 18777c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 18787c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 18797c478bd9Sstevel@tonic-gate */ 18807c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 18817c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 18827c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 18837c478bd9Sstevel@tonic-gate 18847c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 18857c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY)); 18867c478bd9Sstevel@tonic-gate 18877c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 18887c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 18897c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 18907c478bd9Sstevel@tonic-gate return (0); 18917c478bd9Sstevel@tonic-gate } 18927c478bd9Sstevel@tonic-gate 18937c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 18947c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 18957c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 18967c478bd9Sstevel@tonic-gate return (EAGAIN); 18977c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 18987c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 18997c478bd9Sstevel@tonic-gate 1, 0); 19007c478bd9Sstevel@tonic-gate return (0); 19017c478bd9Sstevel@tonic-gate } 19027c478bd9Sstevel@tonic-gate return (EBUSY); /* LOCK_ERRORCHECK */ 19037c478bd9Sstevel@tonic-gate } 19047c478bd9Sstevel@tonic-gate 19057c478bd9Sstevel@tonic-gate return (slow_trylock(mp, self)); 19067c478bd9Sstevel@tonic-gate } 19077c478bd9Sstevel@tonic-gate 19087c478bd9Sstevel@tonic-gate /* else do it the long way */ 19097c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, NULL, MUTEX_TRY)); 19107c478bd9Sstevel@tonic-gate } 19117c478bd9Sstevel@tonic-gate 19127c478bd9Sstevel@tonic-gate int 19137c478bd9Sstevel@tonic-gate mutex_unlock_internal(mutex_t *mp) 19147c478bd9Sstevel@tonic-gate { 19157c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 19167c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 19177c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 19187c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp; 19197c478bd9Sstevel@tonic-gate int error; 19207c478bd9Sstevel@tonic-gate lwpid_t lwpid; 19217c478bd9Sstevel@tonic-gate 19227c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp)) 19237c478bd9Sstevel@tonic-gate return (EPERM); 19247c478bd9Sstevel@tonic-gate 19257c478bd9Sstevel@tonic-gate if (self->ul_error_detection && !mutex_is_held(mp)) 19267c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_unlock", NULL, NULL); 19277c478bd9Sstevel@tonic-gate 19287c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 19297c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 19307c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 19317c478bd9Sstevel@tonic-gate return (0); 19327c478bd9Sstevel@tonic-gate } 19337c478bd9Sstevel@tonic-gate 19347c478bd9Sstevel@tonic-gate if ((msp = MUTEX_STATS(mp, udp)) != NULL) 19357c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 19367c478bd9Sstevel@tonic-gate 19377c478bd9Sstevel@tonic-gate if (mtype & 19387c478bd9Sstevel@tonic-gate (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) { 19397c478bd9Sstevel@tonic-gate no_preempt(self); 19407c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 19417c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 19427c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 19437c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) { 19447c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 19457c478bd9Sstevel@tonic-gate error = ___lwp_mutex_unlock(mp); 19467c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS_ROBUST) { 19477c478bd9Sstevel@tonic-gate error = ___lwp_mutex_unlock(mp); 19487c478bd9Sstevel@tonic-gate } else { 19497c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) 19507c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 19517c478bd9Sstevel@tonic-gate error = 0; 19527c478bd9Sstevel@tonic-gate } 19537c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_PROTECT) { 19547c478bd9Sstevel@tonic-gate if (_ceil_mylist_del(mp)) 19557c478bd9Sstevel@tonic-gate _ceil_prio_waive(); 19567c478bd9Sstevel@tonic-gate } 19577c478bd9Sstevel@tonic-gate preempt(self); 19587c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) { 19597c478bd9Sstevel@tonic-gate if (mp->mutex_lockword & WAITERMASK) 19607c478bd9Sstevel@tonic-gate mutex_unlock_process(mp); 19617c478bd9Sstevel@tonic-gate else { 19627c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 19637c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 19647c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 19657c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) { 19667c478bd9Sstevel@tonic-gate no_preempt(self); 19677c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 19687c478bd9Sstevel@tonic-gate preempt(self); 19697c478bd9Sstevel@tonic-gate } 19707c478bd9Sstevel@tonic-gate } 19717c478bd9Sstevel@tonic-gate error = 0; 19727c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 19737c478bd9Sstevel@tonic-gate if ((lwpid = mutex_unlock_queue(mp)) != 0) { 19747c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 19757c478bd9Sstevel@tonic-gate preempt(self); 19767c478bd9Sstevel@tonic-gate } 19777c478bd9Sstevel@tonic-gate error = 0; 19787c478bd9Sstevel@tonic-gate } 19797c478bd9Sstevel@tonic-gate 19807c478bd9Sstevel@tonic-gate return (error); 19817c478bd9Sstevel@tonic-gate } 19827c478bd9Sstevel@tonic-gate 19837c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_unlock = __mutex_unlock 19847c478bd9Sstevel@tonic-gate #pragma weak mutex_unlock = __mutex_unlock 19857c478bd9Sstevel@tonic-gate #pragma weak _mutex_unlock = __mutex_unlock 19867c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_unlock = __mutex_unlock 19877c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_unlock = __mutex_unlock 19887c478bd9Sstevel@tonic-gate int 19897c478bd9Sstevel@tonic-gate __mutex_unlock(mutex_t *mp) 19907c478bd9Sstevel@tonic-gate { 19917c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 19927c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 19937c478bd9Sstevel@tonic-gate uberflags_t *gflags; 19947c478bd9Sstevel@tonic-gate lwpid_t lwpid; 19957c478bd9Sstevel@tonic-gate int mtype; 19967c478bd9Sstevel@tonic-gate short el; 19977c478bd9Sstevel@tonic-gate 19987c478bd9Sstevel@tonic-gate /* 19997c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 20007c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 20017c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 20027c478bd9Sstevel@tonic-gate * and the process has only a single thread. 20037c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 20047c478bd9Sstevel@tonic-gate */ 20057c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 20067c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 20077c478bd9Sstevel@tonic-gate if (mtype) { 20087c478bd9Sstevel@tonic-gate /* 20097c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 20107c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 20117c478bd9Sstevel@tonic-gate */ 20127c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 20137c478bd9Sstevel@tonic-gate return (EPERM); 20147c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 20157c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 20167c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 20177c478bd9Sstevel@tonic-gate return (0); 20187c478bd9Sstevel@tonic-gate } 20197c478bd9Sstevel@tonic-gate } 20207c478bd9Sstevel@tonic-gate /* 20217c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 20227c478bd9Sstevel@tonic-gate * Also, there can be no waiters. 20237c478bd9Sstevel@tonic-gate */ 20247c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 20257c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 20267c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 20277c478bd9Sstevel@tonic-gate return (0); 20287c478bd9Sstevel@tonic-gate } 20297c478bd9Sstevel@tonic-gate 20307c478bd9Sstevel@tonic-gate /* 20317c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 20327c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 20337c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 20347c478bd9Sstevel@tonic-gate */ 20357c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL) { 20367c478bd9Sstevel@tonic-gate if (((el = gflags->uf_trs_ted) | mtype) == 0) { 20377c478bd9Sstevel@tonic-gate fast_unlock: 20387c478bd9Sstevel@tonic-gate if (!(mp->mutex_lockword & WAITERMASK)) { 20397c478bd9Sstevel@tonic-gate /* no waiter exists right now */ 20407c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 20417c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 20427c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & 20437c478bd9Sstevel@tonic-gate WAITERMASK) { 20447c478bd9Sstevel@tonic-gate /* a waiter suddenly appeared */ 20457c478bd9Sstevel@tonic-gate no_preempt(self); 20467c478bd9Sstevel@tonic-gate if ((lwpid = mutex_wakeup(mp)) != 0) 20477c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 20487c478bd9Sstevel@tonic-gate preempt(self); 20497c478bd9Sstevel@tonic-gate } 20507c478bd9Sstevel@tonic-gate } else if ((lwpid = mutex_unlock_queue(mp)) != 0) { 20517c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 20527c478bd9Sstevel@tonic-gate preempt(self); 20537c478bd9Sstevel@tonic-gate } 20547c478bd9Sstevel@tonic-gate return (0); 20557c478bd9Sstevel@tonic-gate } 20567c478bd9Sstevel@tonic-gate if (el) /* error detection or lock statistics */ 20577c478bd9Sstevel@tonic-gate goto slow_unlock; 20587c478bd9Sstevel@tonic-gate if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 20597c478bd9Sstevel@tonic-gate /* 20607c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 20617c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 20627c478bd9Sstevel@tonic-gate */ 20637c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 20647c478bd9Sstevel@tonic-gate return (EPERM); 20657c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 20667c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 20677c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 20687c478bd9Sstevel@tonic-gate return (0); 20697c478bd9Sstevel@tonic-gate } 20707c478bd9Sstevel@tonic-gate goto fast_unlock; 20717c478bd9Sstevel@tonic-gate } 20727c478bd9Sstevel@tonic-gate if ((mtype & 20737c478bd9Sstevel@tonic-gate ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 20747c478bd9Sstevel@tonic-gate /* 20757c478bd9Sstevel@tonic-gate * At this point we know that zero, one, or both of the 20767c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and 20777c478bd9Sstevel@tonic-gate * that the USYNC_PROCESS flag is set. 20787c478bd9Sstevel@tonic-gate */ 20797c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp)) 20807c478bd9Sstevel@tonic-gate return (EPERM); 20817c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 20827c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 20837c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 20847c478bd9Sstevel@tonic-gate return (0); 20857c478bd9Sstevel@tonic-gate } 20867c478bd9Sstevel@tonic-gate if (mp->mutex_lockword & WAITERMASK) 20877c478bd9Sstevel@tonic-gate mutex_unlock_process(mp); 20887c478bd9Sstevel@tonic-gate else { 20897c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 20907c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 20917c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 20927c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & 20937c478bd9Sstevel@tonic-gate WAITERMASK) { 20947c478bd9Sstevel@tonic-gate no_preempt(self); 20957c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 20967c478bd9Sstevel@tonic-gate preempt(self); 20977c478bd9Sstevel@tonic-gate } 20987c478bd9Sstevel@tonic-gate } 20997c478bd9Sstevel@tonic-gate return (0); 21007c478bd9Sstevel@tonic-gate } 21017c478bd9Sstevel@tonic-gate } 21027c478bd9Sstevel@tonic-gate 21037c478bd9Sstevel@tonic-gate /* else do it the long way */ 21047c478bd9Sstevel@tonic-gate slow_unlock: 21057c478bd9Sstevel@tonic-gate return (mutex_unlock_internal(mp)); 21067c478bd9Sstevel@tonic-gate } 21077c478bd9Sstevel@tonic-gate 21087c478bd9Sstevel@tonic-gate /* 21097c478bd9Sstevel@tonic-gate * Internally to the library, almost all mutex lock/unlock actions 21107c478bd9Sstevel@tonic-gate * go through these lmutex_ functions, to protect critical regions. 21117c478bd9Sstevel@tonic-gate * We replicate a bit of code from __mutex_lock() and __mutex_unlock() 21127c478bd9Sstevel@tonic-gate * to make these functions faster since we know that the mutex type 21137c478bd9Sstevel@tonic-gate * of all internal locks is USYNC_THREAD. We also know that internal 21147c478bd9Sstevel@tonic-gate * locking can never fail, so we panic if it does. 21157c478bd9Sstevel@tonic-gate */ 21167c478bd9Sstevel@tonic-gate void 21177c478bd9Sstevel@tonic-gate lmutex_lock(mutex_t *mp) 21187c478bd9Sstevel@tonic-gate { 21197c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 21207c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 21217c478bd9Sstevel@tonic-gate 21227c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 21237c478bd9Sstevel@tonic-gate 21247c478bd9Sstevel@tonic-gate enter_critical(self); 21257c478bd9Sstevel@tonic-gate /* 21267c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 21277c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 21287c478bd9Sstevel@tonic-gate */ 21297c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 21307c478bd9Sstevel@tonic-gate /* 21317c478bd9Sstevel@tonic-gate * Only one thread exists; the mutex must be free. 21327c478bd9Sstevel@tonic-gate */ 21337c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_lockw == 0); 21347c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 21357c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 21367c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 21377c478bd9Sstevel@tonic-gate } else { 21387c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 21397c478bd9Sstevel@tonic-gate 21407c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 21417c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 21427c478bd9Sstevel@tonic-gate 21437c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 21447c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 21457c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 21467c478bd9Sstevel@tonic-gate } else if (mutex_trylock_adaptive(mp) != 0) { 21477c478bd9Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL); 21487c478bd9Sstevel@tonic-gate } 21497c478bd9Sstevel@tonic-gate 21507c478bd9Sstevel@tonic-gate if (msp) 21517c478bd9Sstevel@tonic-gate record_begin_hold(msp); 21527c478bd9Sstevel@tonic-gate } 21537c478bd9Sstevel@tonic-gate } 21547c478bd9Sstevel@tonic-gate 21557c478bd9Sstevel@tonic-gate void 21567c478bd9Sstevel@tonic-gate lmutex_unlock(mutex_t *mp) 21577c478bd9Sstevel@tonic-gate { 21587c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 21597c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 21607c478bd9Sstevel@tonic-gate 21617c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 21627c478bd9Sstevel@tonic-gate 21637c478bd9Sstevel@tonic-gate /* 21647c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 21657c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 21667c478bd9Sstevel@tonic-gate */ 21677c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 21687c478bd9Sstevel@tonic-gate /* 21697c478bd9Sstevel@tonic-gate * Only one thread exists so there can be no waiters. 21707c478bd9Sstevel@tonic-gate */ 21717c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 21727c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 21737c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 21747c478bd9Sstevel@tonic-gate } else { 21757c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 21767c478bd9Sstevel@tonic-gate lwpid_t lwpid; 21777c478bd9Sstevel@tonic-gate 21787c478bd9Sstevel@tonic-gate if (msp) 21797c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 21807c478bd9Sstevel@tonic-gate if ((lwpid = mutex_unlock_queue(mp)) != 0) { 21817c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 21827c478bd9Sstevel@tonic-gate preempt(self); 21837c478bd9Sstevel@tonic-gate } 21847c478bd9Sstevel@tonic-gate } 21857c478bd9Sstevel@tonic-gate exit_critical(self); 21867c478bd9Sstevel@tonic-gate } 21877c478bd9Sstevel@tonic-gate 21887c478bd9Sstevel@tonic-gate static int 21897c478bd9Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg) 21907c478bd9Sstevel@tonic-gate { 21917c478bd9Sstevel@tonic-gate /* 21927c478bd9Sstevel@tonic-gate * There is an inherent data race in the current ownership design. 21937c478bd9Sstevel@tonic-gate * The mutex_owner and mutex_ownerpid fields cannot be set or tested 21947c478bd9Sstevel@tonic-gate * atomically as a pair. The original implementation tested each 21957c478bd9Sstevel@tonic-gate * field just once. This was exposed to trivial false positives in 21967c478bd9Sstevel@tonic-gate * the case of multiple multithreaded processes with thread addresses 21977c478bd9Sstevel@tonic-gate * in common. To close the window to an acceptable level we now use a 21987c478bd9Sstevel@tonic-gate * sequence of five tests: pid-thr-pid-thr-pid. This ensures that any 21997c478bd9Sstevel@tonic-gate * single interruption will still leave one uninterrupted sequence of 22007c478bd9Sstevel@tonic-gate * pid-thr-pid tests intact. 22017c478bd9Sstevel@tonic-gate * 22027c478bd9Sstevel@tonic-gate * It is assumed that all updates are always ordered thr-pid and that 22037c478bd9Sstevel@tonic-gate * we have TSO hardware. 22047c478bd9Sstevel@tonic-gate */ 22057c478bd9Sstevel@tonic-gate volatile mutex_t *mp = (volatile mutex_t *)mparg; 22067c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 22077c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 22087c478bd9Sstevel@tonic-gate 22097c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid != udp->pid) 22107c478bd9Sstevel@tonic-gate return (0); 22117c478bd9Sstevel@tonic-gate 22127c478bd9Sstevel@tonic-gate if (!MUTEX_OWNED(mp, self)) 22137c478bd9Sstevel@tonic-gate return (0); 22147c478bd9Sstevel@tonic-gate 22157c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid != udp->pid) 22167c478bd9Sstevel@tonic-gate return (0); 22177c478bd9Sstevel@tonic-gate 22187c478bd9Sstevel@tonic-gate if (!MUTEX_OWNED(mp, self)) 22197c478bd9Sstevel@tonic-gate return (0); 22207c478bd9Sstevel@tonic-gate 22217c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid != udp->pid) 22227c478bd9Sstevel@tonic-gate return (0); 22237c478bd9Sstevel@tonic-gate 22247c478bd9Sstevel@tonic-gate return (1); 22257c478bd9Sstevel@tonic-gate } 22267c478bd9Sstevel@tonic-gate 22277c478bd9Sstevel@tonic-gate /* 22287c478bd9Sstevel@tonic-gate * Some crufty old programs define their own version of _mutex_held() 22297c478bd9Sstevel@tonic-gate * to be simply return(1). This breaks internal libc logic, so we 22307c478bd9Sstevel@tonic-gate * define a private version for exclusive use by libc, mutex_is_held(), 22317c478bd9Sstevel@tonic-gate * and also a new public function, __mutex_held(), to be used in new 22327c478bd9Sstevel@tonic-gate * code to circumvent these crufty old programs. 22337c478bd9Sstevel@tonic-gate */ 22347c478bd9Sstevel@tonic-gate #pragma weak mutex_held = mutex_is_held 22357c478bd9Sstevel@tonic-gate #pragma weak _mutex_held = mutex_is_held 22367c478bd9Sstevel@tonic-gate #pragma weak __mutex_held = mutex_is_held 22377c478bd9Sstevel@tonic-gate int 22387c478bd9Sstevel@tonic-gate mutex_is_held(mutex_t *mp) 22397c478bd9Sstevel@tonic-gate { 22407c478bd9Sstevel@tonic-gate if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) 22417c478bd9Sstevel@tonic-gate return (shared_mutex_held(mp)); 22427c478bd9Sstevel@tonic-gate return (MUTEX_OWNED(mp, curthread)); 22437c478bd9Sstevel@tonic-gate } 22447c478bd9Sstevel@tonic-gate 22457c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_destroy = __mutex_destroy 22467c478bd9Sstevel@tonic-gate #pragma weak mutex_destroy = __mutex_destroy 22477c478bd9Sstevel@tonic-gate #pragma weak _mutex_destroy = __mutex_destroy 22487c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_destroy = __mutex_destroy 22497c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_destroy = __mutex_destroy 22507c478bd9Sstevel@tonic-gate int 22517c478bd9Sstevel@tonic-gate __mutex_destroy(mutex_t *mp) 22527c478bd9Sstevel@tonic-gate { 22537c478bd9Sstevel@tonic-gate mp->mutex_magic = 0; 22547c478bd9Sstevel@tonic-gate mp->mutex_flag &= ~LOCK_INITED; 22557c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(mp); 22567c478bd9Sstevel@tonic-gate return (0); 22577c478bd9Sstevel@tonic-gate } 22587c478bd9Sstevel@tonic-gate 22597c478bd9Sstevel@tonic-gate /* 22607c478bd9Sstevel@tonic-gate * Spin locks are separate from ordinary mutexes, 22617c478bd9Sstevel@tonic-gate * but we use the same data structure for them. 22627c478bd9Sstevel@tonic-gate */ 22637c478bd9Sstevel@tonic-gate 22647c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_init = _pthread_spin_init 22657c478bd9Sstevel@tonic-gate int 22667c478bd9Sstevel@tonic-gate _pthread_spin_init(pthread_spinlock_t *lock, int pshared) 22677c478bd9Sstevel@tonic-gate { 22687c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 22697c478bd9Sstevel@tonic-gate 22707c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 22717c478bd9Sstevel@tonic-gate if (pshared == PTHREAD_PROCESS_SHARED) 22727c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_PROCESS; 22737c478bd9Sstevel@tonic-gate else 22747c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_THREAD; 22757c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 22767c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 22777c478bd9Sstevel@tonic-gate return (0); 22787c478bd9Sstevel@tonic-gate } 22797c478bd9Sstevel@tonic-gate 22807c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_destroy = _pthread_spin_destroy 22817c478bd9Sstevel@tonic-gate int 22827c478bd9Sstevel@tonic-gate _pthread_spin_destroy(pthread_spinlock_t *lock) 22837c478bd9Sstevel@tonic-gate { 22847c478bd9Sstevel@tonic-gate (void) _memset(lock, 0, sizeof (*lock)); 22857c478bd9Sstevel@tonic-gate return (0); 22867c478bd9Sstevel@tonic-gate } 22877c478bd9Sstevel@tonic-gate 22887c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_trylock = _pthread_spin_trylock 22897c478bd9Sstevel@tonic-gate int 22907c478bd9Sstevel@tonic-gate _pthread_spin_trylock(pthread_spinlock_t *lock) 22917c478bd9Sstevel@tonic-gate { 22927c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 22937c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 22947c478bd9Sstevel@tonic-gate int error = 0; 22957c478bd9Sstevel@tonic-gate 22967c478bd9Sstevel@tonic-gate no_preempt(self); 22977c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) != 0) 22987c478bd9Sstevel@tonic-gate error = EBUSY; 22997c478bd9Sstevel@tonic-gate else { 23007c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 23017c478bd9Sstevel@tonic-gate if (mp->mutex_type == USYNC_PROCESS) 23027c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = self->ul_uberdata->pid; 23037c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 23047c478bd9Sstevel@tonic-gate } 23057c478bd9Sstevel@tonic-gate preempt(self); 23067c478bd9Sstevel@tonic-gate return (error); 23077c478bd9Sstevel@tonic-gate } 23087c478bd9Sstevel@tonic-gate 23097c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_lock = _pthread_spin_lock 23107c478bd9Sstevel@tonic-gate int 23117c478bd9Sstevel@tonic-gate _pthread_spin_lock(pthread_spinlock_t *lock) 23127c478bd9Sstevel@tonic-gate { 23137c478bd9Sstevel@tonic-gate volatile uint8_t *lockp = 23147c478bd9Sstevel@tonic-gate (volatile uint8_t *)&((mutex_t *)lock)->mutex_lockw; 23157c478bd9Sstevel@tonic-gate 23167c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 23177c478bd9Sstevel@tonic-gate /* 23187c478bd9Sstevel@tonic-gate * We don't care whether the owner is running on a processor. 23197c478bd9Sstevel@tonic-gate * We just spin because that's what this interface requires. 23207c478bd9Sstevel@tonic-gate */ 23217c478bd9Sstevel@tonic-gate for (;;) { 23227c478bd9Sstevel@tonic-gate if (*lockp == 0) { /* lock byte appears to be clear */ 23237c478bd9Sstevel@tonic-gate if (_pthread_spin_trylock(lock) == 0) 23247c478bd9Sstevel@tonic-gate return (0); 23257c478bd9Sstevel@tonic-gate } 23267c478bd9Sstevel@tonic-gate SMT_PAUSE(); 23277c478bd9Sstevel@tonic-gate } 23287c478bd9Sstevel@tonic-gate } 23297c478bd9Sstevel@tonic-gate 23307c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_unlock = _pthread_spin_unlock 23317c478bd9Sstevel@tonic-gate int 23327c478bd9Sstevel@tonic-gate _pthread_spin_unlock(pthread_spinlock_t *lock) 23337c478bd9Sstevel@tonic-gate { 23347c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 23357c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 23367c478bd9Sstevel@tonic-gate 23377c478bd9Sstevel@tonic-gate no_preempt(self); 23387c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 23397c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 23407c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 23417c478bd9Sstevel@tonic-gate (void) swap32(&mp->mutex_lockword, 0); 23427c478bd9Sstevel@tonic-gate preempt(self); 23437c478bd9Sstevel@tonic-gate return (0); 23447c478bd9Sstevel@tonic-gate } 23457c478bd9Sstevel@tonic-gate 23467c478bd9Sstevel@tonic-gate #pragma weak cond_init = _cond_init 23477c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 23487c478bd9Sstevel@tonic-gate int 23497c478bd9Sstevel@tonic-gate _cond_init(cond_t *cvp, int type, void *arg) 23507c478bd9Sstevel@tonic-gate { 23517c478bd9Sstevel@tonic-gate if (type != USYNC_THREAD && type != USYNC_PROCESS) 23527c478bd9Sstevel@tonic-gate return (EINVAL); 23537c478bd9Sstevel@tonic-gate (void) _memset(cvp, 0, sizeof (*cvp)); 23547c478bd9Sstevel@tonic-gate cvp->cond_type = (uint16_t)type; 23557c478bd9Sstevel@tonic-gate cvp->cond_magic = COND_MAGIC; 23567c478bd9Sstevel@tonic-gate return (0); 23577c478bd9Sstevel@tonic-gate } 23587c478bd9Sstevel@tonic-gate 23597c478bd9Sstevel@tonic-gate /* 23607c478bd9Sstevel@tonic-gate * cond_sleep_queue(): utility function for cond_wait_queue(). 23617c478bd9Sstevel@tonic-gate * 23627c478bd9Sstevel@tonic-gate * Go to sleep on a condvar sleep queue, expect to be waked up 23637c478bd9Sstevel@tonic-gate * by someone calling cond_signal() or cond_broadcast() or due 23647c478bd9Sstevel@tonic-gate * to receiving a UNIX signal or being cancelled, or just simply 23657c478bd9Sstevel@tonic-gate * due to a spurious wakeup (like someome calling forkall()). 23667c478bd9Sstevel@tonic-gate * 23677c478bd9Sstevel@tonic-gate * The associated mutex is *not* reacquired before returning. 23687c478bd9Sstevel@tonic-gate * That must be done by the caller of cond_sleep_queue(). 23697c478bd9Sstevel@tonic-gate */ 23707c478bd9Sstevel@tonic-gate int 23717c478bd9Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 23727c478bd9Sstevel@tonic-gate { 23737c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 23747c478bd9Sstevel@tonic-gate queue_head_t *qp; 23757c478bd9Sstevel@tonic-gate queue_head_t *mqp; 23767c478bd9Sstevel@tonic-gate lwpid_t lwpid; 23777c478bd9Sstevel@tonic-gate int signalled; 23787c478bd9Sstevel@tonic-gate int error; 23797c478bd9Sstevel@tonic-gate 23807c478bd9Sstevel@tonic-gate /* 23817c478bd9Sstevel@tonic-gate * Put ourself on the CV sleep queue, unlock the mutex, then 23827c478bd9Sstevel@tonic-gate * park ourself and unpark a candidate lwp to grab the mutex. 23837c478bd9Sstevel@tonic-gate * We must go onto the CV sleep queue before dropping the 23847c478bd9Sstevel@tonic-gate * mutex in order to guarantee atomicity of the operation. 23857c478bd9Sstevel@tonic-gate */ 23867c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 23877c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 23887c478bd9Sstevel@tonic-gate enqueue(qp, self, cvp, CV); 23897c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 1; 23907c478bd9Sstevel@tonic-gate self->ul_cvmutex = mp; 23917c478bd9Sstevel@tonic-gate self->ul_cv_wake = (tsp != NULL); 23927c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 23937c478bd9Sstevel@tonic-gate lwpid = mutex_unlock_queue(mp); 23947c478bd9Sstevel@tonic-gate for (;;) { 23957c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 23967c478bd9Sstevel@tonic-gate queue_unlock(qp); 23977c478bd9Sstevel@tonic-gate if (lwpid != 0) { 23987c478bd9Sstevel@tonic-gate lwpid = preempt_unpark(self, lwpid); 23997c478bd9Sstevel@tonic-gate preempt(self); 24007c478bd9Sstevel@tonic-gate } 24017c478bd9Sstevel@tonic-gate /* 24027c478bd9Sstevel@tonic-gate * We may have a deferred signal present, 24037c478bd9Sstevel@tonic-gate * in which case we should return EINTR. 24047c478bd9Sstevel@tonic-gate * Also, we may have received a SIGCANCEL; if so 24057c478bd9Sstevel@tonic-gate * and we are cancelable we should return EINTR. 24067c478bd9Sstevel@tonic-gate * We force an immediate EINTR return from 24077c478bd9Sstevel@tonic-gate * __lwp_park() by turning our parking flag off. 24087c478bd9Sstevel@tonic-gate */ 24097c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 24107c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 24117c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 24127c478bd9Sstevel@tonic-gate /* 24137c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 24147c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 24157c478bd9Sstevel@tonic-gate */ 24167c478bd9Sstevel@tonic-gate error = __lwp_park(tsp, lwpid); 24177c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 24187c478bd9Sstevel@tonic-gate lwpid = 0; /* unpark the other lwp only once */ 24197c478bd9Sstevel@tonic-gate /* 24207c478bd9Sstevel@tonic-gate * We were waked up by cond_signal(), cond_broadcast(), 24217c478bd9Sstevel@tonic-gate * by an interrupt or timeout (EINTR or ETIME), 24227c478bd9Sstevel@tonic-gate * or we may just have gotten a spurious wakeup. 24237c478bd9Sstevel@tonic-gate */ 24247c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 24257c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 24267c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) 24277c478bd9Sstevel@tonic-gate break; 24287c478bd9Sstevel@tonic-gate /* 24297c478bd9Sstevel@tonic-gate * We are on either the condvar sleep queue or the 24307c478bd9Sstevel@tonic-gate * mutex sleep queue. If we are on the mutex sleep 24317c478bd9Sstevel@tonic-gate * queue, continue sleeping. If we are on the condvar 24327c478bd9Sstevel@tonic-gate * sleep queue, break out of the sleep if we were 24337c478bd9Sstevel@tonic-gate * interrupted or we timed out (EINTR or ETIME). 24347c478bd9Sstevel@tonic-gate * Else this is a spurious wakeup; continue the loop. 24357c478bd9Sstevel@tonic-gate */ 24367c478bd9Sstevel@tonic-gate if (self->ul_sleepq == mqp) /* mutex queue */ 24377c478bd9Sstevel@tonic-gate tsp = NULL; 24387c478bd9Sstevel@tonic-gate else if (self->ul_sleepq == qp) { /* condvar queue */ 24397c478bd9Sstevel@tonic-gate if (error) { 24407c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = dequeue_self(qp, cvp); 24417c478bd9Sstevel@tonic-gate break; 24427c478bd9Sstevel@tonic-gate } 24437c478bd9Sstevel@tonic-gate /* 24447c478bd9Sstevel@tonic-gate * Else a spurious wakeup on the condvar queue. 24457c478bd9Sstevel@tonic-gate * __lwp_park() has already adjusted the timeout. 24467c478bd9Sstevel@tonic-gate */ 24477c478bd9Sstevel@tonic-gate } else { 24487c478bd9Sstevel@tonic-gate thr_panic("cond_sleep_queue(): thread not on queue"); 24497c478bd9Sstevel@tonic-gate } 24507c478bd9Sstevel@tonic-gate queue_unlock(mqp); 24517c478bd9Sstevel@tonic-gate } 24527c478bd9Sstevel@tonic-gate 24537c478bd9Sstevel@tonic-gate self->ul_sp = 0; 24547c478bd9Sstevel@tonic-gate ASSERT(self->ul_cvmutex == NULL && self->ul_cv_wake == 0); 24557c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 24567c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 24577c478bd9Sstevel@tonic-gate 24587c478bd9Sstevel@tonic-gate signalled = self->ul_signalled; 24597c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 24607c478bd9Sstevel@tonic-gate queue_unlock(qp); 24617c478bd9Sstevel@tonic-gate queue_unlock(mqp); 24627c478bd9Sstevel@tonic-gate 24637c478bd9Sstevel@tonic-gate /* 24647c478bd9Sstevel@tonic-gate * If we were concurrently cond_signal()d and any of: 24657c478bd9Sstevel@tonic-gate * received a UNIX signal, were cancelled, or got a timeout, 24667c478bd9Sstevel@tonic-gate * then perform another cond_signal() to avoid consuming it. 24677c478bd9Sstevel@tonic-gate */ 24687c478bd9Sstevel@tonic-gate if (error && signalled) 24697c478bd9Sstevel@tonic-gate (void) cond_signal_internal(cvp); 24707c478bd9Sstevel@tonic-gate 24717c478bd9Sstevel@tonic-gate return (error); 24727c478bd9Sstevel@tonic-gate } 24737c478bd9Sstevel@tonic-gate 24747c478bd9Sstevel@tonic-gate int 24757c478bd9Sstevel@tonic-gate cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp, 24767c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp) 24777c478bd9Sstevel@tonic-gate { 24787c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 24797c478bd9Sstevel@tonic-gate int error; 24807c478bd9Sstevel@tonic-gate 24817c478bd9Sstevel@tonic-gate /* 24827c478bd9Sstevel@tonic-gate * The old thread library was programmed to defer signals 24837c478bd9Sstevel@tonic-gate * while in cond_wait() so that the associated mutex would 24847c478bd9Sstevel@tonic-gate * be guaranteed to be held when the application signal 24857c478bd9Sstevel@tonic-gate * handler was invoked. 24867c478bd9Sstevel@tonic-gate * 24877c478bd9Sstevel@tonic-gate * We do not behave this way by default; the state of the 24887c478bd9Sstevel@tonic-gate * associated mutex in the signal handler is undefined. 24897c478bd9Sstevel@tonic-gate * 24907c478bd9Sstevel@tonic-gate * To accommodate applications that depend on the old 24917c478bd9Sstevel@tonic-gate * behavior, the _THREAD_COND_WAIT_DEFER environment 24927c478bd9Sstevel@tonic-gate * variable can be set to 1 and we will behave in the 24937c478bd9Sstevel@tonic-gate * old way with respect to cond_wait(). 24947c478bd9Sstevel@tonic-gate */ 24957c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 24967c478bd9Sstevel@tonic-gate sigoff(self); 24977c478bd9Sstevel@tonic-gate 24987c478bd9Sstevel@tonic-gate error = cond_sleep_queue(cvp, mp, tsp); 24997c478bd9Sstevel@tonic-gate 25007c478bd9Sstevel@tonic-gate /* 25017c478bd9Sstevel@tonic-gate * Reacquire the mutex. 25027c478bd9Sstevel@tonic-gate */ 25037c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 25047c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 25057c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 25067c478bd9Sstevel@tonic-gate } else if (mutex_trylock_adaptive(mp) != 0) { 25077c478bd9Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL); 25087c478bd9Sstevel@tonic-gate } 25097c478bd9Sstevel@tonic-gate 25107c478bd9Sstevel@tonic-gate if (msp) 25117c478bd9Sstevel@tonic-gate record_begin_hold(msp); 25127c478bd9Sstevel@tonic-gate 25137c478bd9Sstevel@tonic-gate /* 25147c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 25157c478bd9Sstevel@tonic-gate */ 25167c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 25177c478bd9Sstevel@tonic-gate sigon(self); 25187c478bd9Sstevel@tonic-gate 25197c478bd9Sstevel@tonic-gate return (error); 25207c478bd9Sstevel@tonic-gate } 25217c478bd9Sstevel@tonic-gate 25227c478bd9Sstevel@tonic-gate /* 25237c478bd9Sstevel@tonic-gate * cond_sleep_kernel(): utility function for cond_wait_kernel(). 25247c478bd9Sstevel@tonic-gate * See the comment ahead of cond_sleep_queue(), above. 25257c478bd9Sstevel@tonic-gate */ 25267c478bd9Sstevel@tonic-gate int 25277c478bd9Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 25287c478bd9Sstevel@tonic-gate { 25297c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 25307c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 25317c478bd9Sstevel@tonic-gate int error; 25327c478bd9Sstevel@tonic-gate 25337c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_PROTECT) { 25347c478bd9Sstevel@tonic-gate if (_ceil_mylist_del(mp)) 25357c478bd9Sstevel@tonic-gate _ceil_prio_waive(); 25367c478bd9Sstevel@tonic-gate } 25377c478bd9Sstevel@tonic-gate 25387c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 25397c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 25407c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 25417c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 25427c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) 25437c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 25447c478bd9Sstevel@tonic-gate /* 25457c478bd9Sstevel@tonic-gate * ___lwp_cond_wait() returns immediately with EINTR if 25467c478bd9Sstevel@tonic-gate * set_parking_flag(self,0) is called on this lwp before it 25477c478bd9Sstevel@tonic-gate * goes to sleep in the kernel. sigacthandler() calls this 25487c478bd9Sstevel@tonic-gate * when a deferred signal is noted. This assures that we don't 25497c478bd9Sstevel@tonic-gate * get stuck in ___lwp_cond_wait() with all signals blocked 25507c478bd9Sstevel@tonic-gate * due to taking a deferred signal before going to sleep. 25517c478bd9Sstevel@tonic-gate */ 25527c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 25537c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 25547c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 25557c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 25567c478bd9Sstevel@tonic-gate error = ___lwp_cond_wait(cvp, mp, tsp, 1); 25577c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 25587c478bd9Sstevel@tonic-gate self->ul_sp = 0; 25597c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 25607c478bd9Sstevel@tonic-gate return (error); 25617c478bd9Sstevel@tonic-gate } 25627c478bd9Sstevel@tonic-gate 25637c478bd9Sstevel@tonic-gate int 25647c478bd9Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 25657c478bd9Sstevel@tonic-gate { 25667c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 25677c478bd9Sstevel@tonic-gate int error; 25687c478bd9Sstevel@tonic-gate int merror; 25697c478bd9Sstevel@tonic-gate 25707c478bd9Sstevel@tonic-gate /* 25717c478bd9Sstevel@tonic-gate * See the large comment in cond_wait_queue(), above. 25727c478bd9Sstevel@tonic-gate */ 25737c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 25747c478bd9Sstevel@tonic-gate sigoff(self); 25757c478bd9Sstevel@tonic-gate 25767c478bd9Sstevel@tonic-gate error = cond_sleep_kernel(cvp, mp, tsp); 25777c478bd9Sstevel@tonic-gate 25787c478bd9Sstevel@tonic-gate /* 25797c478bd9Sstevel@tonic-gate * Override the return code from ___lwp_cond_wait() 25807c478bd9Sstevel@tonic-gate * with any non-zero return code from mutex_lock(). 25817c478bd9Sstevel@tonic-gate * This addresses robust lock failures in particular; 25827c478bd9Sstevel@tonic-gate * the caller must see the EOWNERDEAD or ENOTRECOVERABLE 25837c478bd9Sstevel@tonic-gate * errors in order to take corrective action. 25847c478bd9Sstevel@tonic-gate */ 25857c478bd9Sstevel@tonic-gate if ((merror = _private_mutex_lock(mp)) != 0) 25867c478bd9Sstevel@tonic-gate error = merror; 25877c478bd9Sstevel@tonic-gate 25887c478bd9Sstevel@tonic-gate /* 25897c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 25907c478bd9Sstevel@tonic-gate */ 25917c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 25927c478bd9Sstevel@tonic-gate sigon(self); 25937c478bd9Sstevel@tonic-gate 25947c478bd9Sstevel@tonic-gate return (error); 25957c478bd9Sstevel@tonic-gate } 25967c478bd9Sstevel@tonic-gate 25977c478bd9Sstevel@tonic-gate /* 25987c478bd9Sstevel@tonic-gate * Common code for _cond_wait() and _cond_timedwait() 25997c478bd9Sstevel@tonic-gate */ 26007c478bd9Sstevel@tonic-gate int 26017c478bd9Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 26027c478bd9Sstevel@tonic-gate { 26037c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 26047c478bd9Sstevel@tonic-gate hrtime_t begin_sleep = 0; 26057c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 26067c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 26077c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 26087c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 26097c478bd9Sstevel@tonic-gate uint8_t rcount; 26107c478bd9Sstevel@tonic-gate int error = 0; 26117c478bd9Sstevel@tonic-gate 26127c478bd9Sstevel@tonic-gate /* 26137c478bd9Sstevel@tonic-gate * The SUSV3 Posix spec for pthread_cond_timedwait() states: 26147c478bd9Sstevel@tonic-gate * Except in the case of [ETIMEDOUT], all these error checks 26157c478bd9Sstevel@tonic-gate * shall act as if they were performed immediately at the 26167c478bd9Sstevel@tonic-gate * beginning of processing for the function and shall cause 26177c478bd9Sstevel@tonic-gate * an error return, in effect, prior to modifying the state 26187c478bd9Sstevel@tonic-gate * of the mutex specified by mutex or the condition variable 26197c478bd9Sstevel@tonic-gate * specified by cond. 26207c478bd9Sstevel@tonic-gate * Therefore, we must return EINVAL now if the timout is invalid. 26217c478bd9Sstevel@tonic-gate */ 26227c478bd9Sstevel@tonic-gate if (tsp != NULL && 26237c478bd9Sstevel@tonic-gate (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC)) 26247c478bd9Sstevel@tonic-gate return (EINVAL); 26257c478bd9Sstevel@tonic-gate 26267c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 26277c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 26287c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 26297c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 26307c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = cvp; 26317c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 26327c478bd9Sstevel@tonic-gate self->ul_sp = 0; 26337c478bd9Sstevel@tonic-gate } 26347c478bd9Sstevel@tonic-gate if (csp) { 26357c478bd9Sstevel@tonic-gate if (tsp) 26367c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait); 26377c478bd9Sstevel@tonic-gate else 26387c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_wait); 26397c478bd9Sstevel@tonic-gate } 26407c478bd9Sstevel@tonic-gate if (msp) 26417c478bd9Sstevel@tonic-gate begin_sleep = record_hold_time(msp); 26427c478bd9Sstevel@tonic-gate else if (csp) 26437c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 26447c478bd9Sstevel@tonic-gate 26457c478bd9Sstevel@tonic-gate if (self->ul_error_detection) { 26467c478bd9Sstevel@tonic-gate if (!mutex_is_held(mp)) 26477c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, NULL); 26487c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) 26497c478bd9Sstevel@tonic-gate lock_error(mp, "recursive mutex in cond_wait", 26507c478bd9Sstevel@tonic-gate cvp, NULL); 26517c478bd9Sstevel@tonic-gate if (cvp->cond_type & USYNC_PROCESS) { 26527c478bd9Sstevel@tonic-gate if (!(mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))) 26537c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 26547c478bd9Sstevel@tonic-gate "condvar process-shared, " 26557c478bd9Sstevel@tonic-gate "mutex process-private"); 26567c478bd9Sstevel@tonic-gate } else { 26577c478bd9Sstevel@tonic-gate if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) 26587c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 26597c478bd9Sstevel@tonic-gate "condvar process-private, " 26607c478bd9Sstevel@tonic-gate "mutex process-shared"); 26617c478bd9Sstevel@tonic-gate } 26627c478bd9Sstevel@tonic-gate } 26637c478bd9Sstevel@tonic-gate 26647c478bd9Sstevel@tonic-gate /* 26657c478bd9Sstevel@tonic-gate * We deal with recursive mutexes by completely 26667c478bd9Sstevel@tonic-gate * dropping the lock and restoring the recursion 26677c478bd9Sstevel@tonic-gate * count after waking up. This is arguably wrong, 26687c478bd9Sstevel@tonic-gate * but it obeys the principle of least astonishment. 26697c478bd9Sstevel@tonic-gate */ 26707c478bd9Sstevel@tonic-gate rcount = mp->mutex_rcount; 26717c478bd9Sstevel@tonic-gate mp->mutex_rcount = 0; 26727c478bd9Sstevel@tonic-gate if ((mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST | 26737c478bd9Sstevel@tonic-gate PTHREAD_PRIO_INHERIT | PTHREAD_PRIO_PROTECT)) | 26747c478bd9Sstevel@tonic-gate (cvp->cond_type & USYNC_PROCESS)) 26757c478bd9Sstevel@tonic-gate error = cond_wait_kernel(cvp, mp, tsp); 26767c478bd9Sstevel@tonic-gate else 26777c478bd9Sstevel@tonic-gate error = cond_wait_queue(cvp, mp, tsp, msp); 26787c478bd9Sstevel@tonic-gate mp->mutex_rcount = rcount; 26797c478bd9Sstevel@tonic-gate 26807c478bd9Sstevel@tonic-gate if (csp) { 26817c478bd9Sstevel@tonic-gate hrtime_t lapse = gethrtime() - begin_sleep; 26827c478bd9Sstevel@tonic-gate if (tsp == NULL) 26837c478bd9Sstevel@tonic-gate csp->cond_wait_sleep_time += lapse; 26847c478bd9Sstevel@tonic-gate else { 26857c478bd9Sstevel@tonic-gate csp->cond_timedwait_sleep_time += lapse; 26867c478bd9Sstevel@tonic-gate if (error == ETIME) 26877c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait_timeout); 26887c478bd9Sstevel@tonic-gate } 26897c478bd9Sstevel@tonic-gate } 26907c478bd9Sstevel@tonic-gate return (error); 26917c478bd9Sstevel@tonic-gate } 26927c478bd9Sstevel@tonic-gate 26937c478bd9Sstevel@tonic-gate /* 26947c478bd9Sstevel@tonic-gate * cond_wait() is a cancellation point but _cond_wait() is not. 26957c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 26967c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 26977c478bd9Sstevel@tonic-gate */ 26987c478bd9Sstevel@tonic-gate int 26997c478bd9Sstevel@tonic-gate _cond_wait(cond_t *cvp, mutex_t *mp) 27007c478bd9Sstevel@tonic-gate { 27017c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 27027c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 27037c478bd9Sstevel@tonic-gate uberflags_t *gflags; 27047c478bd9Sstevel@tonic-gate 27057c478bd9Sstevel@tonic-gate /* 27067c478bd9Sstevel@tonic-gate * Optimize the common case of USYNC_THREAD plus 27077c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, and no event tracing. 27087c478bd9Sstevel@tonic-gate */ 27097c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 27107c478bd9Sstevel@tonic-gate (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted | 27117c478bd9Sstevel@tonic-gate self->ul_td_events_enable | 27127c478bd9Sstevel@tonic-gate udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0) 27137c478bd9Sstevel@tonic-gate return (cond_wait_queue(cvp, mp, NULL, NULL)); 27147c478bd9Sstevel@tonic-gate 27157c478bd9Sstevel@tonic-gate /* 27167c478bd9Sstevel@tonic-gate * Else do it the long way. 27177c478bd9Sstevel@tonic-gate */ 27187c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, NULL)); 27197c478bd9Sstevel@tonic-gate } 27207c478bd9Sstevel@tonic-gate 27217c478bd9Sstevel@tonic-gate int 27227c478bd9Sstevel@tonic-gate cond_wait(cond_t *cvp, mutex_t *mp) 27237c478bd9Sstevel@tonic-gate { 27247c478bd9Sstevel@tonic-gate int error; 27257c478bd9Sstevel@tonic-gate 27267c478bd9Sstevel@tonic-gate _cancelon(); 27277c478bd9Sstevel@tonic-gate error = _cond_wait(cvp, mp); 27287c478bd9Sstevel@tonic-gate if (error == EINTR) 27297c478bd9Sstevel@tonic-gate _canceloff(); 27307c478bd9Sstevel@tonic-gate else 27317c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 27327c478bd9Sstevel@tonic-gate return (error); 27337c478bd9Sstevel@tonic-gate } 27347c478bd9Sstevel@tonic-gate 27357c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_wait = _pthread_cond_wait 27367c478bd9Sstevel@tonic-gate int 27377c478bd9Sstevel@tonic-gate _pthread_cond_wait(cond_t *cvp, mutex_t *mp) 27387c478bd9Sstevel@tonic-gate { 27397c478bd9Sstevel@tonic-gate int error; 27407c478bd9Sstevel@tonic-gate 27417c478bd9Sstevel@tonic-gate error = cond_wait(cvp, mp); 27427c478bd9Sstevel@tonic-gate return ((error == EINTR)? 0 : error); 27437c478bd9Sstevel@tonic-gate } 27447c478bd9Sstevel@tonic-gate 27457c478bd9Sstevel@tonic-gate /* 27467c478bd9Sstevel@tonic-gate * cond_timedwait() is a cancellation point but _cond_timedwait() is not. 27477c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 27487c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 27497c478bd9Sstevel@tonic-gate */ 27507c478bd9Sstevel@tonic-gate int 27517c478bd9Sstevel@tonic-gate _cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 27527c478bd9Sstevel@tonic-gate { 27537c478bd9Sstevel@tonic-gate clockid_t clock_id = cvp->cond_clockid; 27547c478bd9Sstevel@tonic-gate timespec_t reltime; 27557c478bd9Sstevel@tonic-gate int error; 27567c478bd9Sstevel@tonic-gate 27577c478bd9Sstevel@tonic-gate if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES) 27587c478bd9Sstevel@tonic-gate clock_id = CLOCK_REALTIME; 27597c478bd9Sstevel@tonic-gate abstime_to_reltime(clock_id, abstime, &reltime); 27607c478bd9Sstevel@tonic-gate error = cond_wait_common(cvp, mp, &reltime); 27617c478bd9Sstevel@tonic-gate if (error == ETIME && clock_id == CLOCK_HIGHRES) { 27627c478bd9Sstevel@tonic-gate /* 27637c478bd9Sstevel@tonic-gate * Don't return ETIME if we didn't really get a timeout. 27647c478bd9Sstevel@tonic-gate * This can happen if we return because someone resets 27657c478bd9Sstevel@tonic-gate * the system clock. Just return zero in this case, 27667c478bd9Sstevel@tonic-gate * giving a spurious wakeup but not a timeout. 27677c478bd9Sstevel@tonic-gate */ 27687c478bd9Sstevel@tonic-gate if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC + 27697c478bd9Sstevel@tonic-gate abstime->tv_nsec > gethrtime()) 27707c478bd9Sstevel@tonic-gate error = 0; 27717c478bd9Sstevel@tonic-gate } 27727c478bd9Sstevel@tonic-gate return (error); 27737c478bd9Sstevel@tonic-gate } 27747c478bd9Sstevel@tonic-gate 27757c478bd9Sstevel@tonic-gate int 27767c478bd9Sstevel@tonic-gate cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 27777c478bd9Sstevel@tonic-gate { 27787c478bd9Sstevel@tonic-gate int error; 27797c478bd9Sstevel@tonic-gate 27807c478bd9Sstevel@tonic-gate _cancelon(); 27817c478bd9Sstevel@tonic-gate error = _cond_timedwait(cvp, mp, abstime); 27827c478bd9Sstevel@tonic-gate if (error == EINTR) 27837c478bd9Sstevel@tonic-gate _canceloff(); 27847c478bd9Sstevel@tonic-gate else 27857c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 27867c478bd9Sstevel@tonic-gate return (error); 27877c478bd9Sstevel@tonic-gate } 27887c478bd9Sstevel@tonic-gate 27897c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_timedwait = _pthread_cond_timedwait 27907c478bd9Sstevel@tonic-gate int 27917c478bd9Sstevel@tonic-gate _pthread_cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 27927c478bd9Sstevel@tonic-gate { 27937c478bd9Sstevel@tonic-gate int error; 27947c478bd9Sstevel@tonic-gate 27957c478bd9Sstevel@tonic-gate error = cond_timedwait(cvp, mp, abstime); 27967c478bd9Sstevel@tonic-gate if (error == ETIME) 27977c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 27987c478bd9Sstevel@tonic-gate else if (error == EINTR) 27997c478bd9Sstevel@tonic-gate error = 0; 28007c478bd9Sstevel@tonic-gate return (error); 28017c478bd9Sstevel@tonic-gate } 28027c478bd9Sstevel@tonic-gate 28037c478bd9Sstevel@tonic-gate /* 28047c478bd9Sstevel@tonic-gate * cond_reltimedwait() is a cancellation point but _cond_reltimedwait() 28057c478bd9Sstevel@tonic-gate * is not. System libraries call the non-cancellation version. 28067c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 28077c478bd9Sstevel@tonic-gate */ 28087c478bd9Sstevel@tonic-gate int 28097c478bd9Sstevel@tonic-gate _cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 28107c478bd9Sstevel@tonic-gate { 28117c478bd9Sstevel@tonic-gate timespec_t tslocal = *reltime; 28127c478bd9Sstevel@tonic-gate 28137c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, &tslocal)); 28147c478bd9Sstevel@tonic-gate } 28157c478bd9Sstevel@tonic-gate 28167c478bd9Sstevel@tonic-gate #pragma weak cond_reltimedwait = _cond_reltimedwait_cancel 28177c478bd9Sstevel@tonic-gate int 28187c478bd9Sstevel@tonic-gate _cond_reltimedwait_cancel(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 28197c478bd9Sstevel@tonic-gate { 28207c478bd9Sstevel@tonic-gate int error; 28217c478bd9Sstevel@tonic-gate 28227c478bd9Sstevel@tonic-gate _cancelon(); 28237c478bd9Sstevel@tonic-gate error = _cond_reltimedwait(cvp, mp, reltime); 28247c478bd9Sstevel@tonic-gate if (error == EINTR) 28257c478bd9Sstevel@tonic-gate _canceloff(); 28267c478bd9Sstevel@tonic-gate else 28277c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 28287c478bd9Sstevel@tonic-gate return (error); 28297c478bd9Sstevel@tonic-gate } 28307c478bd9Sstevel@tonic-gate 28317c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_reltimedwait_np = _pthread_cond_reltimedwait_np 28327c478bd9Sstevel@tonic-gate int 28337c478bd9Sstevel@tonic-gate _pthread_cond_reltimedwait_np(cond_t *cvp, mutex_t *mp, 28347c478bd9Sstevel@tonic-gate const timespec_t *reltime) 28357c478bd9Sstevel@tonic-gate { 28367c478bd9Sstevel@tonic-gate int error; 28377c478bd9Sstevel@tonic-gate 28387c478bd9Sstevel@tonic-gate error = _cond_reltimedwait_cancel(cvp, mp, reltime); 28397c478bd9Sstevel@tonic-gate if (error == ETIME) 28407c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 28417c478bd9Sstevel@tonic-gate else if (error == EINTR) 28427c478bd9Sstevel@tonic-gate error = 0; 28437c478bd9Sstevel@tonic-gate return (error); 28447c478bd9Sstevel@tonic-gate } 28457c478bd9Sstevel@tonic-gate 28467c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_signal = cond_signal_internal 28477c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_signal = cond_signal_internal 28487c478bd9Sstevel@tonic-gate #pragma weak cond_signal = cond_signal_internal 28497c478bd9Sstevel@tonic-gate #pragma weak _cond_signal = cond_signal_internal 28507c478bd9Sstevel@tonic-gate int 28517c478bd9Sstevel@tonic-gate cond_signal_internal(cond_t *cvp) 28527c478bd9Sstevel@tonic-gate { 28537c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 28547c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 28557c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 28567c478bd9Sstevel@tonic-gate int error = 0; 28577c478bd9Sstevel@tonic-gate queue_head_t *qp; 28587c478bd9Sstevel@tonic-gate mutex_t *mp; 28597c478bd9Sstevel@tonic-gate queue_head_t *mqp; 28607c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 28617c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 28627c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 28637c478bd9Sstevel@tonic-gate ulwp_t *next; 28647c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 28657c478bd9Sstevel@tonic-gate ulwp_t *susprev; 28667c478bd9Sstevel@tonic-gate 28677c478bd9Sstevel@tonic-gate if (csp) 28687c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_signal); 28697c478bd9Sstevel@tonic-gate 28707c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 28717c478bd9Sstevel@tonic-gate error = __lwp_cond_signal(cvp); 28727c478bd9Sstevel@tonic-gate 28737c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 28747c478bd9Sstevel@tonic-gate return (error); 28757c478bd9Sstevel@tonic-gate 28767c478bd9Sstevel@tonic-gate /* 28777c478bd9Sstevel@tonic-gate * Move someone from the condvar sleep queue to the mutex sleep 28787c478bd9Sstevel@tonic-gate * queue for the mutex that he will acquire on being waked up. 28797c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex he will acquire. 28807c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if his ul_cv_wake flag 28817c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark him. 28827c478bd9Sstevel@tonic-gate */ 28837c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 28847c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 28857c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 28867c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == cvp) { 28877c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 28887c478bd9Sstevel@tonic-gate break; 28897c478bd9Sstevel@tonic-gate /* 28907c478bd9Sstevel@tonic-gate * Try not to dequeue a suspended thread. 28917c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 28927c478bd9Sstevel@tonic-gate */ 28937c478bd9Sstevel@tonic-gate if (suspp == NULL) { 28947c478bd9Sstevel@tonic-gate suspp = ulwpp; 28957c478bd9Sstevel@tonic-gate susprev = prev; 28967c478bd9Sstevel@tonic-gate } 28977c478bd9Sstevel@tonic-gate } 28987c478bd9Sstevel@tonic-gate } 28997c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 29007c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 29017c478bd9Sstevel@tonic-gate prev = susprev; 29027c478bd9Sstevel@tonic-gate suspp = NULL; 29037c478bd9Sstevel@tonic-gate } 29047c478bd9Sstevel@tonic-gate if (ulwp == NULL) { /* no one on the sleep queue */ 29057c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 29067c478bd9Sstevel@tonic-gate queue_unlock(qp); 29077c478bd9Sstevel@tonic-gate return (error); 29087c478bd9Sstevel@tonic-gate } 29097c478bd9Sstevel@tonic-gate /* 29107c478bd9Sstevel@tonic-gate * Scan the remainder of the CV queue for another waiter. 29117c478bd9Sstevel@tonic-gate */ 29127c478bd9Sstevel@tonic-gate if (suspp != NULL) { 29137c478bd9Sstevel@tonic-gate next = *suspp; 29147c478bd9Sstevel@tonic-gate } else { 29157c478bd9Sstevel@tonic-gate for (next = ulwp->ul_link; next != NULL; next = next->ul_link) 29167c478bd9Sstevel@tonic-gate if (next->ul_wchan == cvp) 29177c478bd9Sstevel@tonic-gate break; 29187c478bd9Sstevel@tonic-gate } 29197c478bd9Sstevel@tonic-gate if (next == NULL) 29207c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 29217c478bd9Sstevel@tonic-gate 29227c478bd9Sstevel@tonic-gate /* 29237c478bd9Sstevel@tonic-gate * Inform the thread that he was the recipient of a cond_signal(). 29247c478bd9Sstevel@tonic-gate * This lets him deal with cond_signal() and, concurrently, 29257c478bd9Sstevel@tonic-gate * one or more of a cancellation, a UNIX signal, or a timeout. 29267c478bd9Sstevel@tonic-gate * These latter conditions must not consume a cond_signal(). 29277c478bd9Sstevel@tonic-gate */ 29287c478bd9Sstevel@tonic-gate ulwp->ul_signalled = 1; 29297c478bd9Sstevel@tonic-gate 29307c478bd9Sstevel@tonic-gate /* 29317c478bd9Sstevel@tonic-gate * Dequeue the waiter but leave his ul_sleepq non-NULL 29327c478bd9Sstevel@tonic-gate * while we move him to the mutex queue so that he can 29337c478bd9Sstevel@tonic-gate * deal properly with spurious wakeups. 29347c478bd9Sstevel@tonic-gate */ 29357c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 29367c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 29377c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 29387c478bd9Sstevel@tonic-gate qp->qh_qlen--; 29397c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 29407c478bd9Sstevel@tonic-gate 29417c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* the mutex he will acquire */ 29427c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 29437c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 29447c478bd9Sstevel@tonic-gate 29457c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 29467c478bd9Sstevel@tonic-gate lwpid_t lwpid = ulwp->ul_lwpid; 29477c478bd9Sstevel@tonic-gate 29487c478bd9Sstevel@tonic-gate no_preempt(self); 29497c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 29507c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 29517c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 29527c478bd9Sstevel@tonic-gate queue_unlock(qp); 29537c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 29547c478bd9Sstevel@tonic-gate preempt(self); 29557c478bd9Sstevel@tonic-gate } else { 29567c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 29577c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 29587c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 29597c478bd9Sstevel@tonic-gate queue_unlock(mqp); 29607c478bd9Sstevel@tonic-gate queue_unlock(qp); 29617c478bd9Sstevel@tonic-gate } 29627c478bd9Sstevel@tonic-gate 29637c478bd9Sstevel@tonic-gate return (error); 29647c478bd9Sstevel@tonic-gate } 29657c478bd9Sstevel@tonic-gate 29667c478bd9Sstevel@tonic-gate #define MAXLWPS 128 /* max remembered lwpids before overflow */ 29677c478bd9Sstevel@tonic-gate #define NEWLWPS 2048 /* max remembered lwpids at first overflow */ 29687c478bd9Sstevel@tonic-gate 29697c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_broadcast = cond_broadcast_internal 29707c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_broadcast = cond_broadcast_internal 29717c478bd9Sstevel@tonic-gate #pragma weak cond_broadcast = cond_broadcast_internal 29727c478bd9Sstevel@tonic-gate #pragma weak _cond_broadcast = cond_broadcast_internal 29737c478bd9Sstevel@tonic-gate int 29747c478bd9Sstevel@tonic-gate cond_broadcast_internal(cond_t *cvp) 29757c478bd9Sstevel@tonic-gate { 29767c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 29777c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 29787c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 29797c478bd9Sstevel@tonic-gate int error = 0; 29807c478bd9Sstevel@tonic-gate queue_head_t *qp; 29817c478bd9Sstevel@tonic-gate mutex_t *mp; 29827c478bd9Sstevel@tonic-gate queue_head_t *mqp; 29837c478bd9Sstevel@tonic-gate mutex_t *mp_cache = NULL; 29847c478bd9Sstevel@tonic-gate queue_head_t *mqp_cache = NULL; 29857c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 29867c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 29877c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 29887c478bd9Sstevel@tonic-gate lwpid_t buffer[MAXLWPS]; 29897c478bd9Sstevel@tonic-gate lwpid_t *lwpid = buffer; 29907c478bd9Sstevel@tonic-gate int nlwpid = 0; 29917c478bd9Sstevel@tonic-gate int maxlwps = MAXLWPS; 29927c478bd9Sstevel@tonic-gate 29937c478bd9Sstevel@tonic-gate if (csp) 29947c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_broadcast); 29957c478bd9Sstevel@tonic-gate 29967c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 29977c478bd9Sstevel@tonic-gate error = __lwp_cond_broadcast(cvp); 29987c478bd9Sstevel@tonic-gate 29997c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 30007c478bd9Sstevel@tonic-gate return (error); 30017c478bd9Sstevel@tonic-gate 30027c478bd9Sstevel@tonic-gate /* 30037c478bd9Sstevel@tonic-gate * Move everyone from the condvar sleep queue to the mutex sleep 30047c478bd9Sstevel@tonic-gate * queue for the mutex that they will acquire on being waked up. 30057c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex they will acquire. 30067c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if their ul_cv_wake flag 30077c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark them. 30087c478bd9Sstevel@tonic-gate * 30097c478bd9Sstevel@tonic-gate * We keep track of lwpids that are to be unparked in lwpid[]. 30107c478bd9Sstevel@tonic-gate * __lwp_unpark_all() is called to unpark all of them after 30117c478bd9Sstevel@tonic-gate * they have been removed from the sleep queue and the sleep 30127c478bd9Sstevel@tonic-gate * queue lock has been dropped. If we run out of space in our 30137c478bd9Sstevel@tonic-gate * on-stack buffer, we need to allocate more but we can't call 30147c478bd9Sstevel@tonic-gate * lmalloc() because we are holding a queue lock when the overflow 30157c478bd9Sstevel@tonic-gate * occurs and lmalloc() acquires a lock. We can't use alloca() 30167c478bd9Sstevel@tonic-gate * either because the application may have allocated a small stack 30177c478bd9Sstevel@tonic-gate * and we don't want to overrun the stack. So we use the mmap() 30187c478bd9Sstevel@tonic-gate * system call directly since that path acquires no locks. 30197c478bd9Sstevel@tonic-gate */ 30207c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 30217c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 30227c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 30237c478bd9Sstevel@tonic-gate while ((ulwp = *ulwpp) != NULL) { 30247c478bd9Sstevel@tonic-gate 30257c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan != cvp) { 30267c478bd9Sstevel@tonic-gate prev = ulwp; 30277c478bd9Sstevel@tonic-gate ulwpp = &ulwp->ul_link; 30287c478bd9Sstevel@tonic-gate continue; 30297c478bd9Sstevel@tonic-gate } 30307c478bd9Sstevel@tonic-gate 30317c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 30327c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 30337c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 30347c478bd9Sstevel@tonic-gate qp->qh_qlen--; 30357c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 30367c478bd9Sstevel@tonic-gate 30377c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* his mutex */ 30387c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 30397c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 30407c478bd9Sstevel@tonic-gate 30417c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 30427c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 30437c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 30447c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 30457c478bd9Sstevel@tonic-gate if (nlwpid == maxlwps) { 30467c478bd9Sstevel@tonic-gate /* 30477c478bd9Sstevel@tonic-gate * Allocate NEWLWPS ids on the first overflow. 30487c478bd9Sstevel@tonic-gate * Double the allocation each time after that. 30497c478bd9Sstevel@tonic-gate */ 30507c478bd9Sstevel@tonic-gate int newlwps = (lwpid == buffer)? NEWLWPS : 30517c478bd9Sstevel@tonic-gate 2 * maxlwps; 30527c478bd9Sstevel@tonic-gate void *vaddr = _private_mmap(NULL, 30537c478bd9Sstevel@tonic-gate newlwps * sizeof (lwpid_t), 30547c478bd9Sstevel@tonic-gate PROT_READ|PROT_WRITE, 30557c478bd9Sstevel@tonic-gate MAP_PRIVATE|MAP_ANON, -1, (off_t)0); 30567c478bd9Sstevel@tonic-gate if (vaddr == MAP_FAILED) { 30577c478bd9Sstevel@tonic-gate /* 30587c478bd9Sstevel@tonic-gate * Let's hope this never happens. 30597c478bd9Sstevel@tonic-gate * If it does, then we have a terrible 30607c478bd9Sstevel@tonic-gate * thundering herd on our hands. 30617c478bd9Sstevel@tonic-gate */ 30627c478bd9Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid); 30637c478bd9Sstevel@tonic-gate nlwpid = 0; 30647c478bd9Sstevel@tonic-gate } else { 30657c478bd9Sstevel@tonic-gate (void) _memcpy(vaddr, lwpid, 30667c478bd9Sstevel@tonic-gate maxlwps * sizeof (lwpid_t)); 30677c478bd9Sstevel@tonic-gate if (lwpid != buffer) 30687c478bd9Sstevel@tonic-gate (void) _private_munmap(lwpid, 30697c478bd9Sstevel@tonic-gate maxlwps * sizeof (lwpid_t)); 30707c478bd9Sstevel@tonic-gate lwpid = vaddr; 30717c478bd9Sstevel@tonic-gate maxlwps = newlwps; 30727c478bd9Sstevel@tonic-gate } 30737c478bd9Sstevel@tonic-gate } 30747c478bd9Sstevel@tonic-gate lwpid[nlwpid++] = ulwp->ul_lwpid; 30757c478bd9Sstevel@tonic-gate } else { 30767c478bd9Sstevel@tonic-gate if (mp != mp_cache) { 30777c478bd9Sstevel@tonic-gate if (mqp_cache != NULL) 30787c478bd9Sstevel@tonic-gate queue_unlock(mqp_cache); 30797c478bd9Sstevel@tonic-gate mqp_cache = queue_lock(mp, MX); 30807c478bd9Sstevel@tonic-gate mp_cache = mp; 30817c478bd9Sstevel@tonic-gate } 30827c478bd9Sstevel@tonic-gate mqp = mqp_cache; 30837c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 30847c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 30857c478bd9Sstevel@tonic-gate } 30867c478bd9Sstevel@tonic-gate } 30877c478bd9Sstevel@tonic-gate if (mqp_cache != NULL) 30887c478bd9Sstevel@tonic-gate queue_unlock(mqp_cache); 30897c478bd9Sstevel@tonic-gate queue_unlock(qp); 30907c478bd9Sstevel@tonic-gate if (nlwpid) { 30917c478bd9Sstevel@tonic-gate if (nlwpid == 1) 30927c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid[0]); 30937c478bd9Sstevel@tonic-gate else 30947c478bd9Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid); 30957c478bd9Sstevel@tonic-gate } 30967c478bd9Sstevel@tonic-gate if (lwpid != buffer) 30977c478bd9Sstevel@tonic-gate (void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t)); 30987c478bd9Sstevel@tonic-gate 30997c478bd9Sstevel@tonic-gate return (error); 31007c478bd9Sstevel@tonic-gate } 31017c478bd9Sstevel@tonic-gate 31027c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_destroy = _cond_destroy 31037c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_destroy = _cond_destroy 31047c478bd9Sstevel@tonic-gate #pragma weak cond_destroy = _cond_destroy 31057c478bd9Sstevel@tonic-gate int 31067c478bd9Sstevel@tonic-gate _cond_destroy(cond_t *cvp) 31077c478bd9Sstevel@tonic-gate { 31087c478bd9Sstevel@tonic-gate cvp->cond_magic = 0; 31097c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(cvp); 31107c478bd9Sstevel@tonic-gate return (0); 31117c478bd9Sstevel@tonic-gate } 31127c478bd9Sstevel@tonic-gate 31137c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 31147c478bd9Sstevel@tonic-gate void 31157c478bd9Sstevel@tonic-gate assert_no_libc_locks_held(void) 31167c478bd9Sstevel@tonic-gate { 31177c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 31187c478bd9Sstevel@tonic-gate } 31197c478bd9Sstevel@tonic-gate #endif 31207c478bd9Sstevel@tonic-gate 31217c478bd9Sstevel@tonic-gate /* protected by link_lock */ 31227c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin; 31237c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin2; 31247c478bd9Sstevel@tonic-gate uint64_t spin_lock_sleep; 31257c478bd9Sstevel@tonic-gate uint64_t spin_lock_wakeup; 31267c478bd9Sstevel@tonic-gate 31277c478bd9Sstevel@tonic-gate /* 31287c478bd9Sstevel@tonic-gate * Record spin lock statistics. 31297c478bd9Sstevel@tonic-gate * Called by a thread exiting itself in thrp_exit(). 31307c478bd9Sstevel@tonic-gate * Also called via atexit() from the thread calling 31317c478bd9Sstevel@tonic-gate * exit() to do all the other threads as well. 31327c478bd9Sstevel@tonic-gate */ 31337c478bd9Sstevel@tonic-gate void 31347c478bd9Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp) 31357c478bd9Sstevel@tonic-gate { 31367c478bd9Sstevel@tonic-gate spin_lock_spin += ulwp->ul_spin_lock_spin; 31377c478bd9Sstevel@tonic-gate spin_lock_spin2 += ulwp->ul_spin_lock_spin2; 31387c478bd9Sstevel@tonic-gate spin_lock_sleep += ulwp->ul_spin_lock_sleep; 31397c478bd9Sstevel@tonic-gate spin_lock_wakeup += ulwp->ul_spin_lock_wakeup; 31407c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin = 0; 31417c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin2 = 0; 31427c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_sleep = 0; 31437c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_wakeup = 0; 31447c478bd9Sstevel@tonic-gate } 31457c478bd9Sstevel@tonic-gate 31467c478bd9Sstevel@tonic-gate /* 31477c478bd9Sstevel@tonic-gate * atexit function: dump the queue statistics to stderr. 31487c478bd9Sstevel@tonic-gate */ 3149*e8031f0aSraf #if !defined(__lint) 3150*e8031f0aSraf #define fprintf _fprintf 3151*e8031f0aSraf #endif 31527c478bd9Sstevel@tonic-gate #include <stdio.h> 31537c478bd9Sstevel@tonic-gate void 31547c478bd9Sstevel@tonic-gate dump_queue_statistics(void) 31557c478bd9Sstevel@tonic-gate { 31567c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 31577c478bd9Sstevel@tonic-gate queue_head_t *qp; 31587c478bd9Sstevel@tonic-gate int qn; 31597c478bd9Sstevel@tonic-gate uint64_t spin_lock_total = 0; 31607c478bd9Sstevel@tonic-gate 31617c478bd9Sstevel@tonic-gate if (udp->queue_head == NULL || thread_queue_dump == 0) 31627c478bd9Sstevel@tonic-gate return; 31637c478bd9Sstevel@tonic-gate 31647c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 || 31657c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 31667c478bd9Sstevel@tonic-gate return; 31677c478bd9Sstevel@tonic-gate for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) { 31687c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 31697c478bd9Sstevel@tonic-gate continue; 31707c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 31717c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 31727c478bd9Sstevel@tonic-gate (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 31737c478bd9Sstevel@tonic-gate return; 31747c478bd9Sstevel@tonic-gate } 31757c478bd9Sstevel@tonic-gate 31767c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 || 31777c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 31787c478bd9Sstevel@tonic-gate return; 31797c478bd9Sstevel@tonic-gate for (qn = 0; qn < QHASHSIZE; qn++, qp++) { 31807c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 31817c478bd9Sstevel@tonic-gate continue; 31827c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 31837c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 31847c478bd9Sstevel@tonic-gate (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 31857c478bd9Sstevel@tonic-gate return; 31867c478bd9Sstevel@tonic-gate } 31877c478bd9Sstevel@tonic-gate 31887c478bd9Sstevel@tonic-gate (void) fprintf(stderr, "\n spin_lock_total = %10llu\n", 31897c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_total); 31907c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin = %10llu\n", 31917c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_spin); 31927c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin2 = %10llu\n", 31937c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_spin2); 31947c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_sleep = %10llu\n", 31957c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_sleep); 31967c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_wakeup = %10llu\n", 31977c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_wakeup); 31987c478bd9Sstevel@tonic-gate } 3199