17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*2be60c5eSraf * Common Development and Distribution License (the "License"). 6*2be60c5eSraf * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21e8031f0aSraf 227c478bd9Sstevel@tonic-gate /* 23e8031f0aSraf * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/sdt.h> 307c478bd9Sstevel@tonic-gate 317c478bd9Sstevel@tonic-gate #include "lint.h" 327c478bd9Sstevel@tonic-gate #include "thr_uberdata.h" 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate /* 357c478bd9Sstevel@tonic-gate * This mutex is initialized to be held by lwp#1. 367c478bd9Sstevel@tonic-gate * It is used to block a thread that has returned from a mutex_lock() 377c478bd9Sstevel@tonic-gate * of a PTHREAD_PRIO_INHERIT mutex with an unrecoverable error. 387c478bd9Sstevel@tonic-gate */ 397c478bd9Sstevel@tonic-gate mutex_t stall_mutex = DEFAULTMUTEX; 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate static int shared_mutex_held(mutex_t *); 427c478bd9Sstevel@tonic-gate 437c478bd9Sstevel@tonic-gate /* 447c478bd9Sstevel@tonic-gate * Lock statistics support functions. 457c478bd9Sstevel@tonic-gate */ 467c478bd9Sstevel@tonic-gate void 477c478bd9Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp) 487c478bd9Sstevel@tonic-gate { 497c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_lock); 507c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = gethrtime(); 517c478bd9Sstevel@tonic-gate } 527c478bd9Sstevel@tonic-gate 537c478bd9Sstevel@tonic-gate hrtime_t 547c478bd9Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp) 557c478bd9Sstevel@tonic-gate { 567c478bd9Sstevel@tonic-gate hrtime_t now = gethrtime(); 577c478bd9Sstevel@tonic-gate 587c478bd9Sstevel@tonic-gate if (msp->mutex_begin_hold) 597c478bd9Sstevel@tonic-gate msp->mutex_hold_time += now - msp->mutex_begin_hold; 607c478bd9Sstevel@tonic-gate msp->mutex_begin_hold = 0; 617c478bd9Sstevel@tonic-gate return (now); 627c478bd9Sstevel@tonic-gate } 637c478bd9Sstevel@tonic-gate 647c478bd9Sstevel@tonic-gate /* 657c478bd9Sstevel@tonic-gate * Called once at library initialization. 667c478bd9Sstevel@tonic-gate */ 677c478bd9Sstevel@tonic-gate void 687c478bd9Sstevel@tonic-gate mutex_setup(void) 697c478bd9Sstevel@tonic-gate { 707c478bd9Sstevel@tonic-gate if (set_lock_byte(&stall_mutex.mutex_lockw)) 717c478bd9Sstevel@tonic-gate thr_panic("mutex_setup() cannot acquire stall_mutex"); 727c478bd9Sstevel@tonic-gate stall_mutex.mutex_owner = (uintptr_t)curthread; 737c478bd9Sstevel@tonic-gate } 747c478bd9Sstevel@tonic-gate 757c478bd9Sstevel@tonic-gate /* 767c478bd9Sstevel@tonic-gate * The default spin counts of 1000 and 500 are experimentally determined. 777c478bd9Sstevel@tonic-gate * On sun4u machines with any number of processors they could be raised 787c478bd9Sstevel@tonic-gate * to 10,000 but that (experimentally) makes almost no difference. 797c478bd9Sstevel@tonic-gate * The environment variables: 807c478bd9Sstevel@tonic-gate * _THREAD_ADAPTIVE_SPIN=count 817c478bd9Sstevel@tonic-gate * _THREAD_RELEASE_SPIN=count 827c478bd9Sstevel@tonic-gate * can be used to override and set the counts in the range [0 .. 1,000,000]. 837c478bd9Sstevel@tonic-gate */ 847c478bd9Sstevel@tonic-gate int thread_adaptive_spin = 1000; 857c478bd9Sstevel@tonic-gate uint_t thread_max_spinners = 100; 867c478bd9Sstevel@tonic-gate int thread_release_spin = 500; 877c478bd9Sstevel@tonic-gate int thread_queue_verify = 0; 887c478bd9Sstevel@tonic-gate static int ncpus; 897c478bd9Sstevel@tonic-gate 907c478bd9Sstevel@tonic-gate /* 917c478bd9Sstevel@tonic-gate * Distinguish spinning for queue locks from spinning for regular locks. 927c478bd9Sstevel@tonic-gate * The environment variable: 937c478bd9Sstevel@tonic-gate * _THREAD_QUEUE_SPIN=count 947c478bd9Sstevel@tonic-gate * can be used to override and set the count in the range [0 .. 1,000,000]. 957c478bd9Sstevel@tonic-gate * There is no release spin concept for queue locks. 967c478bd9Sstevel@tonic-gate */ 977c478bd9Sstevel@tonic-gate int thread_queue_spin = 1000; 987c478bd9Sstevel@tonic-gate 997c478bd9Sstevel@tonic-gate /* 1007c478bd9Sstevel@tonic-gate * Use the otherwise-unused 'mutex_ownerpid' field of a USYNC_THREAD 1017c478bd9Sstevel@tonic-gate * mutex to be a count of adaptive spins in progress. 1027c478bd9Sstevel@tonic-gate */ 1037c478bd9Sstevel@tonic-gate #define mutex_spinners mutex_ownerpid 1047c478bd9Sstevel@tonic-gate 1057c478bd9Sstevel@tonic-gate void 1067c478bd9Sstevel@tonic-gate _mutex_set_typeattr(mutex_t *mp, int attr) 1077c478bd9Sstevel@tonic-gate { 1087c478bd9Sstevel@tonic-gate mp->mutex_type |= (uint8_t)attr; 1097c478bd9Sstevel@tonic-gate } 1107c478bd9Sstevel@tonic-gate 1117c478bd9Sstevel@tonic-gate /* 1127c478bd9Sstevel@tonic-gate * 'type' can be one of USYNC_THREAD or USYNC_PROCESS, possibly 1137c478bd9Sstevel@tonic-gate * augmented by the flags LOCK_RECURSIVE and/or LOCK_ERRORCHECK, 1147c478bd9Sstevel@tonic-gate * or it can be USYNC_PROCESS_ROBUST with no extra flags. 1157c478bd9Sstevel@tonic-gate */ 1167c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_init = __mutex_init 1177c478bd9Sstevel@tonic-gate #pragma weak mutex_init = __mutex_init 1187c478bd9Sstevel@tonic-gate #pragma weak _mutex_init = __mutex_init 1197c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 1207c478bd9Sstevel@tonic-gate int 1217c478bd9Sstevel@tonic-gate __mutex_init(mutex_t *mp, int type, void *arg) 1227c478bd9Sstevel@tonic-gate { 1237c478bd9Sstevel@tonic-gate int error; 1247c478bd9Sstevel@tonic-gate 1257c478bd9Sstevel@tonic-gate switch (type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) { 1267c478bd9Sstevel@tonic-gate case USYNC_THREAD: 1277c478bd9Sstevel@tonic-gate case USYNC_PROCESS: 1287c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 1297c478bd9Sstevel@tonic-gate mp->mutex_type = (uint8_t)type; 1307c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 1317c478bd9Sstevel@tonic-gate error = 0; 1327c478bd9Sstevel@tonic-gate break; 1337c478bd9Sstevel@tonic-gate case USYNC_PROCESS_ROBUST: 1347c478bd9Sstevel@tonic-gate if (type & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) 1357c478bd9Sstevel@tonic-gate error = EINVAL; 1367c478bd9Sstevel@tonic-gate else 1377c478bd9Sstevel@tonic-gate error = ___lwp_mutex_init(mp, type); 1387c478bd9Sstevel@tonic-gate break; 1397c478bd9Sstevel@tonic-gate default: 1407c478bd9Sstevel@tonic-gate error = EINVAL; 1417c478bd9Sstevel@tonic-gate break; 1427c478bd9Sstevel@tonic-gate } 1437c478bd9Sstevel@tonic-gate if (error == 0) 1447c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 1457c478bd9Sstevel@tonic-gate return (error); 1467c478bd9Sstevel@tonic-gate } 1477c478bd9Sstevel@tonic-gate 1487c478bd9Sstevel@tonic-gate /* 1497c478bd9Sstevel@tonic-gate * Delete mp from list of ceil mutexes owned by curthread. 1507c478bd9Sstevel@tonic-gate * Return 1 if the head of the chain was updated. 1517c478bd9Sstevel@tonic-gate */ 1527c478bd9Sstevel@tonic-gate int 1537c478bd9Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp) 1547c478bd9Sstevel@tonic-gate { 1557c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1567c478bd9Sstevel@tonic-gate mxchain_t **mcpp; 1577c478bd9Sstevel@tonic-gate mxchain_t *mcp; 1587c478bd9Sstevel@tonic-gate 1597c478bd9Sstevel@tonic-gate mcpp = &self->ul_mxchain; 1607c478bd9Sstevel@tonic-gate while ((*mcpp)->mxchain_mx != mp) 1617c478bd9Sstevel@tonic-gate mcpp = &(*mcpp)->mxchain_next; 1627c478bd9Sstevel@tonic-gate mcp = *mcpp; 1637c478bd9Sstevel@tonic-gate *mcpp = mcp->mxchain_next; 1647c478bd9Sstevel@tonic-gate lfree(mcp, sizeof (*mcp)); 1657c478bd9Sstevel@tonic-gate return (mcpp == &self->ul_mxchain); 1667c478bd9Sstevel@tonic-gate } 1677c478bd9Sstevel@tonic-gate 1687c478bd9Sstevel@tonic-gate /* 1697c478bd9Sstevel@tonic-gate * Add mp to head of list of ceil mutexes owned by curthread. 1707c478bd9Sstevel@tonic-gate * Return ENOMEM if no memory could be allocated. 1717c478bd9Sstevel@tonic-gate */ 1727c478bd9Sstevel@tonic-gate int 1737c478bd9Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp) 1747c478bd9Sstevel@tonic-gate { 1757c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1767c478bd9Sstevel@tonic-gate mxchain_t *mcp; 1777c478bd9Sstevel@tonic-gate 1787c478bd9Sstevel@tonic-gate if ((mcp = lmalloc(sizeof (*mcp))) == NULL) 1797c478bd9Sstevel@tonic-gate return (ENOMEM); 1807c478bd9Sstevel@tonic-gate mcp->mxchain_mx = mp; 1817c478bd9Sstevel@tonic-gate mcp->mxchain_next = self->ul_mxchain; 1827c478bd9Sstevel@tonic-gate self->ul_mxchain = mcp; 1837c478bd9Sstevel@tonic-gate return (0); 1847c478bd9Sstevel@tonic-gate } 1857c478bd9Sstevel@tonic-gate 1867c478bd9Sstevel@tonic-gate /* 1877c478bd9Sstevel@tonic-gate * Inherit priority from ceiling. The inheritance impacts the effective 1887c478bd9Sstevel@tonic-gate * priority, not the assigned priority. See _thread_setschedparam_main(). 1897c478bd9Sstevel@tonic-gate */ 1907c478bd9Sstevel@tonic-gate void 1917c478bd9Sstevel@tonic-gate _ceil_prio_inherit(int ceil) 1927c478bd9Sstevel@tonic-gate { 1937c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 1947c478bd9Sstevel@tonic-gate struct sched_param param; 1957c478bd9Sstevel@tonic-gate 1967c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 1977c478bd9Sstevel@tonic-gate param.sched_priority = ceil; 1987c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 1997c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) { 2007c478bd9Sstevel@tonic-gate /* 2017c478bd9Sstevel@tonic-gate * Panic since unclear what error code to return. 2027c478bd9Sstevel@tonic-gate * If we do return the error codes returned by above 2037c478bd9Sstevel@tonic-gate * called routine, update the man page... 2047c478bd9Sstevel@tonic-gate */ 2057c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2067c478bd9Sstevel@tonic-gate } 2077c478bd9Sstevel@tonic-gate } 2087c478bd9Sstevel@tonic-gate 2097c478bd9Sstevel@tonic-gate /* 2107c478bd9Sstevel@tonic-gate * Waive inherited ceiling priority. Inherit from head of owned ceiling locks 2117c478bd9Sstevel@tonic-gate * if holding at least one ceiling lock. If no ceiling locks are held at this 2127c478bd9Sstevel@tonic-gate * point, disinherit completely, reverting back to assigned priority. 2137c478bd9Sstevel@tonic-gate */ 2147c478bd9Sstevel@tonic-gate void 2157c478bd9Sstevel@tonic-gate _ceil_prio_waive(void) 2167c478bd9Sstevel@tonic-gate { 2177c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2187c478bd9Sstevel@tonic-gate struct sched_param param; 2197c478bd9Sstevel@tonic-gate 2207c478bd9Sstevel@tonic-gate (void) _memset(¶m, 0, sizeof (param)); 2217c478bd9Sstevel@tonic-gate if (self->ul_mxchain == NULL) { 2227c478bd9Sstevel@tonic-gate /* 2237c478bd9Sstevel@tonic-gate * No ceil locks held. Zero the epri, revert back to ul_pri. 2247c478bd9Sstevel@tonic-gate * Since thread's hash lock is not held, one cannot just 2257c478bd9Sstevel@tonic-gate * read ul_pri here...do it in the called routine... 2267c478bd9Sstevel@tonic-gate */ 2277c478bd9Sstevel@tonic-gate param.sched_priority = self->ul_pri; /* ignored */ 2287c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2297c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_DISINHERIT)) 2307c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2317c478bd9Sstevel@tonic-gate } else { 2327c478bd9Sstevel@tonic-gate /* 2337c478bd9Sstevel@tonic-gate * Set priority to that of the mutex at the head 2347c478bd9Sstevel@tonic-gate * of the ceilmutex chain. 2357c478bd9Sstevel@tonic-gate */ 2367c478bd9Sstevel@tonic-gate param.sched_priority = 2377c478bd9Sstevel@tonic-gate self->ul_mxchain->mxchain_mx->mutex_ceiling; 2387c478bd9Sstevel@tonic-gate if (_thread_setschedparam_main(self->ul_lwpid, 2397c478bd9Sstevel@tonic-gate self->ul_policy, ¶m, PRIO_INHERIT)) 2407c478bd9Sstevel@tonic-gate thr_panic("_thread_setschedparam_main() fails"); 2417c478bd9Sstevel@tonic-gate } 2427c478bd9Sstevel@tonic-gate } 2437c478bd9Sstevel@tonic-gate 2447c478bd9Sstevel@tonic-gate /* 2457c478bd9Sstevel@tonic-gate * Non-preemptive spin locks. Used by queue_lock(). 2467c478bd9Sstevel@tonic-gate * No lock statistics are gathered for these locks. 2477c478bd9Sstevel@tonic-gate */ 2487c478bd9Sstevel@tonic-gate void 2497c478bd9Sstevel@tonic-gate spin_lock_set(mutex_t *mp) 2507c478bd9Sstevel@tonic-gate { 2517c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2527c478bd9Sstevel@tonic-gate 2537c478bd9Sstevel@tonic-gate no_preempt(self); 2547c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 2557c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2567c478bd9Sstevel@tonic-gate return; 2577c478bd9Sstevel@tonic-gate } 2587c478bd9Sstevel@tonic-gate /* 2597c478bd9Sstevel@tonic-gate * Spin for a while, attempting to acquire the lock. 2607c478bd9Sstevel@tonic-gate */ 2617c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin != UINT_MAX) 2627c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin++; 2637c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 2647c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 2657c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2667c478bd9Sstevel@tonic-gate return; 2677c478bd9Sstevel@tonic-gate } 2687c478bd9Sstevel@tonic-gate /* 2697c478bd9Sstevel@tonic-gate * Try harder if we were previously at a no premption level. 2707c478bd9Sstevel@tonic-gate */ 2717c478bd9Sstevel@tonic-gate if (self->ul_preempt > 1) { 2727c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_spin2 != UINT_MAX) 2737c478bd9Sstevel@tonic-gate self->ul_spin_lock_spin2++; 2747c478bd9Sstevel@tonic-gate if (mutex_queuelock_adaptive(mp) == 0 || 2757c478bd9Sstevel@tonic-gate set_lock_byte(&mp->mutex_lockw) == 0) { 2767c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2777c478bd9Sstevel@tonic-gate return; 2787c478bd9Sstevel@tonic-gate } 2797c478bd9Sstevel@tonic-gate } 2807c478bd9Sstevel@tonic-gate /* 2817c478bd9Sstevel@tonic-gate * Give up and block in the kernel for the mutex. 2827c478bd9Sstevel@tonic-gate */ 2837c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_sleep != UINT_MAX) 2847c478bd9Sstevel@tonic-gate self->ul_spin_lock_sleep++; 2857c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_timedlock(mp, NULL); 2867c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 2877c478bd9Sstevel@tonic-gate } 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate void 2907c478bd9Sstevel@tonic-gate spin_lock_clear(mutex_t *mp) 2917c478bd9Sstevel@tonic-gate { 2927c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 2937c478bd9Sstevel@tonic-gate 2947c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 2957c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) { 2967c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 2977c478bd9Sstevel@tonic-gate if (self->ul_spin_lock_wakeup != UINT_MAX) 2987c478bd9Sstevel@tonic-gate self->ul_spin_lock_wakeup++; 2997c478bd9Sstevel@tonic-gate } 3007c478bd9Sstevel@tonic-gate preempt(self); 3017c478bd9Sstevel@tonic-gate } 3027c478bd9Sstevel@tonic-gate 3037c478bd9Sstevel@tonic-gate /* 3047c478bd9Sstevel@tonic-gate * Allocate the sleep queue hash table. 3057c478bd9Sstevel@tonic-gate */ 3067c478bd9Sstevel@tonic-gate void 3077c478bd9Sstevel@tonic-gate queue_alloc(void) 3087c478bd9Sstevel@tonic-gate { 3097c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3107c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 3117c478bd9Sstevel@tonic-gate void *data; 3127c478bd9Sstevel@tonic-gate int i; 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate /* 3157c478bd9Sstevel@tonic-gate * No locks are needed; we call here only when single-threaded. 3167c478bd9Sstevel@tonic-gate */ 3177c478bd9Sstevel@tonic-gate ASSERT(self == udp->ulwp_one); 3187c478bd9Sstevel@tonic-gate ASSERT(!udp->uberflags.uf_mt); 3197c478bd9Sstevel@tonic-gate if ((data = _private_mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t), 3207c478bd9Sstevel@tonic-gate PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0)) 3217c478bd9Sstevel@tonic-gate == MAP_FAILED) 3227c478bd9Sstevel@tonic-gate thr_panic("cannot allocate thread queue_head table"); 3237c478bd9Sstevel@tonic-gate udp->queue_head = (queue_head_t *)data; 3247c478bd9Sstevel@tonic-gate for (i = 0; i < 2 * QHASHSIZE; i++) 3257c478bd9Sstevel@tonic-gate udp->queue_head[i].qh_lock.mutex_magic = MUTEX_MAGIC; 3267c478bd9Sstevel@tonic-gate } 3277c478bd9Sstevel@tonic-gate 3287c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 3297c478bd9Sstevel@tonic-gate 3307c478bd9Sstevel@tonic-gate /* 3317c478bd9Sstevel@tonic-gate * Debugging: verify correctness of a sleep queue. 3327c478bd9Sstevel@tonic-gate */ 3337c478bd9Sstevel@tonic-gate void 3347c478bd9Sstevel@tonic-gate QVERIFY(queue_head_t *qp) 3357c478bd9Sstevel@tonic-gate { 3367c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 3377c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 3387c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 3397c478bd9Sstevel@tonic-gate ulwp_t *prev; 3407c478bd9Sstevel@tonic-gate uint_t index; 3417c478bd9Sstevel@tonic-gate uint32_t cnt = 0; 3427c478bd9Sstevel@tonic-gate char qtype; 3437c478bd9Sstevel@tonic-gate void *wchan; 3447c478bd9Sstevel@tonic-gate 3457c478bd9Sstevel@tonic-gate ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE); 3467c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 3477c478bd9Sstevel@tonic-gate ASSERT((qp->qh_head != NULL && qp->qh_tail != NULL) || 3487c478bd9Sstevel@tonic-gate (qp->qh_head == NULL && qp->qh_tail == NULL)); 3497c478bd9Sstevel@tonic-gate if (!thread_queue_verify) 3507c478bd9Sstevel@tonic-gate return; 3517c478bd9Sstevel@tonic-gate /* real expensive stuff, only for _THREAD_QUEUE_VERIFY */ 3527c478bd9Sstevel@tonic-gate qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV; 3537c478bd9Sstevel@tonic-gate for (prev = NULL, ulwp = qp->qh_head; ulwp != NULL; 3547c478bd9Sstevel@tonic-gate prev = ulwp, ulwp = ulwp->ul_link, cnt++) { 3557c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_qtype == qtype); 3567c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_wchan != NULL); 3577c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq == qp); 3587c478bd9Sstevel@tonic-gate wchan = ulwp->ul_wchan; 3597c478bd9Sstevel@tonic-gate index = QUEUE_HASH(wchan, qtype); 3607c478bd9Sstevel@tonic-gate ASSERT(&udp->queue_head[index] == qp); 3617c478bd9Sstevel@tonic-gate } 3627c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == prev); 3637c478bd9Sstevel@tonic-gate ASSERT(qp->qh_qlen == cnt); 3647c478bd9Sstevel@tonic-gate } 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate #else /* THREAD_DEBUG */ 3677c478bd9Sstevel@tonic-gate 3687c478bd9Sstevel@tonic-gate #define QVERIFY(qp) 3697c478bd9Sstevel@tonic-gate 3707c478bd9Sstevel@tonic-gate #endif /* THREAD_DEBUG */ 3717c478bd9Sstevel@tonic-gate 3727c478bd9Sstevel@tonic-gate /* 3737c478bd9Sstevel@tonic-gate * Acquire a queue head. 3747c478bd9Sstevel@tonic-gate */ 3757c478bd9Sstevel@tonic-gate queue_head_t * 3767c478bd9Sstevel@tonic-gate queue_lock(void *wchan, int qtype) 3777c478bd9Sstevel@tonic-gate { 3787c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 3797c478bd9Sstevel@tonic-gate queue_head_t *qp; 3807c478bd9Sstevel@tonic-gate 3817c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 3827c478bd9Sstevel@tonic-gate 3837c478bd9Sstevel@tonic-gate /* 3847c478bd9Sstevel@tonic-gate * It is possible that we could be called while still single-threaded. 3857c478bd9Sstevel@tonic-gate * If so, we call queue_alloc() to allocate the queue_head[] array. 3867c478bd9Sstevel@tonic-gate */ 3877c478bd9Sstevel@tonic-gate if ((qp = udp->queue_head) == NULL) { 3887c478bd9Sstevel@tonic-gate queue_alloc(); 3897c478bd9Sstevel@tonic-gate qp = udp->queue_head; 3907c478bd9Sstevel@tonic-gate } 3917c478bd9Sstevel@tonic-gate qp += QUEUE_HASH(wchan, qtype); 3927c478bd9Sstevel@tonic-gate spin_lock_set(&qp->qh_lock); 3937c478bd9Sstevel@tonic-gate /* 3947c478bd9Sstevel@tonic-gate * At once per nanosecond, qh_lockcount will wrap after 512 years. 3957c478bd9Sstevel@tonic-gate * Were we to care about this, we could peg the value at UINT64_MAX. 3967c478bd9Sstevel@tonic-gate */ 3977c478bd9Sstevel@tonic-gate qp->qh_lockcount++; 3987c478bd9Sstevel@tonic-gate QVERIFY(qp); 3997c478bd9Sstevel@tonic-gate return (qp); 4007c478bd9Sstevel@tonic-gate } 4017c478bd9Sstevel@tonic-gate 4027c478bd9Sstevel@tonic-gate /* 4037c478bd9Sstevel@tonic-gate * Release a queue head. 4047c478bd9Sstevel@tonic-gate */ 4057c478bd9Sstevel@tonic-gate void 4067c478bd9Sstevel@tonic-gate queue_unlock(queue_head_t *qp) 4077c478bd9Sstevel@tonic-gate { 4087c478bd9Sstevel@tonic-gate QVERIFY(qp); 4097c478bd9Sstevel@tonic-gate spin_lock_clear(&qp->qh_lock); 4107c478bd9Sstevel@tonic-gate } 4117c478bd9Sstevel@tonic-gate 4127c478bd9Sstevel@tonic-gate /* 4137c478bd9Sstevel@tonic-gate * For rwlock queueing, we must queue writers ahead of readers of the 4147c478bd9Sstevel@tonic-gate * same priority. We do this by making writers appear to have a half 4157c478bd9Sstevel@tonic-gate * point higher priority for purposes of priority comparisons below. 4167c478bd9Sstevel@tonic-gate */ 4177c478bd9Sstevel@tonic-gate #define CMP_PRIO(ulwp) ((real_priority(ulwp) << 1) + (ulwp)->ul_writer) 4187c478bd9Sstevel@tonic-gate 4197c478bd9Sstevel@tonic-gate void 4207c478bd9Sstevel@tonic-gate enqueue(queue_head_t *qp, ulwp_t *ulwp, void *wchan, int qtype) 4217c478bd9Sstevel@tonic-gate { 4227c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 4237c478bd9Sstevel@tonic-gate ulwp_t *next; 4247c478bd9Sstevel@tonic-gate int pri = CMP_PRIO(ulwp); 4257c478bd9Sstevel@tonic-gate int force_fifo = (qtype & FIFOQ); 4267c478bd9Sstevel@tonic-gate int do_fifo; 4277c478bd9Sstevel@tonic-gate 4287c478bd9Sstevel@tonic-gate qtype &= ~FIFOQ; 4297c478bd9Sstevel@tonic-gate ASSERT(qtype == MX || qtype == CV); 4307c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 4317c478bd9Sstevel@tonic-gate ASSERT(ulwp->ul_sleepq != qp); 4327c478bd9Sstevel@tonic-gate 4337c478bd9Sstevel@tonic-gate /* 4347c478bd9Sstevel@tonic-gate * LIFO queue ordering is unfair and can lead to starvation, 4357c478bd9Sstevel@tonic-gate * but it gives better performance for heavily contended locks. 4367c478bd9Sstevel@tonic-gate * We use thread_queue_fifo (range is 0..8) to determine 4377c478bd9Sstevel@tonic-gate * the frequency of FIFO vs LIFO queuing: 4387c478bd9Sstevel@tonic-gate * 0 : every 256th time (almost always LIFO) 4397c478bd9Sstevel@tonic-gate * 1 : every 128th time 4407c478bd9Sstevel@tonic-gate * 2 : every 64th time 4417c478bd9Sstevel@tonic-gate * 3 : every 32nd time 4427c478bd9Sstevel@tonic-gate * 4 : every 16th time (the default value, mostly LIFO) 4437c478bd9Sstevel@tonic-gate * 5 : every 8th time 4447c478bd9Sstevel@tonic-gate * 6 : every 4th time 4457c478bd9Sstevel@tonic-gate * 7 : every 2nd time 4467c478bd9Sstevel@tonic-gate * 8 : every time (never LIFO, always FIFO) 4477c478bd9Sstevel@tonic-gate * Note that there is always some degree of FIFO ordering. 4487c478bd9Sstevel@tonic-gate * This breaks live lock conditions that occur in applications 4497c478bd9Sstevel@tonic-gate * that are written assuming (incorrectly) that threads acquire 4507c478bd9Sstevel@tonic-gate * locks fairly, that is, in roughly round-robin order. 4517c478bd9Sstevel@tonic-gate * In any event, the queue is maintained in priority order. 4527c478bd9Sstevel@tonic-gate * 4537c478bd9Sstevel@tonic-gate * If we are given the FIFOQ flag in qtype, fifo queueing is forced. 4547c478bd9Sstevel@tonic-gate * SUSV3 requires this for semaphores. 4557c478bd9Sstevel@tonic-gate */ 4567c478bd9Sstevel@tonic-gate do_fifo = (force_fifo || 4577c478bd9Sstevel@tonic-gate ((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0); 4587c478bd9Sstevel@tonic-gate 4597c478bd9Sstevel@tonic-gate if (qp->qh_head == NULL) { 4607c478bd9Sstevel@tonic-gate /* 4617c478bd9Sstevel@tonic-gate * The queue is empty. LIFO/FIFO doesn't matter. 4627c478bd9Sstevel@tonic-gate */ 4637c478bd9Sstevel@tonic-gate ASSERT(qp->qh_tail == NULL); 4647c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 4657c478bd9Sstevel@tonic-gate } else if (do_fifo) { 4667c478bd9Sstevel@tonic-gate /* 4677c478bd9Sstevel@tonic-gate * Enqueue after the last thread whose priority is greater 4687c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 4697c478bd9Sstevel@tonic-gate * Attempt first to go directly onto the tail of the queue. 4707c478bd9Sstevel@tonic-gate */ 4717c478bd9Sstevel@tonic-gate if (pri <= CMP_PRIO(qp->qh_tail)) 4727c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_tail->ul_link; 4737c478bd9Sstevel@tonic-gate else { 4747c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 4757c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 4767c478bd9Sstevel@tonic-gate if (pri > CMP_PRIO(next)) 4777c478bd9Sstevel@tonic-gate break; 4787c478bd9Sstevel@tonic-gate } 4797c478bd9Sstevel@tonic-gate } else { 4807c478bd9Sstevel@tonic-gate /* 4817c478bd9Sstevel@tonic-gate * Enqueue before the first thread whose priority is less 4827c478bd9Sstevel@tonic-gate * than or equal to the priority of the thread being queued. 4837c478bd9Sstevel@tonic-gate * Hopefully we can go directly onto the head of the queue. 4847c478bd9Sstevel@tonic-gate */ 4857c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL; 4867c478bd9Sstevel@tonic-gate ulwpp = &next->ul_link) 4877c478bd9Sstevel@tonic-gate if (pri >= CMP_PRIO(next)) 4887c478bd9Sstevel@tonic-gate break; 4897c478bd9Sstevel@tonic-gate } 4907c478bd9Sstevel@tonic-gate if ((ulwp->ul_link = *ulwpp) == NULL) 4917c478bd9Sstevel@tonic-gate qp->qh_tail = ulwp; 4927c478bd9Sstevel@tonic-gate *ulwpp = ulwp; 4937c478bd9Sstevel@tonic-gate 4947c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = qp; 4957c478bd9Sstevel@tonic-gate ulwp->ul_wchan = wchan; 4967c478bd9Sstevel@tonic-gate ulwp->ul_qtype = qtype; 4977c478bd9Sstevel@tonic-gate if (qp->qh_qmax < ++qp->qh_qlen) 4987c478bd9Sstevel@tonic-gate qp->qh_qmax = qp->qh_qlen; 4997c478bd9Sstevel@tonic-gate } 5007c478bd9Sstevel@tonic-gate 5017c478bd9Sstevel@tonic-gate /* 5027c478bd9Sstevel@tonic-gate * Return a pointer to the queue slot of the 5037c478bd9Sstevel@tonic-gate * highest priority thread on the queue. 5047c478bd9Sstevel@tonic-gate * On return, prevp, if not NULL, will contain a pointer 5057c478bd9Sstevel@tonic-gate * to the thread's predecessor on the queue 5067c478bd9Sstevel@tonic-gate */ 5077c478bd9Sstevel@tonic-gate static ulwp_t ** 5087c478bd9Sstevel@tonic-gate queue_slot(queue_head_t *qp, void *wchan, int *more, ulwp_t **prevp) 5097c478bd9Sstevel@tonic-gate { 5107c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 5117c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 5127c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 5137c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 5147c478bd9Sstevel@tonic-gate ulwp_t *susprev; 5157c478bd9Sstevel@tonic-gate 5167c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread)); 5177c478bd9Sstevel@tonic-gate 5187c478bd9Sstevel@tonic-gate /* 5197c478bd9Sstevel@tonic-gate * Find a waiter on the sleep queue. 5207c478bd9Sstevel@tonic-gate */ 5217c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 5227c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 5237c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 5247c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 5257c478bd9Sstevel@tonic-gate break; 5267c478bd9Sstevel@tonic-gate /* 5277c478bd9Sstevel@tonic-gate * Try not to return a suspended thread. 5287c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 5297c478bd9Sstevel@tonic-gate */ 5307c478bd9Sstevel@tonic-gate if (suspp == NULL) { 5317c478bd9Sstevel@tonic-gate suspp = ulwpp; 5327c478bd9Sstevel@tonic-gate susprev = prev; 5337c478bd9Sstevel@tonic-gate } 5347c478bd9Sstevel@tonic-gate } 5357c478bd9Sstevel@tonic-gate } 5367c478bd9Sstevel@tonic-gate 5377c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 5387c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 5397c478bd9Sstevel@tonic-gate prev = susprev; 5407c478bd9Sstevel@tonic-gate suspp = NULL; 5417c478bd9Sstevel@tonic-gate } 5427c478bd9Sstevel@tonic-gate if (ulwp == NULL) { 5437c478bd9Sstevel@tonic-gate if (more != NULL) 5447c478bd9Sstevel@tonic-gate *more = 0; 5457c478bd9Sstevel@tonic-gate return (NULL); 5467c478bd9Sstevel@tonic-gate } 5477c478bd9Sstevel@tonic-gate 5487c478bd9Sstevel@tonic-gate if (prevp != NULL) 5497c478bd9Sstevel@tonic-gate *prevp = prev; 5507c478bd9Sstevel@tonic-gate if (more == NULL) 5517c478bd9Sstevel@tonic-gate return (ulwpp); 5527c478bd9Sstevel@tonic-gate 5537c478bd9Sstevel@tonic-gate /* 5547c478bd9Sstevel@tonic-gate * Scan the remainder of the queue for another waiter. 5557c478bd9Sstevel@tonic-gate */ 5567c478bd9Sstevel@tonic-gate if (suspp != NULL) { 5577c478bd9Sstevel@tonic-gate *more = 1; 5587c478bd9Sstevel@tonic-gate return (ulwpp); 5597c478bd9Sstevel@tonic-gate } 5607c478bd9Sstevel@tonic-gate for (ulwp = ulwp->ul_link; ulwp != NULL; ulwp = ulwp->ul_link) { 5617c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) { 5627c478bd9Sstevel@tonic-gate *more = 1; 5637c478bd9Sstevel@tonic-gate return (ulwpp); 5647c478bd9Sstevel@tonic-gate } 5657c478bd9Sstevel@tonic-gate } 5667c478bd9Sstevel@tonic-gate 5677c478bd9Sstevel@tonic-gate *more = 0; 5687c478bd9Sstevel@tonic-gate return (ulwpp); 5697c478bd9Sstevel@tonic-gate } 5707c478bd9Sstevel@tonic-gate 5717c478bd9Sstevel@tonic-gate ulwp_t * 5727c478bd9Sstevel@tonic-gate dequeue(queue_head_t *qp, void *wchan, int *more) 5737c478bd9Sstevel@tonic-gate { 5747c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 5757c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 5767c478bd9Sstevel@tonic-gate ulwp_t *prev; 5777c478bd9Sstevel@tonic-gate 5787c478bd9Sstevel@tonic-gate if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL) 5797c478bd9Sstevel@tonic-gate return (NULL); 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate /* 5827c478bd9Sstevel@tonic-gate * Dequeue the waiter. 5837c478bd9Sstevel@tonic-gate */ 5847c478bd9Sstevel@tonic-gate ulwp = *ulwpp; 5857c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 5867c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 5877c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 5887c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 5897c478bd9Sstevel@tonic-gate qp->qh_qlen--; 5907c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 5917c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 5927c478bd9Sstevel@tonic-gate 5937c478bd9Sstevel@tonic-gate return (ulwp); 5947c478bd9Sstevel@tonic-gate } 5957c478bd9Sstevel@tonic-gate 5967c478bd9Sstevel@tonic-gate /* 5977c478bd9Sstevel@tonic-gate * Return a pointer to the highest priority thread sleeping on wchan. 5987c478bd9Sstevel@tonic-gate */ 5997c478bd9Sstevel@tonic-gate ulwp_t * 6007c478bd9Sstevel@tonic-gate queue_waiter(queue_head_t *qp, void *wchan) 6017c478bd9Sstevel@tonic-gate { 6027c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 6037c478bd9Sstevel@tonic-gate 6047c478bd9Sstevel@tonic-gate if ((ulwpp = queue_slot(qp, wchan, NULL, NULL)) == NULL) 6057c478bd9Sstevel@tonic-gate return (NULL); 6067c478bd9Sstevel@tonic-gate return (*ulwpp); 6077c478bd9Sstevel@tonic-gate } 6087c478bd9Sstevel@tonic-gate 6097c478bd9Sstevel@tonic-gate uint8_t 6107c478bd9Sstevel@tonic-gate dequeue_self(queue_head_t *qp, void *wchan) 6117c478bd9Sstevel@tonic-gate { 6127c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 6137c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 6147c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 6157c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 6167c478bd9Sstevel@tonic-gate int found = 0; 6177c478bd9Sstevel@tonic-gate int more = 0; 6187c478bd9Sstevel@tonic-gate 6197c478bd9Sstevel@tonic-gate ASSERT(MUTEX_OWNED(&qp->qh_lock, self)); 6207c478bd9Sstevel@tonic-gate 6217c478bd9Sstevel@tonic-gate /* find self on the sleep queue */ 6227c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 6237c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 6247c478bd9Sstevel@tonic-gate if (ulwp == self) { 6257c478bd9Sstevel@tonic-gate /* dequeue ourself */ 6267c478bd9Sstevel@tonic-gate *ulwpp = self->ul_link; 6277c478bd9Sstevel@tonic-gate if (qp->qh_tail == self) 6287c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 6297c478bd9Sstevel@tonic-gate qp->qh_qlen--; 6307c478bd9Sstevel@tonic-gate ASSERT(self->ul_wchan == wchan); 6317c478bd9Sstevel@tonic-gate self->ul_cvmutex = NULL; 6327c478bd9Sstevel@tonic-gate self->ul_sleepq = NULL; 6337c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 6347c478bd9Sstevel@tonic-gate self->ul_cv_wake = 0; 6357c478bd9Sstevel@tonic-gate self->ul_link = NULL; 6367c478bd9Sstevel@tonic-gate found = 1; 6377c478bd9Sstevel@tonic-gate break; 6387c478bd9Sstevel@tonic-gate } 6397c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 6407c478bd9Sstevel@tonic-gate more = 1; 6417c478bd9Sstevel@tonic-gate } 6427c478bd9Sstevel@tonic-gate 6437c478bd9Sstevel@tonic-gate if (!found) 6447c478bd9Sstevel@tonic-gate thr_panic("dequeue_self(): curthread not found on queue"); 6457c478bd9Sstevel@tonic-gate 6467c478bd9Sstevel@tonic-gate if (more) 6477c478bd9Sstevel@tonic-gate return (1); 6487c478bd9Sstevel@tonic-gate 6497c478bd9Sstevel@tonic-gate /* scan the remainder of the queue for another waiter */ 6507c478bd9Sstevel@tonic-gate for (ulwp = *ulwpp; ulwp != NULL; ulwp = ulwp->ul_link) { 6517c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == wchan) 6527c478bd9Sstevel@tonic-gate return (1); 6537c478bd9Sstevel@tonic-gate } 6547c478bd9Sstevel@tonic-gate 6557c478bd9Sstevel@tonic-gate return (0); 6567c478bd9Sstevel@tonic-gate } 6577c478bd9Sstevel@tonic-gate 6587c478bd9Sstevel@tonic-gate /* 6597c478bd9Sstevel@tonic-gate * Called from call_user_handler() and _thrp_suspend() to take 6607c478bd9Sstevel@tonic-gate * ourself off of our sleep queue so we can grab locks. 6617c478bd9Sstevel@tonic-gate */ 6627c478bd9Sstevel@tonic-gate void 6637c478bd9Sstevel@tonic-gate unsleep_self(void) 6647c478bd9Sstevel@tonic-gate { 6657c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 6667c478bd9Sstevel@tonic-gate queue_head_t *qp; 6677c478bd9Sstevel@tonic-gate 6687c478bd9Sstevel@tonic-gate /* 6697c478bd9Sstevel@tonic-gate * Calling enter_critical()/exit_critical() here would lead 6707c478bd9Sstevel@tonic-gate * to recursion. Just manipulate self->ul_critical directly. 6717c478bd9Sstevel@tonic-gate */ 6727c478bd9Sstevel@tonic-gate self->ul_critical++; 6737c478bd9Sstevel@tonic-gate self->ul_writer = 0; 6747c478bd9Sstevel@tonic-gate while (self->ul_sleepq != NULL) { 6757c478bd9Sstevel@tonic-gate qp = queue_lock(self->ul_wchan, self->ul_qtype); 6767c478bd9Sstevel@tonic-gate /* 6777c478bd9Sstevel@tonic-gate * We may have been moved from a CV queue to a 6787c478bd9Sstevel@tonic-gate * mutex queue while we were attempting queue_lock(). 6797c478bd9Sstevel@tonic-gate * If so, just loop around and try again. 6807c478bd9Sstevel@tonic-gate * dequeue_self() clears self->ul_sleepq. 6817c478bd9Sstevel@tonic-gate */ 6827c478bd9Sstevel@tonic-gate if (qp == self->ul_sleepq) 6837c478bd9Sstevel@tonic-gate (void) dequeue_self(qp, self->ul_wchan); 6847c478bd9Sstevel@tonic-gate queue_unlock(qp); 6857c478bd9Sstevel@tonic-gate } 6867c478bd9Sstevel@tonic-gate self->ul_critical--; 6877c478bd9Sstevel@tonic-gate } 6887c478bd9Sstevel@tonic-gate 6897c478bd9Sstevel@tonic-gate /* 6907c478bd9Sstevel@tonic-gate * Common code for calling the the ___lwp_mutex_timedlock() system call. 6917c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 6927c478bd9Sstevel@tonic-gate */ 6937c478bd9Sstevel@tonic-gate int 6947c478bd9Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp) 6957c478bd9Sstevel@tonic-gate { 6967c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 6977c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 6987c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 6997c478bd9Sstevel@tonic-gate int error; 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 7027c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 7037c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 7047c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 7057c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 7067c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 7077c478bd9Sstevel@tonic-gate } 7087c478bd9Sstevel@tonic-gate if (msp) { 7097c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 7107c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 7117c478bd9Sstevel@tonic-gate } 7127c478bd9Sstevel@tonic-gate 7137c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 7147c478bd9Sstevel@tonic-gate 7157c478bd9Sstevel@tonic-gate for (;;) { 7167c478bd9Sstevel@tonic-gate if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0) { 7177c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 7187c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, error); 7197c478bd9Sstevel@tonic-gate break; 7207c478bd9Sstevel@tonic-gate } 7217c478bd9Sstevel@tonic-gate 7227c478bd9Sstevel@tonic-gate if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) { 7237c478bd9Sstevel@tonic-gate /* 7247c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 7257c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 7267c478bd9Sstevel@tonic-gate */ 7277c478bd9Sstevel@tonic-gate enter_critical(self); 7287c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 7297c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 7307c478bd9Sstevel@tonic-gate exit_critical(self); 7317c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 7327c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 7337c478bd9Sstevel@tonic-gate 0, 0); 7347c478bd9Sstevel@tonic-gate break; 7357c478bd9Sstevel@tonic-gate } 7367c478bd9Sstevel@tonic-gate exit_critical(self); 7377c478bd9Sstevel@tonic-gate } else { 7387c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 7397c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 7407c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 7417c478bd9Sstevel@tonic-gate break; 7427c478bd9Sstevel@tonic-gate } 7437c478bd9Sstevel@tonic-gate } 7447c478bd9Sstevel@tonic-gate if (msp) 7457c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 7467c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 7477c478bd9Sstevel@tonic-gate self->ul_sp = 0; 7487c478bd9Sstevel@tonic-gate 7497c478bd9Sstevel@tonic-gate return (error); 7507c478bd9Sstevel@tonic-gate } 7517c478bd9Sstevel@tonic-gate 7527c478bd9Sstevel@tonic-gate /* 7537c478bd9Sstevel@tonic-gate * Common code for calling the ___lwp_mutex_trylock() system call. 7547c478bd9Sstevel@tonic-gate * Returns with mutex_owner and mutex_ownerpid set correctly. 7557c478bd9Sstevel@tonic-gate */ 7567c478bd9Sstevel@tonic-gate int 7577c478bd9Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp) 7587c478bd9Sstevel@tonic-gate { 7597c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 7607c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 7617c478bd9Sstevel@tonic-gate int error; 7627c478bd9Sstevel@tonic-gate 7637c478bd9Sstevel@tonic-gate for (;;) { 7647c478bd9Sstevel@tonic-gate if ((error = ___lwp_mutex_trylock(mp)) != 0) { 7657c478bd9Sstevel@tonic-gate if (error != EBUSY) { 7667c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 7677c478bd9Sstevel@tonic-gate error); 7687c478bd9Sstevel@tonic-gate } 7697c478bd9Sstevel@tonic-gate break; 7707c478bd9Sstevel@tonic-gate } 7717c478bd9Sstevel@tonic-gate 7727c478bd9Sstevel@tonic-gate if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) { 7737c478bd9Sstevel@tonic-gate /* 7747c478bd9Sstevel@tonic-gate * Defend against forkall(). We may be the child, 7757c478bd9Sstevel@tonic-gate * in which case we don't actually own the mutex. 7767c478bd9Sstevel@tonic-gate */ 7777c478bd9Sstevel@tonic-gate enter_critical(self); 7787c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid == udp->pid) { 7797c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 7807c478bd9Sstevel@tonic-gate exit_critical(self); 7817c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 7827c478bd9Sstevel@tonic-gate 0, 0); 7837c478bd9Sstevel@tonic-gate break; 7847c478bd9Sstevel@tonic-gate } 7857c478bd9Sstevel@tonic-gate exit_critical(self); 7867c478bd9Sstevel@tonic-gate } else { 7877c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 7887c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 7897c478bd9Sstevel@tonic-gate break; 7907c478bd9Sstevel@tonic-gate } 7917c478bd9Sstevel@tonic-gate } 7927c478bd9Sstevel@tonic-gate 7937c478bd9Sstevel@tonic-gate return (error); 7947c478bd9Sstevel@tonic-gate } 7957c478bd9Sstevel@tonic-gate 7967c478bd9Sstevel@tonic-gate volatile sc_shared_t * 7977c478bd9Sstevel@tonic-gate setup_schedctl(void) 7987c478bd9Sstevel@tonic-gate { 7997c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 8007c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 8017c478bd9Sstevel@tonic-gate sc_shared_t *tmp; 8027c478bd9Sstevel@tonic-gate 8037c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */ 8047c478bd9Sstevel@tonic-gate !self->ul_vfork && /* not a child of vfork() */ 8057c478bd9Sstevel@tonic-gate !self->ul_schedctl_called) { /* haven't been called before */ 8067c478bd9Sstevel@tonic-gate enter_critical(self); 8077c478bd9Sstevel@tonic-gate self->ul_schedctl_called = &self->ul_uberdata->uberflags; 8087c478bd9Sstevel@tonic-gate if ((tmp = __schedctl()) != (sc_shared_t *)(-1)) 8097c478bd9Sstevel@tonic-gate self->ul_schedctl = scp = tmp; 8107c478bd9Sstevel@tonic-gate exit_critical(self); 8117c478bd9Sstevel@tonic-gate } 8127c478bd9Sstevel@tonic-gate /* 8137c478bd9Sstevel@tonic-gate * Unless the call to setup_schedctl() is surrounded 8147c478bd9Sstevel@tonic-gate * by enter_critical()/exit_critical(), the address 8157c478bd9Sstevel@tonic-gate * we are returning could be invalid due to a forkall() 8167c478bd9Sstevel@tonic-gate * having occurred in another thread. 8177c478bd9Sstevel@tonic-gate */ 8187c478bd9Sstevel@tonic-gate return (scp); 8197c478bd9Sstevel@tonic-gate } 8207c478bd9Sstevel@tonic-gate 8217c478bd9Sstevel@tonic-gate /* 8227c478bd9Sstevel@tonic-gate * Interfaces from libsched, incorporated into libc. 8237c478bd9Sstevel@tonic-gate * libsched.so.1 is now a filter library onto libc. 8247c478bd9Sstevel@tonic-gate */ 8257c478bd9Sstevel@tonic-gate #pragma weak schedctl_lookup = _schedctl_init 8267c478bd9Sstevel@tonic-gate #pragma weak _schedctl_lookup = _schedctl_init 8277c478bd9Sstevel@tonic-gate #pragma weak schedctl_init = _schedctl_init 8287c478bd9Sstevel@tonic-gate schedctl_t * 8297c478bd9Sstevel@tonic-gate _schedctl_init(void) 8307c478bd9Sstevel@tonic-gate { 8317c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = setup_schedctl(); 8327c478bd9Sstevel@tonic-gate return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl); 8337c478bd9Sstevel@tonic-gate } 8347c478bd9Sstevel@tonic-gate 8357c478bd9Sstevel@tonic-gate #pragma weak schedctl_exit = _schedctl_exit 8367c478bd9Sstevel@tonic-gate void 8377c478bd9Sstevel@tonic-gate _schedctl_exit(void) 8387c478bd9Sstevel@tonic-gate { 8397c478bd9Sstevel@tonic-gate } 8407c478bd9Sstevel@tonic-gate 8417c478bd9Sstevel@tonic-gate /* 8427c478bd9Sstevel@tonic-gate * Contract private interface for java. 8437c478bd9Sstevel@tonic-gate * Set up the schedctl data if it doesn't exist yet. 8447c478bd9Sstevel@tonic-gate * Return a pointer to the pointer to the schedctl data. 8457c478bd9Sstevel@tonic-gate */ 8467c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile * 8477c478bd9Sstevel@tonic-gate _thr_schedctl(void) 8487c478bd9Sstevel@tonic-gate { 8497c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 8507c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile *ptr; 8517c478bd9Sstevel@tonic-gate 8527c478bd9Sstevel@tonic-gate if (self->ul_vfork) 8537c478bd9Sstevel@tonic-gate return (NULL); 8547c478bd9Sstevel@tonic-gate if (*(ptr = &self->ul_schedctl) == NULL) 8557c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 8567c478bd9Sstevel@tonic-gate return (ptr); 8577c478bd9Sstevel@tonic-gate } 8587c478bd9Sstevel@tonic-gate 8597c478bd9Sstevel@tonic-gate /* 8607c478bd9Sstevel@tonic-gate * Block signals and attempt to block preemption. 8617c478bd9Sstevel@tonic-gate * no_preempt()/preempt() must be used in pairs but can be nested. 8627c478bd9Sstevel@tonic-gate */ 8637c478bd9Sstevel@tonic-gate void 8647c478bd9Sstevel@tonic-gate no_preempt(ulwp_t *self) 8657c478bd9Sstevel@tonic-gate { 8667c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 8677c478bd9Sstevel@tonic-gate 8687c478bd9Sstevel@tonic-gate if (self->ul_preempt++ == 0) { 8697c478bd9Sstevel@tonic-gate enter_critical(self); 8707c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL || 8717c478bd9Sstevel@tonic-gate (scp = setup_schedctl()) != NULL) { 8727c478bd9Sstevel@tonic-gate /* 8737c478bd9Sstevel@tonic-gate * Save the pre-existing preempt value. 8747c478bd9Sstevel@tonic-gate */ 8757c478bd9Sstevel@tonic-gate self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt; 8767c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = 1; 8777c478bd9Sstevel@tonic-gate } 8787c478bd9Sstevel@tonic-gate } 8797c478bd9Sstevel@tonic-gate } 8807c478bd9Sstevel@tonic-gate 8817c478bd9Sstevel@tonic-gate /* 8827c478bd9Sstevel@tonic-gate * Undo the effects of no_preempt(). 8837c478bd9Sstevel@tonic-gate */ 8847c478bd9Sstevel@tonic-gate void 8857c478bd9Sstevel@tonic-gate preempt(ulwp_t *self) 8867c478bd9Sstevel@tonic-gate { 8877c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 8887c478bd9Sstevel@tonic-gate 8897c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt > 0); 8907c478bd9Sstevel@tonic-gate if (--self->ul_preempt == 0) { 8917c478bd9Sstevel@tonic-gate if ((scp = self->ul_schedctl) != NULL) { 8927c478bd9Sstevel@tonic-gate /* 8937c478bd9Sstevel@tonic-gate * Restore the pre-existing preempt value. 8947c478bd9Sstevel@tonic-gate */ 8957c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt; 8967c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield && 8977c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_nopreempt == 0) { 8987c478bd9Sstevel@tonic-gate lwp_yield(); 8997c478bd9Sstevel@tonic-gate if (scp->sc_preemptctl.sc_yield) { 9007c478bd9Sstevel@tonic-gate /* 9017c478bd9Sstevel@tonic-gate * Shouldn't happen. This is either 9027c478bd9Sstevel@tonic-gate * a race condition or the thread 9037c478bd9Sstevel@tonic-gate * just entered the real-time class. 9047c478bd9Sstevel@tonic-gate */ 9057c478bd9Sstevel@tonic-gate lwp_yield(); 9067c478bd9Sstevel@tonic-gate scp->sc_preemptctl.sc_yield = 0; 9077c478bd9Sstevel@tonic-gate } 9087c478bd9Sstevel@tonic-gate } 9097c478bd9Sstevel@tonic-gate } 9107c478bd9Sstevel@tonic-gate exit_critical(self); 9117c478bd9Sstevel@tonic-gate } 9127c478bd9Sstevel@tonic-gate } 9137c478bd9Sstevel@tonic-gate 9147c478bd9Sstevel@tonic-gate /* 9157c478bd9Sstevel@tonic-gate * If a call to preempt() would cause the current thread to yield or to 9167c478bd9Sstevel@tonic-gate * take deferred actions in exit_critical(), then unpark the specified 9177c478bd9Sstevel@tonic-gate * lwp so it can run while we delay. Return the original lwpid if the 9187c478bd9Sstevel@tonic-gate * unpark was not performed, else return zero. The tests are a repeat 9197c478bd9Sstevel@tonic-gate * of some of the tests in preempt(), above. This is a statistical 9207c478bd9Sstevel@tonic-gate * optimization solely for cond_sleep_queue(), below. 9217c478bd9Sstevel@tonic-gate */ 9227c478bd9Sstevel@tonic-gate static lwpid_t 9237c478bd9Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid) 9247c478bd9Sstevel@tonic-gate { 9257c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp = self->ul_schedctl; 9267c478bd9Sstevel@tonic-gate 9277c478bd9Sstevel@tonic-gate ASSERT(self->ul_preempt == 1 && self->ul_critical > 0); 9287c478bd9Sstevel@tonic-gate if ((scp != NULL && scp->sc_preemptctl.sc_yield) || 9297c478bd9Sstevel@tonic-gate (self->ul_curplease && self->ul_critical == 1)) { 9307c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 9317c478bd9Sstevel@tonic-gate lwpid = 0; 9327c478bd9Sstevel@tonic-gate } 9337c478bd9Sstevel@tonic-gate return (lwpid); 9347c478bd9Sstevel@tonic-gate } 9357c478bd9Sstevel@tonic-gate 9367c478bd9Sstevel@tonic-gate /* 9377c478bd9Sstevel@tonic-gate * Spin for a while, trying to grab the lock. We know that we 9387c478bd9Sstevel@tonic-gate * failed set_lock_byte(&mp->mutex_lockw) once before coming here. 9397c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 9407c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread. 9417c478bd9Sstevel@tonic-gate */ 9427c478bd9Sstevel@tonic-gate int 9437c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mutex_t *mp) 9447c478bd9Sstevel@tonic-gate { 9457c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 9467c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 9477c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 9487c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 9497c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 9507c478bd9Sstevel@tonic-gate int count, max = self->ul_adaptive_spin; 9517c478bd9Sstevel@tonic-gate 9527c478bd9Sstevel@tonic-gate ASSERT(!(mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))); 9537c478bd9Sstevel@tonic-gate 9547c478bd9Sstevel@tonic-gate if (max == 0 || (mp->mutex_spinners >= self->ul_max_spinners)) 9557c478bd9Sstevel@tonic-gate return (EBUSY); 9567c478bd9Sstevel@tonic-gate 9577c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 9587c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 9597c478bd9Sstevel@tonic-gate 9607c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__spin, mp); 9617c478bd9Sstevel@tonic-gate 9627c478bd9Sstevel@tonic-gate /* 9637c478bd9Sstevel@tonic-gate * This spin loop is unfair to lwps that have already dropped into 9647c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a highly-contended mutex. 9657c478bd9Sstevel@tonic-gate * This is just too bad. The adaptive spin algorithm is intended 9667c478bd9Sstevel@tonic-gate * to allow programs with highly-contended locks (that is, broken 9677c478bd9Sstevel@tonic-gate * programs) to execute with reasonable speed despite their contention. 9687c478bd9Sstevel@tonic-gate * Being fair would reduce the speed of such programs and well-written 9697c478bd9Sstevel@tonic-gate * programs will not suffer in any case. 9707c478bd9Sstevel@tonic-gate */ 9717c478bd9Sstevel@tonic-gate enter_critical(self); /* protects ul_schedctl */ 9727c478bd9Sstevel@tonic-gate incr32(&mp->mutex_spinners); 9737c478bd9Sstevel@tonic-gate for (count = 0; count < max; count++) { 9747c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) { 9757c478bd9Sstevel@tonic-gate *ownerp = (uintptr_t)self; 9767c478bd9Sstevel@tonic-gate decr32(&mp->mutex_spinners); 9777c478bd9Sstevel@tonic-gate exit_critical(self); 9787c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__spun, 1, count); 9797c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count); 9807c478bd9Sstevel@tonic-gate return (0); 9817c478bd9Sstevel@tonic-gate } 9827c478bd9Sstevel@tonic-gate SMT_PAUSE(); 9837c478bd9Sstevel@tonic-gate /* 9847c478bd9Sstevel@tonic-gate * Stop spinning if the mutex owner is not running on 9857c478bd9Sstevel@tonic-gate * a processor; it will not drop the lock any time soon 9867c478bd9Sstevel@tonic-gate * and we would just be wasting time to keep spinning. 9877c478bd9Sstevel@tonic-gate * 9887c478bd9Sstevel@tonic-gate * Note that we are looking at another thread (ulwp_t) 9897c478bd9Sstevel@tonic-gate * without ensuring that the other thread does not exit. 9907c478bd9Sstevel@tonic-gate * The scheme relies on ulwp_t structures never being 9917c478bd9Sstevel@tonic-gate * deallocated by the library (the library employs a free 9927c478bd9Sstevel@tonic-gate * list of ulwp_t structs that are reused when new threads 9937c478bd9Sstevel@tonic-gate * are created) and on schedctl shared memory never being 9947c478bd9Sstevel@tonic-gate * deallocated once created via __schedctl(). 9957c478bd9Sstevel@tonic-gate * 9967c478bd9Sstevel@tonic-gate * Thus, the worst that can happen when the spinning thread 9977c478bd9Sstevel@tonic-gate * looks at the owner's schedctl data is that it is looking 9987c478bd9Sstevel@tonic-gate * at some other thread's schedctl data. This almost never 9997c478bd9Sstevel@tonic-gate * happens and is benign when it does. 10007c478bd9Sstevel@tonic-gate */ 10017c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 10027c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 10037c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 10047c478bd9Sstevel@tonic-gate break; 10057c478bd9Sstevel@tonic-gate } 10067c478bd9Sstevel@tonic-gate decr32(&mp->mutex_spinners); 10077c478bd9Sstevel@tonic-gate exit_critical(self); 10087c478bd9Sstevel@tonic-gate 10097c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__spun, 0, count); 10107c478bd9Sstevel@tonic-gate 10117c478bd9Sstevel@tonic-gate return (EBUSY); 10127c478bd9Sstevel@tonic-gate } 10137c478bd9Sstevel@tonic-gate 10147c478bd9Sstevel@tonic-gate /* 10157c478bd9Sstevel@tonic-gate * Same as mutex_trylock_adaptive(), except specifically for queue locks. 10167c478bd9Sstevel@tonic-gate * The owner field is not set here; the caller (spin_lock_set()) sets it. 10177c478bd9Sstevel@tonic-gate */ 10187c478bd9Sstevel@tonic-gate int 10197c478bd9Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp) 10207c478bd9Sstevel@tonic-gate { 10217c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 10227c478bd9Sstevel@tonic-gate volatile sc_shared_t *scp; 10237c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 10247c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 10257c478bd9Sstevel@tonic-gate int count = curthread->ul_queue_spin; 10267c478bd9Sstevel@tonic-gate 10277c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 10287c478bd9Sstevel@tonic-gate 10297c478bd9Sstevel@tonic-gate if (count == 0) 10307c478bd9Sstevel@tonic-gate return (EBUSY); 10317c478bd9Sstevel@tonic-gate 10327c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 10337c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 10347c478bd9Sstevel@tonic-gate while (--count >= 0) { 10357c478bd9Sstevel@tonic-gate if (*lockp == 0 && set_lock_byte(lockp) == 0) 10367c478bd9Sstevel@tonic-gate return (0); 10377c478bd9Sstevel@tonic-gate SMT_PAUSE(); 10387c478bd9Sstevel@tonic-gate if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL && 10397c478bd9Sstevel@tonic-gate ((scp = ulwp->ul_schedctl) == NULL || 10407c478bd9Sstevel@tonic-gate scp->sc_state != SC_ONPROC)) 10417c478bd9Sstevel@tonic-gate break; 10427c478bd9Sstevel@tonic-gate } 10437c478bd9Sstevel@tonic-gate 10447c478bd9Sstevel@tonic-gate return (EBUSY); 10457c478bd9Sstevel@tonic-gate } 10467c478bd9Sstevel@tonic-gate 10477c478bd9Sstevel@tonic-gate /* 10487c478bd9Sstevel@tonic-gate * Like mutex_trylock_adaptive(), but for process-shared mutexes. 10497c478bd9Sstevel@tonic-gate * Spin for a while, trying to grab the lock. We know that we 10507c478bd9Sstevel@tonic-gate * failed set_lock_byte(&mp->mutex_lockw) once before coming here. 10517c478bd9Sstevel@tonic-gate * If this fails, return EBUSY and let the caller deal with it. 10527c478bd9Sstevel@tonic-gate * If this succeeds, return 0 with mutex_owner set to curthread 10537c478bd9Sstevel@tonic-gate * and mutex_ownerpid set to the current pid. 10547c478bd9Sstevel@tonic-gate */ 10557c478bd9Sstevel@tonic-gate int 10567c478bd9Sstevel@tonic-gate mutex_trylock_process(mutex_t *mp) 10577c478bd9Sstevel@tonic-gate { 10587c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 10597c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 10607c478bd9Sstevel@tonic-gate int count; 10617c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 10627c478bd9Sstevel@tonic-gate volatile uint64_t *ownerp; 10637c478bd9Sstevel@tonic-gate volatile int32_t *pidp; 10647c478bd9Sstevel@tonic-gate pid_t pid, newpid; 10657c478bd9Sstevel@tonic-gate uint64_t owner, newowner; 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate if ((count = ncpus) == 0) 10687c478bd9Sstevel@tonic-gate count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 10697c478bd9Sstevel@tonic-gate count = (count > 1)? self->ul_adaptive_spin : 0; 10707c478bd9Sstevel@tonic-gate 10717c478bd9Sstevel@tonic-gate ASSERT((mp->mutex_type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 10727c478bd9Sstevel@tonic-gate USYNC_PROCESS); 10737c478bd9Sstevel@tonic-gate 10747c478bd9Sstevel@tonic-gate if (count == 0) 10757c478bd9Sstevel@tonic-gate return (EBUSY); 10767c478bd9Sstevel@tonic-gate 10777c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 10787c478bd9Sstevel@tonic-gate ownerp = (volatile uint64_t *)&mp->mutex_owner; 10797c478bd9Sstevel@tonic-gate pidp = (volatile int32_t *)&mp->mutex_ownerpid; 10807c478bd9Sstevel@tonic-gate owner = *ownerp; 10817c478bd9Sstevel@tonic-gate pid = *pidp; 10827c478bd9Sstevel@tonic-gate /* 10837c478bd9Sstevel@tonic-gate * This is a process-shared mutex. 10847c478bd9Sstevel@tonic-gate * We cannot know if the owner is running on a processor. 10857c478bd9Sstevel@tonic-gate * We just spin and hope that it is on a processor. 10867c478bd9Sstevel@tonic-gate */ 10877c478bd9Sstevel@tonic-gate while (--count >= 0) { 10887c478bd9Sstevel@tonic-gate if (*lockp == 0) { 10897c478bd9Sstevel@tonic-gate enter_critical(self); 10907c478bd9Sstevel@tonic-gate if (set_lock_byte(lockp) == 0) { 10917c478bd9Sstevel@tonic-gate *ownerp = (uintptr_t)self; 10927c478bd9Sstevel@tonic-gate *pidp = udp->pid; 10937c478bd9Sstevel@tonic-gate exit_critical(self); 10947c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 10957c478bd9Sstevel@tonic-gate 0, 0); 10967c478bd9Sstevel@tonic-gate return (0); 10977c478bd9Sstevel@tonic-gate } 10987c478bd9Sstevel@tonic-gate exit_critical(self); 10997c478bd9Sstevel@tonic-gate } else if ((newowner = *ownerp) == owner && 11007c478bd9Sstevel@tonic-gate (newpid = *pidp) == pid) { 11017c478bd9Sstevel@tonic-gate SMT_PAUSE(); 11027c478bd9Sstevel@tonic-gate continue; 11037c478bd9Sstevel@tonic-gate } 11047c478bd9Sstevel@tonic-gate /* 11057c478bd9Sstevel@tonic-gate * The owner of the lock changed; start the count over again. 11067c478bd9Sstevel@tonic-gate * This may be too aggressive; it needs testing. 11077c478bd9Sstevel@tonic-gate */ 11087c478bd9Sstevel@tonic-gate owner = newowner; 11097c478bd9Sstevel@tonic-gate pid = newpid; 11107c478bd9Sstevel@tonic-gate count = self->ul_adaptive_spin; 11117c478bd9Sstevel@tonic-gate } 11127c478bd9Sstevel@tonic-gate 11137c478bd9Sstevel@tonic-gate return (EBUSY); 11147c478bd9Sstevel@tonic-gate } 11157c478bd9Sstevel@tonic-gate 11167c478bd9Sstevel@tonic-gate /* 11177c478bd9Sstevel@tonic-gate * Mutex wakeup code for releasing a USYNC_THREAD mutex. 11187c478bd9Sstevel@tonic-gate * Returns the lwpid of the thread that was dequeued, if any. 11197c478bd9Sstevel@tonic-gate * The caller of mutex_wakeup() must call __lwp_unpark(lwpid) 11207c478bd9Sstevel@tonic-gate * to wake up the specified lwp. 11217c478bd9Sstevel@tonic-gate */ 11227c478bd9Sstevel@tonic-gate lwpid_t 11237c478bd9Sstevel@tonic-gate mutex_wakeup(mutex_t *mp) 11247c478bd9Sstevel@tonic-gate { 11257c478bd9Sstevel@tonic-gate lwpid_t lwpid = 0; 11267c478bd9Sstevel@tonic-gate queue_head_t *qp; 11277c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 11287c478bd9Sstevel@tonic-gate int more; 11297c478bd9Sstevel@tonic-gate 11307c478bd9Sstevel@tonic-gate /* 11317c478bd9Sstevel@tonic-gate * Dequeue a waiter from the sleep queue. Don't touch the mutex 11327c478bd9Sstevel@tonic-gate * waiters bit if no one was found on the queue because the mutex 11337c478bd9Sstevel@tonic-gate * might have been deallocated or reallocated for another purpose. 11347c478bd9Sstevel@tonic-gate */ 11357c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 11367c478bd9Sstevel@tonic-gate if ((ulwp = dequeue(qp, mp, &more)) != NULL) { 11377c478bd9Sstevel@tonic-gate lwpid = ulwp->ul_lwpid; 11387c478bd9Sstevel@tonic-gate mp->mutex_waiters = (more? 1 : 0); 11397c478bd9Sstevel@tonic-gate } 11407c478bd9Sstevel@tonic-gate queue_unlock(qp); 11417c478bd9Sstevel@tonic-gate return (lwpid); 11427c478bd9Sstevel@tonic-gate } 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate /* 11457c478bd9Sstevel@tonic-gate * Spin for a while, testing to see if the lock has been grabbed. 11467c478bd9Sstevel@tonic-gate * If this fails, call mutex_wakeup() to release a waiter. 11477c478bd9Sstevel@tonic-gate */ 11487c478bd9Sstevel@tonic-gate lwpid_t 11497c478bd9Sstevel@tonic-gate mutex_unlock_queue(mutex_t *mp) 11507c478bd9Sstevel@tonic-gate { 11517c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 11527c478bd9Sstevel@tonic-gate uint32_t *lockw = &mp->mutex_lockword; 11537c478bd9Sstevel@tonic-gate lwpid_t lwpid; 11547c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 11557c478bd9Sstevel@tonic-gate volatile uint32_t *spinp; 11567c478bd9Sstevel@tonic-gate int count; 11577c478bd9Sstevel@tonic-gate 11587c478bd9Sstevel@tonic-gate /* 11597c478bd9Sstevel@tonic-gate * We use the swap primitive to clear the lock, but we must 11607c478bd9Sstevel@tonic-gate * atomically retain the waiters bit for the remainder of this 11617c478bd9Sstevel@tonic-gate * code to work. We first check to see if the waiters bit is 11627c478bd9Sstevel@tonic-gate * set and if so clear the lock by swapping in a word containing 11637c478bd9Sstevel@tonic-gate * only the waiters bit. This could produce a false positive test 11647c478bd9Sstevel@tonic-gate * for whether there are waiters that need to be waked up, but 11657c478bd9Sstevel@tonic-gate * this just causes an extra call to mutex_wakeup() to do nothing. 11667c478bd9Sstevel@tonic-gate * The opposite case is more delicate: If there are no waiters, 11677c478bd9Sstevel@tonic-gate * we swap in a zero lock byte and a zero waiters bit. The result 11687c478bd9Sstevel@tonic-gate * of the swap could indicate that there really was a waiter so in 11697c478bd9Sstevel@tonic-gate * this case we go directly to mutex_wakeup() without performing 11707c478bd9Sstevel@tonic-gate * any of the adaptive code because the waiter bit has been cleared 11717c478bd9Sstevel@tonic-gate * and the adaptive code is unreliable in this case. 11727c478bd9Sstevel@tonic-gate */ 11737c478bd9Sstevel@tonic-gate if (!(*lockw & WAITERMASK)) { /* no waiter exists right now */ 11747c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 11757c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 11767c478bd9Sstevel@tonic-gate if (!(swap32(lockw, 0) & WAITERMASK)) /* still no waiters */ 11777c478bd9Sstevel@tonic-gate return (0); 11787c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 11797c478bd9Sstevel@tonic-gate lwpid = mutex_wakeup(mp); 11807c478bd9Sstevel@tonic-gate } else { 11817c478bd9Sstevel@tonic-gate no_preempt(self); /* ensure a prompt wakeup */ 11827c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 11837c478bd9Sstevel@tonic-gate spinp = (volatile uint32_t *)&mp->mutex_spinners; 11847c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 11857c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 11867c478bd9Sstevel@tonic-gate (void) swap32(lockw, WAITER); /* clear lock, retain waiter */ 11877c478bd9Sstevel@tonic-gate 11887c478bd9Sstevel@tonic-gate /* 11897c478bd9Sstevel@tonic-gate * We spin here fewer times than mutex_trylock_adaptive(). 11907c478bd9Sstevel@tonic-gate * We are trying to balance two conflicting goals: 11917c478bd9Sstevel@tonic-gate * 1. Avoid waking up anyone if a spinning thread 11927c478bd9Sstevel@tonic-gate * grabs the lock. 11937c478bd9Sstevel@tonic-gate * 2. Wake up a sleeping thread promptly to get on 11947c478bd9Sstevel@tonic-gate * with useful work. 11957c478bd9Sstevel@tonic-gate * We don't spin at all if there is no acquiring spinner; 11967c478bd9Sstevel@tonic-gate * (mp->mutex_spinners is non-zero if there are spinners). 11977c478bd9Sstevel@tonic-gate */ 11987c478bd9Sstevel@tonic-gate for (count = self->ul_release_spin; 11997c478bd9Sstevel@tonic-gate *spinp && count > 0; count--) { 12007c478bd9Sstevel@tonic-gate /* 12017c478bd9Sstevel@tonic-gate * There is a waiter that we will have to wake 12027c478bd9Sstevel@tonic-gate * up unless someone else grabs the lock while 12037c478bd9Sstevel@tonic-gate * we are busy spinning. Like the spin loop in 12047c478bd9Sstevel@tonic-gate * mutex_trylock_adaptive(), this spin loop is 12057c478bd9Sstevel@tonic-gate * unfair to lwps that have already dropped into 12067c478bd9Sstevel@tonic-gate * the kernel to sleep. They will starve on a 12077c478bd9Sstevel@tonic-gate * highly-contended mutex. Too bad. 12087c478bd9Sstevel@tonic-gate */ 12097c478bd9Sstevel@tonic-gate if (*lockp != 0) { /* somebody grabbed the lock */ 12107c478bd9Sstevel@tonic-gate preempt(self); 12117c478bd9Sstevel@tonic-gate return (0); 12127c478bd9Sstevel@tonic-gate } 12137c478bd9Sstevel@tonic-gate SMT_PAUSE(); 12147c478bd9Sstevel@tonic-gate } 12157c478bd9Sstevel@tonic-gate 12167c478bd9Sstevel@tonic-gate /* 12177c478bd9Sstevel@tonic-gate * No one grabbed the lock. 12187c478bd9Sstevel@tonic-gate * Wake up some lwp that is waiting for it. 12197c478bd9Sstevel@tonic-gate */ 12207c478bd9Sstevel@tonic-gate mp->mutex_waiters = 0; 12217c478bd9Sstevel@tonic-gate lwpid = mutex_wakeup(mp); 12227c478bd9Sstevel@tonic-gate } 12237c478bd9Sstevel@tonic-gate 12247c478bd9Sstevel@tonic-gate if (lwpid == 0) 12257c478bd9Sstevel@tonic-gate preempt(self); 12267c478bd9Sstevel@tonic-gate return (lwpid); 12277c478bd9Sstevel@tonic-gate } 12287c478bd9Sstevel@tonic-gate 12297c478bd9Sstevel@tonic-gate /* 12307c478bd9Sstevel@tonic-gate * Like mutex_unlock_queue(), but for process-shared mutexes. 12317c478bd9Sstevel@tonic-gate * We tested the waiters field before calling here and it was non-zero. 12327c478bd9Sstevel@tonic-gate */ 12337c478bd9Sstevel@tonic-gate void 12347c478bd9Sstevel@tonic-gate mutex_unlock_process(mutex_t *mp) 12357c478bd9Sstevel@tonic-gate { 12367c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 12377c478bd9Sstevel@tonic-gate int count; 12387c478bd9Sstevel@tonic-gate volatile uint8_t *lockp; 12397c478bd9Sstevel@tonic-gate 12407c478bd9Sstevel@tonic-gate /* 12417c478bd9Sstevel@tonic-gate * See the comments in mutex_unlock_queue(), above. 12427c478bd9Sstevel@tonic-gate */ 12437c478bd9Sstevel@tonic-gate if ((count = ncpus) == 0) 12447c478bd9Sstevel@tonic-gate count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN); 12457c478bd9Sstevel@tonic-gate count = (count > 1)? self->ul_release_spin : 0; 12467c478bd9Sstevel@tonic-gate no_preempt(self); 12477c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 12487c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 12497c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 12507c478bd9Sstevel@tonic-gate if (count == 0) { 12517c478bd9Sstevel@tonic-gate /* clear lock, test waiter */ 12527c478bd9Sstevel@tonic-gate if (!(swap32(&mp->mutex_lockword, 0) & WAITERMASK)) { 12537c478bd9Sstevel@tonic-gate /* no waiters now */ 12547c478bd9Sstevel@tonic-gate preempt(self); 12557c478bd9Sstevel@tonic-gate return; 12567c478bd9Sstevel@tonic-gate } 12577c478bd9Sstevel@tonic-gate } else { 12587c478bd9Sstevel@tonic-gate /* clear lock, retain waiter */ 12597c478bd9Sstevel@tonic-gate (void) swap32(&mp->mutex_lockword, WAITER); 12607c478bd9Sstevel@tonic-gate lockp = (volatile uint8_t *)&mp->mutex_lockw; 12617c478bd9Sstevel@tonic-gate while (--count >= 0) { 12627c478bd9Sstevel@tonic-gate if (*lockp != 0) { 12637c478bd9Sstevel@tonic-gate /* somebody grabbed the lock */ 12647c478bd9Sstevel@tonic-gate preempt(self); 12657c478bd9Sstevel@tonic-gate return; 12667c478bd9Sstevel@tonic-gate } 12677c478bd9Sstevel@tonic-gate SMT_PAUSE(); 12687c478bd9Sstevel@tonic-gate } 12697c478bd9Sstevel@tonic-gate /* 12707c478bd9Sstevel@tonic-gate * We must clear the waiters field before going 12717c478bd9Sstevel@tonic-gate * to the kernel, else it could remain set forever. 12727c478bd9Sstevel@tonic-gate */ 12737c478bd9Sstevel@tonic-gate mp->mutex_waiters = 0; 12747c478bd9Sstevel@tonic-gate } 12757c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 12767c478bd9Sstevel@tonic-gate preempt(self); 12777c478bd9Sstevel@tonic-gate } 12787c478bd9Sstevel@tonic-gate 12797c478bd9Sstevel@tonic-gate /* 12807c478bd9Sstevel@tonic-gate * Return the real priority of a thread. 12817c478bd9Sstevel@tonic-gate */ 12827c478bd9Sstevel@tonic-gate int 12837c478bd9Sstevel@tonic-gate real_priority(ulwp_t *ulwp) 12847c478bd9Sstevel@tonic-gate { 12857c478bd9Sstevel@tonic-gate if (ulwp->ul_epri == 0) 12867c478bd9Sstevel@tonic-gate return (ulwp->ul_mappedpri? ulwp->ul_mappedpri : ulwp->ul_pri); 12877c478bd9Sstevel@tonic-gate return (ulwp->ul_emappedpri? ulwp->ul_emappedpri : ulwp->ul_epri); 12887c478bd9Sstevel@tonic-gate } 12897c478bd9Sstevel@tonic-gate 12907c478bd9Sstevel@tonic-gate void 12917c478bd9Sstevel@tonic-gate stall(void) 12927c478bd9Sstevel@tonic-gate { 12937c478bd9Sstevel@tonic-gate for (;;) 12947c478bd9Sstevel@tonic-gate (void) mutex_lock_kernel(&stall_mutex, NULL, NULL); 12957c478bd9Sstevel@tonic-gate } 12967c478bd9Sstevel@tonic-gate 12977c478bd9Sstevel@tonic-gate /* 12987c478bd9Sstevel@tonic-gate * Acquire a USYNC_THREAD mutex via user-level sleep queues. 12997c478bd9Sstevel@tonic-gate * We failed set_lock_byte(&mp->mutex_lockw) before coming here. 13007c478bd9Sstevel@tonic-gate * Returns with mutex_owner set correctly. 13017c478bd9Sstevel@tonic-gate */ 13027c478bd9Sstevel@tonic-gate int 13037c478bd9Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp, 13047c478bd9Sstevel@tonic-gate timespec_t *tsp) 13057c478bd9Sstevel@tonic-gate { 13067c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 13077c478bd9Sstevel@tonic-gate queue_head_t *qp; 13087c478bd9Sstevel@tonic-gate hrtime_t begin_sleep; 13097c478bd9Sstevel@tonic-gate int error = 0; 13107c478bd9Sstevel@tonic-gate 13117c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 13127c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 13137c478bd9Sstevel@tonic-gate self->ul_wchan = mp; 13147c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 13157c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = mp; 13167c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 13177c478bd9Sstevel@tonic-gate } 13187c478bd9Sstevel@tonic-gate if (msp) { 13197c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_sleep); 13207c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 13217c478bd9Sstevel@tonic-gate } 13227c478bd9Sstevel@tonic-gate 13237c478bd9Sstevel@tonic-gate DTRACE_PROBE1(plockstat, mutex__block, mp); 13247c478bd9Sstevel@tonic-gate 13257c478bd9Sstevel@tonic-gate /* 13267c478bd9Sstevel@tonic-gate * Put ourself on the sleep queue, and while we are 13277c478bd9Sstevel@tonic-gate * unable to grab the lock, go park in the kernel. 13287c478bd9Sstevel@tonic-gate * Take ourself off the sleep queue after we acquire the lock. 13297c478bd9Sstevel@tonic-gate * The waiter bit can be set/cleared only while holding the queue lock. 13307c478bd9Sstevel@tonic-gate */ 13317c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 13327c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 13337c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 13347c478bd9Sstevel@tonic-gate for (;;) { 13357c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 13367c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 13377c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 13387c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 13397c478bd9Sstevel@tonic-gate mp->mutex_waiters = dequeue_self(qp, mp); 13407c478bd9Sstevel@tonic-gate break; 13417c478bd9Sstevel@tonic-gate } 13427c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 13437c478bd9Sstevel@tonic-gate queue_unlock(qp); 13447c478bd9Sstevel@tonic-gate /* 13457c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 13467c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 13477c478bd9Sstevel@tonic-gate */ 13487c478bd9Sstevel@tonic-gate if ((error = __lwp_park(tsp, 0)) == EINTR) 13497c478bd9Sstevel@tonic-gate error = 0; 13507c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 13517c478bd9Sstevel@tonic-gate /* 13527c478bd9Sstevel@tonic-gate * We could have taken a signal or suspended ourself. 13537c478bd9Sstevel@tonic-gate * If we did, then we removed ourself from the queue. 13547c478bd9Sstevel@tonic-gate * Someone else may have removed us from the queue 13557c478bd9Sstevel@tonic-gate * as a consequence of mutex_unlock(). We may have 13567c478bd9Sstevel@tonic-gate * gotten a timeout from __lwp_park(). Or we may still 13577c478bd9Sstevel@tonic-gate * be on the queue and this is just a spurious wakeup. 13587c478bd9Sstevel@tonic-gate */ 13597c478bd9Sstevel@tonic-gate qp = queue_lock(mp, MX); 13607c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) { 13617c478bd9Sstevel@tonic-gate if (error) { 13627c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 13637c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 13647c478bd9Sstevel@tonic-gate error); 13657c478bd9Sstevel@tonic-gate break; 13667c478bd9Sstevel@tonic-gate } 13677c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 13687c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 13697c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1); 13707c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 13717c478bd9Sstevel@tonic-gate 0, 0); 13727c478bd9Sstevel@tonic-gate break; 13737c478bd9Sstevel@tonic-gate } 13747c478bd9Sstevel@tonic-gate enqueue(qp, self, mp, MX); 13757c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 13767c478bd9Sstevel@tonic-gate } 13777c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == qp && 13787c478bd9Sstevel@tonic-gate self->ul_qtype == MX && 13797c478bd9Sstevel@tonic-gate self->ul_wchan == mp); 13807c478bd9Sstevel@tonic-gate if (error) { 13817c478bd9Sstevel@tonic-gate mp->mutex_waiters = dequeue_self(qp, mp); 13827c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0); 13837c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, error); 13847c478bd9Sstevel@tonic-gate break; 13857c478bd9Sstevel@tonic-gate } 13867c478bd9Sstevel@tonic-gate } 13877c478bd9Sstevel@tonic-gate 13887c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 13897c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 13907c478bd9Sstevel@tonic-gate self->ul_sp = 0; 13917c478bd9Sstevel@tonic-gate 13927c478bd9Sstevel@tonic-gate queue_unlock(qp); 13937c478bd9Sstevel@tonic-gate if (msp) 13947c478bd9Sstevel@tonic-gate msp->mutex_sleep_time += gethrtime() - begin_sleep; 13957c478bd9Sstevel@tonic-gate 13967c478bd9Sstevel@tonic-gate ASSERT(error == 0 || error == EINVAL || error == ETIME); 13977c478bd9Sstevel@tonic-gate return (error); 13987c478bd9Sstevel@tonic-gate } 13997c478bd9Sstevel@tonic-gate 14007c478bd9Sstevel@tonic-gate /* 14017c478bd9Sstevel@tonic-gate * Returns with mutex_owner set correctly. 14027c478bd9Sstevel@tonic-gate */ 14037c478bd9Sstevel@tonic-gate int 14047c478bd9Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try) 14057c478bd9Sstevel@tonic-gate { 14067c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 14077c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 14087c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 14097c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 14107c478bd9Sstevel@tonic-gate int error = 0; 14117c478bd9Sstevel@tonic-gate 14127c478bd9Sstevel@tonic-gate ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK); 14137c478bd9Sstevel@tonic-gate 14147c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 14157c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 14167c478bd9Sstevel@tonic-gate 14177c478bd9Sstevel@tonic-gate if (msp && try == MUTEX_TRY) 14187c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try); 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) { 14217c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 14227c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) { 14237c478bd9Sstevel@tonic-gate error = EAGAIN; 14247c478bd9Sstevel@tonic-gate } else { 14257c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 14267c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 14277c478bd9Sstevel@tonic-gate 1, 0); 14287c478bd9Sstevel@tonic-gate return (0); 14297c478bd9Sstevel@tonic-gate } 14307c478bd9Sstevel@tonic-gate } else if (try == MUTEX_TRY) { 14317c478bd9Sstevel@tonic-gate return (EBUSY); 14327c478bd9Sstevel@tonic-gate } else { 14337c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 14347c478bd9Sstevel@tonic-gate return (EDEADLK); 14357c478bd9Sstevel@tonic-gate } 14367c478bd9Sstevel@tonic-gate } 14377c478bd9Sstevel@tonic-gate 14387c478bd9Sstevel@tonic-gate if (self->ul_error_detection && try == MUTEX_LOCK && 14397c478bd9Sstevel@tonic-gate tsp == NULL && mutex_is_held(mp)) 14407c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_lock", NULL, NULL); 14417c478bd9Sstevel@tonic-gate 14427c478bd9Sstevel@tonic-gate if (mtype & 14437c478bd9Sstevel@tonic-gate (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) { 14447c478bd9Sstevel@tonic-gate uint8_t ceil; 14457c478bd9Sstevel@tonic-gate int myprio; 14467c478bd9Sstevel@tonic-gate 14477c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_PROTECT) { 14487c478bd9Sstevel@tonic-gate ceil = mp->mutex_ceiling; 14497c478bd9Sstevel@tonic-gate ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0); 14507c478bd9Sstevel@tonic-gate myprio = real_priority(self); 14517c478bd9Sstevel@tonic-gate if (myprio > ceil) { 14527c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 14537c478bd9Sstevel@tonic-gate EINVAL); 14547c478bd9Sstevel@tonic-gate return (EINVAL); 14557c478bd9Sstevel@tonic-gate } 14567c478bd9Sstevel@tonic-gate if ((error = _ceil_mylist_add(mp)) != 0) { 14577c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, 14587c478bd9Sstevel@tonic-gate error); 14597c478bd9Sstevel@tonic-gate return (error); 14607c478bd9Sstevel@tonic-gate } 14617c478bd9Sstevel@tonic-gate if (myprio < ceil) 14627c478bd9Sstevel@tonic-gate _ceil_prio_inherit(ceil); 14637c478bd9Sstevel@tonic-gate } 14647c478bd9Sstevel@tonic-gate 14657c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) { 14667c478bd9Sstevel@tonic-gate /* go straight to the kernel */ 14677c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) 14687c478bd9Sstevel@tonic-gate error = mutex_trylock_kernel(mp); 14697c478bd9Sstevel@tonic-gate else /* MUTEX_LOCK */ 14707c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 14717c478bd9Sstevel@tonic-gate /* 14727c478bd9Sstevel@tonic-gate * The kernel never sets or clears the lock byte 14737c478bd9Sstevel@tonic-gate * for PTHREAD_PRIO_INHERIT mutexes. 14747c478bd9Sstevel@tonic-gate * Set it here for debugging consistency. 14757c478bd9Sstevel@tonic-gate */ 14767c478bd9Sstevel@tonic-gate switch (error) { 14777c478bd9Sstevel@tonic-gate case 0: 14787c478bd9Sstevel@tonic-gate case EOWNERDEAD: 14797c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 14807c478bd9Sstevel@tonic-gate break; 14817c478bd9Sstevel@tonic-gate } 14827c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS_ROBUST) { 14837c478bd9Sstevel@tonic-gate /* go straight to the kernel */ 14847c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) 14857c478bd9Sstevel@tonic-gate error = mutex_trylock_kernel(mp); 14867c478bd9Sstevel@tonic-gate else /* MUTEX_LOCK */ 14877c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 14887c478bd9Sstevel@tonic-gate } else { /* PTHREAD_PRIO_PROTECT */ 14897c478bd9Sstevel@tonic-gate /* 14907c478bd9Sstevel@tonic-gate * Try once at user level before going to the kernel. 14917c478bd9Sstevel@tonic-gate * If this is a process shared mutex then protect 14927c478bd9Sstevel@tonic-gate * against forkall() while setting mp->mutex_ownerpid. 14937c478bd9Sstevel@tonic-gate */ 14947c478bd9Sstevel@tonic-gate if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) { 14957c478bd9Sstevel@tonic-gate enter_critical(self); 14967c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 14977c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 14987c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 14997c478bd9Sstevel@tonic-gate exit_critical(self); 15007c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, 15017c478bd9Sstevel@tonic-gate mutex__acquire, mp, 0, 0); 15027c478bd9Sstevel@tonic-gate } else { 15037c478bd9Sstevel@tonic-gate exit_critical(self); 15047c478bd9Sstevel@tonic-gate error = EBUSY; 15057c478bd9Sstevel@tonic-gate } 15067c478bd9Sstevel@tonic-gate } else { 15077c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 15087c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15097c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, 15107c478bd9Sstevel@tonic-gate mutex__acquire, mp, 0, 0); 15117c478bd9Sstevel@tonic-gate } else { 15127c478bd9Sstevel@tonic-gate error = EBUSY; 15137c478bd9Sstevel@tonic-gate } 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate if (error && try == MUTEX_LOCK) 15167c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 15177c478bd9Sstevel@tonic-gate } 15187c478bd9Sstevel@tonic-gate 15197c478bd9Sstevel@tonic-gate if (error) { 15207c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) { 15217c478bd9Sstevel@tonic-gate switch (error) { 15227c478bd9Sstevel@tonic-gate case EOWNERDEAD: 15237c478bd9Sstevel@tonic-gate case ENOTRECOVERABLE: 15247c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_MUTEX_ROBUST_NP) 15257c478bd9Sstevel@tonic-gate break; 15267c478bd9Sstevel@tonic-gate if (error == EOWNERDEAD) { 15277c478bd9Sstevel@tonic-gate /* 15287c478bd9Sstevel@tonic-gate * We own the mutex; unlock it. 15297c478bd9Sstevel@tonic-gate * It becomes ENOTRECOVERABLE. 15307c478bd9Sstevel@tonic-gate * All waiters are waked up. 15317c478bd9Sstevel@tonic-gate */ 15327c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 15337c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 15347c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, 15357c478bd9Sstevel@tonic-gate mutex__release, mp, 0); 15367c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 15377c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_unlock(mp); 15387c478bd9Sstevel@tonic-gate } 15397c478bd9Sstevel@tonic-gate /* FALLTHROUGH */ 15407c478bd9Sstevel@tonic-gate case EDEADLK: 15417c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) 15427c478bd9Sstevel@tonic-gate stall(); 15437c478bd9Sstevel@tonic-gate error = EBUSY; 15447c478bd9Sstevel@tonic-gate break; 15457c478bd9Sstevel@tonic-gate } 15467c478bd9Sstevel@tonic-gate } 15477c478bd9Sstevel@tonic-gate if ((mtype & PTHREAD_PRIO_PROTECT) && 15487c478bd9Sstevel@tonic-gate error != EOWNERDEAD) { 15497c478bd9Sstevel@tonic-gate (void) _ceil_mylist_del(mp); 15507c478bd9Sstevel@tonic-gate if (myprio < ceil) 15517c478bd9Sstevel@tonic-gate _ceil_prio_waive(); 15527c478bd9Sstevel@tonic-gate } 15537c478bd9Sstevel@tonic-gate } 15547c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) { 15557c478bd9Sstevel@tonic-gate /* 15567c478bd9Sstevel@tonic-gate * This is a process shared mutex. Protect against 15577c478bd9Sstevel@tonic-gate * forkall() while setting mp->mutex_ownerpid. 15587c478bd9Sstevel@tonic-gate */ 15597c478bd9Sstevel@tonic-gate enter_critical(self); 15607c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 15617c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15627c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 15637c478bd9Sstevel@tonic-gate exit_critical(self); 15647c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 15657c478bd9Sstevel@tonic-gate } else { 15667c478bd9Sstevel@tonic-gate /* try a little harder */ 15677c478bd9Sstevel@tonic-gate exit_critical(self); 15687c478bd9Sstevel@tonic-gate error = mutex_trylock_process(mp); 15697c478bd9Sstevel@tonic-gate } 15707c478bd9Sstevel@tonic-gate if (error && try == MUTEX_LOCK) 15717c478bd9Sstevel@tonic-gate error = mutex_lock_kernel(mp, tsp, msp); 15727c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 15737c478bd9Sstevel@tonic-gate /* try once */ 15747c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 15757c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15767c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 15777c478bd9Sstevel@tonic-gate } else { 15787c478bd9Sstevel@tonic-gate /* try a little harder if we don't own the mutex */ 15797c478bd9Sstevel@tonic-gate error = EBUSY; 15807c478bd9Sstevel@tonic-gate if (MUTEX_OWNER(mp) != self) 15817c478bd9Sstevel@tonic-gate error = mutex_trylock_adaptive(mp); 15827c478bd9Sstevel@tonic-gate if (error && try == MUTEX_LOCK) /* go park */ 15837c478bd9Sstevel@tonic-gate error = mutex_lock_queue(self, msp, mp, tsp); 15847c478bd9Sstevel@tonic-gate } 15857c478bd9Sstevel@tonic-gate } 15867c478bd9Sstevel@tonic-gate 15877c478bd9Sstevel@tonic-gate switch (error) { 15887c478bd9Sstevel@tonic-gate case EOWNERDEAD: 15897c478bd9Sstevel@tonic-gate case ELOCKUNMAPPED: 15907c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 15917c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 15927c478bd9Sstevel@tonic-gate /* FALLTHROUGH */ 15937c478bd9Sstevel@tonic-gate case 0: 15947c478bd9Sstevel@tonic-gate if (msp) 15957c478bd9Sstevel@tonic-gate record_begin_hold(msp); 15967c478bd9Sstevel@tonic-gate break; 15977c478bd9Sstevel@tonic-gate default: 15987c478bd9Sstevel@tonic-gate if (try == MUTEX_TRY) { 15997c478bd9Sstevel@tonic-gate if (msp) 16007c478bd9Sstevel@tonic-gate tdb_incr(msp->mutex_try_fail); 16017c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 16027c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 16037c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 16047c478bd9Sstevel@tonic-gate } 16057c478bd9Sstevel@tonic-gate } 16067c478bd9Sstevel@tonic-gate break; 16077c478bd9Sstevel@tonic-gate } 16087c478bd9Sstevel@tonic-gate 16097c478bd9Sstevel@tonic-gate return (error); 16107c478bd9Sstevel@tonic-gate } 16117c478bd9Sstevel@tonic-gate 16127c478bd9Sstevel@tonic-gate int 16137c478bd9Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try) 16147c478bd9Sstevel@tonic-gate { 16157c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 16167c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 16177c478bd9Sstevel@tonic-gate 16187c478bd9Sstevel@tonic-gate /* 16197c478bd9Sstevel@tonic-gate * We know that USYNC_PROCESS is set in mtype and that 16207c478bd9Sstevel@tonic-gate * zero, one, or both of the flags LOCK_RECURSIVE and 16217c478bd9Sstevel@tonic-gate * LOCK_ERRORCHECK are set, and that no other flags are set. 16227c478bd9Sstevel@tonic-gate */ 16237c478bd9Sstevel@tonic-gate enter_critical(self); 16247c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 16257c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 16267c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = udp->pid; 16277c478bd9Sstevel@tonic-gate exit_critical(self); 16287c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 16297c478bd9Sstevel@tonic-gate return (0); 16307c478bd9Sstevel@tonic-gate } 16317c478bd9Sstevel@tonic-gate exit_critical(self); 16327c478bd9Sstevel@tonic-gate 16337c478bd9Sstevel@tonic-gate if ((mtype & ~USYNC_PROCESS) && shared_mutex_held(mp)) { 16347c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 16357c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 16367c478bd9Sstevel@tonic-gate return (EAGAIN); 16377c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 16387c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0); 16397c478bd9Sstevel@tonic-gate return (0); 16407c478bd9Sstevel@tonic-gate } 16417c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) { 16427c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 16437c478bd9Sstevel@tonic-gate return (EDEADLK); 16447c478bd9Sstevel@tonic-gate } 16457c478bd9Sstevel@tonic-gate return (EBUSY); 16467c478bd9Sstevel@tonic-gate } 16477c478bd9Sstevel@tonic-gate 16487c478bd9Sstevel@tonic-gate /* try a little harder if we don't own the mutex */ 16497c478bd9Sstevel@tonic-gate if (!shared_mutex_held(mp) && mutex_trylock_process(mp) == 0) 16507c478bd9Sstevel@tonic-gate return (0); 16517c478bd9Sstevel@tonic-gate 16527c478bd9Sstevel@tonic-gate if (try == MUTEX_LOCK) 16537c478bd9Sstevel@tonic-gate return (mutex_lock_kernel(mp, tsp, NULL)); 16547c478bd9Sstevel@tonic-gate 16557c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 16567c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 16577c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 16587c478bd9Sstevel@tonic-gate } 16597c478bd9Sstevel@tonic-gate return (EBUSY); 16607c478bd9Sstevel@tonic-gate } 16617c478bd9Sstevel@tonic-gate 16627c478bd9Sstevel@tonic-gate static int 16637c478bd9Sstevel@tonic-gate slow_lock(ulwp_t *self, mutex_t *mp, timespec_t *tsp) 16647c478bd9Sstevel@tonic-gate { 16657c478bd9Sstevel@tonic-gate int error = 0; 16667c478bd9Sstevel@tonic-gate 16677c478bd9Sstevel@tonic-gate if (MUTEX_OWNER(mp) == self || mutex_trylock_adaptive(mp) != 0) 16687c478bd9Sstevel@tonic-gate error = mutex_lock_queue(self, NULL, mp, tsp); 16697c478bd9Sstevel@tonic-gate return (error); 16707c478bd9Sstevel@tonic-gate } 16717c478bd9Sstevel@tonic-gate 16727c478bd9Sstevel@tonic-gate int 16737c478bd9Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp) 16747c478bd9Sstevel@tonic-gate { 16757c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 16767c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 16777c478bd9Sstevel@tonic-gate uberflags_t *gflags; 16787c478bd9Sstevel@tonic-gate int mtype; 16797c478bd9Sstevel@tonic-gate 16807c478bd9Sstevel@tonic-gate /* 16817c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 16827c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 16837c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 16847c478bd9Sstevel@tonic-gate * and the process has only a single thread. 16857c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 16867c478bd9Sstevel@tonic-gate */ 16877c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 16887c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 16897c478bd9Sstevel@tonic-gate /* 16907c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 16917c478bd9Sstevel@tonic-gate */ 16927c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 16937c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 16947c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 16957c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 16967c478bd9Sstevel@tonic-gate return (0); 16977c478bd9Sstevel@tonic-gate } 16987c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 16997c478bd9Sstevel@tonic-gate /* 17007c478bd9Sstevel@tonic-gate * LOCK_RECURSIVE, LOCK_ERRORCHECK, or both. 17017c478bd9Sstevel@tonic-gate */ 17027c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 17037c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 17047c478bd9Sstevel@tonic-gate return (EAGAIN); 17057c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 17067c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 17077c478bd9Sstevel@tonic-gate 1, 0); 17087c478bd9Sstevel@tonic-gate return (0); 17097c478bd9Sstevel@tonic-gate } 17107c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 17117c478bd9Sstevel@tonic-gate return (EDEADLK); /* LOCK_ERRORCHECK */ 17127c478bd9Sstevel@tonic-gate } 17137c478bd9Sstevel@tonic-gate /* 17147c478bd9Sstevel@tonic-gate * We have reached a deadlock, probably because the 17157c478bd9Sstevel@tonic-gate * process is executing non-async-signal-safe code in 17167c478bd9Sstevel@tonic-gate * a signal handler and is attempting to acquire a lock 17177c478bd9Sstevel@tonic-gate * that it already owns. This is not surprising, given 17187c478bd9Sstevel@tonic-gate * bad programming practices over the years that has 17197c478bd9Sstevel@tonic-gate * resulted in applications calling printf() and such 17207c478bd9Sstevel@tonic-gate * in their signal handlers. Unless the user has told 17217c478bd9Sstevel@tonic-gate * us that the signal handlers are safe by setting: 17227c478bd9Sstevel@tonic-gate * export _THREAD_ASYNC_SAFE=1 17237c478bd9Sstevel@tonic-gate * we return EDEADLK rather than actually deadlocking. 17247c478bd9Sstevel@tonic-gate */ 17257c478bd9Sstevel@tonic-gate if (tsp == NULL && 17267c478bd9Sstevel@tonic-gate MUTEX_OWNER(mp) == self && !self->ul_async_safe) { 17277c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 17287c478bd9Sstevel@tonic-gate return (EDEADLK); 17297c478bd9Sstevel@tonic-gate } 17307c478bd9Sstevel@tonic-gate } 17317c478bd9Sstevel@tonic-gate 17327c478bd9Sstevel@tonic-gate /* 17337c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 17347c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 17357c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 17367c478bd9Sstevel@tonic-gate */ 17377c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 17387c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 17397c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 17407c478bd9Sstevel@tonic-gate 17417c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 17427c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK)); 17437c478bd9Sstevel@tonic-gate 17447c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 17457c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 17467c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 17477c478bd9Sstevel@tonic-gate return (0); 17487c478bd9Sstevel@tonic-gate } 17497c478bd9Sstevel@tonic-gate 17507c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 17517c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 17527c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 17537c478bd9Sstevel@tonic-gate return (EAGAIN); 17547c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 17557c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 17567c478bd9Sstevel@tonic-gate 1, 0); 17577c478bd9Sstevel@tonic-gate return (0); 17587c478bd9Sstevel@tonic-gate } 17597c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK); 17607c478bd9Sstevel@tonic-gate return (EDEADLK); /* LOCK_ERRORCHECK */ 17617c478bd9Sstevel@tonic-gate } 17627c478bd9Sstevel@tonic-gate 17637c478bd9Sstevel@tonic-gate return (slow_lock(self, mp, tsp)); 17647c478bd9Sstevel@tonic-gate } 17657c478bd9Sstevel@tonic-gate 17667c478bd9Sstevel@tonic-gate /* else do it the long way */ 17677c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, tsp, MUTEX_LOCK)); 17687c478bd9Sstevel@tonic-gate } 17697c478bd9Sstevel@tonic-gate 17707c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_lock = __mutex_lock 17717c478bd9Sstevel@tonic-gate #pragma weak mutex_lock = __mutex_lock 17727c478bd9Sstevel@tonic-gate #pragma weak _mutex_lock = __mutex_lock 17737c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_lock = __mutex_lock 17747c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_lock = __mutex_lock 17757c478bd9Sstevel@tonic-gate int 17767c478bd9Sstevel@tonic-gate __mutex_lock(mutex_t *mp) 17777c478bd9Sstevel@tonic-gate { 17787c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 17797c478bd9Sstevel@tonic-gate return (mutex_lock_impl(mp, NULL)); 17807c478bd9Sstevel@tonic-gate } 17817c478bd9Sstevel@tonic-gate 17827c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_timedlock = _pthread_mutex_timedlock 17837c478bd9Sstevel@tonic-gate int 17847c478bd9Sstevel@tonic-gate _pthread_mutex_timedlock(mutex_t *mp, const timespec_t *abstime) 17857c478bd9Sstevel@tonic-gate { 17867c478bd9Sstevel@tonic-gate timespec_t tslocal; 17877c478bd9Sstevel@tonic-gate int error; 17887c478bd9Sstevel@tonic-gate 17897c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 17907c478bd9Sstevel@tonic-gate abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal); 17917c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 17927c478bd9Sstevel@tonic-gate if (error == ETIME) 17937c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 17947c478bd9Sstevel@tonic-gate return (error); 17957c478bd9Sstevel@tonic-gate } 17967c478bd9Sstevel@tonic-gate 17977c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_reltimedlock_np = _pthread_mutex_reltimedlock_np 17987c478bd9Sstevel@tonic-gate int 17997c478bd9Sstevel@tonic-gate _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime) 18007c478bd9Sstevel@tonic-gate { 18017c478bd9Sstevel@tonic-gate timespec_t tslocal; 18027c478bd9Sstevel@tonic-gate int error; 18037c478bd9Sstevel@tonic-gate 18047c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 18057c478bd9Sstevel@tonic-gate tslocal = *reltime; 18067c478bd9Sstevel@tonic-gate error = mutex_lock_impl(mp, &tslocal); 18077c478bd9Sstevel@tonic-gate if (error == ETIME) 18087c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 18097c478bd9Sstevel@tonic-gate return (error); 18107c478bd9Sstevel@tonic-gate } 18117c478bd9Sstevel@tonic-gate 18127c478bd9Sstevel@tonic-gate static int 18137c478bd9Sstevel@tonic-gate slow_trylock(mutex_t *mp, ulwp_t *self) 18147c478bd9Sstevel@tonic-gate { 18157c478bd9Sstevel@tonic-gate if (MUTEX_OWNER(mp) == self || 18167c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mp) != 0) { 18177c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 18187c478bd9Sstevel@tonic-gate 18197c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_LOCK_TRY, udp)) { 18207c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_LOCK_TRY; 18217c478bd9Sstevel@tonic-gate tdb_event(TD_LOCK_TRY, udp); 18227c478bd9Sstevel@tonic-gate } 18237c478bd9Sstevel@tonic-gate return (EBUSY); 18247c478bd9Sstevel@tonic-gate } 18257c478bd9Sstevel@tonic-gate return (0); 18267c478bd9Sstevel@tonic-gate } 18277c478bd9Sstevel@tonic-gate 18287c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_trylock = __mutex_trylock 18297c478bd9Sstevel@tonic-gate #pragma weak mutex_trylock = __mutex_trylock 18307c478bd9Sstevel@tonic-gate #pragma weak _mutex_trylock = __mutex_trylock 18317c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_trylock = __mutex_trylock 18327c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_trylock = __mutex_trylock 18337c478bd9Sstevel@tonic-gate int 18347c478bd9Sstevel@tonic-gate __mutex_trylock(mutex_t *mp) 18357c478bd9Sstevel@tonic-gate { 18367c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 18377c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 18387c478bd9Sstevel@tonic-gate uberflags_t *gflags; 18397c478bd9Sstevel@tonic-gate int mtype; 18407c478bd9Sstevel@tonic-gate 18417c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 18427c478bd9Sstevel@tonic-gate /* 18437c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 18447c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 18457c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 18467c478bd9Sstevel@tonic-gate * and the process has only a single thread. 18477c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 18487c478bd9Sstevel@tonic-gate */ 18497c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 18507c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 18517c478bd9Sstevel@tonic-gate /* 18527c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 18537c478bd9Sstevel@tonic-gate */ 18547c478bd9Sstevel@tonic-gate if (mp->mutex_lockw == 0) { 18557c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 18567c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 18577c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 18587c478bd9Sstevel@tonic-gate return (0); 18597c478bd9Sstevel@tonic-gate } 18607c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 18617c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 18627c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 18637c478bd9Sstevel@tonic-gate return (EAGAIN); 18647c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 18657c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 18667c478bd9Sstevel@tonic-gate 1, 0); 18677c478bd9Sstevel@tonic-gate return (0); 18687c478bd9Sstevel@tonic-gate } 18697c478bd9Sstevel@tonic-gate return (EDEADLK); /* LOCK_ERRORCHECK */ 18707c478bd9Sstevel@tonic-gate } 18717c478bd9Sstevel@tonic-gate return (EBUSY); 18727c478bd9Sstevel@tonic-gate } 18737c478bd9Sstevel@tonic-gate 18747c478bd9Sstevel@tonic-gate /* 18757c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 18767c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 18777c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 18787c478bd9Sstevel@tonic-gate */ 18797c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 18807c478bd9Sstevel@tonic-gate (gflags->uf_trs_ted | 18817c478bd9Sstevel@tonic-gate (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) { 18827c478bd9Sstevel@tonic-gate 18837c478bd9Sstevel@tonic-gate if (mtype & USYNC_PROCESS) 18847c478bd9Sstevel@tonic-gate return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY)); 18857c478bd9Sstevel@tonic-gate 18867c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 18877c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 18887c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 18897c478bd9Sstevel@tonic-gate return (0); 18907c478bd9Sstevel@tonic-gate } 18917c478bd9Sstevel@tonic-gate 18927c478bd9Sstevel@tonic-gate if (mtype && MUTEX_OWNER(mp) == self) { 18937c478bd9Sstevel@tonic-gate if (mtype & LOCK_RECURSIVE) { 18947c478bd9Sstevel@tonic-gate if (mp->mutex_rcount == RECURSION_MAX) 18957c478bd9Sstevel@tonic-gate return (EAGAIN); 18967c478bd9Sstevel@tonic-gate mp->mutex_rcount++; 18977c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 18987c478bd9Sstevel@tonic-gate 1, 0); 18997c478bd9Sstevel@tonic-gate return (0); 19007c478bd9Sstevel@tonic-gate } 19017c478bd9Sstevel@tonic-gate return (EBUSY); /* LOCK_ERRORCHECK */ 19027c478bd9Sstevel@tonic-gate } 19037c478bd9Sstevel@tonic-gate 19047c478bd9Sstevel@tonic-gate return (slow_trylock(mp, self)); 19057c478bd9Sstevel@tonic-gate } 19067c478bd9Sstevel@tonic-gate 19077c478bd9Sstevel@tonic-gate /* else do it the long way */ 19087c478bd9Sstevel@tonic-gate return (mutex_lock_internal(mp, NULL, MUTEX_TRY)); 19097c478bd9Sstevel@tonic-gate } 19107c478bd9Sstevel@tonic-gate 19117c478bd9Sstevel@tonic-gate int 19127c478bd9Sstevel@tonic-gate mutex_unlock_internal(mutex_t *mp) 19137c478bd9Sstevel@tonic-gate { 19147c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 19157c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 19167c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 19177c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp; 19187c478bd9Sstevel@tonic-gate int error; 19197c478bd9Sstevel@tonic-gate lwpid_t lwpid; 19207c478bd9Sstevel@tonic-gate 19217c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp)) 19227c478bd9Sstevel@tonic-gate return (EPERM); 19237c478bd9Sstevel@tonic-gate 19247c478bd9Sstevel@tonic-gate if (self->ul_error_detection && !mutex_is_held(mp)) 19257c478bd9Sstevel@tonic-gate lock_error(mp, "mutex_unlock", NULL, NULL); 19267c478bd9Sstevel@tonic-gate 19277c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 19287c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 19297c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 19307c478bd9Sstevel@tonic-gate return (0); 19317c478bd9Sstevel@tonic-gate } 19327c478bd9Sstevel@tonic-gate 19337c478bd9Sstevel@tonic-gate if ((msp = MUTEX_STATS(mp, udp)) != NULL) 19347c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 19357c478bd9Sstevel@tonic-gate 19367c478bd9Sstevel@tonic-gate if (mtype & 19377c478bd9Sstevel@tonic-gate (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) { 19387c478bd9Sstevel@tonic-gate no_preempt(self); 19397c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 19407c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 19417c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 19427c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) { 19437c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 19447c478bd9Sstevel@tonic-gate error = ___lwp_mutex_unlock(mp); 19457c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS_ROBUST) { 19467c478bd9Sstevel@tonic-gate error = ___lwp_mutex_unlock(mp); 19477c478bd9Sstevel@tonic-gate } else { 19487c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) 19497c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 19507c478bd9Sstevel@tonic-gate error = 0; 19517c478bd9Sstevel@tonic-gate } 19527c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_PROTECT) { 19537c478bd9Sstevel@tonic-gate if (_ceil_mylist_del(mp)) 19547c478bd9Sstevel@tonic-gate _ceil_prio_waive(); 19557c478bd9Sstevel@tonic-gate } 19567c478bd9Sstevel@tonic-gate preempt(self); 19577c478bd9Sstevel@tonic-gate } else if (mtype & USYNC_PROCESS) { 19587c478bd9Sstevel@tonic-gate if (mp->mutex_lockword & WAITERMASK) 19597c478bd9Sstevel@tonic-gate mutex_unlock_process(mp); 19607c478bd9Sstevel@tonic-gate else { 19617c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 19627c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 19637c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 19647c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) { 19657c478bd9Sstevel@tonic-gate no_preempt(self); 19667c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 19677c478bd9Sstevel@tonic-gate preempt(self); 19687c478bd9Sstevel@tonic-gate } 19697c478bd9Sstevel@tonic-gate } 19707c478bd9Sstevel@tonic-gate error = 0; 19717c478bd9Sstevel@tonic-gate } else { /* USYNC_THREAD */ 19727c478bd9Sstevel@tonic-gate if ((lwpid = mutex_unlock_queue(mp)) != 0) { 19737c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 19747c478bd9Sstevel@tonic-gate preempt(self); 19757c478bd9Sstevel@tonic-gate } 19767c478bd9Sstevel@tonic-gate error = 0; 19777c478bd9Sstevel@tonic-gate } 19787c478bd9Sstevel@tonic-gate 19797c478bd9Sstevel@tonic-gate return (error); 19807c478bd9Sstevel@tonic-gate } 19817c478bd9Sstevel@tonic-gate 19827c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_unlock = __mutex_unlock 19837c478bd9Sstevel@tonic-gate #pragma weak mutex_unlock = __mutex_unlock 19847c478bd9Sstevel@tonic-gate #pragma weak _mutex_unlock = __mutex_unlock 19857c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_unlock = __mutex_unlock 19867c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_unlock = __mutex_unlock 19877c478bd9Sstevel@tonic-gate int 19887c478bd9Sstevel@tonic-gate __mutex_unlock(mutex_t *mp) 19897c478bd9Sstevel@tonic-gate { 19907c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 19917c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 19927c478bd9Sstevel@tonic-gate uberflags_t *gflags; 19937c478bd9Sstevel@tonic-gate lwpid_t lwpid; 19947c478bd9Sstevel@tonic-gate int mtype; 19957c478bd9Sstevel@tonic-gate short el; 19967c478bd9Sstevel@tonic-gate 19977c478bd9Sstevel@tonic-gate /* 19987c478bd9Sstevel@tonic-gate * Optimize the case of USYNC_THREAD, including 19997c478bd9Sstevel@tonic-gate * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases, 20007c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, 20017c478bd9Sstevel@tonic-gate * and the process has only a single thread. 20027c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 20037c478bd9Sstevel@tonic-gate */ 20047c478bd9Sstevel@tonic-gate if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) | 20057c478bd9Sstevel@tonic-gate udp->uberflags.uf_all) == 0) { 20067c478bd9Sstevel@tonic-gate if (mtype) { 20077c478bd9Sstevel@tonic-gate /* 20087c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 20097c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 20107c478bd9Sstevel@tonic-gate */ 20117c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 20127c478bd9Sstevel@tonic-gate return (EPERM); 20137c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 20147c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 20157c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 20167c478bd9Sstevel@tonic-gate return (0); 20177c478bd9Sstevel@tonic-gate } 20187c478bd9Sstevel@tonic-gate } 20197c478bd9Sstevel@tonic-gate /* 20207c478bd9Sstevel@tonic-gate * Only one thread exists so we don't need an atomic operation. 20217c478bd9Sstevel@tonic-gate * Also, there can be no waiters. 20227c478bd9Sstevel@tonic-gate */ 20237c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 20247c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 20257c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 20267c478bd9Sstevel@tonic-gate return (0); 20277c478bd9Sstevel@tonic-gate } 20287c478bd9Sstevel@tonic-gate 20297c478bd9Sstevel@tonic-gate /* 20307c478bd9Sstevel@tonic-gate * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS, 20317c478bd9Sstevel@tonic-gate * no error detection, and no lock statistics. 20327c478bd9Sstevel@tonic-gate * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases. 20337c478bd9Sstevel@tonic-gate */ 20347c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL) { 20357c478bd9Sstevel@tonic-gate if (((el = gflags->uf_trs_ted) | mtype) == 0) { 20367c478bd9Sstevel@tonic-gate fast_unlock: 20377c478bd9Sstevel@tonic-gate if (!(mp->mutex_lockword & WAITERMASK)) { 20387c478bd9Sstevel@tonic-gate /* no waiter exists right now */ 20397c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 20407c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 20417c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & 20427c478bd9Sstevel@tonic-gate WAITERMASK) { 20437c478bd9Sstevel@tonic-gate /* a waiter suddenly appeared */ 20447c478bd9Sstevel@tonic-gate no_preempt(self); 20457c478bd9Sstevel@tonic-gate if ((lwpid = mutex_wakeup(mp)) != 0) 20467c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 20477c478bd9Sstevel@tonic-gate preempt(self); 20487c478bd9Sstevel@tonic-gate } 20497c478bd9Sstevel@tonic-gate } else if ((lwpid = mutex_unlock_queue(mp)) != 0) { 20507c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 20517c478bd9Sstevel@tonic-gate preempt(self); 20527c478bd9Sstevel@tonic-gate } 20537c478bd9Sstevel@tonic-gate return (0); 20547c478bd9Sstevel@tonic-gate } 20557c478bd9Sstevel@tonic-gate if (el) /* error detection or lock statistics */ 20567c478bd9Sstevel@tonic-gate goto slow_unlock; 20577c478bd9Sstevel@tonic-gate if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 20587c478bd9Sstevel@tonic-gate /* 20597c478bd9Sstevel@tonic-gate * At this point we know that one or both of the 20607c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set. 20617c478bd9Sstevel@tonic-gate */ 20627c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self)) 20637c478bd9Sstevel@tonic-gate return (EPERM); 20647c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 20657c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 20667c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 20677c478bd9Sstevel@tonic-gate return (0); 20687c478bd9Sstevel@tonic-gate } 20697c478bd9Sstevel@tonic-gate goto fast_unlock; 20707c478bd9Sstevel@tonic-gate } 20717c478bd9Sstevel@tonic-gate if ((mtype & 20727c478bd9Sstevel@tonic-gate ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) { 20737c478bd9Sstevel@tonic-gate /* 20747c478bd9Sstevel@tonic-gate * At this point we know that zero, one, or both of the 20757c478bd9Sstevel@tonic-gate * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and 20767c478bd9Sstevel@tonic-gate * that the USYNC_PROCESS flag is set. 20777c478bd9Sstevel@tonic-gate */ 20787c478bd9Sstevel@tonic-gate if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp)) 20797c478bd9Sstevel@tonic-gate return (EPERM); 20807c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) { 20817c478bd9Sstevel@tonic-gate mp->mutex_rcount--; 20827c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 1); 20837c478bd9Sstevel@tonic-gate return (0); 20847c478bd9Sstevel@tonic-gate } 20857c478bd9Sstevel@tonic-gate if (mp->mutex_lockword & WAITERMASK) 20867c478bd9Sstevel@tonic-gate mutex_unlock_process(mp); 20877c478bd9Sstevel@tonic-gate else { 20887c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 20897c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 20907c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 20917c478bd9Sstevel@tonic-gate if (swap32(&mp->mutex_lockword, 0) & 20927c478bd9Sstevel@tonic-gate WAITERMASK) { 20937c478bd9Sstevel@tonic-gate no_preempt(self); 20947c478bd9Sstevel@tonic-gate (void) ___lwp_mutex_wakeup(mp); 20957c478bd9Sstevel@tonic-gate preempt(self); 20967c478bd9Sstevel@tonic-gate } 20977c478bd9Sstevel@tonic-gate } 20987c478bd9Sstevel@tonic-gate return (0); 20997c478bd9Sstevel@tonic-gate } 21007c478bd9Sstevel@tonic-gate } 21017c478bd9Sstevel@tonic-gate 21027c478bd9Sstevel@tonic-gate /* else do it the long way */ 21037c478bd9Sstevel@tonic-gate slow_unlock: 21047c478bd9Sstevel@tonic-gate return (mutex_unlock_internal(mp)); 21057c478bd9Sstevel@tonic-gate } 21067c478bd9Sstevel@tonic-gate 21077c478bd9Sstevel@tonic-gate /* 21087c478bd9Sstevel@tonic-gate * Internally to the library, almost all mutex lock/unlock actions 21097c478bd9Sstevel@tonic-gate * go through these lmutex_ functions, to protect critical regions. 21107c478bd9Sstevel@tonic-gate * We replicate a bit of code from __mutex_lock() and __mutex_unlock() 21117c478bd9Sstevel@tonic-gate * to make these functions faster since we know that the mutex type 21127c478bd9Sstevel@tonic-gate * of all internal locks is USYNC_THREAD. We also know that internal 21137c478bd9Sstevel@tonic-gate * locking can never fail, so we panic if it does. 21147c478bd9Sstevel@tonic-gate */ 21157c478bd9Sstevel@tonic-gate void 21167c478bd9Sstevel@tonic-gate lmutex_lock(mutex_t *mp) 21177c478bd9Sstevel@tonic-gate { 21187c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 21197c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 21207c478bd9Sstevel@tonic-gate 21217c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 21227c478bd9Sstevel@tonic-gate 21237c478bd9Sstevel@tonic-gate enter_critical(self); 21247c478bd9Sstevel@tonic-gate /* 21257c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 21267c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 21277c478bd9Sstevel@tonic-gate */ 21287c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 21297c478bd9Sstevel@tonic-gate /* 21307c478bd9Sstevel@tonic-gate * Only one thread exists; the mutex must be free. 21317c478bd9Sstevel@tonic-gate */ 21327c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_lockw == 0); 21337c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKSET; 21347c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 21357c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 21367c478bd9Sstevel@tonic-gate } else { 21377c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 21387c478bd9Sstevel@tonic-gate 21397c478bd9Sstevel@tonic-gate if (!self->ul_schedctl_called) 21407c478bd9Sstevel@tonic-gate (void) setup_schedctl(); 21417c478bd9Sstevel@tonic-gate 21427c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 21437c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 21447c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 21457c478bd9Sstevel@tonic-gate } else if (mutex_trylock_adaptive(mp) != 0) { 21467c478bd9Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL); 21477c478bd9Sstevel@tonic-gate } 21487c478bd9Sstevel@tonic-gate 21497c478bd9Sstevel@tonic-gate if (msp) 21507c478bd9Sstevel@tonic-gate record_begin_hold(msp); 21517c478bd9Sstevel@tonic-gate } 21527c478bd9Sstevel@tonic-gate } 21537c478bd9Sstevel@tonic-gate 21547c478bd9Sstevel@tonic-gate void 21557c478bd9Sstevel@tonic-gate lmutex_unlock(mutex_t *mp) 21567c478bd9Sstevel@tonic-gate { 21577c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 21587c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 21597c478bd9Sstevel@tonic-gate 21607c478bd9Sstevel@tonic-gate ASSERT(mp->mutex_type == USYNC_THREAD); 21617c478bd9Sstevel@tonic-gate 21627c478bd9Sstevel@tonic-gate /* 21637c478bd9Sstevel@tonic-gate * Optimize the case of no lock statistics and only a single thread. 21647c478bd9Sstevel@tonic-gate * (Most likely a traditional single-threaded application.) 21657c478bd9Sstevel@tonic-gate */ 21667c478bd9Sstevel@tonic-gate if (udp->uberflags.uf_all == 0) { 21677c478bd9Sstevel@tonic-gate /* 21687c478bd9Sstevel@tonic-gate * Only one thread exists so there can be no waiters. 21697c478bd9Sstevel@tonic-gate */ 21707c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 21717c478bd9Sstevel@tonic-gate mp->mutex_lockword = 0; 21727c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 21737c478bd9Sstevel@tonic-gate } else { 21747c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 21757c478bd9Sstevel@tonic-gate lwpid_t lwpid; 21767c478bd9Sstevel@tonic-gate 21777c478bd9Sstevel@tonic-gate if (msp) 21787c478bd9Sstevel@tonic-gate (void) record_hold_time(msp); 21797c478bd9Sstevel@tonic-gate if ((lwpid = mutex_unlock_queue(mp)) != 0) { 21807c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 21817c478bd9Sstevel@tonic-gate preempt(self); 21827c478bd9Sstevel@tonic-gate } 21837c478bd9Sstevel@tonic-gate } 21847c478bd9Sstevel@tonic-gate exit_critical(self); 21857c478bd9Sstevel@tonic-gate } 21867c478bd9Sstevel@tonic-gate 21877c478bd9Sstevel@tonic-gate static int 21887c478bd9Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg) 21897c478bd9Sstevel@tonic-gate { 21907c478bd9Sstevel@tonic-gate /* 21917c478bd9Sstevel@tonic-gate * There is an inherent data race in the current ownership design. 21927c478bd9Sstevel@tonic-gate * The mutex_owner and mutex_ownerpid fields cannot be set or tested 21937c478bd9Sstevel@tonic-gate * atomically as a pair. The original implementation tested each 21947c478bd9Sstevel@tonic-gate * field just once. This was exposed to trivial false positives in 21957c478bd9Sstevel@tonic-gate * the case of multiple multithreaded processes with thread addresses 21967c478bd9Sstevel@tonic-gate * in common. To close the window to an acceptable level we now use a 21977c478bd9Sstevel@tonic-gate * sequence of five tests: pid-thr-pid-thr-pid. This ensures that any 21987c478bd9Sstevel@tonic-gate * single interruption will still leave one uninterrupted sequence of 21997c478bd9Sstevel@tonic-gate * pid-thr-pid tests intact. 22007c478bd9Sstevel@tonic-gate * 22017c478bd9Sstevel@tonic-gate * It is assumed that all updates are always ordered thr-pid and that 22027c478bd9Sstevel@tonic-gate * we have TSO hardware. 22037c478bd9Sstevel@tonic-gate */ 22047c478bd9Sstevel@tonic-gate volatile mutex_t *mp = (volatile mutex_t *)mparg; 22057c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 22067c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 22077c478bd9Sstevel@tonic-gate 22087c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid != udp->pid) 22097c478bd9Sstevel@tonic-gate return (0); 22107c478bd9Sstevel@tonic-gate 22117c478bd9Sstevel@tonic-gate if (!MUTEX_OWNED(mp, self)) 22127c478bd9Sstevel@tonic-gate return (0); 22137c478bd9Sstevel@tonic-gate 22147c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid != udp->pid) 22157c478bd9Sstevel@tonic-gate return (0); 22167c478bd9Sstevel@tonic-gate 22177c478bd9Sstevel@tonic-gate if (!MUTEX_OWNED(mp, self)) 22187c478bd9Sstevel@tonic-gate return (0); 22197c478bd9Sstevel@tonic-gate 22207c478bd9Sstevel@tonic-gate if (mp->mutex_ownerpid != udp->pid) 22217c478bd9Sstevel@tonic-gate return (0); 22227c478bd9Sstevel@tonic-gate 22237c478bd9Sstevel@tonic-gate return (1); 22247c478bd9Sstevel@tonic-gate } 22257c478bd9Sstevel@tonic-gate 22267c478bd9Sstevel@tonic-gate /* 22277c478bd9Sstevel@tonic-gate * Some crufty old programs define their own version of _mutex_held() 22287c478bd9Sstevel@tonic-gate * to be simply return(1). This breaks internal libc logic, so we 22297c478bd9Sstevel@tonic-gate * define a private version for exclusive use by libc, mutex_is_held(), 22307c478bd9Sstevel@tonic-gate * and also a new public function, __mutex_held(), to be used in new 22317c478bd9Sstevel@tonic-gate * code to circumvent these crufty old programs. 22327c478bd9Sstevel@tonic-gate */ 22337c478bd9Sstevel@tonic-gate #pragma weak mutex_held = mutex_is_held 22347c478bd9Sstevel@tonic-gate #pragma weak _mutex_held = mutex_is_held 22357c478bd9Sstevel@tonic-gate #pragma weak __mutex_held = mutex_is_held 22367c478bd9Sstevel@tonic-gate int 22377c478bd9Sstevel@tonic-gate mutex_is_held(mutex_t *mp) 22387c478bd9Sstevel@tonic-gate { 22397c478bd9Sstevel@tonic-gate if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) 22407c478bd9Sstevel@tonic-gate return (shared_mutex_held(mp)); 22417c478bd9Sstevel@tonic-gate return (MUTEX_OWNED(mp, curthread)); 22427c478bd9Sstevel@tonic-gate } 22437c478bd9Sstevel@tonic-gate 22447c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_destroy = __mutex_destroy 22457c478bd9Sstevel@tonic-gate #pragma weak mutex_destroy = __mutex_destroy 22467c478bd9Sstevel@tonic-gate #pragma weak _mutex_destroy = __mutex_destroy 22477c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_destroy = __mutex_destroy 22487c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_destroy = __mutex_destroy 22497c478bd9Sstevel@tonic-gate int 22507c478bd9Sstevel@tonic-gate __mutex_destroy(mutex_t *mp) 22517c478bd9Sstevel@tonic-gate { 22527c478bd9Sstevel@tonic-gate mp->mutex_magic = 0; 22537c478bd9Sstevel@tonic-gate mp->mutex_flag &= ~LOCK_INITED; 22547c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(mp); 22557c478bd9Sstevel@tonic-gate return (0); 22567c478bd9Sstevel@tonic-gate } 22577c478bd9Sstevel@tonic-gate 22587c478bd9Sstevel@tonic-gate /* 22597c478bd9Sstevel@tonic-gate * Spin locks are separate from ordinary mutexes, 22607c478bd9Sstevel@tonic-gate * but we use the same data structure for them. 22617c478bd9Sstevel@tonic-gate */ 22627c478bd9Sstevel@tonic-gate 22637c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_init = _pthread_spin_init 22647c478bd9Sstevel@tonic-gate int 22657c478bd9Sstevel@tonic-gate _pthread_spin_init(pthread_spinlock_t *lock, int pshared) 22667c478bd9Sstevel@tonic-gate { 22677c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 22687c478bd9Sstevel@tonic-gate 22697c478bd9Sstevel@tonic-gate (void) _memset(mp, 0, sizeof (*mp)); 22707c478bd9Sstevel@tonic-gate if (pshared == PTHREAD_PROCESS_SHARED) 22717c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_PROCESS; 22727c478bd9Sstevel@tonic-gate else 22737c478bd9Sstevel@tonic-gate mp->mutex_type = USYNC_THREAD; 22747c478bd9Sstevel@tonic-gate mp->mutex_flag = LOCK_INITED; 22757c478bd9Sstevel@tonic-gate mp->mutex_magic = MUTEX_MAGIC; 22767c478bd9Sstevel@tonic-gate return (0); 22777c478bd9Sstevel@tonic-gate } 22787c478bd9Sstevel@tonic-gate 22797c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_destroy = _pthread_spin_destroy 22807c478bd9Sstevel@tonic-gate int 22817c478bd9Sstevel@tonic-gate _pthread_spin_destroy(pthread_spinlock_t *lock) 22827c478bd9Sstevel@tonic-gate { 22837c478bd9Sstevel@tonic-gate (void) _memset(lock, 0, sizeof (*lock)); 22847c478bd9Sstevel@tonic-gate return (0); 22857c478bd9Sstevel@tonic-gate } 22867c478bd9Sstevel@tonic-gate 22877c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_trylock = _pthread_spin_trylock 22887c478bd9Sstevel@tonic-gate int 22897c478bd9Sstevel@tonic-gate _pthread_spin_trylock(pthread_spinlock_t *lock) 22907c478bd9Sstevel@tonic-gate { 22917c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 22927c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 22937c478bd9Sstevel@tonic-gate int error = 0; 22947c478bd9Sstevel@tonic-gate 22957c478bd9Sstevel@tonic-gate no_preempt(self); 22967c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) != 0) 22977c478bd9Sstevel@tonic-gate error = EBUSY; 22987c478bd9Sstevel@tonic-gate else { 22997c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 23007c478bd9Sstevel@tonic-gate if (mp->mutex_type == USYNC_PROCESS) 23017c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = self->ul_uberdata->pid; 23027c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 23037c478bd9Sstevel@tonic-gate } 23047c478bd9Sstevel@tonic-gate preempt(self); 23057c478bd9Sstevel@tonic-gate return (error); 23067c478bd9Sstevel@tonic-gate } 23077c478bd9Sstevel@tonic-gate 23087c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_lock = _pthread_spin_lock 23097c478bd9Sstevel@tonic-gate int 23107c478bd9Sstevel@tonic-gate _pthread_spin_lock(pthread_spinlock_t *lock) 23117c478bd9Sstevel@tonic-gate { 23127c478bd9Sstevel@tonic-gate volatile uint8_t *lockp = 23137c478bd9Sstevel@tonic-gate (volatile uint8_t *)&((mutex_t *)lock)->mutex_lockw; 23147c478bd9Sstevel@tonic-gate 23157c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 23167c478bd9Sstevel@tonic-gate /* 23177c478bd9Sstevel@tonic-gate * We don't care whether the owner is running on a processor. 23187c478bd9Sstevel@tonic-gate * We just spin because that's what this interface requires. 23197c478bd9Sstevel@tonic-gate */ 23207c478bd9Sstevel@tonic-gate for (;;) { 23217c478bd9Sstevel@tonic-gate if (*lockp == 0) { /* lock byte appears to be clear */ 23227c478bd9Sstevel@tonic-gate if (_pthread_spin_trylock(lock) == 0) 23237c478bd9Sstevel@tonic-gate return (0); 23247c478bd9Sstevel@tonic-gate } 23257c478bd9Sstevel@tonic-gate SMT_PAUSE(); 23267c478bd9Sstevel@tonic-gate } 23277c478bd9Sstevel@tonic-gate } 23287c478bd9Sstevel@tonic-gate 23297c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_unlock = _pthread_spin_unlock 23307c478bd9Sstevel@tonic-gate int 23317c478bd9Sstevel@tonic-gate _pthread_spin_unlock(pthread_spinlock_t *lock) 23327c478bd9Sstevel@tonic-gate { 23337c478bd9Sstevel@tonic-gate mutex_t *mp = (mutex_t *)lock; 23347c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 23357c478bd9Sstevel@tonic-gate 23367c478bd9Sstevel@tonic-gate no_preempt(self); 23377c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 23387c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 23397c478bd9Sstevel@tonic-gate DTRACE_PROBE2(plockstat, mutex__release, mp, 0); 23407c478bd9Sstevel@tonic-gate (void) swap32(&mp->mutex_lockword, 0); 23417c478bd9Sstevel@tonic-gate preempt(self); 23427c478bd9Sstevel@tonic-gate return (0); 23437c478bd9Sstevel@tonic-gate } 23447c478bd9Sstevel@tonic-gate 23457c478bd9Sstevel@tonic-gate #pragma weak cond_init = _cond_init 23467c478bd9Sstevel@tonic-gate /* ARGSUSED2 */ 23477c478bd9Sstevel@tonic-gate int 23487c478bd9Sstevel@tonic-gate _cond_init(cond_t *cvp, int type, void *arg) 23497c478bd9Sstevel@tonic-gate { 23507c478bd9Sstevel@tonic-gate if (type != USYNC_THREAD && type != USYNC_PROCESS) 23517c478bd9Sstevel@tonic-gate return (EINVAL); 23527c478bd9Sstevel@tonic-gate (void) _memset(cvp, 0, sizeof (*cvp)); 23537c478bd9Sstevel@tonic-gate cvp->cond_type = (uint16_t)type; 23547c478bd9Sstevel@tonic-gate cvp->cond_magic = COND_MAGIC; 23557c478bd9Sstevel@tonic-gate return (0); 23567c478bd9Sstevel@tonic-gate } 23577c478bd9Sstevel@tonic-gate 23587c478bd9Sstevel@tonic-gate /* 23597c478bd9Sstevel@tonic-gate * cond_sleep_queue(): utility function for cond_wait_queue(). 23607c478bd9Sstevel@tonic-gate * 23617c478bd9Sstevel@tonic-gate * Go to sleep on a condvar sleep queue, expect to be waked up 23627c478bd9Sstevel@tonic-gate * by someone calling cond_signal() or cond_broadcast() or due 23637c478bd9Sstevel@tonic-gate * to receiving a UNIX signal or being cancelled, or just simply 23647c478bd9Sstevel@tonic-gate * due to a spurious wakeup (like someome calling forkall()). 23657c478bd9Sstevel@tonic-gate * 23667c478bd9Sstevel@tonic-gate * The associated mutex is *not* reacquired before returning. 23677c478bd9Sstevel@tonic-gate * That must be done by the caller of cond_sleep_queue(). 23687c478bd9Sstevel@tonic-gate */ 23697c478bd9Sstevel@tonic-gate int 23707c478bd9Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 23717c478bd9Sstevel@tonic-gate { 23727c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 23737c478bd9Sstevel@tonic-gate queue_head_t *qp; 23747c478bd9Sstevel@tonic-gate queue_head_t *mqp; 23757c478bd9Sstevel@tonic-gate lwpid_t lwpid; 23767c478bd9Sstevel@tonic-gate int signalled; 23777c478bd9Sstevel@tonic-gate int error; 23787c478bd9Sstevel@tonic-gate 23797c478bd9Sstevel@tonic-gate /* 23807c478bd9Sstevel@tonic-gate * Put ourself on the CV sleep queue, unlock the mutex, then 23817c478bd9Sstevel@tonic-gate * park ourself and unpark a candidate lwp to grab the mutex. 23827c478bd9Sstevel@tonic-gate * We must go onto the CV sleep queue before dropping the 23837c478bd9Sstevel@tonic-gate * mutex in order to guarantee atomicity of the operation. 23847c478bd9Sstevel@tonic-gate */ 23857c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 23867c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 23877c478bd9Sstevel@tonic-gate enqueue(qp, self, cvp, CV); 23887c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 1; 23897c478bd9Sstevel@tonic-gate self->ul_cvmutex = mp; 23907c478bd9Sstevel@tonic-gate self->ul_cv_wake = (tsp != NULL); 23917c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 23927c478bd9Sstevel@tonic-gate lwpid = mutex_unlock_queue(mp); 23937c478bd9Sstevel@tonic-gate for (;;) { 23947c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 23957c478bd9Sstevel@tonic-gate queue_unlock(qp); 23967c478bd9Sstevel@tonic-gate if (lwpid != 0) { 23977c478bd9Sstevel@tonic-gate lwpid = preempt_unpark(self, lwpid); 23987c478bd9Sstevel@tonic-gate preempt(self); 23997c478bd9Sstevel@tonic-gate } 24007c478bd9Sstevel@tonic-gate /* 24017c478bd9Sstevel@tonic-gate * We may have a deferred signal present, 24027c478bd9Sstevel@tonic-gate * in which case we should return EINTR. 24037c478bd9Sstevel@tonic-gate * Also, we may have received a SIGCANCEL; if so 24047c478bd9Sstevel@tonic-gate * and we are cancelable we should return EINTR. 24057c478bd9Sstevel@tonic-gate * We force an immediate EINTR return from 24067c478bd9Sstevel@tonic-gate * __lwp_park() by turning our parking flag off. 24077c478bd9Sstevel@tonic-gate */ 24087c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 24097c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 24107c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 24117c478bd9Sstevel@tonic-gate /* 24127c478bd9Sstevel@tonic-gate * __lwp_park() will return the residual time in tsp 24137c478bd9Sstevel@tonic-gate * if we are unparked before the timeout expires. 24147c478bd9Sstevel@tonic-gate */ 24157c478bd9Sstevel@tonic-gate error = __lwp_park(tsp, lwpid); 24167c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 24177c478bd9Sstevel@tonic-gate lwpid = 0; /* unpark the other lwp only once */ 24187c478bd9Sstevel@tonic-gate /* 24197c478bd9Sstevel@tonic-gate * We were waked up by cond_signal(), cond_broadcast(), 24207c478bd9Sstevel@tonic-gate * by an interrupt or timeout (EINTR or ETIME), 24217c478bd9Sstevel@tonic-gate * or we may just have gotten a spurious wakeup. 24227c478bd9Sstevel@tonic-gate */ 24237c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 24247c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 24257c478bd9Sstevel@tonic-gate if (self->ul_sleepq == NULL) 24267c478bd9Sstevel@tonic-gate break; 24277c478bd9Sstevel@tonic-gate /* 24287c478bd9Sstevel@tonic-gate * We are on either the condvar sleep queue or the 2429*2be60c5eSraf * mutex sleep queue. Break out of the sleep if we 2430*2be60c5eSraf * were interrupted or we timed out (EINTR or ETIME). 24317c478bd9Sstevel@tonic-gate * Else this is a spurious wakeup; continue the loop. 24327c478bd9Sstevel@tonic-gate */ 2433*2be60c5eSraf if (self->ul_sleepq == mqp) { /* mutex queue */ 2434*2be60c5eSraf if (error) { 2435*2be60c5eSraf mp->mutex_waiters = dequeue_self(mqp, mp); 2436*2be60c5eSraf break; 2437*2be60c5eSraf } 2438*2be60c5eSraf tsp = NULL; /* no more timeout */ 2439*2be60c5eSraf } else if (self->ul_sleepq == qp) { /* condvar queue */ 24407c478bd9Sstevel@tonic-gate if (error) { 24417c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = dequeue_self(qp, cvp); 24427c478bd9Sstevel@tonic-gate break; 24437c478bd9Sstevel@tonic-gate } 24447c478bd9Sstevel@tonic-gate /* 24457c478bd9Sstevel@tonic-gate * Else a spurious wakeup on the condvar queue. 24467c478bd9Sstevel@tonic-gate * __lwp_park() has already adjusted the timeout. 24477c478bd9Sstevel@tonic-gate */ 24487c478bd9Sstevel@tonic-gate } else { 24497c478bd9Sstevel@tonic-gate thr_panic("cond_sleep_queue(): thread not on queue"); 24507c478bd9Sstevel@tonic-gate } 24517c478bd9Sstevel@tonic-gate queue_unlock(mqp); 24527c478bd9Sstevel@tonic-gate } 24537c478bd9Sstevel@tonic-gate 24547c478bd9Sstevel@tonic-gate self->ul_sp = 0; 24557c478bd9Sstevel@tonic-gate ASSERT(self->ul_cvmutex == NULL && self->ul_cv_wake == 0); 24567c478bd9Sstevel@tonic-gate ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL && 24577c478bd9Sstevel@tonic-gate self->ul_wchan == NULL); 24587c478bd9Sstevel@tonic-gate 24597c478bd9Sstevel@tonic-gate signalled = self->ul_signalled; 24607c478bd9Sstevel@tonic-gate self->ul_signalled = 0; 24617c478bd9Sstevel@tonic-gate queue_unlock(qp); 24627c478bd9Sstevel@tonic-gate queue_unlock(mqp); 24637c478bd9Sstevel@tonic-gate 24647c478bd9Sstevel@tonic-gate /* 24657c478bd9Sstevel@tonic-gate * If we were concurrently cond_signal()d and any of: 24667c478bd9Sstevel@tonic-gate * received a UNIX signal, were cancelled, or got a timeout, 24677c478bd9Sstevel@tonic-gate * then perform another cond_signal() to avoid consuming it. 24687c478bd9Sstevel@tonic-gate */ 24697c478bd9Sstevel@tonic-gate if (error && signalled) 24707c478bd9Sstevel@tonic-gate (void) cond_signal_internal(cvp); 24717c478bd9Sstevel@tonic-gate 24727c478bd9Sstevel@tonic-gate return (error); 24737c478bd9Sstevel@tonic-gate } 24747c478bd9Sstevel@tonic-gate 24757c478bd9Sstevel@tonic-gate int 24767c478bd9Sstevel@tonic-gate cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp, 24777c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp) 24787c478bd9Sstevel@tonic-gate { 24797c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 24807c478bd9Sstevel@tonic-gate int error; 24817c478bd9Sstevel@tonic-gate 24827c478bd9Sstevel@tonic-gate /* 24837c478bd9Sstevel@tonic-gate * The old thread library was programmed to defer signals 24847c478bd9Sstevel@tonic-gate * while in cond_wait() so that the associated mutex would 24857c478bd9Sstevel@tonic-gate * be guaranteed to be held when the application signal 24867c478bd9Sstevel@tonic-gate * handler was invoked. 24877c478bd9Sstevel@tonic-gate * 24887c478bd9Sstevel@tonic-gate * We do not behave this way by default; the state of the 24897c478bd9Sstevel@tonic-gate * associated mutex in the signal handler is undefined. 24907c478bd9Sstevel@tonic-gate * 24917c478bd9Sstevel@tonic-gate * To accommodate applications that depend on the old 24927c478bd9Sstevel@tonic-gate * behavior, the _THREAD_COND_WAIT_DEFER environment 24937c478bd9Sstevel@tonic-gate * variable can be set to 1 and we will behave in the 24947c478bd9Sstevel@tonic-gate * old way with respect to cond_wait(). 24957c478bd9Sstevel@tonic-gate */ 24967c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 24977c478bd9Sstevel@tonic-gate sigoff(self); 24987c478bd9Sstevel@tonic-gate 24997c478bd9Sstevel@tonic-gate error = cond_sleep_queue(cvp, mp, tsp); 25007c478bd9Sstevel@tonic-gate 25017c478bd9Sstevel@tonic-gate /* 25027c478bd9Sstevel@tonic-gate * Reacquire the mutex. 25037c478bd9Sstevel@tonic-gate */ 25047c478bd9Sstevel@tonic-gate if (set_lock_byte(&mp->mutex_lockw) == 0) { 25057c478bd9Sstevel@tonic-gate mp->mutex_owner = (uintptr_t)self; 25067c478bd9Sstevel@tonic-gate DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0); 25077c478bd9Sstevel@tonic-gate } else if (mutex_trylock_adaptive(mp) != 0) { 25087c478bd9Sstevel@tonic-gate (void) mutex_lock_queue(self, msp, mp, NULL); 25097c478bd9Sstevel@tonic-gate } 25107c478bd9Sstevel@tonic-gate 25117c478bd9Sstevel@tonic-gate if (msp) 25127c478bd9Sstevel@tonic-gate record_begin_hold(msp); 25137c478bd9Sstevel@tonic-gate 25147c478bd9Sstevel@tonic-gate /* 25157c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 25167c478bd9Sstevel@tonic-gate */ 25177c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 25187c478bd9Sstevel@tonic-gate sigon(self); 25197c478bd9Sstevel@tonic-gate 25207c478bd9Sstevel@tonic-gate return (error); 25217c478bd9Sstevel@tonic-gate } 25227c478bd9Sstevel@tonic-gate 25237c478bd9Sstevel@tonic-gate /* 25247c478bd9Sstevel@tonic-gate * cond_sleep_kernel(): utility function for cond_wait_kernel(). 25257c478bd9Sstevel@tonic-gate * See the comment ahead of cond_sleep_queue(), above. 25267c478bd9Sstevel@tonic-gate */ 25277c478bd9Sstevel@tonic-gate int 25287c478bd9Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 25297c478bd9Sstevel@tonic-gate { 25307c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 25317c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 25327c478bd9Sstevel@tonic-gate int error; 25337c478bd9Sstevel@tonic-gate 25347c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_PROTECT) { 25357c478bd9Sstevel@tonic-gate if (_ceil_mylist_del(mp)) 25367c478bd9Sstevel@tonic-gate _ceil_prio_waive(); 25377c478bd9Sstevel@tonic-gate } 25387c478bd9Sstevel@tonic-gate 25397c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 25407c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 25417c478bd9Sstevel@tonic-gate mp->mutex_owner = 0; 25427c478bd9Sstevel@tonic-gate mp->mutex_ownerpid = 0; 25437c478bd9Sstevel@tonic-gate if (mtype & PTHREAD_PRIO_INHERIT) 25447c478bd9Sstevel@tonic-gate mp->mutex_lockw = LOCKCLEAR; 25457c478bd9Sstevel@tonic-gate /* 25467c478bd9Sstevel@tonic-gate * ___lwp_cond_wait() returns immediately with EINTR if 25477c478bd9Sstevel@tonic-gate * set_parking_flag(self,0) is called on this lwp before it 25487c478bd9Sstevel@tonic-gate * goes to sleep in the kernel. sigacthandler() calls this 25497c478bd9Sstevel@tonic-gate * when a deferred signal is noted. This assures that we don't 25507c478bd9Sstevel@tonic-gate * get stuck in ___lwp_cond_wait() with all signals blocked 25517c478bd9Sstevel@tonic-gate * due to taking a deferred signal before going to sleep. 25527c478bd9Sstevel@tonic-gate */ 25537c478bd9Sstevel@tonic-gate set_parking_flag(self, 1); 25547c478bd9Sstevel@tonic-gate if (self->ul_cursig != 0 || 25557c478bd9Sstevel@tonic-gate (self->ul_cancelable && self->ul_cancel_pending)) 25567c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 25577c478bd9Sstevel@tonic-gate error = ___lwp_cond_wait(cvp, mp, tsp, 1); 25587c478bd9Sstevel@tonic-gate set_parking_flag(self, 0); 25597c478bd9Sstevel@tonic-gate self->ul_sp = 0; 25607c478bd9Sstevel@tonic-gate self->ul_wchan = NULL; 25617c478bd9Sstevel@tonic-gate return (error); 25627c478bd9Sstevel@tonic-gate } 25637c478bd9Sstevel@tonic-gate 25647c478bd9Sstevel@tonic-gate int 25657c478bd9Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 25667c478bd9Sstevel@tonic-gate { 25677c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 25687c478bd9Sstevel@tonic-gate int error; 25697c478bd9Sstevel@tonic-gate int merror; 25707c478bd9Sstevel@tonic-gate 25717c478bd9Sstevel@tonic-gate /* 25727c478bd9Sstevel@tonic-gate * See the large comment in cond_wait_queue(), above. 25737c478bd9Sstevel@tonic-gate */ 25747c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 25757c478bd9Sstevel@tonic-gate sigoff(self); 25767c478bd9Sstevel@tonic-gate 25777c478bd9Sstevel@tonic-gate error = cond_sleep_kernel(cvp, mp, tsp); 25787c478bd9Sstevel@tonic-gate 25797c478bd9Sstevel@tonic-gate /* 25807c478bd9Sstevel@tonic-gate * Override the return code from ___lwp_cond_wait() 25817c478bd9Sstevel@tonic-gate * with any non-zero return code from mutex_lock(). 25827c478bd9Sstevel@tonic-gate * This addresses robust lock failures in particular; 25837c478bd9Sstevel@tonic-gate * the caller must see the EOWNERDEAD or ENOTRECOVERABLE 25847c478bd9Sstevel@tonic-gate * errors in order to take corrective action. 25857c478bd9Sstevel@tonic-gate */ 25867c478bd9Sstevel@tonic-gate if ((merror = _private_mutex_lock(mp)) != 0) 25877c478bd9Sstevel@tonic-gate error = merror; 25887c478bd9Sstevel@tonic-gate 25897c478bd9Sstevel@tonic-gate /* 25907c478bd9Sstevel@tonic-gate * Take any deferred signal now, after we have reacquired the mutex. 25917c478bd9Sstevel@tonic-gate */ 25927c478bd9Sstevel@tonic-gate if (self->ul_cond_wait_defer) 25937c478bd9Sstevel@tonic-gate sigon(self); 25947c478bd9Sstevel@tonic-gate 25957c478bd9Sstevel@tonic-gate return (error); 25967c478bd9Sstevel@tonic-gate } 25977c478bd9Sstevel@tonic-gate 25987c478bd9Sstevel@tonic-gate /* 25997c478bd9Sstevel@tonic-gate * Common code for _cond_wait() and _cond_timedwait() 26007c478bd9Sstevel@tonic-gate */ 26017c478bd9Sstevel@tonic-gate int 26027c478bd9Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp) 26037c478bd9Sstevel@tonic-gate { 26047c478bd9Sstevel@tonic-gate int mtype = mp->mutex_type; 26057c478bd9Sstevel@tonic-gate hrtime_t begin_sleep = 0; 26067c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 26077c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 26087c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 26097c478bd9Sstevel@tonic-gate tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp); 26107c478bd9Sstevel@tonic-gate uint8_t rcount; 26117c478bd9Sstevel@tonic-gate int error = 0; 26127c478bd9Sstevel@tonic-gate 26137c478bd9Sstevel@tonic-gate /* 26147c478bd9Sstevel@tonic-gate * The SUSV3 Posix spec for pthread_cond_timedwait() states: 26157c478bd9Sstevel@tonic-gate * Except in the case of [ETIMEDOUT], all these error checks 26167c478bd9Sstevel@tonic-gate * shall act as if they were performed immediately at the 26177c478bd9Sstevel@tonic-gate * beginning of processing for the function and shall cause 26187c478bd9Sstevel@tonic-gate * an error return, in effect, prior to modifying the state 26197c478bd9Sstevel@tonic-gate * of the mutex specified by mutex or the condition variable 26207c478bd9Sstevel@tonic-gate * specified by cond. 26217c478bd9Sstevel@tonic-gate * Therefore, we must return EINVAL now if the timout is invalid. 26227c478bd9Sstevel@tonic-gate */ 26237c478bd9Sstevel@tonic-gate if (tsp != NULL && 26247c478bd9Sstevel@tonic-gate (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC)) 26257c478bd9Sstevel@tonic-gate return (EINVAL); 26267c478bd9Sstevel@tonic-gate 26277c478bd9Sstevel@tonic-gate if (__td_event_report(self, TD_SLEEP, udp)) { 26287c478bd9Sstevel@tonic-gate self->ul_sp = stkptr(); 26297c478bd9Sstevel@tonic-gate self->ul_wchan = cvp; 26307c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventnum = TD_SLEEP; 26317c478bd9Sstevel@tonic-gate self->ul_td_evbuf.eventdata = cvp; 26327c478bd9Sstevel@tonic-gate tdb_event(TD_SLEEP, udp); 26337c478bd9Sstevel@tonic-gate self->ul_sp = 0; 26347c478bd9Sstevel@tonic-gate } 26357c478bd9Sstevel@tonic-gate if (csp) { 26367c478bd9Sstevel@tonic-gate if (tsp) 26377c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait); 26387c478bd9Sstevel@tonic-gate else 26397c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_wait); 26407c478bd9Sstevel@tonic-gate } 26417c478bd9Sstevel@tonic-gate if (msp) 26427c478bd9Sstevel@tonic-gate begin_sleep = record_hold_time(msp); 26437c478bd9Sstevel@tonic-gate else if (csp) 26447c478bd9Sstevel@tonic-gate begin_sleep = gethrtime(); 26457c478bd9Sstevel@tonic-gate 26467c478bd9Sstevel@tonic-gate if (self->ul_error_detection) { 26477c478bd9Sstevel@tonic-gate if (!mutex_is_held(mp)) 26487c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, NULL); 26497c478bd9Sstevel@tonic-gate if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) 26507c478bd9Sstevel@tonic-gate lock_error(mp, "recursive mutex in cond_wait", 26517c478bd9Sstevel@tonic-gate cvp, NULL); 26527c478bd9Sstevel@tonic-gate if (cvp->cond_type & USYNC_PROCESS) { 26537c478bd9Sstevel@tonic-gate if (!(mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))) 26547c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 26557c478bd9Sstevel@tonic-gate "condvar process-shared, " 26567c478bd9Sstevel@tonic-gate "mutex process-private"); 26577c478bd9Sstevel@tonic-gate } else { 26587c478bd9Sstevel@tonic-gate if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) 26597c478bd9Sstevel@tonic-gate lock_error(mp, "cond_wait", cvp, 26607c478bd9Sstevel@tonic-gate "condvar process-private, " 26617c478bd9Sstevel@tonic-gate "mutex process-shared"); 26627c478bd9Sstevel@tonic-gate } 26637c478bd9Sstevel@tonic-gate } 26647c478bd9Sstevel@tonic-gate 26657c478bd9Sstevel@tonic-gate /* 26667c478bd9Sstevel@tonic-gate * We deal with recursive mutexes by completely 26677c478bd9Sstevel@tonic-gate * dropping the lock and restoring the recursion 26687c478bd9Sstevel@tonic-gate * count after waking up. This is arguably wrong, 26697c478bd9Sstevel@tonic-gate * but it obeys the principle of least astonishment. 26707c478bd9Sstevel@tonic-gate */ 26717c478bd9Sstevel@tonic-gate rcount = mp->mutex_rcount; 26727c478bd9Sstevel@tonic-gate mp->mutex_rcount = 0; 26737c478bd9Sstevel@tonic-gate if ((mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST | 26747c478bd9Sstevel@tonic-gate PTHREAD_PRIO_INHERIT | PTHREAD_PRIO_PROTECT)) | 26757c478bd9Sstevel@tonic-gate (cvp->cond_type & USYNC_PROCESS)) 26767c478bd9Sstevel@tonic-gate error = cond_wait_kernel(cvp, mp, tsp); 26777c478bd9Sstevel@tonic-gate else 26787c478bd9Sstevel@tonic-gate error = cond_wait_queue(cvp, mp, tsp, msp); 26797c478bd9Sstevel@tonic-gate mp->mutex_rcount = rcount; 26807c478bd9Sstevel@tonic-gate 26817c478bd9Sstevel@tonic-gate if (csp) { 26827c478bd9Sstevel@tonic-gate hrtime_t lapse = gethrtime() - begin_sleep; 26837c478bd9Sstevel@tonic-gate if (tsp == NULL) 26847c478bd9Sstevel@tonic-gate csp->cond_wait_sleep_time += lapse; 26857c478bd9Sstevel@tonic-gate else { 26867c478bd9Sstevel@tonic-gate csp->cond_timedwait_sleep_time += lapse; 26877c478bd9Sstevel@tonic-gate if (error == ETIME) 26887c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_timedwait_timeout); 26897c478bd9Sstevel@tonic-gate } 26907c478bd9Sstevel@tonic-gate } 26917c478bd9Sstevel@tonic-gate return (error); 26927c478bd9Sstevel@tonic-gate } 26937c478bd9Sstevel@tonic-gate 26947c478bd9Sstevel@tonic-gate /* 26957c478bd9Sstevel@tonic-gate * cond_wait() is a cancellation point but _cond_wait() is not. 26967c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 26977c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 26987c478bd9Sstevel@tonic-gate */ 26997c478bd9Sstevel@tonic-gate int 27007c478bd9Sstevel@tonic-gate _cond_wait(cond_t *cvp, mutex_t *mp) 27017c478bd9Sstevel@tonic-gate { 27027c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 27037c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 27047c478bd9Sstevel@tonic-gate uberflags_t *gflags; 27057c478bd9Sstevel@tonic-gate 27067c478bd9Sstevel@tonic-gate /* 27077c478bd9Sstevel@tonic-gate * Optimize the common case of USYNC_THREAD plus 27087c478bd9Sstevel@tonic-gate * no error detection, no lock statistics, and no event tracing. 27097c478bd9Sstevel@tonic-gate */ 27107c478bd9Sstevel@tonic-gate if ((gflags = self->ul_schedctl_called) != NULL && 27117c478bd9Sstevel@tonic-gate (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted | 27127c478bd9Sstevel@tonic-gate self->ul_td_events_enable | 27137c478bd9Sstevel@tonic-gate udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0) 27147c478bd9Sstevel@tonic-gate return (cond_wait_queue(cvp, mp, NULL, NULL)); 27157c478bd9Sstevel@tonic-gate 27167c478bd9Sstevel@tonic-gate /* 27177c478bd9Sstevel@tonic-gate * Else do it the long way. 27187c478bd9Sstevel@tonic-gate */ 27197c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, NULL)); 27207c478bd9Sstevel@tonic-gate } 27217c478bd9Sstevel@tonic-gate 27227c478bd9Sstevel@tonic-gate int 27237c478bd9Sstevel@tonic-gate cond_wait(cond_t *cvp, mutex_t *mp) 27247c478bd9Sstevel@tonic-gate { 27257c478bd9Sstevel@tonic-gate int error; 27267c478bd9Sstevel@tonic-gate 27277c478bd9Sstevel@tonic-gate _cancelon(); 27287c478bd9Sstevel@tonic-gate error = _cond_wait(cvp, mp); 27297c478bd9Sstevel@tonic-gate if (error == EINTR) 27307c478bd9Sstevel@tonic-gate _canceloff(); 27317c478bd9Sstevel@tonic-gate else 27327c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 27337c478bd9Sstevel@tonic-gate return (error); 27347c478bd9Sstevel@tonic-gate } 27357c478bd9Sstevel@tonic-gate 27367c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_wait = _pthread_cond_wait 27377c478bd9Sstevel@tonic-gate int 27387c478bd9Sstevel@tonic-gate _pthread_cond_wait(cond_t *cvp, mutex_t *mp) 27397c478bd9Sstevel@tonic-gate { 27407c478bd9Sstevel@tonic-gate int error; 27417c478bd9Sstevel@tonic-gate 27427c478bd9Sstevel@tonic-gate error = cond_wait(cvp, mp); 27437c478bd9Sstevel@tonic-gate return ((error == EINTR)? 0 : error); 27447c478bd9Sstevel@tonic-gate } 27457c478bd9Sstevel@tonic-gate 27467c478bd9Sstevel@tonic-gate /* 27477c478bd9Sstevel@tonic-gate * cond_timedwait() is a cancellation point but _cond_timedwait() is not. 27487c478bd9Sstevel@tonic-gate * System libraries call the non-cancellation version. 27497c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 27507c478bd9Sstevel@tonic-gate */ 27517c478bd9Sstevel@tonic-gate int 27527c478bd9Sstevel@tonic-gate _cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 27537c478bd9Sstevel@tonic-gate { 27547c478bd9Sstevel@tonic-gate clockid_t clock_id = cvp->cond_clockid; 27557c478bd9Sstevel@tonic-gate timespec_t reltime; 27567c478bd9Sstevel@tonic-gate int error; 27577c478bd9Sstevel@tonic-gate 27587c478bd9Sstevel@tonic-gate if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES) 27597c478bd9Sstevel@tonic-gate clock_id = CLOCK_REALTIME; 27607c478bd9Sstevel@tonic-gate abstime_to_reltime(clock_id, abstime, &reltime); 27617c478bd9Sstevel@tonic-gate error = cond_wait_common(cvp, mp, &reltime); 27627c478bd9Sstevel@tonic-gate if (error == ETIME && clock_id == CLOCK_HIGHRES) { 27637c478bd9Sstevel@tonic-gate /* 27647c478bd9Sstevel@tonic-gate * Don't return ETIME if we didn't really get a timeout. 27657c478bd9Sstevel@tonic-gate * This can happen if we return because someone resets 27667c478bd9Sstevel@tonic-gate * the system clock. Just return zero in this case, 27677c478bd9Sstevel@tonic-gate * giving a spurious wakeup but not a timeout. 27687c478bd9Sstevel@tonic-gate */ 27697c478bd9Sstevel@tonic-gate if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC + 27707c478bd9Sstevel@tonic-gate abstime->tv_nsec > gethrtime()) 27717c478bd9Sstevel@tonic-gate error = 0; 27727c478bd9Sstevel@tonic-gate } 27737c478bd9Sstevel@tonic-gate return (error); 27747c478bd9Sstevel@tonic-gate } 27757c478bd9Sstevel@tonic-gate 27767c478bd9Sstevel@tonic-gate int 27777c478bd9Sstevel@tonic-gate cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 27787c478bd9Sstevel@tonic-gate { 27797c478bd9Sstevel@tonic-gate int error; 27807c478bd9Sstevel@tonic-gate 27817c478bd9Sstevel@tonic-gate _cancelon(); 27827c478bd9Sstevel@tonic-gate error = _cond_timedwait(cvp, mp, abstime); 27837c478bd9Sstevel@tonic-gate if (error == EINTR) 27847c478bd9Sstevel@tonic-gate _canceloff(); 27857c478bd9Sstevel@tonic-gate else 27867c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 27877c478bd9Sstevel@tonic-gate return (error); 27887c478bd9Sstevel@tonic-gate } 27897c478bd9Sstevel@tonic-gate 27907c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_timedwait = _pthread_cond_timedwait 27917c478bd9Sstevel@tonic-gate int 27927c478bd9Sstevel@tonic-gate _pthread_cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime) 27937c478bd9Sstevel@tonic-gate { 27947c478bd9Sstevel@tonic-gate int error; 27957c478bd9Sstevel@tonic-gate 27967c478bd9Sstevel@tonic-gate error = cond_timedwait(cvp, mp, abstime); 27977c478bd9Sstevel@tonic-gate if (error == ETIME) 27987c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 27997c478bd9Sstevel@tonic-gate else if (error == EINTR) 28007c478bd9Sstevel@tonic-gate error = 0; 28017c478bd9Sstevel@tonic-gate return (error); 28027c478bd9Sstevel@tonic-gate } 28037c478bd9Sstevel@tonic-gate 28047c478bd9Sstevel@tonic-gate /* 28057c478bd9Sstevel@tonic-gate * cond_reltimedwait() is a cancellation point but _cond_reltimedwait() 28067c478bd9Sstevel@tonic-gate * is not. System libraries call the non-cancellation version. 28077c478bd9Sstevel@tonic-gate * It is expected that only applications call the cancellation version. 28087c478bd9Sstevel@tonic-gate */ 28097c478bd9Sstevel@tonic-gate int 28107c478bd9Sstevel@tonic-gate _cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 28117c478bd9Sstevel@tonic-gate { 28127c478bd9Sstevel@tonic-gate timespec_t tslocal = *reltime; 28137c478bd9Sstevel@tonic-gate 28147c478bd9Sstevel@tonic-gate return (cond_wait_common(cvp, mp, &tslocal)); 28157c478bd9Sstevel@tonic-gate } 28167c478bd9Sstevel@tonic-gate 28177c478bd9Sstevel@tonic-gate #pragma weak cond_reltimedwait = _cond_reltimedwait_cancel 28187c478bd9Sstevel@tonic-gate int 28197c478bd9Sstevel@tonic-gate _cond_reltimedwait_cancel(cond_t *cvp, mutex_t *mp, const timespec_t *reltime) 28207c478bd9Sstevel@tonic-gate { 28217c478bd9Sstevel@tonic-gate int error; 28227c478bd9Sstevel@tonic-gate 28237c478bd9Sstevel@tonic-gate _cancelon(); 28247c478bd9Sstevel@tonic-gate error = _cond_reltimedwait(cvp, mp, reltime); 28257c478bd9Sstevel@tonic-gate if (error == EINTR) 28267c478bd9Sstevel@tonic-gate _canceloff(); 28277c478bd9Sstevel@tonic-gate else 28287c478bd9Sstevel@tonic-gate _canceloff_nocancel(); 28297c478bd9Sstevel@tonic-gate return (error); 28307c478bd9Sstevel@tonic-gate } 28317c478bd9Sstevel@tonic-gate 28327c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_reltimedwait_np = _pthread_cond_reltimedwait_np 28337c478bd9Sstevel@tonic-gate int 28347c478bd9Sstevel@tonic-gate _pthread_cond_reltimedwait_np(cond_t *cvp, mutex_t *mp, 28357c478bd9Sstevel@tonic-gate const timespec_t *reltime) 28367c478bd9Sstevel@tonic-gate { 28377c478bd9Sstevel@tonic-gate int error; 28387c478bd9Sstevel@tonic-gate 28397c478bd9Sstevel@tonic-gate error = _cond_reltimedwait_cancel(cvp, mp, reltime); 28407c478bd9Sstevel@tonic-gate if (error == ETIME) 28417c478bd9Sstevel@tonic-gate error = ETIMEDOUT; 28427c478bd9Sstevel@tonic-gate else if (error == EINTR) 28437c478bd9Sstevel@tonic-gate error = 0; 28447c478bd9Sstevel@tonic-gate return (error); 28457c478bd9Sstevel@tonic-gate } 28467c478bd9Sstevel@tonic-gate 28477c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_signal = cond_signal_internal 28487c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_signal = cond_signal_internal 28497c478bd9Sstevel@tonic-gate #pragma weak cond_signal = cond_signal_internal 28507c478bd9Sstevel@tonic-gate #pragma weak _cond_signal = cond_signal_internal 28517c478bd9Sstevel@tonic-gate int 28527c478bd9Sstevel@tonic-gate cond_signal_internal(cond_t *cvp) 28537c478bd9Sstevel@tonic-gate { 28547c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 28557c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 28567c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 28577c478bd9Sstevel@tonic-gate int error = 0; 28587c478bd9Sstevel@tonic-gate queue_head_t *qp; 28597c478bd9Sstevel@tonic-gate mutex_t *mp; 28607c478bd9Sstevel@tonic-gate queue_head_t *mqp; 28617c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 28627c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 28637c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 28647c478bd9Sstevel@tonic-gate ulwp_t *next; 28657c478bd9Sstevel@tonic-gate ulwp_t **suspp = NULL; 28667c478bd9Sstevel@tonic-gate ulwp_t *susprev; 28677c478bd9Sstevel@tonic-gate 28687c478bd9Sstevel@tonic-gate if (csp) 28697c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_signal); 28707c478bd9Sstevel@tonic-gate 28717c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 28727c478bd9Sstevel@tonic-gate error = __lwp_cond_signal(cvp); 28737c478bd9Sstevel@tonic-gate 28747c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 28757c478bd9Sstevel@tonic-gate return (error); 28767c478bd9Sstevel@tonic-gate 28777c478bd9Sstevel@tonic-gate /* 28787c478bd9Sstevel@tonic-gate * Move someone from the condvar sleep queue to the mutex sleep 28797c478bd9Sstevel@tonic-gate * queue for the mutex that he will acquire on being waked up. 28807c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex he will acquire. 28817c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if his ul_cv_wake flag 28827c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark him. 28837c478bd9Sstevel@tonic-gate */ 28847c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 28857c478bd9Sstevel@tonic-gate for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL; 28867c478bd9Sstevel@tonic-gate prev = ulwp, ulwpp = &ulwp->ul_link) { 28877c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan == cvp) { 28887c478bd9Sstevel@tonic-gate if (!ulwp->ul_stop) 28897c478bd9Sstevel@tonic-gate break; 28907c478bd9Sstevel@tonic-gate /* 28917c478bd9Sstevel@tonic-gate * Try not to dequeue a suspended thread. 28927c478bd9Sstevel@tonic-gate * This mimics the old libthread's behavior. 28937c478bd9Sstevel@tonic-gate */ 28947c478bd9Sstevel@tonic-gate if (suspp == NULL) { 28957c478bd9Sstevel@tonic-gate suspp = ulwpp; 28967c478bd9Sstevel@tonic-gate susprev = prev; 28977c478bd9Sstevel@tonic-gate } 28987c478bd9Sstevel@tonic-gate } 28997c478bd9Sstevel@tonic-gate } 29007c478bd9Sstevel@tonic-gate if (ulwp == NULL && suspp != NULL) { 29017c478bd9Sstevel@tonic-gate ulwp = *(ulwpp = suspp); 29027c478bd9Sstevel@tonic-gate prev = susprev; 29037c478bd9Sstevel@tonic-gate suspp = NULL; 29047c478bd9Sstevel@tonic-gate } 29057c478bd9Sstevel@tonic-gate if (ulwp == NULL) { /* no one on the sleep queue */ 29067c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 29077c478bd9Sstevel@tonic-gate queue_unlock(qp); 29087c478bd9Sstevel@tonic-gate return (error); 29097c478bd9Sstevel@tonic-gate } 29107c478bd9Sstevel@tonic-gate /* 29117c478bd9Sstevel@tonic-gate * Scan the remainder of the CV queue for another waiter. 29127c478bd9Sstevel@tonic-gate */ 29137c478bd9Sstevel@tonic-gate if (suspp != NULL) { 29147c478bd9Sstevel@tonic-gate next = *suspp; 29157c478bd9Sstevel@tonic-gate } else { 29167c478bd9Sstevel@tonic-gate for (next = ulwp->ul_link; next != NULL; next = next->ul_link) 29177c478bd9Sstevel@tonic-gate if (next->ul_wchan == cvp) 29187c478bd9Sstevel@tonic-gate break; 29197c478bd9Sstevel@tonic-gate } 29207c478bd9Sstevel@tonic-gate if (next == NULL) 29217c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 29227c478bd9Sstevel@tonic-gate 29237c478bd9Sstevel@tonic-gate /* 29247c478bd9Sstevel@tonic-gate * Inform the thread that he was the recipient of a cond_signal(). 29257c478bd9Sstevel@tonic-gate * This lets him deal with cond_signal() and, concurrently, 29267c478bd9Sstevel@tonic-gate * one or more of a cancellation, a UNIX signal, or a timeout. 29277c478bd9Sstevel@tonic-gate * These latter conditions must not consume a cond_signal(). 29287c478bd9Sstevel@tonic-gate */ 29297c478bd9Sstevel@tonic-gate ulwp->ul_signalled = 1; 29307c478bd9Sstevel@tonic-gate 29317c478bd9Sstevel@tonic-gate /* 29327c478bd9Sstevel@tonic-gate * Dequeue the waiter but leave his ul_sleepq non-NULL 29337c478bd9Sstevel@tonic-gate * while we move him to the mutex queue so that he can 29347c478bd9Sstevel@tonic-gate * deal properly with spurious wakeups. 29357c478bd9Sstevel@tonic-gate */ 29367c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 29377c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 29387c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 29397c478bd9Sstevel@tonic-gate qp->qh_qlen--; 29407c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 29417c478bd9Sstevel@tonic-gate 29427c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* the mutex he will acquire */ 29437c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 29447c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 29457c478bd9Sstevel@tonic-gate 29467c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 29477c478bd9Sstevel@tonic-gate lwpid_t lwpid = ulwp->ul_lwpid; 29487c478bd9Sstevel@tonic-gate 29497c478bd9Sstevel@tonic-gate no_preempt(self); 29507c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 29517c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 29527c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 29537c478bd9Sstevel@tonic-gate queue_unlock(qp); 29547c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid); 29557c478bd9Sstevel@tonic-gate preempt(self); 29567c478bd9Sstevel@tonic-gate } else { 29577c478bd9Sstevel@tonic-gate mqp = queue_lock(mp, MX); 29587c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 29597c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 29607c478bd9Sstevel@tonic-gate queue_unlock(mqp); 29617c478bd9Sstevel@tonic-gate queue_unlock(qp); 29627c478bd9Sstevel@tonic-gate } 29637c478bd9Sstevel@tonic-gate 29647c478bd9Sstevel@tonic-gate return (error); 29657c478bd9Sstevel@tonic-gate } 29667c478bd9Sstevel@tonic-gate 29677c478bd9Sstevel@tonic-gate #define MAXLWPS 128 /* max remembered lwpids before overflow */ 29687c478bd9Sstevel@tonic-gate #define NEWLWPS 2048 /* max remembered lwpids at first overflow */ 29697c478bd9Sstevel@tonic-gate 29707c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_broadcast = cond_broadcast_internal 29717c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_broadcast = cond_broadcast_internal 29727c478bd9Sstevel@tonic-gate #pragma weak cond_broadcast = cond_broadcast_internal 29737c478bd9Sstevel@tonic-gate #pragma weak _cond_broadcast = cond_broadcast_internal 29747c478bd9Sstevel@tonic-gate int 29757c478bd9Sstevel@tonic-gate cond_broadcast_internal(cond_t *cvp) 29767c478bd9Sstevel@tonic-gate { 29777c478bd9Sstevel@tonic-gate ulwp_t *self = curthread; 29787c478bd9Sstevel@tonic-gate uberdata_t *udp = self->ul_uberdata; 29797c478bd9Sstevel@tonic-gate tdb_cond_stats_t *csp = COND_STATS(cvp, udp); 29807c478bd9Sstevel@tonic-gate int error = 0; 29817c478bd9Sstevel@tonic-gate queue_head_t *qp; 29827c478bd9Sstevel@tonic-gate mutex_t *mp; 29837c478bd9Sstevel@tonic-gate queue_head_t *mqp; 29847c478bd9Sstevel@tonic-gate mutex_t *mp_cache = NULL; 29857c478bd9Sstevel@tonic-gate queue_head_t *mqp_cache = NULL; 29867c478bd9Sstevel@tonic-gate ulwp_t **ulwpp; 29877c478bd9Sstevel@tonic-gate ulwp_t *ulwp; 29887c478bd9Sstevel@tonic-gate ulwp_t *prev = NULL; 29897c478bd9Sstevel@tonic-gate lwpid_t buffer[MAXLWPS]; 29907c478bd9Sstevel@tonic-gate lwpid_t *lwpid = buffer; 29917c478bd9Sstevel@tonic-gate int nlwpid = 0; 29927c478bd9Sstevel@tonic-gate int maxlwps = MAXLWPS; 29937c478bd9Sstevel@tonic-gate 29947c478bd9Sstevel@tonic-gate if (csp) 29957c478bd9Sstevel@tonic-gate tdb_incr(csp->cond_broadcast); 29967c478bd9Sstevel@tonic-gate 29977c478bd9Sstevel@tonic-gate if (cvp->cond_waiters_kernel) /* someone sleeping in the kernel? */ 29987c478bd9Sstevel@tonic-gate error = __lwp_cond_broadcast(cvp); 29997c478bd9Sstevel@tonic-gate 30007c478bd9Sstevel@tonic-gate if (!cvp->cond_waiters_user) /* no one sleeping at user-level */ 30017c478bd9Sstevel@tonic-gate return (error); 30027c478bd9Sstevel@tonic-gate 30037c478bd9Sstevel@tonic-gate /* 30047c478bd9Sstevel@tonic-gate * Move everyone from the condvar sleep queue to the mutex sleep 30057c478bd9Sstevel@tonic-gate * queue for the mutex that they will acquire on being waked up. 30067c478bd9Sstevel@tonic-gate * We can do this only if we own the mutex they will acquire. 30077c478bd9Sstevel@tonic-gate * If we do not own the mutex, or if their ul_cv_wake flag 30087c478bd9Sstevel@tonic-gate * is set, just dequeue and unpark them. 30097c478bd9Sstevel@tonic-gate * 30107c478bd9Sstevel@tonic-gate * We keep track of lwpids that are to be unparked in lwpid[]. 30117c478bd9Sstevel@tonic-gate * __lwp_unpark_all() is called to unpark all of them after 30127c478bd9Sstevel@tonic-gate * they have been removed from the sleep queue and the sleep 30137c478bd9Sstevel@tonic-gate * queue lock has been dropped. If we run out of space in our 30147c478bd9Sstevel@tonic-gate * on-stack buffer, we need to allocate more but we can't call 30157c478bd9Sstevel@tonic-gate * lmalloc() because we are holding a queue lock when the overflow 30167c478bd9Sstevel@tonic-gate * occurs and lmalloc() acquires a lock. We can't use alloca() 30177c478bd9Sstevel@tonic-gate * either because the application may have allocated a small stack 30187c478bd9Sstevel@tonic-gate * and we don't want to overrun the stack. So we use the mmap() 30197c478bd9Sstevel@tonic-gate * system call directly since that path acquires no locks. 30207c478bd9Sstevel@tonic-gate */ 30217c478bd9Sstevel@tonic-gate qp = queue_lock(cvp, CV); 30227c478bd9Sstevel@tonic-gate cvp->cond_waiters_user = 0; 30237c478bd9Sstevel@tonic-gate ulwpp = &qp->qh_head; 30247c478bd9Sstevel@tonic-gate while ((ulwp = *ulwpp) != NULL) { 30257c478bd9Sstevel@tonic-gate 30267c478bd9Sstevel@tonic-gate if (ulwp->ul_wchan != cvp) { 30277c478bd9Sstevel@tonic-gate prev = ulwp; 30287c478bd9Sstevel@tonic-gate ulwpp = &ulwp->ul_link; 30297c478bd9Sstevel@tonic-gate continue; 30307c478bd9Sstevel@tonic-gate } 30317c478bd9Sstevel@tonic-gate 30327c478bd9Sstevel@tonic-gate *ulwpp = ulwp->ul_link; 30337c478bd9Sstevel@tonic-gate if (qp->qh_tail == ulwp) 30347c478bd9Sstevel@tonic-gate qp->qh_tail = prev; 30357c478bd9Sstevel@tonic-gate qp->qh_qlen--; 30367c478bd9Sstevel@tonic-gate ulwp->ul_link = NULL; 30377c478bd9Sstevel@tonic-gate 30387c478bd9Sstevel@tonic-gate mp = ulwp->ul_cvmutex; /* his mutex */ 30397c478bd9Sstevel@tonic-gate ulwp->ul_cvmutex = NULL; 30407c478bd9Sstevel@tonic-gate ASSERT(mp != NULL); 30417c478bd9Sstevel@tonic-gate 30427c478bd9Sstevel@tonic-gate if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) { 30437c478bd9Sstevel@tonic-gate ulwp->ul_sleepq = NULL; 30447c478bd9Sstevel@tonic-gate ulwp->ul_wchan = NULL; 30457c478bd9Sstevel@tonic-gate ulwp->ul_cv_wake = 0; 30467c478bd9Sstevel@tonic-gate if (nlwpid == maxlwps) { 30477c478bd9Sstevel@tonic-gate /* 30487c478bd9Sstevel@tonic-gate * Allocate NEWLWPS ids on the first overflow. 30497c478bd9Sstevel@tonic-gate * Double the allocation each time after that. 30507c478bd9Sstevel@tonic-gate */ 30517c478bd9Sstevel@tonic-gate int newlwps = (lwpid == buffer)? NEWLWPS : 30527c478bd9Sstevel@tonic-gate 2 * maxlwps; 30537c478bd9Sstevel@tonic-gate void *vaddr = _private_mmap(NULL, 30547c478bd9Sstevel@tonic-gate newlwps * sizeof (lwpid_t), 30557c478bd9Sstevel@tonic-gate PROT_READ|PROT_WRITE, 30567c478bd9Sstevel@tonic-gate MAP_PRIVATE|MAP_ANON, -1, (off_t)0); 30577c478bd9Sstevel@tonic-gate if (vaddr == MAP_FAILED) { 30587c478bd9Sstevel@tonic-gate /* 30597c478bd9Sstevel@tonic-gate * Let's hope this never happens. 30607c478bd9Sstevel@tonic-gate * If it does, then we have a terrible 30617c478bd9Sstevel@tonic-gate * thundering herd on our hands. 30627c478bd9Sstevel@tonic-gate */ 30637c478bd9Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid); 30647c478bd9Sstevel@tonic-gate nlwpid = 0; 30657c478bd9Sstevel@tonic-gate } else { 30667c478bd9Sstevel@tonic-gate (void) _memcpy(vaddr, lwpid, 30677c478bd9Sstevel@tonic-gate maxlwps * sizeof (lwpid_t)); 30687c478bd9Sstevel@tonic-gate if (lwpid != buffer) 30697c478bd9Sstevel@tonic-gate (void) _private_munmap(lwpid, 30707c478bd9Sstevel@tonic-gate maxlwps * sizeof (lwpid_t)); 30717c478bd9Sstevel@tonic-gate lwpid = vaddr; 30727c478bd9Sstevel@tonic-gate maxlwps = newlwps; 30737c478bd9Sstevel@tonic-gate } 30747c478bd9Sstevel@tonic-gate } 30757c478bd9Sstevel@tonic-gate lwpid[nlwpid++] = ulwp->ul_lwpid; 30767c478bd9Sstevel@tonic-gate } else { 30777c478bd9Sstevel@tonic-gate if (mp != mp_cache) { 30787c478bd9Sstevel@tonic-gate if (mqp_cache != NULL) 30797c478bd9Sstevel@tonic-gate queue_unlock(mqp_cache); 30807c478bd9Sstevel@tonic-gate mqp_cache = queue_lock(mp, MX); 30817c478bd9Sstevel@tonic-gate mp_cache = mp; 30827c478bd9Sstevel@tonic-gate } 30837c478bd9Sstevel@tonic-gate mqp = mqp_cache; 30847c478bd9Sstevel@tonic-gate enqueue(mqp, ulwp, mp, MX); 30857c478bd9Sstevel@tonic-gate mp->mutex_waiters = 1; 30867c478bd9Sstevel@tonic-gate } 30877c478bd9Sstevel@tonic-gate } 30887c478bd9Sstevel@tonic-gate if (mqp_cache != NULL) 30897c478bd9Sstevel@tonic-gate queue_unlock(mqp_cache); 30907c478bd9Sstevel@tonic-gate queue_unlock(qp); 30917c478bd9Sstevel@tonic-gate if (nlwpid) { 30927c478bd9Sstevel@tonic-gate if (nlwpid == 1) 30937c478bd9Sstevel@tonic-gate (void) __lwp_unpark(lwpid[0]); 30947c478bd9Sstevel@tonic-gate else 30957c478bd9Sstevel@tonic-gate (void) __lwp_unpark_all(lwpid, nlwpid); 30967c478bd9Sstevel@tonic-gate } 30977c478bd9Sstevel@tonic-gate if (lwpid != buffer) 30987c478bd9Sstevel@tonic-gate (void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t)); 30997c478bd9Sstevel@tonic-gate 31007c478bd9Sstevel@tonic-gate return (error); 31017c478bd9Sstevel@tonic-gate } 31027c478bd9Sstevel@tonic-gate 31037c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_destroy = _cond_destroy 31047c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_destroy = _cond_destroy 31057c478bd9Sstevel@tonic-gate #pragma weak cond_destroy = _cond_destroy 31067c478bd9Sstevel@tonic-gate int 31077c478bd9Sstevel@tonic-gate _cond_destroy(cond_t *cvp) 31087c478bd9Sstevel@tonic-gate { 31097c478bd9Sstevel@tonic-gate cvp->cond_magic = 0; 31107c478bd9Sstevel@tonic-gate tdb_sync_obj_deregister(cvp); 31117c478bd9Sstevel@tonic-gate return (0); 31127c478bd9Sstevel@tonic-gate } 31137c478bd9Sstevel@tonic-gate 31147c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG) 31157c478bd9Sstevel@tonic-gate void 31167c478bd9Sstevel@tonic-gate assert_no_libc_locks_held(void) 31177c478bd9Sstevel@tonic-gate { 31187c478bd9Sstevel@tonic-gate ASSERT(!curthread->ul_critical || curthread->ul_bindflags); 31197c478bd9Sstevel@tonic-gate } 31207c478bd9Sstevel@tonic-gate #endif 31217c478bd9Sstevel@tonic-gate 31227c478bd9Sstevel@tonic-gate /* protected by link_lock */ 31237c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin; 31247c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin2; 31257c478bd9Sstevel@tonic-gate uint64_t spin_lock_sleep; 31267c478bd9Sstevel@tonic-gate uint64_t spin_lock_wakeup; 31277c478bd9Sstevel@tonic-gate 31287c478bd9Sstevel@tonic-gate /* 31297c478bd9Sstevel@tonic-gate * Record spin lock statistics. 31307c478bd9Sstevel@tonic-gate * Called by a thread exiting itself in thrp_exit(). 31317c478bd9Sstevel@tonic-gate * Also called via atexit() from the thread calling 31327c478bd9Sstevel@tonic-gate * exit() to do all the other threads as well. 31337c478bd9Sstevel@tonic-gate */ 31347c478bd9Sstevel@tonic-gate void 31357c478bd9Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp) 31367c478bd9Sstevel@tonic-gate { 31377c478bd9Sstevel@tonic-gate spin_lock_spin += ulwp->ul_spin_lock_spin; 31387c478bd9Sstevel@tonic-gate spin_lock_spin2 += ulwp->ul_spin_lock_spin2; 31397c478bd9Sstevel@tonic-gate spin_lock_sleep += ulwp->ul_spin_lock_sleep; 31407c478bd9Sstevel@tonic-gate spin_lock_wakeup += ulwp->ul_spin_lock_wakeup; 31417c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin = 0; 31427c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_spin2 = 0; 31437c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_sleep = 0; 31447c478bd9Sstevel@tonic-gate ulwp->ul_spin_lock_wakeup = 0; 31457c478bd9Sstevel@tonic-gate } 31467c478bd9Sstevel@tonic-gate 31477c478bd9Sstevel@tonic-gate /* 31487c478bd9Sstevel@tonic-gate * atexit function: dump the queue statistics to stderr. 31497c478bd9Sstevel@tonic-gate */ 3150e8031f0aSraf #if !defined(__lint) 3151e8031f0aSraf #define fprintf _fprintf 3152e8031f0aSraf #endif 31537c478bd9Sstevel@tonic-gate #include <stdio.h> 31547c478bd9Sstevel@tonic-gate void 31557c478bd9Sstevel@tonic-gate dump_queue_statistics(void) 31567c478bd9Sstevel@tonic-gate { 31577c478bd9Sstevel@tonic-gate uberdata_t *udp = curthread->ul_uberdata; 31587c478bd9Sstevel@tonic-gate queue_head_t *qp; 31597c478bd9Sstevel@tonic-gate int qn; 31607c478bd9Sstevel@tonic-gate uint64_t spin_lock_total = 0; 31617c478bd9Sstevel@tonic-gate 31627c478bd9Sstevel@tonic-gate if (udp->queue_head == NULL || thread_queue_dump == 0) 31637c478bd9Sstevel@tonic-gate return; 31647c478bd9Sstevel@tonic-gate 31657c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 || 31667c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 31677c478bd9Sstevel@tonic-gate return; 31687c478bd9Sstevel@tonic-gate for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) { 31697c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 31707c478bd9Sstevel@tonic-gate continue; 31717c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 31727c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 31737c478bd9Sstevel@tonic-gate (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 31747c478bd9Sstevel@tonic-gate return; 31757c478bd9Sstevel@tonic-gate } 31767c478bd9Sstevel@tonic-gate 31777c478bd9Sstevel@tonic-gate if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 || 31787c478bd9Sstevel@tonic-gate fprintf(stderr, "queue# lockcount max qlen\n") < 0) 31797c478bd9Sstevel@tonic-gate return; 31807c478bd9Sstevel@tonic-gate for (qn = 0; qn < QHASHSIZE; qn++, qp++) { 31817c478bd9Sstevel@tonic-gate if (qp->qh_lockcount == 0) 31827c478bd9Sstevel@tonic-gate continue; 31837c478bd9Sstevel@tonic-gate spin_lock_total += qp->qh_lockcount; 31847c478bd9Sstevel@tonic-gate if (fprintf(stderr, "%5d %12llu%12u\n", qn, 31857c478bd9Sstevel@tonic-gate (u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0) 31867c478bd9Sstevel@tonic-gate return; 31877c478bd9Sstevel@tonic-gate } 31887c478bd9Sstevel@tonic-gate 31897c478bd9Sstevel@tonic-gate (void) fprintf(stderr, "\n spin_lock_total = %10llu\n", 31907c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_total); 31917c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin = %10llu\n", 31927c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_spin); 31937c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_spin2 = %10llu\n", 31947c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_spin2); 31957c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_sleep = %10llu\n", 31967c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_sleep); 31977c478bd9Sstevel@tonic-gate (void) fprintf(stderr, " spin_lock_wakeup = %10llu\n", 31987c478bd9Sstevel@tonic-gate (u_longlong_t)spin_lock_wakeup); 31997c478bd9Sstevel@tonic-gate } 3200