xref: /illumos-gate/usr/src/lib/libc/port/threads/synch.c (revision e8031f0a8ed0e45c6d8847c5e09424e66fd34a4b)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
57c478bd9Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
67c478bd9Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
77c478bd9Sstevel@tonic-gate  * with the License.
87c478bd9Sstevel@tonic-gate  *
97c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
107c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
117c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
127c478bd9Sstevel@tonic-gate  * and limitations under the License.
137c478bd9Sstevel@tonic-gate  *
147c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
157c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
167c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
177c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
187c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
197c478bd9Sstevel@tonic-gate  *
207c478bd9Sstevel@tonic-gate  * CDDL HEADER END
217c478bd9Sstevel@tonic-gate  */
22*e8031f0aSraf 
237c478bd9Sstevel@tonic-gate /*
24*e8031f0aSraf  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
257c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
267c478bd9Sstevel@tonic-gate  */
277c478bd9Sstevel@tonic-gate 
287c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
297c478bd9Sstevel@tonic-gate 
307c478bd9Sstevel@tonic-gate #include <sys/sdt.h>
317c478bd9Sstevel@tonic-gate 
327c478bd9Sstevel@tonic-gate #include "lint.h"
337c478bd9Sstevel@tonic-gate #include "thr_uberdata.h"
347c478bd9Sstevel@tonic-gate 
357c478bd9Sstevel@tonic-gate /*
367c478bd9Sstevel@tonic-gate  * This mutex is initialized to be held by lwp#1.
377c478bd9Sstevel@tonic-gate  * It is used to block a thread that has returned from a mutex_lock()
387c478bd9Sstevel@tonic-gate  * of a PTHREAD_PRIO_INHERIT mutex with an unrecoverable error.
397c478bd9Sstevel@tonic-gate  */
407c478bd9Sstevel@tonic-gate mutex_t	stall_mutex = DEFAULTMUTEX;
417c478bd9Sstevel@tonic-gate 
427c478bd9Sstevel@tonic-gate static int shared_mutex_held(mutex_t *);
437c478bd9Sstevel@tonic-gate 
447c478bd9Sstevel@tonic-gate /*
457c478bd9Sstevel@tonic-gate  * Lock statistics support functions.
467c478bd9Sstevel@tonic-gate  */
477c478bd9Sstevel@tonic-gate void
487c478bd9Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp)
497c478bd9Sstevel@tonic-gate {
507c478bd9Sstevel@tonic-gate 	tdb_incr(msp->mutex_lock);
517c478bd9Sstevel@tonic-gate 	msp->mutex_begin_hold = gethrtime();
527c478bd9Sstevel@tonic-gate }
537c478bd9Sstevel@tonic-gate 
547c478bd9Sstevel@tonic-gate hrtime_t
557c478bd9Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp)
567c478bd9Sstevel@tonic-gate {
577c478bd9Sstevel@tonic-gate 	hrtime_t now = gethrtime();
587c478bd9Sstevel@tonic-gate 
597c478bd9Sstevel@tonic-gate 	if (msp->mutex_begin_hold)
607c478bd9Sstevel@tonic-gate 		msp->mutex_hold_time += now - msp->mutex_begin_hold;
617c478bd9Sstevel@tonic-gate 	msp->mutex_begin_hold = 0;
627c478bd9Sstevel@tonic-gate 	return (now);
637c478bd9Sstevel@tonic-gate }
647c478bd9Sstevel@tonic-gate 
657c478bd9Sstevel@tonic-gate /*
667c478bd9Sstevel@tonic-gate  * Called once at library initialization.
677c478bd9Sstevel@tonic-gate  */
687c478bd9Sstevel@tonic-gate void
697c478bd9Sstevel@tonic-gate mutex_setup(void)
707c478bd9Sstevel@tonic-gate {
717c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&stall_mutex.mutex_lockw))
727c478bd9Sstevel@tonic-gate 		thr_panic("mutex_setup() cannot acquire stall_mutex");
737c478bd9Sstevel@tonic-gate 	stall_mutex.mutex_owner = (uintptr_t)curthread;
747c478bd9Sstevel@tonic-gate }
757c478bd9Sstevel@tonic-gate 
767c478bd9Sstevel@tonic-gate /*
777c478bd9Sstevel@tonic-gate  * The default spin counts of 1000 and 500 are experimentally determined.
787c478bd9Sstevel@tonic-gate  * On sun4u machines with any number of processors they could be raised
797c478bd9Sstevel@tonic-gate  * to 10,000 but that (experimentally) makes almost no difference.
807c478bd9Sstevel@tonic-gate  * The environment variables:
817c478bd9Sstevel@tonic-gate  *	_THREAD_ADAPTIVE_SPIN=count
827c478bd9Sstevel@tonic-gate  *	_THREAD_RELEASE_SPIN=count
837c478bd9Sstevel@tonic-gate  * can be used to override and set the counts in the range [0 .. 1,000,000].
847c478bd9Sstevel@tonic-gate  */
857c478bd9Sstevel@tonic-gate int	thread_adaptive_spin = 1000;
867c478bd9Sstevel@tonic-gate uint_t	thread_max_spinners = 100;
877c478bd9Sstevel@tonic-gate int	thread_release_spin = 500;
887c478bd9Sstevel@tonic-gate int	thread_queue_verify = 0;
897c478bd9Sstevel@tonic-gate static	int	ncpus;
907c478bd9Sstevel@tonic-gate 
917c478bd9Sstevel@tonic-gate /*
927c478bd9Sstevel@tonic-gate  * Distinguish spinning for queue locks from spinning for regular locks.
937c478bd9Sstevel@tonic-gate  * The environment variable:
947c478bd9Sstevel@tonic-gate  *	_THREAD_QUEUE_SPIN=count
957c478bd9Sstevel@tonic-gate  * can be used to override and set the count in the range [0 .. 1,000,000].
967c478bd9Sstevel@tonic-gate  * There is no release spin concept for queue locks.
977c478bd9Sstevel@tonic-gate  */
987c478bd9Sstevel@tonic-gate int	thread_queue_spin = 1000;
997c478bd9Sstevel@tonic-gate 
1007c478bd9Sstevel@tonic-gate /*
1017c478bd9Sstevel@tonic-gate  * Use the otherwise-unused 'mutex_ownerpid' field of a USYNC_THREAD
1027c478bd9Sstevel@tonic-gate  * mutex to be a count of adaptive spins in progress.
1037c478bd9Sstevel@tonic-gate  */
1047c478bd9Sstevel@tonic-gate #define	mutex_spinners	mutex_ownerpid
1057c478bd9Sstevel@tonic-gate 
1067c478bd9Sstevel@tonic-gate void
1077c478bd9Sstevel@tonic-gate _mutex_set_typeattr(mutex_t *mp, int attr)
1087c478bd9Sstevel@tonic-gate {
1097c478bd9Sstevel@tonic-gate 	mp->mutex_type |= (uint8_t)attr;
1107c478bd9Sstevel@tonic-gate }
1117c478bd9Sstevel@tonic-gate 
1127c478bd9Sstevel@tonic-gate /*
1137c478bd9Sstevel@tonic-gate  * 'type' can be one of USYNC_THREAD or USYNC_PROCESS, possibly
1147c478bd9Sstevel@tonic-gate  * augmented by the flags LOCK_RECURSIVE and/or LOCK_ERRORCHECK,
1157c478bd9Sstevel@tonic-gate  * or it can be USYNC_PROCESS_ROBUST with no extra flags.
1167c478bd9Sstevel@tonic-gate  */
1177c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_init = __mutex_init
1187c478bd9Sstevel@tonic-gate #pragma weak mutex_init = __mutex_init
1197c478bd9Sstevel@tonic-gate #pragma weak _mutex_init = __mutex_init
1207c478bd9Sstevel@tonic-gate /* ARGSUSED2 */
1217c478bd9Sstevel@tonic-gate int
1227c478bd9Sstevel@tonic-gate __mutex_init(mutex_t *mp, int type, void *arg)
1237c478bd9Sstevel@tonic-gate {
1247c478bd9Sstevel@tonic-gate 	int error;
1257c478bd9Sstevel@tonic-gate 
1267c478bd9Sstevel@tonic-gate 	switch (type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) {
1277c478bd9Sstevel@tonic-gate 	case USYNC_THREAD:
1287c478bd9Sstevel@tonic-gate 	case USYNC_PROCESS:
1297c478bd9Sstevel@tonic-gate 		(void) _memset(mp, 0, sizeof (*mp));
1307c478bd9Sstevel@tonic-gate 		mp->mutex_type = (uint8_t)type;
1317c478bd9Sstevel@tonic-gate 		mp->mutex_flag = LOCK_INITED;
1327c478bd9Sstevel@tonic-gate 		error = 0;
1337c478bd9Sstevel@tonic-gate 		break;
1347c478bd9Sstevel@tonic-gate 	case USYNC_PROCESS_ROBUST:
1357c478bd9Sstevel@tonic-gate 		if (type & (LOCK_RECURSIVE|LOCK_ERRORCHECK))
1367c478bd9Sstevel@tonic-gate 			error = EINVAL;
1377c478bd9Sstevel@tonic-gate 		else
1387c478bd9Sstevel@tonic-gate 			error = ___lwp_mutex_init(mp, type);
1397c478bd9Sstevel@tonic-gate 		break;
1407c478bd9Sstevel@tonic-gate 	default:
1417c478bd9Sstevel@tonic-gate 		error = EINVAL;
1427c478bd9Sstevel@tonic-gate 		break;
1437c478bd9Sstevel@tonic-gate 	}
1447c478bd9Sstevel@tonic-gate 	if (error == 0)
1457c478bd9Sstevel@tonic-gate 		mp->mutex_magic = MUTEX_MAGIC;
1467c478bd9Sstevel@tonic-gate 	return (error);
1477c478bd9Sstevel@tonic-gate }
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate /*
1507c478bd9Sstevel@tonic-gate  * Delete mp from list of ceil mutexes owned by curthread.
1517c478bd9Sstevel@tonic-gate  * Return 1 if the head of the chain was updated.
1527c478bd9Sstevel@tonic-gate  */
1537c478bd9Sstevel@tonic-gate int
1547c478bd9Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp)
1557c478bd9Sstevel@tonic-gate {
1567c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1577c478bd9Sstevel@tonic-gate 	mxchain_t **mcpp;
1587c478bd9Sstevel@tonic-gate 	mxchain_t *mcp;
1597c478bd9Sstevel@tonic-gate 
1607c478bd9Sstevel@tonic-gate 	mcpp = &self->ul_mxchain;
1617c478bd9Sstevel@tonic-gate 	while ((*mcpp)->mxchain_mx != mp)
1627c478bd9Sstevel@tonic-gate 		mcpp = &(*mcpp)->mxchain_next;
1637c478bd9Sstevel@tonic-gate 	mcp = *mcpp;
1647c478bd9Sstevel@tonic-gate 	*mcpp = mcp->mxchain_next;
1657c478bd9Sstevel@tonic-gate 	lfree(mcp, sizeof (*mcp));
1667c478bd9Sstevel@tonic-gate 	return (mcpp == &self->ul_mxchain);
1677c478bd9Sstevel@tonic-gate }
1687c478bd9Sstevel@tonic-gate 
1697c478bd9Sstevel@tonic-gate /*
1707c478bd9Sstevel@tonic-gate  * Add mp to head of list of ceil mutexes owned by curthread.
1717c478bd9Sstevel@tonic-gate  * Return ENOMEM if no memory could be allocated.
1727c478bd9Sstevel@tonic-gate  */
1737c478bd9Sstevel@tonic-gate int
1747c478bd9Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp)
1757c478bd9Sstevel@tonic-gate {
1767c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1777c478bd9Sstevel@tonic-gate 	mxchain_t *mcp;
1787c478bd9Sstevel@tonic-gate 
1797c478bd9Sstevel@tonic-gate 	if ((mcp = lmalloc(sizeof (*mcp))) == NULL)
1807c478bd9Sstevel@tonic-gate 		return (ENOMEM);
1817c478bd9Sstevel@tonic-gate 	mcp->mxchain_mx = mp;
1827c478bd9Sstevel@tonic-gate 	mcp->mxchain_next = self->ul_mxchain;
1837c478bd9Sstevel@tonic-gate 	self->ul_mxchain = mcp;
1847c478bd9Sstevel@tonic-gate 	return (0);
1857c478bd9Sstevel@tonic-gate }
1867c478bd9Sstevel@tonic-gate 
1877c478bd9Sstevel@tonic-gate /*
1887c478bd9Sstevel@tonic-gate  * Inherit priority from ceiling.  The inheritance impacts the effective
1897c478bd9Sstevel@tonic-gate  * priority, not the assigned priority.  See _thread_setschedparam_main().
1907c478bd9Sstevel@tonic-gate  */
1917c478bd9Sstevel@tonic-gate void
1927c478bd9Sstevel@tonic-gate _ceil_prio_inherit(int ceil)
1937c478bd9Sstevel@tonic-gate {
1947c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1957c478bd9Sstevel@tonic-gate 	struct sched_param param;
1967c478bd9Sstevel@tonic-gate 
1977c478bd9Sstevel@tonic-gate 	(void) _memset(&param, 0, sizeof (param));
1987c478bd9Sstevel@tonic-gate 	param.sched_priority = ceil;
1997c478bd9Sstevel@tonic-gate 	if (_thread_setschedparam_main(self->ul_lwpid,
2007c478bd9Sstevel@tonic-gate 	    self->ul_policy, &param, PRIO_INHERIT)) {
2017c478bd9Sstevel@tonic-gate 		/*
2027c478bd9Sstevel@tonic-gate 		 * Panic since unclear what error code to return.
2037c478bd9Sstevel@tonic-gate 		 * If we do return the error codes returned by above
2047c478bd9Sstevel@tonic-gate 		 * called routine, update the man page...
2057c478bd9Sstevel@tonic-gate 		 */
2067c478bd9Sstevel@tonic-gate 		thr_panic("_thread_setschedparam_main() fails");
2077c478bd9Sstevel@tonic-gate 	}
2087c478bd9Sstevel@tonic-gate }
2097c478bd9Sstevel@tonic-gate 
2107c478bd9Sstevel@tonic-gate /*
2117c478bd9Sstevel@tonic-gate  * Waive inherited ceiling priority.  Inherit from head of owned ceiling locks
2127c478bd9Sstevel@tonic-gate  * if holding at least one ceiling lock.  If no ceiling locks are held at this
2137c478bd9Sstevel@tonic-gate  * point, disinherit completely, reverting back to assigned priority.
2147c478bd9Sstevel@tonic-gate  */
2157c478bd9Sstevel@tonic-gate void
2167c478bd9Sstevel@tonic-gate _ceil_prio_waive(void)
2177c478bd9Sstevel@tonic-gate {
2187c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2197c478bd9Sstevel@tonic-gate 	struct sched_param param;
2207c478bd9Sstevel@tonic-gate 
2217c478bd9Sstevel@tonic-gate 	(void) _memset(&param, 0, sizeof (param));
2227c478bd9Sstevel@tonic-gate 	if (self->ul_mxchain == NULL) {
2237c478bd9Sstevel@tonic-gate 		/*
2247c478bd9Sstevel@tonic-gate 		 * No ceil locks held.  Zero the epri, revert back to ul_pri.
2257c478bd9Sstevel@tonic-gate 		 * Since thread's hash lock is not held, one cannot just
2267c478bd9Sstevel@tonic-gate 		 * read ul_pri here...do it in the called routine...
2277c478bd9Sstevel@tonic-gate 		 */
2287c478bd9Sstevel@tonic-gate 		param.sched_priority = self->ul_pri;	/* ignored */
2297c478bd9Sstevel@tonic-gate 		if (_thread_setschedparam_main(self->ul_lwpid,
2307c478bd9Sstevel@tonic-gate 		    self->ul_policy, &param, PRIO_DISINHERIT))
2317c478bd9Sstevel@tonic-gate 			thr_panic("_thread_setschedparam_main() fails");
2327c478bd9Sstevel@tonic-gate 	} else {
2337c478bd9Sstevel@tonic-gate 		/*
2347c478bd9Sstevel@tonic-gate 		 * Set priority to that of the mutex at the head
2357c478bd9Sstevel@tonic-gate 		 * of the ceilmutex chain.
2367c478bd9Sstevel@tonic-gate 		 */
2377c478bd9Sstevel@tonic-gate 		param.sched_priority =
2387c478bd9Sstevel@tonic-gate 		    self->ul_mxchain->mxchain_mx->mutex_ceiling;
2397c478bd9Sstevel@tonic-gate 		if (_thread_setschedparam_main(self->ul_lwpid,
2407c478bd9Sstevel@tonic-gate 		    self->ul_policy, &param, PRIO_INHERIT))
2417c478bd9Sstevel@tonic-gate 			thr_panic("_thread_setschedparam_main() fails");
2427c478bd9Sstevel@tonic-gate 	}
2437c478bd9Sstevel@tonic-gate }
2447c478bd9Sstevel@tonic-gate 
2457c478bd9Sstevel@tonic-gate /*
2467c478bd9Sstevel@tonic-gate  * Non-preemptive spin locks.  Used by queue_lock().
2477c478bd9Sstevel@tonic-gate  * No lock statistics are gathered for these locks.
2487c478bd9Sstevel@tonic-gate  */
2497c478bd9Sstevel@tonic-gate void
2507c478bd9Sstevel@tonic-gate spin_lock_set(mutex_t *mp)
2517c478bd9Sstevel@tonic-gate {
2527c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2537c478bd9Sstevel@tonic-gate 
2547c478bd9Sstevel@tonic-gate 	no_preempt(self);
2557c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
2567c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
2577c478bd9Sstevel@tonic-gate 		return;
2587c478bd9Sstevel@tonic-gate 	}
2597c478bd9Sstevel@tonic-gate 	/*
2607c478bd9Sstevel@tonic-gate 	 * Spin for a while, attempting to acquire the lock.
2617c478bd9Sstevel@tonic-gate 	 */
2627c478bd9Sstevel@tonic-gate 	if (self->ul_spin_lock_spin != UINT_MAX)
2637c478bd9Sstevel@tonic-gate 		self->ul_spin_lock_spin++;
2647c478bd9Sstevel@tonic-gate 	if (mutex_queuelock_adaptive(mp) == 0 ||
2657c478bd9Sstevel@tonic-gate 	    set_lock_byte(&mp->mutex_lockw) == 0) {
2667c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
2677c478bd9Sstevel@tonic-gate 		return;
2687c478bd9Sstevel@tonic-gate 	}
2697c478bd9Sstevel@tonic-gate 	/*
2707c478bd9Sstevel@tonic-gate 	 * Try harder if we were previously at a no premption level.
2717c478bd9Sstevel@tonic-gate 	 */
2727c478bd9Sstevel@tonic-gate 	if (self->ul_preempt > 1) {
2737c478bd9Sstevel@tonic-gate 		if (self->ul_spin_lock_spin2 != UINT_MAX)
2747c478bd9Sstevel@tonic-gate 			self->ul_spin_lock_spin2++;
2757c478bd9Sstevel@tonic-gate 		if (mutex_queuelock_adaptive(mp) == 0 ||
2767c478bd9Sstevel@tonic-gate 		    set_lock_byte(&mp->mutex_lockw) == 0) {
2777c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
2787c478bd9Sstevel@tonic-gate 			return;
2797c478bd9Sstevel@tonic-gate 		}
2807c478bd9Sstevel@tonic-gate 	}
2817c478bd9Sstevel@tonic-gate 	/*
2827c478bd9Sstevel@tonic-gate 	 * Give up and block in the kernel for the mutex.
2837c478bd9Sstevel@tonic-gate 	 */
2847c478bd9Sstevel@tonic-gate 	if (self->ul_spin_lock_sleep != UINT_MAX)
2857c478bd9Sstevel@tonic-gate 		self->ul_spin_lock_sleep++;
2867c478bd9Sstevel@tonic-gate 	(void) ___lwp_mutex_timedlock(mp, NULL);
2877c478bd9Sstevel@tonic-gate 	mp->mutex_owner = (uintptr_t)self;
2887c478bd9Sstevel@tonic-gate }
2897c478bd9Sstevel@tonic-gate 
2907c478bd9Sstevel@tonic-gate void
2917c478bd9Sstevel@tonic-gate spin_lock_clear(mutex_t *mp)
2927c478bd9Sstevel@tonic-gate {
2937c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2947c478bd9Sstevel@tonic-gate 
2957c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
2967c478bd9Sstevel@tonic-gate 	if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) {
2977c478bd9Sstevel@tonic-gate 		(void) ___lwp_mutex_wakeup(mp);
2987c478bd9Sstevel@tonic-gate 		if (self->ul_spin_lock_wakeup != UINT_MAX)
2997c478bd9Sstevel@tonic-gate 			self->ul_spin_lock_wakeup++;
3007c478bd9Sstevel@tonic-gate 	}
3017c478bd9Sstevel@tonic-gate 	preempt(self);
3027c478bd9Sstevel@tonic-gate }
3037c478bd9Sstevel@tonic-gate 
3047c478bd9Sstevel@tonic-gate /*
3057c478bd9Sstevel@tonic-gate  * Allocate the sleep queue hash table.
3067c478bd9Sstevel@tonic-gate  */
3077c478bd9Sstevel@tonic-gate void
3087c478bd9Sstevel@tonic-gate queue_alloc(void)
3097c478bd9Sstevel@tonic-gate {
3107c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
3117c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
3127c478bd9Sstevel@tonic-gate 	void *data;
3137c478bd9Sstevel@tonic-gate 	int i;
3147c478bd9Sstevel@tonic-gate 
3157c478bd9Sstevel@tonic-gate 	/*
3167c478bd9Sstevel@tonic-gate 	 * No locks are needed; we call here only when single-threaded.
3177c478bd9Sstevel@tonic-gate 	 */
3187c478bd9Sstevel@tonic-gate 	ASSERT(self == udp->ulwp_one);
3197c478bd9Sstevel@tonic-gate 	ASSERT(!udp->uberflags.uf_mt);
3207c478bd9Sstevel@tonic-gate 	if ((data = _private_mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t),
3217c478bd9Sstevel@tonic-gate 	    PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0))
3227c478bd9Sstevel@tonic-gate 	    == MAP_FAILED)
3237c478bd9Sstevel@tonic-gate 		thr_panic("cannot allocate thread queue_head table");
3247c478bd9Sstevel@tonic-gate 	udp->queue_head = (queue_head_t *)data;
3257c478bd9Sstevel@tonic-gate 	for (i = 0; i < 2 * QHASHSIZE; i++)
3267c478bd9Sstevel@tonic-gate 		udp->queue_head[i].qh_lock.mutex_magic = MUTEX_MAGIC;
3277c478bd9Sstevel@tonic-gate }
3287c478bd9Sstevel@tonic-gate 
3297c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG)
3307c478bd9Sstevel@tonic-gate 
3317c478bd9Sstevel@tonic-gate /*
3327c478bd9Sstevel@tonic-gate  * Debugging: verify correctness of a sleep queue.
3337c478bd9Sstevel@tonic-gate  */
3347c478bd9Sstevel@tonic-gate void
3357c478bd9Sstevel@tonic-gate QVERIFY(queue_head_t *qp)
3367c478bd9Sstevel@tonic-gate {
3377c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
3387c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
3397c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
3407c478bd9Sstevel@tonic-gate 	ulwp_t *prev;
3417c478bd9Sstevel@tonic-gate 	uint_t index;
3427c478bd9Sstevel@tonic-gate 	uint32_t cnt = 0;
3437c478bd9Sstevel@tonic-gate 	char qtype;
3447c478bd9Sstevel@tonic-gate 	void *wchan;
3457c478bd9Sstevel@tonic-gate 
3467c478bd9Sstevel@tonic-gate 	ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE);
3477c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
3487c478bd9Sstevel@tonic-gate 	ASSERT((qp->qh_head != NULL && qp->qh_tail != NULL) ||
3497c478bd9Sstevel@tonic-gate 		(qp->qh_head == NULL && qp->qh_tail == NULL));
3507c478bd9Sstevel@tonic-gate 	if (!thread_queue_verify)
3517c478bd9Sstevel@tonic-gate 		return;
3527c478bd9Sstevel@tonic-gate 	/* real expensive stuff, only for _THREAD_QUEUE_VERIFY */
3537c478bd9Sstevel@tonic-gate 	qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV;
3547c478bd9Sstevel@tonic-gate 	for (prev = NULL, ulwp = qp->qh_head; ulwp != NULL;
3557c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwp = ulwp->ul_link, cnt++) {
3567c478bd9Sstevel@tonic-gate 		ASSERT(ulwp->ul_qtype == qtype);
3577c478bd9Sstevel@tonic-gate 		ASSERT(ulwp->ul_wchan != NULL);
3587c478bd9Sstevel@tonic-gate 		ASSERT(ulwp->ul_sleepq == qp);
3597c478bd9Sstevel@tonic-gate 		wchan = ulwp->ul_wchan;
3607c478bd9Sstevel@tonic-gate 		index = QUEUE_HASH(wchan, qtype);
3617c478bd9Sstevel@tonic-gate 		ASSERT(&udp->queue_head[index] == qp);
3627c478bd9Sstevel@tonic-gate 	}
3637c478bd9Sstevel@tonic-gate 	ASSERT(qp->qh_tail == prev);
3647c478bd9Sstevel@tonic-gate 	ASSERT(qp->qh_qlen == cnt);
3657c478bd9Sstevel@tonic-gate }
3667c478bd9Sstevel@tonic-gate 
3677c478bd9Sstevel@tonic-gate #else	/* THREAD_DEBUG */
3687c478bd9Sstevel@tonic-gate 
3697c478bd9Sstevel@tonic-gate #define	QVERIFY(qp)
3707c478bd9Sstevel@tonic-gate 
3717c478bd9Sstevel@tonic-gate #endif	/* THREAD_DEBUG */
3727c478bd9Sstevel@tonic-gate 
3737c478bd9Sstevel@tonic-gate /*
3747c478bd9Sstevel@tonic-gate  * Acquire a queue head.
3757c478bd9Sstevel@tonic-gate  */
3767c478bd9Sstevel@tonic-gate queue_head_t *
3777c478bd9Sstevel@tonic-gate queue_lock(void *wchan, int qtype)
3787c478bd9Sstevel@tonic-gate {
3797c478bd9Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
3807c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
3817c478bd9Sstevel@tonic-gate 
3827c478bd9Sstevel@tonic-gate 	ASSERT(qtype == MX || qtype == CV);
3837c478bd9Sstevel@tonic-gate 
3847c478bd9Sstevel@tonic-gate 	/*
3857c478bd9Sstevel@tonic-gate 	 * It is possible that we could be called while still single-threaded.
3867c478bd9Sstevel@tonic-gate 	 * If so, we call queue_alloc() to allocate the queue_head[] array.
3877c478bd9Sstevel@tonic-gate 	 */
3887c478bd9Sstevel@tonic-gate 	if ((qp = udp->queue_head) == NULL) {
3897c478bd9Sstevel@tonic-gate 		queue_alloc();
3907c478bd9Sstevel@tonic-gate 		qp = udp->queue_head;
3917c478bd9Sstevel@tonic-gate 	}
3927c478bd9Sstevel@tonic-gate 	qp += QUEUE_HASH(wchan, qtype);
3937c478bd9Sstevel@tonic-gate 	spin_lock_set(&qp->qh_lock);
3947c478bd9Sstevel@tonic-gate 	/*
3957c478bd9Sstevel@tonic-gate 	 * At once per nanosecond, qh_lockcount will wrap after 512 years.
3967c478bd9Sstevel@tonic-gate 	 * Were we to care about this, we could peg the value at UINT64_MAX.
3977c478bd9Sstevel@tonic-gate 	 */
3987c478bd9Sstevel@tonic-gate 	qp->qh_lockcount++;
3997c478bd9Sstevel@tonic-gate 	QVERIFY(qp);
4007c478bd9Sstevel@tonic-gate 	return (qp);
4017c478bd9Sstevel@tonic-gate }
4027c478bd9Sstevel@tonic-gate 
4037c478bd9Sstevel@tonic-gate /*
4047c478bd9Sstevel@tonic-gate  * Release a queue head.
4057c478bd9Sstevel@tonic-gate  */
4067c478bd9Sstevel@tonic-gate void
4077c478bd9Sstevel@tonic-gate queue_unlock(queue_head_t *qp)
4087c478bd9Sstevel@tonic-gate {
4097c478bd9Sstevel@tonic-gate 	QVERIFY(qp);
4107c478bd9Sstevel@tonic-gate 	spin_lock_clear(&qp->qh_lock);
4117c478bd9Sstevel@tonic-gate }
4127c478bd9Sstevel@tonic-gate 
4137c478bd9Sstevel@tonic-gate /*
4147c478bd9Sstevel@tonic-gate  * For rwlock queueing, we must queue writers ahead of readers of the
4157c478bd9Sstevel@tonic-gate  * same priority.  We do this by making writers appear to have a half
4167c478bd9Sstevel@tonic-gate  * point higher priority for purposes of priority comparisons below.
4177c478bd9Sstevel@tonic-gate  */
4187c478bd9Sstevel@tonic-gate #define	CMP_PRIO(ulwp)	((real_priority(ulwp) << 1) + (ulwp)->ul_writer)
4197c478bd9Sstevel@tonic-gate 
4207c478bd9Sstevel@tonic-gate void
4217c478bd9Sstevel@tonic-gate enqueue(queue_head_t *qp, ulwp_t *ulwp, void *wchan, int qtype)
4227c478bd9Sstevel@tonic-gate {
4237c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
4247c478bd9Sstevel@tonic-gate 	ulwp_t *next;
4257c478bd9Sstevel@tonic-gate 	int pri = CMP_PRIO(ulwp);
4267c478bd9Sstevel@tonic-gate 	int force_fifo = (qtype & FIFOQ);
4277c478bd9Sstevel@tonic-gate 	int do_fifo;
4287c478bd9Sstevel@tonic-gate 
4297c478bd9Sstevel@tonic-gate 	qtype &= ~FIFOQ;
4307c478bd9Sstevel@tonic-gate 	ASSERT(qtype == MX || qtype == CV);
4317c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
4327c478bd9Sstevel@tonic-gate 	ASSERT(ulwp->ul_sleepq != qp);
4337c478bd9Sstevel@tonic-gate 
4347c478bd9Sstevel@tonic-gate 	/*
4357c478bd9Sstevel@tonic-gate 	 * LIFO queue ordering is unfair and can lead to starvation,
4367c478bd9Sstevel@tonic-gate 	 * but it gives better performance for heavily contended locks.
4377c478bd9Sstevel@tonic-gate 	 * We use thread_queue_fifo (range is 0..8) to determine
4387c478bd9Sstevel@tonic-gate 	 * the frequency of FIFO vs LIFO queuing:
4397c478bd9Sstevel@tonic-gate 	 *	0 : every 256th time	(almost always LIFO)
4407c478bd9Sstevel@tonic-gate 	 *	1 : every 128th time
4417c478bd9Sstevel@tonic-gate 	 *	2 : every 64th  time
4427c478bd9Sstevel@tonic-gate 	 *	3 : every 32nd  time
4437c478bd9Sstevel@tonic-gate 	 *	4 : every 16th  time	(the default value, mostly LIFO)
4447c478bd9Sstevel@tonic-gate 	 *	5 : every 8th   time
4457c478bd9Sstevel@tonic-gate 	 *	6 : every 4th   time
4467c478bd9Sstevel@tonic-gate 	 *	7 : every 2nd   time
4477c478bd9Sstevel@tonic-gate 	 *	8 : every time		(never LIFO, always FIFO)
4487c478bd9Sstevel@tonic-gate 	 * Note that there is always some degree of FIFO ordering.
4497c478bd9Sstevel@tonic-gate 	 * This breaks live lock conditions that occur in applications
4507c478bd9Sstevel@tonic-gate 	 * that are written assuming (incorrectly) that threads acquire
4517c478bd9Sstevel@tonic-gate 	 * locks fairly, that is, in roughly round-robin order.
4527c478bd9Sstevel@tonic-gate 	 * In any event, the queue is maintained in priority order.
4537c478bd9Sstevel@tonic-gate 	 *
4547c478bd9Sstevel@tonic-gate 	 * If we are given the FIFOQ flag in qtype, fifo queueing is forced.
4557c478bd9Sstevel@tonic-gate 	 * SUSV3 requires this for semaphores.
4567c478bd9Sstevel@tonic-gate 	 */
4577c478bd9Sstevel@tonic-gate 	do_fifo = (force_fifo ||
4587c478bd9Sstevel@tonic-gate 		((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0);
4597c478bd9Sstevel@tonic-gate 
4607c478bd9Sstevel@tonic-gate 	if (qp->qh_head == NULL) {
4617c478bd9Sstevel@tonic-gate 		/*
4627c478bd9Sstevel@tonic-gate 		 * The queue is empty.  LIFO/FIFO doesn't matter.
4637c478bd9Sstevel@tonic-gate 		 */
4647c478bd9Sstevel@tonic-gate 		ASSERT(qp->qh_tail == NULL);
4657c478bd9Sstevel@tonic-gate 		ulwpp = &qp->qh_head;
4667c478bd9Sstevel@tonic-gate 	} else if (do_fifo) {
4677c478bd9Sstevel@tonic-gate 		/*
4687c478bd9Sstevel@tonic-gate 		 * Enqueue after the last thread whose priority is greater
4697c478bd9Sstevel@tonic-gate 		 * than or equal to the priority of the thread being queued.
4707c478bd9Sstevel@tonic-gate 		 * Attempt first to go directly onto the tail of the queue.
4717c478bd9Sstevel@tonic-gate 		 */
4727c478bd9Sstevel@tonic-gate 		if (pri <= CMP_PRIO(qp->qh_tail))
4737c478bd9Sstevel@tonic-gate 			ulwpp = &qp->qh_tail->ul_link;
4747c478bd9Sstevel@tonic-gate 		else {
4757c478bd9Sstevel@tonic-gate 			for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL;
4767c478bd9Sstevel@tonic-gate 			    ulwpp = &next->ul_link)
4777c478bd9Sstevel@tonic-gate 				if (pri > CMP_PRIO(next))
4787c478bd9Sstevel@tonic-gate 					break;
4797c478bd9Sstevel@tonic-gate 		}
4807c478bd9Sstevel@tonic-gate 	} else {
4817c478bd9Sstevel@tonic-gate 		/*
4827c478bd9Sstevel@tonic-gate 		 * Enqueue before the first thread whose priority is less
4837c478bd9Sstevel@tonic-gate 		 * than or equal to the priority of the thread being queued.
4847c478bd9Sstevel@tonic-gate 		 * Hopefully we can go directly onto the head of the queue.
4857c478bd9Sstevel@tonic-gate 		 */
4867c478bd9Sstevel@tonic-gate 		for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL;
4877c478bd9Sstevel@tonic-gate 		    ulwpp = &next->ul_link)
4887c478bd9Sstevel@tonic-gate 			if (pri >= CMP_PRIO(next))
4897c478bd9Sstevel@tonic-gate 				break;
4907c478bd9Sstevel@tonic-gate 	}
4917c478bd9Sstevel@tonic-gate 	if ((ulwp->ul_link = *ulwpp) == NULL)
4927c478bd9Sstevel@tonic-gate 		qp->qh_tail = ulwp;
4937c478bd9Sstevel@tonic-gate 	*ulwpp = ulwp;
4947c478bd9Sstevel@tonic-gate 
4957c478bd9Sstevel@tonic-gate 	ulwp->ul_sleepq = qp;
4967c478bd9Sstevel@tonic-gate 	ulwp->ul_wchan = wchan;
4977c478bd9Sstevel@tonic-gate 	ulwp->ul_qtype = qtype;
4987c478bd9Sstevel@tonic-gate 	if (qp->qh_qmax < ++qp->qh_qlen)
4997c478bd9Sstevel@tonic-gate 		qp->qh_qmax = qp->qh_qlen;
5007c478bd9Sstevel@tonic-gate }
5017c478bd9Sstevel@tonic-gate 
5027c478bd9Sstevel@tonic-gate /*
5037c478bd9Sstevel@tonic-gate  * Return a pointer to the queue slot of the
5047c478bd9Sstevel@tonic-gate  * highest priority thread on the queue.
5057c478bd9Sstevel@tonic-gate  * On return, prevp, if not NULL, will contain a pointer
5067c478bd9Sstevel@tonic-gate  * to the thread's predecessor on the queue
5077c478bd9Sstevel@tonic-gate  */
5087c478bd9Sstevel@tonic-gate static ulwp_t **
5097c478bd9Sstevel@tonic-gate queue_slot(queue_head_t *qp, void *wchan, int *more, ulwp_t **prevp)
5107c478bd9Sstevel@tonic-gate {
5117c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
5127c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
5137c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
5147c478bd9Sstevel@tonic-gate 	ulwp_t **suspp = NULL;
5157c478bd9Sstevel@tonic-gate 	ulwp_t *susprev;
5167c478bd9Sstevel@tonic-gate 
5177c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
5187c478bd9Sstevel@tonic-gate 
5197c478bd9Sstevel@tonic-gate 	/*
5207c478bd9Sstevel@tonic-gate 	 * Find a waiter on the sleep queue.
5217c478bd9Sstevel@tonic-gate 	 */
5227c478bd9Sstevel@tonic-gate 	for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL;
5237c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
5247c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan) {
5257c478bd9Sstevel@tonic-gate 			if (!ulwp->ul_stop)
5267c478bd9Sstevel@tonic-gate 				break;
5277c478bd9Sstevel@tonic-gate 			/*
5287c478bd9Sstevel@tonic-gate 			 * Try not to return a suspended thread.
5297c478bd9Sstevel@tonic-gate 			 * This mimics the old libthread's behavior.
5307c478bd9Sstevel@tonic-gate 			 */
5317c478bd9Sstevel@tonic-gate 			if (suspp == NULL) {
5327c478bd9Sstevel@tonic-gate 				suspp = ulwpp;
5337c478bd9Sstevel@tonic-gate 				susprev = prev;
5347c478bd9Sstevel@tonic-gate 			}
5357c478bd9Sstevel@tonic-gate 		}
5367c478bd9Sstevel@tonic-gate 	}
5377c478bd9Sstevel@tonic-gate 
5387c478bd9Sstevel@tonic-gate 	if (ulwp == NULL && suspp != NULL) {
5397c478bd9Sstevel@tonic-gate 		ulwp = *(ulwpp = suspp);
5407c478bd9Sstevel@tonic-gate 		prev = susprev;
5417c478bd9Sstevel@tonic-gate 		suspp = NULL;
5427c478bd9Sstevel@tonic-gate 	}
5437c478bd9Sstevel@tonic-gate 	if (ulwp == NULL) {
5447c478bd9Sstevel@tonic-gate 		if (more != NULL)
5457c478bd9Sstevel@tonic-gate 			*more = 0;
5467c478bd9Sstevel@tonic-gate 		return (NULL);
5477c478bd9Sstevel@tonic-gate 	}
5487c478bd9Sstevel@tonic-gate 
5497c478bd9Sstevel@tonic-gate 	if (prevp != NULL)
5507c478bd9Sstevel@tonic-gate 		*prevp = prev;
5517c478bd9Sstevel@tonic-gate 	if (more == NULL)
5527c478bd9Sstevel@tonic-gate 		return (ulwpp);
5537c478bd9Sstevel@tonic-gate 
5547c478bd9Sstevel@tonic-gate 	/*
5557c478bd9Sstevel@tonic-gate 	 * Scan the remainder of the queue for another waiter.
5567c478bd9Sstevel@tonic-gate 	 */
5577c478bd9Sstevel@tonic-gate 	if (suspp != NULL) {
5587c478bd9Sstevel@tonic-gate 		*more = 1;
5597c478bd9Sstevel@tonic-gate 		return (ulwpp);
5607c478bd9Sstevel@tonic-gate 	}
5617c478bd9Sstevel@tonic-gate 	for (ulwp = ulwp->ul_link; ulwp != NULL; ulwp = ulwp->ul_link) {
5627c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan) {
5637c478bd9Sstevel@tonic-gate 			*more = 1;
5647c478bd9Sstevel@tonic-gate 			return (ulwpp);
5657c478bd9Sstevel@tonic-gate 		}
5667c478bd9Sstevel@tonic-gate 	}
5677c478bd9Sstevel@tonic-gate 
5687c478bd9Sstevel@tonic-gate 	*more = 0;
5697c478bd9Sstevel@tonic-gate 	return (ulwpp);
5707c478bd9Sstevel@tonic-gate }
5717c478bd9Sstevel@tonic-gate 
5727c478bd9Sstevel@tonic-gate ulwp_t *
5737c478bd9Sstevel@tonic-gate dequeue(queue_head_t *qp, void *wchan, int *more)
5747c478bd9Sstevel@tonic-gate {
5757c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
5767c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
5777c478bd9Sstevel@tonic-gate 	ulwp_t *prev;
5787c478bd9Sstevel@tonic-gate 
5797c478bd9Sstevel@tonic-gate 	if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL)
5807c478bd9Sstevel@tonic-gate 		return (NULL);
5817c478bd9Sstevel@tonic-gate 
5827c478bd9Sstevel@tonic-gate 	/*
5837c478bd9Sstevel@tonic-gate 	 * Dequeue the waiter.
5847c478bd9Sstevel@tonic-gate 	 */
5857c478bd9Sstevel@tonic-gate 	ulwp = *ulwpp;
5867c478bd9Sstevel@tonic-gate 	*ulwpp = ulwp->ul_link;
5877c478bd9Sstevel@tonic-gate 	ulwp->ul_link = NULL;
5887c478bd9Sstevel@tonic-gate 	if (qp->qh_tail == ulwp)
5897c478bd9Sstevel@tonic-gate 		qp->qh_tail = prev;
5907c478bd9Sstevel@tonic-gate 	qp->qh_qlen--;
5917c478bd9Sstevel@tonic-gate 	ulwp->ul_sleepq = NULL;
5927c478bd9Sstevel@tonic-gate 	ulwp->ul_wchan = NULL;
5937c478bd9Sstevel@tonic-gate 
5947c478bd9Sstevel@tonic-gate 	return (ulwp);
5957c478bd9Sstevel@tonic-gate }
5967c478bd9Sstevel@tonic-gate 
5977c478bd9Sstevel@tonic-gate /*
5987c478bd9Sstevel@tonic-gate  * Return a pointer to the highest priority thread sleeping on wchan.
5997c478bd9Sstevel@tonic-gate  */
6007c478bd9Sstevel@tonic-gate ulwp_t *
6017c478bd9Sstevel@tonic-gate queue_waiter(queue_head_t *qp, void *wchan)
6027c478bd9Sstevel@tonic-gate {
6037c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
6047c478bd9Sstevel@tonic-gate 
6057c478bd9Sstevel@tonic-gate 	if ((ulwpp = queue_slot(qp, wchan, NULL, NULL)) == NULL)
6067c478bd9Sstevel@tonic-gate 		return (NULL);
6077c478bd9Sstevel@tonic-gate 	return (*ulwpp);
6087c478bd9Sstevel@tonic-gate }
6097c478bd9Sstevel@tonic-gate 
6107c478bd9Sstevel@tonic-gate uint8_t
6117c478bd9Sstevel@tonic-gate dequeue_self(queue_head_t *qp, void *wchan)
6127c478bd9Sstevel@tonic-gate {
6137c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
6147c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
6157c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
6167c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
6177c478bd9Sstevel@tonic-gate 	int found = 0;
6187c478bd9Sstevel@tonic-gate 	int more = 0;
6197c478bd9Sstevel@tonic-gate 
6207c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
6217c478bd9Sstevel@tonic-gate 
6227c478bd9Sstevel@tonic-gate 	/* find self on the sleep queue */
6237c478bd9Sstevel@tonic-gate 	for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL;
6247c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
6257c478bd9Sstevel@tonic-gate 		if (ulwp == self) {
6267c478bd9Sstevel@tonic-gate 			/* dequeue ourself */
6277c478bd9Sstevel@tonic-gate 			*ulwpp = self->ul_link;
6287c478bd9Sstevel@tonic-gate 			if (qp->qh_tail == self)
6297c478bd9Sstevel@tonic-gate 				qp->qh_tail = prev;
6307c478bd9Sstevel@tonic-gate 			qp->qh_qlen--;
6317c478bd9Sstevel@tonic-gate 			ASSERT(self->ul_wchan == wchan);
6327c478bd9Sstevel@tonic-gate 			self->ul_cvmutex = NULL;
6337c478bd9Sstevel@tonic-gate 			self->ul_sleepq = NULL;
6347c478bd9Sstevel@tonic-gate 			self->ul_wchan = NULL;
6357c478bd9Sstevel@tonic-gate 			self->ul_cv_wake = 0;
6367c478bd9Sstevel@tonic-gate 			self->ul_link = NULL;
6377c478bd9Sstevel@tonic-gate 			found = 1;
6387c478bd9Sstevel@tonic-gate 			break;
6397c478bd9Sstevel@tonic-gate 		}
6407c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan)
6417c478bd9Sstevel@tonic-gate 			more = 1;
6427c478bd9Sstevel@tonic-gate 	}
6437c478bd9Sstevel@tonic-gate 
6447c478bd9Sstevel@tonic-gate 	if (!found)
6457c478bd9Sstevel@tonic-gate 		thr_panic("dequeue_self(): curthread not found on queue");
6467c478bd9Sstevel@tonic-gate 
6477c478bd9Sstevel@tonic-gate 	if (more)
6487c478bd9Sstevel@tonic-gate 		return (1);
6497c478bd9Sstevel@tonic-gate 
6507c478bd9Sstevel@tonic-gate 	/* scan the remainder of the queue for another waiter */
6517c478bd9Sstevel@tonic-gate 	for (ulwp = *ulwpp; ulwp != NULL; ulwp = ulwp->ul_link) {
6527c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan)
6537c478bd9Sstevel@tonic-gate 			return (1);
6547c478bd9Sstevel@tonic-gate 	}
6557c478bd9Sstevel@tonic-gate 
6567c478bd9Sstevel@tonic-gate 	return (0);
6577c478bd9Sstevel@tonic-gate }
6587c478bd9Sstevel@tonic-gate 
6597c478bd9Sstevel@tonic-gate /*
6607c478bd9Sstevel@tonic-gate  * Called from call_user_handler() and _thrp_suspend() to take
6617c478bd9Sstevel@tonic-gate  * ourself off of our sleep queue so we can grab locks.
6627c478bd9Sstevel@tonic-gate  */
6637c478bd9Sstevel@tonic-gate void
6647c478bd9Sstevel@tonic-gate unsleep_self(void)
6657c478bd9Sstevel@tonic-gate {
6667c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
6677c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
6687c478bd9Sstevel@tonic-gate 
6697c478bd9Sstevel@tonic-gate 	/*
6707c478bd9Sstevel@tonic-gate 	 * Calling enter_critical()/exit_critical() here would lead
6717c478bd9Sstevel@tonic-gate 	 * to recursion.  Just manipulate self->ul_critical directly.
6727c478bd9Sstevel@tonic-gate 	 */
6737c478bd9Sstevel@tonic-gate 	self->ul_critical++;
6747c478bd9Sstevel@tonic-gate 	self->ul_writer = 0;
6757c478bd9Sstevel@tonic-gate 	while (self->ul_sleepq != NULL) {
6767c478bd9Sstevel@tonic-gate 		qp = queue_lock(self->ul_wchan, self->ul_qtype);
6777c478bd9Sstevel@tonic-gate 		/*
6787c478bd9Sstevel@tonic-gate 		 * We may have been moved from a CV queue to a
6797c478bd9Sstevel@tonic-gate 		 * mutex queue while we were attempting queue_lock().
6807c478bd9Sstevel@tonic-gate 		 * If so, just loop around and try again.
6817c478bd9Sstevel@tonic-gate 		 * dequeue_self() clears self->ul_sleepq.
6827c478bd9Sstevel@tonic-gate 		 */
6837c478bd9Sstevel@tonic-gate 		if (qp == self->ul_sleepq)
6847c478bd9Sstevel@tonic-gate 			(void) dequeue_self(qp, self->ul_wchan);
6857c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
6867c478bd9Sstevel@tonic-gate 	}
6877c478bd9Sstevel@tonic-gate 	self->ul_critical--;
6887c478bd9Sstevel@tonic-gate }
6897c478bd9Sstevel@tonic-gate 
6907c478bd9Sstevel@tonic-gate /*
6917c478bd9Sstevel@tonic-gate  * Common code for calling the the ___lwp_mutex_timedlock() system call.
6927c478bd9Sstevel@tonic-gate  * Returns with mutex_owner and mutex_ownerpid set correctly.
6937c478bd9Sstevel@tonic-gate  */
6947c478bd9Sstevel@tonic-gate int
6957c478bd9Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
6967c478bd9Sstevel@tonic-gate {
6977c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
6987c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
6997c478bd9Sstevel@tonic-gate 	hrtime_t begin_sleep;
7007c478bd9Sstevel@tonic-gate 	int error;
7017c478bd9Sstevel@tonic-gate 
7027c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
7037c478bd9Sstevel@tonic-gate 	self->ul_wchan = mp;
7047c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
7057c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
7067c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = mp;
7077c478bd9Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
7087c478bd9Sstevel@tonic-gate 	}
7097c478bd9Sstevel@tonic-gate 	if (msp) {
7107c478bd9Sstevel@tonic-gate 		tdb_incr(msp->mutex_sleep);
7117c478bd9Sstevel@tonic-gate 		begin_sleep = gethrtime();
7127c478bd9Sstevel@tonic-gate 	}
7137c478bd9Sstevel@tonic-gate 
7147c478bd9Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__block, mp);
7157c478bd9Sstevel@tonic-gate 
7167c478bd9Sstevel@tonic-gate 	for (;;) {
7177c478bd9Sstevel@tonic-gate 		if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0) {
7187c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
7197c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
7207c478bd9Sstevel@tonic-gate 			break;
7217c478bd9Sstevel@tonic-gate 		}
7227c478bd9Sstevel@tonic-gate 
7237c478bd9Sstevel@tonic-gate 		if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
7247c478bd9Sstevel@tonic-gate 			/*
7257c478bd9Sstevel@tonic-gate 			 * Defend against forkall().  We may be the child,
7267c478bd9Sstevel@tonic-gate 			 * in which case we don't actually own the mutex.
7277c478bd9Sstevel@tonic-gate 			 */
7287c478bd9Sstevel@tonic-gate 			enter_critical(self);
7297c478bd9Sstevel@tonic-gate 			if (mp->mutex_ownerpid == udp->pid) {
7307c478bd9Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
7317c478bd9Sstevel@tonic-gate 				exit_critical(self);
7327c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
7337c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
7347c478bd9Sstevel@tonic-gate 				    0, 0);
7357c478bd9Sstevel@tonic-gate 				break;
7367c478bd9Sstevel@tonic-gate 			}
7377c478bd9Sstevel@tonic-gate 			exit_critical(self);
7387c478bd9Sstevel@tonic-gate 		} else {
7397c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
7407c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
7417c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
7427c478bd9Sstevel@tonic-gate 			break;
7437c478bd9Sstevel@tonic-gate 		}
7447c478bd9Sstevel@tonic-gate 	}
7457c478bd9Sstevel@tonic-gate 	if (msp)
7467c478bd9Sstevel@tonic-gate 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
7477c478bd9Sstevel@tonic-gate 	self->ul_wchan = NULL;
7487c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
7497c478bd9Sstevel@tonic-gate 
7507c478bd9Sstevel@tonic-gate 	return (error);
7517c478bd9Sstevel@tonic-gate }
7527c478bd9Sstevel@tonic-gate 
7537c478bd9Sstevel@tonic-gate /*
7547c478bd9Sstevel@tonic-gate  * Common code for calling the ___lwp_mutex_trylock() system call.
7557c478bd9Sstevel@tonic-gate  * Returns with mutex_owner and mutex_ownerpid set correctly.
7567c478bd9Sstevel@tonic-gate  */
7577c478bd9Sstevel@tonic-gate int
7587c478bd9Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp)
7597c478bd9Sstevel@tonic-gate {
7607c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
7617c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
7627c478bd9Sstevel@tonic-gate 	int error;
7637c478bd9Sstevel@tonic-gate 
7647c478bd9Sstevel@tonic-gate 	for (;;) {
7657c478bd9Sstevel@tonic-gate 		if ((error = ___lwp_mutex_trylock(mp)) != 0) {
7667c478bd9Sstevel@tonic-gate 			if (error != EBUSY) {
7677c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
7687c478bd9Sstevel@tonic-gate 				    error);
7697c478bd9Sstevel@tonic-gate 			}
7707c478bd9Sstevel@tonic-gate 			break;
7717c478bd9Sstevel@tonic-gate 		}
7727c478bd9Sstevel@tonic-gate 
7737c478bd9Sstevel@tonic-gate 		if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
7747c478bd9Sstevel@tonic-gate 			/*
7757c478bd9Sstevel@tonic-gate 			 * Defend against forkall().  We may be the child,
7767c478bd9Sstevel@tonic-gate 			 * in which case we don't actually own the mutex.
7777c478bd9Sstevel@tonic-gate 			 */
7787c478bd9Sstevel@tonic-gate 			enter_critical(self);
7797c478bd9Sstevel@tonic-gate 			if (mp->mutex_ownerpid == udp->pid) {
7807c478bd9Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
7817c478bd9Sstevel@tonic-gate 				exit_critical(self);
7827c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
7837c478bd9Sstevel@tonic-gate 				    0, 0);
7847c478bd9Sstevel@tonic-gate 				break;
7857c478bd9Sstevel@tonic-gate 			}
7867c478bd9Sstevel@tonic-gate 			exit_critical(self);
7877c478bd9Sstevel@tonic-gate 		} else {
7887c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
7897c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
7907c478bd9Sstevel@tonic-gate 			break;
7917c478bd9Sstevel@tonic-gate 		}
7927c478bd9Sstevel@tonic-gate 	}
7937c478bd9Sstevel@tonic-gate 
7947c478bd9Sstevel@tonic-gate 	return (error);
7957c478bd9Sstevel@tonic-gate }
7967c478bd9Sstevel@tonic-gate 
7977c478bd9Sstevel@tonic-gate volatile sc_shared_t *
7987c478bd9Sstevel@tonic-gate setup_schedctl(void)
7997c478bd9Sstevel@tonic-gate {
8007c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
8017c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
8027c478bd9Sstevel@tonic-gate 	sc_shared_t *tmp;
8037c478bd9Sstevel@tonic-gate 
8047c478bd9Sstevel@tonic-gate 	if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */
8057c478bd9Sstevel@tonic-gate 	    !self->ul_vfork &&			/* not a child of vfork() */
8067c478bd9Sstevel@tonic-gate 	    !self->ul_schedctl_called) {	/* haven't been called before */
8077c478bd9Sstevel@tonic-gate 		enter_critical(self);
8087c478bd9Sstevel@tonic-gate 		self->ul_schedctl_called = &self->ul_uberdata->uberflags;
8097c478bd9Sstevel@tonic-gate 		if ((tmp = __schedctl()) != (sc_shared_t *)(-1))
8107c478bd9Sstevel@tonic-gate 			self->ul_schedctl = scp = tmp;
8117c478bd9Sstevel@tonic-gate 		exit_critical(self);
8127c478bd9Sstevel@tonic-gate 	}
8137c478bd9Sstevel@tonic-gate 	/*
8147c478bd9Sstevel@tonic-gate 	 * Unless the call to setup_schedctl() is surrounded
8157c478bd9Sstevel@tonic-gate 	 * by enter_critical()/exit_critical(), the address
8167c478bd9Sstevel@tonic-gate 	 * we are returning could be invalid due to a forkall()
8177c478bd9Sstevel@tonic-gate 	 * having occurred in another thread.
8187c478bd9Sstevel@tonic-gate 	 */
8197c478bd9Sstevel@tonic-gate 	return (scp);
8207c478bd9Sstevel@tonic-gate }
8217c478bd9Sstevel@tonic-gate 
8227c478bd9Sstevel@tonic-gate /*
8237c478bd9Sstevel@tonic-gate  * Interfaces from libsched, incorporated into libc.
8247c478bd9Sstevel@tonic-gate  * libsched.so.1 is now a filter library onto libc.
8257c478bd9Sstevel@tonic-gate  */
8267c478bd9Sstevel@tonic-gate #pragma weak schedctl_lookup = _schedctl_init
8277c478bd9Sstevel@tonic-gate #pragma weak _schedctl_lookup = _schedctl_init
8287c478bd9Sstevel@tonic-gate #pragma weak schedctl_init = _schedctl_init
8297c478bd9Sstevel@tonic-gate schedctl_t *
8307c478bd9Sstevel@tonic-gate _schedctl_init(void)
8317c478bd9Sstevel@tonic-gate {
8327c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp = setup_schedctl();
8337c478bd9Sstevel@tonic-gate 	return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl);
8347c478bd9Sstevel@tonic-gate }
8357c478bd9Sstevel@tonic-gate 
8367c478bd9Sstevel@tonic-gate #pragma weak schedctl_exit = _schedctl_exit
8377c478bd9Sstevel@tonic-gate void
8387c478bd9Sstevel@tonic-gate _schedctl_exit(void)
8397c478bd9Sstevel@tonic-gate {
8407c478bd9Sstevel@tonic-gate }
8417c478bd9Sstevel@tonic-gate 
8427c478bd9Sstevel@tonic-gate /*
8437c478bd9Sstevel@tonic-gate  * Contract private interface for java.
8447c478bd9Sstevel@tonic-gate  * Set up the schedctl data if it doesn't exist yet.
8457c478bd9Sstevel@tonic-gate  * Return a pointer to the pointer to the schedctl data.
8467c478bd9Sstevel@tonic-gate  */
8477c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile *
8487c478bd9Sstevel@tonic-gate _thr_schedctl(void)
8497c478bd9Sstevel@tonic-gate {
8507c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
8517c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *volatile *ptr;
8527c478bd9Sstevel@tonic-gate 
8537c478bd9Sstevel@tonic-gate 	if (self->ul_vfork)
8547c478bd9Sstevel@tonic-gate 		return (NULL);
8557c478bd9Sstevel@tonic-gate 	if (*(ptr = &self->ul_schedctl) == NULL)
8567c478bd9Sstevel@tonic-gate 		(void) setup_schedctl();
8577c478bd9Sstevel@tonic-gate 	return (ptr);
8587c478bd9Sstevel@tonic-gate }
8597c478bd9Sstevel@tonic-gate 
8607c478bd9Sstevel@tonic-gate /*
8617c478bd9Sstevel@tonic-gate  * Block signals and attempt to block preemption.
8627c478bd9Sstevel@tonic-gate  * no_preempt()/preempt() must be used in pairs but can be nested.
8637c478bd9Sstevel@tonic-gate  */
8647c478bd9Sstevel@tonic-gate void
8657c478bd9Sstevel@tonic-gate no_preempt(ulwp_t *self)
8667c478bd9Sstevel@tonic-gate {
8677c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
8687c478bd9Sstevel@tonic-gate 
8697c478bd9Sstevel@tonic-gate 	if (self->ul_preempt++ == 0) {
8707c478bd9Sstevel@tonic-gate 		enter_critical(self);
8717c478bd9Sstevel@tonic-gate 		if ((scp = self->ul_schedctl) != NULL ||
8727c478bd9Sstevel@tonic-gate 		    (scp = setup_schedctl()) != NULL) {
8737c478bd9Sstevel@tonic-gate 			/*
8747c478bd9Sstevel@tonic-gate 			 * Save the pre-existing preempt value.
8757c478bd9Sstevel@tonic-gate 			 */
8767c478bd9Sstevel@tonic-gate 			self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt;
8777c478bd9Sstevel@tonic-gate 			scp->sc_preemptctl.sc_nopreempt = 1;
8787c478bd9Sstevel@tonic-gate 		}
8797c478bd9Sstevel@tonic-gate 	}
8807c478bd9Sstevel@tonic-gate }
8817c478bd9Sstevel@tonic-gate 
8827c478bd9Sstevel@tonic-gate /*
8837c478bd9Sstevel@tonic-gate  * Undo the effects of no_preempt().
8847c478bd9Sstevel@tonic-gate  */
8857c478bd9Sstevel@tonic-gate void
8867c478bd9Sstevel@tonic-gate preempt(ulwp_t *self)
8877c478bd9Sstevel@tonic-gate {
8887c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
8897c478bd9Sstevel@tonic-gate 
8907c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_preempt > 0);
8917c478bd9Sstevel@tonic-gate 	if (--self->ul_preempt == 0) {
8927c478bd9Sstevel@tonic-gate 		if ((scp = self->ul_schedctl) != NULL) {
8937c478bd9Sstevel@tonic-gate 			/*
8947c478bd9Sstevel@tonic-gate 			 * Restore the pre-existing preempt value.
8957c478bd9Sstevel@tonic-gate 			 */
8967c478bd9Sstevel@tonic-gate 			scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt;
8977c478bd9Sstevel@tonic-gate 			if (scp->sc_preemptctl.sc_yield &&
8987c478bd9Sstevel@tonic-gate 			    scp->sc_preemptctl.sc_nopreempt == 0) {
8997c478bd9Sstevel@tonic-gate 				lwp_yield();
9007c478bd9Sstevel@tonic-gate 				if (scp->sc_preemptctl.sc_yield) {
9017c478bd9Sstevel@tonic-gate 					/*
9027c478bd9Sstevel@tonic-gate 					 * Shouldn't happen.  This is either
9037c478bd9Sstevel@tonic-gate 					 * a race condition or the thread
9047c478bd9Sstevel@tonic-gate 					 * just entered the real-time class.
9057c478bd9Sstevel@tonic-gate 					 */
9067c478bd9Sstevel@tonic-gate 					lwp_yield();
9077c478bd9Sstevel@tonic-gate 					scp->sc_preemptctl.sc_yield = 0;
9087c478bd9Sstevel@tonic-gate 				}
9097c478bd9Sstevel@tonic-gate 			}
9107c478bd9Sstevel@tonic-gate 		}
9117c478bd9Sstevel@tonic-gate 		exit_critical(self);
9127c478bd9Sstevel@tonic-gate 	}
9137c478bd9Sstevel@tonic-gate }
9147c478bd9Sstevel@tonic-gate 
9157c478bd9Sstevel@tonic-gate /*
9167c478bd9Sstevel@tonic-gate  * If a call to preempt() would cause the current thread to yield or to
9177c478bd9Sstevel@tonic-gate  * take deferred actions in exit_critical(), then unpark the specified
9187c478bd9Sstevel@tonic-gate  * lwp so it can run while we delay.  Return the original lwpid if the
9197c478bd9Sstevel@tonic-gate  * unpark was not performed, else return zero.  The tests are a repeat
9207c478bd9Sstevel@tonic-gate  * of some of the tests in preempt(), above.  This is a statistical
9217c478bd9Sstevel@tonic-gate  * optimization solely for cond_sleep_queue(), below.
9227c478bd9Sstevel@tonic-gate  */
9237c478bd9Sstevel@tonic-gate static lwpid_t
9247c478bd9Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid)
9257c478bd9Sstevel@tonic-gate {
9267c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp = self->ul_schedctl;
9277c478bd9Sstevel@tonic-gate 
9287c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_preempt == 1 && self->ul_critical > 0);
9297c478bd9Sstevel@tonic-gate 	if ((scp != NULL && scp->sc_preemptctl.sc_yield) ||
9307c478bd9Sstevel@tonic-gate 	    (self->ul_curplease && self->ul_critical == 1)) {
9317c478bd9Sstevel@tonic-gate 		(void) __lwp_unpark(lwpid);
9327c478bd9Sstevel@tonic-gate 		lwpid = 0;
9337c478bd9Sstevel@tonic-gate 	}
9347c478bd9Sstevel@tonic-gate 	return (lwpid);
9357c478bd9Sstevel@tonic-gate }
9367c478bd9Sstevel@tonic-gate 
9377c478bd9Sstevel@tonic-gate /*
9387c478bd9Sstevel@tonic-gate  * Spin for a while, trying to grab the lock.  We know that we
9397c478bd9Sstevel@tonic-gate  * failed set_lock_byte(&mp->mutex_lockw) once before coming here.
9407c478bd9Sstevel@tonic-gate  * If this fails, return EBUSY and let the caller deal with it.
9417c478bd9Sstevel@tonic-gate  * If this succeeds, return 0 with mutex_owner set to curthread.
9427c478bd9Sstevel@tonic-gate  */
9437c478bd9Sstevel@tonic-gate int
9447c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mutex_t *mp)
9457c478bd9Sstevel@tonic-gate {
9467c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
9477c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
9487c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
9497c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
9507c478bd9Sstevel@tonic-gate 	volatile uint64_t *ownerp;
9517c478bd9Sstevel@tonic-gate 	int count, max = self->ul_adaptive_spin;
9527c478bd9Sstevel@tonic-gate 
9537c478bd9Sstevel@tonic-gate 	ASSERT(!(mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)));
9547c478bd9Sstevel@tonic-gate 
9557c478bd9Sstevel@tonic-gate 	if (max == 0 || (mp->mutex_spinners >= self->ul_max_spinners))
9567c478bd9Sstevel@tonic-gate 		return (EBUSY);
9577c478bd9Sstevel@tonic-gate 
9587c478bd9Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
9597c478bd9Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
9607c478bd9Sstevel@tonic-gate 
9617c478bd9Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
9627c478bd9Sstevel@tonic-gate 
9637c478bd9Sstevel@tonic-gate 	/*
9647c478bd9Sstevel@tonic-gate 	 * This spin loop is unfair to lwps that have already dropped into
9657c478bd9Sstevel@tonic-gate 	 * the kernel to sleep.  They will starve on a highly-contended mutex.
9667c478bd9Sstevel@tonic-gate 	 * This is just too bad.  The adaptive spin algorithm is intended
9677c478bd9Sstevel@tonic-gate 	 * to allow programs with highly-contended locks (that is, broken
9687c478bd9Sstevel@tonic-gate 	 * programs) to execute with reasonable speed despite their contention.
9697c478bd9Sstevel@tonic-gate 	 * Being fair would reduce the speed of such programs and well-written
9707c478bd9Sstevel@tonic-gate 	 * programs will not suffer in any case.
9717c478bd9Sstevel@tonic-gate 	 */
9727c478bd9Sstevel@tonic-gate 	enter_critical(self);		/* protects ul_schedctl */
9737c478bd9Sstevel@tonic-gate 	incr32(&mp->mutex_spinners);
9747c478bd9Sstevel@tonic-gate 	for (count = 0; count < max; count++) {
9757c478bd9Sstevel@tonic-gate 		if (*lockp == 0 && set_lock_byte(lockp) == 0) {
9767c478bd9Sstevel@tonic-gate 			*ownerp = (uintptr_t)self;
9777c478bd9Sstevel@tonic-gate 			decr32(&mp->mutex_spinners);
9787c478bd9Sstevel@tonic-gate 			exit_critical(self);
9797c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__spun, 1, count);
9807c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
9817c478bd9Sstevel@tonic-gate 			return (0);
9827c478bd9Sstevel@tonic-gate 		}
9837c478bd9Sstevel@tonic-gate 		SMT_PAUSE();
9847c478bd9Sstevel@tonic-gate 		/*
9857c478bd9Sstevel@tonic-gate 		 * Stop spinning if the mutex owner is not running on
9867c478bd9Sstevel@tonic-gate 		 * a processor; it will not drop the lock any time soon
9877c478bd9Sstevel@tonic-gate 		 * and we would just be wasting time to keep spinning.
9887c478bd9Sstevel@tonic-gate 		 *
9897c478bd9Sstevel@tonic-gate 		 * Note that we are looking at another thread (ulwp_t)
9907c478bd9Sstevel@tonic-gate 		 * without ensuring that the other thread does not exit.
9917c478bd9Sstevel@tonic-gate 		 * The scheme relies on ulwp_t structures never being
9927c478bd9Sstevel@tonic-gate 		 * deallocated by the library (the library employs a free
9937c478bd9Sstevel@tonic-gate 		 * list of ulwp_t structs that are reused when new threads
9947c478bd9Sstevel@tonic-gate 		 * are created) and on schedctl shared memory never being
9957c478bd9Sstevel@tonic-gate 		 * deallocated once created via __schedctl().
9967c478bd9Sstevel@tonic-gate 		 *
9977c478bd9Sstevel@tonic-gate 		 * Thus, the worst that can happen when the spinning thread
9987c478bd9Sstevel@tonic-gate 		 * looks at the owner's schedctl data is that it is looking
9997c478bd9Sstevel@tonic-gate 		 * at some other thread's schedctl data.  This almost never
10007c478bd9Sstevel@tonic-gate 		 * happens and is benign when it does.
10017c478bd9Sstevel@tonic-gate 		 */
10027c478bd9Sstevel@tonic-gate 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
10037c478bd9Sstevel@tonic-gate 		    ((scp = ulwp->ul_schedctl) == NULL ||
10047c478bd9Sstevel@tonic-gate 		    scp->sc_state != SC_ONPROC))
10057c478bd9Sstevel@tonic-gate 			break;
10067c478bd9Sstevel@tonic-gate 	}
10077c478bd9Sstevel@tonic-gate 	decr32(&mp->mutex_spinners);
10087c478bd9Sstevel@tonic-gate 	exit_critical(self);
10097c478bd9Sstevel@tonic-gate 
10107c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__spun, 0, count);
10117c478bd9Sstevel@tonic-gate 
10127c478bd9Sstevel@tonic-gate 	return (EBUSY);
10137c478bd9Sstevel@tonic-gate }
10147c478bd9Sstevel@tonic-gate 
10157c478bd9Sstevel@tonic-gate /*
10167c478bd9Sstevel@tonic-gate  * Same as mutex_trylock_adaptive(), except specifically for queue locks.
10177c478bd9Sstevel@tonic-gate  * The owner field is not set here; the caller (spin_lock_set()) sets it.
10187c478bd9Sstevel@tonic-gate  */
10197c478bd9Sstevel@tonic-gate int
10207c478bd9Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp)
10217c478bd9Sstevel@tonic-gate {
10227c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
10237c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
10247c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
10257c478bd9Sstevel@tonic-gate 	volatile uint64_t *ownerp;
10267c478bd9Sstevel@tonic-gate 	int count = curthread->ul_queue_spin;
10277c478bd9Sstevel@tonic-gate 
10287c478bd9Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
10297c478bd9Sstevel@tonic-gate 
10307c478bd9Sstevel@tonic-gate 	if (count == 0)
10317c478bd9Sstevel@tonic-gate 		return (EBUSY);
10327c478bd9Sstevel@tonic-gate 
10337c478bd9Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
10347c478bd9Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
10357c478bd9Sstevel@tonic-gate 	while (--count >= 0) {
10367c478bd9Sstevel@tonic-gate 		if (*lockp == 0 && set_lock_byte(lockp) == 0)
10377c478bd9Sstevel@tonic-gate 			return (0);
10387c478bd9Sstevel@tonic-gate 		SMT_PAUSE();
10397c478bd9Sstevel@tonic-gate 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
10407c478bd9Sstevel@tonic-gate 		    ((scp = ulwp->ul_schedctl) == NULL ||
10417c478bd9Sstevel@tonic-gate 		    scp->sc_state != SC_ONPROC))
10427c478bd9Sstevel@tonic-gate 			break;
10437c478bd9Sstevel@tonic-gate 	}
10447c478bd9Sstevel@tonic-gate 
10457c478bd9Sstevel@tonic-gate 	return (EBUSY);
10467c478bd9Sstevel@tonic-gate }
10477c478bd9Sstevel@tonic-gate 
10487c478bd9Sstevel@tonic-gate /*
10497c478bd9Sstevel@tonic-gate  * Like mutex_trylock_adaptive(), but for process-shared mutexes.
10507c478bd9Sstevel@tonic-gate  * Spin for a while, trying to grab the lock.  We know that we
10517c478bd9Sstevel@tonic-gate  * failed set_lock_byte(&mp->mutex_lockw) once before coming here.
10527c478bd9Sstevel@tonic-gate  * If this fails, return EBUSY and let the caller deal with it.
10537c478bd9Sstevel@tonic-gate  * If this succeeds, return 0 with mutex_owner set to curthread
10547c478bd9Sstevel@tonic-gate  * and mutex_ownerpid set to the current pid.
10557c478bd9Sstevel@tonic-gate  */
10567c478bd9Sstevel@tonic-gate int
10577c478bd9Sstevel@tonic-gate mutex_trylock_process(mutex_t *mp)
10587c478bd9Sstevel@tonic-gate {
10597c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
10607c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
10617c478bd9Sstevel@tonic-gate 	int count;
10627c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
10637c478bd9Sstevel@tonic-gate 	volatile uint64_t *ownerp;
10647c478bd9Sstevel@tonic-gate 	volatile int32_t *pidp;
10657c478bd9Sstevel@tonic-gate 	pid_t pid, newpid;
10667c478bd9Sstevel@tonic-gate 	uint64_t owner, newowner;
10677c478bd9Sstevel@tonic-gate 
10687c478bd9Sstevel@tonic-gate 	if ((count = ncpus) == 0)
10697c478bd9Sstevel@tonic-gate 		count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
10707c478bd9Sstevel@tonic-gate 	count = (count > 1)? self->ul_adaptive_spin : 0;
10717c478bd9Sstevel@tonic-gate 
10727c478bd9Sstevel@tonic-gate 	ASSERT((mp->mutex_type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) ==
10737c478bd9Sstevel@tonic-gate 		USYNC_PROCESS);
10747c478bd9Sstevel@tonic-gate 
10757c478bd9Sstevel@tonic-gate 	if (count == 0)
10767c478bd9Sstevel@tonic-gate 		return (EBUSY);
10777c478bd9Sstevel@tonic-gate 
10787c478bd9Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
10797c478bd9Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
10807c478bd9Sstevel@tonic-gate 	pidp = (volatile int32_t *)&mp->mutex_ownerpid;
10817c478bd9Sstevel@tonic-gate 	owner = *ownerp;
10827c478bd9Sstevel@tonic-gate 	pid = *pidp;
10837c478bd9Sstevel@tonic-gate 	/*
10847c478bd9Sstevel@tonic-gate 	 * This is a process-shared mutex.
10857c478bd9Sstevel@tonic-gate 	 * We cannot know if the owner is running on a processor.
10867c478bd9Sstevel@tonic-gate 	 * We just spin and hope that it is on a processor.
10877c478bd9Sstevel@tonic-gate 	 */
10887c478bd9Sstevel@tonic-gate 	while (--count >= 0) {
10897c478bd9Sstevel@tonic-gate 		if (*lockp == 0) {
10907c478bd9Sstevel@tonic-gate 			enter_critical(self);
10917c478bd9Sstevel@tonic-gate 			if (set_lock_byte(lockp) == 0) {
10927c478bd9Sstevel@tonic-gate 				*ownerp = (uintptr_t)self;
10937c478bd9Sstevel@tonic-gate 				*pidp = udp->pid;
10947c478bd9Sstevel@tonic-gate 				exit_critical(self);
10957c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
10967c478bd9Sstevel@tonic-gate 				    0, 0);
10977c478bd9Sstevel@tonic-gate 				return (0);
10987c478bd9Sstevel@tonic-gate 			}
10997c478bd9Sstevel@tonic-gate 			exit_critical(self);
11007c478bd9Sstevel@tonic-gate 		} else if ((newowner = *ownerp) == owner &&
11017c478bd9Sstevel@tonic-gate 		    (newpid = *pidp) == pid) {
11027c478bd9Sstevel@tonic-gate 			SMT_PAUSE();
11037c478bd9Sstevel@tonic-gate 			continue;
11047c478bd9Sstevel@tonic-gate 		}
11057c478bd9Sstevel@tonic-gate 		/*
11067c478bd9Sstevel@tonic-gate 		 * The owner of the lock changed; start the count over again.
11077c478bd9Sstevel@tonic-gate 		 * This may be too aggressive; it needs testing.
11087c478bd9Sstevel@tonic-gate 		 */
11097c478bd9Sstevel@tonic-gate 		owner = newowner;
11107c478bd9Sstevel@tonic-gate 		pid = newpid;
11117c478bd9Sstevel@tonic-gate 		count = self->ul_adaptive_spin;
11127c478bd9Sstevel@tonic-gate 	}
11137c478bd9Sstevel@tonic-gate 
11147c478bd9Sstevel@tonic-gate 	return (EBUSY);
11157c478bd9Sstevel@tonic-gate }
11167c478bd9Sstevel@tonic-gate 
11177c478bd9Sstevel@tonic-gate /*
11187c478bd9Sstevel@tonic-gate  * Mutex wakeup code for releasing a USYNC_THREAD mutex.
11197c478bd9Sstevel@tonic-gate  * Returns the lwpid of the thread that was dequeued, if any.
11207c478bd9Sstevel@tonic-gate  * The caller of mutex_wakeup() must call __lwp_unpark(lwpid)
11217c478bd9Sstevel@tonic-gate  * to wake up the specified lwp.
11227c478bd9Sstevel@tonic-gate  */
11237c478bd9Sstevel@tonic-gate lwpid_t
11247c478bd9Sstevel@tonic-gate mutex_wakeup(mutex_t *mp)
11257c478bd9Sstevel@tonic-gate {
11267c478bd9Sstevel@tonic-gate 	lwpid_t lwpid = 0;
11277c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
11287c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
11297c478bd9Sstevel@tonic-gate 	int more;
11307c478bd9Sstevel@tonic-gate 
11317c478bd9Sstevel@tonic-gate 	/*
11327c478bd9Sstevel@tonic-gate 	 * Dequeue a waiter from the sleep queue.  Don't touch the mutex
11337c478bd9Sstevel@tonic-gate 	 * waiters bit if no one was found on the queue because the mutex
11347c478bd9Sstevel@tonic-gate 	 * might have been deallocated or reallocated for another purpose.
11357c478bd9Sstevel@tonic-gate 	 */
11367c478bd9Sstevel@tonic-gate 	qp = queue_lock(mp, MX);
11377c478bd9Sstevel@tonic-gate 	if ((ulwp = dequeue(qp, mp, &more)) != NULL) {
11387c478bd9Sstevel@tonic-gate 		lwpid = ulwp->ul_lwpid;
11397c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = (more? 1 : 0);
11407c478bd9Sstevel@tonic-gate 	}
11417c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
11427c478bd9Sstevel@tonic-gate 	return (lwpid);
11437c478bd9Sstevel@tonic-gate }
11447c478bd9Sstevel@tonic-gate 
11457c478bd9Sstevel@tonic-gate /*
11467c478bd9Sstevel@tonic-gate  * Spin for a while, testing to see if the lock has been grabbed.
11477c478bd9Sstevel@tonic-gate  * If this fails, call mutex_wakeup() to release a waiter.
11487c478bd9Sstevel@tonic-gate  */
11497c478bd9Sstevel@tonic-gate lwpid_t
11507c478bd9Sstevel@tonic-gate mutex_unlock_queue(mutex_t *mp)
11517c478bd9Sstevel@tonic-gate {
11527c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
11537c478bd9Sstevel@tonic-gate 	uint32_t *lockw = &mp->mutex_lockword;
11547c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
11557c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
11567c478bd9Sstevel@tonic-gate 	volatile uint32_t *spinp;
11577c478bd9Sstevel@tonic-gate 	int count;
11587c478bd9Sstevel@tonic-gate 
11597c478bd9Sstevel@tonic-gate 	/*
11607c478bd9Sstevel@tonic-gate 	 * We use the swap primitive to clear the lock, but we must
11617c478bd9Sstevel@tonic-gate 	 * atomically retain the waiters bit for the remainder of this
11627c478bd9Sstevel@tonic-gate 	 * code to work.  We first check to see if the waiters bit is
11637c478bd9Sstevel@tonic-gate 	 * set and if so clear the lock by swapping in a word containing
11647c478bd9Sstevel@tonic-gate 	 * only the waiters bit.  This could produce a false positive test
11657c478bd9Sstevel@tonic-gate 	 * for whether there are waiters that need to be waked up, but
11667c478bd9Sstevel@tonic-gate 	 * this just causes an extra call to mutex_wakeup() to do nothing.
11677c478bd9Sstevel@tonic-gate 	 * The opposite case is more delicate:  If there are no waiters,
11687c478bd9Sstevel@tonic-gate 	 * we swap in a zero lock byte and a zero waiters bit.  The result
11697c478bd9Sstevel@tonic-gate 	 * of the swap could indicate that there really was a waiter so in
11707c478bd9Sstevel@tonic-gate 	 * this case we go directly to mutex_wakeup() without performing
11717c478bd9Sstevel@tonic-gate 	 * any of the adaptive code because the waiter bit has been cleared
11727c478bd9Sstevel@tonic-gate 	 * and the adaptive code is unreliable in this case.
11737c478bd9Sstevel@tonic-gate 	 */
11747c478bd9Sstevel@tonic-gate 	if (!(*lockw & WAITERMASK)) {	/* no waiter exists right now */
11757c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
11767c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
11777c478bd9Sstevel@tonic-gate 		if (!(swap32(lockw, 0) & WAITERMASK))	/* still no waiters */
11787c478bd9Sstevel@tonic-gate 			return (0);
11797c478bd9Sstevel@tonic-gate 		no_preempt(self);	/* ensure a prompt wakeup */
11807c478bd9Sstevel@tonic-gate 		lwpid = mutex_wakeup(mp);
11817c478bd9Sstevel@tonic-gate 	} else {
11827c478bd9Sstevel@tonic-gate 		no_preempt(self);	/* ensure a prompt wakeup */
11837c478bd9Sstevel@tonic-gate 		lockp = (volatile uint8_t *)&mp->mutex_lockw;
11847c478bd9Sstevel@tonic-gate 		spinp = (volatile uint32_t *)&mp->mutex_spinners;
11857c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
11867c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
11877c478bd9Sstevel@tonic-gate 		(void) swap32(lockw, WAITER);	/* clear lock, retain waiter */
11887c478bd9Sstevel@tonic-gate 
11897c478bd9Sstevel@tonic-gate 		/*
11907c478bd9Sstevel@tonic-gate 		 * We spin here fewer times than mutex_trylock_adaptive().
11917c478bd9Sstevel@tonic-gate 		 * We are trying to balance two conflicting goals:
11927c478bd9Sstevel@tonic-gate 		 * 1. Avoid waking up anyone if a spinning thread
11937c478bd9Sstevel@tonic-gate 		 *    grabs the lock.
11947c478bd9Sstevel@tonic-gate 		 * 2. Wake up a sleeping thread promptly to get on
11957c478bd9Sstevel@tonic-gate 		 *    with useful work.
11967c478bd9Sstevel@tonic-gate 		 * We don't spin at all if there is no acquiring spinner;
11977c478bd9Sstevel@tonic-gate 		 * (mp->mutex_spinners is non-zero if there are spinners).
11987c478bd9Sstevel@tonic-gate 		 */
11997c478bd9Sstevel@tonic-gate 		for (count = self->ul_release_spin;
12007c478bd9Sstevel@tonic-gate 		    *spinp && count > 0; count--) {
12017c478bd9Sstevel@tonic-gate 			/*
12027c478bd9Sstevel@tonic-gate 			 * There is a waiter that we will have to wake
12037c478bd9Sstevel@tonic-gate 			 * up unless someone else grabs the lock while
12047c478bd9Sstevel@tonic-gate 			 * we are busy spinning.  Like the spin loop in
12057c478bd9Sstevel@tonic-gate 			 * mutex_trylock_adaptive(), this spin loop is
12067c478bd9Sstevel@tonic-gate 			 * unfair to lwps that have already dropped into
12077c478bd9Sstevel@tonic-gate 			 * the kernel to sleep.  They will starve on a
12087c478bd9Sstevel@tonic-gate 			 * highly-contended mutex.  Too bad.
12097c478bd9Sstevel@tonic-gate 			 */
12107c478bd9Sstevel@tonic-gate 			if (*lockp != 0) {	/* somebody grabbed the lock */
12117c478bd9Sstevel@tonic-gate 				preempt(self);
12127c478bd9Sstevel@tonic-gate 				return (0);
12137c478bd9Sstevel@tonic-gate 			}
12147c478bd9Sstevel@tonic-gate 			SMT_PAUSE();
12157c478bd9Sstevel@tonic-gate 		}
12167c478bd9Sstevel@tonic-gate 
12177c478bd9Sstevel@tonic-gate 		/*
12187c478bd9Sstevel@tonic-gate 		 * No one grabbed the lock.
12197c478bd9Sstevel@tonic-gate 		 * Wake up some lwp that is waiting for it.
12207c478bd9Sstevel@tonic-gate 		 */
12217c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = 0;
12227c478bd9Sstevel@tonic-gate 		lwpid = mutex_wakeup(mp);
12237c478bd9Sstevel@tonic-gate 	}
12247c478bd9Sstevel@tonic-gate 
12257c478bd9Sstevel@tonic-gate 	if (lwpid == 0)
12267c478bd9Sstevel@tonic-gate 		preempt(self);
12277c478bd9Sstevel@tonic-gate 	return (lwpid);
12287c478bd9Sstevel@tonic-gate }
12297c478bd9Sstevel@tonic-gate 
12307c478bd9Sstevel@tonic-gate /*
12317c478bd9Sstevel@tonic-gate  * Like mutex_unlock_queue(), but for process-shared mutexes.
12327c478bd9Sstevel@tonic-gate  * We tested the waiters field before calling here and it was non-zero.
12337c478bd9Sstevel@tonic-gate  */
12347c478bd9Sstevel@tonic-gate void
12357c478bd9Sstevel@tonic-gate mutex_unlock_process(mutex_t *mp)
12367c478bd9Sstevel@tonic-gate {
12377c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
12387c478bd9Sstevel@tonic-gate 	int count;
12397c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
12407c478bd9Sstevel@tonic-gate 
12417c478bd9Sstevel@tonic-gate 	/*
12427c478bd9Sstevel@tonic-gate 	 * See the comments in mutex_unlock_queue(), above.
12437c478bd9Sstevel@tonic-gate 	 */
12447c478bd9Sstevel@tonic-gate 	if ((count = ncpus) == 0)
12457c478bd9Sstevel@tonic-gate 		count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
12467c478bd9Sstevel@tonic-gate 	count = (count > 1)? self->ul_release_spin : 0;
12477c478bd9Sstevel@tonic-gate 	no_preempt(self);
12487c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
12497c478bd9Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
12507c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
12517c478bd9Sstevel@tonic-gate 	if (count == 0) {
12527c478bd9Sstevel@tonic-gate 		/* clear lock, test waiter */
12537c478bd9Sstevel@tonic-gate 		if (!(swap32(&mp->mutex_lockword, 0) & WAITERMASK)) {
12547c478bd9Sstevel@tonic-gate 			/* no waiters now */
12557c478bd9Sstevel@tonic-gate 			preempt(self);
12567c478bd9Sstevel@tonic-gate 			return;
12577c478bd9Sstevel@tonic-gate 		}
12587c478bd9Sstevel@tonic-gate 	} else {
12597c478bd9Sstevel@tonic-gate 		/* clear lock, retain waiter */
12607c478bd9Sstevel@tonic-gate 		(void) swap32(&mp->mutex_lockword, WAITER);
12617c478bd9Sstevel@tonic-gate 		lockp = (volatile uint8_t *)&mp->mutex_lockw;
12627c478bd9Sstevel@tonic-gate 		while (--count >= 0) {
12637c478bd9Sstevel@tonic-gate 			if (*lockp != 0) {
12647c478bd9Sstevel@tonic-gate 				/* somebody grabbed the lock */
12657c478bd9Sstevel@tonic-gate 				preempt(self);
12667c478bd9Sstevel@tonic-gate 				return;
12677c478bd9Sstevel@tonic-gate 			}
12687c478bd9Sstevel@tonic-gate 			SMT_PAUSE();
12697c478bd9Sstevel@tonic-gate 		}
12707c478bd9Sstevel@tonic-gate 		/*
12717c478bd9Sstevel@tonic-gate 		 * We must clear the waiters field before going
12727c478bd9Sstevel@tonic-gate 		 * to the kernel, else it could remain set forever.
12737c478bd9Sstevel@tonic-gate 		 */
12747c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = 0;
12757c478bd9Sstevel@tonic-gate 	}
12767c478bd9Sstevel@tonic-gate 	(void) ___lwp_mutex_wakeup(mp);
12777c478bd9Sstevel@tonic-gate 	preempt(self);
12787c478bd9Sstevel@tonic-gate }
12797c478bd9Sstevel@tonic-gate 
12807c478bd9Sstevel@tonic-gate /*
12817c478bd9Sstevel@tonic-gate  * Return the real priority of a thread.
12827c478bd9Sstevel@tonic-gate  */
12837c478bd9Sstevel@tonic-gate int
12847c478bd9Sstevel@tonic-gate real_priority(ulwp_t *ulwp)
12857c478bd9Sstevel@tonic-gate {
12867c478bd9Sstevel@tonic-gate 	if (ulwp->ul_epri == 0)
12877c478bd9Sstevel@tonic-gate 		return (ulwp->ul_mappedpri? ulwp->ul_mappedpri : ulwp->ul_pri);
12887c478bd9Sstevel@tonic-gate 	return (ulwp->ul_emappedpri? ulwp->ul_emappedpri : ulwp->ul_epri);
12897c478bd9Sstevel@tonic-gate }
12907c478bd9Sstevel@tonic-gate 
12917c478bd9Sstevel@tonic-gate void
12927c478bd9Sstevel@tonic-gate stall(void)
12937c478bd9Sstevel@tonic-gate {
12947c478bd9Sstevel@tonic-gate 	for (;;)
12957c478bd9Sstevel@tonic-gate 		(void) mutex_lock_kernel(&stall_mutex, NULL, NULL);
12967c478bd9Sstevel@tonic-gate }
12977c478bd9Sstevel@tonic-gate 
12987c478bd9Sstevel@tonic-gate /*
12997c478bd9Sstevel@tonic-gate  * Acquire a USYNC_THREAD mutex via user-level sleep queues.
13007c478bd9Sstevel@tonic-gate  * We failed set_lock_byte(&mp->mutex_lockw) before coming here.
13017c478bd9Sstevel@tonic-gate  * Returns with mutex_owner set correctly.
13027c478bd9Sstevel@tonic-gate  */
13037c478bd9Sstevel@tonic-gate int
13047c478bd9Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
13057c478bd9Sstevel@tonic-gate 	timespec_t *tsp)
13067c478bd9Sstevel@tonic-gate {
13077c478bd9Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
13087c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
13097c478bd9Sstevel@tonic-gate 	hrtime_t begin_sleep;
13107c478bd9Sstevel@tonic-gate 	int error = 0;
13117c478bd9Sstevel@tonic-gate 
13127c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
13137c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
13147c478bd9Sstevel@tonic-gate 		self->ul_wchan = mp;
13157c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
13167c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = mp;
13177c478bd9Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
13187c478bd9Sstevel@tonic-gate 	}
13197c478bd9Sstevel@tonic-gate 	if (msp) {
13207c478bd9Sstevel@tonic-gate 		tdb_incr(msp->mutex_sleep);
13217c478bd9Sstevel@tonic-gate 		begin_sleep = gethrtime();
13227c478bd9Sstevel@tonic-gate 	}
13237c478bd9Sstevel@tonic-gate 
13247c478bd9Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__block, mp);
13257c478bd9Sstevel@tonic-gate 
13267c478bd9Sstevel@tonic-gate 	/*
13277c478bd9Sstevel@tonic-gate 	 * Put ourself on the sleep queue, and while we are
13287c478bd9Sstevel@tonic-gate 	 * unable to grab the lock, go park in the kernel.
13297c478bd9Sstevel@tonic-gate 	 * Take ourself off the sleep queue after we acquire the lock.
13307c478bd9Sstevel@tonic-gate 	 * The waiter bit can be set/cleared only while holding the queue lock.
13317c478bd9Sstevel@tonic-gate 	 */
13327c478bd9Sstevel@tonic-gate 	qp = queue_lock(mp, MX);
13337c478bd9Sstevel@tonic-gate 	enqueue(qp, self, mp, MX);
13347c478bd9Sstevel@tonic-gate 	mp->mutex_waiters = 1;
13357c478bd9Sstevel@tonic-gate 	for (;;) {
13367c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
13377c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
13387c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
13397c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
13407c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = dequeue_self(qp, mp);
13417c478bd9Sstevel@tonic-gate 			break;
13427c478bd9Sstevel@tonic-gate 		}
13437c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 1);
13447c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
13457c478bd9Sstevel@tonic-gate 		/*
13467c478bd9Sstevel@tonic-gate 		 * __lwp_park() will return the residual time in tsp
13477c478bd9Sstevel@tonic-gate 		 * if we are unparked before the timeout expires.
13487c478bd9Sstevel@tonic-gate 		 */
13497c478bd9Sstevel@tonic-gate 		if ((error = __lwp_park(tsp, 0)) == EINTR)
13507c478bd9Sstevel@tonic-gate 			error = 0;
13517c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 0);
13527c478bd9Sstevel@tonic-gate 		/*
13537c478bd9Sstevel@tonic-gate 		 * We could have taken a signal or suspended ourself.
13547c478bd9Sstevel@tonic-gate 		 * If we did, then we removed ourself from the queue.
13557c478bd9Sstevel@tonic-gate 		 * Someone else may have removed us from the queue
13567c478bd9Sstevel@tonic-gate 		 * as a consequence of mutex_unlock().  We may have
13577c478bd9Sstevel@tonic-gate 		 * gotten a timeout from __lwp_park().  Or we may still
13587c478bd9Sstevel@tonic-gate 		 * be on the queue and this is just a spurious wakeup.
13597c478bd9Sstevel@tonic-gate 		 */
13607c478bd9Sstevel@tonic-gate 		qp = queue_lock(mp, MX);
13617c478bd9Sstevel@tonic-gate 		if (self->ul_sleepq == NULL) {
13627c478bd9Sstevel@tonic-gate 			if (error) {
13637c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
13647c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
13657c478bd9Sstevel@tonic-gate 				    error);
13667c478bd9Sstevel@tonic-gate 				break;
13677c478bd9Sstevel@tonic-gate 			}
13687c478bd9Sstevel@tonic-gate 			if (set_lock_byte(&mp->mutex_lockw) == 0) {
13697c478bd9Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
13707c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
13717c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
13727c478bd9Sstevel@tonic-gate 				    0, 0);
13737c478bd9Sstevel@tonic-gate 				break;
13747c478bd9Sstevel@tonic-gate 			}
13757c478bd9Sstevel@tonic-gate 			enqueue(qp, self, mp, MX);
13767c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = 1;
13777c478bd9Sstevel@tonic-gate 		}
13787c478bd9Sstevel@tonic-gate 		ASSERT(self->ul_sleepq == qp &&
13797c478bd9Sstevel@tonic-gate 		    self->ul_qtype == MX &&
13807c478bd9Sstevel@tonic-gate 		    self->ul_wchan == mp);
13817c478bd9Sstevel@tonic-gate 		if (error) {
13827c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = dequeue_self(qp, mp);
13837c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
13847c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
13857c478bd9Sstevel@tonic-gate 			break;
13867c478bd9Sstevel@tonic-gate 		}
13877c478bd9Sstevel@tonic-gate 	}
13887c478bd9Sstevel@tonic-gate 
13897c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
13907c478bd9Sstevel@tonic-gate 	    self->ul_wchan == NULL);
13917c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
13927c478bd9Sstevel@tonic-gate 
13937c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
13947c478bd9Sstevel@tonic-gate 	if (msp)
13957c478bd9Sstevel@tonic-gate 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
13967c478bd9Sstevel@tonic-gate 
13977c478bd9Sstevel@tonic-gate 	ASSERT(error == 0 || error == EINVAL || error == ETIME);
13987c478bd9Sstevel@tonic-gate 	return (error);
13997c478bd9Sstevel@tonic-gate }
14007c478bd9Sstevel@tonic-gate 
14017c478bd9Sstevel@tonic-gate /*
14027c478bd9Sstevel@tonic-gate  * Returns with mutex_owner set correctly.
14037c478bd9Sstevel@tonic-gate  */
14047c478bd9Sstevel@tonic-gate int
14057c478bd9Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try)
14067c478bd9Sstevel@tonic-gate {
14077c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
14087c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
14097c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
14107c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
14117c478bd9Sstevel@tonic-gate 	int error = 0;
14127c478bd9Sstevel@tonic-gate 
14137c478bd9Sstevel@tonic-gate 	ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
14147c478bd9Sstevel@tonic-gate 
14157c478bd9Sstevel@tonic-gate 	if (!self->ul_schedctl_called)
14167c478bd9Sstevel@tonic-gate 		(void) setup_schedctl();
14177c478bd9Sstevel@tonic-gate 
14187c478bd9Sstevel@tonic-gate 	if (msp && try == MUTEX_TRY)
14197c478bd9Sstevel@tonic-gate 		tdb_incr(msp->mutex_try);
14207c478bd9Sstevel@tonic-gate 
14217c478bd9Sstevel@tonic-gate 	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) {
14227c478bd9Sstevel@tonic-gate 		if (mtype & LOCK_RECURSIVE) {
14237c478bd9Sstevel@tonic-gate 			if (mp->mutex_rcount == RECURSION_MAX) {
14247c478bd9Sstevel@tonic-gate 				error = EAGAIN;
14257c478bd9Sstevel@tonic-gate 			} else {
14267c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
14277c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
14287c478bd9Sstevel@tonic-gate 				    1, 0);
14297c478bd9Sstevel@tonic-gate 				return (0);
14307c478bd9Sstevel@tonic-gate 			}
14317c478bd9Sstevel@tonic-gate 		} else if (try == MUTEX_TRY) {
14327c478bd9Sstevel@tonic-gate 			return (EBUSY);
14337c478bd9Sstevel@tonic-gate 		} else {
14347c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
14357c478bd9Sstevel@tonic-gate 			return (EDEADLK);
14367c478bd9Sstevel@tonic-gate 		}
14377c478bd9Sstevel@tonic-gate 	}
14387c478bd9Sstevel@tonic-gate 
14397c478bd9Sstevel@tonic-gate 	if (self->ul_error_detection && try == MUTEX_LOCK &&
14407c478bd9Sstevel@tonic-gate 	    tsp == NULL && mutex_is_held(mp))
14417c478bd9Sstevel@tonic-gate 		lock_error(mp, "mutex_lock", NULL, NULL);
14427c478bd9Sstevel@tonic-gate 
14437c478bd9Sstevel@tonic-gate 	if (mtype &
14447c478bd9Sstevel@tonic-gate 	    (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) {
14457c478bd9Sstevel@tonic-gate 		uint8_t ceil;
14467c478bd9Sstevel@tonic-gate 		int myprio;
14477c478bd9Sstevel@tonic-gate 
14487c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_PROTECT) {
14497c478bd9Sstevel@tonic-gate 			ceil = mp->mutex_ceiling;
14507c478bd9Sstevel@tonic-gate 			ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0);
14517c478bd9Sstevel@tonic-gate 			myprio = real_priority(self);
14527c478bd9Sstevel@tonic-gate 			if (myprio > ceil) {
14537c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
14547c478bd9Sstevel@tonic-gate 				    EINVAL);
14557c478bd9Sstevel@tonic-gate 				return (EINVAL);
14567c478bd9Sstevel@tonic-gate 			}
14577c478bd9Sstevel@tonic-gate 			if ((error = _ceil_mylist_add(mp)) != 0) {
14587c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
14597c478bd9Sstevel@tonic-gate 				    error);
14607c478bd9Sstevel@tonic-gate 				return (error);
14617c478bd9Sstevel@tonic-gate 			}
14627c478bd9Sstevel@tonic-gate 			if (myprio < ceil)
14637c478bd9Sstevel@tonic-gate 				_ceil_prio_inherit(ceil);
14647c478bd9Sstevel@tonic-gate 		}
14657c478bd9Sstevel@tonic-gate 
14667c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_INHERIT) {
14677c478bd9Sstevel@tonic-gate 			/* go straight to the kernel */
14687c478bd9Sstevel@tonic-gate 			if (try == MUTEX_TRY)
14697c478bd9Sstevel@tonic-gate 				error = mutex_trylock_kernel(mp);
14707c478bd9Sstevel@tonic-gate 			else	/* MUTEX_LOCK */
14717c478bd9Sstevel@tonic-gate 				error = mutex_lock_kernel(mp, tsp, msp);
14727c478bd9Sstevel@tonic-gate 			/*
14737c478bd9Sstevel@tonic-gate 			 * The kernel never sets or clears the lock byte
14747c478bd9Sstevel@tonic-gate 			 * for PTHREAD_PRIO_INHERIT mutexes.
14757c478bd9Sstevel@tonic-gate 			 * Set it here for debugging consistency.
14767c478bd9Sstevel@tonic-gate 			 */
14777c478bd9Sstevel@tonic-gate 			switch (error) {
14787c478bd9Sstevel@tonic-gate 			case 0:
14797c478bd9Sstevel@tonic-gate 			case EOWNERDEAD:
14807c478bd9Sstevel@tonic-gate 				mp->mutex_lockw = LOCKSET;
14817c478bd9Sstevel@tonic-gate 				break;
14827c478bd9Sstevel@tonic-gate 			}
14837c478bd9Sstevel@tonic-gate 		} else if (mtype & USYNC_PROCESS_ROBUST) {
14847c478bd9Sstevel@tonic-gate 			/* go straight to the kernel */
14857c478bd9Sstevel@tonic-gate 			if (try == MUTEX_TRY)
14867c478bd9Sstevel@tonic-gate 				error = mutex_trylock_kernel(mp);
14877c478bd9Sstevel@tonic-gate 			else	/* MUTEX_LOCK */
14887c478bd9Sstevel@tonic-gate 				error = mutex_lock_kernel(mp, tsp, msp);
14897c478bd9Sstevel@tonic-gate 		} else {	/* PTHREAD_PRIO_PROTECT */
14907c478bd9Sstevel@tonic-gate 			/*
14917c478bd9Sstevel@tonic-gate 			 * Try once at user level before going to the kernel.
14927c478bd9Sstevel@tonic-gate 			 * If this is a process shared mutex then protect
14937c478bd9Sstevel@tonic-gate 			 * against forkall() while setting mp->mutex_ownerpid.
14947c478bd9Sstevel@tonic-gate 			 */
14957c478bd9Sstevel@tonic-gate 			if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
14967c478bd9Sstevel@tonic-gate 				enter_critical(self);
14977c478bd9Sstevel@tonic-gate 				if (set_lock_byte(&mp->mutex_lockw) == 0) {
14987c478bd9Sstevel@tonic-gate 					mp->mutex_owner = (uintptr_t)self;
14997c478bd9Sstevel@tonic-gate 					mp->mutex_ownerpid = udp->pid;
15007c478bd9Sstevel@tonic-gate 					exit_critical(self);
15017c478bd9Sstevel@tonic-gate 					DTRACE_PROBE3(plockstat,
15027c478bd9Sstevel@tonic-gate 					    mutex__acquire, mp, 0, 0);
15037c478bd9Sstevel@tonic-gate 				} else {
15047c478bd9Sstevel@tonic-gate 					exit_critical(self);
15057c478bd9Sstevel@tonic-gate 					error = EBUSY;
15067c478bd9Sstevel@tonic-gate 				}
15077c478bd9Sstevel@tonic-gate 			} else {
15087c478bd9Sstevel@tonic-gate 				if (set_lock_byte(&mp->mutex_lockw) == 0) {
15097c478bd9Sstevel@tonic-gate 					mp->mutex_owner = (uintptr_t)self;
15107c478bd9Sstevel@tonic-gate 					DTRACE_PROBE3(plockstat,
15117c478bd9Sstevel@tonic-gate 					    mutex__acquire, mp, 0, 0);
15127c478bd9Sstevel@tonic-gate 				} else {
15137c478bd9Sstevel@tonic-gate 					error = EBUSY;
15147c478bd9Sstevel@tonic-gate 				}
15157c478bd9Sstevel@tonic-gate 			}
15167c478bd9Sstevel@tonic-gate 			if (error && try == MUTEX_LOCK)
15177c478bd9Sstevel@tonic-gate 				error = mutex_lock_kernel(mp, tsp, msp);
15187c478bd9Sstevel@tonic-gate 		}
15197c478bd9Sstevel@tonic-gate 
15207c478bd9Sstevel@tonic-gate 		if (error) {
15217c478bd9Sstevel@tonic-gate 			if (mtype & PTHREAD_PRIO_INHERIT) {
15227c478bd9Sstevel@tonic-gate 				switch (error) {
15237c478bd9Sstevel@tonic-gate 				case EOWNERDEAD:
15247c478bd9Sstevel@tonic-gate 				case ENOTRECOVERABLE:
15257c478bd9Sstevel@tonic-gate 					if (mtype & PTHREAD_MUTEX_ROBUST_NP)
15267c478bd9Sstevel@tonic-gate 						break;
15277c478bd9Sstevel@tonic-gate 					if (error == EOWNERDEAD) {
15287c478bd9Sstevel@tonic-gate 						/*
15297c478bd9Sstevel@tonic-gate 						 * We own the mutex; unlock it.
15307c478bd9Sstevel@tonic-gate 						 * It becomes ENOTRECOVERABLE.
15317c478bd9Sstevel@tonic-gate 						 * All waiters are waked up.
15327c478bd9Sstevel@tonic-gate 						 */
15337c478bd9Sstevel@tonic-gate 						mp->mutex_owner = 0;
15347c478bd9Sstevel@tonic-gate 						mp->mutex_ownerpid = 0;
15357c478bd9Sstevel@tonic-gate 						DTRACE_PROBE2(plockstat,
15367c478bd9Sstevel@tonic-gate 						    mutex__release, mp, 0);
15377c478bd9Sstevel@tonic-gate 						mp->mutex_lockw = LOCKCLEAR;
15387c478bd9Sstevel@tonic-gate 						(void) ___lwp_mutex_unlock(mp);
15397c478bd9Sstevel@tonic-gate 					}
15407c478bd9Sstevel@tonic-gate 					/* FALLTHROUGH */
15417c478bd9Sstevel@tonic-gate 				case EDEADLK:
15427c478bd9Sstevel@tonic-gate 					if (try == MUTEX_LOCK)
15437c478bd9Sstevel@tonic-gate 						stall();
15447c478bd9Sstevel@tonic-gate 					error = EBUSY;
15457c478bd9Sstevel@tonic-gate 					break;
15467c478bd9Sstevel@tonic-gate 				}
15477c478bd9Sstevel@tonic-gate 			}
15487c478bd9Sstevel@tonic-gate 			if ((mtype & PTHREAD_PRIO_PROTECT) &&
15497c478bd9Sstevel@tonic-gate 			    error != EOWNERDEAD) {
15507c478bd9Sstevel@tonic-gate 				(void) _ceil_mylist_del(mp);
15517c478bd9Sstevel@tonic-gate 				if (myprio < ceil)
15527c478bd9Sstevel@tonic-gate 					_ceil_prio_waive();
15537c478bd9Sstevel@tonic-gate 			}
15547c478bd9Sstevel@tonic-gate 		}
15557c478bd9Sstevel@tonic-gate 	} else if (mtype & USYNC_PROCESS) {
15567c478bd9Sstevel@tonic-gate 		/*
15577c478bd9Sstevel@tonic-gate 		 * This is a process shared mutex.  Protect against
15587c478bd9Sstevel@tonic-gate 		 * forkall() while setting mp->mutex_ownerpid.
15597c478bd9Sstevel@tonic-gate 		 */
15607c478bd9Sstevel@tonic-gate 		enter_critical(self);
15617c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
15627c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
15637c478bd9Sstevel@tonic-gate 			mp->mutex_ownerpid = udp->pid;
15647c478bd9Sstevel@tonic-gate 			exit_critical(self);
15657c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
15667c478bd9Sstevel@tonic-gate 		} else {
15677c478bd9Sstevel@tonic-gate 			/* try a little harder */
15687c478bd9Sstevel@tonic-gate 			exit_critical(self);
15697c478bd9Sstevel@tonic-gate 			error = mutex_trylock_process(mp);
15707c478bd9Sstevel@tonic-gate 		}
15717c478bd9Sstevel@tonic-gate 		if (error && try == MUTEX_LOCK)
15727c478bd9Sstevel@tonic-gate 			error = mutex_lock_kernel(mp, tsp, msp);
15737c478bd9Sstevel@tonic-gate 	} else  {	/* USYNC_THREAD */
15747c478bd9Sstevel@tonic-gate 		/* try once */
15757c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
15767c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
15777c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
15787c478bd9Sstevel@tonic-gate 		} else {
15797c478bd9Sstevel@tonic-gate 			/* try a little harder if we don't own the mutex */
15807c478bd9Sstevel@tonic-gate 			error = EBUSY;
15817c478bd9Sstevel@tonic-gate 			if (MUTEX_OWNER(mp) != self)
15827c478bd9Sstevel@tonic-gate 				error = mutex_trylock_adaptive(mp);
15837c478bd9Sstevel@tonic-gate 			if (error && try == MUTEX_LOCK)		/* go park */
15847c478bd9Sstevel@tonic-gate 				error = mutex_lock_queue(self, msp, mp, tsp);
15857c478bd9Sstevel@tonic-gate 		}
15867c478bd9Sstevel@tonic-gate 	}
15877c478bd9Sstevel@tonic-gate 
15887c478bd9Sstevel@tonic-gate 	switch (error) {
15897c478bd9Sstevel@tonic-gate 	case EOWNERDEAD:
15907c478bd9Sstevel@tonic-gate 	case ELOCKUNMAPPED:
15917c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
15927c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
15937c478bd9Sstevel@tonic-gate 		/* FALLTHROUGH */
15947c478bd9Sstevel@tonic-gate 	case 0:
15957c478bd9Sstevel@tonic-gate 		if (msp)
15967c478bd9Sstevel@tonic-gate 			record_begin_hold(msp);
15977c478bd9Sstevel@tonic-gate 		break;
15987c478bd9Sstevel@tonic-gate 	default:
15997c478bd9Sstevel@tonic-gate 		if (try == MUTEX_TRY) {
16007c478bd9Sstevel@tonic-gate 			if (msp)
16017c478bd9Sstevel@tonic-gate 				tdb_incr(msp->mutex_try_fail);
16027c478bd9Sstevel@tonic-gate 			if (__td_event_report(self, TD_LOCK_TRY, udp)) {
16037c478bd9Sstevel@tonic-gate 				self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
16047c478bd9Sstevel@tonic-gate 				tdb_event(TD_LOCK_TRY, udp);
16057c478bd9Sstevel@tonic-gate 			}
16067c478bd9Sstevel@tonic-gate 		}
16077c478bd9Sstevel@tonic-gate 		break;
16087c478bd9Sstevel@tonic-gate 	}
16097c478bd9Sstevel@tonic-gate 
16107c478bd9Sstevel@tonic-gate 	return (error);
16117c478bd9Sstevel@tonic-gate }
16127c478bd9Sstevel@tonic-gate 
16137c478bd9Sstevel@tonic-gate int
16147c478bd9Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try)
16157c478bd9Sstevel@tonic-gate {
16167c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
16177c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
16187c478bd9Sstevel@tonic-gate 
16197c478bd9Sstevel@tonic-gate 	/*
16207c478bd9Sstevel@tonic-gate 	 * We know that USYNC_PROCESS is set in mtype and that
16217c478bd9Sstevel@tonic-gate 	 * zero, one, or both of the flags LOCK_RECURSIVE and
16227c478bd9Sstevel@tonic-gate 	 * LOCK_ERRORCHECK are set, and that no other flags are set.
16237c478bd9Sstevel@tonic-gate 	 */
16247c478bd9Sstevel@tonic-gate 	enter_critical(self);
16257c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
16267c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
16277c478bd9Sstevel@tonic-gate 		mp->mutex_ownerpid = udp->pid;
16287c478bd9Sstevel@tonic-gate 		exit_critical(self);
16297c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
16307c478bd9Sstevel@tonic-gate 		return (0);
16317c478bd9Sstevel@tonic-gate 	}
16327c478bd9Sstevel@tonic-gate 	exit_critical(self);
16337c478bd9Sstevel@tonic-gate 
16347c478bd9Sstevel@tonic-gate 	if ((mtype & ~USYNC_PROCESS) && shared_mutex_held(mp)) {
16357c478bd9Sstevel@tonic-gate 		if (mtype & LOCK_RECURSIVE) {
16367c478bd9Sstevel@tonic-gate 			if (mp->mutex_rcount == RECURSION_MAX)
16377c478bd9Sstevel@tonic-gate 				return (EAGAIN);
16387c478bd9Sstevel@tonic-gate 			mp->mutex_rcount++;
16397c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0);
16407c478bd9Sstevel@tonic-gate 			return (0);
16417c478bd9Sstevel@tonic-gate 		}
16427c478bd9Sstevel@tonic-gate 		if (try == MUTEX_LOCK) {
16437c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
16447c478bd9Sstevel@tonic-gate 			return (EDEADLK);
16457c478bd9Sstevel@tonic-gate 		}
16467c478bd9Sstevel@tonic-gate 		return (EBUSY);
16477c478bd9Sstevel@tonic-gate 	}
16487c478bd9Sstevel@tonic-gate 
16497c478bd9Sstevel@tonic-gate 	/* try a little harder if we don't own the mutex */
16507c478bd9Sstevel@tonic-gate 	if (!shared_mutex_held(mp) && mutex_trylock_process(mp) == 0)
16517c478bd9Sstevel@tonic-gate 		return (0);
16527c478bd9Sstevel@tonic-gate 
16537c478bd9Sstevel@tonic-gate 	if (try == MUTEX_LOCK)
16547c478bd9Sstevel@tonic-gate 		return (mutex_lock_kernel(mp, tsp, NULL));
16557c478bd9Sstevel@tonic-gate 
16567c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_LOCK_TRY, udp)) {
16577c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
16587c478bd9Sstevel@tonic-gate 		tdb_event(TD_LOCK_TRY, udp);
16597c478bd9Sstevel@tonic-gate 	}
16607c478bd9Sstevel@tonic-gate 	return (EBUSY);
16617c478bd9Sstevel@tonic-gate }
16627c478bd9Sstevel@tonic-gate 
16637c478bd9Sstevel@tonic-gate static int
16647c478bd9Sstevel@tonic-gate slow_lock(ulwp_t *self, mutex_t *mp, timespec_t *tsp)
16657c478bd9Sstevel@tonic-gate {
16667c478bd9Sstevel@tonic-gate 	int error = 0;
16677c478bd9Sstevel@tonic-gate 
16687c478bd9Sstevel@tonic-gate 	if (MUTEX_OWNER(mp) == self || mutex_trylock_adaptive(mp) != 0)
16697c478bd9Sstevel@tonic-gate 		error = mutex_lock_queue(self, NULL, mp, tsp);
16707c478bd9Sstevel@tonic-gate 	return (error);
16717c478bd9Sstevel@tonic-gate }
16727c478bd9Sstevel@tonic-gate 
16737c478bd9Sstevel@tonic-gate int
16747c478bd9Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
16757c478bd9Sstevel@tonic-gate {
16767c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
16777c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
16787c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
16797c478bd9Sstevel@tonic-gate 	int mtype;
16807c478bd9Sstevel@tonic-gate 
16817c478bd9Sstevel@tonic-gate 	/*
16827c478bd9Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
16837c478bd9Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
16847c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics,
16857c478bd9Sstevel@tonic-gate 	 * and the process has only a single thread.
16867c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
16877c478bd9Sstevel@tonic-gate 	 */
16887c478bd9Sstevel@tonic-gate 	if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
16897c478bd9Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
16907c478bd9Sstevel@tonic-gate 		/*
16917c478bd9Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
16927c478bd9Sstevel@tonic-gate 		 */
16937c478bd9Sstevel@tonic-gate 		if (mp->mutex_lockw == 0) {
16947c478bd9Sstevel@tonic-gate 			mp->mutex_lockw = LOCKSET;
16957c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
16967c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
16977c478bd9Sstevel@tonic-gate 			return (0);
16987c478bd9Sstevel@tonic-gate 		}
16997c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
17007c478bd9Sstevel@tonic-gate 			/*
17017c478bd9Sstevel@tonic-gate 			 * LOCK_RECURSIVE, LOCK_ERRORCHECK, or both.
17027c478bd9Sstevel@tonic-gate 			 */
17037c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
17047c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
17057c478bd9Sstevel@tonic-gate 					return (EAGAIN);
17067c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
17077c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
17087c478bd9Sstevel@tonic-gate 				    1, 0);
17097c478bd9Sstevel@tonic-gate 				return (0);
17107c478bd9Sstevel@tonic-gate 			}
17117c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
17127c478bd9Sstevel@tonic-gate 			return (EDEADLK);	/* LOCK_ERRORCHECK */
17137c478bd9Sstevel@tonic-gate 		}
17147c478bd9Sstevel@tonic-gate 		/*
17157c478bd9Sstevel@tonic-gate 		 * We have reached a deadlock, probably because the
17167c478bd9Sstevel@tonic-gate 		 * process is executing non-async-signal-safe code in
17177c478bd9Sstevel@tonic-gate 		 * a signal handler and is attempting to acquire a lock
17187c478bd9Sstevel@tonic-gate 		 * that it already owns.  This is not surprising, given
17197c478bd9Sstevel@tonic-gate 		 * bad programming practices over the years that has
17207c478bd9Sstevel@tonic-gate 		 * resulted in applications calling printf() and such
17217c478bd9Sstevel@tonic-gate 		 * in their signal handlers.  Unless the user has told
17227c478bd9Sstevel@tonic-gate 		 * us that the signal handlers are safe by setting:
17237c478bd9Sstevel@tonic-gate 		 *	export _THREAD_ASYNC_SAFE=1
17247c478bd9Sstevel@tonic-gate 		 * we return EDEADLK rather than actually deadlocking.
17257c478bd9Sstevel@tonic-gate 		 */
17267c478bd9Sstevel@tonic-gate 		if (tsp == NULL &&
17277c478bd9Sstevel@tonic-gate 		    MUTEX_OWNER(mp) == self && !self->ul_async_safe) {
17287c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
17297c478bd9Sstevel@tonic-gate 			return (EDEADLK);
17307c478bd9Sstevel@tonic-gate 		}
17317c478bd9Sstevel@tonic-gate 	}
17327c478bd9Sstevel@tonic-gate 
17337c478bd9Sstevel@tonic-gate 	/*
17347c478bd9Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
17357c478bd9Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
17367c478bd9Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
17377c478bd9Sstevel@tonic-gate 	 */
17387c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
17397c478bd9Sstevel@tonic-gate 	    (gflags->uf_trs_ted |
17407c478bd9Sstevel@tonic-gate 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
17417c478bd9Sstevel@tonic-gate 
17427c478bd9Sstevel@tonic-gate 		if (mtype & USYNC_PROCESS)
17437c478bd9Sstevel@tonic-gate 			return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK));
17447c478bd9Sstevel@tonic-gate 
17457c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
17467c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
17477c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
17487c478bd9Sstevel@tonic-gate 			return (0);
17497c478bd9Sstevel@tonic-gate 		}
17507c478bd9Sstevel@tonic-gate 
17517c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
17527c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
17537c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
17547c478bd9Sstevel@tonic-gate 					return (EAGAIN);
17557c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
17567c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
17577c478bd9Sstevel@tonic-gate 				    1, 0);
17587c478bd9Sstevel@tonic-gate 				return (0);
17597c478bd9Sstevel@tonic-gate 			}
17607c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
17617c478bd9Sstevel@tonic-gate 			return (EDEADLK);	/* LOCK_ERRORCHECK */
17627c478bd9Sstevel@tonic-gate 		}
17637c478bd9Sstevel@tonic-gate 
17647c478bd9Sstevel@tonic-gate 		return (slow_lock(self, mp, tsp));
17657c478bd9Sstevel@tonic-gate 	}
17667c478bd9Sstevel@tonic-gate 
17677c478bd9Sstevel@tonic-gate 	/* else do it the long way */
17687c478bd9Sstevel@tonic-gate 	return (mutex_lock_internal(mp, tsp, MUTEX_LOCK));
17697c478bd9Sstevel@tonic-gate }
17707c478bd9Sstevel@tonic-gate 
17717c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_lock = __mutex_lock
17727c478bd9Sstevel@tonic-gate #pragma weak mutex_lock = __mutex_lock
17737c478bd9Sstevel@tonic-gate #pragma weak _mutex_lock = __mutex_lock
17747c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_lock = __mutex_lock
17757c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_lock = __mutex_lock
17767c478bd9Sstevel@tonic-gate int
17777c478bd9Sstevel@tonic-gate __mutex_lock(mutex_t *mp)
17787c478bd9Sstevel@tonic-gate {
17797c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
17807c478bd9Sstevel@tonic-gate 	return (mutex_lock_impl(mp, NULL));
17817c478bd9Sstevel@tonic-gate }
17827c478bd9Sstevel@tonic-gate 
17837c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_timedlock = _pthread_mutex_timedlock
17847c478bd9Sstevel@tonic-gate int
17857c478bd9Sstevel@tonic-gate _pthread_mutex_timedlock(mutex_t *mp, const timespec_t *abstime)
17867c478bd9Sstevel@tonic-gate {
17877c478bd9Sstevel@tonic-gate 	timespec_t tslocal;
17887c478bd9Sstevel@tonic-gate 	int error;
17897c478bd9Sstevel@tonic-gate 
17907c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
17917c478bd9Sstevel@tonic-gate 	abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
17927c478bd9Sstevel@tonic-gate 	error = mutex_lock_impl(mp, &tslocal);
17937c478bd9Sstevel@tonic-gate 	if (error == ETIME)
17947c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
17957c478bd9Sstevel@tonic-gate 	return (error);
17967c478bd9Sstevel@tonic-gate }
17977c478bd9Sstevel@tonic-gate 
17987c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_reltimedlock_np = _pthread_mutex_reltimedlock_np
17997c478bd9Sstevel@tonic-gate int
18007c478bd9Sstevel@tonic-gate _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime)
18017c478bd9Sstevel@tonic-gate {
18027c478bd9Sstevel@tonic-gate 	timespec_t tslocal;
18037c478bd9Sstevel@tonic-gate 	int error;
18047c478bd9Sstevel@tonic-gate 
18057c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
18067c478bd9Sstevel@tonic-gate 	tslocal = *reltime;
18077c478bd9Sstevel@tonic-gate 	error = mutex_lock_impl(mp, &tslocal);
18087c478bd9Sstevel@tonic-gate 	if (error == ETIME)
18097c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
18107c478bd9Sstevel@tonic-gate 	return (error);
18117c478bd9Sstevel@tonic-gate }
18127c478bd9Sstevel@tonic-gate 
18137c478bd9Sstevel@tonic-gate static int
18147c478bd9Sstevel@tonic-gate slow_trylock(mutex_t *mp, ulwp_t *self)
18157c478bd9Sstevel@tonic-gate {
18167c478bd9Sstevel@tonic-gate 	if (MUTEX_OWNER(mp) == self ||
18177c478bd9Sstevel@tonic-gate 	    mutex_trylock_adaptive(mp) != 0) {
18187c478bd9Sstevel@tonic-gate 		uberdata_t *udp = self->ul_uberdata;
18197c478bd9Sstevel@tonic-gate 
18207c478bd9Sstevel@tonic-gate 		if (__td_event_report(self, TD_LOCK_TRY, udp)) {
18217c478bd9Sstevel@tonic-gate 			self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
18227c478bd9Sstevel@tonic-gate 			tdb_event(TD_LOCK_TRY, udp);
18237c478bd9Sstevel@tonic-gate 		}
18247c478bd9Sstevel@tonic-gate 		return (EBUSY);
18257c478bd9Sstevel@tonic-gate 	}
18267c478bd9Sstevel@tonic-gate 	return (0);
18277c478bd9Sstevel@tonic-gate }
18287c478bd9Sstevel@tonic-gate 
18297c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_trylock = __mutex_trylock
18307c478bd9Sstevel@tonic-gate #pragma weak mutex_trylock = __mutex_trylock
18317c478bd9Sstevel@tonic-gate #pragma weak _mutex_trylock = __mutex_trylock
18327c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_trylock = __mutex_trylock
18337c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_trylock = __mutex_trylock
18347c478bd9Sstevel@tonic-gate int
18357c478bd9Sstevel@tonic-gate __mutex_trylock(mutex_t *mp)
18367c478bd9Sstevel@tonic-gate {
18377c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
18387c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
18397c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
18407c478bd9Sstevel@tonic-gate 	int mtype;
18417c478bd9Sstevel@tonic-gate 
18427c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
18437c478bd9Sstevel@tonic-gate 	/*
18447c478bd9Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
18457c478bd9Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
18467c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics,
18477c478bd9Sstevel@tonic-gate 	 * and the process has only a single thread.
18487c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
18497c478bd9Sstevel@tonic-gate 	 */
18507c478bd9Sstevel@tonic-gate 	if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
18517c478bd9Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
18527c478bd9Sstevel@tonic-gate 		/*
18537c478bd9Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
18547c478bd9Sstevel@tonic-gate 		 */
18557c478bd9Sstevel@tonic-gate 		if (mp->mutex_lockw == 0) {
18567c478bd9Sstevel@tonic-gate 			mp->mutex_lockw = LOCKSET;
18577c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
18587c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
18597c478bd9Sstevel@tonic-gate 			return (0);
18607c478bd9Sstevel@tonic-gate 		}
18617c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
18627c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
18637c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
18647c478bd9Sstevel@tonic-gate 					return (EAGAIN);
18657c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
18667c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
18677c478bd9Sstevel@tonic-gate 				    1, 0);
18687c478bd9Sstevel@tonic-gate 				return (0);
18697c478bd9Sstevel@tonic-gate 			}
18707c478bd9Sstevel@tonic-gate 			return (EDEADLK);	/* LOCK_ERRORCHECK */
18717c478bd9Sstevel@tonic-gate 		}
18727c478bd9Sstevel@tonic-gate 		return (EBUSY);
18737c478bd9Sstevel@tonic-gate 	}
18747c478bd9Sstevel@tonic-gate 
18757c478bd9Sstevel@tonic-gate 	/*
18767c478bd9Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
18777c478bd9Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
18787c478bd9Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
18797c478bd9Sstevel@tonic-gate 	 */
18807c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
18817c478bd9Sstevel@tonic-gate 	    (gflags->uf_trs_ted |
18827c478bd9Sstevel@tonic-gate 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
18837c478bd9Sstevel@tonic-gate 
18847c478bd9Sstevel@tonic-gate 		if (mtype & USYNC_PROCESS)
18857c478bd9Sstevel@tonic-gate 			return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY));
18867c478bd9Sstevel@tonic-gate 
18877c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
18887c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
18897c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
18907c478bd9Sstevel@tonic-gate 			return (0);
18917c478bd9Sstevel@tonic-gate 		}
18927c478bd9Sstevel@tonic-gate 
18937c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
18947c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
18957c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
18967c478bd9Sstevel@tonic-gate 					return (EAGAIN);
18977c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
18987c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
18997c478bd9Sstevel@tonic-gate 				    1, 0);
19007c478bd9Sstevel@tonic-gate 				return (0);
19017c478bd9Sstevel@tonic-gate 			}
19027c478bd9Sstevel@tonic-gate 			return (EBUSY);		/* LOCK_ERRORCHECK */
19037c478bd9Sstevel@tonic-gate 		}
19047c478bd9Sstevel@tonic-gate 
19057c478bd9Sstevel@tonic-gate 		return (slow_trylock(mp, self));
19067c478bd9Sstevel@tonic-gate 	}
19077c478bd9Sstevel@tonic-gate 
19087c478bd9Sstevel@tonic-gate 	/* else do it the long way */
19097c478bd9Sstevel@tonic-gate 	return (mutex_lock_internal(mp, NULL, MUTEX_TRY));
19107c478bd9Sstevel@tonic-gate }
19117c478bd9Sstevel@tonic-gate 
19127c478bd9Sstevel@tonic-gate int
19137c478bd9Sstevel@tonic-gate mutex_unlock_internal(mutex_t *mp)
19147c478bd9Sstevel@tonic-gate {
19157c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
19167c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
19177c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
19187c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp;
19197c478bd9Sstevel@tonic-gate 	int error;
19207c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
19217c478bd9Sstevel@tonic-gate 
19227c478bd9Sstevel@tonic-gate 	if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp))
19237c478bd9Sstevel@tonic-gate 		return (EPERM);
19247c478bd9Sstevel@tonic-gate 
19257c478bd9Sstevel@tonic-gate 	if (self->ul_error_detection && !mutex_is_held(mp))
19267c478bd9Sstevel@tonic-gate 		lock_error(mp, "mutex_unlock", NULL, NULL);
19277c478bd9Sstevel@tonic-gate 
19287c478bd9Sstevel@tonic-gate 	if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
19297c478bd9Sstevel@tonic-gate 		mp->mutex_rcount--;
19307c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
19317c478bd9Sstevel@tonic-gate 		return (0);
19327c478bd9Sstevel@tonic-gate 	}
19337c478bd9Sstevel@tonic-gate 
19347c478bd9Sstevel@tonic-gate 	if ((msp = MUTEX_STATS(mp, udp)) != NULL)
19357c478bd9Sstevel@tonic-gate 		(void) record_hold_time(msp);
19367c478bd9Sstevel@tonic-gate 
19377c478bd9Sstevel@tonic-gate 	if (mtype &
19387c478bd9Sstevel@tonic-gate 	    (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) {
19397c478bd9Sstevel@tonic-gate 		no_preempt(self);
19407c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
19417c478bd9Sstevel@tonic-gate 		mp->mutex_ownerpid = 0;
19427c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
19437c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_INHERIT) {
19447c478bd9Sstevel@tonic-gate 			mp->mutex_lockw = LOCKCLEAR;
19457c478bd9Sstevel@tonic-gate 			error = ___lwp_mutex_unlock(mp);
19467c478bd9Sstevel@tonic-gate 		} else if (mtype & USYNC_PROCESS_ROBUST) {
19477c478bd9Sstevel@tonic-gate 			error = ___lwp_mutex_unlock(mp);
19487c478bd9Sstevel@tonic-gate 		} else {
19497c478bd9Sstevel@tonic-gate 			if (swap32(&mp->mutex_lockword, 0) & WAITERMASK)
19507c478bd9Sstevel@tonic-gate 				(void) ___lwp_mutex_wakeup(mp);
19517c478bd9Sstevel@tonic-gate 			error = 0;
19527c478bd9Sstevel@tonic-gate 		}
19537c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_PROTECT) {
19547c478bd9Sstevel@tonic-gate 			if (_ceil_mylist_del(mp))
19557c478bd9Sstevel@tonic-gate 				_ceil_prio_waive();
19567c478bd9Sstevel@tonic-gate 		}
19577c478bd9Sstevel@tonic-gate 		preempt(self);
19587c478bd9Sstevel@tonic-gate 	} else if (mtype & USYNC_PROCESS) {
19597c478bd9Sstevel@tonic-gate 		if (mp->mutex_lockword & WAITERMASK)
19607c478bd9Sstevel@tonic-gate 			mutex_unlock_process(mp);
19617c478bd9Sstevel@tonic-gate 		else {
19627c478bd9Sstevel@tonic-gate 			mp->mutex_owner = 0;
19637c478bd9Sstevel@tonic-gate 			mp->mutex_ownerpid = 0;
19647c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
19657c478bd9Sstevel@tonic-gate 			if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) {
19667c478bd9Sstevel@tonic-gate 				no_preempt(self);
19677c478bd9Sstevel@tonic-gate 				(void) ___lwp_mutex_wakeup(mp);
19687c478bd9Sstevel@tonic-gate 				preempt(self);
19697c478bd9Sstevel@tonic-gate 			}
19707c478bd9Sstevel@tonic-gate 		}
19717c478bd9Sstevel@tonic-gate 		error = 0;
19727c478bd9Sstevel@tonic-gate 	} else {	/* USYNC_THREAD */
19737c478bd9Sstevel@tonic-gate 		if ((lwpid = mutex_unlock_queue(mp)) != 0) {
19747c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid);
19757c478bd9Sstevel@tonic-gate 			preempt(self);
19767c478bd9Sstevel@tonic-gate 		}
19777c478bd9Sstevel@tonic-gate 		error = 0;
19787c478bd9Sstevel@tonic-gate 	}
19797c478bd9Sstevel@tonic-gate 
19807c478bd9Sstevel@tonic-gate 	return (error);
19817c478bd9Sstevel@tonic-gate }
19827c478bd9Sstevel@tonic-gate 
19837c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_unlock = __mutex_unlock
19847c478bd9Sstevel@tonic-gate #pragma weak mutex_unlock = __mutex_unlock
19857c478bd9Sstevel@tonic-gate #pragma weak _mutex_unlock = __mutex_unlock
19867c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_unlock = __mutex_unlock
19877c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_unlock = __mutex_unlock
19887c478bd9Sstevel@tonic-gate int
19897c478bd9Sstevel@tonic-gate __mutex_unlock(mutex_t *mp)
19907c478bd9Sstevel@tonic-gate {
19917c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
19927c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
19937c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
19947c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
19957c478bd9Sstevel@tonic-gate 	int mtype;
19967c478bd9Sstevel@tonic-gate 	short el;
19977c478bd9Sstevel@tonic-gate 
19987c478bd9Sstevel@tonic-gate 	/*
19997c478bd9Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
20007c478bd9Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
20017c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics,
20027c478bd9Sstevel@tonic-gate 	 * and the process has only a single thread.
20037c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
20047c478bd9Sstevel@tonic-gate 	 */
20057c478bd9Sstevel@tonic-gate 	if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
20067c478bd9Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
20077c478bd9Sstevel@tonic-gate 		if (mtype) {
20087c478bd9Sstevel@tonic-gate 			/*
20097c478bd9Sstevel@tonic-gate 			 * At this point we know that one or both of the
20107c478bd9Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
20117c478bd9Sstevel@tonic-gate 			 */
20127c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
20137c478bd9Sstevel@tonic-gate 				return (EPERM);
20147c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
20157c478bd9Sstevel@tonic-gate 				mp->mutex_rcount--;
20167c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
20177c478bd9Sstevel@tonic-gate 				return (0);
20187c478bd9Sstevel@tonic-gate 			}
20197c478bd9Sstevel@tonic-gate 		}
20207c478bd9Sstevel@tonic-gate 		/*
20217c478bd9Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
20227c478bd9Sstevel@tonic-gate 		 * Also, there can be no waiters.
20237c478bd9Sstevel@tonic-gate 		 */
20247c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
20257c478bd9Sstevel@tonic-gate 		mp->mutex_lockword = 0;
20267c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
20277c478bd9Sstevel@tonic-gate 		return (0);
20287c478bd9Sstevel@tonic-gate 	}
20297c478bd9Sstevel@tonic-gate 
20307c478bd9Sstevel@tonic-gate 	/*
20317c478bd9Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
20327c478bd9Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
20337c478bd9Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
20347c478bd9Sstevel@tonic-gate 	 */
20357c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL) {
20367c478bd9Sstevel@tonic-gate 		if (((el = gflags->uf_trs_ted) | mtype) == 0) {
20377c478bd9Sstevel@tonic-gate fast_unlock:
20387c478bd9Sstevel@tonic-gate 			if (!(mp->mutex_lockword & WAITERMASK)) {
20397c478bd9Sstevel@tonic-gate 				/* no waiter exists right now */
20407c478bd9Sstevel@tonic-gate 				mp->mutex_owner = 0;
20417c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
20427c478bd9Sstevel@tonic-gate 				if (swap32(&mp->mutex_lockword, 0) &
20437c478bd9Sstevel@tonic-gate 				    WAITERMASK) {
20447c478bd9Sstevel@tonic-gate 					/* a waiter suddenly appeared */
20457c478bd9Sstevel@tonic-gate 					no_preempt(self);
20467c478bd9Sstevel@tonic-gate 					if ((lwpid = mutex_wakeup(mp)) != 0)
20477c478bd9Sstevel@tonic-gate 						(void) __lwp_unpark(lwpid);
20487c478bd9Sstevel@tonic-gate 					preempt(self);
20497c478bd9Sstevel@tonic-gate 				}
20507c478bd9Sstevel@tonic-gate 			} else if ((lwpid = mutex_unlock_queue(mp)) != 0) {
20517c478bd9Sstevel@tonic-gate 				(void) __lwp_unpark(lwpid);
20527c478bd9Sstevel@tonic-gate 				preempt(self);
20537c478bd9Sstevel@tonic-gate 			}
20547c478bd9Sstevel@tonic-gate 			return (0);
20557c478bd9Sstevel@tonic-gate 		}
20567c478bd9Sstevel@tonic-gate 		if (el)		/* error detection or lock statistics */
20577c478bd9Sstevel@tonic-gate 			goto slow_unlock;
20587c478bd9Sstevel@tonic-gate 		if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
20597c478bd9Sstevel@tonic-gate 			/*
20607c478bd9Sstevel@tonic-gate 			 * At this point we know that one or both of the
20617c478bd9Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
20627c478bd9Sstevel@tonic-gate 			 */
20637c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
20647c478bd9Sstevel@tonic-gate 				return (EPERM);
20657c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
20667c478bd9Sstevel@tonic-gate 				mp->mutex_rcount--;
20677c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
20687c478bd9Sstevel@tonic-gate 				return (0);
20697c478bd9Sstevel@tonic-gate 			}
20707c478bd9Sstevel@tonic-gate 			goto fast_unlock;
20717c478bd9Sstevel@tonic-gate 		}
20727c478bd9Sstevel@tonic-gate 		if ((mtype &
20737c478bd9Sstevel@tonic-gate 		    ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
20747c478bd9Sstevel@tonic-gate 			/*
20757c478bd9Sstevel@tonic-gate 			 * At this point we know that zero, one, or both of the
20767c478bd9Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and
20777c478bd9Sstevel@tonic-gate 			 * that the USYNC_PROCESS flag is set.
20787c478bd9Sstevel@tonic-gate 			 */
20797c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp))
20807c478bd9Sstevel@tonic-gate 				return (EPERM);
20817c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
20827c478bd9Sstevel@tonic-gate 				mp->mutex_rcount--;
20837c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
20847c478bd9Sstevel@tonic-gate 				return (0);
20857c478bd9Sstevel@tonic-gate 			}
20867c478bd9Sstevel@tonic-gate 			if (mp->mutex_lockword & WAITERMASK)
20877c478bd9Sstevel@tonic-gate 				mutex_unlock_process(mp);
20887c478bd9Sstevel@tonic-gate 			else {
20897c478bd9Sstevel@tonic-gate 				mp->mutex_owner = 0;
20907c478bd9Sstevel@tonic-gate 				mp->mutex_ownerpid = 0;
20917c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
20927c478bd9Sstevel@tonic-gate 				if (swap32(&mp->mutex_lockword, 0) &
20937c478bd9Sstevel@tonic-gate 				    WAITERMASK) {
20947c478bd9Sstevel@tonic-gate 					no_preempt(self);
20957c478bd9Sstevel@tonic-gate 					(void) ___lwp_mutex_wakeup(mp);
20967c478bd9Sstevel@tonic-gate 					preempt(self);
20977c478bd9Sstevel@tonic-gate 				}
20987c478bd9Sstevel@tonic-gate 			}
20997c478bd9Sstevel@tonic-gate 			return (0);
21007c478bd9Sstevel@tonic-gate 		}
21017c478bd9Sstevel@tonic-gate 	}
21027c478bd9Sstevel@tonic-gate 
21037c478bd9Sstevel@tonic-gate 	/* else do it the long way */
21047c478bd9Sstevel@tonic-gate slow_unlock:
21057c478bd9Sstevel@tonic-gate 	return (mutex_unlock_internal(mp));
21067c478bd9Sstevel@tonic-gate }
21077c478bd9Sstevel@tonic-gate 
21087c478bd9Sstevel@tonic-gate /*
21097c478bd9Sstevel@tonic-gate  * Internally to the library, almost all mutex lock/unlock actions
21107c478bd9Sstevel@tonic-gate  * go through these lmutex_ functions, to protect critical regions.
21117c478bd9Sstevel@tonic-gate  * We replicate a bit of code from __mutex_lock() and __mutex_unlock()
21127c478bd9Sstevel@tonic-gate  * to make these functions faster since we know that the mutex type
21137c478bd9Sstevel@tonic-gate  * of all internal locks is USYNC_THREAD.  We also know that internal
21147c478bd9Sstevel@tonic-gate  * locking can never fail, so we panic if it does.
21157c478bd9Sstevel@tonic-gate  */
21167c478bd9Sstevel@tonic-gate void
21177c478bd9Sstevel@tonic-gate lmutex_lock(mutex_t *mp)
21187c478bd9Sstevel@tonic-gate {
21197c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
21207c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
21217c478bd9Sstevel@tonic-gate 
21227c478bd9Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
21237c478bd9Sstevel@tonic-gate 
21247c478bd9Sstevel@tonic-gate 	enter_critical(self);
21257c478bd9Sstevel@tonic-gate 	/*
21267c478bd9Sstevel@tonic-gate 	 * Optimize the case of no lock statistics and only a single thread.
21277c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
21287c478bd9Sstevel@tonic-gate 	 */
21297c478bd9Sstevel@tonic-gate 	if (udp->uberflags.uf_all == 0) {
21307c478bd9Sstevel@tonic-gate 		/*
21317c478bd9Sstevel@tonic-gate 		 * Only one thread exists; the mutex must be free.
21327c478bd9Sstevel@tonic-gate 		 */
21337c478bd9Sstevel@tonic-gate 		ASSERT(mp->mutex_lockw == 0);
21347c478bd9Sstevel@tonic-gate 		mp->mutex_lockw = LOCKSET;
21357c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
21367c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
21377c478bd9Sstevel@tonic-gate 	} else {
21387c478bd9Sstevel@tonic-gate 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
21397c478bd9Sstevel@tonic-gate 
21407c478bd9Sstevel@tonic-gate 		if (!self->ul_schedctl_called)
21417c478bd9Sstevel@tonic-gate 			(void) setup_schedctl();
21427c478bd9Sstevel@tonic-gate 
21437c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
21447c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
21457c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
21467c478bd9Sstevel@tonic-gate 		} else if (mutex_trylock_adaptive(mp) != 0) {
21477c478bd9Sstevel@tonic-gate 			(void) mutex_lock_queue(self, msp, mp, NULL);
21487c478bd9Sstevel@tonic-gate 		}
21497c478bd9Sstevel@tonic-gate 
21507c478bd9Sstevel@tonic-gate 		if (msp)
21517c478bd9Sstevel@tonic-gate 			record_begin_hold(msp);
21527c478bd9Sstevel@tonic-gate 	}
21537c478bd9Sstevel@tonic-gate }
21547c478bd9Sstevel@tonic-gate 
21557c478bd9Sstevel@tonic-gate void
21567c478bd9Sstevel@tonic-gate lmutex_unlock(mutex_t *mp)
21577c478bd9Sstevel@tonic-gate {
21587c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
21597c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
21607c478bd9Sstevel@tonic-gate 
21617c478bd9Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
21627c478bd9Sstevel@tonic-gate 
21637c478bd9Sstevel@tonic-gate 	/*
21647c478bd9Sstevel@tonic-gate 	 * Optimize the case of no lock statistics and only a single thread.
21657c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
21667c478bd9Sstevel@tonic-gate 	 */
21677c478bd9Sstevel@tonic-gate 	if (udp->uberflags.uf_all == 0) {
21687c478bd9Sstevel@tonic-gate 		/*
21697c478bd9Sstevel@tonic-gate 		 * Only one thread exists so there can be no waiters.
21707c478bd9Sstevel@tonic-gate 		 */
21717c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
21727c478bd9Sstevel@tonic-gate 		mp->mutex_lockword = 0;
21737c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
21747c478bd9Sstevel@tonic-gate 	} else {
21757c478bd9Sstevel@tonic-gate 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
21767c478bd9Sstevel@tonic-gate 		lwpid_t lwpid;
21777c478bd9Sstevel@tonic-gate 
21787c478bd9Sstevel@tonic-gate 		if (msp)
21797c478bd9Sstevel@tonic-gate 			(void) record_hold_time(msp);
21807c478bd9Sstevel@tonic-gate 		if ((lwpid = mutex_unlock_queue(mp)) != 0) {
21817c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid);
21827c478bd9Sstevel@tonic-gate 			preempt(self);
21837c478bd9Sstevel@tonic-gate 		}
21847c478bd9Sstevel@tonic-gate 	}
21857c478bd9Sstevel@tonic-gate 	exit_critical(self);
21867c478bd9Sstevel@tonic-gate }
21877c478bd9Sstevel@tonic-gate 
21887c478bd9Sstevel@tonic-gate static int
21897c478bd9Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg)
21907c478bd9Sstevel@tonic-gate {
21917c478bd9Sstevel@tonic-gate 	/*
21927c478bd9Sstevel@tonic-gate 	 * There is an inherent data race in the current ownership design.
21937c478bd9Sstevel@tonic-gate 	 * The mutex_owner and mutex_ownerpid fields cannot be set or tested
21947c478bd9Sstevel@tonic-gate 	 * atomically as a pair. The original implementation tested each
21957c478bd9Sstevel@tonic-gate 	 * field just once. This was exposed to trivial false positives in
21967c478bd9Sstevel@tonic-gate 	 * the case of multiple multithreaded processes with thread addresses
21977c478bd9Sstevel@tonic-gate 	 * in common. To close the window to an acceptable level we now use a
21987c478bd9Sstevel@tonic-gate 	 * sequence of five tests: pid-thr-pid-thr-pid. This ensures that any
21997c478bd9Sstevel@tonic-gate 	 * single interruption will still leave one uninterrupted sequence of
22007c478bd9Sstevel@tonic-gate 	 * pid-thr-pid tests intact.
22017c478bd9Sstevel@tonic-gate 	 *
22027c478bd9Sstevel@tonic-gate 	 * It is assumed that all updates are always ordered thr-pid and that
22037c478bd9Sstevel@tonic-gate 	 * we have TSO hardware.
22047c478bd9Sstevel@tonic-gate 	 */
22057c478bd9Sstevel@tonic-gate 	volatile mutex_t *mp = (volatile mutex_t *)mparg;
22067c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
22077c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
22087c478bd9Sstevel@tonic-gate 
22097c478bd9Sstevel@tonic-gate 	if (mp->mutex_ownerpid != udp->pid)
22107c478bd9Sstevel@tonic-gate 		return (0);
22117c478bd9Sstevel@tonic-gate 
22127c478bd9Sstevel@tonic-gate 	if (!MUTEX_OWNED(mp, self))
22137c478bd9Sstevel@tonic-gate 		return (0);
22147c478bd9Sstevel@tonic-gate 
22157c478bd9Sstevel@tonic-gate 	if (mp->mutex_ownerpid != udp->pid)
22167c478bd9Sstevel@tonic-gate 		return (0);
22177c478bd9Sstevel@tonic-gate 
22187c478bd9Sstevel@tonic-gate 	if (!MUTEX_OWNED(mp, self))
22197c478bd9Sstevel@tonic-gate 		return (0);
22207c478bd9Sstevel@tonic-gate 
22217c478bd9Sstevel@tonic-gate 	if (mp->mutex_ownerpid != udp->pid)
22227c478bd9Sstevel@tonic-gate 		return (0);
22237c478bd9Sstevel@tonic-gate 
22247c478bd9Sstevel@tonic-gate 	return (1);
22257c478bd9Sstevel@tonic-gate }
22267c478bd9Sstevel@tonic-gate 
22277c478bd9Sstevel@tonic-gate /*
22287c478bd9Sstevel@tonic-gate  * Some crufty old programs define their own version of _mutex_held()
22297c478bd9Sstevel@tonic-gate  * to be simply return(1).  This breaks internal libc logic, so we
22307c478bd9Sstevel@tonic-gate  * define a private version for exclusive use by libc, mutex_is_held(),
22317c478bd9Sstevel@tonic-gate  * and also a new public function, __mutex_held(), to be used in new
22327c478bd9Sstevel@tonic-gate  * code to circumvent these crufty old programs.
22337c478bd9Sstevel@tonic-gate  */
22347c478bd9Sstevel@tonic-gate #pragma weak mutex_held = mutex_is_held
22357c478bd9Sstevel@tonic-gate #pragma weak _mutex_held = mutex_is_held
22367c478bd9Sstevel@tonic-gate #pragma weak __mutex_held = mutex_is_held
22377c478bd9Sstevel@tonic-gate int
22387c478bd9Sstevel@tonic-gate mutex_is_held(mutex_t *mp)
22397c478bd9Sstevel@tonic-gate {
22407c478bd9Sstevel@tonic-gate 	if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))
22417c478bd9Sstevel@tonic-gate 		return (shared_mutex_held(mp));
22427c478bd9Sstevel@tonic-gate 	return (MUTEX_OWNED(mp, curthread));
22437c478bd9Sstevel@tonic-gate }
22447c478bd9Sstevel@tonic-gate 
22457c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_destroy = __mutex_destroy
22467c478bd9Sstevel@tonic-gate #pragma weak mutex_destroy = __mutex_destroy
22477c478bd9Sstevel@tonic-gate #pragma weak _mutex_destroy = __mutex_destroy
22487c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_destroy = __mutex_destroy
22497c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_destroy = __mutex_destroy
22507c478bd9Sstevel@tonic-gate int
22517c478bd9Sstevel@tonic-gate __mutex_destroy(mutex_t *mp)
22527c478bd9Sstevel@tonic-gate {
22537c478bd9Sstevel@tonic-gate 	mp->mutex_magic = 0;
22547c478bd9Sstevel@tonic-gate 	mp->mutex_flag &= ~LOCK_INITED;
22557c478bd9Sstevel@tonic-gate 	tdb_sync_obj_deregister(mp);
22567c478bd9Sstevel@tonic-gate 	return (0);
22577c478bd9Sstevel@tonic-gate }
22587c478bd9Sstevel@tonic-gate 
22597c478bd9Sstevel@tonic-gate /*
22607c478bd9Sstevel@tonic-gate  * Spin locks are separate from ordinary mutexes,
22617c478bd9Sstevel@tonic-gate  * but we use the same data structure for them.
22627c478bd9Sstevel@tonic-gate  */
22637c478bd9Sstevel@tonic-gate 
22647c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_init = _pthread_spin_init
22657c478bd9Sstevel@tonic-gate int
22667c478bd9Sstevel@tonic-gate _pthread_spin_init(pthread_spinlock_t *lock, int pshared)
22677c478bd9Sstevel@tonic-gate {
22687c478bd9Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
22697c478bd9Sstevel@tonic-gate 
22707c478bd9Sstevel@tonic-gate 	(void) _memset(mp, 0, sizeof (*mp));
22717c478bd9Sstevel@tonic-gate 	if (pshared == PTHREAD_PROCESS_SHARED)
22727c478bd9Sstevel@tonic-gate 		mp->mutex_type = USYNC_PROCESS;
22737c478bd9Sstevel@tonic-gate 	else
22747c478bd9Sstevel@tonic-gate 		mp->mutex_type = USYNC_THREAD;
22757c478bd9Sstevel@tonic-gate 	mp->mutex_flag = LOCK_INITED;
22767c478bd9Sstevel@tonic-gate 	mp->mutex_magic = MUTEX_MAGIC;
22777c478bd9Sstevel@tonic-gate 	return (0);
22787c478bd9Sstevel@tonic-gate }
22797c478bd9Sstevel@tonic-gate 
22807c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_destroy = _pthread_spin_destroy
22817c478bd9Sstevel@tonic-gate int
22827c478bd9Sstevel@tonic-gate _pthread_spin_destroy(pthread_spinlock_t *lock)
22837c478bd9Sstevel@tonic-gate {
22847c478bd9Sstevel@tonic-gate 	(void) _memset(lock, 0, sizeof (*lock));
22857c478bd9Sstevel@tonic-gate 	return (0);
22867c478bd9Sstevel@tonic-gate }
22877c478bd9Sstevel@tonic-gate 
22887c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_trylock = _pthread_spin_trylock
22897c478bd9Sstevel@tonic-gate int
22907c478bd9Sstevel@tonic-gate _pthread_spin_trylock(pthread_spinlock_t *lock)
22917c478bd9Sstevel@tonic-gate {
22927c478bd9Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
22937c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
22947c478bd9Sstevel@tonic-gate 	int error = 0;
22957c478bd9Sstevel@tonic-gate 
22967c478bd9Sstevel@tonic-gate 	no_preempt(self);
22977c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) != 0)
22987c478bd9Sstevel@tonic-gate 		error = EBUSY;
22997c478bd9Sstevel@tonic-gate 	else {
23007c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
23017c478bd9Sstevel@tonic-gate 		if (mp->mutex_type == USYNC_PROCESS)
23027c478bd9Sstevel@tonic-gate 			mp->mutex_ownerpid = self->ul_uberdata->pid;
23037c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
23047c478bd9Sstevel@tonic-gate 	}
23057c478bd9Sstevel@tonic-gate 	preempt(self);
23067c478bd9Sstevel@tonic-gate 	return (error);
23077c478bd9Sstevel@tonic-gate }
23087c478bd9Sstevel@tonic-gate 
23097c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_lock = _pthread_spin_lock
23107c478bd9Sstevel@tonic-gate int
23117c478bd9Sstevel@tonic-gate _pthread_spin_lock(pthread_spinlock_t *lock)
23127c478bd9Sstevel@tonic-gate {
23137c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp =
23147c478bd9Sstevel@tonic-gate 		(volatile uint8_t *)&((mutex_t *)lock)->mutex_lockw;
23157c478bd9Sstevel@tonic-gate 
23167c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23177c478bd9Sstevel@tonic-gate 	/*
23187c478bd9Sstevel@tonic-gate 	 * We don't care whether the owner is running on a processor.
23197c478bd9Sstevel@tonic-gate 	 * We just spin because that's what this interface requires.
23207c478bd9Sstevel@tonic-gate 	 */
23217c478bd9Sstevel@tonic-gate 	for (;;) {
23227c478bd9Sstevel@tonic-gate 		if (*lockp == 0) {	/* lock byte appears to be clear */
23237c478bd9Sstevel@tonic-gate 			if (_pthread_spin_trylock(lock) == 0)
23247c478bd9Sstevel@tonic-gate 				return (0);
23257c478bd9Sstevel@tonic-gate 		}
23267c478bd9Sstevel@tonic-gate 		SMT_PAUSE();
23277c478bd9Sstevel@tonic-gate 	}
23287c478bd9Sstevel@tonic-gate }
23297c478bd9Sstevel@tonic-gate 
23307c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_unlock = _pthread_spin_unlock
23317c478bd9Sstevel@tonic-gate int
23327c478bd9Sstevel@tonic-gate _pthread_spin_unlock(pthread_spinlock_t *lock)
23337c478bd9Sstevel@tonic-gate {
23347c478bd9Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
23357c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
23367c478bd9Sstevel@tonic-gate 
23377c478bd9Sstevel@tonic-gate 	no_preempt(self);
23387c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
23397c478bd9Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
23407c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
23417c478bd9Sstevel@tonic-gate 	(void) swap32(&mp->mutex_lockword, 0);
23427c478bd9Sstevel@tonic-gate 	preempt(self);
23437c478bd9Sstevel@tonic-gate 	return (0);
23447c478bd9Sstevel@tonic-gate }
23457c478bd9Sstevel@tonic-gate 
23467c478bd9Sstevel@tonic-gate #pragma weak cond_init = _cond_init
23477c478bd9Sstevel@tonic-gate /* ARGSUSED2 */
23487c478bd9Sstevel@tonic-gate int
23497c478bd9Sstevel@tonic-gate _cond_init(cond_t *cvp, int type, void *arg)
23507c478bd9Sstevel@tonic-gate {
23517c478bd9Sstevel@tonic-gate 	if (type != USYNC_THREAD && type != USYNC_PROCESS)
23527c478bd9Sstevel@tonic-gate 		return (EINVAL);
23537c478bd9Sstevel@tonic-gate 	(void) _memset(cvp, 0, sizeof (*cvp));
23547c478bd9Sstevel@tonic-gate 	cvp->cond_type = (uint16_t)type;
23557c478bd9Sstevel@tonic-gate 	cvp->cond_magic = COND_MAGIC;
23567c478bd9Sstevel@tonic-gate 	return (0);
23577c478bd9Sstevel@tonic-gate }
23587c478bd9Sstevel@tonic-gate 
23597c478bd9Sstevel@tonic-gate /*
23607c478bd9Sstevel@tonic-gate  * cond_sleep_queue(): utility function for cond_wait_queue().
23617c478bd9Sstevel@tonic-gate  *
23627c478bd9Sstevel@tonic-gate  * Go to sleep on a condvar sleep queue, expect to be waked up
23637c478bd9Sstevel@tonic-gate  * by someone calling cond_signal() or cond_broadcast() or due
23647c478bd9Sstevel@tonic-gate  * to receiving a UNIX signal or being cancelled, or just simply
23657c478bd9Sstevel@tonic-gate  * due to a spurious wakeup (like someome calling forkall()).
23667c478bd9Sstevel@tonic-gate  *
23677c478bd9Sstevel@tonic-gate  * The associated mutex is *not* reacquired before returning.
23687c478bd9Sstevel@tonic-gate  * That must be done by the caller of cond_sleep_queue().
23697c478bd9Sstevel@tonic-gate  */
23707c478bd9Sstevel@tonic-gate int
23717c478bd9Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
23727c478bd9Sstevel@tonic-gate {
23737c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
23747c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
23757c478bd9Sstevel@tonic-gate 	queue_head_t *mqp;
23767c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
23777c478bd9Sstevel@tonic-gate 	int signalled;
23787c478bd9Sstevel@tonic-gate 	int error;
23797c478bd9Sstevel@tonic-gate 
23807c478bd9Sstevel@tonic-gate 	/*
23817c478bd9Sstevel@tonic-gate 	 * Put ourself on the CV sleep queue, unlock the mutex, then
23827c478bd9Sstevel@tonic-gate 	 * park ourself and unpark a candidate lwp to grab the mutex.
23837c478bd9Sstevel@tonic-gate 	 * We must go onto the CV sleep queue before dropping the
23847c478bd9Sstevel@tonic-gate 	 * mutex in order to guarantee atomicity of the operation.
23857c478bd9Sstevel@tonic-gate 	 */
23867c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
23877c478bd9Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
23887c478bd9Sstevel@tonic-gate 	enqueue(qp, self, cvp, CV);
23897c478bd9Sstevel@tonic-gate 	cvp->cond_waiters_user = 1;
23907c478bd9Sstevel@tonic-gate 	self->ul_cvmutex = mp;
23917c478bd9Sstevel@tonic-gate 	self->ul_cv_wake = (tsp != NULL);
23927c478bd9Sstevel@tonic-gate 	self->ul_signalled = 0;
23937c478bd9Sstevel@tonic-gate 	lwpid = mutex_unlock_queue(mp);
23947c478bd9Sstevel@tonic-gate 	for (;;) {
23957c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 1);
23967c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
23977c478bd9Sstevel@tonic-gate 		if (lwpid != 0) {
23987c478bd9Sstevel@tonic-gate 			lwpid = preempt_unpark(self, lwpid);
23997c478bd9Sstevel@tonic-gate 			preempt(self);
24007c478bd9Sstevel@tonic-gate 		}
24017c478bd9Sstevel@tonic-gate 		/*
24027c478bd9Sstevel@tonic-gate 		 * We may have a deferred signal present,
24037c478bd9Sstevel@tonic-gate 		 * in which case we should return EINTR.
24047c478bd9Sstevel@tonic-gate 		 * Also, we may have received a SIGCANCEL; if so
24057c478bd9Sstevel@tonic-gate 		 * and we are cancelable we should return EINTR.
24067c478bd9Sstevel@tonic-gate 		 * We force an immediate EINTR return from
24077c478bd9Sstevel@tonic-gate 		 * __lwp_park() by turning our parking flag off.
24087c478bd9Sstevel@tonic-gate 		 */
24097c478bd9Sstevel@tonic-gate 		if (self->ul_cursig != 0 ||
24107c478bd9Sstevel@tonic-gate 		    (self->ul_cancelable && self->ul_cancel_pending))
24117c478bd9Sstevel@tonic-gate 			set_parking_flag(self, 0);
24127c478bd9Sstevel@tonic-gate 		/*
24137c478bd9Sstevel@tonic-gate 		 * __lwp_park() will return the residual time in tsp
24147c478bd9Sstevel@tonic-gate 		 * if we are unparked before the timeout expires.
24157c478bd9Sstevel@tonic-gate 		 */
24167c478bd9Sstevel@tonic-gate 		error = __lwp_park(tsp, lwpid);
24177c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 0);
24187c478bd9Sstevel@tonic-gate 		lwpid = 0;	/* unpark the other lwp only once */
24197c478bd9Sstevel@tonic-gate 		/*
24207c478bd9Sstevel@tonic-gate 		 * We were waked up by cond_signal(), cond_broadcast(),
24217c478bd9Sstevel@tonic-gate 		 * by an interrupt or timeout (EINTR or ETIME),
24227c478bd9Sstevel@tonic-gate 		 * or we may just have gotten a spurious wakeup.
24237c478bd9Sstevel@tonic-gate 		 */
24247c478bd9Sstevel@tonic-gate 		qp = queue_lock(cvp, CV);
24257c478bd9Sstevel@tonic-gate 		mqp = queue_lock(mp, MX);
24267c478bd9Sstevel@tonic-gate 		if (self->ul_sleepq == NULL)
24277c478bd9Sstevel@tonic-gate 			break;
24287c478bd9Sstevel@tonic-gate 		/*
24297c478bd9Sstevel@tonic-gate 		 * We are on either the condvar sleep queue or the
24307c478bd9Sstevel@tonic-gate 		 * mutex sleep queue.  If we are on the mutex sleep
24317c478bd9Sstevel@tonic-gate 		 * queue, continue sleeping.  If we are on the condvar
24327c478bd9Sstevel@tonic-gate 		 * sleep queue, break out of the sleep if we were
24337c478bd9Sstevel@tonic-gate 		 * interrupted or we timed out (EINTR or ETIME).
24347c478bd9Sstevel@tonic-gate 		 * Else this is a spurious wakeup; continue the loop.
24357c478bd9Sstevel@tonic-gate 		 */
24367c478bd9Sstevel@tonic-gate 		if (self->ul_sleepq == mqp)		/* mutex queue */
24377c478bd9Sstevel@tonic-gate 			tsp = NULL;
24387c478bd9Sstevel@tonic-gate 		else if (self->ul_sleepq == qp) {	/* condvar queue */
24397c478bd9Sstevel@tonic-gate 			if (error) {
24407c478bd9Sstevel@tonic-gate 				cvp->cond_waiters_user = dequeue_self(qp, cvp);
24417c478bd9Sstevel@tonic-gate 				break;
24427c478bd9Sstevel@tonic-gate 			}
24437c478bd9Sstevel@tonic-gate 			/*
24447c478bd9Sstevel@tonic-gate 			 * Else a spurious wakeup on the condvar queue.
24457c478bd9Sstevel@tonic-gate 			 * __lwp_park() has already adjusted the timeout.
24467c478bd9Sstevel@tonic-gate 			 */
24477c478bd9Sstevel@tonic-gate 		} else {
24487c478bd9Sstevel@tonic-gate 			thr_panic("cond_sleep_queue(): thread not on queue");
24497c478bd9Sstevel@tonic-gate 		}
24507c478bd9Sstevel@tonic-gate 		queue_unlock(mqp);
24517c478bd9Sstevel@tonic-gate 	}
24527c478bd9Sstevel@tonic-gate 
24537c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
24547c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_cvmutex == NULL && self->ul_cv_wake == 0);
24557c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
24567c478bd9Sstevel@tonic-gate 	    self->ul_wchan == NULL);
24577c478bd9Sstevel@tonic-gate 
24587c478bd9Sstevel@tonic-gate 	signalled = self->ul_signalled;
24597c478bd9Sstevel@tonic-gate 	self->ul_signalled = 0;
24607c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
24617c478bd9Sstevel@tonic-gate 	queue_unlock(mqp);
24627c478bd9Sstevel@tonic-gate 
24637c478bd9Sstevel@tonic-gate 	/*
24647c478bd9Sstevel@tonic-gate 	 * If we were concurrently cond_signal()d and any of:
24657c478bd9Sstevel@tonic-gate 	 * received a UNIX signal, were cancelled, or got a timeout,
24667c478bd9Sstevel@tonic-gate 	 * then perform another cond_signal() to avoid consuming it.
24677c478bd9Sstevel@tonic-gate 	 */
24687c478bd9Sstevel@tonic-gate 	if (error && signalled)
24697c478bd9Sstevel@tonic-gate 		(void) cond_signal_internal(cvp);
24707c478bd9Sstevel@tonic-gate 
24717c478bd9Sstevel@tonic-gate 	return (error);
24727c478bd9Sstevel@tonic-gate }
24737c478bd9Sstevel@tonic-gate 
24747c478bd9Sstevel@tonic-gate int
24757c478bd9Sstevel@tonic-gate cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp,
24767c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp)
24777c478bd9Sstevel@tonic-gate {
24787c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
24797c478bd9Sstevel@tonic-gate 	int error;
24807c478bd9Sstevel@tonic-gate 
24817c478bd9Sstevel@tonic-gate 	/*
24827c478bd9Sstevel@tonic-gate 	 * The old thread library was programmed to defer signals
24837c478bd9Sstevel@tonic-gate 	 * while in cond_wait() so that the associated mutex would
24847c478bd9Sstevel@tonic-gate 	 * be guaranteed to be held when the application signal
24857c478bd9Sstevel@tonic-gate 	 * handler was invoked.
24867c478bd9Sstevel@tonic-gate 	 *
24877c478bd9Sstevel@tonic-gate 	 * We do not behave this way by default; the state of the
24887c478bd9Sstevel@tonic-gate 	 * associated mutex in the signal handler is undefined.
24897c478bd9Sstevel@tonic-gate 	 *
24907c478bd9Sstevel@tonic-gate 	 * To accommodate applications that depend on the old
24917c478bd9Sstevel@tonic-gate 	 * behavior, the _THREAD_COND_WAIT_DEFER environment
24927c478bd9Sstevel@tonic-gate 	 * variable can be set to 1 and we will behave in the
24937c478bd9Sstevel@tonic-gate 	 * old way with respect to cond_wait().
24947c478bd9Sstevel@tonic-gate 	 */
24957c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
24967c478bd9Sstevel@tonic-gate 		sigoff(self);
24977c478bd9Sstevel@tonic-gate 
24987c478bd9Sstevel@tonic-gate 	error = cond_sleep_queue(cvp, mp, tsp);
24997c478bd9Sstevel@tonic-gate 
25007c478bd9Sstevel@tonic-gate 	/*
25017c478bd9Sstevel@tonic-gate 	 * Reacquire the mutex.
25027c478bd9Sstevel@tonic-gate 	 */
25037c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
25047c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
25057c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
25067c478bd9Sstevel@tonic-gate 	} else if (mutex_trylock_adaptive(mp) != 0) {
25077c478bd9Sstevel@tonic-gate 		(void) mutex_lock_queue(self, msp, mp, NULL);
25087c478bd9Sstevel@tonic-gate 	}
25097c478bd9Sstevel@tonic-gate 
25107c478bd9Sstevel@tonic-gate 	if (msp)
25117c478bd9Sstevel@tonic-gate 		record_begin_hold(msp);
25127c478bd9Sstevel@tonic-gate 
25137c478bd9Sstevel@tonic-gate 	/*
25147c478bd9Sstevel@tonic-gate 	 * Take any deferred signal now, after we have reacquired the mutex.
25157c478bd9Sstevel@tonic-gate 	 */
25167c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
25177c478bd9Sstevel@tonic-gate 		sigon(self);
25187c478bd9Sstevel@tonic-gate 
25197c478bd9Sstevel@tonic-gate 	return (error);
25207c478bd9Sstevel@tonic-gate }
25217c478bd9Sstevel@tonic-gate 
25227c478bd9Sstevel@tonic-gate /*
25237c478bd9Sstevel@tonic-gate  * cond_sleep_kernel(): utility function for cond_wait_kernel().
25247c478bd9Sstevel@tonic-gate  * See the comment ahead of cond_sleep_queue(), above.
25257c478bd9Sstevel@tonic-gate  */
25267c478bd9Sstevel@tonic-gate int
25277c478bd9Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
25287c478bd9Sstevel@tonic-gate {
25297c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
25307c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
25317c478bd9Sstevel@tonic-gate 	int error;
25327c478bd9Sstevel@tonic-gate 
25337c478bd9Sstevel@tonic-gate 	if (mtype & PTHREAD_PRIO_PROTECT) {
25347c478bd9Sstevel@tonic-gate 		if (_ceil_mylist_del(mp))
25357c478bd9Sstevel@tonic-gate 			_ceil_prio_waive();
25367c478bd9Sstevel@tonic-gate 	}
25377c478bd9Sstevel@tonic-gate 
25387c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
25397c478bd9Sstevel@tonic-gate 	self->ul_wchan = cvp;
25407c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
25417c478bd9Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
25427c478bd9Sstevel@tonic-gate 	if (mtype & PTHREAD_PRIO_INHERIT)
25437c478bd9Sstevel@tonic-gate 		mp->mutex_lockw = LOCKCLEAR;
25447c478bd9Sstevel@tonic-gate 	/*
25457c478bd9Sstevel@tonic-gate 	 * ___lwp_cond_wait() returns immediately with EINTR if
25467c478bd9Sstevel@tonic-gate 	 * set_parking_flag(self,0) is called on this lwp before it
25477c478bd9Sstevel@tonic-gate 	 * goes to sleep in the kernel.  sigacthandler() calls this
25487c478bd9Sstevel@tonic-gate 	 * when a deferred signal is noted.  This assures that we don't
25497c478bd9Sstevel@tonic-gate 	 * get stuck in ___lwp_cond_wait() with all signals blocked
25507c478bd9Sstevel@tonic-gate 	 * due to taking a deferred signal before going to sleep.
25517c478bd9Sstevel@tonic-gate 	 */
25527c478bd9Sstevel@tonic-gate 	set_parking_flag(self, 1);
25537c478bd9Sstevel@tonic-gate 	if (self->ul_cursig != 0 ||
25547c478bd9Sstevel@tonic-gate 	    (self->ul_cancelable && self->ul_cancel_pending))
25557c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 0);
25567c478bd9Sstevel@tonic-gate 	error = ___lwp_cond_wait(cvp, mp, tsp, 1);
25577c478bd9Sstevel@tonic-gate 	set_parking_flag(self, 0);
25587c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
25597c478bd9Sstevel@tonic-gate 	self->ul_wchan = NULL;
25607c478bd9Sstevel@tonic-gate 	return (error);
25617c478bd9Sstevel@tonic-gate }
25627c478bd9Sstevel@tonic-gate 
25637c478bd9Sstevel@tonic-gate int
25647c478bd9Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
25657c478bd9Sstevel@tonic-gate {
25667c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
25677c478bd9Sstevel@tonic-gate 	int error;
25687c478bd9Sstevel@tonic-gate 	int merror;
25697c478bd9Sstevel@tonic-gate 
25707c478bd9Sstevel@tonic-gate 	/*
25717c478bd9Sstevel@tonic-gate 	 * See the large comment in cond_wait_queue(), above.
25727c478bd9Sstevel@tonic-gate 	 */
25737c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
25747c478bd9Sstevel@tonic-gate 		sigoff(self);
25757c478bd9Sstevel@tonic-gate 
25767c478bd9Sstevel@tonic-gate 	error = cond_sleep_kernel(cvp, mp, tsp);
25777c478bd9Sstevel@tonic-gate 
25787c478bd9Sstevel@tonic-gate 	/*
25797c478bd9Sstevel@tonic-gate 	 * Override the return code from ___lwp_cond_wait()
25807c478bd9Sstevel@tonic-gate 	 * with any non-zero return code from mutex_lock().
25817c478bd9Sstevel@tonic-gate 	 * This addresses robust lock failures in particular;
25827c478bd9Sstevel@tonic-gate 	 * the caller must see the EOWNERDEAD or ENOTRECOVERABLE
25837c478bd9Sstevel@tonic-gate 	 * errors in order to take corrective action.
25847c478bd9Sstevel@tonic-gate 	 */
25857c478bd9Sstevel@tonic-gate 	if ((merror = _private_mutex_lock(mp)) != 0)
25867c478bd9Sstevel@tonic-gate 		error = merror;
25877c478bd9Sstevel@tonic-gate 
25887c478bd9Sstevel@tonic-gate 	/*
25897c478bd9Sstevel@tonic-gate 	 * Take any deferred signal now, after we have reacquired the mutex.
25907c478bd9Sstevel@tonic-gate 	 */
25917c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
25927c478bd9Sstevel@tonic-gate 		sigon(self);
25937c478bd9Sstevel@tonic-gate 
25947c478bd9Sstevel@tonic-gate 	return (error);
25957c478bd9Sstevel@tonic-gate }
25967c478bd9Sstevel@tonic-gate 
25977c478bd9Sstevel@tonic-gate /*
25987c478bd9Sstevel@tonic-gate  * Common code for _cond_wait() and _cond_timedwait()
25997c478bd9Sstevel@tonic-gate  */
26007c478bd9Sstevel@tonic-gate int
26017c478bd9Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
26027c478bd9Sstevel@tonic-gate {
26037c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
26047c478bd9Sstevel@tonic-gate 	hrtime_t begin_sleep = 0;
26057c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
26067c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
26077c478bd9Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
26087c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
26097c478bd9Sstevel@tonic-gate 	uint8_t rcount;
26107c478bd9Sstevel@tonic-gate 	int error = 0;
26117c478bd9Sstevel@tonic-gate 
26127c478bd9Sstevel@tonic-gate 	/*
26137c478bd9Sstevel@tonic-gate 	 * The SUSV3 Posix spec for pthread_cond_timedwait() states:
26147c478bd9Sstevel@tonic-gate 	 *	Except in the case of [ETIMEDOUT], all these error checks
26157c478bd9Sstevel@tonic-gate 	 *	shall act as if they were performed immediately at the
26167c478bd9Sstevel@tonic-gate 	 *	beginning of processing for the function and shall cause
26177c478bd9Sstevel@tonic-gate 	 *	an error return, in effect, prior to modifying the state
26187c478bd9Sstevel@tonic-gate 	 *	of the mutex specified by mutex or the condition variable
26197c478bd9Sstevel@tonic-gate 	 *	specified by cond.
26207c478bd9Sstevel@tonic-gate 	 * Therefore, we must return EINVAL now if the timout is invalid.
26217c478bd9Sstevel@tonic-gate 	 */
26227c478bd9Sstevel@tonic-gate 	if (tsp != NULL &&
26237c478bd9Sstevel@tonic-gate 	    (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC))
26247c478bd9Sstevel@tonic-gate 		return (EINVAL);
26257c478bd9Sstevel@tonic-gate 
26267c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
26277c478bd9Sstevel@tonic-gate 		self->ul_sp = stkptr();
26287c478bd9Sstevel@tonic-gate 		self->ul_wchan = cvp;
26297c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
26307c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = cvp;
26317c478bd9Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
26327c478bd9Sstevel@tonic-gate 		self->ul_sp = 0;
26337c478bd9Sstevel@tonic-gate 	}
26347c478bd9Sstevel@tonic-gate 	if (csp) {
26357c478bd9Sstevel@tonic-gate 		if (tsp)
26367c478bd9Sstevel@tonic-gate 			tdb_incr(csp->cond_timedwait);
26377c478bd9Sstevel@tonic-gate 		else
26387c478bd9Sstevel@tonic-gate 			tdb_incr(csp->cond_wait);
26397c478bd9Sstevel@tonic-gate 	}
26407c478bd9Sstevel@tonic-gate 	if (msp)
26417c478bd9Sstevel@tonic-gate 		begin_sleep = record_hold_time(msp);
26427c478bd9Sstevel@tonic-gate 	else if (csp)
26437c478bd9Sstevel@tonic-gate 		begin_sleep = gethrtime();
26447c478bd9Sstevel@tonic-gate 
26457c478bd9Sstevel@tonic-gate 	if (self->ul_error_detection) {
26467c478bd9Sstevel@tonic-gate 		if (!mutex_is_held(mp))
26477c478bd9Sstevel@tonic-gate 			lock_error(mp, "cond_wait", cvp, NULL);
26487c478bd9Sstevel@tonic-gate 		if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0)
26497c478bd9Sstevel@tonic-gate 			lock_error(mp, "recursive mutex in cond_wait",
26507c478bd9Sstevel@tonic-gate 				cvp, NULL);
26517c478bd9Sstevel@tonic-gate 		if (cvp->cond_type & USYNC_PROCESS) {
26527c478bd9Sstevel@tonic-gate 			if (!(mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)))
26537c478bd9Sstevel@tonic-gate 				lock_error(mp, "cond_wait", cvp,
26547c478bd9Sstevel@tonic-gate 					"condvar process-shared, "
26557c478bd9Sstevel@tonic-gate 					"mutex process-private");
26567c478bd9Sstevel@tonic-gate 		} else {
26577c478bd9Sstevel@tonic-gate 			if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))
26587c478bd9Sstevel@tonic-gate 				lock_error(mp, "cond_wait", cvp,
26597c478bd9Sstevel@tonic-gate 					"condvar process-private, "
26607c478bd9Sstevel@tonic-gate 					"mutex process-shared");
26617c478bd9Sstevel@tonic-gate 		}
26627c478bd9Sstevel@tonic-gate 	}
26637c478bd9Sstevel@tonic-gate 
26647c478bd9Sstevel@tonic-gate 	/*
26657c478bd9Sstevel@tonic-gate 	 * We deal with recursive mutexes by completely
26667c478bd9Sstevel@tonic-gate 	 * dropping the lock and restoring the recursion
26677c478bd9Sstevel@tonic-gate 	 * count after waking up.  This is arguably wrong,
26687c478bd9Sstevel@tonic-gate 	 * but it obeys the principle of least astonishment.
26697c478bd9Sstevel@tonic-gate 	 */
26707c478bd9Sstevel@tonic-gate 	rcount = mp->mutex_rcount;
26717c478bd9Sstevel@tonic-gate 	mp->mutex_rcount = 0;
26727c478bd9Sstevel@tonic-gate 	if ((mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST |
26737c478bd9Sstevel@tonic-gate 	    PTHREAD_PRIO_INHERIT | PTHREAD_PRIO_PROTECT)) |
26747c478bd9Sstevel@tonic-gate 	    (cvp->cond_type & USYNC_PROCESS))
26757c478bd9Sstevel@tonic-gate 		error = cond_wait_kernel(cvp, mp, tsp);
26767c478bd9Sstevel@tonic-gate 	else
26777c478bd9Sstevel@tonic-gate 		error = cond_wait_queue(cvp, mp, tsp, msp);
26787c478bd9Sstevel@tonic-gate 	mp->mutex_rcount = rcount;
26797c478bd9Sstevel@tonic-gate 
26807c478bd9Sstevel@tonic-gate 	if (csp) {
26817c478bd9Sstevel@tonic-gate 		hrtime_t lapse = gethrtime() - begin_sleep;
26827c478bd9Sstevel@tonic-gate 		if (tsp == NULL)
26837c478bd9Sstevel@tonic-gate 			csp->cond_wait_sleep_time += lapse;
26847c478bd9Sstevel@tonic-gate 		else {
26857c478bd9Sstevel@tonic-gate 			csp->cond_timedwait_sleep_time += lapse;
26867c478bd9Sstevel@tonic-gate 			if (error == ETIME)
26877c478bd9Sstevel@tonic-gate 				tdb_incr(csp->cond_timedwait_timeout);
26887c478bd9Sstevel@tonic-gate 		}
26897c478bd9Sstevel@tonic-gate 	}
26907c478bd9Sstevel@tonic-gate 	return (error);
26917c478bd9Sstevel@tonic-gate }
26927c478bd9Sstevel@tonic-gate 
26937c478bd9Sstevel@tonic-gate /*
26947c478bd9Sstevel@tonic-gate  * cond_wait() is a cancellation point but _cond_wait() is not.
26957c478bd9Sstevel@tonic-gate  * System libraries call the non-cancellation version.
26967c478bd9Sstevel@tonic-gate  * It is expected that only applications call the cancellation version.
26977c478bd9Sstevel@tonic-gate  */
26987c478bd9Sstevel@tonic-gate int
26997c478bd9Sstevel@tonic-gate _cond_wait(cond_t *cvp, mutex_t *mp)
27007c478bd9Sstevel@tonic-gate {
27017c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
27027c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
27037c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
27047c478bd9Sstevel@tonic-gate 
27057c478bd9Sstevel@tonic-gate 	/*
27067c478bd9Sstevel@tonic-gate 	 * Optimize the common case of USYNC_THREAD plus
27077c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics, and no event tracing.
27087c478bd9Sstevel@tonic-gate 	 */
27097c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
27107c478bd9Sstevel@tonic-gate 	    (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted |
27117c478bd9Sstevel@tonic-gate 	    self->ul_td_events_enable |
27127c478bd9Sstevel@tonic-gate 	    udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0)
27137c478bd9Sstevel@tonic-gate 		return (cond_wait_queue(cvp, mp, NULL, NULL));
27147c478bd9Sstevel@tonic-gate 
27157c478bd9Sstevel@tonic-gate 	/*
27167c478bd9Sstevel@tonic-gate 	 * Else do it the long way.
27177c478bd9Sstevel@tonic-gate 	 */
27187c478bd9Sstevel@tonic-gate 	return (cond_wait_common(cvp, mp, NULL));
27197c478bd9Sstevel@tonic-gate }
27207c478bd9Sstevel@tonic-gate 
27217c478bd9Sstevel@tonic-gate int
27227c478bd9Sstevel@tonic-gate cond_wait(cond_t *cvp, mutex_t *mp)
27237c478bd9Sstevel@tonic-gate {
27247c478bd9Sstevel@tonic-gate 	int error;
27257c478bd9Sstevel@tonic-gate 
27267c478bd9Sstevel@tonic-gate 	_cancelon();
27277c478bd9Sstevel@tonic-gate 	error = _cond_wait(cvp, mp);
27287c478bd9Sstevel@tonic-gate 	if (error == EINTR)
27297c478bd9Sstevel@tonic-gate 		_canceloff();
27307c478bd9Sstevel@tonic-gate 	else
27317c478bd9Sstevel@tonic-gate 		_canceloff_nocancel();
27327c478bd9Sstevel@tonic-gate 	return (error);
27337c478bd9Sstevel@tonic-gate }
27347c478bd9Sstevel@tonic-gate 
27357c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_wait = _pthread_cond_wait
27367c478bd9Sstevel@tonic-gate int
27377c478bd9Sstevel@tonic-gate _pthread_cond_wait(cond_t *cvp, mutex_t *mp)
27387c478bd9Sstevel@tonic-gate {
27397c478bd9Sstevel@tonic-gate 	int error;
27407c478bd9Sstevel@tonic-gate 
27417c478bd9Sstevel@tonic-gate 	error = cond_wait(cvp, mp);
27427c478bd9Sstevel@tonic-gate 	return ((error == EINTR)? 0 : error);
27437c478bd9Sstevel@tonic-gate }
27447c478bd9Sstevel@tonic-gate 
27457c478bd9Sstevel@tonic-gate /*
27467c478bd9Sstevel@tonic-gate  * cond_timedwait() is a cancellation point but _cond_timedwait() is not.
27477c478bd9Sstevel@tonic-gate  * System libraries call the non-cancellation version.
27487c478bd9Sstevel@tonic-gate  * It is expected that only applications call the cancellation version.
27497c478bd9Sstevel@tonic-gate  */
27507c478bd9Sstevel@tonic-gate int
27517c478bd9Sstevel@tonic-gate _cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
27527c478bd9Sstevel@tonic-gate {
27537c478bd9Sstevel@tonic-gate 	clockid_t clock_id = cvp->cond_clockid;
27547c478bd9Sstevel@tonic-gate 	timespec_t reltime;
27557c478bd9Sstevel@tonic-gate 	int error;
27567c478bd9Sstevel@tonic-gate 
27577c478bd9Sstevel@tonic-gate 	if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES)
27587c478bd9Sstevel@tonic-gate 		clock_id = CLOCK_REALTIME;
27597c478bd9Sstevel@tonic-gate 	abstime_to_reltime(clock_id, abstime, &reltime);
27607c478bd9Sstevel@tonic-gate 	error = cond_wait_common(cvp, mp, &reltime);
27617c478bd9Sstevel@tonic-gate 	if (error == ETIME && clock_id == CLOCK_HIGHRES) {
27627c478bd9Sstevel@tonic-gate 		/*
27637c478bd9Sstevel@tonic-gate 		 * Don't return ETIME if we didn't really get a timeout.
27647c478bd9Sstevel@tonic-gate 		 * This can happen if we return because someone resets
27657c478bd9Sstevel@tonic-gate 		 * the system clock.  Just return zero in this case,
27667c478bd9Sstevel@tonic-gate 		 * giving a spurious wakeup but not a timeout.
27677c478bd9Sstevel@tonic-gate 		 */
27687c478bd9Sstevel@tonic-gate 		if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC +
27697c478bd9Sstevel@tonic-gate 		    abstime->tv_nsec > gethrtime())
27707c478bd9Sstevel@tonic-gate 			error = 0;
27717c478bd9Sstevel@tonic-gate 	}
27727c478bd9Sstevel@tonic-gate 	return (error);
27737c478bd9Sstevel@tonic-gate }
27747c478bd9Sstevel@tonic-gate 
27757c478bd9Sstevel@tonic-gate int
27767c478bd9Sstevel@tonic-gate cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
27777c478bd9Sstevel@tonic-gate {
27787c478bd9Sstevel@tonic-gate 	int error;
27797c478bd9Sstevel@tonic-gate 
27807c478bd9Sstevel@tonic-gate 	_cancelon();
27817c478bd9Sstevel@tonic-gate 	error = _cond_timedwait(cvp, mp, abstime);
27827c478bd9Sstevel@tonic-gate 	if (error == EINTR)
27837c478bd9Sstevel@tonic-gate 		_canceloff();
27847c478bd9Sstevel@tonic-gate 	else
27857c478bd9Sstevel@tonic-gate 		_canceloff_nocancel();
27867c478bd9Sstevel@tonic-gate 	return (error);
27877c478bd9Sstevel@tonic-gate }
27887c478bd9Sstevel@tonic-gate 
27897c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_timedwait = _pthread_cond_timedwait
27907c478bd9Sstevel@tonic-gate int
27917c478bd9Sstevel@tonic-gate _pthread_cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
27927c478bd9Sstevel@tonic-gate {
27937c478bd9Sstevel@tonic-gate 	int error;
27947c478bd9Sstevel@tonic-gate 
27957c478bd9Sstevel@tonic-gate 	error = cond_timedwait(cvp, mp, abstime);
27967c478bd9Sstevel@tonic-gate 	if (error == ETIME)
27977c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
27987c478bd9Sstevel@tonic-gate 	else if (error == EINTR)
27997c478bd9Sstevel@tonic-gate 		error = 0;
28007c478bd9Sstevel@tonic-gate 	return (error);
28017c478bd9Sstevel@tonic-gate }
28027c478bd9Sstevel@tonic-gate 
28037c478bd9Sstevel@tonic-gate /*
28047c478bd9Sstevel@tonic-gate  * cond_reltimedwait() is a cancellation point but _cond_reltimedwait()
28057c478bd9Sstevel@tonic-gate  * is not.  System libraries call the non-cancellation version.
28067c478bd9Sstevel@tonic-gate  * It is expected that only applications call the cancellation version.
28077c478bd9Sstevel@tonic-gate  */
28087c478bd9Sstevel@tonic-gate int
28097c478bd9Sstevel@tonic-gate _cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
28107c478bd9Sstevel@tonic-gate {
28117c478bd9Sstevel@tonic-gate 	timespec_t tslocal = *reltime;
28127c478bd9Sstevel@tonic-gate 
28137c478bd9Sstevel@tonic-gate 	return (cond_wait_common(cvp, mp, &tslocal));
28147c478bd9Sstevel@tonic-gate }
28157c478bd9Sstevel@tonic-gate 
28167c478bd9Sstevel@tonic-gate #pragma weak cond_reltimedwait = _cond_reltimedwait_cancel
28177c478bd9Sstevel@tonic-gate int
28187c478bd9Sstevel@tonic-gate _cond_reltimedwait_cancel(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
28197c478bd9Sstevel@tonic-gate {
28207c478bd9Sstevel@tonic-gate 	int error;
28217c478bd9Sstevel@tonic-gate 
28227c478bd9Sstevel@tonic-gate 	_cancelon();
28237c478bd9Sstevel@tonic-gate 	error = _cond_reltimedwait(cvp, mp, reltime);
28247c478bd9Sstevel@tonic-gate 	if (error == EINTR)
28257c478bd9Sstevel@tonic-gate 		_canceloff();
28267c478bd9Sstevel@tonic-gate 	else
28277c478bd9Sstevel@tonic-gate 		_canceloff_nocancel();
28287c478bd9Sstevel@tonic-gate 	return (error);
28297c478bd9Sstevel@tonic-gate }
28307c478bd9Sstevel@tonic-gate 
28317c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_reltimedwait_np = _pthread_cond_reltimedwait_np
28327c478bd9Sstevel@tonic-gate int
28337c478bd9Sstevel@tonic-gate _pthread_cond_reltimedwait_np(cond_t *cvp, mutex_t *mp,
28347c478bd9Sstevel@tonic-gate 	const timespec_t *reltime)
28357c478bd9Sstevel@tonic-gate {
28367c478bd9Sstevel@tonic-gate 	int error;
28377c478bd9Sstevel@tonic-gate 
28387c478bd9Sstevel@tonic-gate 	error = _cond_reltimedwait_cancel(cvp, mp, reltime);
28397c478bd9Sstevel@tonic-gate 	if (error == ETIME)
28407c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
28417c478bd9Sstevel@tonic-gate 	else if (error == EINTR)
28427c478bd9Sstevel@tonic-gate 		error = 0;
28437c478bd9Sstevel@tonic-gate 	return (error);
28447c478bd9Sstevel@tonic-gate }
28457c478bd9Sstevel@tonic-gate 
28467c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_signal = cond_signal_internal
28477c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_signal = cond_signal_internal
28487c478bd9Sstevel@tonic-gate #pragma weak cond_signal = cond_signal_internal
28497c478bd9Sstevel@tonic-gate #pragma weak _cond_signal = cond_signal_internal
28507c478bd9Sstevel@tonic-gate int
28517c478bd9Sstevel@tonic-gate cond_signal_internal(cond_t *cvp)
28527c478bd9Sstevel@tonic-gate {
28537c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
28547c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
28557c478bd9Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
28567c478bd9Sstevel@tonic-gate 	int error = 0;
28577c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
28587c478bd9Sstevel@tonic-gate 	mutex_t *mp;
28597c478bd9Sstevel@tonic-gate 	queue_head_t *mqp;
28607c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
28617c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
28627c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
28637c478bd9Sstevel@tonic-gate 	ulwp_t *next;
28647c478bd9Sstevel@tonic-gate 	ulwp_t **suspp = NULL;
28657c478bd9Sstevel@tonic-gate 	ulwp_t *susprev;
28667c478bd9Sstevel@tonic-gate 
28677c478bd9Sstevel@tonic-gate 	if (csp)
28687c478bd9Sstevel@tonic-gate 		tdb_incr(csp->cond_signal);
28697c478bd9Sstevel@tonic-gate 
28707c478bd9Sstevel@tonic-gate 	if (cvp->cond_waiters_kernel)	/* someone sleeping in the kernel? */
28717c478bd9Sstevel@tonic-gate 		error = __lwp_cond_signal(cvp);
28727c478bd9Sstevel@tonic-gate 
28737c478bd9Sstevel@tonic-gate 	if (!cvp->cond_waiters_user)	/* no one sleeping at user-level */
28747c478bd9Sstevel@tonic-gate 		return (error);
28757c478bd9Sstevel@tonic-gate 
28767c478bd9Sstevel@tonic-gate 	/*
28777c478bd9Sstevel@tonic-gate 	 * Move someone from the condvar sleep queue to the mutex sleep
28787c478bd9Sstevel@tonic-gate 	 * queue for the mutex that he will acquire on being waked up.
28797c478bd9Sstevel@tonic-gate 	 * We can do this only if we own the mutex he will acquire.
28807c478bd9Sstevel@tonic-gate 	 * If we do not own the mutex, or if his ul_cv_wake flag
28817c478bd9Sstevel@tonic-gate 	 * is set, just dequeue and unpark him.
28827c478bd9Sstevel@tonic-gate 	 */
28837c478bd9Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
28847c478bd9Sstevel@tonic-gate 	for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL;
28857c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
28867c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == cvp) {
28877c478bd9Sstevel@tonic-gate 			if (!ulwp->ul_stop)
28887c478bd9Sstevel@tonic-gate 				break;
28897c478bd9Sstevel@tonic-gate 			/*
28907c478bd9Sstevel@tonic-gate 			 * Try not to dequeue a suspended thread.
28917c478bd9Sstevel@tonic-gate 			 * This mimics the old libthread's behavior.
28927c478bd9Sstevel@tonic-gate 			 */
28937c478bd9Sstevel@tonic-gate 			if (suspp == NULL) {
28947c478bd9Sstevel@tonic-gate 				suspp = ulwpp;
28957c478bd9Sstevel@tonic-gate 				susprev = prev;
28967c478bd9Sstevel@tonic-gate 			}
28977c478bd9Sstevel@tonic-gate 		}
28987c478bd9Sstevel@tonic-gate 	}
28997c478bd9Sstevel@tonic-gate 	if (ulwp == NULL && suspp != NULL) {
29007c478bd9Sstevel@tonic-gate 		ulwp = *(ulwpp = suspp);
29017c478bd9Sstevel@tonic-gate 		prev = susprev;
29027c478bd9Sstevel@tonic-gate 		suspp = NULL;
29037c478bd9Sstevel@tonic-gate 	}
29047c478bd9Sstevel@tonic-gate 	if (ulwp == NULL) {	/* no one on the sleep queue */
29057c478bd9Sstevel@tonic-gate 		cvp->cond_waiters_user = 0;
29067c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
29077c478bd9Sstevel@tonic-gate 		return (error);
29087c478bd9Sstevel@tonic-gate 	}
29097c478bd9Sstevel@tonic-gate 	/*
29107c478bd9Sstevel@tonic-gate 	 * Scan the remainder of the CV queue for another waiter.
29117c478bd9Sstevel@tonic-gate 	 */
29127c478bd9Sstevel@tonic-gate 	if (suspp != NULL) {
29137c478bd9Sstevel@tonic-gate 		next = *suspp;
29147c478bd9Sstevel@tonic-gate 	} else {
29157c478bd9Sstevel@tonic-gate 		for (next = ulwp->ul_link; next != NULL; next = next->ul_link)
29167c478bd9Sstevel@tonic-gate 			if (next->ul_wchan == cvp)
29177c478bd9Sstevel@tonic-gate 				break;
29187c478bd9Sstevel@tonic-gate 	}
29197c478bd9Sstevel@tonic-gate 	if (next == NULL)
29207c478bd9Sstevel@tonic-gate 		cvp->cond_waiters_user = 0;
29217c478bd9Sstevel@tonic-gate 
29227c478bd9Sstevel@tonic-gate 	/*
29237c478bd9Sstevel@tonic-gate 	 * Inform the thread that he was the recipient of a cond_signal().
29247c478bd9Sstevel@tonic-gate 	 * This lets him deal with cond_signal() and, concurrently,
29257c478bd9Sstevel@tonic-gate 	 * one or more of a cancellation, a UNIX signal, or a timeout.
29267c478bd9Sstevel@tonic-gate 	 * These latter conditions must not consume a cond_signal().
29277c478bd9Sstevel@tonic-gate 	 */
29287c478bd9Sstevel@tonic-gate 	ulwp->ul_signalled = 1;
29297c478bd9Sstevel@tonic-gate 
29307c478bd9Sstevel@tonic-gate 	/*
29317c478bd9Sstevel@tonic-gate 	 * Dequeue the waiter but leave his ul_sleepq non-NULL
29327c478bd9Sstevel@tonic-gate 	 * while we move him to the mutex queue so that he can
29337c478bd9Sstevel@tonic-gate 	 * deal properly with spurious wakeups.
29347c478bd9Sstevel@tonic-gate 	 */
29357c478bd9Sstevel@tonic-gate 	*ulwpp = ulwp->ul_link;
29367c478bd9Sstevel@tonic-gate 	if (qp->qh_tail == ulwp)
29377c478bd9Sstevel@tonic-gate 		qp->qh_tail = prev;
29387c478bd9Sstevel@tonic-gate 	qp->qh_qlen--;
29397c478bd9Sstevel@tonic-gate 	ulwp->ul_link = NULL;
29407c478bd9Sstevel@tonic-gate 
29417c478bd9Sstevel@tonic-gate 	mp = ulwp->ul_cvmutex;		/* the mutex he will acquire */
29427c478bd9Sstevel@tonic-gate 	ulwp->ul_cvmutex = NULL;
29437c478bd9Sstevel@tonic-gate 	ASSERT(mp != NULL);
29447c478bd9Sstevel@tonic-gate 
29457c478bd9Sstevel@tonic-gate 	if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
29467c478bd9Sstevel@tonic-gate 		lwpid_t lwpid = ulwp->ul_lwpid;
29477c478bd9Sstevel@tonic-gate 
29487c478bd9Sstevel@tonic-gate 		no_preempt(self);
29497c478bd9Sstevel@tonic-gate 		ulwp->ul_sleepq = NULL;
29507c478bd9Sstevel@tonic-gate 		ulwp->ul_wchan = NULL;
29517c478bd9Sstevel@tonic-gate 		ulwp->ul_cv_wake = 0;
29527c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
29537c478bd9Sstevel@tonic-gate 		(void) __lwp_unpark(lwpid);
29547c478bd9Sstevel@tonic-gate 		preempt(self);
29557c478bd9Sstevel@tonic-gate 	} else {
29567c478bd9Sstevel@tonic-gate 		mqp = queue_lock(mp, MX);
29577c478bd9Sstevel@tonic-gate 		enqueue(mqp, ulwp, mp, MX);
29587c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = 1;
29597c478bd9Sstevel@tonic-gate 		queue_unlock(mqp);
29607c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
29617c478bd9Sstevel@tonic-gate 	}
29627c478bd9Sstevel@tonic-gate 
29637c478bd9Sstevel@tonic-gate 	return (error);
29647c478bd9Sstevel@tonic-gate }
29657c478bd9Sstevel@tonic-gate 
29667c478bd9Sstevel@tonic-gate #define	MAXLWPS	128	/* max remembered lwpids before overflow */
29677c478bd9Sstevel@tonic-gate #define	NEWLWPS	2048	/* max remembered lwpids at first overflow */
29687c478bd9Sstevel@tonic-gate 
29697c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_broadcast = cond_broadcast_internal
29707c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_broadcast = cond_broadcast_internal
29717c478bd9Sstevel@tonic-gate #pragma weak cond_broadcast = cond_broadcast_internal
29727c478bd9Sstevel@tonic-gate #pragma weak _cond_broadcast = cond_broadcast_internal
29737c478bd9Sstevel@tonic-gate int
29747c478bd9Sstevel@tonic-gate cond_broadcast_internal(cond_t *cvp)
29757c478bd9Sstevel@tonic-gate {
29767c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
29777c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
29787c478bd9Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
29797c478bd9Sstevel@tonic-gate 	int error = 0;
29807c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
29817c478bd9Sstevel@tonic-gate 	mutex_t *mp;
29827c478bd9Sstevel@tonic-gate 	queue_head_t *mqp;
29837c478bd9Sstevel@tonic-gate 	mutex_t *mp_cache = NULL;
29847c478bd9Sstevel@tonic-gate 	queue_head_t *mqp_cache = NULL;
29857c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
29867c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
29877c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
29887c478bd9Sstevel@tonic-gate 	lwpid_t buffer[MAXLWPS];
29897c478bd9Sstevel@tonic-gate 	lwpid_t *lwpid = buffer;
29907c478bd9Sstevel@tonic-gate 	int nlwpid = 0;
29917c478bd9Sstevel@tonic-gate 	int maxlwps = MAXLWPS;
29927c478bd9Sstevel@tonic-gate 
29937c478bd9Sstevel@tonic-gate 	if (csp)
29947c478bd9Sstevel@tonic-gate 		tdb_incr(csp->cond_broadcast);
29957c478bd9Sstevel@tonic-gate 
29967c478bd9Sstevel@tonic-gate 	if (cvp->cond_waiters_kernel)	/* someone sleeping in the kernel? */
29977c478bd9Sstevel@tonic-gate 		error = __lwp_cond_broadcast(cvp);
29987c478bd9Sstevel@tonic-gate 
29997c478bd9Sstevel@tonic-gate 	if (!cvp->cond_waiters_user)	/* no one sleeping at user-level */
30007c478bd9Sstevel@tonic-gate 		return (error);
30017c478bd9Sstevel@tonic-gate 
30027c478bd9Sstevel@tonic-gate 	/*
30037c478bd9Sstevel@tonic-gate 	 * Move everyone from the condvar sleep queue to the mutex sleep
30047c478bd9Sstevel@tonic-gate 	 * queue for the mutex that they will acquire on being waked up.
30057c478bd9Sstevel@tonic-gate 	 * We can do this only if we own the mutex they will acquire.
30067c478bd9Sstevel@tonic-gate 	 * If we do not own the mutex, or if their ul_cv_wake flag
30077c478bd9Sstevel@tonic-gate 	 * is set, just dequeue and unpark them.
30087c478bd9Sstevel@tonic-gate 	 *
30097c478bd9Sstevel@tonic-gate 	 * We keep track of lwpids that are to be unparked in lwpid[].
30107c478bd9Sstevel@tonic-gate 	 * __lwp_unpark_all() is called to unpark all of them after
30117c478bd9Sstevel@tonic-gate 	 * they have been removed from the sleep queue and the sleep
30127c478bd9Sstevel@tonic-gate 	 * queue lock has been dropped.  If we run out of space in our
30137c478bd9Sstevel@tonic-gate 	 * on-stack buffer, we need to allocate more but we can't call
30147c478bd9Sstevel@tonic-gate 	 * lmalloc() because we are holding a queue lock when the overflow
30157c478bd9Sstevel@tonic-gate 	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
30167c478bd9Sstevel@tonic-gate 	 * either because the application may have allocated a small stack
30177c478bd9Sstevel@tonic-gate 	 * and we don't want to overrun the stack.  So we use the mmap()
30187c478bd9Sstevel@tonic-gate 	 * system call directly since that path acquires no locks.
30197c478bd9Sstevel@tonic-gate 	 */
30207c478bd9Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
30217c478bd9Sstevel@tonic-gate 	cvp->cond_waiters_user = 0;
30227c478bd9Sstevel@tonic-gate 	ulwpp = &qp->qh_head;
30237c478bd9Sstevel@tonic-gate 	while ((ulwp = *ulwpp) != NULL) {
30247c478bd9Sstevel@tonic-gate 
30257c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan != cvp) {
30267c478bd9Sstevel@tonic-gate 			prev = ulwp;
30277c478bd9Sstevel@tonic-gate 			ulwpp = &ulwp->ul_link;
30287c478bd9Sstevel@tonic-gate 			continue;
30297c478bd9Sstevel@tonic-gate 		}
30307c478bd9Sstevel@tonic-gate 
30317c478bd9Sstevel@tonic-gate 		*ulwpp = ulwp->ul_link;
30327c478bd9Sstevel@tonic-gate 		if (qp->qh_tail == ulwp)
30337c478bd9Sstevel@tonic-gate 			qp->qh_tail = prev;
30347c478bd9Sstevel@tonic-gate 		qp->qh_qlen--;
30357c478bd9Sstevel@tonic-gate 		ulwp->ul_link = NULL;
30367c478bd9Sstevel@tonic-gate 
30377c478bd9Sstevel@tonic-gate 		mp = ulwp->ul_cvmutex;		/* his mutex */
30387c478bd9Sstevel@tonic-gate 		ulwp->ul_cvmutex = NULL;
30397c478bd9Sstevel@tonic-gate 		ASSERT(mp != NULL);
30407c478bd9Sstevel@tonic-gate 
30417c478bd9Sstevel@tonic-gate 		if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
30427c478bd9Sstevel@tonic-gate 			ulwp->ul_sleepq = NULL;
30437c478bd9Sstevel@tonic-gate 			ulwp->ul_wchan = NULL;
30447c478bd9Sstevel@tonic-gate 			ulwp->ul_cv_wake = 0;
30457c478bd9Sstevel@tonic-gate 			if (nlwpid == maxlwps) {
30467c478bd9Sstevel@tonic-gate 				/*
30477c478bd9Sstevel@tonic-gate 				 * Allocate NEWLWPS ids on the first overflow.
30487c478bd9Sstevel@tonic-gate 				 * Double the allocation each time after that.
30497c478bd9Sstevel@tonic-gate 				 */
30507c478bd9Sstevel@tonic-gate 				int newlwps = (lwpid == buffer)? NEWLWPS :
30517c478bd9Sstevel@tonic-gate 						2 * maxlwps;
30527c478bd9Sstevel@tonic-gate 				void *vaddr = _private_mmap(NULL,
30537c478bd9Sstevel@tonic-gate 					newlwps * sizeof (lwpid_t),
30547c478bd9Sstevel@tonic-gate 					PROT_READ|PROT_WRITE,
30557c478bd9Sstevel@tonic-gate 					MAP_PRIVATE|MAP_ANON, -1, (off_t)0);
30567c478bd9Sstevel@tonic-gate 				if (vaddr == MAP_FAILED) {
30577c478bd9Sstevel@tonic-gate 					/*
30587c478bd9Sstevel@tonic-gate 					 * Let's hope this never happens.
30597c478bd9Sstevel@tonic-gate 					 * If it does, then we have a terrible
30607c478bd9Sstevel@tonic-gate 					 * thundering herd on our hands.
30617c478bd9Sstevel@tonic-gate 					 */
30627c478bd9Sstevel@tonic-gate 					(void) __lwp_unpark_all(lwpid, nlwpid);
30637c478bd9Sstevel@tonic-gate 					nlwpid = 0;
30647c478bd9Sstevel@tonic-gate 				} else {
30657c478bd9Sstevel@tonic-gate 					(void) _memcpy(vaddr, lwpid,
30667c478bd9Sstevel@tonic-gate 						maxlwps * sizeof (lwpid_t));
30677c478bd9Sstevel@tonic-gate 					if (lwpid != buffer)
30687c478bd9Sstevel@tonic-gate 						(void) _private_munmap(lwpid,
30697c478bd9Sstevel@tonic-gate 						    maxlwps * sizeof (lwpid_t));
30707c478bd9Sstevel@tonic-gate 					lwpid = vaddr;
30717c478bd9Sstevel@tonic-gate 					maxlwps = newlwps;
30727c478bd9Sstevel@tonic-gate 				}
30737c478bd9Sstevel@tonic-gate 			}
30747c478bd9Sstevel@tonic-gate 			lwpid[nlwpid++] = ulwp->ul_lwpid;
30757c478bd9Sstevel@tonic-gate 		} else {
30767c478bd9Sstevel@tonic-gate 			if (mp != mp_cache) {
30777c478bd9Sstevel@tonic-gate 				if (mqp_cache != NULL)
30787c478bd9Sstevel@tonic-gate 					queue_unlock(mqp_cache);
30797c478bd9Sstevel@tonic-gate 				mqp_cache = queue_lock(mp, MX);
30807c478bd9Sstevel@tonic-gate 				mp_cache = mp;
30817c478bd9Sstevel@tonic-gate 			}
30827c478bd9Sstevel@tonic-gate 			mqp = mqp_cache;
30837c478bd9Sstevel@tonic-gate 			enqueue(mqp, ulwp, mp, MX);
30847c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = 1;
30857c478bd9Sstevel@tonic-gate 		}
30867c478bd9Sstevel@tonic-gate 	}
30877c478bd9Sstevel@tonic-gate 	if (mqp_cache != NULL)
30887c478bd9Sstevel@tonic-gate 		queue_unlock(mqp_cache);
30897c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
30907c478bd9Sstevel@tonic-gate 	if (nlwpid) {
30917c478bd9Sstevel@tonic-gate 		if (nlwpid == 1)
30927c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid[0]);
30937c478bd9Sstevel@tonic-gate 		else
30947c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark_all(lwpid, nlwpid);
30957c478bd9Sstevel@tonic-gate 	}
30967c478bd9Sstevel@tonic-gate 	if (lwpid != buffer)
30977c478bd9Sstevel@tonic-gate 		(void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t));
30987c478bd9Sstevel@tonic-gate 
30997c478bd9Sstevel@tonic-gate 	return (error);
31007c478bd9Sstevel@tonic-gate }
31017c478bd9Sstevel@tonic-gate 
31027c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_destroy = _cond_destroy
31037c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_destroy = _cond_destroy
31047c478bd9Sstevel@tonic-gate #pragma weak cond_destroy = _cond_destroy
31057c478bd9Sstevel@tonic-gate int
31067c478bd9Sstevel@tonic-gate _cond_destroy(cond_t *cvp)
31077c478bd9Sstevel@tonic-gate {
31087c478bd9Sstevel@tonic-gate 	cvp->cond_magic = 0;
31097c478bd9Sstevel@tonic-gate 	tdb_sync_obj_deregister(cvp);
31107c478bd9Sstevel@tonic-gate 	return (0);
31117c478bd9Sstevel@tonic-gate }
31127c478bd9Sstevel@tonic-gate 
31137c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG)
31147c478bd9Sstevel@tonic-gate void
31157c478bd9Sstevel@tonic-gate assert_no_libc_locks_held(void)
31167c478bd9Sstevel@tonic-gate {
31177c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
31187c478bd9Sstevel@tonic-gate }
31197c478bd9Sstevel@tonic-gate #endif
31207c478bd9Sstevel@tonic-gate 
31217c478bd9Sstevel@tonic-gate /* protected by link_lock */
31227c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin;
31237c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin2;
31247c478bd9Sstevel@tonic-gate uint64_t spin_lock_sleep;
31257c478bd9Sstevel@tonic-gate uint64_t spin_lock_wakeup;
31267c478bd9Sstevel@tonic-gate 
31277c478bd9Sstevel@tonic-gate /*
31287c478bd9Sstevel@tonic-gate  * Record spin lock statistics.
31297c478bd9Sstevel@tonic-gate  * Called by a thread exiting itself in thrp_exit().
31307c478bd9Sstevel@tonic-gate  * Also called via atexit() from the thread calling
31317c478bd9Sstevel@tonic-gate  * exit() to do all the other threads as well.
31327c478bd9Sstevel@tonic-gate  */
31337c478bd9Sstevel@tonic-gate void
31347c478bd9Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp)
31357c478bd9Sstevel@tonic-gate {
31367c478bd9Sstevel@tonic-gate 	spin_lock_spin += ulwp->ul_spin_lock_spin;
31377c478bd9Sstevel@tonic-gate 	spin_lock_spin2 += ulwp->ul_spin_lock_spin2;
31387c478bd9Sstevel@tonic-gate 	spin_lock_sleep += ulwp->ul_spin_lock_sleep;
31397c478bd9Sstevel@tonic-gate 	spin_lock_wakeup += ulwp->ul_spin_lock_wakeup;
31407c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_spin = 0;
31417c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_spin2 = 0;
31427c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_sleep = 0;
31437c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_wakeup = 0;
31447c478bd9Sstevel@tonic-gate }
31457c478bd9Sstevel@tonic-gate 
31467c478bd9Sstevel@tonic-gate /*
31477c478bd9Sstevel@tonic-gate  * atexit function:  dump the queue statistics to stderr.
31487c478bd9Sstevel@tonic-gate  */
3149*e8031f0aSraf #if !defined(__lint)
3150*e8031f0aSraf #define	fprintf	_fprintf
3151*e8031f0aSraf #endif
31527c478bd9Sstevel@tonic-gate #include <stdio.h>
31537c478bd9Sstevel@tonic-gate void
31547c478bd9Sstevel@tonic-gate dump_queue_statistics(void)
31557c478bd9Sstevel@tonic-gate {
31567c478bd9Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
31577c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
31587c478bd9Sstevel@tonic-gate 	int qn;
31597c478bd9Sstevel@tonic-gate 	uint64_t spin_lock_total = 0;
31607c478bd9Sstevel@tonic-gate 
31617c478bd9Sstevel@tonic-gate 	if (udp->queue_head == NULL || thread_queue_dump == 0)
31627c478bd9Sstevel@tonic-gate 		return;
31637c478bd9Sstevel@tonic-gate 
31647c478bd9Sstevel@tonic-gate 	if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 ||
31657c478bd9Sstevel@tonic-gate 	    fprintf(stderr, "queue#   lockcount    max qlen\n") < 0)
31667c478bd9Sstevel@tonic-gate 		return;
31677c478bd9Sstevel@tonic-gate 	for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) {
31687c478bd9Sstevel@tonic-gate 		if (qp->qh_lockcount == 0)
31697c478bd9Sstevel@tonic-gate 			continue;
31707c478bd9Sstevel@tonic-gate 		spin_lock_total += qp->qh_lockcount;
31717c478bd9Sstevel@tonic-gate 		if (fprintf(stderr, "%5d %12llu%12u\n", qn,
31727c478bd9Sstevel@tonic-gate 			(u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0)
31737c478bd9Sstevel@tonic-gate 				return;
31747c478bd9Sstevel@tonic-gate 	}
31757c478bd9Sstevel@tonic-gate 
31767c478bd9Sstevel@tonic-gate 	if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 ||
31777c478bd9Sstevel@tonic-gate 	    fprintf(stderr, "queue#   lockcount    max qlen\n") < 0)
31787c478bd9Sstevel@tonic-gate 		return;
31797c478bd9Sstevel@tonic-gate 	for (qn = 0; qn < QHASHSIZE; qn++, qp++) {
31807c478bd9Sstevel@tonic-gate 		if (qp->qh_lockcount == 0)
31817c478bd9Sstevel@tonic-gate 			continue;
31827c478bd9Sstevel@tonic-gate 		spin_lock_total += qp->qh_lockcount;
31837c478bd9Sstevel@tonic-gate 		if (fprintf(stderr, "%5d %12llu%12u\n", qn,
31847c478bd9Sstevel@tonic-gate 			(u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0)
31857c478bd9Sstevel@tonic-gate 				return;
31867c478bd9Sstevel@tonic-gate 	}
31877c478bd9Sstevel@tonic-gate 
31887c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "\n  spin_lock_total  = %10llu\n",
31897c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_total);
31907c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_spin   = %10llu\n",
31917c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_spin);
31927c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_spin2  = %10llu\n",
31937c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_spin2);
31947c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_sleep  = %10llu\n",
31957c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_sleep);
31967c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_wakeup = %10llu\n",
31977c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_wakeup);
31987c478bd9Sstevel@tonic-gate }
3199