xref: /illumos-gate/usr/src/lib/libc/port/threads/synch.c (revision 2be60c5e8b4c40abfc7e3201cd6774851edf0ea0)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*2be60c5eSraf  * Common Development and Distribution License (the "License").
6*2be60c5eSraf  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
21e8031f0aSraf 
227c478bd9Sstevel@tonic-gate /*
23e8031f0aSraf  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
287c478bd9Sstevel@tonic-gate 
297c478bd9Sstevel@tonic-gate #include <sys/sdt.h>
307c478bd9Sstevel@tonic-gate 
317c478bd9Sstevel@tonic-gate #include "lint.h"
327c478bd9Sstevel@tonic-gate #include "thr_uberdata.h"
337c478bd9Sstevel@tonic-gate 
347c478bd9Sstevel@tonic-gate /*
357c478bd9Sstevel@tonic-gate  * This mutex is initialized to be held by lwp#1.
367c478bd9Sstevel@tonic-gate  * It is used to block a thread that has returned from a mutex_lock()
377c478bd9Sstevel@tonic-gate  * of a PTHREAD_PRIO_INHERIT mutex with an unrecoverable error.
387c478bd9Sstevel@tonic-gate  */
397c478bd9Sstevel@tonic-gate mutex_t	stall_mutex = DEFAULTMUTEX;
407c478bd9Sstevel@tonic-gate 
417c478bd9Sstevel@tonic-gate static int shared_mutex_held(mutex_t *);
427c478bd9Sstevel@tonic-gate 
437c478bd9Sstevel@tonic-gate /*
447c478bd9Sstevel@tonic-gate  * Lock statistics support functions.
457c478bd9Sstevel@tonic-gate  */
467c478bd9Sstevel@tonic-gate void
477c478bd9Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp)
487c478bd9Sstevel@tonic-gate {
497c478bd9Sstevel@tonic-gate 	tdb_incr(msp->mutex_lock);
507c478bd9Sstevel@tonic-gate 	msp->mutex_begin_hold = gethrtime();
517c478bd9Sstevel@tonic-gate }
527c478bd9Sstevel@tonic-gate 
537c478bd9Sstevel@tonic-gate hrtime_t
547c478bd9Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp)
557c478bd9Sstevel@tonic-gate {
567c478bd9Sstevel@tonic-gate 	hrtime_t now = gethrtime();
577c478bd9Sstevel@tonic-gate 
587c478bd9Sstevel@tonic-gate 	if (msp->mutex_begin_hold)
597c478bd9Sstevel@tonic-gate 		msp->mutex_hold_time += now - msp->mutex_begin_hold;
607c478bd9Sstevel@tonic-gate 	msp->mutex_begin_hold = 0;
617c478bd9Sstevel@tonic-gate 	return (now);
627c478bd9Sstevel@tonic-gate }
637c478bd9Sstevel@tonic-gate 
647c478bd9Sstevel@tonic-gate /*
657c478bd9Sstevel@tonic-gate  * Called once at library initialization.
667c478bd9Sstevel@tonic-gate  */
677c478bd9Sstevel@tonic-gate void
687c478bd9Sstevel@tonic-gate mutex_setup(void)
697c478bd9Sstevel@tonic-gate {
707c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&stall_mutex.mutex_lockw))
717c478bd9Sstevel@tonic-gate 		thr_panic("mutex_setup() cannot acquire stall_mutex");
727c478bd9Sstevel@tonic-gate 	stall_mutex.mutex_owner = (uintptr_t)curthread;
737c478bd9Sstevel@tonic-gate }
747c478bd9Sstevel@tonic-gate 
757c478bd9Sstevel@tonic-gate /*
767c478bd9Sstevel@tonic-gate  * The default spin counts of 1000 and 500 are experimentally determined.
777c478bd9Sstevel@tonic-gate  * On sun4u machines with any number of processors they could be raised
787c478bd9Sstevel@tonic-gate  * to 10,000 but that (experimentally) makes almost no difference.
797c478bd9Sstevel@tonic-gate  * The environment variables:
807c478bd9Sstevel@tonic-gate  *	_THREAD_ADAPTIVE_SPIN=count
817c478bd9Sstevel@tonic-gate  *	_THREAD_RELEASE_SPIN=count
827c478bd9Sstevel@tonic-gate  * can be used to override and set the counts in the range [0 .. 1,000,000].
837c478bd9Sstevel@tonic-gate  */
847c478bd9Sstevel@tonic-gate int	thread_adaptive_spin = 1000;
857c478bd9Sstevel@tonic-gate uint_t	thread_max_spinners = 100;
867c478bd9Sstevel@tonic-gate int	thread_release_spin = 500;
877c478bd9Sstevel@tonic-gate int	thread_queue_verify = 0;
887c478bd9Sstevel@tonic-gate static	int	ncpus;
897c478bd9Sstevel@tonic-gate 
907c478bd9Sstevel@tonic-gate /*
917c478bd9Sstevel@tonic-gate  * Distinguish spinning for queue locks from spinning for regular locks.
927c478bd9Sstevel@tonic-gate  * The environment variable:
937c478bd9Sstevel@tonic-gate  *	_THREAD_QUEUE_SPIN=count
947c478bd9Sstevel@tonic-gate  * can be used to override and set the count in the range [0 .. 1,000,000].
957c478bd9Sstevel@tonic-gate  * There is no release spin concept for queue locks.
967c478bd9Sstevel@tonic-gate  */
977c478bd9Sstevel@tonic-gate int	thread_queue_spin = 1000;
987c478bd9Sstevel@tonic-gate 
997c478bd9Sstevel@tonic-gate /*
1007c478bd9Sstevel@tonic-gate  * Use the otherwise-unused 'mutex_ownerpid' field of a USYNC_THREAD
1017c478bd9Sstevel@tonic-gate  * mutex to be a count of adaptive spins in progress.
1027c478bd9Sstevel@tonic-gate  */
1037c478bd9Sstevel@tonic-gate #define	mutex_spinners	mutex_ownerpid
1047c478bd9Sstevel@tonic-gate 
1057c478bd9Sstevel@tonic-gate void
1067c478bd9Sstevel@tonic-gate _mutex_set_typeattr(mutex_t *mp, int attr)
1077c478bd9Sstevel@tonic-gate {
1087c478bd9Sstevel@tonic-gate 	mp->mutex_type |= (uint8_t)attr;
1097c478bd9Sstevel@tonic-gate }
1107c478bd9Sstevel@tonic-gate 
1117c478bd9Sstevel@tonic-gate /*
1127c478bd9Sstevel@tonic-gate  * 'type' can be one of USYNC_THREAD or USYNC_PROCESS, possibly
1137c478bd9Sstevel@tonic-gate  * augmented by the flags LOCK_RECURSIVE and/or LOCK_ERRORCHECK,
1147c478bd9Sstevel@tonic-gate  * or it can be USYNC_PROCESS_ROBUST with no extra flags.
1157c478bd9Sstevel@tonic-gate  */
1167c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_init = __mutex_init
1177c478bd9Sstevel@tonic-gate #pragma weak mutex_init = __mutex_init
1187c478bd9Sstevel@tonic-gate #pragma weak _mutex_init = __mutex_init
1197c478bd9Sstevel@tonic-gate /* ARGSUSED2 */
1207c478bd9Sstevel@tonic-gate int
1217c478bd9Sstevel@tonic-gate __mutex_init(mutex_t *mp, int type, void *arg)
1227c478bd9Sstevel@tonic-gate {
1237c478bd9Sstevel@tonic-gate 	int error;
1247c478bd9Sstevel@tonic-gate 
1257c478bd9Sstevel@tonic-gate 	switch (type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) {
1267c478bd9Sstevel@tonic-gate 	case USYNC_THREAD:
1277c478bd9Sstevel@tonic-gate 	case USYNC_PROCESS:
1287c478bd9Sstevel@tonic-gate 		(void) _memset(mp, 0, sizeof (*mp));
1297c478bd9Sstevel@tonic-gate 		mp->mutex_type = (uint8_t)type;
1307c478bd9Sstevel@tonic-gate 		mp->mutex_flag = LOCK_INITED;
1317c478bd9Sstevel@tonic-gate 		error = 0;
1327c478bd9Sstevel@tonic-gate 		break;
1337c478bd9Sstevel@tonic-gate 	case USYNC_PROCESS_ROBUST:
1347c478bd9Sstevel@tonic-gate 		if (type & (LOCK_RECURSIVE|LOCK_ERRORCHECK))
1357c478bd9Sstevel@tonic-gate 			error = EINVAL;
1367c478bd9Sstevel@tonic-gate 		else
1377c478bd9Sstevel@tonic-gate 			error = ___lwp_mutex_init(mp, type);
1387c478bd9Sstevel@tonic-gate 		break;
1397c478bd9Sstevel@tonic-gate 	default:
1407c478bd9Sstevel@tonic-gate 		error = EINVAL;
1417c478bd9Sstevel@tonic-gate 		break;
1427c478bd9Sstevel@tonic-gate 	}
1437c478bd9Sstevel@tonic-gate 	if (error == 0)
1447c478bd9Sstevel@tonic-gate 		mp->mutex_magic = MUTEX_MAGIC;
1457c478bd9Sstevel@tonic-gate 	return (error);
1467c478bd9Sstevel@tonic-gate }
1477c478bd9Sstevel@tonic-gate 
1487c478bd9Sstevel@tonic-gate /*
1497c478bd9Sstevel@tonic-gate  * Delete mp from list of ceil mutexes owned by curthread.
1507c478bd9Sstevel@tonic-gate  * Return 1 if the head of the chain was updated.
1517c478bd9Sstevel@tonic-gate  */
1527c478bd9Sstevel@tonic-gate int
1537c478bd9Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp)
1547c478bd9Sstevel@tonic-gate {
1557c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1567c478bd9Sstevel@tonic-gate 	mxchain_t **mcpp;
1577c478bd9Sstevel@tonic-gate 	mxchain_t *mcp;
1587c478bd9Sstevel@tonic-gate 
1597c478bd9Sstevel@tonic-gate 	mcpp = &self->ul_mxchain;
1607c478bd9Sstevel@tonic-gate 	while ((*mcpp)->mxchain_mx != mp)
1617c478bd9Sstevel@tonic-gate 		mcpp = &(*mcpp)->mxchain_next;
1627c478bd9Sstevel@tonic-gate 	mcp = *mcpp;
1637c478bd9Sstevel@tonic-gate 	*mcpp = mcp->mxchain_next;
1647c478bd9Sstevel@tonic-gate 	lfree(mcp, sizeof (*mcp));
1657c478bd9Sstevel@tonic-gate 	return (mcpp == &self->ul_mxchain);
1667c478bd9Sstevel@tonic-gate }
1677c478bd9Sstevel@tonic-gate 
1687c478bd9Sstevel@tonic-gate /*
1697c478bd9Sstevel@tonic-gate  * Add mp to head of list of ceil mutexes owned by curthread.
1707c478bd9Sstevel@tonic-gate  * Return ENOMEM if no memory could be allocated.
1717c478bd9Sstevel@tonic-gate  */
1727c478bd9Sstevel@tonic-gate int
1737c478bd9Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp)
1747c478bd9Sstevel@tonic-gate {
1757c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1767c478bd9Sstevel@tonic-gate 	mxchain_t *mcp;
1777c478bd9Sstevel@tonic-gate 
1787c478bd9Sstevel@tonic-gate 	if ((mcp = lmalloc(sizeof (*mcp))) == NULL)
1797c478bd9Sstevel@tonic-gate 		return (ENOMEM);
1807c478bd9Sstevel@tonic-gate 	mcp->mxchain_mx = mp;
1817c478bd9Sstevel@tonic-gate 	mcp->mxchain_next = self->ul_mxchain;
1827c478bd9Sstevel@tonic-gate 	self->ul_mxchain = mcp;
1837c478bd9Sstevel@tonic-gate 	return (0);
1847c478bd9Sstevel@tonic-gate }
1857c478bd9Sstevel@tonic-gate 
1867c478bd9Sstevel@tonic-gate /*
1877c478bd9Sstevel@tonic-gate  * Inherit priority from ceiling.  The inheritance impacts the effective
1887c478bd9Sstevel@tonic-gate  * priority, not the assigned priority.  See _thread_setschedparam_main().
1897c478bd9Sstevel@tonic-gate  */
1907c478bd9Sstevel@tonic-gate void
1917c478bd9Sstevel@tonic-gate _ceil_prio_inherit(int ceil)
1927c478bd9Sstevel@tonic-gate {
1937c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1947c478bd9Sstevel@tonic-gate 	struct sched_param param;
1957c478bd9Sstevel@tonic-gate 
1967c478bd9Sstevel@tonic-gate 	(void) _memset(&param, 0, sizeof (param));
1977c478bd9Sstevel@tonic-gate 	param.sched_priority = ceil;
1987c478bd9Sstevel@tonic-gate 	if (_thread_setschedparam_main(self->ul_lwpid,
1997c478bd9Sstevel@tonic-gate 	    self->ul_policy, &param, PRIO_INHERIT)) {
2007c478bd9Sstevel@tonic-gate 		/*
2017c478bd9Sstevel@tonic-gate 		 * Panic since unclear what error code to return.
2027c478bd9Sstevel@tonic-gate 		 * If we do return the error codes returned by above
2037c478bd9Sstevel@tonic-gate 		 * called routine, update the man page...
2047c478bd9Sstevel@tonic-gate 		 */
2057c478bd9Sstevel@tonic-gate 		thr_panic("_thread_setschedparam_main() fails");
2067c478bd9Sstevel@tonic-gate 	}
2077c478bd9Sstevel@tonic-gate }
2087c478bd9Sstevel@tonic-gate 
2097c478bd9Sstevel@tonic-gate /*
2107c478bd9Sstevel@tonic-gate  * Waive inherited ceiling priority.  Inherit from head of owned ceiling locks
2117c478bd9Sstevel@tonic-gate  * if holding at least one ceiling lock.  If no ceiling locks are held at this
2127c478bd9Sstevel@tonic-gate  * point, disinherit completely, reverting back to assigned priority.
2137c478bd9Sstevel@tonic-gate  */
2147c478bd9Sstevel@tonic-gate void
2157c478bd9Sstevel@tonic-gate _ceil_prio_waive(void)
2167c478bd9Sstevel@tonic-gate {
2177c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2187c478bd9Sstevel@tonic-gate 	struct sched_param param;
2197c478bd9Sstevel@tonic-gate 
2207c478bd9Sstevel@tonic-gate 	(void) _memset(&param, 0, sizeof (param));
2217c478bd9Sstevel@tonic-gate 	if (self->ul_mxchain == NULL) {
2227c478bd9Sstevel@tonic-gate 		/*
2237c478bd9Sstevel@tonic-gate 		 * No ceil locks held.  Zero the epri, revert back to ul_pri.
2247c478bd9Sstevel@tonic-gate 		 * Since thread's hash lock is not held, one cannot just
2257c478bd9Sstevel@tonic-gate 		 * read ul_pri here...do it in the called routine...
2267c478bd9Sstevel@tonic-gate 		 */
2277c478bd9Sstevel@tonic-gate 		param.sched_priority = self->ul_pri;	/* ignored */
2287c478bd9Sstevel@tonic-gate 		if (_thread_setschedparam_main(self->ul_lwpid,
2297c478bd9Sstevel@tonic-gate 		    self->ul_policy, &param, PRIO_DISINHERIT))
2307c478bd9Sstevel@tonic-gate 			thr_panic("_thread_setschedparam_main() fails");
2317c478bd9Sstevel@tonic-gate 	} else {
2327c478bd9Sstevel@tonic-gate 		/*
2337c478bd9Sstevel@tonic-gate 		 * Set priority to that of the mutex at the head
2347c478bd9Sstevel@tonic-gate 		 * of the ceilmutex chain.
2357c478bd9Sstevel@tonic-gate 		 */
2367c478bd9Sstevel@tonic-gate 		param.sched_priority =
2377c478bd9Sstevel@tonic-gate 		    self->ul_mxchain->mxchain_mx->mutex_ceiling;
2387c478bd9Sstevel@tonic-gate 		if (_thread_setschedparam_main(self->ul_lwpid,
2397c478bd9Sstevel@tonic-gate 		    self->ul_policy, &param, PRIO_INHERIT))
2407c478bd9Sstevel@tonic-gate 			thr_panic("_thread_setschedparam_main() fails");
2417c478bd9Sstevel@tonic-gate 	}
2427c478bd9Sstevel@tonic-gate }
2437c478bd9Sstevel@tonic-gate 
2447c478bd9Sstevel@tonic-gate /*
2457c478bd9Sstevel@tonic-gate  * Non-preemptive spin locks.  Used by queue_lock().
2467c478bd9Sstevel@tonic-gate  * No lock statistics are gathered for these locks.
2477c478bd9Sstevel@tonic-gate  */
2487c478bd9Sstevel@tonic-gate void
2497c478bd9Sstevel@tonic-gate spin_lock_set(mutex_t *mp)
2507c478bd9Sstevel@tonic-gate {
2517c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2527c478bd9Sstevel@tonic-gate 
2537c478bd9Sstevel@tonic-gate 	no_preempt(self);
2547c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
2557c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
2567c478bd9Sstevel@tonic-gate 		return;
2577c478bd9Sstevel@tonic-gate 	}
2587c478bd9Sstevel@tonic-gate 	/*
2597c478bd9Sstevel@tonic-gate 	 * Spin for a while, attempting to acquire the lock.
2607c478bd9Sstevel@tonic-gate 	 */
2617c478bd9Sstevel@tonic-gate 	if (self->ul_spin_lock_spin != UINT_MAX)
2627c478bd9Sstevel@tonic-gate 		self->ul_spin_lock_spin++;
2637c478bd9Sstevel@tonic-gate 	if (mutex_queuelock_adaptive(mp) == 0 ||
2647c478bd9Sstevel@tonic-gate 	    set_lock_byte(&mp->mutex_lockw) == 0) {
2657c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
2667c478bd9Sstevel@tonic-gate 		return;
2677c478bd9Sstevel@tonic-gate 	}
2687c478bd9Sstevel@tonic-gate 	/*
2697c478bd9Sstevel@tonic-gate 	 * Try harder if we were previously at a no premption level.
2707c478bd9Sstevel@tonic-gate 	 */
2717c478bd9Sstevel@tonic-gate 	if (self->ul_preempt > 1) {
2727c478bd9Sstevel@tonic-gate 		if (self->ul_spin_lock_spin2 != UINT_MAX)
2737c478bd9Sstevel@tonic-gate 			self->ul_spin_lock_spin2++;
2747c478bd9Sstevel@tonic-gate 		if (mutex_queuelock_adaptive(mp) == 0 ||
2757c478bd9Sstevel@tonic-gate 		    set_lock_byte(&mp->mutex_lockw) == 0) {
2767c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
2777c478bd9Sstevel@tonic-gate 			return;
2787c478bd9Sstevel@tonic-gate 		}
2797c478bd9Sstevel@tonic-gate 	}
2807c478bd9Sstevel@tonic-gate 	/*
2817c478bd9Sstevel@tonic-gate 	 * Give up and block in the kernel for the mutex.
2827c478bd9Sstevel@tonic-gate 	 */
2837c478bd9Sstevel@tonic-gate 	if (self->ul_spin_lock_sleep != UINT_MAX)
2847c478bd9Sstevel@tonic-gate 		self->ul_spin_lock_sleep++;
2857c478bd9Sstevel@tonic-gate 	(void) ___lwp_mutex_timedlock(mp, NULL);
2867c478bd9Sstevel@tonic-gate 	mp->mutex_owner = (uintptr_t)self;
2877c478bd9Sstevel@tonic-gate }
2887c478bd9Sstevel@tonic-gate 
2897c478bd9Sstevel@tonic-gate void
2907c478bd9Sstevel@tonic-gate spin_lock_clear(mutex_t *mp)
2917c478bd9Sstevel@tonic-gate {
2927c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
2957c478bd9Sstevel@tonic-gate 	if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) {
2967c478bd9Sstevel@tonic-gate 		(void) ___lwp_mutex_wakeup(mp);
2977c478bd9Sstevel@tonic-gate 		if (self->ul_spin_lock_wakeup != UINT_MAX)
2987c478bd9Sstevel@tonic-gate 			self->ul_spin_lock_wakeup++;
2997c478bd9Sstevel@tonic-gate 	}
3007c478bd9Sstevel@tonic-gate 	preempt(self);
3017c478bd9Sstevel@tonic-gate }
3027c478bd9Sstevel@tonic-gate 
3037c478bd9Sstevel@tonic-gate /*
3047c478bd9Sstevel@tonic-gate  * Allocate the sleep queue hash table.
3057c478bd9Sstevel@tonic-gate  */
3067c478bd9Sstevel@tonic-gate void
3077c478bd9Sstevel@tonic-gate queue_alloc(void)
3087c478bd9Sstevel@tonic-gate {
3097c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
3107c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
3117c478bd9Sstevel@tonic-gate 	void *data;
3127c478bd9Sstevel@tonic-gate 	int i;
3137c478bd9Sstevel@tonic-gate 
3147c478bd9Sstevel@tonic-gate 	/*
3157c478bd9Sstevel@tonic-gate 	 * No locks are needed; we call here only when single-threaded.
3167c478bd9Sstevel@tonic-gate 	 */
3177c478bd9Sstevel@tonic-gate 	ASSERT(self == udp->ulwp_one);
3187c478bd9Sstevel@tonic-gate 	ASSERT(!udp->uberflags.uf_mt);
3197c478bd9Sstevel@tonic-gate 	if ((data = _private_mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t),
3207c478bd9Sstevel@tonic-gate 	    PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0))
3217c478bd9Sstevel@tonic-gate 	    == MAP_FAILED)
3227c478bd9Sstevel@tonic-gate 		thr_panic("cannot allocate thread queue_head table");
3237c478bd9Sstevel@tonic-gate 	udp->queue_head = (queue_head_t *)data;
3247c478bd9Sstevel@tonic-gate 	for (i = 0; i < 2 * QHASHSIZE; i++)
3257c478bd9Sstevel@tonic-gate 		udp->queue_head[i].qh_lock.mutex_magic = MUTEX_MAGIC;
3267c478bd9Sstevel@tonic-gate }
3277c478bd9Sstevel@tonic-gate 
3287c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG)
3297c478bd9Sstevel@tonic-gate 
3307c478bd9Sstevel@tonic-gate /*
3317c478bd9Sstevel@tonic-gate  * Debugging: verify correctness of a sleep queue.
3327c478bd9Sstevel@tonic-gate  */
3337c478bd9Sstevel@tonic-gate void
3347c478bd9Sstevel@tonic-gate QVERIFY(queue_head_t *qp)
3357c478bd9Sstevel@tonic-gate {
3367c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
3377c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
3387c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
3397c478bd9Sstevel@tonic-gate 	ulwp_t *prev;
3407c478bd9Sstevel@tonic-gate 	uint_t index;
3417c478bd9Sstevel@tonic-gate 	uint32_t cnt = 0;
3427c478bd9Sstevel@tonic-gate 	char qtype;
3437c478bd9Sstevel@tonic-gate 	void *wchan;
3447c478bd9Sstevel@tonic-gate 
3457c478bd9Sstevel@tonic-gate 	ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE);
3467c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
3477c478bd9Sstevel@tonic-gate 	ASSERT((qp->qh_head != NULL && qp->qh_tail != NULL) ||
3487c478bd9Sstevel@tonic-gate 		(qp->qh_head == NULL && qp->qh_tail == NULL));
3497c478bd9Sstevel@tonic-gate 	if (!thread_queue_verify)
3507c478bd9Sstevel@tonic-gate 		return;
3517c478bd9Sstevel@tonic-gate 	/* real expensive stuff, only for _THREAD_QUEUE_VERIFY */
3527c478bd9Sstevel@tonic-gate 	qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV;
3537c478bd9Sstevel@tonic-gate 	for (prev = NULL, ulwp = qp->qh_head; ulwp != NULL;
3547c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwp = ulwp->ul_link, cnt++) {
3557c478bd9Sstevel@tonic-gate 		ASSERT(ulwp->ul_qtype == qtype);
3567c478bd9Sstevel@tonic-gate 		ASSERT(ulwp->ul_wchan != NULL);
3577c478bd9Sstevel@tonic-gate 		ASSERT(ulwp->ul_sleepq == qp);
3587c478bd9Sstevel@tonic-gate 		wchan = ulwp->ul_wchan;
3597c478bd9Sstevel@tonic-gate 		index = QUEUE_HASH(wchan, qtype);
3607c478bd9Sstevel@tonic-gate 		ASSERT(&udp->queue_head[index] == qp);
3617c478bd9Sstevel@tonic-gate 	}
3627c478bd9Sstevel@tonic-gate 	ASSERT(qp->qh_tail == prev);
3637c478bd9Sstevel@tonic-gate 	ASSERT(qp->qh_qlen == cnt);
3647c478bd9Sstevel@tonic-gate }
3657c478bd9Sstevel@tonic-gate 
3667c478bd9Sstevel@tonic-gate #else	/* THREAD_DEBUG */
3677c478bd9Sstevel@tonic-gate 
3687c478bd9Sstevel@tonic-gate #define	QVERIFY(qp)
3697c478bd9Sstevel@tonic-gate 
3707c478bd9Sstevel@tonic-gate #endif	/* THREAD_DEBUG */
3717c478bd9Sstevel@tonic-gate 
3727c478bd9Sstevel@tonic-gate /*
3737c478bd9Sstevel@tonic-gate  * Acquire a queue head.
3747c478bd9Sstevel@tonic-gate  */
3757c478bd9Sstevel@tonic-gate queue_head_t *
3767c478bd9Sstevel@tonic-gate queue_lock(void *wchan, int qtype)
3777c478bd9Sstevel@tonic-gate {
3787c478bd9Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
3797c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
3807c478bd9Sstevel@tonic-gate 
3817c478bd9Sstevel@tonic-gate 	ASSERT(qtype == MX || qtype == CV);
3827c478bd9Sstevel@tonic-gate 
3837c478bd9Sstevel@tonic-gate 	/*
3847c478bd9Sstevel@tonic-gate 	 * It is possible that we could be called while still single-threaded.
3857c478bd9Sstevel@tonic-gate 	 * If so, we call queue_alloc() to allocate the queue_head[] array.
3867c478bd9Sstevel@tonic-gate 	 */
3877c478bd9Sstevel@tonic-gate 	if ((qp = udp->queue_head) == NULL) {
3887c478bd9Sstevel@tonic-gate 		queue_alloc();
3897c478bd9Sstevel@tonic-gate 		qp = udp->queue_head;
3907c478bd9Sstevel@tonic-gate 	}
3917c478bd9Sstevel@tonic-gate 	qp += QUEUE_HASH(wchan, qtype);
3927c478bd9Sstevel@tonic-gate 	spin_lock_set(&qp->qh_lock);
3937c478bd9Sstevel@tonic-gate 	/*
3947c478bd9Sstevel@tonic-gate 	 * At once per nanosecond, qh_lockcount will wrap after 512 years.
3957c478bd9Sstevel@tonic-gate 	 * Were we to care about this, we could peg the value at UINT64_MAX.
3967c478bd9Sstevel@tonic-gate 	 */
3977c478bd9Sstevel@tonic-gate 	qp->qh_lockcount++;
3987c478bd9Sstevel@tonic-gate 	QVERIFY(qp);
3997c478bd9Sstevel@tonic-gate 	return (qp);
4007c478bd9Sstevel@tonic-gate }
4017c478bd9Sstevel@tonic-gate 
4027c478bd9Sstevel@tonic-gate /*
4037c478bd9Sstevel@tonic-gate  * Release a queue head.
4047c478bd9Sstevel@tonic-gate  */
4057c478bd9Sstevel@tonic-gate void
4067c478bd9Sstevel@tonic-gate queue_unlock(queue_head_t *qp)
4077c478bd9Sstevel@tonic-gate {
4087c478bd9Sstevel@tonic-gate 	QVERIFY(qp);
4097c478bd9Sstevel@tonic-gate 	spin_lock_clear(&qp->qh_lock);
4107c478bd9Sstevel@tonic-gate }
4117c478bd9Sstevel@tonic-gate 
4127c478bd9Sstevel@tonic-gate /*
4137c478bd9Sstevel@tonic-gate  * For rwlock queueing, we must queue writers ahead of readers of the
4147c478bd9Sstevel@tonic-gate  * same priority.  We do this by making writers appear to have a half
4157c478bd9Sstevel@tonic-gate  * point higher priority for purposes of priority comparisons below.
4167c478bd9Sstevel@tonic-gate  */
4177c478bd9Sstevel@tonic-gate #define	CMP_PRIO(ulwp)	((real_priority(ulwp) << 1) + (ulwp)->ul_writer)
4187c478bd9Sstevel@tonic-gate 
4197c478bd9Sstevel@tonic-gate void
4207c478bd9Sstevel@tonic-gate enqueue(queue_head_t *qp, ulwp_t *ulwp, void *wchan, int qtype)
4217c478bd9Sstevel@tonic-gate {
4227c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
4237c478bd9Sstevel@tonic-gate 	ulwp_t *next;
4247c478bd9Sstevel@tonic-gate 	int pri = CMP_PRIO(ulwp);
4257c478bd9Sstevel@tonic-gate 	int force_fifo = (qtype & FIFOQ);
4267c478bd9Sstevel@tonic-gate 	int do_fifo;
4277c478bd9Sstevel@tonic-gate 
4287c478bd9Sstevel@tonic-gate 	qtype &= ~FIFOQ;
4297c478bd9Sstevel@tonic-gate 	ASSERT(qtype == MX || qtype == CV);
4307c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
4317c478bd9Sstevel@tonic-gate 	ASSERT(ulwp->ul_sleepq != qp);
4327c478bd9Sstevel@tonic-gate 
4337c478bd9Sstevel@tonic-gate 	/*
4347c478bd9Sstevel@tonic-gate 	 * LIFO queue ordering is unfair and can lead to starvation,
4357c478bd9Sstevel@tonic-gate 	 * but it gives better performance for heavily contended locks.
4367c478bd9Sstevel@tonic-gate 	 * We use thread_queue_fifo (range is 0..8) to determine
4377c478bd9Sstevel@tonic-gate 	 * the frequency of FIFO vs LIFO queuing:
4387c478bd9Sstevel@tonic-gate 	 *	0 : every 256th time	(almost always LIFO)
4397c478bd9Sstevel@tonic-gate 	 *	1 : every 128th time
4407c478bd9Sstevel@tonic-gate 	 *	2 : every 64th  time
4417c478bd9Sstevel@tonic-gate 	 *	3 : every 32nd  time
4427c478bd9Sstevel@tonic-gate 	 *	4 : every 16th  time	(the default value, mostly LIFO)
4437c478bd9Sstevel@tonic-gate 	 *	5 : every 8th   time
4447c478bd9Sstevel@tonic-gate 	 *	6 : every 4th   time
4457c478bd9Sstevel@tonic-gate 	 *	7 : every 2nd   time
4467c478bd9Sstevel@tonic-gate 	 *	8 : every time		(never LIFO, always FIFO)
4477c478bd9Sstevel@tonic-gate 	 * Note that there is always some degree of FIFO ordering.
4487c478bd9Sstevel@tonic-gate 	 * This breaks live lock conditions that occur in applications
4497c478bd9Sstevel@tonic-gate 	 * that are written assuming (incorrectly) that threads acquire
4507c478bd9Sstevel@tonic-gate 	 * locks fairly, that is, in roughly round-robin order.
4517c478bd9Sstevel@tonic-gate 	 * In any event, the queue is maintained in priority order.
4527c478bd9Sstevel@tonic-gate 	 *
4537c478bd9Sstevel@tonic-gate 	 * If we are given the FIFOQ flag in qtype, fifo queueing is forced.
4547c478bd9Sstevel@tonic-gate 	 * SUSV3 requires this for semaphores.
4557c478bd9Sstevel@tonic-gate 	 */
4567c478bd9Sstevel@tonic-gate 	do_fifo = (force_fifo ||
4577c478bd9Sstevel@tonic-gate 		((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0);
4587c478bd9Sstevel@tonic-gate 
4597c478bd9Sstevel@tonic-gate 	if (qp->qh_head == NULL) {
4607c478bd9Sstevel@tonic-gate 		/*
4617c478bd9Sstevel@tonic-gate 		 * The queue is empty.  LIFO/FIFO doesn't matter.
4627c478bd9Sstevel@tonic-gate 		 */
4637c478bd9Sstevel@tonic-gate 		ASSERT(qp->qh_tail == NULL);
4647c478bd9Sstevel@tonic-gate 		ulwpp = &qp->qh_head;
4657c478bd9Sstevel@tonic-gate 	} else if (do_fifo) {
4667c478bd9Sstevel@tonic-gate 		/*
4677c478bd9Sstevel@tonic-gate 		 * Enqueue after the last thread whose priority is greater
4687c478bd9Sstevel@tonic-gate 		 * than or equal to the priority of the thread being queued.
4697c478bd9Sstevel@tonic-gate 		 * Attempt first to go directly onto the tail of the queue.
4707c478bd9Sstevel@tonic-gate 		 */
4717c478bd9Sstevel@tonic-gate 		if (pri <= CMP_PRIO(qp->qh_tail))
4727c478bd9Sstevel@tonic-gate 			ulwpp = &qp->qh_tail->ul_link;
4737c478bd9Sstevel@tonic-gate 		else {
4747c478bd9Sstevel@tonic-gate 			for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL;
4757c478bd9Sstevel@tonic-gate 			    ulwpp = &next->ul_link)
4767c478bd9Sstevel@tonic-gate 				if (pri > CMP_PRIO(next))
4777c478bd9Sstevel@tonic-gate 					break;
4787c478bd9Sstevel@tonic-gate 		}
4797c478bd9Sstevel@tonic-gate 	} else {
4807c478bd9Sstevel@tonic-gate 		/*
4817c478bd9Sstevel@tonic-gate 		 * Enqueue before the first thread whose priority is less
4827c478bd9Sstevel@tonic-gate 		 * than or equal to the priority of the thread being queued.
4837c478bd9Sstevel@tonic-gate 		 * Hopefully we can go directly onto the head of the queue.
4847c478bd9Sstevel@tonic-gate 		 */
4857c478bd9Sstevel@tonic-gate 		for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL;
4867c478bd9Sstevel@tonic-gate 		    ulwpp = &next->ul_link)
4877c478bd9Sstevel@tonic-gate 			if (pri >= CMP_PRIO(next))
4887c478bd9Sstevel@tonic-gate 				break;
4897c478bd9Sstevel@tonic-gate 	}
4907c478bd9Sstevel@tonic-gate 	if ((ulwp->ul_link = *ulwpp) == NULL)
4917c478bd9Sstevel@tonic-gate 		qp->qh_tail = ulwp;
4927c478bd9Sstevel@tonic-gate 	*ulwpp = ulwp;
4937c478bd9Sstevel@tonic-gate 
4947c478bd9Sstevel@tonic-gate 	ulwp->ul_sleepq = qp;
4957c478bd9Sstevel@tonic-gate 	ulwp->ul_wchan = wchan;
4967c478bd9Sstevel@tonic-gate 	ulwp->ul_qtype = qtype;
4977c478bd9Sstevel@tonic-gate 	if (qp->qh_qmax < ++qp->qh_qlen)
4987c478bd9Sstevel@tonic-gate 		qp->qh_qmax = qp->qh_qlen;
4997c478bd9Sstevel@tonic-gate }
5007c478bd9Sstevel@tonic-gate 
5017c478bd9Sstevel@tonic-gate /*
5027c478bd9Sstevel@tonic-gate  * Return a pointer to the queue slot of the
5037c478bd9Sstevel@tonic-gate  * highest priority thread on the queue.
5047c478bd9Sstevel@tonic-gate  * On return, prevp, if not NULL, will contain a pointer
5057c478bd9Sstevel@tonic-gate  * to the thread's predecessor on the queue
5067c478bd9Sstevel@tonic-gate  */
5077c478bd9Sstevel@tonic-gate static ulwp_t **
5087c478bd9Sstevel@tonic-gate queue_slot(queue_head_t *qp, void *wchan, int *more, ulwp_t **prevp)
5097c478bd9Sstevel@tonic-gate {
5107c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
5117c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
5127c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
5137c478bd9Sstevel@tonic-gate 	ulwp_t **suspp = NULL;
5147c478bd9Sstevel@tonic-gate 	ulwp_t *susprev;
5157c478bd9Sstevel@tonic-gate 
5167c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
5177c478bd9Sstevel@tonic-gate 
5187c478bd9Sstevel@tonic-gate 	/*
5197c478bd9Sstevel@tonic-gate 	 * Find a waiter on the sleep queue.
5207c478bd9Sstevel@tonic-gate 	 */
5217c478bd9Sstevel@tonic-gate 	for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL;
5227c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
5237c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan) {
5247c478bd9Sstevel@tonic-gate 			if (!ulwp->ul_stop)
5257c478bd9Sstevel@tonic-gate 				break;
5267c478bd9Sstevel@tonic-gate 			/*
5277c478bd9Sstevel@tonic-gate 			 * Try not to return a suspended thread.
5287c478bd9Sstevel@tonic-gate 			 * This mimics the old libthread's behavior.
5297c478bd9Sstevel@tonic-gate 			 */
5307c478bd9Sstevel@tonic-gate 			if (suspp == NULL) {
5317c478bd9Sstevel@tonic-gate 				suspp = ulwpp;
5327c478bd9Sstevel@tonic-gate 				susprev = prev;
5337c478bd9Sstevel@tonic-gate 			}
5347c478bd9Sstevel@tonic-gate 		}
5357c478bd9Sstevel@tonic-gate 	}
5367c478bd9Sstevel@tonic-gate 
5377c478bd9Sstevel@tonic-gate 	if (ulwp == NULL && suspp != NULL) {
5387c478bd9Sstevel@tonic-gate 		ulwp = *(ulwpp = suspp);
5397c478bd9Sstevel@tonic-gate 		prev = susprev;
5407c478bd9Sstevel@tonic-gate 		suspp = NULL;
5417c478bd9Sstevel@tonic-gate 	}
5427c478bd9Sstevel@tonic-gate 	if (ulwp == NULL) {
5437c478bd9Sstevel@tonic-gate 		if (more != NULL)
5447c478bd9Sstevel@tonic-gate 			*more = 0;
5457c478bd9Sstevel@tonic-gate 		return (NULL);
5467c478bd9Sstevel@tonic-gate 	}
5477c478bd9Sstevel@tonic-gate 
5487c478bd9Sstevel@tonic-gate 	if (prevp != NULL)
5497c478bd9Sstevel@tonic-gate 		*prevp = prev;
5507c478bd9Sstevel@tonic-gate 	if (more == NULL)
5517c478bd9Sstevel@tonic-gate 		return (ulwpp);
5527c478bd9Sstevel@tonic-gate 
5537c478bd9Sstevel@tonic-gate 	/*
5547c478bd9Sstevel@tonic-gate 	 * Scan the remainder of the queue for another waiter.
5557c478bd9Sstevel@tonic-gate 	 */
5567c478bd9Sstevel@tonic-gate 	if (suspp != NULL) {
5577c478bd9Sstevel@tonic-gate 		*more = 1;
5587c478bd9Sstevel@tonic-gate 		return (ulwpp);
5597c478bd9Sstevel@tonic-gate 	}
5607c478bd9Sstevel@tonic-gate 	for (ulwp = ulwp->ul_link; ulwp != NULL; ulwp = ulwp->ul_link) {
5617c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan) {
5627c478bd9Sstevel@tonic-gate 			*more = 1;
5637c478bd9Sstevel@tonic-gate 			return (ulwpp);
5647c478bd9Sstevel@tonic-gate 		}
5657c478bd9Sstevel@tonic-gate 	}
5667c478bd9Sstevel@tonic-gate 
5677c478bd9Sstevel@tonic-gate 	*more = 0;
5687c478bd9Sstevel@tonic-gate 	return (ulwpp);
5697c478bd9Sstevel@tonic-gate }
5707c478bd9Sstevel@tonic-gate 
5717c478bd9Sstevel@tonic-gate ulwp_t *
5727c478bd9Sstevel@tonic-gate dequeue(queue_head_t *qp, void *wchan, int *more)
5737c478bd9Sstevel@tonic-gate {
5747c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
5757c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
5767c478bd9Sstevel@tonic-gate 	ulwp_t *prev;
5777c478bd9Sstevel@tonic-gate 
5787c478bd9Sstevel@tonic-gate 	if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL)
5797c478bd9Sstevel@tonic-gate 		return (NULL);
5807c478bd9Sstevel@tonic-gate 
5817c478bd9Sstevel@tonic-gate 	/*
5827c478bd9Sstevel@tonic-gate 	 * Dequeue the waiter.
5837c478bd9Sstevel@tonic-gate 	 */
5847c478bd9Sstevel@tonic-gate 	ulwp = *ulwpp;
5857c478bd9Sstevel@tonic-gate 	*ulwpp = ulwp->ul_link;
5867c478bd9Sstevel@tonic-gate 	ulwp->ul_link = NULL;
5877c478bd9Sstevel@tonic-gate 	if (qp->qh_tail == ulwp)
5887c478bd9Sstevel@tonic-gate 		qp->qh_tail = prev;
5897c478bd9Sstevel@tonic-gate 	qp->qh_qlen--;
5907c478bd9Sstevel@tonic-gate 	ulwp->ul_sleepq = NULL;
5917c478bd9Sstevel@tonic-gate 	ulwp->ul_wchan = NULL;
5927c478bd9Sstevel@tonic-gate 
5937c478bd9Sstevel@tonic-gate 	return (ulwp);
5947c478bd9Sstevel@tonic-gate }
5957c478bd9Sstevel@tonic-gate 
5967c478bd9Sstevel@tonic-gate /*
5977c478bd9Sstevel@tonic-gate  * Return a pointer to the highest priority thread sleeping on wchan.
5987c478bd9Sstevel@tonic-gate  */
5997c478bd9Sstevel@tonic-gate ulwp_t *
6007c478bd9Sstevel@tonic-gate queue_waiter(queue_head_t *qp, void *wchan)
6017c478bd9Sstevel@tonic-gate {
6027c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
6037c478bd9Sstevel@tonic-gate 
6047c478bd9Sstevel@tonic-gate 	if ((ulwpp = queue_slot(qp, wchan, NULL, NULL)) == NULL)
6057c478bd9Sstevel@tonic-gate 		return (NULL);
6067c478bd9Sstevel@tonic-gate 	return (*ulwpp);
6077c478bd9Sstevel@tonic-gate }
6087c478bd9Sstevel@tonic-gate 
6097c478bd9Sstevel@tonic-gate uint8_t
6107c478bd9Sstevel@tonic-gate dequeue_self(queue_head_t *qp, void *wchan)
6117c478bd9Sstevel@tonic-gate {
6127c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
6137c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
6147c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
6157c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
6167c478bd9Sstevel@tonic-gate 	int found = 0;
6177c478bd9Sstevel@tonic-gate 	int more = 0;
6187c478bd9Sstevel@tonic-gate 
6197c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
6207c478bd9Sstevel@tonic-gate 
6217c478bd9Sstevel@tonic-gate 	/* find self on the sleep queue */
6227c478bd9Sstevel@tonic-gate 	for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL;
6237c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
6247c478bd9Sstevel@tonic-gate 		if (ulwp == self) {
6257c478bd9Sstevel@tonic-gate 			/* dequeue ourself */
6267c478bd9Sstevel@tonic-gate 			*ulwpp = self->ul_link;
6277c478bd9Sstevel@tonic-gate 			if (qp->qh_tail == self)
6287c478bd9Sstevel@tonic-gate 				qp->qh_tail = prev;
6297c478bd9Sstevel@tonic-gate 			qp->qh_qlen--;
6307c478bd9Sstevel@tonic-gate 			ASSERT(self->ul_wchan == wchan);
6317c478bd9Sstevel@tonic-gate 			self->ul_cvmutex = NULL;
6327c478bd9Sstevel@tonic-gate 			self->ul_sleepq = NULL;
6337c478bd9Sstevel@tonic-gate 			self->ul_wchan = NULL;
6347c478bd9Sstevel@tonic-gate 			self->ul_cv_wake = 0;
6357c478bd9Sstevel@tonic-gate 			self->ul_link = NULL;
6367c478bd9Sstevel@tonic-gate 			found = 1;
6377c478bd9Sstevel@tonic-gate 			break;
6387c478bd9Sstevel@tonic-gate 		}
6397c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan)
6407c478bd9Sstevel@tonic-gate 			more = 1;
6417c478bd9Sstevel@tonic-gate 	}
6427c478bd9Sstevel@tonic-gate 
6437c478bd9Sstevel@tonic-gate 	if (!found)
6447c478bd9Sstevel@tonic-gate 		thr_panic("dequeue_self(): curthread not found on queue");
6457c478bd9Sstevel@tonic-gate 
6467c478bd9Sstevel@tonic-gate 	if (more)
6477c478bd9Sstevel@tonic-gate 		return (1);
6487c478bd9Sstevel@tonic-gate 
6497c478bd9Sstevel@tonic-gate 	/* scan the remainder of the queue for another waiter */
6507c478bd9Sstevel@tonic-gate 	for (ulwp = *ulwpp; ulwp != NULL; ulwp = ulwp->ul_link) {
6517c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan)
6527c478bd9Sstevel@tonic-gate 			return (1);
6537c478bd9Sstevel@tonic-gate 	}
6547c478bd9Sstevel@tonic-gate 
6557c478bd9Sstevel@tonic-gate 	return (0);
6567c478bd9Sstevel@tonic-gate }
6577c478bd9Sstevel@tonic-gate 
6587c478bd9Sstevel@tonic-gate /*
6597c478bd9Sstevel@tonic-gate  * Called from call_user_handler() and _thrp_suspend() to take
6607c478bd9Sstevel@tonic-gate  * ourself off of our sleep queue so we can grab locks.
6617c478bd9Sstevel@tonic-gate  */
6627c478bd9Sstevel@tonic-gate void
6637c478bd9Sstevel@tonic-gate unsleep_self(void)
6647c478bd9Sstevel@tonic-gate {
6657c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
6667c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
6677c478bd9Sstevel@tonic-gate 
6687c478bd9Sstevel@tonic-gate 	/*
6697c478bd9Sstevel@tonic-gate 	 * Calling enter_critical()/exit_critical() here would lead
6707c478bd9Sstevel@tonic-gate 	 * to recursion.  Just manipulate self->ul_critical directly.
6717c478bd9Sstevel@tonic-gate 	 */
6727c478bd9Sstevel@tonic-gate 	self->ul_critical++;
6737c478bd9Sstevel@tonic-gate 	self->ul_writer = 0;
6747c478bd9Sstevel@tonic-gate 	while (self->ul_sleepq != NULL) {
6757c478bd9Sstevel@tonic-gate 		qp = queue_lock(self->ul_wchan, self->ul_qtype);
6767c478bd9Sstevel@tonic-gate 		/*
6777c478bd9Sstevel@tonic-gate 		 * We may have been moved from a CV queue to a
6787c478bd9Sstevel@tonic-gate 		 * mutex queue while we were attempting queue_lock().
6797c478bd9Sstevel@tonic-gate 		 * If so, just loop around and try again.
6807c478bd9Sstevel@tonic-gate 		 * dequeue_self() clears self->ul_sleepq.
6817c478bd9Sstevel@tonic-gate 		 */
6827c478bd9Sstevel@tonic-gate 		if (qp == self->ul_sleepq)
6837c478bd9Sstevel@tonic-gate 			(void) dequeue_self(qp, self->ul_wchan);
6847c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
6857c478bd9Sstevel@tonic-gate 	}
6867c478bd9Sstevel@tonic-gate 	self->ul_critical--;
6877c478bd9Sstevel@tonic-gate }
6887c478bd9Sstevel@tonic-gate 
6897c478bd9Sstevel@tonic-gate /*
6907c478bd9Sstevel@tonic-gate  * Common code for calling the the ___lwp_mutex_timedlock() system call.
6917c478bd9Sstevel@tonic-gate  * Returns with mutex_owner and mutex_ownerpid set correctly.
6927c478bd9Sstevel@tonic-gate  */
6937c478bd9Sstevel@tonic-gate int
6947c478bd9Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
6957c478bd9Sstevel@tonic-gate {
6967c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
6977c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
6987c478bd9Sstevel@tonic-gate 	hrtime_t begin_sleep;
6997c478bd9Sstevel@tonic-gate 	int error;
7007c478bd9Sstevel@tonic-gate 
7017c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
7027c478bd9Sstevel@tonic-gate 	self->ul_wchan = mp;
7037c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
7047c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
7057c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = mp;
7067c478bd9Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
7077c478bd9Sstevel@tonic-gate 	}
7087c478bd9Sstevel@tonic-gate 	if (msp) {
7097c478bd9Sstevel@tonic-gate 		tdb_incr(msp->mutex_sleep);
7107c478bd9Sstevel@tonic-gate 		begin_sleep = gethrtime();
7117c478bd9Sstevel@tonic-gate 	}
7127c478bd9Sstevel@tonic-gate 
7137c478bd9Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__block, mp);
7147c478bd9Sstevel@tonic-gate 
7157c478bd9Sstevel@tonic-gate 	for (;;) {
7167c478bd9Sstevel@tonic-gate 		if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0) {
7177c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
7187c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
7197c478bd9Sstevel@tonic-gate 			break;
7207c478bd9Sstevel@tonic-gate 		}
7217c478bd9Sstevel@tonic-gate 
7227c478bd9Sstevel@tonic-gate 		if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
7237c478bd9Sstevel@tonic-gate 			/*
7247c478bd9Sstevel@tonic-gate 			 * Defend against forkall().  We may be the child,
7257c478bd9Sstevel@tonic-gate 			 * in which case we don't actually own the mutex.
7267c478bd9Sstevel@tonic-gate 			 */
7277c478bd9Sstevel@tonic-gate 			enter_critical(self);
7287c478bd9Sstevel@tonic-gate 			if (mp->mutex_ownerpid == udp->pid) {
7297c478bd9Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
7307c478bd9Sstevel@tonic-gate 				exit_critical(self);
7317c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
7327c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
7337c478bd9Sstevel@tonic-gate 				    0, 0);
7347c478bd9Sstevel@tonic-gate 				break;
7357c478bd9Sstevel@tonic-gate 			}
7367c478bd9Sstevel@tonic-gate 			exit_critical(self);
7377c478bd9Sstevel@tonic-gate 		} else {
7387c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
7397c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
7407c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
7417c478bd9Sstevel@tonic-gate 			break;
7427c478bd9Sstevel@tonic-gate 		}
7437c478bd9Sstevel@tonic-gate 	}
7447c478bd9Sstevel@tonic-gate 	if (msp)
7457c478bd9Sstevel@tonic-gate 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
7467c478bd9Sstevel@tonic-gate 	self->ul_wchan = NULL;
7477c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
7487c478bd9Sstevel@tonic-gate 
7497c478bd9Sstevel@tonic-gate 	return (error);
7507c478bd9Sstevel@tonic-gate }
7517c478bd9Sstevel@tonic-gate 
7527c478bd9Sstevel@tonic-gate /*
7537c478bd9Sstevel@tonic-gate  * Common code for calling the ___lwp_mutex_trylock() system call.
7547c478bd9Sstevel@tonic-gate  * Returns with mutex_owner and mutex_ownerpid set correctly.
7557c478bd9Sstevel@tonic-gate  */
7567c478bd9Sstevel@tonic-gate int
7577c478bd9Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp)
7587c478bd9Sstevel@tonic-gate {
7597c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
7607c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
7617c478bd9Sstevel@tonic-gate 	int error;
7627c478bd9Sstevel@tonic-gate 
7637c478bd9Sstevel@tonic-gate 	for (;;) {
7647c478bd9Sstevel@tonic-gate 		if ((error = ___lwp_mutex_trylock(mp)) != 0) {
7657c478bd9Sstevel@tonic-gate 			if (error != EBUSY) {
7667c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
7677c478bd9Sstevel@tonic-gate 				    error);
7687c478bd9Sstevel@tonic-gate 			}
7697c478bd9Sstevel@tonic-gate 			break;
7707c478bd9Sstevel@tonic-gate 		}
7717c478bd9Sstevel@tonic-gate 
7727c478bd9Sstevel@tonic-gate 		if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
7737c478bd9Sstevel@tonic-gate 			/*
7747c478bd9Sstevel@tonic-gate 			 * Defend against forkall().  We may be the child,
7757c478bd9Sstevel@tonic-gate 			 * in which case we don't actually own the mutex.
7767c478bd9Sstevel@tonic-gate 			 */
7777c478bd9Sstevel@tonic-gate 			enter_critical(self);
7787c478bd9Sstevel@tonic-gate 			if (mp->mutex_ownerpid == udp->pid) {
7797c478bd9Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
7807c478bd9Sstevel@tonic-gate 				exit_critical(self);
7817c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
7827c478bd9Sstevel@tonic-gate 				    0, 0);
7837c478bd9Sstevel@tonic-gate 				break;
7847c478bd9Sstevel@tonic-gate 			}
7857c478bd9Sstevel@tonic-gate 			exit_critical(self);
7867c478bd9Sstevel@tonic-gate 		} else {
7877c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
7887c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
7897c478bd9Sstevel@tonic-gate 			break;
7907c478bd9Sstevel@tonic-gate 		}
7917c478bd9Sstevel@tonic-gate 	}
7927c478bd9Sstevel@tonic-gate 
7937c478bd9Sstevel@tonic-gate 	return (error);
7947c478bd9Sstevel@tonic-gate }
7957c478bd9Sstevel@tonic-gate 
7967c478bd9Sstevel@tonic-gate volatile sc_shared_t *
7977c478bd9Sstevel@tonic-gate setup_schedctl(void)
7987c478bd9Sstevel@tonic-gate {
7997c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
8007c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
8017c478bd9Sstevel@tonic-gate 	sc_shared_t *tmp;
8027c478bd9Sstevel@tonic-gate 
8037c478bd9Sstevel@tonic-gate 	if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */
8047c478bd9Sstevel@tonic-gate 	    !self->ul_vfork &&			/* not a child of vfork() */
8057c478bd9Sstevel@tonic-gate 	    !self->ul_schedctl_called) {	/* haven't been called before */
8067c478bd9Sstevel@tonic-gate 		enter_critical(self);
8077c478bd9Sstevel@tonic-gate 		self->ul_schedctl_called = &self->ul_uberdata->uberflags;
8087c478bd9Sstevel@tonic-gate 		if ((tmp = __schedctl()) != (sc_shared_t *)(-1))
8097c478bd9Sstevel@tonic-gate 			self->ul_schedctl = scp = tmp;
8107c478bd9Sstevel@tonic-gate 		exit_critical(self);
8117c478bd9Sstevel@tonic-gate 	}
8127c478bd9Sstevel@tonic-gate 	/*
8137c478bd9Sstevel@tonic-gate 	 * Unless the call to setup_schedctl() is surrounded
8147c478bd9Sstevel@tonic-gate 	 * by enter_critical()/exit_critical(), the address
8157c478bd9Sstevel@tonic-gate 	 * we are returning could be invalid due to a forkall()
8167c478bd9Sstevel@tonic-gate 	 * having occurred in another thread.
8177c478bd9Sstevel@tonic-gate 	 */
8187c478bd9Sstevel@tonic-gate 	return (scp);
8197c478bd9Sstevel@tonic-gate }
8207c478bd9Sstevel@tonic-gate 
8217c478bd9Sstevel@tonic-gate /*
8227c478bd9Sstevel@tonic-gate  * Interfaces from libsched, incorporated into libc.
8237c478bd9Sstevel@tonic-gate  * libsched.so.1 is now a filter library onto libc.
8247c478bd9Sstevel@tonic-gate  */
8257c478bd9Sstevel@tonic-gate #pragma weak schedctl_lookup = _schedctl_init
8267c478bd9Sstevel@tonic-gate #pragma weak _schedctl_lookup = _schedctl_init
8277c478bd9Sstevel@tonic-gate #pragma weak schedctl_init = _schedctl_init
8287c478bd9Sstevel@tonic-gate schedctl_t *
8297c478bd9Sstevel@tonic-gate _schedctl_init(void)
8307c478bd9Sstevel@tonic-gate {
8317c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp = setup_schedctl();
8327c478bd9Sstevel@tonic-gate 	return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl);
8337c478bd9Sstevel@tonic-gate }
8347c478bd9Sstevel@tonic-gate 
8357c478bd9Sstevel@tonic-gate #pragma weak schedctl_exit = _schedctl_exit
8367c478bd9Sstevel@tonic-gate void
8377c478bd9Sstevel@tonic-gate _schedctl_exit(void)
8387c478bd9Sstevel@tonic-gate {
8397c478bd9Sstevel@tonic-gate }
8407c478bd9Sstevel@tonic-gate 
8417c478bd9Sstevel@tonic-gate /*
8427c478bd9Sstevel@tonic-gate  * Contract private interface for java.
8437c478bd9Sstevel@tonic-gate  * Set up the schedctl data if it doesn't exist yet.
8447c478bd9Sstevel@tonic-gate  * Return a pointer to the pointer to the schedctl data.
8457c478bd9Sstevel@tonic-gate  */
8467c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile *
8477c478bd9Sstevel@tonic-gate _thr_schedctl(void)
8487c478bd9Sstevel@tonic-gate {
8497c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
8507c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *volatile *ptr;
8517c478bd9Sstevel@tonic-gate 
8527c478bd9Sstevel@tonic-gate 	if (self->ul_vfork)
8537c478bd9Sstevel@tonic-gate 		return (NULL);
8547c478bd9Sstevel@tonic-gate 	if (*(ptr = &self->ul_schedctl) == NULL)
8557c478bd9Sstevel@tonic-gate 		(void) setup_schedctl();
8567c478bd9Sstevel@tonic-gate 	return (ptr);
8577c478bd9Sstevel@tonic-gate }
8587c478bd9Sstevel@tonic-gate 
8597c478bd9Sstevel@tonic-gate /*
8607c478bd9Sstevel@tonic-gate  * Block signals and attempt to block preemption.
8617c478bd9Sstevel@tonic-gate  * no_preempt()/preempt() must be used in pairs but can be nested.
8627c478bd9Sstevel@tonic-gate  */
8637c478bd9Sstevel@tonic-gate void
8647c478bd9Sstevel@tonic-gate no_preempt(ulwp_t *self)
8657c478bd9Sstevel@tonic-gate {
8667c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
8677c478bd9Sstevel@tonic-gate 
8687c478bd9Sstevel@tonic-gate 	if (self->ul_preempt++ == 0) {
8697c478bd9Sstevel@tonic-gate 		enter_critical(self);
8707c478bd9Sstevel@tonic-gate 		if ((scp = self->ul_schedctl) != NULL ||
8717c478bd9Sstevel@tonic-gate 		    (scp = setup_schedctl()) != NULL) {
8727c478bd9Sstevel@tonic-gate 			/*
8737c478bd9Sstevel@tonic-gate 			 * Save the pre-existing preempt value.
8747c478bd9Sstevel@tonic-gate 			 */
8757c478bd9Sstevel@tonic-gate 			self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt;
8767c478bd9Sstevel@tonic-gate 			scp->sc_preemptctl.sc_nopreempt = 1;
8777c478bd9Sstevel@tonic-gate 		}
8787c478bd9Sstevel@tonic-gate 	}
8797c478bd9Sstevel@tonic-gate }
8807c478bd9Sstevel@tonic-gate 
8817c478bd9Sstevel@tonic-gate /*
8827c478bd9Sstevel@tonic-gate  * Undo the effects of no_preempt().
8837c478bd9Sstevel@tonic-gate  */
8847c478bd9Sstevel@tonic-gate void
8857c478bd9Sstevel@tonic-gate preempt(ulwp_t *self)
8867c478bd9Sstevel@tonic-gate {
8877c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
8887c478bd9Sstevel@tonic-gate 
8897c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_preempt > 0);
8907c478bd9Sstevel@tonic-gate 	if (--self->ul_preempt == 0) {
8917c478bd9Sstevel@tonic-gate 		if ((scp = self->ul_schedctl) != NULL) {
8927c478bd9Sstevel@tonic-gate 			/*
8937c478bd9Sstevel@tonic-gate 			 * Restore the pre-existing preempt value.
8947c478bd9Sstevel@tonic-gate 			 */
8957c478bd9Sstevel@tonic-gate 			scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt;
8967c478bd9Sstevel@tonic-gate 			if (scp->sc_preemptctl.sc_yield &&
8977c478bd9Sstevel@tonic-gate 			    scp->sc_preemptctl.sc_nopreempt == 0) {
8987c478bd9Sstevel@tonic-gate 				lwp_yield();
8997c478bd9Sstevel@tonic-gate 				if (scp->sc_preemptctl.sc_yield) {
9007c478bd9Sstevel@tonic-gate 					/*
9017c478bd9Sstevel@tonic-gate 					 * Shouldn't happen.  This is either
9027c478bd9Sstevel@tonic-gate 					 * a race condition or the thread
9037c478bd9Sstevel@tonic-gate 					 * just entered the real-time class.
9047c478bd9Sstevel@tonic-gate 					 */
9057c478bd9Sstevel@tonic-gate 					lwp_yield();
9067c478bd9Sstevel@tonic-gate 					scp->sc_preemptctl.sc_yield = 0;
9077c478bd9Sstevel@tonic-gate 				}
9087c478bd9Sstevel@tonic-gate 			}
9097c478bd9Sstevel@tonic-gate 		}
9107c478bd9Sstevel@tonic-gate 		exit_critical(self);
9117c478bd9Sstevel@tonic-gate 	}
9127c478bd9Sstevel@tonic-gate }
9137c478bd9Sstevel@tonic-gate 
9147c478bd9Sstevel@tonic-gate /*
9157c478bd9Sstevel@tonic-gate  * If a call to preempt() would cause the current thread to yield or to
9167c478bd9Sstevel@tonic-gate  * take deferred actions in exit_critical(), then unpark the specified
9177c478bd9Sstevel@tonic-gate  * lwp so it can run while we delay.  Return the original lwpid if the
9187c478bd9Sstevel@tonic-gate  * unpark was not performed, else return zero.  The tests are a repeat
9197c478bd9Sstevel@tonic-gate  * of some of the tests in preempt(), above.  This is a statistical
9207c478bd9Sstevel@tonic-gate  * optimization solely for cond_sleep_queue(), below.
9217c478bd9Sstevel@tonic-gate  */
9227c478bd9Sstevel@tonic-gate static lwpid_t
9237c478bd9Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid)
9247c478bd9Sstevel@tonic-gate {
9257c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp = self->ul_schedctl;
9267c478bd9Sstevel@tonic-gate 
9277c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_preempt == 1 && self->ul_critical > 0);
9287c478bd9Sstevel@tonic-gate 	if ((scp != NULL && scp->sc_preemptctl.sc_yield) ||
9297c478bd9Sstevel@tonic-gate 	    (self->ul_curplease && self->ul_critical == 1)) {
9307c478bd9Sstevel@tonic-gate 		(void) __lwp_unpark(lwpid);
9317c478bd9Sstevel@tonic-gate 		lwpid = 0;
9327c478bd9Sstevel@tonic-gate 	}
9337c478bd9Sstevel@tonic-gate 	return (lwpid);
9347c478bd9Sstevel@tonic-gate }
9357c478bd9Sstevel@tonic-gate 
9367c478bd9Sstevel@tonic-gate /*
9377c478bd9Sstevel@tonic-gate  * Spin for a while, trying to grab the lock.  We know that we
9387c478bd9Sstevel@tonic-gate  * failed set_lock_byte(&mp->mutex_lockw) once before coming here.
9397c478bd9Sstevel@tonic-gate  * If this fails, return EBUSY and let the caller deal with it.
9407c478bd9Sstevel@tonic-gate  * If this succeeds, return 0 with mutex_owner set to curthread.
9417c478bd9Sstevel@tonic-gate  */
9427c478bd9Sstevel@tonic-gate int
9437c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mutex_t *mp)
9447c478bd9Sstevel@tonic-gate {
9457c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
9467c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
9477c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
9487c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
9497c478bd9Sstevel@tonic-gate 	volatile uint64_t *ownerp;
9507c478bd9Sstevel@tonic-gate 	int count, max = self->ul_adaptive_spin;
9517c478bd9Sstevel@tonic-gate 
9527c478bd9Sstevel@tonic-gate 	ASSERT(!(mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)));
9537c478bd9Sstevel@tonic-gate 
9547c478bd9Sstevel@tonic-gate 	if (max == 0 || (mp->mutex_spinners >= self->ul_max_spinners))
9557c478bd9Sstevel@tonic-gate 		return (EBUSY);
9567c478bd9Sstevel@tonic-gate 
9577c478bd9Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
9587c478bd9Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
9597c478bd9Sstevel@tonic-gate 
9607c478bd9Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
9617c478bd9Sstevel@tonic-gate 
9627c478bd9Sstevel@tonic-gate 	/*
9637c478bd9Sstevel@tonic-gate 	 * This spin loop is unfair to lwps that have already dropped into
9647c478bd9Sstevel@tonic-gate 	 * the kernel to sleep.  They will starve on a highly-contended mutex.
9657c478bd9Sstevel@tonic-gate 	 * This is just too bad.  The adaptive spin algorithm is intended
9667c478bd9Sstevel@tonic-gate 	 * to allow programs with highly-contended locks (that is, broken
9677c478bd9Sstevel@tonic-gate 	 * programs) to execute with reasonable speed despite their contention.
9687c478bd9Sstevel@tonic-gate 	 * Being fair would reduce the speed of such programs and well-written
9697c478bd9Sstevel@tonic-gate 	 * programs will not suffer in any case.
9707c478bd9Sstevel@tonic-gate 	 */
9717c478bd9Sstevel@tonic-gate 	enter_critical(self);		/* protects ul_schedctl */
9727c478bd9Sstevel@tonic-gate 	incr32(&mp->mutex_spinners);
9737c478bd9Sstevel@tonic-gate 	for (count = 0; count < max; count++) {
9747c478bd9Sstevel@tonic-gate 		if (*lockp == 0 && set_lock_byte(lockp) == 0) {
9757c478bd9Sstevel@tonic-gate 			*ownerp = (uintptr_t)self;
9767c478bd9Sstevel@tonic-gate 			decr32(&mp->mutex_spinners);
9777c478bd9Sstevel@tonic-gate 			exit_critical(self);
9787c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__spun, 1, count);
9797c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
9807c478bd9Sstevel@tonic-gate 			return (0);
9817c478bd9Sstevel@tonic-gate 		}
9827c478bd9Sstevel@tonic-gate 		SMT_PAUSE();
9837c478bd9Sstevel@tonic-gate 		/*
9847c478bd9Sstevel@tonic-gate 		 * Stop spinning if the mutex owner is not running on
9857c478bd9Sstevel@tonic-gate 		 * a processor; it will not drop the lock any time soon
9867c478bd9Sstevel@tonic-gate 		 * and we would just be wasting time to keep spinning.
9877c478bd9Sstevel@tonic-gate 		 *
9887c478bd9Sstevel@tonic-gate 		 * Note that we are looking at another thread (ulwp_t)
9897c478bd9Sstevel@tonic-gate 		 * without ensuring that the other thread does not exit.
9907c478bd9Sstevel@tonic-gate 		 * The scheme relies on ulwp_t structures never being
9917c478bd9Sstevel@tonic-gate 		 * deallocated by the library (the library employs a free
9927c478bd9Sstevel@tonic-gate 		 * list of ulwp_t structs that are reused when new threads
9937c478bd9Sstevel@tonic-gate 		 * are created) and on schedctl shared memory never being
9947c478bd9Sstevel@tonic-gate 		 * deallocated once created via __schedctl().
9957c478bd9Sstevel@tonic-gate 		 *
9967c478bd9Sstevel@tonic-gate 		 * Thus, the worst that can happen when the spinning thread
9977c478bd9Sstevel@tonic-gate 		 * looks at the owner's schedctl data is that it is looking
9987c478bd9Sstevel@tonic-gate 		 * at some other thread's schedctl data.  This almost never
9997c478bd9Sstevel@tonic-gate 		 * happens and is benign when it does.
10007c478bd9Sstevel@tonic-gate 		 */
10017c478bd9Sstevel@tonic-gate 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
10027c478bd9Sstevel@tonic-gate 		    ((scp = ulwp->ul_schedctl) == NULL ||
10037c478bd9Sstevel@tonic-gate 		    scp->sc_state != SC_ONPROC))
10047c478bd9Sstevel@tonic-gate 			break;
10057c478bd9Sstevel@tonic-gate 	}
10067c478bd9Sstevel@tonic-gate 	decr32(&mp->mutex_spinners);
10077c478bd9Sstevel@tonic-gate 	exit_critical(self);
10087c478bd9Sstevel@tonic-gate 
10097c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__spun, 0, count);
10107c478bd9Sstevel@tonic-gate 
10117c478bd9Sstevel@tonic-gate 	return (EBUSY);
10127c478bd9Sstevel@tonic-gate }
10137c478bd9Sstevel@tonic-gate 
10147c478bd9Sstevel@tonic-gate /*
10157c478bd9Sstevel@tonic-gate  * Same as mutex_trylock_adaptive(), except specifically for queue locks.
10167c478bd9Sstevel@tonic-gate  * The owner field is not set here; the caller (spin_lock_set()) sets it.
10177c478bd9Sstevel@tonic-gate  */
10187c478bd9Sstevel@tonic-gate int
10197c478bd9Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp)
10207c478bd9Sstevel@tonic-gate {
10217c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
10227c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
10237c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
10247c478bd9Sstevel@tonic-gate 	volatile uint64_t *ownerp;
10257c478bd9Sstevel@tonic-gate 	int count = curthread->ul_queue_spin;
10267c478bd9Sstevel@tonic-gate 
10277c478bd9Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
10287c478bd9Sstevel@tonic-gate 
10297c478bd9Sstevel@tonic-gate 	if (count == 0)
10307c478bd9Sstevel@tonic-gate 		return (EBUSY);
10317c478bd9Sstevel@tonic-gate 
10327c478bd9Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
10337c478bd9Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
10347c478bd9Sstevel@tonic-gate 	while (--count >= 0) {
10357c478bd9Sstevel@tonic-gate 		if (*lockp == 0 && set_lock_byte(lockp) == 0)
10367c478bd9Sstevel@tonic-gate 			return (0);
10377c478bd9Sstevel@tonic-gate 		SMT_PAUSE();
10387c478bd9Sstevel@tonic-gate 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
10397c478bd9Sstevel@tonic-gate 		    ((scp = ulwp->ul_schedctl) == NULL ||
10407c478bd9Sstevel@tonic-gate 		    scp->sc_state != SC_ONPROC))
10417c478bd9Sstevel@tonic-gate 			break;
10427c478bd9Sstevel@tonic-gate 	}
10437c478bd9Sstevel@tonic-gate 
10447c478bd9Sstevel@tonic-gate 	return (EBUSY);
10457c478bd9Sstevel@tonic-gate }
10467c478bd9Sstevel@tonic-gate 
10477c478bd9Sstevel@tonic-gate /*
10487c478bd9Sstevel@tonic-gate  * Like mutex_trylock_adaptive(), but for process-shared mutexes.
10497c478bd9Sstevel@tonic-gate  * Spin for a while, trying to grab the lock.  We know that we
10507c478bd9Sstevel@tonic-gate  * failed set_lock_byte(&mp->mutex_lockw) once before coming here.
10517c478bd9Sstevel@tonic-gate  * If this fails, return EBUSY and let the caller deal with it.
10527c478bd9Sstevel@tonic-gate  * If this succeeds, return 0 with mutex_owner set to curthread
10537c478bd9Sstevel@tonic-gate  * and mutex_ownerpid set to the current pid.
10547c478bd9Sstevel@tonic-gate  */
10557c478bd9Sstevel@tonic-gate int
10567c478bd9Sstevel@tonic-gate mutex_trylock_process(mutex_t *mp)
10577c478bd9Sstevel@tonic-gate {
10587c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
10597c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
10607c478bd9Sstevel@tonic-gate 	int count;
10617c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
10627c478bd9Sstevel@tonic-gate 	volatile uint64_t *ownerp;
10637c478bd9Sstevel@tonic-gate 	volatile int32_t *pidp;
10647c478bd9Sstevel@tonic-gate 	pid_t pid, newpid;
10657c478bd9Sstevel@tonic-gate 	uint64_t owner, newowner;
10667c478bd9Sstevel@tonic-gate 
10677c478bd9Sstevel@tonic-gate 	if ((count = ncpus) == 0)
10687c478bd9Sstevel@tonic-gate 		count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
10697c478bd9Sstevel@tonic-gate 	count = (count > 1)? self->ul_adaptive_spin : 0;
10707c478bd9Sstevel@tonic-gate 
10717c478bd9Sstevel@tonic-gate 	ASSERT((mp->mutex_type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) ==
10727c478bd9Sstevel@tonic-gate 		USYNC_PROCESS);
10737c478bd9Sstevel@tonic-gate 
10747c478bd9Sstevel@tonic-gate 	if (count == 0)
10757c478bd9Sstevel@tonic-gate 		return (EBUSY);
10767c478bd9Sstevel@tonic-gate 
10777c478bd9Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
10787c478bd9Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
10797c478bd9Sstevel@tonic-gate 	pidp = (volatile int32_t *)&mp->mutex_ownerpid;
10807c478bd9Sstevel@tonic-gate 	owner = *ownerp;
10817c478bd9Sstevel@tonic-gate 	pid = *pidp;
10827c478bd9Sstevel@tonic-gate 	/*
10837c478bd9Sstevel@tonic-gate 	 * This is a process-shared mutex.
10847c478bd9Sstevel@tonic-gate 	 * We cannot know if the owner is running on a processor.
10857c478bd9Sstevel@tonic-gate 	 * We just spin and hope that it is on a processor.
10867c478bd9Sstevel@tonic-gate 	 */
10877c478bd9Sstevel@tonic-gate 	while (--count >= 0) {
10887c478bd9Sstevel@tonic-gate 		if (*lockp == 0) {
10897c478bd9Sstevel@tonic-gate 			enter_critical(self);
10907c478bd9Sstevel@tonic-gate 			if (set_lock_byte(lockp) == 0) {
10917c478bd9Sstevel@tonic-gate 				*ownerp = (uintptr_t)self;
10927c478bd9Sstevel@tonic-gate 				*pidp = udp->pid;
10937c478bd9Sstevel@tonic-gate 				exit_critical(self);
10947c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
10957c478bd9Sstevel@tonic-gate 				    0, 0);
10967c478bd9Sstevel@tonic-gate 				return (0);
10977c478bd9Sstevel@tonic-gate 			}
10987c478bd9Sstevel@tonic-gate 			exit_critical(self);
10997c478bd9Sstevel@tonic-gate 		} else if ((newowner = *ownerp) == owner &&
11007c478bd9Sstevel@tonic-gate 		    (newpid = *pidp) == pid) {
11017c478bd9Sstevel@tonic-gate 			SMT_PAUSE();
11027c478bd9Sstevel@tonic-gate 			continue;
11037c478bd9Sstevel@tonic-gate 		}
11047c478bd9Sstevel@tonic-gate 		/*
11057c478bd9Sstevel@tonic-gate 		 * The owner of the lock changed; start the count over again.
11067c478bd9Sstevel@tonic-gate 		 * This may be too aggressive; it needs testing.
11077c478bd9Sstevel@tonic-gate 		 */
11087c478bd9Sstevel@tonic-gate 		owner = newowner;
11097c478bd9Sstevel@tonic-gate 		pid = newpid;
11107c478bd9Sstevel@tonic-gate 		count = self->ul_adaptive_spin;
11117c478bd9Sstevel@tonic-gate 	}
11127c478bd9Sstevel@tonic-gate 
11137c478bd9Sstevel@tonic-gate 	return (EBUSY);
11147c478bd9Sstevel@tonic-gate }
11157c478bd9Sstevel@tonic-gate 
11167c478bd9Sstevel@tonic-gate /*
11177c478bd9Sstevel@tonic-gate  * Mutex wakeup code for releasing a USYNC_THREAD mutex.
11187c478bd9Sstevel@tonic-gate  * Returns the lwpid of the thread that was dequeued, if any.
11197c478bd9Sstevel@tonic-gate  * The caller of mutex_wakeup() must call __lwp_unpark(lwpid)
11207c478bd9Sstevel@tonic-gate  * to wake up the specified lwp.
11217c478bd9Sstevel@tonic-gate  */
11227c478bd9Sstevel@tonic-gate lwpid_t
11237c478bd9Sstevel@tonic-gate mutex_wakeup(mutex_t *mp)
11247c478bd9Sstevel@tonic-gate {
11257c478bd9Sstevel@tonic-gate 	lwpid_t lwpid = 0;
11267c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
11277c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
11287c478bd9Sstevel@tonic-gate 	int more;
11297c478bd9Sstevel@tonic-gate 
11307c478bd9Sstevel@tonic-gate 	/*
11317c478bd9Sstevel@tonic-gate 	 * Dequeue a waiter from the sleep queue.  Don't touch the mutex
11327c478bd9Sstevel@tonic-gate 	 * waiters bit if no one was found on the queue because the mutex
11337c478bd9Sstevel@tonic-gate 	 * might have been deallocated or reallocated for another purpose.
11347c478bd9Sstevel@tonic-gate 	 */
11357c478bd9Sstevel@tonic-gate 	qp = queue_lock(mp, MX);
11367c478bd9Sstevel@tonic-gate 	if ((ulwp = dequeue(qp, mp, &more)) != NULL) {
11377c478bd9Sstevel@tonic-gate 		lwpid = ulwp->ul_lwpid;
11387c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = (more? 1 : 0);
11397c478bd9Sstevel@tonic-gate 	}
11407c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
11417c478bd9Sstevel@tonic-gate 	return (lwpid);
11427c478bd9Sstevel@tonic-gate }
11437c478bd9Sstevel@tonic-gate 
11447c478bd9Sstevel@tonic-gate /*
11457c478bd9Sstevel@tonic-gate  * Spin for a while, testing to see if the lock has been grabbed.
11467c478bd9Sstevel@tonic-gate  * If this fails, call mutex_wakeup() to release a waiter.
11477c478bd9Sstevel@tonic-gate  */
11487c478bd9Sstevel@tonic-gate lwpid_t
11497c478bd9Sstevel@tonic-gate mutex_unlock_queue(mutex_t *mp)
11507c478bd9Sstevel@tonic-gate {
11517c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
11527c478bd9Sstevel@tonic-gate 	uint32_t *lockw = &mp->mutex_lockword;
11537c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
11547c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
11557c478bd9Sstevel@tonic-gate 	volatile uint32_t *spinp;
11567c478bd9Sstevel@tonic-gate 	int count;
11577c478bd9Sstevel@tonic-gate 
11587c478bd9Sstevel@tonic-gate 	/*
11597c478bd9Sstevel@tonic-gate 	 * We use the swap primitive to clear the lock, but we must
11607c478bd9Sstevel@tonic-gate 	 * atomically retain the waiters bit for the remainder of this
11617c478bd9Sstevel@tonic-gate 	 * code to work.  We first check to see if the waiters bit is
11627c478bd9Sstevel@tonic-gate 	 * set and if so clear the lock by swapping in a word containing
11637c478bd9Sstevel@tonic-gate 	 * only the waiters bit.  This could produce a false positive test
11647c478bd9Sstevel@tonic-gate 	 * for whether there are waiters that need to be waked up, but
11657c478bd9Sstevel@tonic-gate 	 * this just causes an extra call to mutex_wakeup() to do nothing.
11667c478bd9Sstevel@tonic-gate 	 * The opposite case is more delicate:  If there are no waiters,
11677c478bd9Sstevel@tonic-gate 	 * we swap in a zero lock byte and a zero waiters bit.  The result
11687c478bd9Sstevel@tonic-gate 	 * of the swap could indicate that there really was a waiter so in
11697c478bd9Sstevel@tonic-gate 	 * this case we go directly to mutex_wakeup() without performing
11707c478bd9Sstevel@tonic-gate 	 * any of the adaptive code because the waiter bit has been cleared
11717c478bd9Sstevel@tonic-gate 	 * and the adaptive code is unreliable in this case.
11727c478bd9Sstevel@tonic-gate 	 */
11737c478bd9Sstevel@tonic-gate 	if (!(*lockw & WAITERMASK)) {	/* no waiter exists right now */
11747c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
11757c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
11767c478bd9Sstevel@tonic-gate 		if (!(swap32(lockw, 0) & WAITERMASK))	/* still no waiters */
11777c478bd9Sstevel@tonic-gate 			return (0);
11787c478bd9Sstevel@tonic-gate 		no_preempt(self);	/* ensure a prompt wakeup */
11797c478bd9Sstevel@tonic-gate 		lwpid = mutex_wakeup(mp);
11807c478bd9Sstevel@tonic-gate 	} else {
11817c478bd9Sstevel@tonic-gate 		no_preempt(self);	/* ensure a prompt wakeup */
11827c478bd9Sstevel@tonic-gate 		lockp = (volatile uint8_t *)&mp->mutex_lockw;
11837c478bd9Sstevel@tonic-gate 		spinp = (volatile uint32_t *)&mp->mutex_spinners;
11847c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
11857c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
11867c478bd9Sstevel@tonic-gate 		(void) swap32(lockw, WAITER);	/* clear lock, retain waiter */
11877c478bd9Sstevel@tonic-gate 
11887c478bd9Sstevel@tonic-gate 		/*
11897c478bd9Sstevel@tonic-gate 		 * We spin here fewer times than mutex_trylock_adaptive().
11907c478bd9Sstevel@tonic-gate 		 * We are trying to balance two conflicting goals:
11917c478bd9Sstevel@tonic-gate 		 * 1. Avoid waking up anyone if a spinning thread
11927c478bd9Sstevel@tonic-gate 		 *    grabs the lock.
11937c478bd9Sstevel@tonic-gate 		 * 2. Wake up a sleeping thread promptly to get on
11947c478bd9Sstevel@tonic-gate 		 *    with useful work.
11957c478bd9Sstevel@tonic-gate 		 * We don't spin at all if there is no acquiring spinner;
11967c478bd9Sstevel@tonic-gate 		 * (mp->mutex_spinners is non-zero if there are spinners).
11977c478bd9Sstevel@tonic-gate 		 */
11987c478bd9Sstevel@tonic-gate 		for (count = self->ul_release_spin;
11997c478bd9Sstevel@tonic-gate 		    *spinp && count > 0; count--) {
12007c478bd9Sstevel@tonic-gate 			/*
12017c478bd9Sstevel@tonic-gate 			 * There is a waiter that we will have to wake
12027c478bd9Sstevel@tonic-gate 			 * up unless someone else grabs the lock while
12037c478bd9Sstevel@tonic-gate 			 * we are busy spinning.  Like the spin loop in
12047c478bd9Sstevel@tonic-gate 			 * mutex_trylock_adaptive(), this spin loop is
12057c478bd9Sstevel@tonic-gate 			 * unfair to lwps that have already dropped into
12067c478bd9Sstevel@tonic-gate 			 * the kernel to sleep.  They will starve on a
12077c478bd9Sstevel@tonic-gate 			 * highly-contended mutex.  Too bad.
12087c478bd9Sstevel@tonic-gate 			 */
12097c478bd9Sstevel@tonic-gate 			if (*lockp != 0) {	/* somebody grabbed the lock */
12107c478bd9Sstevel@tonic-gate 				preempt(self);
12117c478bd9Sstevel@tonic-gate 				return (0);
12127c478bd9Sstevel@tonic-gate 			}
12137c478bd9Sstevel@tonic-gate 			SMT_PAUSE();
12147c478bd9Sstevel@tonic-gate 		}
12157c478bd9Sstevel@tonic-gate 
12167c478bd9Sstevel@tonic-gate 		/*
12177c478bd9Sstevel@tonic-gate 		 * No one grabbed the lock.
12187c478bd9Sstevel@tonic-gate 		 * Wake up some lwp that is waiting for it.
12197c478bd9Sstevel@tonic-gate 		 */
12207c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = 0;
12217c478bd9Sstevel@tonic-gate 		lwpid = mutex_wakeup(mp);
12227c478bd9Sstevel@tonic-gate 	}
12237c478bd9Sstevel@tonic-gate 
12247c478bd9Sstevel@tonic-gate 	if (lwpid == 0)
12257c478bd9Sstevel@tonic-gate 		preempt(self);
12267c478bd9Sstevel@tonic-gate 	return (lwpid);
12277c478bd9Sstevel@tonic-gate }
12287c478bd9Sstevel@tonic-gate 
12297c478bd9Sstevel@tonic-gate /*
12307c478bd9Sstevel@tonic-gate  * Like mutex_unlock_queue(), but for process-shared mutexes.
12317c478bd9Sstevel@tonic-gate  * We tested the waiters field before calling here and it was non-zero.
12327c478bd9Sstevel@tonic-gate  */
12337c478bd9Sstevel@tonic-gate void
12347c478bd9Sstevel@tonic-gate mutex_unlock_process(mutex_t *mp)
12357c478bd9Sstevel@tonic-gate {
12367c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
12377c478bd9Sstevel@tonic-gate 	int count;
12387c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
12397c478bd9Sstevel@tonic-gate 
12407c478bd9Sstevel@tonic-gate 	/*
12417c478bd9Sstevel@tonic-gate 	 * See the comments in mutex_unlock_queue(), above.
12427c478bd9Sstevel@tonic-gate 	 */
12437c478bd9Sstevel@tonic-gate 	if ((count = ncpus) == 0)
12447c478bd9Sstevel@tonic-gate 		count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
12457c478bd9Sstevel@tonic-gate 	count = (count > 1)? self->ul_release_spin : 0;
12467c478bd9Sstevel@tonic-gate 	no_preempt(self);
12477c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
12487c478bd9Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
12497c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
12507c478bd9Sstevel@tonic-gate 	if (count == 0) {
12517c478bd9Sstevel@tonic-gate 		/* clear lock, test waiter */
12527c478bd9Sstevel@tonic-gate 		if (!(swap32(&mp->mutex_lockword, 0) & WAITERMASK)) {
12537c478bd9Sstevel@tonic-gate 			/* no waiters now */
12547c478bd9Sstevel@tonic-gate 			preempt(self);
12557c478bd9Sstevel@tonic-gate 			return;
12567c478bd9Sstevel@tonic-gate 		}
12577c478bd9Sstevel@tonic-gate 	} else {
12587c478bd9Sstevel@tonic-gate 		/* clear lock, retain waiter */
12597c478bd9Sstevel@tonic-gate 		(void) swap32(&mp->mutex_lockword, WAITER);
12607c478bd9Sstevel@tonic-gate 		lockp = (volatile uint8_t *)&mp->mutex_lockw;
12617c478bd9Sstevel@tonic-gate 		while (--count >= 0) {
12627c478bd9Sstevel@tonic-gate 			if (*lockp != 0) {
12637c478bd9Sstevel@tonic-gate 				/* somebody grabbed the lock */
12647c478bd9Sstevel@tonic-gate 				preempt(self);
12657c478bd9Sstevel@tonic-gate 				return;
12667c478bd9Sstevel@tonic-gate 			}
12677c478bd9Sstevel@tonic-gate 			SMT_PAUSE();
12687c478bd9Sstevel@tonic-gate 		}
12697c478bd9Sstevel@tonic-gate 		/*
12707c478bd9Sstevel@tonic-gate 		 * We must clear the waiters field before going
12717c478bd9Sstevel@tonic-gate 		 * to the kernel, else it could remain set forever.
12727c478bd9Sstevel@tonic-gate 		 */
12737c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = 0;
12747c478bd9Sstevel@tonic-gate 	}
12757c478bd9Sstevel@tonic-gate 	(void) ___lwp_mutex_wakeup(mp);
12767c478bd9Sstevel@tonic-gate 	preempt(self);
12777c478bd9Sstevel@tonic-gate }
12787c478bd9Sstevel@tonic-gate 
12797c478bd9Sstevel@tonic-gate /*
12807c478bd9Sstevel@tonic-gate  * Return the real priority of a thread.
12817c478bd9Sstevel@tonic-gate  */
12827c478bd9Sstevel@tonic-gate int
12837c478bd9Sstevel@tonic-gate real_priority(ulwp_t *ulwp)
12847c478bd9Sstevel@tonic-gate {
12857c478bd9Sstevel@tonic-gate 	if (ulwp->ul_epri == 0)
12867c478bd9Sstevel@tonic-gate 		return (ulwp->ul_mappedpri? ulwp->ul_mappedpri : ulwp->ul_pri);
12877c478bd9Sstevel@tonic-gate 	return (ulwp->ul_emappedpri? ulwp->ul_emappedpri : ulwp->ul_epri);
12887c478bd9Sstevel@tonic-gate }
12897c478bd9Sstevel@tonic-gate 
12907c478bd9Sstevel@tonic-gate void
12917c478bd9Sstevel@tonic-gate stall(void)
12927c478bd9Sstevel@tonic-gate {
12937c478bd9Sstevel@tonic-gate 	for (;;)
12947c478bd9Sstevel@tonic-gate 		(void) mutex_lock_kernel(&stall_mutex, NULL, NULL);
12957c478bd9Sstevel@tonic-gate }
12967c478bd9Sstevel@tonic-gate 
12977c478bd9Sstevel@tonic-gate /*
12987c478bd9Sstevel@tonic-gate  * Acquire a USYNC_THREAD mutex via user-level sleep queues.
12997c478bd9Sstevel@tonic-gate  * We failed set_lock_byte(&mp->mutex_lockw) before coming here.
13007c478bd9Sstevel@tonic-gate  * Returns with mutex_owner set correctly.
13017c478bd9Sstevel@tonic-gate  */
13027c478bd9Sstevel@tonic-gate int
13037c478bd9Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
13047c478bd9Sstevel@tonic-gate 	timespec_t *tsp)
13057c478bd9Sstevel@tonic-gate {
13067c478bd9Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
13077c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
13087c478bd9Sstevel@tonic-gate 	hrtime_t begin_sleep;
13097c478bd9Sstevel@tonic-gate 	int error = 0;
13107c478bd9Sstevel@tonic-gate 
13117c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
13127c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
13137c478bd9Sstevel@tonic-gate 		self->ul_wchan = mp;
13147c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
13157c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = mp;
13167c478bd9Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
13177c478bd9Sstevel@tonic-gate 	}
13187c478bd9Sstevel@tonic-gate 	if (msp) {
13197c478bd9Sstevel@tonic-gate 		tdb_incr(msp->mutex_sleep);
13207c478bd9Sstevel@tonic-gate 		begin_sleep = gethrtime();
13217c478bd9Sstevel@tonic-gate 	}
13227c478bd9Sstevel@tonic-gate 
13237c478bd9Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__block, mp);
13247c478bd9Sstevel@tonic-gate 
13257c478bd9Sstevel@tonic-gate 	/*
13267c478bd9Sstevel@tonic-gate 	 * Put ourself on the sleep queue, and while we are
13277c478bd9Sstevel@tonic-gate 	 * unable to grab the lock, go park in the kernel.
13287c478bd9Sstevel@tonic-gate 	 * Take ourself off the sleep queue after we acquire the lock.
13297c478bd9Sstevel@tonic-gate 	 * The waiter bit can be set/cleared only while holding the queue lock.
13307c478bd9Sstevel@tonic-gate 	 */
13317c478bd9Sstevel@tonic-gate 	qp = queue_lock(mp, MX);
13327c478bd9Sstevel@tonic-gate 	enqueue(qp, self, mp, MX);
13337c478bd9Sstevel@tonic-gate 	mp->mutex_waiters = 1;
13347c478bd9Sstevel@tonic-gate 	for (;;) {
13357c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
13367c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
13377c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
13387c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
13397c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = dequeue_self(qp, mp);
13407c478bd9Sstevel@tonic-gate 			break;
13417c478bd9Sstevel@tonic-gate 		}
13427c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 1);
13437c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
13447c478bd9Sstevel@tonic-gate 		/*
13457c478bd9Sstevel@tonic-gate 		 * __lwp_park() will return the residual time in tsp
13467c478bd9Sstevel@tonic-gate 		 * if we are unparked before the timeout expires.
13477c478bd9Sstevel@tonic-gate 		 */
13487c478bd9Sstevel@tonic-gate 		if ((error = __lwp_park(tsp, 0)) == EINTR)
13497c478bd9Sstevel@tonic-gate 			error = 0;
13507c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 0);
13517c478bd9Sstevel@tonic-gate 		/*
13527c478bd9Sstevel@tonic-gate 		 * We could have taken a signal or suspended ourself.
13537c478bd9Sstevel@tonic-gate 		 * If we did, then we removed ourself from the queue.
13547c478bd9Sstevel@tonic-gate 		 * Someone else may have removed us from the queue
13557c478bd9Sstevel@tonic-gate 		 * as a consequence of mutex_unlock().  We may have
13567c478bd9Sstevel@tonic-gate 		 * gotten a timeout from __lwp_park().  Or we may still
13577c478bd9Sstevel@tonic-gate 		 * be on the queue and this is just a spurious wakeup.
13587c478bd9Sstevel@tonic-gate 		 */
13597c478bd9Sstevel@tonic-gate 		qp = queue_lock(mp, MX);
13607c478bd9Sstevel@tonic-gate 		if (self->ul_sleepq == NULL) {
13617c478bd9Sstevel@tonic-gate 			if (error) {
13627c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
13637c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
13647c478bd9Sstevel@tonic-gate 				    error);
13657c478bd9Sstevel@tonic-gate 				break;
13667c478bd9Sstevel@tonic-gate 			}
13677c478bd9Sstevel@tonic-gate 			if (set_lock_byte(&mp->mutex_lockw) == 0) {
13687c478bd9Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
13697c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
13707c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
13717c478bd9Sstevel@tonic-gate 				    0, 0);
13727c478bd9Sstevel@tonic-gate 				break;
13737c478bd9Sstevel@tonic-gate 			}
13747c478bd9Sstevel@tonic-gate 			enqueue(qp, self, mp, MX);
13757c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = 1;
13767c478bd9Sstevel@tonic-gate 		}
13777c478bd9Sstevel@tonic-gate 		ASSERT(self->ul_sleepq == qp &&
13787c478bd9Sstevel@tonic-gate 		    self->ul_qtype == MX &&
13797c478bd9Sstevel@tonic-gate 		    self->ul_wchan == mp);
13807c478bd9Sstevel@tonic-gate 		if (error) {
13817c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = dequeue_self(qp, mp);
13827c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
13837c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
13847c478bd9Sstevel@tonic-gate 			break;
13857c478bd9Sstevel@tonic-gate 		}
13867c478bd9Sstevel@tonic-gate 	}
13877c478bd9Sstevel@tonic-gate 
13887c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
13897c478bd9Sstevel@tonic-gate 	    self->ul_wchan == NULL);
13907c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
13917c478bd9Sstevel@tonic-gate 
13927c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
13937c478bd9Sstevel@tonic-gate 	if (msp)
13947c478bd9Sstevel@tonic-gate 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
13957c478bd9Sstevel@tonic-gate 
13967c478bd9Sstevel@tonic-gate 	ASSERT(error == 0 || error == EINVAL || error == ETIME);
13977c478bd9Sstevel@tonic-gate 	return (error);
13987c478bd9Sstevel@tonic-gate }
13997c478bd9Sstevel@tonic-gate 
14007c478bd9Sstevel@tonic-gate /*
14017c478bd9Sstevel@tonic-gate  * Returns with mutex_owner set correctly.
14027c478bd9Sstevel@tonic-gate  */
14037c478bd9Sstevel@tonic-gate int
14047c478bd9Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try)
14057c478bd9Sstevel@tonic-gate {
14067c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
14077c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
14087c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
14097c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
14107c478bd9Sstevel@tonic-gate 	int error = 0;
14117c478bd9Sstevel@tonic-gate 
14127c478bd9Sstevel@tonic-gate 	ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
14137c478bd9Sstevel@tonic-gate 
14147c478bd9Sstevel@tonic-gate 	if (!self->ul_schedctl_called)
14157c478bd9Sstevel@tonic-gate 		(void) setup_schedctl();
14167c478bd9Sstevel@tonic-gate 
14177c478bd9Sstevel@tonic-gate 	if (msp && try == MUTEX_TRY)
14187c478bd9Sstevel@tonic-gate 		tdb_incr(msp->mutex_try);
14197c478bd9Sstevel@tonic-gate 
14207c478bd9Sstevel@tonic-gate 	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) {
14217c478bd9Sstevel@tonic-gate 		if (mtype & LOCK_RECURSIVE) {
14227c478bd9Sstevel@tonic-gate 			if (mp->mutex_rcount == RECURSION_MAX) {
14237c478bd9Sstevel@tonic-gate 				error = EAGAIN;
14247c478bd9Sstevel@tonic-gate 			} else {
14257c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
14267c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
14277c478bd9Sstevel@tonic-gate 				    1, 0);
14287c478bd9Sstevel@tonic-gate 				return (0);
14297c478bd9Sstevel@tonic-gate 			}
14307c478bd9Sstevel@tonic-gate 		} else if (try == MUTEX_TRY) {
14317c478bd9Sstevel@tonic-gate 			return (EBUSY);
14327c478bd9Sstevel@tonic-gate 		} else {
14337c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
14347c478bd9Sstevel@tonic-gate 			return (EDEADLK);
14357c478bd9Sstevel@tonic-gate 		}
14367c478bd9Sstevel@tonic-gate 	}
14377c478bd9Sstevel@tonic-gate 
14387c478bd9Sstevel@tonic-gate 	if (self->ul_error_detection && try == MUTEX_LOCK &&
14397c478bd9Sstevel@tonic-gate 	    tsp == NULL && mutex_is_held(mp))
14407c478bd9Sstevel@tonic-gate 		lock_error(mp, "mutex_lock", NULL, NULL);
14417c478bd9Sstevel@tonic-gate 
14427c478bd9Sstevel@tonic-gate 	if (mtype &
14437c478bd9Sstevel@tonic-gate 	    (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) {
14447c478bd9Sstevel@tonic-gate 		uint8_t ceil;
14457c478bd9Sstevel@tonic-gate 		int myprio;
14467c478bd9Sstevel@tonic-gate 
14477c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_PROTECT) {
14487c478bd9Sstevel@tonic-gate 			ceil = mp->mutex_ceiling;
14497c478bd9Sstevel@tonic-gate 			ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0);
14507c478bd9Sstevel@tonic-gate 			myprio = real_priority(self);
14517c478bd9Sstevel@tonic-gate 			if (myprio > ceil) {
14527c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
14537c478bd9Sstevel@tonic-gate 				    EINVAL);
14547c478bd9Sstevel@tonic-gate 				return (EINVAL);
14557c478bd9Sstevel@tonic-gate 			}
14567c478bd9Sstevel@tonic-gate 			if ((error = _ceil_mylist_add(mp)) != 0) {
14577c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
14587c478bd9Sstevel@tonic-gate 				    error);
14597c478bd9Sstevel@tonic-gate 				return (error);
14607c478bd9Sstevel@tonic-gate 			}
14617c478bd9Sstevel@tonic-gate 			if (myprio < ceil)
14627c478bd9Sstevel@tonic-gate 				_ceil_prio_inherit(ceil);
14637c478bd9Sstevel@tonic-gate 		}
14647c478bd9Sstevel@tonic-gate 
14657c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_INHERIT) {
14667c478bd9Sstevel@tonic-gate 			/* go straight to the kernel */
14677c478bd9Sstevel@tonic-gate 			if (try == MUTEX_TRY)
14687c478bd9Sstevel@tonic-gate 				error = mutex_trylock_kernel(mp);
14697c478bd9Sstevel@tonic-gate 			else	/* MUTEX_LOCK */
14707c478bd9Sstevel@tonic-gate 				error = mutex_lock_kernel(mp, tsp, msp);
14717c478bd9Sstevel@tonic-gate 			/*
14727c478bd9Sstevel@tonic-gate 			 * The kernel never sets or clears the lock byte
14737c478bd9Sstevel@tonic-gate 			 * for PTHREAD_PRIO_INHERIT mutexes.
14747c478bd9Sstevel@tonic-gate 			 * Set it here for debugging consistency.
14757c478bd9Sstevel@tonic-gate 			 */
14767c478bd9Sstevel@tonic-gate 			switch (error) {
14777c478bd9Sstevel@tonic-gate 			case 0:
14787c478bd9Sstevel@tonic-gate 			case EOWNERDEAD:
14797c478bd9Sstevel@tonic-gate 				mp->mutex_lockw = LOCKSET;
14807c478bd9Sstevel@tonic-gate 				break;
14817c478bd9Sstevel@tonic-gate 			}
14827c478bd9Sstevel@tonic-gate 		} else if (mtype & USYNC_PROCESS_ROBUST) {
14837c478bd9Sstevel@tonic-gate 			/* go straight to the kernel */
14847c478bd9Sstevel@tonic-gate 			if (try == MUTEX_TRY)
14857c478bd9Sstevel@tonic-gate 				error = mutex_trylock_kernel(mp);
14867c478bd9Sstevel@tonic-gate 			else	/* MUTEX_LOCK */
14877c478bd9Sstevel@tonic-gate 				error = mutex_lock_kernel(mp, tsp, msp);
14887c478bd9Sstevel@tonic-gate 		} else {	/* PTHREAD_PRIO_PROTECT */
14897c478bd9Sstevel@tonic-gate 			/*
14907c478bd9Sstevel@tonic-gate 			 * Try once at user level before going to the kernel.
14917c478bd9Sstevel@tonic-gate 			 * If this is a process shared mutex then protect
14927c478bd9Sstevel@tonic-gate 			 * against forkall() while setting mp->mutex_ownerpid.
14937c478bd9Sstevel@tonic-gate 			 */
14947c478bd9Sstevel@tonic-gate 			if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
14957c478bd9Sstevel@tonic-gate 				enter_critical(self);
14967c478bd9Sstevel@tonic-gate 				if (set_lock_byte(&mp->mutex_lockw) == 0) {
14977c478bd9Sstevel@tonic-gate 					mp->mutex_owner = (uintptr_t)self;
14987c478bd9Sstevel@tonic-gate 					mp->mutex_ownerpid = udp->pid;
14997c478bd9Sstevel@tonic-gate 					exit_critical(self);
15007c478bd9Sstevel@tonic-gate 					DTRACE_PROBE3(plockstat,
15017c478bd9Sstevel@tonic-gate 					    mutex__acquire, mp, 0, 0);
15027c478bd9Sstevel@tonic-gate 				} else {
15037c478bd9Sstevel@tonic-gate 					exit_critical(self);
15047c478bd9Sstevel@tonic-gate 					error = EBUSY;
15057c478bd9Sstevel@tonic-gate 				}
15067c478bd9Sstevel@tonic-gate 			} else {
15077c478bd9Sstevel@tonic-gate 				if (set_lock_byte(&mp->mutex_lockw) == 0) {
15087c478bd9Sstevel@tonic-gate 					mp->mutex_owner = (uintptr_t)self;
15097c478bd9Sstevel@tonic-gate 					DTRACE_PROBE3(plockstat,
15107c478bd9Sstevel@tonic-gate 					    mutex__acquire, mp, 0, 0);
15117c478bd9Sstevel@tonic-gate 				} else {
15127c478bd9Sstevel@tonic-gate 					error = EBUSY;
15137c478bd9Sstevel@tonic-gate 				}
15147c478bd9Sstevel@tonic-gate 			}
15157c478bd9Sstevel@tonic-gate 			if (error && try == MUTEX_LOCK)
15167c478bd9Sstevel@tonic-gate 				error = mutex_lock_kernel(mp, tsp, msp);
15177c478bd9Sstevel@tonic-gate 		}
15187c478bd9Sstevel@tonic-gate 
15197c478bd9Sstevel@tonic-gate 		if (error) {
15207c478bd9Sstevel@tonic-gate 			if (mtype & PTHREAD_PRIO_INHERIT) {
15217c478bd9Sstevel@tonic-gate 				switch (error) {
15227c478bd9Sstevel@tonic-gate 				case EOWNERDEAD:
15237c478bd9Sstevel@tonic-gate 				case ENOTRECOVERABLE:
15247c478bd9Sstevel@tonic-gate 					if (mtype & PTHREAD_MUTEX_ROBUST_NP)
15257c478bd9Sstevel@tonic-gate 						break;
15267c478bd9Sstevel@tonic-gate 					if (error == EOWNERDEAD) {
15277c478bd9Sstevel@tonic-gate 						/*
15287c478bd9Sstevel@tonic-gate 						 * We own the mutex; unlock it.
15297c478bd9Sstevel@tonic-gate 						 * It becomes ENOTRECOVERABLE.
15307c478bd9Sstevel@tonic-gate 						 * All waiters are waked up.
15317c478bd9Sstevel@tonic-gate 						 */
15327c478bd9Sstevel@tonic-gate 						mp->mutex_owner = 0;
15337c478bd9Sstevel@tonic-gate 						mp->mutex_ownerpid = 0;
15347c478bd9Sstevel@tonic-gate 						DTRACE_PROBE2(plockstat,
15357c478bd9Sstevel@tonic-gate 						    mutex__release, mp, 0);
15367c478bd9Sstevel@tonic-gate 						mp->mutex_lockw = LOCKCLEAR;
15377c478bd9Sstevel@tonic-gate 						(void) ___lwp_mutex_unlock(mp);
15387c478bd9Sstevel@tonic-gate 					}
15397c478bd9Sstevel@tonic-gate 					/* FALLTHROUGH */
15407c478bd9Sstevel@tonic-gate 				case EDEADLK:
15417c478bd9Sstevel@tonic-gate 					if (try == MUTEX_LOCK)
15427c478bd9Sstevel@tonic-gate 						stall();
15437c478bd9Sstevel@tonic-gate 					error = EBUSY;
15447c478bd9Sstevel@tonic-gate 					break;
15457c478bd9Sstevel@tonic-gate 				}
15467c478bd9Sstevel@tonic-gate 			}
15477c478bd9Sstevel@tonic-gate 			if ((mtype & PTHREAD_PRIO_PROTECT) &&
15487c478bd9Sstevel@tonic-gate 			    error != EOWNERDEAD) {
15497c478bd9Sstevel@tonic-gate 				(void) _ceil_mylist_del(mp);
15507c478bd9Sstevel@tonic-gate 				if (myprio < ceil)
15517c478bd9Sstevel@tonic-gate 					_ceil_prio_waive();
15527c478bd9Sstevel@tonic-gate 			}
15537c478bd9Sstevel@tonic-gate 		}
15547c478bd9Sstevel@tonic-gate 	} else if (mtype & USYNC_PROCESS) {
15557c478bd9Sstevel@tonic-gate 		/*
15567c478bd9Sstevel@tonic-gate 		 * This is a process shared mutex.  Protect against
15577c478bd9Sstevel@tonic-gate 		 * forkall() while setting mp->mutex_ownerpid.
15587c478bd9Sstevel@tonic-gate 		 */
15597c478bd9Sstevel@tonic-gate 		enter_critical(self);
15607c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
15617c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
15627c478bd9Sstevel@tonic-gate 			mp->mutex_ownerpid = udp->pid;
15637c478bd9Sstevel@tonic-gate 			exit_critical(self);
15647c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
15657c478bd9Sstevel@tonic-gate 		} else {
15667c478bd9Sstevel@tonic-gate 			/* try a little harder */
15677c478bd9Sstevel@tonic-gate 			exit_critical(self);
15687c478bd9Sstevel@tonic-gate 			error = mutex_trylock_process(mp);
15697c478bd9Sstevel@tonic-gate 		}
15707c478bd9Sstevel@tonic-gate 		if (error && try == MUTEX_LOCK)
15717c478bd9Sstevel@tonic-gate 			error = mutex_lock_kernel(mp, tsp, msp);
15727c478bd9Sstevel@tonic-gate 	} else  {	/* USYNC_THREAD */
15737c478bd9Sstevel@tonic-gate 		/* try once */
15747c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
15757c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
15767c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
15777c478bd9Sstevel@tonic-gate 		} else {
15787c478bd9Sstevel@tonic-gate 			/* try a little harder if we don't own the mutex */
15797c478bd9Sstevel@tonic-gate 			error = EBUSY;
15807c478bd9Sstevel@tonic-gate 			if (MUTEX_OWNER(mp) != self)
15817c478bd9Sstevel@tonic-gate 				error = mutex_trylock_adaptive(mp);
15827c478bd9Sstevel@tonic-gate 			if (error && try == MUTEX_LOCK)		/* go park */
15837c478bd9Sstevel@tonic-gate 				error = mutex_lock_queue(self, msp, mp, tsp);
15847c478bd9Sstevel@tonic-gate 		}
15857c478bd9Sstevel@tonic-gate 	}
15867c478bd9Sstevel@tonic-gate 
15877c478bd9Sstevel@tonic-gate 	switch (error) {
15887c478bd9Sstevel@tonic-gate 	case EOWNERDEAD:
15897c478bd9Sstevel@tonic-gate 	case ELOCKUNMAPPED:
15907c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
15917c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
15927c478bd9Sstevel@tonic-gate 		/* FALLTHROUGH */
15937c478bd9Sstevel@tonic-gate 	case 0:
15947c478bd9Sstevel@tonic-gate 		if (msp)
15957c478bd9Sstevel@tonic-gate 			record_begin_hold(msp);
15967c478bd9Sstevel@tonic-gate 		break;
15977c478bd9Sstevel@tonic-gate 	default:
15987c478bd9Sstevel@tonic-gate 		if (try == MUTEX_TRY) {
15997c478bd9Sstevel@tonic-gate 			if (msp)
16007c478bd9Sstevel@tonic-gate 				tdb_incr(msp->mutex_try_fail);
16017c478bd9Sstevel@tonic-gate 			if (__td_event_report(self, TD_LOCK_TRY, udp)) {
16027c478bd9Sstevel@tonic-gate 				self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
16037c478bd9Sstevel@tonic-gate 				tdb_event(TD_LOCK_TRY, udp);
16047c478bd9Sstevel@tonic-gate 			}
16057c478bd9Sstevel@tonic-gate 		}
16067c478bd9Sstevel@tonic-gate 		break;
16077c478bd9Sstevel@tonic-gate 	}
16087c478bd9Sstevel@tonic-gate 
16097c478bd9Sstevel@tonic-gate 	return (error);
16107c478bd9Sstevel@tonic-gate }
16117c478bd9Sstevel@tonic-gate 
16127c478bd9Sstevel@tonic-gate int
16137c478bd9Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try)
16147c478bd9Sstevel@tonic-gate {
16157c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
16167c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
16177c478bd9Sstevel@tonic-gate 
16187c478bd9Sstevel@tonic-gate 	/*
16197c478bd9Sstevel@tonic-gate 	 * We know that USYNC_PROCESS is set in mtype and that
16207c478bd9Sstevel@tonic-gate 	 * zero, one, or both of the flags LOCK_RECURSIVE and
16217c478bd9Sstevel@tonic-gate 	 * LOCK_ERRORCHECK are set, and that no other flags are set.
16227c478bd9Sstevel@tonic-gate 	 */
16237c478bd9Sstevel@tonic-gate 	enter_critical(self);
16247c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
16257c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
16267c478bd9Sstevel@tonic-gate 		mp->mutex_ownerpid = udp->pid;
16277c478bd9Sstevel@tonic-gate 		exit_critical(self);
16287c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
16297c478bd9Sstevel@tonic-gate 		return (0);
16307c478bd9Sstevel@tonic-gate 	}
16317c478bd9Sstevel@tonic-gate 	exit_critical(self);
16327c478bd9Sstevel@tonic-gate 
16337c478bd9Sstevel@tonic-gate 	if ((mtype & ~USYNC_PROCESS) && shared_mutex_held(mp)) {
16347c478bd9Sstevel@tonic-gate 		if (mtype & LOCK_RECURSIVE) {
16357c478bd9Sstevel@tonic-gate 			if (mp->mutex_rcount == RECURSION_MAX)
16367c478bd9Sstevel@tonic-gate 				return (EAGAIN);
16377c478bd9Sstevel@tonic-gate 			mp->mutex_rcount++;
16387c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0);
16397c478bd9Sstevel@tonic-gate 			return (0);
16407c478bd9Sstevel@tonic-gate 		}
16417c478bd9Sstevel@tonic-gate 		if (try == MUTEX_LOCK) {
16427c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
16437c478bd9Sstevel@tonic-gate 			return (EDEADLK);
16447c478bd9Sstevel@tonic-gate 		}
16457c478bd9Sstevel@tonic-gate 		return (EBUSY);
16467c478bd9Sstevel@tonic-gate 	}
16477c478bd9Sstevel@tonic-gate 
16487c478bd9Sstevel@tonic-gate 	/* try a little harder if we don't own the mutex */
16497c478bd9Sstevel@tonic-gate 	if (!shared_mutex_held(mp) && mutex_trylock_process(mp) == 0)
16507c478bd9Sstevel@tonic-gate 		return (0);
16517c478bd9Sstevel@tonic-gate 
16527c478bd9Sstevel@tonic-gate 	if (try == MUTEX_LOCK)
16537c478bd9Sstevel@tonic-gate 		return (mutex_lock_kernel(mp, tsp, NULL));
16547c478bd9Sstevel@tonic-gate 
16557c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_LOCK_TRY, udp)) {
16567c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
16577c478bd9Sstevel@tonic-gate 		tdb_event(TD_LOCK_TRY, udp);
16587c478bd9Sstevel@tonic-gate 	}
16597c478bd9Sstevel@tonic-gate 	return (EBUSY);
16607c478bd9Sstevel@tonic-gate }
16617c478bd9Sstevel@tonic-gate 
16627c478bd9Sstevel@tonic-gate static int
16637c478bd9Sstevel@tonic-gate slow_lock(ulwp_t *self, mutex_t *mp, timespec_t *tsp)
16647c478bd9Sstevel@tonic-gate {
16657c478bd9Sstevel@tonic-gate 	int error = 0;
16667c478bd9Sstevel@tonic-gate 
16677c478bd9Sstevel@tonic-gate 	if (MUTEX_OWNER(mp) == self || mutex_trylock_adaptive(mp) != 0)
16687c478bd9Sstevel@tonic-gate 		error = mutex_lock_queue(self, NULL, mp, tsp);
16697c478bd9Sstevel@tonic-gate 	return (error);
16707c478bd9Sstevel@tonic-gate }
16717c478bd9Sstevel@tonic-gate 
16727c478bd9Sstevel@tonic-gate int
16737c478bd9Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
16747c478bd9Sstevel@tonic-gate {
16757c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
16767c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
16777c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
16787c478bd9Sstevel@tonic-gate 	int mtype;
16797c478bd9Sstevel@tonic-gate 
16807c478bd9Sstevel@tonic-gate 	/*
16817c478bd9Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
16827c478bd9Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
16837c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics,
16847c478bd9Sstevel@tonic-gate 	 * and the process has only a single thread.
16857c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
16867c478bd9Sstevel@tonic-gate 	 */
16877c478bd9Sstevel@tonic-gate 	if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
16887c478bd9Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
16897c478bd9Sstevel@tonic-gate 		/*
16907c478bd9Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
16917c478bd9Sstevel@tonic-gate 		 */
16927c478bd9Sstevel@tonic-gate 		if (mp->mutex_lockw == 0) {
16937c478bd9Sstevel@tonic-gate 			mp->mutex_lockw = LOCKSET;
16947c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
16957c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
16967c478bd9Sstevel@tonic-gate 			return (0);
16977c478bd9Sstevel@tonic-gate 		}
16987c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
16997c478bd9Sstevel@tonic-gate 			/*
17007c478bd9Sstevel@tonic-gate 			 * LOCK_RECURSIVE, LOCK_ERRORCHECK, or both.
17017c478bd9Sstevel@tonic-gate 			 */
17027c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
17037c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
17047c478bd9Sstevel@tonic-gate 					return (EAGAIN);
17057c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
17067c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
17077c478bd9Sstevel@tonic-gate 				    1, 0);
17087c478bd9Sstevel@tonic-gate 				return (0);
17097c478bd9Sstevel@tonic-gate 			}
17107c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
17117c478bd9Sstevel@tonic-gate 			return (EDEADLK);	/* LOCK_ERRORCHECK */
17127c478bd9Sstevel@tonic-gate 		}
17137c478bd9Sstevel@tonic-gate 		/*
17147c478bd9Sstevel@tonic-gate 		 * We have reached a deadlock, probably because the
17157c478bd9Sstevel@tonic-gate 		 * process is executing non-async-signal-safe code in
17167c478bd9Sstevel@tonic-gate 		 * a signal handler and is attempting to acquire a lock
17177c478bd9Sstevel@tonic-gate 		 * that it already owns.  This is not surprising, given
17187c478bd9Sstevel@tonic-gate 		 * bad programming practices over the years that has
17197c478bd9Sstevel@tonic-gate 		 * resulted in applications calling printf() and such
17207c478bd9Sstevel@tonic-gate 		 * in their signal handlers.  Unless the user has told
17217c478bd9Sstevel@tonic-gate 		 * us that the signal handlers are safe by setting:
17227c478bd9Sstevel@tonic-gate 		 *	export _THREAD_ASYNC_SAFE=1
17237c478bd9Sstevel@tonic-gate 		 * we return EDEADLK rather than actually deadlocking.
17247c478bd9Sstevel@tonic-gate 		 */
17257c478bd9Sstevel@tonic-gate 		if (tsp == NULL &&
17267c478bd9Sstevel@tonic-gate 		    MUTEX_OWNER(mp) == self && !self->ul_async_safe) {
17277c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
17287c478bd9Sstevel@tonic-gate 			return (EDEADLK);
17297c478bd9Sstevel@tonic-gate 		}
17307c478bd9Sstevel@tonic-gate 	}
17317c478bd9Sstevel@tonic-gate 
17327c478bd9Sstevel@tonic-gate 	/*
17337c478bd9Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
17347c478bd9Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
17357c478bd9Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
17367c478bd9Sstevel@tonic-gate 	 */
17377c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
17387c478bd9Sstevel@tonic-gate 	    (gflags->uf_trs_ted |
17397c478bd9Sstevel@tonic-gate 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
17407c478bd9Sstevel@tonic-gate 
17417c478bd9Sstevel@tonic-gate 		if (mtype & USYNC_PROCESS)
17427c478bd9Sstevel@tonic-gate 			return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK));
17437c478bd9Sstevel@tonic-gate 
17447c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
17457c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
17467c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
17477c478bd9Sstevel@tonic-gate 			return (0);
17487c478bd9Sstevel@tonic-gate 		}
17497c478bd9Sstevel@tonic-gate 
17507c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
17517c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
17527c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
17537c478bd9Sstevel@tonic-gate 					return (EAGAIN);
17547c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
17557c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
17567c478bd9Sstevel@tonic-gate 				    1, 0);
17577c478bd9Sstevel@tonic-gate 				return (0);
17587c478bd9Sstevel@tonic-gate 			}
17597c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
17607c478bd9Sstevel@tonic-gate 			return (EDEADLK);	/* LOCK_ERRORCHECK */
17617c478bd9Sstevel@tonic-gate 		}
17627c478bd9Sstevel@tonic-gate 
17637c478bd9Sstevel@tonic-gate 		return (slow_lock(self, mp, tsp));
17647c478bd9Sstevel@tonic-gate 	}
17657c478bd9Sstevel@tonic-gate 
17667c478bd9Sstevel@tonic-gate 	/* else do it the long way */
17677c478bd9Sstevel@tonic-gate 	return (mutex_lock_internal(mp, tsp, MUTEX_LOCK));
17687c478bd9Sstevel@tonic-gate }
17697c478bd9Sstevel@tonic-gate 
17707c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_lock = __mutex_lock
17717c478bd9Sstevel@tonic-gate #pragma weak mutex_lock = __mutex_lock
17727c478bd9Sstevel@tonic-gate #pragma weak _mutex_lock = __mutex_lock
17737c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_lock = __mutex_lock
17747c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_lock = __mutex_lock
17757c478bd9Sstevel@tonic-gate int
17767c478bd9Sstevel@tonic-gate __mutex_lock(mutex_t *mp)
17777c478bd9Sstevel@tonic-gate {
17787c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
17797c478bd9Sstevel@tonic-gate 	return (mutex_lock_impl(mp, NULL));
17807c478bd9Sstevel@tonic-gate }
17817c478bd9Sstevel@tonic-gate 
17827c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_timedlock = _pthread_mutex_timedlock
17837c478bd9Sstevel@tonic-gate int
17847c478bd9Sstevel@tonic-gate _pthread_mutex_timedlock(mutex_t *mp, const timespec_t *abstime)
17857c478bd9Sstevel@tonic-gate {
17867c478bd9Sstevel@tonic-gate 	timespec_t tslocal;
17877c478bd9Sstevel@tonic-gate 	int error;
17887c478bd9Sstevel@tonic-gate 
17897c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
17907c478bd9Sstevel@tonic-gate 	abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
17917c478bd9Sstevel@tonic-gate 	error = mutex_lock_impl(mp, &tslocal);
17927c478bd9Sstevel@tonic-gate 	if (error == ETIME)
17937c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
17947c478bd9Sstevel@tonic-gate 	return (error);
17957c478bd9Sstevel@tonic-gate }
17967c478bd9Sstevel@tonic-gate 
17977c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_reltimedlock_np = _pthread_mutex_reltimedlock_np
17987c478bd9Sstevel@tonic-gate int
17997c478bd9Sstevel@tonic-gate _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime)
18007c478bd9Sstevel@tonic-gate {
18017c478bd9Sstevel@tonic-gate 	timespec_t tslocal;
18027c478bd9Sstevel@tonic-gate 	int error;
18037c478bd9Sstevel@tonic-gate 
18047c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
18057c478bd9Sstevel@tonic-gate 	tslocal = *reltime;
18067c478bd9Sstevel@tonic-gate 	error = mutex_lock_impl(mp, &tslocal);
18077c478bd9Sstevel@tonic-gate 	if (error == ETIME)
18087c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
18097c478bd9Sstevel@tonic-gate 	return (error);
18107c478bd9Sstevel@tonic-gate }
18117c478bd9Sstevel@tonic-gate 
18127c478bd9Sstevel@tonic-gate static int
18137c478bd9Sstevel@tonic-gate slow_trylock(mutex_t *mp, ulwp_t *self)
18147c478bd9Sstevel@tonic-gate {
18157c478bd9Sstevel@tonic-gate 	if (MUTEX_OWNER(mp) == self ||
18167c478bd9Sstevel@tonic-gate 	    mutex_trylock_adaptive(mp) != 0) {
18177c478bd9Sstevel@tonic-gate 		uberdata_t *udp = self->ul_uberdata;
18187c478bd9Sstevel@tonic-gate 
18197c478bd9Sstevel@tonic-gate 		if (__td_event_report(self, TD_LOCK_TRY, udp)) {
18207c478bd9Sstevel@tonic-gate 			self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
18217c478bd9Sstevel@tonic-gate 			tdb_event(TD_LOCK_TRY, udp);
18227c478bd9Sstevel@tonic-gate 		}
18237c478bd9Sstevel@tonic-gate 		return (EBUSY);
18247c478bd9Sstevel@tonic-gate 	}
18257c478bd9Sstevel@tonic-gate 	return (0);
18267c478bd9Sstevel@tonic-gate }
18277c478bd9Sstevel@tonic-gate 
18287c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_trylock = __mutex_trylock
18297c478bd9Sstevel@tonic-gate #pragma weak mutex_trylock = __mutex_trylock
18307c478bd9Sstevel@tonic-gate #pragma weak _mutex_trylock = __mutex_trylock
18317c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_trylock = __mutex_trylock
18327c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_trylock = __mutex_trylock
18337c478bd9Sstevel@tonic-gate int
18347c478bd9Sstevel@tonic-gate __mutex_trylock(mutex_t *mp)
18357c478bd9Sstevel@tonic-gate {
18367c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
18377c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
18387c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
18397c478bd9Sstevel@tonic-gate 	int mtype;
18407c478bd9Sstevel@tonic-gate 
18417c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
18427c478bd9Sstevel@tonic-gate 	/*
18437c478bd9Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
18447c478bd9Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
18457c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics,
18467c478bd9Sstevel@tonic-gate 	 * and the process has only a single thread.
18477c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
18487c478bd9Sstevel@tonic-gate 	 */
18497c478bd9Sstevel@tonic-gate 	if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
18507c478bd9Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
18517c478bd9Sstevel@tonic-gate 		/*
18527c478bd9Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
18537c478bd9Sstevel@tonic-gate 		 */
18547c478bd9Sstevel@tonic-gate 		if (mp->mutex_lockw == 0) {
18557c478bd9Sstevel@tonic-gate 			mp->mutex_lockw = LOCKSET;
18567c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
18577c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
18587c478bd9Sstevel@tonic-gate 			return (0);
18597c478bd9Sstevel@tonic-gate 		}
18607c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
18617c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
18627c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
18637c478bd9Sstevel@tonic-gate 					return (EAGAIN);
18647c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
18657c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
18667c478bd9Sstevel@tonic-gate 				    1, 0);
18677c478bd9Sstevel@tonic-gate 				return (0);
18687c478bd9Sstevel@tonic-gate 			}
18697c478bd9Sstevel@tonic-gate 			return (EDEADLK);	/* LOCK_ERRORCHECK */
18707c478bd9Sstevel@tonic-gate 		}
18717c478bd9Sstevel@tonic-gate 		return (EBUSY);
18727c478bd9Sstevel@tonic-gate 	}
18737c478bd9Sstevel@tonic-gate 
18747c478bd9Sstevel@tonic-gate 	/*
18757c478bd9Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
18767c478bd9Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
18777c478bd9Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
18787c478bd9Sstevel@tonic-gate 	 */
18797c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
18807c478bd9Sstevel@tonic-gate 	    (gflags->uf_trs_ted |
18817c478bd9Sstevel@tonic-gate 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
18827c478bd9Sstevel@tonic-gate 
18837c478bd9Sstevel@tonic-gate 		if (mtype & USYNC_PROCESS)
18847c478bd9Sstevel@tonic-gate 			return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY));
18857c478bd9Sstevel@tonic-gate 
18867c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
18877c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
18887c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
18897c478bd9Sstevel@tonic-gate 			return (0);
18907c478bd9Sstevel@tonic-gate 		}
18917c478bd9Sstevel@tonic-gate 
18927c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
18937c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
18947c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
18957c478bd9Sstevel@tonic-gate 					return (EAGAIN);
18967c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
18977c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
18987c478bd9Sstevel@tonic-gate 				    1, 0);
18997c478bd9Sstevel@tonic-gate 				return (0);
19007c478bd9Sstevel@tonic-gate 			}
19017c478bd9Sstevel@tonic-gate 			return (EBUSY);		/* LOCK_ERRORCHECK */
19027c478bd9Sstevel@tonic-gate 		}
19037c478bd9Sstevel@tonic-gate 
19047c478bd9Sstevel@tonic-gate 		return (slow_trylock(mp, self));
19057c478bd9Sstevel@tonic-gate 	}
19067c478bd9Sstevel@tonic-gate 
19077c478bd9Sstevel@tonic-gate 	/* else do it the long way */
19087c478bd9Sstevel@tonic-gate 	return (mutex_lock_internal(mp, NULL, MUTEX_TRY));
19097c478bd9Sstevel@tonic-gate }
19107c478bd9Sstevel@tonic-gate 
19117c478bd9Sstevel@tonic-gate int
19127c478bd9Sstevel@tonic-gate mutex_unlock_internal(mutex_t *mp)
19137c478bd9Sstevel@tonic-gate {
19147c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
19157c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
19167c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
19177c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp;
19187c478bd9Sstevel@tonic-gate 	int error;
19197c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
19207c478bd9Sstevel@tonic-gate 
19217c478bd9Sstevel@tonic-gate 	if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp))
19227c478bd9Sstevel@tonic-gate 		return (EPERM);
19237c478bd9Sstevel@tonic-gate 
19247c478bd9Sstevel@tonic-gate 	if (self->ul_error_detection && !mutex_is_held(mp))
19257c478bd9Sstevel@tonic-gate 		lock_error(mp, "mutex_unlock", NULL, NULL);
19267c478bd9Sstevel@tonic-gate 
19277c478bd9Sstevel@tonic-gate 	if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
19287c478bd9Sstevel@tonic-gate 		mp->mutex_rcount--;
19297c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
19307c478bd9Sstevel@tonic-gate 		return (0);
19317c478bd9Sstevel@tonic-gate 	}
19327c478bd9Sstevel@tonic-gate 
19337c478bd9Sstevel@tonic-gate 	if ((msp = MUTEX_STATS(mp, udp)) != NULL)
19347c478bd9Sstevel@tonic-gate 		(void) record_hold_time(msp);
19357c478bd9Sstevel@tonic-gate 
19367c478bd9Sstevel@tonic-gate 	if (mtype &
19377c478bd9Sstevel@tonic-gate 	    (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) {
19387c478bd9Sstevel@tonic-gate 		no_preempt(self);
19397c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
19407c478bd9Sstevel@tonic-gate 		mp->mutex_ownerpid = 0;
19417c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
19427c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_INHERIT) {
19437c478bd9Sstevel@tonic-gate 			mp->mutex_lockw = LOCKCLEAR;
19447c478bd9Sstevel@tonic-gate 			error = ___lwp_mutex_unlock(mp);
19457c478bd9Sstevel@tonic-gate 		} else if (mtype & USYNC_PROCESS_ROBUST) {
19467c478bd9Sstevel@tonic-gate 			error = ___lwp_mutex_unlock(mp);
19477c478bd9Sstevel@tonic-gate 		} else {
19487c478bd9Sstevel@tonic-gate 			if (swap32(&mp->mutex_lockword, 0) & WAITERMASK)
19497c478bd9Sstevel@tonic-gate 				(void) ___lwp_mutex_wakeup(mp);
19507c478bd9Sstevel@tonic-gate 			error = 0;
19517c478bd9Sstevel@tonic-gate 		}
19527c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_PROTECT) {
19537c478bd9Sstevel@tonic-gate 			if (_ceil_mylist_del(mp))
19547c478bd9Sstevel@tonic-gate 				_ceil_prio_waive();
19557c478bd9Sstevel@tonic-gate 		}
19567c478bd9Sstevel@tonic-gate 		preempt(self);
19577c478bd9Sstevel@tonic-gate 	} else if (mtype & USYNC_PROCESS) {
19587c478bd9Sstevel@tonic-gate 		if (mp->mutex_lockword & WAITERMASK)
19597c478bd9Sstevel@tonic-gate 			mutex_unlock_process(mp);
19607c478bd9Sstevel@tonic-gate 		else {
19617c478bd9Sstevel@tonic-gate 			mp->mutex_owner = 0;
19627c478bd9Sstevel@tonic-gate 			mp->mutex_ownerpid = 0;
19637c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
19647c478bd9Sstevel@tonic-gate 			if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) {
19657c478bd9Sstevel@tonic-gate 				no_preempt(self);
19667c478bd9Sstevel@tonic-gate 				(void) ___lwp_mutex_wakeup(mp);
19677c478bd9Sstevel@tonic-gate 				preempt(self);
19687c478bd9Sstevel@tonic-gate 			}
19697c478bd9Sstevel@tonic-gate 		}
19707c478bd9Sstevel@tonic-gate 		error = 0;
19717c478bd9Sstevel@tonic-gate 	} else {	/* USYNC_THREAD */
19727c478bd9Sstevel@tonic-gate 		if ((lwpid = mutex_unlock_queue(mp)) != 0) {
19737c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid);
19747c478bd9Sstevel@tonic-gate 			preempt(self);
19757c478bd9Sstevel@tonic-gate 		}
19767c478bd9Sstevel@tonic-gate 		error = 0;
19777c478bd9Sstevel@tonic-gate 	}
19787c478bd9Sstevel@tonic-gate 
19797c478bd9Sstevel@tonic-gate 	return (error);
19807c478bd9Sstevel@tonic-gate }
19817c478bd9Sstevel@tonic-gate 
19827c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_unlock = __mutex_unlock
19837c478bd9Sstevel@tonic-gate #pragma weak mutex_unlock = __mutex_unlock
19847c478bd9Sstevel@tonic-gate #pragma weak _mutex_unlock = __mutex_unlock
19857c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_unlock = __mutex_unlock
19867c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_unlock = __mutex_unlock
19877c478bd9Sstevel@tonic-gate int
19887c478bd9Sstevel@tonic-gate __mutex_unlock(mutex_t *mp)
19897c478bd9Sstevel@tonic-gate {
19907c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
19917c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
19927c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
19937c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
19947c478bd9Sstevel@tonic-gate 	int mtype;
19957c478bd9Sstevel@tonic-gate 	short el;
19967c478bd9Sstevel@tonic-gate 
19977c478bd9Sstevel@tonic-gate 	/*
19987c478bd9Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
19997c478bd9Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
20007c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics,
20017c478bd9Sstevel@tonic-gate 	 * and the process has only a single thread.
20027c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
20037c478bd9Sstevel@tonic-gate 	 */
20047c478bd9Sstevel@tonic-gate 	if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
20057c478bd9Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
20067c478bd9Sstevel@tonic-gate 		if (mtype) {
20077c478bd9Sstevel@tonic-gate 			/*
20087c478bd9Sstevel@tonic-gate 			 * At this point we know that one or both of the
20097c478bd9Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
20107c478bd9Sstevel@tonic-gate 			 */
20117c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
20127c478bd9Sstevel@tonic-gate 				return (EPERM);
20137c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
20147c478bd9Sstevel@tonic-gate 				mp->mutex_rcount--;
20157c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
20167c478bd9Sstevel@tonic-gate 				return (0);
20177c478bd9Sstevel@tonic-gate 			}
20187c478bd9Sstevel@tonic-gate 		}
20197c478bd9Sstevel@tonic-gate 		/*
20207c478bd9Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
20217c478bd9Sstevel@tonic-gate 		 * Also, there can be no waiters.
20227c478bd9Sstevel@tonic-gate 		 */
20237c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
20247c478bd9Sstevel@tonic-gate 		mp->mutex_lockword = 0;
20257c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
20267c478bd9Sstevel@tonic-gate 		return (0);
20277c478bd9Sstevel@tonic-gate 	}
20287c478bd9Sstevel@tonic-gate 
20297c478bd9Sstevel@tonic-gate 	/*
20307c478bd9Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
20317c478bd9Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
20327c478bd9Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
20337c478bd9Sstevel@tonic-gate 	 */
20347c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL) {
20357c478bd9Sstevel@tonic-gate 		if (((el = gflags->uf_trs_ted) | mtype) == 0) {
20367c478bd9Sstevel@tonic-gate fast_unlock:
20377c478bd9Sstevel@tonic-gate 			if (!(mp->mutex_lockword & WAITERMASK)) {
20387c478bd9Sstevel@tonic-gate 				/* no waiter exists right now */
20397c478bd9Sstevel@tonic-gate 				mp->mutex_owner = 0;
20407c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
20417c478bd9Sstevel@tonic-gate 				if (swap32(&mp->mutex_lockword, 0) &
20427c478bd9Sstevel@tonic-gate 				    WAITERMASK) {
20437c478bd9Sstevel@tonic-gate 					/* a waiter suddenly appeared */
20447c478bd9Sstevel@tonic-gate 					no_preempt(self);
20457c478bd9Sstevel@tonic-gate 					if ((lwpid = mutex_wakeup(mp)) != 0)
20467c478bd9Sstevel@tonic-gate 						(void) __lwp_unpark(lwpid);
20477c478bd9Sstevel@tonic-gate 					preempt(self);
20487c478bd9Sstevel@tonic-gate 				}
20497c478bd9Sstevel@tonic-gate 			} else if ((lwpid = mutex_unlock_queue(mp)) != 0) {
20507c478bd9Sstevel@tonic-gate 				(void) __lwp_unpark(lwpid);
20517c478bd9Sstevel@tonic-gate 				preempt(self);
20527c478bd9Sstevel@tonic-gate 			}
20537c478bd9Sstevel@tonic-gate 			return (0);
20547c478bd9Sstevel@tonic-gate 		}
20557c478bd9Sstevel@tonic-gate 		if (el)		/* error detection or lock statistics */
20567c478bd9Sstevel@tonic-gate 			goto slow_unlock;
20577c478bd9Sstevel@tonic-gate 		if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
20587c478bd9Sstevel@tonic-gate 			/*
20597c478bd9Sstevel@tonic-gate 			 * At this point we know that one or both of the
20607c478bd9Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
20617c478bd9Sstevel@tonic-gate 			 */
20627c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
20637c478bd9Sstevel@tonic-gate 				return (EPERM);
20647c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
20657c478bd9Sstevel@tonic-gate 				mp->mutex_rcount--;
20667c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
20677c478bd9Sstevel@tonic-gate 				return (0);
20687c478bd9Sstevel@tonic-gate 			}
20697c478bd9Sstevel@tonic-gate 			goto fast_unlock;
20707c478bd9Sstevel@tonic-gate 		}
20717c478bd9Sstevel@tonic-gate 		if ((mtype &
20727c478bd9Sstevel@tonic-gate 		    ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
20737c478bd9Sstevel@tonic-gate 			/*
20747c478bd9Sstevel@tonic-gate 			 * At this point we know that zero, one, or both of the
20757c478bd9Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and
20767c478bd9Sstevel@tonic-gate 			 * that the USYNC_PROCESS flag is set.
20777c478bd9Sstevel@tonic-gate 			 */
20787c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp))
20797c478bd9Sstevel@tonic-gate 				return (EPERM);
20807c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
20817c478bd9Sstevel@tonic-gate 				mp->mutex_rcount--;
20827c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
20837c478bd9Sstevel@tonic-gate 				return (0);
20847c478bd9Sstevel@tonic-gate 			}
20857c478bd9Sstevel@tonic-gate 			if (mp->mutex_lockword & WAITERMASK)
20867c478bd9Sstevel@tonic-gate 				mutex_unlock_process(mp);
20877c478bd9Sstevel@tonic-gate 			else {
20887c478bd9Sstevel@tonic-gate 				mp->mutex_owner = 0;
20897c478bd9Sstevel@tonic-gate 				mp->mutex_ownerpid = 0;
20907c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
20917c478bd9Sstevel@tonic-gate 				if (swap32(&mp->mutex_lockword, 0) &
20927c478bd9Sstevel@tonic-gate 				    WAITERMASK) {
20937c478bd9Sstevel@tonic-gate 					no_preempt(self);
20947c478bd9Sstevel@tonic-gate 					(void) ___lwp_mutex_wakeup(mp);
20957c478bd9Sstevel@tonic-gate 					preempt(self);
20967c478bd9Sstevel@tonic-gate 				}
20977c478bd9Sstevel@tonic-gate 			}
20987c478bd9Sstevel@tonic-gate 			return (0);
20997c478bd9Sstevel@tonic-gate 		}
21007c478bd9Sstevel@tonic-gate 	}
21017c478bd9Sstevel@tonic-gate 
21027c478bd9Sstevel@tonic-gate 	/* else do it the long way */
21037c478bd9Sstevel@tonic-gate slow_unlock:
21047c478bd9Sstevel@tonic-gate 	return (mutex_unlock_internal(mp));
21057c478bd9Sstevel@tonic-gate }
21067c478bd9Sstevel@tonic-gate 
21077c478bd9Sstevel@tonic-gate /*
21087c478bd9Sstevel@tonic-gate  * Internally to the library, almost all mutex lock/unlock actions
21097c478bd9Sstevel@tonic-gate  * go through these lmutex_ functions, to protect critical regions.
21107c478bd9Sstevel@tonic-gate  * We replicate a bit of code from __mutex_lock() and __mutex_unlock()
21117c478bd9Sstevel@tonic-gate  * to make these functions faster since we know that the mutex type
21127c478bd9Sstevel@tonic-gate  * of all internal locks is USYNC_THREAD.  We also know that internal
21137c478bd9Sstevel@tonic-gate  * locking can never fail, so we panic if it does.
21147c478bd9Sstevel@tonic-gate  */
21157c478bd9Sstevel@tonic-gate void
21167c478bd9Sstevel@tonic-gate lmutex_lock(mutex_t *mp)
21177c478bd9Sstevel@tonic-gate {
21187c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
21197c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
21207c478bd9Sstevel@tonic-gate 
21217c478bd9Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
21227c478bd9Sstevel@tonic-gate 
21237c478bd9Sstevel@tonic-gate 	enter_critical(self);
21247c478bd9Sstevel@tonic-gate 	/*
21257c478bd9Sstevel@tonic-gate 	 * Optimize the case of no lock statistics and only a single thread.
21267c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
21277c478bd9Sstevel@tonic-gate 	 */
21287c478bd9Sstevel@tonic-gate 	if (udp->uberflags.uf_all == 0) {
21297c478bd9Sstevel@tonic-gate 		/*
21307c478bd9Sstevel@tonic-gate 		 * Only one thread exists; the mutex must be free.
21317c478bd9Sstevel@tonic-gate 		 */
21327c478bd9Sstevel@tonic-gate 		ASSERT(mp->mutex_lockw == 0);
21337c478bd9Sstevel@tonic-gate 		mp->mutex_lockw = LOCKSET;
21347c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
21357c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
21367c478bd9Sstevel@tonic-gate 	} else {
21377c478bd9Sstevel@tonic-gate 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
21387c478bd9Sstevel@tonic-gate 
21397c478bd9Sstevel@tonic-gate 		if (!self->ul_schedctl_called)
21407c478bd9Sstevel@tonic-gate 			(void) setup_schedctl();
21417c478bd9Sstevel@tonic-gate 
21427c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
21437c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
21447c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
21457c478bd9Sstevel@tonic-gate 		} else if (mutex_trylock_adaptive(mp) != 0) {
21467c478bd9Sstevel@tonic-gate 			(void) mutex_lock_queue(self, msp, mp, NULL);
21477c478bd9Sstevel@tonic-gate 		}
21487c478bd9Sstevel@tonic-gate 
21497c478bd9Sstevel@tonic-gate 		if (msp)
21507c478bd9Sstevel@tonic-gate 			record_begin_hold(msp);
21517c478bd9Sstevel@tonic-gate 	}
21527c478bd9Sstevel@tonic-gate }
21537c478bd9Sstevel@tonic-gate 
21547c478bd9Sstevel@tonic-gate void
21557c478bd9Sstevel@tonic-gate lmutex_unlock(mutex_t *mp)
21567c478bd9Sstevel@tonic-gate {
21577c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
21587c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
21597c478bd9Sstevel@tonic-gate 
21607c478bd9Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
21617c478bd9Sstevel@tonic-gate 
21627c478bd9Sstevel@tonic-gate 	/*
21637c478bd9Sstevel@tonic-gate 	 * Optimize the case of no lock statistics and only a single thread.
21647c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
21657c478bd9Sstevel@tonic-gate 	 */
21667c478bd9Sstevel@tonic-gate 	if (udp->uberflags.uf_all == 0) {
21677c478bd9Sstevel@tonic-gate 		/*
21687c478bd9Sstevel@tonic-gate 		 * Only one thread exists so there can be no waiters.
21697c478bd9Sstevel@tonic-gate 		 */
21707c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
21717c478bd9Sstevel@tonic-gate 		mp->mutex_lockword = 0;
21727c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
21737c478bd9Sstevel@tonic-gate 	} else {
21747c478bd9Sstevel@tonic-gate 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
21757c478bd9Sstevel@tonic-gate 		lwpid_t lwpid;
21767c478bd9Sstevel@tonic-gate 
21777c478bd9Sstevel@tonic-gate 		if (msp)
21787c478bd9Sstevel@tonic-gate 			(void) record_hold_time(msp);
21797c478bd9Sstevel@tonic-gate 		if ((lwpid = mutex_unlock_queue(mp)) != 0) {
21807c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid);
21817c478bd9Sstevel@tonic-gate 			preempt(self);
21827c478bd9Sstevel@tonic-gate 		}
21837c478bd9Sstevel@tonic-gate 	}
21847c478bd9Sstevel@tonic-gate 	exit_critical(self);
21857c478bd9Sstevel@tonic-gate }
21867c478bd9Sstevel@tonic-gate 
21877c478bd9Sstevel@tonic-gate static int
21887c478bd9Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg)
21897c478bd9Sstevel@tonic-gate {
21907c478bd9Sstevel@tonic-gate 	/*
21917c478bd9Sstevel@tonic-gate 	 * There is an inherent data race in the current ownership design.
21927c478bd9Sstevel@tonic-gate 	 * The mutex_owner and mutex_ownerpid fields cannot be set or tested
21937c478bd9Sstevel@tonic-gate 	 * atomically as a pair. The original implementation tested each
21947c478bd9Sstevel@tonic-gate 	 * field just once. This was exposed to trivial false positives in
21957c478bd9Sstevel@tonic-gate 	 * the case of multiple multithreaded processes with thread addresses
21967c478bd9Sstevel@tonic-gate 	 * in common. To close the window to an acceptable level we now use a
21977c478bd9Sstevel@tonic-gate 	 * sequence of five tests: pid-thr-pid-thr-pid. This ensures that any
21987c478bd9Sstevel@tonic-gate 	 * single interruption will still leave one uninterrupted sequence of
21997c478bd9Sstevel@tonic-gate 	 * pid-thr-pid tests intact.
22007c478bd9Sstevel@tonic-gate 	 *
22017c478bd9Sstevel@tonic-gate 	 * It is assumed that all updates are always ordered thr-pid and that
22027c478bd9Sstevel@tonic-gate 	 * we have TSO hardware.
22037c478bd9Sstevel@tonic-gate 	 */
22047c478bd9Sstevel@tonic-gate 	volatile mutex_t *mp = (volatile mutex_t *)mparg;
22057c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
22067c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
22077c478bd9Sstevel@tonic-gate 
22087c478bd9Sstevel@tonic-gate 	if (mp->mutex_ownerpid != udp->pid)
22097c478bd9Sstevel@tonic-gate 		return (0);
22107c478bd9Sstevel@tonic-gate 
22117c478bd9Sstevel@tonic-gate 	if (!MUTEX_OWNED(mp, self))
22127c478bd9Sstevel@tonic-gate 		return (0);
22137c478bd9Sstevel@tonic-gate 
22147c478bd9Sstevel@tonic-gate 	if (mp->mutex_ownerpid != udp->pid)
22157c478bd9Sstevel@tonic-gate 		return (0);
22167c478bd9Sstevel@tonic-gate 
22177c478bd9Sstevel@tonic-gate 	if (!MUTEX_OWNED(mp, self))
22187c478bd9Sstevel@tonic-gate 		return (0);
22197c478bd9Sstevel@tonic-gate 
22207c478bd9Sstevel@tonic-gate 	if (mp->mutex_ownerpid != udp->pid)
22217c478bd9Sstevel@tonic-gate 		return (0);
22227c478bd9Sstevel@tonic-gate 
22237c478bd9Sstevel@tonic-gate 	return (1);
22247c478bd9Sstevel@tonic-gate }
22257c478bd9Sstevel@tonic-gate 
22267c478bd9Sstevel@tonic-gate /*
22277c478bd9Sstevel@tonic-gate  * Some crufty old programs define their own version of _mutex_held()
22287c478bd9Sstevel@tonic-gate  * to be simply return(1).  This breaks internal libc logic, so we
22297c478bd9Sstevel@tonic-gate  * define a private version for exclusive use by libc, mutex_is_held(),
22307c478bd9Sstevel@tonic-gate  * and also a new public function, __mutex_held(), to be used in new
22317c478bd9Sstevel@tonic-gate  * code to circumvent these crufty old programs.
22327c478bd9Sstevel@tonic-gate  */
22337c478bd9Sstevel@tonic-gate #pragma weak mutex_held = mutex_is_held
22347c478bd9Sstevel@tonic-gate #pragma weak _mutex_held = mutex_is_held
22357c478bd9Sstevel@tonic-gate #pragma weak __mutex_held = mutex_is_held
22367c478bd9Sstevel@tonic-gate int
22377c478bd9Sstevel@tonic-gate mutex_is_held(mutex_t *mp)
22387c478bd9Sstevel@tonic-gate {
22397c478bd9Sstevel@tonic-gate 	if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))
22407c478bd9Sstevel@tonic-gate 		return (shared_mutex_held(mp));
22417c478bd9Sstevel@tonic-gate 	return (MUTEX_OWNED(mp, curthread));
22427c478bd9Sstevel@tonic-gate }
22437c478bd9Sstevel@tonic-gate 
22447c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_destroy = __mutex_destroy
22457c478bd9Sstevel@tonic-gate #pragma weak mutex_destroy = __mutex_destroy
22467c478bd9Sstevel@tonic-gate #pragma weak _mutex_destroy = __mutex_destroy
22477c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_destroy = __mutex_destroy
22487c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_destroy = __mutex_destroy
22497c478bd9Sstevel@tonic-gate int
22507c478bd9Sstevel@tonic-gate __mutex_destroy(mutex_t *mp)
22517c478bd9Sstevel@tonic-gate {
22527c478bd9Sstevel@tonic-gate 	mp->mutex_magic = 0;
22537c478bd9Sstevel@tonic-gate 	mp->mutex_flag &= ~LOCK_INITED;
22547c478bd9Sstevel@tonic-gate 	tdb_sync_obj_deregister(mp);
22557c478bd9Sstevel@tonic-gate 	return (0);
22567c478bd9Sstevel@tonic-gate }
22577c478bd9Sstevel@tonic-gate 
22587c478bd9Sstevel@tonic-gate /*
22597c478bd9Sstevel@tonic-gate  * Spin locks are separate from ordinary mutexes,
22607c478bd9Sstevel@tonic-gate  * but we use the same data structure for them.
22617c478bd9Sstevel@tonic-gate  */
22627c478bd9Sstevel@tonic-gate 
22637c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_init = _pthread_spin_init
22647c478bd9Sstevel@tonic-gate int
22657c478bd9Sstevel@tonic-gate _pthread_spin_init(pthread_spinlock_t *lock, int pshared)
22667c478bd9Sstevel@tonic-gate {
22677c478bd9Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
22687c478bd9Sstevel@tonic-gate 
22697c478bd9Sstevel@tonic-gate 	(void) _memset(mp, 0, sizeof (*mp));
22707c478bd9Sstevel@tonic-gate 	if (pshared == PTHREAD_PROCESS_SHARED)
22717c478bd9Sstevel@tonic-gate 		mp->mutex_type = USYNC_PROCESS;
22727c478bd9Sstevel@tonic-gate 	else
22737c478bd9Sstevel@tonic-gate 		mp->mutex_type = USYNC_THREAD;
22747c478bd9Sstevel@tonic-gate 	mp->mutex_flag = LOCK_INITED;
22757c478bd9Sstevel@tonic-gate 	mp->mutex_magic = MUTEX_MAGIC;
22767c478bd9Sstevel@tonic-gate 	return (0);
22777c478bd9Sstevel@tonic-gate }
22787c478bd9Sstevel@tonic-gate 
22797c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_destroy = _pthread_spin_destroy
22807c478bd9Sstevel@tonic-gate int
22817c478bd9Sstevel@tonic-gate _pthread_spin_destroy(pthread_spinlock_t *lock)
22827c478bd9Sstevel@tonic-gate {
22837c478bd9Sstevel@tonic-gate 	(void) _memset(lock, 0, sizeof (*lock));
22847c478bd9Sstevel@tonic-gate 	return (0);
22857c478bd9Sstevel@tonic-gate }
22867c478bd9Sstevel@tonic-gate 
22877c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_trylock = _pthread_spin_trylock
22887c478bd9Sstevel@tonic-gate int
22897c478bd9Sstevel@tonic-gate _pthread_spin_trylock(pthread_spinlock_t *lock)
22907c478bd9Sstevel@tonic-gate {
22917c478bd9Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
22927c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
22937c478bd9Sstevel@tonic-gate 	int error = 0;
22947c478bd9Sstevel@tonic-gate 
22957c478bd9Sstevel@tonic-gate 	no_preempt(self);
22967c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) != 0)
22977c478bd9Sstevel@tonic-gate 		error = EBUSY;
22987c478bd9Sstevel@tonic-gate 	else {
22997c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
23007c478bd9Sstevel@tonic-gate 		if (mp->mutex_type == USYNC_PROCESS)
23017c478bd9Sstevel@tonic-gate 			mp->mutex_ownerpid = self->ul_uberdata->pid;
23027c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
23037c478bd9Sstevel@tonic-gate 	}
23047c478bd9Sstevel@tonic-gate 	preempt(self);
23057c478bd9Sstevel@tonic-gate 	return (error);
23067c478bd9Sstevel@tonic-gate }
23077c478bd9Sstevel@tonic-gate 
23087c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_lock = _pthread_spin_lock
23097c478bd9Sstevel@tonic-gate int
23107c478bd9Sstevel@tonic-gate _pthread_spin_lock(pthread_spinlock_t *lock)
23117c478bd9Sstevel@tonic-gate {
23127c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp =
23137c478bd9Sstevel@tonic-gate 		(volatile uint8_t *)&((mutex_t *)lock)->mutex_lockw;
23147c478bd9Sstevel@tonic-gate 
23157c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
23167c478bd9Sstevel@tonic-gate 	/*
23177c478bd9Sstevel@tonic-gate 	 * We don't care whether the owner is running on a processor.
23187c478bd9Sstevel@tonic-gate 	 * We just spin because that's what this interface requires.
23197c478bd9Sstevel@tonic-gate 	 */
23207c478bd9Sstevel@tonic-gate 	for (;;) {
23217c478bd9Sstevel@tonic-gate 		if (*lockp == 0) {	/* lock byte appears to be clear */
23227c478bd9Sstevel@tonic-gate 			if (_pthread_spin_trylock(lock) == 0)
23237c478bd9Sstevel@tonic-gate 				return (0);
23247c478bd9Sstevel@tonic-gate 		}
23257c478bd9Sstevel@tonic-gate 		SMT_PAUSE();
23267c478bd9Sstevel@tonic-gate 	}
23277c478bd9Sstevel@tonic-gate }
23287c478bd9Sstevel@tonic-gate 
23297c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_unlock = _pthread_spin_unlock
23307c478bd9Sstevel@tonic-gate int
23317c478bd9Sstevel@tonic-gate _pthread_spin_unlock(pthread_spinlock_t *lock)
23327c478bd9Sstevel@tonic-gate {
23337c478bd9Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
23347c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
23357c478bd9Sstevel@tonic-gate 
23367c478bd9Sstevel@tonic-gate 	no_preempt(self);
23377c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
23387c478bd9Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
23397c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
23407c478bd9Sstevel@tonic-gate 	(void) swap32(&mp->mutex_lockword, 0);
23417c478bd9Sstevel@tonic-gate 	preempt(self);
23427c478bd9Sstevel@tonic-gate 	return (0);
23437c478bd9Sstevel@tonic-gate }
23447c478bd9Sstevel@tonic-gate 
23457c478bd9Sstevel@tonic-gate #pragma weak cond_init = _cond_init
23467c478bd9Sstevel@tonic-gate /* ARGSUSED2 */
23477c478bd9Sstevel@tonic-gate int
23487c478bd9Sstevel@tonic-gate _cond_init(cond_t *cvp, int type, void *arg)
23497c478bd9Sstevel@tonic-gate {
23507c478bd9Sstevel@tonic-gate 	if (type != USYNC_THREAD && type != USYNC_PROCESS)
23517c478bd9Sstevel@tonic-gate 		return (EINVAL);
23527c478bd9Sstevel@tonic-gate 	(void) _memset(cvp, 0, sizeof (*cvp));
23537c478bd9Sstevel@tonic-gate 	cvp->cond_type = (uint16_t)type;
23547c478bd9Sstevel@tonic-gate 	cvp->cond_magic = COND_MAGIC;
23557c478bd9Sstevel@tonic-gate 	return (0);
23567c478bd9Sstevel@tonic-gate }
23577c478bd9Sstevel@tonic-gate 
23587c478bd9Sstevel@tonic-gate /*
23597c478bd9Sstevel@tonic-gate  * cond_sleep_queue(): utility function for cond_wait_queue().
23607c478bd9Sstevel@tonic-gate  *
23617c478bd9Sstevel@tonic-gate  * Go to sleep on a condvar sleep queue, expect to be waked up
23627c478bd9Sstevel@tonic-gate  * by someone calling cond_signal() or cond_broadcast() or due
23637c478bd9Sstevel@tonic-gate  * to receiving a UNIX signal or being cancelled, or just simply
23647c478bd9Sstevel@tonic-gate  * due to a spurious wakeup (like someome calling forkall()).
23657c478bd9Sstevel@tonic-gate  *
23667c478bd9Sstevel@tonic-gate  * The associated mutex is *not* reacquired before returning.
23677c478bd9Sstevel@tonic-gate  * That must be done by the caller of cond_sleep_queue().
23687c478bd9Sstevel@tonic-gate  */
23697c478bd9Sstevel@tonic-gate int
23707c478bd9Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
23717c478bd9Sstevel@tonic-gate {
23727c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
23737c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
23747c478bd9Sstevel@tonic-gate 	queue_head_t *mqp;
23757c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
23767c478bd9Sstevel@tonic-gate 	int signalled;
23777c478bd9Sstevel@tonic-gate 	int error;
23787c478bd9Sstevel@tonic-gate 
23797c478bd9Sstevel@tonic-gate 	/*
23807c478bd9Sstevel@tonic-gate 	 * Put ourself on the CV sleep queue, unlock the mutex, then
23817c478bd9Sstevel@tonic-gate 	 * park ourself and unpark a candidate lwp to grab the mutex.
23827c478bd9Sstevel@tonic-gate 	 * We must go onto the CV sleep queue before dropping the
23837c478bd9Sstevel@tonic-gate 	 * mutex in order to guarantee atomicity of the operation.
23847c478bd9Sstevel@tonic-gate 	 */
23857c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
23867c478bd9Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
23877c478bd9Sstevel@tonic-gate 	enqueue(qp, self, cvp, CV);
23887c478bd9Sstevel@tonic-gate 	cvp->cond_waiters_user = 1;
23897c478bd9Sstevel@tonic-gate 	self->ul_cvmutex = mp;
23907c478bd9Sstevel@tonic-gate 	self->ul_cv_wake = (tsp != NULL);
23917c478bd9Sstevel@tonic-gate 	self->ul_signalled = 0;
23927c478bd9Sstevel@tonic-gate 	lwpid = mutex_unlock_queue(mp);
23937c478bd9Sstevel@tonic-gate 	for (;;) {
23947c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 1);
23957c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
23967c478bd9Sstevel@tonic-gate 		if (lwpid != 0) {
23977c478bd9Sstevel@tonic-gate 			lwpid = preempt_unpark(self, lwpid);
23987c478bd9Sstevel@tonic-gate 			preempt(self);
23997c478bd9Sstevel@tonic-gate 		}
24007c478bd9Sstevel@tonic-gate 		/*
24017c478bd9Sstevel@tonic-gate 		 * We may have a deferred signal present,
24027c478bd9Sstevel@tonic-gate 		 * in which case we should return EINTR.
24037c478bd9Sstevel@tonic-gate 		 * Also, we may have received a SIGCANCEL; if so
24047c478bd9Sstevel@tonic-gate 		 * and we are cancelable we should return EINTR.
24057c478bd9Sstevel@tonic-gate 		 * We force an immediate EINTR return from
24067c478bd9Sstevel@tonic-gate 		 * __lwp_park() by turning our parking flag off.
24077c478bd9Sstevel@tonic-gate 		 */
24087c478bd9Sstevel@tonic-gate 		if (self->ul_cursig != 0 ||
24097c478bd9Sstevel@tonic-gate 		    (self->ul_cancelable && self->ul_cancel_pending))
24107c478bd9Sstevel@tonic-gate 			set_parking_flag(self, 0);
24117c478bd9Sstevel@tonic-gate 		/*
24127c478bd9Sstevel@tonic-gate 		 * __lwp_park() will return the residual time in tsp
24137c478bd9Sstevel@tonic-gate 		 * if we are unparked before the timeout expires.
24147c478bd9Sstevel@tonic-gate 		 */
24157c478bd9Sstevel@tonic-gate 		error = __lwp_park(tsp, lwpid);
24167c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 0);
24177c478bd9Sstevel@tonic-gate 		lwpid = 0;	/* unpark the other lwp only once */
24187c478bd9Sstevel@tonic-gate 		/*
24197c478bd9Sstevel@tonic-gate 		 * We were waked up by cond_signal(), cond_broadcast(),
24207c478bd9Sstevel@tonic-gate 		 * by an interrupt or timeout (EINTR or ETIME),
24217c478bd9Sstevel@tonic-gate 		 * or we may just have gotten a spurious wakeup.
24227c478bd9Sstevel@tonic-gate 		 */
24237c478bd9Sstevel@tonic-gate 		qp = queue_lock(cvp, CV);
24247c478bd9Sstevel@tonic-gate 		mqp = queue_lock(mp, MX);
24257c478bd9Sstevel@tonic-gate 		if (self->ul_sleepq == NULL)
24267c478bd9Sstevel@tonic-gate 			break;
24277c478bd9Sstevel@tonic-gate 		/*
24287c478bd9Sstevel@tonic-gate 		 * We are on either the condvar sleep queue or the
2429*2be60c5eSraf 		 * mutex sleep queue.  Break out of the sleep if we
2430*2be60c5eSraf 		 * were interrupted or we timed out (EINTR or ETIME).
24317c478bd9Sstevel@tonic-gate 		 * Else this is a spurious wakeup; continue the loop.
24327c478bd9Sstevel@tonic-gate 		 */
2433*2be60c5eSraf 		if (self->ul_sleepq == mqp) {		/* mutex queue */
2434*2be60c5eSraf 			if (error) {
2435*2be60c5eSraf 				mp->mutex_waiters = dequeue_self(mqp, mp);
2436*2be60c5eSraf 				break;
2437*2be60c5eSraf 			}
2438*2be60c5eSraf 			tsp = NULL;	/* no more timeout */
2439*2be60c5eSraf 		} else if (self->ul_sleepq == qp) {	/* condvar queue */
24407c478bd9Sstevel@tonic-gate 			if (error) {
24417c478bd9Sstevel@tonic-gate 				cvp->cond_waiters_user = dequeue_self(qp, cvp);
24427c478bd9Sstevel@tonic-gate 				break;
24437c478bd9Sstevel@tonic-gate 			}
24447c478bd9Sstevel@tonic-gate 			/*
24457c478bd9Sstevel@tonic-gate 			 * Else a spurious wakeup on the condvar queue.
24467c478bd9Sstevel@tonic-gate 			 * __lwp_park() has already adjusted the timeout.
24477c478bd9Sstevel@tonic-gate 			 */
24487c478bd9Sstevel@tonic-gate 		} else {
24497c478bd9Sstevel@tonic-gate 			thr_panic("cond_sleep_queue(): thread not on queue");
24507c478bd9Sstevel@tonic-gate 		}
24517c478bd9Sstevel@tonic-gate 		queue_unlock(mqp);
24527c478bd9Sstevel@tonic-gate 	}
24537c478bd9Sstevel@tonic-gate 
24547c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
24557c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_cvmutex == NULL && self->ul_cv_wake == 0);
24567c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
24577c478bd9Sstevel@tonic-gate 	    self->ul_wchan == NULL);
24587c478bd9Sstevel@tonic-gate 
24597c478bd9Sstevel@tonic-gate 	signalled = self->ul_signalled;
24607c478bd9Sstevel@tonic-gate 	self->ul_signalled = 0;
24617c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
24627c478bd9Sstevel@tonic-gate 	queue_unlock(mqp);
24637c478bd9Sstevel@tonic-gate 
24647c478bd9Sstevel@tonic-gate 	/*
24657c478bd9Sstevel@tonic-gate 	 * If we were concurrently cond_signal()d and any of:
24667c478bd9Sstevel@tonic-gate 	 * received a UNIX signal, were cancelled, or got a timeout,
24677c478bd9Sstevel@tonic-gate 	 * then perform another cond_signal() to avoid consuming it.
24687c478bd9Sstevel@tonic-gate 	 */
24697c478bd9Sstevel@tonic-gate 	if (error && signalled)
24707c478bd9Sstevel@tonic-gate 		(void) cond_signal_internal(cvp);
24717c478bd9Sstevel@tonic-gate 
24727c478bd9Sstevel@tonic-gate 	return (error);
24737c478bd9Sstevel@tonic-gate }
24747c478bd9Sstevel@tonic-gate 
24757c478bd9Sstevel@tonic-gate int
24767c478bd9Sstevel@tonic-gate cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp,
24777c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp)
24787c478bd9Sstevel@tonic-gate {
24797c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
24807c478bd9Sstevel@tonic-gate 	int error;
24817c478bd9Sstevel@tonic-gate 
24827c478bd9Sstevel@tonic-gate 	/*
24837c478bd9Sstevel@tonic-gate 	 * The old thread library was programmed to defer signals
24847c478bd9Sstevel@tonic-gate 	 * while in cond_wait() so that the associated mutex would
24857c478bd9Sstevel@tonic-gate 	 * be guaranteed to be held when the application signal
24867c478bd9Sstevel@tonic-gate 	 * handler was invoked.
24877c478bd9Sstevel@tonic-gate 	 *
24887c478bd9Sstevel@tonic-gate 	 * We do not behave this way by default; the state of the
24897c478bd9Sstevel@tonic-gate 	 * associated mutex in the signal handler is undefined.
24907c478bd9Sstevel@tonic-gate 	 *
24917c478bd9Sstevel@tonic-gate 	 * To accommodate applications that depend on the old
24927c478bd9Sstevel@tonic-gate 	 * behavior, the _THREAD_COND_WAIT_DEFER environment
24937c478bd9Sstevel@tonic-gate 	 * variable can be set to 1 and we will behave in the
24947c478bd9Sstevel@tonic-gate 	 * old way with respect to cond_wait().
24957c478bd9Sstevel@tonic-gate 	 */
24967c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
24977c478bd9Sstevel@tonic-gate 		sigoff(self);
24987c478bd9Sstevel@tonic-gate 
24997c478bd9Sstevel@tonic-gate 	error = cond_sleep_queue(cvp, mp, tsp);
25007c478bd9Sstevel@tonic-gate 
25017c478bd9Sstevel@tonic-gate 	/*
25027c478bd9Sstevel@tonic-gate 	 * Reacquire the mutex.
25037c478bd9Sstevel@tonic-gate 	 */
25047c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
25057c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
25067c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
25077c478bd9Sstevel@tonic-gate 	} else if (mutex_trylock_adaptive(mp) != 0) {
25087c478bd9Sstevel@tonic-gate 		(void) mutex_lock_queue(self, msp, mp, NULL);
25097c478bd9Sstevel@tonic-gate 	}
25107c478bd9Sstevel@tonic-gate 
25117c478bd9Sstevel@tonic-gate 	if (msp)
25127c478bd9Sstevel@tonic-gate 		record_begin_hold(msp);
25137c478bd9Sstevel@tonic-gate 
25147c478bd9Sstevel@tonic-gate 	/*
25157c478bd9Sstevel@tonic-gate 	 * Take any deferred signal now, after we have reacquired the mutex.
25167c478bd9Sstevel@tonic-gate 	 */
25177c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
25187c478bd9Sstevel@tonic-gate 		sigon(self);
25197c478bd9Sstevel@tonic-gate 
25207c478bd9Sstevel@tonic-gate 	return (error);
25217c478bd9Sstevel@tonic-gate }
25227c478bd9Sstevel@tonic-gate 
25237c478bd9Sstevel@tonic-gate /*
25247c478bd9Sstevel@tonic-gate  * cond_sleep_kernel(): utility function for cond_wait_kernel().
25257c478bd9Sstevel@tonic-gate  * See the comment ahead of cond_sleep_queue(), above.
25267c478bd9Sstevel@tonic-gate  */
25277c478bd9Sstevel@tonic-gate int
25287c478bd9Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
25297c478bd9Sstevel@tonic-gate {
25307c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
25317c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
25327c478bd9Sstevel@tonic-gate 	int error;
25337c478bd9Sstevel@tonic-gate 
25347c478bd9Sstevel@tonic-gate 	if (mtype & PTHREAD_PRIO_PROTECT) {
25357c478bd9Sstevel@tonic-gate 		if (_ceil_mylist_del(mp))
25367c478bd9Sstevel@tonic-gate 			_ceil_prio_waive();
25377c478bd9Sstevel@tonic-gate 	}
25387c478bd9Sstevel@tonic-gate 
25397c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
25407c478bd9Sstevel@tonic-gate 	self->ul_wchan = cvp;
25417c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
25427c478bd9Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
25437c478bd9Sstevel@tonic-gate 	if (mtype & PTHREAD_PRIO_INHERIT)
25447c478bd9Sstevel@tonic-gate 		mp->mutex_lockw = LOCKCLEAR;
25457c478bd9Sstevel@tonic-gate 	/*
25467c478bd9Sstevel@tonic-gate 	 * ___lwp_cond_wait() returns immediately with EINTR if
25477c478bd9Sstevel@tonic-gate 	 * set_parking_flag(self,0) is called on this lwp before it
25487c478bd9Sstevel@tonic-gate 	 * goes to sleep in the kernel.  sigacthandler() calls this
25497c478bd9Sstevel@tonic-gate 	 * when a deferred signal is noted.  This assures that we don't
25507c478bd9Sstevel@tonic-gate 	 * get stuck in ___lwp_cond_wait() with all signals blocked
25517c478bd9Sstevel@tonic-gate 	 * due to taking a deferred signal before going to sleep.
25527c478bd9Sstevel@tonic-gate 	 */
25537c478bd9Sstevel@tonic-gate 	set_parking_flag(self, 1);
25547c478bd9Sstevel@tonic-gate 	if (self->ul_cursig != 0 ||
25557c478bd9Sstevel@tonic-gate 	    (self->ul_cancelable && self->ul_cancel_pending))
25567c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 0);
25577c478bd9Sstevel@tonic-gate 	error = ___lwp_cond_wait(cvp, mp, tsp, 1);
25587c478bd9Sstevel@tonic-gate 	set_parking_flag(self, 0);
25597c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
25607c478bd9Sstevel@tonic-gate 	self->ul_wchan = NULL;
25617c478bd9Sstevel@tonic-gate 	return (error);
25627c478bd9Sstevel@tonic-gate }
25637c478bd9Sstevel@tonic-gate 
25647c478bd9Sstevel@tonic-gate int
25657c478bd9Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
25667c478bd9Sstevel@tonic-gate {
25677c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
25687c478bd9Sstevel@tonic-gate 	int error;
25697c478bd9Sstevel@tonic-gate 	int merror;
25707c478bd9Sstevel@tonic-gate 
25717c478bd9Sstevel@tonic-gate 	/*
25727c478bd9Sstevel@tonic-gate 	 * See the large comment in cond_wait_queue(), above.
25737c478bd9Sstevel@tonic-gate 	 */
25747c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
25757c478bd9Sstevel@tonic-gate 		sigoff(self);
25767c478bd9Sstevel@tonic-gate 
25777c478bd9Sstevel@tonic-gate 	error = cond_sleep_kernel(cvp, mp, tsp);
25787c478bd9Sstevel@tonic-gate 
25797c478bd9Sstevel@tonic-gate 	/*
25807c478bd9Sstevel@tonic-gate 	 * Override the return code from ___lwp_cond_wait()
25817c478bd9Sstevel@tonic-gate 	 * with any non-zero return code from mutex_lock().
25827c478bd9Sstevel@tonic-gate 	 * This addresses robust lock failures in particular;
25837c478bd9Sstevel@tonic-gate 	 * the caller must see the EOWNERDEAD or ENOTRECOVERABLE
25847c478bd9Sstevel@tonic-gate 	 * errors in order to take corrective action.
25857c478bd9Sstevel@tonic-gate 	 */
25867c478bd9Sstevel@tonic-gate 	if ((merror = _private_mutex_lock(mp)) != 0)
25877c478bd9Sstevel@tonic-gate 		error = merror;
25887c478bd9Sstevel@tonic-gate 
25897c478bd9Sstevel@tonic-gate 	/*
25907c478bd9Sstevel@tonic-gate 	 * Take any deferred signal now, after we have reacquired the mutex.
25917c478bd9Sstevel@tonic-gate 	 */
25927c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
25937c478bd9Sstevel@tonic-gate 		sigon(self);
25947c478bd9Sstevel@tonic-gate 
25957c478bd9Sstevel@tonic-gate 	return (error);
25967c478bd9Sstevel@tonic-gate }
25977c478bd9Sstevel@tonic-gate 
25987c478bd9Sstevel@tonic-gate /*
25997c478bd9Sstevel@tonic-gate  * Common code for _cond_wait() and _cond_timedwait()
26007c478bd9Sstevel@tonic-gate  */
26017c478bd9Sstevel@tonic-gate int
26027c478bd9Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
26037c478bd9Sstevel@tonic-gate {
26047c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
26057c478bd9Sstevel@tonic-gate 	hrtime_t begin_sleep = 0;
26067c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
26077c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
26087c478bd9Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
26097c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
26107c478bd9Sstevel@tonic-gate 	uint8_t rcount;
26117c478bd9Sstevel@tonic-gate 	int error = 0;
26127c478bd9Sstevel@tonic-gate 
26137c478bd9Sstevel@tonic-gate 	/*
26147c478bd9Sstevel@tonic-gate 	 * The SUSV3 Posix spec for pthread_cond_timedwait() states:
26157c478bd9Sstevel@tonic-gate 	 *	Except in the case of [ETIMEDOUT], all these error checks
26167c478bd9Sstevel@tonic-gate 	 *	shall act as if they were performed immediately at the
26177c478bd9Sstevel@tonic-gate 	 *	beginning of processing for the function and shall cause
26187c478bd9Sstevel@tonic-gate 	 *	an error return, in effect, prior to modifying the state
26197c478bd9Sstevel@tonic-gate 	 *	of the mutex specified by mutex or the condition variable
26207c478bd9Sstevel@tonic-gate 	 *	specified by cond.
26217c478bd9Sstevel@tonic-gate 	 * Therefore, we must return EINVAL now if the timout is invalid.
26227c478bd9Sstevel@tonic-gate 	 */
26237c478bd9Sstevel@tonic-gate 	if (tsp != NULL &&
26247c478bd9Sstevel@tonic-gate 	    (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC))
26257c478bd9Sstevel@tonic-gate 		return (EINVAL);
26267c478bd9Sstevel@tonic-gate 
26277c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
26287c478bd9Sstevel@tonic-gate 		self->ul_sp = stkptr();
26297c478bd9Sstevel@tonic-gate 		self->ul_wchan = cvp;
26307c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
26317c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = cvp;
26327c478bd9Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
26337c478bd9Sstevel@tonic-gate 		self->ul_sp = 0;
26347c478bd9Sstevel@tonic-gate 	}
26357c478bd9Sstevel@tonic-gate 	if (csp) {
26367c478bd9Sstevel@tonic-gate 		if (tsp)
26377c478bd9Sstevel@tonic-gate 			tdb_incr(csp->cond_timedwait);
26387c478bd9Sstevel@tonic-gate 		else
26397c478bd9Sstevel@tonic-gate 			tdb_incr(csp->cond_wait);
26407c478bd9Sstevel@tonic-gate 	}
26417c478bd9Sstevel@tonic-gate 	if (msp)
26427c478bd9Sstevel@tonic-gate 		begin_sleep = record_hold_time(msp);
26437c478bd9Sstevel@tonic-gate 	else if (csp)
26447c478bd9Sstevel@tonic-gate 		begin_sleep = gethrtime();
26457c478bd9Sstevel@tonic-gate 
26467c478bd9Sstevel@tonic-gate 	if (self->ul_error_detection) {
26477c478bd9Sstevel@tonic-gate 		if (!mutex_is_held(mp))
26487c478bd9Sstevel@tonic-gate 			lock_error(mp, "cond_wait", cvp, NULL);
26497c478bd9Sstevel@tonic-gate 		if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0)
26507c478bd9Sstevel@tonic-gate 			lock_error(mp, "recursive mutex in cond_wait",
26517c478bd9Sstevel@tonic-gate 				cvp, NULL);
26527c478bd9Sstevel@tonic-gate 		if (cvp->cond_type & USYNC_PROCESS) {
26537c478bd9Sstevel@tonic-gate 			if (!(mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)))
26547c478bd9Sstevel@tonic-gate 				lock_error(mp, "cond_wait", cvp,
26557c478bd9Sstevel@tonic-gate 					"condvar process-shared, "
26567c478bd9Sstevel@tonic-gate 					"mutex process-private");
26577c478bd9Sstevel@tonic-gate 		} else {
26587c478bd9Sstevel@tonic-gate 			if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))
26597c478bd9Sstevel@tonic-gate 				lock_error(mp, "cond_wait", cvp,
26607c478bd9Sstevel@tonic-gate 					"condvar process-private, "
26617c478bd9Sstevel@tonic-gate 					"mutex process-shared");
26627c478bd9Sstevel@tonic-gate 		}
26637c478bd9Sstevel@tonic-gate 	}
26647c478bd9Sstevel@tonic-gate 
26657c478bd9Sstevel@tonic-gate 	/*
26667c478bd9Sstevel@tonic-gate 	 * We deal with recursive mutexes by completely
26677c478bd9Sstevel@tonic-gate 	 * dropping the lock and restoring the recursion
26687c478bd9Sstevel@tonic-gate 	 * count after waking up.  This is arguably wrong,
26697c478bd9Sstevel@tonic-gate 	 * but it obeys the principle of least astonishment.
26707c478bd9Sstevel@tonic-gate 	 */
26717c478bd9Sstevel@tonic-gate 	rcount = mp->mutex_rcount;
26727c478bd9Sstevel@tonic-gate 	mp->mutex_rcount = 0;
26737c478bd9Sstevel@tonic-gate 	if ((mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST |
26747c478bd9Sstevel@tonic-gate 	    PTHREAD_PRIO_INHERIT | PTHREAD_PRIO_PROTECT)) |
26757c478bd9Sstevel@tonic-gate 	    (cvp->cond_type & USYNC_PROCESS))
26767c478bd9Sstevel@tonic-gate 		error = cond_wait_kernel(cvp, mp, tsp);
26777c478bd9Sstevel@tonic-gate 	else
26787c478bd9Sstevel@tonic-gate 		error = cond_wait_queue(cvp, mp, tsp, msp);
26797c478bd9Sstevel@tonic-gate 	mp->mutex_rcount = rcount;
26807c478bd9Sstevel@tonic-gate 
26817c478bd9Sstevel@tonic-gate 	if (csp) {
26827c478bd9Sstevel@tonic-gate 		hrtime_t lapse = gethrtime() - begin_sleep;
26837c478bd9Sstevel@tonic-gate 		if (tsp == NULL)
26847c478bd9Sstevel@tonic-gate 			csp->cond_wait_sleep_time += lapse;
26857c478bd9Sstevel@tonic-gate 		else {
26867c478bd9Sstevel@tonic-gate 			csp->cond_timedwait_sleep_time += lapse;
26877c478bd9Sstevel@tonic-gate 			if (error == ETIME)
26887c478bd9Sstevel@tonic-gate 				tdb_incr(csp->cond_timedwait_timeout);
26897c478bd9Sstevel@tonic-gate 		}
26907c478bd9Sstevel@tonic-gate 	}
26917c478bd9Sstevel@tonic-gate 	return (error);
26927c478bd9Sstevel@tonic-gate }
26937c478bd9Sstevel@tonic-gate 
26947c478bd9Sstevel@tonic-gate /*
26957c478bd9Sstevel@tonic-gate  * cond_wait() is a cancellation point but _cond_wait() is not.
26967c478bd9Sstevel@tonic-gate  * System libraries call the non-cancellation version.
26977c478bd9Sstevel@tonic-gate  * It is expected that only applications call the cancellation version.
26987c478bd9Sstevel@tonic-gate  */
26997c478bd9Sstevel@tonic-gate int
27007c478bd9Sstevel@tonic-gate _cond_wait(cond_t *cvp, mutex_t *mp)
27017c478bd9Sstevel@tonic-gate {
27027c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
27037c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
27047c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
27057c478bd9Sstevel@tonic-gate 
27067c478bd9Sstevel@tonic-gate 	/*
27077c478bd9Sstevel@tonic-gate 	 * Optimize the common case of USYNC_THREAD plus
27087c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics, and no event tracing.
27097c478bd9Sstevel@tonic-gate 	 */
27107c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
27117c478bd9Sstevel@tonic-gate 	    (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted |
27127c478bd9Sstevel@tonic-gate 	    self->ul_td_events_enable |
27137c478bd9Sstevel@tonic-gate 	    udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0)
27147c478bd9Sstevel@tonic-gate 		return (cond_wait_queue(cvp, mp, NULL, NULL));
27157c478bd9Sstevel@tonic-gate 
27167c478bd9Sstevel@tonic-gate 	/*
27177c478bd9Sstevel@tonic-gate 	 * Else do it the long way.
27187c478bd9Sstevel@tonic-gate 	 */
27197c478bd9Sstevel@tonic-gate 	return (cond_wait_common(cvp, mp, NULL));
27207c478bd9Sstevel@tonic-gate }
27217c478bd9Sstevel@tonic-gate 
27227c478bd9Sstevel@tonic-gate int
27237c478bd9Sstevel@tonic-gate cond_wait(cond_t *cvp, mutex_t *mp)
27247c478bd9Sstevel@tonic-gate {
27257c478bd9Sstevel@tonic-gate 	int error;
27267c478bd9Sstevel@tonic-gate 
27277c478bd9Sstevel@tonic-gate 	_cancelon();
27287c478bd9Sstevel@tonic-gate 	error = _cond_wait(cvp, mp);
27297c478bd9Sstevel@tonic-gate 	if (error == EINTR)
27307c478bd9Sstevel@tonic-gate 		_canceloff();
27317c478bd9Sstevel@tonic-gate 	else
27327c478bd9Sstevel@tonic-gate 		_canceloff_nocancel();
27337c478bd9Sstevel@tonic-gate 	return (error);
27347c478bd9Sstevel@tonic-gate }
27357c478bd9Sstevel@tonic-gate 
27367c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_wait = _pthread_cond_wait
27377c478bd9Sstevel@tonic-gate int
27387c478bd9Sstevel@tonic-gate _pthread_cond_wait(cond_t *cvp, mutex_t *mp)
27397c478bd9Sstevel@tonic-gate {
27407c478bd9Sstevel@tonic-gate 	int error;
27417c478bd9Sstevel@tonic-gate 
27427c478bd9Sstevel@tonic-gate 	error = cond_wait(cvp, mp);
27437c478bd9Sstevel@tonic-gate 	return ((error == EINTR)? 0 : error);
27447c478bd9Sstevel@tonic-gate }
27457c478bd9Sstevel@tonic-gate 
27467c478bd9Sstevel@tonic-gate /*
27477c478bd9Sstevel@tonic-gate  * cond_timedwait() is a cancellation point but _cond_timedwait() is not.
27487c478bd9Sstevel@tonic-gate  * System libraries call the non-cancellation version.
27497c478bd9Sstevel@tonic-gate  * It is expected that only applications call the cancellation version.
27507c478bd9Sstevel@tonic-gate  */
27517c478bd9Sstevel@tonic-gate int
27527c478bd9Sstevel@tonic-gate _cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
27537c478bd9Sstevel@tonic-gate {
27547c478bd9Sstevel@tonic-gate 	clockid_t clock_id = cvp->cond_clockid;
27557c478bd9Sstevel@tonic-gate 	timespec_t reltime;
27567c478bd9Sstevel@tonic-gate 	int error;
27577c478bd9Sstevel@tonic-gate 
27587c478bd9Sstevel@tonic-gate 	if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES)
27597c478bd9Sstevel@tonic-gate 		clock_id = CLOCK_REALTIME;
27607c478bd9Sstevel@tonic-gate 	abstime_to_reltime(clock_id, abstime, &reltime);
27617c478bd9Sstevel@tonic-gate 	error = cond_wait_common(cvp, mp, &reltime);
27627c478bd9Sstevel@tonic-gate 	if (error == ETIME && clock_id == CLOCK_HIGHRES) {
27637c478bd9Sstevel@tonic-gate 		/*
27647c478bd9Sstevel@tonic-gate 		 * Don't return ETIME if we didn't really get a timeout.
27657c478bd9Sstevel@tonic-gate 		 * This can happen if we return because someone resets
27667c478bd9Sstevel@tonic-gate 		 * the system clock.  Just return zero in this case,
27677c478bd9Sstevel@tonic-gate 		 * giving a spurious wakeup but not a timeout.
27687c478bd9Sstevel@tonic-gate 		 */
27697c478bd9Sstevel@tonic-gate 		if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC +
27707c478bd9Sstevel@tonic-gate 		    abstime->tv_nsec > gethrtime())
27717c478bd9Sstevel@tonic-gate 			error = 0;
27727c478bd9Sstevel@tonic-gate 	}
27737c478bd9Sstevel@tonic-gate 	return (error);
27747c478bd9Sstevel@tonic-gate }
27757c478bd9Sstevel@tonic-gate 
27767c478bd9Sstevel@tonic-gate int
27777c478bd9Sstevel@tonic-gate cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
27787c478bd9Sstevel@tonic-gate {
27797c478bd9Sstevel@tonic-gate 	int error;
27807c478bd9Sstevel@tonic-gate 
27817c478bd9Sstevel@tonic-gate 	_cancelon();
27827c478bd9Sstevel@tonic-gate 	error = _cond_timedwait(cvp, mp, abstime);
27837c478bd9Sstevel@tonic-gate 	if (error == EINTR)
27847c478bd9Sstevel@tonic-gate 		_canceloff();
27857c478bd9Sstevel@tonic-gate 	else
27867c478bd9Sstevel@tonic-gate 		_canceloff_nocancel();
27877c478bd9Sstevel@tonic-gate 	return (error);
27887c478bd9Sstevel@tonic-gate }
27897c478bd9Sstevel@tonic-gate 
27907c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_timedwait = _pthread_cond_timedwait
27917c478bd9Sstevel@tonic-gate int
27927c478bd9Sstevel@tonic-gate _pthread_cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
27937c478bd9Sstevel@tonic-gate {
27947c478bd9Sstevel@tonic-gate 	int error;
27957c478bd9Sstevel@tonic-gate 
27967c478bd9Sstevel@tonic-gate 	error = cond_timedwait(cvp, mp, abstime);
27977c478bd9Sstevel@tonic-gate 	if (error == ETIME)
27987c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
27997c478bd9Sstevel@tonic-gate 	else if (error == EINTR)
28007c478bd9Sstevel@tonic-gate 		error = 0;
28017c478bd9Sstevel@tonic-gate 	return (error);
28027c478bd9Sstevel@tonic-gate }
28037c478bd9Sstevel@tonic-gate 
28047c478bd9Sstevel@tonic-gate /*
28057c478bd9Sstevel@tonic-gate  * cond_reltimedwait() is a cancellation point but _cond_reltimedwait()
28067c478bd9Sstevel@tonic-gate  * is not.  System libraries call the non-cancellation version.
28077c478bd9Sstevel@tonic-gate  * It is expected that only applications call the cancellation version.
28087c478bd9Sstevel@tonic-gate  */
28097c478bd9Sstevel@tonic-gate int
28107c478bd9Sstevel@tonic-gate _cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
28117c478bd9Sstevel@tonic-gate {
28127c478bd9Sstevel@tonic-gate 	timespec_t tslocal = *reltime;
28137c478bd9Sstevel@tonic-gate 
28147c478bd9Sstevel@tonic-gate 	return (cond_wait_common(cvp, mp, &tslocal));
28157c478bd9Sstevel@tonic-gate }
28167c478bd9Sstevel@tonic-gate 
28177c478bd9Sstevel@tonic-gate #pragma weak cond_reltimedwait = _cond_reltimedwait_cancel
28187c478bd9Sstevel@tonic-gate int
28197c478bd9Sstevel@tonic-gate _cond_reltimedwait_cancel(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
28207c478bd9Sstevel@tonic-gate {
28217c478bd9Sstevel@tonic-gate 	int error;
28227c478bd9Sstevel@tonic-gate 
28237c478bd9Sstevel@tonic-gate 	_cancelon();
28247c478bd9Sstevel@tonic-gate 	error = _cond_reltimedwait(cvp, mp, reltime);
28257c478bd9Sstevel@tonic-gate 	if (error == EINTR)
28267c478bd9Sstevel@tonic-gate 		_canceloff();
28277c478bd9Sstevel@tonic-gate 	else
28287c478bd9Sstevel@tonic-gate 		_canceloff_nocancel();
28297c478bd9Sstevel@tonic-gate 	return (error);
28307c478bd9Sstevel@tonic-gate }
28317c478bd9Sstevel@tonic-gate 
28327c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_reltimedwait_np = _pthread_cond_reltimedwait_np
28337c478bd9Sstevel@tonic-gate int
28347c478bd9Sstevel@tonic-gate _pthread_cond_reltimedwait_np(cond_t *cvp, mutex_t *mp,
28357c478bd9Sstevel@tonic-gate 	const timespec_t *reltime)
28367c478bd9Sstevel@tonic-gate {
28377c478bd9Sstevel@tonic-gate 	int error;
28387c478bd9Sstevel@tonic-gate 
28397c478bd9Sstevel@tonic-gate 	error = _cond_reltimedwait_cancel(cvp, mp, reltime);
28407c478bd9Sstevel@tonic-gate 	if (error == ETIME)
28417c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
28427c478bd9Sstevel@tonic-gate 	else if (error == EINTR)
28437c478bd9Sstevel@tonic-gate 		error = 0;
28447c478bd9Sstevel@tonic-gate 	return (error);
28457c478bd9Sstevel@tonic-gate }
28467c478bd9Sstevel@tonic-gate 
28477c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_signal = cond_signal_internal
28487c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_signal = cond_signal_internal
28497c478bd9Sstevel@tonic-gate #pragma weak cond_signal = cond_signal_internal
28507c478bd9Sstevel@tonic-gate #pragma weak _cond_signal = cond_signal_internal
28517c478bd9Sstevel@tonic-gate int
28527c478bd9Sstevel@tonic-gate cond_signal_internal(cond_t *cvp)
28537c478bd9Sstevel@tonic-gate {
28547c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
28557c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
28567c478bd9Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
28577c478bd9Sstevel@tonic-gate 	int error = 0;
28587c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
28597c478bd9Sstevel@tonic-gate 	mutex_t *mp;
28607c478bd9Sstevel@tonic-gate 	queue_head_t *mqp;
28617c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
28627c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
28637c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
28647c478bd9Sstevel@tonic-gate 	ulwp_t *next;
28657c478bd9Sstevel@tonic-gate 	ulwp_t **suspp = NULL;
28667c478bd9Sstevel@tonic-gate 	ulwp_t *susprev;
28677c478bd9Sstevel@tonic-gate 
28687c478bd9Sstevel@tonic-gate 	if (csp)
28697c478bd9Sstevel@tonic-gate 		tdb_incr(csp->cond_signal);
28707c478bd9Sstevel@tonic-gate 
28717c478bd9Sstevel@tonic-gate 	if (cvp->cond_waiters_kernel)	/* someone sleeping in the kernel? */
28727c478bd9Sstevel@tonic-gate 		error = __lwp_cond_signal(cvp);
28737c478bd9Sstevel@tonic-gate 
28747c478bd9Sstevel@tonic-gate 	if (!cvp->cond_waiters_user)	/* no one sleeping at user-level */
28757c478bd9Sstevel@tonic-gate 		return (error);
28767c478bd9Sstevel@tonic-gate 
28777c478bd9Sstevel@tonic-gate 	/*
28787c478bd9Sstevel@tonic-gate 	 * Move someone from the condvar sleep queue to the mutex sleep
28797c478bd9Sstevel@tonic-gate 	 * queue for the mutex that he will acquire on being waked up.
28807c478bd9Sstevel@tonic-gate 	 * We can do this only if we own the mutex he will acquire.
28817c478bd9Sstevel@tonic-gate 	 * If we do not own the mutex, or if his ul_cv_wake flag
28827c478bd9Sstevel@tonic-gate 	 * is set, just dequeue and unpark him.
28837c478bd9Sstevel@tonic-gate 	 */
28847c478bd9Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
28857c478bd9Sstevel@tonic-gate 	for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL;
28867c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
28877c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == cvp) {
28887c478bd9Sstevel@tonic-gate 			if (!ulwp->ul_stop)
28897c478bd9Sstevel@tonic-gate 				break;
28907c478bd9Sstevel@tonic-gate 			/*
28917c478bd9Sstevel@tonic-gate 			 * Try not to dequeue a suspended thread.
28927c478bd9Sstevel@tonic-gate 			 * This mimics the old libthread's behavior.
28937c478bd9Sstevel@tonic-gate 			 */
28947c478bd9Sstevel@tonic-gate 			if (suspp == NULL) {
28957c478bd9Sstevel@tonic-gate 				suspp = ulwpp;
28967c478bd9Sstevel@tonic-gate 				susprev = prev;
28977c478bd9Sstevel@tonic-gate 			}
28987c478bd9Sstevel@tonic-gate 		}
28997c478bd9Sstevel@tonic-gate 	}
29007c478bd9Sstevel@tonic-gate 	if (ulwp == NULL && suspp != NULL) {
29017c478bd9Sstevel@tonic-gate 		ulwp = *(ulwpp = suspp);
29027c478bd9Sstevel@tonic-gate 		prev = susprev;
29037c478bd9Sstevel@tonic-gate 		suspp = NULL;
29047c478bd9Sstevel@tonic-gate 	}
29057c478bd9Sstevel@tonic-gate 	if (ulwp == NULL) {	/* no one on the sleep queue */
29067c478bd9Sstevel@tonic-gate 		cvp->cond_waiters_user = 0;
29077c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
29087c478bd9Sstevel@tonic-gate 		return (error);
29097c478bd9Sstevel@tonic-gate 	}
29107c478bd9Sstevel@tonic-gate 	/*
29117c478bd9Sstevel@tonic-gate 	 * Scan the remainder of the CV queue for another waiter.
29127c478bd9Sstevel@tonic-gate 	 */
29137c478bd9Sstevel@tonic-gate 	if (suspp != NULL) {
29147c478bd9Sstevel@tonic-gate 		next = *suspp;
29157c478bd9Sstevel@tonic-gate 	} else {
29167c478bd9Sstevel@tonic-gate 		for (next = ulwp->ul_link; next != NULL; next = next->ul_link)
29177c478bd9Sstevel@tonic-gate 			if (next->ul_wchan == cvp)
29187c478bd9Sstevel@tonic-gate 				break;
29197c478bd9Sstevel@tonic-gate 	}
29207c478bd9Sstevel@tonic-gate 	if (next == NULL)
29217c478bd9Sstevel@tonic-gate 		cvp->cond_waiters_user = 0;
29227c478bd9Sstevel@tonic-gate 
29237c478bd9Sstevel@tonic-gate 	/*
29247c478bd9Sstevel@tonic-gate 	 * Inform the thread that he was the recipient of a cond_signal().
29257c478bd9Sstevel@tonic-gate 	 * This lets him deal with cond_signal() and, concurrently,
29267c478bd9Sstevel@tonic-gate 	 * one or more of a cancellation, a UNIX signal, or a timeout.
29277c478bd9Sstevel@tonic-gate 	 * These latter conditions must not consume a cond_signal().
29287c478bd9Sstevel@tonic-gate 	 */
29297c478bd9Sstevel@tonic-gate 	ulwp->ul_signalled = 1;
29307c478bd9Sstevel@tonic-gate 
29317c478bd9Sstevel@tonic-gate 	/*
29327c478bd9Sstevel@tonic-gate 	 * Dequeue the waiter but leave his ul_sleepq non-NULL
29337c478bd9Sstevel@tonic-gate 	 * while we move him to the mutex queue so that he can
29347c478bd9Sstevel@tonic-gate 	 * deal properly with spurious wakeups.
29357c478bd9Sstevel@tonic-gate 	 */
29367c478bd9Sstevel@tonic-gate 	*ulwpp = ulwp->ul_link;
29377c478bd9Sstevel@tonic-gate 	if (qp->qh_tail == ulwp)
29387c478bd9Sstevel@tonic-gate 		qp->qh_tail = prev;
29397c478bd9Sstevel@tonic-gate 	qp->qh_qlen--;
29407c478bd9Sstevel@tonic-gate 	ulwp->ul_link = NULL;
29417c478bd9Sstevel@tonic-gate 
29427c478bd9Sstevel@tonic-gate 	mp = ulwp->ul_cvmutex;		/* the mutex he will acquire */
29437c478bd9Sstevel@tonic-gate 	ulwp->ul_cvmutex = NULL;
29447c478bd9Sstevel@tonic-gate 	ASSERT(mp != NULL);
29457c478bd9Sstevel@tonic-gate 
29467c478bd9Sstevel@tonic-gate 	if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
29477c478bd9Sstevel@tonic-gate 		lwpid_t lwpid = ulwp->ul_lwpid;
29487c478bd9Sstevel@tonic-gate 
29497c478bd9Sstevel@tonic-gate 		no_preempt(self);
29507c478bd9Sstevel@tonic-gate 		ulwp->ul_sleepq = NULL;
29517c478bd9Sstevel@tonic-gate 		ulwp->ul_wchan = NULL;
29527c478bd9Sstevel@tonic-gate 		ulwp->ul_cv_wake = 0;
29537c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
29547c478bd9Sstevel@tonic-gate 		(void) __lwp_unpark(lwpid);
29557c478bd9Sstevel@tonic-gate 		preempt(self);
29567c478bd9Sstevel@tonic-gate 	} else {
29577c478bd9Sstevel@tonic-gate 		mqp = queue_lock(mp, MX);
29587c478bd9Sstevel@tonic-gate 		enqueue(mqp, ulwp, mp, MX);
29597c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = 1;
29607c478bd9Sstevel@tonic-gate 		queue_unlock(mqp);
29617c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
29627c478bd9Sstevel@tonic-gate 	}
29637c478bd9Sstevel@tonic-gate 
29647c478bd9Sstevel@tonic-gate 	return (error);
29657c478bd9Sstevel@tonic-gate }
29667c478bd9Sstevel@tonic-gate 
29677c478bd9Sstevel@tonic-gate #define	MAXLWPS	128	/* max remembered lwpids before overflow */
29687c478bd9Sstevel@tonic-gate #define	NEWLWPS	2048	/* max remembered lwpids at first overflow */
29697c478bd9Sstevel@tonic-gate 
29707c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_broadcast = cond_broadcast_internal
29717c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_broadcast = cond_broadcast_internal
29727c478bd9Sstevel@tonic-gate #pragma weak cond_broadcast = cond_broadcast_internal
29737c478bd9Sstevel@tonic-gate #pragma weak _cond_broadcast = cond_broadcast_internal
29747c478bd9Sstevel@tonic-gate int
29757c478bd9Sstevel@tonic-gate cond_broadcast_internal(cond_t *cvp)
29767c478bd9Sstevel@tonic-gate {
29777c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
29787c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
29797c478bd9Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
29807c478bd9Sstevel@tonic-gate 	int error = 0;
29817c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
29827c478bd9Sstevel@tonic-gate 	mutex_t *mp;
29837c478bd9Sstevel@tonic-gate 	queue_head_t *mqp;
29847c478bd9Sstevel@tonic-gate 	mutex_t *mp_cache = NULL;
29857c478bd9Sstevel@tonic-gate 	queue_head_t *mqp_cache = NULL;
29867c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
29877c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
29887c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
29897c478bd9Sstevel@tonic-gate 	lwpid_t buffer[MAXLWPS];
29907c478bd9Sstevel@tonic-gate 	lwpid_t *lwpid = buffer;
29917c478bd9Sstevel@tonic-gate 	int nlwpid = 0;
29927c478bd9Sstevel@tonic-gate 	int maxlwps = MAXLWPS;
29937c478bd9Sstevel@tonic-gate 
29947c478bd9Sstevel@tonic-gate 	if (csp)
29957c478bd9Sstevel@tonic-gate 		tdb_incr(csp->cond_broadcast);
29967c478bd9Sstevel@tonic-gate 
29977c478bd9Sstevel@tonic-gate 	if (cvp->cond_waiters_kernel)	/* someone sleeping in the kernel? */
29987c478bd9Sstevel@tonic-gate 		error = __lwp_cond_broadcast(cvp);
29997c478bd9Sstevel@tonic-gate 
30007c478bd9Sstevel@tonic-gate 	if (!cvp->cond_waiters_user)	/* no one sleeping at user-level */
30017c478bd9Sstevel@tonic-gate 		return (error);
30027c478bd9Sstevel@tonic-gate 
30037c478bd9Sstevel@tonic-gate 	/*
30047c478bd9Sstevel@tonic-gate 	 * Move everyone from the condvar sleep queue to the mutex sleep
30057c478bd9Sstevel@tonic-gate 	 * queue for the mutex that they will acquire on being waked up.
30067c478bd9Sstevel@tonic-gate 	 * We can do this only if we own the mutex they will acquire.
30077c478bd9Sstevel@tonic-gate 	 * If we do not own the mutex, or if their ul_cv_wake flag
30087c478bd9Sstevel@tonic-gate 	 * is set, just dequeue and unpark them.
30097c478bd9Sstevel@tonic-gate 	 *
30107c478bd9Sstevel@tonic-gate 	 * We keep track of lwpids that are to be unparked in lwpid[].
30117c478bd9Sstevel@tonic-gate 	 * __lwp_unpark_all() is called to unpark all of them after
30127c478bd9Sstevel@tonic-gate 	 * they have been removed from the sleep queue and the sleep
30137c478bd9Sstevel@tonic-gate 	 * queue lock has been dropped.  If we run out of space in our
30147c478bd9Sstevel@tonic-gate 	 * on-stack buffer, we need to allocate more but we can't call
30157c478bd9Sstevel@tonic-gate 	 * lmalloc() because we are holding a queue lock when the overflow
30167c478bd9Sstevel@tonic-gate 	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
30177c478bd9Sstevel@tonic-gate 	 * either because the application may have allocated a small stack
30187c478bd9Sstevel@tonic-gate 	 * and we don't want to overrun the stack.  So we use the mmap()
30197c478bd9Sstevel@tonic-gate 	 * system call directly since that path acquires no locks.
30207c478bd9Sstevel@tonic-gate 	 */
30217c478bd9Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
30227c478bd9Sstevel@tonic-gate 	cvp->cond_waiters_user = 0;
30237c478bd9Sstevel@tonic-gate 	ulwpp = &qp->qh_head;
30247c478bd9Sstevel@tonic-gate 	while ((ulwp = *ulwpp) != NULL) {
30257c478bd9Sstevel@tonic-gate 
30267c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan != cvp) {
30277c478bd9Sstevel@tonic-gate 			prev = ulwp;
30287c478bd9Sstevel@tonic-gate 			ulwpp = &ulwp->ul_link;
30297c478bd9Sstevel@tonic-gate 			continue;
30307c478bd9Sstevel@tonic-gate 		}
30317c478bd9Sstevel@tonic-gate 
30327c478bd9Sstevel@tonic-gate 		*ulwpp = ulwp->ul_link;
30337c478bd9Sstevel@tonic-gate 		if (qp->qh_tail == ulwp)
30347c478bd9Sstevel@tonic-gate 			qp->qh_tail = prev;
30357c478bd9Sstevel@tonic-gate 		qp->qh_qlen--;
30367c478bd9Sstevel@tonic-gate 		ulwp->ul_link = NULL;
30377c478bd9Sstevel@tonic-gate 
30387c478bd9Sstevel@tonic-gate 		mp = ulwp->ul_cvmutex;		/* his mutex */
30397c478bd9Sstevel@tonic-gate 		ulwp->ul_cvmutex = NULL;
30407c478bd9Sstevel@tonic-gate 		ASSERT(mp != NULL);
30417c478bd9Sstevel@tonic-gate 
30427c478bd9Sstevel@tonic-gate 		if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
30437c478bd9Sstevel@tonic-gate 			ulwp->ul_sleepq = NULL;
30447c478bd9Sstevel@tonic-gate 			ulwp->ul_wchan = NULL;
30457c478bd9Sstevel@tonic-gate 			ulwp->ul_cv_wake = 0;
30467c478bd9Sstevel@tonic-gate 			if (nlwpid == maxlwps) {
30477c478bd9Sstevel@tonic-gate 				/*
30487c478bd9Sstevel@tonic-gate 				 * Allocate NEWLWPS ids on the first overflow.
30497c478bd9Sstevel@tonic-gate 				 * Double the allocation each time after that.
30507c478bd9Sstevel@tonic-gate 				 */
30517c478bd9Sstevel@tonic-gate 				int newlwps = (lwpid == buffer)? NEWLWPS :
30527c478bd9Sstevel@tonic-gate 						2 * maxlwps;
30537c478bd9Sstevel@tonic-gate 				void *vaddr = _private_mmap(NULL,
30547c478bd9Sstevel@tonic-gate 					newlwps * sizeof (lwpid_t),
30557c478bd9Sstevel@tonic-gate 					PROT_READ|PROT_WRITE,
30567c478bd9Sstevel@tonic-gate 					MAP_PRIVATE|MAP_ANON, -1, (off_t)0);
30577c478bd9Sstevel@tonic-gate 				if (vaddr == MAP_FAILED) {
30587c478bd9Sstevel@tonic-gate 					/*
30597c478bd9Sstevel@tonic-gate 					 * Let's hope this never happens.
30607c478bd9Sstevel@tonic-gate 					 * If it does, then we have a terrible
30617c478bd9Sstevel@tonic-gate 					 * thundering herd on our hands.
30627c478bd9Sstevel@tonic-gate 					 */
30637c478bd9Sstevel@tonic-gate 					(void) __lwp_unpark_all(lwpid, nlwpid);
30647c478bd9Sstevel@tonic-gate 					nlwpid = 0;
30657c478bd9Sstevel@tonic-gate 				} else {
30667c478bd9Sstevel@tonic-gate 					(void) _memcpy(vaddr, lwpid,
30677c478bd9Sstevel@tonic-gate 						maxlwps * sizeof (lwpid_t));
30687c478bd9Sstevel@tonic-gate 					if (lwpid != buffer)
30697c478bd9Sstevel@tonic-gate 						(void) _private_munmap(lwpid,
30707c478bd9Sstevel@tonic-gate 						    maxlwps * sizeof (lwpid_t));
30717c478bd9Sstevel@tonic-gate 					lwpid = vaddr;
30727c478bd9Sstevel@tonic-gate 					maxlwps = newlwps;
30737c478bd9Sstevel@tonic-gate 				}
30747c478bd9Sstevel@tonic-gate 			}
30757c478bd9Sstevel@tonic-gate 			lwpid[nlwpid++] = ulwp->ul_lwpid;
30767c478bd9Sstevel@tonic-gate 		} else {
30777c478bd9Sstevel@tonic-gate 			if (mp != mp_cache) {
30787c478bd9Sstevel@tonic-gate 				if (mqp_cache != NULL)
30797c478bd9Sstevel@tonic-gate 					queue_unlock(mqp_cache);
30807c478bd9Sstevel@tonic-gate 				mqp_cache = queue_lock(mp, MX);
30817c478bd9Sstevel@tonic-gate 				mp_cache = mp;
30827c478bd9Sstevel@tonic-gate 			}
30837c478bd9Sstevel@tonic-gate 			mqp = mqp_cache;
30847c478bd9Sstevel@tonic-gate 			enqueue(mqp, ulwp, mp, MX);
30857c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = 1;
30867c478bd9Sstevel@tonic-gate 		}
30877c478bd9Sstevel@tonic-gate 	}
30887c478bd9Sstevel@tonic-gate 	if (mqp_cache != NULL)
30897c478bd9Sstevel@tonic-gate 		queue_unlock(mqp_cache);
30907c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
30917c478bd9Sstevel@tonic-gate 	if (nlwpid) {
30927c478bd9Sstevel@tonic-gate 		if (nlwpid == 1)
30937c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid[0]);
30947c478bd9Sstevel@tonic-gate 		else
30957c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark_all(lwpid, nlwpid);
30967c478bd9Sstevel@tonic-gate 	}
30977c478bd9Sstevel@tonic-gate 	if (lwpid != buffer)
30987c478bd9Sstevel@tonic-gate 		(void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t));
30997c478bd9Sstevel@tonic-gate 
31007c478bd9Sstevel@tonic-gate 	return (error);
31017c478bd9Sstevel@tonic-gate }
31027c478bd9Sstevel@tonic-gate 
31037c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_destroy = _cond_destroy
31047c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_destroy = _cond_destroy
31057c478bd9Sstevel@tonic-gate #pragma weak cond_destroy = _cond_destroy
31067c478bd9Sstevel@tonic-gate int
31077c478bd9Sstevel@tonic-gate _cond_destroy(cond_t *cvp)
31087c478bd9Sstevel@tonic-gate {
31097c478bd9Sstevel@tonic-gate 	cvp->cond_magic = 0;
31107c478bd9Sstevel@tonic-gate 	tdb_sync_obj_deregister(cvp);
31117c478bd9Sstevel@tonic-gate 	return (0);
31127c478bd9Sstevel@tonic-gate }
31137c478bd9Sstevel@tonic-gate 
31147c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG)
31157c478bd9Sstevel@tonic-gate void
31167c478bd9Sstevel@tonic-gate assert_no_libc_locks_held(void)
31177c478bd9Sstevel@tonic-gate {
31187c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
31197c478bd9Sstevel@tonic-gate }
31207c478bd9Sstevel@tonic-gate #endif
31217c478bd9Sstevel@tonic-gate 
31227c478bd9Sstevel@tonic-gate /* protected by link_lock */
31237c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin;
31247c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin2;
31257c478bd9Sstevel@tonic-gate uint64_t spin_lock_sleep;
31267c478bd9Sstevel@tonic-gate uint64_t spin_lock_wakeup;
31277c478bd9Sstevel@tonic-gate 
31287c478bd9Sstevel@tonic-gate /*
31297c478bd9Sstevel@tonic-gate  * Record spin lock statistics.
31307c478bd9Sstevel@tonic-gate  * Called by a thread exiting itself in thrp_exit().
31317c478bd9Sstevel@tonic-gate  * Also called via atexit() from the thread calling
31327c478bd9Sstevel@tonic-gate  * exit() to do all the other threads as well.
31337c478bd9Sstevel@tonic-gate  */
31347c478bd9Sstevel@tonic-gate void
31357c478bd9Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp)
31367c478bd9Sstevel@tonic-gate {
31377c478bd9Sstevel@tonic-gate 	spin_lock_spin += ulwp->ul_spin_lock_spin;
31387c478bd9Sstevel@tonic-gate 	spin_lock_spin2 += ulwp->ul_spin_lock_spin2;
31397c478bd9Sstevel@tonic-gate 	spin_lock_sleep += ulwp->ul_spin_lock_sleep;
31407c478bd9Sstevel@tonic-gate 	spin_lock_wakeup += ulwp->ul_spin_lock_wakeup;
31417c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_spin = 0;
31427c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_spin2 = 0;
31437c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_sleep = 0;
31447c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_wakeup = 0;
31457c478bd9Sstevel@tonic-gate }
31467c478bd9Sstevel@tonic-gate 
31477c478bd9Sstevel@tonic-gate /*
31487c478bd9Sstevel@tonic-gate  * atexit function:  dump the queue statistics to stderr.
31497c478bd9Sstevel@tonic-gate  */
3150e8031f0aSraf #if !defined(__lint)
3151e8031f0aSraf #define	fprintf	_fprintf
3152e8031f0aSraf #endif
31537c478bd9Sstevel@tonic-gate #include <stdio.h>
31547c478bd9Sstevel@tonic-gate void
31557c478bd9Sstevel@tonic-gate dump_queue_statistics(void)
31567c478bd9Sstevel@tonic-gate {
31577c478bd9Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
31587c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
31597c478bd9Sstevel@tonic-gate 	int qn;
31607c478bd9Sstevel@tonic-gate 	uint64_t spin_lock_total = 0;
31617c478bd9Sstevel@tonic-gate 
31627c478bd9Sstevel@tonic-gate 	if (udp->queue_head == NULL || thread_queue_dump == 0)
31637c478bd9Sstevel@tonic-gate 		return;
31647c478bd9Sstevel@tonic-gate 
31657c478bd9Sstevel@tonic-gate 	if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 ||
31667c478bd9Sstevel@tonic-gate 	    fprintf(stderr, "queue#   lockcount    max qlen\n") < 0)
31677c478bd9Sstevel@tonic-gate 		return;
31687c478bd9Sstevel@tonic-gate 	for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) {
31697c478bd9Sstevel@tonic-gate 		if (qp->qh_lockcount == 0)
31707c478bd9Sstevel@tonic-gate 			continue;
31717c478bd9Sstevel@tonic-gate 		spin_lock_total += qp->qh_lockcount;
31727c478bd9Sstevel@tonic-gate 		if (fprintf(stderr, "%5d %12llu%12u\n", qn,
31737c478bd9Sstevel@tonic-gate 			(u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0)
31747c478bd9Sstevel@tonic-gate 				return;
31757c478bd9Sstevel@tonic-gate 	}
31767c478bd9Sstevel@tonic-gate 
31777c478bd9Sstevel@tonic-gate 	if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 ||
31787c478bd9Sstevel@tonic-gate 	    fprintf(stderr, "queue#   lockcount    max qlen\n") < 0)
31797c478bd9Sstevel@tonic-gate 		return;
31807c478bd9Sstevel@tonic-gate 	for (qn = 0; qn < QHASHSIZE; qn++, qp++) {
31817c478bd9Sstevel@tonic-gate 		if (qp->qh_lockcount == 0)
31827c478bd9Sstevel@tonic-gate 			continue;
31837c478bd9Sstevel@tonic-gate 		spin_lock_total += qp->qh_lockcount;
31847c478bd9Sstevel@tonic-gate 		if (fprintf(stderr, "%5d %12llu%12u\n", qn,
31857c478bd9Sstevel@tonic-gate 			(u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0)
31867c478bd9Sstevel@tonic-gate 				return;
31877c478bd9Sstevel@tonic-gate 	}
31887c478bd9Sstevel@tonic-gate 
31897c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "\n  spin_lock_total  = %10llu\n",
31907c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_total);
31917c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_spin   = %10llu\n",
31927c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_spin);
31937c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_spin2  = %10llu\n",
31947c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_spin2);
31957c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_sleep  = %10llu\n",
31967c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_sleep);
31977c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_wakeup = %10llu\n",
31987c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_wakeup);
31997c478bd9Sstevel@tonic-gate }
3200