xref: /illumos-gate/usr/src/lib/libc/port/threads/synch.c (revision 7c478bd95313f5f23a4c958a745db2134aa0324)
1*7c478bd9Sstevel@tonic-gate /*
2*7c478bd9Sstevel@tonic-gate  * CDDL HEADER START
3*7c478bd9Sstevel@tonic-gate  *
4*7c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*7c478bd9Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
6*7c478bd9Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
7*7c478bd9Sstevel@tonic-gate  * with the License.
8*7c478bd9Sstevel@tonic-gate  *
9*7c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10*7c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
11*7c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
12*7c478bd9Sstevel@tonic-gate  * and limitations under the License.
13*7c478bd9Sstevel@tonic-gate  *
14*7c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
15*7c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16*7c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
17*7c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
18*7c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
19*7c478bd9Sstevel@tonic-gate  *
20*7c478bd9Sstevel@tonic-gate  * CDDL HEADER END
21*7c478bd9Sstevel@tonic-gate  */
22*7c478bd9Sstevel@tonic-gate /*
23*7c478bd9Sstevel@tonic-gate  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24*7c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
25*7c478bd9Sstevel@tonic-gate  */
26*7c478bd9Sstevel@tonic-gate 
27*7c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
28*7c478bd9Sstevel@tonic-gate 
29*7c478bd9Sstevel@tonic-gate #include <sys/sdt.h>
30*7c478bd9Sstevel@tonic-gate 
31*7c478bd9Sstevel@tonic-gate #include "lint.h"
32*7c478bd9Sstevel@tonic-gate #include "thr_uberdata.h"
33*7c478bd9Sstevel@tonic-gate 
34*7c478bd9Sstevel@tonic-gate /*
35*7c478bd9Sstevel@tonic-gate  * This mutex is initialized to be held by lwp#1.
36*7c478bd9Sstevel@tonic-gate  * It is used to block a thread that has returned from a mutex_lock()
37*7c478bd9Sstevel@tonic-gate  * of a PTHREAD_PRIO_INHERIT mutex with an unrecoverable error.
38*7c478bd9Sstevel@tonic-gate  */
39*7c478bd9Sstevel@tonic-gate mutex_t	stall_mutex = DEFAULTMUTEX;
40*7c478bd9Sstevel@tonic-gate 
41*7c478bd9Sstevel@tonic-gate static int shared_mutex_held(mutex_t *);
42*7c478bd9Sstevel@tonic-gate 
43*7c478bd9Sstevel@tonic-gate /*
44*7c478bd9Sstevel@tonic-gate  * Lock statistics support functions.
45*7c478bd9Sstevel@tonic-gate  */
46*7c478bd9Sstevel@tonic-gate void
47*7c478bd9Sstevel@tonic-gate record_begin_hold(tdb_mutex_stats_t *msp)
48*7c478bd9Sstevel@tonic-gate {
49*7c478bd9Sstevel@tonic-gate 	tdb_incr(msp->mutex_lock);
50*7c478bd9Sstevel@tonic-gate 	msp->mutex_begin_hold = gethrtime();
51*7c478bd9Sstevel@tonic-gate }
52*7c478bd9Sstevel@tonic-gate 
53*7c478bd9Sstevel@tonic-gate hrtime_t
54*7c478bd9Sstevel@tonic-gate record_hold_time(tdb_mutex_stats_t *msp)
55*7c478bd9Sstevel@tonic-gate {
56*7c478bd9Sstevel@tonic-gate 	hrtime_t now = gethrtime();
57*7c478bd9Sstevel@tonic-gate 
58*7c478bd9Sstevel@tonic-gate 	if (msp->mutex_begin_hold)
59*7c478bd9Sstevel@tonic-gate 		msp->mutex_hold_time += now - msp->mutex_begin_hold;
60*7c478bd9Sstevel@tonic-gate 	msp->mutex_begin_hold = 0;
61*7c478bd9Sstevel@tonic-gate 	return (now);
62*7c478bd9Sstevel@tonic-gate }
63*7c478bd9Sstevel@tonic-gate 
64*7c478bd9Sstevel@tonic-gate /*
65*7c478bd9Sstevel@tonic-gate  * Called once at library initialization.
66*7c478bd9Sstevel@tonic-gate  */
67*7c478bd9Sstevel@tonic-gate void
68*7c478bd9Sstevel@tonic-gate mutex_setup(void)
69*7c478bd9Sstevel@tonic-gate {
70*7c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&stall_mutex.mutex_lockw))
71*7c478bd9Sstevel@tonic-gate 		thr_panic("mutex_setup() cannot acquire stall_mutex");
72*7c478bd9Sstevel@tonic-gate 	stall_mutex.mutex_owner = (uintptr_t)curthread;
73*7c478bd9Sstevel@tonic-gate }
74*7c478bd9Sstevel@tonic-gate 
75*7c478bd9Sstevel@tonic-gate /*
76*7c478bd9Sstevel@tonic-gate  * The default spin counts of 1000 and 500 are experimentally determined.
77*7c478bd9Sstevel@tonic-gate  * On sun4u machines with any number of processors they could be raised
78*7c478bd9Sstevel@tonic-gate  * to 10,000 but that (experimentally) makes almost no difference.
79*7c478bd9Sstevel@tonic-gate  * The environment variables:
80*7c478bd9Sstevel@tonic-gate  *	_THREAD_ADAPTIVE_SPIN=count
81*7c478bd9Sstevel@tonic-gate  *	_THREAD_RELEASE_SPIN=count
82*7c478bd9Sstevel@tonic-gate  * can be used to override and set the counts in the range [0 .. 1,000,000].
83*7c478bd9Sstevel@tonic-gate  */
84*7c478bd9Sstevel@tonic-gate int	thread_adaptive_spin = 1000;
85*7c478bd9Sstevel@tonic-gate uint_t	thread_max_spinners = 100;
86*7c478bd9Sstevel@tonic-gate int	thread_release_spin = 500;
87*7c478bd9Sstevel@tonic-gate int	thread_queue_verify = 0;
88*7c478bd9Sstevel@tonic-gate static	int	ncpus;
89*7c478bd9Sstevel@tonic-gate 
90*7c478bd9Sstevel@tonic-gate /*
91*7c478bd9Sstevel@tonic-gate  * Distinguish spinning for queue locks from spinning for regular locks.
92*7c478bd9Sstevel@tonic-gate  * The environment variable:
93*7c478bd9Sstevel@tonic-gate  *	_THREAD_QUEUE_SPIN=count
94*7c478bd9Sstevel@tonic-gate  * can be used to override and set the count in the range [0 .. 1,000,000].
95*7c478bd9Sstevel@tonic-gate  * There is no release spin concept for queue locks.
96*7c478bd9Sstevel@tonic-gate  */
97*7c478bd9Sstevel@tonic-gate int	thread_queue_spin = 1000;
98*7c478bd9Sstevel@tonic-gate 
99*7c478bd9Sstevel@tonic-gate /*
100*7c478bd9Sstevel@tonic-gate  * Use the otherwise-unused 'mutex_ownerpid' field of a USYNC_THREAD
101*7c478bd9Sstevel@tonic-gate  * mutex to be a count of adaptive spins in progress.
102*7c478bd9Sstevel@tonic-gate  */
103*7c478bd9Sstevel@tonic-gate #define	mutex_spinners	mutex_ownerpid
104*7c478bd9Sstevel@tonic-gate 
105*7c478bd9Sstevel@tonic-gate void
106*7c478bd9Sstevel@tonic-gate _mutex_set_typeattr(mutex_t *mp, int attr)
107*7c478bd9Sstevel@tonic-gate {
108*7c478bd9Sstevel@tonic-gate 	mp->mutex_type |= (uint8_t)attr;
109*7c478bd9Sstevel@tonic-gate }
110*7c478bd9Sstevel@tonic-gate 
111*7c478bd9Sstevel@tonic-gate /*
112*7c478bd9Sstevel@tonic-gate  * 'type' can be one of USYNC_THREAD or USYNC_PROCESS, possibly
113*7c478bd9Sstevel@tonic-gate  * augmented by the flags LOCK_RECURSIVE and/or LOCK_ERRORCHECK,
114*7c478bd9Sstevel@tonic-gate  * or it can be USYNC_PROCESS_ROBUST with no extra flags.
115*7c478bd9Sstevel@tonic-gate  */
116*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_init = __mutex_init
117*7c478bd9Sstevel@tonic-gate #pragma weak mutex_init = __mutex_init
118*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_init = __mutex_init
119*7c478bd9Sstevel@tonic-gate /* ARGSUSED2 */
120*7c478bd9Sstevel@tonic-gate int
121*7c478bd9Sstevel@tonic-gate __mutex_init(mutex_t *mp, int type, void *arg)
122*7c478bd9Sstevel@tonic-gate {
123*7c478bd9Sstevel@tonic-gate 	int error;
124*7c478bd9Sstevel@tonic-gate 
125*7c478bd9Sstevel@tonic-gate 	switch (type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) {
126*7c478bd9Sstevel@tonic-gate 	case USYNC_THREAD:
127*7c478bd9Sstevel@tonic-gate 	case USYNC_PROCESS:
128*7c478bd9Sstevel@tonic-gate 		(void) _memset(mp, 0, sizeof (*mp));
129*7c478bd9Sstevel@tonic-gate 		mp->mutex_type = (uint8_t)type;
130*7c478bd9Sstevel@tonic-gate 		mp->mutex_flag = LOCK_INITED;
131*7c478bd9Sstevel@tonic-gate 		error = 0;
132*7c478bd9Sstevel@tonic-gate 		break;
133*7c478bd9Sstevel@tonic-gate 	case USYNC_PROCESS_ROBUST:
134*7c478bd9Sstevel@tonic-gate 		if (type & (LOCK_RECURSIVE|LOCK_ERRORCHECK))
135*7c478bd9Sstevel@tonic-gate 			error = EINVAL;
136*7c478bd9Sstevel@tonic-gate 		else
137*7c478bd9Sstevel@tonic-gate 			error = ___lwp_mutex_init(mp, type);
138*7c478bd9Sstevel@tonic-gate 		break;
139*7c478bd9Sstevel@tonic-gate 	default:
140*7c478bd9Sstevel@tonic-gate 		error = EINVAL;
141*7c478bd9Sstevel@tonic-gate 		break;
142*7c478bd9Sstevel@tonic-gate 	}
143*7c478bd9Sstevel@tonic-gate 	if (error == 0)
144*7c478bd9Sstevel@tonic-gate 		mp->mutex_magic = MUTEX_MAGIC;
145*7c478bd9Sstevel@tonic-gate 	return (error);
146*7c478bd9Sstevel@tonic-gate }
147*7c478bd9Sstevel@tonic-gate 
148*7c478bd9Sstevel@tonic-gate /*
149*7c478bd9Sstevel@tonic-gate  * Delete mp from list of ceil mutexes owned by curthread.
150*7c478bd9Sstevel@tonic-gate  * Return 1 if the head of the chain was updated.
151*7c478bd9Sstevel@tonic-gate  */
152*7c478bd9Sstevel@tonic-gate int
153*7c478bd9Sstevel@tonic-gate _ceil_mylist_del(mutex_t *mp)
154*7c478bd9Sstevel@tonic-gate {
155*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
156*7c478bd9Sstevel@tonic-gate 	mxchain_t **mcpp;
157*7c478bd9Sstevel@tonic-gate 	mxchain_t *mcp;
158*7c478bd9Sstevel@tonic-gate 
159*7c478bd9Sstevel@tonic-gate 	mcpp = &self->ul_mxchain;
160*7c478bd9Sstevel@tonic-gate 	while ((*mcpp)->mxchain_mx != mp)
161*7c478bd9Sstevel@tonic-gate 		mcpp = &(*mcpp)->mxchain_next;
162*7c478bd9Sstevel@tonic-gate 	mcp = *mcpp;
163*7c478bd9Sstevel@tonic-gate 	*mcpp = mcp->mxchain_next;
164*7c478bd9Sstevel@tonic-gate 	lfree(mcp, sizeof (*mcp));
165*7c478bd9Sstevel@tonic-gate 	return (mcpp == &self->ul_mxchain);
166*7c478bd9Sstevel@tonic-gate }
167*7c478bd9Sstevel@tonic-gate 
168*7c478bd9Sstevel@tonic-gate /*
169*7c478bd9Sstevel@tonic-gate  * Add mp to head of list of ceil mutexes owned by curthread.
170*7c478bd9Sstevel@tonic-gate  * Return ENOMEM if no memory could be allocated.
171*7c478bd9Sstevel@tonic-gate  */
172*7c478bd9Sstevel@tonic-gate int
173*7c478bd9Sstevel@tonic-gate _ceil_mylist_add(mutex_t *mp)
174*7c478bd9Sstevel@tonic-gate {
175*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
176*7c478bd9Sstevel@tonic-gate 	mxchain_t *mcp;
177*7c478bd9Sstevel@tonic-gate 
178*7c478bd9Sstevel@tonic-gate 	if ((mcp = lmalloc(sizeof (*mcp))) == NULL)
179*7c478bd9Sstevel@tonic-gate 		return (ENOMEM);
180*7c478bd9Sstevel@tonic-gate 	mcp->mxchain_mx = mp;
181*7c478bd9Sstevel@tonic-gate 	mcp->mxchain_next = self->ul_mxchain;
182*7c478bd9Sstevel@tonic-gate 	self->ul_mxchain = mcp;
183*7c478bd9Sstevel@tonic-gate 	return (0);
184*7c478bd9Sstevel@tonic-gate }
185*7c478bd9Sstevel@tonic-gate 
186*7c478bd9Sstevel@tonic-gate /*
187*7c478bd9Sstevel@tonic-gate  * Inherit priority from ceiling.  The inheritance impacts the effective
188*7c478bd9Sstevel@tonic-gate  * priority, not the assigned priority.  See _thread_setschedparam_main().
189*7c478bd9Sstevel@tonic-gate  */
190*7c478bd9Sstevel@tonic-gate void
191*7c478bd9Sstevel@tonic-gate _ceil_prio_inherit(int ceil)
192*7c478bd9Sstevel@tonic-gate {
193*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
194*7c478bd9Sstevel@tonic-gate 	struct sched_param param;
195*7c478bd9Sstevel@tonic-gate 
196*7c478bd9Sstevel@tonic-gate 	(void) _memset(&param, 0, sizeof (param));
197*7c478bd9Sstevel@tonic-gate 	param.sched_priority = ceil;
198*7c478bd9Sstevel@tonic-gate 	if (_thread_setschedparam_main(self->ul_lwpid,
199*7c478bd9Sstevel@tonic-gate 	    self->ul_policy, &param, PRIO_INHERIT)) {
200*7c478bd9Sstevel@tonic-gate 		/*
201*7c478bd9Sstevel@tonic-gate 		 * Panic since unclear what error code to return.
202*7c478bd9Sstevel@tonic-gate 		 * If we do return the error codes returned by above
203*7c478bd9Sstevel@tonic-gate 		 * called routine, update the man page...
204*7c478bd9Sstevel@tonic-gate 		 */
205*7c478bd9Sstevel@tonic-gate 		thr_panic("_thread_setschedparam_main() fails");
206*7c478bd9Sstevel@tonic-gate 	}
207*7c478bd9Sstevel@tonic-gate }
208*7c478bd9Sstevel@tonic-gate 
209*7c478bd9Sstevel@tonic-gate /*
210*7c478bd9Sstevel@tonic-gate  * Waive inherited ceiling priority.  Inherit from head of owned ceiling locks
211*7c478bd9Sstevel@tonic-gate  * if holding at least one ceiling lock.  If no ceiling locks are held at this
212*7c478bd9Sstevel@tonic-gate  * point, disinherit completely, reverting back to assigned priority.
213*7c478bd9Sstevel@tonic-gate  */
214*7c478bd9Sstevel@tonic-gate void
215*7c478bd9Sstevel@tonic-gate _ceil_prio_waive(void)
216*7c478bd9Sstevel@tonic-gate {
217*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
218*7c478bd9Sstevel@tonic-gate 	struct sched_param param;
219*7c478bd9Sstevel@tonic-gate 
220*7c478bd9Sstevel@tonic-gate 	(void) _memset(&param, 0, sizeof (param));
221*7c478bd9Sstevel@tonic-gate 	if (self->ul_mxchain == NULL) {
222*7c478bd9Sstevel@tonic-gate 		/*
223*7c478bd9Sstevel@tonic-gate 		 * No ceil locks held.  Zero the epri, revert back to ul_pri.
224*7c478bd9Sstevel@tonic-gate 		 * Since thread's hash lock is not held, one cannot just
225*7c478bd9Sstevel@tonic-gate 		 * read ul_pri here...do it in the called routine...
226*7c478bd9Sstevel@tonic-gate 		 */
227*7c478bd9Sstevel@tonic-gate 		param.sched_priority = self->ul_pri;	/* ignored */
228*7c478bd9Sstevel@tonic-gate 		if (_thread_setschedparam_main(self->ul_lwpid,
229*7c478bd9Sstevel@tonic-gate 		    self->ul_policy, &param, PRIO_DISINHERIT))
230*7c478bd9Sstevel@tonic-gate 			thr_panic("_thread_setschedparam_main() fails");
231*7c478bd9Sstevel@tonic-gate 	} else {
232*7c478bd9Sstevel@tonic-gate 		/*
233*7c478bd9Sstevel@tonic-gate 		 * Set priority to that of the mutex at the head
234*7c478bd9Sstevel@tonic-gate 		 * of the ceilmutex chain.
235*7c478bd9Sstevel@tonic-gate 		 */
236*7c478bd9Sstevel@tonic-gate 		param.sched_priority =
237*7c478bd9Sstevel@tonic-gate 		    self->ul_mxchain->mxchain_mx->mutex_ceiling;
238*7c478bd9Sstevel@tonic-gate 		if (_thread_setschedparam_main(self->ul_lwpid,
239*7c478bd9Sstevel@tonic-gate 		    self->ul_policy, &param, PRIO_INHERIT))
240*7c478bd9Sstevel@tonic-gate 			thr_panic("_thread_setschedparam_main() fails");
241*7c478bd9Sstevel@tonic-gate 	}
242*7c478bd9Sstevel@tonic-gate }
243*7c478bd9Sstevel@tonic-gate 
244*7c478bd9Sstevel@tonic-gate /*
245*7c478bd9Sstevel@tonic-gate  * Non-preemptive spin locks.  Used by queue_lock().
246*7c478bd9Sstevel@tonic-gate  * No lock statistics are gathered for these locks.
247*7c478bd9Sstevel@tonic-gate  */
248*7c478bd9Sstevel@tonic-gate void
249*7c478bd9Sstevel@tonic-gate spin_lock_set(mutex_t *mp)
250*7c478bd9Sstevel@tonic-gate {
251*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
252*7c478bd9Sstevel@tonic-gate 
253*7c478bd9Sstevel@tonic-gate 	no_preempt(self);
254*7c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
255*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
256*7c478bd9Sstevel@tonic-gate 		return;
257*7c478bd9Sstevel@tonic-gate 	}
258*7c478bd9Sstevel@tonic-gate 	/*
259*7c478bd9Sstevel@tonic-gate 	 * Spin for a while, attempting to acquire the lock.
260*7c478bd9Sstevel@tonic-gate 	 */
261*7c478bd9Sstevel@tonic-gate 	if (self->ul_spin_lock_spin != UINT_MAX)
262*7c478bd9Sstevel@tonic-gate 		self->ul_spin_lock_spin++;
263*7c478bd9Sstevel@tonic-gate 	if (mutex_queuelock_adaptive(mp) == 0 ||
264*7c478bd9Sstevel@tonic-gate 	    set_lock_byte(&mp->mutex_lockw) == 0) {
265*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
266*7c478bd9Sstevel@tonic-gate 		return;
267*7c478bd9Sstevel@tonic-gate 	}
268*7c478bd9Sstevel@tonic-gate 	/*
269*7c478bd9Sstevel@tonic-gate 	 * Try harder if we were previously at a no premption level.
270*7c478bd9Sstevel@tonic-gate 	 */
271*7c478bd9Sstevel@tonic-gate 	if (self->ul_preempt > 1) {
272*7c478bd9Sstevel@tonic-gate 		if (self->ul_spin_lock_spin2 != UINT_MAX)
273*7c478bd9Sstevel@tonic-gate 			self->ul_spin_lock_spin2++;
274*7c478bd9Sstevel@tonic-gate 		if (mutex_queuelock_adaptive(mp) == 0 ||
275*7c478bd9Sstevel@tonic-gate 		    set_lock_byte(&mp->mutex_lockw) == 0) {
276*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
277*7c478bd9Sstevel@tonic-gate 			return;
278*7c478bd9Sstevel@tonic-gate 		}
279*7c478bd9Sstevel@tonic-gate 	}
280*7c478bd9Sstevel@tonic-gate 	/*
281*7c478bd9Sstevel@tonic-gate 	 * Give up and block in the kernel for the mutex.
282*7c478bd9Sstevel@tonic-gate 	 */
283*7c478bd9Sstevel@tonic-gate 	if (self->ul_spin_lock_sleep != UINT_MAX)
284*7c478bd9Sstevel@tonic-gate 		self->ul_spin_lock_sleep++;
285*7c478bd9Sstevel@tonic-gate 	(void) ___lwp_mutex_timedlock(mp, NULL);
286*7c478bd9Sstevel@tonic-gate 	mp->mutex_owner = (uintptr_t)self;
287*7c478bd9Sstevel@tonic-gate }
288*7c478bd9Sstevel@tonic-gate 
289*7c478bd9Sstevel@tonic-gate void
290*7c478bd9Sstevel@tonic-gate spin_lock_clear(mutex_t *mp)
291*7c478bd9Sstevel@tonic-gate {
292*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
293*7c478bd9Sstevel@tonic-gate 
294*7c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
295*7c478bd9Sstevel@tonic-gate 	if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) {
296*7c478bd9Sstevel@tonic-gate 		(void) ___lwp_mutex_wakeup(mp);
297*7c478bd9Sstevel@tonic-gate 		if (self->ul_spin_lock_wakeup != UINT_MAX)
298*7c478bd9Sstevel@tonic-gate 			self->ul_spin_lock_wakeup++;
299*7c478bd9Sstevel@tonic-gate 	}
300*7c478bd9Sstevel@tonic-gate 	preempt(self);
301*7c478bd9Sstevel@tonic-gate }
302*7c478bd9Sstevel@tonic-gate 
303*7c478bd9Sstevel@tonic-gate /*
304*7c478bd9Sstevel@tonic-gate  * Allocate the sleep queue hash table.
305*7c478bd9Sstevel@tonic-gate  */
306*7c478bd9Sstevel@tonic-gate void
307*7c478bd9Sstevel@tonic-gate queue_alloc(void)
308*7c478bd9Sstevel@tonic-gate {
309*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
310*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
311*7c478bd9Sstevel@tonic-gate 	void *data;
312*7c478bd9Sstevel@tonic-gate 	int i;
313*7c478bd9Sstevel@tonic-gate 
314*7c478bd9Sstevel@tonic-gate 	/*
315*7c478bd9Sstevel@tonic-gate 	 * No locks are needed; we call here only when single-threaded.
316*7c478bd9Sstevel@tonic-gate 	 */
317*7c478bd9Sstevel@tonic-gate 	ASSERT(self == udp->ulwp_one);
318*7c478bd9Sstevel@tonic-gate 	ASSERT(!udp->uberflags.uf_mt);
319*7c478bd9Sstevel@tonic-gate 	if ((data = _private_mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t),
320*7c478bd9Sstevel@tonic-gate 	    PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0))
321*7c478bd9Sstevel@tonic-gate 	    == MAP_FAILED)
322*7c478bd9Sstevel@tonic-gate 		thr_panic("cannot allocate thread queue_head table");
323*7c478bd9Sstevel@tonic-gate 	udp->queue_head = (queue_head_t *)data;
324*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < 2 * QHASHSIZE; i++)
325*7c478bd9Sstevel@tonic-gate 		udp->queue_head[i].qh_lock.mutex_magic = MUTEX_MAGIC;
326*7c478bd9Sstevel@tonic-gate }
327*7c478bd9Sstevel@tonic-gate 
328*7c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG)
329*7c478bd9Sstevel@tonic-gate 
330*7c478bd9Sstevel@tonic-gate /*
331*7c478bd9Sstevel@tonic-gate  * Debugging: verify correctness of a sleep queue.
332*7c478bd9Sstevel@tonic-gate  */
333*7c478bd9Sstevel@tonic-gate void
334*7c478bd9Sstevel@tonic-gate QVERIFY(queue_head_t *qp)
335*7c478bd9Sstevel@tonic-gate {
336*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
337*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
338*7c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
339*7c478bd9Sstevel@tonic-gate 	ulwp_t *prev;
340*7c478bd9Sstevel@tonic-gate 	uint_t index;
341*7c478bd9Sstevel@tonic-gate 	uint32_t cnt = 0;
342*7c478bd9Sstevel@tonic-gate 	char qtype;
343*7c478bd9Sstevel@tonic-gate 	void *wchan;
344*7c478bd9Sstevel@tonic-gate 
345*7c478bd9Sstevel@tonic-gate 	ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE);
346*7c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
347*7c478bd9Sstevel@tonic-gate 	ASSERT((qp->qh_head != NULL && qp->qh_tail != NULL) ||
348*7c478bd9Sstevel@tonic-gate 		(qp->qh_head == NULL && qp->qh_tail == NULL));
349*7c478bd9Sstevel@tonic-gate 	if (!thread_queue_verify)
350*7c478bd9Sstevel@tonic-gate 		return;
351*7c478bd9Sstevel@tonic-gate 	/* real expensive stuff, only for _THREAD_QUEUE_VERIFY */
352*7c478bd9Sstevel@tonic-gate 	qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV;
353*7c478bd9Sstevel@tonic-gate 	for (prev = NULL, ulwp = qp->qh_head; ulwp != NULL;
354*7c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwp = ulwp->ul_link, cnt++) {
355*7c478bd9Sstevel@tonic-gate 		ASSERT(ulwp->ul_qtype == qtype);
356*7c478bd9Sstevel@tonic-gate 		ASSERT(ulwp->ul_wchan != NULL);
357*7c478bd9Sstevel@tonic-gate 		ASSERT(ulwp->ul_sleepq == qp);
358*7c478bd9Sstevel@tonic-gate 		wchan = ulwp->ul_wchan;
359*7c478bd9Sstevel@tonic-gate 		index = QUEUE_HASH(wchan, qtype);
360*7c478bd9Sstevel@tonic-gate 		ASSERT(&udp->queue_head[index] == qp);
361*7c478bd9Sstevel@tonic-gate 	}
362*7c478bd9Sstevel@tonic-gate 	ASSERT(qp->qh_tail == prev);
363*7c478bd9Sstevel@tonic-gate 	ASSERT(qp->qh_qlen == cnt);
364*7c478bd9Sstevel@tonic-gate }
365*7c478bd9Sstevel@tonic-gate 
366*7c478bd9Sstevel@tonic-gate #else	/* THREAD_DEBUG */
367*7c478bd9Sstevel@tonic-gate 
368*7c478bd9Sstevel@tonic-gate #define	QVERIFY(qp)
369*7c478bd9Sstevel@tonic-gate 
370*7c478bd9Sstevel@tonic-gate #endif	/* THREAD_DEBUG */
371*7c478bd9Sstevel@tonic-gate 
372*7c478bd9Sstevel@tonic-gate /*
373*7c478bd9Sstevel@tonic-gate  * Acquire a queue head.
374*7c478bd9Sstevel@tonic-gate  */
375*7c478bd9Sstevel@tonic-gate queue_head_t *
376*7c478bd9Sstevel@tonic-gate queue_lock(void *wchan, int qtype)
377*7c478bd9Sstevel@tonic-gate {
378*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
379*7c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
380*7c478bd9Sstevel@tonic-gate 
381*7c478bd9Sstevel@tonic-gate 	ASSERT(qtype == MX || qtype == CV);
382*7c478bd9Sstevel@tonic-gate 
383*7c478bd9Sstevel@tonic-gate 	/*
384*7c478bd9Sstevel@tonic-gate 	 * It is possible that we could be called while still single-threaded.
385*7c478bd9Sstevel@tonic-gate 	 * If so, we call queue_alloc() to allocate the queue_head[] array.
386*7c478bd9Sstevel@tonic-gate 	 */
387*7c478bd9Sstevel@tonic-gate 	if ((qp = udp->queue_head) == NULL) {
388*7c478bd9Sstevel@tonic-gate 		queue_alloc();
389*7c478bd9Sstevel@tonic-gate 		qp = udp->queue_head;
390*7c478bd9Sstevel@tonic-gate 	}
391*7c478bd9Sstevel@tonic-gate 	qp += QUEUE_HASH(wchan, qtype);
392*7c478bd9Sstevel@tonic-gate 	spin_lock_set(&qp->qh_lock);
393*7c478bd9Sstevel@tonic-gate 	/*
394*7c478bd9Sstevel@tonic-gate 	 * At once per nanosecond, qh_lockcount will wrap after 512 years.
395*7c478bd9Sstevel@tonic-gate 	 * Were we to care about this, we could peg the value at UINT64_MAX.
396*7c478bd9Sstevel@tonic-gate 	 */
397*7c478bd9Sstevel@tonic-gate 	qp->qh_lockcount++;
398*7c478bd9Sstevel@tonic-gate 	QVERIFY(qp);
399*7c478bd9Sstevel@tonic-gate 	return (qp);
400*7c478bd9Sstevel@tonic-gate }
401*7c478bd9Sstevel@tonic-gate 
402*7c478bd9Sstevel@tonic-gate /*
403*7c478bd9Sstevel@tonic-gate  * Release a queue head.
404*7c478bd9Sstevel@tonic-gate  */
405*7c478bd9Sstevel@tonic-gate void
406*7c478bd9Sstevel@tonic-gate queue_unlock(queue_head_t *qp)
407*7c478bd9Sstevel@tonic-gate {
408*7c478bd9Sstevel@tonic-gate 	QVERIFY(qp);
409*7c478bd9Sstevel@tonic-gate 	spin_lock_clear(&qp->qh_lock);
410*7c478bd9Sstevel@tonic-gate }
411*7c478bd9Sstevel@tonic-gate 
412*7c478bd9Sstevel@tonic-gate /*
413*7c478bd9Sstevel@tonic-gate  * For rwlock queueing, we must queue writers ahead of readers of the
414*7c478bd9Sstevel@tonic-gate  * same priority.  We do this by making writers appear to have a half
415*7c478bd9Sstevel@tonic-gate  * point higher priority for purposes of priority comparisons below.
416*7c478bd9Sstevel@tonic-gate  */
417*7c478bd9Sstevel@tonic-gate #define	CMP_PRIO(ulwp)	((real_priority(ulwp) << 1) + (ulwp)->ul_writer)
418*7c478bd9Sstevel@tonic-gate 
419*7c478bd9Sstevel@tonic-gate void
420*7c478bd9Sstevel@tonic-gate enqueue(queue_head_t *qp, ulwp_t *ulwp, void *wchan, int qtype)
421*7c478bd9Sstevel@tonic-gate {
422*7c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
423*7c478bd9Sstevel@tonic-gate 	ulwp_t *next;
424*7c478bd9Sstevel@tonic-gate 	int pri = CMP_PRIO(ulwp);
425*7c478bd9Sstevel@tonic-gate 	int force_fifo = (qtype & FIFOQ);
426*7c478bd9Sstevel@tonic-gate 	int do_fifo;
427*7c478bd9Sstevel@tonic-gate 
428*7c478bd9Sstevel@tonic-gate 	qtype &= ~FIFOQ;
429*7c478bd9Sstevel@tonic-gate 	ASSERT(qtype == MX || qtype == CV);
430*7c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
431*7c478bd9Sstevel@tonic-gate 	ASSERT(ulwp->ul_sleepq != qp);
432*7c478bd9Sstevel@tonic-gate 
433*7c478bd9Sstevel@tonic-gate 	/*
434*7c478bd9Sstevel@tonic-gate 	 * LIFO queue ordering is unfair and can lead to starvation,
435*7c478bd9Sstevel@tonic-gate 	 * but it gives better performance for heavily contended locks.
436*7c478bd9Sstevel@tonic-gate 	 * We use thread_queue_fifo (range is 0..8) to determine
437*7c478bd9Sstevel@tonic-gate 	 * the frequency of FIFO vs LIFO queuing:
438*7c478bd9Sstevel@tonic-gate 	 *	0 : every 256th time	(almost always LIFO)
439*7c478bd9Sstevel@tonic-gate 	 *	1 : every 128th time
440*7c478bd9Sstevel@tonic-gate 	 *	2 : every 64th  time
441*7c478bd9Sstevel@tonic-gate 	 *	3 : every 32nd  time
442*7c478bd9Sstevel@tonic-gate 	 *	4 : every 16th  time	(the default value, mostly LIFO)
443*7c478bd9Sstevel@tonic-gate 	 *	5 : every 8th   time
444*7c478bd9Sstevel@tonic-gate 	 *	6 : every 4th   time
445*7c478bd9Sstevel@tonic-gate 	 *	7 : every 2nd   time
446*7c478bd9Sstevel@tonic-gate 	 *	8 : every time		(never LIFO, always FIFO)
447*7c478bd9Sstevel@tonic-gate 	 * Note that there is always some degree of FIFO ordering.
448*7c478bd9Sstevel@tonic-gate 	 * This breaks live lock conditions that occur in applications
449*7c478bd9Sstevel@tonic-gate 	 * that are written assuming (incorrectly) that threads acquire
450*7c478bd9Sstevel@tonic-gate 	 * locks fairly, that is, in roughly round-robin order.
451*7c478bd9Sstevel@tonic-gate 	 * In any event, the queue is maintained in priority order.
452*7c478bd9Sstevel@tonic-gate 	 *
453*7c478bd9Sstevel@tonic-gate 	 * If we are given the FIFOQ flag in qtype, fifo queueing is forced.
454*7c478bd9Sstevel@tonic-gate 	 * SUSV3 requires this for semaphores.
455*7c478bd9Sstevel@tonic-gate 	 */
456*7c478bd9Sstevel@tonic-gate 	do_fifo = (force_fifo ||
457*7c478bd9Sstevel@tonic-gate 		((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0);
458*7c478bd9Sstevel@tonic-gate 
459*7c478bd9Sstevel@tonic-gate 	if (qp->qh_head == NULL) {
460*7c478bd9Sstevel@tonic-gate 		/*
461*7c478bd9Sstevel@tonic-gate 		 * The queue is empty.  LIFO/FIFO doesn't matter.
462*7c478bd9Sstevel@tonic-gate 		 */
463*7c478bd9Sstevel@tonic-gate 		ASSERT(qp->qh_tail == NULL);
464*7c478bd9Sstevel@tonic-gate 		ulwpp = &qp->qh_head;
465*7c478bd9Sstevel@tonic-gate 	} else if (do_fifo) {
466*7c478bd9Sstevel@tonic-gate 		/*
467*7c478bd9Sstevel@tonic-gate 		 * Enqueue after the last thread whose priority is greater
468*7c478bd9Sstevel@tonic-gate 		 * than or equal to the priority of the thread being queued.
469*7c478bd9Sstevel@tonic-gate 		 * Attempt first to go directly onto the tail of the queue.
470*7c478bd9Sstevel@tonic-gate 		 */
471*7c478bd9Sstevel@tonic-gate 		if (pri <= CMP_PRIO(qp->qh_tail))
472*7c478bd9Sstevel@tonic-gate 			ulwpp = &qp->qh_tail->ul_link;
473*7c478bd9Sstevel@tonic-gate 		else {
474*7c478bd9Sstevel@tonic-gate 			for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL;
475*7c478bd9Sstevel@tonic-gate 			    ulwpp = &next->ul_link)
476*7c478bd9Sstevel@tonic-gate 				if (pri > CMP_PRIO(next))
477*7c478bd9Sstevel@tonic-gate 					break;
478*7c478bd9Sstevel@tonic-gate 		}
479*7c478bd9Sstevel@tonic-gate 	} else {
480*7c478bd9Sstevel@tonic-gate 		/*
481*7c478bd9Sstevel@tonic-gate 		 * Enqueue before the first thread whose priority is less
482*7c478bd9Sstevel@tonic-gate 		 * than or equal to the priority of the thread being queued.
483*7c478bd9Sstevel@tonic-gate 		 * Hopefully we can go directly onto the head of the queue.
484*7c478bd9Sstevel@tonic-gate 		 */
485*7c478bd9Sstevel@tonic-gate 		for (ulwpp = &qp->qh_head; (next = *ulwpp) != NULL;
486*7c478bd9Sstevel@tonic-gate 		    ulwpp = &next->ul_link)
487*7c478bd9Sstevel@tonic-gate 			if (pri >= CMP_PRIO(next))
488*7c478bd9Sstevel@tonic-gate 				break;
489*7c478bd9Sstevel@tonic-gate 	}
490*7c478bd9Sstevel@tonic-gate 	if ((ulwp->ul_link = *ulwpp) == NULL)
491*7c478bd9Sstevel@tonic-gate 		qp->qh_tail = ulwp;
492*7c478bd9Sstevel@tonic-gate 	*ulwpp = ulwp;
493*7c478bd9Sstevel@tonic-gate 
494*7c478bd9Sstevel@tonic-gate 	ulwp->ul_sleepq = qp;
495*7c478bd9Sstevel@tonic-gate 	ulwp->ul_wchan = wchan;
496*7c478bd9Sstevel@tonic-gate 	ulwp->ul_qtype = qtype;
497*7c478bd9Sstevel@tonic-gate 	if (qp->qh_qmax < ++qp->qh_qlen)
498*7c478bd9Sstevel@tonic-gate 		qp->qh_qmax = qp->qh_qlen;
499*7c478bd9Sstevel@tonic-gate }
500*7c478bd9Sstevel@tonic-gate 
501*7c478bd9Sstevel@tonic-gate /*
502*7c478bd9Sstevel@tonic-gate  * Return a pointer to the queue slot of the
503*7c478bd9Sstevel@tonic-gate  * highest priority thread on the queue.
504*7c478bd9Sstevel@tonic-gate  * On return, prevp, if not NULL, will contain a pointer
505*7c478bd9Sstevel@tonic-gate  * to the thread's predecessor on the queue
506*7c478bd9Sstevel@tonic-gate  */
507*7c478bd9Sstevel@tonic-gate static ulwp_t **
508*7c478bd9Sstevel@tonic-gate queue_slot(queue_head_t *qp, void *wchan, int *more, ulwp_t **prevp)
509*7c478bd9Sstevel@tonic-gate {
510*7c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
511*7c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
512*7c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
513*7c478bd9Sstevel@tonic-gate 	ulwp_t **suspp = NULL;
514*7c478bd9Sstevel@tonic-gate 	ulwp_t *susprev;
515*7c478bd9Sstevel@tonic-gate 
516*7c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
517*7c478bd9Sstevel@tonic-gate 
518*7c478bd9Sstevel@tonic-gate 	/*
519*7c478bd9Sstevel@tonic-gate 	 * Find a waiter on the sleep queue.
520*7c478bd9Sstevel@tonic-gate 	 */
521*7c478bd9Sstevel@tonic-gate 	for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL;
522*7c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
523*7c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan) {
524*7c478bd9Sstevel@tonic-gate 			if (!ulwp->ul_stop)
525*7c478bd9Sstevel@tonic-gate 				break;
526*7c478bd9Sstevel@tonic-gate 			/*
527*7c478bd9Sstevel@tonic-gate 			 * Try not to return a suspended thread.
528*7c478bd9Sstevel@tonic-gate 			 * This mimics the old libthread's behavior.
529*7c478bd9Sstevel@tonic-gate 			 */
530*7c478bd9Sstevel@tonic-gate 			if (suspp == NULL) {
531*7c478bd9Sstevel@tonic-gate 				suspp = ulwpp;
532*7c478bd9Sstevel@tonic-gate 				susprev = prev;
533*7c478bd9Sstevel@tonic-gate 			}
534*7c478bd9Sstevel@tonic-gate 		}
535*7c478bd9Sstevel@tonic-gate 	}
536*7c478bd9Sstevel@tonic-gate 
537*7c478bd9Sstevel@tonic-gate 	if (ulwp == NULL && suspp != NULL) {
538*7c478bd9Sstevel@tonic-gate 		ulwp = *(ulwpp = suspp);
539*7c478bd9Sstevel@tonic-gate 		prev = susprev;
540*7c478bd9Sstevel@tonic-gate 		suspp = NULL;
541*7c478bd9Sstevel@tonic-gate 	}
542*7c478bd9Sstevel@tonic-gate 	if (ulwp == NULL) {
543*7c478bd9Sstevel@tonic-gate 		if (more != NULL)
544*7c478bd9Sstevel@tonic-gate 			*more = 0;
545*7c478bd9Sstevel@tonic-gate 		return (NULL);
546*7c478bd9Sstevel@tonic-gate 	}
547*7c478bd9Sstevel@tonic-gate 
548*7c478bd9Sstevel@tonic-gate 	if (prevp != NULL)
549*7c478bd9Sstevel@tonic-gate 		*prevp = prev;
550*7c478bd9Sstevel@tonic-gate 	if (more == NULL)
551*7c478bd9Sstevel@tonic-gate 		return (ulwpp);
552*7c478bd9Sstevel@tonic-gate 
553*7c478bd9Sstevel@tonic-gate 	/*
554*7c478bd9Sstevel@tonic-gate 	 * Scan the remainder of the queue for another waiter.
555*7c478bd9Sstevel@tonic-gate 	 */
556*7c478bd9Sstevel@tonic-gate 	if (suspp != NULL) {
557*7c478bd9Sstevel@tonic-gate 		*more = 1;
558*7c478bd9Sstevel@tonic-gate 		return (ulwpp);
559*7c478bd9Sstevel@tonic-gate 	}
560*7c478bd9Sstevel@tonic-gate 	for (ulwp = ulwp->ul_link; ulwp != NULL; ulwp = ulwp->ul_link) {
561*7c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan) {
562*7c478bd9Sstevel@tonic-gate 			*more = 1;
563*7c478bd9Sstevel@tonic-gate 			return (ulwpp);
564*7c478bd9Sstevel@tonic-gate 		}
565*7c478bd9Sstevel@tonic-gate 	}
566*7c478bd9Sstevel@tonic-gate 
567*7c478bd9Sstevel@tonic-gate 	*more = 0;
568*7c478bd9Sstevel@tonic-gate 	return (ulwpp);
569*7c478bd9Sstevel@tonic-gate }
570*7c478bd9Sstevel@tonic-gate 
571*7c478bd9Sstevel@tonic-gate ulwp_t *
572*7c478bd9Sstevel@tonic-gate dequeue(queue_head_t *qp, void *wchan, int *more)
573*7c478bd9Sstevel@tonic-gate {
574*7c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
575*7c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
576*7c478bd9Sstevel@tonic-gate 	ulwp_t *prev;
577*7c478bd9Sstevel@tonic-gate 
578*7c478bd9Sstevel@tonic-gate 	if ((ulwpp = queue_slot(qp, wchan, more, &prev)) == NULL)
579*7c478bd9Sstevel@tonic-gate 		return (NULL);
580*7c478bd9Sstevel@tonic-gate 
581*7c478bd9Sstevel@tonic-gate 	/*
582*7c478bd9Sstevel@tonic-gate 	 * Dequeue the waiter.
583*7c478bd9Sstevel@tonic-gate 	 */
584*7c478bd9Sstevel@tonic-gate 	ulwp = *ulwpp;
585*7c478bd9Sstevel@tonic-gate 	*ulwpp = ulwp->ul_link;
586*7c478bd9Sstevel@tonic-gate 	ulwp->ul_link = NULL;
587*7c478bd9Sstevel@tonic-gate 	if (qp->qh_tail == ulwp)
588*7c478bd9Sstevel@tonic-gate 		qp->qh_tail = prev;
589*7c478bd9Sstevel@tonic-gate 	qp->qh_qlen--;
590*7c478bd9Sstevel@tonic-gate 	ulwp->ul_sleepq = NULL;
591*7c478bd9Sstevel@tonic-gate 	ulwp->ul_wchan = NULL;
592*7c478bd9Sstevel@tonic-gate 
593*7c478bd9Sstevel@tonic-gate 	return (ulwp);
594*7c478bd9Sstevel@tonic-gate }
595*7c478bd9Sstevel@tonic-gate 
596*7c478bd9Sstevel@tonic-gate /*
597*7c478bd9Sstevel@tonic-gate  * Return a pointer to the highest priority thread sleeping on wchan.
598*7c478bd9Sstevel@tonic-gate  */
599*7c478bd9Sstevel@tonic-gate ulwp_t *
600*7c478bd9Sstevel@tonic-gate queue_waiter(queue_head_t *qp, void *wchan)
601*7c478bd9Sstevel@tonic-gate {
602*7c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
603*7c478bd9Sstevel@tonic-gate 
604*7c478bd9Sstevel@tonic-gate 	if ((ulwpp = queue_slot(qp, wchan, NULL, NULL)) == NULL)
605*7c478bd9Sstevel@tonic-gate 		return (NULL);
606*7c478bd9Sstevel@tonic-gate 	return (*ulwpp);
607*7c478bd9Sstevel@tonic-gate }
608*7c478bd9Sstevel@tonic-gate 
609*7c478bd9Sstevel@tonic-gate uint8_t
610*7c478bd9Sstevel@tonic-gate dequeue_self(queue_head_t *qp, void *wchan)
611*7c478bd9Sstevel@tonic-gate {
612*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
613*7c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
614*7c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
615*7c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
616*7c478bd9Sstevel@tonic-gate 	int found = 0;
617*7c478bd9Sstevel@tonic-gate 	int more = 0;
618*7c478bd9Sstevel@tonic-gate 
619*7c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
620*7c478bd9Sstevel@tonic-gate 
621*7c478bd9Sstevel@tonic-gate 	/* find self on the sleep queue */
622*7c478bd9Sstevel@tonic-gate 	for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL;
623*7c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
624*7c478bd9Sstevel@tonic-gate 		if (ulwp == self) {
625*7c478bd9Sstevel@tonic-gate 			/* dequeue ourself */
626*7c478bd9Sstevel@tonic-gate 			*ulwpp = self->ul_link;
627*7c478bd9Sstevel@tonic-gate 			if (qp->qh_tail == self)
628*7c478bd9Sstevel@tonic-gate 				qp->qh_tail = prev;
629*7c478bd9Sstevel@tonic-gate 			qp->qh_qlen--;
630*7c478bd9Sstevel@tonic-gate 			ASSERT(self->ul_wchan == wchan);
631*7c478bd9Sstevel@tonic-gate 			self->ul_cvmutex = NULL;
632*7c478bd9Sstevel@tonic-gate 			self->ul_sleepq = NULL;
633*7c478bd9Sstevel@tonic-gate 			self->ul_wchan = NULL;
634*7c478bd9Sstevel@tonic-gate 			self->ul_cv_wake = 0;
635*7c478bd9Sstevel@tonic-gate 			self->ul_link = NULL;
636*7c478bd9Sstevel@tonic-gate 			found = 1;
637*7c478bd9Sstevel@tonic-gate 			break;
638*7c478bd9Sstevel@tonic-gate 		}
639*7c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan)
640*7c478bd9Sstevel@tonic-gate 			more = 1;
641*7c478bd9Sstevel@tonic-gate 	}
642*7c478bd9Sstevel@tonic-gate 
643*7c478bd9Sstevel@tonic-gate 	if (!found)
644*7c478bd9Sstevel@tonic-gate 		thr_panic("dequeue_self(): curthread not found on queue");
645*7c478bd9Sstevel@tonic-gate 
646*7c478bd9Sstevel@tonic-gate 	if (more)
647*7c478bd9Sstevel@tonic-gate 		return (1);
648*7c478bd9Sstevel@tonic-gate 
649*7c478bd9Sstevel@tonic-gate 	/* scan the remainder of the queue for another waiter */
650*7c478bd9Sstevel@tonic-gate 	for (ulwp = *ulwpp; ulwp != NULL; ulwp = ulwp->ul_link) {
651*7c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == wchan)
652*7c478bd9Sstevel@tonic-gate 			return (1);
653*7c478bd9Sstevel@tonic-gate 	}
654*7c478bd9Sstevel@tonic-gate 
655*7c478bd9Sstevel@tonic-gate 	return (0);
656*7c478bd9Sstevel@tonic-gate }
657*7c478bd9Sstevel@tonic-gate 
658*7c478bd9Sstevel@tonic-gate /*
659*7c478bd9Sstevel@tonic-gate  * Called from call_user_handler() and _thrp_suspend() to take
660*7c478bd9Sstevel@tonic-gate  * ourself off of our sleep queue so we can grab locks.
661*7c478bd9Sstevel@tonic-gate  */
662*7c478bd9Sstevel@tonic-gate void
663*7c478bd9Sstevel@tonic-gate unsleep_self(void)
664*7c478bd9Sstevel@tonic-gate {
665*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
666*7c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
667*7c478bd9Sstevel@tonic-gate 
668*7c478bd9Sstevel@tonic-gate 	/*
669*7c478bd9Sstevel@tonic-gate 	 * Calling enter_critical()/exit_critical() here would lead
670*7c478bd9Sstevel@tonic-gate 	 * to recursion.  Just manipulate self->ul_critical directly.
671*7c478bd9Sstevel@tonic-gate 	 */
672*7c478bd9Sstevel@tonic-gate 	self->ul_critical++;
673*7c478bd9Sstevel@tonic-gate 	self->ul_writer = 0;
674*7c478bd9Sstevel@tonic-gate 	while (self->ul_sleepq != NULL) {
675*7c478bd9Sstevel@tonic-gate 		qp = queue_lock(self->ul_wchan, self->ul_qtype);
676*7c478bd9Sstevel@tonic-gate 		/*
677*7c478bd9Sstevel@tonic-gate 		 * We may have been moved from a CV queue to a
678*7c478bd9Sstevel@tonic-gate 		 * mutex queue while we were attempting queue_lock().
679*7c478bd9Sstevel@tonic-gate 		 * If so, just loop around and try again.
680*7c478bd9Sstevel@tonic-gate 		 * dequeue_self() clears self->ul_sleepq.
681*7c478bd9Sstevel@tonic-gate 		 */
682*7c478bd9Sstevel@tonic-gate 		if (qp == self->ul_sleepq)
683*7c478bd9Sstevel@tonic-gate 			(void) dequeue_self(qp, self->ul_wchan);
684*7c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
685*7c478bd9Sstevel@tonic-gate 	}
686*7c478bd9Sstevel@tonic-gate 	self->ul_critical--;
687*7c478bd9Sstevel@tonic-gate }
688*7c478bd9Sstevel@tonic-gate 
689*7c478bd9Sstevel@tonic-gate /*
690*7c478bd9Sstevel@tonic-gate  * Common code for calling the the ___lwp_mutex_timedlock() system call.
691*7c478bd9Sstevel@tonic-gate  * Returns with mutex_owner and mutex_ownerpid set correctly.
692*7c478bd9Sstevel@tonic-gate  */
693*7c478bd9Sstevel@tonic-gate int
694*7c478bd9Sstevel@tonic-gate mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
695*7c478bd9Sstevel@tonic-gate {
696*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
697*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
698*7c478bd9Sstevel@tonic-gate 	hrtime_t begin_sleep;
699*7c478bd9Sstevel@tonic-gate 	int error;
700*7c478bd9Sstevel@tonic-gate 
701*7c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
702*7c478bd9Sstevel@tonic-gate 	self->ul_wchan = mp;
703*7c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
704*7c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
705*7c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = mp;
706*7c478bd9Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
707*7c478bd9Sstevel@tonic-gate 	}
708*7c478bd9Sstevel@tonic-gate 	if (msp) {
709*7c478bd9Sstevel@tonic-gate 		tdb_incr(msp->mutex_sleep);
710*7c478bd9Sstevel@tonic-gate 		begin_sleep = gethrtime();
711*7c478bd9Sstevel@tonic-gate 	}
712*7c478bd9Sstevel@tonic-gate 
713*7c478bd9Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__block, mp);
714*7c478bd9Sstevel@tonic-gate 
715*7c478bd9Sstevel@tonic-gate 	for (;;) {
716*7c478bd9Sstevel@tonic-gate 		if ((error = ___lwp_mutex_timedlock(mp, tsp)) != 0) {
717*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
718*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
719*7c478bd9Sstevel@tonic-gate 			break;
720*7c478bd9Sstevel@tonic-gate 		}
721*7c478bd9Sstevel@tonic-gate 
722*7c478bd9Sstevel@tonic-gate 		if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
723*7c478bd9Sstevel@tonic-gate 			/*
724*7c478bd9Sstevel@tonic-gate 			 * Defend against forkall().  We may be the child,
725*7c478bd9Sstevel@tonic-gate 			 * in which case we don't actually own the mutex.
726*7c478bd9Sstevel@tonic-gate 			 */
727*7c478bd9Sstevel@tonic-gate 			enter_critical(self);
728*7c478bd9Sstevel@tonic-gate 			if (mp->mutex_ownerpid == udp->pid) {
729*7c478bd9Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
730*7c478bd9Sstevel@tonic-gate 				exit_critical(self);
731*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
732*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
733*7c478bd9Sstevel@tonic-gate 				    0, 0);
734*7c478bd9Sstevel@tonic-gate 				break;
735*7c478bd9Sstevel@tonic-gate 			}
736*7c478bd9Sstevel@tonic-gate 			exit_critical(self);
737*7c478bd9Sstevel@tonic-gate 		} else {
738*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
739*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
740*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
741*7c478bd9Sstevel@tonic-gate 			break;
742*7c478bd9Sstevel@tonic-gate 		}
743*7c478bd9Sstevel@tonic-gate 	}
744*7c478bd9Sstevel@tonic-gate 	if (msp)
745*7c478bd9Sstevel@tonic-gate 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
746*7c478bd9Sstevel@tonic-gate 	self->ul_wchan = NULL;
747*7c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
748*7c478bd9Sstevel@tonic-gate 
749*7c478bd9Sstevel@tonic-gate 	return (error);
750*7c478bd9Sstevel@tonic-gate }
751*7c478bd9Sstevel@tonic-gate 
752*7c478bd9Sstevel@tonic-gate /*
753*7c478bd9Sstevel@tonic-gate  * Common code for calling the ___lwp_mutex_trylock() system call.
754*7c478bd9Sstevel@tonic-gate  * Returns with mutex_owner and mutex_ownerpid set correctly.
755*7c478bd9Sstevel@tonic-gate  */
756*7c478bd9Sstevel@tonic-gate int
757*7c478bd9Sstevel@tonic-gate mutex_trylock_kernel(mutex_t *mp)
758*7c478bd9Sstevel@tonic-gate {
759*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
760*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
761*7c478bd9Sstevel@tonic-gate 	int error;
762*7c478bd9Sstevel@tonic-gate 
763*7c478bd9Sstevel@tonic-gate 	for (;;) {
764*7c478bd9Sstevel@tonic-gate 		if ((error = ___lwp_mutex_trylock(mp)) != 0) {
765*7c478bd9Sstevel@tonic-gate 			if (error != EBUSY) {
766*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
767*7c478bd9Sstevel@tonic-gate 				    error);
768*7c478bd9Sstevel@tonic-gate 			}
769*7c478bd9Sstevel@tonic-gate 			break;
770*7c478bd9Sstevel@tonic-gate 		}
771*7c478bd9Sstevel@tonic-gate 
772*7c478bd9Sstevel@tonic-gate 		if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
773*7c478bd9Sstevel@tonic-gate 			/*
774*7c478bd9Sstevel@tonic-gate 			 * Defend against forkall().  We may be the child,
775*7c478bd9Sstevel@tonic-gate 			 * in which case we don't actually own the mutex.
776*7c478bd9Sstevel@tonic-gate 			 */
777*7c478bd9Sstevel@tonic-gate 			enter_critical(self);
778*7c478bd9Sstevel@tonic-gate 			if (mp->mutex_ownerpid == udp->pid) {
779*7c478bd9Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
780*7c478bd9Sstevel@tonic-gate 				exit_critical(self);
781*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
782*7c478bd9Sstevel@tonic-gate 				    0, 0);
783*7c478bd9Sstevel@tonic-gate 				break;
784*7c478bd9Sstevel@tonic-gate 			}
785*7c478bd9Sstevel@tonic-gate 			exit_critical(self);
786*7c478bd9Sstevel@tonic-gate 		} else {
787*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
788*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
789*7c478bd9Sstevel@tonic-gate 			break;
790*7c478bd9Sstevel@tonic-gate 		}
791*7c478bd9Sstevel@tonic-gate 	}
792*7c478bd9Sstevel@tonic-gate 
793*7c478bd9Sstevel@tonic-gate 	return (error);
794*7c478bd9Sstevel@tonic-gate }
795*7c478bd9Sstevel@tonic-gate 
796*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *
797*7c478bd9Sstevel@tonic-gate setup_schedctl(void)
798*7c478bd9Sstevel@tonic-gate {
799*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
800*7c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
801*7c478bd9Sstevel@tonic-gate 	sc_shared_t *tmp;
802*7c478bd9Sstevel@tonic-gate 
803*7c478bd9Sstevel@tonic-gate 	if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */
804*7c478bd9Sstevel@tonic-gate 	    !self->ul_vfork &&			/* not a child of vfork() */
805*7c478bd9Sstevel@tonic-gate 	    !self->ul_schedctl_called) {	/* haven't been called before */
806*7c478bd9Sstevel@tonic-gate 		enter_critical(self);
807*7c478bd9Sstevel@tonic-gate 		self->ul_schedctl_called = &self->ul_uberdata->uberflags;
808*7c478bd9Sstevel@tonic-gate 		if ((tmp = __schedctl()) != (sc_shared_t *)(-1))
809*7c478bd9Sstevel@tonic-gate 			self->ul_schedctl = scp = tmp;
810*7c478bd9Sstevel@tonic-gate 		exit_critical(self);
811*7c478bd9Sstevel@tonic-gate 	}
812*7c478bd9Sstevel@tonic-gate 	/*
813*7c478bd9Sstevel@tonic-gate 	 * Unless the call to setup_schedctl() is surrounded
814*7c478bd9Sstevel@tonic-gate 	 * by enter_critical()/exit_critical(), the address
815*7c478bd9Sstevel@tonic-gate 	 * we are returning could be invalid due to a forkall()
816*7c478bd9Sstevel@tonic-gate 	 * having occurred in another thread.
817*7c478bd9Sstevel@tonic-gate 	 */
818*7c478bd9Sstevel@tonic-gate 	return (scp);
819*7c478bd9Sstevel@tonic-gate }
820*7c478bd9Sstevel@tonic-gate 
821*7c478bd9Sstevel@tonic-gate /*
822*7c478bd9Sstevel@tonic-gate  * Interfaces from libsched, incorporated into libc.
823*7c478bd9Sstevel@tonic-gate  * libsched.so.1 is now a filter library onto libc.
824*7c478bd9Sstevel@tonic-gate  */
825*7c478bd9Sstevel@tonic-gate #pragma weak schedctl_lookup = _schedctl_init
826*7c478bd9Sstevel@tonic-gate #pragma weak _schedctl_lookup = _schedctl_init
827*7c478bd9Sstevel@tonic-gate #pragma weak schedctl_init = _schedctl_init
828*7c478bd9Sstevel@tonic-gate schedctl_t *
829*7c478bd9Sstevel@tonic-gate _schedctl_init(void)
830*7c478bd9Sstevel@tonic-gate {
831*7c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp = setup_schedctl();
832*7c478bd9Sstevel@tonic-gate 	return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl);
833*7c478bd9Sstevel@tonic-gate }
834*7c478bd9Sstevel@tonic-gate 
835*7c478bd9Sstevel@tonic-gate #pragma weak schedctl_exit = _schedctl_exit
836*7c478bd9Sstevel@tonic-gate void
837*7c478bd9Sstevel@tonic-gate _schedctl_exit(void)
838*7c478bd9Sstevel@tonic-gate {
839*7c478bd9Sstevel@tonic-gate }
840*7c478bd9Sstevel@tonic-gate 
841*7c478bd9Sstevel@tonic-gate /*
842*7c478bd9Sstevel@tonic-gate  * Contract private interface for java.
843*7c478bd9Sstevel@tonic-gate  * Set up the schedctl data if it doesn't exist yet.
844*7c478bd9Sstevel@tonic-gate  * Return a pointer to the pointer to the schedctl data.
845*7c478bd9Sstevel@tonic-gate  */
846*7c478bd9Sstevel@tonic-gate volatile sc_shared_t *volatile *
847*7c478bd9Sstevel@tonic-gate _thr_schedctl(void)
848*7c478bd9Sstevel@tonic-gate {
849*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
850*7c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *volatile *ptr;
851*7c478bd9Sstevel@tonic-gate 
852*7c478bd9Sstevel@tonic-gate 	if (self->ul_vfork)
853*7c478bd9Sstevel@tonic-gate 		return (NULL);
854*7c478bd9Sstevel@tonic-gate 	if (*(ptr = &self->ul_schedctl) == NULL)
855*7c478bd9Sstevel@tonic-gate 		(void) setup_schedctl();
856*7c478bd9Sstevel@tonic-gate 	return (ptr);
857*7c478bd9Sstevel@tonic-gate }
858*7c478bd9Sstevel@tonic-gate 
859*7c478bd9Sstevel@tonic-gate /*
860*7c478bd9Sstevel@tonic-gate  * Block signals and attempt to block preemption.
861*7c478bd9Sstevel@tonic-gate  * no_preempt()/preempt() must be used in pairs but can be nested.
862*7c478bd9Sstevel@tonic-gate  */
863*7c478bd9Sstevel@tonic-gate void
864*7c478bd9Sstevel@tonic-gate no_preempt(ulwp_t *self)
865*7c478bd9Sstevel@tonic-gate {
866*7c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
867*7c478bd9Sstevel@tonic-gate 
868*7c478bd9Sstevel@tonic-gate 	if (self->ul_preempt++ == 0) {
869*7c478bd9Sstevel@tonic-gate 		enter_critical(self);
870*7c478bd9Sstevel@tonic-gate 		if ((scp = self->ul_schedctl) != NULL ||
871*7c478bd9Sstevel@tonic-gate 		    (scp = setup_schedctl()) != NULL) {
872*7c478bd9Sstevel@tonic-gate 			/*
873*7c478bd9Sstevel@tonic-gate 			 * Save the pre-existing preempt value.
874*7c478bd9Sstevel@tonic-gate 			 */
875*7c478bd9Sstevel@tonic-gate 			self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt;
876*7c478bd9Sstevel@tonic-gate 			scp->sc_preemptctl.sc_nopreempt = 1;
877*7c478bd9Sstevel@tonic-gate 		}
878*7c478bd9Sstevel@tonic-gate 	}
879*7c478bd9Sstevel@tonic-gate }
880*7c478bd9Sstevel@tonic-gate 
881*7c478bd9Sstevel@tonic-gate /*
882*7c478bd9Sstevel@tonic-gate  * Undo the effects of no_preempt().
883*7c478bd9Sstevel@tonic-gate  */
884*7c478bd9Sstevel@tonic-gate void
885*7c478bd9Sstevel@tonic-gate preempt(ulwp_t *self)
886*7c478bd9Sstevel@tonic-gate {
887*7c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
888*7c478bd9Sstevel@tonic-gate 
889*7c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_preempt > 0);
890*7c478bd9Sstevel@tonic-gate 	if (--self->ul_preempt == 0) {
891*7c478bd9Sstevel@tonic-gate 		if ((scp = self->ul_schedctl) != NULL) {
892*7c478bd9Sstevel@tonic-gate 			/*
893*7c478bd9Sstevel@tonic-gate 			 * Restore the pre-existing preempt value.
894*7c478bd9Sstevel@tonic-gate 			 */
895*7c478bd9Sstevel@tonic-gate 			scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt;
896*7c478bd9Sstevel@tonic-gate 			if (scp->sc_preemptctl.sc_yield &&
897*7c478bd9Sstevel@tonic-gate 			    scp->sc_preemptctl.sc_nopreempt == 0) {
898*7c478bd9Sstevel@tonic-gate 				lwp_yield();
899*7c478bd9Sstevel@tonic-gate 				if (scp->sc_preemptctl.sc_yield) {
900*7c478bd9Sstevel@tonic-gate 					/*
901*7c478bd9Sstevel@tonic-gate 					 * Shouldn't happen.  This is either
902*7c478bd9Sstevel@tonic-gate 					 * a race condition or the thread
903*7c478bd9Sstevel@tonic-gate 					 * just entered the real-time class.
904*7c478bd9Sstevel@tonic-gate 					 */
905*7c478bd9Sstevel@tonic-gate 					lwp_yield();
906*7c478bd9Sstevel@tonic-gate 					scp->sc_preemptctl.sc_yield = 0;
907*7c478bd9Sstevel@tonic-gate 				}
908*7c478bd9Sstevel@tonic-gate 			}
909*7c478bd9Sstevel@tonic-gate 		}
910*7c478bd9Sstevel@tonic-gate 		exit_critical(self);
911*7c478bd9Sstevel@tonic-gate 	}
912*7c478bd9Sstevel@tonic-gate }
913*7c478bd9Sstevel@tonic-gate 
914*7c478bd9Sstevel@tonic-gate /*
915*7c478bd9Sstevel@tonic-gate  * If a call to preempt() would cause the current thread to yield or to
916*7c478bd9Sstevel@tonic-gate  * take deferred actions in exit_critical(), then unpark the specified
917*7c478bd9Sstevel@tonic-gate  * lwp so it can run while we delay.  Return the original lwpid if the
918*7c478bd9Sstevel@tonic-gate  * unpark was not performed, else return zero.  The tests are a repeat
919*7c478bd9Sstevel@tonic-gate  * of some of the tests in preempt(), above.  This is a statistical
920*7c478bd9Sstevel@tonic-gate  * optimization solely for cond_sleep_queue(), below.
921*7c478bd9Sstevel@tonic-gate  */
922*7c478bd9Sstevel@tonic-gate static lwpid_t
923*7c478bd9Sstevel@tonic-gate preempt_unpark(ulwp_t *self, lwpid_t lwpid)
924*7c478bd9Sstevel@tonic-gate {
925*7c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp = self->ul_schedctl;
926*7c478bd9Sstevel@tonic-gate 
927*7c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_preempt == 1 && self->ul_critical > 0);
928*7c478bd9Sstevel@tonic-gate 	if ((scp != NULL && scp->sc_preemptctl.sc_yield) ||
929*7c478bd9Sstevel@tonic-gate 	    (self->ul_curplease && self->ul_critical == 1)) {
930*7c478bd9Sstevel@tonic-gate 		(void) __lwp_unpark(lwpid);
931*7c478bd9Sstevel@tonic-gate 		lwpid = 0;
932*7c478bd9Sstevel@tonic-gate 	}
933*7c478bd9Sstevel@tonic-gate 	return (lwpid);
934*7c478bd9Sstevel@tonic-gate }
935*7c478bd9Sstevel@tonic-gate 
936*7c478bd9Sstevel@tonic-gate /*
937*7c478bd9Sstevel@tonic-gate  * Spin for a while, trying to grab the lock.  We know that we
938*7c478bd9Sstevel@tonic-gate  * failed set_lock_byte(&mp->mutex_lockw) once before coming here.
939*7c478bd9Sstevel@tonic-gate  * If this fails, return EBUSY and let the caller deal with it.
940*7c478bd9Sstevel@tonic-gate  * If this succeeds, return 0 with mutex_owner set to curthread.
941*7c478bd9Sstevel@tonic-gate  */
942*7c478bd9Sstevel@tonic-gate int
943*7c478bd9Sstevel@tonic-gate mutex_trylock_adaptive(mutex_t *mp)
944*7c478bd9Sstevel@tonic-gate {
945*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
946*7c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
947*7c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
948*7c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
949*7c478bd9Sstevel@tonic-gate 	volatile uint64_t *ownerp;
950*7c478bd9Sstevel@tonic-gate 	int count, max = self->ul_adaptive_spin;
951*7c478bd9Sstevel@tonic-gate 
952*7c478bd9Sstevel@tonic-gate 	ASSERT(!(mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)));
953*7c478bd9Sstevel@tonic-gate 
954*7c478bd9Sstevel@tonic-gate 	if (max == 0 || (mp->mutex_spinners >= self->ul_max_spinners))
955*7c478bd9Sstevel@tonic-gate 		return (EBUSY);
956*7c478bd9Sstevel@tonic-gate 
957*7c478bd9Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
958*7c478bd9Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
959*7c478bd9Sstevel@tonic-gate 
960*7c478bd9Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
961*7c478bd9Sstevel@tonic-gate 
962*7c478bd9Sstevel@tonic-gate 	/*
963*7c478bd9Sstevel@tonic-gate 	 * This spin loop is unfair to lwps that have already dropped into
964*7c478bd9Sstevel@tonic-gate 	 * the kernel to sleep.  They will starve on a highly-contended mutex.
965*7c478bd9Sstevel@tonic-gate 	 * This is just too bad.  The adaptive spin algorithm is intended
966*7c478bd9Sstevel@tonic-gate 	 * to allow programs with highly-contended locks (that is, broken
967*7c478bd9Sstevel@tonic-gate 	 * programs) to execute with reasonable speed despite their contention.
968*7c478bd9Sstevel@tonic-gate 	 * Being fair would reduce the speed of such programs and well-written
969*7c478bd9Sstevel@tonic-gate 	 * programs will not suffer in any case.
970*7c478bd9Sstevel@tonic-gate 	 */
971*7c478bd9Sstevel@tonic-gate 	enter_critical(self);		/* protects ul_schedctl */
972*7c478bd9Sstevel@tonic-gate 	incr32(&mp->mutex_spinners);
973*7c478bd9Sstevel@tonic-gate 	for (count = 0; count < max; count++) {
974*7c478bd9Sstevel@tonic-gate 		if (*lockp == 0 && set_lock_byte(lockp) == 0) {
975*7c478bd9Sstevel@tonic-gate 			*ownerp = (uintptr_t)self;
976*7c478bd9Sstevel@tonic-gate 			decr32(&mp->mutex_spinners);
977*7c478bd9Sstevel@tonic-gate 			exit_critical(self);
978*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__spun, 1, count);
979*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
980*7c478bd9Sstevel@tonic-gate 			return (0);
981*7c478bd9Sstevel@tonic-gate 		}
982*7c478bd9Sstevel@tonic-gate 		SMT_PAUSE();
983*7c478bd9Sstevel@tonic-gate 		/*
984*7c478bd9Sstevel@tonic-gate 		 * Stop spinning if the mutex owner is not running on
985*7c478bd9Sstevel@tonic-gate 		 * a processor; it will not drop the lock any time soon
986*7c478bd9Sstevel@tonic-gate 		 * and we would just be wasting time to keep spinning.
987*7c478bd9Sstevel@tonic-gate 		 *
988*7c478bd9Sstevel@tonic-gate 		 * Note that we are looking at another thread (ulwp_t)
989*7c478bd9Sstevel@tonic-gate 		 * without ensuring that the other thread does not exit.
990*7c478bd9Sstevel@tonic-gate 		 * The scheme relies on ulwp_t structures never being
991*7c478bd9Sstevel@tonic-gate 		 * deallocated by the library (the library employs a free
992*7c478bd9Sstevel@tonic-gate 		 * list of ulwp_t structs that are reused when new threads
993*7c478bd9Sstevel@tonic-gate 		 * are created) and on schedctl shared memory never being
994*7c478bd9Sstevel@tonic-gate 		 * deallocated once created via __schedctl().
995*7c478bd9Sstevel@tonic-gate 		 *
996*7c478bd9Sstevel@tonic-gate 		 * Thus, the worst that can happen when the spinning thread
997*7c478bd9Sstevel@tonic-gate 		 * looks at the owner's schedctl data is that it is looking
998*7c478bd9Sstevel@tonic-gate 		 * at some other thread's schedctl data.  This almost never
999*7c478bd9Sstevel@tonic-gate 		 * happens and is benign when it does.
1000*7c478bd9Sstevel@tonic-gate 		 */
1001*7c478bd9Sstevel@tonic-gate 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
1002*7c478bd9Sstevel@tonic-gate 		    ((scp = ulwp->ul_schedctl) == NULL ||
1003*7c478bd9Sstevel@tonic-gate 		    scp->sc_state != SC_ONPROC))
1004*7c478bd9Sstevel@tonic-gate 			break;
1005*7c478bd9Sstevel@tonic-gate 	}
1006*7c478bd9Sstevel@tonic-gate 	decr32(&mp->mutex_spinners);
1007*7c478bd9Sstevel@tonic-gate 	exit_critical(self);
1008*7c478bd9Sstevel@tonic-gate 
1009*7c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__spun, 0, count);
1010*7c478bd9Sstevel@tonic-gate 
1011*7c478bd9Sstevel@tonic-gate 	return (EBUSY);
1012*7c478bd9Sstevel@tonic-gate }
1013*7c478bd9Sstevel@tonic-gate 
1014*7c478bd9Sstevel@tonic-gate /*
1015*7c478bd9Sstevel@tonic-gate  * Same as mutex_trylock_adaptive(), except specifically for queue locks.
1016*7c478bd9Sstevel@tonic-gate  * The owner field is not set here; the caller (spin_lock_set()) sets it.
1017*7c478bd9Sstevel@tonic-gate  */
1018*7c478bd9Sstevel@tonic-gate int
1019*7c478bd9Sstevel@tonic-gate mutex_queuelock_adaptive(mutex_t *mp)
1020*7c478bd9Sstevel@tonic-gate {
1021*7c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
1022*7c478bd9Sstevel@tonic-gate 	volatile sc_shared_t *scp;
1023*7c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
1024*7c478bd9Sstevel@tonic-gate 	volatile uint64_t *ownerp;
1025*7c478bd9Sstevel@tonic-gate 	int count = curthread->ul_queue_spin;
1026*7c478bd9Sstevel@tonic-gate 
1027*7c478bd9Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
1028*7c478bd9Sstevel@tonic-gate 
1029*7c478bd9Sstevel@tonic-gate 	if (count == 0)
1030*7c478bd9Sstevel@tonic-gate 		return (EBUSY);
1031*7c478bd9Sstevel@tonic-gate 
1032*7c478bd9Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
1033*7c478bd9Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
1034*7c478bd9Sstevel@tonic-gate 	while (--count >= 0) {
1035*7c478bd9Sstevel@tonic-gate 		if (*lockp == 0 && set_lock_byte(lockp) == 0)
1036*7c478bd9Sstevel@tonic-gate 			return (0);
1037*7c478bd9Sstevel@tonic-gate 		SMT_PAUSE();
1038*7c478bd9Sstevel@tonic-gate 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
1039*7c478bd9Sstevel@tonic-gate 		    ((scp = ulwp->ul_schedctl) == NULL ||
1040*7c478bd9Sstevel@tonic-gate 		    scp->sc_state != SC_ONPROC))
1041*7c478bd9Sstevel@tonic-gate 			break;
1042*7c478bd9Sstevel@tonic-gate 	}
1043*7c478bd9Sstevel@tonic-gate 
1044*7c478bd9Sstevel@tonic-gate 	return (EBUSY);
1045*7c478bd9Sstevel@tonic-gate }
1046*7c478bd9Sstevel@tonic-gate 
1047*7c478bd9Sstevel@tonic-gate /*
1048*7c478bd9Sstevel@tonic-gate  * Like mutex_trylock_adaptive(), but for process-shared mutexes.
1049*7c478bd9Sstevel@tonic-gate  * Spin for a while, trying to grab the lock.  We know that we
1050*7c478bd9Sstevel@tonic-gate  * failed set_lock_byte(&mp->mutex_lockw) once before coming here.
1051*7c478bd9Sstevel@tonic-gate  * If this fails, return EBUSY and let the caller deal with it.
1052*7c478bd9Sstevel@tonic-gate  * If this succeeds, return 0 with mutex_owner set to curthread
1053*7c478bd9Sstevel@tonic-gate  * and mutex_ownerpid set to the current pid.
1054*7c478bd9Sstevel@tonic-gate  */
1055*7c478bd9Sstevel@tonic-gate int
1056*7c478bd9Sstevel@tonic-gate mutex_trylock_process(mutex_t *mp)
1057*7c478bd9Sstevel@tonic-gate {
1058*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1059*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
1060*7c478bd9Sstevel@tonic-gate 	int count;
1061*7c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
1062*7c478bd9Sstevel@tonic-gate 	volatile uint64_t *ownerp;
1063*7c478bd9Sstevel@tonic-gate 	volatile int32_t *pidp;
1064*7c478bd9Sstevel@tonic-gate 	pid_t pid, newpid;
1065*7c478bd9Sstevel@tonic-gate 	uint64_t owner, newowner;
1066*7c478bd9Sstevel@tonic-gate 
1067*7c478bd9Sstevel@tonic-gate 	if ((count = ncpus) == 0)
1068*7c478bd9Sstevel@tonic-gate 		count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
1069*7c478bd9Sstevel@tonic-gate 	count = (count > 1)? self->ul_adaptive_spin : 0;
1070*7c478bd9Sstevel@tonic-gate 
1071*7c478bd9Sstevel@tonic-gate 	ASSERT((mp->mutex_type & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) ==
1072*7c478bd9Sstevel@tonic-gate 		USYNC_PROCESS);
1073*7c478bd9Sstevel@tonic-gate 
1074*7c478bd9Sstevel@tonic-gate 	if (count == 0)
1075*7c478bd9Sstevel@tonic-gate 		return (EBUSY);
1076*7c478bd9Sstevel@tonic-gate 
1077*7c478bd9Sstevel@tonic-gate 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
1078*7c478bd9Sstevel@tonic-gate 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
1079*7c478bd9Sstevel@tonic-gate 	pidp = (volatile int32_t *)&mp->mutex_ownerpid;
1080*7c478bd9Sstevel@tonic-gate 	owner = *ownerp;
1081*7c478bd9Sstevel@tonic-gate 	pid = *pidp;
1082*7c478bd9Sstevel@tonic-gate 	/*
1083*7c478bd9Sstevel@tonic-gate 	 * This is a process-shared mutex.
1084*7c478bd9Sstevel@tonic-gate 	 * We cannot know if the owner is running on a processor.
1085*7c478bd9Sstevel@tonic-gate 	 * We just spin and hope that it is on a processor.
1086*7c478bd9Sstevel@tonic-gate 	 */
1087*7c478bd9Sstevel@tonic-gate 	while (--count >= 0) {
1088*7c478bd9Sstevel@tonic-gate 		if (*lockp == 0) {
1089*7c478bd9Sstevel@tonic-gate 			enter_critical(self);
1090*7c478bd9Sstevel@tonic-gate 			if (set_lock_byte(lockp) == 0) {
1091*7c478bd9Sstevel@tonic-gate 				*ownerp = (uintptr_t)self;
1092*7c478bd9Sstevel@tonic-gate 				*pidp = udp->pid;
1093*7c478bd9Sstevel@tonic-gate 				exit_critical(self);
1094*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
1095*7c478bd9Sstevel@tonic-gate 				    0, 0);
1096*7c478bd9Sstevel@tonic-gate 				return (0);
1097*7c478bd9Sstevel@tonic-gate 			}
1098*7c478bd9Sstevel@tonic-gate 			exit_critical(self);
1099*7c478bd9Sstevel@tonic-gate 		} else if ((newowner = *ownerp) == owner &&
1100*7c478bd9Sstevel@tonic-gate 		    (newpid = *pidp) == pid) {
1101*7c478bd9Sstevel@tonic-gate 			SMT_PAUSE();
1102*7c478bd9Sstevel@tonic-gate 			continue;
1103*7c478bd9Sstevel@tonic-gate 		}
1104*7c478bd9Sstevel@tonic-gate 		/*
1105*7c478bd9Sstevel@tonic-gate 		 * The owner of the lock changed; start the count over again.
1106*7c478bd9Sstevel@tonic-gate 		 * This may be too aggressive; it needs testing.
1107*7c478bd9Sstevel@tonic-gate 		 */
1108*7c478bd9Sstevel@tonic-gate 		owner = newowner;
1109*7c478bd9Sstevel@tonic-gate 		pid = newpid;
1110*7c478bd9Sstevel@tonic-gate 		count = self->ul_adaptive_spin;
1111*7c478bd9Sstevel@tonic-gate 	}
1112*7c478bd9Sstevel@tonic-gate 
1113*7c478bd9Sstevel@tonic-gate 	return (EBUSY);
1114*7c478bd9Sstevel@tonic-gate }
1115*7c478bd9Sstevel@tonic-gate 
1116*7c478bd9Sstevel@tonic-gate /*
1117*7c478bd9Sstevel@tonic-gate  * Mutex wakeup code for releasing a USYNC_THREAD mutex.
1118*7c478bd9Sstevel@tonic-gate  * Returns the lwpid of the thread that was dequeued, if any.
1119*7c478bd9Sstevel@tonic-gate  * The caller of mutex_wakeup() must call __lwp_unpark(lwpid)
1120*7c478bd9Sstevel@tonic-gate  * to wake up the specified lwp.
1121*7c478bd9Sstevel@tonic-gate  */
1122*7c478bd9Sstevel@tonic-gate lwpid_t
1123*7c478bd9Sstevel@tonic-gate mutex_wakeup(mutex_t *mp)
1124*7c478bd9Sstevel@tonic-gate {
1125*7c478bd9Sstevel@tonic-gate 	lwpid_t lwpid = 0;
1126*7c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
1127*7c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
1128*7c478bd9Sstevel@tonic-gate 	int more;
1129*7c478bd9Sstevel@tonic-gate 
1130*7c478bd9Sstevel@tonic-gate 	/*
1131*7c478bd9Sstevel@tonic-gate 	 * Dequeue a waiter from the sleep queue.  Don't touch the mutex
1132*7c478bd9Sstevel@tonic-gate 	 * waiters bit if no one was found on the queue because the mutex
1133*7c478bd9Sstevel@tonic-gate 	 * might have been deallocated or reallocated for another purpose.
1134*7c478bd9Sstevel@tonic-gate 	 */
1135*7c478bd9Sstevel@tonic-gate 	qp = queue_lock(mp, MX);
1136*7c478bd9Sstevel@tonic-gate 	if ((ulwp = dequeue(qp, mp, &more)) != NULL) {
1137*7c478bd9Sstevel@tonic-gate 		lwpid = ulwp->ul_lwpid;
1138*7c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = (more? 1 : 0);
1139*7c478bd9Sstevel@tonic-gate 	}
1140*7c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
1141*7c478bd9Sstevel@tonic-gate 	return (lwpid);
1142*7c478bd9Sstevel@tonic-gate }
1143*7c478bd9Sstevel@tonic-gate 
1144*7c478bd9Sstevel@tonic-gate /*
1145*7c478bd9Sstevel@tonic-gate  * Spin for a while, testing to see if the lock has been grabbed.
1146*7c478bd9Sstevel@tonic-gate  * If this fails, call mutex_wakeup() to release a waiter.
1147*7c478bd9Sstevel@tonic-gate  */
1148*7c478bd9Sstevel@tonic-gate lwpid_t
1149*7c478bd9Sstevel@tonic-gate mutex_unlock_queue(mutex_t *mp)
1150*7c478bd9Sstevel@tonic-gate {
1151*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1152*7c478bd9Sstevel@tonic-gate 	uint32_t *lockw = &mp->mutex_lockword;
1153*7c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
1154*7c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
1155*7c478bd9Sstevel@tonic-gate 	volatile uint32_t *spinp;
1156*7c478bd9Sstevel@tonic-gate 	int count;
1157*7c478bd9Sstevel@tonic-gate 
1158*7c478bd9Sstevel@tonic-gate 	/*
1159*7c478bd9Sstevel@tonic-gate 	 * We use the swap primitive to clear the lock, but we must
1160*7c478bd9Sstevel@tonic-gate 	 * atomically retain the waiters bit for the remainder of this
1161*7c478bd9Sstevel@tonic-gate 	 * code to work.  We first check to see if the waiters bit is
1162*7c478bd9Sstevel@tonic-gate 	 * set and if so clear the lock by swapping in a word containing
1163*7c478bd9Sstevel@tonic-gate 	 * only the waiters bit.  This could produce a false positive test
1164*7c478bd9Sstevel@tonic-gate 	 * for whether there are waiters that need to be waked up, but
1165*7c478bd9Sstevel@tonic-gate 	 * this just causes an extra call to mutex_wakeup() to do nothing.
1166*7c478bd9Sstevel@tonic-gate 	 * The opposite case is more delicate:  If there are no waiters,
1167*7c478bd9Sstevel@tonic-gate 	 * we swap in a zero lock byte and a zero waiters bit.  The result
1168*7c478bd9Sstevel@tonic-gate 	 * of the swap could indicate that there really was a waiter so in
1169*7c478bd9Sstevel@tonic-gate 	 * this case we go directly to mutex_wakeup() without performing
1170*7c478bd9Sstevel@tonic-gate 	 * any of the adaptive code because the waiter bit has been cleared
1171*7c478bd9Sstevel@tonic-gate 	 * and the adaptive code is unreliable in this case.
1172*7c478bd9Sstevel@tonic-gate 	 */
1173*7c478bd9Sstevel@tonic-gate 	if (!(*lockw & WAITERMASK)) {	/* no waiter exists right now */
1174*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
1175*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
1176*7c478bd9Sstevel@tonic-gate 		if (!(swap32(lockw, 0) & WAITERMASK))	/* still no waiters */
1177*7c478bd9Sstevel@tonic-gate 			return (0);
1178*7c478bd9Sstevel@tonic-gate 		no_preempt(self);	/* ensure a prompt wakeup */
1179*7c478bd9Sstevel@tonic-gate 		lwpid = mutex_wakeup(mp);
1180*7c478bd9Sstevel@tonic-gate 	} else {
1181*7c478bd9Sstevel@tonic-gate 		no_preempt(self);	/* ensure a prompt wakeup */
1182*7c478bd9Sstevel@tonic-gate 		lockp = (volatile uint8_t *)&mp->mutex_lockw;
1183*7c478bd9Sstevel@tonic-gate 		spinp = (volatile uint32_t *)&mp->mutex_spinners;
1184*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
1185*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
1186*7c478bd9Sstevel@tonic-gate 		(void) swap32(lockw, WAITER);	/* clear lock, retain waiter */
1187*7c478bd9Sstevel@tonic-gate 
1188*7c478bd9Sstevel@tonic-gate 		/*
1189*7c478bd9Sstevel@tonic-gate 		 * We spin here fewer times than mutex_trylock_adaptive().
1190*7c478bd9Sstevel@tonic-gate 		 * We are trying to balance two conflicting goals:
1191*7c478bd9Sstevel@tonic-gate 		 * 1. Avoid waking up anyone if a spinning thread
1192*7c478bd9Sstevel@tonic-gate 		 *    grabs the lock.
1193*7c478bd9Sstevel@tonic-gate 		 * 2. Wake up a sleeping thread promptly to get on
1194*7c478bd9Sstevel@tonic-gate 		 *    with useful work.
1195*7c478bd9Sstevel@tonic-gate 		 * We don't spin at all if there is no acquiring spinner;
1196*7c478bd9Sstevel@tonic-gate 		 * (mp->mutex_spinners is non-zero if there are spinners).
1197*7c478bd9Sstevel@tonic-gate 		 */
1198*7c478bd9Sstevel@tonic-gate 		for (count = self->ul_release_spin;
1199*7c478bd9Sstevel@tonic-gate 		    *spinp && count > 0; count--) {
1200*7c478bd9Sstevel@tonic-gate 			/*
1201*7c478bd9Sstevel@tonic-gate 			 * There is a waiter that we will have to wake
1202*7c478bd9Sstevel@tonic-gate 			 * up unless someone else grabs the lock while
1203*7c478bd9Sstevel@tonic-gate 			 * we are busy spinning.  Like the spin loop in
1204*7c478bd9Sstevel@tonic-gate 			 * mutex_trylock_adaptive(), this spin loop is
1205*7c478bd9Sstevel@tonic-gate 			 * unfair to lwps that have already dropped into
1206*7c478bd9Sstevel@tonic-gate 			 * the kernel to sleep.  They will starve on a
1207*7c478bd9Sstevel@tonic-gate 			 * highly-contended mutex.  Too bad.
1208*7c478bd9Sstevel@tonic-gate 			 */
1209*7c478bd9Sstevel@tonic-gate 			if (*lockp != 0) {	/* somebody grabbed the lock */
1210*7c478bd9Sstevel@tonic-gate 				preempt(self);
1211*7c478bd9Sstevel@tonic-gate 				return (0);
1212*7c478bd9Sstevel@tonic-gate 			}
1213*7c478bd9Sstevel@tonic-gate 			SMT_PAUSE();
1214*7c478bd9Sstevel@tonic-gate 		}
1215*7c478bd9Sstevel@tonic-gate 
1216*7c478bd9Sstevel@tonic-gate 		/*
1217*7c478bd9Sstevel@tonic-gate 		 * No one grabbed the lock.
1218*7c478bd9Sstevel@tonic-gate 		 * Wake up some lwp that is waiting for it.
1219*7c478bd9Sstevel@tonic-gate 		 */
1220*7c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = 0;
1221*7c478bd9Sstevel@tonic-gate 		lwpid = mutex_wakeup(mp);
1222*7c478bd9Sstevel@tonic-gate 	}
1223*7c478bd9Sstevel@tonic-gate 
1224*7c478bd9Sstevel@tonic-gate 	if (lwpid == 0)
1225*7c478bd9Sstevel@tonic-gate 		preempt(self);
1226*7c478bd9Sstevel@tonic-gate 	return (lwpid);
1227*7c478bd9Sstevel@tonic-gate }
1228*7c478bd9Sstevel@tonic-gate 
1229*7c478bd9Sstevel@tonic-gate /*
1230*7c478bd9Sstevel@tonic-gate  * Like mutex_unlock_queue(), but for process-shared mutexes.
1231*7c478bd9Sstevel@tonic-gate  * We tested the waiters field before calling here and it was non-zero.
1232*7c478bd9Sstevel@tonic-gate  */
1233*7c478bd9Sstevel@tonic-gate void
1234*7c478bd9Sstevel@tonic-gate mutex_unlock_process(mutex_t *mp)
1235*7c478bd9Sstevel@tonic-gate {
1236*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1237*7c478bd9Sstevel@tonic-gate 	int count;
1238*7c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp;
1239*7c478bd9Sstevel@tonic-gate 
1240*7c478bd9Sstevel@tonic-gate 	/*
1241*7c478bd9Sstevel@tonic-gate 	 * See the comments in mutex_unlock_queue(), above.
1242*7c478bd9Sstevel@tonic-gate 	 */
1243*7c478bd9Sstevel@tonic-gate 	if ((count = ncpus) == 0)
1244*7c478bd9Sstevel@tonic-gate 		count = ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
1245*7c478bd9Sstevel@tonic-gate 	count = (count > 1)? self->ul_release_spin : 0;
1246*7c478bd9Sstevel@tonic-gate 	no_preempt(self);
1247*7c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
1248*7c478bd9Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
1249*7c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
1250*7c478bd9Sstevel@tonic-gate 	if (count == 0) {
1251*7c478bd9Sstevel@tonic-gate 		/* clear lock, test waiter */
1252*7c478bd9Sstevel@tonic-gate 		if (!(swap32(&mp->mutex_lockword, 0) & WAITERMASK)) {
1253*7c478bd9Sstevel@tonic-gate 			/* no waiters now */
1254*7c478bd9Sstevel@tonic-gate 			preempt(self);
1255*7c478bd9Sstevel@tonic-gate 			return;
1256*7c478bd9Sstevel@tonic-gate 		}
1257*7c478bd9Sstevel@tonic-gate 	} else {
1258*7c478bd9Sstevel@tonic-gate 		/* clear lock, retain waiter */
1259*7c478bd9Sstevel@tonic-gate 		(void) swap32(&mp->mutex_lockword, WAITER);
1260*7c478bd9Sstevel@tonic-gate 		lockp = (volatile uint8_t *)&mp->mutex_lockw;
1261*7c478bd9Sstevel@tonic-gate 		while (--count >= 0) {
1262*7c478bd9Sstevel@tonic-gate 			if (*lockp != 0) {
1263*7c478bd9Sstevel@tonic-gate 				/* somebody grabbed the lock */
1264*7c478bd9Sstevel@tonic-gate 				preempt(self);
1265*7c478bd9Sstevel@tonic-gate 				return;
1266*7c478bd9Sstevel@tonic-gate 			}
1267*7c478bd9Sstevel@tonic-gate 			SMT_PAUSE();
1268*7c478bd9Sstevel@tonic-gate 		}
1269*7c478bd9Sstevel@tonic-gate 		/*
1270*7c478bd9Sstevel@tonic-gate 		 * We must clear the waiters field before going
1271*7c478bd9Sstevel@tonic-gate 		 * to the kernel, else it could remain set forever.
1272*7c478bd9Sstevel@tonic-gate 		 */
1273*7c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = 0;
1274*7c478bd9Sstevel@tonic-gate 	}
1275*7c478bd9Sstevel@tonic-gate 	(void) ___lwp_mutex_wakeup(mp);
1276*7c478bd9Sstevel@tonic-gate 	preempt(self);
1277*7c478bd9Sstevel@tonic-gate }
1278*7c478bd9Sstevel@tonic-gate 
1279*7c478bd9Sstevel@tonic-gate /*
1280*7c478bd9Sstevel@tonic-gate  * Return the real priority of a thread.
1281*7c478bd9Sstevel@tonic-gate  */
1282*7c478bd9Sstevel@tonic-gate int
1283*7c478bd9Sstevel@tonic-gate real_priority(ulwp_t *ulwp)
1284*7c478bd9Sstevel@tonic-gate {
1285*7c478bd9Sstevel@tonic-gate 	if (ulwp->ul_epri == 0)
1286*7c478bd9Sstevel@tonic-gate 		return (ulwp->ul_mappedpri? ulwp->ul_mappedpri : ulwp->ul_pri);
1287*7c478bd9Sstevel@tonic-gate 	return (ulwp->ul_emappedpri? ulwp->ul_emappedpri : ulwp->ul_epri);
1288*7c478bd9Sstevel@tonic-gate }
1289*7c478bd9Sstevel@tonic-gate 
1290*7c478bd9Sstevel@tonic-gate void
1291*7c478bd9Sstevel@tonic-gate stall(void)
1292*7c478bd9Sstevel@tonic-gate {
1293*7c478bd9Sstevel@tonic-gate 	for (;;)
1294*7c478bd9Sstevel@tonic-gate 		(void) mutex_lock_kernel(&stall_mutex, NULL, NULL);
1295*7c478bd9Sstevel@tonic-gate }
1296*7c478bd9Sstevel@tonic-gate 
1297*7c478bd9Sstevel@tonic-gate /*
1298*7c478bd9Sstevel@tonic-gate  * Acquire a USYNC_THREAD mutex via user-level sleep queues.
1299*7c478bd9Sstevel@tonic-gate  * We failed set_lock_byte(&mp->mutex_lockw) before coming here.
1300*7c478bd9Sstevel@tonic-gate  * Returns with mutex_owner set correctly.
1301*7c478bd9Sstevel@tonic-gate  */
1302*7c478bd9Sstevel@tonic-gate int
1303*7c478bd9Sstevel@tonic-gate mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
1304*7c478bd9Sstevel@tonic-gate 	timespec_t *tsp)
1305*7c478bd9Sstevel@tonic-gate {
1306*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
1307*7c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
1308*7c478bd9Sstevel@tonic-gate 	hrtime_t begin_sleep;
1309*7c478bd9Sstevel@tonic-gate 	int error = 0;
1310*7c478bd9Sstevel@tonic-gate 
1311*7c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
1312*7c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
1313*7c478bd9Sstevel@tonic-gate 		self->ul_wchan = mp;
1314*7c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
1315*7c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = mp;
1316*7c478bd9Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
1317*7c478bd9Sstevel@tonic-gate 	}
1318*7c478bd9Sstevel@tonic-gate 	if (msp) {
1319*7c478bd9Sstevel@tonic-gate 		tdb_incr(msp->mutex_sleep);
1320*7c478bd9Sstevel@tonic-gate 		begin_sleep = gethrtime();
1321*7c478bd9Sstevel@tonic-gate 	}
1322*7c478bd9Sstevel@tonic-gate 
1323*7c478bd9Sstevel@tonic-gate 	DTRACE_PROBE1(plockstat, mutex__block, mp);
1324*7c478bd9Sstevel@tonic-gate 
1325*7c478bd9Sstevel@tonic-gate 	/*
1326*7c478bd9Sstevel@tonic-gate 	 * Put ourself on the sleep queue, and while we are
1327*7c478bd9Sstevel@tonic-gate 	 * unable to grab the lock, go park in the kernel.
1328*7c478bd9Sstevel@tonic-gate 	 * Take ourself off the sleep queue after we acquire the lock.
1329*7c478bd9Sstevel@tonic-gate 	 * The waiter bit can be set/cleared only while holding the queue lock.
1330*7c478bd9Sstevel@tonic-gate 	 */
1331*7c478bd9Sstevel@tonic-gate 	qp = queue_lock(mp, MX);
1332*7c478bd9Sstevel@tonic-gate 	enqueue(qp, self, mp, MX);
1333*7c478bd9Sstevel@tonic-gate 	mp->mutex_waiters = 1;
1334*7c478bd9Sstevel@tonic-gate 	for (;;) {
1335*7c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
1336*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
1337*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
1338*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1339*7c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = dequeue_self(qp, mp);
1340*7c478bd9Sstevel@tonic-gate 			break;
1341*7c478bd9Sstevel@tonic-gate 		}
1342*7c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 1);
1343*7c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
1344*7c478bd9Sstevel@tonic-gate 		/*
1345*7c478bd9Sstevel@tonic-gate 		 * __lwp_park() will return the residual time in tsp
1346*7c478bd9Sstevel@tonic-gate 		 * if we are unparked before the timeout expires.
1347*7c478bd9Sstevel@tonic-gate 		 */
1348*7c478bd9Sstevel@tonic-gate 		if ((error = __lwp_park(tsp, 0)) == EINTR)
1349*7c478bd9Sstevel@tonic-gate 			error = 0;
1350*7c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 0);
1351*7c478bd9Sstevel@tonic-gate 		/*
1352*7c478bd9Sstevel@tonic-gate 		 * We could have taken a signal or suspended ourself.
1353*7c478bd9Sstevel@tonic-gate 		 * If we did, then we removed ourself from the queue.
1354*7c478bd9Sstevel@tonic-gate 		 * Someone else may have removed us from the queue
1355*7c478bd9Sstevel@tonic-gate 		 * as a consequence of mutex_unlock().  We may have
1356*7c478bd9Sstevel@tonic-gate 		 * gotten a timeout from __lwp_park().  Or we may still
1357*7c478bd9Sstevel@tonic-gate 		 * be on the queue and this is just a spurious wakeup.
1358*7c478bd9Sstevel@tonic-gate 		 */
1359*7c478bd9Sstevel@tonic-gate 		qp = queue_lock(mp, MX);
1360*7c478bd9Sstevel@tonic-gate 		if (self->ul_sleepq == NULL) {
1361*7c478bd9Sstevel@tonic-gate 			if (error) {
1362*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
1363*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
1364*7c478bd9Sstevel@tonic-gate 				    error);
1365*7c478bd9Sstevel@tonic-gate 				break;
1366*7c478bd9Sstevel@tonic-gate 			}
1367*7c478bd9Sstevel@tonic-gate 			if (set_lock_byte(&mp->mutex_lockw) == 0) {
1368*7c478bd9Sstevel@tonic-gate 				mp->mutex_owner = (uintptr_t)self;
1369*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
1370*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
1371*7c478bd9Sstevel@tonic-gate 				    0, 0);
1372*7c478bd9Sstevel@tonic-gate 				break;
1373*7c478bd9Sstevel@tonic-gate 			}
1374*7c478bd9Sstevel@tonic-gate 			enqueue(qp, self, mp, MX);
1375*7c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = 1;
1376*7c478bd9Sstevel@tonic-gate 		}
1377*7c478bd9Sstevel@tonic-gate 		ASSERT(self->ul_sleepq == qp &&
1378*7c478bd9Sstevel@tonic-gate 		    self->ul_qtype == MX &&
1379*7c478bd9Sstevel@tonic-gate 		    self->ul_wchan == mp);
1380*7c478bd9Sstevel@tonic-gate 		if (error) {
1381*7c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = dequeue_self(qp, mp);
1382*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
1383*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
1384*7c478bd9Sstevel@tonic-gate 			break;
1385*7c478bd9Sstevel@tonic-gate 		}
1386*7c478bd9Sstevel@tonic-gate 	}
1387*7c478bd9Sstevel@tonic-gate 
1388*7c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
1389*7c478bd9Sstevel@tonic-gate 	    self->ul_wchan == NULL);
1390*7c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
1391*7c478bd9Sstevel@tonic-gate 
1392*7c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
1393*7c478bd9Sstevel@tonic-gate 	if (msp)
1394*7c478bd9Sstevel@tonic-gate 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
1395*7c478bd9Sstevel@tonic-gate 
1396*7c478bd9Sstevel@tonic-gate 	ASSERT(error == 0 || error == EINVAL || error == ETIME);
1397*7c478bd9Sstevel@tonic-gate 	return (error);
1398*7c478bd9Sstevel@tonic-gate }
1399*7c478bd9Sstevel@tonic-gate 
1400*7c478bd9Sstevel@tonic-gate /*
1401*7c478bd9Sstevel@tonic-gate  * Returns with mutex_owner set correctly.
1402*7c478bd9Sstevel@tonic-gate  */
1403*7c478bd9Sstevel@tonic-gate int
1404*7c478bd9Sstevel@tonic-gate mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try)
1405*7c478bd9Sstevel@tonic-gate {
1406*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1407*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
1408*7c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
1409*7c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
1410*7c478bd9Sstevel@tonic-gate 	int error = 0;
1411*7c478bd9Sstevel@tonic-gate 
1412*7c478bd9Sstevel@tonic-gate 	ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
1413*7c478bd9Sstevel@tonic-gate 
1414*7c478bd9Sstevel@tonic-gate 	if (!self->ul_schedctl_called)
1415*7c478bd9Sstevel@tonic-gate 		(void) setup_schedctl();
1416*7c478bd9Sstevel@tonic-gate 
1417*7c478bd9Sstevel@tonic-gate 	if (msp && try == MUTEX_TRY)
1418*7c478bd9Sstevel@tonic-gate 		tdb_incr(msp->mutex_try);
1419*7c478bd9Sstevel@tonic-gate 
1420*7c478bd9Sstevel@tonic-gate 	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_is_held(mp)) {
1421*7c478bd9Sstevel@tonic-gate 		if (mtype & LOCK_RECURSIVE) {
1422*7c478bd9Sstevel@tonic-gate 			if (mp->mutex_rcount == RECURSION_MAX) {
1423*7c478bd9Sstevel@tonic-gate 				error = EAGAIN;
1424*7c478bd9Sstevel@tonic-gate 			} else {
1425*7c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
1426*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
1427*7c478bd9Sstevel@tonic-gate 				    1, 0);
1428*7c478bd9Sstevel@tonic-gate 				return (0);
1429*7c478bd9Sstevel@tonic-gate 			}
1430*7c478bd9Sstevel@tonic-gate 		} else if (try == MUTEX_TRY) {
1431*7c478bd9Sstevel@tonic-gate 			return (EBUSY);
1432*7c478bd9Sstevel@tonic-gate 		} else {
1433*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
1434*7c478bd9Sstevel@tonic-gate 			return (EDEADLK);
1435*7c478bd9Sstevel@tonic-gate 		}
1436*7c478bd9Sstevel@tonic-gate 	}
1437*7c478bd9Sstevel@tonic-gate 
1438*7c478bd9Sstevel@tonic-gate 	if (self->ul_error_detection && try == MUTEX_LOCK &&
1439*7c478bd9Sstevel@tonic-gate 	    tsp == NULL && mutex_is_held(mp))
1440*7c478bd9Sstevel@tonic-gate 		lock_error(mp, "mutex_lock", NULL, NULL);
1441*7c478bd9Sstevel@tonic-gate 
1442*7c478bd9Sstevel@tonic-gate 	if (mtype &
1443*7c478bd9Sstevel@tonic-gate 	    (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) {
1444*7c478bd9Sstevel@tonic-gate 		uint8_t ceil;
1445*7c478bd9Sstevel@tonic-gate 		int myprio;
1446*7c478bd9Sstevel@tonic-gate 
1447*7c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_PROTECT) {
1448*7c478bd9Sstevel@tonic-gate 			ceil = mp->mutex_ceiling;
1449*7c478bd9Sstevel@tonic-gate 			ASSERT(_validate_rt_prio(SCHED_FIFO, ceil) == 0);
1450*7c478bd9Sstevel@tonic-gate 			myprio = real_priority(self);
1451*7c478bd9Sstevel@tonic-gate 			if (myprio > ceil) {
1452*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
1453*7c478bd9Sstevel@tonic-gate 				    EINVAL);
1454*7c478bd9Sstevel@tonic-gate 				return (EINVAL);
1455*7c478bd9Sstevel@tonic-gate 			}
1456*7c478bd9Sstevel@tonic-gate 			if ((error = _ceil_mylist_add(mp)) != 0) {
1457*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__error, mp,
1458*7c478bd9Sstevel@tonic-gate 				    error);
1459*7c478bd9Sstevel@tonic-gate 				return (error);
1460*7c478bd9Sstevel@tonic-gate 			}
1461*7c478bd9Sstevel@tonic-gate 			if (myprio < ceil)
1462*7c478bd9Sstevel@tonic-gate 				_ceil_prio_inherit(ceil);
1463*7c478bd9Sstevel@tonic-gate 		}
1464*7c478bd9Sstevel@tonic-gate 
1465*7c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_INHERIT) {
1466*7c478bd9Sstevel@tonic-gate 			/* go straight to the kernel */
1467*7c478bd9Sstevel@tonic-gate 			if (try == MUTEX_TRY)
1468*7c478bd9Sstevel@tonic-gate 				error = mutex_trylock_kernel(mp);
1469*7c478bd9Sstevel@tonic-gate 			else	/* MUTEX_LOCK */
1470*7c478bd9Sstevel@tonic-gate 				error = mutex_lock_kernel(mp, tsp, msp);
1471*7c478bd9Sstevel@tonic-gate 			/*
1472*7c478bd9Sstevel@tonic-gate 			 * The kernel never sets or clears the lock byte
1473*7c478bd9Sstevel@tonic-gate 			 * for PTHREAD_PRIO_INHERIT mutexes.
1474*7c478bd9Sstevel@tonic-gate 			 * Set it here for debugging consistency.
1475*7c478bd9Sstevel@tonic-gate 			 */
1476*7c478bd9Sstevel@tonic-gate 			switch (error) {
1477*7c478bd9Sstevel@tonic-gate 			case 0:
1478*7c478bd9Sstevel@tonic-gate 			case EOWNERDEAD:
1479*7c478bd9Sstevel@tonic-gate 				mp->mutex_lockw = LOCKSET;
1480*7c478bd9Sstevel@tonic-gate 				break;
1481*7c478bd9Sstevel@tonic-gate 			}
1482*7c478bd9Sstevel@tonic-gate 		} else if (mtype & USYNC_PROCESS_ROBUST) {
1483*7c478bd9Sstevel@tonic-gate 			/* go straight to the kernel */
1484*7c478bd9Sstevel@tonic-gate 			if (try == MUTEX_TRY)
1485*7c478bd9Sstevel@tonic-gate 				error = mutex_trylock_kernel(mp);
1486*7c478bd9Sstevel@tonic-gate 			else	/* MUTEX_LOCK */
1487*7c478bd9Sstevel@tonic-gate 				error = mutex_lock_kernel(mp, tsp, msp);
1488*7c478bd9Sstevel@tonic-gate 		} else {	/* PTHREAD_PRIO_PROTECT */
1489*7c478bd9Sstevel@tonic-gate 			/*
1490*7c478bd9Sstevel@tonic-gate 			 * Try once at user level before going to the kernel.
1491*7c478bd9Sstevel@tonic-gate 			 * If this is a process shared mutex then protect
1492*7c478bd9Sstevel@tonic-gate 			 * against forkall() while setting mp->mutex_ownerpid.
1493*7c478bd9Sstevel@tonic-gate 			 */
1494*7c478bd9Sstevel@tonic-gate 			if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)) {
1495*7c478bd9Sstevel@tonic-gate 				enter_critical(self);
1496*7c478bd9Sstevel@tonic-gate 				if (set_lock_byte(&mp->mutex_lockw) == 0) {
1497*7c478bd9Sstevel@tonic-gate 					mp->mutex_owner = (uintptr_t)self;
1498*7c478bd9Sstevel@tonic-gate 					mp->mutex_ownerpid = udp->pid;
1499*7c478bd9Sstevel@tonic-gate 					exit_critical(self);
1500*7c478bd9Sstevel@tonic-gate 					DTRACE_PROBE3(plockstat,
1501*7c478bd9Sstevel@tonic-gate 					    mutex__acquire, mp, 0, 0);
1502*7c478bd9Sstevel@tonic-gate 				} else {
1503*7c478bd9Sstevel@tonic-gate 					exit_critical(self);
1504*7c478bd9Sstevel@tonic-gate 					error = EBUSY;
1505*7c478bd9Sstevel@tonic-gate 				}
1506*7c478bd9Sstevel@tonic-gate 			} else {
1507*7c478bd9Sstevel@tonic-gate 				if (set_lock_byte(&mp->mutex_lockw) == 0) {
1508*7c478bd9Sstevel@tonic-gate 					mp->mutex_owner = (uintptr_t)self;
1509*7c478bd9Sstevel@tonic-gate 					DTRACE_PROBE3(plockstat,
1510*7c478bd9Sstevel@tonic-gate 					    mutex__acquire, mp, 0, 0);
1511*7c478bd9Sstevel@tonic-gate 				} else {
1512*7c478bd9Sstevel@tonic-gate 					error = EBUSY;
1513*7c478bd9Sstevel@tonic-gate 				}
1514*7c478bd9Sstevel@tonic-gate 			}
1515*7c478bd9Sstevel@tonic-gate 			if (error && try == MUTEX_LOCK)
1516*7c478bd9Sstevel@tonic-gate 				error = mutex_lock_kernel(mp, tsp, msp);
1517*7c478bd9Sstevel@tonic-gate 		}
1518*7c478bd9Sstevel@tonic-gate 
1519*7c478bd9Sstevel@tonic-gate 		if (error) {
1520*7c478bd9Sstevel@tonic-gate 			if (mtype & PTHREAD_PRIO_INHERIT) {
1521*7c478bd9Sstevel@tonic-gate 				switch (error) {
1522*7c478bd9Sstevel@tonic-gate 				case EOWNERDEAD:
1523*7c478bd9Sstevel@tonic-gate 				case ENOTRECOVERABLE:
1524*7c478bd9Sstevel@tonic-gate 					if (mtype & PTHREAD_MUTEX_ROBUST_NP)
1525*7c478bd9Sstevel@tonic-gate 						break;
1526*7c478bd9Sstevel@tonic-gate 					if (error == EOWNERDEAD) {
1527*7c478bd9Sstevel@tonic-gate 						/*
1528*7c478bd9Sstevel@tonic-gate 						 * We own the mutex; unlock it.
1529*7c478bd9Sstevel@tonic-gate 						 * It becomes ENOTRECOVERABLE.
1530*7c478bd9Sstevel@tonic-gate 						 * All waiters are waked up.
1531*7c478bd9Sstevel@tonic-gate 						 */
1532*7c478bd9Sstevel@tonic-gate 						mp->mutex_owner = 0;
1533*7c478bd9Sstevel@tonic-gate 						mp->mutex_ownerpid = 0;
1534*7c478bd9Sstevel@tonic-gate 						DTRACE_PROBE2(plockstat,
1535*7c478bd9Sstevel@tonic-gate 						    mutex__release, mp, 0);
1536*7c478bd9Sstevel@tonic-gate 						mp->mutex_lockw = LOCKCLEAR;
1537*7c478bd9Sstevel@tonic-gate 						(void) ___lwp_mutex_unlock(mp);
1538*7c478bd9Sstevel@tonic-gate 					}
1539*7c478bd9Sstevel@tonic-gate 					/* FALLTHROUGH */
1540*7c478bd9Sstevel@tonic-gate 				case EDEADLK:
1541*7c478bd9Sstevel@tonic-gate 					if (try == MUTEX_LOCK)
1542*7c478bd9Sstevel@tonic-gate 						stall();
1543*7c478bd9Sstevel@tonic-gate 					error = EBUSY;
1544*7c478bd9Sstevel@tonic-gate 					break;
1545*7c478bd9Sstevel@tonic-gate 				}
1546*7c478bd9Sstevel@tonic-gate 			}
1547*7c478bd9Sstevel@tonic-gate 			if ((mtype & PTHREAD_PRIO_PROTECT) &&
1548*7c478bd9Sstevel@tonic-gate 			    error != EOWNERDEAD) {
1549*7c478bd9Sstevel@tonic-gate 				(void) _ceil_mylist_del(mp);
1550*7c478bd9Sstevel@tonic-gate 				if (myprio < ceil)
1551*7c478bd9Sstevel@tonic-gate 					_ceil_prio_waive();
1552*7c478bd9Sstevel@tonic-gate 			}
1553*7c478bd9Sstevel@tonic-gate 		}
1554*7c478bd9Sstevel@tonic-gate 	} else if (mtype & USYNC_PROCESS) {
1555*7c478bd9Sstevel@tonic-gate 		/*
1556*7c478bd9Sstevel@tonic-gate 		 * This is a process shared mutex.  Protect against
1557*7c478bd9Sstevel@tonic-gate 		 * forkall() while setting mp->mutex_ownerpid.
1558*7c478bd9Sstevel@tonic-gate 		 */
1559*7c478bd9Sstevel@tonic-gate 		enter_critical(self);
1560*7c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
1561*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
1562*7c478bd9Sstevel@tonic-gate 			mp->mutex_ownerpid = udp->pid;
1563*7c478bd9Sstevel@tonic-gate 			exit_critical(self);
1564*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1565*7c478bd9Sstevel@tonic-gate 		} else {
1566*7c478bd9Sstevel@tonic-gate 			/* try a little harder */
1567*7c478bd9Sstevel@tonic-gate 			exit_critical(self);
1568*7c478bd9Sstevel@tonic-gate 			error = mutex_trylock_process(mp);
1569*7c478bd9Sstevel@tonic-gate 		}
1570*7c478bd9Sstevel@tonic-gate 		if (error && try == MUTEX_LOCK)
1571*7c478bd9Sstevel@tonic-gate 			error = mutex_lock_kernel(mp, tsp, msp);
1572*7c478bd9Sstevel@tonic-gate 	} else  {	/* USYNC_THREAD */
1573*7c478bd9Sstevel@tonic-gate 		/* try once */
1574*7c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
1575*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
1576*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1577*7c478bd9Sstevel@tonic-gate 		} else {
1578*7c478bd9Sstevel@tonic-gate 			/* try a little harder if we don't own the mutex */
1579*7c478bd9Sstevel@tonic-gate 			error = EBUSY;
1580*7c478bd9Sstevel@tonic-gate 			if (MUTEX_OWNER(mp) != self)
1581*7c478bd9Sstevel@tonic-gate 				error = mutex_trylock_adaptive(mp);
1582*7c478bd9Sstevel@tonic-gate 			if (error && try == MUTEX_LOCK)		/* go park */
1583*7c478bd9Sstevel@tonic-gate 				error = mutex_lock_queue(self, msp, mp, tsp);
1584*7c478bd9Sstevel@tonic-gate 		}
1585*7c478bd9Sstevel@tonic-gate 	}
1586*7c478bd9Sstevel@tonic-gate 
1587*7c478bd9Sstevel@tonic-gate 	switch (error) {
1588*7c478bd9Sstevel@tonic-gate 	case EOWNERDEAD:
1589*7c478bd9Sstevel@tonic-gate 	case ELOCKUNMAPPED:
1590*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
1591*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1592*7c478bd9Sstevel@tonic-gate 		/* FALLTHROUGH */
1593*7c478bd9Sstevel@tonic-gate 	case 0:
1594*7c478bd9Sstevel@tonic-gate 		if (msp)
1595*7c478bd9Sstevel@tonic-gate 			record_begin_hold(msp);
1596*7c478bd9Sstevel@tonic-gate 		break;
1597*7c478bd9Sstevel@tonic-gate 	default:
1598*7c478bd9Sstevel@tonic-gate 		if (try == MUTEX_TRY) {
1599*7c478bd9Sstevel@tonic-gate 			if (msp)
1600*7c478bd9Sstevel@tonic-gate 				tdb_incr(msp->mutex_try_fail);
1601*7c478bd9Sstevel@tonic-gate 			if (__td_event_report(self, TD_LOCK_TRY, udp)) {
1602*7c478bd9Sstevel@tonic-gate 				self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
1603*7c478bd9Sstevel@tonic-gate 				tdb_event(TD_LOCK_TRY, udp);
1604*7c478bd9Sstevel@tonic-gate 			}
1605*7c478bd9Sstevel@tonic-gate 		}
1606*7c478bd9Sstevel@tonic-gate 		break;
1607*7c478bd9Sstevel@tonic-gate 	}
1608*7c478bd9Sstevel@tonic-gate 
1609*7c478bd9Sstevel@tonic-gate 	return (error);
1610*7c478bd9Sstevel@tonic-gate }
1611*7c478bd9Sstevel@tonic-gate 
1612*7c478bd9Sstevel@tonic-gate int
1613*7c478bd9Sstevel@tonic-gate fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try)
1614*7c478bd9Sstevel@tonic-gate {
1615*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1616*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
1617*7c478bd9Sstevel@tonic-gate 
1618*7c478bd9Sstevel@tonic-gate 	/*
1619*7c478bd9Sstevel@tonic-gate 	 * We know that USYNC_PROCESS is set in mtype and that
1620*7c478bd9Sstevel@tonic-gate 	 * zero, one, or both of the flags LOCK_RECURSIVE and
1621*7c478bd9Sstevel@tonic-gate 	 * LOCK_ERRORCHECK are set, and that no other flags are set.
1622*7c478bd9Sstevel@tonic-gate 	 */
1623*7c478bd9Sstevel@tonic-gate 	enter_critical(self);
1624*7c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
1625*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
1626*7c478bd9Sstevel@tonic-gate 		mp->mutex_ownerpid = udp->pid;
1627*7c478bd9Sstevel@tonic-gate 		exit_critical(self);
1628*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1629*7c478bd9Sstevel@tonic-gate 		return (0);
1630*7c478bd9Sstevel@tonic-gate 	}
1631*7c478bd9Sstevel@tonic-gate 	exit_critical(self);
1632*7c478bd9Sstevel@tonic-gate 
1633*7c478bd9Sstevel@tonic-gate 	if ((mtype & ~USYNC_PROCESS) && shared_mutex_held(mp)) {
1634*7c478bd9Sstevel@tonic-gate 		if (mtype & LOCK_RECURSIVE) {
1635*7c478bd9Sstevel@tonic-gate 			if (mp->mutex_rcount == RECURSION_MAX)
1636*7c478bd9Sstevel@tonic-gate 				return (EAGAIN);
1637*7c478bd9Sstevel@tonic-gate 			mp->mutex_rcount++;
1638*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0);
1639*7c478bd9Sstevel@tonic-gate 			return (0);
1640*7c478bd9Sstevel@tonic-gate 		}
1641*7c478bd9Sstevel@tonic-gate 		if (try == MUTEX_LOCK) {
1642*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
1643*7c478bd9Sstevel@tonic-gate 			return (EDEADLK);
1644*7c478bd9Sstevel@tonic-gate 		}
1645*7c478bd9Sstevel@tonic-gate 		return (EBUSY);
1646*7c478bd9Sstevel@tonic-gate 	}
1647*7c478bd9Sstevel@tonic-gate 
1648*7c478bd9Sstevel@tonic-gate 	/* try a little harder if we don't own the mutex */
1649*7c478bd9Sstevel@tonic-gate 	if (!shared_mutex_held(mp) && mutex_trylock_process(mp) == 0)
1650*7c478bd9Sstevel@tonic-gate 		return (0);
1651*7c478bd9Sstevel@tonic-gate 
1652*7c478bd9Sstevel@tonic-gate 	if (try == MUTEX_LOCK)
1653*7c478bd9Sstevel@tonic-gate 		return (mutex_lock_kernel(mp, tsp, NULL));
1654*7c478bd9Sstevel@tonic-gate 
1655*7c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_LOCK_TRY, udp)) {
1656*7c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
1657*7c478bd9Sstevel@tonic-gate 		tdb_event(TD_LOCK_TRY, udp);
1658*7c478bd9Sstevel@tonic-gate 	}
1659*7c478bd9Sstevel@tonic-gate 	return (EBUSY);
1660*7c478bd9Sstevel@tonic-gate }
1661*7c478bd9Sstevel@tonic-gate 
1662*7c478bd9Sstevel@tonic-gate static int
1663*7c478bd9Sstevel@tonic-gate slow_lock(ulwp_t *self, mutex_t *mp, timespec_t *tsp)
1664*7c478bd9Sstevel@tonic-gate {
1665*7c478bd9Sstevel@tonic-gate 	int error = 0;
1666*7c478bd9Sstevel@tonic-gate 
1667*7c478bd9Sstevel@tonic-gate 	if (MUTEX_OWNER(mp) == self || mutex_trylock_adaptive(mp) != 0)
1668*7c478bd9Sstevel@tonic-gate 		error = mutex_lock_queue(self, NULL, mp, tsp);
1669*7c478bd9Sstevel@tonic-gate 	return (error);
1670*7c478bd9Sstevel@tonic-gate }
1671*7c478bd9Sstevel@tonic-gate 
1672*7c478bd9Sstevel@tonic-gate int
1673*7c478bd9Sstevel@tonic-gate mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
1674*7c478bd9Sstevel@tonic-gate {
1675*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1676*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
1677*7c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
1678*7c478bd9Sstevel@tonic-gate 	int mtype;
1679*7c478bd9Sstevel@tonic-gate 
1680*7c478bd9Sstevel@tonic-gate 	/*
1681*7c478bd9Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
1682*7c478bd9Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
1683*7c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics,
1684*7c478bd9Sstevel@tonic-gate 	 * and the process has only a single thread.
1685*7c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
1686*7c478bd9Sstevel@tonic-gate 	 */
1687*7c478bd9Sstevel@tonic-gate 	if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
1688*7c478bd9Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
1689*7c478bd9Sstevel@tonic-gate 		/*
1690*7c478bd9Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
1691*7c478bd9Sstevel@tonic-gate 		 */
1692*7c478bd9Sstevel@tonic-gate 		if (mp->mutex_lockw == 0) {
1693*7c478bd9Sstevel@tonic-gate 			mp->mutex_lockw = LOCKSET;
1694*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
1695*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1696*7c478bd9Sstevel@tonic-gate 			return (0);
1697*7c478bd9Sstevel@tonic-gate 		}
1698*7c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
1699*7c478bd9Sstevel@tonic-gate 			/*
1700*7c478bd9Sstevel@tonic-gate 			 * LOCK_RECURSIVE, LOCK_ERRORCHECK, or both.
1701*7c478bd9Sstevel@tonic-gate 			 */
1702*7c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
1703*7c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
1704*7c478bd9Sstevel@tonic-gate 					return (EAGAIN);
1705*7c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
1706*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
1707*7c478bd9Sstevel@tonic-gate 				    1, 0);
1708*7c478bd9Sstevel@tonic-gate 				return (0);
1709*7c478bd9Sstevel@tonic-gate 			}
1710*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
1711*7c478bd9Sstevel@tonic-gate 			return (EDEADLK);	/* LOCK_ERRORCHECK */
1712*7c478bd9Sstevel@tonic-gate 		}
1713*7c478bd9Sstevel@tonic-gate 		/*
1714*7c478bd9Sstevel@tonic-gate 		 * We have reached a deadlock, probably because the
1715*7c478bd9Sstevel@tonic-gate 		 * process is executing non-async-signal-safe code in
1716*7c478bd9Sstevel@tonic-gate 		 * a signal handler and is attempting to acquire a lock
1717*7c478bd9Sstevel@tonic-gate 		 * that it already owns.  This is not surprising, given
1718*7c478bd9Sstevel@tonic-gate 		 * bad programming practices over the years that has
1719*7c478bd9Sstevel@tonic-gate 		 * resulted in applications calling printf() and such
1720*7c478bd9Sstevel@tonic-gate 		 * in their signal handlers.  Unless the user has told
1721*7c478bd9Sstevel@tonic-gate 		 * us that the signal handlers are safe by setting:
1722*7c478bd9Sstevel@tonic-gate 		 *	export _THREAD_ASYNC_SAFE=1
1723*7c478bd9Sstevel@tonic-gate 		 * we return EDEADLK rather than actually deadlocking.
1724*7c478bd9Sstevel@tonic-gate 		 */
1725*7c478bd9Sstevel@tonic-gate 		if (tsp == NULL &&
1726*7c478bd9Sstevel@tonic-gate 		    MUTEX_OWNER(mp) == self && !self->ul_async_safe) {
1727*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
1728*7c478bd9Sstevel@tonic-gate 			return (EDEADLK);
1729*7c478bd9Sstevel@tonic-gate 		}
1730*7c478bd9Sstevel@tonic-gate 	}
1731*7c478bd9Sstevel@tonic-gate 
1732*7c478bd9Sstevel@tonic-gate 	/*
1733*7c478bd9Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
1734*7c478bd9Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
1735*7c478bd9Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
1736*7c478bd9Sstevel@tonic-gate 	 */
1737*7c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
1738*7c478bd9Sstevel@tonic-gate 	    (gflags->uf_trs_ted |
1739*7c478bd9Sstevel@tonic-gate 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
1740*7c478bd9Sstevel@tonic-gate 
1741*7c478bd9Sstevel@tonic-gate 		if (mtype & USYNC_PROCESS)
1742*7c478bd9Sstevel@tonic-gate 			return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK));
1743*7c478bd9Sstevel@tonic-gate 
1744*7c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
1745*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
1746*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1747*7c478bd9Sstevel@tonic-gate 			return (0);
1748*7c478bd9Sstevel@tonic-gate 		}
1749*7c478bd9Sstevel@tonic-gate 
1750*7c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
1751*7c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
1752*7c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
1753*7c478bd9Sstevel@tonic-gate 					return (EAGAIN);
1754*7c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
1755*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
1756*7c478bd9Sstevel@tonic-gate 				    1, 0);
1757*7c478bd9Sstevel@tonic-gate 				return (0);
1758*7c478bd9Sstevel@tonic-gate 			}
1759*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
1760*7c478bd9Sstevel@tonic-gate 			return (EDEADLK);	/* LOCK_ERRORCHECK */
1761*7c478bd9Sstevel@tonic-gate 		}
1762*7c478bd9Sstevel@tonic-gate 
1763*7c478bd9Sstevel@tonic-gate 		return (slow_lock(self, mp, tsp));
1764*7c478bd9Sstevel@tonic-gate 	}
1765*7c478bd9Sstevel@tonic-gate 
1766*7c478bd9Sstevel@tonic-gate 	/* else do it the long way */
1767*7c478bd9Sstevel@tonic-gate 	return (mutex_lock_internal(mp, tsp, MUTEX_LOCK));
1768*7c478bd9Sstevel@tonic-gate }
1769*7c478bd9Sstevel@tonic-gate 
1770*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_lock = __mutex_lock
1771*7c478bd9Sstevel@tonic-gate #pragma weak mutex_lock = __mutex_lock
1772*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_lock = __mutex_lock
1773*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_lock = __mutex_lock
1774*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_lock = __mutex_lock
1775*7c478bd9Sstevel@tonic-gate int
1776*7c478bd9Sstevel@tonic-gate __mutex_lock(mutex_t *mp)
1777*7c478bd9Sstevel@tonic-gate {
1778*7c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
1779*7c478bd9Sstevel@tonic-gate 	return (mutex_lock_impl(mp, NULL));
1780*7c478bd9Sstevel@tonic-gate }
1781*7c478bd9Sstevel@tonic-gate 
1782*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_timedlock = _pthread_mutex_timedlock
1783*7c478bd9Sstevel@tonic-gate int
1784*7c478bd9Sstevel@tonic-gate _pthread_mutex_timedlock(mutex_t *mp, const timespec_t *abstime)
1785*7c478bd9Sstevel@tonic-gate {
1786*7c478bd9Sstevel@tonic-gate 	timespec_t tslocal;
1787*7c478bd9Sstevel@tonic-gate 	int error;
1788*7c478bd9Sstevel@tonic-gate 
1789*7c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
1790*7c478bd9Sstevel@tonic-gate 	abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
1791*7c478bd9Sstevel@tonic-gate 	error = mutex_lock_impl(mp, &tslocal);
1792*7c478bd9Sstevel@tonic-gate 	if (error == ETIME)
1793*7c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
1794*7c478bd9Sstevel@tonic-gate 	return (error);
1795*7c478bd9Sstevel@tonic-gate }
1796*7c478bd9Sstevel@tonic-gate 
1797*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_reltimedlock_np = _pthread_mutex_reltimedlock_np
1798*7c478bd9Sstevel@tonic-gate int
1799*7c478bd9Sstevel@tonic-gate _pthread_mutex_reltimedlock_np(mutex_t *mp, const timespec_t *reltime)
1800*7c478bd9Sstevel@tonic-gate {
1801*7c478bd9Sstevel@tonic-gate 	timespec_t tslocal;
1802*7c478bd9Sstevel@tonic-gate 	int error;
1803*7c478bd9Sstevel@tonic-gate 
1804*7c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
1805*7c478bd9Sstevel@tonic-gate 	tslocal = *reltime;
1806*7c478bd9Sstevel@tonic-gate 	error = mutex_lock_impl(mp, &tslocal);
1807*7c478bd9Sstevel@tonic-gate 	if (error == ETIME)
1808*7c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
1809*7c478bd9Sstevel@tonic-gate 	return (error);
1810*7c478bd9Sstevel@tonic-gate }
1811*7c478bd9Sstevel@tonic-gate 
1812*7c478bd9Sstevel@tonic-gate static int
1813*7c478bd9Sstevel@tonic-gate slow_trylock(mutex_t *mp, ulwp_t *self)
1814*7c478bd9Sstevel@tonic-gate {
1815*7c478bd9Sstevel@tonic-gate 	if (MUTEX_OWNER(mp) == self ||
1816*7c478bd9Sstevel@tonic-gate 	    mutex_trylock_adaptive(mp) != 0) {
1817*7c478bd9Sstevel@tonic-gate 		uberdata_t *udp = self->ul_uberdata;
1818*7c478bd9Sstevel@tonic-gate 
1819*7c478bd9Sstevel@tonic-gate 		if (__td_event_report(self, TD_LOCK_TRY, udp)) {
1820*7c478bd9Sstevel@tonic-gate 			self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
1821*7c478bd9Sstevel@tonic-gate 			tdb_event(TD_LOCK_TRY, udp);
1822*7c478bd9Sstevel@tonic-gate 		}
1823*7c478bd9Sstevel@tonic-gate 		return (EBUSY);
1824*7c478bd9Sstevel@tonic-gate 	}
1825*7c478bd9Sstevel@tonic-gate 	return (0);
1826*7c478bd9Sstevel@tonic-gate }
1827*7c478bd9Sstevel@tonic-gate 
1828*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_trylock = __mutex_trylock
1829*7c478bd9Sstevel@tonic-gate #pragma weak mutex_trylock = __mutex_trylock
1830*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_trylock = __mutex_trylock
1831*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_trylock = __mutex_trylock
1832*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_trylock = __mutex_trylock
1833*7c478bd9Sstevel@tonic-gate int
1834*7c478bd9Sstevel@tonic-gate __mutex_trylock(mutex_t *mp)
1835*7c478bd9Sstevel@tonic-gate {
1836*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1837*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
1838*7c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
1839*7c478bd9Sstevel@tonic-gate 	int mtype;
1840*7c478bd9Sstevel@tonic-gate 
1841*7c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
1842*7c478bd9Sstevel@tonic-gate 	/*
1843*7c478bd9Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
1844*7c478bd9Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
1845*7c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics,
1846*7c478bd9Sstevel@tonic-gate 	 * and the process has only a single thread.
1847*7c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
1848*7c478bd9Sstevel@tonic-gate 	 */
1849*7c478bd9Sstevel@tonic-gate 	if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
1850*7c478bd9Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
1851*7c478bd9Sstevel@tonic-gate 		/*
1852*7c478bd9Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
1853*7c478bd9Sstevel@tonic-gate 		 */
1854*7c478bd9Sstevel@tonic-gate 		if (mp->mutex_lockw == 0) {
1855*7c478bd9Sstevel@tonic-gate 			mp->mutex_lockw = LOCKSET;
1856*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
1857*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1858*7c478bd9Sstevel@tonic-gate 			return (0);
1859*7c478bd9Sstevel@tonic-gate 		}
1860*7c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
1861*7c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
1862*7c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
1863*7c478bd9Sstevel@tonic-gate 					return (EAGAIN);
1864*7c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
1865*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
1866*7c478bd9Sstevel@tonic-gate 				    1, 0);
1867*7c478bd9Sstevel@tonic-gate 				return (0);
1868*7c478bd9Sstevel@tonic-gate 			}
1869*7c478bd9Sstevel@tonic-gate 			return (EDEADLK);	/* LOCK_ERRORCHECK */
1870*7c478bd9Sstevel@tonic-gate 		}
1871*7c478bd9Sstevel@tonic-gate 		return (EBUSY);
1872*7c478bd9Sstevel@tonic-gate 	}
1873*7c478bd9Sstevel@tonic-gate 
1874*7c478bd9Sstevel@tonic-gate 	/*
1875*7c478bd9Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
1876*7c478bd9Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
1877*7c478bd9Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
1878*7c478bd9Sstevel@tonic-gate 	 */
1879*7c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
1880*7c478bd9Sstevel@tonic-gate 	    (gflags->uf_trs_ted |
1881*7c478bd9Sstevel@tonic-gate 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
1882*7c478bd9Sstevel@tonic-gate 
1883*7c478bd9Sstevel@tonic-gate 		if (mtype & USYNC_PROCESS)
1884*7c478bd9Sstevel@tonic-gate 			return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY));
1885*7c478bd9Sstevel@tonic-gate 
1886*7c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
1887*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
1888*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1889*7c478bd9Sstevel@tonic-gate 			return (0);
1890*7c478bd9Sstevel@tonic-gate 		}
1891*7c478bd9Sstevel@tonic-gate 
1892*7c478bd9Sstevel@tonic-gate 		if (mtype && MUTEX_OWNER(mp) == self) {
1893*7c478bd9Sstevel@tonic-gate 			if (mtype & LOCK_RECURSIVE) {
1894*7c478bd9Sstevel@tonic-gate 				if (mp->mutex_rcount == RECURSION_MAX)
1895*7c478bd9Sstevel@tonic-gate 					return (EAGAIN);
1896*7c478bd9Sstevel@tonic-gate 				mp->mutex_rcount++;
1897*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE3(plockstat, mutex__acquire, mp,
1898*7c478bd9Sstevel@tonic-gate 				    1, 0);
1899*7c478bd9Sstevel@tonic-gate 				return (0);
1900*7c478bd9Sstevel@tonic-gate 			}
1901*7c478bd9Sstevel@tonic-gate 			return (EBUSY);		/* LOCK_ERRORCHECK */
1902*7c478bd9Sstevel@tonic-gate 		}
1903*7c478bd9Sstevel@tonic-gate 
1904*7c478bd9Sstevel@tonic-gate 		return (slow_trylock(mp, self));
1905*7c478bd9Sstevel@tonic-gate 	}
1906*7c478bd9Sstevel@tonic-gate 
1907*7c478bd9Sstevel@tonic-gate 	/* else do it the long way */
1908*7c478bd9Sstevel@tonic-gate 	return (mutex_lock_internal(mp, NULL, MUTEX_TRY));
1909*7c478bd9Sstevel@tonic-gate }
1910*7c478bd9Sstevel@tonic-gate 
1911*7c478bd9Sstevel@tonic-gate int
1912*7c478bd9Sstevel@tonic-gate mutex_unlock_internal(mutex_t *mp)
1913*7c478bd9Sstevel@tonic-gate {
1914*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1915*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
1916*7c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
1917*7c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp;
1918*7c478bd9Sstevel@tonic-gate 	int error;
1919*7c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
1920*7c478bd9Sstevel@tonic-gate 
1921*7c478bd9Sstevel@tonic-gate 	if ((mtype & LOCK_ERRORCHECK) && !mutex_is_held(mp))
1922*7c478bd9Sstevel@tonic-gate 		return (EPERM);
1923*7c478bd9Sstevel@tonic-gate 
1924*7c478bd9Sstevel@tonic-gate 	if (self->ul_error_detection && !mutex_is_held(mp))
1925*7c478bd9Sstevel@tonic-gate 		lock_error(mp, "mutex_unlock", NULL, NULL);
1926*7c478bd9Sstevel@tonic-gate 
1927*7c478bd9Sstevel@tonic-gate 	if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
1928*7c478bd9Sstevel@tonic-gate 		mp->mutex_rcount--;
1929*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
1930*7c478bd9Sstevel@tonic-gate 		return (0);
1931*7c478bd9Sstevel@tonic-gate 	}
1932*7c478bd9Sstevel@tonic-gate 
1933*7c478bd9Sstevel@tonic-gate 	if ((msp = MUTEX_STATS(mp, udp)) != NULL)
1934*7c478bd9Sstevel@tonic-gate 		(void) record_hold_time(msp);
1935*7c478bd9Sstevel@tonic-gate 
1936*7c478bd9Sstevel@tonic-gate 	if (mtype &
1937*7c478bd9Sstevel@tonic-gate 	    (USYNC_PROCESS_ROBUST|PTHREAD_PRIO_INHERIT|PTHREAD_PRIO_PROTECT)) {
1938*7c478bd9Sstevel@tonic-gate 		no_preempt(self);
1939*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
1940*7c478bd9Sstevel@tonic-gate 		mp->mutex_ownerpid = 0;
1941*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
1942*7c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_INHERIT) {
1943*7c478bd9Sstevel@tonic-gate 			mp->mutex_lockw = LOCKCLEAR;
1944*7c478bd9Sstevel@tonic-gate 			error = ___lwp_mutex_unlock(mp);
1945*7c478bd9Sstevel@tonic-gate 		} else if (mtype & USYNC_PROCESS_ROBUST) {
1946*7c478bd9Sstevel@tonic-gate 			error = ___lwp_mutex_unlock(mp);
1947*7c478bd9Sstevel@tonic-gate 		} else {
1948*7c478bd9Sstevel@tonic-gate 			if (swap32(&mp->mutex_lockword, 0) & WAITERMASK)
1949*7c478bd9Sstevel@tonic-gate 				(void) ___lwp_mutex_wakeup(mp);
1950*7c478bd9Sstevel@tonic-gate 			error = 0;
1951*7c478bd9Sstevel@tonic-gate 		}
1952*7c478bd9Sstevel@tonic-gate 		if (mtype & PTHREAD_PRIO_PROTECT) {
1953*7c478bd9Sstevel@tonic-gate 			if (_ceil_mylist_del(mp))
1954*7c478bd9Sstevel@tonic-gate 				_ceil_prio_waive();
1955*7c478bd9Sstevel@tonic-gate 		}
1956*7c478bd9Sstevel@tonic-gate 		preempt(self);
1957*7c478bd9Sstevel@tonic-gate 	} else if (mtype & USYNC_PROCESS) {
1958*7c478bd9Sstevel@tonic-gate 		if (mp->mutex_lockword & WAITERMASK)
1959*7c478bd9Sstevel@tonic-gate 			mutex_unlock_process(mp);
1960*7c478bd9Sstevel@tonic-gate 		else {
1961*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = 0;
1962*7c478bd9Sstevel@tonic-gate 			mp->mutex_ownerpid = 0;
1963*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
1964*7c478bd9Sstevel@tonic-gate 			if (swap32(&mp->mutex_lockword, 0) & WAITERMASK) {
1965*7c478bd9Sstevel@tonic-gate 				no_preempt(self);
1966*7c478bd9Sstevel@tonic-gate 				(void) ___lwp_mutex_wakeup(mp);
1967*7c478bd9Sstevel@tonic-gate 				preempt(self);
1968*7c478bd9Sstevel@tonic-gate 			}
1969*7c478bd9Sstevel@tonic-gate 		}
1970*7c478bd9Sstevel@tonic-gate 		error = 0;
1971*7c478bd9Sstevel@tonic-gate 	} else {	/* USYNC_THREAD */
1972*7c478bd9Sstevel@tonic-gate 		if ((lwpid = mutex_unlock_queue(mp)) != 0) {
1973*7c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid);
1974*7c478bd9Sstevel@tonic-gate 			preempt(self);
1975*7c478bd9Sstevel@tonic-gate 		}
1976*7c478bd9Sstevel@tonic-gate 		error = 0;
1977*7c478bd9Sstevel@tonic-gate 	}
1978*7c478bd9Sstevel@tonic-gate 
1979*7c478bd9Sstevel@tonic-gate 	return (error);
1980*7c478bd9Sstevel@tonic-gate }
1981*7c478bd9Sstevel@tonic-gate 
1982*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_unlock = __mutex_unlock
1983*7c478bd9Sstevel@tonic-gate #pragma weak mutex_unlock = __mutex_unlock
1984*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_unlock = __mutex_unlock
1985*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_unlock = __mutex_unlock
1986*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_unlock = __mutex_unlock
1987*7c478bd9Sstevel@tonic-gate int
1988*7c478bd9Sstevel@tonic-gate __mutex_unlock(mutex_t *mp)
1989*7c478bd9Sstevel@tonic-gate {
1990*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
1991*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
1992*7c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
1993*7c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
1994*7c478bd9Sstevel@tonic-gate 	int mtype;
1995*7c478bd9Sstevel@tonic-gate 	short el;
1996*7c478bd9Sstevel@tonic-gate 
1997*7c478bd9Sstevel@tonic-gate 	/*
1998*7c478bd9Sstevel@tonic-gate 	 * Optimize the case of USYNC_THREAD, including
1999*7c478bd9Sstevel@tonic-gate 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
2000*7c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics,
2001*7c478bd9Sstevel@tonic-gate 	 * and the process has only a single thread.
2002*7c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
2003*7c478bd9Sstevel@tonic-gate 	 */
2004*7c478bd9Sstevel@tonic-gate 	if ((((mtype = mp->mutex_type) & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
2005*7c478bd9Sstevel@tonic-gate 	    udp->uberflags.uf_all) == 0) {
2006*7c478bd9Sstevel@tonic-gate 		if (mtype) {
2007*7c478bd9Sstevel@tonic-gate 			/*
2008*7c478bd9Sstevel@tonic-gate 			 * At this point we know that one or both of the
2009*7c478bd9Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
2010*7c478bd9Sstevel@tonic-gate 			 */
2011*7c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
2012*7c478bd9Sstevel@tonic-gate 				return (EPERM);
2013*7c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
2014*7c478bd9Sstevel@tonic-gate 				mp->mutex_rcount--;
2015*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
2016*7c478bd9Sstevel@tonic-gate 				return (0);
2017*7c478bd9Sstevel@tonic-gate 			}
2018*7c478bd9Sstevel@tonic-gate 		}
2019*7c478bd9Sstevel@tonic-gate 		/*
2020*7c478bd9Sstevel@tonic-gate 		 * Only one thread exists so we don't need an atomic operation.
2021*7c478bd9Sstevel@tonic-gate 		 * Also, there can be no waiters.
2022*7c478bd9Sstevel@tonic-gate 		 */
2023*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
2024*7c478bd9Sstevel@tonic-gate 		mp->mutex_lockword = 0;
2025*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
2026*7c478bd9Sstevel@tonic-gate 		return (0);
2027*7c478bd9Sstevel@tonic-gate 	}
2028*7c478bd9Sstevel@tonic-gate 
2029*7c478bd9Sstevel@tonic-gate 	/*
2030*7c478bd9Sstevel@tonic-gate 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
2031*7c478bd9Sstevel@tonic-gate 	 * no error detection, and no lock statistics.
2032*7c478bd9Sstevel@tonic-gate 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
2033*7c478bd9Sstevel@tonic-gate 	 */
2034*7c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL) {
2035*7c478bd9Sstevel@tonic-gate 		if (((el = gflags->uf_trs_ted) | mtype) == 0) {
2036*7c478bd9Sstevel@tonic-gate fast_unlock:
2037*7c478bd9Sstevel@tonic-gate 			if (!(mp->mutex_lockword & WAITERMASK)) {
2038*7c478bd9Sstevel@tonic-gate 				/* no waiter exists right now */
2039*7c478bd9Sstevel@tonic-gate 				mp->mutex_owner = 0;
2040*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
2041*7c478bd9Sstevel@tonic-gate 				if (swap32(&mp->mutex_lockword, 0) &
2042*7c478bd9Sstevel@tonic-gate 				    WAITERMASK) {
2043*7c478bd9Sstevel@tonic-gate 					/* a waiter suddenly appeared */
2044*7c478bd9Sstevel@tonic-gate 					no_preempt(self);
2045*7c478bd9Sstevel@tonic-gate 					if ((lwpid = mutex_wakeup(mp)) != 0)
2046*7c478bd9Sstevel@tonic-gate 						(void) __lwp_unpark(lwpid);
2047*7c478bd9Sstevel@tonic-gate 					preempt(self);
2048*7c478bd9Sstevel@tonic-gate 				}
2049*7c478bd9Sstevel@tonic-gate 			} else if ((lwpid = mutex_unlock_queue(mp)) != 0) {
2050*7c478bd9Sstevel@tonic-gate 				(void) __lwp_unpark(lwpid);
2051*7c478bd9Sstevel@tonic-gate 				preempt(self);
2052*7c478bd9Sstevel@tonic-gate 			}
2053*7c478bd9Sstevel@tonic-gate 			return (0);
2054*7c478bd9Sstevel@tonic-gate 		}
2055*7c478bd9Sstevel@tonic-gate 		if (el)		/* error detection or lock statistics */
2056*7c478bd9Sstevel@tonic-gate 			goto slow_unlock;
2057*7c478bd9Sstevel@tonic-gate 		if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
2058*7c478bd9Sstevel@tonic-gate 			/*
2059*7c478bd9Sstevel@tonic-gate 			 * At this point we know that one or both of the
2060*7c478bd9Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
2061*7c478bd9Sstevel@tonic-gate 			 */
2062*7c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
2063*7c478bd9Sstevel@tonic-gate 				return (EPERM);
2064*7c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
2065*7c478bd9Sstevel@tonic-gate 				mp->mutex_rcount--;
2066*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
2067*7c478bd9Sstevel@tonic-gate 				return (0);
2068*7c478bd9Sstevel@tonic-gate 			}
2069*7c478bd9Sstevel@tonic-gate 			goto fast_unlock;
2070*7c478bd9Sstevel@tonic-gate 		}
2071*7c478bd9Sstevel@tonic-gate 		if ((mtype &
2072*7c478bd9Sstevel@tonic-gate 		    ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
2073*7c478bd9Sstevel@tonic-gate 			/*
2074*7c478bd9Sstevel@tonic-gate 			 * At this point we know that zero, one, or both of the
2075*7c478bd9Sstevel@tonic-gate 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and
2076*7c478bd9Sstevel@tonic-gate 			 * that the USYNC_PROCESS flag is set.
2077*7c478bd9Sstevel@tonic-gate 			 */
2078*7c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp))
2079*7c478bd9Sstevel@tonic-gate 				return (EPERM);
2080*7c478bd9Sstevel@tonic-gate 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
2081*7c478bd9Sstevel@tonic-gate 				mp->mutex_rcount--;
2082*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
2083*7c478bd9Sstevel@tonic-gate 				return (0);
2084*7c478bd9Sstevel@tonic-gate 			}
2085*7c478bd9Sstevel@tonic-gate 			if (mp->mutex_lockword & WAITERMASK)
2086*7c478bd9Sstevel@tonic-gate 				mutex_unlock_process(mp);
2087*7c478bd9Sstevel@tonic-gate 			else {
2088*7c478bd9Sstevel@tonic-gate 				mp->mutex_owner = 0;
2089*7c478bd9Sstevel@tonic-gate 				mp->mutex_ownerpid = 0;
2090*7c478bd9Sstevel@tonic-gate 				DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
2091*7c478bd9Sstevel@tonic-gate 				if (swap32(&mp->mutex_lockword, 0) &
2092*7c478bd9Sstevel@tonic-gate 				    WAITERMASK) {
2093*7c478bd9Sstevel@tonic-gate 					no_preempt(self);
2094*7c478bd9Sstevel@tonic-gate 					(void) ___lwp_mutex_wakeup(mp);
2095*7c478bd9Sstevel@tonic-gate 					preempt(self);
2096*7c478bd9Sstevel@tonic-gate 				}
2097*7c478bd9Sstevel@tonic-gate 			}
2098*7c478bd9Sstevel@tonic-gate 			return (0);
2099*7c478bd9Sstevel@tonic-gate 		}
2100*7c478bd9Sstevel@tonic-gate 	}
2101*7c478bd9Sstevel@tonic-gate 
2102*7c478bd9Sstevel@tonic-gate 	/* else do it the long way */
2103*7c478bd9Sstevel@tonic-gate slow_unlock:
2104*7c478bd9Sstevel@tonic-gate 	return (mutex_unlock_internal(mp));
2105*7c478bd9Sstevel@tonic-gate }
2106*7c478bd9Sstevel@tonic-gate 
2107*7c478bd9Sstevel@tonic-gate /*
2108*7c478bd9Sstevel@tonic-gate  * Internally to the library, almost all mutex lock/unlock actions
2109*7c478bd9Sstevel@tonic-gate  * go through these lmutex_ functions, to protect critical regions.
2110*7c478bd9Sstevel@tonic-gate  * We replicate a bit of code from __mutex_lock() and __mutex_unlock()
2111*7c478bd9Sstevel@tonic-gate  * to make these functions faster since we know that the mutex type
2112*7c478bd9Sstevel@tonic-gate  * of all internal locks is USYNC_THREAD.  We also know that internal
2113*7c478bd9Sstevel@tonic-gate  * locking can never fail, so we panic if it does.
2114*7c478bd9Sstevel@tonic-gate  */
2115*7c478bd9Sstevel@tonic-gate void
2116*7c478bd9Sstevel@tonic-gate lmutex_lock(mutex_t *mp)
2117*7c478bd9Sstevel@tonic-gate {
2118*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2119*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
2120*7c478bd9Sstevel@tonic-gate 
2121*7c478bd9Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
2122*7c478bd9Sstevel@tonic-gate 
2123*7c478bd9Sstevel@tonic-gate 	enter_critical(self);
2124*7c478bd9Sstevel@tonic-gate 	/*
2125*7c478bd9Sstevel@tonic-gate 	 * Optimize the case of no lock statistics and only a single thread.
2126*7c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
2127*7c478bd9Sstevel@tonic-gate 	 */
2128*7c478bd9Sstevel@tonic-gate 	if (udp->uberflags.uf_all == 0) {
2129*7c478bd9Sstevel@tonic-gate 		/*
2130*7c478bd9Sstevel@tonic-gate 		 * Only one thread exists; the mutex must be free.
2131*7c478bd9Sstevel@tonic-gate 		 */
2132*7c478bd9Sstevel@tonic-gate 		ASSERT(mp->mutex_lockw == 0);
2133*7c478bd9Sstevel@tonic-gate 		mp->mutex_lockw = LOCKSET;
2134*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
2135*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2136*7c478bd9Sstevel@tonic-gate 	} else {
2137*7c478bd9Sstevel@tonic-gate 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
2138*7c478bd9Sstevel@tonic-gate 
2139*7c478bd9Sstevel@tonic-gate 		if (!self->ul_schedctl_called)
2140*7c478bd9Sstevel@tonic-gate 			(void) setup_schedctl();
2141*7c478bd9Sstevel@tonic-gate 
2142*7c478bd9Sstevel@tonic-gate 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
2143*7c478bd9Sstevel@tonic-gate 			mp->mutex_owner = (uintptr_t)self;
2144*7c478bd9Sstevel@tonic-gate 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2145*7c478bd9Sstevel@tonic-gate 		} else if (mutex_trylock_adaptive(mp) != 0) {
2146*7c478bd9Sstevel@tonic-gate 			(void) mutex_lock_queue(self, msp, mp, NULL);
2147*7c478bd9Sstevel@tonic-gate 		}
2148*7c478bd9Sstevel@tonic-gate 
2149*7c478bd9Sstevel@tonic-gate 		if (msp)
2150*7c478bd9Sstevel@tonic-gate 			record_begin_hold(msp);
2151*7c478bd9Sstevel@tonic-gate 	}
2152*7c478bd9Sstevel@tonic-gate }
2153*7c478bd9Sstevel@tonic-gate 
2154*7c478bd9Sstevel@tonic-gate void
2155*7c478bd9Sstevel@tonic-gate lmutex_unlock(mutex_t *mp)
2156*7c478bd9Sstevel@tonic-gate {
2157*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2158*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
2159*7c478bd9Sstevel@tonic-gate 
2160*7c478bd9Sstevel@tonic-gate 	ASSERT(mp->mutex_type == USYNC_THREAD);
2161*7c478bd9Sstevel@tonic-gate 
2162*7c478bd9Sstevel@tonic-gate 	/*
2163*7c478bd9Sstevel@tonic-gate 	 * Optimize the case of no lock statistics and only a single thread.
2164*7c478bd9Sstevel@tonic-gate 	 * (Most likely a traditional single-threaded application.)
2165*7c478bd9Sstevel@tonic-gate 	 */
2166*7c478bd9Sstevel@tonic-gate 	if (udp->uberflags.uf_all == 0) {
2167*7c478bd9Sstevel@tonic-gate 		/*
2168*7c478bd9Sstevel@tonic-gate 		 * Only one thread exists so there can be no waiters.
2169*7c478bd9Sstevel@tonic-gate 		 */
2170*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = 0;
2171*7c478bd9Sstevel@tonic-gate 		mp->mutex_lockword = 0;
2172*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
2173*7c478bd9Sstevel@tonic-gate 	} else {
2174*7c478bd9Sstevel@tonic-gate 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
2175*7c478bd9Sstevel@tonic-gate 		lwpid_t lwpid;
2176*7c478bd9Sstevel@tonic-gate 
2177*7c478bd9Sstevel@tonic-gate 		if (msp)
2178*7c478bd9Sstevel@tonic-gate 			(void) record_hold_time(msp);
2179*7c478bd9Sstevel@tonic-gate 		if ((lwpid = mutex_unlock_queue(mp)) != 0) {
2180*7c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid);
2181*7c478bd9Sstevel@tonic-gate 			preempt(self);
2182*7c478bd9Sstevel@tonic-gate 		}
2183*7c478bd9Sstevel@tonic-gate 	}
2184*7c478bd9Sstevel@tonic-gate 	exit_critical(self);
2185*7c478bd9Sstevel@tonic-gate }
2186*7c478bd9Sstevel@tonic-gate 
2187*7c478bd9Sstevel@tonic-gate static int
2188*7c478bd9Sstevel@tonic-gate shared_mutex_held(mutex_t *mparg)
2189*7c478bd9Sstevel@tonic-gate {
2190*7c478bd9Sstevel@tonic-gate 	/*
2191*7c478bd9Sstevel@tonic-gate 	 * There is an inherent data race in the current ownership design.
2192*7c478bd9Sstevel@tonic-gate 	 * The mutex_owner and mutex_ownerpid fields cannot be set or tested
2193*7c478bd9Sstevel@tonic-gate 	 * atomically as a pair. The original implementation tested each
2194*7c478bd9Sstevel@tonic-gate 	 * field just once. This was exposed to trivial false positives in
2195*7c478bd9Sstevel@tonic-gate 	 * the case of multiple multithreaded processes with thread addresses
2196*7c478bd9Sstevel@tonic-gate 	 * in common. To close the window to an acceptable level we now use a
2197*7c478bd9Sstevel@tonic-gate 	 * sequence of five tests: pid-thr-pid-thr-pid. This ensures that any
2198*7c478bd9Sstevel@tonic-gate 	 * single interruption will still leave one uninterrupted sequence of
2199*7c478bd9Sstevel@tonic-gate 	 * pid-thr-pid tests intact.
2200*7c478bd9Sstevel@tonic-gate 	 *
2201*7c478bd9Sstevel@tonic-gate 	 * It is assumed that all updates are always ordered thr-pid and that
2202*7c478bd9Sstevel@tonic-gate 	 * we have TSO hardware.
2203*7c478bd9Sstevel@tonic-gate 	 */
2204*7c478bd9Sstevel@tonic-gate 	volatile mutex_t *mp = (volatile mutex_t *)mparg;
2205*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2206*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
2207*7c478bd9Sstevel@tonic-gate 
2208*7c478bd9Sstevel@tonic-gate 	if (mp->mutex_ownerpid != udp->pid)
2209*7c478bd9Sstevel@tonic-gate 		return (0);
2210*7c478bd9Sstevel@tonic-gate 
2211*7c478bd9Sstevel@tonic-gate 	if (!MUTEX_OWNED(mp, self))
2212*7c478bd9Sstevel@tonic-gate 		return (0);
2213*7c478bd9Sstevel@tonic-gate 
2214*7c478bd9Sstevel@tonic-gate 	if (mp->mutex_ownerpid != udp->pid)
2215*7c478bd9Sstevel@tonic-gate 		return (0);
2216*7c478bd9Sstevel@tonic-gate 
2217*7c478bd9Sstevel@tonic-gate 	if (!MUTEX_OWNED(mp, self))
2218*7c478bd9Sstevel@tonic-gate 		return (0);
2219*7c478bd9Sstevel@tonic-gate 
2220*7c478bd9Sstevel@tonic-gate 	if (mp->mutex_ownerpid != udp->pid)
2221*7c478bd9Sstevel@tonic-gate 		return (0);
2222*7c478bd9Sstevel@tonic-gate 
2223*7c478bd9Sstevel@tonic-gate 	return (1);
2224*7c478bd9Sstevel@tonic-gate }
2225*7c478bd9Sstevel@tonic-gate 
2226*7c478bd9Sstevel@tonic-gate /*
2227*7c478bd9Sstevel@tonic-gate  * Some crufty old programs define their own version of _mutex_held()
2228*7c478bd9Sstevel@tonic-gate  * to be simply return(1).  This breaks internal libc logic, so we
2229*7c478bd9Sstevel@tonic-gate  * define a private version for exclusive use by libc, mutex_is_held(),
2230*7c478bd9Sstevel@tonic-gate  * and also a new public function, __mutex_held(), to be used in new
2231*7c478bd9Sstevel@tonic-gate  * code to circumvent these crufty old programs.
2232*7c478bd9Sstevel@tonic-gate  */
2233*7c478bd9Sstevel@tonic-gate #pragma weak mutex_held = mutex_is_held
2234*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_held = mutex_is_held
2235*7c478bd9Sstevel@tonic-gate #pragma weak __mutex_held = mutex_is_held
2236*7c478bd9Sstevel@tonic-gate int
2237*7c478bd9Sstevel@tonic-gate mutex_is_held(mutex_t *mp)
2238*7c478bd9Sstevel@tonic-gate {
2239*7c478bd9Sstevel@tonic-gate 	if (mp->mutex_type & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))
2240*7c478bd9Sstevel@tonic-gate 		return (shared_mutex_held(mp));
2241*7c478bd9Sstevel@tonic-gate 	return (MUTEX_OWNED(mp, curthread));
2242*7c478bd9Sstevel@tonic-gate }
2243*7c478bd9Sstevel@tonic-gate 
2244*7c478bd9Sstevel@tonic-gate #pragma weak _private_mutex_destroy = __mutex_destroy
2245*7c478bd9Sstevel@tonic-gate #pragma weak mutex_destroy = __mutex_destroy
2246*7c478bd9Sstevel@tonic-gate #pragma weak _mutex_destroy = __mutex_destroy
2247*7c478bd9Sstevel@tonic-gate #pragma weak pthread_mutex_destroy = __mutex_destroy
2248*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_mutex_destroy = __mutex_destroy
2249*7c478bd9Sstevel@tonic-gate int
2250*7c478bd9Sstevel@tonic-gate __mutex_destroy(mutex_t *mp)
2251*7c478bd9Sstevel@tonic-gate {
2252*7c478bd9Sstevel@tonic-gate 	mp->mutex_magic = 0;
2253*7c478bd9Sstevel@tonic-gate 	mp->mutex_flag &= ~LOCK_INITED;
2254*7c478bd9Sstevel@tonic-gate 	tdb_sync_obj_deregister(mp);
2255*7c478bd9Sstevel@tonic-gate 	return (0);
2256*7c478bd9Sstevel@tonic-gate }
2257*7c478bd9Sstevel@tonic-gate 
2258*7c478bd9Sstevel@tonic-gate /*
2259*7c478bd9Sstevel@tonic-gate  * Spin locks are separate from ordinary mutexes,
2260*7c478bd9Sstevel@tonic-gate  * but we use the same data structure for them.
2261*7c478bd9Sstevel@tonic-gate  */
2262*7c478bd9Sstevel@tonic-gate 
2263*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_init = _pthread_spin_init
2264*7c478bd9Sstevel@tonic-gate int
2265*7c478bd9Sstevel@tonic-gate _pthread_spin_init(pthread_spinlock_t *lock, int pshared)
2266*7c478bd9Sstevel@tonic-gate {
2267*7c478bd9Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
2268*7c478bd9Sstevel@tonic-gate 
2269*7c478bd9Sstevel@tonic-gate 	(void) _memset(mp, 0, sizeof (*mp));
2270*7c478bd9Sstevel@tonic-gate 	if (pshared == PTHREAD_PROCESS_SHARED)
2271*7c478bd9Sstevel@tonic-gate 		mp->mutex_type = USYNC_PROCESS;
2272*7c478bd9Sstevel@tonic-gate 	else
2273*7c478bd9Sstevel@tonic-gate 		mp->mutex_type = USYNC_THREAD;
2274*7c478bd9Sstevel@tonic-gate 	mp->mutex_flag = LOCK_INITED;
2275*7c478bd9Sstevel@tonic-gate 	mp->mutex_magic = MUTEX_MAGIC;
2276*7c478bd9Sstevel@tonic-gate 	return (0);
2277*7c478bd9Sstevel@tonic-gate }
2278*7c478bd9Sstevel@tonic-gate 
2279*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_destroy = _pthread_spin_destroy
2280*7c478bd9Sstevel@tonic-gate int
2281*7c478bd9Sstevel@tonic-gate _pthread_spin_destroy(pthread_spinlock_t *lock)
2282*7c478bd9Sstevel@tonic-gate {
2283*7c478bd9Sstevel@tonic-gate 	(void) _memset(lock, 0, sizeof (*lock));
2284*7c478bd9Sstevel@tonic-gate 	return (0);
2285*7c478bd9Sstevel@tonic-gate }
2286*7c478bd9Sstevel@tonic-gate 
2287*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_trylock = _pthread_spin_trylock
2288*7c478bd9Sstevel@tonic-gate int
2289*7c478bd9Sstevel@tonic-gate _pthread_spin_trylock(pthread_spinlock_t *lock)
2290*7c478bd9Sstevel@tonic-gate {
2291*7c478bd9Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
2292*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2293*7c478bd9Sstevel@tonic-gate 	int error = 0;
2294*7c478bd9Sstevel@tonic-gate 
2295*7c478bd9Sstevel@tonic-gate 	no_preempt(self);
2296*7c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) != 0)
2297*7c478bd9Sstevel@tonic-gate 		error = EBUSY;
2298*7c478bd9Sstevel@tonic-gate 	else {
2299*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
2300*7c478bd9Sstevel@tonic-gate 		if (mp->mutex_type == USYNC_PROCESS)
2301*7c478bd9Sstevel@tonic-gate 			mp->mutex_ownerpid = self->ul_uberdata->pid;
2302*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2303*7c478bd9Sstevel@tonic-gate 	}
2304*7c478bd9Sstevel@tonic-gate 	preempt(self);
2305*7c478bd9Sstevel@tonic-gate 	return (error);
2306*7c478bd9Sstevel@tonic-gate }
2307*7c478bd9Sstevel@tonic-gate 
2308*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_lock = _pthread_spin_lock
2309*7c478bd9Sstevel@tonic-gate int
2310*7c478bd9Sstevel@tonic-gate _pthread_spin_lock(pthread_spinlock_t *lock)
2311*7c478bd9Sstevel@tonic-gate {
2312*7c478bd9Sstevel@tonic-gate 	volatile uint8_t *lockp =
2313*7c478bd9Sstevel@tonic-gate 		(volatile uint8_t *)&((mutex_t *)lock)->mutex_lockw;
2314*7c478bd9Sstevel@tonic-gate 
2315*7c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
2316*7c478bd9Sstevel@tonic-gate 	/*
2317*7c478bd9Sstevel@tonic-gate 	 * We don't care whether the owner is running on a processor.
2318*7c478bd9Sstevel@tonic-gate 	 * We just spin because that's what this interface requires.
2319*7c478bd9Sstevel@tonic-gate 	 */
2320*7c478bd9Sstevel@tonic-gate 	for (;;) {
2321*7c478bd9Sstevel@tonic-gate 		if (*lockp == 0) {	/* lock byte appears to be clear */
2322*7c478bd9Sstevel@tonic-gate 			if (_pthread_spin_trylock(lock) == 0)
2323*7c478bd9Sstevel@tonic-gate 				return (0);
2324*7c478bd9Sstevel@tonic-gate 		}
2325*7c478bd9Sstevel@tonic-gate 		SMT_PAUSE();
2326*7c478bd9Sstevel@tonic-gate 	}
2327*7c478bd9Sstevel@tonic-gate }
2328*7c478bd9Sstevel@tonic-gate 
2329*7c478bd9Sstevel@tonic-gate #pragma weak pthread_spin_unlock = _pthread_spin_unlock
2330*7c478bd9Sstevel@tonic-gate int
2331*7c478bd9Sstevel@tonic-gate _pthread_spin_unlock(pthread_spinlock_t *lock)
2332*7c478bd9Sstevel@tonic-gate {
2333*7c478bd9Sstevel@tonic-gate 	mutex_t *mp = (mutex_t *)lock;
2334*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2335*7c478bd9Sstevel@tonic-gate 
2336*7c478bd9Sstevel@tonic-gate 	no_preempt(self);
2337*7c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
2338*7c478bd9Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
2339*7c478bd9Sstevel@tonic-gate 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
2340*7c478bd9Sstevel@tonic-gate 	(void) swap32(&mp->mutex_lockword, 0);
2341*7c478bd9Sstevel@tonic-gate 	preempt(self);
2342*7c478bd9Sstevel@tonic-gate 	return (0);
2343*7c478bd9Sstevel@tonic-gate }
2344*7c478bd9Sstevel@tonic-gate 
2345*7c478bd9Sstevel@tonic-gate #pragma weak cond_init = _cond_init
2346*7c478bd9Sstevel@tonic-gate /* ARGSUSED2 */
2347*7c478bd9Sstevel@tonic-gate int
2348*7c478bd9Sstevel@tonic-gate _cond_init(cond_t *cvp, int type, void *arg)
2349*7c478bd9Sstevel@tonic-gate {
2350*7c478bd9Sstevel@tonic-gate 	if (type != USYNC_THREAD && type != USYNC_PROCESS)
2351*7c478bd9Sstevel@tonic-gate 		return (EINVAL);
2352*7c478bd9Sstevel@tonic-gate 	(void) _memset(cvp, 0, sizeof (*cvp));
2353*7c478bd9Sstevel@tonic-gate 	cvp->cond_type = (uint16_t)type;
2354*7c478bd9Sstevel@tonic-gate 	cvp->cond_magic = COND_MAGIC;
2355*7c478bd9Sstevel@tonic-gate 	return (0);
2356*7c478bd9Sstevel@tonic-gate }
2357*7c478bd9Sstevel@tonic-gate 
2358*7c478bd9Sstevel@tonic-gate /*
2359*7c478bd9Sstevel@tonic-gate  * cond_sleep_queue(): utility function for cond_wait_queue().
2360*7c478bd9Sstevel@tonic-gate  *
2361*7c478bd9Sstevel@tonic-gate  * Go to sleep on a condvar sleep queue, expect to be waked up
2362*7c478bd9Sstevel@tonic-gate  * by someone calling cond_signal() or cond_broadcast() or due
2363*7c478bd9Sstevel@tonic-gate  * to receiving a UNIX signal or being cancelled, or just simply
2364*7c478bd9Sstevel@tonic-gate  * due to a spurious wakeup (like someome calling forkall()).
2365*7c478bd9Sstevel@tonic-gate  *
2366*7c478bd9Sstevel@tonic-gate  * The associated mutex is *not* reacquired before returning.
2367*7c478bd9Sstevel@tonic-gate  * That must be done by the caller of cond_sleep_queue().
2368*7c478bd9Sstevel@tonic-gate  */
2369*7c478bd9Sstevel@tonic-gate int
2370*7c478bd9Sstevel@tonic-gate cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
2371*7c478bd9Sstevel@tonic-gate {
2372*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2373*7c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
2374*7c478bd9Sstevel@tonic-gate 	queue_head_t *mqp;
2375*7c478bd9Sstevel@tonic-gate 	lwpid_t lwpid;
2376*7c478bd9Sstevel@tonic-gate 	int signalled;
2377*7c478bd9Sstevel@tonic-gate 	int error;
2378*7c478bd9Sstevel@tonic-gate 
2379*7c478bd9Sstevel@tonic-gate 	/*
2380*7c478bd9Sstevel@tonic-gate 	 * Put ourself on the CV sleep queue, unlock the mutex, then
2381*7c478bd9Sstevel@tonic-gate 	 * park ourself and unpark a candidate lwp to grab the mutex.
2382*7c478bd9Sstevel@tonic-gate 	 * We must go onto the CV sleep queue before dropping the
2383*7c478bd9Sstevel@tonic-gate 	 * mutex in order to guarantee atomicity of the operation.
2384*7c478bd9Sstevel@tonic-gate 	 */
2385*7c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
2386*7c478bd9Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
2387*7c478bd9Sstevel@tonic-gate 	enqueue(qp, self, cvp, CV);
2388*7c478bd9Sstevel@tonic-gate 	cvp->cond_waiters_user = 1;
2389*7c478bd9Sstevel@tonic-gate 	self->ul_cvmutex = mp;
2390*7c478bd9Sstevel@tonic-gate 	self->ul_cv_wake = (tsp != NULL);
2391*7c478bd9Sstevel@tonic-gate 	self->ul_signalled = 0;
2392*7c478bd9Sstevel@tonic-gate 	lwpid = mutex_unlock_queue(mp);
2393*7c478bd9Sstevel@tonic-gate 	for (;;) {
2394*7c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 1);
2395*7c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
2396*7c478bd9Sstevel@tonic-gate 		if (lwpid != 0) {
2397*7c478bd9Sstevel@tonic-gate 			lwpid = preempt_unpark(self, lwpid);
2398*7c478bd9Sstevel@tonic-gate 			preempt(self);
2399*7c478bd9Sstevel@tonic-gate 		}
2400*7c478bd9Sstevel@tonic-gate 		/*
2401*7c478bd9Sstevel@tonic-gate 		 * We may have a deferred signal present,
2402*7c478bd9Sstevel@tonic-gate 		 * in which case we should return EINTR.
2403*7c478bd9Sstevel@tonic-gate 		 * Also, we may have received a SIGCANCEL; if so
2404*7c478bd9Sstevel@tonic-gate 		 * and we are cancelable we should return EINTR.
2405*7c478bd9Sstevel@tonic-gate 		 * We force an immediate EINTR return from
2406*7c478bd9Sstevel@tonic-gate 		 * __lwp_park() by turning our parking flag off.
2407*7c478bd9Sstevel@tonic-gate 		 */
2408*7c478bd9Sstevel@tonic-gate 		if (self->ul_cursig != 0 ||
2409*7c478bd9Sstevel@tonic-gate 		    (self->ul_cancelable && self->ul_cancel_pending))
2410*7c478bd9Sstevel@tonic-gate 			set_parking_flag(self, 0);
2411*7c478bd9Sstevel@tonic-gate 		/*
2412*7c478bd9Sstevel@tonic-gate 		 * __lwp_park() will return the residual time in tsp
2413*7c478bd9Sstevel@tonic-gate 		 * if we are unparked before the timeout expires.
2414*7c478bd9Sstevel@tonic-gate 		 */
2415*7c478bd9Sstevel@tonic-gate 		error = __lwp_park(tsp, lwpid);
2416*7c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 0);
2417*7c478bd9Sstevel@tonic-gate 		lwpid = 0;	/* unpark the other lwp only once */
2418*7c478bd9Sstevel@tonic-gate 		/*
2419*7c478bd9Sstevel@tonic-gate 		 * We were waked up by cond_signal(), cond_broadcast(),
2420*7c478bd9Sstevel@tonic-gate 		 * by an interrupt or timeout (EINTR or ETIME),
2421*7c478bd9Sstevel@tonic-gate 		 * or we may just have gotten a spurious wakeup.
2422*7c478bd9Sstevel@tonic-gate 		 */
2423*7c478bd9Sstevel@tonic-gate 		qp = queue_lock(cvp, CV);
2424*7c478bd9Sstevel@tonic-gate 		mqp = queue_lock(mp, MX);
2425*7c478bd9Sstevel@tonic-gate 		if (self->ul_sleepq == NULL)
2426*7c478bd9Sstevel@tonic-gate 			break;
2427*7c478bd9Sstevel@tonic-gate 		/*
2428*7c478bd9Sstevel@tonic-gate 		 * We are on either the condvar sleep queue or the
2429*7c478bd9Sstevel@tonic-gate 		 * mutex sleep queue.  If we are on the mutex sleep
2430*7c478bd9Sstevel@tonic-gate 		 * queue, continue sleeping.  If we are on the condvar
2431*7c478bd9Sstevel@tonic-gate 		 * sleep queue, break out of the sleep if we were
2432*7c478bd9Sstevel@tonic-gate 		 * interrupted or we timed out (EINTR or ETIME).
2433*7c478bd9Sstevel@tonic-gate 		 * Else this is a spurious wakeup; continue the loop.
2434*7c478bd9Sstevel@tonic-gate 		 */
2435*7c478bd9Sstevel@tonic-gate 		if (self->ul_sleepq == mqp)		/* mutex queue */
2436*7c478bd9Sstevel@tonic-gate 			tsp = NULL;
2437*7c478bd9Sstevel@tonic-gate 		else if (self->ul_sleepq == qp) {	/* condvar queue */
2438*7c478bd9Sstevel@tonic-gate 			if (error) {
2439*7c478bd9Sstevel@tonic-gate 				cvp->cond_waiters_user = dequeue_self(qp, cvp);
2440*7c478bd9Sstevel@tonic-gate 				break;
2441*7c478bd9Sstevel@tonic-gate 			}
2442*7c478bd9Sstevel@tonic-gate 			/*
2443*7c478bd9Sstevel@tonic-gate 			 * Else a spurious wakeup on the condvar queue.
2444*7c478bd9Sstevel@tonic-gate 			 * __lwp_park() has already adjusted the timeout.
2445*7c478bd9Sstevel@tonic-gate 			 */
2446*7c478bd9Sstevel@tonic-gate 		} else {
2447*7c478bd9Sstevel@tonic-gate 			thr_panic("cond_sleep_queue(): thread not on queue");
2448*7c478bd9Sstevel@tonic-gate 		}
2449*7c478bd9Sstevel@tonic-gate 		queue_unlock(mqp);
2450*7c478bd9Sstevel@tonic-gate 	}
2451*7c478bd9Sstevel@tonic-gate 
2452*7c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
2453*7c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_cvmutex == NULL && self->ul_cv_wake == 0);
2454*7c478bd9Sstevel@tonic-gate 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
2455*7c478bd9Sstevel@tonic-gate 	    self->ul_wchan == NULL);
2456*7c478bd9Sstevel@tonic-gate 
2457*7c478bd9Sstevel@tonic-gate 	signalled = self->ul_signalled;
2458*7c478bd9Sstevel@tonic-gate 	self->ul_signalled = 0;
2459*7c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
2460*7c478bd9Sstevel@tonic-gate 	queue_unlock(mqp);
2461*7c478bd9Sstevel@tonic-gate 
2462*7c478bd9Sstevel@tonic-gate 	/*
2463*7c478bd9Sstevel@tonic-gate 	 * If we were concurrently cond_signal()d and any of:
2464*7c478bd9Sstevel@tonic-gate 	 * received a UNIX signal, were cancelled, or got a timeout,
2465*7c478bd9Sstevel@tonic-gate 	 * then perform another cond_signal() to avoid consuming it.
2466*7c478bd9Sstevel@tonic-gate 	 */
2467*7c478bd9Sstevel@tonic-gate 	if (error && signalled)
2468*7c478bd9Sstevel@tonic-gate 		(void) cond_signal_internal(cvp);
2469*7c478bd9Sstevel@tonic-gate 
2470*7c478bd9Sstevel@tonic-gate 	return (error);
2471*7c478bd9Sstevel@tonic-gate }
2472*7c478bd9Sstevel@tonic-gate 
2473*7c478bd9Sstevel@tonic-gate int
2474*7c478bd9Sstevel@tonic-gate cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp,
2475*7c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp)
2476*7c478bd9Sstevel@tonic-gate {
2477*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2478*7c478bd9Sstevel@tonic-gate 	int error;
2479*7c478bd9Sstevel@tonic-gate 
2480*7c478bd9Sstevel@tonic-gate 	/*
2481*7c478bd9Sstevel@tonic-gate 	 * The old thread library was programmed to defer signals
2482*7c478bd9Sstevel@tonic-gate 	 * while in cond_wait() so that the associated mutex would
2483*7c478bd9Sstevel@tonic-gate 	 * be guaranteed to be held when the application signal
2484*7c478bd9Sstevel@tonic-gate 	 * handler was invoked.
2485*7c478bd9Sstevel@tonic-gate 	 *
2486*7c478bd9Sstevel@tonic-gate 	 * We do not behave this way by default; the state of the
2487*7c478bd9Sstevel@tonic-gate 	 * associated mutex in the signal handler is undefined.
2488*7c478bd9Sstevel@tonic-gate 	 *
2489*7c478bd9Sstevel@tonic-gate 	 * To accommodate applications that depend on the old
2490*7c478bd9Sstevel@tonic-gate 	 * behavior, the _THREAD_COND_WAIT_DEFER environment
2491*7c478bd9Sstevel@tonic-gate 	 * variable can be set to 1 and we will behave in the
2492*7c478bd9Sstevel@tonic-gate 	 * old way with respect to cond_wait().
2493*7c478bd9Sstevel@tonic-gate 	 */
2494*7c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
2495*7c478bd9Sstevel@tonic-gate 		sigoff(self);
2496*7c478bd9Sstevel@tonic-gate 
2497*7c478bd9Sstevel@tonic-gate 	error = cond_sleep_queue(cvp, mp, tsp);
2498*7c478bd9Sstevel@tonic-gate 
2499*7c478bd9Sstevel@tonic-gate 	/*
2500*7c478bd9Sstevel@tonic-gate 	 * Reacquire the mutex.
2501*7c478bd9Sstevel@tonic-gate 	 */
2502*7c478bd9Sstevel@tonic-gate 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
2503*7c478bd9Sstevel@tonic-gate 		mp->mutex_owner = (uintptr_t)self;
2504*7c478bd9Sstevel@tonic-gate 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2505*7c478bd9Sstevel@tonic-gate 	} else if (mutex_trylock_adaptive(mp) != 0) {
2506*7c478bd9Sstevel@tonic-gate 		(void) mutex_lock_queue(self, msp, mp, NULL);
2507*7c478bd9Sstevel@tonic-gate 	}
2508*7c478bd9Sstevel@tonic-gate 
2509*7c478bd9Sstevel@tonic-gate 	if (msp)
2510*7c478bd9Sstevel@tonic-gate 		record_begin_hold(msp);
2511*7c478bd9Sstevel@tonic-gate 
2512*7c478bd9Sstevel@tonic-gate 	/*
2513*7c478bd9Sstevel@tonic-gate 	 * Take any deferred signal now, after we have reacquired the mutex.
2514*7c478bd9Sstevel@tonic-gate 	 */
2515*7c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
2516*7c478bd9Sstevel@tonic-gate 		sigon(self);
2517*7c478bd9Sstevel@tonic-gate 
2518*7c478bd9Sstevel@tonic-gate 	return (error);
2519*7c478bd9Sstevel@tonic-gate }
2520*7c478bd9Sstevel@tonic-gate 
2521*7c478bd9Sstevel@tonic-gate /*
2522*7c478bd9Sstevel@tonic-gate  * cond_sleep_kernel(): utility function for cond_wait_kernel().
2523*7c478bd9Sstevel@tonic-gate  * See the comment ahead of cond_sleep_queue(), above.
2524*7c478bd9Sstevel@tonic-gate  */
2525*7c478bd9Sstevel@tonic-gate int
2526*7c478bd9Sstevel@tonic-gate cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
2527*7c478bd9Sstevel@tonic-gate {
2528*7c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
2529*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2530*7c478bd9Sstevel@tonic-gate 	int error;
2531*7c478bd9Sstevel@tonic-gate 
2532*7c478bd9Sstevel@tonic-gate 	if (mtype & PTHREAD_PRIO_PROTECT) {
2533*7c478bd9Sstevel@tonic-gate 		if (_ceil_mylist_del(mp))
2534*7c478bd9Sstevel@tonic-gate 			_ceil_prio_waive();
2535*7c478bd9Sstevel@tonic-gate 	}
2536*7c478bd9Sstevel@tonic-gate 
2537*7c478bd9Sstevel@tonic-gate 	self->ul_sp = stkptr();
2538*7c478bd9Sstevel@tonic-gate 	self->ul_wchan = cvp;
2539*7c478bd9Sstevel@tonic-gate 	mp->mutex_owner = 0;
2540*7c478bd9Sstevel@tonic-gate 	mp->mutex_ownerpid = 0;
2541*7c478bd9Sstevel@tonic-gate 	if (mtype & PTHREAD_PRIO_INHERIT)
2542*7c478bd9Sstevel@tonic-gate 		mp->mutex_lockw = LOCKCLEAR;
2543*7c478bd9Sstevel@tonic-gate 	/*
2544*7c478bd9Sstevel@tonic-gate 	 * ___lwp_cond_wait() returns immediately with EINTR if
2545*7c478bd9Sstevel@tonic-gate 	 * set_parking_flag(self,0) is called on this lwp before it
2546*7c478bd9Sstevel@tonic-gate 	 * goes to sleep in the kernel.  sigacthandler() calls this
2547*7c478bd9Sstevel@tonic-gate 	 * when a deferred signal is noted.  This assures that we don't
2548*7c478bd9Sstevel@tonic-gate 	 * get stuck in ___lwp_cond_wait() with all signals blocked
2549*7c478bd9Sstevel@tonic-gate 	 * due to taking a deferred signal before going to sleep.
2550*7c478bd9Sstevel@tonic-gate 	 */
2551*7c478bd9Sstevel@tonic-gate 	set_parking_flag(self, 1);
2552*7c478bd9Sstevel@tonic-gate 	if (self->ul_cursig != 0 ||
2553*7c478bd9Sstevel@tonic-gate 	    (self->ul_cancelable && self->ul_cancel_pending))
2554*7c478bd9Sstevel@tonic-gate 		set_parking_flag(self, 0);
2555*7c478bd9Sstevel@tonic-gate 	error = ___lwp_cond_wait(cvp, mp, tsp, 1);
2556*7c478bd9Sstevel@tonic-gate 	set_parking_flag(self, 0);
2557*7c478bd9Sstevel@tonic-gate 	self->ul_sp = 0;
2558*7c478bd9Sstevel@tonic-gate 	self->ul_wchan = NULL;
2559*7c478bd9Sstevel@tonic-gate 	return (error);
2560*7c478bd9Sstevel@tonic-gate }
2561*7c478bd9Sstevel@tonic-gate 
2562*7c478bd9Sstevel@tonic-gate int
2563*7c478bd9Sstevel@tonic-gate cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
2564*7c478bd9Sstevel@tonic-gate {
2565*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2566*7c478bd9Sstevel@tonic-gate 	int error;
2567*7c478bd9Sstevel@tonic-gate 	int merror;
2568*7c478bd9Sstevel@tonic-gate 
2569*7c478bd9Sstevel@tonic-gate 	/*
2570*7c478bd9Sstevel@tonic-gate 	 * See the large comment in cond_wait_queue(), above.
2571*7c478bd9Sstevel@tonic-gate 	 */
2572*7c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
2573*7c478bd9Sstevel@tonic-gate 		sigoff(self);
2574*7c478bd9Sstevel@tonic-gate 
2575*7c478bd9Sstevel@tonic-gate 	error = cond_sleep_kernel(cvp, mp, tsp);
2576*7c478bd9Sstevel@tonic-gate 
2577*7c478bd9Sstevel@tonic-gate 	/*
2578*7c478bd9Sstevel@tonic-gate 	 * Override the return code from ___lwp_cond_wait()
2579*7c478bd9Sstevel@tonic-gate 	 * with any non-zero return code from mutex_lock().
2580*7c478bd9Sstevel@tonic-gate 	 * This addresses robust lock failures in particular;
2581*7c478bd9Sstevel@tonic-gate 	 * the caller must see the EOWNERDEAD or ENOTRECOVERABLE
2582*7c478bd9Sstevel@tonic-gate 	 * errors in order to take corrective action.
2583*7c478bd9Sstevel@tonic-gate 	 */
2584*7c478bd9Sstevel@tonic-gate 	if ((merror = _private_mutex_lock(mp)) != 0)
2585*7c478bd9Sstevel@tonic-gate 		error = merror;
2586*7c478bd9Sstevel@tonic-gate 
2587*7c478bd9Sstevel@tonic-gate 	/*
2588*7c478bd9Sstevel@tonic-gate 	 * Take any deferred signal now, after we have reacquired the mutex.
2589*7c478bd9Sstevel@tonic-gate 	 */
2590*7c478bd9Sstevel@tonic-gate 	if (self->ul_cond_wait_defer)
2591*7c478bd9Sstevel@tonic-gate 		sigon(self);
2592*7c478bd9Sstevel@tonic-gate 
2593*7c478bd9Sstevel@tonic-gate 	return (error);
2594*7c478bd9Sstevel@tonic-gate }
2595*7c478bd9Sstevel@tonic-gate 
2596*7c478bd9Sstevel@tonic-gate /*
2597*7c478bd9Sstevel@tonic-gate  * Common code for _cond_wait() and _cond_timedwait()
2598*7c478bd9Sstevel@tonic-gate  */
2599*7c478bd9Sstevel@tonic-gate int
2600*7c478bd9Sstevel@tonic-gate cond_wait_common(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
2601*7c478bd9Sstevel@tonic-gate {
2602*7c478bd9Sstevel@tonic-gate 	int mtype = mp->mutex_type;
2603*7c478bd9Sstevel@tonic-gate 	hrtime_t begin_sleep = 0;
2604*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2605*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
2606*7c478bd9Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
2607*7c478bd9Sstevel@tonic-gate 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
2608*7c478bd9Sstevel@tonic-gate 	uint8_t rcount;
2609*7c478bd9Sstevel@tonic-gate 	int error = 0;
2610*7c478bd9Sstevel@tonic-gate 
2611*7c478bd9Sstevel@tonic-gate 	/*
2612*7c478bd9Sstevel@tonic-gate 	 * The SUSV3 Posix spec for pthread_cond_timedwait() states:
2613*7c478bd9Sstevel@tonic-gate 	 *	Except in the case of [ETIMEDOUT], all these error checks
2614*7c478bd9Sstevel@tonic-gate 	 *	shall act as if they were performed immediately at the
2615*7c478bd9Sstevel@tonic-gate 	 *	beginning of processing for the function and shall cause
2616*7c478bd9Sstevel@tonic-gate 	 *	an error return, in effect, prior to modifying the state
2617*7c478bd9Sstevel@tonic-gate 	 *	of the mutex specified by mutex or the condition variable
2618*7c478bd9Sstevel@tonic-gate 	 *	specified by cond.
2619*7c478bd9Sstevel@tonic-gate 	 * Therefore, we must return EINVAL now if the timout is invalid.
2620*7c478bd9Sstevel@tonic-gate 	 */
2621*7c478bd9Sstevel@tonic-gate 	if (tsp != NULL &&
2622*7c478bd9Sstevel@tonic-gate 	    (tsp->tv_sec < 0 || (ulong_t)tsp->tv_nsec >= NANOSEC))
2623*7c478bd9Sstevel@tonic-gate 		return (EINVAL);
2624*7c478bd9Sstevel@tonic-gate 
2625*7c478bd9Sstevel@tonic-gate 	if (__td_event_report(self, TD_SLEEP, udp)) {
2626*7c478bd9Sstevel@tonic-gate 		self->ul_sp = stkptr();
2627*7c478bd9Sstevel@tonic-gate 		self->ul_wchan = cvp;
2628*7c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventnum = TD_SLEEP;
2629*7c478bd9Sstevel@tonic-gate 		self->ul_td_evbuf.eventdata = cvp;
2630*7c478bd9Sstevel@tonic-gate 		tdb_event(TD_SLEEP, udp);
2631*7c478bd9Sstevel@tonic-gate 		self->ul_sp = 0;
2632*7c478bd9Sstevel@tonic-gate 	}
2633*7c478bd9Sstevel@tonic-gate 	if (csp) {
2634*7c478bd9Sstevel@tonic-gate 		if (tsp)
2635*7c478bd9Sstevel@tonic-gate 			tdb_incr(csp->cond_timedwait);
2636*7c478bd9Sstevel@tonic-gate 		else
2637*7c478bd9Sstevel@tonic-gate 			tdb_incr(csp->cond_wait);
2638*7c478bd9Sstevel@tonic-gate 	}
2639*7c478bd9Sstevel@tonic-gate 	if (msp)
2640*7c478bd9Sstevel@tonic-gate 		begin_sleep = record_hold_time(msp);
2641*7c478bd9Sstevel@tonic-gate 	else if (csp)
2642*7c478bd9Sstevel@tonic-gate 		begin_sleep = gethrtime();
2643*7c478bd9Sstevel@tonic-gate 
2644*7c478bd9Sstevel@tonic-gate 	if (self->ul_error_detection) {
2645*7c478bd9Sstevel@tonic-gate 		if (!mutex_is_held(mp))
2646*7c478bd9Sstevel@tonic-gate 			lock_error(mp, "cond_wait", cvp, NULL);
2647*7c478bd9Sstevel@tonic-gate 		if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0)
2648*7c478bd9Sstevel@tonic-gate 			lock_error(mp, "recursive mutex in cond_wait",
2649*7c478bd9Sstevel@tonic-gate 				cvp, NULL);
2650*7c478bd9Sstevel@tonic-gate 		if (cvp->cond_type & USYNC_PROCESS) {
2651*7c478bd9Sstevel@tonic-gate 			if (!(mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST)))
2652*7c478bd9Sstevel@tonic-gate 				lock_error(mp, "cond_wait", cvp,
2653*7c478bd9Sstevel@tonic-gate 					"condvar process-shared, "
2654*7c478bd9Sstevel@tonic-gate 					"mutex process-private");
2655*7c478bd9Sstevel@tonic-gate 		} else {
2656*7c478bd9Sstevel@tonic-gate 			if (mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST))
2657*7c478bd9Sstevel@tonic-gate 				lock_error(mp, "cond_wait", cvp,
2658*7c478bd9Sstevel@tonic-gate 					"condvar process-private, "
2659*7c478bd9Sstevel@tonic-gate 					"mutex process-shared");
2660*7c478bd9Sstevel@tonic-gate 		}
2661*7c478bd9Sstevel@tonic-gate 	}
2662*7c478bd9Sstevel@tonic-gate 
2663*7c478bd9Sstevel@tonic-gate 	/*
2664*7c478bd9Sstevel@tonic-gate 	 * We deal with recursive mutexes by completely
2665*7c478bd9Sstevel@tonic-gate 	 * dropping the lock and restoring the recursion
2666*7c478bd9Sstevel@tonic-gate 	 * count after waking up.  This is arguably wrong,
2667*7c478bd9Sstevel@tonic-gate 	 * but it obeys the principle of least astonishment.
2668*7c478bd9Sstevel@tonic-gate 	 */
2669*7c478bd9Sstevel@tonic-gate 	rcount = mp->mutex_rcount;
2670*7c478bd9Sstevel@tonic-gate 	mp->mutex_rcount = 0;
2671*7c478bd9Sstevel@tonic-gate 	if ((mtype & (USYNC_PROCESS | USYNC_PROCESS_ROBUST |
2672*7c478bd9Sstevel@tonic-gate 	    PTHREAD_PRIO_INHERIT | PTHREAD_PRIO_PROTECT)) |
2673*7c478bd9Sstevel@tonic-gate 	    (cvp->cond_type & USYNC_PROCESS))
2674*7c478bd9Sstevel@tonic-gate 		error = cond_wait_kernel(cvp, mp, tsp);
2675*7c478bd9Sstevel@tonic-gate 	else
2676*7c478bd9Sstevel@tonic-gate 		error = cond_wait_queue(cvp, mp, tsp, msp);
2677*7c478bd9Sstevel@tonic-gate 	mp->mutex_rcount = rcount;
2678*7c478bd9Sstevel@tonic-gate 
2679*7c478bd9Sstevel@tonic-gate 	if (csp) {
2680*7c478bd9Sstevel@tonic-gate 		hrtime_t lapse = gethrtime() - begin_sleep;
2681*7c478bd9Sstevel@tonic-gate 		if (tsp == NULL)
2682*7c478bd9Sstevel@tonic-gate 			csp->cond_wait_sleep_time += lapse;
2683*7c478bd9Sstevel@tonic-gate 		else {
2684*7c478bd9Sstevel@tonic-gate 			csp->cond_timedwait_sleep_time += lapse;
2685*7c478bd9Sstevel@tonic-gate 			if (error == ETIME)
2686*7c478bd9Sstevel@tonic-gate 				tdb_incr(csp->cond_timedwait_timeout);
2687*7c478bd9Sstevel@tonic-gate 		}
2688*7c478bd9Sstevel@tonic-gate 	}
2689*7c478bd9Sstevel@tonic-gate 	return (error);
2690*7c478bd9Sstevel@tonic-gate }
2691*7c478bd9Sstevel@tonic-gate 
2692*7c478bd9Sstevel@tonic-gate /*
2693*7c478bd9Sstevel@tonic-gate  * cond_wait() is a cancellation point but _cond_wait() is not.
2694*7c478bd9Sstevel@tonic-gate  * System libraries call the non-cancellation version.
2695*7c478bd9Sstevel@tonic-gate  * It is expected that only applications call the cancellation version.
2696*7c478bd9Sstevel@tonic-gate  */
2697*7c478bd9Sstevel@tonic-gate int
2698*7c478bd9Sstevel@tonic-gate _cond_wait(cond_t *cvp, mutex_t *mp)
2699*7c478bd9Sstevel@tonic-gate {
2700*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2701*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
2702*7c478bd9Sstevel@tonic-gate 	uberflags_t *gflags;
2703*7c478bd9Sstevel@tonic-gate 
2704*7c478bd9Sstevel@tonic-gate 	/*
2705*7c478bd9Sstevel@tonic-gate 	 * Optimize the common case of USYNC_THREAD plus
2706*7c478bd9Sstevel@tonic-gate 	 * no error detection, no lock statistics, and no event tracing.
2707*7c478bd9Sstevel@tonic-gate 	 */
2708*7c478bd9Sstevel@tonic-gate 	if ((gflags = self->ul_schedctl_called) != NULL &&
2709*7c478bd9Sstevel@tonic-gate 	    (cvp->cond_type | mp->mutex_type | gflags->uf_trs_ted |
2710*7c478bd9Sstevel@tonic-gate 	    self->ul_td_events_enable |
2711*7c478bd9Sstevel@tonic-gate 	    udp->tdb.tdb_ev_global_mask.event_bits[0]) == 0)
2712*7c478bd9Sstevel@tonic-gate 		return (cond_wait_queue(cvp, mp, NULL, NULL));
2713*7c478bd9Sstevel@tonic-gate 
2714*7c478bd9Sstevel@tonic-gate 	/*
2715*7c478bd9Sstevel@tonic-gate 	 * Else do it the long way.
2716*7c478bd9Sstevel@tonic-gate 	 */
2717*7c478bd9Sstevel@tonic-gate 	return (cond_wait_common(cvp, mp, NULL));
2718*7c478bd9Sstevel@tonic-gate }
2719*7c478bd9Sstevel@tonic-gate 
2720*7c478bd9Sstevel@tonic-gate int
2721*7c478bd9Sstevel@tonic-gate cond_wait(cond_t *cvp, mutex_t *mp)
2722*7c478bd9Sstevel@tonic-gate {
2723*7c478bd9Sstevel@tonic-gate 	int error;
2724*7c478bd9Sstevel@tonic-gate 
2725*7c478bd9Sstevel@tonic-gate 	_cancelon();
2726*7c478bd9Sstevel@tonic-gate 	error = _cond_wait(cvp, mp);
2727*7c478bd9Sstevel@tonic-gate 	if (error == EINTR)
2728*7c478bd9Sstevel@tonic-gate 		_canceloff();
2729*7c478bd9Sstevel@tonic-gate 	else
2730*7c478bd9Sstevel@tonic-gate 		_canceloff_nocancel();
2731*7c478bd9Sstevel@tonic-gate 	return (error);
2732*7c478bd9Sstevel@tonic-gate }
2733*7c478bd9Sstevel@tonic-gate 
2734*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_wait = _pthread_cond_wait
2735*7c478bd9Sstevel@tonic-gate int
2736*7c478bd9Sstevel@tonic-gate _pthread_cond_wait(cond_t *cvp, mutex_t *mp)
2737*7c478bd9Sstevel@tonic-gate {
2738*7c478bd9Sstevel@tonic-gate 	int error;
2739*7c478bd9Sstevel@tonic-gate 
2740*7c478bd9Sstevel@tonic-gate 	error = cond_wait(cvp, mp);
2741*7c478bd9Sstevel@tonic-gate 	return ((error == EINTR)? 0 : error);
2742*7c478bd9Sstevel@tonic-gate }
2743*7c478bd9Sstevel@tonic-gate 
2744*7c478bd9Sstevel@tonic-gate /*
2745*7c478bd9Sstevel@tonic-gate  * cond_timedwait() is a cancellation point but _cond_timedwait() is not.
2746*7c478bd9Sstevel@tonic-gate  * System libraries call the non-cancellation version.
2747*7c478bd9Sstevel@tonic-gate  * It is expected that only applications call the cancellation version.
2748*7c478bd9Sstevel@tonic-gate  */
2749*7c478bd9Sstevel@tonic-gate int
2750*7c478bd9Sstevel@tonic-gate _cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
2751*7c478bd9Sstevel@tonic-gate {
2752*7c478bd9Sstevel@tonic-gate 	clockid_t clock_id = cvp->cond_clockid;
2753*7c478bd9Sstevel@tonic-gate 	timespec_t reltime;
2754*7c478bd9Sstevel@tonic-gate 	int error;
2755*7c478bd9Sstevel@tonic-gate 
2756*7c478bd9Sstevel@tonic-gate 	if (clock_id != CLOCK_REALTIME && clock_id != CLOCK_HIGHRES)
2757*7c478bd9Sstevel@tonic-gate 		clock_id = CLOCK_REALTIME;
2758*7c478bd9Sstevel@tonic-gate 	abstime_to_reltime(clock_id, abstime, &reltime);
2759*7c478bd9Sstevel@tonic-gate 	error = cond_wait_common(cvp, mp, &reltime);
2760*7c478bd9Sstevel@tonic-gate 	if (error == ETIME && clock_id == CLOCK_HIGHRES) {
2761*7c478bd9Sstevel@tonic-gate 		/*
2762*7c478bd9Sstevel@tonic-gate 		 * Don't return ETIME if we didn't really get a timeout.
2763*7c478bd9Sstevel@tonic-gate 		 * This can happen if we return because someone resets
2764*7c478bd9Sstevel@tonic-gate 		 * the system clock.  Just return zero in this case,
2765*7c478bd9Sstevel@tonic-gate 		 * giving a spurious wakeup but not a timeout.
2766*7c478bd9Sstevel@tonic-gate 		 */
2767*7c478bd9Sstevel@tonic-gate 		if ((hrtime_t)(uint32_t)abstime->tv_sec * NANOSEC +
2768*7c478bd9Sstevel@tonic-gate 		    abstime->tv_nsec > gethrtime())
2769*7c478bd9Sstevel@tonic-gate 			error = 0;
2770*7c478bd9Sstevel@tonic-gate 	}
2771*7c478bd9Sstevel@tonic-gate 	return (error);
2772*7c478bd9Sstevel@tonic-gate }
2773*7c478bd9Sstevel@tonic-gate 
2774*7c478bd9Sstevel@tonic-gate int
2775*7c478bd9Sstevel@tonic-gate cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
2776*7c478bd9Sstevel@tonic-gate {
2777*7c478bd9Sstevel@tonic-gate 	int error;
2778*7c478bd9Sstevel@tonic-gate 
2779*7c478bd9Sstevel@tonic-gate 	_cancelon();
2780*7c478bd9Sstevel@tonic-gate 	error = _cond_timedwait(cvp, mp, abstime);
2781*7c478bd9Sstevel@tonic-gate 	if (error == EINTR)
2782*7c478bd9Sstevel@tonic-gate 		_canceloff();
2783*7c478bd9Sstevel@tonic-gate 	else
2784*7c478bd9Sstevel@tonic-gate 		_canceloff_nocancel();
2785*7c478bd9Sstevel@tonic-gate 	return (error);
2786*7c478bd9Sstevel@tonic-gate }
2787*7c478bd9Sstevel@tonic-gate 
2788*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_timedwait = _pthread_cond_timedwait
2789*7c478bd9Sstevel@tonic-gate int
2790*7c478bd9Sstevel@tonic-gate _pthread_cond_timedwait(cond_t *cvp, mutex_t *mp, const timespec_t *abstime)
2791*7c478bd9Sstevel@tonic-gate {
2792*7c478bd9Sstevel@tonic-gate 	int error;
2793*7c478bd9Sstevel@tonic-gate 
2794*7c478bd9Sstevel@tonic-gate 	error = cond_timedwait(cvp, mp, abstime);
2795*7c478bd9Sstevel@tonic-gate 	if (error == ETIME)
2796*7c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
2797*7c478bd9Sstevel@tonic-gate 	else if (error == EINTR)
2798*7c478bd9Sstevel@tonic-gate 		error = 0;
2799*7c478bd9Sstevel@tonic-gate 	return (error);
2800*7c478bd9Sstevel@tonic-gate }
2801*7c478bd9Sstevel@tonic-gate 
2802*7c478bd9Sstevel@tonic-gate /*
2803*7c478bd9Sstevel@tonic-gate  * cond_reltimedwait() is a cancellation point but _cond_reltimedwait()
2804*7c478bd9Sstevel@tonic-gate  * is not.  System libraries call the non-cancellation version.
2805*7c478bd9Sstevel@tonic-gate  * It is expected that only applications call the cancellation version.
2806*7c478bd9Sstevel@tonic-gate  */
2807*7c478bd9Sstevel@tonic-gate int
2808*7c478bd9Sstevel@tonic-gate _cond_reltimedwait(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
2809*7c478bd9Sstevel@tonic-gate {
2810*7c478bd9Sstevel@tonic-gate 	timespec_t tslocal = *reltime;
2811*7c478bd9Sstevel@tonic-gate 
2812*7c478bd9Sstevel@tonic-gate 	return (cond_wait_common(cvp, mp, &tslocal));
2813*7c478bd9Sstevel@tonic-gate }
2814*7c478bd9Sstevel@tonic-gate 
2815*7c478bd9Sstevel@tonic-gate #pragma weak cond_reltimedwait = _cond_reltimedwait_cancel
2816*7c478bd9Sstevel@tonic-gate int
2817*7c478bd9Sstevel@tonic-gate _cond_reltimedwait_cancel(cond_t *cvp, mutex_t *mp, const timespec_t *reltime)
2818*7c478bd9Sstevel@tonic-gate {
2819*7c478bd9Sstevel@tonic-gate 	int error;
2820*7c478bd9Sstevel@tonic-gate 
2821*7c478bd9Sstevel@tonic-gate 	_cancelon();
2822*7c478bd9Sstevel@tonic-gate 	error = _cond_reltimedwait(cvp, mp, reltime);
2823*7c478bd9Sstevel@tonic-gate 	if (error == EINTR)
2824*7c478bd9Sstevel@tonic-gate 		_canceloff();
2825*7c478bd9Sstevel@tonic-gate 	else
2826*7c478bd9Sstevel@tonic-gate 		_canceloff_nocancel();
2827*7c478bd9Sstevel@tonic-gate 	return (error);
2828*7c478bd9Sstevel@tonic-gate }
2829*7c478bd9Sstevel@tonic-gate 
2830*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_reltimedwait_np = _pthread_cond_reltimedwait_np
2831*7c478bd9Sstevel@tonic-gate int
2832*7c478bd9Sstevel@tonic-gate _pthread_cond_reltimedwait_np(cond_t *cvp, mutex_t *mp,
2833*7c478bd9Sstevel@tonic-gate 	const timespec_t *reltime)
2834*7c478bd9Sstevel@tonic-gate {
2835*7c478bd9Sstevel@tonic-gate 	int error;
2836*7c478bd9Sstevel@tonic-gate 
2837*7c478bd9Sstevel@tonic-gate 	error = _cond_reltimedwait_cancel(cvp, mp, reltime);
2838*7c478bd9Sstevel@tonic-gate 	if (error == ETIME)
2839*7c478bd9Sstevel@tonic-gate 		error = ETIMEDOUT;
2840*7c478bd9Sstevel@tonic-gate 	else if (error == EINTR)
2841*7c478bd9Sstevel@tonic-gate 		error = 0;
2842*7c478bd9Sstevel@tonic-gate 	return (error);
2843*7c478bd9Sstevel@tonic-gate }
2844*7c478bd9Sstevel@tonic-gate 
2845*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_signal = cond_signal_internal
2846*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_signal = cond_signal_internal
2847*7c478bd9Sstevel@tonic-gate #pragma weak cond_signal = cond_signal_internal
2848*7c478bd9Sstevel@tonic-gate #pragma weak _cond_signal = cond_signal_internal
2849*7c478bd9Sstevel@tonic-gate int
2850*7c478bd9Sstevel@tonic-gate cond_signal_internal(cond_t *cvp)
2851*7c478bd9Sstevel@tonic-gate {
2852*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2853*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
2854*7c478bd9Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
2855*7c478bd9Sstevel@tonic-gate 	int error = 0;
2856*7c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
2857*7c478bd9Sstevel@tonic-gate 	mutex_t *mp;
2858*7c478bd9Sstevel@tonic-gate 	queue_head_t *mqp;
2859*7c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
2860*7c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
2861*7c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
2862*7c478bd9Sstevel@tonic-gate 	ulwp_t *next;
2863*7c478bd9Sstevel@tonic-gate 	ulwp_t **suspp = NULL;
2864*7c478bd9Sstevel@tonic-gate 	ulwp_t *susprev;
2865*7c478bd9Sstevel@tonic-gate 
2866*7c478bd9Sstevel@tonic-gate 	if (csp)
2867*7c478bd9Sstevel@tonic-gate 		tdb_incr(csp->cond_signal);
2868*7c478bd9Sstevel@tonic-gate 
2869*7c478bd9Sstevel@tonic-gate 	if (cvp->cond_waiters_kernel)	/* someone sleeping in the kernel? */
2870*7c478bd9Sstevel@tonic-gate 		error = __lwp_cond_signal(cvp);
2871*7c478bd9Sstevel@tonic-gate 
2872*7c478bd9Sstevel@tonic-gate 	if (!cvp->cond_waiters_user)	/* no one sleeping at user-level */
2873*7c478bd9Sstevel@tonic-gate 		return (error);
2874*7c478bd9Sstevel@tonic-gate 
2875*7c478bd9Sstevel@tonic-gate 	/*
2876*7c478bd9Sstevel@tonic-gate 	 * Move someone from the condvar sleep queue to the mutex sleep
2877*7c478bd9Sstevel@tonic-gate 	 * queue for the mutex that he will acquire on being waked up.
2878*7c478bd9Sstevel@tonic-gate 	 * We can do this only if we own the mutex he will acquire.
2879*7c478bd9Sstevel@tonic-gate 	 * If we do not own the mutex, or if his ul_cv_wake flag
2880*7c478bd9Sstevel@tonic-gate 	 * is set, just dequeue and unpark him.
2881*7c478bd9Sstevel@tonic-gate 	 */
2882*7c478bd9Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
2883*7c478bd9Sstevel@tonic-gate 	for (ulwpp = &qp->qh_head; (ulwp = *ulwpp) != NULL;
2884*7c478bd9Sstevel@tonic-gate 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
2885*7c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan == cvp) {
2886*7c478bd9Sstevel@tonic-gate 			if (!ulwp->ul_stop)
2887*7c478bd9Sstevel@tonic-gate 				break;
2888*7c478bd9Sstevel@tonic-gate 			/*
2889*7c478bd9Sstevel@tonic-gate 			 * Try not to dequeue a suspended thread.
2890*7c478bd9Sstevel@tonic-gate 			 * This mimics the old libthread's behavior.
2891*7c478bd9Sstevel@tonic-gate 			 */
2892*7c478bd9Sstevel@tonic-gate 			if (suspp == NULL) {
2893*7c478bd9Sstevel@tonic-gate 				suspp = ulwpp;
2894*7c478bd9Sstevel@tonic-gate 				susprev = prev;
2895*7c478bd9Sstevel@tonic-gate 			}
2896*7c478bd9Sstevel@tonic-gate 		}
2897*7c478bd9Sstevel@tonic-gate 	}
2898*7c478bd9Sstevel@tonic-gate 	if (ulwp == NULL && suspp != NULL) {
2899*7c478bd9Sstevel@tonic-gate 		ulwp = *(ulwpp = suspp);
2900*7c478bd9Sstevel@tonic-gate 		prev = susprev;
2901*7c478bd9Sstevel@tonic-gate 		suspp = NULL;
2902*7c478bd9Sstevel@tonic-gate 	}
2903*7c478bd9Sstevel@tonic-gate 	if (ulwp == NULL) {	/* no one on the sleep queue */
2904*7c478bd9Sstevel@tonic-gate 		cvp->cond_waiters_user = 0;
2905*7c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
2906*7c478bd9Sstevel@tonic-gate 		return (error);
2907*7c478bd9Sstevel@tonic-gate 	}
2908*7c478bd9Sstevel@tonic-gate 	/*
2909*7c478bd9Sstevel@tonic-gate 	 * Scan the remainder of the CV queue for another waiter.
2910*7c478bd9Sstevel@tonic-gate 	 */
2911*7c478bd9Sstevel@tonic-gate 	if (suspp != NULL) {
2912*7c478bd9Sstevel@tonic-gate 		next = *suspp;
2913*7c478bd9Sstevel@tonic-gate 	} else {
2914*7c478bd9Sstevel@tonic-gate 		for (next = ulwp->ul_link; next != NULL; next = next->ul_link)
2915*7c478bd9Sstevel@tonic-gate 			if (next->ul_wchan == cvp)
2916*7c478bd9Sstevel@tonic-gate 				break;
2917*7c478bd9Sstevel@tonic-gate 	}
2918*7c478bd9Sstevel@tonic-gate 	if (next == NULL)
2919*7c478bd9Sstevel@tonic-gate 		cvp->cond_waiters_user = 0;
2920*7c478bd9Sstevel@tonic-gate 
2921*7c478bd9Sstevel@tonic-gate 	/*
2922*7c478bd9Sstevel@tonic-gate 	 * Inform the thread that he was the recipient of a cond_signal().
2923*7c478bd9Sstevel@tonic-gate 	 * This lets him deal with cond_signal() and, concurrently,
2924*7c478bd9Sstevel@tonic-gate 	 * one or more of a cancellation, a UNIX signal, or a timeout.
2925*7c478bd9Sstevel@tonic-gate 	 * These latter conditions must not consume a cond_signal().
2926*7c478bd9Sstevel@tonic-gate 	 */
2927*7c478bd9Sstevel@tonic-gate 	ulwp->ul_signalled = 1;
2928*7c478bd9Sstevel@tonic-gate 
2929*7c478bd9Sstevel@tonic-gate 	/*
2930*7c478bd9Sstevel@tonic-gate 	 * Dequeue the waiter but leave his ul_sleepq non-NULL
2931*7c478bd9Sstevel@tonic-gate 	 * while we move him to the mutex queue so that he can
2932*7c478bd9Sstevel@tonic-gate 	 * deal properly with spurious wakeups.
2933*7c478bd9Sstevel@tonic-gate 	 */
2934*7c478bd9Sstevel@tonic-gate 	*ulwpp = ulwp->ul_link;
2935*7c478bd9Sstevel@tonic-gate 	if (qp->qh_tail == ulwp)
2936*7c478bd9Sstevel@tonic-gate 		qp->qh_tail = prev;
2937*7c478bd9Sstevel@tonic-gate 	qp->qh_qlen--;
2938*7c478bd9Sstevel@tonic-gate 	ulwp->ul_link = NULL;
2939*7c478bd9Sstevel@tonic-gate 
2940*7c478bd9Sstevel@tonic-gate 	mp = ulwp->ul_cvmutex;		/* the mutex he will acquire */
2941*7c478bd9Sstevel@tonic-gate 	ulwp->ul_cvmutex = NULL;
2942*7c478bd9Sstevel@tonic-gate 	ASSERT(mp != NULL);
2943*7c478bd9Sstevel@tonic-gate 
2944*7c478bd9Sstevel@tonic-gate 	if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
2945*7c478bd9Sstevel@tonic-gate 		lwpid_t lwpid = ulwp->ul_lwpid;
2946*7c478bd9Sstevel@tonic-gate 
2947*7c478bd9Sstevel@tonic-gate 		no_preempt(self);
2948*7c478bd9Sstevel@tonic-gate 		ulwp->ul_sleepq = NULL;
2949*7c478bd9Sstevel@tonic-gate 		ulwp->ul_wchan = NULL;
2950*7c478bd9Sstevel@tonic-gate 		ulwp->ul_cv_wake = 0;
2951*7c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
2952*7c478bd9Sstevel@tonic-gate 		(void) __lwp_unpark(lwpid);
2953*7c478bd9Sstevel@tonic-gate 		preempt(self);
2954*7c478bd9Sstevel@tonic-gate 	} else {
2955*7c478bd9Sstevel@tonic-gate 		mqp = queue_lock(mp, MX);
2956*7c478bd9Sstevel@tonic-gate 		enqueue(mqp, ulwp, mp, MX);
2957*7c478bd9Sstevel@tonic-gate 		mp->mutex_waiters = 1;
2958*7c478bd9Sstevel@tonic-gate 		queue_unlock(mqp);
2959*7c478bd9Sstevel@tonic-gate 		queue_unlock(qp);
2960*7c478bd9Sstevel@tonic-gate 	}
2961*7c478bd9Sstevel@tonic-gate 
2962*7c478bd9Sstevel@tonic-gate 	return (error);
2963*7c478bd9Sstevel@tonic-gate }
2964*7c478bd9Sstevel@tonic-gate 
2965*7c478bd9Sstevel@tonic-gate #define	MAXLWPS	128	/* max remembered lwpids before overflow */
2966*7c478bd9Sstevel@tonic-gate #define	NEWLWPS	2048	/* max remembered lwpids at first overflow */
2967*7c478bd9Sstevel@tonic-gate 
2968*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_broadcast = cond_broadcast_internal
2969*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_broadcast = cond_broadcast_internal
2970*7c478bd9Sstevel@tonic-gate #pragma weak cond_broadcast = cond_broadcast_internal
2971*7c478bd9Sstevel@tonic-gate #pragma weak _cond_broadcast = cond_broadcast_internal
2972*7c478bd9Sstevel@tonic-gate int
2973*7c478bd9Sstevel@tonic-gate cond_broadcast_internal(cond_t *cvp)
2974*7c478bd9Sstevel@tonic-gate {
2975*7c478bd9Sstevel@tonic-gate 	ulwp_t *self = curthread;
2976*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = self->ul_uberdata;
2977*7c478bd9Sstevel@tonic-gate 	tdb_cond_stats_t *csp = COND_STATS(cvp, udp);
2978*7c478bd9Sstevel@tonic-gate 	int error = 0;
2979*7c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
2980*7c478bd9Sstevel@tonic-gate 	mutex_t *mp;
2981*7c478bd9Sstevel@tonic-gate 	queue_head_t *mqp;
2982*7c478bd9Sstevel@tonic-gate 	mutex_t *mp_cache = NULL;
2983*7c478bd9Sstevel@tonic-gate 	queue_head_t *mqp_cache = NULL;
2984*7c478bd9Sstevel@tonic-gate 	ulwp_t **ulwpp;
2985*7c478bd9Sstevel@tonic-gate 	ulwp_t *ulwp;
2986*7c478bd9Sstevel@tonic-gate 	ulwp_t *prev = NULL;
2987*7c478bd9Sstevel@tonic-gate 	lwpid_t buffer[MAXLWPS];
2988*7c478bd9Sstevel@tonic-gate 	lwpid_t *lwpid = buffer;
2989*7c478bd9Sstevel@tonic-gate 	int nlwpid = 0;
2990*7c478bd9Sstevel@tonic-gate 	int maxlwps = MAXLWPS;
2991*7c478bd9Sstevel@tonic-gate 
2992*7c478bd9Sstevel@tonic-gate 	if (csp)
2993*7c478bd9Sstevel@tonic-gate 		tdb_incr(csp->cond_broadcast);
2994*7c478bd9Sstevel@tonic-gate 
2995*7c478bd9Sstevel@tonic-gate 	if (cvp->cond_waiters_kernel)	/* someone sleeping in the kernel? */
2996*7c478bd9Sstevel@tonic-gate 		error = __lwp_cond_broadcast(cvp);
2997*7c478bd9Sstevel@tonic-gate 
2998*7c478bd9Sstevel@tonic-gate 	if (!cvp->cond_waiters_user)	/* no one sleeping at user-level */
2999*7c478bd9Sstevel@tonic-gate 		return (error);
3000*7c478bd9Sstevel@tonic-gate 
3001*7c478bd9Sstevel@tonic-gate 	/*
3002*7c478bd9Sstevel@tonic-gate 	 * Move everyone from the condvar sleep queue to the mutex sleep
3003*7c478bd9Sstevel@tonic-gate 	 * queue for the mutex that they will acquire on being waked up.
3004*7c478bd9Sstevel@tonic-gate 	 * We can do this only if we own the mutex they will acquire.
3005*7c478bd9Sstevel@tonic-gate 	 * If we do not own the mutex, or if their ul_cv_wake flag
3006*7c478bd9Sstevel@tonic-gate 	 * is set, just dequeue and unpark them.
3007*7c478bd9Sstevel@tonic-gate 	 *
3008*7c478bd9Sstevel@tonic-gate 	 * We keep track of lwpids that are to be unparked in lwpid[].
3009*7c478bd9Sstevel@tonic-gate 	 * __lwp_unpark_all() is called to unpark all of them after
3010*7c478bd9Sstevel@tonic-gate 	 * they have been removed from the sleep queue and the sleep
3011*7c478bd9Sstevel@tonic-gate 	 * queue lock has been dropped.  If we run out of space in our
3012*7c478bd9Sstevel@tonic-gate 	 * on-stack buffer, we need to allocate more but we can't call
3013*7c478bd9Sstevel@tonic-gate 	 * lmalloc() because we are holding a queue lock when the overflow
3014*7c478bd9Sstevel@tonic-gate 	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
3015*7c478bd9Sstevel@tonic-gate 	 * either because the application may have allocated a small stack
3016*7c478bd9Sstevel@tonic-gate 	 * and we don't want to overrun the stack.  So we use the mmap()
3017*7c478bd9Sstevel@tonic-gate 	 * system call directly since that path acquires no locks.
3018*7c478bd9Sstevel@tonic-gate 	 */
3019*7c478bd9Sstevel@tonic-gate 	qp = queue_lock(cvp, CV);
3020*7c478bd9Sstevel@tonic-gate 	cvp->cond_waiters_user = 0;
3021*7c478bd9Sstevel@tonic-gate 	ulwpp = &qp->qh_head;
3022*7c478bd9Sstevel@tonic-gate 	while ((ulwp = *ulwpp) != NULL) {
3023*7c478bd9Sstevel@tonic-gate 
3024*7c478bd9Sstevel@tonic-gate 		if (ulwp->ul_wchan != cvp) {
3025*7c478bd9Sstevel@tonic-gate 			prev = ulwp;
3026*7c478bd9Sstevel@tonic-gate 			ulwpp = &ulwp->ul_link;
3027*7c478bd9Sstevel@tonic-gate 			continue;
3028*7c478bd9Sstevel@tonic-gate 		}
3029*7c478bd9Sstevel@tonic-gate 
3030*7c478bd9Sstevel@tonic-gate 		*ulwpp = ulwp->ul_link;
3031*7c478bd9Sstevel@tonic-gate 		if (qp->qh_tail == ulwp)
3032*7c478bd9Sstevel@tonic-gate 			qp->qh_tail = prev;
3033*7c478bd9Sstevel@tonic-gate 		qp->qh_qlen--;
3034*7c478bd9Sstevel@tonic-gate 		ulwp->ul_link = NULL;
3035*7c478bd9Sstevel@tonic-gate 
3036*7c478bd9Sstevel@tonic-gate 		mp = ulwp->ul_cvmutex;		/* his mutex */
3037*7c478bd9Sstevel@tonic-gate 		ulwp->ul_cvmutex = NULL;
3038*7c478bd9Sstevel@tonic-gate 		ASSERT(mp != NULL);
3039*7c478bd9Sstevel@tonic-gate 
3040*7c478bd9Sstevel@tonic-gate 		if (ulwp->ul_cv_wake || !MUTEX_OWNED(mp, self)) {
3041*7c478bd9Sstevel@tonic-gate 			ulwp->ul_sleepq = NULL;
3042*7c478bd9Sstevel@tonic-gate 			ulwp->ul_wchan = NULL;
3043*7c478bd9Sstevel@tonic-gate 			ulwp->ul_cv_wake = 0;
3044*7c478bd9Sstevel@tonic-gate 			if (nlwpid == maxlwps) {
3045*7c478bd9Sstevel@tonic-gate 				/*
3046*7c478bd9Sstevel@tonic-gate 				 * Allocate NEWLWPS ids on the first overflow.
3047*7c478bd9Sstevel@tonic-gate 				 * Double the allocation each time after that.
3048*7c478bd9Sstevel@tonic-gate 				 */
3049*7c478bd9Sstevel@tonic-gate 				int newlwps = (lwpid == buffer)? NEWLWPS :
3050*7c478bd9Sstevel@tonic-gate 						2 * maxlwps;
3051*7c478bd9Sstevel@tonic-gate 				void *vaddr = _private_mmap(NULL,
3052*7c478bd9Sstevel@tonic-gate 					newlwps * sizeof (lwpid_t),
3053*7c478bd9Sstevel@tonic-gate 					PROT_READ|PROT_WRITE,
3054*7c478bd9Sstevel@tonic-gate 					MAP_PRIVATE|MAP_ANON, -1, (off_t)0);
3055*7c478bd9Sstevel@tonic-gate 				if (vaddr == MAP_FAILED) {
3056*7c478bd9Sstevel@tonic-gate 					/*
3057*7c478bd9Sstevel@tonic-gate 					 * Let's hope this never happens.
3058*7c478bd9Sstevel@tonic-gate 					 * If it does, then we have a terrible
3059*7c478bd9Sstevel@tonic-gate 					 * thundering herd on our hands.
3060*7c478bd9Sstevel@tonic-gate 					 */
3061*7c478bd9Sstevel@tonic-gate 					(void) __lwp_unpark_all(lwpid, nlwpid);
3062*7c478bd9Sstevel@tonic-gate 					nlwpid = 0;
3063*7c478bd9Sstevel@tonic-gate 				} else {
3064*7c478bd9Sstevel@tonic-gate 					(void) _memcpy(vaddr, lwpid,
3065*7c478bd9Sstevel@tonic-gate 						maxlwps * sizeof (lwpid_t));
3066*7c478bd9Sstevel@tonic-gate 					if (lwpid != buffer)
3067*7c478bd9Sstevel@tonic-gate 						(void) _private_munmap(lwpid,
3068*7c478bd9Sstevel@tonic-gate 						    maxlwps * sizeof (lwpid_t));
3069*7c478bd9Sstevel@tonic-gate 					lwpid = vaddr;
3070*7c478bd9Sstevel@tonic-gate 					maxlwps = newlwps;
3071*7c478bd9Sstevel@tonic-gate 				}
3072*7c478bd9Sstevel@tonic-gate 			}
3073*7c478bd9Sstevel@tonic-gate 			lwpid[nlwpid++] = ulwp->ul_lwpid;
3074*7c478bd9Sstevel@tonic-gate 		} else {
3075*7c478bd9Sstevel@tonic-gate 			if (mp != mp_cache) {
3076*7c478bd9Sstevel@tonic-gate 				if (mqp_cache != NULL)
3077*7c478bd9Sstevel@tonic-gate 					queue_unlock(mqp_cache);
3078*7c478bd9Sstevel@tonic-gate 				mqp_cache = queue_lock(mp, MX);
3079*7c478bd9Sstevel@tonic-gate 				mp_cache = mp;
3080*7c478bd9Sstevel@tonic-gate 			}
3081*7c478bd9Sstevel@tonic-gate 			mqp = mqp_cache;
3082*7c478bd9Sstevel@tonic-gate 			enqueue(mqp, ulwp, mp, MX);
3083*7c478bd9Sstevel@tonic-gate 			mp->mutex_waiters = 1;
3084*7c478bd9Sstevel@tonic-gate 		}
3085*7c478bd9Sstevel@tonic-gate 	}
3086*7c478bd9Sstevel@tonic-gate 	if (mqp_cache != NULL)
3087*7c478bd9Sstevel@tonic-gate 		queue_unlock(mqp_cache);
3088*7c478bd9Sstevel@tonic-gate 	queue_unlock(qp);
3089*7c478bd9Sstevel@tonic-gate 	if (nlwpid) {
3090*7c478bd9Sstevel@tonic-gate 		if (nlwpid == 1)
3091*7c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark(lwpid[0]);
3092*7c478bd9Sstevel@tonic-gate 		else
3093*7c478bd9Sstevel@tonic-gate 			(void) __lwp_unpark_all(lwpid, nlwpid);
3094*7c478bd9Sstevel@tonic-gate 	}
3095*7c478bd9Sstevel@tonic-gate 	if (lwpid != buffer)
3096*7c478bd9Sstevel@tonic-gate 		(void) _private_munmap(lwpid, maxlwps * sizeof (lwpid_t));
3097*7c478bd9Sstevel@tonic-gate 
3098*7c478bd9Sstevel@tonic-gate 	return (error);
3099*7c478bd9Sstevel@tonic-gate }
3100*7c478bd9Sstevel@tonic-gate 
3101*7c478bd9Sstevel@tonic-gate #pragma weak pthread_cond_destroy = _cond_destroy
3102*7c478bd9Sstevel@tonic-gate #pragma weak _pthread_cond_destroy = _cond_destroy
3103*7c478bd9Sstevel@tonic-gate #pragma weak cond_destroy = _cond_destroy
3104*7c478bd9Sstevel@tonic-gate int
3105*7c478bd9Sstevel@tonic-gate _cond_destroy(cond_t *cvp)
3106*7c478bd9Sstevel@tonic-gate {
3107*7c478bd9Sstevel@tonic-gate 	cvp->cond_magic = 0;
3108*7c478bd9Sstevel@tonic-gate 	tdb_sync_obj_deregister(cvp);
3109*7c478bd9Sstevel@tonic-gate 	return (0);
3110*7c478bd9Sstevel@tonic-gate }
3111*7c478bd9Sstevel@tonic-gate 
3112*7c478bd9Sstevel@tonic-gate #if defined(THREAD_DEBUG)
3113*7c478bd9Sstevel@tonic-gate void
3114*7c478bd9Sstevel@tonic-gate assert_no_libc_locks_held(void)
3115*7c478bd9Sstevel@tonic-gate {
3116*7c478bd9Sstevel@tonic-gate 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
3117*7c478bd9Sstevel@tonic-gate }
3118*7c478bd9Sstevel@tonic-gate #endif
3119*7c478bd9Sstevel@tonic-gate 
3120*7c478bd9Sstevel@tonic-gate /* protected by link_lock */
3121*7c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin;
3122*7c478bd9Sstevel@tonic-gate uint64_t spin_lock_spin2;
3123*7c478bd9Sstevel@tonic-gate uint64_t spin_lock_sleep;
3124*7c478bd9Sstevel@tonic-gate uint64_t spin_lock_wakeup;
3125*7c478bd9Sstevel@tonic-gate 
3126*7c478bd9Sstevel@tonic-gate /*
3127*7c478bd9Sstevel@tonic-gate  * Record spin lock statistics.
3128*7c478bd9Sstevel@tonic-gate  * Called by a thread exiting itself in thrp_exit().
3129*7c478bd9Sstevel@tonic-gate  * Also called via atexit() from the thread calling
3130*7c478bd9Sstevel@tonic-gate  * exit() to do all the other threads as well.
3131*7c478bd9Sstevel@tonic-gate  */
3132*7c478bd9Sstevel@tonic-gate void
3133*7c478bd9Sstevel@tonic-gate record_spin_locks(ulwp_t *ulwp)
3134*7c478bd9Sstevel@tonic-gate {
3135*7c478bd9Sstevel@tonic-gate 	spin_lock_spin += ulwp->ul_spin_lock_spin;
3136*7c478bd9Sstevel@tonic-gate 	spin_lock_spin2 += ulwp->ul_spin_lock_spin2;
3137*7c478bd9Sstevel@tonic-gate 	spin_lock_sleep += ulwp->ul_spin_lock_sleep;
3138*7c478bd9Sstevel@tonic-gate 	spin_lock_wakeup += ulwp->ul_spin_lock_wakeup;
3139*7c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_spin = 0;
3140*7c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_spin2 = 0;
3141*7c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_sleep = 0;
3142*7c478bd9Sstevel@tonic-gate 	ulwp->ul_spin_lock_wakeup = 0;
3143*7c478bd9Sstevel@tonic-gate }
3144*7c478bd9Sstevel@tonic-gate 
3145*7c478bd9Sstevel@tonic-gate /*
3146*7c478bd9Sstevel@tonic-gate  * atexit function:  dump the queue statistics to stderr.
3147*7c478bd9Sstevel@tonic-gate  */
3148*7c478bd9Sstevel@tonic-gate #include <stdio.h>
3149*7c478bd9Sstevel@tonic-gate void
3150*7c478bd9Sstevel@tonic-gate dump_queue_statistics(void)
3151*7c478bd9Sstevel@tonic-gate {
3152*7c478bd9Sstevel@tonic-gate 	uberdata_t *udp = curthread->ul_uberdata;
3153*7c478bd9Sstevel@tonic-gate 	queue_head_t *qp;
3154*7c478bd9Sstevel@tonic-gate 	int qn;
3155*7c478bd9Sstevel@tonic-gate 	uint64_t spin_lock_total = 0;
3156*7c478bd9Sstevel@tonic-gate 
3157*7c478bd9Sstevel@tonic-gate 	if (udp->queue_head == NULL || thread_queue_dump == 0)
3158*7c478bd9Sstevel@tonic-gate 		return;
3159*7c478bd9Sstevel@tonic-gate 
3160*7c478bd9Sstevel@tonic-gate 	if (fprintf(stderr, "\n%5d mutex queues:\n", QHASHSIZE) < 0 ||
3161*7c478bd9Sstevel@tonic-gate 	    fprintf(stderr, "queue#   lockcount    max qlen\n") < 0)
3162*7c478bd9Sstevel@tonic-gate 		return;
3163*7c478bd9Sstevel@tonic-gate 	for (qn = 0, qp = udp->queue_head; qn < QHASHSIZE; qn++, qp++) {
3164*7c478bd9Sstevel@tonic-gate 		if (qp->qh_lockcount == 0)
3165*7c478bd9Sstevel@tonic-gate 			continue;
3166*7c478bd9Sstevel@tonic-gate 		spin_lock_total += qp->qh_lockcount;
3167*7c478bd9Sstevel@tonic-gate 		if (fprintf(stderr, "%5d %12llu%12u\n", qn,
3168*7c478bd9Sstevel@tonic-gate 			(u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0)
3169*7c478bd9Sstevel@tonic-gate 				return;
3170*7c478bd9Sstevel@tonic-gate 	}
3171*7c478bd9Sstevel@tonic-gate 
3172*7c478bd9Sstevel@tonic-gate 	if (fprintf(stderr, "\n%5d condvar queues:\n", QHASHSIZE) < 0 ||
3173*7c478bd9Sstevel@tonic-gate 	    fprintf(stderr, "queue#   lockcount    max qlen\n") < 0)
3174*7c478bd9Sstevel@tonic-gate 		return;
3175*7c478bd9Sstevel@tonic-gate 	for (qn = 0; qn < QHASHSIZE; qn++, qp++) {
3176*7c478bd9Sstevel@tonic-gate 		if (qp->qh_lockcount == 0)
3177*7c478bd9Sstevel@tonic-gate 			continue;
3178*7c478bd9Sstevel@tonic-gate 		spin_lock_total += qp->qh_lockcount;
3179*7c478bd9Sstevel@tonic-gate 		if (fprintf(stderr, "%5d %12llu%12u\n", qn,
3180*7c478bd9Sstevel@tonic-gate 			(u_longlong_t)qp->qh_lockcount, qp->qh_qmax) < 0)
3181*7c478bd9Sstevel@tonic-gate 				return;
3182*7c478bd9Sstevel@tonic-gate 	}
3183*7c478bd9Sstevel@tonic-gate 
3184*7c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "\n  spin_lock_total  = %10llu\n",
3185*7c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_total);
3186*7c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_spin   = %10llu\n",
3187*7c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_spin);
3188*7c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_spin2  = %10llu\n",
3189*7c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_spin2);
3190*7c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_sleep  = %10llu\n",
3191*7c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_sleep);
3192*7c478bd9Sstevel@tonic-gate 	(void) fprintf(stderr, "  spin_lock_wakeup = %10llu\n",
3193*7c478bd9Sstevel@tonic-gate 		(u_longlong_t)spin_lock_wakeup);
3194*7c478bd9Sstevel@tonic-gate }
3195