xref: /illumos-gate/usr/src/lib/libc/port/threads/synch.c (revision 48bbca81)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  * Copyright 2015, Joyent, Inc.
26  * Copyright (c) 2016 by Delphix. All rights reserved.
27  */
28 
29 #include "lint.h"
30 #include "thr_uberdata.h"
31 #include <sys/rtpriocntl.h>
32 #include <sys/sdt.h>
33 #include <atomic.h>
34 
35 #if defined(THREAD_DEBUG)
36 #define	INCR32(x)	(((x) != UINT32_MAX)? (x)++ : 0)
37 #define	INCR(x)		((x)++)
38 #define	DECR(x)		((x)--)
39 #define	MAXINCR(m, x)	((m < ++x)? (m = x) : 0)
40 #else
41 #define	INCR32(x)
42 #define	INCR(x)
43 #define	DECR(x)
44 #define	MAXINCR(m, x)
45 #endif
46 
47 /*
48  * This mutex is initialized to be held by lwp#1.
49  * It is used to block a thread that has returned from a mutex_lock()
50  * of a LOCK_PRIO_INHERIT mutex with an unrecoverable error.
51  */
52 mutex_t	stall_mutex = DEFAULTMUTEX;
53 
54 static int shared_mutex_held(mutex_t *);
55 static int mutex_queuelock_adaptive(mutex_t *);
56 static void mutex_wakeup_all(mutex_t *);
57 
58 /*
59  * Lock statistics support functions.
60  */
61 void
record_begin_hold(tdb_mutex_stats_t * msp)62 record_begin_hold(tdb_mutex_stats_t *msp)
63 {
64 	tdb_incr(msp->mutex_lock);
65 	msp->mutex_begin_hold = gethrtime();
66 }
67 
68 hrtime_t
record_hold_time(tdb_mutex_stats_t * msp)69 record_hold_time(tdb_mutex_stats_t *msp)
70 {
71 	hrtime_t now = gethrtime();
72 
73 	if (msp->mutex_begin_hold)
74 		msp->mutex_hold_time += now - msp->mutex_begin_hold;
75 	msp->mutex_begin_hold = 0;
76 	return (now);
77 }
78 
79 /*
80  * Called once at library initialization.
81  */
82 void
mutex_setup(void)83 mutex_setup(void)
84 {
85 	if (set_lock_byte(&stall_mutex.mutex_lockw))
86 		thr_panic("mutex_setup() cannot acquire stall_mutex");
87 	stall_mutex.mutex_owner = (uintptr_t)curthread;
88 }
89 
90 /*
91  * The default spin count of 1000 is experimentally determined.
92  * On sun4u machines with any number of processors it could be raised
93  * to 10,000 but that (experimentally) makes almost no difference.
94  * The environment variable:
95  *	_THREAD_ADAPTIVE_SPIN=count
96  * can be used to override and set the count in the range [0 .. 1,000,000].
97  */
98 int	thread_adaptive_spin = 1000;
99 uint_t	thread_max_spinners = 100;
100 int	thread_queue_verify = 0;
101 static	int	ncpus;
102 
103 /*
104  * Distinguish spinning for queue locks from spinning for regular locks.
105  * We try harder to acquire queue locks by spinning.
106  * The environment variable:
107  *	_THREAD_QUEUE_SPIN=count
108  * can be used to override and set the count in the range [0 .. 1,000,000].
109  */
110 int	thread_queue_spin = 10000;
111 
112 #define	ALL_ATTRIBUTES				\
113 	(LOCK_RECURSIVE | LOCK_ERRORCHECK |	\
114 	LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT |	\
115 	LOCK_ROBUST)
116 
117 /*
118  * 'type' can be one of USYNC_THREAD, USYNC_PROCESS, or USYNC_PROCESS_ROBUST,
119  * augmented by zero or more the flags:
120  *	LOCK_RECURSIVE
121  *	LOCK_ERRORCHECK
122  *	LOCK_PRIO_INHERIT
123  *	LOCK_PRIO_PROTECT
124  *	LOCK_ROBUST
125  */
126 #pragma weak _mutex_init = mutex_init
127 /* ARGSUSED2 */
128 int
mutex_init(mutex_t * mp,int type,void * arg)129 mutex_init(mutex_t *mp, int type, void *arg)
130 {
131 	int basetype = (type & ~ALL_ATTRIBUTES);
132 	const pcclass_t *pccp;
133 	int error = 0;
134 	int ceil;
135 
136 	if (basetype == USYNC_PROCESS_ROBUST) {
137 		/*
138 		 * USYNC_PROCESS_ROBUST is a deprecated historical type.
139 		 * We change it into (USYNC_PROCESS | LOCK_ROBUST) but
140 		 * retain the USYNC_PROCESS_ROBUST flag so we can return
141 		 * ELOCKUNMAPPED when necessary (only USYNC_PROCESS_ROBUST
142 		 * mutexes will ever draw ELOCKUNMAPPED).
143 		 */
144 		type |= (USYNC_PROCESS | LOCK_ROBUST);
145 		basetype = USYNC_PROCESS;
146 	}
147 
148 	if (type & LOCK_PRIO_PROTECT)
149 		pccp = get_info_by_policy(SCHED_FIFO);
150 	if ((basetype != USYNC_THREAD && basetype != USYNC_PROCESS) ||
151 	    (type & (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT))
152 	    == (LOCK_PRIO_INHERIT | LOCK_PRIO_PROTECT) ||
153 	    ((type & LOCK_PRIO_PROTECT) &&
154 	    ((ceil = *(int *)arg) < pccp->pcc_primin ||
155 	    ceil > pccp->pcc_primax))) {
156 		error = EINVAL;
157 	} else if (type & LOCK_ROBUST) {
158 		/*
159 		 * Callers of mutex_init() with the LOCK_ROBUST attribute
160 		 * are required to pass an initially all-zero mutex.
161 		 * Multiple calls to mutex_init() are allowed; all but
162 		 * the first return EBUSY.  A call to mutex_init() is
163 		 * allowed to make an inconsistent robust lock consistent
164 		 * (for historical usage, even though the proper interface
165 		 * for this is mutex_consistent()).  Note that we use
166 		 * atomic_or_16() to set the LOCK_INITED flag so as
167 		 * not to disturb surrounding bits (LOCK_OWNERDEAD, etc).
168 		 */
169 		if (!(mp->mutex_flag & LOCK_INITED)) {
170 			mp->mutex_type = (uint8_t)type;
171 			atomic_or_16(&mp->mutex_flag, LOCK_INITED);
172 			mp->mutex_magic = MUTEX_MAGIC;
173 		} else if (type != mp->mutex_type ||
174 		    ((type & LOCK_PRIO_PROTECT) && mp->mutex_ceiling != ceil)) {
175 			error = EINVAL;
176 		} else if (mutex_consistent(mp) != 0) {
177 			error = EBUSY;
178 		}
179 		/* register a process robust mutex with the kernel */
180 		if (basetype == USYNC_PROCESS)
181 			register_lock(mp);
182 	} else {
183 		(void) memset(mp, 0, sizeof (*mp));
184 		mp->mutex_type = (uint8_t)type;
185 		mp->mutex_flag = LOCK_INITED;
186 		mp->mutex_magic = MUTEX_MAGIC;
187 	}
188 
189 	if (error == 0 && (type & LOCK_PRIO_PROTECT)) {
190 		mp->mutex_ceiling = ceil;
191 	}
192 
193 	/*
194 	 * This should be at the beginning of the function,
195 	 * but for the sake of old broken applications that
196 	 * do not have proper alignment for their mutexes
197 	 * (and don't check the return code from mutex_init),
198 	 * we put it here, after initializing the mutex regardless.
199 	 */
200 	if (error == 0 &&
201 	    ((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
202 	    curthread->ul_misaligned == 0)
203 		error = EINVAL;
204 
205 	return (error);
206 }
207 
208 /*
209  * Delete mp from list of ceiling mutexes owned by curthread.
210  * Return 1 if the head of the chain was updated.
211  */
212 int
_ceil_mylist_del(mutex_t * mp)213 _ceil_mylist_del(mutex_t *mp)
214 {
215 	ulwp_t *self = curthread;
216 	mxchain_t **mcpp;
217 	mxchain_t *mcp;
218 
219 	for (mcpp = &self->ul_mxchain;
220 	    (mcp = *mcpp) != NULL;
221 	    mcpp = &mcp->mxchain_next) {
222 		if (mcp->mxchain_mx == mp) {
223 			*mcpp = mcp->mxchain_next;
224 			lfree(mcp, sizeof (*mcp));
225 			return (mcpp == &self->ul_mxchain);
226 		}
227 	}
228 	return (0);
229 }
230 
231 /*
232  * Add mp to the list of ceiling mutexes owned by curthread.
233  * Return ENOMEM if no memory could be allocated.
234  */
235 int
_ceil_mylist_add(mutex_t * mp)236 _ceil_mylist_add(mutex_t *mp)
237 {
238 	ulwp_t *self = curthread;
239 	mxchain_t *mcp;
240 
241 	if ((mcp = lmalloc(sizeof (*mcp))) == NULL)
242 		return (ENOMEM);
243 	mcp->mxchain_mx = mp;
244 	mcp->mxchain_next = self->ul_mxchain;
245 	self->ul_mxchain = mcp;
246 	return (0);
247 }
248 
249 /*
250  * Helper function for _ceil_prio_inherit() and _ceil_prio_waive(), below.
251  */
252 static void
set_rt_priority(ulwp_t * self,int prio)253 set_rt_priority(ulwp_t *self, int prio)
254 {
255 	pcparms_t pcparm;
256 
257 	pcparm.pc_cid = self->ul_rtclassid;
258 	((rtparms_t *)pcparm.pc_clparms)->rt_tqnsecs = RT_NOCHANGE;
259 	((rtparms_t *)pcparm.pc_clparms)->rt_pri = prio;
260 	(void) priocntl(P_LWPID, self->ul_lwpid, PC_SETPARMS, &pcparm);
261 }
262 
263 /*
264  * Inherit priority from ceiling.
265  * This changes the effective priority, not the assigned priority.
266  */
267 void
_ceil_prio_inherit(int prio)268 _ceil_prio_inherit(int prio)
269 {
270 	ulwp_t *self = curthread;
271 
272 	self->ul_epri = prio;
273 	set_rt_priority(self, prio);
274 }
275 
276 /*
277  * Waive inherited ceiling priority.  Inherit from head of owned ceiling locks
278  * if holding at least one ceiling lock.  If no ceiling locks are held at this
279  * point, disinherit completely, reverting back to assigned priority.
280  */
281 void
_ceil_prio_waive(void)282 _ceil_prio_waive(void)
283 {
284 	ulwp_t *self = curthread;
285 	mxchain_t *mcp = self->ul_mxchain;
286 	int prio;
287 
288 	if (mcp == NULL) {
289 		prio = self->ul_pri;
290 		self->ul_epri = 0;
291 	} else {
292 		prio = mcp->mxchain_mx->mutex_ceiling;
293 		self->ul_epri = prio;
294 	}
295 	set_rt_priority(self, prio);
296 }
297 
298 /*
299  * Clear the lock byte.  Retain the waiters byte and the spinners byte.
300  * Return the old value of the lock word.
301  */
302 static uint32_t
clear_lockbyte(volatile uint32_t * lockword)303 clear_lockbyte(volatile uint32_t *lockword)
304 {
305 	uint32_t old;
306 	uint32_t new;
307 
308 	do {
309 		old = *lockword;
310 		new = old & ~LOCKMASK;
311 	} while (atomic_cas_32(lockword, old, new) != old);
312 
313 	return (old);
314 }
315 
316 /*
317  * Same as clear_lockbyte(), but operates on mutex_lockword64.
318  * The mutex_ownerpid field is cleared along with the lock byte.
319  */
320 static uint64_t
clear_lockbyte64(volatile uint64_t * lockword64)321 clear_lockbyte64(volatile uint64_t *lockword64)
322 {
323 	uint64_t old;
324 	uint64_t new;
325 
326 	do {
327 		old = *lockword64;
328 		new = old & ~LOCKMASK64;
329 	} while (atomic_cas_64(lockword64, old, new) != old);
330 
331 	return (old);
332 }
333 
334 /*
335  * Similar to set_lock_byte(), which only tries to set the lock byte.
336  * Here, we attempt to set the lock byte AND the mutex_ownerpid, keeping
337  * the remaining bytes constant.  This atomic operation is required for the
338  * correctness of process-shared robust locks, otherwise there would be
339  * a window or vulnerability in which the lock byte had been set but the
340  * mutex_ownerpid had not yet been set.  If the process were to die in
341  * this window of vulnerability (due to some other thread calling exit()
342  * or the process receiving a fatal signal), the mutex would be left locked
343  * but without a process-ID to determine which process was holding the lock.
344  * The kernel would then be unable to mark the robust mutex as LOCK_OWNERDEAD
345  * when the process died.  For all other cases of process-shared locks, this
346  * operation is just a convenience, for the sake of common code.
347  *
348  * This operation requires process-shared robust locks to be properly
349  * aligned on an 8-byte boundary, at least on sparc machines, lest the
350  * operation incur an alignment fault.  This is automatic when locks
351  * are declared properly using the mutex_t or pthread_mutex_t data types
352  * and the application does not allocate dynamic memory on less than an
353  * 8-byte boundary.  See the 'horrible hack' comments below for cases
354  * dealing with such broken applications.
355  */
356 static int
set_lock_byte64(volatile uint64_t * lockword64,pid_t ownerpid)357 set_lock_byte64(volatile uint64_t *lockword64, pid_t ownerpid)
358 {
359 	uint64_t old;
360 	uint64_t new;
361 
362 	old = *lockword64 & ~LOCKMASK64;
363 	new = old | ((uint64_t)(uint_t)ownerpid << PIDSHIFT) | LOCKBYTE64;
364 	if (atomic_cas_64(lockword64, old, new) == old)
365 		return (LOCKCLEAR);
366 
367 	return (LOCKSET);
368 }
369 
370 /*
371  * Increment the spinners count in the mutex lock word.
372  * Return 0 on success.  Return -1 if the count would overflow.
373  */
374 static int
spinners_incr(volatile uint32_t * lockword,uint8_t max_spinners)375 spinners_incr(volatile uint32_t *lockword, uint8_t max_spinners)
376 {
377 	uint32_t old;
378 	uint32_t new;
379 
380 	do {
381 		old = *lockword;
382 		if (((old & SPINNERMASK) >> SPINNERSHIFT) >= max_spinners)
383 			return (-1);
384 		new = old + (1 << SPINNERSHIFT);
385 	} while (atomic_cas_32(lockword, old, new) != old);
386 
387 	return (0);
388 }
389 
390 /*
391  * Decrement the spinners count in the mutex lock word.
392  * Return the new value of the lock word.
393  */
394 static uint32_t
spinners_decr(volatile uint32_t * lockword)395 spinners_decr(volatile uint32_t *lockword)
396 {
397 	uint32_t old;
398 	uint32_t new;
399 
400 	do {
401 		new = old = *lockword;
402 		if (new & SPINNERMASK)
403 			new -= (1 << SPINNERSHIFT);
404 	} while (atomic_cas_32(lockword, old, new) != old);
405 
406 	return (new);
407 }
408 
409 /*
410  * Non-preemptive spin locks.  Used by queue_lock().
411  * No lock statistics are gathered for these locks.
412  * No DTrace probes are provided for these locks.
413  */
414 void
spin_lock_set(mutex_t * mp)415 spin_lock_set(mutex_t *mp)
416 {
417 	ulwp_t *self = curthread;
418 
419 	no_preempt(self);
420 	if (set_lock_byte(&mp->mutex_lockw) == 0) {
421 		mp->mutex_owner = (uintptr_t)self;
422 		return;
423 	}
424 	/*
425 	 * Spin for a while, attempting to acquire the lock.
426 	 */
427 	INCR32(self->ul_spin_lock_spin);
428 	if (mutex_queuelock_adaptive(mp) == 0 ||
429 	    set_lock_byte(&mp->mutex_lockw) == 0) {
430 		mp->mutex_owner = (uintptr_t)self;
431 		return;
432 	}
433 	/*
434 	 * Try harder if we were previously at a no premption level.
435 	 */
436 	if (self->ul_preempt > 1) {
437 		INCR32(self->ul_spin_lock_spin2);
438 		if (mutex_queuelock_adaptive(mp) == 0 ||
439 		    set_lock_byte(&mp->mutex_lockw) == 0) {
440 			mp->mutex_owner = (uintptr_t)self;
441 			return;
442 		}
443 	}
444 	/*
445 	 * Give up and block in the kernel for the mutex.
446 	 */
447 	INCR32(self->ul_spin_lock_sleep);
448 	(void) ___lwp_mutex_timedlock(mp, NULL, self);
449 }
450 
451 void
spin_lock_clear(mutex_t * mp)452 spin_lock_clear(mutex_t *mp)
453 {
454 	ulwp_t *self = curthread;
455 
456 	mp->mutex_owner = 0;
457 	if (atomic_swap_32(&mp->mutex_lockword, 0) & WAITERMASK) {
458 		(void) ___lwp_mutex_wakeup(mp, 0);
459 		INCR32(self->ul_spin_lock_wakeup);
460 	}
461 	preempt(self);
462 }
463 
464 /*
465  * Allocate the sleep queue hash table.
466  */
467 void
queue_alloc(void)468 queue_alloc(void)
469 {
470 	ulwp_t *self = curthread;
471 	uberdata_t *udp = self->ul_uberdata;
472 	queue_head_t *qp;
473 	void *data;
474 	int i;
475 
476 	/*
477 	 * No locks are needed; we call here only when single-threaded.
478 	 */
479 	ASSERT(self == udp->ulwp_one);
480 	ASSERT(!udp->uberflags.uf_mt);
481 	if ((data = mmap(NULL, 2 * QHASHSIZE * sizeof (queue_head_t),
482 	    PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, (off_t)0))
483 	    == MAP_FAILED)
484 		thr_panic("cannot allocate thread queue_head table");
485 	udp->queue_head = qp = (queue_head_t *)data;
486 	for (i = 0; i < 2 * QHASHSIZE; qp++, i++) {
487 		qp->qh_type = (i < QHASHSIZE)? MX : CV;
488 		qp->qh_lock.mutex_flag = LOCK_INITED;
489 		qp->qh_lock.mutex_magic = MUTEX_MAGIC;
490 		qp->qh_hlist = &qp->qh_def_root;
491 #if defined(THREAD_DEBUG)
492 		qp->qh_hlen = 1;
493 		qp->qh_hmax = 1;
494 #endif
495 	}
496 }
497 
498 #if defined(THREAD_DEBUG)
499 
500 /*
501  * Debugging: verify correctness of a sleep queue.
502  */
503 void
QVERIFY(queue_head_t * qp)504 QVERIFY(queue_head_t *qp)
505 {
506 	ulwp_t *self = curthread;
507 	uberdata_t *udp = self->ul_uberdata;
508 	queue_root_t *qrp;
509 	ulwp_t *ulwp;
510 	ulwp_t *prev;
511 	uint_t index;
512 	uint32_t cnt;
513 	char qtype;
514 	void *wchan;
515 
516 	ASSERT(qp >= udp->queue_head && (qp - udp->queue_head) < 2 * QHASHSIZE);
517 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
518 	for (cnt = 0, qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next) {
519 		cnt++;
520 		ASSERT((qrp->qr_head != NULL && qrp->qr_tail != NULL) ||
521 		    (qrp->qr_head == NULL && qrp->qr_tail == NULL));
522 	}
523 	ASSERT(qp->qh_hlen == cnt && qp->qh_hmax >= cnt);
524 	qtype = ((qp - udp->queue_head) < QHASHSIZE)? MX : CV;
525 	ASSERT(qp->qh_type == qtype);
526 	if (!thread_queue_verify)
527 		return;
528 	/* real expensive stuff, only for _THREAD_QUEUE_VERIFY */
529 	for (cnt = 0, qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next) {
530 		for (prev = NULL, ulwp = qrp->qr_head; ulwp != NULL;
531 		    prev = ulwp, ulwp = ulwp->ul_link) {
532 			cnt++;
533 			if (ulwp->ul_writer)
534 				ASSERT(prev == NULL || prev->ul_writer);
535 			ASSERT(ulwp->ul_qtype == qtype);
536 			ASSERT(ulwp->ul_wchan != NULL);
537 			ASSERT(ulwp->ul_sleepq == qp);
538 			wchan = ulwp->ul_wchan;
539 			ASSERT(qrp->qr_wchan == wchan);
540 			index = QUEUE_HASH(wchan, qtype);
541 			ASSERT(&udp->queue_head[index] == qp);
542 		}
543 		ASSERT(qrp->qr_tail == prev);
544 	}
545 	ASSERT(qp->qh_qlen == cnt);
546 }
547 
548 #else	/* THREAD_DEBUG */
549 
550 #define	QVERIFY(qp)
551 
552 #endif	/* THREAD_DEBUG */
553 
554 /*
555  * Acquire a queue head.
556  */
557 queue_head_t *
queue_lock(void * wchan,int qtype)558 queue_lock(void *wchan, int qtype)
559 {
560 	uberdata_t *udp = curthread->ul_uberdata;
561 	queue_head_t *qp;
562 	queue_root_t *qrp;
563 
564 	ASSERT(qtype == MX || qtype == CV);
565 
566 	/*
567 	 * It is possible that we could be called while still single-threaded.
568 	 * If so, we call queue_alloc() to allocate the queue_head[] array.
569 	 */
570 	if ((qp = udp->queue_head) == NULL) {
571 		queue_alloc();
572 		qp = udp->queue_head;
573 	}
574 	qp += QUEUE_HASH(wchan, qtype);
575 	spin_lock_set(&qp->qh_lock);
576 	for (qrp = qp->qh_hlist; qrp != NULL; qrp = qrp->qr_next)
577 		if (qrp->qr_wchan == wchan)
578 			break;
579 	if (qrp == NULL && qp->qh_def_root.qr_head == NULL) {
580 		/* the default queue root is available; use it */
581 		qrp = &qp->qh_def_root;
582 		qrp->qr_wchan = wchan;
583 		ASSERT(qrp->qr_next == NULL);
584 		ASSERT(qrp->qr_tail == NULL &&
585 		    qrp->qr_rtcount == 0 && qrp->qr_qlen == 0);
586 	}
587 	qp->qh_wchan = wchan;	/* valid until queue_unlock() is called */
588 	qp->qh_root = qrp;	/* valid until queue_unlock() is called */
589 	INCR32(qp->qh_lockcount);
590 	QVERIFY(qp);
591 	return (qp);
592 }
593 
594 /*
595  * Release a queue head.
596  */
597 void
queue_unlock(queue_head_t * qp)598 queue_unlock(queue_head_t *qp)
599 {
600 	QVERIFY(qp);
601 	spin_lock_clear(&qp->qh_lock);
602 }
603 
604 /*
605  * For rwlock queueing, we must queue writers ahead of readers of the
606  * same priority.  We do this by making writers appear to have a half
607  * point higher priority for purposes of priority comparisons below.
608  */
609 #define	CMP_PRIO(ulwp)	((real_priority(ulwp) << 1) + (ulwp)->ul_writer)
610 
611 void
enqueue(queue_head_t * qp,ulwp_t * ulwp,int force_fifo)612 enqueue(queue_head_t *qp, ulwp_t *ulwp, int force_fifo)
613 {
614 	queue_root_t *qrp;
615 	ulwp_t **ulwpp;
616 	ulwp_t *next;
617 	int pri = CMP_PRIO(ulwp);
618 
619 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
620 	ASSERT(ulwp->ul_sleepq != qp);
621 
622 	if ((qrp = qp->qh_root) == NULL) {
623 		/* use the thread's queue root for the linkage */
624 		qrp = &ulwp->ul_queue_root;
625 		qrp->qr_next = qp->qh_hlist;
626 		qrp->qr_prev = NULL;
627 		qrp->qr_head = NULL;
628 		qrp->qr_tail = NULL;
629 		qrp->qr_wchan = qp->qh_wchan;
630 		qrp->qr_rtcount = 0;
631 		qrp->qr_qlen = 0;
632 		qrp->qr_qmax = 0;
633 		qp->qh_hlist->qr_prev = qrp;
634 		qp->qh_hlist = qrp;
635 		qp->qh_root = qrp;
636 		MAXINCR(qp->qh_hmax, qp->qh_hlen);
637 	}
638 
639 	/*
640 	 * LIFO queue ordering is unfair and can lead to starvation,
641 	 * but it gives better performance for heavily contended locks.
642 	 * We use thread_queue_fifo (range is 0..8) to determine
643 	 * the frequency of FIFO vs LIFO queuing:
644 	 *	0 : every 256th time	(almost always LIFO)
645 	 *	1 : every 128th time
646 	 *	2 : every 64th  time
647 	 *	3 : every 32nd  time
648 	 *	4 : every 16th  time	(the default value, mostly LIFO)
649 	 *	5 : every 8th   time
650 	 *	6 : every 4th   time
651 	 *	7 : every 2nd   time
652 	 *	8 : every time		(never LIFO, always FIFO)
653 	 * Note that there is always some degree of FIFO ordering.
654 	 * This breaks live lock conditions that occur in applications
655 	 * that are written assuming (incorrectly) that threads acquire
656 	 * locks fairly, that is, in roughly round-robin order.
657 	 * In any event, the queue is maintained in kernel priority order.
658 	 *
659 	 * If force_fifo is non-zero, fifo queueing is forced.
660 	 * SUSV3 requires this for semaphores.
661 	 */
662 	if (qrp->qr_head == NULL) {
663 		/*
664 		 * The queue is empty.  LIFO/FIFO doesn't matter.
665 		 */
666 		ASSERT(qrp->qr_tail == NULL);
667 		ulwpp = &qrp->qr_head;
668 	} else if (force_fifo |
669 	    (((++qp->qh_qcnt << curthread->ul_queue_fifo) & 0xff) == 0)) {
670 		/*
671 		 * Enqueue after the last thread whose priority is greater
672 		 * than or equal to the priority of the thread being queued.
673 		 * Attempt first to go directly onto the tail of the queue.
674 		 */
675 		if (pri <= CMP_PRIO(qrp->qr_tail))
676 			ulwpp = &qrp->qr_tail->ul_link;
677 		else {
678 			for (ulwpp = &qrp->qr_head; (next = *ulwpp) != NULL;
679 			    ulwpp = &next->ul_link)
680 				if (pri > CMP_PRIO(next))
681 					break;
682 		}
683 	} else {
684 		/*
685 		 * Enqueue before the first thread whose priority is less
686 		 * than or equal to the priority of the thread being queued.
687 		 * Hopefully we can go directly onto the head of the queue.
688 		 */
689 		for (ulwpp = &qrp->qr_head; (next = *ulwpp) != NULL;
690 		    ulwpp = &next->ul_link)
691 			if (pri >= CMP_PRIO(next))
692 				break;
693 	}
694 	if ((ulwp->ul_link = *ulwpp) == NULL)
695 		qrp->qr_tail = ulwp;
696 	*ulwpp = ulwp;
697 
698 	ulwp->ul_sleepq = qp;
699 	ulwp->ul_wchan = qp->qh_wchan;
700 	ulwp->ul_qtype = qp->qh_type;
701 	if ((ulwp->ul_schedctl != NULL &&
702 	    ulwp->ul_schedctl->sc_cid == ulwp->ul_rtclassid) |
703 	    ulwp->ul_pilocks) {
704 		ulwp->ul_rtqueued = 1;
705 		qrp->qr_rtcount++;
706 	}
707 	MAXINCR(qrp->qr_qmax, qrp->qr_qlen);
708 	MAXINCR(qp->qh_qmax, qp->qh_qlen);
709 }
710 
711 /*
712  * Helper function for queue_slot() and queue_slot_rt().
713  * Try to find a non-suspended thread on the queue.
714  */
715 static ulwp_t **
queue_slot_runnable(ulwp_t ** ulwpp,ulwp_t ** prevp,int rt)716 queue_slot_runnable(ulwp_t **ulwpp, ulwp_t **prevp, int rt)
717 {
718 	ulwp_t *ulwp;
719 	ulwp_t **foundpp = NULL;
720 	int priority = -1;
721 	ulwp_t *prev;
722 	int tpri;
723 
724 	for (prev = NULL;
725 	    (ulwp = *ulwpp) != NULL;
726 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
727 		if (ulwp->ul_stop)	/* skip suspended threads */
728 			continue;
729 		tpri = rt? CMP_PRIO(ulwp) : 0;
730 		if (tpri > priority) {
731 			foundpp = ulwpp;
732 			*prevp = prev;
733 			priority = tpri;
734 			if (!rt)
735 				break;
736 		}
737 	}
738 	return (foundpp);
739 }
740 
741 /*
742  * For real-time, we search the entire queue because the dispatch
743  * (kernel) priorities may have changed since enqueueing.
744  */
745 static ulwp_t **
queue_slot_rt(ulwp_t ** ulwpp_org,ulwp_t ** prevp)746 queue_slot_rt(ulwp_t **ulwpp_org, ulwp_t **prevp)
747 {
748 	ulwp_t **ulwpp = ulwpp_org;
749 	ulwp_t *ulwp = *ulwpp;
750 	ulwp_t **foundpp = ulwpp;
751 	int priority = CMP_PRIO(ulwp);
752 	ulwp_t *prev;
753 	int tpri;
754 
755 	for (prev = ulwp, ulwpp = &ulwp->ul_link;
756 	    (ulwp = *ulwpp) != NULL;
757 	    prev = ulwp, ulwpp = &ulwp->ul_link) {
758 		tpri = CMP_PRIO(ulwp);
759 		if (tpri > priority) {
760 			foundpp = ulwpp;
761 			*prevp = prev;
762 			priority = tpri;
763 		}
764 	}
765 	ulwp = *foundpp;
766 
767 	/*
768 	 * Try not to return a suspended thread.
769 	 * This mimics the old libthread's behavior.
770 	 */
771 	if (ulwp->ul_stop &&
772 	    (ulwpp = queue_slot_runnable(ulwpp_org, prevp, 1)) != NULL) {
773 		foundpp = ulwpp;
774 		ulwp = *foundpp;
775 	}
776 	ulwp->ul_rt = 1;
777 	return (foundpp);
778 }
779 
780 ulwp_t **
queue_slot(queue_head_t * qp,ulwp_t ** prevp,int * more)781 queue_slot(queue_head_t *qp, ulwp_t **prevp, int *more)
782 {
783 	queue_root_t *qrp;
784 	ulwp_t **ulwpp;
785 	ulwp_t *ulwp;
786 	int rt;
787 
788 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
789 
790 	if ((qrp = qp->qh_root) == NULL || (ulwp = qrp->qr_head) == NULL) {
791 		*more = 0;
792 		return (NULL);		/* no lwps on the queue */
793 	}
794 	rt = (qrp->qr_rtcount != 0);
795 	*prevp = NULL;
796 	if (ulwp->ul_link == NULL) {	/* only one lwp on the queue */
797 		*more = 0;
798 		ulwp->ul_rt = rt;
799 		return (&qrp->qr_head);
800 	}
801 	*more = 1;
802 
803 	if (rt)		/* real-time queue */
804 		return (queue_slot_rt(&qrp->qr_head, prevp));
805 	/*
806 	 * Try not to return a suspended thread.
807 	 * This mimics the old libthread's behavior.
808 	 */
809 	if (ulwp->ul_stop &&
810 	    (ulwpp = queue_slot_runnable(&qrp->qr_head, prevp, 0)) != NULL) {
811 		ulwp = *ulwpp;
812 		ulwp->ul_rt = 0;
813 		return (ulwpp);
814 	}
815 	/*
816 	 * The common case; just pick the first thread on the queue.
817 	 */
818 	ulwp->ul_rt = 0;
819 	return (&qrp->qr_head);
820 }
821 
822 /*
823  * Common code for unlinking an lwp from a user-level sleep queue.
824  */
825 void
queue_unlink(queue_head_t * qp,ulwp_t ** ulwpp,ulwp_t * prev)826 queue_unlink(queue_head_t *qp, ulwp_t **ulwpp, ulwp_t *prev)
827 {
828 	queue_root_t *qrp = qp->qh_root;
829 	queue_root_t *nqrp;
830 	ulwp_t *ulwp = *ulwpp;
831 	ulwp_t *next;
832 
833 	ASSERT(MUTEX_OWNED(&qp->qh_lock, curthread));
834 	ASSERT(qp->qh_wchan != NULL && ulwp->ul_wchan == qp->qh_wchan);
835 
836 	DECR(qp->qh_qlen);
837 	DECR(qrp->qr_qlen);
838 	if (ulwp->ul_rtqueued) {
839 		ulwp->ul_rtqueued = 0;
840 		qrp->qr_rtcount--;
841 	}
842 	next = ulwp->ul_link;
843 	*ulwpp = next;
844 	ulwp->ul_link = NULL;
845 	if (qrp->qr_tail == ulwp)
846 		qrp->qr_tail = prev;
847 	if (qrp == &ulwp->ul_queue_root) {
848 		/*
849 		 * We can't continue to use the unlinked thread's
850 		 * queue root for the linkage.
851 		 */
852 		queue_root_t *qr_next = qrp->qr_next;
853 		queue_root_t *qr_prev = qrp->qr_prev;
854 
855 		if (qrp->qr_tail) {
856 			/* switch to using the last thread's queue root */
857 			ASSERT(qrp->qr_qlen != 0);
858 			nqrp = &qrp->qr_tail->ul_queue_root;
859 			*nqrp = *qrp;
860 			if (qr_next)
861 				qr_next->qr_prev = nqrp;
862 			if (qr_prev)
863 				qr_prev->qr_next = nqrp;
864 			else
865 				qp->qh_hlist = nqrp;
866 			qp->qh_root = nqrp;
867 		} else {
868 			/* empty queue root; just delete from the hash list */
869 			ASSERT(qrp->qr_qlen == 0);
870 			if (qr_next)
871 				qr_next->qr_prev = qr_prev;
872 			if (qr_prev)
873 				qr_prev->qr_next = qr_next;
874 			else
875 				qp->qh_hlist = qr_next;
876 			qp->qh_root = NULL;
877 			DECR(qp->qh_hlen);
878 		}
879 	}
880 }
881 
882 ulwp_t *
dequeue(queue_head_t * qp,int * more)883 dequeue(queue_head_t *qp, int *more)
884 {
885 	ulwp_t **ulwpp;
886 	ulwp_t *ulwp;
887 	ulwp_t *prev;
888 
889 	if ((ulwpp = queue_slot(qp, &prev, more)) == NULL)
890 		return (NULL);
891 	ulwp = *ulwpp;
892 	queue_unlink(qp, ulwpp, prev);
893 	ulwp->ul_sleepq = NULL;
894 	ulwp->ul_wchan = NULL;
895 	return (ulwp);
896 }
897 
898 /*
899  * Return a pointer to the highest priority thread sleeping on wchan.
900  */
901 ulwp_t *
queue_waiter(queue_head_t * qp)902 queue_waiter(queue_head_t *qp)
903 {
904 	ulwp_t **ulwpp;
905 	ulwp_t *prev;
906 	int more;
907 
908 	if ((ulwpp = queue_slot(qp, &prev, &more)) == NULL)
909 		return (NULL);
910 	return (*ulwpp);
911 }
912 
913 int
dequeue_self(queue_head_t * qp)914 dequeue_self(queue_head_t *qp)
915 {
916 	ulwp_t *self = curthread;
917 	queue_root_t *qrp;
918 	ulwp_t **ulwpp;
919 	ulwp_t *ulwp;
920 	ulwp_t *prev;
921 	int found = 0;
922 
923 	ASSERT(MUTEX_OWNED(&qp->qh_lock, self));
924 
925 	/* find self on the sleep queue */
926 	if ((qrp = qp->qh_root) != NULL) {
927 		for (prev = NULL, ulwpp = &qrp->qr_head;
928 		    (ulwp = *ulwpp) != NULL;
929 		    prev = ulwp, ulwpp = &ulwp->ul_link) {
930 			if (ulwp == self) {
931 				queue_unlink(qp, ulwpp, prev);
932 				self->ul_cvmutex = NULL;
933 				self->ul_sleepq = NULL;
934 				self->ul_wchan = NULL;
935 				found = 1;
936 				break;
937 			}
938 		}
939 	}
940 
941 	if (!found)
942 		thr_panic("dequeue_self(): curthread not found on queue");
943 
944 	return ((qrp = qp->qh_root) != NULL && qrp->qr_head != NULL);
945 }
946 
947 /*
948  * Called from call_user_handler() and _thrp_suspend() to take
949  * ourself off of our sleep queue so we can grab locks.
950  */
951 void
unsleep_self(void)952 unsleep_self(void)
953 {
954 	ulwp_t *self = curthread;
955 	queue_head_t *qp;
956 
957 	/*
958 	 * Calling enter_critical()/exit_critical() here would lead
959 	 * to recursion.  Just manipulate self->ul_critical directly.
960 	 */
961 	self->ul_critical++;
962 	while (self->ul_sleepq != NULL) {
963 		qp = queue_lock(self->ul_wchan, self->ul_qtype);
964 		/*
965 		 * We may have been moved from a CV queue to a
966 		 * mutex queue while we were attempting queue_lock().
967 		 * If so, just loop around and try again.
968 		 * dequeue_self() clears self->ul_sleepq.
969 		 */
970 		if (qp == self->ul_sleepq)
971 			(void) dequeue_self(qp);
972 		queue_unlock(qp);
973 	}
974 	self->ul_writer = 0;
975 	self->ul_critical--;
976 }
977 
978 /*
979  * Common code for calling the the ___lwp_mutex_timedlock() system call.
980  * Returns with mutex_owner and mutex_ownerpid set correctly.
981  */
982 static int
mutex_lock_kernel(mutex_t * mp,timespec_t * tsp,tdb_mutex_stats_t * msp)983 mutex_lock_kernel(mutex_t *mp, timespec_t *tsp, tdb_mutex_stats_t *msp)
984 {
985 	ulwp_t *self = curthread;
986 	uberdata_t *udp = self->ul_uberdata;
987 	int mtype = mp->mutex_type;
988 	hrtime_t begin_sleep;
989 	int acquired;
990 	int error;
991 
992 	self->ul_sp = stkptr();
993 	self->ul_wchan = mp;
994 	if (__td_event_report(self, TD_SLEEP, udp)) {
995 		self->ul_td_evbuf.eventnum = TD_SLEEP;
996 		self->ul_td_evbuf.eventdata = mp;
997 		tdb_event(TD_SLEEP, udp);
998 	}
999 	if (msp) {
1000 		tdb_incr(msp->mutex_sleep);
1001 		begin_sleep = gethrtime();
1002 	}
1003 
1004 	DTRACE_PROBE1(plockstat, mutex__block, mp);
1005 
1006 	for (;;) {
1007 		/*
1008 		 * A return value of EOWNERDEAD or ELOCKUNMAPPED
1009 		 * means we successfully acquired the lock.
1010 		 */
1011 		if ((error = ___lwp_mutex_timedlock(mp, tsp, self)) != 0 &&
1012 		    error != EOWNERDEAD && error != ELOCKUNMAPPED) {
1013 			acquired = 0;
1014 			break;
1015 		}
1016 
1017 		if (mtype & USYNC_PROCESS) {
1018 			/*
1019 			 * Defend against forkall().  We may be the child,
1020 			 * in which case we don't actually own the mutex.
1021 			 */
1022 			enter_critical(self);
1023 			if (mp->mutex_ownerpid == udp->pid) {
1024 				exit_critical(self);
1025 				acquired = 1;
1026 				break;
1027 			}
1028 			exit_critical(self);
1029 		} else {
1030 			acquired = 1;
1031 			break;
1032 		}
1033 	}
1034 
1035 	if (msp)
1036 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
1037 	self->ul_wchan = NULL;
1038 	self->ul_sp = 0;
1039 
1040 	if (acquired) {
1041 		ASSERT(mp->mutex_owner == (uintptr_t)self);
1042 		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
1043 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1044 	} else {
1045 		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
1046 		DTRACE_PROBE2(plockstat, mutex__error, mp, error);
1047 	}
1048 
1049 	return (error);
1050 }
1051 
1052 /*
1053  * Common code for calling the ___lwp_mutex_trylock() system call.
1054  * Returns with mutex_owner and mutex_ownerpid set correctly.
1055  */
1056 int
mutex_trylock_kernel(mutex_t * mp)1057 mutex_trylock_kernel(mutex_t *mp)
1058 {
1059 	ulwp_t *self = curthread;
1060 	uberdata_t *udp = self->ul_uberdata;
1061 	int mtype = mp->mutex_type;
1062 	int error;
1063 	int acquired;
1064 
1065 	for (;;) {
1066 		/*
1067 		 * A return value of EOWNERDEAD or ELOCKUNMAPPED
1068 		 * means we successfully acquired the lock.
1069 		 */
1070 		if ((error = ___lwp_mutex_trylock(mp, self)) != 0 &&
1071 		    error != EOWNERDEAD && error != ELOCKUNMAPPED) {
1072 			acquired = 0;
1073 			break;
1074 		}
1075 
1076 		if (mtype & USYNC_PROCESS) {
1077 			/*
1078 			 * Defend against forkall().  We may be the child,
1079 			 * in which case we don't actually own the mutex.
1080 			 */
1081 			enter_critical(self);
1082 			if (mp->mutex_ownerpid == udp->pid) {
1083 				exit_critical(self);
1084 				acquired = 1;
1085 				break;
1086 			}
1087 			exit_critical(self);
1088 		} else {
1089 			acquired = 1;
1090 			break;
1091 		}
1092 	}
1093 
1094 	if (acquired) {
1095 		ASSERT(mp->mutex_owner == (uintptr_t)self);
1096 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1097 	} else if (error != EBUSY) {
1098 		DTRACE_PROBE2(plockstat, mutex__error, mp, error);
1099 	}
1100 
1101 	return (error);
1102 }
1103 
1104 volatile sc_shared_t *
setup_schedctl(void)1105 setup_schedctl(void)
1106 {
1107 	ulwp_t *self = curthread;
1108 	volatile sc_shared_t *scp;
1109 	sc_shared_t *tmp;
1110 
1111 	if ((scp = self->ul_schedctl) == NULL && /* no shared state yet */
1112 	    !self->ul_vfork &&			/* not a child of vfork() */
1113 	    !self->ul_schedctl_called) {	/* haven't been called before */
1114 		enter_critical(self);
1115 		self->ul_schedctl_called = &self->ul_uberdata->uberflags;
1116 		if ((tmp = __schedctl()) != (sc_shared_t *)(-1))
1117 			self->ul_schedctl = scp = tmp;
1118 		exit_critical(self);
1119 	}
1120 	/*
1121 	 * Unless the call to setup_schedctl() is surrounded
1122 	 * by enter_critical()/exit_critical(), the address
1123 	 * we are returning could be invalid due to a forkall()
1124 	 * having occurred in another thread.
1125 	 */
1126 	return (scp);
1127 }
1128 
1129 /*
1130  * Interfaces from libsched, incorporated into libc.
1131  * libsched.so.1 is now a filter library onto libc.
1132  */
1133 #pragma weak schedctl_lookup = schedctl_init
1134 schedctl_t *
schedctl_init(void)1135 schedctl_init(void)
1136 {
1137 	volatile sc_shared_t *scp = setup_schedctl();
1138 	return ((scp == NULL)? NULL : (schedctl_t *)&scp->sc_preemptctl);
1139 }
1140 
1141 void
schedctl_exit(void)1142 schedctl_exit(void)
1143 {
1144 }
1145 
1146 /*
1147  * Contract private interface for java.
1148  * Set up the schedctl data if it doesn't exist yet.
1149  * Return a pointer to the pointer to the schedctl data.
1150  */
1151 volatile sc_shared_t *volatile *
_thr_schedctl(void)1152 _thr_schedctl(void)
1153 {
1154 	ulwp_t *self = curthread;
1155 	volatile sc_shared_t *volatile *ptr;
1156 
1157 	if (self->ul_vfork)
1158 		return (NULL);
1159 	if (*(ptr = &self->ul_schedctl) == NULL)
1160 		(void) setup_schedctl();
1161 	return (ptr);
1162 }
1163 
1164 /*
1165  * Block signals and attempt to block preemption.
1166  * no_preempt()/preempt() must be used in pairs but can be nested.
1167  */
1168 void
no_preempt(ulwp_t * self)1169 no_preempt(ulwp_t *self)
1170 {
1171 	volatile sc_shared_t *scp;
1172 
1173 	if (self->ul_preempt++ == 0) {
1174 		enter_critical(self);
1175 		if ((scp = self->ul_schedctl) != NULL ||
1176 		    (scp = setup_schedctl()) != NULL) {
1177 			/*
1178 			 * Save the pre-existing preempt value.
1179 			 */
1180 			self->ul_savpreempt = scp->sc_preemptctl.sc_nopreempt;
1181 			scp->sc_preemptctl.sc_nopreempt = 1;
1182 		}
1183 	}
1184 }
1185 
1186 /*
1187  * Undo the effects of no_preempt().
1188  */
1189 void
preempt(ulwp_t * self)1190 preempt(ulwp_t *self)
1191 {
1192 	volatile sc_shared_t *scp;
1193 
1194 	ASSERT(self->ul_preempt > 0);
1195 	if (--self->ul_preempt == 0) {
1196 		if ((scp = self->ul_schedctl) != NULL) {
1197 			/*
1198 			 * Restore the pre-existing preempt value.
1199 			 */
1200 			scp->sc_preemptctl.sc_nopreempt = self->ul_savpreempt;
1201 			if (scp->sc_preemptctl.sc_yield &&
1202 			    scp->sc_preemptctl.sc_nopreempt == 0) {
1203 				yield();
1204 				if (scp->sc_preemptctl.sc_yield) {
1205 					/*
1206 					 * Shouldn't happen.  This is either
1207 					 * a race condition or the thread
1208 					 * just entered the real-time class.
1209 					 */
1210 					yield();
1211 					scp->sc_preemptctl.sc_yield = 0;
1212 				}
1213 			}
1214 		}
1215 		exit_critical(self);
1216 	}
1217 }
1218 
1219 /*
1220  * If a call to preempt() would cause the current thread to yield or to
1221  * take deferred actions in exit_critical(), then unpark the specified
1222  * lwp so it can run while we delay.  Return the original lwpid if the
1223  * unpark was not performed, else return zero.  The tests are a repeat
1224  * of some of the tests in preempt(), above.  This is a statistical
1225  * optimization solely for cond_sleep_queue(), below.
1226  */
1227 static lwpid_t
preempt_unpark(ulwp_t * self,lwpid_t lwpid)1228 preempt_unpark(ulwp_t *self, lwpid_t lwpid)
1229 {
1230 	volatile sc_shared_t *scp = self->ul_schedctl;
1231 
1232 	ASSERT(self->ul_preempt == 1 && self->ul_critical > 0);
1233 	if ((scp != NULL && scp->sc_preemptctl.sc_yield) ||
1234 	    (self->ul_curplease && self->ul_critical == 1)) {
1235 		(void) __lwp_unpark(lwpid);
1236 		lwpid = 0;
1237 	}
1238 	return (lwpid);
1239 }
1240 
1241 /*
1242  * Spin for a while (if 'tryhard' is true), trying to grab the lock.
1243  * If this fails, return EBUSY and let the caller deal with it.
1244  * If this succeeds, return 0 with mutex_owner set to curthread.
1245  */
1246 static int
mutex_trylock_adaptive(mutex_t * mp,int tryhard)1247 mutex_trylock_adaptive(mutex_t *mp, int tryhard)
1248 {
1249 	ulwp_t *self = curthread;
1250 	int error = EBUSY;
1251 	ulwp_t *ulwp;
1252 	volatile sc_shared_t *scp;
1253 	volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw;
1254 	volatile uint64_t *ownerp = (volatile uint64_t *)&mp->mutex_owner;
1255 	uint32_t new_lockword;
1256 	int count = 0;
1257 	int max_count;
1258 	uint8_t max_spinners;
1259 
1260 	ASSERT(!(mp->mutex_type & USYNC_PROCESS));
1261 
1262 	if (MUTEX_OWNED(mp, self))
1263 		return (EBUSY);
1264 
1265 	enter_critical(self);
1266 
1267 	/* short-cut, not definitive (see below) */
1268 	if (mp->mutex_flag & LOCK_NOTRECOVERABLE) {
1269 		ASSERT(mp->mutex_type & LOCK_ROBUST);
1270 		error = ENOTRECOVERABLE;
1271 		goto done;
1272 	}
1273 
1274 	/*
1275 	 * Make one attempt to acquire the lock before
1276 	 * incurring the overhead of the spin loop.
1277 	 */
1278 	if (set_lock_byte(lockp) == 0) {
1279 		*ownerp = (uintptr_t)self;
1280 		error = 0;
1281 		goto done;
1282 	}
1283 	if (!tryhard)
1284 		goto done;
1285 	if (ncpus == 0)
1286 		ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
1287 	if ((max_spinners = self->ul_max_spinners) >= ncpus)
1288 		max_spinners = ncpus - 1;
1289 	max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0;
1290 	if (max_count == 0)
1291 		goto done;
1292 
1293 	/*
1294 	 * This spin loop is unfair to lwps that have already dropped into
1295 	 * the kernel to sleep.  They will starve on a highly-contended mutex.
1296 	 * This is just too bad.  The adaptive spin algorithm is intended
1297 	 * to allow programs with highly-contended locks (that is, broken
1298 	 * programs) to execute with reasonable speed despite their contention.
1299 	 * Being fair would reduce the speed of such programs and well-written
1300 	 * programs will not suffer in any case.
1301 	 */
1302 	if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1)
1303 		goto done;
1304 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
1305 	for (count = 1; ; count++) {
1306 		if (*lockp == 0 && set_lock_byte(lockp) == 0) {
1307 			*ownerp = (uintptr_t)self;
1308 			error = 0;
1309 			break;
1310 		}
1311 		if (count == max_count)
1312 			break;
1313 		SMT_PAUSE();
1314 		/*
1315 		 * Stop spinning if the mutex owner is not running on
1316 		 * a processor; it will not drop the lock any time soon
1317 		 * and we would just be wasting time to keep spinning.
1318 		 *
1319 		 * Note that we are looking at another thread (ulwp_t)
1320 		 * without ensuring that the other thread does not exit.
1321 		 * The scheme relies on ulwp_t structures never being
1322 		 * deallocated by the library (the library employs a free
1323 		 * list of ulwp_t structs that are reused when new threads
1324 		 * are created) and on schedctl shared memory never being
1325 		 * deallocated once created via __schedctl().
1326 		 *
1327 		 * Thus, the worst that can happen when the spinning thread
1328 		 * looks at the owner's schedctl data is that it is looking
1329 		 * at some other thread's schedctl data.  This almost never
1330 		 * happens and is benign when it does.
1331 		 */
1332 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
1333 		    ((scp = ulwp->ul_schedctl) == NULL ||
1334 		    scp->sc_state != SC_ONPROC))
1335 			break;
1336 	}
1337 	new_lockword = spinners_decr(&mp->mutex_lockword);
1338 	if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) {
1339 		/*
1340 		 * We haven't yet acquired the lock, the lock
1341 		 * is free, and there are no other spinners.
1342 		 * Make one final attempt to acquire the lock.
1343 		 *
1344 		 * This isn't strictly necessary since mutex_lock_queue()
1345 		 * (the next action this thread will take if it doesn't
1346 		 * acquire the lock here) makes one attempt to acquire
1347 		 * the lock before putting the thread to sleep.
1348 		 *
1349 		 * If the next action for this thread (on failure here)
1350 		 * were not to call mutex_lock_queue(), this would be
1351 		 * necessary for correctness, to avoid ending up with an
1352 		 * unheld mutex with waiters but no one to wake them up.
1353 		 */
1354 		if (set_lock_byte(lockp) == 0) {
1355 			*ownerp = (uintptr_t)self;
1356 			error = 0;
1357 		}
1358 		count++;
1359 	}
1360 
1361 done:
1362 	if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
1363 		ASSERT(mp->mutex_type & LOCK_ROBUST);
1364 		/*
1365 		 * We shouldn't own the mutex.
1366 		 * Just clear the lock; everyone has already been waked up.
1367 		 */
1368 		*ownerp = 0;
1369 		(void) clear_lockbyte(&mp->mutex_lockword);
1370 		error = ENOTRECOVERABLE;
1371 	}
1372 
1373 	exit_critical(self);
1374 
1375 	if (error) {
1376 		if (count) {
1377 			DTRACE_PROBE3(plockstat, mutex__spun, mp, 0, count);
1378 		}
1379 		if (error != EBUSY) {
1380 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
1381 		}
1382 	} else {
1383 		if (count) {
1384 			DTRACE_PROBE3(plockstat, mutex__spun, mp, 1, count);
1385 		}
1386 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
1387 		if (mp->mutex_flag & LOCK_OWNERDEAD) {
1388 			ASSERT(mp->mutex_type & LOCK_ROBUST);
1389 			error = EOWNERDEAD;
1390 		}
1391 	}
1392 
1393 	return (error);
1394 }
1395 
1396 /*
1397  * Same as mutex_trylock_adaptive(), except specifically for queue locks.
1398  * The owner field is not set here; the caller (spin_lock_set()) sets it.
1399  */
1400 static int
mutex_queuelock_adaptive(mutex_t * mp)1401 mutex_queuelock_adaptive(mutex_t *mp)
1402 {
1403 	ulwp_t *ulwp;
1404 	volatile sc_shared_t *scp;
1405 	volatile uint8_t *lockp;
1406 	volatile uint64_t *ownerp;
1407 	int count = curthread->ul_queue_spin;
1408 
1409 	ASSERT(mp->mutex_type == USYNC_THREAD);
1410 
1411 	if (count == 0)
1412 		return (EBUSY);
1413 
1414 	lockp = (volatile uint8_t *)&mp->mutex_lockw;
1415 	ownerp = (volatile uint64_t *)&mp->mutex_owner;
1416 	while (--count >= 0) {
1417 		if (*lockp == 0 && set_lock_byte(lockp) == 0)
1418 			return (0);
1419 		SMT_PAUSE();
1420 		if ((ulwp = (ulwp_t *)(uintptr_t)*ownerp) != NULL &&
1421 		    ((scp = ulwp->ul_schedctl) == NULL ||
1422 		    scp->sc_state != SC_ONPROC))
1423 			break;
1424 	}
1425 
1426 	return (EBUSY);
1427 }
1428 
1429 /*
1430  * Like mutex_trylock_adaptive(), but for process-shared mutexes.
1431  * Spin for a while (if 'tryhard' is true), trying to grab the lock.
1432  * If this fails, return EBUSY and let the caller deal with it.
1433  * If this succeeds, return 0 with mutex_owner set to curthread
1434  * and mutex_ownerpid set to the current pid.
1435  */
1436 static int
mutex_trylock_process(mutex_t * mp,int tryhard)1437 mutex_trylock_process(mutex_t *mp, int tryhard)
1438 {
1439 	ulwp_t *self = curthread;
1440 	uberdata_t *udp = self->ul_uberdata;
1441 	int error = EBUSY;
1442 	volatile uint64_t *lockp = (volatile uint64_t *)&mp->mutex_lockword64;
1443 	uint32_t new_lockword;
1444 	int count = 0;
1445 	int max_count;
1446 	uint8_t max_spinners;
1447 
1448 #if defined(__sparc) && !defined(_LP64)
1449 	/* horrible hack, necessary only on 32-bit sparc */
1450 	int fix_alignment_problem =
1451 	    (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
1452 	    self->ul_misaligned && !(mp->mutex_type & LOCK_ROBUST));
1453 #endif
1454 
1455 	ASSERT(mp->mutex_type & USYNC_PROCESS);
1456 
1457 	if (shared_mutex_held(mp))
1458 		return (EBUSY);
1459 
1460 	enter_critical(self);
1461 
1462 	/* short-cut, not definitive (see below) */
1463 	if (mp->mutex_flag & LOCK_NOTRECOVERABLE) {
1464 		ASSERT(mp->mutex_type & LOCK_ROBUST);
1465 		error = ENOTRECOVERABLE;
1466 		goto done;
1467 	}
1468 
1469 	/*
1470 	 * Make one attempt to acquire the lock before
1471 	 * incurring the overhead of the spin loop.
1472 	 */
1473 #if defined(__sparc) && !defined(_LP64)
1474 	/* horrible hack, necessary only on 32-bit sparc */
1475 	if (fix_alignment_problem) {
1476 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
1477 			mp->mutex_ownerpid = udp->pid;
1478 			mp->mutex_owner = (uintptr_t)self;
1479 			error = 0;
1480 			goto done;
1481 		}
1482 	} else
1483 #endif
1484 	if (set_lock_byte64(lockp, udp->pid) == 0) {
1485 		mp->mutex_owner = (uintptr_t)self;
1486 		/* mp->mutex_ownerpid was set by set_lock_byte64() */
1487 		error = 0;
1488 		goto done;
1489 	}
1490 	if (!tryhard)
1491 		goto done;
1492 	if (ncpus == 0)
1493 		ncpus = (int)_sysconf(_SC_NPROCESSORS_ONLN);
1494 	if ((max_spinners = self->ul_max_spinners) >= ncpus)
1495 		max_spinners = ncpus - 1;
1496 	max_count = (max_spinners != 0)? self->ul_adaptive_spin : 0;
1497 	if (max_count == 0)
1498 		goto done;
1499 
1500 	/*
1501 	 * This is a process-shared mutex.
1502 	 * We cannot know if the owner is running on a processor.
1503 	 * We just spin and hope that it is on a processor.
1504 	 */
1505 	if (spinners_incr(&mp->mutex_lockword, max_spinners) == -1)
1506 		goto done;
1507 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
1508 	for (count = 1; ; count++) {
1509 #if defined(__sparc) && !defined(_LP64)
1510 		/* horrible hack, necessary only on 32-bit sparc */
1511 		if (fix_alignment_problem) {
1512 			if ((*lockp & LOCKMASK64) == 0 &&
1513 			    set_lock_byte(&mp->mutex_lockw) == 0) {
1514 				mp->mutex_ownerpid = udp->pid;
1515 				mp->mutex_owner = (uintptr_t)self;
1516 				error = 0;
1517 				break;
1518 			}
1519 		} else
1520 #endif
1521 		if ((*lockp & LOCKMASK64) == 0 &&
1522 		    set_lock_byte64(lockp, udp->pid) == 0) {
1523 			mp->mutex_owner = (uintptr_t)self;
1524 			/* mp->mutex_ownerpid was set by set_lock_byte64() */
1525 			error = 0;
1526 			break;
1527 		}
1528 		if (count == max_count)
1529 			break;
1530 		SMT_PAUSE();
1531 	}
1532 	new_lockword = spinners_decr(&mp->mutex_lockword);
1533 	if (error && (new_lockword & (LOCKMASK | SPINNERMASK)) == 0) {
1534 		/*
1535 		 * We haven't yet acquired the lock, the lock
1536 		 * is free, and there are no other spinners.
1537 		 * Make one final attempt to acquire the lock.
1538 		 *
1539 		 * This isn't strictly necessary since mutex_lock_kernel()
1540 		 * (the next action this thread will take if it doesn't
1541 		 * acquire the lock here) makes one attempt to acquire
1542 		 * the lock before putting the thread to sleep.
1543 		 *
1544 		 * If the next action for this thread (on failure here)
1545 		 * were not to call mutex_lock_kernel(), this would be
1546 		 * necessary for correctness, to avoid ending up with an
1547 		 * unheld mutex with waiters but no one to wake them up.
1548 		 */
1549 #if defined(__sparc) && !defined(_LP64)
1550 		/* horrible hack, necessary only on 32-bit sparc */
1551 		if (fix_alignment_problem) {
1552 			if (set_lock_byte(&mp->mutex_lockw) == 0) {
1553 				mp->mutex_ownerpid = udp->pid;
1554 				mp->mutex_owner = (uintptr_t)self;
1555 				error = 0;
1556 			}
1557 		} else
1558 #endif
1559 		if (set_lock_byte64(lockp, udp->pid) == 0) {
1560 			mp->mutex_owner = (uintptr_t)self;
1561 			/* mp->mutex_ownerpid was set by set_lock_byte64() */
1562 			error = 0;
1563 		}
1564 		count++;
1565 	}
1566 
1567 done:
1568 	if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
1569 		ASSERT(mp->mutex_type & LOCK_ROBUST);
1570 		/*
1571 		 * We shouldn't own the mutex.
1572 		 * Just clear the lock; everyone has already been waked up.
1573 		 */
1574 		mp->mutex_owner = 0;
1575 		/* mp->mutex_ownerpid is cleared by clear_lockbyte64() */
1576 		(void) clear_lockbyte64(&mp->mutex_lockword64);
1577 		error = ENOTRECOVERABLE;
1578 	}
1579 
1580 	exit_critical(self);
1581 
1582 	if (error) {
1583 		if (count) {
1584 			DTRACE_PROBE3(plockstat, mutex__spun, mp, 0, count);
1585 		}
1586 		if (error != EBUSY) {
1587 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
1588 		}
1589 	} else {
1590 		if (count) {
1591 			DTRACE_PROBE3(plockstat, mutex__spun, mp, 1, count);
1592 		}
1593 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
1594 		if (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED)) {
1595 			ASSERT(mp->mutex_type & LOCK_ROBUST);
1596 			if (mp->mutex_flag & LOCK_OWNERDEAD)
1597 				error = EOWNERDEAD;
1598 			else if (mp->mutex_type & USYNC_PROCESS_ROBUST)
1599 				error = ELOCKUNMAPPED;
1600 			else
1601 				error = EOWNERDEAD;
1602 		}
1603 	}
1604 
1605 	return (error);
1606 }
1607 
1608 /*
1609  * Mutex wakeup code for releasing a USYNC_THREAD mutex.
1610  * Returns the lwpid of the thread that was dequeued, if any.
1611  * The caller of mutex_wakeup() must call __lwp_unpark(lwpid)
1612  * to wake up the specified lwp.
1613  */
1614 static lwpid_t
mutex_wakeup(mutex_t * mp)1615 mutex_wakeup(mutex_t *mp)
1616 {
1617 	lwpid_t lwpid = 0;
1618 	int more;
1619 	queue_head_t *qp;
1620 	ulwp_t *ulwp;
1621 
1622 	/*
1623 	 * Dequeue a waiter from the sleep queue.  Don't touch the mutex
1624 	 * waiters bit if no one was found on the queue because the mutex
1625 	 * might have been deallocated or reallocated for another purpose.
1626 	 */
1627 	qp = queue_lock(mp, MX);
1628 	if ((ulwp = dequeue(qp, &more)) != NULL) {
1629 		lwpid = ulwp->ul_lwpid;
1630 		mp->mutex_waiters = more;
1631 	}
1632 	queue_unlock(qp);
1633 	return (lwpid);
1634 }
1635 
1636 /*
1637  * Mutex wakeup code for releasing all waiters on a USYNC_THREAD mutex.
1638  */
1639 static void
mutex_wakeup_all(mutex_t * mp)1640 mutex_wakeup_all(mutex_t *mp)
1641 {
1642 	queue_head_t *qp;
1643 	queue_root_t *qrp;
1644 	int nlwpid = 0;
1645 	int maxlwps = MAXLWPS;
1646 	ulwp_t *ulwp;
1647 	lwpid_t buffer[MAXLWPS];
1648 	lwpid_t *lwpid = buffer;
1649 
1650 	/*
1651 	 * Walk the list of waiters and prepare to wake up all of them.
1652 	 * The waiters flag has already been cleared from the mutex.
1653 	 *
1654 	 * We keep track of lwpids that are to be unparked in lwpid[].
1655 	 * __lwp_unpark_all() is called to unpark all of them after
1656 	 * they have been removed from the sleep queue and the sleep
1657 	 * queue lock has been dropped.  If we run out of space in our
1658 	 * on-stack buffer, we need to allocate more but we can't call
1659 	 * lmalloc() because we are holding a queue lock when the overflow
1660 	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
1661 	 * either because the application may have allocated a small
1662 	 * stack and we don't want to overrun the stack.  So we call
1663 	 * alloc_lwpids() to allocate a bigger buffer using the mmap()
1664 	 * system call directly since that path acquires no locks.
1665 	 */
1666 	qp = queue_lock(mp, MX);
1667 	for (;;) {
1668 		if ((qrp = qp->qh_root) == NULL ||
1669 		    (ulwp = qrp->qr_head) == NULL)
1670 			break;
1671 		ASSERT(ulwp->ul_wchan == mp);
1672 		queue_unlink(qp, &qrp->qr_head, NULL);
1673 		ulwp->ul_sleepq = NULL;
1674 		ulwp->ul_wchan = NULL;
1675 		if (nlwpid == maxlwps)
1676 			lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
1677 		lwpid[nlwpid++] = ulwp->ul_lwpid;
1678 	}
1679 
1680 	if (nlwpid == 0) {
1681 		queue_unlock(qp);
1682 	} else {
1683 		mp->mutex_waiters = 0;
1684 		no_preempt(curthread);
1685 		queue_unlock(qp);
1686 		if (nlwpid == 1)
1687 			(void) __lwp_unpark(lwpid[0]);
1688 		else
1689 			(void) __lwp_unpark_all(lwpid, nlwpid);
1690 		preempt(curthread);
1691 	}
1692 
1693 	if (lwpid != buffer)
1694 		(void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t));
1695 }
1696 
1697 /*
1698  * Release a process-private mutex.
1699  * As an optimization, if there are waiters but there are also spinners
1700  * attempting to acquire the mutex, then don't bother waking up a waiter;
1701  * one of the spinners will acquire the mutex soon and it would be a waste
1702  * of resources to wake up some thread just to have it spin for a while
1703  * and then possibly go back to sleep.  See mutex_trylock_adaptive().
1704  */
1705 static lwpid_t
mutex_unlock_queue(mutex_t * mp,int release_all)1706 mutex_unlock_queue(mutex_t *mp, int release_all)
1707 {
1708 	ulwp_t *self = curthread;
1709 	lwpid_t lwpid = 0;
1710 	uint32_t old_lockword;
1711 
1712 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
1713 	sigoff(self);
1714 	mp->mutex_owner = 0;
1715 	old_lockword = clear_lockbyte(&mp->mutex_lockword);
1716 	if ((old_lockword & WAITERMASK) &&
1717 	    (release_all || (old_lockword & SPINNERMASK) == 0)) {
1718 		no_preempt(self);	/* ensure a prompt wakeup */
1719 		if (release_all)
1720 			mutex_wakeup_all(mp);
1721 		else
1722 			lwpid = mutex_wakeup(mp);
1723 		if (lwpid == 0)
1724 			preempt(self);
1725 	}
1726 	sigon(self);
1727 	return (lwpid);
1728 }
1729 
1730 /*
1731  * Like mutex_unlock_queue(), but for process-shared mutexes.
1732  */
1733 static void
mutex_unlock_process(mutex_t * mp,int release_all)1734 mutex_unlock_process(mutex_t *mp, int release_all)
1735 {
1736 	ulwp_t *self = curthread;
1737 	uint64_t old_lockword64;
1738 
1739 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
1740 	sigoff(self);
1741 	mp->mutex_owner = 0;
1742 #if defined(__sparc) && !defined(_LP64)
1743 	/* horrible hack, necessary only on 32-bit sparc */
1744 	if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
1745 	    self->ul_misaligned && !(mp->mutex_type & LOCK_ROBUST)) {
1746 		uint32_t old_lockword;
1747 		mp->mutex_ownerpid = 0;
1748 		old_lockword = clear_lockbyte(&mp->mutex_lockword);
1749 		if ((old_lockword & WAITERMASK) &&
1750 		    (release_all || (old_lockword & SPINNERMASK) == 0)) {
1751 			no_preempt(self);	/* ensure a prompt wakeup */
1752 			(void) ___lwp_mutex_wakeup(mp, release_all);
1753 			preempt(self);
1754 		}
1755 		sigon(self);
1756 		return;
1757 	}
1758 #endif
1759 	/* mp->mutex_ownerpid is cleared by clear_lockbyte64() */
1760 	old_lockword64 = clear_lockbyte64(&mp->mutex_lockword64);
1761 	if ((old_lockword64 & WAITERMASK64) &&
1762 	    (release_all || (old_lockword64 & SPINNERMASK64) == 0)) {
1763 		no_preempt(self);	/* ensure a prompt wakeup */
1764 		(void) ___lwp_mutex_wakeup(mp, release_all);
1765 		preempt(self);
1766 	}
1767 	sigon(self);
1768 }
1769 
1770 void
stall(void)1771 stall(void)
1772 {
1773 	for (;;)
1774 		(void) mutex_lock_kernel(&stall_mutex, NULL, NULL);
1775 }
1776 
1777 /*
1778  * Acquire a USYNC_THREAD mutex via user-level sleep queues.
1779  * We failed set_lock_byte(&mp->mutex_lockw) before coming here.
1780  * If successful, returns with mutex_owner set correctly.
1781  */
1782 int
mutex_lock_queue(ulwp_t * self,tdb_mutex_stats_t * msp,mutex_t * mp,timespec_t * tsp)1783 mutex_lock_queue(ulwp_t *self, tdb_mutex_stats_t *msp, mutex_t *mp,
1784 	timespec_t *tsp)
1785 {
1786 	uberdata_t *udp = curthread->ul_uberdata;
1787 	queue_head_t *qp;
1788 	hrtime_t begin_sleep;
1789 	int error = 0;
1790 
1791 	self->ul_sp = stkptr();
1792 	if (__td_event_report(self, TD_SLEEP, udp)) {
1793 		self->ul_wchan = mp;
1794 		self->ul_td_evbuf.eventnum = TD_SLEEP;
1795 		self->ul_td_evbuf.eventdata = mp;
1796 		tdb_event(TD_SLEEP, udp);
1797 	}
1798 	if (msp) {
1799 		tdb_incr(msp->mutex_sleep);
1800 		begin_sleep = gethrtime();
1801 	}
1802 
1803 	DTRACE_PROBE1(plockstat, mutex__block, mp);
1804 
1805 	/*
1806 	 * Put ourself on the sleep queue, and while we are
1807 	 * unable to grab the lock, go park in the kernel.
1808 	 * Take ourself off the sleep queue after we acquire the lock.
1809 	 * The waiter bit can be set/cleared only while holding the queue lock.
1810 	 */
1811 	qp = queue_lock(mp, MX);
1812 	enqueue(qp, self, 0);
1813 	mp->mutex_waiters = 1;
1814 	for (;;) {
1815 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
1816 			mp->mutex_owner = (uintptr_t)self;
1817 			mp->mutex_waiters = dequeue_self(qp);
1818 			break;
1819 		}
1820 		set_parking_flag(self, 1);
1821 		queue_unlock(qp);
1822 		/*
1823 		 * __lwp_park() will return the residual time in tsp
1824 		 * if we are unparked before the timeout expires.
1825 		 */
1826 		error = __lwp_park(tsp, 0);
1827 		set_parking_flag(self, 0);
1828 		/*
1829 		 * We could have taken a signal or suspended ourself.
1830 		 * If we did, then we removed ourself from the queue.
1831 		 * Someone else may have removed us from the queue
1832 		 * as a consequence of mutex_unlock().  We may have
1833 		 * gotten a timeout from __lwp_park().  Or we may still
1834 		 * be on the queue and this is just a spurious wakeup.
1835 		 */
1836 		qp = queue_lock(mp, MX);
1837 		if (self->ul_sleepq == NULL) {
1838 			if (error) {
1839 				mp->mutex_waiters = queue_waiter(qp)? 1 : 0;
1840 				if (error != EINTR)
1841 					break;
1842 				error = 0;
1843 			}
1844 			if (set_lock_byte(&mp->mutex_lockw) == 0) {
1845 				mp->mutex_owner = (uintptr_t)self;
1846 				break;
1847 			}
1848 			enqueue(qp, self, 0);
1849 			mp->mutex_waiters = 1;
1850 		}
1851 		ASSERT(self->ul_sleepq == qp &&
1852 		    self->ul_qtype == MX &&
1853 		    self->ul_wchan == mp);
1854 		if (error) {
1855 			if (error != EINTR) {
1856 				mp->mutex_waiters = dequeue_self(qp);
1857 				break;
1858 			}
1859 			error = 0;
1860 		}
1861 	}
1862 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
1863 	    self->ul_wchan == NULL);
1864 	self->ul_sp = 0;
1865 
1866 	ASSERT(error == 0 || error == EINVAL || error == ETIME);
1867 
1868 	if (error == 0 && (mp->mutex_flag & LOCK_NOTRECOVERABLE)) {
1869 		ASSERT(mp->mutex_type & LOCK_ROBUST);
1870 		/*
1871 		 * We shouldn't own the mutex.
1872 		 * Just clear the lock; everyone has already been waked up.
1873 		 */
1874 		mp->mutex_owner = 0;
1875 		(void) clear_lockbyte(&mp->mutex_lockword);
1876 		error = ENOTRECOVERABLE;
1877 	}
1878 
1879 	queue_unlock(qp);
1880 
1881 	if (msp)
1882 		msp->mutex_sleep_time += gethrtime() - begin_sleep;
1883 
1884 	if (error) {
1885 		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 0);
1886 		DTRACE_PROBE2(plockstat, mutex__error, mp, error);
1887 	} else {
1888 		DTRACE_PROBE2(plockstat, mutex__blocked, mp, 1);
1889 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
1890 		if (mp->mutex_flag & LOCK_OWNERDEAD) {
1891 			ASSERT(mp->mutex_type & LOCK_ROBUST);
1892 			error = EOWNERDEAD;
1893 		}
1894 	}
1895 
1896 	return (error);
1897 }
1898 
1899 static int
mutex_recursion(mutex_t * mp,int mtype,int try)1900 mutex_recursion(mutex_t *mp, int mtype, int try)
1901 {
1902 	ASSERT(mutex_held(mp));
1903 	ASSERT(mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK));
1904 	ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
1905 
1906 	if (mtype & LOCK_RECURSIVE) {
1907 		if (mp->mutex_rcount == RECURSION_MAX) {
1908 			DTRACE_PROBE2(plockstat, mutex__error, mp, EAGAIN);
1909 			return (EAGAIN);
1910 		}
1911 		mp->mutex_rcount++;
1912 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 1, 0);
1913 		return (0);
1914 	}
1915 	if (try == MUTEX_LOCK) {
1916 		DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
1917 		return (EDEADLK);
1918 	}
1919 	return (EBUSY);
1920 }
1921 
1922 /*
1923  * Register this USYNC_PROCESS|LOCK_ROBUST mutex with the kernel so
1924  * it can apply LOCK_OWNERDEAD|LOCK_UNMAPPED if it becomes necessary.
1925  * We use tdb_hash_lock here and in the synch object tracking code in
1926  * the tdb_agent.c file.  There is no conflict between these two usages.
1927  */
1928 void
register_lock(mutex_t * mp)1929 register_lock(mutex_t *mp)
1930 {
1931 	uberdata_t *udp = curthread->ul_uberdata;
1932 	uint_t hash = LOCK_HASH(mp);
1933 	robust_t *rlp;
1934 	robust_t *invalid;
1935 	robust_t **rlpp;
1936 	robust_t **table;
1937 
1938 	if ((table = udp->robustlocks) == NULL) {
1939 		lmutex_lock(&udp->tdb_hash_lock);
1940 		if ((table = udp->robustlocks) == NULL) {
1941 			table = lmalloc(LOCKHASHSZ * sizeof (robust_t *));
1942 			membar_producer();
1943 			udp->robustlocks = table;
1944 		}
1945 		lmutex_unlock(&udp->tdb_hash_lock);
1946 	}
1947 	membar_consumer();
1948 
1949 	/*
1950 	 * First search the registered table with no locks held.
1951 	 * This is safe because the table never shrinks
1952 	 * and we can only get a false negative.
1953 	 */
1954 	for (rlp = table[hash]; rlp != NULL; rlp = rlp->robust_next) {
1955 		if (rlp->robust_lock == mp)	/* already registered */
1956 			return;
1957 	}
1958 
1959 	/*
1960 	 * The lock was not found.
1961 	 * Repeat the operation with tdb_hash_lock held.
1962 	 */
1963 	lmutex_lock(&udp->tdb_hash_lock);
1964 
1965 	invalid = NULL;
1966 	for (rlpp = &table[hash];
1967 	    (rlp = *rlpp) != NULL;
1968 	    rlpp = &rlp->robust_next) {
1969 		if (rlp->robust_lock == mp) {	/* already registered */
1970 			lmutex_unlock(&udp->tdb_hash_lock);
1971 			return;
1972 		}
1973 		/* remember the first invalid entry, if any */
1974 		if (rlp->robust_lock == INVALID_ADDR && invalid == NULL)
1975 			invalid = rlp;
1976 	}
1977 
1978 	/*
1979 	 * The lock has never been registered.
1980 	 * Add it to the table and register it now.
1981 	 */
1982 	if ((rlp = invalid) != NULL) {
1983 		/*
1984 		 * Reuse the invalid entry we found above.
1985 		 * The linkages are still correct.
1986 		 */
1987 		rlp->robust_lock = mp;
1988 		membar_producer();
1989 	} else {
1990 		/*
1991 		 * Allocate a new entry and add it to
1992 		 * the hash table and to the global list.
1993 		 */
1994 		rlp = lmalloc(sizeof (*rlp));
1995 		rlp->robust_lock = mp;
1996 		rlp->robust_next = NULL;
1997 		rlp->robust_list = udp->robustlist;
1998 		udp->robustlist = rlp;
1999 		membar_producer();
2000 		*rlpp = rlp;
2001 	}
2002 
2003 	lmutex_unlock(&udp->tdb_hash_lock);
2004 
2005 	(void) ___lwp_mutex_register(mp, &rlp->robust_lock);
2006 }
2007 
2008 /*
2009  * This is called in the child of fork()/forkall() to start over
2010  * with a clean slate.  (Each process must register its own locks.)
2011  * No locks are needed because all other threads are suspended or gone.
2012  */
2013 void
unregister_locks(void)2014 unregister_locks(void)
2015 {
2016 	uberdata_t *udp = curthread->ul_uberdata;
2017 	robust_t **table;
2018 	robust_t *rlp;
2019 	robust_t *next;
2020 
2021 	/*
2022 	 * Do this first, before calling lfree().
2023 	 */
2024 	table = udp->robustlocks;
2025 	udp->robustlocks = NULL;
2026 	rlp = udp->robustlist;
2027 	udp->robustlist = NULL;
2028 
2029 	/*
2030 	 * Do this by traversing the global list, not the hash table.
2031 	 */
2032 	while (rlp != NULL) {
2033 		next = rlp->robust_list;
2034 		lfree(rlp, sizeof (*rlp));
2035 		rlp = next;
2036 	}
2037 	if (table != NULL)
2038 		lfree(table, LOCKHASHSZ * sizeof (robust_t *));
2039 }
2040 
2041 /*
2042  * Returns with mutex_owner set correctly.
2043  */
2044 int
mutex_lock_internal(mutex_t * mp,timespec_t * tsp,int try)2045 mutex_lock_internal(mutex_t *mp, timespec_t *tsp, int try)
2046 {
2047 	ulwp_t *self = curthread;
2048 	uberdata_t *udp = self->ul_uberdata;
2049 	int mtype = mp->mutex_type;
2050 	tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
2051 	int error = 0;
2052 	int noceil = try & MUTEX_NOCEIL;
2053 	uint8_t ceil;
2054 	int myprio;
2055 
2056 	try &= ~MUTEX_NOCEIL;
2057 	ASSERT(try == MUTEX_TRY || try == MUTEX_LOCK);
2058 
2059 	if (!self->ul_schedctl_called)
2060 		(void) setup_schedctl();
2061 
2062 	if (msp && try == MUTEX_TRY)
2063 		tdb_incr(msp->mutex_try);
2064 
2065 	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && mutex_held(mp))
2066 		return (mutex_recursion(mp, mtype, try));
2067 
2068 	if (self->ul_error_detection && try == MUTEX_LOCK &&
2069 	    tsp == NULL && mutex_held(mp))
2070 		lock_error(mp, "mutex_lock", NULL, NULL);
2071 
2072 	if ((mtype & LOCK_PRIO_PROTECT) && noceil == 0) {
2073 		update_sched(self);
2074 		if (self->ul_cid != self->ul_rtclassid) {
2075 			DTRACE_PROBE2(plockstat, mutex__error, mp, EPERM);
2076 			return (EPERM);
2077 		}
2078 		ceil = mp->mutex_ceiling;
2079 		myprio = self->ul_epri? self->ul_epri : self->ul_pri;
2080 		if (myprio > ceil) {
2081 			DTRACE_PROBE2(plockstat, mutex__error, mp, EINVAL);
2082 			return (EINVAL);
2083 		}
2084 		if ((error = _ceil_mylist_add(mp)) != 0) {
2085 			DTRACE_PROBE2(plockstat, mutex__error, mp, error);
2086 			return (error);
2087 		}
2088 		if (myprio < ceil)
2089 			_ceil_prio_inherit(ceil);
2090 	}
2091 
2092 	if ((mtype & (USYNC_PROCESS | LOCK_ROBUST))
2093 	    == (USYNC_PROCESS | LOCK_ROBUST))
2094 		register_lock(mp);
2095 
2096 	if (mtype & LOCK_PRIO_INHERIT) {
2097 		/* go straight to the kernel */
2098 		if (try == MUTEX_TRY)
2099 			error = mutex_trylock_kernel(mp);
2100 		else	/* MUTEX_LOCK */
2101 			error = mutex_lock_kernel(mp, tsp, msp);
2102 		/*
2103 		 * The kernel never sets or clears the lock byte
2104 		 * for LOCK_PRIO_INHERIT mutexes.
2105 		 * Set it here for consistency.
2106 		 */
2107 		switch (error) {
2108 		case 0:
2109 			self->ul_pilocks++;
2110 			mp->mutex_lockw = LOCKSET;
2111 			break;
2112 		case EOWNERDEAD:
2113 		case ELOCKUNMAPPED:
2114 			self->ul_pilocks++;
2115 			mp->mutex_lockw = LOCKSET;
2116 			/* FALLTHROUGH */
2117 		case ENOTRECOVERABLE:
2118 			ASSERT(mtype & LOCK_ROBUST);
2119 			break;
2120 		case EDEADLK:
2121 			if (try == MUTEX_TRY) {
2122 				error = EBUSY;
2123 			} else if (tsp != NULL) {	/* simulate a timeout */
2124 				/*
2125 				 * Note: mutex_timedlock() never returns EINTR.
2126 				 */
2127 				timespec_t ts = *tsp;
2128 				timespec_t rts;
2129 
2130 				while (__nanosleep(&ts, &rts) == EINTR)
2131 					ts = rts;
2132 				error = ETIME;
2133 			} else {		/* simulate a deadlock */
2134 				stall();
2135 			}
2136 			break;
2137 		}
2138 	} else if (mtype & USYNC_PROCESS) {
2139 		error = mutex_trylock_process(mp, try == MUTEX_LOCK);
2140 		if (error == EBUSY && try == MUTEX_LOCK)
2141 			error = mutex_lock_kernel(mp, tsp, msp);
2142 	} else {	/* USYNC_THREAD */
2143 		error = mutex_trylock_adaptive(mp, try == MUTEX_LOCK);
2144 		if (error == EBUSY && try == MUTEX_LOCK)
2145 			error = mutex_lock_queue(self, msp, mp, tsp);
2146 	}
2147 
2148 	switch (error) {
2149 	case 0:
2150 	case EOWNERDEAD:
2151 	case ELOCKUNMAPPED:
2152 		if (mtype & LOCK_ROBUST)
2153 			remember_lock(mp);
2154 		if (msp)
2155 			record_begin_hold(msp);
2156 		break;
2157 	default:
2158 		if ((mtype & LOCK_PRIO_PROTECT) && noceil == 0) {
2159 			(void) _ceil_mylist_del(mp);
2160 			if (myprio < ceil)
2161 				_ceil_prio_waive();
2162 		}
2163 		if (try == MUTEX_TRY) {
2164 			if (msp)
2165 				tdb_incr(msp->mutex_try_fail);
2166 			if (__td_event_report(self, TD_LOCK_TRY, udp)) {
2167 				self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
2168 				tdb_event(TD_LOCK_TRY, udp);
2169 			}
2170 		}
2171 		break;
2172 	}
2173 
2174 	return (error);
2175 }
2176 
2177 int
fast_process_lock(mutex_t * mp,timespec_t * tsp,int mtype,int try)2178 fast_process_lock(mutex_t *mp, timespec_t *tsp, int mtype, int try)
2179 {
2180 	ulwp_t *self = curthread;
2181 	uberdata_t *udp = self->ul_uberdata;
2182 
2183 	/*
2184 	 * We know that USYNC_PROCESS is set in mtype and that
2185 	 * zero, one, or both of the flags LOCK_RECURSIVE and
2186 	 * LOCK_ERRORCHECK are set, and that no other flags are set.
2187 	 */
2188 	ASSERT((mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0);
2189 	enter_critical(self);
2190 #if defined(__sparc) && !defined(_LP64)
2191 	/* horrible hack, necessary only on 32-bit sparc */
2192 	if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
2193 	    self->ul_misaligned) {
2194 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
2195 			mp->mutex_ownerpid = udp->pid;
2196 			mp->mutex_owner = (uintptr_t)self;
2197 			exit_critical(self);
2198 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2199 			return (0);
2200 		}
2201 	} else
2202 #endif
2203 	if (set_lock_byte64(&mp->mutex_lockword64, udp->pid) == 0) {
2204 		mp->mutex_owner = (uintptr_t)self;
2205 		/* mp->mutex_ownerpid was set by set_lock_byte64() */
2206 		exit_critical(self);
2207 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2208 		return (0);
2209 	}
2210 	exit_critical(self);
2211 
2212 	if ((mtype & (LOCK_RECURSIVE|LOCK_ERRORCHECK)) && shared_mutex_held(mp))
2213 		return (mutex_recursion(mp, mtype, try));
2214 
2215 	if (try == MUTEX_LOCK) {
2216 		if (mutex_trylock_process(mp, 1) == 0)
2217 			return (0);
2218 		return (mutex_lock_kernel(mp, tsp, NULL));
2219 	}
2220 
2221 	if (__td_event_report(self, TD_LOCK_TRY, udp)) {
2222 		self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
2223 		tdb_event(TD_LOCK_TRY, udp);
2224 	}
2225 	return (EBUSY);
2226 }
2227 
2228 static int
mutex_lock_impl(mutex_t * mp,timespec_t * tsp)2229 mutex_lock_impl(mutex_t *mp, timespec_t *tsp)
2230 {
2231 	ulwp_t *self = curthread;
2232 	int mtype = mp->mutex_type;
2233 	uberflags_t *gflags;
2234 
2235 	if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
2236 	    self->ul_error_detection && self->ul_misaligned == 0)
2237 		lock_error(mp, "mutex_lock", NULL, "mutex is misaligned");
2238 
2239 	/*
2240 	 * Optimize the case of USYNC_THREAD, including
2241 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
2242 	 * no error detection, no lock statistics,
2243 	 * and the process has only a single thread.
2244 	 * (Most likely a traditional single-threaded application.)
2245 	 */
2246 	if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
2247 	    self->ul_uberdata->uberflags.uf_all) == 0) {
2248 		/*
2249 		 * Only one thread exists so we don't need an atomic operation.
2250 		 * We do, however, need to protect against signals.
2251 		 */
2252 		if (mp->mutex_lockw == 0) {
2253 			sigoff(self);
2254 			mp->mutex_lockw = LOCKSET;
2255 			mp->mutex_owner = (uintptr_t)self;
2256 			sigon(self);
2257 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2258 			return (0);
2259 		}
2260 		if (mtype && MUTEX_OWNER(mp) == self)
2261 			return (mutex_recursion(mp, mtype, MUTEX_LOCK));
2262 		/*
2263 		 * We have reached a deadlock, probably because the
2264 		 * process is executing non-async-signal-safe code in
2265 		 * a signal handler and is attempting to acquire a lock
2266 		 * that it already owns.  This is not surprising, given
2267 		 * bad programming practices over the years that has
2268 		 * resulted in applications calling printf() and such
2269 		 * in their signal handlers.  Unless the user has told
2270 		 * us that the signal handlers are safe by setting:
2271 		 *	export _THREAD_ASYNC_SAFE=1
2272 		 * we return EDEADLK rather than actually deadlocking.
2273 		 */
2274 		if (tsp == NULL &&
2275 		    MUTEX_OWNER(mp) == self && !self->ul_async_safe) {
2276 			DTRACE_PROBE2(plockstat, mutex__error, mp, EDEADLK);
2277 			return (EDEADLK);
2278 		}
2279 	}
2280 
2281 	/*
2282 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
2283 	 * no error detection, and no lock statistics.
2284 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
2285 	 */
2286 	if ((gflags = self->ul_schedctl_called) != NULL &&
2287 	    (gflags->uf_trs_ted |
2288 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
2289 		if (mtype & USYNC_PROCESS)
2290 			return (fast_process_lock(mp, tsp, mtype, MUTEX_LOCK));
2291 		sigoff(self);
2292 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
2293 			mp->mutex_owner = (uintptr_t)self;
2294 			sigon(self);
2295 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2296 			return (0);
2297 		}
2298 		sigon(self);
2299 		if (mtype && MUTEX_OWNER(mp) == self)
2300 			return (mutex_recursion(mp, mtype, MUTEX_LOCK));
2301 		if (mutex_trylock_adaptive(mp, 1) != 0)
2302 			return (mutex_lock_queue(self, NULL, mp, tsp));
2303 		return (0);
2304 	}
2305 
2306 	/* else do it the long way */
2307 	return (mutex_lock_internal(mp, tsp, MUTEX_LOCK));
2308 }
2309 
2310 #pragma weak pthread_mutex_lock = mutex_lock
2311 #pragma weak _mutex_lock = mutex_lock
2312 int
mutex_lock(mutex_t * mp)2313 mutex_lock(mutex_t *mp)
2314 {
2315 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
2316 	return (mutex_lock_impl(mp, NULL));
2317 }
2318 
2319 void
mutex_enter(mutex_t * mp)2320 mutex_enter(mutex_t *mp)
2321 {
2322 	int ret;
2323 	int attr = mp->mutex_type & ALL_ATTRIBUTES;
2324 
2325 	/*
2326 	 * Require LOCK_ERRORCHECK, accept LOCK_RECURSIVE.
2327 	 */
2328 	if (attr != LOCK_ERRORCHECK &&
2329 	    attr != (LOCK_ERRORCHECK | LOCK_RECURSIVE)) {
2330 		mutex_panic(mp, "mutex_enter: bad mutex type");
2331 	}
2332 	ret = mutex_lock(mp);
2333 	if (ret == EDEADLK) {
2334 		mutex_panic(mp, "recursive mutex_enter");
2335 	} else if (ret == EAGAIN) {
2336 		mutex_panic(mp, "excessive recursive mutex_enter");
2337 	} else if (ret != 0) {
2338 		mutex_panic(mp, "unknown mutex_enter failure");
2339 	}
2340 }
2341 
2342 int
pthread_mutex_timedlock(pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD abstime)2343 pthread_mutex_timedlock(pthread_mutex_t *_RESTRICT_KYWD mp,
2344 	const struct timespec *_RESTRICT_KYWD abstime)
2345 {
2346 	timespec_t tslocal;
2347 	int error;
2348 
2349 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
2350 	abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
2351 	error = mutex_lock_impl((mutex_t *)mp, &tslocal);
2352 	if (error == ETIME)
2353 		error = ETIMEDOUT;
2354 	return (error);
2355 }
2356 
2357 int
pthread_mutex_reltimedlock_np(pthread_mutex_t * _RESTRICT_KYWD mp,const struct timespec * _RESTRICT_KYWD reltime)2358 pthread_mutex_reltimedlock_np(pthread_mutex_t *_RESTRICT_KYWD mp,
2359 	const struct timespec *_RESTRICT_KYWD reltime)
2360 {
2361 	timespec_t tslocal;
2362 	int error;
2363 
2364 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
2365 	tslocal = *reltime;
2366 	error = mutex_lock_impl((mutex_t *)mp, &tslocal);
2367 	if (error == ETIME)
2368 		error = ETIMEDOUT;
2369 	return (error);
2370 }
2371 
2372 #pragma weak pthread_mutex_trylock = mutex_trylock
2373 int
mutex_trylock(mutex_t * mp)2374 mutex_trylock(mutex_t *mp)
2375 {
2376 	ulwp_t *self = curthread;
2377 	uberdata_t *udp = self->ul_uberdata;
2378 	int mtype = mp->mutex_type;
2379 	uberflags_t *gflags;
2380 
2381 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
2382 
2383 	/*
2384 	 * Optimize the case of USYNC_THREAD, including
2385 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
2386 	 * no error detection, no lock statistics,
2387 	 * and the process has only a single thread.
2388 	 * (Most likely a traditional single-threaded application.)
2389 	 */
2390 	if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
2391 	    udp->uberflags.uf_all) == 0) {
2392 		/*
2393 		 * Only one thread exists so we don't need an atomic operation.
2394 		 * We do, however, need to protect against signals.
2395 		 */
2396 		if (mp->mutex_lockw == 0) {
2397 			sigoff(self);
2398 			mp->mutex_lockw = LOCKSET;
2399 			mp->mutex_owner = (uintptr_t)self;
2400 			sigon(self);
2401 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2402 			return (0);
2403 		}
2404 		if (mtype && MUTEX_OWNER(mp) == self)
2405 			return (mutex_recursion(mp, mtype, MUTEX_TRY));
2406 		return (EBUSY);
2407 	}
2408 
2409 	/*
2410 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
2411 	 * no error detection, and no lock statistics.
2412 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
2413 	 */
2414 	if ((gflags = self->ul_schedctl_called) != NULL &&
2415 	    (gflags->uf_trs_ted |
2416 	    (mtype & ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK))) == 0) {
2417 		if (mtype & USYNC_PROCESS)
2418 			return (fast_process_lock(mp, NULL, mtype, MUTEX_TRY));
2419 		sigoff(self);
2420 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
2421 			mp->mutex_owner = (uintptr_t)self;
2422 			sigon(self);
2423 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2424 			return (0);
2425 		}
2426 		sigon(self);
2427 		if (mtype && MUTEX_OWNER(mp) == self)
2428 			return (mutex_recursion(mp, mtype, MUTEX_TRY));
2429 		if (__td_event_report(self, TD_LOCK_TRY, udp)) {
2430 			self->ul_td_evbuf.eventnum = TD_LOCK_TRY;
2431 			tdb_event(TD_LOCK_TRY, udp);
2432 		}
2433 		return (EBUSY);
2434 	}
2435 
2436 	/* else do it the long way */
2437 	return (mutex_lock_internal(mp, NULL, MUTEX_TRY));
2438 }
2439 
2440 int
mutex_unlock_internal(mutex_t * mp,int retain_robust_flags)2441 mutex_unlock_internal(mutex_t *mp, int retain_robust_flags)
2442 {
2443 	ulwp_t *self = curthread;
2444 	uberdata_t *udp = self->ul_uberdata;
2445 	int mtype = mp->mutex_type;
2446 	tdb_mutex_stats_t *msp;
2447 	int error = 0;
2448 	int release_all;
2449 	lwpid_t lwpid;
2450 
2451 	if ((mtype & (LOCK_ERRORCHECK | LOCK_ROBUST)) &&
2452 	    !mutex_held(mp))
2453 		return (EPERM);
2454 
2455 	if (self->ul_error_detection && !mutex_held(mp))
2456 		lock_error(mp, "mutex_unlock", NULL, NULL);
2457 
2458 	if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
2459 		mp->mutex_rcount--;
2460 		DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
2461 		return (0);
2462 	}
2463 
2464 	if ((msp = MUTEX_STATS(mp, udp)) != NULL)
2465 		(void) record_hold_time(msp);
2466 
2467 	if (!retain_robust_flags && !(mtype & LOCK_PRIO_INHERIT) &&
2468 	    (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) {
2469 		ASSERT(mtype & LOCK_ROBUST);
2470 		mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
2471 		mp->mutex_flag |= LOCK_NOTRECOVERABLE;
2472 	}
2473 	release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0);
2474 
2475 	if (mtype & LOCK_PRIO_INHERIT) {
2476 		no_preempt(self);
2477 		mp->mutex_owner = 0;
2478 		/* mp->mutex_ownerpid is cleared by ___lwp_mutex_unlock() */
2479 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
2480 		mp->mutex_lockw = LOCKCLEAR;
2481 		self->ul_pilocks--;
2482 		error = ___lwp_mutex_unlock(mp);
2483 		preempt(self);
2484 	} else if (mtype & USYNC_PROCESS) {
2485 		mutex_unlock_process(mp, release_all);
2486 	} else {	/* USYNC_THREAD */
2487 		if ((lwpid = mutex_unlock_queue(mp, release_all)) != 0) {
2488 			(void) __lwp_unpark(lwpid);
2489 			preempt(self);
2490 		}
2491 	}
2492 
2493 	if (mtype & LOCK_ROBUST)
2494 		forget_lock(mp);
2495 
2496 	if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp))
2497 		_ceil_prio_waive();
2498 
2499 	return (error);
2500 }
2501 
2502 #pragma weak pthread_mutex_unlock = mutex_unlock
2503 #pragma weak _mutex_unlock = mutex_unlock
2504 int
mutex_unlock(mutex_t * mp)2505 mutex_unlock(mutex_t *mp)
2506 {
2507 	ulwp_t *self = curthread;
2508 	int mtype = mp->mutex_type;
2509 	uberflags_t *gflags;
2510 	lwpid_t lwpid;
2511 	short el;
2512 
2513 	/*
2514 	 * Optimize the case of USYNC_THREAD, including
2515 	 * the LOCK_RECURSIVE and LOCK_ERRORCHECK cases,
2516 	 * no error detection, no lock statistics,
2517 	 * and the process has only a single thread.
2518 	 * (Most likely a traditional single-threaded application.)
2519 	 */
2520 	if (((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) |
2521 	    self->ul_uberdata->uberflags.uf_all) == 0) {
2522 		if (mtype) {
2523 			/*
2524 			 * At this point we know that one or both of the
2525 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
2526 			 */
2527 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
2528 				return (EPERM);
2529 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
2530 				mp->mutex_rcount--;
2531 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
2532 				return (0);
2533 			}
2534 		}
2535 		/*
2536 		 * Only one thread exists so we don't need an atomic operation.
2537 		 * Also, there can be no waiters.
2538 		 */
2539 		sigoff(self);
2540 		mp->mutex_owner = 0;
2541 		mp->mutex_lockword = 0;
2542 		sigon(self);
2543 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
2544 		return (0);
2545 	}
2546 
2547 	/*
2548 	 * Optimize the common cases of USYNC_THREAD or USYNC_PROCESS,
2549 	 * no error detection, and no lock statistics.
2550 	 * Include LOCK_RECURSIVE and LOCK_ERRORCHECK cases.
2551 	 */
2552 	if ((gflags = self->ul_schedctl_called) != NULL) {
2553 		if (((el = gflags->uf_trs_ted) | mtype) == 0) {
2554 fast_unlock:
2555 			if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) {
2556 				(void) __lwp_unpark(lwpid);
2557 				preempt(self);
2558 			}
2559 			return (0);
2560 		}
2561 		if (el)		/* error detection or lock statistics */
2562 			goto slow_unlock;
2563 		if ((mtype & ~(LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
2564 			/*
2565 			 * At this point we know that one or both of the
2566 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set.
2567 			 */
2568 			if ((mtype & LOCK_ERRORCHECK) && !MUTEX_OWNED(mp, self))
2569 				return (EPERM);
2570 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
2571 				mp->mutex_rcount--;
2572 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
2573 				return (0);
2574 			}
2575 			goto fast_unlock;
2576 		}
2577 		if ((mtype &
2578 		    ~(USYNC_PROCESS|LOCK_RECURSIVE|LOCK_ERRORCHECK)) == 0) {
2579 			/*
2580 			 * At this point we know that zero, one, or both of the
2581 			 * flags LOCK_RECURSIVE or LOCK_ERRORCHECK is set and
2582 			 * that the USYNC_PROCESS flag is set.
2583 			 */
2584 			if ((mtype & LOCK_ERRORCHECK) && !shared_mutex_held(mp))
2585 				return (EPERM);
2586 			if ((mtype & LOCK_RECURSIVE) && mp->mutex_rcount != 0) {
2587 				mp->mutex_rcount--;
2588 				DTRACE_PROBE2(plockstat, mutex__release, mp, 1);
2589 				return (0);
2590 			}
2591 			mutex_unlock_process(mp, 0);
2592 			return (0);
2593 		}
2594 	}
2595 
2596 	/* else do it the long way */
2597 slow_unlock:
2598 	return (mutex_unlock_internal(mp, 0));
2599 }
2600 
2601 void
mutex_exit(mutex_t * mp)2602 mutex_exit(mutex_t *mp)
2603 {
2604 	int ret;
2605 	int attr = mp->mutex_type & ALL_ATTRIBUTES;
2606 
2607 	if (attr != LOCK_ERRORCHECK &&
2608 	    attr != (LOCK_ERRORCHECK | LOCK_RECURSIVE)) {
2609 		mutex_panic(mp, "mutex_exit: bad mutex type");
2610 	}
2611 	ret = mutex_unlock(mp);
2612 	if (ret == EPERM) {
2613 		mutex_panic(mp, "mutex_exit: not owner");
2614 	} else if (ret != 0) {
2615 		mutex_panic(mp, "unknown mutex_exit failure");
2616 	}
2617 
2618 }
2619 
2620 /*
2621  * Internally to the library, almost all mutex lock/unlock actions
2622  * go through these lmutex_ functions, to protect critical regions.
2623  * We replicate a bit of code from mutex_lock() and mutex_unlock()
2624  * to make these functions faster since we know that the mutex type
2625  * of all internal locks is USYNC_THREAD.  We also know that internal
2626  * locking can never fail, so we panic if it does.
2627  */
2628 void
lmutex_lock(mutex_t * mp)2629 lmutex_lock(mutex_t *mp)
2630 {
2631 	ulwp_t *self = curthread;
2632 	uberdata_t *udp = self->ul_uberdata;
2633 
2634 	ASSERT(mp->mutex_type == USYNC_THREAD);
2635 
2636 	enter_critical(self);
2637 	/*
2638 	 * Optimize the case of no lock statistics and only a single thread.
2639 	 * (Most likely a traditional single-threaded application.)
2640 	 */
2641 	if (udp->uberflags.uf_all == 0) {
2642 		/*
2643 		 * Only one thread exists; the mutex must be free.
2644 		 */
2645 		ASSERT(mp->mutex_lockw == 0);
2646 		mp->mutex_lockw = LOCKSET;
2647 		mp->mutex_owner = (uintptr_t)self;
2648 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2649 	} else {
2650 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
2651 
2652 		if (!self->ul_schedctl_called)
2653 			(void) setup_schedctl();
2654 
2655 		if (set_lock_byte(&mp->mutex_lockw) == 0) {
2656 			mp->mutex_owner = (uintptr_t)self;
2657 			DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2658 		} else if (mutex_trylock_adaptive(mp, 1) != 0) {
2659 			(void) mutex_lock_queue(self, msp, mp, NULL);
2660 		}
2661 
2662 		if (msp)
2663 			record_begin_hold(msp);
2664 	}
2665 }
2666 
2667 void
lmutex_unlock(mutex_t * mp)2668 lmutex_unlock(mutex_t *mp)
2669 {
2670 	ulwp_t *self = curthread;
2671 	uberdata_t *udp = self->ul_uberdata;
2672 
2673 	ASSERT(mp->mutex_type == USYNC_THREAD);
2674 
2675 	/*
2676 	 * Optimize the case of no lock statistics and only a single thread.
2677 	 * (Most likely a traditional single-threaded application.)
2678 	 */
2679 	if (udp->uberflags.uf_all == 0) {
2680 		/*
2681 		 * Only one thread exists so there can be no waiters.
2682 		 */
2683 		mp->mutex_owner = 0;
2684 		mp->mutex_lockword = 0;
2685 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
2686 	} else {
2687 		tdb_mutex_stats_t *msp = MUTEX_STATS(mp, udp);
2688 		lwpid_t lwpid;
2689 
2690 		if (msp)
2691 			(void) record_hold_time(msp);
2692 		if ((lwpid = mutex_unlock_queue(mp, 0)) != 0) {
2693 			(void) __lwp_unpark(lwpid);
2694 			preempt(self);
2695 		}
2696 	}
2697 	exit_critical(self);
2698 }
2699 
2700 /*
2701  * For specialized code in libc, like the asynchronous i/o code,
2702  * the following sig_*() locking primitives are used in order
2703  * to make the code asynchronous signal safe.  Signals are
2704  * deferred while locks acquired by these functions are held.
2705  */
2706 void
sig_mutex_lock(mutex_t * mp)2707 sig_mutex_lock(mutex_t *mp)
2708 {
2709 	ulwp_t *self = curthread;
2710 
2711 	sigoff(self);
2712 	(void) mutex_lock(mp);
2713 }
2714 
2715 void
sig_mutex_unlock(mutex_t * mp)2716 sig_mutex_unlock(mutex_t *mp)
2717 {
2718 	ulwp_t *self = curthread;
2719 
2720 	(void) mutex_unlock(mp);
2721 	sigon(self);
2722 }
2723 
2724 int
sig_mutex_trylock(mutex_t * mp)2725 sig_mutex_trylock(mutex_t *mp)
2726 {
2727 	ulwp_t *self = curthread;
2728 	int error;
2729 
2730 	sigoff(self);
2731 	if ((error = mutex_trylock(mp)) != 0)
2732 		sigon(self);
2733 	return (error);
2734 }
2735 
2736 /*
2737  * sig_cond_wait() is a cancellation point.
2738  */
2739 int
sig_cond_wait(cond_t * cv,mutex_t * mp)2740 sig_cond_wait(cond_t *cv, mutex_t *mp)
2741 {
2742 	int error;
2743 
2744 	ASSERT(curthread->ul_sigdefer != 0);
2745 	pthread_testcancel();
2746 	error = __cond_wait(cv, mp);
2747 	if (error == EINTR && curthread->ul_cursig) {
2748 		sig_mutex_unlock(mp);
2749 		/* take the deferred signal here */
2750 		sig_mutex_lock(mp);
2751 	}
2752 	pthread_testcancel();
2753 	return (error);
2754 }
2755 
2756 /*
2757  * sig_cond_reltimedwait() is a cancellation point.
2758  */
2759 int
sig_cond_reltimedwait(cond_t * cv,mutex_t * mp,const timespec_t * ts)2760 sig_cond_reltimedwait(cond_t *cv, mutex_t *mp, const timespec_t *ts)
2761 {
2762 	int error;
2763 
2764 	ASSERT(curthread->ul_sigdefer != 0);
2765 	pthread_testcancel();
2766 	error = __cond_reltimedwait(cv, mp, ts);
2767 	if (error == EINTR && curthread->ul_cursig) {
2768 		sig_mutex_unlock(mp);
2769 		/* take the deferred signal here */
2770 		sig_mutex_lock(mp);
2771 	}
2772 	pthread_testcancel();
2773 	return (error);
2774 }
2775 
2776 /*
2777  * For specialized code in libc, like the stdio code.
2778  * the following cancel_safe_*() locking primitives are used in
2779  * order to make the code cancellation-safe.  Cancellation is
2780  * deferred while locks acquired by these functions are held.
2781  */
2782 void
cancel_safe_mutex_lock(mutex_t * mp)2783 cancel_safe_mutex_lock(mutex_t *mp)
2784 {
2785 	(void) mutex_lock(mp);
2786 	curthread->ul_libc_locks++;
2787 }
2788 
2789 int
cancel_safe_mutex_trylock(mutex_t * mp)2790 cancel_safe_mutex_trylock(mutex_t *mp)
2791 {
2792 	int error;
2793 
2794 	if ((error = mutex_trylock(mp)) == 0)
2795 		curthread->ul_libc_locks++;
2796 	return (error);
2797 }
2798 
2799 void
cancel_safe_mutex_unlock(mutex_t * mp)2800 cancel_safe_mutex_unlock(mutex_t *mp)
2801 {
2802 	ulwp_t *self = curthread;
2803 
2804 	ASSERT(self->ul_libc_locks != 0);
2805 
2806 	(void) mutex_unlock(mp);
2807 
2808 	/*
2809 	 * Decrement the count of locks held by cancel_safe_mutex_lock().
2810 	 * If we are then in a position to terminate cleanly and
2811 	 * if there is a pending cancellation and cancellation
2812 	 * is not disabled and we received EINTR from a recent
2813 	 * system call then perform the cancellation action now.
2814 	 */
2815 	if (--self->ul_libc_locks == 0 &&
2816 	    !(self->ul_vfork | self->ul_nocancel |
2817 	    self->ul_critical | self->ul_sigdefer) &&
2818 	    cancel_active())
2819 		pthread_exit(PTHREAD_CANCELED);
2820 }
2821 
2822 static int
shared_mutex_held(mutex_t * mparg)2823 shared_mutex_held(mutex_t *mparg)
2824 {
2825 	/*
2826 	 * The 'volatile' is necessary to make sure the compiler doesn't
2827 	 * reorder the tests of the various components of the mutex.
2828 	 * They must be tested in this order:
2829 	 *	mutex_lockw
2830 	 *	mutex_owner
2831 	 *	mutex_ownerpid
2832 	 * This relies on the fact that everywhere mutex_lockw is cleared,
2833 	 * mutex_owner and mutex_ownerpid are cleared before mutex_lockw
2834 	 * is cleared, and that everywhere mutex_lockw is set, mutex_owner
2835 	 * and mutex_ownerpid are set after mutex_lockw is set, and that
2836 	 * mutex_lockw is set or cleared with a memory barrier.
2837 	 */
2838 	volatile mutex_t *mp = (volatile mutex_t *)mparg;
2839 	ulwp_t *self = curthread;
2840 	uberdata_t *udp = self->ul_uberdata;
2841 
2842 	return (MUTEX_OWNED(mp, self) && mp->mutex_ownerpid == udp->pid);
2843 }
2844 
2845 #pragma weak _mutex_held = mutex_held
2846 int
mutex_held(mutex_t * mparg)2847 mutex_held(mutex_t *mparg)
2848 {
2849 	volatile mutex_t *mp = (volatile mutex_t *)mparg;
2850 
2851 	if (mparg->mutex_type & USYNC_PROCESS)
2852 		return (shared_mutex_held(mparg));
2853 	return (MUTEX_OWNED(mp, curthread));
2854 }
2855 
2856 #pragma weak pthread_mutex_destroy = mutex_destroy
2857 #pragma weak _mutex_destroy = mutex_destroy
2858 int
mutex_destroy(mutex_t * mp)2859 mutex_destroy(mutex_t *mp)
2860 {
2861 	if (mp->mutex_type & USYNC_PROCESS)
2862 		forget_lock(mp);
2863 	(void) memset(mp, 0, sizeof (*mp));
2864 	tdb_sync_obj_deregister(mp);
2865 	return (0);
2866 }
2867 
2868 #pragma weak pthread_mutex_consistent_np = mutex_consistent
2869 #pragma weak pthread_mutex_consistent = mutex_consistent
2870 int
mutex_consistent(mutex_t * mp)2871 mutex_consistent(mutex_t *mp)
2872 {
2873 	/*
2874 	 * Do this only for an inconsistent, initialized robust lock
2875 	 * that we hold.  For all other cases, return EINVAL.
2876 	 */
2877 	if (mutex_held(mp) &&
2878 	    (mp->mutex_type & LOCK_ROBUST) &&
2879 	    (mp->mutex_flag & LOCK_INITED) &&
2880 	    (mp->mutex_flag & (LOCK_OWNERDEAD | LOCK_UNMAPPED))) {
2881 		mp->mutex_flag &= ~(LOCK_OWNERDEAD | LOCK_UNMAPPED);
2882 		mp->mutex_rcount = 0;
2883 		return (0);
2884 	}
2885 	return (EINVAL);
2886 }
2887 
2888 /*
2889  * Spin locks are separate from ordinary mutexes,
2890  * but we use the same data structure for them.
2891  */
2892 
2893 int
pthread_spin_init(pthread_spinlock_t * lock,int pshared)2894 pthread_spin_init(pthread_spinlock_t *lock, int pshared)
2895 {
2896 	mutex_t *mp = (mutex_t *)lock;
2897 
2898 	(void) memset(mp, 0, sizeof (*mp));
2899 	if (pshared == PTHREAD_PROCESS_SHARED)
2900 		mp->mutex_type = USYNC_PROCESS;
2901 	else
2902 		mp->mutex_type = USYNC_THREAD;
2903 	mp->mutex_flag = LOCK_INITED;
2904 	mp->mutex_magic = MUTEX_MAGIC;
2905 
2906 	/*
2907 	 * This should be at the beginning of the function,
2908 	 * but for the sake of old broken applications that
2909 	 * do not have proper alignment for their mutexes
2910 	 * (and don't check the return code from pthread_spin_init),
2911 	 * we put it here, after initializing the mutex regardless.
2912 	 */
2913 	if (((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1)) &&
2914 	    curthread->ul_misaligned == 0)
2915 		return (EINVAL);
2916 
2917 	return (0);
2918 }
2919 
2920 int
pthread_spin_destroy(pthread_spinlock_t * lock)2921 pthread_spin_destroy(pthread_spinlock_t *lock)
2922 {
2923 	(void) memset(lock, 0, sizeof (*lock));
2924 	return (0);
2925 }
2926 
2927 int
pthread_spin_trylock(pthread_spinlock_t * lock)2928 pthread_spin_trylock(pthread_spinlock_t *lock)
2929 {
2930 	mutex_t *mp = (mutex_t *)lock;
2931 	ulwp_t *self = curthread;
2932 	int error = 0;
2933 
2934 	no_preempt(self);
2935 	if (set_lock_byte(&mp->mutex_lockw) != 0)
2936 		error = EBUSY;
2937 	else {
2938 		mp->mutex_owner = (uintptr_t)self;
2939 		if (mp->mutex_type == USYNC_PROCESS)
2940 			mp->mutex_ownerpid = self->ul_uberdata->pid;
2941 		DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, 0);
2942 	}
2943 	preempt(self);
2944 	return (error);
2945 }
2946 
2947 int
pthread_spin_lock(pthread_spinlock_t * lock)2948 pthread_spin_lock(pthread_spinlock_t *lock)
2949 {
2950 	mutex_t *mp = (mutex_t *)lock;
2951 	ulwp_t *self = curthread;
2952 	volatile uint8_t *lockp = (volatile uint8_t *)&mp->mutex_lockw;
2953 	int count = 0;
2954 
2955 	ASSERT(!self->ul_critical || self->ul_bindflags);
2956 
2957 	DTRACE_PROBE1(plockstat, mutex__spin, mp);
2958 
2959 	/*
2960 	 * We don't care whether the owner is running on a processor.
2961 	 * We just spin because that's what this interface requires.
2962 	 */
2963 	for (;;) {
2964 		if (*lockp == 0) {	/* lock byte appears to be clear */
2965 			no_preempt(self);
2966 			if (set_lock_byte(lockp) == 0)
2967 				break;
2968 			preempt(self);
2969 		}
2970 		if (count < INT_MAX)
2971 			count++;
2972 		SMT_PAUSE();
2973 	}
2974 	mp->mutex_owner = (uintptr_t)self;
2975 	if (mp->mutex_type == USYNC_PROCESS)
2976 		mp->mutex_ownerpid = self->ul_uberdata->pid;
2977 	preempt(self);
2978 	if (count) {
2979 		DTRACE_PROBE3(plockstat, mutex__spun, mp, 1, count);
2980 	}
2981 	DTRACE_PROBE3(plockstat, mutex__acquire, mp, 0, count);
2982 	return (0);
2983 }
2984 
2985 int
pthread_spin_unlock(pthread_spinlock_t * lock)2986 pthread_spin_unlock(pthread_spinlock_t *lock)
2987 {
2988 	mutex_t *mp = (mutex_t *)lock;
2989 	ulwp_t *self = curthread;
2990 
2991 	no_preempt(self);
2992 	mp->mutex_owner = 0;
2993 	mp->mutex_ownerpid = 0;
2994 	DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
2995 	(void) atomic_swap_32(&mp->mutex_lockword, 0);
2996 	preempt(self);
2997 	return (0);
2998 }
2999 
3000 #define	INITIAL_LOCKS	8	/* initial size of ul_heldlocks.array */
3001 
3002 /*
3003  * Find/allocate an entry for 'lock' in our array of held locks.
3004  */
3005 static mutex_t **
find_lock_entry(mutex_t * lock)3006 find_lock_entry(mutex_t *lock)
3007 {
3008 	ulwp_t *self = curthread;
3009 	mutex_t **remembered = NULL;
3010 	mutex_t **lockptr;
3011 	uint_t nlocks;
3012 
3013 	if ((nlocks = self->ul_heldlockcnt) != 0)
3014 		lockptr = self->ul_heldlocks.array;
3015 	else {
3016 		nlocks = 1;
3017 		lockptr = &self->ul_heldlocks.single;
3018 	}
3019 
3020 	for (; nlocks; nlocks--, lockptr++) {
3021 		if (*lockptr == lock)
3022 			return (lockptr);
3023 		if (*lockptr == NULL && remembered == NULL)
3024 			remembered = lockptr;
3025 	}
3026 	if (remembered != NULL) {
3027 		*remembered = lock;
3028 		return (remembered);
3029 	}
3030 
3031 	/*
3032 	 * No entry available.  Allocate more space, converting
3033 	 * the single entry into an array of entries if necessary.
3034 	 */
3035 	if ((nlocks = self->ul_heldlockcnt) == 0) {
3036 		/*
3037 		 * Initial allocation of the array.
3038 		 * Convert the single entry into an array.
3039 		 */
3040 		self->ul_heldlockcnt = nlocks = INITIAL_LOCKS;
3041 		lockptr = lmalloc(nlocks * sizeof (mutex_t *));
3042 		/*
3043 		 * The single entry becomes the first entry in the array.
3044 		 */
3045 		*lockptr = self->ul_heldlocks.single;
3046 		self->ul_heldlocks.array = lockptr;
3047 		/*
3048 		 * Return the next available entry in the array.
3049 		 */
3050 		*++lockptr = lock;
3051 		return (lockptr);
3052 	}
3053 	/*
3054 	 * Reallocate the array, double the size each time.
3055 	 */
3056 	lockptr = lmalloc(nlocks * 2 * sizeof (mutex_t *));
3057 	(void) memcpy(lockptr, self->ul_heldlocks.array,
3058 	    nlocks * sizeof (mutex_t *));
3059 	lfree(self->ul_heldlocks.array, nlocks * sizeof (mutex_t *));
3060 	self->ul_heldlocks.array = lockptr;
3061 	self->ul_heldlockcnt *= 2;
3062 	/*
3063 	 * Return the next available entry in the newly allocated array.
3064 	 */
3065 	*(lockptr += nlocks) = lock;
3066 	return (lockptr);
3067 }
3068 
3069 /*
3070  * Insert 'lock' into our list of held locks.
3071  * Currently only used for LOCK_ROBUST mutexes.
3072  */
3073 void
remember_lock(mutex_t * lock)3074 remember_lock(mutex_t *lock)
3075 {
3076 	(void) find_lock_entry(lock);
3077 }
3078 
3079 /*
3080  * Remove 'lock' from our list of held locks.
3081  * Currently only used for LOCK_ROBUST mutexes.
3082  */
3083 void
forget_lock(mutex_t * lock)3084 forget_lock(mutex_t *lock)
3085 {
3086 	*find_lock_entry(lock) = NULL;
3087 }
3088 
3089 /*
3090  * Free the array of held locks.
3091  */
3092 void
heldlock_free(ulwp_t * ulwp)3093 heldlock_free(ulwp_t *ulwp)
3094 {
3095 	uint_t nlocks;
3096 
3097 	if ((nlocks = ulwp->ul_heldlockcnt) != 0)
3098 		lfree(ulwp->ul_heldlocks.array, nlocks * sizeof (mutex_t *));
3099 	ulwp->ul_heldlockcnt = 0;
3100 	ulwp->ul_heldlocks.array = NULL;
3101 }
3102 
3103 /*
3104  * Mark all held LOCK_ROBUST mutexes LOCK_OWNERDEAD.
3105  * Called from _thrp_exit() to deal with abandoned locks.
3106  */
3107 void
heldlock_exit(void)3108 heldlock_exit(void)
3109 {
3110 	ulwp_t *self = curthread;
3111 	mutex_t **lockptr;
3112 	uint_t nlocks;
3113 	mutex_t *mp;
3114 
3115 	if ((nlocks = self->ul_heldlockcnt) != 0)
3116 		lockptr = self->ul_heldlocks.array;
3117 	else {
3118 		nlocks = 1;
3119 		lockptr = &self->ul_heldlocks.single;
3120 	}
3121 
3122 	for (; nlocks; nlocks--, lockptr++) {
3123 		/*
3124 		 * The kernel takes care of transitioning held
3125 		 * LOCK_PRIO_INHERIT mutexes to LOCK_OWNERDEAD.
3126 		 * We avoid that case here.
3127 		 */
3128 		if ((mp = *lockptr) != NULL &&
3129 		    mutex_held(mp) &&
3130 		    (mp->mutex_type & (LOCK_ROBUST | LOCK_PRIO_INHERIT)) ==
3131 		    LOCK_ROBUST) {
3132 			mp->mutex_rcount = 0;
3133 			if (!(mp->mutex_flag & LOCK_UNMAPPED))
3134 				mp->mutex_flag |= LOCK_OWNERDEAD;
3135 			(void) mutex_unlock_internal(mp, 1);
3136 		}
3137 	}
3138 
3139 	heldlock_free(self);
3140 }
3141 
3142 #pragma weak _cond_init = cond_init
3143 /* ARGSUSED2 */
3144 int
cond_init(cond_t * cvp,int type,void * arg)3145 cond_init(cond_t *cvp, int type, void *arg)
3146 {
3147 	if (type != USYNC_THREAD && type != USYNC_PROCESS)
3148 		return (EINVAL);
3149 	(void) memset(cvp, 0, sizeof (*cvp));
3150 	cvp->cond_type = (uint16_t)type;
3151 	cvp->cond_magic = COND_MAGIC;
3152 
3153 	/*
3154 	 * This should be at the beginning of the function,
3155 	 * but for the sake of old broken applications that
3156 	 * do not have proper alignment for their condvars
3157 	 * (and don't check the return code from cond_init),
3158 	 * we put it here, after initializing the condvar regardless.
3159 	 */
3160 	if (((uintptr_t)cvp & (_LONG_LONG_ALIGNMENT - 1)) &&
3161 	    curthread->ul_misaligned == 0)
3162 		return (EINVAL);
3163 
3164 	return (0);
3165 }
3166 
3167 /*
3168  * cond_sleep_queue(): utility function for cond_wait_queue().
3169  *
3170  * Go to sleep on a condvar sleep queue, expect to be waked up
3171  * by someone calling cond_signal() or cond_broadcast() or due
3172  * to receiving a UNIX signal or being cancelled, or just simply
3173  * due to a spurious wakeup (like someome calling forkall()).
3174  *
3175  * The associated mutex is *not* reacquired before returning.
3176  * That must be done by the caller of cond_sleep_queue().
3177  */
3178 static int
cond_sleep_queue(cond_t * cvp,mutex_t * mp,timespec_t * tsp)3179 cond_sleep_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
3180 {
3181 	ulwp_t *self = curthread;
3182 	queue_head_t *qp;
3183 	queue_head_t *mqp;
3184 	lwpid_t lwpid;
3185 	int signalled;
3186 	int error;
3187 	int cv_wake;
3188 	int release_all;
3189 
3190 	/*
3191 	 * Put ourself on the CV sleep queue, unlock the mutex, then
3192 	 * park ourself and unpark a candidate lwp to grab the mutex.
3193 	 * We must go onto the CV sleep queue before dropping the
3194 	 * mutex in order to guarantee atomicity of the operation.
3195 	 */
3196 	self->ul_sp = stkptr();
3197 	qp = queue_lock(cvp, CV);
3198 	enqueue(qp, self, 0);
3199 	cvp->cond_waiters_user = 1;
3200 	self->ul_cvmutex = mp;
3201 	self->ul_cv_wake = cv_wake = (tsp != NULL);
3202 	self->ul_signalled = 0;
3203 	if (mp->mutex_flag & LOCK_OWNERDEAD) {
3204 		mp->mutex_flag &= ~LOCK_OWNERDEAD;
3205 		mp->mutex_flag |= LOCK_NOTRECOVERABLE;
3206 	}
3207 	release_all = ((mp->mutex_flag & LOCK_NOTRECOVERABLE) != 0);
3208 	lwpid = mutex_unlock_queue(mp, release_all);
3209 	for (;;) {
3210 		set_parking_flag(self, 1);
3211 		queue_unlock(qp);
3212 		if (lwpid != 0) {
3213 			lwpid = preempt_unpark(self, lwpid);
3214 			preempt(self);
3215 		}
3216 		/*
3217 		 * We may have a deferred signal present,
3218 		 * in which case we should return EINTR.
3219 		 * Also, we may have received a SIGCANCEL; if so
3220 		 * and we are cancelable we should return EINTR.
3221 		 * We force an immediate EINTR return from
3222 		 * __lwp_park() by turning our parking flag off.
3223 		 */
3224 		if (self->ul_cursig != 0 ||
3225 		    (self->ul_cancelable && self->ul_cancel_pending))
3226 			set_parking_flag(self, 0);
3227 		/*
3228 		 * __lwp_park() will return the residual time in tsp
3229 		 * if we are unparked before the timeout expires.
3230 		 */
3231 		error = __lwp_park(tsp, lwpid);
3232 		set_parking_flag(self, 0);
3233 		lwpid = 0;	/* unpark the other lwp only once */
3234 		/*
3235 		 * We were waked up by cond_signal(), cond_broadcast(),
3236 		 * by an interrupt or timeout (EINTR or ETIME),
3237 		 * or we may just have gotten a spurious wakeup.
3238 		 */
3239 		qp = queue_lock(cvp, CV);
3240 		if (!cv_wake)
3241 			mqp = queue_lock(mp, MX);
3242 		if (self->ul_sleepq == NULL)
3243 			break;
3244 		/*
3245 		 * We are on either the condvar sleep queue or the
3246 		 * mutex sleep queue.  Break out of the sleep if we
3247 		 * were interrupted or we timed out (EINTR or ETIME).
3248 		 * Else this is a spurious wakeup; continue the loop.
3249 		 */
3250 		if (!cv_wake && self->ul_sleepq == mqp) { /* mutex queue */
3251 			if (error) {
3252 				mp->mutex_waiters = dequeue_self(mqp);
3253 				break;
3254 			}
3255 			tsp = NULL;	/* no more timeout */
3256 		} else if (self->ul_sleepq == qp) {	/* condvar queue */
3257 			if (error) {
3258 				cvp->cond_waiters_user = dequeue_self(qp);
3259 				break;
3260 			}
3261 			/*
3262 			 * Else a spurious wakeup on the condvar queue.
3263 			 * __lwp_park() has already adjusted the timeout.
3264 			 */
3265 		} else {
3266 			thr_panic("cond_sleep_queue(): thread not on queue");
3267 		}
3268 		if (!cv_wake)
3269 			queue_unlock(mqp);
3270 	}
3271 
3272 	self->ul_sp = 0;
3273 	self->ul_cv_wake = 0;
3274 	ASSERT(self->ul_cvmutex == NULL);
3275 	ASSERT(self->ul_sleepq == NULL && self->ul_link == NULL &&
3276 	    self->ul_wchan == NULL);
3277 
3278 	signalled = self->ul_signalled;
3279 	self->ul_signalled = 0;
3280 	queue_unlock(qp);
3281 	if (!cv_wake)
3282 		queue_unlock(mqp);
3283 
3284 	/*
3285 	 * If we were concurrently cond_signal()d and any of:
3286 	 * received a UNIX signal, were cancelled, or got a timeout,
3287 	 * then perform another cond_signal() to avoid consuming it.
3288 	 */
3289 	if (error && signalled)
3290 		(void) cond_signal(cvp);
3291 
3292 	return (error);
3293 }
3294 
3295 static void
cond_wait_check_alignment(cond_t * cvp,mutex_t * mp)3296 cond_wait_check_alignment(cond_t *cvp, mutex_t *mp)
3297 {
3298 	if ((uintptr_t)mp & (_LONG_LONG_ALIGNMENT - 1))
3299 		lock_error(mp, "cond_wait", cvp, "mutex is misaligned");
3300 	if ((uintptr_t)cvp & (_LONG_LONG_ALIGNMENT - 1))
3301 		lock_error(mp, "cond_wait", cvp, "condvar is misaligned");
3302 }
3303 
3304 int
cond_wait_queue(cond_t * cvp,mutex_t * mp,timespec_t * tsp)3305 cond_wait_queue(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
3306 {
3307 	ulwp_t *self = curthread;
3308 	int error;
3309 	int merror;
3310 
3311 	if (self->ul_error_detection && self->ul_misaligned == 0)
3312 		cond_wait_check_alignment(cvp, mp);
3313 
3314 	/*
3315 	 * The old thread library was programmed to defer signals
3316 	 * while in cond_wait() so that the associated mutex would
3317 	 * be guaranteed to be held when the application signal
3318 	 * handler was invoked.
3319 	 *
3320 	 * We do not behave this way by default; the state of the
3321 	 * associated mutex in the signal handler is undefined.
3322 	 *
3323 	 * To accommodate applications that depend on the old
3324 	 * behavior, the _THREAD_COND_WAIT_DEFER environment
3325 	 * variable can be set to 1 and we will behave in the
3326 	 * old way with respect to cond_wait().
3327 	 */
3328 	if (self->ul_cond_wait_defer)
3329 		sigoff(self);
3330 
3331 	error = cond_sleep_queue(cvp, mp, tsp);
3332 
3333 	/*
3334 	 * Reacquire the mutex.
3335 	 */
3336 	if ((merror = mutex_lock_impl(mp, NULL)) != 0)
3337 		error = merror;
3338 
3339 	/*
3340 	 * Take any deferred signal now, after we have reacquired the mutex.
3341 	 */
3342 	if (self->ul_cond_wait_defer)
3343 		sigon(self);
3344 
3345 	return (error);
3346 }
3347 
3348 /*
3349  * cond_sleep_kernel(): utility function for cond_wait_kernel().
3350  * See the comment ahead of cond_sleep_queue(), above.
3351  */
3352 static int
cond_sleep_kernel(cond_t * cvp,mutex_t * mp,timespec_t * tsp)3353 cond_sleep_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
3354 {
3355 	int mtype = mp->mutex_type;
3356 	ulwp_t *self = curthread;
3357 	int error;
3358 
3359 	if ((mtype & LOCK_PRIO_PROTECT) && _ceil_mylist_del(mp))
3360 		_ceil_prio_waive();
3361 
3362 	self->ul_sp = stkptr();
3363 	self->ul_wchan = cvp;
3364 	sigoff(self);
3365 	mp->mutex_owner = 0;
3366 	/* mp->mutex_ownerpid is cleared by ___lwp_cond_wait() */
3367 	if (mtype & LOCK_PRIO_INHERIT) {
3368 		mp->mutex_lockw = LOCKCLEAR;
3369 		self->ul_pilocks--;
3370 	}
3371 	/*
3372 	 * ___lwp_cond_wait() returns immediately with EINTR if
3373 	 * set_parking_flag(self,0) is called on this lwp before it
3374 	 * goes to sleep in the kernel.  sigacthandler() calls this
3375 	 * when a deferred signal is noted.  This assures that we don't
3376 	 * get stuck in ___lwp_cond_wait() with all signals blocked
3377 	 * due to taking a deferred signal before going to sleep.
3378 	 */
3379 	set_parking_flag(self, 1);
3380 	if (self->ul_cursig != 0 ||
3381 	    (self->ul_cancelable && self->ul_cancel_pending))
3382 		set_parking_flag(self, 0);
3383 	error = ___lwp_cond_wait(cvp, mp, tsp, 1);
3384 	set_parking_flag(self, 0);
3385 	sigon(self);
3386 	self->ul_sp = 0;
3387 	self->ul_wchan = NULL;
3388 	return (error);
3389 }
3390 
3391 int
cond_wait_kernel(cond_t * cvp,mutex_t * mp,timespec_t * tsp)3392 cond_wait_kernel(cond_t *cvp, mutex_t *mp, timespec_t *tsp)
3393 {
3394 	ulwp_t *self = curthread;
3395 	int error;
3396 	int merror;
3397 
3398 	if (self->ul_error_detection && self->ul_misaligned == 0)
3399 		cond_wait_check_alignment(cvp, mp);
3400 
3401 	/*
3402 	 * See the large comment in cond_wait_queue(), above.
3403 	 */
3404 	if (