xref: /illumos-gate/usr/src/uts/common/os/mutex.c (revision 374ae87f60894937d3c6e53ec4a739188e702ea5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * Big Theory Statement for mutual exclusion locking primitives.
30  *
31  * A mutex serializes multiple threads so that only one thread
32  * (the "owner" of the mutex) is active at a time.  See mutex(9F)
33  * for a full description of the interfaces and programming model.
34  * The rest of this comment describes the implementation.
35  *
36  * Mutexes come in two flavors: adaptive and spin.  mutex_init(9F)
37  * determines the type based solely on the iblock cookie (PIL) argument.
38  * PIL > LOCK_LEVEL implies a spin lock; everything else is adaptive.
39  *
40  * Spin mutexes block interrupts and spin until the lock becomes available.
41  * A thread may not sleep, or call any function that might sleep, while
42  * holding a spin mutex.  With few exceptions, spin mutexes should only
43  * be used to synchronize with interrupt handlers.
44  *
45  * Adaptive mutexes (the default type) spin if the owner is running on
46  * another CPU and block otherwise.  This policy is based on the assumption
47  * that mutex hold times are typically short enough that the time spent
48  * spinning is less than the time it takes to block.  If you need mutual
49  * exclusion semantics with long hold times, consider an rwlock(9F) as
50  * RW_WRITER.  Better still, reconsider the algorithm: if it requires
51  * mutual exclusion for long periods of time, it's probably not scalable.
52  *
53  * Adaptive mutexes are overwhelmingly more common than spin mutexes,
54  * so mutex_enter() assumes that the lock is adaptive.  We get away
55  * with this by structuring mutexes so that an attempt to acquire a
56  * spin mutex as adaptive always fails.  When mutex_enter() fails
57  * it punts to mutex_vector_enter(), which does all the hard stuff.
58  *
59  * mutex_vector_enter() first checks the type.  If it's spin mutex,
60  * we just call lock_set_spl() and return.  If it's an adaptive mutex,
61  * we check to see what the owner is doing.  If the owner is running,
62  * we spin until the lock becomes available; if not, we mark the lock
63  * as having waiters and block.
64  *
65  * Blocking on a mutex is surprisingly delicate dance because, for speed,
66  * mutex_exit() doesn't use an atomic instruction.  Thus we have to work
67  * a little harder in the (rarely-executed) blocking path to make sure
68  * we don't block on a mutex that's just been released -- otherwise we
69  * might never be woken up.
70  *
71  * The logic for synchronizing mutex_vector_enter() with mutex_exit()
72  * in the face of preemption and relaxed memory ordering is as follows:
73  *
74  * (1) Preemption in the middle of mutex_exit() must cause mutex_exit()
75  *     to restart.  Each platform must enforce this by checking the
76  *     interrupted PC in the interrupt handler (or on return from trap --
77  *     whichever is more convenient for the platform).  If the PC
78  *     lies within the critical region of mutex_exit(), the interrupt
79  *     handler must reset the PC back to the beginning of mutex_exit().
80  *     The critical region consists of all instructions up to, but not
81  *     including, the store that clears the lock (which, of course,
82  *     must never be executed twice.)
83  *
84  *     This ensures that the owner will always check for waiters after
85  *     resuming from a previous preemption.
86  *
87  * (2) A thread resuming in mutex_exit() does (at least) the following:
88  *
89  *	when resuming:	set CPU_THREAD = owner
90  *			membar #StoreLoad
91  *
92  *	in mutex_exit:	check waiters bit; do wakeup if set
93  *			membar #LoadStore|#StoreStore
94  *			clear owner
95  *			(at this point, other threads may or may not grab
96  *			the lock, and we may or may not reacquire it)
97  *
98  *	when blocking:	membar #StoreStore (due to disp_lock_enter())
99  *			set CPU_THREAD = (possibly) someone else
100  *
101  * (3) A thread blocking in mutex_vector_enter() does the following:
102  *
103  *			set waiters bit
104  *			membar #StoreLoad (via membar_enter())
105  *			check CPU_THREAD for owner's t_cpu
106  *				continue if owner running
107  *			membar #LoadLoad (via membar_consumer())
108  *			check owner and waiters bit; abort if either changed
109  *			block
110  *
111  * Thus the global memory orderings for (2) and (3) are as follows:
112  *
113  * (2M) mutex_exit() memory order:
114  *
115  *			STORE	CPU_THREAD = owner
116  *			LOAD	waiters bit
117  *			STORE	owner = NULL
118  *			STORE	CPU_THREAD = (possibly) someone else
119  *
120  * (3M) mutex_vector_enter() memory order:
121  *
122  *			STORE	waiters bit = 1
123  *			LOAD	CPU_THREAD for each CPU
124  *			LOAD	owner and waiters bit
125  *
126  * It has been verified by exhaustive simulation that all possible global
127  * memory orderings of (2M) interleaved with (3M) result in correct
128  * behavior.  Moreover, these ordering constraints are minimal: changing
129  * the ordering of anything in (2M) or (3M) breaks the algorithm, creating
130  * windows for missed wakeups.  Note: the possibility that other threads
131  * may grab the lock after the owner drops it can be factored out of the
132  * memory ordering analysis because mutex_vector_enter() won't block
133  * if the lock isn't still owned by the same thread.
134  *
135  * The only requirements of code outside the mutex implementation are
136  * (1) mutex_exit() preemption fixup in interrupt handlers or trap return,
137  * (2) a membar #StoreLoad after setting CPU_THREAD in resume(),
138  * (3) mutex_owner_running() preemption fixup in interrupt handlers
139  * or trap returns.
140  * Note: idle threads cannot grab adaptive locks (since they cannot block),
141  * so the membar may be safely omitted when resuming an idle thread.
142  *
143  * When a mutex has waiters, mutex_vector_exit() has several options:
144  *
145  * (1) Choose a waiter and make that thread the owner before waking it;
146  *     this is known as "direct handoff" of ownership.
147  *
148  * (2) Drop the lock and wake one waiter.
149  *
150  * (3) Drop the lock, clear the waiters bit, and wake all waiters.
151  *
152  * In many ways (1) is the cleanest solution, but if a lock is moderately
153  * contended it defeats the adaptive spin logic.  If we make some other
154  * thread the owner, but he's not ONPROC yet, then all other threads on
155  * other cpus that try to get the lock will conclude that the owner is
156  * blocked, so they'll block too.  And so on -- it escalates quickly,
157  * with every thread taking the blocking path rather than the spin path.
158  * Thus, direct handoff is *not* a good idea for adaptive mutexes.
159  *
160  * Option (2) is the next most natural-seeming option, but it has several
161  * annoying properties.  If there's more than one waiter, we must preserve
162  * the waiters bit on an unheld lock.  On cas-capable platforms, where
163  * the waiters bit is part of the lock word, this means that both 0x0
164  * and 0x1 represent unheld locks, so we have to cas against *both*.
165  * Priority inheritance also gets more complicated, because a lock can
166  * have waiters but no owner to whom priority can be willed.  So while
167  * it is possible to make option (2) work, it's surprisingly vile.
168  *
169  * Option (3), the least-intuitive at first glance, is what we actually do.
170  * It has the advantage that because you always wake all waiters, you
171  * never have to preserve the waiters bit.  Waking all waiters seems like
172  * begging for a thundering herd problem, but consider: under option (2),
173  * every thread that grabs and drops the lock will wake one waiter -- so
174  * if the lock is fairly active, all waiters will be awakened very quickly
175  * anyway.  Moreover, this is how adaptive locks are *supposed* to work.
176  * The blocking case is rare; the more common case (by 3-4 orders of
177  * magnitude) is that one or more threads spin waiting to get the lock.
178  * Only direct handoff can prevent the thundering herd problem, but as
179  * mentioned earlier, that would tend to defeat the adaptive spin logic.
180  * In practice, option (3) works well because the blocking case is rare.
181  */
182 
183 /*
184  * delayed lock retry with exponential delay for spin locks
185  *
186  * It is noted above that for both the spin locks and the adaptive locks,
187  * spinning is the dominate mode of operation.  So long as there is only
188  * one thread waiting on a lock, the naive spin loop works very well in
189  * cache based architectures.  The lock data structure is pulled into the
190  * cache of the processor with the waiting/spinning thread and no further
191  * memory traffic is generated until the lock is released.  Unfortunately,
192  * once two or more threads are waiting on a lock, the naive spin has
193  * the property of generating maximum memory traffic from each spinning
194  * thread as the spinning threads contend for the lock data structure.
195  *
196  * By executing a delay loop before retrying a lock, a waiting thread
197  * can reduce its memory traffic by a large factor, depending on the
198  * size of the delay loop.  A large delay loop greatly reduced the memory
199  * traffic, but has the drawback of having a period of time when
200  * no thread is attempting to gain the lock even though several threads
201  * might be waiting.  A small delay loop has the drawback of not
202  * much reduction in memory traffic, but reduces the potential idle time.
203  * The theory of the exponential delay code is to start with a short
204  * delay loop and double the waiting time on each iteration, up to
205  * a preselected maximum.
206  */
207 
208 #include <sys/param.h>
209 #include <sys/time.h>
210 #include <sys/cpuvar.h>
211 #include <sys/thread.h>
212 #include <sys/debug.h>
213 #include <sys/cmn_err.h>
214 #include <sys/sobject.h>
215 #include <sys/turnstile.h>
216 #include <sys/systm.h>
217 #include <sys/mutex_impl.h>
218 #include <sys/spl.h>
219 #include <sys/lockstat.h>
220 #include <sys/atomic.h>
221 #include <sys/cpu.h>
222 #include <sys/stack.h>
223 #include <sys/archsystm.h>
224 #include <sys/machsystm.h>
225 #include <sys/x_call.h>
226 
227 /*
228  * The sobj_ops vector exports a set of functions needed when a thread
229  * is asleep on a synchronization object of this type.
230  */
231 static sobj_ops_t mutex_sobj_ops = {
232 	SOBJ_MUTEX, mutex_owner, turnstile_stay_asleep, turnstile_change_pri
233 };
234 
235 /*
236  * If the system panics on a mutex, save the address of the offending
237  * mutex in panic_mutex_addr, and save the contents in panic_mutex.
238  */
239 static mutex_impl_t panic_mutex;
240 static mutex_impl_t *panic_mutex_addr;
241 
242 static void
243 mutex_panic(char *msg, mutex_impl_t *lp)
244 {
245 	if (panicstr)
246 		return;
247 
248 	if (casptr(&panic_mutex_addr, NULL, lp) == NULL)
249 		panic_mutex = *lp;
250 
251 	panic("%s, lp=%p owner=%p thread=%p",
252 	    msg, lp, MUTEX_OWNER(&panic_mutex), curthread);
253 }
254 
255 /* "tunables" for per-platform backoff constants. */
256 uint_t mutex_backoff_cap = 0;
257 ushort_t mutex_backoff_base = MUTEX_BACKOFF_BASE;
258 ushort_t mutex_cap_factor = MUTEX_CAP_FACTOR;
259 uchar_t mutex_backoff_shift = MUTEX_BACKOFF_SHIFT;
260 
261 void
262 mutex_sync(void)
263 {
264 	MUTEX_SYNC();
265 }
266 
267 /* calculate the backoff interval */
268 uint_t
269 default_lock_backoff(uint_t backoff)
270 {
271 	uint_t cap;		/* backoff cap calculated */
272 
273 	if (backoff == 0) {
274 		backoff = mutex_backoff_base;
275 		/* first call just sets the base */
276 		return (backoff);
277 	}
278 
279 	/* set cap */
280 	if (mutex_backoff_cap == 0) {
281 		/*
282 		 * For a contended lock, in the worst case a load + cas may
283 		 * be queued  at the controller for each contending CPU.
284 		 * Therefore, to avoid queueing, the accesses for all CPUS must
285 		 * be spread out in time over an interval of (ncpu *
286 		 * cap-factor).  Maximum backoff is set to this value, and
287 		 * actual backoff is a random number from 0 to the current max.
288 		 */
289 		cap = ncpus_online * mutex_cap_factor;
290 	} else {
291 		cap = mutex_backoff_cap;
292 	}
293 
294 	/* calculate new backoff value */
295 	backoff <<= mutex_backoff_shift;	/* increase backoff */
296 	if (backoff > cap) {
297 		if (cap < mutex_backoff_base)
298 			backoff = mutex_backoff_base;
299 		else
300 			backoff = cap;
301 	}
302 
303 	return (backoff);
304 }
305 
306 /*
307  * default delay function for mutexes.
308  */
309 void
310 default_lock_delay(uint_t backoff)
311 {
312 	ulong_t rnd;		/* random factor */
313 	uint_t cur_backoff;	/* calculated backoff */
314 	uint_t backctr;
315 
316 	/*
317 	 * Modify backoff by a random amount to avoid lockstep, and to
318 	 * make it probable that some thread gets a small backoff, and
319 	 * re-checks quickly
320 	 */
321 	rnd = (((long)curthread >> PTR24_LSB) ^ (long)MUTEX_GETTICK());
322 	cur_backoff = (uint_t)(rnd % (backoff - mutex_backoff_base + 1)) +
323 	    mutex_backoff_base;
324 
325 	/*
326 	 * Delay before trying
327 	 * to touch the mutex data structure.
328 	 */
329 	for (backctr = cur_backoff; backctr; backctr--) {
330 		MUTEX_DELAY();
331 	};
332 }
333 
334 uint_t (*mutex_lock_backoff)(uint_t) = default_lock_backoff;
335 void (*mutex_lock_delay)(uint_t) = default_lock_delay;
336 void (*mutex_delay)(void) = mutex_delay_default;
337 
338 /*
339  * mutex_vector_enter() is called from the assembly mutex_enter() routine
340  * if the lock is held or is not of type MUTEX_ADAPTIVE.
341  */
342 void
343 mutex_vector_enter(mutex_impl_t *lp)
344 {
345 	kthread_id_t	owner;
346 	kthread_id_t	lastowner = MUTEX_NO_OWNER; /* track owner changes */
347 	hrtime_t	sleep_time = 0;	/* how long we slept */
348 	hrtime_t	spin_time = 0;	/* how long we spun */
349 	cpu_t 		*cpup;
350 	turnstile_t	*ts;
351 	volatile mutex_impl_t *vlp = (volatile mutex_impl_t *)lp;
352 	uint_t		backoff = 0;	/* current backoff */
353 	int		changecnt = 0;	/* count of owner changes */
354 
355 	ASSERT_STACK_ALIGNED();
356 
357 	if (MUTEX_TYPE_SPIN(lp)) {
358 		lock_set_spl(&lp->m_spin.m_spinlock, lp->m_spin.m_minspl,
359 		    &lp->m_spin.m_oldspl);
360 		return;
361 	}
362 
363 	if (!MUTEX_TYPE_ADAPTIVE(lp)) {
364 		mutex_panic("mutex_enter: bad mutex", lp);
365 		return;
366 	}
367 
368 	/*
369 	 * Adaptive mutexes must not be acquired from above LOCK_LEVEL.
370 	 * We can migrate after loading CPU but before checking CPU_ON_INTR,
371 	 * so we must verify by disabling preemption and loading CPU again.
372 	 */
373 	cpup = CPU;
374 	if (CPU_ON_INTR(cpup) && !panicstr) {
375 		kpreempt_disable();
376 		if (CPU_ON_INTR(CPU))
377 			mutex_panic("mutex_enter: adaptive at high PIL", lp);
378 		kpreempt_enable();
379 	}
380 
381 	CPU_STATS_ADDQ(cpup, sys, mutex_adenters, 1);
382 
383 	spin_time = LOCKSTAT_START_TIME(LS_MUTEX_ENTER_SPIN);
384 
385 	backoff = mutex_lock_backoff(0);	/* set base backoff */
386 	for (;;) {
387 		mutex_lock_delay(backoff); /* backoff delay */
388 
389 		if (panicstr)
390 			return;
391 
392 		if ((owner = MUTEX_OWNER(vlp)) == NULL) {
393 			if (mutex_adaptive_tryenter(lp)) {
394 				break;
395 			}
396 			/* increase backoff only on failed attempt. */
397 			backoff = mutex_lock_backoff(backoff);
398 			changecnt++;
399 			continue;
400 		} else if (lastowner != owner) {
401 			lastowner = owner;
402 			backoff = mutex_lock_backoff(backoff);
403 			changecnt++;
404 		}
405 
406 		if (changecnt >= ncpus_online) {
407 			backoff = mutex_lock_backoff(0);
408 			changecnt = 0;
409 		}
410 
411 		if (owner == curthread)
412 			mutex_panic("recursive mutex_enter", lp);
413 
414 		/*
415 		 * If lock is held but owner is not yet set, spin.
416 		 * (Only relevant for platforms that don't have cas.)
417 		 */
418 		if (owner == MUTEX_NO_OWNER)
419 			continue;
420 
421 		if (mutex_owner_running(lp) != NULL)  {
422 			continue;
423 		}
424 
425 		/*
426 		 * The owner appears not to be running, so block.
427 		 * See the Big Theory Statement for memory ordering issues.
428 		 */
429 		ts = turnstile_lookup(lp);
430 		MUTEX_SET_WAITERS(lp);
431 		membar_enter();
432 
433 		/*
434 		 * Recheck whether owner is running after waiters bit hits
435 		 * global visibility (above).  If owner is running, spin.
436 		 */
437 		if (mutex_owner_running(lp) != NULL) {
438 			turnstile_exit(lp);
439 			continue;
440 		}
441 		membar_consumer();
442 
443 		/*
444 		 * If owner and waiters bit are unchanged, block.
445 		 */
446 		if (MUTEX_OWNER(vlp) == owner && MUTEX_HAS_WAITERS(vlp)) {
447 			sleep_time -= gethrtime();
448 			(void) turnstile_block(ts, TS_WRITER_Q, lp,
449 			    &mutex_sobj_ops, NULL, NULL);
450 			sleep_time += gethrtime();
451 			/* reset backoff after turnstile */
452 			backoff = mutex_lock_backoff(0);
453 		} else {
454 			turnstile_exit(lp);
455 		}
456 	}
457 
458 	ASSERT(MUTEX_OWNER(lp) == curthread);
459 
460 	if (sleep_time != 0) {
461 		/*
462 		 * Note, sleep time is the sum of all the sleeping we
463 		 * did.
464 		 */
465 		LOCKSTAT_RECORD(LS_MUTEX_ENTER_BLOCK, lp, sleep_time);
466 	}
467 
468 	/* record spin time, don't count sleep time */
469 	if (spin_time != 0) {
470 		LOCKSTAT_RECORD_TIME(LS_MUTEX_ENTER_SPIN, lp,
471 		    spin_time + sleep_time);
472 	}
473 
474 	LOCKSTAT_RECORD0(LS_MUTEX_ENTER_ACQUIRE, lp);
475 }
476 
477 /*
478  * mutex_vector_tryenter() is called from the assembly mutex_tryenter()
479  * routine if the lock is held or is not of type MUTEX_ADAPTIVE.
480  */
481 int
482 mutex_vector_tryenter(mutex_impl_t *lp)
483 {
484 	int s;
485 
486 	if (MUTEX_TYPE_ADAPTIVE(lp))
487 		return (0);		/* we already tried in assembly */
488 
489 	if (!MUTEX_TYPE_SPIN(lp)) {
490 		mutex_panic("mutex_tryenter: bad mutex", lp);
491 		return (0);
492 	}
493 
494 	s = splr(lp->m_spin.m_minspl);
495 	if (lock_try(&lp->m_spin.m_spinlock)) {
496 		lp->m_spin.m_oldspl = (ushort_t)s;
497 		return (1);
498 	}
499 	splx(s);
500 	return (0);
501 }
502 
503 /*
504  * mutex_vector_exit() is called from mutex_exit() if the lock is not
505  * adaptive, has waiters, or is not owned by the current thread (panic).
506  */
507 void
508 mutex_vector_exit(mutex_impl_t *lp)
509 {
510 	turnstile_t *ts;
511 
512 	if (MUTEX_TYPE_SPIN(lp)) {
513 		lock_clear_splx(&lp->m_spin.m_spinlock, lp->m_spin.m_oldspl);
514 		return;
515 	}
516 
517 	if (MUTEX_OWNER(lp) != curthread) {
518 		mutex_panic("mutex_exit: not owner", lp);
519 		return;
520 	}
521 
522 	ts = turnstile_lookup(lp);
523 	MUTEX_CLEAR_LOCK_AND_WAITERS(lp);
524 	if (ts == NULL)
525 		turnstile_exit(lp);
526 	else
527 		turnstile_wakeup(ts, TS_WRITER_Q, ts->ts_waiters, NULL);
528 	LOCKSTAT_RECORD0(LS_MUTEX_EXIT_RELEASE, lp);
529 }
530 
531 int
532 mutex_owned(kmutex_t *mp)
533 {
534 	mutex_impl_t *lp = (mutex_impl_t *)mp;
535 
536 	if (panicstr)
537 		return (1);
538 
539 	if (MUTEX_TYPE_ADAPTIVE(lp))
540 		return (MUTEX_OWNER(lp) == curthread);
541 	return (LOCK_HELD(&lp->m_spin.m_spinlock));
542 }
543 
544 kthread_t *
545 mutex_owner(kmutex_t *mp)
546 {
547 	mutex_impl_t *lp = (mutex_impl_t *)mp;
548 	kthread_id_t t;
549 
550 	if (MUTEX_TYPE_ADAPTIVE(lp) && (t = MUTEX_OWNER(lp)) != MUTEX_NO_OWNER)
551 		return (t);
552 	return (NULL);
553 }
554 
555 /*
556  * The iblock cookie 'ibc' is the spl level associated with the lock;
557  * this alone determines whether the lock will be ADAPTIVE or SPIN.
558  *
559  * Adaptive mutexes created in zeroed memory do not need to call
560  * mutex_init() as their allocation in this fashion guarantees
561  * their initialization.
562  *   eg adaptive mutexes created as static within the BSS or allocated
563  *      by kmem_zalloc().
564  */
565 /* ARGSUSED */
566 void
567 mutex_init(kmutex_t *mp, char *name, kmutex_type_t type, void *ibc)
568 {
569 	mutex_impl_t *lp = (mutex_impl_t *)mp;
570 
571 	ASSERT(ibc < (void *)KERNELBASE);	/* see 1215173 */
572 
573 	if ((intptr_t)ibc > ipltospl(LOCK_LEVEL) && ibc < (void *)KERNELBASE) {
574 		ASSERT(type != MUTEX_ADAPTIVE && type != MUTEX_DEFAULT);
575 		MUTEX_SET_TYPE(lp, MUTEX_SPIN);
576 		LOCK_INIT_CLEAR(&lp->m_spin.m_spinlock);
577 		LOCK_INIT_HELD(&lp->m_spin.m_dummylock);
578 		lp->m_spin.m_minspl = (int)(intptr_t)ibc;
579 	} else {
580 		ASSERT(type != MUTEX_SPIN);
581 		MUTEX_SET_TYPE(lp, MUTEX_ADAPTIVE);
582 		MUTEX_CLEAR_LOCK_AND_WAITERS(lp);
583 	}
584 }
585 
586 void
587 mutex_destroy(kmutex_t *mp)
588 {
589 	mutex_impl_t *lp = (mutex_impl_t *)mp;
590 
591 	if (lp->m_owner == 0 && !MUTEX_HAS_WAITERS(lp)) {
592 		MUTEX_DESTROY(lp);
593 	} else if (MUTEX_TYPE_SPIN(lp)) {
594 		LOCKSTAT_RECORD0(LS_MUTEX_DESTROY_RELEASE, lp);
595 		MUTEX_DESTROY(lp);
596 	} else if (MUTEX_TYPE_ADAPTIVE(lp)) {
597 		LOCKSTAT_RECORD0(LS_MUTEX_DESTROY_RELEASE, lp);
598 		if (MUTEX_OWNER(lp) != curthread)
599 			mutex_panic("mutex_destroy: not owner", lp);
600 		if (MUTEX_HAS_WAITERS(lp)) {
601 			turnstile_t *ts = turnstile_lookup(lp);
602 			turnstile_exit(lp);
603 			if (ts != NULL)
604 				mutex_panic("mutex_destroy: has waiters", lp);
605 		}
606 		MUTEX_DESTROY(lp);
607 	} else {
608 		mutex_panic("mutex_destroy: bad mutex", lp);
609 	}
610 }
611 
612 /*
613  * Simple C support for the cases where spin locks miss on the first try.
614  */
615 void
616 lock_set_spin(lock_t *lp)
617 {
618 	int loop_count = 0;
619 	uint_t backoff = 0;	/* current backoff */
620 	hrtime_t spin_time = 0;	/* how long we spun */
621 
622 	if (panicstr)
623 		return;
624 
625 	if (ncpus == 1)
626 		panic("lock_set: %p lock held and only one CPU", lp);
627 
628 	spin_time = LOCKSTAT_START_TIME(LS_LOCK_SET_SPIN);
629 
630 	while (LOCK_HELD(lp) || !lock_spin_try(lp)) {
631 		if (panicstr)
632 			return;
633 		loop_count++;
634 
635 		if (ncpus_online == loop_count) {
636 			backoff = mutex_lock_backoff(0);
637 			loop_count = 0;
638 		} else {
639 			backoff = mutex_lock_backoff(backoff);
640 		}
641 		mutex_lock_delay(backoff);
642 	}
643 
644 	LOCKSTAT_RECORD_TIME(LS_LOCK_SET_SPIN, lp, spin_time);
645 
646 	LOCKSTAT_RECORD0(LS_LOCK_SET_ACQUIRE, lp);
647 }
648 
649 void
650 lock_set_spl_spin(lock_t *lp, int new_pil, ushort_t *old_pil_addr, int old_pil)
651 {
652 	int loop_count = 0;
653 	uint_t backoff = 0;	/* current backoff */
654 	hrtime_t spin_time = 0;	/* how long we spun */
655 
656 	if (panicstr)
657 		return;
658 
659 	if (ncpus == 1)
660 		panic("lock_set_spl: %p lock held and only one CPU", lp);
661 
662 	ASSERT(new_pil > LOCK_LEVEL);
663 
664 	spin_time = LOCKSTAT_START_TIME(LS_LOCK_SET_SPL_SPIN);
665 
666 	do {
667 		splx(old_pil);
668 		while (LOCK_HELD(lp)) {
669 			loop_count++;
670 
671 			if (panicstr) {
672 				*old_pil_addr = (ushort_t)splr(new_pil);
673 				return;
674 			}
675 			if (ncpus_online == loop_count) {
676 				backoff = mutex_lock_backoff(0);
677 				loop_count = 0;
678 			} else {
679 				backoff = mutex_lock_backoff(backoff);
680 			}
681 			mutex_lock_delay(backoff);
682 		}
683 		old_pil = splr(new_pil);
684 	} while (!lock_spin_try(lp));
685 
686 	*old_pil_addr = (ushort_t)old_pil;
687 
688 	LOCKSTAT_RECORD_TIME(LS_LOCK_SET_SPL_SPIN, lp, spin_time);
689 
690 	LOCKSTAT_RECORD0(LS_LOCK_SET_SPL_ACQUIRE, lp);
691 }
692