17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
5575a742pt * Common Development and Distribution License (the "License").
6575a742pt * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
22575a742pt * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
237c478bdstevel@tonic-gate * Use is subject to license terms.
249b0bb79John Levon *
259b0bb79John Levon * Copyright 2020 Joyent, Inc.
267c478bdstevel@tonic-gate */
277c478bdstevel@tonic-gate
287c478bdstevel@tonic-gate#include "assym.h"
297c478bdstevel@tonic-gate
307c478bdstevel@tonic-gate#include <sys/t_lock.h>
317c478bdstevel@tonic-gate#include <sys/mutex.h>
327c478bdstevel@tonic-gate#include <sys/mutex_impl.h>
337c478bdstevel@tonic-gate#include <sys/rwlock_impl.h>
347c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
357c478bdstevel@tonic-gate#include <sys/machlock.h>
367c478bdstevel@tonic-gate#include <sys/machthread.h>
377c478bdstevel@tonic-gate#include <sys/lockstat.h>
387c478bdstevel@tonic-gate
397c478bdstevel@tonic-gate/* #define DEBUG */
407c478bdstevel@tonic-gate
417c478bdstevel@tonic-gate#ifdef DEBUG
427c478bdstevel@tonic-gate#include <sys/machparam.h>
437c478bdstevel@tonic-gate#endif /* DEBUG */
447c478bdstevel@tonic-gate
457c478bdstevel@tonic-gate/************************************************************************
467c478bdstevel@tonic-gate *		ATOMIC OPERATIONS
477c478bdstevel@tonic-gate */
487c478bdstevel@tonic-gate
497c478bdstevel@tonic-gate/*
507c478bdstevel@tonic-gate * uint8_t	ldstub(uint8_t *cp)
517c478bdstevel@tonic-gate *
527c478bdstevel@tonic-gate * Store 0xFF at the specified location, and return its previous content.
537c478bdstevel@tonic-gate */
547c478bdstevel@tonic-gate
557c478bdstevel@tonic-gate	ENTRY(ldstub)
567c478bdstevel@tonic-gate	retl
577c478bdstevel@tonic-gate	ldstub	[%o0], %o0
587c478bdstevel@tonic-gate	SET_SIZE(ldstub)
597c478bdstevel@tonic-gate
607c478bdstevel@tonic-gate/************************************************************************
617c478bdstevel@tonic-gate *		MEMORY BARRIERS -- see atomic.h for full descriptions.
627c478bdstevel@tonic-gate */
637c478bdstevel@tonic-gate
647c478bdstevel@tonic-gate#ifdef SF_ERRATA_51
657c478bdstevel@tonic-gate	.align 32
667c478bdstevel@tonic-gate	ENTRY(membar_return)
677c478bdstevel@tonic-gate	retl
687c478bdstevel@tonic-gate	nop
697c478bdstevel@tonic-gate	SET_SIZE(membar_return)
707c478bdstevel@tonic-gate#define	MEMBAR_RETURN	ba,pt %icc, membar_return
717c478bdstevel@tonic-gate#else
727c478bdstevel@tonic-gate#define	MEMBAR_RETURN	retl
737c478bdstevel@tonic-gate#endif
747c478bdstevel@tonic-gate
757c478bdstevel@tonic-gate	ENTRY(membar_enter)
767c478bdstevel@tonic-gate	MEMBAR_RETURN
777c478bdstevel@tonic-gate	membar	#StoreLoad|#StoreStore
787c478bdstevel@tonic-gate	SET_SIZE(membar_enter)
797c478bdstevel@tonic-gate
807c478bdstevel@tonic-gate	ENTRY(membar_exit)
817c478bdstevel@tonic-gate	MEMBAR_RETURN
827c478bdstevel@tonic-gate	membar	#LoadStore|#StoreStore
837c478bdstevel@tonic-gate	SET_SIZE(membar_exit)
847c478bdstevel@tonic-gate
857c478bdstevel@tonic-gate	ENTRY(membar_producer)
867c478bdstevel@tonic-gate	MEMBAR_RETURN
877c478bdstevel@tonic-gate	membar	#StoreStore
887c478bdstevel@tonic-gate	SET_SIZE(membar_producer)
897c478bdstevel@tonic-gate
907c478bdstevel@tonic-gate	ENTRY(membar_consumer)
917c478bdstevel@tonic-gate	MEMBAR_RETURN
927c478bdstevel@tonic-gate	membar	#LoadLoad
937c478bdstevel@tonic-gate	SET_SIZE(membar_consumer)
947c478bdstevel@tonic-gate
957c478bdstevel@tonic-gate/************************************************************************
967c478bdstevel@tonic-gate *		MINIMUM LOCKS
977c478bdstevel@tonic-gate */
987c478bdstevel@tonic-gate
997c478bdstevel@tonic-gate/*
1007c478bdstevel@tonic-gate * lock_try(lp), ulock_try(lp)
1019b0bb79John Levon * - returns non-zero on success.
1029b0bb79John Levon * - doesn't block interrupts so don't use this to spin on a lock.
1039b0bb79John Levon * - uses "0xFF is busy, anything else is free" model.
1047c478bdstevel@tonic-gate *
1059b0bb79John Levon * ulock_try() is for a lock in the user address space.
1067c478bdstevel@tonic-gate */
1077c478bdstevel@tonic-gate
1087c478bdstevel@tonic-gate	.align	32
1097c478bdstevel@tonic-gate	ENTRY(lock_try)
1107c478bdstevel@tonic-gate	ldstub	[%o0], %o1		! try to set lock, get value in %o1
1117c478bdstevel@tonic-gate	brnz,pn	%o1, 1f
1127c478bdstevel@tonic-gate	membar	#LoadLoad
1137c478bdstevel@tonic-gate.lock_try_lockstat_patch_point:
1147c478bdstevel@tonic-gate	retl
1157c478bdstevel@tonic-gate	or	%o0, 1, %o0		! ensure lo32 != 0
1167c478bdstevel@tonic-gate1:
1177c478bdstevel@tonic-gate	retl
1187c478bdstevel@tonic-gate	clr	%o0
1197c478bdstevel@tonic-gate	SET_SIZE(lock_try)
1207c478bdstevel@tonic-gate
1217c478bdstevel@tonic-gate	.align	32
1227c478bdstevel@tonic-gate	ENTRY(lock_spin_try)
1237c478bdstevel@tonic-gate	ldstub	[%o0], %o1		! try to set lock, get value in %o1
1247c478bdstevel@tonic-gate	brnz,pn	%o1, 1f
1257c478bdstevel@tonic-gate	membar	#LoadLoad
1267c478bdstevel@tonic-gate	retl
1277c478bdstevel@tonic-gate	or	%o0, 1, %o0		! ensure lo32 != 0
1287c478bdstevel@tonic-gate1:
1297c478bdstevel@tonic-gate	retl
1307c478bdstevel@tonic-gate	clr	%o0
1317c478bdstevel@tonic-gate	SET_SIZE(lock_spin_try)
1327c478bdstevel@tonic-gate
1337c478bdstevel@tonic-gate	.align	32
1347c478bdstevel@tonic-gate	ENTRY(lock_set)
1357c478bdstevel@tonic-gate	ldstub	[%o0], %o1
1367c478bdstevel@tonic-gate	brnz,pn	%o1, 1f			! go to C for the hard case
1377c478bdstevel@tonic-gate	membar	#LoadLoad
1387c478bdstevel@tonic-gate.lock_set_lockstat_patch_point:
1397c478bdstevel@tonic-gate	retl
1407c478bdstevel@tonic-gate	nop
1417c478bdstevel@tonic-gate1:
1427c478bdstevel@tonic-gate	sethi	%hi(lock_set_spin), %o2	! load up for jump to C
1437c478bdstevel@tonic-gate	jmp	%o2 + %lo(lock_set_spin)
1447c478bdstevel@tonic-gate	nop				! delay: do nothing
1457c478bdstevel@tonic-gate	SET_SIZE(lock_set)
1467c478bdstevel@tonic-gate
1477c478bdstevel@tonic-gate	ENTRY(lock_clear)
1487c478bdstevel@tonic-gate	membar	#LoadStore|#StoreStore
1497c478bdstevel@tonic-gate.lock_clear_lockstat_patch_point:
1507c478bdstevel@tonic-gate	retl
1517c478bdstevel@tonic-gate	clrb	[%o0]
1527c478bdstevel@tonic-gate	SET_SIZE(lock_clear)
1537c478bdstevel@tonic-gate
1547c478bdstevel@tonic-gate	.align	32
1557c478bdstevel@tonic-gate	ENTRY(ulock_try)
1567c478bdstevel@tonic-gate	ldstuba	[%o0]ASI_USER, %o1	! try to set lock, get value in %o1
1577c478bdstevel@tonic-gate	xor	%o1, 0xff, %o0		! delay - return non-zero if success
1587c478bdstevel@tonic-gate	retl
1597c478bdstevel@tonic-gate	  membar	#LoadLoad
1607c478bdstevel@tonic-gate	SET_SIZE(ulock_try)
1617c478bdstevel@tonic-gate
1627c478bdstevel@tonic-gate	ENTRY(ulock_clear)
1637c478bdstevel@tonic-gate	membar  #LoadStore|#StoreStore
1647c478bdstevel@tonic-gate	retl
1657c478bdstevel@tonic-gate	  stba	%g0, [%o0]ASI_USER	! clear lock
1667c478bdstevel@tonic-gate	SET_SIZE(ulock_clear)
1677c478bdstevel@tonic-gate
1687c478bdstevel@tonic-gate
1697c478bdstevel@tonic-gate/*
1707c478bdstevel@tonic-gate * lock_set_spl(lp, new_pil, *old_pil_addr)
1716a0b121Patrick Mooney *	Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr.
1727c478bdstevel@tonic-gate */
1737c478bdstevel@tonic-gate
1747c478bdstevel@tonic-gate	ENTRY(lock_set_spl)
1757c478bdstevel@tonic-gate	rdpr	%pil, %o3			! %o3 = current pil
1767c478bdstevel@tonic-gate	cmp	%o3, %o1			! is current pil high enough?
1777c478bdstevel@tonic-gate	bl,a,pt %icc, 1f			! if not, write %pil in delay
1787c478bdstevel@tonic-gate	wrpr	%g0, %o1, %pil
1797c478bdstevel@tonic-gate1:
1807c478bdstevel@tonic-gate	ldstub	[%o0], %o4			! try the lock
1817c478bdstevel@tonic-gate	brnz,pn	%o4, 2f				! go to C for the miss case
1827c478bdstevel@tonic-gate	membar	#LoadLoad
1837c478bdstevel@tonic-gate.lock_set_spl_lockstat_patch_point:
1847c478bdstevel@tonic-gate	retl
1857c478bdstevel@tonic-gate	sth	%o3, [%o2]			! delay - save original pil
1867c478bdstevel@tonic-gate2:
1877c478bdstevel@tonic-gate	sethi	%hi(lock_set_spl_spin), %o5	! load up jmp to C
1887c478bdstevel@tonic-gate	jmp	%o5 + %lo(lock_set_spl_spin)	! jmp to lock_set_spl_spin
1897c478bdstevel@tonic-gate	nop					! delay: do nothing
1907c478bdstevel@tonic-gate	SET_SIZE(lock_set_spl)
1917c478bdstevel@tonic-gate
1927c478bdstevel@tonic-gate/*
1937c478bdstevel@tonic-gate * lock_clear_splx(lp, s)
1947c478bdstevel@tonic-gate */
1957c478bdstevel@tonic-gate
1967c478bdstevel@tonic-gate	ENTRY(lock_clear_splx)
1977c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o2	! get CPU pointer
1987c478bdstevel@tonic-gate	membar	#LoadStore|#StoreStore
1997c478bdstevel@tonic-gate	ld	[%o2 + CPU_BASE_SPL], %o2
2007c478bdstevel@tonic-gate	clrb	[%o0]				! clear lock
2017c478bdstevel@tonic-gate	cmp	%o2, %o1			! compare new to base
2027c478bdstevel@tonic-gate	movl	%xcc, %o1, %o2			! use new pri if base is less
2037c478bdstevel@tonic-gate.lock_clear_splx_lockstat_patch_point:
2047c478bdstevel@tonic-gate	retl
2057c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pil
2067c478bdstevel@tonic-gate	SET_SIZE(lock_clear_splx)
2077c478bdstevel@tonic-gate
2087c478bdstevel@tonic-gate/*
2097c478bdstevel@tonic-gate * mutex_enter() and mutex_exit().
2106a0b121Patrick Mooney *
2117c478bdstevel@tonic-gate * These routines handle the simple cases of mutex_enter() (adaptive
2127c478bdstevel@tonic-gate * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
2137c478bdstevel@tonic-gate * If anything complicated is going on we punt to mutex_vector_enter().
2147c478bdstevel@tonic-gate *
2157c478bdstevel@tonic-gate * mutex_tryenter() is similar to mutex_enter() but returns zero if
2167c478bdstevel@tonic-gate * the lock cannot be acquired, nonzero on success.
2177c478bdstevel@tonic-gate *
2187c478bdstevel@tonic-gate * If mutex_exit() gets preempted in the window between checking waiters
2197c478bdstevel@tonic-gate * and clearing the lock, we can miss wakeups.  Disabling preemption
2207c478bdstevel@tonic-gate * in the mutex code is prohibitively expensive, so instead we detect
2217c478bdstevel@tonic-gate * mutex preemption by examining the trapped PC in the interrupt path.
2227c478bdstevel@tonic-gate * If we interrupt a thread in mutex_exit() that has not yet cleared
2237c478bdstevel@tonic-gate * the lock, pil_interrupt() resets its PC back to the beginning of
2247c478bdstevel@tonic-gate * mutex_exit() so it will check again for waiters when it resumes.
2257c478bdstevel@tonic-gate *
2267c478bdstevel@tonic-gate * The lockstat code below is activated when the lockstat driver
2277c478bdstevel@tonic-gate * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
2287c478bdstevel@tonic-gate * Note that we don't need to test lockstat_event_mask here -- we won't
2297c478bdstevel@tonic-gate * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
2307c478bdstevel@tonic-gate */
2317c478bdstevel@tonic-gate
2327c478bdstevel@tonic-gate	.align	32
2337c478bdstevel@tonic-gate	ENTRY(mutex_enter)
2347c478bdstevel@tonic-gate	mov	THREAD_REG, %o1
2357c478bdstevel@tonic-gate	casx	[%o0], %g0, %o1			! try to acquire as adaptive
2367c478bdstevel@tonic-gate	brnz,pn	%o1, 1f				! locked or wrong type
2377c478bdstevel@tonic-gate	membar	#LoadLoad
2387c478bdstevel@tonic-gate.mutex_enter_lockstat_patch_point:
2397c478bdstevel@tonic-gate	retl
2407c478bdstevel@tonic-gate	nop
2417c478bdstevel@tonic-gate1:
2427c478bdstevel@tonic-gate	sethi	%hi(mutex_vector_enter), %o2	! load up for jump to C
2437c478bdstevel@tonic-gate	jmp	%o2 + %lo(mutex_vector_enter)
2447c478bdstevel@tonic-gate	nop
2457c478bdstevel@tonic-gate	SET_SIZE(mutex_enter)
2467c478bdstevel@tonic-gate
2477c478bdstevel@tonic-gate	ENTRY(mutex_tryenter)
2487c478bdstevel@tonic-gate	mov	THREAD_REG, %o1
2497c478bdstevel@tonic-gate	casx	[%o0], %g0, %o1			! try to acquire as adaptive
2507c478bdstevel@tonic-gate	brnz,pn	%o1, 1f				! locked or wrong type continue
2517c478bdstevel@tonic-gate	membar	#LoadLoad
2527c478bdstevel@tonic-gate.mutex_tryenter_lockstat_patch_point:
2537c478bdstevel@tonic-gate	retl
2547c478bdstevel@tonic-gate	or	%o0, 1, %o0			! ensure lo32 != 0
2557c478bdstevel@tonic-gate1:
2567c478bdstevel@tonic-gate	sethi	%hi(mutex_vector_tryenter), %o2		! hi bits
2577c478bdstevel@tonic-gate	jmp	%o2 + %lo(mutex_vector_tryenter)	! go to C
2587c478bdstevel@tonic-gate	nop
2597c478bdstevel@tonic-gate	SET_SIZE(mutex_tryenter)
2607c478bdstevel@tonic-gate
2617c478bdstevel@tonic-gate	ENTRY(mutex_adaptive_tryenter)
2627c478bdstevel@tonic-gate	mov	THREAD_REG, %o1
2637c478bdstevel@tonic-gate	casx	[%o0], %g0, %o1			! try to acquire as adaptive
2647c478bdstevel@tonic-gate	brnz,pn	%o1, 0f				! locked or wrong type
2657c478bdstevel@tonic-gate	membar	#LoadLoad
2667c478bdstevel@tonic-gate	retl
2677c478bdstevel@tonic-gate	or	%o0, 1, %o0			! ensure lo32 != 0
2687c478bdstevel@tonic-gate0:
2697c478bdstevel@tonic-gate	retl
2707c478bdstevel@tonic-gate	mov	%g0, %o0
2717c478bdstevel@tonic-gate	SET_SIZE(mutex_adaptive_tryenter)
2727c478bdstevel@tonic-gate
273575a742pt	! these need to be together and cache aligned for performance.
274575a742pt	.align 64
2757c478bdstevel@tonic-gate	.global	mutex_exit_critical_size
2767c478bdstevel@tonic-gate	.global	mutex_exit_critical_start
277575a742pt	.global mutex_owner_running_critical_size
278575a742pt	.global mutex_owner_running_critical_start
2797c478bdstevel@tonic-gate
2807c478bdstevel@tonic-gatemutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start
2817c478bdstevel@tonic-gate
2827c478bdstevel@tonic-gate	.align	32
2837c478bdstevel@tonic-gate
2847c478bdstevel@tonic-gate	ENTRY(mutex_exit)
2857c478bdstevel@tonic-gatemutex_exit_critical_start:		! If we are interrupted, restart here
2867c478bdstevel@tonic-gate	ldn	[%o0], %o1		! get the owner field
2877c478bdstevel@tonic-gate	membar	#LoadStore|#StoreStore
2887c478bdstevel@tonic-gate	cmp	THREAD_REG, %o1		! do we own lock with no waiters?
2897c478bdstevel@tonic-gate	be,a,pt	%ncc, 1f		! if so, drive on ...
2907c478bdstevel@tonic-gate	stn	%g0, [%o0]		! delay: clear lock if we owned it
2917c478bdstevel@tonic-gate.mutex_exit_critical_end:		! for pil_interrupt() hook
2927c478bdstevel@tonic-gate	ba,a,pt	%xcc, mutex_vector_exit	! go to C for the hard cases
2937c478bdstevel@tonic-gate1:
2947c478bdstevel@tonic-gate.mutex_exit_lockstat_patch_point:
2957c478bdstevel@tonic-gate	retl
2967c478bdstevel@tonic-gate	nop
2977c478bdstevel@tonic-gate	SET_SIZE(mutex_exit)
2987c478bdstevel@tonic-gate
299575a742ptmutex_owner_running_critical_size = .mutex_owner_running_critical_end - mutex_owner_running_critical_start
300575a742pt
301575a742pt	.align  32
302575a742pt
303575a742pt	ENTRY(mutex_owner_running)
304575a742ptmutex_owner_running_critical_start:	! If interrupted restart here
305575a742pt	ldn	[%o0], %o1		! get the owner field
306575a742pt	and	%o1, MUTEX_THREAD, %o1	! remove the waiters bit if any
307575a742pt	brz,pn	%o1, 1f			! if so, drive on ...
308575a742pt	nop
309575a742pt	ldn	[%o1+T_CPU], %o2	! get owner->t_cpu
310575a742pt	ldn	[%o2+CPU_THREAD], %o3	! get owner->t_cpu->cpu_thread
311575a742pt.mutex_owner_running_critical_end:	! for pil_interrupt() hook
312575a742pt	cmp	%o1, %o3		! owner == running thread?
313575a742pt	be,a,pt	%xcc, 2f		! yes, go return cpu
314575a742pt	nop
315575a742pt1:
316575a742pt	retl
317575a742pt	mov	%g0, %o0		! return 0 (owner not running)
318575a742pt2:
319575a742pt	retl
320575a742pt	mov	%o2, %o0		! owner running, return cpu
321575a742pt	SET_SIZE(mutex_owner_running)
322575a742pt
3237c478bdstevel@tonic-gate/*
3247c478bdstevel@tonic-gate * rw_enter() and rw_exit().
3256a0b121Patrick Mooney *
3267c478bdstevel@tonic-gate * These routines handle the simple cases of rw_enter (write-locking an unheld
3277c478bdstevel@tonic-gate * lock or read-locking a lock that's neither write-locked nor write-wanted)
3287c478bdstevel@tonic-gate * and rw_exit (no waiters or not the last reader).  If anything complicated
3297c478bdstevel@tonic-gate * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
3307c478bdstevel@tonic-gate */
3317c478bdstevel@tonic-gate
3327c478bdstevel@tonic-gate	.align	16
3337c478bdstevel@tonic-gate	ENTRY(rw_enter)
3347c478bdstevel@tonic-gate	cmp	%o1, RW_WRITER			! entering as writer?
3357c478bdstevel@tonic-gate	be,a,pn	%icc, 2f			! if so, go do it ...
3367c478bdstevel@tonic-gate	or	THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner
3377c478bdstevel@tonic-gate	ldn	[%o0], %o4			! %o4 = old lock value
3387c478bdstevel@tonic-gate1:
3397c478bdstevel@tonic-gate	andcc	%o4, RW_WRITE_CLAIMED, %g0	! write-locked or write-wanted?
3406a0b121Patrick Mooney	bz,pt	%xcc, 3f			! if so, prepare to block
3417c478bdstevel@tonic-gate	add	%o4, RW_READ_LOCK, %o5		! delay: increment hold count
3427c478bdstevel@tonic-gate	sethi	%hi(rw_enter_sleep), %o2	! load up jump
3437c478bdstevel@tonic-gate	jmp	%o2 + %lo(rw_enter_sleep)	! jmp to rw_enter_sleep
3447c478bdstevel@tonic-gate	nop					! delay: do nothing
3457c478bdstevel@tonic-gate3:
3467c478bdstevel@tonic-gate	casx	[%o0], %o4, %o5			! try to grab read lock
3477c478bdstevel@tonic-gate	cmp	%o4, %o5			! did we get it?
348374ae87svemuri#ifdef sun4v
349374ae87svemuri	be,a,pt %xcc, 0f
350374ae87svemuri	membar  #LoadLoad
351374ae87svemuri	sethi	%hi(rw_enter_sleep), %o2	! load up jump
352374ae87svemuri	jmp	%o2 + %lo(rw_enter_sleep)	! jmp to rw_enter_sleep
353374ae87svemuri	nop					! delay: do nothing
354374ae87svemuri0:
355374ae87svemuri#else /* sun4v */
3567c478bdstevel@tonic-gate	bne,pn	%xcc, 1b			! if not, try again
3577c478bdstevel@tonic-gate	mov	%o5, %o4			! delay: %o4 = old lock value
3587c478bdstevel@tonic-gate	membar	#LoadLoad
359374ae87svemuri#endif /* sun4v */
3607c478bdstevel@tonic-gate.rw_read_enter_lockstat_patch_point:
3617c478bdstevel@tonic-gate	retl
3627c478bdstevel@tonic-gate	nop
3637c478bdstevel@tonic-gate2:
3647c478bdstevel@tonic-gate	casx	[%o0], %g0, %o5			! try to grab write lock
3657c478bdstevel@tonic-gate	brz,pt %o5, 4f				! branch around if we got it
3667c478bdstevel@tonic-gate	membar	#LoadLoad			! done regardless of where we go
3677c478bdstevel@tonic-gate	sethi	%hi(rw_enter_sleep), %o2
3687c478bdstevel@tonic-gate	jmp	%o2 + %lo(rw_enter_sleep)	! jump to rw_enter_sleep if not
3697c478bdstevel@tonic-gate	nop					! delay: do nothing
3707c478bdstevel@tonic-gate4:
3717c478bdstevel@tonic-gate.rw_write_enter_lockstat_patch_point:
3727c478bdstevel@tonic-gate	retl
3737c478bdstevel@tonic-gate	nop
3747c478bdstevel@tonic-gate	SET_SIZE(rw_enter)
3757c478bdstevel@tonic-gate
3767c478bdstevel@tonic-gate	.align	16
3777c478bdstevel@tonic-gate	ENTRY(rw_exit)
3787c478bdstevel@tonic-gate	ldn	[%o0], %o4			! %o4 = old lock value
3797c478bdstevel@tonic-gate	membar	#LoadStore|#StoreStore		! membar_exit()
3807c478bdstevel@tonic-gate	subcc	%o4, RW_READ_LOCK, %o5		! %o5 = new lock value if reader
3817c478bdstevel@tonic-gate	bnz,pn	%xcc, 2f			! single reader, no waiters?
3827c478bdstevel@tonic-gate	clr	%o1
3837c478bdstevel@tonic-gate1:
3847c478bdstevel@tonic-gate	srl	%o4, RW_HOLD_COUNT_SHIFT, %o3	! %o3 = hold count (lockstat)
3857c478bdstevel@tonic-gate	casx	[%o0], %o4, %o5			! try to drop lock
3867c478bdstevel@tonic-gate	cmp	%o4, %o5			! did we succeed?
3877c478bdstevel@tonic-gate	bne,pn	%xcc, rw_exit_wakeup		! if not, go to C
3886a0b121Patrick Mooney	nop					! delay: do nothing
3897c478bdstevel@tonic-gate.rw_read_exit_lockstat_patch_point:
3907c478bdstevel@tonic-gate	retl
3916a0b121Patrick Mooney	nop					! delay: do nothing
3927c478bdstevel@tonic-gate2:
3937c478bdstevel@tonic-gate	andcc	%o4, RW_WRITE_LOCKED, %g0	! are we a writer?
3947c478bdstevel@tonic-gate	bnz,a,pt %xcc, 3f
3957c478bdstevel@tonic-gate	or	THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
3967c478bdstevel@tonic-gate	cmp	%o5, RW_READ_LOCK		! would lock still be held?
3977c478bdstevel@tonic-gate	bge,pt	%xcc, 1b			! if so, go ahead and drop it
3987c478bdstevel@tonic-gate	nop
3997c478bdstevel@tonic-gate	ba,pt	%xcc, rw_exit_wakeup		! otherwise, wake waiters
4007c478bdstevel@tonic-gate	nop
4017c478bdstevel@tonic-gate3:
4027c478bdstevel@tonic-gate	casx	[%o0], %o4, %o1			! try to drop write lock
4037c478bdstevel@tonic-gate	cmp	%o4, %o1			! did we succeed?
4047c478bdstevel@tonic-gate	bne,pn	%xcc, rw_exit_wakeup		! if not, go to C
4057c478bdstevel@tonic-gate	nop
4067c478bdstevel@tonic-gate.rw_write_exit_lockstat_patch_point:
4077c478bdstevel@tonic-gate	retl
4087c478bdstevel@tonic-gate	nop
4097c478bdstevel@tonic-gate	SET_SIZE(rw_exit)
4107c478bdstevel@tonic-gate
4117c478bdstevel@tonic-gate#define	RETL			0x81c3e008
4127c478bdstevel@tonic-gate#define	NOP			0x01000000
4137c478bdstevel@tonic-gate#define BA			0x10800000
4147c478bdstevel@tonic-gate
4157c478bdstevel@tonic-gate#define	DISP22			((1 << 22) - 1)
4167c478bdstevel@tonic-gate#define	ANNUL			0x20000000
4177c478bdstevel@tonic-gate
4187c478bdstevel@tonic-gate#define	HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs)		\
4197c478bdstevel@tonic-gate	ba	1f;							\
4207c478bdstevel@tonic-gate	rd	%pc, %o0;						\
4217c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME), %sp;				\
4227c478bdstevel@tonic-gate	set	lockstat_probemap, %l1;					\
4237c478bdstevel@tonic-gate	ld	[%l1 + (event * DTRACE_IDSIZE)], %o0;			\
4247c478bdstevel@tonic-gate	brz,pn	%o0, 0f;						\
4257c478bdstevel@tonic-gate	ldub	[THREAD_REG + T_LOCKSTAT], %l0;				\
4267c478bdstevel@tonic-gate	add	%l0, 1, %l2;						\
4277c478bdstevel@tonic-gate	stub	%l2, [THREAD_REG + T_LOCKSTAT];				\
4287c478bdstevel@tonic-gate	set	lockstat_probe, %g1;					\
4297c478bdstevel@tonic-gate	ld	[%l1 + (event * DTRACE_IDSIZE)], %o0;			\
4307c478bdstevel@tonic-gate	brz,a,pn %o0, 0f;						\
4317c478bdstevel@tonic-gate	stub	%l0, [THREAD_REG + T_LOCKSTAT];				\
4327c478bdstevel@tonic-gate	ldn	[%g1], %g2;						\
4337c478bdstevel@tonic-gate	mov	rs, %o2;						\
4347c478bdstevel@tonic-gate	jmpl	%g2, %o7;						\
4357c478bdstevel@tonic-gate	mov	%i0, %o1;						\
4367c478bdstevel@tonic-gate	stub	%l0, [THREAD_REG + T_LOCKSTAT];				\
4377c478bdstevel@tonic-gate0:	ret;								\
4387c478bdstevel@tonic-gate	restore	%g0, 1, %o0;	/* for mutex_tryenter / lock_try */	\
4397c478bdstevel@tonic-gate1:	set	addr, %o1;						\
4407c478bdstevel@tonic-gate	sub	%o0, %o1, %o0;						\
4417c478bdstevel@tonic-gate	srl	%o0, 2, %o0;						\
4427c478bdstevel@tonic-gate	inc	%o0;							\
4437c478bdstevel@tonic-gate	set	DISP22, %o1;						\
4447c478bdstevel@tonic-gate	and	%o1, %o0, %o0;						\
4457c478bdstevel@tonic-gate	set	BA, %o1;						\
4467c478bdstevel@tonic-gate	or	%o1, %o0, %o0;						\
4477c478bdstevel@tonic-gate	sethi	%hi(annul), %o2;					\
4487c478bdstevel@tonic-gate	add	%o0, %o2, %o2;						\
4497c478bdstevel@tonic-gate	set	addr, %o0;						\
4507c478bdstevel@tonic-gate	set	normal_instr, %o1;					\
4517c478bdstevel@tonic-gate	ld	[%i0 + (event * DTRACE_IDSIZE)], %o3;			\
4527c478bdstevel@tonic-gate	tst	%o3;							\
4537c478bdstevel@tonic-gate	movnz	%icc, %o2, %o1;						\
4547c478bdstevel@tonic-gate	call	hot_patch_kernel_text;					\
4557c478bdstevel@tonic-gate	mov	4, %o2;							\
4567c478bdstevel@tonic-gate	membar	#Sync
4577c478bdstevel@tonic-gate
4587c478bdstevel@tonic-gate#define	HOT_PATCH(addr, event, normal_instr)	\
4597c478bdstevel@tonic-gate	HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1)
4607c478bdstevel@tonic-gate
4617c478bdstevel@tonic-gate#define	HOT_PATCH_ARG(addr, event, normal_instr, arg)	\
4627c478bdstevel@tonic-gate	HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg)
4637c478bdstevel@tonic-gate
4647c478bdstevel@tonic-gate#define HOT_PATCH_ANNULLED(addr, event, normal_instr)	\
4657c478bdstevel@tonic-gate	HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1)
4667c478bdstevel@tonic-gate
4677c478bdstevel@tonic-gate	ENTRY(lockstat_hot_patch)
4687c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME), %sp
4697c478bdstevel@tonic-gate	set	lockstat_probemap, %i0
4707c478bdstevel@tonic-gate	HOT_PATCH(.mutex_enter_lockstat_patch_point,
4717c478bdstevel@tonic-gate		LS_MUTEX_ENTER_ACQUIRE, RETL)
4727c478bdstevel@tonic-gate	HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point,
4737c478bdstevel@tonic-gate		LS_MUTEX_TRYENTER_ACQUIRE, RETL)
4747c478bdstevel@tonic-gate	HOT_PATCH(.mutex_exit_lockstat_patch_point,
4757c478bdstevel@tonic-gate		LS_MUTEX_EXIT_RELEASE, RETL)
4767c478bdstevel@tonic-gate	HOT_PATCH(.rw_write_enter_lockstat_patch_point,
4777c478bdstevel@tonic-gate		LS_RW_ENTER_ACQUIRE, RETL)
4787c478bdstevel@tonic-gate	HOT_PATCH(.rw_read_enter_lockstat_patch_point,
4797c478bdstevel@tonic-gate		LS_RW_ENTER_ACQUIRE, RETL)
4807c478bdstevel@tonic-gate	HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point,
4817c478bdstevel@tonic-gate		LS_RW_EXIT_RELEASE, RETL, RW_WRITER)
4827c478bdstevel@tonic-gate	HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point,
4837c478bdstevel@tonic-gate		LS_RW_EXIT_RELEASE, RETL, RW_READER)
4847c478bdstevel@tonic-gate	HOT_PATCH(.lock_set_lockstat_patch_point,
4857c478bdstevel@tonic-gate		LS_LOCK_SET_ACQUIRE, RETL)
4867c478bdstevel@tonic-gate	HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point,
4877c478bdstevel@tonic-gate		LS_LOCK_TRY_ACQUIRE, RETL)
4887c478bdstevel@tonic-gate	HOT_PATCH(.lock_clear_lockstat_patch_point,
4897c478bdstevel@tonic-gate		LS_LOCK_CLEAR_RELEASE, RETL)
4907c478bdstevel@tonic-gate	HOT_PATCH(.lock_set_spl_lockstat_patch_point,
4917c478bdstevel@tonic-gate		LS_LOCK_SET_SPL_ACQUIRE, RETL)
4927c478bdstevel@tonic-gate	HOT_PATCH(.lock_clear_splx_lockstat_patch_point,
4937c478bdstevel@tonic-gate		LS_LOCK_CLEAR_SPLX_RELEASE, RETL)
4947c478bdstevel@tonic-gate	ret
4957c478bdstevel@tonic-gate	restore
4967c478bdstevel@tonic-gate	SET_SIZE(lockstat_hot_patch)
4977c478bdstevel@tonic-gate
4987c478bdstevel@tonic-gate/*
4997c478bdstevel@tonic-gate * asm_mutex_spin_enter(mutex_t *)
5007c478bdstevel@tonic-gate *
5017c478bdstevel@tonic-gate * For use by assembly interrupt handler only.
5027c478bdstevel@tonic-gate * Does not change spl, since the interrupt handler is assumed to be
5037c478bdstevel@tonic-gate * running at high level already.
5047c478bdstevel@tonic-gate * Traps may be off, so cannot panic.
5057c478bdstevel@tonic-gate * Does not keep statistics on the lock.
5067c478bdstevel@tonic-gate *
5077c478bdstevel@tonic-gate * Entry:	%l6 - points to mutex
5086a0b121Patrick Mooney *		%l7 - address of call (returns to %l7+8)
5097c478bdstevel@tonic-gate * Uses:	%l6, %l5
5107c478bdstevel@tonic-gate */
5117c478bdstevel@tonic-gate	.align 16
5127c478bdstevel@tonic-gate	ENTRY_NP(asm_mutex_spin_enter)
5137c478bdstevel@tonic-gate	ldstub	[%l6 + M_SPINLOCK], %l5	! try to set lock, get value in %l5
5147c478bdstevel@tonic-gate1:
5157c478bdstevel@tonic-gate	tst	%l5
5167c478bdstevel@tonic-gate	bnz	3f			! lock already held - go spin
5177c478bdstevel@tonic-gate	nop
5186a0b121Patrick Mooney2:
5197c478bdstevel@tonic-gate	jmp	%l7 + 8			! return
5207c478bdstevel@tonic-gate	membar	#LoadLoad
5217c478bdstevel@tonic-gate	!
5227c478bdstevel@tonic-gate	! Spin on lock without using an atomic operation to prevent the caches
5237c478bdstevel@tonic-gate	! from unnecessarily moving ownership of the line around.
5247c478bdstevel@tonic-gate	!
5257c478bdstevel@tonic-gate3:
5267c478bdstevel@tonic-gate	ldub	[%l6 + M_SPINLOCK], %l5
5277c478bdstevel@tonic-gate4:
5287c478bdstevel@tonic-gate	tst	%l5
5297c478bdstevel@tonic-gate	bz,a	1b			! lock appears to be free, try again
5307c478bdstevel@tonic-gate	ldstub	[%l6 + M_SPINLOCK], %l5	! delay slot - try to set lock
5317c478bdstevel@tonic-gate
5327c478bdstevel@tonic-gate	sethi	%hi(panicstr) , %l5
5337c478bdstevel@tonic-gate	ldn	[%l5 + %lo(panicstr)], %l5
5346a0b121Patrick Mooney	tst	%l5
5357c478bdstevel@tonic-gate	bnz	2b			! after panic, feign success
5367c478bdstevel@tonic-gate	nop
5377c478bdstevel@tonic-gate	b	4b
5387c478bdstevel@tonic-gate	ldub	[%l6 + M_SPINLOCK], %l5	! delay - reload lock
5397c478bdstevel@tonic-gate	SET_SIZE(asm_mutex_spin_enter)
5407c478bdstevel@tonic-gate
5417c478bdstevel@tonic-gate/*
5427c478bdstevel@tonic-gate * asm_mutex_spin_exit(mutex_t *)
5437c478bdstevel@tonic-gate *
5447c478bdstevel@tonic-gate * For use by assembly interrupt handler only.
5457c478bdstevel@tonic-gate * Does not change spl, since the interrupt handler is assumed to be
5467c478bdstevel@tonic-gate * running at high level already.
5477c478bdstevel@tonic-gate *
5487c478bdstevel@tonic-gate * Entry:	%l6 - points to mutex
5496a0b121Patrick Mooney *		%l7 - address of call (returns to %l7+8)
5507c478bdstevel@tonic-gate * Uses:	none
5517c478bdstevel@tonic-gate */
5527c478bdstevel@tonic-gate	ENTRY_NP(asm_mutex_spin_exit)
5537c478bdstevel@tonic-gate	membar	#LoadStore|#StoreStore
5547c478bdstevel@tonic-gate	jmp	%l7 + 8			! return
5557c478bdstevel@tonic-gate	clrb	[%l6 + M_SPINLOCK]	! delay - clear lock
5567c478bdstevel@tonic-gate	SET_SIZE(asm_mutex_spin_exit)
5577c478bdstevel@tonic-gate
5587c478bdstevel@tonic-gate/*
5597c478bdstevel@tonic-gate * thread_onproc()
5607c478bdstevel@tonic-gate * Set thread in onproc state for the specified CPU.
5617c478bdstevel@tonic-gate * Also set the thread lock pointer to the CPU's onproc lock.
5627c478bdstevel@tonic-gate * Since the new lock isn't held, the store ordering is important.
5637c478bdstevel@tonic-gate * If not done in assembler, the compiler could reorder the stores.
5647c478bdstevel@tonic-gate */
5657c478bdstevel@tonic-gate
5667c478bdstevel@tonic-gate	ENTRY(thread_onproc)
5677c478bdstevel@tonic-gate	set	TS_ONPROC, %o2		! TS_ONPROC state
5687c478bdstevel@tonic-gate	st	%o2, [%o0 + T_STATE]	! store state
5697c478bdstevel@tonic-gate	add	%o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running
5707c478bdstevel@tonic-gate	retl				! return
5717c478bdstevel@tonic-gate	stn	%o3, [%o0 + T_LOCKP]	! delay - store new lock pointer
5727c478bdstevel@tonic-gate	SET_SIZE(thread_onproc)
5737c478bdstevel@tonic-gate
574575a742pt/* delay function used in some mutex code - just do 3 nop cas ops */
575575a742pt	ENTRY(cas_delay)
576575a742pt	casx [%o0], %g0, %g0
577575a742pt	casx [%o0], %g0, %g0
578575a742pt	retl
579575a742pt	casx [%o0], %g0, %g0
580575a742pt	SET_SIZE(cas_delay)
581575a742pt
582575a742pt/*
583575a742pt * alternative delay function for some niagara processors.   The rd
584575a742pt * instruction uses less resources than casx on those cpus.
585575a742pt */
586575a742pt	ENTRY(rdccr_delay)
587575a742pt	rd	%ccr, %g0
588575a742pt	rd	%ccr, %g0
589575a742pt	retl
590575a742pt	rd	%ccr, %g0
591575a742pt	SET_SIZE(rdccr_delay)
592575a742pt
593575a742pt/*
594575a742pt * mutex_delay_default(void)
595575a742pt * Spins for approx a few hundred processor cycles and returns to caller.
596575a742pt */
597575a742pt
598575a742pt	ENTRY(mutex_delay_default)
599575a742pt	mov	72,%o0
600575a742pt1:	brgz	%o0, 1b
601575a742pt	dec	%o0
602575a742pt	retl
603575a742pt	nop
604575a742pt	SET_SIZE(mutex_delay_default)
605575a742pt
606