17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
54df4bd6bs * Common Development and Distribution License (the "License").
64df4bd6bs * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
22023e71dHaik Aftandilian * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bdstevel@tonic-gate * Use is subject to license terms.
247c478bdstevel@tonic-gate */
257c478bdstevel@tonic-gate
267c478bdstevel@tonic-gate/*
277c478bdstevel@tonic-gate * General assembly language routines.
287c478bdstevel@tonic-gate * It is the intent of this file to contain routines that are
297c478bdstevel@tonic-gate * independent of the specific kernel architecture, and those that are
307c478bdstevel@tonic-gate * common across kernel architectures.
317c478bdstevel@tonic-gate * As architectures diverge, and implementations of specific
327c478bdstevel@tonic-gate * architecture-dependent routines change, the routines should be moved
337c478bdstevel@tonic-gate * from this file into the respective ../`arch -k`/subr.s file.
347c478bdstevel@tonic-gate * Or, if you want to be really nice, move them to a file whose
357c478bdstevel@tonic-gate * name has something to do with the routine you are moving.
367c478bdstevel@tonic-gate */
377c478bdstevel@tonic-gate
387c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
397c478bdstevel@tonic-gate#include <sys/privregs.h>
407c478bdstevel@tonic-gate#include <sys/machparam.h>	/* To get SYSBASE and PAGESIZE */
417c478bdstevel@tonic-gate#include <sys/machthread.h>
427c478bdstevel@tonic-gate#include <sys/clock.h>
437c478bdstevel@tonic-gate#include <sys/psr_compat.h>
447c478bdstevel@tonic-gate#include <sys/isa_defs.h>
457c478bdstevel@tonic-gate#include <sys/dditypes.h>
467c478bdstevel@tonic-gate#include <sys/panic.h>
477c478bdstevel@tonic-gate#include <sys/machlock.h>
487c478bdstevel@tonic-gate#include <sys/ontrap.h>
497c478bdstevel@tonic-gate
507c478bdstevel@tonic-gate#include "assym.h"
517c478bdstevel@tonic-gate
527c478bdstevel@tonic-gate	.seg	".text"
537c478bdstevel@tonic-gate	.align	4
547c478bdstevel@tonic-gate
557c478bdstevel@tonic-gate/*
567c478bdstevel@tonic-gate * Macro to raise processor priority level.
577c478bdstevel@tonic-gate * Avoid dropping processor priority if already at high level.
587c478bdstevel@tonic-gate * Also avoid going below CPU->cpu_base_spl, which could've just been set by
597c478bdstevel@tonic-gate * a higher-level interrupt thread that just blocked.
607c478bdstevel@tonic-gate *
617c478bdstevel@tonic-gate * level can be %o0 (not other regs used here) or a constant.
627c478bdstevel@tonic-gate */
637c478bdstevel@tonic-gate#define	RAISE(level) \
647c478bdstevel@tonic-gate	rdpr	%pil, %o1;		/* get current PIL */		\
657c478bdstevel@tonic-gate	cmp	%o1, level;		/* is PIL high enough? */	\
667c478bdstevel@tonic-gate	bge	1f;			/* yes, return */		\
677c478bdstevel@tonic-gate	nop;								\
687c478bdstevel@tonic-gate	wrpr	%g0, PIL_MAX, %pil;	/* freeze CPU_BASE_SPL */	\
697c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o2;				\
707c478bdstevel@tonic-gate	ld	[%o2 + CPU_BASE_SPL], %o2;				\
717c478bdstevel@tonic-gate	cmp	%o2, level;		/* compare new to base */	\
727c478bdstevel@tonic-gate	movl	%xcc, level, %o2;	/* use new if base lower */	\
737c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pil;						\
747c478bdstevel@tonic-gate1:									\
757c478bdstevel@tonic-gate	retl;								\
767c478bdstevel@tonic-gate	mov	%o1, %o0		/* return old PIL */
777c478bdstevel@tonic-gate
787c478bdstevel@tonic-gate/*
797c478bdstevel@tonic-gate * Macro to raise processor priority level to level >= DISP_LEVEL.
807c478bdstevel@tonic-gate * Doesn't require comparison to CPU->cpu_base_spl.
817c478bdstevel@tonic-gate *
827c478bdstevel@tonic-gate * newpil can be %o0 (not other regs used here) or a constant.
837c478bdstevel@tonic-gate */
847c478bdstevel@tonic-gate#define	RAISE_HIGH(level) \
857c478bdstevel@tonic-gate	rdpr	%pil, %o1;		/* get current PIL */		\
867c478bdstevel@tonic-gate	cmp	%o1, level;		/* is PIL high enough? */	\
877c478bdstevel@tonic-gate	bge	1f;			/* yes, return */		\
887c478bdstevel@tonic-gate	nop;								\
897c478bdstevel@tonic-gate	wrpr	%g0, level, %pil;	/* use chose value */		\
907c478bdstevel@tonic-gate1:									\
917c478bdstevel@tonic-gate	retl;								\
927c478bdstevel@tonic-gate	mov	%o1, %o0		/* return old PIL */
937c478bdstevel@tonic-gate
947c478bdstevel@tonic-gate/*
957c478bdstevel@tonic-gate * Macro to set the priority to a specified level.
967c478bdstevel@tonic-gate * Avoid dropping the priority below CPU->cpu_base_spl.
977c478bdstevel@tonic-gate *
987c478bdstevel@tonic-gate * newpil can be %o0 (not other regs used here) or a constant with
997c478bdstevel@tonic-gate * the new PIL in the PSR_PIL field of the level arg.
1007c478bdstevel@tonic-gate */
1017c478bdstevel@tonic-gate#define SETPRI(level) \
1027c478bdstevel@tonic-gate	rdpr	%pil, %o1;		/* get current PIL */		\
1037c478bdstevel@tonic-gate	wrpr	%g0, PIL_MAX, %pil;	/* freeze CPU_BASE_SPL */	\
1047c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o2;				\
1057c478bdstevel@tonic-gate	ld	[%o2 + CPU_BASE_SPL], %o2;				\
1067c478bdstevel@tonic-gate	cmp	%o2, level;		/* compare new to base */	\
1077c478bdstevel@tonic-gate	movl	%xcc, level, %o2;	/* use new if base lower */	\
1087c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pil;						\
1097c478bdstevel@tonic-gate	retl;								\
1107c478bdstevel@tonic-gate	mov	%o1, %o0		/* return old PIL */
1117c478bdstevel@tonic-gate
1127c478bdstevel@tonic-gate/*
1137c478bdstevel@tonic-gate * Macro to set the priority to a specified level at or above LOCK_LEVEL.
1147c478bdstevel@tonic-gate * Doesn't require comparison to CPU->cpu_base_spl.
1157c478bdstevel@tonic-gate *
1167c478bdstevel@tonic-gate * newpil can be %o0 (not other regs used here) or a constant with
1177c478bdstevel@tonic-gate * the new PIL in the PSR_PIL field of the level arg.
1187c478bdstevel@tonic-gate */
1197c478bdstevel@tonic-gate#define	SETPRI_HIGH(level) \
1207c478bdstevel@tonic-gate	rdpr	%pil, %o1;		/* get current PIL */		\
1217c478bdstevel@tonic-gate	wrpr	%g0, level, %pil;					\
1227c478bdstevel@tonic-gate	retl;								\
1237c478bdstevel@tonic-gate	mov	%o1, %o0		/* return old PIL */
1247c478bdstevel@tonic-gate
1257c478bdstevel@tonic-gate	/*
1267c478bdstevel@tonic-gate	 * Berkley 4.3 introduced symbolically named interrupt levels
1277c478bdstevel@tonic-gate	 * as a way deal with priority in a machine independent fashion.
1287c478bdstevel@tonic-gate	 * Numbered priorities are machine specific, and should be
1297c478bdstevel@tonic-gate	 * discouraged where possible.
1307c478bdstevel@tonic-gate	 *
1317c478bdstevel@tonic-gate	 * Note, for the machine specific priorities there are
1327c478bdstevel@tonic-gate	 * examples listed for devices that use a particular priority.
1337c478bdstevel@tonic-gate	 * It should not be construed that all devices of that
1347c478bdstevel@tonic-gate	 * type should be at that priority.  It is currently were
1357c478bdstevel@tonic-gate	 * the current devices fit into the priority scheme based
1367c478bdstevel@tonic-gate	 * upon time criticalness.
1377c478bdstevel@tonic-gate	 *
1387c478bdstevel@tonic-gate	 * The underlying assumption of these assignments is that
1397c478bdstevel@tonic-gate	 * SPARC9 IPL 10 is the highest level from which a device
1407c478bdstevel@tonic-gate	 * routine can call wakeup.  Devices that interrupt from higher
1417c478bdstevel@tonic-gate	 * levels are restricted in what they can do.  If they need
1427c478bdstevel@tonic-gate	 * kernels services they should schedule a routine at a lower
1437c478bdstevel@tonic-gate	 * level (via software interrupt) to do the required
1447c478bdstevel@tonic-gate	 * processing.
1457c478bdstevel@tonic-gate	 *
1467c478bdstevel@tonic-gate	 * Examples of this higher usage:
1477c478bdstevel@tonic-gate	 *	Level	Usage
1487c478bdstevel@tonic-gate	 *	15	Asynchronous memory exceptions
1497c478bdstevel@tonic-gate	 *	14	Profiling clock (and PROM uart polling clock)
1507c478bdstevel@tonic-gate	 *	13	Audio device
1517c478bdstevel@tonic-gate	 *	12	Serial ports
1527c478bdstevel@tonic-gate	 *	11	Floppy controller
1537c478bdstevel@tonic-gate	 *
1547c478bdstevel@tonic-gate	 * The serial ports request lower level processing on level 6.
1557c478bdstevel@tonic-gate	 * Audio and floppy request lower level processing on level 4.
1567c478bdstevel@tonic-gate	 *
1577c478bdstevel@tonic-gate	 * Also, almost all splN routines (where N is a number or a
1587c478bdstevel@tonic-gate	 * mnemonic) will do a RAISE(), on the assumption that they are
1597c478bdstevel@tonic-gate	 * never used to lower our priority.
1607c478bdstevel@tonic-gate	 * The exceptions are:
1617c478bdstevel@tonic-gate	 *	spl8()		Because you can't be above 15 to begin with!
1627c478bdstevel@tonic-gate	 *	splzs()		Because this is used at boot time to lower our
1637c478bdstevel@tonic-gate	 *			priority, to allow the PROM to poll the uart.
1647c478bdstevel@tonic-gate	 *	spl0()		Used to lower priority to 0.
1657c478bdstevel@tonic-gate	 */
1667c478bdstevel@tonic-gate
1677c478bdstevel@tonic-gate	/* locks out all interrupts, including memory errors */
1687c478bdstevel@tonic-gate	ENTRY(spl8)
1697c478bdstevel@tonic-gate	SETPRI_HIGH(15)
1707c478bdstevel@tonic-gate	SET_SIZE(spl8)
1717c478bdstevel@tonic-gate
1727c478bdstevel@tonic-gate	/* just below the level that profiling runs */
1737c478bdstevel@tonic-gate	ENTRY(spl7)
1747c478bdstevel@tonic-gate	RAISE_HIGH(13)
1757c478bdstevel@tonic-gate	SET_SIZE(spl7)
1767c478bdstevel@tonic-gate
1777c478bdstevel@tonic-gate	/* sun specific - highest priority onboard serial i/o zs ports */
1787c478bdstevel@tonic-gate	ENTRY(splzs)
1797c478bdstevel@tonic-gate	SETPRI_HIGH(12)	/* Can't be a RAISE, as it's used to lower us */
1807c478bdstevel@tonic-gate	SET_SIZE(splzs)
1817c478bdstevel@tonic-gate
1827c478bdstevel@tonic-gate	/*
1837c478bdstevel@tonic-gate	 * should lock out clocks and all interrupts,
1847c478bdstevel@tonic-gate	 * as you can see, there are exceptions
1857c478bdstevel@tonic-gate	 */
1867c478bdstevel@tonic-gate	ENTRY(splhi)
1877c478bdstevel@tonic-gate	ALTENTRY(splhigh)
1887c478bdstevel@tonic-gate	ALTENTRY(spl6)
1897c478bdstevel@tonic-gate	ALTENTRY(i_ddi_splhigh)
1907c478bdstevel@tonic-gate	RAISE_HIGH(DISP_LEVEL)
1917c478bdstevel@tonic-gate	SET_SIZE(i_ddi_splhigh)
1927c478bdstevel@tonic-gate	SET_SIZE(spl6)
1937c478bdstevel@tonic-gate	SET_SIZE(splhigh)
1947c478bdstevel@tonic-gate	SET_SIZE(splhi)
1957c478bdstevel@tonic-gate
1967c478bdstevel@tonic-gate	/* allow all interrupts */
1977c478bdstevel@tonic-gate	ENTRY(spl0)
1987c478bdstevel@tonic-gate	SETPRI(0)
1997c478bdstevel@tonic-gate	SET_SIZE(spl0)
2007c478bdstevel@tonic-gate
2017c478bdstevel@tonic-gate/*
2027c478bdstevel@tonic-gate * splx - set PIL back to that indicated by the old %pil passed as an argument,
2037c478bdstevel@tonic-gate * or to the CPU's base priority, whichever is higher.
2047c478bdstevel@tonic-gate */
2057c478bdstevel@tonic-gate
2067c478bdstevel@tonic-gate	ENTRY(splx)
2077c478bdstevel@tonic-gate	ALTENTRY(i_ddi_splx)
2087c478bdstevel@tonic-gate	SETPRI(%o0)		/* set PIL */
2097c478bdstevel@tonic-gate	SET_SIZE(i_ddi_splx)
2107c478bdstevel@tonic-gate	SET_SIZE(splx)
2117c478bdstevel@tonic-gate
2127c478bdstevel@tonic-gate/*
2137c478bdstevel@tonic-gate * splr()
2147c478bdstevel@tonic-gate *
2157c478bdstevel@tonic-gate * splr is like splx but will only raise the priority and never drop it
2167c478bdstevel@tonic-gate * Be careful not to set priority lower than CPU->cpu_base_pri,
2177c478bdstevel@tonic-gate * even though it seems we're raising the priority, it could be set higher
2187c478bdstevel@tonic-gate * at any time by an interrupt routine, so we must block interrupts and
2197c478bdstevel@tonic-gate * look at CPU->cpu_base_pri.
2207c478bdstevel@tonic-gate */
2217c478bdstevel@tonic-gate
2227c478bdstevel@tonic-gate	ENTRY(splr)
2237c478bdstevel@tonic-gate	RAISE(%o0)
2247c478bdstevel@tonic-gate	SET_SIZE(splr)
2257c478bdstevel@tonic-gate
2267c478bdstevel@tonic-gate/*
2277c478bdstevel@tonic-gate * on_fault()
2287c478bdstevel@tonic-gate * Catch lofault faults. Like setjmp except it returns one
2297c478bdstevel@tonic-gate * if code following causes uncorrectable fault. Turned off
2307c478bdstevel@tonic-gate * by calling no_fault().
2317c478bdstevel@tonic-gate */
2327c478bdstevel@tonic-gate
2337c478bdstevel@tonic-gate	ENTRY(on_fault)
2347c478bdstevel@tonic-gate	membar	#Sync			! sync error barrier (see copy.s)
2357c478bdstevel@tonic-gate	stn	%o0, [THREAD_REG + T_ONFAULT]
2367c478bdstevel@tonic-gate	set	catch_fault, %o1
2377c478bdstevel@tonic-gate	b	setjmp			! let setjmp do the rest
2387c478bdstevel@tonic-gate	stn	%o1, [THREAD_REG + T_LOFAULT]	! put catch_fault in t_lofault
2397c478bdstevel@tonic-gate
2407c478bdstevel@tonic-gatecatch_fault:
2417c478bdstevel@tonic-gate	save	%sp, -SA(WINDOWSIZE), %sp ! goto next window so that we can rtn
2427c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_ONFAULT], %o0
2437c478bdstevel@tonic-gate	membar	#Sync				! sync error barrier
2447c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_ONFAULT]	! turn off onfault
2457c478bdstevel@tonic-gate	b	longjmp			! let longjmp do the rest
2467c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_LOFAULT]	! turn off lofault
2477c478bdstevel@tonic-gate	SET_SIZE(on_fault)
2487c478bdstevel@tonic-gate
2497c478bdstevel@tonic-gate/*
2507c478bdstevel@tonic-gate * no_fault()
2517c478bdstevel@tonic-gate * turn off fault catching.
2527c478bdstevel@tonic-gate */
2537c478bdstevel@tonic-gate
2547c478bdstevel@tonic-gate	ENTRY(no_fault)
2557c478bdstevel@tonic-gate	membar	#Sync				! sync error barrier
2567c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_ONFAULT]
2577c478bdstevel@tonic-gate	retl
2587c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_LOFAULT]	! turn off lofault
2597c478bdstevel@tonic-gate	SET_SIZE(no_fault)
2607c478bdstevel@tonic-gate
2617c478bdstevel@tonic-gate/*
2627c478bdstevel@tonic-gate * Default trampoline code for on_trap() (see <sys/ontrap.h>).  On sparcv9,
2637c478bdstevel@tonic-gate * the trap code will complete trap processing but reset the return %pc to
2647c478bdstevel@tonic-gate * ot_trampoline, which will by default be set to the address of this code.
2657c478bdstevel@tonic-gate * We longjmp(&curthread->t_ontrap->ot_jmpbuf) to return back to on_trap().
2667c478bdstevel@tonic-gate */
2677c478bdstevel@tonic-gate
2687c478bdstevel@tonic-gate	ENTRY(on_trap_trampoline)
2697c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_ONTRAP], %o0
2707c478bdstevel@tonic-gate	b	longjmp
2717c478bdstevel@tonic-gate	add	%o0, OT_JMPBUF, %o0
2727c478bdstevel@tonic-gate	SET_SIZE(on_trap_trampoline)
2737c478bdstevel@tonic-gate
2747c478bdstevel@tonic-gate/*
2757c478bdstevel@tonic-gate * Push a new element on to the t_ontrap stack.  Refer to <sys/ontrap.h> for
2767c478bdstevel@tonic-gate * more information about the on_trap() mechanism.  If the on_trap_data is the
2777c478bdstevel@tonic-gate * same as the topmost stack element, we just modify that element.
2787c478bdstevel@tonic-gate * On UltraSPARC, we need to issue a membar #Sync before modifying t_ontrap.
2797c478bdstevel@tonic-gate * The issue barrier is defined to force all deferred errors to complete before
2807c478bdstevel@tonic-gate * we go any further.  We want these errors to be processed before we modify
2817c478bdstevel@tonic-gate * our current error protection.
2827c478bdstevel@tonic-gate */
2837c478bdstevel@tonic-gate
2847c478bdstevel@tonic-gate	ENTRY(on_trap)
2857c478bdstevel@tonic-gate	membar	#Sync				! force error barrier
2867c478bdstevel@tonic-gate	sth	%o1, [%o0 + OT_PROT]		! ot_prot = prot
2877c478bdstevel@tonic-gate	sth	%g0, [%o0 + OT_TRAP]		! ot_trap = 0
2887c478bdstevel@tonic-gate	set	on_trap_trampoline, %o2		! %o2 = &on_trap_trampoline
2897c478bdstevel@tonic-gate	stn	%o2, [%o0 + OT_TRAMPOLINE]	! ot_trampoline = %o2
2907c478bdstevel@tonic-gate	stn	%g0, [%o0 + OT_HANDLE]		! ot_handle = NULL
2917c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_ONTRAP], %o2	! %o2 = curthread->t_ontrap
2927c478bdstevel@tonic-gate	cmp	%o0, %o2			! if (otp == %o2)
2937c478bdstevel@tonic-gate	be	0f				!    don't modify t_ontrap
2947c478bdstevel@tonic-gate	stn	%g0, [%o0 + OT_PAD1]		! delay - ot_pad1 = NULL
2957c478bdstevel@tonic-gate
2967c478bdstevel@tonic-gate	stn	%o2, [%o0 + OT_PREV]		! ot_prev = t_ontrap
2977c478bdstevel@tonic-gate	membar	#Sync				! force error barrier
2987c478bdstevel@tonic-gate	stn	%o0, [THREAD_REG + T_ONTRAP]	! t_ontrap = otp
2997c478bdstevel@tonic-gate
3007c478bdstevel@tonic-gate0:	b	setjmp				! let setjmp do the rest
3017c478bdstevel@tonic-gate	add	%o0, OT_JMPBUF, %o0		! %o0 = &ot_jmpbuf
3027c478bdstevel@tonic-gate	SET_SIZE(on_trap)
3037c478bdstevel@tonic-gate
3047c478bdstevel@tonic-gate/*
3057c478bdstevel@tonic-gate * Setjmp and longjmp implement non-local gotos using state vectors
3067c478bdstevel@tonic-gate * type label_t.
3077c478bdstevel@tonic-gate */
3087c478bdstevel@tonic-gate
3097c478bdstevel@tonic-gate	ENTRY(setjmp)
3107c478bdstevel@tonic-gate	stn	%o7, [%o0 + L_PC]	! save return address
3117c478bdstevel@tonic-gate	stn	%sp, [%o0 + L_SP]	! save stack ptr
3127c478bdstevel@tonic-gate	retl
3137c478bdstevel@tonic-gate	clr	%o0			! return 0
3147c478bdstevel@tonic-gate	SET_SIZE(setjmp)
3157c478bdstevel@tonic-gate
3167c478bdstevel@tonic-gate
3177c478bdstevel@tonic-gate	ENTRY(longjmp)
3187c478bdstevel@tonic-gate	!
3197c478bdstevel@tonic-gate        ! The following save is required so that an extra register
3207c478bdstevel@tonic-gate        ! window is flushed.  Flushw flushes nwindows-2
3217c478bdstevel@tonic-gate        ! register windows.  If setjmp and longjmp are called from
3227c478bdstevel@tonic-gate        ! within the same window, that window will not get pushed
3237c478bdstevel@tonic-gate        ! out onto the stack without the extra save below.  Tail call
3247c478bdstevel@tonic-gate        ! optimization can lead to callers of longjmp executing
3257c478bdstevel@tonic-gate        ! from a window that could be the same as the setjmp,
3267c478bdstevel@tonic-gate        ! thus the need for the following save.
3277c478bdstevel@tonic-gate        !
3287c478bdstevel@tonic-gate	save    %sp, -SA(MINFRAME), %sp
3297c478bdstevel@tonic-gate	flushw				! flush all but this window
3307c478bdstevel@tonic-gate	ldn	[%i0 + L_PC], %i7	! restore return addr
3317c478bdstevel@tonic-gate	ldn	[%i0 + L_SP], %fp	! restore sp for dest on foreign stack
3327c478bdstevel@tonic-gate	ret				! return 1
3337c478bdstevel@tonic-gate	restore	%g0, 1, %o0		! takes underflow, switches stacks
3347c478bdstevel@tonic-gate	SET_SIZE(longjmp)
3357c478bdstevel@tonic-gate
3367c478bdstevel@tonic-gate/*
3377c478bdstevel@tonic-gate * movtuc(length, from, to, table)
3387c478bdstevel@tonic-gate *
3397c478bdstevel@tonic-gate * VAX movtuc instruction (sort of).
3407c478bdstevel@tonic-gate */
3417c478bdstevel@tonic-gate
3427c478bdstevel@tonic-gate	ENTRY(movtuc)
3437c478bdstevel@tonic-gate	tst     %o0
3447c478bdstevel@tonic-gate	ble,pn	%ncc, 2f		! check length
3457c478bdstevel@tonic-gate	clr     %o4
3467c478bdstevel@tonic-gate
3477c478bdstevel@tonic-gate	ldub    [%o1 + %o4], %g1        ! get next byte in string
3487c478bdstevel@tonic-gate0:
3497c478bdstevel@tonic-gate	ldub    [%o3 + %g1], %g1        ! get corresponding table entry
3507c478bdstevel@tonic-gate	tst     %g1                     ! escape char?
3517c478bdstevel@tonic-gate	bnz     1f
3527c478bdstevel@tonic-gate	stb     %g1, [%o2 + %o4]        ! delay slot, store it
3537c478bdstevel@tonic-gate
3547c478bdstevel@tonic-gate	retl                            ! return (bytes moved)
3557c478bdstevel@tonic-gate	mov     %o4, %o0
3567c478bdstevel@tonic-gate1:
3577c478bdstevel@tonic-gate	inc     %o4                     ! increment index
3587c478bdstevel@tonic-gate	cmp     %o4, %o0                ! index < length ?
3597c478bdstevel@tonic-gate	bl,a,pt	%ncc, 0b
3607c478bdstevel@tonic-gate	ldub    [%o1 + %o4], %g1        ! delay slot, get next byte in string
3617c478bdstevel@tonic-gate2:
3627c478bdstevel@tonic-gate	retl                            ! return (bytes moved)
3637c478bdstevel@tonic-gate	mov     %o4, %o0
3647c478bdstevel@tonic-gate	SET_SIZE(movtuc)
3657c478bdstevel@tonic-gate
3667c478bdstevel@tonic-gate/*
3677c478bdstevel@tonic-gate * scanc(length, string, table, mask)
3687c478bdstevel@tonic-gate *
3697c478bdstevel@tonic-gate * VAX scanc instruction.
3707c478bdstevel@tonic-gate */
3717c478bdstevel@tonic-gate
3727c478bdstevel@tonic-gate	ENTRY(scanc)
3737c478bdstevel@tonic-gate	tst	%o0
3747c478bdstevel@tonic-gate	ble,pn	%ncc, 1f		! check length
3757c478bdstevel@tonic-gate	clr	%o4
3767c478bdstevel@tonic-gate0:
3777c478bdstevel@tonic-gate	ldub	[%o1 + %o4], %g1	! get next byte in string
3787c478bdstevel@tonic-gate	cmp	%o4, %o0		! interlock slot, index < length ?
3797c478bdstevel@tonic-gate	ldub	[%o2 + %g1], %g1	! get corresponding table entry
3807c478bdstevel@tonic-gate	bge,pn	%ncc, 1f		! interlock slot
3817c478bdstevel@tonic-gate	btst	%o3, %g1		! apply the mask
3827c478bdstevel@tonic-gate	bz,a	0b
3837c478bdstevel@tonic-gate	inc	%o4			! delay slot, increment index
3847c478bdstevel@tonic-gate1:
3857c478bdstevel@tonic-gate	retl				! return(length - index)
3867c478bdstevel@tonic-gate	sub	%o0, %o4, %o0
3877c478bdstevel@tonic-gate	SET_SIZE(scanc)
3887c478bdstevel@tonic-gate
3897c478bdstevel@tonic-gate/*
3907c478bdstevel@tonic-gate * if a() calls b() calls caller(),
3917c478bdstevel@tonic-gate * caller() returns return address in a().
3927c478bdstevel@tonic-gate */
3937c478bdstevel@tonic-gate
3947c478bdstevel@tonic-gate	ENTRY(caller)
3957c478bdstevel@tonic-gate	retl
3967c478bdstevel@tonic-gate	mov	%i7, %o0
3977c478bdstevel@tonic-gate	SET_SIZE(caller)
3987c478bdstevel@tonic-gate
3997c478bdstevel@tonic-gate/*
4007c478bdstevel@tonic-gate * if a() calls callee(), callee() returns the
4017c478bdstevel@tonic-gate * return address in a();
4027c478bdstevel@tonic-gate */
4037c478bdstevel@tonic-gate
4047c478bdstevel@tonic-gate	ENTRY(callee)
4057c478bdstevel@tonic-gate	retl
4067c478bdstevel@tonic-gate	mov	%o7, %o0
4077c478bdstevel@tonic-gate	SET_SIZE(callee)
4087c478bdstevel@tonic-gate
4097c478bdstevel@tonic-gate/*
4107c478bdstevel@tonic-gate * return the current frame pointer
4117c478bdstevel@tonic-gate */
4127c478bdstevel@tonic-gate
4137c478bdstevel@tonic-gate	ENTRY(getfp)
4147c478bdstevel@tonic-gate	retl
4157c478bdstevel@tonic-gate	mov	%fp, %o0
4167c478bdstevel@tonic-gate	SET_SIZE(getfp)
4177c478bdstevel@tonic-gate
4187c478bdstevel@tonic-gate/*
4197c478bdstevel@tonic-gate * Get vector base register
4207c478bdstevel@tonic-gate */
4217c478bdstevel@tonic-gate
4227c478bdstevel@tonic-gate	ENTRY(gettbr)
4237c478bdstevel@tonic-gate	retl
4247c478bdstevel@tonic-gate	mov     %tbr, %o0
4257c478bdstevel@tonic-gate	SET_SIZE(gettbr)
4267c478bdstevel@tonic-gate
4277c478bdstevel@tonic-gate/*
4287c478bdstevel@tonic-gate * Get processor state register, V9 faked to look like V8.
4297c478bdstevel@tonic-gate * Note: does not provide ccr.xcc and provides FPRS.FEF instead of
4307c478bdstevel@tonic-gate * PSTATE.PEF, because PSTATE.PEF is always on in order to allow the
4317c478bdstevel@tonic-gate * libc_psr memcpy routines to run without hitting the fp_disabled trap.
4327c478bdstevel@tonic-gate */
4337c478bdstevel@tonic-gate
4347c478bdstevel@tonic-gate	ENTRY(getpsr)
4357c478bdstevel@tonic-gate	rd	%ccr, %o1			! get ccr
4367c478bdstevel@tonic-gate        sll	%o1, PSR_ICC_SHIFT, %o0		! move icc to V8 psr.icc
4377c478bdstevel@tonic-gate	rd	%fprs, %o1			! get fprs
4387c478bdstevel@tonic-gate	and	%o1, FPRS_FEF, %o1		! mask out dirty upper/lower
4397c478bdstevel@tonic-gate	sllx	%o1, PSR_FPRS_FEF_SHIFT, %o1	! shift fef to V8 psr.ef
4407c478bdstevel@tonic-gate        or	%o0, %o1, %o0			! or into psr.ef
4417c478bdstevel@tonic-gate        set	V9_PSR_IMPLVER, %o1		! SI assigned impl/ver: 0xef
4427c478bdstevel@tonic-gate        retl
4437c478bdstevel@tonic-gate        or	%o0, %o1, %o0			! or into psr.impl/ver
4447c478bdstevel@tonic-gate	SET_SIZE(getpsr)
4457c478bdstevel@tonic-gate
4467c478bdstevel@tonic-gate/*
4477c478bdstevel@tonic-gate * Get current processor interrupt level
4487c478bdstevel@tonic-gate */
4497c478bdstevel@tonic-gate
4507c478bdstevel@tonic-gate	ENTRY(getpil)
4517c478bdstevel@tonic-gate	retl
4527c478bdstevel@tonic-gate	rdpr	%pil, %o0
4537c478bdstevel@tonic-gate	SET_SIZE(getpil)
4547c478bdstevel@tonic-gate
4557c478bdstevel@tonic-gate	ENTRY(setpil)
4567c478bdstevel@tonic-gate	retl
4577c478bdstevel@tonic-gate	wrpr	%g0, %o0, %pil
4587c478bdstevel@tonic-gate	SET_SIZE(setpil)
4597c478bdstevel@tonic-gate
4607c478bdstevel@tonic-gate
4617c478bdstevel@tonic-gate/*
4627c478bdstevel@tonic-gate * _insque(entryp, predp)
4637c478bdstevel@tonic-gate *
4647c478bdstevel@tonic-gate * Insert entryp after predp in a doubly linked list.
4657c478bdstevel@tonic-gate */
4667c478bdstevel@tonic-gate
4677c478bdstevel@tonic-gate	ENTRY(_insque)
4687c478bdstevel@tonic-gate	ldn	[%o1], %g1		! predp->forw
4697c478bdstevel@tonic-gate	stn	%o1, [%o0 + CPTRSIZE]	! entryp->back = predp
4707c478bdstevel@tonic-gate	stn	%g1, [%o0]		! entryp->forw = predp->forw
4717c478bdstevel@tonic-gate	stn	%o0, [%o1]		! predp->forw = entryp
4727c478bdstevel@tonic-gate	retl
4737c478bdstevel@tonic-gate	stn	%o0, [%g1 + CPTRSIZE]	! predp->forw->back = entryp
4747c478bdstevel@tonic-gate	SET_SIZE(_insque)
4757c478bdstevel@tonic-gate
4767c478bdstevel@tonic-gate/*
4777c478bdstevel@tonic-gate * _remque(entryp)
4787c478bdstevel@tonic-gate *
4797c478bdstevel@tonic-gate * Remove entryp from a doubly linked list
4807c478bdstevel@tonic-gate */
4817c478bdstevel@tonic-gate
4827c478bdstevel@tonic-gate	ENTRY(_remque)
4837c478bdstevel@tonic-gate	ldn	[%o0], %g1		! entryp->forw
4847c478bdstevel@tonic-gate	ldn	[%o0 + CPTRSIZE], %g2	! entryp->back
4857c478bdstevel@tonic-gate	stn	%g1, [%g2]		! entryp->back->forw = entryp->forw
4867c478bdstevel@tonic-gate	retl
4877c478bdstevel@tonic-gate	stn	%g2, [%g1 + CPTRSIZE]	! entryp->forw->back = entryp->back
4887c478bdstevel@tonic-gate	SET_SIZE(_remque)
4897c478bdstevel@tonic-gate
4907c478bdstevel@tonic-gate
4917c478bdstevel@tonic-gate/*
4927c478bdstevel@tonic-gate * strlen(str)
4937c478bdstevel@tonic-gate *
4947c478bdstevel@tonic-gate * Returns the number of non-NULL bytes in string argument.
4957c478bdstevel@tonic-gate *
4967c478bdstevel@tonic-gate * XXX -  why is this here, rather than the traditional file?
4977c478bdstevel@tonic-gate *	  why does it have local labels which don't start with a `.'?
4987c478bdstevel@tonic-gate */
4997c478bdstevel@tonic-gate
5007c478bdstevel@tonic-gate	ENTRY(strlen)
5017c478bdstevel@tonic-gate	mov	%o0, %o1
5027c478bdstevel@tonic-gate	andcc	%o1, 3, %o3		! is src word aligned
5037c478bdstevel@tonic-gate	bz	$nowalgnd
5047c478bdstevel@tonic-gate	clr	%o0			! length of non-zero bytes
5057c478bdstevel@tonic-gate	cmp	%o3, 2			! is src half-word aligned
5067c478bdstevel@tonic-gate	be	$s2algn
5077c478bdstevel@tonic-gate	cmp	%o3, 3			! src is byte aligned
5087c478bdstevel@tonic-gate	ldub	[%o1], %o3		! move 1 or 3 bytes to align it
5097c478bdstevel@tonic-gate	inc	1, %o1			! in either case, safe to do a byte
5107c478bdstevel@tonic-gate	be	$s3algn
5117c478bdstevel@tonic-gate	tst	%o3
5127c478bdstevel@tonic-gate$s1algn:
5137c478bdstevel@tonic-gate	bnz,a	$s2algn			! now go align dest
5147c478bdstevel@tonic-gate	inc	1, %o0
5157c478bdstevel@tonic-gate	b,a	$done
5167c478bdstevel@tonic-gate
5177c478bdstevel@tonic-gate$s2algn:
5187c478bdstevel@tonic-gate	lduh	[%o1], %o3		! know src is half-byte aligned
5197c478bdstevel@tonic-gate	inc	2, %o1
5207c478bdstevel@tonic-gate	srl	%o3, 8, %o4
5217c478bdstevel@tonic-gate	tst	%o4			! is the first byte zero
5227c478bdstevel@tonic-gate	bnz,a	1f
5237c478bdstevel@tonic-gate	inc	%o0
5247c478bdstevel@tonic-gate	b,a	$done
5257c478bdstevel@tonic-gate1:	andcc	%o3, 0xff, %o3		! is the second byte zero
5267c478bdstevel@tonic-gate	bnz,a	$nowalgnd
5277c478bdstevel@tonic-gate	inc	%o0
5287c478bdstevel@tonic-gate	b,a	$done
5297c478bdstevel@tonic-gate$s3algn:
5307c478bdstevel@tonic-gate	bnz,a	$nowalgnd
5317c478bdstevel@tonic-gate	inc	1, %o0
5327c478bdstevel@tonic-gate	b,a	$done
5337c478bdstevel@tonic-gate
5347c478bdstevel@tonic-gate$nowalgnd:
5357c478bdstevel@tonic-gate	! use trick to check if any read bytes of a word are zero
5367c478bdstevel@tonic-gate	! the following two constants will generate "byte carries"
5377c478bdstevel@tonic-gate	! and check if any bit in a byte is set, if all characters
5387c478bdstevel@tonic-gate	! are 7bits (unsigned) this allways works, otherwise
5397c478bdstevel@tonic-gate	! there is a specil case that rarely happens, see below
5407c478bdstevel@tonic-gate
5417c478bdstevel@tonic-gate	set	0x7efefeff, %o3
5427c478bdstevel@tonic-gate	set	0x81010100, %o4
5437c478bdstevel@tonic-gate
5447c478bdstevel@tonic-gate3:	ld	[%o1], %o2		! main loop
5457c478bdstevel@tonic-gate	inc	4, %o1
5467c478bdstevel@tonic-gate	add	%o2, %o3, %o5		! generate byte-carries
5477c478bdstevel@tonic-gate	xor	%o5, %o2, %o5		! see if orignal bits set
5487c478bdstevel@tonic-gate	and	%o5, %o4, %o5
5497c478bdstevel@tonic-gate	cmp	%o5, %o4		! if ==,  no zero bytes
5507c478bdstevel@tonic-gate	be,a	3b
5517c478bdstevel@tonic-gate	inc	4, %o0
5527c478bdstevel@tonic-gate
5537c478bdstevel@tonic-gate	! check for the zero byte and increment the count appropriately
5547c478bdstevel@tonic-gate	! some information (the carry bit) is lost if bit 31
5557c478bdstevel@tonic-gate	! was set (very rare), if this is the rare condition,
5567c478bdstevel@tonic-gate	! return to the main loop again
5577c478bdstevel@tonic-gate
5587c478bdstevel@tonic-gate	sethi	%hi(0xff000000), %o5	! mask used to test for terminator
5597c478bdstevel@tonic-gate	andcc	%o2, %o5, %g0		! check if first byte was zero
5607c478bdstevel@tonic-gate	bnz	1f
5617c478bdstevel@tonic-gate	srl	%o5, 8, %o5
5627c478bdstevel@tonic-gate$done:
5637c478bdstevel@tonic-gate	retl
5647c478bdstevel@tonic-gate	nop
5657c478bdstevel@tonic-gate1:	andcc	%o2, %o5, %g0		! check if second byte was zero
5667c478bdstevel@tonic-gate	bnz	1f
5677c478bdstevel@tonic-gate	srl	%o5, 8, %o5
5687c478bdstevel@tonic-gate$done1:
5697c478bdstevel@tonic-gate	retl
5707c478bdstevel@tonic-gate	inc	%o0
5717c478bdstevel@tonic-gate1:	andcc 	%o2, %o5, %g0		! check if third byte was zero
5727c478bdstevel@tonic-gate	bnz	1f
5737c478bdstevel@tonic-gate	andcc	%o2, 0xff, %g0		! check if last byte is zero
5747c478bdstevel@tonic-gate$done2:
5757c478bdstevel@tonic-gate	retl
5767c478bdstevel@tonic-gate	inc	2, %o0
5777c478bdstevel@tonic-gate1:	bnz,a	3b
5787c478bdstevel@tonic-gate	inc	4, %o0			! count of bytes
5797c478bdstevel@tonic-gate$done3:
5807c478bdstevel@tonic-gate	retl
5817c478bdstevel@tonic-gate	inc	3, %o0
5827c478bdstevel@tonic-gate	SET_SIZE(strlen)
5837c478bdstevel@tonic-gate
5847c478bdstevel@tonic-gate/*
5857c478bdstevel@tonic-gate * Provide a C callable interface to the membar instruction.
5867c478bdstevel@tonic-gate */
5877c478bdstevel@tonic-gate
5887c478bdstevel@tonic-gate	ENTRY(membar_ldld)
5897c478bdstevel@tonic-gate	retl
5907c478bdstevel@tonic-gate	membar	#LoadLoad
5917c478bdstevel@tonic-gate	SET_SIZE(membar_ldld)
5927c478bdstevel@tonic-gate
5937c478bdstevel@tonic-gate	ENTRY(membar_stld)
5947c478bdstevel@tonic-gate	retl
5957c478bdstevel@tonic-gate	membar	#StoreLoad
5967c478bdstevel@tonic-gate	SET_SIZE(membar_stld)
5977c478bdstevel@tonic-gate
5987c478bdstevel@tonic-gate	ENTRY(membar_ldst)
5997c478bdstevel@tonic-gate	retl
6007c478bdstevel@tonic-gate	membar	#LoadStore
6017c478bdstevel@tonic-gate	SET_SIZE(membar_ldst)
6027c478bdstevel@tonic-gate
6037c478bdstevel@tonic-gate	ENTRY(membar_stst)
6047c478bdstevel@tonic-gate	retl
6057c478bdstevel@tonic-gate	membar	#StoreStore
6067c478bdstevel@tonic-gate	SET_SIZE(membar_stst)
6077c478bdstevel@tonic-gate
6087c478bdstevel@tonic-gate	ENTRY(membar_ldld_stld)
6097c478bdstevel@tonic-gate	ALTENTRY(membar_stld_ldld)
6107c478bdstevel@tonic-gate	retl
6117c478bdstevel@tonic-gate	membar	#LoadLoad|#StoreLoad
6127c478bdstevel@tonic-gate	SET_SIZE(membar_stld_ldld)
6137c478bdstevel@tonic-gate	SET_SIZE(membar_ldld_stld)
6147c478bdstevel@tonic-gate
6157c478bdstevel@tonic-gate	ENTRY(membar_ldld_ldst)
6167c478bdstevel@tonic-gate	ALTENTRY(membar_ldst_ldld)
6177c478bdstevel@tonic-gate	retl
6187c478bdstevel@tonic-gate	membar	#LoadLoad|#LoadStore
6197c478bdstevel@tonic-gate	SET_SIZE(membar_ldst_ldld)
6207c478bdstevel@tonic-gate	SET_SIZE(membar_ldld_ldst)
6217c478bdstevel@tonic-gate
6227c478bdstevel@tonic-gate	ENTRY(membar_ldld_stst)
6237c478bdstevel@tonic-gate	ALTENTRY(membar_stst_ldld)
6247c478bdstevel@tonic-gate	retl
6257c478bdstevel@tonic-gate	membar	#LoadLoad|#StoreStore
6267c478bdstevel@tonic-gate	SET_SIZE(membar_stst_ldld)
6277c478bdstevel@tonic-gate	SET_SIZE(membar_ldld_stst)
6287c478bdstevel@tonic-gate
6297c478bdstevel@tonic-gate	ENTRY(membar_stld_ldst)
6307c478bdstevel@tonic-gate	ALTENTRY(membar_ldst_stld)
6317c478bdstevel@tonic-gate	retl
6327c478bdstevel@tonic-gate	membar	#StoreLoad|#LoadStore
6337c478bdstevel@tonic-gate	SET_SIZE(membar_ldst_stld)
6347c478bdstevel@tonic-gate	SET_SIZE(membar_stld_ldst)
6357c478bdstevel@tonic-gate
6367c478bdstevel@tonic-gate	ENTRY(membar_stld_stst)
6377c478bdstevel@tonic-gate	ALTENTRY(membar_stst_stld)
6387c478bdstevel@tonic-gate	retl
6397c478bdstevel@tonic-gate	membar	#StoreLoad|#StoreStore
6407c478bdstevel@tonic-gate	SET_SIZE(membar_stst_stld)
6417c478bdstevel@tonic-gate	SET_SIZE(membar_stld_stst)
6427c478bdstevel@tonic-gate
6437c478bdstevel@tonic-gate	ENTRY(membar_ldst_stst)
6447c478bdstevel@tonic-gate	ALTENTRY(membar_stst_ldst)
6457c478bdstevel@tonic-gate	retl
6467c478bdstevel@tonic-gate	membar	#LoadStore|#StoreStore
6477c478bdstevel@tonic-gate	SET_SIZE(membar_stst_ldst)
6487c478bdstevel@tonic-gate	SET_SIZE(membar_ldst_stst)
6497c478bdstevel@tonic-gate
6507c478bdstevel@tonic-gate	ENTRY(membar_lookaside)
6517c478bdstevel@tonic-gate	retl
6527c478bdstevel@tonic-gate	membar	#Lookaside
6537c478bdstevel@tonic-gate	SET_SIZE(membar_lookaside)
6547c478bdstevel@tonic-gate
6557c478bdstevel@tonic-gate	ENTRY(membar_memissue)
6567c478bdstevel@tonic-gate	retl
6577c478bdstevel@tonic-gate	membar	#MemIssue
6587c478bdstevel@tonic-gate	SET_SIZE(membar_memissue)
6597c478bdstevel@tonic-gate
6607c478bdstevel@tonic-gate	ENTRY(membar_sync)
6617c478bdstevel@tonic-gate	retl
6627c478bdstevel@tonic-gate	membar	#Sync
6637c478bdstevel@tonic-gate	SET_SIZE(membar_sync)
6647c478bdstevel@tonic-gate
6657c478bdstevel@tonic-gate
6667c478bdstevel@tonic-gate/*
6677c478bdstevel@tonic-gate * Since all of the fuword() variants are so similar, we have a macro to spit
6687c478bdstevel@tonic-gate * them out.
6697c478bdstevel@tonic-gate */
6707c478bdstevel@tonic-gate
6717c478bdstevel@tonic-gate#define	FUWORD(NAME, LOAD, STORE, COPYOP)	\
6727c478bdstevel@tonic-gate	ENTRY(NAME);				\
6737c478bdstevel@tonic-gate	sethi	%hi(1f), %o5;			\
6747c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_LOFAULT], %o3;	\
6757c478bdstevel@tonic-gate	or	%o5, %lo(1f), %o5;		\
6767c478bdstevel@tonic-gate	membar	#Sync;				\
6777c478bdstevel@tonic-gate	stn	%o5, [THREAD_REG + T_LOFAULT];	\
6787c478bdstevel@tonic-gate	LOAD	[%o0]ASI_USER, %o2;		\
6797c478bdstevel@tonic-gate	membar	#Sync;				\
6807c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LOFAULT];	\
6817c478bdstevel@tonic-gate	mov	0, %o0;				\
6827c478bdstevel@tonic-gate	retl;					\
6837c478bdstevel@tonic-gate	STORE	%o2, [%o1];			\
6847c478bdstevel@tonic-gate1:						\
6857c478bdstevel@tonic-gate	membar	#Sync;				\
6867c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LOFAULT];	\
6877c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_COPYOPS], %o2;	\
6887c478bdstevel@tonic-gate	brz	%o2, 2f;			\
6897c478bdstevel@tonic-gate	nop;					\
6907c478bdstevel@tonic-gate	ldn	[%o2 + COPYOP], %g1;		\
6917c478bdstevel@tonic-gate	jmp	%g1;				\
6927c478bdstevel@tonic-gate	nop;					\
6937c478bdstevel@tonic-gate2:						\
6947c478bdstevel@tonic-gate	retl;					\
6957c478bdstevel@tonic-gate	mov	-1, %o0;			\
6967c478bdstevel@tonic-gate	SET_SIZE(NAME)
6977c478bdstevel@tonic-gate
6987c478bdstevel@tonic-gate	FUWORD(fuword64, ldxa, stx, CP_FUWORD64)
6997c478bdstevel@tonic-gate	FUWORD(fuword32, lda, st, CP_FUWORD32)
7007c478bdstevel@tonic-gate	FUWORD(fuword16, lduha, sth, CP_FUWORD16)
7017c478bdstevel@tonic-gate	FUWORD(fuword8, lduba, stb, CP_FUWORD8)
7027c478bdstevel@tonic-gate
7037c478bdstevel@tonic-gate
7047c478bdstevel@tonic-gate/*
7057c478bdstevel@tonic-gate * Since all of the suword() variants are so similar, we have a macro to spit
7067c478bdstevel@tonic-gate * them out.
7077c478bdstevel@tonic-gate */
7087c478bdstevel@tonic-gate
7097c478bdstevel@tonic-gate#define	SUWORD(NAME, STORE, COPYOP)		\
7107c478bdstevel@tonic-gate	ENTRY(NAME)				\
7117c478bdstevel@tonic-gate	sethi	%hi(1f), %o5;			\
7127c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_LOFAULT], %o3;	\
7137c478bdstevel@tonic-gate	or	%o5, %lo(1f), %o5;		\
7147c478bdstevel@tonic-gate	membar	#Sync;				\
7157c478bdstevel@tonic-gate	stn	%o5, [THREAD_REG + T_LOFAULT];	\
7167c478bdstevel@tonic-gate	STORE	%o1, [%o0]ASI_USER;		\
7177c478bdstevel@tonic-gate	membar	#Sync;				\
7187c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LOFAULT];	\
7197c478bdstevel@tonic-gate	retl;					\
7207c478bdstevel@tonic-gate	clr	%o0;				\
7217c478bdstevel@tonic-gate1:						\
7227c478bdstevel@tonic-gate	membar	#Sync;				\
7237c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LOFAULT];	\
7247c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_COPYOPS], %o2;	\
7257c478bdstevel@tonic-gate	brz	%o2, 2f;			\
7267c478bdstevel@tonic-gate	nop;					\
7277c478bdstevel@tonic-gate	ldn	[%o2 + COPYOP], %g1;		\
7287c478bdstevel@tonic-gate	jmp	%g1;				\
7297c478bdstevel@tonic-gate	nop;					\
7307c478bdstevel@tonic-gate2:						\
7317c478bdstevel@tonic-gate	retl;					\
7327c478bdstevel@tonic-gate	mov	-1, %o0;			\
7337c478bdstevel@tonic-gate	SET_SIZE(NAME)
7347c478bdstevel@tonic-gate
7357c478bdstevel@tonic-gate	SUWORD(suword64, stxa, CP_SUWORD64)
7367c478bdstevel@tonic-gate	SUWORD(suword32, sta, CP_SUWORD32)
7377c478bdstevel@tonic-gate	SUWORD(suword16, stha, CP_SUWORD16)
7387c478bdstevel@tonic-gate	SUWORD(suword8, stba, CP_SUWORD8)
7397c478bdstevel@tonic-gate
7407c478bdstevel@tonic-gate	ENTRY(fuword8_noerr)
7417c478bdstevel@tonic-gate	lduba	[%o0]ASI_USER, %o0
7427c478bdstevel@tonic-gate	retl
7437c478bdstevel@tonic-gate	stb	%o0, [%o1]
7447c478bdstevel@tonic-gate	SET_SIZE(fuword8_noerr)
7457c478bdstevel@tonic-gate
7467c478bdstevel@tonic-gate	ENTRY(fuword16_noerr)
7477c478bdstevel@tonic-gate	lduha	[%o0]ASI_USER, %o0
7487c478bdstevel@tonic-gate	retl
7497c478bdstevel@tonic-gate	sth	%o0, [%o1]
7507c478bdstevel@tonic-gate	SET_SIZE(fuword16_noerr)
7517c478bdstevel@tonic-gate
7527c478bdstevel@tonic-gate	ENTRY(fuword32_noerr)
7537c478bdstevel@tonic-gate	lda	[%o0]ASI_USER, %o0
7547c478bdstevel@tonic-gate	retl
7557c478bdstevel@tonic-gate	st	%o0, [%o1]
7567c478bdstevel@tonic-gate	SET_SIZE(fuword32_noerr)
7577c478bdstevel@tonic-gate
7587c478bdstevel@tonic-gate	ENTRY(fuword64_noerr)
7597c478bdstevel@tonic-gate	ldxa	[%o0]ASI_USER, %o0
7607c478bdstevel@tonic-gate	retl
7617c478bdstevel@tonic-gate	stx	%o0, [%o1]
7627c478bdstevel@tonic-gate	SET_SIZE(fuword64_noerr)
7637c478bdstevel@tonic-gate
7647c478bdstevel@tonic-gate	ENTRY(suword8_noerr)
7657c478bdstevel@tonic-gate	retl
7667c478bdstevel@tonic-gate	stba	%o1, [%o0]ASI_USER
7677c478bdstevel@tonic-gate	SET_SIZE(suword8_noerr)
7687c478bdstevel@tonic-gate
7697c478bdstevel@tonic-gate	ENTRY(suword16_noerr)
7707c478bdstevel@tonic-gate	retl
7717c478bdstevel@tonic-gate	stha	%o1, [%o0]ASI_USER
7727c478bdstevel@tonic-gate	SET_SIZE(suword16_noerr)
7737c478bdstevel@tonic-gate
7747c478bdstevel@tonic-gate	ENTRY(suword32_noerr)
7757c478bdstevel@tonic-gate	retl
7767c478bdstevel@tonic-gate	sta	%o1, [%o0]ASI_USER
7777c478bdstevel@tonic-gate	SET_SIZE(suword32_noerr)
7787c478bdstevel@tonic-gate
7797c478bdstevel@tonic-gate	ENTRY(suword64_noerr)
7807c478bdstevel@tonic-gate	retl
7817c478bdstevel@tonic-gate	stxa	%o1, [%o0]ASI_USER
7827c478bdstevel@tonic-gate	SET_SIZE(suword64_noerr)
7837c478bdstevel@tonic-gate
7847c478bdstevel@tonic-gate	.weak	subyte
7857c478bdstevel@tonic-gate	subyte=suword8
7867c478bdstevel@tonic-gate	.weak	subyte_noerr
7877c478bdstevel@tonic-gate	subyte_noerr=suword8_noerr
7887c478bdstevel@tonic-gate#ifdef _LP64
7897c478bdstevel@tonic-gate	.weak	fulword
7907c478bdstevel@tonic-gate	fulword=fuword64
7917c478bdstevel@tonic-gate	.weak	fulword_noerr
7927c478bdstevel@tonic-gate	fulword_noerr=fuword64_noerr
7937c478bdstevel@tonic-gate	.weak	sulword
7947c478bdstevel@tonic-gate	sulword=suword64
7957c478bdstevel@tonic-gate	.weak	sulword_noerr
7967c478bdstevel@tonic-gate	sulword_noerr=suword64_noerr
7977c478bdstevel@tonic-gate#else
7987c478bdstevel@tonic-gate	.weak	fulword
7997c478bdstevel@tonic-gate	fulword=fuword32
8007c478bdstevel@tonic-gate	.weak	fulword_noerr
8017c478bdstevel@tonic-gate	fulword_noerr=fuword32_noerr
8027c478bdstevel@tonic-gate	.weak	sulword
8037c478bdstevel@tonic-gate	sulword=suword32
8047c478bdstevel@tonic-gate	.weak	sulword_noerr
8057c478bdstevel@tonic-gate	sulword_noerr=suword32_noerr
8067c478bdstevel@tonic-gate#endif	/* LP64 */
8077c478bdstevel@tonic-gate
808023e71dHaik Aftandilian/*
809023e71dHaik Aftandilian * We define rdtick here, but not for sun4v. On sun4v systems, the %tick
810023e71dHaik Aftandilian * and %stick should not be read directly without considering the tick
811023e71dHaik Aftandilian * and stick offset kernel variables introduced to support sun4v OS
812023e71dHaik Aftandilian * suspension.
813023e71dHaik Aftandilian */
814023e71dHaik Aftandilian#if !defined (sun4v)
8157c478bdstevel@tonic-gate
8167c478bdstevel@tonic-gate	ENTRY(rdtick)
8177c478bdstevel@tonic-gate	retl
8187c478bdstevel@tonic-gate	rd	%tick, %o0
8197c478bdstevel@tonic-gate        SET_SIZE(rdtick)
820023e71dHaik Aftandilian
821023e71dHaik Aftandilian#endif /* !sun4v */
8227c478bdstevel@tonic-gate
8237c478bdstevel@tonic-gate/*
8247c478bdstevel@tonic-gate * Set tba to given address, no side effects.
8257c478bdstevel@tonic-gate */
8267c478bdstevel@tonic-gate
8277c478bdstevel@tonic-gate	ENTRY(set_tba)
8287c478bdstevel@tonic-gate	mov	%o0, %o1
8297c478bdstevel@tonic-gate	rdpr	%tba, %o0
8307c478bdstevel@tonic-gate	wrpr	%o1, %tba
8317c478bdstevel@tonic-gate	retl
8327c478bdstevel@tonic-gate	nop
8337c478bdstevel@tonic-gate	SET_SIZE(set_tba)
8347c478bdstevel@tonic-gate
8357c478bdstevel@tonic-gate	ENTRY(get_tba)
8367c478bdstevel@tonic-gate	retl
8377c478bdstevel@tonic-gate	rdpr	%tba, %o0
8387c478bdstevel@tonic-gate	SET_SIZE(get_tba)
8397c478bdstevel@tonic-gate
8407c478bdstevel@tonic-gate	ENTRY_NP(setpstate)
8417c478bdstevel@tonic-gate	retl
8427c478bdstevel@tonic-gate	wrpr	%g0, %o0, %pstate
8437c478bdstevel@tonic-gate	SET_SIZE(setpstate)
8447c478bdstevel@tonic-gate
8457c478bdstevel@tonic-gate	ENTRY_NP(getpstate)
8467c478bdstevel@tonic-gate	retl
8477c478bdstevel@tonic-gate	rdpr	%pstate, %o0
8487c478bdstevel@tonic-gate	SET_SIZE(getpstate)
8497c478bdstevel@tonic-gate
8507c478bdstevel@tonic-gate	ENTRY_NP(dtrace_interrupt_disable)
8517c478bdstevel@tonic-gate	rdpr	%pstate, %o0
8527c478bdstevel@tonic-gate	andn	%o0, PSTATE_IE, %o1
8537c478bdstevel@tonic-gate	retl
8547c478bdstevel@tonic-gate	wrpr	%g0, %o1, %pstate
8557c478bdstevel@tonic-gate	SET_SIZE(dtrace_interrupt_disable)
8567c478bdstevel@tonic-gate
8577c478bdstevel@tonic-gate	ENTRY_NP(dtrace_interrupt_enable)
8587c478bdstevel@tonic-gate	retl
8597c478bdstevel@tonic-gate	wrpr	%g0, %o0, %pstate
8607c478bdstevel@tonic-gate	SET_SIZE(dtrace_interrupt_enable)
8617c478bdstevel@tonic-gate
8627c478bdstevel@tonic-gate#ifdef SF_ERRATA_51
8637c478bdstevel@tonic-gate	.align 32
8647c478bdstevel@tonic-gate	ENTRY(dtrace_membar_return)
8657c478bdstevel@tonic-gate	retl
8667c478bdstevel@tonic-gate	nop
8677c478bdstevel@tonic-gate	SET_SIZE(dtrace_membar_return)
8687c478bdstevel@tonic-gate#define	DTRACE_MEMBAR_RETURN	ba,pt %icc, dtrace_membar_return
8697c478bdstevel@tonic-gate#else
8707c478bdstevel@tonic-gate#define	DTRACE_MEMBAR_RETURN	retl
8717c478bdstevel@tonic-gate#endif
8727c478bdstevel@tonic-gate
8737c478bdstevel@tonic-gate	ENTRY(dtrace_membar_producer)
8747c478bdstevel@tonic-gate	DTRACE_MEMBAR_RETURN
8757c478bdstevel@tonic-gate	membar	#StoreStore
8767c478bdstevel@tonic-gate	SET_SIZE(dtrace_membar_producer)
8777c478bdstevel@tonic-gate
8787c478bdstevel@tonic-gate	ENTRY(dtrace_membar_consumer)
8797c478bdstevel@tonic-gate	DTRACE_MEMBAR_RETURN
8807c478bdstevel@tonic-gate	membar	#LoadLoad
8817c478bdstevel@tonic-gate	SET_SIZE(dtrace_membar_consumer)
8827c478bdstevel@tonic-gate
8837c478bdstevel@tonic-gate	ENTRY_NP(dtrace_flush_windows)
8847c478bdstevel@tonic-gate	retl
8857c478bdstevel@tonic-gate	flushw
8867c478bdstevel@tonic-gate	SET_SIZE(dtrace_flush_windows)
8877c478bdstevel@tonic-gate
8887c478bdstevel@tonic-gate	/*
8897c478bdstevel@tonic-gate	 * %g1	pcstack
8907c478bdstevel@tonic-gate	 * %g2	iteration count
8917c478bdstevel@tonic-gate	 * %g3	final %fp
8927c478bdstevel@tonic-gate	 * %g4	final %i7
8937c478bdstevel@tonic-gate	 * %g5	saved %cwp (so we can get back to the original window)
8947c478bdstevel@tonic-gate	 *
8957c478bdstevel@tonic-gate	 * %o0	pcstack / return value (iteration count)
8967c478bdstevel@tonic-gate	 * %o1	limit / saved %cansave
8977c478bdstevel@tonic-gate	 * %o2	lastfp
8987c478bdstevel@tonic-gate	 * %o3	lastpc
8997c478bdstevel@tonic-gate	 * %o4	saved %canrestore
9007c478bdstevel@tonic-gate	 * %o5	saved %pstate (to restore interrupts)
9017c478bdstevel@tonic-gate	 *
9027c478bdstevel@tonic-gate	 * Note:  The frame pointer returned via lastfp is safe to use as
9037c478bdstevel@tonic-gate	 *	long as getpcstack_top() returns either (0) or a value less
9047c478bdstevel@tonic-gate	 *	than (limit).
9057c478bdstevel@tonic-gate	 */
9067c478bdstevel@tonic-gate	ENTRY_NP(getpcstack_top)
9077c478bdstevel@tonic-gate
9087c478bdstevel@tonic-gate	rdpr	%pstate, %o5
9097c478bdstevel@tonic-gate	andn	%o5, PSTATE_IE, %g1
9107c478bdstevel@tonic-gate	wrpr	%g0, %g1, %pstate	! disable interrupts
9117c478bdstevel@tonic-gate
9127c478bdstevel@tonic-gate	mov	%o0, %g1		! we need the pcstack pointer while
9137c478bdstevel@tonic-gate					! we're visiting other windows
9147c478bdstevel@tonic-gate
9157c478bdstevel@tonic-gate	rdpr	%canrestore, %g2	! number of available windows
9167c478bdstevel@tonic-gate	sub	%g2, 1, %g2		! account for skipped frame
9177c478bdstevel@tonic-gate	cmp	%g2, %o1		! compare with limit
9187c478bdstevel@tonic-gate	movg	%icc, %o1, %g2		! %g2 = min(%canrestore-1, limit)
9197c478bdstevel@tonic-gate
9207c478bdstevel@tonic-gate	brlez,a,pn %g2, 3f		! Use slow path if count <= 0 --
9217c478bdstevel@tonic-gate	clr	%o0			! return zero.
9227c478bdstevel@tonic-gate
9237c478bdstevel@tonic-gate	mov	%g2, %o0		! set up return value
9247c478bdstevel@tonic-gate
9257c478bdstevel@tonic-gate	rdpr	%cwp, %g5		! remember the register window state
9267c478bdstevel@tonic-gate	rdpr	%cansave, %o1		! 'restore' changes, so we can undo
9277c478bdstevel@tonic-gate	rdpr	%canrestore, %o4	! its effects when we finish.
9287c478bdstevel@tonic-gate
9297c478bdstevel@tonic-gate	restore				! skip caller's frame
9307c478bdstevel@tonic-gate1:
9317c478bdstevel@tonic-gate	st	%i7, [%g1]		! stash return address in pcstack
9327c478bdstevel@tonic-gate	restore				! go to the next frame
9337c478bdstevel@tonic-gate	subcc	%g2, 1, %g2		! decrement the count
9347c478bdstevel@tonic-gate	bnz,pt	%icc, 1b		! loop until count reaches 0
9357c478bdstevel@tonic-gate	add	%g1, 4, %g1		! increment pcstack
9367c478bdstevel@tonic-gate
9377c478bdstevel@tonic-gate	mov	%i6, %g3		! copy the final %fp and return PC
9387c478bdstevel@tonic-gate	mov	%i7, %g4		! aside so we can return them to our
9397c478bdstevel@tonic-gate					! caller
9407c478bdstevel@tonic-gate
9417c478bdstevel@tonic-gate	wrpr	%g0, %g5, %cwp		! jump back to the original window
9427c478bdstevel@tonic-gate	wrpr	%g0, %o1, %cansave	! and restore the original register
9437c478bdstevel@tonic-gate	wrpr	%g0, %o4, %canrestore	! window state.
9447c478bdstevel@tonic-gate2:
9457c478bdstevel@tonic-gate	stn	%g3, [%o2]		! store the frame pointer and pc
9467c478bdstevel@tonic-gate	st	%g4, [%o3]		! so our caller can continue the trace
9477c478bdstevel@tonic-gate
9487c478bdstevel@tonic-gate	retl				! return to caller
9497c478bdstevel@tonic-gate	wrpr	%g0, %o5, %pstate	! restore interrupts
9507c478bdstevel@tonic-gate
9517c478bdstevel@tonic-gate3:
9527c478bdstevel@tonic-gate	flushw				! flush register windows, then
9537c478bdstevel@tonic-gate	ldn	[%fp + STACK_BIAS + 14*CLONGSIZE], %g3	! load initial fp
9547c478bdstevel@tonic-gate	ba	2b
9557c478bdstevel@tonic-gate	ldn	[%fp + STACK_BIAS + 15*CLONGSIZE], %g4	! and pc
9567c478bdstevel@tonic-gate	SET_SIZE(getpcstack_top)
9577c478bdstevel@tonic-gate
9587c478bdstevel@tonic-gate	ENTRY_NP(setwstate)
9597c478bdstevel@tonic-gate	retl
9607c478bdstevel@tonic-gate	wrpr	%g0, %o0, %wstate
9617c478bdstevel@tonic-gate	SET_SIZE(setwstate)
9627c478bdstevel@tonic-gate
9637c478bdstevel@tonic-gate
9647c478bdstevel@tonic-gate	ENTRY_NP(getwstate)
9657c478bdstevel@tonic-gate	retl
9667c478bdstevel@tonic-gate	rdpr	%wstate, %o0
9677c478bdstevel@tonic-gate	SET_SIZE(getwstate)
9687c478bdstevel@tonic-gate
9697c478bdstevel@tonic-gate
9707c478bdstevel@tonic-gate/*
9717c478bdstevel@tonic-gate * int panic_trigger(int *tp)
9727c478bdstevel@tonic-gate *
9737c478bdstevel@tonic-gate * A panic trigger is a word which is updated atomically and can only be set
9747c478bdstevel@tonic-gate * once.  We atomically store 0xFF into the high byte and load the old value.
9757c478bdstevel@tonic-gate * If the byte was 0xFF, the trigger has already been activated and we fail.
9767c478bdstevel@tonic-gate * If the previous value was 0 or not 0xFF, we succeed.  This allows a
9777c478bdstevel@tonic-gate * partially corrupt trigger to still trigger correctly.  DTrace has its own
9787c478bdstevel@tonic-gate * version of this function to allow it to panic correctly from probe context.
9797c478bdstevel@tonic-gate */
9807c478bdstevel@tonic-gate
9817c478bdstevel@tonic-gate	ENTRY_NP(panic_trigger)
9827c478bdstevel@tonic-gate	ldstub	[%o0], %o0		! store 0xFF, load byte into %o0
9837c478bdstevel@tonic-gate	cmp	%o0, 0xFF		! compare %o0 to 0xFF
9847c478bdstevel@tonic-gate	set	1, %o1			! %o1 = 1
9857c478bdstevel@tonic-gate	be,a	0f			! if (%o0 == 0xFF) goto 0f (else annul)
9867c478bdstevel@tonic-gate	set	0, %o1			! delay - %o1 = 0
9877c478bdstevel@tonic-gate0:	retl
9887c478bdstevel@tonic-gate	mov	%o1, %o0		! return (%o1);
9897c478bdstevel@tonic-gate	SET_SIZE(panic_trigger)
9907c478bdstevel@tonic-gate
9917c478bdstevel@tonic-gate	ENTRY_NP(dtrace_panic_trigger)
9927c478bdstevel@tonic-gate	ldstub	[%o0], %o0		! store 0xFF, load byte into %o0
9937c478bdstevel@tonic-gate	cmp	%o0, 0xFF		! compare %o0 to 0xFF
9947c478bdstevel@tonic-gate	set	1, %o1			! %o1 = 1
9957c478bdstevel@tonic-gate	be,a	0f			! if (%o0 == 0xFF) goto 0f (else annul)
9967c478bdstevel@tonic-gate	set	0, %o1			! delay - %o1 = 0
9977c478bdstevel@tonic-gate0:	retl
9987c478bdstevel@tonic-gate	mov	%o1, %o0		! return (%o1);
9997c478bdstevel@tonic-gate	SET_SIZE(dtrace_panic_trigger)
10007c478bdstevel@tonic-gate
10017c478bdstevel@tonic-gate/*
10027c478bdstevel@tonic-gate * void vpanic(const char *format, va_list alist)
10037c478bdstevel@tonic-gate *
10047c478bdstevel@tonic-gate * The panic() and cmn_err() functions invoke vpanic() as a common entry point
10057c478bdstevel@tonic-gate * into the panic code implemented in panicsys().  vpanic() is responsible
10067c478bdstevel@tonic-gate * for passing through the format string and arguments, and constructing a
10077c478bdstevel@tonic-gate * regs structure on the stack into which it saves the current register
10087c478bdstevel@tonic-gate * values.  If we are not dying due to a fatal trap, these registers will
10097c478bdstevel@tonic-gate * then be preserved in panicbuf as the current processor state.  Before
10107c478bdstevel@tonic-gate * invoking panicsys(), vpanic() activates the first panic trigger (see
10117c478bdstevel@tonic-gate * common/os/panic.c) and switches to the panic_stack if successful.  Note that
10127c478bdstevel@tonic-gate * DTrace takes a slightly different panic path if it must panic from probe
10137c478bdstevel@tonic-gate * context.  Instead of calling panic, it calls into dtrace_vpanic(), which
10147c478bdstevel@tonic-gate * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
10157c478bdstevel@tonic-gate * branches back into vpanic().
10167c478bdstevel@tonic-gate */
10177c478bdstevel@tonic-gate
10187c478bdstevel@tonic-gate	ENTRY_NP(vpanic)
10197c478bdstevel@tonic-gate
10207c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME + REGSIZE), %sp	! save and allocate regs
10217c478bdstevel@tonic-gate
10227c478bdstevel@tonic-gate	!
10237c478bdstevel@tonic-gate	! The v9 struct regs has a 64-bit r_tstate field, which we use here
10247c478bdstevel@tonic-gate	! to store the %ccr, %asi, %pstate, and %cwp as they would appear
10257c478bdstevel@tonic-gate	! in %tstate if a trap occurred.  We leave it up to the debugger to
10267c478bdstevel@tonic-gate	! realize what happened and extract the register values.
10277c478bdstevel@tonic-gate	!
10287c478bdstevel@tonic-gate	rd	%ccr, %l0				! %l0 = %ccr
10297c478bdstevel@tonic-gate	sllx	%l0, TSTATE_CCR_SHIFT, %l0		! %l0 <<= CCR_SHIFT
10307c478bdstevel@tonic-gate	rd	%asi, %l1				! %l1 = %asi
10317c478bdstevel@tonic-gate	sllx	%l1, TSTATE_ASI_SHIFT, %l1		! %l1 <<= ASI_SHIFT
10327c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
10337c478bdstevel@tonic-gate	rdpr	%pstate, %l1				! %l1 = %pstate
10347c478bdstevel@tonic-gate	sllx	%l1, TSTATE_PSTATE_SHIFT, %l1		! %l1 <<= PSTATE_SHIFT
10357c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
10367c478bdstevel@tonic-gate	rdpr	%cwp, %l1				! %l1 = %cwp
10377c478bdstevel@tonic-gate	sllx	%l1, TSTATE_CWP_SHIFT, %l1		! %l1 <<= CWP_SHIFT
10387c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
10397c478bdstevel@tonic-gate
10407c478bdstevel@tonic-gate	set	vpanic, %l1				! %l1 = %pc (vpanic)
10417c478bdstevel@tonic-gate	add	%l1, 4, %l2				! %l2 = %npc (vpanic+4)
10427c478bdstevel@tonic-gate	rd	%y, %l3					! %l3 = %y
10437c478bdstevel@tonic-gate	!
10447c478bdstevel@tonic-gate	! Flush register windows before panic_trigger() in order to avoid a
10457c478bdstevel@tonic-gate	! problem that a dump hangs if flush_windows() causes another panic.
10467c478bdstevel@tonic-gate	!
10477c478bdstevel@tonic-gate	call	flush_windows
10487c478bdstevel@tonic-gate	nop
10497c478bdstevel@tonic-gate
10507c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %o0
10517c478bdstevel@tonic-gate	call	panic_trigger
10527c478bdstevel@tonic-gate	or	%o0, %lo(panic_quiesce), %o0		! if (!panic_trigger(
10537c478bdstevel@tonic-gate
10547c478bdstevel@tonic-gatevpanic_common:
10557c478bdstevel@tonic-gate	tst	%o0					!     &panic_quiesce))
10567c478bdstevel@tonic-gate	be	0f					!   goto 0f;
10577c478bdstevel@tonic-gate	mov	%o0, %l4				!   delay - %l4 = %o0
10587c478bdstevel@tonic-gate
10597c478bdstevel@tonic-gate	!
10607c478bdstevel@tonic-gate	! If panic_trigger() was successful, we are the first to initiate a
10617c478bdstevel@tonic-gate	! panic: switch to the panic_stack.
10627c478bdstevel@tonic-gate	!
10637c478bdstevel@tonic-gate	set	panic_stack, %o0			! %o0 = panic_stack
10647c478bdstevel@tonic-gate	set	PANICSTKSIZE, %o1			! %o1 = size of stack
10657c478bdstevel@tonic-gate	add	%o0, %o1, %o0				! %o0 = top of stack
10667c478bdstevel@tonic-gate
10677c478bdstevel@tonic-gate	sub	%o0, SA(MINFRAME + REGSIZE) + STACK_BIAS, %sp
10687c478bdstevel@tonic-gate
10697c478bdstevel@tonic-gate	!
10707c478bdstevel@tonic-gate	! Now that we've got everything set up, store each register to its
10717c478bdstevel@tonic-gate	! designated location in the regs structure allocated on the stack.
10727c478bdstevel@tonic-gate	! The register set we store is the equivalent of the registers at
10737c478bdstevel@tonic-gate	! the time the %pc was pointing to vpanic, thus the %i's now contain
10747c478bdstevel@tonic-gate	! what the %o's contained prior to the save instruction.
10757c478bdstevel@tonic-gate	!
10767c478bdstevel@tonic-gate0:	stx	%l0, [%sp + STACK_BIAS + SA(MINFRAME) + TSTATE_OFF]
10777c478bdstevel@tonic-gate	stx	%g1, [%sp + STACK_BIAS + SA(MINFRAME) + G1_OFF]
10787c478bdstevel@tonic-gate	stx	%g2, [%sp + STACK_BIAS + SA(MINFRAME) + G2_OFF]
10797c478bdstevel@tonic-gate	stx	%g3, [%sp + STACK_BIAS + SA(MINFRAME) + G3_OFF]
10807c478bdstevel@tonic-gate	stx	%g4, [%sp + STACK_BIAS + SA(MINFRAME) + G4_OFF]
10817c478bdstevel@tonic-gate	stx	%g5, [%sp + STACK_BIAS + SA(MINFRAME) + G5_OFF]
10827c478bdstevel@tonic-gate	stx	%g6, [%sp + STACK_BIAS + SA(MINFRAME) + G6_OFF]
10837c478bdstevel@tonic-gate	stx	%g7, [%sp + STACK_BIAS + SA(MINFRAME) + G7_OFF]
10847c478bdstevel@tonic-gate	stx	%i0, [%sp + STACK_BIAS + SA(MINFRAME) + O0_OFF]
10857c478bdstevel@tonic-gate	stx	%i1, [%sp + STACK_BIAS + SA(MINFRAME) + O1_OFF]
10867c478bdstevel@tonic-gate	stx	%i2, [%sp + STACK_BIAS + SA(MINFRAME) + O2_OFF]
10877c478bdstevel@tonic-gate	stx	%i3, [%sp + STACK_BIAS + SA(MINFRAME) + O3_OFF]
10887c478bdstevel@tonic-gate	stx	%i4, [%sp + STACK_BIAS + SA(MINFRAME) + O4_OFF]
10897c478bdstevel@tonic-gate	stx	%i5, [%sp + STACK_BIAS + SA(MINFRAME) + O5_OFF]
10907c478bdstevel@tonic-gate	stx	%i6, [%sp + STACK_BIAS + SA(MINFRAME) + O6_OFF]
10917c478bdstevel@tonic-gate	stx	%i7, [%sp + STACK_BIAS + SA(MINFRAME) + O7_OFF]
10927c478bdstevel@tonic-gate	stn	%l1, [%sp + STACK_BIAS + SA(MINFRAME) + PC_OFF]
10937c478bdstevel@tonic-gate	stn	%l2, [%sp + STACK_BIAS + SA(MINFRAME) + NPC_OFF]
10947c478bdstevel@tonic-gate	st	%l3, [%sp + STACK_BIAS + SA(MINFRAME) + Y_OFF]
10957c478bdstevel@tonic-gate
10967c478bdstevel@tonic-gate	mov	%l4, %o3				! %o3 = on_panic_stack
10977c478bdstevel@tonic-gate	add	%sp, STACK_BIAS + SA(MINFRAME), %o2	! %o2 = &regs
10987c478bdstevel@tonic-gate	mov	%i1, %o1				! %o1 = alist
10997c478bdstevel@tonic-gate	call	panicsys				! panicsys();
11007c478bdstevel@tonic-gate	mov	%i0, %o0				! %o0 = format
11017c478bdstevel@tonic-gate	ret
11027c478bdstevel@tonic-gate	restore
11037c478bdstevel@tonic-gate
11047c478bdstevel@tonic-gate	SET_SIZE(vpanic)
11057c478bdstevel@tonic-gate
11067c478bdstevel@tonic-gate	ENTRY_NP(dtrace_vpanic)
11077c478bdstevel@tonic-gate
11087c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME + REGSIZE), %sp	! save and allocate regs
11097c478bdstevel@tonic-gate
11107c478bdstevel@tonic-gate	!
11117c478bdstevel@tonic-gate	! The v9 struct regs has a 64-bit r_tstate field, which we use here
11127c478bdstevel@tonic-gate	! to store the %ccr, %asi, %pstate, and %cwp as they would appear
11137c478bdstevel@tonic-gate	! in %tstate if a trap occurred.  We leave it up to the debugger to
11147c478bdstevel@tonic-gate	! realize what happened and extract the register values.
11157c478bdstevel@tonic-gate	!
11167c478bdstevel@tonic-gate	rd	%ccr, %l0				! %l0 = %ccr
11177c478bdstevel@tonic-gate	sllx	%l0, TSTATE_CCR_SHIFT, %l0		! %l0 <<= CCR_SHIFT
11187c478bdstevel@tonic-gate	rd	%asi, %l1				! %l1 = %asi
11197c478bdstevel@tonic-gate	sllx	%l1, TSTATE_ASI_SHIFT, %l1		! %l1 <<= ASI_SHIFT
11207c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
11217c478bdstevel@tonic-gate	rdpr	%pstate, %l1				! %l1 = %pstate
11227c478bdstevel@tonic-gate	sllx	%l1, TSTATE_PSTATE_SHIFT, %l1		! %l1 <<= PSTATE_SHIFT
11237c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
11247c478bdstevel@tonic-gate	rdpr	%cwp, %l1				! %l1 = %cwp
11257c478bdstevel@tonic-gate	sllx	%l1, TSTATE_CWP_SHIFT, %l1		! %l1 <<= CWP_SHIFT
11267c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
11277c478bdstevel@tonic-gate
11287c478bdstevel@tonic-gate	set	dtrace_vpanic, %l1			! %l1 = %pc (vpanic)
11297c478bdstevel@tonic-gate	add	%l1, 4, %l2				! %l2 = %npc (vpanic+4)
11307c478bdstevel@tonic-gate	rd	%y, %l3					! %l3 = %y
11317c478bdstevel@tonic-gate	!
11327c478bdstevel@tonic-gate	! Flush register windows before panic_trigger() in order to avoid a
11337c478bdstevel@tonic-gate	! problem that a dump hangs if flush_windows() causes another panic.
11347c478bdstevel@tonic-gate	!
11357c478bdstevel@tonic-gate	call	dtrace_flush_windows
11367c478bdstevel@tonic-gate	nop
11377c478bdstevel@tonic-gate
11387c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %o0
11397c478bdstevel@tonic-gate	call	dtrace_panic_trigger
11407c478bdstevel@tonic-gate	or	%o0, %lo(panic_quiesce), %o0		! if (!panic_trigger(
11417c478bdstevel@tonic-gate
11427c478bdstevel@tonic-gate	ba,a	vpanic_common
11437c478bdstevel@tonic-gate	SET_SIZE(dtrace_vpanic)
11447c478bdstevel@tonic-gate
1145bfb7f38kalai	ENTRY(get_subcc_ccr)
1146bfb7f38kalai	wr	%g0, %ccr	! clear condition codes
1147bfb7f38kalai	subcc	%o0, %o1, %g0
1148bfb7f38kalai	retl
1149bfb7f38kalai	rd	%ccr, %o0	! return condition codes
1150bfb7f38kalai	SET_SIZE(get_subcc_ccr)
1151bfb7f38kalai
11524df4bd6bs	ENTRY_NP(ftrace_interrupt_disable)
11534df4bd6bs	rdpr	%pstate, %o0
11544df4bd6bs	andn	%o0, PSTATE_IE, %o1
11554df4bd6bs	retl
11564df4bd6bs	wrpr	%g0, %o1, %pstate
11574df4bd6bs	SET_SIZE(ftrace_interrupt_disable)
11584df4bd6bs
11594df4bd6bs	ENTRY_NP(ftrace_interrupt_enable)
11604df4bd6bs	retl
11614df4bd6bs	wrpr	%g0, %o0, %pstate
11624df4bd6bs	SET_SIZE(ftrace_interrupt_enable)
11634df4bd6bs
1164