17c478bdstevel@tonic-gate/*
27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
54df4bd6bs * Common Development and Distribution License (the "License").
64df4bd6bs * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
217c478bdstevel@tonic-gate/*
22023e71dHaik Aftandilian * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bdstevel@tonic-gate * Use is subject to license terms.
247c478bdstevel@tonic-gate */
257c478bdstevel@tonic-gate
267c478bdstevel@tonic-gate/*
277c478bdstevel@tonic-gate * General assembly language routines.
287c478bdstevel@tonic-gate * It is the intent of this file to contain routines that are
297c478bdstevel@tonic-gate * independent of the specific kernel architecture, and those that are
307c478bdstevel@tonic-gate * common across kernel architectures.
317c478bdstevel@tonic-gate * As architectures diverge, and implementations of specific
327c478bdstevel@tonic-gate * architecture-dependent routines change, the routines should be moved
337c478bdstevel@tonic-gate * from this file into the respective ../`arch -k`/subr.s file.
347c478bdstevel@tonic-gate * Or, if you want to be really nice, move them to a file whose
357c478bdstevel@tonic-gate * name has something to do with the routine you are moving.
367c478bdstevel@tonic-gate */
377c478bdstevel@tonic-gate
387c478bdstevel@tonic-gate#if defined(lint)
397c478bdstevel@tonic-gate#include <sys/types.h>
407c478bdstevel@tonic-gate#include <sys/scb.h>
417c478bdstevel@tonic-gate#include <sys/systm.h>
427c478bdstevel@tonic-gate#include <sys/regset.h>
437c478bdstevel@tonic-gate#include <sys/sunddi.h>
447c478bdstevel@tonic-gate#include <sys/lockstat.h>
457c478bdstevel@tonic-gate#include <sys/dtrace.h>
464df4bd6bs#include <sys/ftrace.h>
477c478bdstevel@tonic-gate#endif	/* lint */
487c478bdstevel@tonic-gate
497c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
507c478bdstevel@tonic-gate#include <sys/privregs.h>
517c478bdstevel@tonic-gate#include <sys/machparam.h>	/* To get SYSBASE and PAGESIZE */
527c478bdstevel@tonic-gate#include <sys/machthread.h>
537c478bdstevel@tonic-gate#include <sys/clock.h>
547c478bdstevel@tonic-gate#include <sys/psr_compat.h>
557c478bdstevel@tonic-gate#include <sys/isa_defs.h>
567c478bdstevel@tonic-gate#include <sys/dditypes.h>
577c478bdstevel@tonic-gate#include <sys/panic.h>
587c478bdstevel@tonic-gate#include <sys/machlock.h>
597c478bdstevel@tonic-gate#include <sys/ontrap.h>
607c478bdstevel@tonic-gate
617c478bdstevel@tonic-gate#if !defined(lint)
627c478bdstevel@tonic-gate#include "assym.h"
637c478bdstevel@tonic-gate
647c478bdstevel@tonic-gate	.seg	".text"
657c478bdstevel@tonic-gate	.align	4
667c478bdstevel@tonic-gate
677c478bdstevel@tonic-gate/*
687c478bdstevel@tonic-gate * Macro to raise processor priority level.
697c478bdstevel@tonic-gate * Avoid dropping processor priority if already at high level.
707c478bdstevel@tonic-gate * Also avoid going below CPU->cpu_base_spl, which could've just been set by
717c478bdstevel@tonic-gate * a higher-level interrupt thread that just blocked.
727c478bdstevel@tonic-gate *
737c478bdstevel@tonic-gate * level can be %o0 (not other regs used here) or a constant.
747c478bdstevel@tonic-gate */
757c478bdstevel@tonic-gate#define	RAISE(level) \
767c478bdstevel@tonic-gate	rdpr	%pil, %o1;		/* get current PIL */		\
777c478bdstevel@tonic-gate	cmp	%o1, level;		/* is PIL high enough? */	\
787c478bdstevel@tonic-gate	bge	1f;			/* yes, return */		\
797c478bdstevel@tonic-gate	nop;								\
807c478bdstevel@tonic-gate	wrpr	%g0, PIL_MAX, %pil;	/* freeze CPU_BASE_SPL */	\
817c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o2;				\
827c478bdstevel@tonic-gate	ld	[%o2 + CPU_BASE_SPL], %o2;				\
837c478bdstevel@tonic-gate	cmp	%o2, level;		/* compare new to base */	\
847c478bdstevel@tonic-gate	movl	%xcc, level, %o2;	/* use new if base lower */	\
857c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pil;						\
867c478bdstevel@tonic-gate1:									\
877c478bdstevel@tonic-gate	retl;								\
887c478bdstevel@tonic-gate	mov	%o1, %o0		/* return old PIL */
897c478bdstevel@tonic-gate
907c478bdstevel@tonic-gate/*
917c478bdstevel@tonic-gate * Macro to raise processor priority level to level >= DISP_LEVEL.
927c478bdstevel@tonic-gate * Doesn't require comparison to CPU->cpu_base_spl.
937c478bdstevel@tonic-gate *
947c478bdstevel@tonic-gate * newpil can be %o0 (not other regs used here) or a constant.
957c478bdstevel@tonic-gate */
967c478bdstevel@tonic-gate#define	RAISE_HIGH(level) \
977c478bdstevel@tonic-gate	rdpr	%pil, %o1;		/* get current PIL */		\
987c478bdstevel@tonic-gate	cmp	%o1, level;		/* is PIL high enough? */	\
997c478bdstevel@tonic-gate	bge	1f;			/* yes, return */		\
1007c478bdstevel@tonic-gate	nop;								\
1017c478bdstevel@tonic-gate	wrpr	%g0, level, %pil;	/* use chose value */		\
1027c478bdstevel@tonic-gate1:									\
1037c478bdstevel@tonic-gate	retl;								\
1047c478bdstevel@tonic-gate	mov	%o1, %o0		/* return old PIL */
1057c478bdstevel@tonic-gate
1067c478bdstevel@tonic-gate/*
1077c478bdstevel@tonic-gate * Macro to set the priority to a specified level.
1087c478bdstevel@tonic-gate * Avoid dropping the priority below CPU->cpu_base_spl.
1097c478bdstevel@tonic-gate *
1107c478bdstevel@tonic-gate * newpil can be %o0 (not other regs used here) or a constant with
1117c478bdstevel@tonic-gate * the new PIL in the PSR_PIL field of the level arg.
1127c478bdstevel@tonic-gate */
1137c478bdstevel@tonic-gate#define SETPRI(level) \
1147c478bdstevel@tonic-gate	rdpr	%pil, %o1;		/* get current PIL */		\
1157c478bdstevel@tonic-gate	wrpr	%g0, PIL_MAX, %pil;	/* freeze CPU_BASE_SPL */	\
1167c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_CPU], %o2;				\
1177c478bdstevel@tonic-gate	ld	[%o2 + CPU_BASE_SPL], %o2;				\
1187c478bdstevel@tonic-gate	cmp	%o2, level;		/* compare new to base */	\
1197c478bdstevel@tonic-gate	movl	%xcc, level, %o2;	/* use new if base lower */	\
1207c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pil;						\
1217c478bdstevel@tonic-gate	retl;								\
1227c478bdstevel@tonic-gate	mov	%o1, %o0		/* return old PIL */
1237c478bdstevel@tonic-gate
1247c478bdstevel@tonic-gate/*
1257c478bdstevel@tonic-gate * Macro to set the priority to a specified level at or above LOCK_LEVEL.
1267c478bdstevel@tonic-gate * Doesn't require comparison to CPU->cpu_base_spl.
1277c478bdstevel@tonic-gate *
1287c478bdstevel@tonic-gate * newpil can be %o0 (not other regs used here) or a constant with
1297c478bdstevel@tonic-gate * the new PIL in the PSR_PIL field of the level arg.
1307c478bdstevel@tonic-gate */
1317c478bdstevel@tonic-gate#define	SETPRI_HIGH(level) \
1327c478bdstevel@tonic-gate	rdpr	%pil, %o1;		/* get current PIL */		\
1337c478bdstevel@tonic-gate	wrpr	%g0, level, %pil;					\
1347c478bdstevel@tonic-gate	retl;								\
1357c478bdstevel@tonic-gate	mov	%o1, %o0		/* return old PIL */
1367c478bdstevel@tonic-gate
1377c478bdstevel@tonic-gate#endif	/* lint */
1387c478bdstevel@tonic-gate
1397c478bdstevel@tonic-gate	/*
1407c478bdstevel@tonic-gate	 * Berkley 4.3 introduced symbolically named interrupt levels
1417c478bdstevel@tonic-gate	 * as a way deal with priority in a machine independent fashion.
1427c478bdstevel@tonic-gate	 * Numbered priorities are machine specific, and should be
1437c478bdstevel@tonic-gate	 * discouraged where possible.
1447c478bdstevel@tonic-gate	 *
1457c478bdstevel@tonic-gate	 * Note, for the machine specific priorities there are
1467c478bdstevel@tonic-gate	 * examples listed for devices that use a particular priority.
1477c478bdstevel@tonic-gate	 * It should not be construed that all devices of that
1487c478bdstevel@tonic-gate	 * type should be at that priority.  It is currently were
1497c478bdstevel@tonic-gate	 * the current devices fit into the priority scheme based
1507c478bdstevel@tonic-gate	 * upon time criticalness.
1517c478bdstevel@tonic-gate	 *
1527c478bdstevel@tonic-gate	 * The underlying assumption of these assignments is that
1537c478bdstevel@tonic-gate	 * SPARC9 IPL 10 is the highest level from which a device
1547c478bdstevel@tonic-gate	 * routine can call wakeup.  Devices that interrupt from higher
1557c478bdstevel@tonic-gate	 * levels are restricted in what they can do.  If they need
1567c478bdstevel@tonic-gate	 * kernels services they should schedule a routine at a lower
1577c478bdstevel@tonic-gate	 * level (via software interrupt) to do the required
1587c478bdstevel@tonic-gate	 * processing.
1597c478bdstevel@tonic-gate	 *
1607c478bdstevel@tonic-gate	 * Examples of this higher usage:
1617c478bdstevel@tonic-gate	 *	Level	Usage
1627c478bdstevel@tonic-gate	 *	15	Asynchronous memory exceptions
1637c478bdstevel@tonic-gate	 *	14	Profiling clock (and PROM uart polling clock)
1647c478bdstevel@tonic-gate	 *	13	Audio device
1657c478bdstevel@tonic-gate	 *	12	Serial ports
1667c478bdstevel@tonic-gate	 *	11	Floppy controller
1677c478bdstevel@tonic-gate	 *
1687c478bdstevel@tonic-gate	 * The serial ports request lower level processing on level 6.
1697c478bdstevel@tonic-gate	 * Audio and floppy request lower level processing on level 4.
1707c478bdstevel@tonic-gate	 *
1717c478bdstevel@tonic-gate	 * Also, almost all splN routines (where N is a number or a
1727c478bdstevel@tonic-gate	 * mnemonic) will do a RAISE(), on the assumption that they are
1737c478bdstevel@tonic-gate	 * never used to lower our priority.
1747c478bdstevel@tonic-gate	 * The exceptions are:
1757c478bdstevel@tonic-gate	 *	spl8()		Because you can't be above 15 to begin with!
1767c478bdstevel@tonic-gate	 *	splzs()		Because this is used at boot time to lower our
1777c478bdstevel@tonic-gate	 *			priority, to allow the PROM to poll the uart.
1787c478bdstevel@tonic-gate	 *	spl0()		Used to lower priority to 0.
1797c478bdstevel@tonic-gate	 */
1807c478bdstevel@tonic-gate
1817c478bdstevel@tonic-gate#if defined(lint)
1827c478bdstevel@tonic-gate
1837c478bdstevel@tonic-gateint spl0(void)		{ return (0); }
1847c478bdstevel@tonic-gateint spl6(void)		{ return (0); }
1857c478bdstevel@tonic-gateint spl7(void)		{ return (0); }
1867c478bdstevel@tonic-gateint spl8(void)		{ return (0); }
1877c478bdstevel@tonic-gateint splhi(void)		{ return (0); }
1887c478bdstevel@tonic-gateint splhigh(void)	{ return (0); }
1897c478bdstevel@tonic-gateint splzs(void)		{ return (0); }
1907c478bdstevel@tonic-gate
1917c478bdstevel@tonic-gate#else	/* lint */
1927c478bdstevel@tonic-gate
1937c478bdstevel@tonic-gate	/* locks out all interrupts, including memory errors */
1947c478bdstevel@tonic-gate	ENTRY(spl8)
1957c478bdstevel@tonic-gate	SETPRI_HIGH(15)
1967c478bdstevel@tonic-gate	SET_SIZE(spl8)
1977c478bdstevel@tonic-gate
1987c478bdstevel@tonic-gate	/* just below the level that profiling runs */
1997c478bdstevel@tonic-gate	ENTRY(spl7)
2007c478bdstevel@tonic-gate	RAISE_HIGH(13)
2017c478bdstevel@tonic-gate	SET_SIZE(spl7)
2027c478bdstevel@tonic-gate
2037c478bdstevel@tonic-gate	/* sun specific - highest priority onboard serial i/o zs ports */
2047c478bdstevel@tonic-gate	ENTRY(splzs)
2057c478bdstevel@tonic-gate	SETPRI_HIGH(12)	/* Can't be a RAISE, as it's used to lower us */
2067c478bdstevel@tonic-gate	SET_SIZE(splzs)
2077c478bdstevel@tonic-gate
2087c478bdstevel@tonic-gate	/*
2097c478bdstevel@tonic-gate	 * should lock out clocks and all interrupts,
2107c478bdstevel@tonic-gate	 * as you can see, there are exceptions
2117c478bdstevel@tonic-gate	 */
2127c478bdstevel@tonic-gate	ENTRY(splhi)
2137c478bdstevel@tonic-gate	ALTENTRY(splhigh)
2147c478bdstevel@tonic-gate	ALTENTRY(spl6)
2157c478bdstevel@tonic-gate	ALTENTRY(i_ddi_splhigh)
2167c478bdstevel@tonic-gate	RAISE_HIGH(DISP_LEVEL)
2177c478bdstevel@tonic-gate	SET_SIZE(i_ddi_splhigh)
2187c478bdstevel@tonic-gate	SET_SIZE(spl6)
2197c478bdstevel@tonic-gate	SET_SIZE(splhigh)
2207c478bdstevel@tonic-gate	SET_SIZE(splhi)
2217c478bdstevel@tonic-gate
2227c478bdstevel@tonic-gate	/* allow all interrupts */
2237c478bdstevel@tonic-gate	ENTRY(spl0)
2247c478bdstevel@tonic-gate	SETPRI(0)
2257c478bdstevel@tonic-gate	SET_SIZE(spl0)
2267c478bdstevel@tonic-gate
2277c478bdstevel@tonic-gate#endif	/* lint */
2287c478bdstevel@tonic-gate
2297c478bdstevel@tonic-gate/*
2307c478bdstevel@tonic-gate * splx - set PIL back to that indicated by the old %pil passed as an argument,
2317c478bdstevel@tonic-gate * or to the CPU's base priority, whichever is higher.
2327c478bdstevel@tonic-gate */
2337c478bdstevel@tonic-gate
2347c478bdstevel@tonic-gate#if defined(lint)
2357c478bdstevel@tonic-gate
2367c478bdstevel@tonic-gate/* ARGSUSED */
2377c478bdstevel@tonic-gatevoid
2387c478bdstevel@tonic-gatesplx(int level)
2397c478bdstevel@tonic-gate{}
2407c478bdstevel@tonic-gate
2417c478bdstevel@tonic-gate#else	/* lint */
2427c478bdstevel@tonic-gate
2437c478bdstevel@tonic-gate	ENTRY(splx)
2447c478bdstevel@tonic-gate	ALTENTRY(i_ddi_splx)
2457c478bdstevel@tonic-gate	SETPRI(%o0)		/* set PIL */
2467c478bdstevel@tonic-gate	SET_SIZE(i_ddi_splx)
2477c478bdstevel@tonic-gate	SET_SIZE(splx)
2487c478bdstevel@tonic-gate
2497c478bdstevel@tonic-gate#endif	/* level */
2507c478bdstevel@tonic-gate
2517c478bdstevel@tonic-gate/*
2527c478bdstevel@tonic-gate * splr()
2537c478bdstevel@tonic-gate *
2547c478bdstevel@tonic-gate * splr is like splx but will only raise the priority and never drop it
2557c478bdstevel@tonic-gate * Be careful not to set priority lower than CPU->cpu_base_pri,
2567c478bdstevel@tonic-gate * even though it seems we're raising the priority, it could be set higher
2577c478bdstevel@tonic-gate * at any time by an interrupt routine, so we must block interrupts and
2587c478bdstevel@tonic-gate * look at CPU->cpu_base_pri.
2597c478bdstevel@tonic-gate */
2607c478bdstevel@tonic-gate
2617c478bdstevel@tonic-gate#if defined(lint)
2627c478bdstevel@tonic-gate
2637c478bdstevel@tonic-gate/* ARGSUSED */
2647c478bdstevel@tonic-gateint
2657c478bdstevel@tonic-gatesplr(int level)
2667c478bdstevel@tonic-gate{ return (0); }
2677c478bdstevel@tonic-gate
2687c478bdstevel@tonic-gate#else	/* lint */
2697c478bdstevel@tonic-gate	ENTRY(splr)
2707c478bdstevel@tonic-gate	RAISE(%o0)
2717c478bdstevel@tonic-gate	SET_SIZE(splr)
2727c478bdstevel@tonic-gate
2737c478bdstevel@tonic-gate#endif	/* lint */
2747c478bdstevel@tonic-gate
2757c478bdstevel@tonic-gate/*
2767c478bdstevel@tonic-gate * on_fault()
2777c478bdstevel@tonic-gate * Catch lofault faults. Like setjmp except it returns one
2787c478bdstevel@tonic-gate * if code following causes uncorrectable fault. Turned off
2797c478bdstevel@tonic-gate * by calling no_fault().
2807c478bdstevel@tonic-gate */
2817c478bdstevel@tonic-gate
2827c478bdstevel@tonic-gate#if defined(lint)
2837c478bdstevel@tonic-gate
2847c478bdstevel@tonic-gate/* ARGSUSED */
2857c478bdstevel@tonic-gateint
2867c478bdstevel@tonic-gateon_fault(label_t *ljb)
2877c478bdstevel@tonic-gate{ return (0); }
2887c478bdstevel@tonic-gate
2897c478bdstevel@tonic-gate#else	/* lint */
2907c478bdstevel@tonic-gate
2917c478bdstevel@tonic-gate	ENTRY(on_fault)
2927c478bdstevel@tonic-gate	membar	#Sync			! sync error barrier (see copy.s)
2937c478bdstevel@tonic-gate	stn	%o0, [THREAD_REG + T_ONFAULT]
2947c478bdstevel@tonic-gate	set	catch_fault, %o1
2957c478bdstevel@tonic-gate	b	setjmp			! let setjmp do the rest
2967c478bdstevel@tonic-gate	stn	%o1, [THREAD_REG + T_LOFAULT]	! put catch_fault in t_lofault
2977c478bdstevel@tonic-gate
2987c478bdstevel@tonic-gatecatch_fault:
2997c478bdstevel@tonic-gate	save	%sp, -SA(WINDOWSIZE), %sp ! goto next window so that we can rtn
3007c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_ONFAULT], %o0
3017c478bdstevel@tonic-gate	membar	#Sync				! sync error barrier
3027c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_ONFAULT]	! turn off onfault
3037c478bdstevel@tonic-gate	b	longjmp			! let longjmp do the rest
3047c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_LOFAULT]	! turn off lofault
3057c478bdstevel@tonic-gate	SET_SIZE(on_fault)
3067c478bdstevel@tonic-gate
3077c478bdstevel@tonic-gate#endif	/* lint */
3087c478bdstevel@tonic-gate
3097c478bdstevel@tonic-gate/*
3107c478bdstevel@tonic-gate * no_fault()
3117c478bdstevel@tonic-gate * turn off fault catching.
3127c478bdstevel@tonic-gate */
3137c478bdstevel@tonic-gate
3147c478bdstevel@tonic-gate#if defined(lint)
3157c478bdstevel@tonic-gate
3167c478bdstevel@tonic-gatevoid
3177c478bdstevel@tonic-gateno_fault(void)
3187c478bdstevel@tonic-gate{}
3197c478bdstevel@tonic-gate
3207c478bdstevel@tonic-gate#else	/* lint */
3217c478bdstevel@tonic-gate
3227c478bdstevel@tonic-gate	ENTRY(no_fault)
3237c478bdstevel@tonic-gate	membar	#Sync				! sync error barrier
3247c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_ONFAULT]
3257c478bdstevel@tonic-gate	retl
3267c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_LOFAULT]	! turn off lofault
3277c478bdstevel@tonic-gate	SET_SIZE(no_fault)
3287c478bdstevel@tonic-gate
3297c478bdstevel@tonic-gate#endif	/* lint */
3307c478bdstevel@tonic-gate
3317c478bdstevel@tonic-gate/*
3327c478bdstevel@tonic-gate * Default trampoline code for on_trap() (see <sys/ontrap.h>).  On sparcv9,
3337c478bdstevel@tonic-gate * the trap code will complete trap processing but reset the return %pc to
3347c478bdstevel@tonic-gate * ot_trampoline, which will by default be set to the address of this code.
3357c478bdstevel@tonic-gate * We longjmp(&curthread->t_ontrap->ot_jmpbuf) to return back to on_trap().
3367c478bdstevel@tonic-gate */
3377c478bdstevel@tonic-gate#if defined(lint)
3387c478bdstevel@tonic-gate
3397c478bdstevel@tonic-gatevoid
3407c478bdstevel@tonic-gateon_trap_trampoline(void)
3417c478bdstevel@tonic-gate{}
3427c478bdstevel@tonic-gate
3437c478bdstevel@tonic-gate#else	/* lint */
3447c478bdstevel@tonic-gate
3457c478bdstevel@tonic-gate	ENTRY(on_trap_trampoline)
3467c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_ONTRAP], %o0
3477c478bdstevel@tonic-gate	b	longjmp
3487c478bdstevel@tonic-gate	add	%o0, OT_JMPBUF, %o0
3497c478bdstevel@tonic-gate	SET_SIZE(on_trap_trampoline)
3507c478bdstevel@tonic-gate
3517c478bdstevel@tonic-gate#endif	/* lint */
3527c478bdstevel@tonic-gate
3537c478bdstevel@tonic-gate/*
3547c478bdstevel@tonic-gate * Push a new element on to the t_ontrap stack.  Refer to <sys/ontrap.h> for
3557c478bdstevel@tonic-gate * more information about the on_trap() mechanism.  If the on_trap_data is the
3567c478bdstevel@tonic-gate * same as the topmost stack element, we just modify that element.
3577c478bdstevel@tonic-gate * On UltraSPARC, we need to issue a membar #Sync before modifying t_ontrap.
3587c478bdstevel@tonic-gate * The issue barrier is defined to force all deferred errors to complete before
3597c478bdstevel@tonic-gate * we go any further.  We want these errors to be processed before we modify
3607c478bdstevel@tonic-gate * our current error protection.
3617c478bdstevel@tonic-gate */
3627c478bdstevel@tonic-gate#if defined(lint)
3637c478bdstevel@tonic-gate
3647c478bdstevel@tonic-gate/*ARGSUSED*/
3657c478bdstevel@tonic-gateint
3667c478bdstevel@tonic-gateon_trap(on_trap_data_t *otp, uint_t prot)
3677c478bdstevel@tonic-gate{ return (0); }
3687c478bdstevel@tonic-gate
3697c478bdstevel@tonic-gate#else	/* lint */
3707c478bdstevel@tonic-gate
3717c478bdstevel@tonic-gate	ENTRY(on_trap)
3727c478bdstevel@tonic-gate	membar	#Sync				! force error barrier
3737c478bdstevel@tonic-gate	sth	%o1, [%o0 + OT_PROT]		! ot_prot = prot
3747c478bdstevel@tonic-gate	sth	%g0, [%o0 + OT_TRAP]		! ot_trap = 0
3757c478bdstevel@tonic-gate	set	on_trap_trampoline, %o2		! %o2 = &on_trap_trampoline
3767c478bdstevel@tonic-gate	stn	%o2, [%o0 + OT_TRAMPOLINE]	! ot_trampoline = %o2
3777c478bdstevel@tonic-gate	stn	%g0, [%o0 + OT_HANDLE]		! ot_handle = NULL
3787c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_ONTRAP], %o2	! %o2 = curthread->t_ontrap
3797c478bdstevel@tonic-gate	cmp	%o0, %o2			! if (otp == %o2)
3807c478bdstevel@tonic-gate	be	0f				!    don't modify t_ontrap
3817c478bdstevel@tonic-gate	stn	%g0, [%o0 + OT_PAD1]		! delay - ot_pad1 = NULL
3827c478bdstevel@tonic-gate
3837c478bdstevel@tonic-gate	stn	%o2, [%o0 + OT_PREV]		! ot_prev = t_ontrap
3847c478bdstevel@tonic-gate	membar	#Sync				! force error barrier
3857c478bdstevel@tonic-gate	stn	%o0, [THREAD_REG + T_ONTRAP]	! t_ontrap = otp
3867c478bdstevel@tonic-gate
3877c478bdstevel@tonic-gate0:	b	setjmp				! let setjmp do the rest
3887c478bdstevel@tonic-gate	add	%o0, OT_JMPBUF, %o0		! %o0 = &ot_jmpbuf
3897c478bdstevel@tonic-gate	SET_SIZE(on_trap)
3907c478bdstevel@tonic-gate
3917c478bdstevel@tonic-gate#endif	/* lint */
3927c478bdstevel@tonic-gate
3937c478bdstevel@tonic-gate/*
3947c478bdstevel@tonic-gate * Setjmp and longjmp implement non-local gotos using state vectors
3957c478bdstevel@tonic-gate * type label_t.
3967c478bdstevel@tonic-gate */
3977c478bdstevel@tonic-gate
3987c478bdstevel@tonic-gate#if defined(lint)
3997c478bdstevel@tonic-gate
4007c478bdstevel@tonic-gate/* ARGSUSED */
4017c478bdstevel@tonic-gateint
4027c478bdstevel@tonic-gatesetjmp(label_t *lp)
4037c478bdstevel@tonic-gate{ return (0); }
4047c478bdstevel@tonic-gate
4057c478bdstevel@tonic-gate#else	/* lint */
4067c478bdstevel@tonic-gate
4077c478bdstevel@tonic-gate	ENTRY(setjmp)
4087c478bdstevel@tonic-gate	stn	%o7, [%o0 + L_PC]	! save return address
4097c478bdstevel@tonic-gate	stn	%sp, [%o0 + L_SP]	! save stack ptr
4107c478bdstevel@tonic-gate	retl
4117c478bdstevel@tonic-gate	clr	%o0			! return 0
4127c478bdstevel@tonic-gate	SET_SIZE(setjmp)
4137c478bdstevel@tonic-gate
4147c478bdstevel@tonic-gate#endif	/* lint */
4157c478bdstevel@tonic-gate
4167c478bdstevel@tonic-gate
4177c478bdstevel@tonic-gate#if defined(lint)
4187c478bdstevel@tonic-gate
4197c478bdstevel@tonic-gate/* ARGSUSED */
4207c478bdstevel@tonic-gatevoid
4217c478bdstevel@tonic-gatelongjmp(label_t *lp)
4227c478bdstevel@tonic-gate{}
4237c478bdstevel@tonic-gate
4247c478bdstevel@tonic-gate#else	/* lint */
4257c478bdstevel@tonic-gate
4267c478bdstevel@tonic-gate	ENTRY(longjmp)
4277c478bdstevel@tonic-gate	!
4287c478bdstevel@tonic-gate        ! The following save is required so that an extra register
4297c478bdstevel@tonic-gate        ! window is flushed.  Flushw flushes nwindows-2
4307c478bdstevel@tonic-gate        ! register windows.  If setjmp and longjmp are called from
4317c478bdstevel@tonic-gate        ! within the same window, that window will not get pushed
4327c478bdstevel@tonic-gate        ! out onto the stack without the extra save below.  Tail call
4337c478bdstevel@tonic-gate        ! optimization can lead to callers of longjmp executing
4347c478bdstevel@tonic-gate        ! from a window that could be the same as the setjmp,
4357c478bdstevel@tonic-gate        ! thus the need for the following save.
4367c478bdstevel@tonic-gate        !
4377c478bdstevel@tonic-gate	save    %sp, -SA(MINFRAME), %sp
4387c478bdstevel@tonic-gate	flushw				! flush all but this window
4397c478bdstevel@tonic-gate	ldn	[%i0 + L_PC], %i7	! restore return addr
4407c478bdstevel@tonic-gate	ldn	[%i0 + L_SP], %fp	! restore sp for dest on foreign stack
4417c478bdstevel@tonic-gate	ret				! return 1
4427c478bdstevel@tonic-gate	restore	%g0, 1, %o0		! takes underflow, switches stacks
4437c478bdstevel@tonic-gate	SET_SIZE(longjmp)
4447c478bdstevel@tonic-gate
4457c478bdstevel@tonic-gate#endif	/* lint */
4467c478bdstevel@tonic-gate
4477c478bdstevel@tonic-gate/*
4487c478bdstevel@tonic-gate * movtuc(length, from, to, table)
4497c478bdstevel@tonic-gate *
4507c478bdstevel@tonic-gate * VAX movtuc instruction (sort of).
4517c478bdstevel@tonic-gate */
4527c478bdstevel@tonic-gate
4537c478bdstevel@tonic-gate#if defined(lint)
4547c478bdstevel@tonic-gate
4557c478bdstevel@tonic-gate/*ARGSUSED*/
4567c478bdstevel@tonic-gateint
4577c478bdstevel@tonic-gatemovtuc(size_t length, u_char *from, u_char *to, u_char table[])
4587c478bdstevel@tonic-gate{ return (0); }
4597c478bdstevel@tonic-gate
4607c478bdstevel@tonic-gate#else	/* lint */
4617c478bdstevel@tonic-gate
4627c478bdstevel@tonic-gate	ENTRY(movtuc)
4637c478bdstevel@tonic-gate	tst     %o0
4647c478bdstevel@tonic-gate	ble,pn	%ncc, 2f		! check length
4657c478bdstevel@tonic-gate	clr     %o4
4667c478bdstevel@tonic-gate
4677c478bdstevel@tonic-gate	ldub    [%o1 + %o4], %g1        ! get next byte in string
4687c478bdstevel@tonic-gate0:
4697c478bdstevel@tonic-gate	ldub    [%o3 + %g1], %g1        ! get corresponding table entry
4707c478bdstevel@tonic-gate	tst     %g1                     ! escape char?
4717c478bdstevel@tonic-gate	bnz     1f
4727c478bdstevel@tonic-gate	stb     %g1, [%o2 + %o4]        ! delay slot, store it
4737c478bdstevel@tonic-gate
4747c478bdstevel@tonic-gate	retl                            ! return (bytes moved)
4757c478bdstevel@tonic-gate	mov     %o4, %o0
4767c478bdstevel@tonic-gate1:
4777c478bdstevel@tonic-gate	inc     %o4                     ! increment index
4787c478bdstevel@tonic-gate	cmp     %o4, %o0                ! index < length ?
4797c478bdstevel@tonic-gate	bl,a,pt	%ncc, 0b
4807c478bdstevel@tonic-gate	ldub    [%o1 + %o4], %g1        ! delay slot, get next byte in string
4817c478bdstevel@tonic-gate2:
4827c478bdstevel@tonic-gate	retl                            ! return (bytes moved)
4837c478bdstevel@tonic-gate	mov     %o4, %o0
4847c478bdstevel@tonic-gate	SET_SIZE(movtuc)
4857c478bdstevel@tonic-gate
4867c478bdstevel@tonic-gate#endif	/* lint */
4877c478bdstevel@tonic-gate
4887c478bdstevel@tonic-gate/*
4897c478bdstevel@tonic-gate * scanc(length, string, table, mask)
4907c478bdstevel@tonic-gate *
4917c478bdstevel@tonic-gate * VAX scanc instruction.
4927c478bdstevel@tonic-gate */
4937c478bdstevel@tonic-gate
4947c478bdstevel@tonic-gate#if defined(lint)
4957c478bdstevel@tonic-gate
4967c478bdstevel@tonic-gate/*ARGSUSED*/
4977c478bdstevel@tonic-gateint
4987c478bdstevel@tonic-gatescanc(size_t length, u_char *string, u_char table[], u_char mask)
4997c478bdstevel@tonic-gate{ return (0); }
5007c478bdstevel@tonic-gate
5017c478bdstevel@tonic-gate#else	/* lint */
5027c478bdstevel@tonic-gate
5037c478bdstevel@tonic-gate	ENTRY(scanc)
5047c478bdstevel@tonic-gate	tst	%o0
5057c478bdstevel@tonic-gate	ble,pn	%ncc, 1f		! check length
5067c478bdstevel@tonic-gate	clr	%o4
5077c478bdstevel@tonic-gate0:
5087c478bdstevel@tonic-gate	ldub	[%o1 + %o4], %g1	! get next byte in string
5097c478bdstevel@tonic-gate	cmp	%o4, %o0		! interlock slot, index < length ?
5107c478bdstevel@tonic-gate	ldub	[%o2 + %g1], %g1	! get corresponding table entry
5117c478bdstevel@tonic-gate	bge,pn	%ncc, 1f		! interlock slot
5127c478bdstevel@tonic-gate	btst	%o3, %g1		! apply the mask
5137c478bdstevel@tonic-gate	bz,a	0b
5147c478bdstevel@tonic-gate	inc	%o4			! delay slot, increment index
5157c478bdstevel@tonic-gate1:
5167c478bdstevel@tonic-gate	retl				! return(length - index)
5177c478bdstevel@tonic-gate	sub	%o0, %o4, %o0
5187c478bdstevel@tonic-gate	SET_SIZE(scanc)
5197c478bdstevel@tonic-gate
5207c478bdstevel@tonic-gate#endif	/* lint */
5217c478bdstevel@tonic-gate
5227c478bdstevel@tonic-gate/*
5237c478bdstevel@tonic-gate * if a() calls b() calls caller(),
5247c478bdstevel@tonic-gate * caller() returns return address in a().
5257c478bdstevel@tonic-gate */
5267c478bdstevel@tonic-gate
5277c478bdstevel@tonic-gate#if defined(lint)
5287c478bdstevel@tonic-gate
5297c478bdstevel@tonic-gatecaddr_t
5307c478bdstevel@tonic-gatecaller(void)
5317c478bdstevel@tonic-gate{ return (0); }
5327c478bdstevel@tonic-gate
5337c478bdstevel@tonic-gate#else	/* lint */
5347c478bdstevel@tonic-gate
5357c478bdstevel@tonic-gate	ENTRY(caller)
5367c478bdstevel@tonic-gate	retl
5377c478bdstevel@tonic-gate	mov	%i7, %o0
5387c478bdstevel@tonic-gate	SET_SIZE(caller)
5397c478bdstevel@tonic-gate
5407c478bdstevel@tonic-gate#endif	/* lint */
5417c478bdstevel@tonic-gate
5427c478bdstevel@tonic-gate/*
5437c478bdstevel@tonic-gate * if a() calls callee(), callee() returns the
5447c478bdstevel@tonic-gate * return address in a();
5457c478bdstevel@tonic-gate */
5467c478bdstevel@tonic-gate
5477c478bdstevel@tonic-gate#if defined(lint)
5487c478bdstevel@tonic-gate
5497c478bdstevel@tonic-gatecaddr_t
5507c478bdstevel@tonic-gatecallee(void)
5517c478bdstevel@tonic-gate{ return (0); }
5527c478bdstevel@tonic-gate
5537c478bdstevel@tonic-gate#else	/* lint */
5547c478bdstevel@tonic-gate
5557c478bdstevel@tonic-gate	ENTRY(callee)
5567c478bdstevel@tonic-gate	retl
5577c478bdstevel@tonic-gate	mov	%o7, %o0
5587c478bdstevel@tonic-gate	SET_SIZE(callee)
5597c478bdstevel@tonic-gate
5607c478bdstevel@tonic-gate#endif	/* lint */
5617c478bdstevel@tonic-gate
5627c478bdstevel@tonic-gate/*
5637c478bdstevel@tonic-gate * return the current frame pointer
5647c478bdstevel@tonic-gate */
5657c478bdstevel@tonic-gate
5667c478bdstevel@tonic-gate#if defined(lint)
5677c478bdstevel@tonic-gate
5687c478bdstevel@tonic-gategreg_t
5697c478bdstevel@tonic-gategetfp(void)
5707c478bdstevel@tonic-gate{ return (0); }
5717c478bdstevel@tonic-gate
5727c478bdstevel@tonic-gate#else	/* lint */
5737c478bdstevel@tonic-gate
5747c478bdstevel@tonic-gate	ENTRY(getfp)
5757c478bdstevel@tonic-gate	retl
5767c478bdstevel@tonic-gate	mov	%fp, %o0
5777c478bdstevel@tonic-gate	SET_SIZE(getfp)
5787c478bdstevel@tonic-gate
5797c478bdstevel@tonic-gate#endif	/* lint */
5807c478bdstevel@tonic-gate
5817c478bdstevel@tonic-gate/*
5827c478bdstevel@tonic-gate * Get vector base register
5837c478bdstevel@tonic-gate */
5847c478bdstevel@tonic-gate
5857c478bdstevel@tonic-gate#if defined(lint)
5867c478bdstevel@tonic-gate
5877c478bdstevel@tonic-gategreg_t
5887c478bdstevel@tonic-gategettbr(void)
5897c478bdstevel@tonic-gate{ return (0); }
5907c478bdstevel@tonic-gate
5917c478bdstevel@tonic-gate#else	/* lint */
5927c478bdstevel@tonic-gate
5937c478bdstevel@tonic-gate	ENTRY(gettbr)
5947c478bdstevel@tonic-gate	retl
5957c478bdstevel@tonic-gate	mov     %tbr, %o0
5967c478bdstevel@tonic-gate	SET_SIZE(gettbr)
5977c478bdstevel@tonic-gate
5987c478bdstevel@tonic-gate#endif	/* lint */
5997c478bdstevel@tonic-gate
6007c478bdstevel@tonic-gate/*
6017c478bdstevel@tonic-gate * Get processor state register, V9 faked to look like V8.
6027c478bdstevel@tonic-gate * Note: does not provide ccr.xcc and provides FPRS.FEF instead of
6037c478bdstevel@tonic-gate * PSTATE.PEF, because PSTATE.PEF is always on in order to allow the
6047c478bdstevel@tonic-gate * libc_psr memcpy routines to run without hitting the fp_disabled trap.
6057c478bdstevel@tonic-gate */
6067c478bdstevel@tonic-gate
6077c478bdstevel@tonic-gate#if defined(lint)
6087c478bdstevel@tonic-gate
6097c478bdstevel@tonic-gategreg_t
6107c478bdstevel@tonic-gategetpsr(void)
6117c478bdstevel@tonic-gate{ return (0); }
6127c478bdstevel@tonic-gate
6137c478bdstevel@tonic-gate#else	/* lint */
6147c478bdstevel@tonic-gate
6157c478bdstevel@tonic-gate	ENTRY(getpsr)
6167c478bdstevel@tonic-gate	rd	%ccr, %o1			! get ccr
6177c478bdstevel@tonic-gate        sll	%o1, PSR_ICC_SHIFT, %o0		! move icc to V8 psr.icc
6187c478bdstevel@tonic-gate	rd	%fprs, %o1			! get fprs
6197c478bdstevel@tonic-gate	and	%o1, FPRS_FEF, %o1		! mask out dirty upper/lower
6207c478bdstevel@tonic-gate	sllx	%o1, PSR_FPRS_FEF_SHIFT, %o1	! shift fef to V8 psr.ef
6217c478bdstevel@tonic-gate        or	%o0, %o1, %o0			! or into psr.ef
6227c478bdstevel@tonic-gate        set	V9_PSR_IMPLVER, %o1		! SI assigned impl/ver: 0xef
6237c478bdstevel@tonic-gate        retl
6247c478bdstevel@tonic-gate        or	%o0, %o1, %o0			! or into psr.impl/ver
6257c478bdstevel@tonic-gate	SET_SIZE(getpsr)
6267c478bdstevel@tonic-gate
6277c478bdstevel@tonic-gate#endif	/* lint */
6287c478bdstevel@tonic-gate
6297c478bdstevel@tonic-gate/*
6307c478bdstevel@tonic-gate * Get current processor interrupt level
6317c478bdstevel@tonic-gate */
6327c478bdstevel@tonic-gate
6337c478bdstevel@tonic-gate#if defined(lint)
6347c478bdstevel@tonic-gate
6357c478bdstevel@tonic-gateu_int
6367c478bdstevel@tonic-gategetpil(void)
6377c478bdstevel@tonic-gate{ return (0); }
6387c478bdstevel@tonic-gate
6397c478bdstevel@tonic-gate#else	/* lint */
6407c478bdstevel@tonic-gate
6417c478bdstevel@tonic-gate	ENTRY(getpil)
6427c478bdstevel@tonic-gate	retl
6437c478bdstevel@tonic-gate	rdpr	%pil, %o0
6447c478bdstevel@tonic-gate	SET_SIZE(getpil)
6457c478bdstevel@tonic-gate
6467c478bdstevel@tonic-gate#endif	/* lint */
6477c478bdstevel@tonic-gate
6487c478bdstevel@tonic-gate#if defined(lint)
6497c478bdstevel@tonic-gate
6507c478bdstevel@tonic-gate/*ARGSUSED*/
6517c478bdstevel@tonic-gatevoid
6527c478bdstevel@tonic-gatesetpil(u_int pil)
6537c478bdstevel@tonic-gate{}
6547c478bdstevel@tonic-gate
6557c478bdstevel@tonic-gate#else	/* lint */
6567c478bdstevel@tonic-gate
6577c478bdstevel@tonic-gate	ENTRY(setpil)
6587c478bdstevel@tonic-gate	retl
6597c478bdstevel@tonic-gate	wrpr	%g0, %o0, %pil
6607c478bdstevel@tonic-gate	SET_SIZE(setpil)
6617c478bdstevel@tonic-gate
6627c478bdstevel@tonic-gate#endif	/* lint */
6637c478bdstevel@tonic-gate
6647c478bdstevel@tonic-gate
6657c478bdstevel@tonic-gate/*
6667c478bdstevel@tonic-gate * _insque(entryp, predp)
6677c478bdstevel@tonic-gate *
6687c478bdstevel@tonic-gate * Insert entryp after predp in a doubly linked list.
6697c478bdstevel@tonic-gate */
6707c478bdstevel@tonic-gate
6717c478bdstevel@tonic-gate#if defined(lint)
6727c478bdstevel@tonic-gate
6737c478bdstevel@tonic-gate/*ARGSUSED*/
6747c478bdstevel@tonic-gatevoid
6757c478bdstevel@tonic-gate_insque(caddr_t entryp, caddr_t predp)
6767c478bdstevel@tonic-gate{}
6777c478bdstevel@tonic-gate
6787c478bdstevel@tonic-gate#else	/* lint */
6797c478bdstevel@tonic-gate
6807c478bdstevel@tonic-gate	ENTRY(_insque)
6817c478bdstevel@tonic-gate	ldn	[%o1], %g1		! predp->forw
6827c478bdstevel@tonic-gate	stn	%o1, [%o0 + CPTRSIZE]	! entryp->back = predp
6837c478bdstevel@tonic-gate	stn	%g1, [%o0]		! entryp->forw = predp->forw
6847c478bdstevel@tonic-gate	stn	%o0, [%o1]		! predp->forw = entryp
6857c478bdstevel@tonic-gate	retl
6867c478bdstevel@tonic-gate	stn	%o0, [%g1 + CPTRSIZE]	! predp->forw->back = entryp
6877c478bdstevel@tonic-gate	SET_SIZE(_insque)
6887c478bdstevel@tonic-gate
6897c478bdstevel@tonic-gate#endif	/* lint */
6907c478bdstevel@tonic-gate
6917c478bdstevel@tonic-gate/*
6927c478bdstevel@tonic-gate * _remque(entryp)
6937c478bdstevel@tonic-gate *
6947c478bdstevel@tonic-gate * Remove entryp from a doubly linked list
6957c478bdstevel@tonic-gate */
6967c478bdstevel@tonic-gate
6977c478bdstevel@tonic-gate#if defined(lint)
6987c478bdstevel@tonic-gate
6997c478bdstevel@tonic-gate/*ARGSUSED*/
7007c478bdstevel@tonic-gatevoid
7017c478bdstevel@tonic-gate_remque(caddr_t entryp)
7027c478bdstevel@tonic-gate{}
7037c478bdstevel@tonic-gate
7047c478bdstevel@tonic-gate#else	/* lint */
7057c478bdstevel@tonic-gate
7067c478bdstevel@tonic-gate	ENTRY(_remque)
7077c478bdstevel@tonic-gate	ldn	[%o0], %g1		! entryp->forw
7087c478bdstevel@tonic-gate	ldn	[%o0 + CPTRSIZE], %g2	! entryp->back
7097c478bdstevel@tonic-gate	stn	%g1, [%g2]		! entryp->back->forw = entryp->forw
7107c478bdstevel@tonic-gate	retl
7117c478bdstevel@tonic-gate	stn	%g2, [%g1 + CPTRSIZE]	! entryp->forw->back = entryp->back
7127c478bdstevel@tonic-gate	SET_SIZE(_remque)
7137c478bdstevel@tonic-gate
7147c478bdstevel@tonic-gate#endif	/* lint */
7157c478bdstevel@tonic-gate
7167c478bdstevel@tonic-gate
7177c478bdstevel@tonic-gate/*
7187c478bdstevel@tonic-gate * strlen(str)
7197c478bdstevel@tonic-gate *
7207c478bdstevel@tonic-gate * Returns the number of non-NULL bytes in string argument.
7217c478bdstevel@tonic-gate *
7227c478bdstevel@tonic-gate * XXX -  why is this here, rather than the traditional file?
7237c478bdstevel@tonic-gate *	  why does it have local labels which don't start with a `.'?
7247c478bdstevel@tonic-gate */
7257c478bdstevel@tonic-gate
7267c478bdstevel@tonic-gate#if defined(lint)
7277c478bdstevel@tonic-gate
7287c478bdstevel@tonic-gate/*ARGSUSED*/
7297c478bdstevel@tonic-gatesize_t
7307c478bdstevel@tonic-gatestrlen(const char *str)
7317c478bdstevel@tonic-gate{ return (0); }
7327c478bdstevel@tonic-gate
7337c478bdstevel@tonic-gate#else	/* lint */
7347c478bdstevel@tonic-gate
7357c478bdstevel@tonic-gate	ENTRY(strlen)
7367c478bdstevel@tonic-gate	mov	%o0, %o1
7377c478bdstevel@tonic-gate	andcc	%o1, 3, %o3		! is src word aligned
7387c478bdstevel@tonic-gate	bz	$nowalgnd
7397c478bdstevel@tonic-gate	clr	%o0			! length of non-zero bytes
7407c478bdstevel@tonic-gate	cmp	%o3, 2			! is src half-word aligned
7417c478bdstevel@tonic-gate	be	$s2algn
7427c478bdstevel@tonic-gate	cmp	%o3, 3			! src is byte aligned
7437c478bdstevel@tonic-gate	ldub	[%o1], %o3		! move 1 or 3 bytes to align it
7447c478bdstevel@tonic-gate	inc	1, %o1			! in either case, safe to do a byte
7457c478bdstevel@tonic-gate	be	$s3algn
7467c478bdstevel@tonic-gate	tst	%o3
7477c478bdstevel@tonic-gate$s1algn:
7487c478bdstevel@tonic-gate	bnz,a	$s2algn			! now go align dest
7497c478bdstevel@tonic-gate	inc	1, %o0
7507c478bdstevel@tonic-gate	b,a	$done
7517c478bdstevel@tonic-gate
7527c478bdstevel@tonic-gate$s2algn:
7537c478bdstevel@tonic-gate	lduh	[%o1], %o3		! know src is half-byte aligned
7547c478bdstevel@tonic-gate	inc	2, %o1
7557c478bdstevel@tonic-gate	srl	%o3, 8, %o4
7567c478bdstevel@tonic-gate	tst	%o4			! is the first byte zero
7577c478bdstevel@tonic-gate	bnz,a	1f
7587c478bdstevel@tonic-gate	inc	%o0
7597c478bdstevel@tonic-gate	b,a	$done
7607c478bdstevel@tonic-gate1:	andcc	%o3, 0xff, %o3		! is the second byte zero
7617c478bdstevel@tonic-gate	bnz,a	$nowalgnd
7627c478bdstevel@tonic-gate	inc	%o0
7637c478bdstevel@tonic-gate	b,a	$done
7647c478bdstevel@tonic-gate$s3algn:
7657c478bdstevel@tonic-gate	bnz,a	$nowalgnd
7667c478bdstevel@tonic-gate	inc	1, %o0
7677c478bdstevel@tonic-gate	b,a	$done
7687c478bdstevel@tonic-gate
7697c478bdstevel@tonic-gate$nowalgnd:
7707c478bdstevel@tonic-gate	! use trick to check if any read bytes of a word are zero
7717c478bdstevel@tonic-gate	! the following two constants will generate "byte carries"
7727c478bdstevel@tonic-gate	! and check if any bit in a byte is set, if all characters
7737c478bdstevel@tonic-gate	! are 7bits (unsigned) this allways works, otherwise
7747c478bdstevel@tonic-gate	! there is a specil case that rarely happens, see below
7757c478bdstevel@tonic-gate
7767c478bdstevel@tonic-gate	set	0x7efefeff, %o3
7777c478bdstevel@tonic-gate	set	0x81010100, %o4
7787c478bdstevel@tonic-gate
7797c478bdstevel@tonic-gate3:	ld	[%o1], %o2		! main loop
7807c478bdstevel@tonic-gate	inc	4, %o1
7817c478bdstevel@tonic-gate	add	%o2, %o3, %o5		! generate byte-carries
7827c478bdstevel@tonic-gate	xor	%o5, %o2, %o5		! see if orignal bits set
7837c478bdstevel@tonic-gate	and	%o5, %o4, %o5
7847c478bdstevel@tonic-gate	cmp	%o5, %o4		! if ==,  no zero bytes
7857c478bdstevel@tonic-gate	be,a	3b
7867c478bdstevel@tonic-gate	inc	4, %o0
7877c478bdstevel@tonic-gate
7887c478bdstevel@tonic-gate	! check for the zero byte and increment the count appropriately
7897c478bdstevel@tonic-gate	! some information (the carry bit) is lost if bit 31
7907c478bdstevel@tonic-gate	! was set (very rare), if this is the rare condition,
7917c478bdstevel@tonic-gate	! return to the main loop again
7927c478bdstevel@tonic-gate
7937c478bdstevel@tonic-gate	sethi	%hi(0xff000000), %o5	! mask used to test for terminator
7947c478bdstevel@tonic-gate	andcc	%o2, %o5, %g0		! check if first byte was zero
7957c478bdstevel@tonic-gate	bnz	1f
7967c478bdstevel@tonic-gate	srl	%o5, 8, %o5
7977c478bdstevel@tonic-gate$done:
7987c478bdstevel@tonic-gate	retl
7997c478bdstevel@tonic-gate	nop
8007c478bdstevel@tonic-gate1:	andcc	%o2, %o5, %g0		! check if second byte was zero
8017c478bdstevel@tonic-gate	bnz	1f
8027c478bdstevel@tonic-gate	srl	%o5, 8, %o5
8037c478bdstevel@tonic-gate$done1:
8047c478bdstevel@tonic-gate	retl
8057c478bdstevel@tonic-gate	inc	%o0
8067c478bdstevel@tonic-gate1:	andcc 	%o2, %o5, %g0		! check if third byte was zero
8077c478bdstevel@tonic-gate	bnz	1f
8087c478bdstevel@tonic-gate	andcc	%o2, 0xff, %g0		! check if last byte is zero
8097c478bdstevel@tonic-gate$done2:
8107c478bdstevel@tonic-gate	retl
8117c478bdstevel@tonic-gate	inc	2, %o0
8127c478bdstevel@tonic-gate1:	bnz,a	3b
8137c478bdstevel@tonic-gate	inc	4, %o0			! count of bytes
8147c478bdstevel@tonic-gate$done3:
8157c478bdstevel@tonic-gate	retl
8167c478bdstevel@tonic-gate	inc	3, %o0
8177c478bdstevel@tonic-gate	SET_SIZE(strlen)
8187c478bdstevel@tonic-gate
8197c478bdstevel@tonic-gate#endif	/* lint */
8207c478bdstevel@tonic-gate
8217c478bdstevel@tonic-gate/*
8227c478bdstevel@tonic-gate * Provide a C callable interface to the membar instruction.
8237c478bdstevel@tonic-gate */
8247c478bdstevel@tonic-gate
8257c478bdstevel@tonic-gate#if defined(lint)
8267c478bdstevel@tonic-gate
8277c478bdstevel@tonic-gatevoid
8287c478bdstevel@tonic-gatemembar_ldld(void)
8297c478bdstevel@tonic-gate{}
8307c478bdstevel@tonic-gate
8317c478bdstevel@tonic-gatevoid
8327c478bdstevel@tonic-gatemembar_stld(void)
8337c478bdstevel@tonic-gate{}
8347c478bdstevel@tonic-gate
8357c478bdstevel@tonic-gatevoid
8367c478bdstevel@tonic-gatemembar_ldst(void)
8377c478bdstevel@tonic-gate{}
8387c478bdstevel@tonic-gate
8397c478bdstevel@tonic-gatevoid
8407c478bdstevel@tonic-gatemembar_stst(void)
8417c478bdstevel@tonic-gate{}
8427c478bdstevel@tonic-gate
8437c478bdstevel@tonic-gatevoid
8447c478bdstevel@tonic-gatemembar_ldld_ldst(void)
8457c478bdstevel@tonic-gate{}
8467c478bdstevel@tonic-gate
8477c478bdstevel@tonic-gatevoid
8487c478bdstevel@tonic-gatemembar_ldld_stld(void)
8497c478bdstevel@tonic-gate{}
8507c478bdstevel@tonic-gate
8517c478bdstevel@tonic-gatevoid
8527c478bdstevel@tonic-gatemembar_ldld_stst(void)
8537c478bdstevel@tonic-gate{}
8547c478bdstevel@tonic-gate
8557c478bdstevel@tonic-gatevoid
8567c478bdstevel@tonic-gatemembar_stld_ldld(void)
8577c478bdstevel@tonic-gate{}
8587c478bdstevel@tonic-gate
8597c478bdstevel@tonic-gatevoid
8607c478bdstevel@tonic-gatemembar_stld_ldst(void)
8617c478bdstevel@tonic-gate{}
8627c478bdstevel@tonic-gate
8637c478bdstevel@tonic-gatevoid
8647c478bdstevel@tonic-gatemembar_stld_stst(void)
8657c478bdstevel@tonic-gate{}
8667c478bdstevel@tonic-gate
8677c478bdstevel@tonic-gatevoid
8687c478bdstevel@tonic-gatemembar_ldst_ldld(void)
8697c478bdstevel@tonic-gate{}
8707c478bdstevel@tonic-gate
8717c478bdstevel@tonic-gatevoid
8727c478bdstevel@tonic-gatemembar_ldst_stld(void)
8737c478bdstevel@tonic-gate{}
8747c478bdstevel@tonic-gate
8757c478bdstevel@tonic-gatevoid
8767c478bdstevel@tonic-gatemembar_ldst_stst(void)
8777c478bdstevel@tonic-gate{}
8787c478bdstevel@tonic-gate
8797c478bdstevel@tonic-gatevoid
8807c478bdstevel@tonic-gatemembar_stst_ldld(void)
8817c478bdstevel@tonic-gate{}
8827c478bdstevel@tonic-gate
8837c478bdstevel@tonic-gatevoid
8847c478bdstevel@tonic-gatemembar_stst_stld(void)
8857c478bdstevel@tonic-gate{}
8867c478bdstevel@tonic-gate
8877c478bdstevel@tonic-gatevoid
8887c478bdstevel@tonic-gatemembar_stst_ldst(void)
8897c478bdstevel@tonic-gate{}
8907c478bdstevel@tonic-gate
8917c478bdstevel@tonic-gatevoid
8927c478bdstevel@tonic-gatemembar_lookaside(void)
8937c478bdstevel@tonic-gate{}
8947c478bdstevel@tonic-gate
8957c478bdstevel@tonic-gatevoid
8967c478bdstevel@tonic-gatemembar_memissue(void)
8977c478bdstevel@tonic-gate{}
8987c478bdstevel@tonic-gate
8997c478bdstevel@tonic-gatevoid
9007c478bdstevel@tonic-gatemembar_sync(void)
9017c478bdstevel@tonic-gate{}
9027c478bdstevel@tonic-gate
9037c478bdstevel@tonic-gate#else
9047c478bdstevel@tonic-gate	ENTRY(membar_ldld)
9057c478bdstevel@tonic-gate	retl
9067c478bdstevel@tonic-gate	membar	#LoadLoad
9077c478bdstevel@tonic-gate	SET_SIZE(membar_ldld)
9087c478bdstevel@tonic-gate
9097c478bdstevel@tonic-gate	ENTRY(membar_stld)
9107c478bdstevel@tonic-gate	retl
9117c478bdstevel@tonic-gate	membar	#StoreLoad
9127c478bdstevel@tonic-gate	SET_SIZE(membar_stld)
9137c478bdstevel@tonic-gate
9147c478bdstevel@tonic-gate	ENTRY(membar_ldst)
9157c478bdstevel@tonic-gate	retl
9167c478bdstevel@tonic-gate	membar	#LoadStore
9177c478bdstevel@tonic-gate	SET_SIZE(membar_ldst)
9187c478bdstevel@tonic-gate
9197c478bdstevel@tonic-gate	ENTRY(membar_stst)
9207c478bdstevel@tonic-gate	retl
9217c478bdstevel@tonic-gate	membar	#StoreStore
9227c478bdstevel@tonic-gate	SET_SIZE(membar_stst)
9237c478bdstevel@tonic-gate
9247c478bdstevel@tonic-gate	ENTRY(membar_ldld_stld)
9257c478bdstevel@tonic-gate	ALTENTRY(membar_stld_ldld)
9267c478bdstevel@tonic-gate	retl
9277c478bdstevel@tonic-gate	membar	#LoadLoad|#StoreLoad
9287c478bdstevel@tonic-gate	SET_SIZE(membar_stld_ldld)
9297c478bdstevel@tonic-gate	SET_SIZE(membar_ldld_stld)
9307c478bdstevel@tonic-gate
9317c478bdstevel@tonic-gate	ENTRY(membar_ldld_ldst)
9327c478bdstevel@tonic-gate	ALTENTRY(membar_ldst_ldld)
9337c478bdstevel@tonic-gate	retl
9347c478bdstevel@tonic-gate	membar	#LoadLoad|#LoadStore
9357c478bdstevel@tonic-gate	SET_SIZE(membar_ldst_ldld)
9367c478bdstevel@tonic-gate	SET_SIZE(membar_ldld_ldst)
9377c478bdstevel@tonic-gate
9387c478bdstevel@tonic-gate	ENTRY(membar_ldld_stst)
9397c478bdstevel@tonic-gate	ALTENTRY(membar_stst_ldld)
9407c478bdstevel@tonic-gate	retl
9417c478bdstevel@tonic-gate	membar	#LoadLoad|#StoreStore
9427c478bdstevel@tonic-gate	SET_SIZE(membar_stst_ldld)
9437c478bdstevel@tonic-gate	SET_SIZE(membar_ldld_stst)
9447c478bdstevel@tonic-gate
9457c478bdstevel@tonic-gate	ENTRY(membar_stld_ldst)
9467c478bdstevel@tonic-gate	ALTENTRY(membar_ldst_stld)
9477c478bdstevel@tonic-gate	retl
9487c478bdstevel@tonic-gate	membar	#StoreLoad|#LoadStore
9497c478bdstevel@tonic-gate	SET_SIZE(membar_ldst_stld)
9507c478bdstevel@tonic-gate	SET_SIZE(membar_stld_ldst)
9517c478bdstevel@tonic-gate
9527c478bdstevel@tonic-gate	ENTRY(membar_stld_stst)
9537c478bdstevel@tonic-gate	ALTENTRY(membar_stst_stld)
9547c478bdstevel@tonic-gate	retl
9557c478bdstevel@tonic-gate	membar	#StoreLoad|#StoreStore
9567c478bdstevel@tonic-gate	SET_SIZE(membar_stst_stld)
9577c478bdstevel@tonic-gate	SET_SIZE(membar_stld_stst)
9587c478bdstevel@tonic-gate
9597c478bdstevel@tonic-gate	ENTRY(membar_ldst_stst)
9607c478bdstevel@tonic-gate	ALTENTRY(membar_stst_ldst)
9617c478bdstevel@tonic-gate	retl
9627c478bdstevel@tonic-gate	membar	#LoadStore|#StoreStore
9637c478bdstevel@tonic-gate	SET_SIZE(membar_stst_ldst)
9647c478bdstevel@tonic-gate	SET_SIZE(membar_ldst_stst)
9657c478bdstevel@tonic-gate
9667c478bdstevel@tonic-gate	ENTRY(membar_lookaside)
9677c478bdstevel@tonic-gate	retl
9687c478bdstevel@tonic-gate	membar	#Lookaside
9697c478bdstevel@tonic-gate	SET_SIZE(membar_lookaside)
9707c478bdstevel@tonic-gate
9717c478bdstevel@tonic-gate	ENTRY(membar_memissue)
9727c478bdstevel@tonic-gate	retl
9737c478bdstevel@tonic-gate	membar	#MemIssue
9747c478bdstevel@tonic-gate	SET_SIZE(membar_memissue)
9757c478bdstevel@tonic-gate
9767c478bdstevel@tonic-gate	ENTRY(membar_sync)
9777c478bdstevel@tonic-gate	retl
9787c478bdstevel@tonic-gate	membar	#Sync
9797c478bdstevel@tonic-gate	SET_SIZE(membar_sync)
9807c478bdstevel@tonic-gate
9817c478bdstevel@tonic-gate#endif	/* lint */
9827c478bdstevel@tonic-gate
9837c478bdstevel@tonic-gate
9847c478bdstevel@tonic-gate#if defined(lint)
9857c478bdstevel@tonic-gate
9867c478bdstevel@tonic-gate/*ARGSUSED*/
9877c478bdstevel@tonic-gateint
9887c478bdstevel@tonic-gatefuword64(const void *addr, uint64_t *dst)
9897c478bdstevel@tonic-gate{ return (0); }
9907c478bdstevel@tonic-gate
9917c478bdstevel@tonic-gate/*ARGSUSED*/
9927c478bdstevel@tonic-gateint
9937c478bdstevel@tonic-gatefuword32(const void *addr, uint32_t *dst)
9947c478bdstevel@tonic-gate{ return (0); }
9957c478bdstevel@tonic-gate
9967c478bdstevel@tonic-gate/*ARGSUSED*/
9977c478bdstevel@tonic-gateint
9987c478bdstevel@tonic-gatefuword16(const void *addr, uint16_t *dst)
9997c478bdstevel@tonic-gate{ return (0); }
10007c478bdstevel@tonic-gate
10017c478bdstevel@tonic-gate/*ARGSUSED*/
10027c478bdstevel@tonic-gateint
10037c478bdstevel@tonic-gatefuword8(const void *addr, uint8_t *dst)
10047c478bdstevel@tonic-gate{ return (0); }
10057c478bdstevel@tonic-gate
10067c478bdstevel@tonic-gate/*ARGSUSED*/
10077c478bdstevel@tonic-gateint
10087c478bdstevel@tonic-gatedtrace_ft_fuword64(const void *addr, uint64_t *dst)
10097c478bdstevel@tonic-gate{ return (0); }
10107c478bdstevel@tonic-gate
10117c478bdstevel@tonic-gate/*ARGSUSED*/
10127c478bdstevel@tonic-gateint
10137c478bdstevel@tonic-gatedtrace_ft_fuword32(const void *addr, uint32_t *dst)
10147c478bdstevel@tonic-gate{ return (0); }
10157c478bdstevel@tonic-gate
10167c478bdstevel@tonic-gate#else	/* lint */
10177c478bdstevel@tonic-gate
10187c478bdstevel@tonic-gate/*
10197c478bdstevel@tonic-gate * Since all of the fuword() variants are so similar, we have a macro to spit
10207c478bdstevel@tonic-gate * them out.
10217c478bdstevel@tonic-gate */
10227c478bdstevel@tonic-gate
10237c478bdstevel@tonic-gate#define	FUWORD(NAME, LOAD, STORE, COPYOP)	\
10247c478bdstevel@tonic-gate	ENTRY(NAME);				\
10257c478bdstevel@tonic-gate	sethi	%hi(1f), %o5;			\
10267c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_LOFAULT], %o3;	\
10277c478bdstevel@tonic-gate	or	%o5, %lo(1f), %o5;		\
10287c478bdstevel@tonic-gate	membar	#Sync;				\
10297c478bdstevel@tonic-gate	stn	%o5, [THREAD_REG + T_LOFAULT];	\
10307c478bdstevel@tonic-gate	LOAD	[%o0]ASI_USER, %o2;		\
10317c478bdstevel@tonic-gate	membar	#Sync;				\
10327c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LOFAULT];	\
10337c478bdstevel@tonic-gate	mov	0, %o0;				\
10347c478bdstevel@tonic-gate	retl;					\
10357c478bdstevel@tonic-gate	STORE	%o2, [%o1];			\
10367c478bdstevel@tonic-gate1:						\
10377c478bdstevel@tonic-gate	membar	#Sync;				\
10387c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LOFAULT];	\
10397c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_COPYOPS], %o2;	\
10407c478bdstevel@tonic-gate	brz	%o2, 2f;			\
10417c478bdstevel@tonic-gate	nop;					\
10427c478bdstevel@tonic-gate	ldn	[%o2 + COPYOP], %g1;		\
10437c478bdstevel@tonic-gate	jmp	%g1;				\
10447c478bdstevel@tonic-gate	nop;					\
10457c478bdstevel@tonic-gate2:						\
10467c478bdstevel@tonic-gate	retl;					\
10477c478bdstevel@tonic-gate	mov	-1, %o0;			\
10487c478bdstevel@tonic-gate	SET_SIZE(NAME)
10497c478bdstevel@tonic-gate
10507c478bdstevel@tonic-gate	FUWORD(fuword64, ldxa, stx, CP_FUWORD64)
10517c478bdstevel@tonic-gate	FUWORD(fuword32, lda, st, CP_FUWORD32)
10527c478bdstevel@tonic-gate	FUWORD(fuword16, lduha, sth, CP_FUWORD16)
10537c478bdstevel@tonic-gate	FUWORD(fuword8, lduba, stb, CP_FUWORD8)
10547c478bdstevel@tonic-gate
10557c478bdstevel@tonic-gate#endif	/* lint */
10567c478bdstevel@tonic-gate
10577c478bdstevel@tonic-gate
10587c478bdstevel@tonic-gate#if defined(lint)
10597c478bdstevel@tonic-gate
10607c478bdstevel@tonic-gate/*ARGSUSED*/
10617c478bdstevel@tonic-gateint
10627c478bdstevel@tonic-gatesuword64(void *addr, uint64_t value)
10637c478bdstevel@tonic-gate{ return (0); }
10647c478bdstevel@tonic-gate
10657c478bdstevel@tonic-gate/*ARGSUSED*/
10667c478bdstevel@tonic-gateint
10677c478bdstevel@tonic-gatesuword32(void *addr, uint32_t value)
10687c478bdstevel@tonic-gate{ return (0); }
10697c478bdstevel@tonic-gate
10707c478bdstevel@tonic-gate/*ARGSUSED*/
10717c478bdstevel@tonic-gateint
10727c478bdstevel@tonic-gatesuword16(void *addr, uint16_t value)
10737c478bdstevel@tonic-gate{ return (0); }
10747c478bdstevel@tonic-gate
10757c478bdstevel@tonic-gate/*ARGSUSED*/
10767c478bdstevel@tonic-gateint
10777c478bdstevel@tonic-gatesuword8(void *addr, uint8_t value)
10787c478bdstevel@tonic-gate{ return (0); }
10797c478bdstevel@tonic-gate
10807c478bdstevel@tonic-gate#else	/* lint */
10817c478bdstevel@tonic-gate
10827c478bdstevel@tonic-gate/*
10837c478bdstevel@tonic-gate * Since all of the suword() variants are so similar, we have a macro to spit
10847c478bdstevel@tonic-gate * them out.
10857c478bdstevel@tonic-gate */
10867c478bdstevel@tonic-gate
10877c478bdstevel@tonic-gate#define	SUWORD(NAME, STORE, COPYOP)		\
10887c478bdstevel@tonic-gate	ENTRY(NAME)				\
10897c478bdstevel@tonic-gate	sethi	%hi(1f), %o5;			\
10907c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_LOFAULT], %o3;	\
10917c478bdstevel@tonic-gate	or	%o5, %lo(1f), %o5;		\
10927c478bdstevel@tonic-gate	membar	#Sync;				\
10937c478bdstevel@tonic-gate	stn	%o5, [THREAD_REG + T_LOFAULT];	\
10947c478bdstevel@tonic-gate	STORE	%o1, [%o0]ASI_USER;		\
10957c478bdstevel@tonic-gate	membar	#Sync;				\
10967c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LOFAULT];	\
10977c478bdstevel@tonic-gate	retl;					\
10987c478bdstevel@tonic-gate	clr	%o0;				\
10997c478bdstevel@tonic-gate1:						\
11007c478bdstevel@tonic-gate	membar	#Sync;				\
11017c478bdstevel@tonic-gate	stn	%o3, [THREAD_REG + T_LOFAULT];	\
11027c478bdstevel@tonic-gate	ldn	[THREAD_REG + T_COPYOPS], %o2;	\
11037c478bdstevel@tonic-gate	brz	%o2, 2f;			\
11047c478bdstevel@tonic-gate	nop;					\
11057c478bdstevel@tonic-gate	ldn	[%o2 + COPYOP], %g1;		\
11067c478bdstevel@tonic-gate	jmp	%g1;				\
11077c478bdstevel@tonic-gate	nop;					\
11087c478bdstevel@tonic-gate2:						\
11097c478bdstevel@tonic-gate	retl;					\
11107c478bdstevel@tonic-gate	mov	-1, %o0;			\
11117c478bdstevel@tonic-gate	SET_SIZE(NAME)
11127c478bdstevel@tonic-gate
11137c478bdstevel@tonic-gate	SUWORD(suword64, stxa, CP_SUWORD64)
11147c478bdstevel@tonic-gate	SUWORD(suword32, sta, CP_SUWORD32)
11157c478bdstevel@tonic-gate	SUWORD(suword16, stha, CP_SUWORD16)
11167c478bdstevel@tonic-gate	SUWORD(suword8, stba, CP_SUWORD8)
11177c478bdstevel@tonic-gate
11187c478bdstevel@tonic-gate#endif	/* lint */
11197c478bdstevel@tonic-gate
11207c478bdstevel@tonic-gate#if defined(lint)
11217c478bdstevel@tonic-gate
11227c478bdstevel@tonic-gate/*ARGSUSED*/
11237c478bdstevel@tonic-gatevoid
11247c478bdstevel@tonic-gatefuword8_noerr(const void *addr, uint8_t *dst)
11257c478bdstevel@tonic-gate{}
11267c478bdstevel@tonic-gate
11277c478bdstevel@tonic-gate/*ARGSUSED*/
11287c478bdstevel@tonic-gatevoid
11297c478bdstevel@tonic-gatefuword16_noerr(const void *addr, uint16_t *dst)
11307c478bdstevel@tonic-gate{}
11317c478bdstevel@tonic-gate
11327c478bdstevel@tonic-gate/*ARGSUSED*/
11337c478bdstevel@tonic-gatevoid
11347c478bdstevel@tonic-gatefuword32_noerr(const void *addr, uint32_t *dst)
11357c478bdstevel@tonic-gate{}
11367c478bdstevel@tonic-gate
11377c478bdstevel@tonic-gate/*ARGSUSED*/
11387c478bdstevel@tonic-gatevoid
11397c478bdstevel@tonic-gatefuword64_noerr(const void *addr, uint64_t *dst)
11407c478bdstevel@tonic-gate{}
11417c478bdstevel@tonic-gate
11427c478bdstevel@tonic-gate#else	/* lint */
11437c478bdstevel@tonic-gate
11447c478bdstevel@tonic-gate	ENTRY(fuword8_noerr)
11457c478bdstevel@tonic-gate	lduba	[%o0]ASI_USER, %o0
11467c478bdstevel@tonic-gate	retl
11477c478bdstevel@tonic-gate	stb	%o0, [%o1]
11487c478bdstevel@tonic-gate	SET_SIZE(fuword8_noerr)
11497c478bdstevel@tonic-gate
11507c478bdstevel@tonic-gate	ENTRY(fuword16_noerr)
11517c478bdstevel@tonic-gate	lduha	[%o0]ASI_USER, %o0
11527c478bdstevel@tonic-gate	retl
11537c478bdstevel@tonic-gate	sth	%o0, [%o1]
11547c478bdstevel@tonic-gate	SET_SIZE(fuword16_noerr)
11557c478bdstevel@tonic-gate
11567c478bdstevel@tonic-gate	ENTRY(fuword32_noerr)
11577c478bdstevel@tonic-gate	lda	[%o0]ASI_USER, %o0
11587c478bdstevel@tonic-gate	retl
11597c478bdstevel@tonic-gate	st	%o0, [%o1]
11607c478bdstevel@tonic-gate	SET_SIZE(fuword32_noerr)
11617c478bdstevel@tonic-gate
11627c478bdstevel@tonic-gate	ENTRY(fuword64_noerr)
11637c478bdstevel@tonic-gate	ldxa	[%o0]ASI_USER, %o0
11647c478bdstevel@tonic-gate	retl
11657c478bdstevel@tonic-gate	stx	%o0, [%o1]
11667c478bdstevel@tonic-gate	SET_SIZE(fuword64_noerr)
11677c478bdstevel@tonic-gate
11687c478bdstevel@tonic-gate#endif	/* lint */
11697c478bdstevel@tonic-gate
11707c478bdstevel@tonic-gate#if defined(lint)
11717c478bdstevel@tonic-gate
11727c478bdstevel@tonic-gate/*ARGSUSED*/
11737c478bdstevel@tonic-gatevoid
11747c478bdstevel@tonic-gatesuword8_noerr(void *addr, uint8_t value)
11757c478bdstevel@tonic-gate{}
11767c478bdstevel@tonic-gate
11777c478bdstevel@tonic-gate/*ARGSUSED*/
11787c478bdstevel@tonic-gatevoid
11797c478bdstevel@tonic-gatesuword16_noerr(void *addr, uint16_t value)
11807c478bdstevel@tonic-gate{}
11817c478bdstevel@tonic-gate
11827c478bdstevel@tonic-gate/*ARGSUSED*/
11837c478bdstevel@tonic-gatevoid
11847c478bdstevel@tonic-gatesuword32_noerr(void *addr, uint32_t value)
11857c478bdstevel@tonic-gate{}
11867c478bdstevel@tonic-gate
11877c478bdstevel@tonic-gate/*ARGSUSED*/
11887c478bdstevel@tonic-gatevoid
11897c478bdstevel@tonic-gatesuword64_noerr(void *addr, uint64_t value)
11907c478bdstevel@tonic-gate{}
11917c478bdstevel@tonic-gate
11927c478bdstevel@tonic-gate#else	/* lint */
11937c478bdstevel@tonic-gate
11947c478bdstevel@tonic-gate	ENTRY(suword8_noerr)
11957c478bdstevel@tonic-gate	retl
11967c478bdstevel@tonic-gate	stba	%o1, [%o0]ASI_USER
11977c478bdstevel@tonic-gate	SET_SIZE(suword8_noerr)
11987c478bdstevel@tonic-gate
11997c478bdstevel@tonic-gate	ENTRY(suword16_noerr)
12007c478bdstevel@tonic-gate	retl
12017c478bdstevel@tonic-gate	stha	%o1, [%o0]ASI_USER
12027c478bdstevel@tonic-gate	SET_SIZE(suword16_noerr)
12037c478bdstevel@tonic-gate
12047c478bdstevel@tonic-gate	ENTRY(suword32_noerr)
12057c478bdstevel@tonic-gate	retl
12067c478bdstevel@tonic-gate	sta	%o1, [%o0]ASI_USER
12077c478bdstevel@tonic-gate	SET_SIZE(suword32_noerr)
12087c478bdstevel@tonic-gate
12097c478bdstevel@tonic-gate	ENTRY(suword64_noerr)
12107c478bdstevel@tonic-gate	retl
12117c478bdstevel@tonic-gate	stxa	%o1, [%o0]ASI_USER
12127c478bdstevel@tonic-gate	SET_SIZE(suword64_noerr)
12137c478bdstevel@tonic-gate
12147c478bdstevel@tonic-gate#endif	/* lint */
12157c478bdstevel@tonic-gate
12167c478bdstevel@tonic-gate#if defined(__lint)
12177c478bdstevel@tonic-gate
12187c478bdstevel@tonic-gate/*ARGSUSED*/
12197c478bdstevel@tonic-gateint
12207c478bdstevel@tonic-gatesubyte(void *addr, uchar_t value)
12217c478bdstevel@tonic-gate{ return (0); }
12227c478bdstevel@tonic-gate
12237c478bdstevel@tonic-gate/*ARGSUSED*/
12247c478bdstevel@tonic-gatevoid
12257c478bdstevel@tonic-gatesubyte_noerr(void *addr, uchar_t value)
12267c478bdstevel@tonic-gate{}
12277c478bdstevel@tonic-gate
12287c478bdstevel@tonic-gate/*ARGSUSED*/
12297c478bdstevel@tonic-gateint
12307c478bdstevel@tonic-gatefulword(const void *addr, ulong_t *valuep)
12317c478bdstevel@tonic-gate{ return (0); }
12327c478bdstevel@tonic-gate
12337c478bdstevel@tonic-gate/*ARGSUSED*/
12347c478bdstevel@tonic-gatevoid
12357c478bdstevel@tonic-gatefulword_noerr(const void *addr, ulong_t *valuep)
12367c478bdstevel@tonic-gate{}
12377c478bdstevel@tonic-gate
12387c478bdstevel@tonic-gate/*ARGSUSED*/
12397c478bdstevel@tonic-gateint
12407c478bdstevel@tonic-gatesulword(void *addr, ulong_t valuep)
12417c478bdstevel@tonic-gate{ return (0); }
12427c478bdstevel@tonic-gate
12437c478bdstevel@tonic-gate/*ARGSUSED*/
12447c478bdstevel@tonic-gatevoid
12457c478bdstevel@tonic-gatesulword_noerr(void *addr, ulong_t valuep)
12467c478bdstevel@tonic-gate{}
12477c478bdstevel@tonic-gate
12487c478bdstevel@tonic-gate#else
12497c478bdstevel@tonic-gate
12507c478bdstevel@tonic-gate	.weak	subyte
12517c478bdstevel@tonic-gate	subyte=suword8
12527c478bdstevel@tonic-gate	.weak	subyte_noerr
12537c478bdstevel@tonic-gate	subyte_noerr=suword8_noerr
12547c478bdstevel@tonic-gate#ifdef _LP64
12557c478bdstevel@tonic-gate	.weak	fulword
12567c478bdstevel@tonic-gate	fulword=fuword64
12577c478bdstevel@tonic-gate	.weak	fulword_noerr
12587c478bdstevel@tonic-gate	fulword_noerr=fuword64_noerr
12597c478bdstevel@tonic-gate	.weak	sulword
12607c478bdstevel@tonic-gate	sulword=suword64
12617c478bdstevel@tonic-gate	.weak	sulword_noerr
12627c478bdstevel@tonic-gate	sulword_noerr=suword64_noerr
12637c478bdstevel@tonic-gate#else
12647c478bdstevel@tonic-gate	.weak	fulword
12657c478bdstevel@tonic-gate	fulword=fuword32
12667c478bdstevel@tonic-gate	.weak	fulword_noerr
12677c478bdstevel@tonic-gate	fulword_noerr=fuword32_noerr
12687c478bdstevel@tonic-gate	.weak	sulword
12697c478bdstevel@tonic-gate	sulword=suword32
12707c478bdstevel@tonic-gate	.weak	sulword_noerr
12717c478bdstevel@tonic-gate	sulword_noerr=suword32_noerr
12727c478bdstevel@tonic-gate#endif	/* LP64 */
12737c478bdstevel@tonic-gate
12747c478bdstevel@tonic-gate#endif	/* lint */
12757c478bdstevel@tonic-gate
1276023e71dHaik Aftandilian/*
1277023e71dHaik Aftandilian * We define rdtick here, but not for sun4v. On sun4v systems, the %tick
1278023e71dHaik Aftandilian * and %stick should not be read directly without considering the tick
1279023e71dHaik Aftandilian * and stick offset kernel variables introduced to support sun4v OS
1280023e71dHaik Aftandilian * suspension.
1281023e71dHaik Aftandilian */
1282023e71dHaik Aftandilian#if !defined (sun4v)
12837c478bdstevel@tonic-gate
12847c478bdstevel@tonic-gate#if defined (lint)
12857c478bdstevel@tonic-gate
12867c478bdstevel@tonic-gatehrtime_t
12877c478bdstevel@tonic-gaterdtick()
12887c478bdstevel@tonic-gate{ return (0); }
12897c478bdstevel@tonic-gate
1290023e71dHaik Aftandilian#else /* lint */
1291023e71dHaik Aftandilian
12927c478bdstevel@tonic-gate	ENTRY(rdtick)
12937c478bdstevel@tonic-gate	retl
12947c478bdstevel@tonic-gate	rd	%tick, %o0
12957c478bdstevel@tonic-gate        SET_SIZE(rdtick)
1296023e71dHaik Aftandilian
1297023e71dHaik Aftandilian#endif /* lint */
1298023e71dHaik Aftandilian
1299023e71dHaik Aftandilian#endif /* !sun4v */
13007c478bdstevel@tonic-gate
13017c478bdstevel@tonic-gate/*
13027c478bdstevel@tonic-gate * Set tba to given address, no side effects.
13037c478bdstevel@tonic-gate */
13047c478bdstevel@tonic-gate#if defined (lint)
13057c478bdstevel@tonic-gate
13067c478bdstevel@tonic-gate/*ARGSUSED*/
13077c478bdstevel@tonic-gatevoid *
13087c478bdstevel@tonic-gateset_tba(void *new_tba)
13097c478bdstevel@tonic-gate{ return (0); }
13107c478bdstevel@tonic-gate
13117c478bdstevel@tonic-gate#else	/* lint */
13127c478bdstevel@tonic-gate
13137c478bdstevel@tonic-gate	ENTRY(set_tba)
13147c478bdstevel@tonic-gate	mov	%o0, %o1
13157c478bdstevel@tonic-gate	rdpr	%tba, %o0
13167c478bdstevel@tonic-gate	wrpr	%o1, %tba
13177c478bdstevel@tonic-gate	retl
13187c478bdstevel@tonic-gate	nop
13197c478bdstevel@tonic-gate	SET_SIZE(set_tba)
13207c478bdstevel@tonic-gate
13217c478bdstevel@tonic-gate#endif	/* lint */
13227c478bdstevel@tonic-gate
13237c478bdstevel@tonic-gate#if defined (lint)
13247c478bdstevel@tonic-gate
13257c478bdstevel@tonic-gate/*ARGSUSED*/
13267c478bdstevel@tonic-gatevoid *
13277c478bdstevel@tonic-gateget_tba()
13287c478bdstevel@tonic-gate{ return (0); }
13297c478bdstevel@tonic-gate
13307c478bdstevel@tonic-gate#else	/* lint */
13317c478bdstevel@tonic-gate
13327c478bdstevel@tonic-gate	ENTRY(get_tba)
13337c478bdstevel@tonic-gate	retl
13347c478bdstevel@tonic-gate	rdpr	%tba, %o0
13357c478bdstevel@tonic-gate	SET_SIZE(get_tba)
13367c478bdstevel@tonic-gate
13377c478bdstevel@tonic-gate#endif	/* lint */
13387c478bdstevel@tonic-gate
13397c478bdstevel@tonic-gate#if defined(lint) || defined(__lint)
13407c478bdstevel@tonic-gate
13417c478bdstevel@tonic-gate/* ARGSUSED */
13427c478bdstevel@tonic-gatevoid
13437c478bdstevel@tonic-gatesetpstate(u_int pstate)
13447c478bdstevel@tonic-gate{}
13457c478bdstevel@tonic-gate
13467c478bdstevel@tonic-gate#else	/* lint */
13477c478bdstevel@tonic-gate
13487c478bdstevel@tonic-gate	ENTRY_NP(setpstate)
13497c478bdstevel@tonic-gate	retl
13507c478bdstevel@tonic-gate	wrpr	%g0, %o0, %pstate
13517c478bdstevel@tonic-gate	SET_SIZE(setpstate)
13527c478bdstevel@tonic-gate
13537c478bdstevel@tonic-gate#endif	/* lint */
13547c478bdstevel@tonic-gate
13557c478bdstevel@tonic-gate#if defined(lint) || defined(__lint)
13567c478bdstevel@tonic-gate
13577c478bdstevel@tonic-gateu_int
13587c478bdstevel@tonic-gategetpstate(void)
13597c478bdstevel@tonic-gate{ return(0); }
13607c478bdstevel@tonic-gate
13617c478bdstevel@tonic-gate#else	/* lint */
13627c478bdstevel@tonic-gate
13637c478bdstevel@tonic-gate	ENTRY_NP(getpstate)
13647c478bdstevel@tonic-gate	retl
13657c478bdstevel@tonic-gate	rdpr	%pstate, %o0
13667c478bdstevel@tonic-gate	SET_SIZE(getpstate)
13677c478bdstevel@tonic-gate
13687c478bdstevel@tonic-gate#endif	/* lint */
13697c478bdstevel@tonic-gate
13707c478bdstevel@tonic-gate#if defined(lint) || defined(__lint)
13717c478bdstevel@tonic-gate
13727c478bdstevel@tonic-gatedtrace_icookie_t
13737c478bdstevel@tonic-gatedtrace_interrupt_disable(void)
13747c478bdstevel@tonic-gate{ return (0); }
13757c478bdstevel@tonic-gate
13767c478bdstevel@tonic-gate#else	/* lint */
13777c478bdstevel@tonic-gate
13787c478bdstevel@tonic-gate	ENTRY_NP(dtrace_interrupt_disable)
13797c478bdstevel@tonic-gate	rdpr	%pstate, %o0
13807c478bdstevel@tonic-gate	andn	%o0, PSTATE_IE, %o1
13817c478bdstevel@tonic-gate	retl
13827c478bdstevel@tonic-gate	wrpr	%g0, %o1, %pstate
13837c478bdstevel@tonic-gate	SET_SIZE(dtrace_interrupt_disable)
13847c478bdstevel@tonic-gate
13857c478bdstevel@tonic-gate#endif	/* lint */
13867c478bdstevel@tonic-gate
13877c478bdstevel@tonic-gate#if defined(lint) || defined(__lint)
13887c478bdstevel@tonic-gate
13897c478bdstevel@tonic-gate/*ARGSUSED*/
13907c478bdstevel@tonic-gatevoid
13917c478bdstevel@tonic-gatedtrace_interrupt_enable(dtrace_icookie_t cookie)
13927c478bdstevel@tonic-gate{}
13937c478bdstevel@tonic-gate
13947c478bdstevel@tonic-gate#else
13957c478bdstevel@tonic-gate
13967c478bdstevel@tonic-gate	ENTRY_NP(dtrace_interrupt_enable)
13977c478bdstevel@tonic-gate	retl
13987c478bdstevel@tonic-gate	wrpr	%g0, %o0, %pstate
13997c478bdstevel@tonic-gate	SET_SIZE(dtrace_interrupt_enable)
14007c478bdstevel@tonic-gate
14017c478bdstevel@tonic-gate#endif /* lint*/
14027c478bdstevel@tonic-gate
14037c478bdstevel@tonic-gate#if defined(lint)
14047c478bdstevel@tonic-gate
14057c478bdstevel@tonic-gatevoid
14067c478bdstevel@tonic-gatedtrace_membar_producer(void)
14077c478bdstevel@tonic-gate{}
14087c478bdstevel@tonic-gate
14097c478bdstevel@tonic-gatevoid
14107c478bdstevel@tonic-gatedtrace_membar_consumer(void)
14117c478bdstevel@tonic-gate{}
14127c478bdstevel@tonic-gate
14137c478bdstevel@tonic-gate#else	/* lint */
14147c478bdstevel@tonic-gate
14157c478bdstevel@tonic-gate#ifdef SF_ERRATA_51
14167c478bdstevel@tonic-gate	.align 32
14177c478bdstevel@tonic-gate	ENTRY(dtrace_membar_return)
14187c478bdstevel@tonic-gate	retl
14197c478bdstevel@tonic-gate	nop
14207c478bdstevel@tonic-gate	SET_SIZE(dtrace_membar_return)
14217c478bdstevel@tonic-gate#define	DTRACE_MEMBAR_RETURN	ba,pt %icc, dtrace_membar_return
14227c478bdstevel@tonic-gate#else
14237c478bdstevel@tonic-gate#define	DTRACE_MEMBAR_RETURN	retl
14247c478bdstevel@tonic-gate#endif
14257c478bdstevel@tonic-gate
14267c478bdstevel@tonic-gate	ENTRY(dtrace_membar_producer)
14277c478bdstevel@tonic-gate	DTRACE_MEMBAR_RETURN
14287c478bdstevel@tonic-gate	membar	#StoreStore
14297c478bdstevel@tonic-gate	SET_SIZE(dtrace_membar_producer)
14307c478bdstevel@tonic-gate
14317c478bdstevel@tonic-gate	ENTRY(dtrace_membar_consumer)
14327c478bdstevel@tonic-gate	DTRACE_MEMBAR_RETURN
14337c478bdstevel@tonic-gate	membar	#LoadLoad
14347c478bdstevel@tonic-gate	SET_SIZE(dtrace_membar_consumer)
14357c478bdstevel@tonic-gate
14367c478bdstevel@tonic-gate#endif	/* lint */
14377c478bdstevel@tonic-gate
14387c478bdstevel@tonic-gate#if defined(lint) || defined(__lint)
14397c478bdstevel@tonic-gate
14407c478bdstevel@tonic-gatevoid
14417c478bdstevel@tonic-gatedtrace_flush_windows(void)
14427c478bdstevel@tonic-gate{}
14437c478bdstevel@tonic-gate
14447c478bdstevel@tonic-gate#else
14457c478bdstevel@tonic-gate
14467c478bdstevel@tonic-gate	ENTRY_NP(dtrace_flush_windows)
14477c478bdstevel@tonic-gate	retl
14487c478bdstevel@tonic-gate	flushw
14497c478bdstevel@tonic-gate	SET_SIZE(dtrace_flush_windows)
14507c478bdstevel@tonic-gate
14517c478bdstevel@tonic-gate#endif	/* lint */
14527c478bdstevel@tonic-gate
14537c478bdstevel@tonic-gate#if defined(lint)
14547c478bdstevel@tonic-gate
14557c478bdstevel@tonic-gate/*ARGSUSED*/
14567c478bdstevel@tonic-gateint
14577c478bdstevel@tonic-gategetpcstack_top(pc_t *pcstack, int limit, uintptr_t *lastfp, pc_t *lastpc)
14587c478bdstevel@tonic-gate{
14597c478bdstevel@tonic-gate	return (0);
14607c478bdstevel@tonic-gate}
14617c478bdstevel@tonic-gate
14627c478bdstevel@tonic-gate#else	/* lint */
14637c478bdstevel@tonic-gate
14647c478bdstevel@tonic-gate	/*
14657c478bdstevel@tonic-gate	 * %g1	pcstack
14667c478bdstevel@tonic-gate	 * %g2	iteration count
14677c478bdstevel@tonic-gate	 * %g3	final %fp
14687c478bdstevel@tonic-gate	 * %g4	final %i7
14697c478bdstevel@tonic-gate	 * %g5	saved %cwp (so we can get back to the original window)
14707c478bdstevel@tonic-gate	 *
14717c478bdstevel@tonic-gate	 * %o0	pcstack / return value (iteration count)
14727c478bdstevel@tonic-gate	 * %o1	limit / saved %cansave
14737c478bdstevel@tonic-gate	 * %o2	lastfp
14747c478bdstevel@tonic-gate	 * %o3	lastpc
14757c478bdstevel@tonic-gate	 * %o4	saved %canrestore
14767c478bdstevel@tonic-gate	 * %o5	saved %pstate (to restore interrupts)
14777c478bdstevel@tonic-gate	 *
14787c478bdstevel@tonic-gate	 * Note:  The frame pointer returned via lastfp is safe to use as
14797c478bdstevel@tonic-gate	 *	long as getpcstack_top() returns either (0) or a value less
14807c478bdstevel@tonic-gate	 *	than (limit).
14817c478bdstevel@tonic-gate	 */
14827c478bdstevel@tonic-gate	ENTRY_NP(getpcstack_top)
14837c478bdstevel@tonic-gate
14847c478bdstevel@tonic-gate	rdpr	%pstate, %o5
14857c478bdstevel@tonic-gate	andn	%o5, PSTATE_IE, %g1
14867c478bdstevel@tonic-gate	wrpr	%g0, %g1, %pstate	! disable interrupts
14877c478bdstevel@tonic-gate
14887c478bdstevel@tonic-gate	mov	%o0, %g1		! we need the pcstack pointer while
14897c478bdstevel@tonic-gate					! we're visiting other windows
14907c478bdstevel@tonic-gate
14917c478bdstevel@tonic-gate	rdpr	%canrestore, %g2	! number of available windows
14927c478bdstevel@tonic-gate	sub	%g2, 1, %g2		! account for skipped frame
14937c478bdstevel@tonic-gate	cmp	%g2, %o1		! compare with limit
14947c478bdstevel@tonic-gate	movg	%icc, %o1, %g2		! %g2 = min(%canrestore-1, limit)
14957c478bdstevel@tonic-gate
14967c478bdstevel@tonic-gate	brlez,a,pn %g2, 3f		! Use slow path if count <= 0 --
14977c478bdstevel@tonic-gate	clr	%o0			! return zero.
14987c478bdstevel@tonic-gate
14997c478bdstevel@tonic-gate	mov	%g2, %o0		! set up return value
15007c478bdstevel@tonic-gate
15017c478bdstevel@tonic-gate	rdpr	%cwp, %g5		! remember the register window state
15027c478bdstevel@tonic-gate	rdpr	%cansave, %o1		! 'restore' changes, so we can undo
15037c478bdstevel@tonic-gate	rdpr	%canrestore, %o4	! its effects when we finish.
15047c478bdstevel@tonic-gate
15057c478bdstevel@tonic-gate	restore				! skip caller's frame
15067c478bdstevel@tonic-gate1:
15077c478bdstevel@tonic-gate	st	%i7, [%g1]		! stash return address in pcstack
15087c478bdstevel@tonic-gate	restore				! go to the next frame
15097c478bdstevel@tonic-gate	subcc	%g2, 1, %g2		! decrement the count
15107c478bdstevel@tonic-gate	bnz,pt	%icc, 1b		! loop until count reaches 0
15117c478bdstevel@tonic-gate	add	%g1, 4, %g1		! increment pcstack
15127c478bdstevel@tonic-gate
15137c478bdstevel@tonic-gate	mov	%i6, %g3		! copy the final %fp and return PC
15147c478bdstevel@tonic-gate	mov	%i7, %g4		! aside so we can return them to our
15157c478bdstevel@tonic-gate					! caller
15167c478bdstevel@tonic-gate
15177c478bdstevel@tonic-gate	wrpr	%g0, %g5, %cwp		! jump back to the original window
15187c478bdstevel@tonic-gate	wrpr	%g0, %o1, %cansave	! and restore the original register
15197c478bdstevel@tonic-gate	wrpr	%g0, %o4, %canrestore	! window state.
15207c478bdstevel@tonic-gate2:
15217c478bdstevel@tonic-gate	stn	%g3, [%o2]		! store the frame pointer and pc
15227c478bdstevel@tonic-gate	st	%g4, [%o3]		! so our caller can continue the trace
15237c478bdstevel@tonic-gate
15247c478bdstevel@tonic-gate	retl				! return to caller
15257c478bdstevel@tonic-gate	wrpr	%g0, %o5, %pstate	! restore interrupts
15267c478bdstevel@tonic-gate
15277c478bdstevel@tonic-gate3:
15287c478bdstevel@tonic-gate	flushw				! flush register windows, then
15297c478bdstevel@tonic-gate	ldn	[%fp + STACK_BIAS + 14*CLONGSIZE], %g3	! load initial fp
15307c478bdstevel@tonic-gate	ba	2b
15317c478bdstevel@tonic-gate	ldn	[%fp + STACK_BIAS + 15*CLONGSIZE], %g4	! and pc
15327c478bdstevel@tonic-gate	SET_SIZE(getpcstack_top)
15337c478bdstevel@tonic-gate
15347c478bdstevel@tonic-gate#endif	/* lint */
15357c478bdstevel@tonic-gate
15367c478bdstevel@tonic-gate#if defined(lint) || defined(__lint)
15377c478bdstevel@tonic-gate
15387c478bdstevel@tonic-gate/* ARGSUSED */
15397c478bdstevel@tonic-gatevoid
15407c478bdstevel@tonic-gatesetwstate(u_int wstate)
15417c478bdstevel@tonic-gate{}
15427c478bdstevel@tonic-gate
15437c478bdstevel@tonic-gate#else	/* lint */
15447c478bdstevel@tonic-gate
15457c478bdstevel@tonic-gate	ENTRY_NP(setwstate)
15467c478bdstevel@tonic-gate	retl
15477c478bdstevel@tonic-gate	wrpr	%g0, %o0, %wstate
15487c478bdstevel@tonic-gate	SET_SIZE(setwstate)
15497c478bdstevel@tonic-gate
15507c478bdstevel@tonic-gate#endif	/* lint */
15517c478bdstevel@tonic-gate
15527c478bdstevel@tonic-gate
15537c478bdstevel@tonic-gate#if defined(lint) || defined(__lint)
15547c478bdstevel@tonic-gate
15557c478bdstevel@tonic-gateu_int
15567c478bdstevel@tonic-gategetwstate(void)
15577c478bdstevel@tonic-gate{ return(0); }
15587c478bdstevel@tonic-gate
15597c478bdstevel@tonic-gate#else	/* lint */
15607c478bdstevel@tonic-gate
15617c478bdstevel@tonic-gate	ENTRY_NP(getwstate)
15627c478bdstevel@tonic-gate	retl
15637c478bdstevel@tonic-gate	rdpr	%wstate, %o0
15647c478bdstevel@tonic-gate	SET_SIZE(getwstate)
15657c478bdstevel@tonic-gate
15667c478bdstevel@tonic-gate#endif	/* lint */
15677c478bdstevel@tonic-gate
15687c478bdstevel@tonic-gate
15697c478bdstevel@tonic-gate/*
15707c478bdstevel@tonic-gate * int panic_trigger(int *tp)
15717c478bdstevel@tonic-gate *
15727c478bdstevel@tonic-gate * A panic trigger is a word which is updated atomically and can only be set
15737c478bdstevel@tonic-gate * once.  We atomically store 0xFF into the high byte and load the old value.
15747c478bdstevel@tonic-gate * If the byte was 0xFF, the trigger has already been activated and we fail.
15757c478bdstevel@tonic-gate * If the previous value was 0 or not 0xFF, we succeed.  This allows a
15767c478bdstevel@tonic-gate * partially corrupt trigger to still trigger correctly.  DTrace has its own
15777c478bdstevel@tonic-gate * version of this function to allow it to panic correctly from probe context.
15787c478bdstevel@tonic-gate */
15797c478bdstevel@tonic-gate#if defined(lint)
15807c478bdstevel@tonic-gate
15817c478bdstevel@tonic-gate/*ARGSUSED*/
15827c478bdstevel@tonic-gateint panic_trigger(int *tp) { return (0); }
15837c478bdstevel@tonic-gate
15847c478bdstevel@tonic-gate/*ARGSUSED*/
15857c478bdstevel@tonic-gateint dtrace_panic_trigger(int *tp) { return (0); }
15867c478bdstevel@tonic-gate
15877c478bdstevel@tonic-gate#else	/* lint */
15887c478bdstevel@tonic-gate
15897c478bdstevel@tonic-gate	ENTRY_NP(panic_trigger)
15907c478bdstevel@tonic-gate	ldstub	[%o0], %o0		! store 0xFF, load byte into %o0
15917c478bdstevel@tonic-gate	cmp	%o0, 0xFF		! compare %o0 to 0xFF
15927c478bdstevel@tonic-gate	set	1, %o1			! %o1 = 1
15937c478bdstevel@tonic-gate	be,a	0f			! if (%o0 == 0xFF) goto 0f (else annul)
15947c478bdstevel@tonic-gate	set	0, %o1			! delay - %o1 = 0
15957c478bdstevel@tonic-gate0:	retl
15967c478bdstevel@tonic-gate	mov	%o1, %o0		! return (%o1);
15977c478bdstevel@tonic-gate	SET_SIZE(panic_trigger)
15987c478bdstevel@tonic-gate
15997c478bdstevel@tonic-gate	ENTRY_NP(dtrace_panic_trigger)
16007c478bdstevel@tonic-gate	ldstub	[%o0], %o0		! store 0xFF, load byte into %o0
16017c478bdstevel@tonic-gate	cmp	%o0, 0xFF		! compare %o0 to 0xFF
16027c478bdstevel@tonic-gate	set	1, %o1			! %o1 = 1
16037c478bdstevel@tonic-gate	be,a	0f			! if (%o0 == 0xFF) goto 0f (else annul)
16047c478bdstevel@tonic-gate	set	0, %o1			! delay - %o1 = 0
16057c478bdstevel@tonic-gate0:	retl
16067c478bdstevel@tonic-gate	mov	%o1, %o0		! return (%o1);
16077c478bdstevel@tonic-gate	SET_SIZE(dtrace_panic_trigger)
16087c478bdstevel@tonic-gate
16097c478bdstevel@tonic-gate#endif	/* lint */
16107c478bdstevel@tonic-gate
16117c478bdstevel@tonic-gate/*
16127c478bdstevel@tonic-gate * void vpanic(const char *format, va_list alist)
16137c478bdstevel@tonic-gate *
16147c478bdstevel@tonic-gate * The panic() and cmn_err() functions invoke vpanic() as a common entry point
16157c478bdstevel@tonic-gate * into the panic code implemented in panicsys().  vpanic() is responsible
16167c478bdstevel@tonic-gate * for passing through the format string and arguments, and constructing a
16177c478bdstevel@tonic-gate * regs structure on the stack into which it saves the current register
16187c478bdstevel@tonic-gate * values.  If we are not dying due to a fatal trap, these registers will
16197c478bdstevel@tonic-gate * then be preserved in panicbuf as the current processor state.  Before
16207c478bdstevel@tonic-gate * invoking panicsys(), vpanic() activates the first panic trigger (see
16217c478bdstevel@tonic-gate * common/os/panic.c) and switches to the panic_stack if successful.  Note that
16227c478bdstevel@tonic-gate * DTrace takes a slightly different panic path if it must panic from probe
16237c478bdstevel@tonic-gate * context.  Instead of calling panic, it calls into dtrace_vpanic(), which
16247c478bdstevel@tonic-gate * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
16257c478bdstevel@tonic-gate * branches back into vpanic().
16267c478bdstevel@tonic-gate */
16277c478bdstevel@tonic-gate#if defined(lint)
16287c478bdstevel@tonic-gate
16297c478bdstevel@tonic-gate/*ARGSUSED*/
16307c478bdstevel@tonic-gatevoid vpanic(const char *format, va_list alist) {}
16317c478bdstevel@tonic-gate
16327c478bdstevel@tonic-gate/*ARGSUSED*/
16337c478bdstevel@tonic-gatevoid dtrace_vpanic(const char *format, va_list alist) {}
16347c478bdstevel@tonic-gate
16357c478bdstevel@tonic-gate#else	/* lint */
16367c478bdstevel@tonic-gate
16377c478bdstevel@tonic-gate	ENTRY_NP(vpanic)
16387c478bdstevel@tonic-gate
16397c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME + REGSIZE), %sp	! save and allocate regs
16407c478bdstevel@tonic-gate
16417c478bdstevel@tonic-gate	!
16427c478bdstevel@tonic-gate	! The v9 struct regs has a 64-bit r_tstate field, which we use here
16437c478bdstevel@tonic-gate	! to store the %ccr, %asi, %pstate, and %cwp as they would appear
16447c478bdstevel@tonic-gate	! in %tstate if a trap occurred.  We leave it up to the debugger to
16457c478bdstevel@tonic-gate	! realize what happened and extract the register values.
16467c478bdstevel@tonic-gate	!
16477c478bdstevel@tonic-gate	rd	%ccr, %l0				! %l0 = %ccr
16487c478bdstevel@tonic-gate	sllx	%l0, TSTATE_CCR_SHIFT, %l0		! %l0 <<= CCR_SHIFT
16497c478bdstevel@tonic-gate	rd	%asi, %l1				! %l1 = %asi
16507c478bdstevel@tonic-gate	sllx	%l1, TSTATE_ASI_SHIFT, %l1		! %l1 <<= ASI_SHIFT
16517c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
16527c478bdstevel@tonic-gate	rdpr	%pstate, %l1				! %l1 = %pstate
16537c478bdstevel@tonic-gate	sllx	%l1, TSTATE_PSTATE_SHIFT, %l1		! %l1 <<= PSTATE_SHIFT
16547c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
16557c478bdstevel@tonic-gate	rdpr	%cwp, %l1				! %l1 = %cwp
16567c478bdstevel@tonic-gate	sllx	%l1, TSTATE_CWP_SHIFT, %l1		! %l1 <<= CWP_SHIFT
16577c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
16587c478bdstevel@tonic-gate
16597c478bdstevel@tonic-gate	set	vpanic, %l1				! %l1 = %pc (vpanic)
16607c478bdstevel@tonic-gate	add	%l1, 4, %l2				! %l2 = %npc (vpanic+4)
16617c478bdstevel@tonic-gate	rd	%y, %l3					! %l3 = %y
16627c478bdstevel@tonic-gate	!
16637c478bdstevel@tonic-gate	! Flush register windows before panic_trigger() in order to avoid a
16647c478bdstevel@tonic-gate	! problem that a dump hangs if flush_windows() causes another panic.
16657c478bdstevel@tonic-gate	!
16667c478bdstevel@tonic-gate	call	flush_windows
16677c478bdstevel@tonic-gate	nop
16687c478bdstevel@tonic-gate
16697c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %o0
16707c478bdstevel@tonic-gate	call	panic_trigger
16717c478bdstevel@tonic-gate	or	%o0, %lo(panic_quiesce), %o0		! if (!panic_trigger(
16727c478bdstevel@tonic-gate
16737c478bdstevel@tonic-gatevpanic_common:
16747c478bdstevel@tonic-gate	tst	%o0					!     &panic_quiesce))
16757c478bdstevel@tonic-gate	be	0f					!   goto 0f;
16767c478bdstevel@tonic-gate	mov	%o0, %l4				!   delay - %l4 = %o0
16777c478bdstevel@tonic-gate
16787c478bdstevel@tonic-gate	!
16797c478bdstevel@tonic-gate	! If panic_trigger() was successful, we are the first to initiate a
16807c478bdstevel@tonic-gate	! panic: switch to the panic_stack.
16817c478bdstevel@tonic-gate	!
16827c478bdstevel@tonic-gate	set	panic_stack, %o0			! %o0 = panic_stack
16837c478bdstevel@tonic-gate	set	PANICSTKSIZE, %o1			! %o1 = size of stack
16847c478bdstevel@tonic-gate	add	%o0, %o1, %o0				! %o0 = top of stack
16857c478bdstevel@tonic-gate
16867c478bdstevel@tonic-gate	sub	%o0, SA(MINFRAME + REGSIZE) + STACK_BIAS, %sp
16877c478bdstevel@tonic-gate
16887c478bdstevel@tonic-gate	!
16897c478bdstevel@tonic-gate	! Now that we've got everything set up, store each register to its
16907c478bdstevel@tonic-gate	! designated location in the regs structure allocated on the stack.
16917c478bdstevel@tonic-gate	! The register set we store is the equivalent of the registers at
16927c478bdstevel@tonic-gate	! the time the %pc was pointing to vpanic, thus the %i's now contain
16937c478bdstevel@tonic-gate	! what the %o's contained prior to the save instruction.
16947c478bdstevel@tonic-gate	!
16957c478bdstevel@tonic-gate0:	stx	%l0, [%sp + STACK_BIAS + SA(MINFRAME) + TSTATE_OFF]
16967c478bdstevel@tonic-gate	stx	%g1, [%sp + STACK_BIAS + SA(MINFRAME) + G1_OFF]
16977c478bdstevel@tonic-gate	stx	%g2, [%sp + STACK_BIAS + SA(MINFRAME) + G2_OFF]
16987c478bdstevel@tonic-gate	stx	%g3, [%sp + STACK_BIAS + SA(MINFRAME) + G3_OFF]
16997c478bdstevel@tonic-gate	stx	%g4, [%sp + STACK_BIAS + SA(MINFRAME) + G4_OFF]
17007c478bdstevel@tonic-gate	stx	%g5, [%sp + STACK_BIAS + SA(MINFRAME) + G5_OFF]
17017c478bdstevel@tonic-gate	stx	%g6, [%sp + STACK_BIAS + SA(MINFRAME) + G6_OFF]
17027c478bdstevel@tonic-gate	stx	%g7, [%sp + STACK_BIAS + SA(MINFRAME) + G7_OFF]
17037c478bdstevel@tonic-gate	stx	%i0, [%sp + STACK_BIAS + SA(MINFRAME) + O0_OFF]
17047c478bdstevel@tonic-gate	stx	%i1, [%sp + STACK_BIAS + SA(MINFRAME) + O1_OFF]
17057c478bdstevel@tonic-gate	stx	%i2, [%sp + STACK_BIAS + SA(MINFRAME) + O2_OFF]
17067c478bdstevel@tonic-gate	stx	%i3, [%sp + STACK_BIAS + SA(MINFRAME) + O3_OFF]
17077c478bdstevel@tonic-gate	stx	%i4, [%sp + STACK_BIAS + SA(MINFRAME) + O4_OFF]
17087c478bdstevel@tonic-gate	stx	%i5, [%sp + STACK_BIAS + SA(MINFRAME) + O5_OFF]
17097c478bdstevel@tonic-gate	stx	%i6, [%sp + STACK_BIAS + SA(MINFRAME) + O6_OFF]
17107c478bdstevel@tonic-gate	stx	%i7, [%sp + STACK_BIAS + SA(MINFRAME) + O7_OFF]
17117c478bdstevel@tonic-gate	stn	%l1, [%sp + STACK_BIAS + SA(MINFRAME) + PC_OFF]
17127c478bdstevel@tonic-gate	stn	%l2, [%sp + STACK_BIAS + SA(MINFRAME) + NPC_OFF]
17137c478bdstevel@tonic-gate	st	%l3, [%sp + STACK_BIAS + SA(MINFRAME) + Y_OFF]
17147c478bdstevel@tonic-gate
17157c478bdstevel@tonic-gate	mov	%l4, %o3				! %o3 = on_panic_stack
17167c478bdstevel@tonic-gate	add	%sp, STACK_BIAS + SA(MINFRAME), %o2	! %o2 = &regs
17177c478bdstevel@tonic-gate	mov	%i1, %o1				! %o1 = alist
17187c478bdstevel@tonic-gate	call	panicsys				! panicsys();
17197c478bdstevel@tonic-gate	mov	%i0, %o0				! %o0 = format
17207c478bdstevel@tonic-gate	ret
17217c478bdstevel@tonic-gate	restore
17227c478bdstevel@tonic-gate
17237c478bdstevel@tonic-gate	SET_SIZE(vpanic)
17247c478bdstevel@tonic-gate
17257c478bdstevel@tonic-gate	ENTRY_NP(dtrace_vpanic)
17267c478bdstevel@tonic-gate
17277c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME + REGSIZE), %sp	! save and allocate regs
17287c478bdstevel@tonic-gate
17297c478bdstevel@tonic-gate	!
17307c478bdstevel@tonic-gate	! The v9 struct regs has a 64-bit r_tstate field, which we use here
17317c478bdstevel@tonic-gate	! to store the %ccr, %asi, %pstate, and %cwp as they would appear
17327c478bdstevel@tonic-gate	! in %tstate if a trap occurred.  We leave it up to the debugger to
17337c478bdstevel@tonic-gate	! realize what happened and extract the register values.
17347c478bdstevel@tonic-gate	!
17357c478bdstevel@tonic-gate	rd	%ccr, %l0				! %l0 = %ccr
17367c478bdstevel@tonic-gate	sllx	%l0, TSTATE_CCR_SHIFT, %l0		! %l0 <<= CCR_SHIFT
17377c478bdstevel@tonic-gate	rd	%asi, %l1				! %l1 = %asi
17387c478bdstevel@tonic-gate	sllx	%l1, TSTATE_ASI_SHIFT, %l1		! %l1 <<= ASI_SHIFT
17397c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
17407c478bdstevel@tonic-gate	rdpr	%pstate, %l1				! %l1 = %pstate
17417c478bdstevel@tonic-gate	sllx	%l1, TSTATE_PSTATE_SHIFT, %l1		! %l1 <<= PSTATE_SHIFT
17427c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
17437c478bdstevel@tonic-gate	rdpr	%cwp, %l1				! %l1 = %cwp
17447c478bdstevel@tonic-gate	sllx	%l1, TSTATE_CWP_SHIFT, %l1		! %l1 <<= CWP_SHIFT
17457c478bdstevel@tonic-gate	or	%l0, %l1, %l0				! %l0 |= %l1
17467c478bdstevel@tonic-gate
17477c478bdstevel@tonic-gate	set	dtrace_vpanic, %l1			! %l1 = %pc (vpanic)
17487c478bdstevel@tonic-gate	add	%l1, 4, %l2				! %l2 = %npc (vpanic+4)
17497c478bdstevel@tonic-gate	rd	%y, %l3					! %l3 = %y
17507c478bdstevel@tonic-gate	!
17517c478bdstevel@tonic-gate	! Flush register windows before panic_trigger() in order to avoid a
17527c478bdstevel@tonic-gate	! problem that a dump hangs if flush_windows() causes another panic.
17537c478bdstevel@tonic-gate	!
17547c478bdstevel@tonic-gate	call	dtrace_flush_windows
17557c478bdstevel@tonic-gate	nop
17567c478bdstevel@tonic-gate
17577c478bdstevel@tonic-gate	sethi	%hi(panic_quiesce), %o0
17587c478bdstevel@tonic-gate	call	dtrace_panic_trigger
17597c478bdstevel@tonic-gate	or	%o0, %lo(panic_quiesce), %o0		! if (!panic_trigger(
17607c478bdstevel@tonic-gate
17617c478bdstevel@tonic-gate	ba,a	vpanic_common
17627c478bdstevel@tonic-gate	SET_SIZE(dtrace_vpanic)
17637c478bdstevel@tonic-gate
17647c478bdstevel@tonic-gate#endif	/* lint */
1765bfb7f38kalai
1766bfb7f38kalai#if defined(lint)
1767bfb7f38kalai
1768bfb7f38kalai/*ARGSUSED*/
1769bfb7f38kalai
1770bfb7f38kalaiuint_t
1771bfb7f38kalaiget_subcc_ccr( uint64_t addrl, uint64_t addrr)
1772bfb7f38kalai{ return (0); }
1773bfb7f38kalai
1774bfb7f38kalai#else   /* lint */
1775bfb7f38kalai
1776bfb7f38kalai	ENTRY(get_subcc_ccr)
1777bfb7f38kalai	wr	%g0, %ccr	! clear condition codes
1778bfb7f38kalai	subcc	%o0, %o1, %g0
1779bfb7f38kalai	retl
1780bfb7f38kalai	rd	%ccr, %o0	! return condition codes
1781bfb7f38kalai	SET_SIZE(get_subcc_ccr)
1782bfb7f38kalai
1783bfb7f38kalai#endif  /* lint */
17844df4bd6bs
17854df4bd6bs#if defined(lint) || defined(__lint)
17864df4bd6bs
17874df4bd6bsftrace_icookie_t
17884df4bd6bsftrace_interrupt_disable(void)
17894df4bd6bs{ return (0); }
17904df4bd6bs
17914df4bd6bs#else	/* lint */
17924df4bd6bs
17934df4bd6bs	ENTRY_NP(ftrace_interrupt_disable)
17944df4bd6bs	rdpr	%pstate, %o0
17954df4bd6bs	andn	%o0, PSTATE_IE, %o1
17964df4bd6bs	retl
17974df4bd6bs	wrpr	%g0, %o1, %pstate
17984df4bd6bs	SET_SIZE(ftrace_interrupt_disable)
17994df4bd6bs
18004df4bd6bs#endif	/* lint */
18014df4bd6bs
18024df4bd6bs#if defined(lint) || defined(__lint)
18034df4bd6bs
18044df4bd6bs/*ARGSUSED*/
18054df4bd6bsvoid
18064df4bd6bsftrace_interrupt_enable(ftrace_icookie_t cookie)
18074df4bd6bs{}
18084df4bd6bs
18094df4bd6bs#else
18104df4bd6bs
18114df4bd6bs	ENTRY_NP(ftrace_interrupt_enable)
18124df4bd6bs	retl
18134df4bd6bs	wrpr	%g0, %o0, %pstate
18144df4bd6bs	SET_SIZE(ftrace_interrupt_enable)
18154df4bd6bs
18164df4bd6bs#endif /* lint*/
1817