xref: /illumos-gate/usr/src/uts/sun4u/cpu/us3_common_asm.S (revision 55fea89d)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2020 Joyent, Inc.
 *
 * Assembly code support for Cheetah/Cheetah+ modules
 */

#include "assym.h"

#include <sys/asm_linkage.h>
#include <sys/mmu.h>
#include <vm/hat_sfmmu.h>
#include <sys/machparam.h>
#include <sys/machcpuvar.h>
#include <sys/machthread.h>
#include <sys/machtrap.h>
#include <sys/privregs.h>
#include <sys/trap.h>
#include <sys/cheetahregs.h>
#include <sys/us3_module.h>
#include <sys/xc_impl.h>
#include <sys/intreg.h>
#include <sys/async.h>
#include <sys/clock.h>
#include <sys/cheetahasm.h>
#include <sys/cmpregs.h>

#ifdef TRAPTRACE
#include <sys/traptrace.h>
#endif /* TRAPTRACE */

/* BEGIN CSTYLED */

#define	DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3)			\
	ldxa	[%g0]ASI_DCU, tmp1					;\
	btst	DCU_DC, tmp1		/* is dcache enabled? */	;\
	bz,pn	%icc, 1f						;\
	ASM_LD(tmp1, dcache_linesize)					;\
	ASM_LD(tmp2, dflush_type)					;\
	cmp	tmp2, FLUSHPAGE_TYPE					;\
	be,pt	%icc, 2f						;\
	nop								;\
	sllx	arg1, CHEETAH_DC_VBIT_SHIFT, arg1/* tag to compare */	;\
	ASM_LD(tmp3, dcache_size)					;\
	cmp	tmp2, FLUSHMATCH_TYPE					;\
	be,pt	%icc, 3f						;\
	nop								;\
	/*								\
	 * flushtype = FLUSHALL_TYPE, flush the whole thing		\
	 * tmp3 = cache size						\
	 * tmp1 = cache line size					\
	 */								\
	sub	tmp3, tmp1, tmp2					;\
4:									\
	stxa	%g0, [tmp2]ASI_DC_TAG					;\
	membar	#Sync							;\
	cmp	%g0, tmp2						;\
	bne,pt	%icc, 4b						;\
	sub	tmp2, tmp1, tmp2					;\
	ba,pt	%icc, 1f						;\
	nop								;\
	/*								\
	 * flushtype = FLUSHPAGE_TYPE					\
	 * arg1 = pfn							\
	 * arg2 = virtual color						\
	 * tmp1 = cache line size					\
	 * tmp2 = tag from cache					\
	 * tmp3 = counter						\
	 */								\
2:									\
	set	MMU_PAGESIZE, tmp3					;\
        sllx    arg1, MMU_PAGESHIFT, arg1  /* pfn to 43 bit PA	   */   ;\
	sub	tmp3, tmp1, tmp3					;\
4:									\
	stxa	%g0, [arg1 + tmp3]ASI_DC_INVAL				;\
	membar	#Sync							;\
5:									\
	cmp	%g0, tmp3						;\
	bnz,pt	%icc, 4b		/* branch if not done */	;\
	sub	tmp3, tmp1, tmp3					;\
	ba,pt	%icc, 1f						;\
	nop								;\
	/*								\
	 * flushtype = FLUSHMATCH_TYPE					\
	 * arg1 = tag to compare against				\
	 * tmp1 = cache line size					\
	 * tmp3 = cache size						\
	 * arg2 = counter						\
	 * tmp2 = cache tag						\
	 */								\
3:									\
	sub	tmp3, tmp1, arg2					;\
4:									\
	ldxa	[arg2]ASI_DC_TAG, tmp2		/* read tag */		;\
	btst	CHEETAH_DC_VBIT_MASK, tmp2				;\
	bz,pn	%icc, 5f		/* br if no valid sub-blocks */	;\
	andn	tmp2, CHEETAH_DC_VBIT_MASK, tmp2 /* clear out v bits */	;\
	cmp	tmp2, arg1						;\
	bne,pn	%icc, 5f		/* branch if tag miss */	;\
	nop								;\
	stxa	%g0, [arg2]ASI_DC_TAG					;\
	membar	#Sync							;\
5:									\
	cmp	%g0, arg2						;\
	bne,pt	%icc, 4b		/* branch if not done */	;\
	sub	arg2, tmp1, arg2					;\
1:

/*
 * macro that flushes the entire dcache color
 * dcache size = 64K, one way 16K
 *
 * In:
 *    arg = virtual color register (not clobbered)
 *    way = way#, can either be a constant or a register (not clobbered)
 *    tmp1, tmp2, tmp3 = scratch registers
 *
 */
#define DCACHE_FLUSHCOLOR(arg, way, tmp1, tmp2, tmp3)			\
	ldxa	[%g0]ASI_DCU, tmp1;					\
	btst	DCU_DC, tmp1;		/* is dcache enabled? */	\
	bz,pn	%icc, 1f;						\
	ASM_LD(tmp1, dcache_linesize)					\
	/*								\
	 * arg = virtual color						\
	 * tmp1 = cache line size					\
	 */								\
	sllx	arg, MMU_PAGESHIFT, tmp2; /* color to dcache page */	\
	mov	way, tmp3;						\
	sllx	tmp3, 14, tmp3;		  /* One way 16K */		\
	or	tmp2, tmp3, tmp3;					\
	set	MMU_PAGESIZE, tmp2;					\
	/*								\
	 * tmp2 = page size						\
	 * tmp3 =  cached page in dcache				\
	 */								\
	sub	tmp2, tmp1, tmp2;					\
2:									\
	stxa	%g0, [tmp3 + tmp2]ASI_DC_TAG;				\
	membar	#Sync;							\
	cmp	%g0, tmp2;						\
	bne,pt	%icc, 2b;						\
	sub	tmp2, tmp1, tmp2;					\
1:

/* END CSTYLED */

/*
 * Cheetah MMU and Cache operations.
 */

	ENTRY_NP(vtag_flushpage)
	/*
	 * flush page from the tlb
	 *
	 * %o0 = vaddr
	 * %o1 = sfmmup
	 */
	rdpr	%pstate, %o5
#ifdef DEBUG
	PANIC_IF_INTR_DISABLED_PSTR(%o5, u3_di_label0, %g1)
#endif /* DEBUG */
	/*
	 * disable ints
	 */
	andn	%o5, PSTATE_IE, %o4
	wrpr	%o4, 0, %pstate

	/*
	 * Then, blow out the tlb
	 * Interrupts are disabled to prevent the primary ctx register
	 * from changing underneath us.
	 */
	sethi   %hi(ksfmmup), %o3
        ldx     [%o3 + %lo(ksfmmup)], %o3
        cmp     %o3, %o1
        bne,pt   %xcc, 1f			! if not kernel as, go to 1
	  sethi	%hi(FLUSH_ADDR), %o3
	/*
	 * For Kernel demaps use primary. type = page implicitly
	 */
	stxa	%g0, [%o0]ASI_DTLB_DEMAP	/* dmmu flush for KCONTEXT */
	stxa	%g0, [%o0]ASI_ITLB_DEMAP	/* immu flush for KCONTEXT */
	flush	%o3
	retl
	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1:
	/*
	 * User demap.  We need to set the primary context properly.
	 * Secondary context cannot be used for Cheetah IMMU.
	 * %o0 = vaddr
	 * %o1 = sfmmup
	 * %o3 = FLUSH_ADDR
	 */
	SFMMU_CPU_CNUM(%o1, %g1, %g2)		! %g1 = sfmmu cnum on this CPU

	ldub	[%o1 + SFMMU_CEXT], %o4		! %o4 = sfmmup->sfmmu_cext
	sll	%o4, CTXREG_EXT_SHIFT, %o4
	or	%g1, %o4, %g1			! %g1 = primary pgsz | cnum

	wrpr	%g0, 1, %tl
	set	MMU_PCONTEXT, %o4
	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %o0, %o0
	ldxa	[%o4]ASI_DMMU, %o2		! %o2 = save old ctxnum
	srlx	%o2, CTXREG_NEXT_SHIFT, %o1	! need to preserve nucleus pgsz
	sllx	%o1, CTXREG_NEXT_SHIFT, %o1	! %o1 = nucleus pgsz
	or	%g1, %o1, %g1			! %g1 = nucleus pgsz | primary pgsz | cnum
	stxa	%g1, [%o4]ASI_DMMU		! wr new ctxum

	stxa	%g0, [%o0]ASI_DTLB_DEMAP
	stxa	%g0, [%o0]ASI_ITLB_DEMAP
	stxa	%o2, [%o4]ASI_DMMU		/* restore old ctxnum */
	flush	%o3
	wrpr	%g0, 0, %tl

	retl
	wrpr	%g0, %o5, %pstate		/* enable interrupts */
	SET_SIZE(vtag_flushpage)

	ENTRY_NP2(vtag_flushall, demap_all)
	/*
	 * flush the tlb
	 */
	sethi	%hi(FLUSH_ADDR), %o3
	set	DEMAP_ALL_TYPE, %g1
	stxa	%g0, [%g1]ASI_DTLB_DEMAP
	stxa	%g0, [%g1]ASI_ITLB_DEMAP
	flush	%o3
	retl
	nop
	SET_SIZE(demap_all)
	SET_SIZE(vtag_flushall)


	ENTRY_NP(vtag_flushpage_tl1)
	/*
	 * x-trap to flush page from tlb and tsb
	 *
	 * %g1 = vaddr, zero-extended on 32-bit kernel
	 * %g2 = sfmmup
	 *
	 * assumes TSBE_TAG = 0
	 */
	srln	%g1, MMU_PAGESHIFT, %g1

	sethi   %hi(ksfmmup), %g3
        ldx     [%g3 + %lo(ksfmmup)], %g3
        cmp     %g3, %g2
        bne,pt	%xcc, 1f                        ! if not kernel as, go to 1
	  slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */

	/* We need to demap in the kernel context */
	or	DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1
	stxa	%g0, [%g1]ASI_DTLB_DEMAP
	stxa	%g0, [%g1]ASI_ITLB_DEMAP
	retry
1:
	/* We need to demap in a user context */
	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1

	SFMMU_CPU_CNUM(%g2, %g6, %g3)	! %g6 = sfmmu cnum on this CPU

	ldub	[%g2 + SFMMU_CEXT], %g4		! %g4 = sfmmup->cext
	sll	%g4, CTXREG_EXT_SHIFT, %g4
	or	%g6, %g4, %g6			! %g6 = pgsz | cnum

	set	MMU_PCONTEXT, %g4
	ldxa	[%g4]ASI_DMMU, %g5		/* rd old ctxnum */
	srlx	%g5, CTXREG_NEXT_SHIFT, %g2	/* %g2 = nucleus pgsz */
	sllx	%g2, CTXREG_NEXT_SHIFT, %g2	/* preserve nucleus pgsz */
	or	%g6, %g2, %g6			/* %g6 = nucleus pgsz | primary pgsz | cnum */
	stxa	%g6, [%g4]ASI_DMMU		/* wr new ctxum */
	stxa	%g0, [%g1]ASI_DTLB_DEMAP
	stxa	%g0, [%g1]ASI_ITLB_DEMAP
	stxa	%g5, [%g4]ASI_DMMU		/* restore old ctxnum */
	retry
	SET_SIZE(vtag_flushpage_tl1)


	ENTRY_NP(vtag_flush_pgcnt_tl1)
	/*
	 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb
	 *
	 * %g1 = vaddr, zero-extended on 32-bit kernel
	 * %g2 = <sfmmup58|pgcnt6>, (pgcnt - 1) is pass'ed in via pgcnt6 bits.
	 *
	 * NOTE: this handler relies on the fact that no
	 *	interrupts or traps can occur during the loop
	 *	issuing the TLB_DEMAP operations. It is assumed
	 *	that interrupts are disabled and this code is
	 *	fetching from the kernel locked text address.
	 *
	 * assumes TSBE_TAG = 0
	 */
	set	SFMMU_PGCNT_MASK, %g4
	and	%g4, %g2, %g3			/* g3 = pgcnt - 1 */
	add	%g3, 1, %g3			/* g3 = pgcnt */

	andn	%g2, SFMMU_PGCNT_MASK, %g2	/* g2 = sfmmup */
	srln	%g1, MMU_PAGESHIFT, %g1

	sethi   %hi(ksfmmup), %g4
        ldx     [%g4 + %lo(ksfmmup)], %g4
        cmp     %g4, %g2
        bne,pn   %xcc, 1f			/* if not kernel as, go to 1 */
	  slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */

	/* We need to demap in the kernel context */
	or	DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1
	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
	sethi   %hi(FLUSH_ADDR), %g5
4:
	stxa	%g0, [%g1]ASI_DTLB_DEMAP
	stxa	%g0, [%g1]ASI_ITLB_DEMAP
	flush	%g5				! flush required by immu

	deccc	%g3				/* decr pgcnt */
	bnz,pt	%icc,4b
	  add	%g1, %g2, %g1			/* next page */
	retry
1:
	/*
	 * We need to demap in a user context
	 *
	 * g2 = sfmmup
	 * g3 = pgcnt
	 */
	SFMMU_CPU_CNUM(%g2, %g5, %g6)		! %g5 = sfmmu cnum on this CPU

	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1

	ldub	[%g2 + SFMMU_CEXT], %g4		! %g4 = sfmmup->cext
	sll	%g4, CTXREG_EXT_SHIFT, %g4
	or	%g5, %g4, %g5

	set	MMU_PCONTEXT, %g4
	ldxa	[%g4]ASI_DMMU, %g6		/* rd old ctxnum */
	srlx	%g6, CTXREG_NEXT_SHIFT, %g2	/* %g2 = nucleus pgsz */
	sllx	%g2, CTXREG_NEXT_SHIFT, %g2	/* preserve nucleus pgsz */
	or	%g5, %g2, %g5			/* %g5 = nucleus pgsz | primary pgsz | cnum */
	stxa	%g5, [%g4]ASI_DMMU		/* wr new ctxum */

	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
	sethi   %hi(FLUSH_ADDR), %g5
3:
	stxa	%g0, [%g1]ASI_DTLB_DEMAP
	stxa	%g0, [%g1]ASI_ITLB_DEMAP
	flush	%g5				! flush required by immu

	deccc	%g3				/* decr pgcnt */
	bnz,pt	%icc,3b
	  add	%g1, %g2, %g1			/* next page */

	stxa	%g6, [%g4]ASI_DMMU		/* restore old ctxnum */
	retry
	SET_SIZE(vtag_flush_pgcnt_tl1)

	ENTRY_NP(vtag_flushall_tl1)
	/*
	 * x-trap to flush tlb
	 */
	set	DEMAP_ALL_TYPE, %g4
	stxa	%g0, [%g4]ASI_DTLB_DEMAP
	stxa	%g0, [%g4]ASI_ITLB_DEMAP
	retry
	SET_SIZE(vtag_flushall_tl1)


/*
 * vac_flushpage(pfnum, color)
 *	Flush 1 8k page of the D-$ with physical page = pfnum
 *	Algorithm:
 *		The cheetah dcache is a 64k psuedo 4 way accaociative cache.
 *		It is virtual indexed, physically tagged cache.
 */
	.seg	".data"
	.align	8
	.global	dflush_type
dflush_type:
	.word	FLUSHPAGE_TYPE

	ENTRY(vac_flushpage)
	/*
	 * flush page from the d$
	 *
	 * %o0 = pfnum, %o1 = color
	 */
	DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4)
	retl
	  nop
	SET_SIZE(vac_flushpage)


	ENTRY_NP(vac_flushpage_tl1)
	/*
	 * x-trap to flush page from the d$
	 *
	 * %g1 = pfnum, %g2 = color
	 */
	DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5)
	retry
	SET_SIZE(vac_flushpage_tl1)


	ENTRY(vac_flushcolor)
	/*
	 * %o0 = vcolor
	 */
	DCACHE_FLUSHCOLOR(%o0, 0, %o1, %o2, %o3)
	DCACHE_FLUSHCOLOR(%o0, 1, %o1, %o2, %o3)
	DCACHE_FLUSHCOLOR(%o0, 2, %o1, %o2, %o3)
	DCACHE_FLUSHCOLOR(%o0, 3, %o1, %o2, %o3)
	retl
	  nop
	SET_SIZE(vac_flushcolor)


	ENTRY(vac_flushcolor_tl1)
	/*
	 * %g1 = vcolor
	 */
	DCACHE_FLUSHCOLOR(%g1, 0, %g2, %g3, %g4)
	DCACHE_FLUSHCOLOR(%g1, 1, %g2, %g3, %g4)
	DCACHE_FLUSHCOLOR(%g1, 2, %g2, %g3, %g4)
	DCACHE_FLUSHCOLOR(%g1, 3, %g2, %g3, %g4)
	retry
	SET_SIZE(vac_flushcolor_tl1)

/*
 * Determine whether or not the IDSR is busy.
 * Entry: no arguments
 * Returns: 1 if busy, 0 otherwise
 */
	ENTRY(idsr_busy)
	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
	clr	%o0
	btst	IDSR_BUSY, %g1
	bz,a,pt	%xcc, 1f
	mov	1, %o0
1:
	retl
	nop
	SET_SIZE(idsr_busy)

	.global _dispatch_status_busy
_dispatch_status_busy:
	.asciz	"ASI_INTR_DISPATCH_STATUS error: busy"
	.align	4

/*
 * Setup interrupt dispatch data registers
 * Entry:
 *	%o0 - function or inumber to call
 *	%o1, %o2 - arguments (2 uint64_t's)
 */
	.seg "text"

	ENTRY(init_mondo)
#ifdef DEBUG
	!
	! IDSR should not be busy at the moment
	!
	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
	btst	IDSR_BUSY, %g1
	bz,pt	%xcc, 1f
	nop
	sethi	%hi(_dispatch_status_busy), %o0
	call	panic
	or	%o0, %lo(_dispatch_status_busy), %o0
#endif /* DEBUG */

	ALTENTRY(init_mondo_nocheck)
	!
	! interrupt vector dispatch data reg 0
	!
1:
	mov	IDDR_0, %g1
	mov	IDDR_1, %g2
	mov	IDDR_2, %g3
	stxa	%o0, [%g1]ASI_INTR_DISPATCH

	!
	! interrupt vector dispatch data reg 1
	!
	stxa	%o1, [%g2]ASI_INTR_DISPATCH

	!
	! interrupt vector dispatch data reg 2
	!
	stxa	%o2, [%g3]ASI_INTR_DISPATCH

	membar	#Sync
	retl
	nop
	SET_SIZE(init_mondo_nocheck)
	SET_SIZE(init_mondo)


#if !(defined(JALAPENO) || defined(SERRANO))

/*
 * Ship mondo to aid using busy/nack pair bn
 */
	ENTRY_NP(shipit)
	sll	%o0, IDCR_PID_SHIFT, %g1	! IDCR<18:14> = agent id
	sll	%o1, IDCR_BN_SHIFT, %g2		! IDCR<28:24> = b/n pair
	or	%g1, IDCR_OFFSET, %g1		! IDCR<13:0> = 0x70
	or	%g1, %g2, %g1
	stxa	%g0, [%g1]ASI_INTR_DISPATCH	! interrupt vector dispatch
	membar	#Sync
	retl
	nop
	SET_SIZE(shipit)

#endif	/* !(JALAPENO || SERRANO) */


/*
 * flush_instr_mem:
 *	Flush 1 page of the I-$ starting at vaddr
 * 	%o0 vaddr
 *	%o1 bytes to be flushed
 * UltraSPARC-III maintains consistency of the on-chip Instruction Cache with
 * the stores from all processors so that a FLUSH instruction is only needed
 * to ensure pipeline is consistent. This means a single flush is sufficient at
 * the end of a sequence of stores that updates the instruction stream to
 * ensure correct operation.
 */

	ENTRY(flush_instr_mem)
	flush	%o0			! address irrelevant
	retl
	nop
	SET_SIZE(flush_instr_mem)


#if defined(CPU_IMP_ECACHE_ASSOC)

	ENTRY(get_ecache_ctrl)
	GET_CPU_IMPL(%o0)
	cmp	%o0, JAGUAR_IMPL
	!
	! Putting an ASI access in the delay slot may
	! cause it to be accessed, even when annulled.
	!
	bne	1f
	  nop
	ldxa	[%g0]ASI_EC_CFG_TIMING, %o0	! read Jaguar shared E$ ctrl reg
	b	2f
	  nop
1:
	ldxa	[%g0]ASI_EC_CTRL, %o0		! read Ch/Ch+ E$ control reg
2:
	retl
	  nop
	SET_SIZE(get_ecache_ctrl)

#endif	/* CPU_IMP_ECACHE_ASSOC */


#if !(defined(JALAPENO) || defined(SERRANO))

/*
 * flush_ecache:
 *	%o0 - 64 bit physical address
 *	%o1 - ecache size
 *	%o2 - ecache linesize
 */

	ENTRY(flush_ecache)

	/*
	 * For certain CPU implementations, we have to flush the L2 cache
	 * before flushing the ecache.
	 */
	PN_L2_FLUSHALL(%g3, %g4, %g5)

	/*
	 * Flush the entire Ecache using displacement flush.
	 */
	ECACHE_FLUSHALL(%o1, %o2, %o0, %o4)

	retl
	nop
	SET_SIZE(flush_ecache)

#endif	/* !(JALAPENO || SERRANO) */


	ENTRY(flush_dcache)
	ASM_LD(%o0, dcache_size)
	ASM_LD(%o1, dcache_linesize)
	CH_DCACHE_FLUSHALL(%o0, %o1, %o2)
	retl
	nop
	SET_SIZE(flush_dcache)


	ENTRY(flush_icache)
	GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, flush_icache_1);
	ld	[%o0 + CHPR_ICACHE_LINESIZE], %o1
	ba,pt	%icc, 2f
	  ld	[%o0 + CHPR_ICACHE_SIZE], %o0
flush_icache_1:
	ASM_LD(%o0, icache_size)
	ASM_LD(%o1, icache_linesize)
2:
	CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4)
	retl
	nop
	SET_SIZE(flush_icache)

	ENTRY(kdi_flush_idcache)
	CH_DCACHE_FLUSHALL(%o0, %o1, %g1)
	CH_ICACHE_FLUSHALL(%o2, %o3, %g1, %g2)
	membar	#Sync
	retl
	nop
	SET_SIZE(kdi_flush_idcache)

	ENTRY(flush_pcache)
	PCACHE_FLUSHALL(%o0, %o1, %o2)
	retl
	nop
	SET_SIZE(flush_pcache)


#if defined(CPU_IMP_L1_CACHE_PARITY)

/*
 * Get dcache data and tag.  The Dcache data is a pointer to a ch_dc_data_t
 * structure (see cheetahregs.h):
 * The Dcache *should* be turned off when this code is executed.
 */
	.align	128
	ENTRY(get_dcache_dtag)
	rdpr	%pstate, %o5
	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
	wrpr	%g0, %o3, %pstate
	b	1f
	  stx	%o0, [%o1 + CH_DC_IDX]

	.align	128
1:
	ldxa	[%o0]ASI_DC_TAG, %o2
	stx	%o2, [%o1 + CH_DC_TAG]
	membar	#Sync
	ldxa	[%o0]ASI_DC_UTAG, %o2
	membar	#Sync
	stx	%o2, [%o1 + CH_DC_UTAG]
	ldxa	[%o0]ASI_DC_SNP_TAG, %o2
	stx	%o2, [%o1 + CH_DC_SNTAG]
	add	%o1, CH_DC_DATA, %o1
	clr	%o3
2:
	membar	#Sync				! required before ASI_DC_DATA
	ldxa	[%o0 + %o3]ASI_DC_DATA, %o2
	membar	#Sync				! required after ASI_DC_DATA
	stx	%o2, [%o1 + %o3]
	cmp	%o3, CH_DC_DATA_REG_SIZE - 8
	blt	2b
	  add	%o3, 8, %o3

	/*
	 * Unlike other CPUs in the family, D$ data parity bits for Panther
	 * do not reside in the microtag. Instead, we have to read them
	 * using the DC_data_parity bit of ASI_DCACHE_DATA. Also, instead
	 * of just having 8 parity bits to protect all 32 bytes of data
	 * per line, we now have 32 bits of parity.
	 */
	GET_CPU_IMPL(%o3)
	cmp	%o3, PANTHER_IMPL
	bne	4f
	  clr	%o3

	/*
	 * move our pointer to the next field where we store parity bits
	 * and add the offset of the last parity byte since we will be
	 * storing all 4 parity bytes within one 64 bit field like this:
	 *
	 * +------+------------+------------+------------+------------+
	 * |  -   | DC_parity  | DC_parity  | DC_parity  | DC_parity  |
	 * |  -   | for word 3 | for word 2 | for word 1 | for word 0 |
	 * +------+------------+------------+------------+------------+
	 *  63:32     31:24        23:16         15:8          7:0
	 */
	add	%o1, CH_DC_PN_DATA_PARITY - CH_DC_DATA + 7, %o1

	/* add the DC_data_parity bit into our working index */
	mov	1, %o2
	sll	%o2, PN_DC_DATA_PARITY_BIT_SHIFT, %o2
	or	%o0, %o2, %o0
3:
	membar	#Sync				! required before ASI_DC_DATA
	ldxa	[%o0 + %o3]ASI_DC_DATA, %o2
	membar	#Sync				! required after ASI_DC_DATA
	stb	%o2, [%o1]
	dec	%o1
	cmp	%o3, CH_DC_DATA_REG_SIZE - 8
	blt	3b
	  add	%o3, 8, %o3
4:
	retl
	  wrpr	%g0, %o5, %pstate
	SET_SIZE(get_dcache_dtag)


/*
 * Get icache data and tag.  The data argument is a pointer to a ch_ic_data_t
 * structure (see cheetahregs.h):
 * The Icache *Must* be turned off when this function is called.
 * This is because diagnostic accesses to the Icache interfere with cache
 * consistency.
 */
	.align	128
	ENTRY(get_icache_dtag)
	rdpr	%pstate, %o5
	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
	wrpr	%g0, %o3, %pstate

	stx	%o0, [%o1 + CH_IC_IDX]
	ldxa	[%o0]ASI_IC_TAG, %o2
	stx	%o2, [%o1 + CH_IC_PATAG]
	add	%o0, CH_ICTAG_UTAG, %o0
	ldxa	[%o0]ASI_IC_TAG, %o2
	add	%o0, (CH_ICTAG_UPPER - CH_ICTAG_UTAG), %o0
	stx	%o2, [%o1 + CH_IC_UTAG]
	ldxa	[%o0]ASI_IC_TAG, %o2
	add	%o0, (CH_ICTAG_LOWER - CH_ICTAG_UPPER), %o0
	stx	%o2, [%o1 + CH_IC_UPPER]
	ldxa	[%o0]ASI_IC_TAG, %o2
	andn	%o0, CH_ICTAG_TMASK, %o0
	stx	%o2, [%o1 + CH_IC_LOWER]
	ldxa	[%o0]ASI_IC_SNP_TAG, %o2
	stx	%o2, [%o1 + CH_IC_SNTAG]
	add	%o1, CH_IC_DATA, %o1
	clr	%o3
2:
	ldxa	[%o0 + %o3]ASI_IC_DATA, %o2
	stx	%o2, [%o1 + %o3]
	cmp	%o3, PN_IC_DATA_REG_SIZE - 8
	blt	2b
	  add	%o3, 8, %o3

	retl
	  wrpr	%g0, %o5, %pstate
	SET_SIZE(get_icache_dtag)

/*
 * Get pcache data and tags.
 * inputs:
 *   pcache_idx	- fully constructed VA for for accessing P$ diagnostic
 *		  registers. Contains PC_way and PC_addr shifted into
 *		  the correct bit positions. See the PRM for more details.
 *   data	- pointer to a ch_pc_data_t
 * structure (see cheetahregs.h):
 */
	.align	128
	ENTRY(get_pcache_dtag)
	rdpr	%pstate, %o5
	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
	wrpr	%g0, %o3, %pstate

	stx	%o0, [%o1 + CH_PC_IDX]
	ldxa	[%o0]ASI_PC_STATUS_DATA, %o2
	stx	%o2, [%o1 + CH_PC_STATUS]
	ldxa	[%o0]ASI_PC_TAG, %o2
	stx	%o2, [%o1 + CH_PC_TAG]
	ldxa	[%o0]ASI_PC_SNP_TAG, %o2
	stx	%o2, [%o1 + CH_PC_SNTAG]
	add	%o1, CH_PC_DATA, %o1
	clr	%o3
2:
	ldxa	[%o0 + %o3]ASI_PC_DATA, %o2
	stx	%o2, [%o1 + %o3]
	cmp	%o3, CH_PC_DATA_REG_SIZE - 8
	blt	2b
	  add	%o3, 8, %o3

	retl
	  wrpr	%g0, %o5, %pstate
	SET_SIZE(get_pcache_dtag)

#endif	/* CPU_IMP_L1_CACHE_PARITY */

/*
 * re-enable the i$, d$, w$, and p$ according to bootup cache state.
 * Turn on WE, HPE, SPE, PE, IC, and DC bits defined as DCU_CACHE.
 *   %o0 - 64 bit constant
 */
	ENTRY(set_dcu)
	stxa	%o0, [%g0]ASI_DCU	! Store to DCU
	flush	%g0	/* flush required after changing the IC bit */
	retl
	nop
	SET_SIZE(set_dcu)


/*
 * Return DCU register.
 */
	ENTRY(get_dcu)
	ldxa	[%g0]ASI_DCU, %o0		/* DCU control register */
	retl
	nop
	SET_SIZE(get_dcu)

/*
 * Cheetah/Cheetah+ level 15 interrupt handler trap table entry.
 *
 * This handler is used to check for softints generated by error trap
 * handlers to report errors.  On Cheetah, this mechanism is used by the
 * Fast ECC at TL>0 error trap handler and, on Cheetah+, by both the Fast
 * ECC at TL>0 error and the I$/D$ parity error at TL>0 trap handlers.
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */

	ENTRY_NP(ch_pil15_interrupt_instr)
	ASM_JMP(%g1, ch_pil15_interrupt)
	SET_SIZE(ch_pil15_interrupt_instr)


	ENTRY_NP(ch_pil15_interrupt)

	/*
	 * Since pil_interrupt is hacked to assume that every level 15
	 * interrupt is generated by the CPU to indicate a performance
	 * counter overflow this gets ugly.  Before calling pil_interrupt
	 * the Error at TL>0 pending status is inspected.  If it is
	 * non-zero, then an error has occurred and it is handled.
	 * Otherwise control is transfered to pil_interrupt.  Note that if
	 * an error is detected pil_interrupt will not be called and
	 * overflow interrupts may be lost causing erroneous performance
	 * measurements.  However, error-recovery will have a detrimental
	 * effect on performance anyway.
	 */
	CPU_INDEX(%g1, %g4)
	set	ch_err_tl1_pending, %g4
	ldub	[%g1 + %g4], %g2
	brz	%g2, 1f
	  nop

	/*
	 * We have a pending TL>0 error, clear the TL>0 pending status.
	 */
	stb	%g0, [%g1 + %g4]

	/*
	 * Clear the softint.
	 */
	mov	1, %g5
	sll	%g5, PIL_15, %g5
	wr	%g5, CLEAR_SOFTINT

	/*
	 * For Cheetah*, call cpu_tl1_error via systrap at PIL 15
	 * to process the Fast ECC/Cache Parity at TL>0 error.  Clear
	 * panic flag (%g2).
	 */
	set	cpu_tl1_error, %g1
	clr	%g2
	ba	sys_trap
	  mov	PIL_15, %g4

1:
	/*
	 * The logout is invalid.
	 *
	 * Call the default interrupt handler.
	 */
	sethi	%hi(pil_interrupt), %g1
	jmp	%g1 + %lo(pil_interrupt)
	  mov	PIL_15, %g4

	SET_SIZE(ch_pil15_interrupt)


/*
 * Error Handling
 *
 * Cheetah provides error checking for all memory access paths between
 * the CPU, External Cache, Cheetah Data Switch and system bus. Error
 * information is logged in the AFSR, (also AFSR_EXT for Panther) and
 * AFAR and one of the following traps is generated (provided that it
 * is enabled in External Cache Error Enable Register) to handle that
 * error:
 * 1. trap 0x70: Precise trap
 *    tt0_fecc for errors at trap level(TL)>=0
 * 2. trap 0x0A and 0x32: Deferred trap
 *    async_err for errors at TL>=0
 * 3. trap 0x63: Disrupting trap
 *    ce_err for errors at TL=0
 *    (Note that trap 0x63 cannot happen at trap level > 0)
 *
 * Trap level one handlers panic the system except for the fast ecc
 * error handler which tries to recover from certain errors.
 */

/*
 * FAST ECC TRAP STRATEGY:
 *
 * Software must handle single and multi bit errors which occur due to data
 * or instruction cache reads from the external cache. A single or multi bit
 * error occuring in one of these situations results in a precise trap.
 *
 * The basic flow of this trap handler is as follows:
 *
 * 1) Record the state and then turn off the Dcache and Icache.  The Dcache
 *    is disabled because bad data could have been installed.  The Icache is
 *    turned off because we want to capture the Icache line related to the
 *    AFAR.
 * 2) Disable trapping on CEEN/NCCEN errors during TL=0 processing.
 * 3) Park sibling core if caches are shared (to avoid race condition while
 *    accessing shared resources such as L3 data staging register during
 *    CPU logout.
 * 4) Read the AFAR and AFSR.
 * 5) If CPU logout structure is not being used, then:
 *    6) Clear all errors from the AFSR.
 *    7) Capture Ecache, Dcache and Icache lines in "CPU log out" structure.
 *    8) Flush Ecache then Flush Dcache and Icache and restore to previous
 *       state.
 *    9) Unpark sibling core if we parked it earlier.
 *    10) call cpu_fast_ecc_error via systrap at PIL 14 unless we're already
 *        running at PIL 15.
 * 6) Otherwise, if CPU logout structure is being used:
 *    7) Incriment the "logout busy count".
 *    8) Flush Ecache then Flush Dcache and Icache and restore to previous
 *       state.
 *    9) Unpark sibling core if we parked it earlier.
 *    10) Issue a retry since the other CPU error logging code will end up
 *       finding this error bit and logging information about it later.
 * 7) Alternatively (to 5 and 6 above), if the cpu_private struct is not
 *    yet initialized such that we can't even check the logout struct, then
 *    we place the clo_flags data into %g2 (sys_trap->have_win arg #1) and
 *    call cpu_fast_ecc_error via systrap. The clo_flags parameter is used
 *    to determine information such as TL, TT, CEEN and NCEEN settings, etc
 *    in the high level trap handler since we don't have access to detailed
 *    logout information in cases where the cpu_private struct is not yet
 *    initialized.
 *
 * We flush the E$ and D$ here on TL=1 code to prevent getting nested
 * Fast ECC traps in the TL=0 code.  If we get a Fast ECC event here in
 * the TL=1 code, we will go to the Fast ECC at TL>0 handler which,
 * since it is uses different code/data from this handler, has a better
 * chance of fixing things up than simply recursing through this code
 * again (this would probably cause an eventual kernel stack overflow).
 * If the Fast ECC at TL>0 handler encounters a Fast ECC error before it
 * can flush the E$ (or the error is a stuck-at bit), we will recurse in
 * the Fast ECC at TL>0 handler and eventually Red Mode.
 *
 * Note that for Cheetah (and only Cheetah), we use alias addresses for
 * flushing rather than ASI accesses (which don't exist on Cheetah).
 * Should we encounter a Fast ECC error within this handler on Cheetah,
 * there's a good chance it's within the ecache_flushaddr buffer (since
 * it's the largest piece of memory we touch in the handler and it is
 * usually kernel text/data).  For that reason the Fast ECC at TL>0
 * handler for Cheetah uses an alternate buffer: ecache_tl1_flushaddr.
 */

/*
 * Cheetah ecc-protected E$ trap (Trap 70) at TL=0
 * tt0_fecc is replaced by fecc_err_instr in cpu_init_trap of the various
 * architecture-specific files.
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */

	ENTRY_NP(fecc_err_instr)
	membar	#Sync			! Cheetah requires membar #Sync

	/*
	 * Save current DCU state.  Turn off the Dcache and Icache.
	 */
	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
	andn	%g1, DCU_DC + DCU_IC, %g4
	stxa	%g4, [%g0]ASI_DCU
	flush	%g0	/* flush required after changing the IC bit */

	ASM_JMP(%g4, fast_ecc_err)
	SET_SIZE(fecc_err_instr)


#if !(defined(JALAPENO) || defined(SERRANO))

	.section ".text"
	.align	64
	ENTRY_NP(fast_ecc_err)

	/*
	 * Turn off CEEN and NCEEN.
	 */
	ldxa	[%g0]ASI_ESTATE_ERR, %g3
	andn	%g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
	stxa	%g4, [%g0]ASI_ESTATE_ERR
	membar	#Sync			! membar sync required

	/*
	 * Check to see whether we need to park our sibling core
	 * before recording diagnostic information from caches
	 * which may be shared by both cores.
	 * We use %g1 to store information about whether or not
	 * we had to park the core (%g1 holds our DCUCR value and
	 * we only use bits from that register which are "reserved"
	 * to keep track of core parking) so that we know whether
	 * or not to unpark later. %g5 and %g4 are scratch registers.
	 */
	PARK_SIBLING_CORE(%g1, %g5, %g4)

	/*
	 * Do the CPU log out capture.
	 *   %g3 = "failed?" return value.
	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
	 *         into this macro via %g4. Output only valid if cpu_private
	 *         struct has not been initialized.
	 *   CHPR_FECCTL0_LOGOUT = cpu logout structure offset input
	 *   %g4 = Trap information stored in the cpu logout flags field
	 *   %g5 = scr1
	 *   %g6 = scr2
	 *   %g3 = scr3
	 *   %g4 = scr4
	 */
	 /* store the CEEN and NCEEN values, TL=0 */
	and	%g3, EN_REG_CEEN + EN_REG_NCEEN, %g4
	set	CHPR_FECCTL0_LOGOUT, %g6
	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)

	/*
	 * Flush the Ecache (and L2 cache for Panther) to get the error out
	 * of the Ecache.  If the UCC or UCU is on a dirty line, then the
	 * following flush will turn that into a WDC or WDU, respectively.
	 */
	PN_L2_FLUSHALL(%g4, %g5, %g6)

	CPU_INDEX(%g4, %g5)
	mulx	%g4, CPU_NODE_SIZE, %g4
	set	cpunodes, %g5
	add	%g4, %g5, %g4
	ld	[%g4 + ECACHE_LINESIZE], %g5
	ld	[%g4 + ECACHE_SIZE], %g4

	ASM_LDX(%g6, ecache_flushaddr)
	ECACHE_FLUSHALL(%g4, %g5, %g6, %g7)

	/*
	 * Flush the Dcache.  Since bad data could have been installed in
	 * the Dcache we must flush it before re-enabling it.
	 */
	ASM_LD(%g5, dcache_size)
	ASM_LD(%g6, dcache_linesize)
	CH_DCACHE_FLUSHALL(%g5, %g6, %g7)

	/*
	 * Flush the Icache.  Since we turned off the Icache to capture the
	 * Icache line it is now stale or corrupted and we must flush it
	 * before re-enabling it.
	 */
	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_5);
	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
	ba,pt	%icc, 6f
	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
fast_ecc_err_5:
	ASM_LD(%g5, icache_size)
	ASM_LD(%g6, icache_linesize)
6:
	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)

	/*
	 * check to see whether we parked our sibling core at the start
	 * of this handler. If so, we need to unpark it here.
	 * We use DCUCR reserved bits (stored in %g1) to keep track of
	 * whether or not we need to unpark. %g5 and %g4 are scratch registers.
	 */
	UNPARK_SIBLING_CORE(%g1, %g5, %g4)

	/*
	 * Restore the Dcache and Icache to the previous state.
	 */
	stxa	%g1, [%g0]ASI_DCU
	flush	%g0	/* flush required after changing the IC bit */

	/*
	 * Make sure our CPU logout operation was successful.
	 */
	cmp	%g3, %g0
	be	8f
	  nop

	/*
	 * If the logout structure had been busy, how many times have
	 * we tried to use it and failed (nesting count)? If we have
	 * already recursed a substantial number of times, then we can
	 * assume things are not going to get better by themselves and
	 * so it would be best to panic.
	 */
	cmp	%g3, CLO_NESTING_MAX
	blt	7f
	  nop

        call ptl1_panic
          mov   PTL1_BAD_ECC, %g1

7:
	/*
	 * Otherwise, if the logout structure was busy but we have not
	 * nested more times than our maximum value, then we simply
	 * issue a retry. Our TL=0 trap handler code will check and
	 * clear the AFSR after it is done logging what is currently
	 * in the logout struct and handle this event at that time.
	 */
	retry
8:
	/*
	 * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're
	 * already at PIL 15.
	 */
	set	cpu_fast_ecc_error, %g1
	rdpr	%pil, %g4
	cmp	%g4, PIL_14
	ba	sys_trap
	  movl	%icc, PIL_14, %g4

	SET_SIZE(fast_ecc_err)

#endif	/* !(JALAPENO || SERRANO) */


/*
 * Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy:
 *
 * The basic flow of this trap handler is as follows:
 *
 * 1) In the "trap 70" trap table code (fecc_err_tl1_instr), generate a
 *    software trap 0 ("ta 0") to buy an extra set of %tpc, etc. which we
 *    will use to save %g1 and %g2.
 * 2) At the software trap 0 at TL>0 trap table code (fecc_err_tl1_cont_instr),
 *    we save %g1+%g2 using %tpc, %tnpc + %tstate and jump to the fast ecc
 *    handler (using the just saved %g1).
 * 3) Turn off the Dcache if it was on and save the state of the Dcache
 *    (whether on or off) in Bit2 (CH_ERR_TSTATE_DC_ON) of %tstate.
 *    NB: we don't turn off the Icache because bad data is not installed nor
 *        will we be doing any diagnostic accesses.
 * 4) compute physical address of the per-cpu/per-tl save area using %g1+%g2
 * 5) Save %g1-%g7 into the per-cpu/per-tl save area (%g1 + %g2 from the
 *    %tpc, %tnpc, %tstate values previously saved).
 * 6) set %tl to %tl - 1.
 * 7) Save the appropriate flags and TPC in the ch_err_tl1_data structure.
 * 8) Save the value of CH_ERR_TSTATE_DC_ON in the ch_err_tl1_tmp field.
 * 9) For Cheetah and Jalapeno, read the AFAR and AFSR and clear.  For
 *    Cheetah+ (and later), read the shadow AFAR and AFSR but don't clear.
 *    Save the values in ch_err_tl1_data.  For Panther, read the shadow
 *    AFSR_EXT and save the value in ch_err_tl1_data.
 * 10) Disable CEEN/NCEEN to prevent any disrupting/deferred errors from
 *    being queued.  We'll report them via the AFSR/AFAR capture in step 13.
 * 11) Flush the Ecache.
 *    NB: the Ecache is flushed assuming the largest possible size with
 *        the smallest possible line size since access to the cpu_nodes may
 *        cause an unrecoverable DTLB miss.
 * 12) Reenable CEEN/NCEEN with the value saved from step 10.
 * 13) For Cheetah and Jalapeno, read the AFAR and AFSR and clear again.
 *    For Cheetah+ (and later), read the primary AFAR and AFSR and now clear.
 *    Save the read AFSR/AFAR values in ch_err_tl1_data.  For Panther,
 *    read and clear the primary AFSR_EXT and save it in ch_err_tl1_data.
 * 14) Flush and re-enable the Dcache if it was on at step 3.
 * 15) Do TRAPTRACE if enabled.
 * 16) Check if a UCU->WDU (or L3_UCU->WDU for Panther) happened, panic if so.
 * 17) Set the event pending flag in ch_err_tl1_pending[CPU]
 * 18) Cause a softint 15.  The pil15_interrupt handler will inspect the
 *    event pending flag and call cpu_tl1_error via systrap if set.
 * 19) Restore the registers from step 5 and issue retry.
 */

/*
 * Cheetah ecc-protected E$ trap (Trap 70) at TL>0
 * tt1_fecc is replaced by fecc_err_tl1_instr in cpu_init_trap of the various
 * architecture-specific files.  This generates a "Software Trap 0" at TL>0,
 * which goes to fecc_err_tl1_cont_instr, and we continue the handling there.
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */

	ENTRY_NP(fecc_err_tl1_instr)
	CH_ERR_TL1_TRAPENTRY(SWTRAP_0);
	SET_SIZE(fecc_err_tl1_instr)

/*
 * Software trap 0 at TL>0.
 * tt1_swtrap0 is replaced by fecc_err_tl1_cont_instr in cpu_init_trap of
 * the various architecture-specific files.  This is used as a continuation
 * of the fast ecc handling where we've bought an extra TL level, so we can
 * use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
 * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
 * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
 * order two bits from %g1 and %g2 respectively).
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */

	ENTRY_NP(fecc_err_tl1_cont_instr)
	CH_ERR_TL1_SWTRAPENTRY(fast_ecc_tl1_err)
	SET_SIZE(fecc_err_tl1_cont_instr)


/*
 * The ce_err function handles disrupting trap type 0x63 at TL=0.
 *
 * AFSR errors bits which cause this trap are:
 *	CE, EMC, EDU:ST, EDC, WDU, WDC, CPU, CPC, IVU, IVC
 *
 * NCEEN Bit of Cheetah External Cache Error Enable Register enables
 * the following AFSR disrupting traps: EDU:ST, WDU, CPU, IVU
 *
 * CEEN Bit of Cheetah External Cache Error Enable Register enables
 * the following AFSR disrupting traps: CE, EMC, EDC, WDC, CPC, IVC
 *
 * Cheetah+ also handles (No additional processing required):
 *    DUE, DTO, DBERR	(NCEEN controlled)
 *    THCE		(CEEN and ET_ECC_en controlled)
 *    TUE		(ET_ECC_en controlled)
 *
 * Panther further adds:
 *    IMU, L3_EDU, L3_WDU, L3_CPU		(NCEEN controlled)
 *    IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE	(CEEN controlled)
 *    TUE_SH, TUE		(NCEEN and L2_tag_ECC_en controlled)
 *    L3_TUE, L3_TUE_SH		(NCEEN and ET_ECC_en controlled)
 *    THCE			(CEEN and L2_tag_ECC_en controlled)
 *    L3_THCE			(CEEN and ET_ECC_en controlled)
 *
 * Steps:
 *	1. Disable hardware corrected disrupting errors only (CEEN)
 *	2. Park sibling core if caches are shared (to avoid race
 *	   condition while accessing shared resources such as L3
 *	   data staging register during CPU logout.
 *	3. If the CPU logout structure is not currently being used:
 *		4. Clear AFSR error bits
 *		5. Capture Ecache, Dcache and Icache lines associated
 *		   with AFAR.
 *		6. Unpark sibling core if we parked it earlier.
 *		7. call cpu_disrupting_error via sys_trap at PIL 14
 *		   unless we're already running at PIL 15.
 *	4. Otherwise, if the CPU logout structure is busy:
 *		5. Incriment "logout busy count" and place into %g3
 *		6. Unpark sibling core if we parked it earlier.
 *		7. Issue a retry since the other CPU error logging
 *		   code will end up finding this error bit and logging
 *		   information about it later.
 *	5. Alternatively (to 3 and 4 above), if the cpu_private struct is
 *         not yet initialized such that we can't even check the logout
 *         struct, then we place the clo_flags data into %g2
 *         (sys_trap->have_win arg #1) and call cpu_disrupting_error via
 *         systrap. The clo_flags parameter is used to determine information
 *         such as TL, TT, CEEN settings, etc in the high level trap
 *         handler since we don't have access to detailed logout information
 *         in cases where the cpu_private struct is not yet initialized.
 *
 * %g3: [ logout busy count ] - arg #2
 * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1
 */

	.align	128
	ENTRY_NP(ce_err)
	membar	#Sync			! Cheetah requires membar #Sync

	/*
	 * Disable trap on hardware corrected errors (CEEN) while at TL=0
	 * to prevent recursion.
	 */
	ldxa	[%g0]ASI_ESTATE_ERR, %g1
	bclr	EN_REG_CEEN, %g1
	stxa	%g1, [%g0]ASI_ESTATE_ERR
	membar	#Sync			! membar sync required

	/*
	 * Save current DCU state.  Turn off Icache to allow capture of
	 * Icache data by DO_CPU_LOGOUT.
	 */
	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
	andn	%g1, DCU_IC, %g4
	stxa	%g4, [%g0]ASI_DCU
	flush	%g0	/* flush required after changing the IC bit */

	/*
	 * Check to see whether we need to park our sibling core
	 * before recording diagnostic information from caches
	 * which may be shared by both cores.
	 * We use %g1 to store information about whether or not
	 * we had to park the core (%g1 holds our DCUCR value and
	 * we only use bits from that register which are "reserved"
	 * to keep track of core parking) so that we know whether
	 * or not to unpark later. %g5 and %g4 are scratch registers.
	 */
	PARK_SIBLING_CORE(%g1, %g5, %g4)

	/*
	 * Do the CPU log out capture.
	 *   %g3 = "failed?" return value.
	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
	 *         into this macro via %g4. Output only valid if cpu_private
	 *         struct has not been initialized.
	 *   CHPR_CECC_LOGOUT = cpu logout structure offset input
	 *   %g4 = Trap information stored in the cpu logout flags field
	 *   %g5 = scr1
	 *   %g6 = scr2
	 *   %g3 = scr3
	 *   %g4 = scr4
	 */
	clr	%g4			! TL=0 bit in afsr
	set	CHPR_CECC_LOGOUT, %g6
	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)

	/*
	 * Flush the Icache.  Since we turned off the Icache to capture the
	 * Icache line it is now stale or corrupted and we must flush it
	 * before re-enabling it.
	 */
	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, ce_err_1);
	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
	ba,pt	%icc, 2f
	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
ce_err_1:
	ASM_LD(%g5, icache_size)
	ASM_LD(%g6, icache_linesize)
2:
	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)

	/*
	 * check to see whether we parked our sibling core at the start
	 * of this handler. If so, we need to unpark it here.
	 * We use DCUCR reserved bits (stored in %g1) to keep track of
	 * whether or not we need to unpark. %g5 and %g4 are scratch registers.
	 */
	UNPARK_SIBLING_CORE(%g1, %g5, %g4)

	/*
	 * Restore Icache to previous state.
	 */
	stxa	%g1, [%g0]ASI_DCU
	flush	%g0	/* flush required after changing the IC bit */

	/*
	 * Make sure our CPU logout operation was successful.
	 */
	cmp	%g3, %g0
	be	4f
	  nop

	/*
	 * If the logout structure had been busy, how many times have
	 * we tried to use it and failed (nesting count)? If we have
	 * already recursed a substantial number of times, then we can
	 * assume things are not going to get better by themselves and
	 * so it would be best to panic.
	 */
	cmp	%g3, CLO_NESTING_MAX
	blt	3f
	  nop

        call ptl1_panic
          mov   PTL1_BAD_ECC, %g1

3:
	/*
	 * Otherwise, if the logout structure was busy but we have not
	 * nested more times than our maximum value, then we simply
	 * issue a retry. Our TL=0 trap handler code will check and
	 * clear the AFSR after it is done logging what is currently
	 * in the logout struct and handle this event at that time.
	 */
	retry
4:
	/*
	 * Call cpu_disrupting_error via systrap at PIL 14 unless we're
	 * already at PIL 15.
	 */
	set	cpu_disrupting_error, %g1
	rdpr	%pil, %g4
	cmp	%g4, PIL_14
	ba	sys_trap
	  movl	%icc, PIL_14, %g4
	SET_SIZE(ce_err)


/*
 * This trap cannot happen at TL>0 which means this routine will never
 * actually be called and so we treat this like a BAD TRAP panic.
 */
	.align	64
	ENTRY_NP(ce_err_tl1)

        call ptl1_panic
          mov   PTL1_BAD_TRAP, %g1

	SET_SIZE(ce_err_tl1)


/*
 * The async_err function handles deferred trap types 0xA
 * (instruction_access_error) and 0x32 (data_access_error) at TL>=0.
 *
 * AFSR errors bits which cause this trap are:
 *	UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR
 * On some platforms, EMU may causes cheetah to pull the error pin
 * never giving Solaris a chance to take a trap.
 *
 * NCEEN Bit of Cheetah External Cache Error Enable Register enables
 * the following AFSR deferred traps: UE, EMU, EDU:BLD, TO, BERR
 *
 * Steps:
 *	1. Disable CEEN and NCEEN errors to prevent recursive errors.
 *	2. Turn D$ off per Cheetah PRM P.5 Note 6, turn I$ off to capture
 *         I$ line in DO_CPU_LOGOUT.
 *	3. Park sibling core if caches are shared (to avoid race
 *	   condition while accessing shared resources such as L3
 *	   data staging register during CPU logout.
 *	4. If the CPU logout structure is not currently being used:
 *		5. Clear AFSR error bits
 *		6. Capture Ecache, Dcache and Icache lines associated
 *		   with AFAR.
 *		7. Unpark sibling core if we parked it earlier.
 *		8. call cpu_deferred_error via sys_trap.
 *	5. Otherwise, if the CPU logout structure is busy:
 *		6. Incriment "logout busy count"
 *		7. Unpark sibling core if we parked it earlier.
 *		8) Issue a retry since the other CPU error logging
 *		   code will end up finding this error bit and logging
 *		   information about it later.
 *      6. Alternatively (to 4 and 5 above), if the cpu_private struct is
 *         not yet initialized such that we can't even check the logout
 *         struct, then we place the clo_flags data into %g2
 *         (sys_trap->have_win arg #1) and call cpu_deferred_error via
 *         systrap. The clo_flags parameter is used to determine information
 *         such as TL, TT, CEEN settings, etc in the high level trap handler
 *         since we don't have access to detailed logout information in cases
 *         where the cpu_private struct is not yet initialized.
 *
 * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1
 * %g3: [ logout busy count ] - arg #2
 */

	ENTRY_NP(async_err)
	membar	#Sync			! Cheetah requires membar #Sync

	/*
	 * Disable CEEN and NCEEN.
	 */
	ldxa	[%g0]ASI_ESTATE_ERR, %g3
	andn	%g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
	stxa	%g4, [%g0]ASI_ESTATE_ERR
	membar	#Sync			! membar sync required

	/*
	 * Save current DCU state.
	 * Disable Icache to allow capture of Icache data by DO_CPU_LOGOUT.
	 * Do this regardless of whether this is a Data Access Error or
	 * Instruction Access Error Trap.
	 * Disable Dcache for both Data Access Error and Instruction Access
	 * Error per Cheetah PRM P.5 Note 6.
	 */
	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
	andn	%g1, DCU_IC + DCU_DC, %g4
	stxa	%g4, [%g0]ASI_DCU
	flush	%g0	/* flush required after changing the IC bit */

	/*
	 * Check to see whether we need to park our sibling core
	 * before recording diagnostic information from caches
	 * which may be shared by both cores.
	 * We use %g1 to store information about whether or not
	 * we had to park the core (%g1 holds our DCUCR value and
	 * we only use bits from that register which are "reserved"
	 * to keep track of core parking) so that we know whether
	 * or not to unpark later. %g6 and %g4 are scratch registers.
	 */
	PARK_SIBLING_CORE(%g1, %g6, %g4)

	/*
	 * Do the CPU logout capture.
	 *
	 *   %g3 = "failed?" return value.
	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
	 *         into this macro via %g4. Output only valid if cpu_private
	 *         struct has not been initialized.
	 *   CHPR_ASYNC_LOGOUT = cpu logout structure offset input
	 *   %g4 = Trap information stored in the cpu logout flags field
	 *   %g5 = scr1
	 *   %g6 = scr2
	 *   %g3 = scr3
	 *   %g4 = scr4
	 */
	andcc	%g5, T_TL1, %g0
	clr	%g6
	movnz	%xcc, 1, %g6			! set %g6 if T_TL1 set
	sllx	%g6, CLO_FLAGS_TL_SHIFT, %g6
	sllx	%g5, CLO_FLAGS_TT_SHIFT, %g4
	set	CLO_FLAGS_TT_MASK, %g2
	and	%g4, %g2, %g4			! ttype
	or	%g6, %g4, %g4			! TT and TL
	and	%g3, EN_REG_CEEN, %g3		! CEEN value
	or	%g3, %g4, %g4			! TT and TL and CEEN
	set	CHPR_ASYNC_LOGOUT, %g6
	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)

	/*
	 * If the logout struct was busy, we may need to pass the
	 * TT, TL, and CEEN information to the TL=0 handler via
	 * systrap parameter so save it off here.
	 */
	cmp	%g3, %g0
	be	1f
	  nop
	sllx	%g4, 32, %g4
	or	%g4, %g3, %g3
1:
	/*
	 * Flush the Icache.  Since we turned off the Icache to capture the
	 * Icache line it is now stale or corrupted and we must flush it
	 * before re-enabling it.
	 */
	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, async_err_1);
	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
	ba,pt	%icc, 2f
	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
async_err_1:
	ASM_LD(%g5, icache_size)
	ASM_LD(%g6, icache_linesize)
2:
	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)

	/*
	 * XXX - Don't we need to flush the Dcache before turning it back
	 *       on to avoid stale or corrupt data? Was this broken?
	 */
	/*
	 * Flush the Dcache before turning it back on since it may now
	 * contain stale or corrupt data.
	 */
	ASM_LD(%g5, dcache_size)
	ASM_LD(%g6, dcache_linesize)
	CH_DCACHE_FLUSHALL(%g5, %g6, %g7)

	/*
	 * check to see whether we parked our sibling core at the start
	 * of this handler. If so, we need to unpark it here.
	 * We use DCUCR reserved bits (stored in %g1) to keep track of
	 * whether or not we need to unpark. %g5 and %g7 are scratch registers.
	 */
	UNPARK_SIBLING_CORE(%g1, %g5, %g7)

	/*
	 * Restore Icache and Dcache to previous state.
	 */
	stxa	%g1, [%g0]ASI_DCU
	flush	%g0	/* flush required after changing the IC bit */

	/*
	 * Make sure our CPU logout operation was successful.
	 */
	cmp	%g3, %g0
	be	4f
	  nop

	/*
	 * If the logout structure had been busy, how many times have
	 * we tried to use it and failed (nesting count)? If we have
	 * already recursed a substantial number of times, then we can
	 * assume things are not going to get better by themselves and
	 * so it would be best to panic.
	 */
	cmp	%g3, CLO_NESTING_MAX
	blt	3f
	  nop

        call ptl1_panic
          mov   PTL1_BAD_ECC, %g1

3:
	/*
	 * Otherwise, if the logout structure was busy but we have not
	 * nested more times than our maximum value, then we simply
	 * issue a retry. Our TL=0 trap handler code will check and
	 * clear the AFSR after it is done logging what is currently
	 * in the logout struct and handle this event at that time.
	 */
	retry
4:
	RESET_USER_RTT_REGS(%g4, %g5, async_err_resetskip)
async_err_resetskip:
	set	cpu_deferred_error, %g1
	ba	sys_trap
	  mov	PIL_15, %g4		! run at pil 15
	SET_SIZE(async_err)

#if defined(CPU_IMP_L1_CACHE_PARITY)

/*
 * D$ parity error trap (trap 71) at TL=0.
 * tt0_dperr is replaced by dcache_parity_instr in cpu_init_trap of
 * the various architecture-specific files.  This merely sets up the
 * arguments for cpu_parity_error and calls it via sys_trap.
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */
	ENTRY_NP(dcache_parity_instr)
	membar	#Sync			! Cheetah+ requires membar #Sync
	set	cpu_parity_error, %g1
	or	%g0, CH_ERR_DPE, %g2
	rdpr	%tpc, %g3
	sethi	%hi(sys_trap), %g7
	jmp	%g7 + %lo(sys_trap)
	  mov	PIL_15, %g4		! run at pil 15
	SET_SIZE(dcache_parity_instr)


/*
 * D$ parity error trap (trap 71) at TL>0.
 * tt1_dperr is replaced by dcache_parity_tl1_instr in cpu_init_trap of
 * the various architecture-specific files.  This generates a "Software
 * Trap 1" at TL>0, which goes to dcache_parity_tl1_cont_instr, and we
 * continue the handling there.
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */
	ENTRY_NP(dcache_parity_tl1_instr)
	CH_ERR_TL1_TRAPENTRY(SWTRAP_1);
	SET_SIZE(dcache_parity_tl1_instr)


/*
 * Software trap 1 at TL>0.
 * tt1_swtrap1 is replaced by dcache_parity_tl1_cont_instr in cpu_init_trap
 * of the various architecture-specific files.  This is used as a continuation
 * of the dcache parity handling where we've bought an extra TL level, so we
 * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
 * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
 * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
 * order two bits from %g1 and %g2 respectively).
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */
	ENTRY_NP(dcache_parity_tl1_cont_instr)
	CH_ERR_TL1_SWTRAPENTRY(dcache_parity_tl1_err);
	SET_SIZE(dcache_parity_tl1_cont_instr)

/*
 * D$ parity error at TL>0 handler
 * We get here via trap 71 at TL>0->Software trap 1 at TL>0.  We enter
 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
 */

	ENTRY_NP(dcache_parity_tl1_err)

	/*
	 * This macro saves all the %g registers in the ch_err_tl1_data
	 * structure, updates the ch_err_tl1_flags and saves the %tpc in
	 * ch_err_tl1_tpc.  At the end of this macro, %g1 will point to
	 * the ch_err_tl1_data structure and %g2 will have the original
	 * flags in the ch_err_tl1_data structure.  All %g registers
	 * except for %g1 and %g2 will be available.
	 */
	CH_ERR_TL1_ENTER(CH_ERR_DPE);

#ifdef TRAPTRACE
	/*
	 * Get current trap trace entry physical pointer.
	 */
	CPU_INDEX(%g6, %g5)
	sll	%g6, TRAPTR_SIZE_SHIFT, %g6
	set	trap_trace_ctl, %g5
	add	%g6, %g5, %g6
	ld	[%g6 + TRAPTR_LIMIT], %g5
	tst	%g5
	be	%icc, dpe_tl1_skip_tt
	  nop
	ldx	[%g6 + TRAPTR_PBASE], %g5
	ld	[%g6 + TRAPTR_OFFSET], %g4
	add	%g5, %g4, %g5

	/*
	 * Create trap trace entry.
	 */
	rd	%asi, %g7
	wr	%g0, TRAPTR_ASI, %asi
	rd	STICK, %g4
	stxa	%g4, [%g5 + TRAP_ENT_TICK]%asi
	rdpr	%tl, %g4
	stha	%g4, [%g5 + TRAP_ENT_TL]%asi
	rdpr	%tt, %g4
	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
	rdpr	%tpc, %g4
	stna	%g4, [%g5 + TRAP_ENT_TPC]%asi
	rdpr	%tstate, %g4
	stxa	%g4, [%g5 + TRAP_ENT_TSTATE]%asi
	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
	stna	%g0, [%g5 + TRAP_ENT_F1]%asi
	stna	%g0, [%g5 + TRAP_ENT_F2]%asi
	stna	%g0, [%g5 + TRAP_ENT_F3]%asi
	stna	%g0, [%g5 + TRAP_ENT_F4]%asi
	wr	%g0, %g7, %asi

	/*
	 * Advance trap trace pointer.
	 */
	ld	[%g6 + TRAPTR_OFFSET], %g5
	ld	[%g6 + TRAPTR_LIMIT], %g4
	st	%g5, [%g6 + TRAPTR_LAST_OFFSET]
	add	%g5, TRAP_ENT_SIZE, %g5
	sub	%g4, TRAP_ENT_SIZE, %g4
	cmp	%g5, %g4
	movge	%icc, 0, %g5
	st	%g5, [%g6 + TRAPTR_OFFSET]
dpe_tl1_skip_tt:
#endif	/* TRAPTRACE */

	/*
	 * I$ and D$ are automatically turned off by HW when the CPU hits
	 * a dcache or icache parity error so we will just leave those two
	 * off for now to avoid repeating this trap.
	 * For Panther, however, since we trap on P$ data parity errors
	 * and HW does not automatically disable P$, we need to disable it
	 * here so that we don't encounter any recursive traps when we
	 * issue the retry.
	 */
	ldxa	[%g0]ASI_DCU, %g3
	mov	1, %g4
	sllx	%g4, DCU_PE_SHIFT, %g4
	andn	%g3, %g4, %g3
	stxa	%g3, [%g0]ASI_DCU
	membar	#Sync

	/*
	 * We fall into this macro if we've successfully logged the error in
	 * the ch_err_tl1_data structure and want the PIL15 softint to pick
	 * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
	 * Restores the %g registers and issues retry.
	 */
	CH_ERR_TL1_EXIT;
	SET_SIZE(dcache_parity_tl1_err)

/*
 * I$ parity error trap (trap 72) at TL=0.
 * tt0_iperr is replaced by icache_parity_instr in cpu_init_trap of
 * the various architecture-specific files.  This merely sets up the
 * arguments for cpu_parity_error and calls it via sys_trap.
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */

	ENTRY_NP(icache_parity_instr)
	membar	#Sync			! Cheetah+ requires membar #Sync
	set	cpu_parity_error, %g1
	or	%g0, CH_ERR_IPE, %g2
	rdpr	%tpc, %g3
	sethi	%hi(sys_trap), %g7
	jmp	%g7 + %lo(sys_trap)
	  mov	PIL_15, %g4		! run at pil 15
	SET_SIZE(icache_parity_instr)

/*
 * I$ parity error trap (trap 72) at TL>0.
 * tt1_iperr is replaced by icache_parity_tl1_instr in cpu_init_trap of
 * the various architecture-specific files.  This generates a "Software
 * Trap 2" at TL>0, which goes to icache_parity_tl1_cont_instr, and we
 * continue the handling there.
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */
	ENTRY_NP(icache_parity_tl1_instr)
	CH_ERR_TL1_TRAPENTRY(SWTRAP_2);
	SET_SIZE(icache_parity_tl1_instr)

/*
 * Software trap 2 at TL>0.
 * tt1_swtrap2 is replaced by icache_parity_tl1_cont_instr in cpu_init_trap
 * of the various architecture-specific files.  This is used as a continuation
 * of the icache parity handling where we've bought an extra TL level, so we
 * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
 * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
 * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
 * order two bits from %g1 and %g2 respectively).
 * NB: Must be 8 instructions or less to fit in trap table and code must
 *     be relocatable.
 */
	ENTRY_NP(icache_parity_tl1_cont_instr)
	CH_ERR_TL1_SWTRAPENTRY(icache_parity_tl1_err);
	SET_SIZE(icache_parity_tl1_cont_instr)


/*
 * I$ parity error at TL>0 handler
 * We get here via trap 72 at TL>0->Software trap 2 at TL>0.  We enter
 * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
 */

	ENTRY_NP(icache_parity_tl1_err)

	/*
	 * This macro saves all the %g registers in the ch_err_tl1_data
	 * structure, updates the ch_err_tl1_flags and saves the %tpc in
	 * ch_err_tl1_tpc.  At the end of this macro, %g1 will point to
	 * the ch_err_tl1_data structure and %g2 will have the original
	 * flags in the ch_err_tl1_data structure.  All %g registers
	 * except for %g1 and %g2 will be available.
	 */
	CH_ERR_TL1_ENTER(CH_ERR_IPE);

#ifdef TRAPTRACE
	/*
	 * Get current trap trace entry physical pointer.
	 */
	CPU_INDEX(%g6, %g5)
	sll	%g6, TRAPTR_SIZE_SHIFT, %g6
	set	trap_trace_ctl, %g5
	add	%g6, %g5, %g6
	ld	[%g6 + TRAPTR_LIMIT], %g5
	tst	%g5
	be	%icc, ipe_tl1_skip_tt
	  nop
	ldx	[%g6 + TRAPTR_PBASE], %g5
	ld	[%g6 + TRAPTR_OFFSET], %g4
	add	%g5, %g4, %g5

	/*
	 * Create trap trace entry.
	 */
	rd	%asi, %g7
	wr	%g0, TRAPTR_ASI, %asi
	rd	STICK, %g4
	stxa	%g4, [%g5 + TRAP_ENT_TICK]%asi
	rdpr	%tl, %g4
	stha	%g4, [%g5 + TRAP_ENT_TL]%asi
	rdpr	%tt, %g4
	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
	rdpr	%tpc, %g4
	stna	%g4, [%g5 + TRAP_ENT_TPC]%asi
	rdpr	%tstate, %g4
	stxa	%g4, [%g5 + TRAP_ENT_TSTATE]%asi
	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
	stna	%g0, [%g5 + TRAP_ENT_F1]%asi
	stna	%g0, [%g5 + TRAP_ENT_F2]%asi
	stna	%g0, [%g5 + TRAP_ENT_F3]%asi
	stna	%g0, [%g5 + TRAP_ENT_F4]%asi
	wr	%g0, %g7, %asi

	/*
	 * Advance trap trace pointer.
	 */
	ld	[%g6 + TRAPTR_OFFSET], %g5
	ld	[%g6 + TRAPTR_LIMIT], %g4
	st	%g5, [%g6 + TRAPTR_LAST_OFFSET]
	add	%g5, TRAP_ENT_SIZE, %g5
	sub	%g4, TRAP_ENT_SIZE, %g4
	cmp	%g5, %g4
	movge	%icc, 0, %g5
	st	%g5, [%g6 + TRAPTR_OFFSET]
ipe_tl1_skip_tt:
#endif	/* TRAPTRACE */

	/*
	 * We fall into this macro if we've successfully logged the error in
	 * the ch_err_tl1_data structure and want the PIL15 softint to pick
	 * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
	 * Restores the %g registers and issues retry.
	 */
	CH_ERR_TL1_EXIT;

	SET_SIZE(icache_parity_tl1_err)

#endif	/* CPU_IMP_L1_CACHE_PARITY */


/*
 * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the
 * tte, the virtual address, and the ctxnum of the specified tlb entry.  They
 * should only be used in places where you have no choice but to look at the
 * tlb itself.
 *
 * Note: These two routines are required by the Estar "cpr" loadable module.
 */

	ENTRY_NP(itlb_rd_entry)
	sllx	%o0, 3, %o0
	ldxa	[%o0]ASI_ITLB_ACCESS, %g1
	stx	%g1, [%o1]
	ldxa	[%o0]ASI_ITLB_TAGREAD, %g2
	set	TAGREAD_CTX_MASK, %o4
	andn	%g2, %o4, %o5
	retl
	  stx	%o5, [%o2]
	SET_SIZE(itlb_rd_entry)


	ENTRY_NP(dtlb_rd_entry)
	sllx	%o0, 3, %o0
	ldxa	[%o0]ASI_DTLB_ACCESS, %g1
	stx	%g1, [%o1]
	ldxa	[%o0]ASI_DTLB_TAGREAD, %g2
	set	TAGREAD_CTX_MASK, %o4
	andn	%g2, %o4, %o5
	retl
	  stx	%o5, [%o2]
	SET_SIZE(dtlb_rd_entry)


#if !(defined(JALAPENO) || defined(SERRANO))

	ENTRY(get_safari_config)
	ldxa	[%g0]ASI_SAFARI_CONFIG, %o0
	retl
	nop
	SET_SIZE(get_safari_config)


	ENTRY(set_safari_config)
	stxa	%o0, [%g0]ASI_SAFARI_CONFIG
	membar	#Sync
	retl
	nop
	SET_SIZE(set_safari_config)

#endif	/* !(JALAPENO || SERRANO) */


	/*
	 * Clear the NPT (non-privileged trap) bit in the %tick/%stick
	 * registers. In an effort to make the change in the
	 * tick/stick counter as consistent as possible, we disable
	 * all interrupts while we're changing the registers. We also
	 * ensure that the read and write instructions are in the same
	 * line in the instruction cache.
	 */
	ENTRY_NP(cpu_clearticknpt)
	rdpr	%pstate, %g1		/* save processor state */
	andn	%g1, PSTATE_IE, %g3	/* turn off */
	wrpr	%g0, %g3, %pstate	/*   interrupts */
	rdpr	%tick, %g2		/* get tick register */
	brgez,pn %g2, 1f		/* if NPT bit off, we're done */
	mov	1, %g3			/* create mask */
	sllx	%g3, 63, %g3		/*   for NPT bit */
	ba,a,pt	%xcc, 2f
	.align	8			/* Ensure rd/wr in same i$ line */
2:
	rdpr	%tick, %g2		/* get tick register */
	wrpr	%g3, %g2, %tick		/* write tick register, */
					/*   clearing NPT bit   */
1:
	rd	STICK, %g2		/* get stick register */
	brgez,pn %g2, 3f		/* if NPT bit off, we're done */
	mov	1, %g3			/* create mask */
	sllx	%g3, 63, %g3		/*   for NPT bit */
	ba,a,pt	%xcc, 4f
	.align	8			/* Ensure rd/wr in same i$ line */
4:
	rd	STICK, %g2		/* get stick register */
	wr	%g3, %g2, STICK		/* write stick register, */
					/*   clearing NPT bit   */
3:
	jmp	%g4 + 4
	wrpr	%g0, %g1, %pstate	/* restore processor state */

	SET_SIZE(cpu_clearticknpt)


#if defined(CPU_IMP_L1_CACHE_PARITY)

/*
 * correct_dcache_parity(size_t size, size_t linesize)
 *
 * Correct D$ data parity by zeroing the data and initializing microtag
 * for all indexes and all ways of the D$.
 *
 */
	ENTRY(correct_dcache_parity)
	/*
	 * Register Usage:
	 *
	 * %o0 = input D$ size
	 * %o1 = input D$ line size
	 * %o2 = scratch
	 * %o3 = scratch
	 * %o4 = scratch
	 */

	sub	%o0, %o1, %o0			! init cache line address

	/*
	 * For Panther CPUs, we also need to clear the data parity bits
	 * using DC_data_parity bit of the ASI_DCACHE_DATA register.
	 */
	GET_CPU_IMPL(%o3)
	cmp	%o3, PANTHER_IMPL
	bne	1f
	  clr	%o3				! zero for non-Panther
	mov	1, %o3
	sll	%o3, PN_DC_DATA_PARITY_BIT_SHIFT, %o3

1:
	/*
	 * Set utag = way since it must be unique within an index.
	 */
	srl	%o0, 14, %o2			! get cache way (DC_way)
	membar	#Sync				! required before ASI_DC_UTAG
	stxa	%o2, [%o0]ASI_DC_UTAG		! set D$ utag = cache way
	membar	#Sync				! required after ASI_DC_UTAG

	/*
	 * Zero line of D$ data (and data parity bits for Panther)
	 */
	sub	%o1, 8, %o2
	or	%o0, %o3, %o4			! same address + DC_data_parity
2:
	membar	#Sync				! required before ASI_DC_DATA
	stxa	%g0, [%o0 + %o2]ASI_DC_DATA	! zero 8 bytes of D$ data
	membar	#Sync				! required after ASI_DC_DATA
	/*
	 * We also clear the parity bits if this is a panther. For non-Panther
	 * CPUs, we simply end up clearing the $data register twice.
	 */
	stxa	%g0, [%o4 + %o2]ASI_DC_DATA
	membar	#Sync

	subcc	%o2, 8, %o2
	bge	2b
	nop

	subcc	%o0, %o1, %o0
	bge	1b
	nop

	retl
	  nop
	SET_SIZE(correct_dcache_parity)

#endif	/* CPU_IMP_L1_CACHE_PARITY */


	ENTRY_NP(stick_timestamp)
	rd	STICK, %g1	! read stick reg
	sllx	%g1, 1, %g1
	srlx	%g1, 1, %g1	! clear npt bit

	retl
	stx     %g1, [%o0]	! store the timestamp
	SET_SIZE(stick_timestamp)


	ENTRY_NP(stick_adj)
	rdpr	%pstate, %g1		! save processor state
	andn	%g1, PSTATE_IE, %g3
	ba	1f			! cache align stick adj
	wrpr	%g0, %g3, %pstate	! turn off interrupts

	.align	16
1:	nop

	rd	STICK, %g4		! read stick reg
	add	%g4, %o0, %o1		! adjust stick with skew
	wr	%o1, %g0, STICK		! write stick reg

	retl
	wrpr	%g1, %pstate		! restore processor state
	SET_SIZE(stick_adj)

	ENTRY_NP(kdi_get_stick)
	rd	STICK, %g1
	stx	%g1, [%o0]
	retl
	mov	%g0, %o0
	SET_SIZE(kdi_get_stick)

/*
 * Invalidate the specified line from the D$.
 *
 * Register usage:
 *	%o0 - index for the invalidation, specifies DC_way and DC_addr
 *
 * ASI_DC_TAG, 0x47, is used in the following manner. A 64-bit value is
 * stored to a particular DC_way and DC_addr in ASI_DC_TAG.
 *
 * The format of the stored 64-bit value is:
 *
 *	+----------+--------+----------+
 *	| Reserved | DC_tag | DC_valid |
 *	+----------+--------+----------+
 *       63      31 30     1	      0
 *
 * DC_tag is the 30-bit physical tag of the associated line.
 * DC_valid is the 1-bit valid field for both the physical and snoop tags.
 *
 * The format of the 64-bit DC_way and DC_addr into ASI_DC_TAG is:
 *
 *	+----------+--------+----------+----------+
 *	| Reserved | DC_way | DC_addr  | Reserved |
 *	+----------+--------+----------+----------+
 *       63      16 15    14 13       5 4        0
 *
 * DC_way is a 2-bit index that selects one of the 4 ways.
 * DC_addr is a 9-bit index that selects one of 512 tag/valid fields.
 *
 * Setting the DC_valid bit to zero for the specified DC_way and
 * DC_addr index into the D$ results in an invalidation of a D$ line.
 */
	ENTRY(dcache_inval_line)
	sll	%o0, 5, %o0		! shift index into DC_way and DC_addr
	stxa	%g0, [%o0]ASI_DC_TAG	! zero the DC_valid and DC_tag bits
	membar	#Sync
	retl
	nop
	SET_SIZE(dcache_inval_line)

/*
 * Invalidate the entire I$
 *
 * Register usage:
 *	%o0 - specifies IC_way, IC_addr, IC_tag
 *	%o1 - scratch
 *	%o2 - used to save and restore DCU value
 *	%o3 - scratch
 *	%o5 - used to save and restore PSTATE
 *
 * Due to the behavior of the I$ control logic when accessing ASI_IC_TAG,
 * the I$ should be turned off. Accesses to ASI_IC_TAG may collide and
 * block out snoops and invalidates to the I$, causing I$ consistency
 * to be broken. Before turning on the I$, all I$ lines must be invalidated.
 *
 * ASI_IC_TAG, 0x67, is used in the following manner. A 64-bit value is
 * stored to a particular IC_way, IC_addr, IC_tag in ASI_IC_TAG. The
 * info below describes store (write) use of ASI_IC_TAG. Note that read
 * use of ASI_IC_TAG behaves differently.
 *
 * The format of the stored 64-bit value is:
 *
 *	+----------+--------+---------------+-----------+
 *	| Reserved | Valid  | IC_vpred<7:0> | Undefined |
 *	+----------+--------+---------------+-----------+
 *       63      55    54    53           46 45        0
 *
 * Valid is the 1-bit valid field for both the physical and snoop tags.
 * IC_vpred is the 8-bit LPB bits for 8 instructions starting at
 *	the 32-byte boundary aligned address specified by IC_addr.
 *
 * The format of the 64-bit IC_way, IC_addr, IC_tag into ASI_IC_TAG is:
 *
 *	+----------+--------+---------+--------+---------+
 *	| Reserved | IC_way | IC_addr | IC_tag |Reserved |
 *	+----------+--------+---------+--------+---------+
 *       63      16 15    14 13      5 4      3 2       0
 *
 * IC_way is a 2-bit index that selects one of the 4 ways.
 * IC_addr[13:6] is an 8-bit index that selects one of 256 valid fields.
 * IC_addr[5] is a "don't care" for a store.
 * IC_tag set to 2 specifies that the stored value is to be interpreted
 *	as containing Valid and IC_vpred as described above.
 *
 * Setting the Valid bit to zero for the specified IC_way and
 * IC_addr index into the I$ results in an invalidation of an I$ line.
 */
	ENTRY(icache_inval_all)
	rdpr	%pstate, %o5
	andn	%o5, PSTATE_IE, %o3
	wrpr	%g0, %o3, %pstate	! clear IE bit

	GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, icache_inval_all_1);
	ld	[%o0 + CHPR_ICACHE_LINESIZE], %o1
	ba,pt	%icc, 2f
	  ld	[%o0 + CHPR_ICACHE_SIZE], %o0
icache_inval_all_1:
	ASM_LD(%o0, icache_size)
	ASM_LD(%o1, icache_linesize)
2:
	CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4)

	retl
	wrpr	%g0, %o5, %pstate	! restore earlier pstate
	SET_SIZE(icache_inval_all)


/*
 * cache_scrubreq_tl1 is the crosstrap handler called on offlined cpus via a
 * crosstrap.  It atomically increments the outstanding request counter and,
 * if there was not already an outstanding request, branches to setsoftint_tl1
 * to enqueue an intr_vec for the given inum.
 */

	! Register usage:
	!
	! Arguments:
	! %g1 - inum
	! %g2 - index into chsm_outstanding array
	!
	! Internal:
	! %g2, %g3, %g5 - scratch
	! %g4 - ptr. to scrub_misc chsm_outstanding[index].
	! %g6 - setsoftint_tl1 address

	ENTRY_NP(cache_scrubreq_tl1)
	mulx	%g2, CHSM_OUTSTANDING_INCR, %g2
	set	CHPR_SCRUB_MISC + CHSM_OUTSTANDING, %g3
	add	%g2, %g3, %g2
	GET_CPU_PRIVATE_PTR(%g2, %g4, %g5, 1f);
	ld	[%g4], %g2		! cpu's chsm_outstanding[index]
	!
	! no need to use atomic instructions for the following
	! increment - we're at tl1
	!
	add	%g2, 0x1, %g3
	brnz,pn	%g2, 1f			! no need to enqueue more intr_vec
	  st	%g3, [%g4]		! delay - store incremented counter
	ASM_JMP(%g6, setsoftint_tl1)
	! not reached
1:
	retry
	SET_SIZE(cache_scrubreq_tl1)


/*
 * Get the error state for the processor.
 * Note that this must not be used at TL>0
 */
	ENTRY(get_cpu_error_state)
#if defined(CHEETAH_PLUS)
	set	ASI_SHADOW_REG_VA, %o2
	ldxa	[%o2]ASI_AFSR, %o1		! shadow afsr reg
	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR]
	ldxa	[%o2]ASI_AFAR, %o1		! shadow afar reg
	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR]
	GET_CPU_IMPL(%o3)	! Only panther has AFSR_EXT registers
	cmp	%o3, PANTHER_IMPL
	bne,a	1f
	  stx	%g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT]	! zero for non-PN
	set	ASI_AFSR_EXT_VA, %o2
	ldxa	[%o2]ASI_AFSR, %o1		! afsr_ext reg
	stx	%o1, [%o0 + CH_CPU_ERRORS_AFSR_EXT]
	set	ASI_SHADOW_AFSR_EXT_VA, %o2
	ldxa	[%o2]ASI_AFSR, %o1		! shadow afsr_ext reg
	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT]
	b	2f
	  nop
1:
	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] ! zero for non-PN
2:
#else	/* CHEETAH_PLUS */
	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR]
	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR]
	stx	%g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT]
	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT]
#endif	/* CHEETAH_PLUS */
#if defined(SERRANO)
	/*
	 * Serrano has an afar2 which captures the address on FRC/FRU errors.
	 * We save this in the afar2 of the register save area.
	 */
	set	ASI_MCU_AFAR2_VA, %o2
	ldxa	[%o2]ASI_MCU_CTRL, %o1
	stx	%o1, [%o0 + CH_CPU_ERRORS_AFAR2]
#endif	/* SERRANO */
	ldxa	[%g0]ASI_AFSR, %o1		! primary afsr reg
	stx	%o1, [%o0 + CH_CPU_ERRORS_AFSR]
	ldxa	[%g0]ASI_AFAR, %o1		! primary afar reg
	retl
	stx	%o1, [%o0 + CH_CPU_ERRORS_AFAR]
	SET_SIZE(get_cpu_error_state)

/*
 * Check a page of memory for errors.
 *
 * Load each 64 byte block from physical memory.
 * Check AFSR after each load to see if an error
 * was caused. If so, log/scrub that error.
 *
 * Used to determine if a page contains
 * CEs when CEEN is disabled.
 */
	ENTRY(cpu_check_block)
	!
	! get a new window with room for the error regs
	!
	save	%sp, -SA(MINFRAME + CH_CPU_ERROR_SIZE), %sp
	srl	%i1, 6, %l4		! clear top bits of psz
					! and divide by 64
	rd	%fprs, %l2		! store FP
	wr	%g0, FPRS_FEF, %fprs	! enable FP
1:
	ldda	[%i0]ASI_BLK_P, %d0	! load a block
	membar	#Sync
	ldxa    [%g0]ASI_AFSR, %l3	! read afsr reg
	brz,a,pt %l3, 2f		! check for error
	nop

	!
	! if error, read the error regs and log it
	!
	call	get_cpu_error_state
	add	%fp, STACK_BIAS - CH_CPU_ERROR_SIZE, %o0

	!
	! cpu_ce_detected(ch_cpu_errors_t *, flag)
	!
	call	cpu_ce_detected		! log the error
	mov	CE_CEEN_TIMEOUT, %o1
2:
	dec	%l4			! next 64-byte block
	brnz,a,pt  %l4, 1b
	add	%i0, 64, %i0		! increment block addr

	wr	%l2, %g0, %fprs		! restore FP
	ret
	restore

	SET_SIZE(cpu_check_block)

/*
 * Perform a cpu logout called from C.  This is used where we did not trap
 * for the error but still want to gather "what we can".  Caller must make
 * sure cpu private area exists and that the indicated logout area is free
 * for use, and that we are unable to migrate cpus.
 */
	ENTRY(cpu_delayed_logout)
	rdpr	%pstate, %o2
	andn	%o2, PSTATE_IE, %o2
	wrpr	%g0, %o2, %pstate		! disable interrupts
	PARK_SIBLING_CORE(%o2, %o3, %o4)	! %o2 has DCU value
	add	%o1, CH_CLO_DATA + CH_CHD_EC_DATA, %o1
	rd	%asi, %g1
	wr	%g0, ASI_P, %asi
	GET_ECACHE_DTAGS(%o0, %o1, %o3, %o4, %o5)
	wr	%g1, %asi
	UNPARK_SIBLING_CORE(%o2, %o3, %o4)	! can use %o2 again
	rdpr	%pstate, %o2
	or	%o2, PSTATE_IE, %o2
	wrpr	%g0, %o2, %pstate
	retl
	  nop
	SET_SIZE(cpu_delayed_logout)

	ENTRY(dtrace_blksuword32)
	save	%sp, -SA(MINFRAME + 4), %sp

	rdpr	%pstate, %l1
	andn	%l1, PSTATE_IE, %l2		! disable interrupts to
	wrpr	%g0, %l2, %pstate		! protect our FPU diddling

	rd	%fprs, %l0
	andcc	%l0, FPRS_FEF, %g0
	bz,a,pt	%xcc, 1f			! if the fpu is disabled
	wr	%g0, FPRS_FEF, %fprs		! ... enable the fpu

	st	%f0, [%fp + STACK_BIAS - 4]	! save %f0 to the stack
1:
	set	0f, %l5
        /*
         * We're about to write a block full or either total garbage
         * (not kernel data, don't worry) or user floating-point data
         * (so it only _looks_ like garbage).
         */
	ld	[%i1], %f0			! modify the block
	membar	#Sync
	stn	%l5, [THREAD_REG + T_LOFAULT]	! set up the lofault handler
	stda	%d0, [%i0]ASI_BLK_COMMIT_S	! store the modified block
	membar	#Sync
	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler

	bz,a,pt	%xcc, 1f
	wr	%g0, %l0, %fprs			! restore %fprs

	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
1:

	wrpr	%g0, %l1, %pstate		! restore interrupts

	ret
	restore	%g0, %g0, %o0

0:
	membar	#Sync
	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler

	bz,a,pt	%xcc, 1f
	wr	%g0, %l0, %fprs			! restore %fprs

	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
1:

	wrpr	%g0, %l1, %pstate		! restore interrupts

	/*
	 * If tryagain is set (%i2) we tail-call dtrace_blksuword32_err()
	 * which deals with watchpoints. Otherwise, just return -1.
	 */
	brnz,pt	%i2, 1f
	nop
	ret
	restore	%g0, -1, %o0
1:
	call	dtrace_blksuword32_err
	restore

	SET_SIZE(dtrace_blksuword32)

#ifdef	CHEETAHPLUS_ERRATUM_25

	/* Claim a chunk of physical address space. */
	ENTRY(claimlines)
1:
	subcc	%o1, %o2, %o1
	add	%o0, %o1, %o3
	bgeu,a,pt	%xcc, 1b
	casxa	[%o3]ASI_MEM, %g0, %g0
	membar  #Sync
	retl
	nop
	SET_SIZE(claimlines)

	/*
	 * CPU feature initialization,
	 * turn BPE off,
	 * get device id.
	 */
	ENTRY(cpu_feature_init)
	save	%sp, -SA(MINFRAME), %sp
	sethi	%hi(cheetah_bpe_off), %o0
	ld	[%o0 + %lo(cheetah_bpe_off)], %o0
	brz	%o0, 1f
	nop
	rd	ASR_DISPATCH_CONTROL, %o0
	andn	%o0, ASR_DISPATCH_CONTROL_BPE, %o0
	wr	%o0, 0, ASR_DISPATCH_CONTROL
1:
	!
	! get the device_id and store the device_id
	! in the appropriate cpunodes structure
	! given the cpus index
	!
	CPU_INDEX(%o0, %o1)
	mulx %o0, CPU_NODE_SIZE, %o0
	set  cpunodes + DEVICE_ID, %o1
	ldxa [%g0] ASI_DEVICE_SERIAL_ID, %o2
	stx  %o2, [%o0 + %o1]
#ifdef	CHEETAHPLUS_ERRATUM_34
	!
	! apply Cheetah+ erratum 34 workaround
	!
	call itlb_erratum34_fixup
	  nop
	call dtlb_erratum34_fixup
	  nop
#endif	/* CHEETAHPLUS_ERRATUM_34 */
	ret
	  restore
	SET_SIZE(cpu_feature_init)

/*
 * Copy a tsb entry atomically, from src to dest.
 * src must be 128 bit aligned.
 */
	ENTRY(copy_tsb_entry)
	ldda	[%o0]ASI_NQUAD_LD, %o2		! %o2 = tag, %o3 = data
	stx	%o2, [%o1]
	stx	%o3, [%o1 + 8 ]
	retl
	nop
	SET_SIZE(copy_tsb_entry)

#endif	/* CHEETAHPLUS_ERRATUM_25 */

#ifdef	CHEETAHPLUS_ERRATUM_34

	!
	! In Cheetah+ erratum 34, under certain conditions an ITLB locked
	! index 0 TTE will erroneously be displaced when a new TTE is
	! loaded via ASI_ITLB_IN.  In order to avoid cheetah+ erratum 34,
	! locked index 0 TTEs must be relocated.
	!
	! NOTE: Care must be taken to avoid an ITLB miss in this routine.
	!
	ENTRY_NP(itlb_erratum34_fixup)
	rdpr	%pstate, %o3
#ifdef DEBUG
	PANIC_IF_INTR_DISABLED_PSTR(%o3, u3_di_label1, %g1)
#endif /* DEBUG */
	wrpr	%o3, PSTATE_IE, %pstate		! Disable interrupts
	ldxa	[%g0]ASI_ITLB_ACCESS, %o1	! %o1 = entry 0 data
	ldxa	[%g0]ASI_ITLB_TAGREAD, %o2	! %o2 = entry 0 tag

	cmp	%o1, %g0			! Is this entry valid?
	bge	%xcc, 1f
	  andcc	%o1, TTE_LCK_INT, %g0		! Is this entry locked?
	bnz	%icc, 2f
	  nop
1:
	retl					! Nope, outta here...
	  wrpr	%g0, %o3, %pstate		! Enable interrupts
2:
	sethi	%hi(FLUSH_ADDR), %o4
	stxa	%g0, [%o2]ASI_ITLB_DEMAP	! Flush this mapping
	flush	%o4				! Flush required for I-MMU
	!
	! Start search from index 1 up.  This is because the kernel force
	! loads its text page at index 15 in sfmmu_kernel_remap() and we
	! don't want our relocated entry evicted later.
	!
	! NOTE: We assume that we'll be successful in finding an unlocked
	! or invalid entry.  If that isn't the case there are bound to
	! bigger problems.
	!
	set	(1 << 3), %g3
3:
	ldxa	[%g3]ASI_ITLB_ACCESS, %o4	! Load TTE from t16
	!
	! If this entry isn't valid, we'll choose to displace it (regardless
	! of the lock bit).
	!
	cmp	%o4, %g0			! TTE is > 0 iff not valid
	bge	%xcc, 4f			! If invalid, go displace
	  andcc	%o4, TTE_LCK_INT, %g0		! Check for lock bit
	bnz,a	%icc, 3b			! If locked, look at next
	  add	%g3, (1 << 3), %g3		!  entry
4:
	!
	! We found an unlocked or invalid entry; we'll explicitly load
	! the former index 0 entry here.
	!
	sethi	%hi(FLUSH_ADDR), %o4
	set	MMU_TAG_ACCESS, %g4
	stxa	%o2, [%g4]ASI_IMMU
	stxa	%o1, [%g3]ASI_ITLB_ACCESS
	flush	%o4				! Flush required for I-MMU
	retl
	  wrpr	%g0, %o3, %pstate		! Enable interrupts
	SET_SIZE(itlb_erratum34_fixup)

	!
	! In Cheetah+ erratum 34, under certain conditions a DTLB locked
	! index 0 TTE will erroneously be displaced when a new TTE is
	! loaded.  In order to avoid cheetah+ erratum 34, locked index 0
	! TTEs must be relocated.
	!
	ENTRY_NP(dtlb_erratum34_fixup)
	rdpr	%pstate, %o3
#ifdef DEBUG
	PANIC_IF_INTR_DISABLED_PSTR(%o3, u3_di_label2, %g1)
#endif /* DEBUG */
	wrpr	%o3, PSTATE_IE, %pstate		! Disable interrupts
	ldxa	[%g0]ASI_DTLB_ACCESS, %o1	! %o1 = entry 0 data
	ldxa	[%g0]ASI_DTLB_TAGREAD, %o2	! %o2 = entry 0 tag

	cmp	%o1, %g0			! Is this entry valid?
	bge	%xcc, 1f
	  andcc	%o1, TTE_LCK_INT, %g0		! Is this entry locked?
	bnz	%icc, 2f
	  nop
1:
	retl					! Nope, outta here...
	  wrpr	%g0, %o3, %pstate		! Enable interrupts
2:
	stxa	%g0, [%o2]ASI_DTLB_DEMAP	! Flush this mapping
	membar	#Sync
	!
	! Start search from index 1 up.
	!
	! NOTE: We assume that we'll be successful in finding an unlocked
	! or invalid entry.  If that isn't the case there are bound to
	! bigger problems.
	!
	set	(1 << 3), %g3
3:
	ldxa	[%g3]ASI_DTLB_ACCESS, %o4	! Load TTE from t16
	!
	! If this entry isn't valid, we'll choose to displace it (regardless
	! of the lock bit).
	!
	cmp	%o4, %g0			! TTE is > 0 iff not valid
	bge	%xcc, 4f			! If invalid, go displace
	  andcc	%o4, TTE_LCK_INT, %g0		! Check for lock bit
	bnz,a	%icc, 3b			! If locked, look at next
	  add	%g3, (1 << 3), %g3		!  entry
4:
	!
	! We found an unlocked or invalid entry; we'll explicitly load
	! the former index 0 entry here.
	!
	set	MMU_TAG_ACCESS, %g4
	stxa	%o2, [%g4]ASI_DMMU
	stxa	%o1, [%g3]ASI_DTLB_ACCESS
	membar	#Sync
	retl
	  wrpr	%g0, %o3, %pstate		! Enable interrupts
	SET_SIZE(dtlb_erratum34_fixup)

#endif	/* CHEETAHPLUS_ERRATUM_34 */