27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
51e2e7a7huah * Common Development and Distribution License (the "License").
61e2e7a7huah * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
22bd963cbEthindra Ramamurthy * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bdstevel@tonic-gate * Use is subject to license terms.
247c478bdstevel@tonic-gate *
259b0bb79John Levon * Copyright 2020 Joyent, Inc.
269b0bb79John Levon *
277c478bdstevel@tonic-gate * Assembly code support for Cheetah/Cheetah+ modules
287c478bdstevel@tonic-gate */
307c478bdstevel@tonic-gate#include "assym.h"
327c478bdstevel@tonic-gate#include <sys/asm_linkage.h>
337c478bdstevel@tonic-gate#include <sys/mmu.h>
347c478bdstevel@tonic-gate#include <vm/hat_sfmmu.h>
357c478bdstevel@tonic-gate#include <sys/machparam.h>
367c478bdstevel@tonic-gate#include <sys/machcpuvar.h>
377c478bdstevel@tonic-gate#include <sys/machthread.h>
387c478bdstevel@tonic-gate#include <sys/machtrap.h>
397c478bdstevel@tonic-gate#include <sys/privregs.h>
407c478bdstevel@tonic-gate#include <sys/trap.h>
417c478bdstevel@tonic-gate#include <sys/cheetahregs.h>
427c478bdstevel@tonic-gate#include <sys/us3_module.h>
437c478bdstevel@tonic-gate#include <sys/xc_impl.h>
447c478bdstevel@tonic-gate#include <sys/intreg.h>
457c478bdstevel@tonic-gate#include <sys/async.h>
467c478bdstevel@tonic-gate#include <sys/clock.h>
477c478bdstevel@tonic-gate#include <sys/cheetahasm.h>
487c478bdstevel@tonic-gate#include <sys/cmpregs.h>
507c478bdstevel@tonic-gate#ifdef TRAPTRACE
517c478bdstevel@tonic-gate#include <sys/traptrace.h>
527c478bdstevel@tonic-gate#endif /* TRAPTRACE */
547c478bdstevel@tonic-gate/* BEGIN CSTYLED */
567c478bdstevel@tonic-gate#define	DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3)			\
577c478bdstevel@tonic-gate	ldxa	[%g0]ASI_DCU, tmp1					;\
587c478bdstevel@tonic-gate	btst	DCU_DC, tmp1		/* is dcache enabled? */	;\
597c478bdstevel@tonic-gate	bz,pn	%icc, 1f						;\
607c478bdstevel@tonic-gate	ASM_LD(tmp1, dcache_linesize)					;\
617c478bdstevel@tonic-gate	ASM_LD(tmp2, dflush_type)					;\
627c478bdstevel@tonic-gate	cmp	tmp2, FLUSHPAGE_TYPE					;\
637c478bdstevel@tonic-gate	be,pt	%icc, 2f						;\
647c478bdstevel@tonic-gate	nop								;\
657c478bdstevel@tonic-gate	sllx	arg1, CHEETAH_DC_VBIT_SHIFT, arg1/* tag to compare */	;\
667c478bdstevel@tonic-gate	ASM_LD(tmp3, dcache_size)					;\
677c478bdstevel@tonic-gate	cmp	tmp2, FLUSHMATCH_TYPE					;\
687c478bdstevel@tonic-gate	be,pt	%icc, 3f						;\
697c478bdstevel@tonic-gate	nop								;\
707c478bdstevel@tonic-gate	/*								\
717c478bdstevel@tonic-gate	 * flushtype = FLUSHALL_TYPE, flush the whole thing		\
727c478bdstevel@tonic-gate	 * tmp3 = cache size						\
737c478bdstevel@tonic-gate	 * tmp1 = cache line size					\
747c478bdstevel@tonic-gate	 */								\
757c478bdstevel@tonic-gate	sub	tmp3, tmp1, tmp2					;\
767c478bdstevel@tonic-gate4:									\
777c478bdstevel@tonic-gate	stxa	%g0, [tmp2]ASI_DC_TAG					;\
787c478bdstevel@tonic-gate	membar	#Sync							;\
797c478bdstevel@tonic-gate	cmp	%g0, tmp2						;\
807c478bdstevel@tonic-gate	bne,pt	%icc, 4b						;\
817c478bdstevel@tonic-gate	sub	tmp2, tmp1, tmp2					;\
827c478bdstevel@tonic-gate	ba,pt	%icc, 1f						;\
837c478bdstevel@tonic-gate	nop								;\
847c478bdstevel@tonic-gate	/*								\
857c478bdstevel@tonic-gate	 * flushtype = FLUSHPAGE_TYPE					\
867c478bdstevel@tonic-gate	 * arg1 = pfn							\
877c478bdstevel@tonic-gate	 * arg2 = virtual color						\
887c478bdstevel@tonic-gate	 * tmp1 = cache line size					\
897c478bdstevel@tonic-gate	 * tmp2 = tag from cache					\
907c478bdstevel@tonic-gate	 * tmp3 = counter						\
917c478bdstevel@tonic-gate	 */								\
927c478bdstevel@tonic-gate2:									\
937c478bdstevel@tonic-gate	set	MMU_PAGESIZE, tmp3					;\
947c478bdstevel@tonic-gate        sllx    arg1, MMU_PAGESHIFT, arg1  /* pfn to 43 bit PA	   */   ;\
957c478bdstevel@tonic-gate	sub	tmp3, tmp1, tmp3					;\
967c478bdstevel@tonic-gate4:									\
977c478bdstevel@tonic-gate	stxa	%g0, [arg1 + tmp3]ASI_DC_INVAL				;\
987c478bdstevel@tonic-gate	membar	#Sync							;\
997c478bdstevel@tonic-gate5:									\
1007c478bdstevel@tonic-gate	cmp	%g0, tmp3						;\
1017c478bdstevel@tonic-gate	bnz,pt	%icc, 4b		/* branch if not done */	;\
1027c478bdstevel@tonic-gate	sub	tmp3, tmp1, tmp3					;\
1037c478bdstevel@tonic-gate	ba,pt	%icc, 1f						;\
1047c478bdstevel@tonic-gate	nop								;\
1057c478bdstevel@tonic-gate	/*								\
1067c478bdstevel@tonic-gate	 * flushtype = FLUSHMATCH_TYPE					\
1077c478bdstevel@tonic-gate	 * arg1 = tag to compare against				\
1087c478bdstevel@tonic-gate	 * tmp1 = cache line size					\
1097c478bdstevel@tonic-gate	 * tmp3 = cache size						\
1107c478bdstevel@tonic-gate	 * arg2 = counter						\
1117c478bdstevel@tonic-gate	 * tmp2 = cache tag						\
1127c478bdstevel@tonic-gate	 */								\
1137c478bdstevel@tonic-gate3:									\
1147c478bdstevel@tonic-gate	sub	tmp3, tmp1, arg2					;\
1157c478bdstevel@tonic-gate4:									\
1167c478bdstevel@tonic-gate	ldxa	[arg2]ASI_DC_TAG, tmp2		/* read tag */		;\
1177c478bdstevel@tonic-gate	btst	CHEETAH_DC_VBIT_MASK, tmp2				;\
1187c478bdstevel@tonic-gate	bz,pn	%icc, 5f		/* br if no valid sub-blocks */	;\
1197c478bdstevel@tonic-gate	andn	tmp2, CHEETAH_DC_VBIT_MASK, tmp2 /* clear out v bits */	;\
1207c478bdstevel@tonic-gate	cmp	tmp2, arg1						;\
1217c478bdstevel@tonic-gate	bne,pn	%icc, 5f		/* branch if tag miss */	;\
1227c478bdstevel@tonic-gate	nop								;\
1237c478bdstevel@tonic-gate	stxa	%g0, [arg2]ASI_DC_TAG					;\
1247c478bdstevel@tonic-gate	membar	#Sync							;\
1257c478bdstevel@tonic-gate5:									\
1267c478bdstevel@tonic-gate	cmp	%g0, arg2						;\
1277c478bdstevel@tonic-gate	bne,pt	%icc, 4b		/* branch if not done */	;\
1287c478bdstevel@tonic-gate	sub	arg2, tmp1, arg2					;\
131444ce08Donghai Qiao/*
132444ce08Donghai Qiao * macro that flushes the entire dcache color
133444ce08Donghai Qiao * dcache size = 64K, one way 16K
134bd963cbEthindra Ramamurthy *
135bd963cbEthindra Ramamurthy * In:
136bd963cbEthindra Ramamurthy *    arg = virtual color register (not clobbered)
137bd963cbEthindra Ramamurthy *    way = way#, can either be a constant or a register (not clobbered)
138bd963cbEthindra Ramamurthy *    tmp1, tmp2, tmp3 = scratch registers
139bd963cbEthindra Ramamurthy *
140444ce08Donghai Qiao */
141444ce08Donghai Qiao#define DCACHE_FLUSHCOLOR(arg, way, tmp1, tmp2, tmp3)			\
142444ce08Donghai Qiao	ldxa	[%g0]ASI_DCU, tmp1;					\
143444ce08Donghai Qiao	btst	DCU_DC, tmp1;		/* is dcache enabled? */	\
144444ce08Donghai Qiao	bz,pn	%icc, 1f;						\
145444ce08Donghai Qiao	ASM_LD(tmp1, dcache_linesize)					\
146444ce08Donghai Qiao	/*								\
147444ce08Donghai Qiao	 * arg = virtual color						\
148444ce08Donghai Qiao	 * tmp1 = cache line size					\
149444ce08Donghai Qiao	 */								\
150bd963cbEthindra Ramamurthy	sllx	arg, MMU_PAGESHIFT, tmp2; /* color to dcache page */	\
151444ce08Donghai Qiao	mov	way, tmp3;						\
152bd963cbEthindra Ramamurthy	sllx	tmp3, 14, tmp3;		  /* One way 16K */		\
153bd963cbEthindra Ramamurthy	or	tmp2, tmp3, tmp3;					\
154bd963cbEthindra Ramamurthy	set	MMU_PAGESIZE, tmp2;					\
155bd963cbEthindra Ramamurthy	/*								\
156bd963cbEthindra Ramamurthy	 * tmp2 = page size						\
157bd963cbEthindra Ramamurthy	 * tmp3 =  cached page in dcache				\
158bd963cbEthindra Ramamurthy	 */								\
159444ce08Donghai Qiao	sub	tmp2, tmp1, tmp2;					\
160444ce08Donghai Qiao2:									\
161bd963cbEthindra Ramamurthy	stxa	%g0, [tmp3 + tmp2]ASI_DC_TAG;				\
162444ce08Donghai Qiao	membar	#Sync;							\
163444ce08Donghai Qiao	cmp	%g0, tmp2;						\
164444ce08Donghai Qiao	bne,pt	%icc, 2b;						\
165bd963cbEthindra Ramamurthy	sub	tmp2, tmp1, tmp2;					\
166444ce08Donghai Qiao1:
1687c478bdstevel@tonic-gate/* END CSTYLED */
1717c478bdstevel@tonic-gate * Cheetah MMU and Cache operations.
1727c478bdstevel@tonic-gate */
1747c478bdstevel@tonic-gate	ENTRY_NP(vtag_flushpage)
1757c478bdstevel@tonic-gate	/*
1767c478bdstevel@tonic-gate	 * flush page from the tlb
1777c478bdstevel@tonic-gate	 *
1787c478bdstevel@tonic-gate	 * %o0 = vaddr
1791e2e7a7huah	 * %o1 = sfmmup
1807c478bdstevel@tonic-gate	 */
1817c478bdstevel@tonic-gate	rdpr	%pstate, %o5
1827c478bdstevel@tonic-gate#ifdef DEBUG
1831e2e7a7huah	PANIC_IF_INTR_DISABLED_PSTR(%o5, u3_di_label0, %g1)
1847c478bdstevel@tonic-gate#endif /* DEBUG */
1857c478bdstevel@tonic-gate	/*
1867c478bdstevel@tonic-gate	 * disable ints
1877c478bdstevel@tonic-gate	 */
1887c478bdstevel@tonic-gate	andn	%o5, PSTATE_IE, %o4
1897c478bdstevel@tonic-gate	wrpr	%o4, 0, %pstate
1917c478bdstevel@tonic-gate	/*
1927c478bdstevel@tonic-gate	 * Then, blow out the tlb
1937c478bdstevel@tonic-gate	 * Interrupts are disabled to prevent the primary ctx register
1947c478bdstevel@tonic-gate	 * from changing underneath us.
1957c478bdstevel@tonic-gate	 */
1961e2e7a7huah	sethi   %hi(ksfmmup), %o3
1971e2e7a7huah        ldx     [%o3 + %lo(ksfmmup)], %o3
1981e2e7a7huah        cmp     %o3, %o1
1991e2e7a7huah        bne,pt   %xcc, 1f			! if not kernel as, go to 1
2001e2e7a7huah	  sethi	%hi(FLUSH_ADDR), %o3
2017c478bdstevel@tonic-gate	/*
2021e2e7a7huah	 * For Kernel demaps use primary. type = page implicitly
2037c478bdstevel@tonic-gate	 */
2047c478bdstevel@tonic-gate	stxa	%g0, [%o0]ASI_DTLB_DEMAP	/* dmmu flush for KCONTEXT */
2057c478bdstevel@tonic-gate	stxa	%g0, [%o0]ASI_ITLB_DEMAP	/* immu flush for KCONTEXT */
2067c478bdstevel@tonic-gate	flush	%o3
2071e2e7a7huah	retl
2081e2e7a7huah	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
2107c478bdstevel@tonic-gate	/*
2117c478bdstevel@tonic-gate	 * User demap.  We need to set the primary context properly.
2127c478bdstevel@tonic-gate	 * Secondary context cannot be used for Cheetah IMMU.
2137c478bdstevel@tonic-gate	 * %o0 = vaddr
2141e2e7a7huah	 * %o1 = sfmmup
2157c478bdstevel@tonic-gate	 * %o3 = FLUSH_ADDR
2167c478bdstevel@tonic-gate	 */
2171e2e7a7huah	SFMMU_CPU_CNUM(%o1, %g1, %g2)		! %g1 = sfmmu cnum on this CPU
2191e2e7a7huah	ldub	[%o1 + SFMMU_CEXT], %o4		! %o4 = sfmmup->sfmmu_cext
2207c478bdstevel@tonic-gate	sll	%o4, CTXREG_EXT_SHIFT, %o4
221febcc4ajimand	or	%g1, %o4, %g1			! %g1 = primary pgsz | cnum
2237c478bdstevel@tonic-gate	wrpr	%g0, 1, %tl
2247c478bdstevel@tonic-gate	set	MMU_PCONTEXT, %o4
2257c478bdstevel@tonic-gate	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %o0, %o0
2261e2e7a7huah	ldxa	[%o4]ASI_DMMU, %o2		! %o2 = save old ctxnum
227febcc4ajimand	srlx	%o2, CTXREG_NEXT_SHIFT, %o1	! need to preserve nucleus pgsz
228febcc4ajimand	sllx	%o1, CTXREG_NEXT_SHIFT, %o1	! %o1 = nucleus pgsz
229febcc4ajimand	or	%g1, %o1, %g1			! %g1 = nucleus pgsz | primary pgsz | cnum
2301e2e7a7huah	stxa	%g1, [%o4]ASI_DMMU		! wr new ctxum
2327c478bdstevel@tonic-gate	stxa	%g0, [%o0]ASI_DTLB_DEMAP
2337c478bdstevel@tonic-gate	stxa	%g0, [%o0]ASI_ITLB_DEMAP
2347c478bdstevel@tonic-gate	stxa	%o2, [%o4]ASI_DMMU		/* restore old ctxnum */
2357c478bdstevel@tonic-gate	flush	%o3
2367c478bdstevel@tonic-gate	wrpr	%g0, 0, %tl
2387c478bdstevel@tonic-gate	retl
2397c478bdstevel@tonic-gate	wrpr	%g0, %o5, %pstate		/* enable interrupts */
2407c478bdstevel@tonic-gate	SET_SIZE(vtag_flushpage)
2427c478bdstevel@tonic-gate	ENTRY_NP2(vtag_flushall, demap_all)
2437c478bdstevel@tonic-gate	/*
2447c478bdstevel@tonic-gate	 * flush the tlb
2457c478bdstevel@tonic-gate	 */
2467c478bdstevel@tonic-gate	sethi	%hi(FLUSH_ADDR), %o3
2477c478bdstevel@tonic-gate	set	DEMAP_ALL_TYPE, %g1
2487c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_DTLB_DEMAP
2497c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_ITLB_DEMAP
2507c478bdstevel@tonic-gate	flush	%o3
2517c478bdstevel@tonic-gate	retl
2527c478bdstevel@tonic-gate	nop
2537c478bdstevel@tonic-gate	SET_SIZE(demap_all)
2547c478bdstevel@tonic-gate	SET_SIZE(vtag_flushall)
2577c478bdstevel@tonic-gate	ENTRY_NP(vtag_flushpage_tl1)
2587c478bdstevel@tonic-gate	/*
2597c478bdstevel@tonic-gate	 * x-trap to flush page from tlb and tsb
2607c478bdstevel@tonic-gate	 *
2617c478bdstevel@tonic-gate	 * %g1 = vaddr, zero-extended on 32-bit kernel
2621e2e7a7huah	 * %g2 = sfmmup
2637c478bdstevel@tonic-gate	 *
2647c478bdstevel@tonic-gate	 * assumes TSBE_TAG = 0
2657c478bdstevel@tonic-gate	 */
2667c478bdstevel@tonic-gate	srln	%g1, MMU_PAGESHIFT, %g1
2681e2e7a7huah	sethi   %hi(ksfmmup), %g3
2691e2e7a7huah        ldx     [%g3 + %lo(ksfmmup)], %g3
2701e2e7a7huah        cmp     %g3, %g2
2711e2e7a7huah        bne,pt	%xcc, 1f                        ! if not kernel as, go to 1
2727c478bdstevel@tonic-gate	  slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
2747c478bdstevel@tonic-gate	/* We need to demap in the kernel context */
2757c478bdstevel@tonic-gate	or	DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1
2767c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_DTLB_DEMAP
2777c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_ITLB_DEMAP
2787c478bdstevel@tonic-gate	retry
2807c478bdstevel@tonic-gate	/* We need to demap in a user context */
2817c478bdstevel@tonic-gate	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1
2831e2e7a7huah	SFMMU_CPU_CNUM(%g2, %g6, %g3)	! %g6 = sfmmu cnum on this CPU
2851e2e7a7huah	ldub	[%g2 + SFMMU_CEXT], %g4		! %g4 = sfmmup->cext
2867c478bdstevel@tonic-gate	sll	%g4, CTXREG_EXT_SHIFT, %g4
2871e2e7a7huah	or	%g6, %g4, %g6			! %g6 = pgsz | cnum
2897c478bdstevel@tonic-gate	set	MMU_PCONTEXT, %g4
2907c478bdstevel@tonic-gate	ldxa	[%g4]ASI_DMMU, %g5		/* rd old ctxnum */
291febcc4ajimand	srlx	%g5, CTXREG_NEXT_SHIFT, %g2	/* %g2 = nucleus pgsz */
292febcc4ajimand	sllx	%g2, CTXREG_NEXT_SHIFT, %g2	/* preserve nucleus pgsz */
293febcc4ajimand	or	%g6, %g2, %g6			/* %g6 = nucleus pgsz | primary pgsz | cnum */
2941e2e7a7huah	stxa	%g6, [%g4]ASI_DMMU		/* wr new ctxum */
2957c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_DTLB_DEMAP
2967c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_ITLB_DEMAP
2977c478bdstevel@tonic-gate	stxa	%g5, [%g4]ASI_DMMU		/* restore old ctxnum */
2987c478bdstevel@tonic-gate	retry
2997c478bdstevel@tonic-gate	SET_SIZE(vtag_flushpage_tl1)
3027c478bdstevel@tonic-gate	ENTRY_NP(vtag_flush_pgcnt_tl1)
3037c478bdstevel@tonic-gate	/*
3047c478bdstevel@tonic-gate	 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb
3057c478bdstevel@tonic-gate	 *
3067c478bdstevel@tonic-gate	 * %g1 = vaddr, zero-extended on 32-bit kernel
3071e2e7a7huah	 * %g2 = <sfmmup58|pgcnt6>, (pgcnt - 1) is pass'ed in via pgcnt6 bits.
3087c478bdstevel@tonic-gate	 *
3097c478bdstevel@tonic-gate	 * NOTE: this handler relies on the fact that no
3107c478bdstevel@tonic-gate	 *	interrupts or traps can occur during the loop
3117c478bdstevel@tonic-gate	 *	issuing the TLB_DEMAP operations. It is assumed
3127c478bdstevel@tonic-gate	 *	that interrupts are disabled and this code is
3137c478bdstevel@tonic-gate	 *	fetching from the kernel locked text address.
3147c478bdstevel@tonic-gate	 *
3157c478bdstevel@tonic-gate	 * assumes TSBE_TAG = 0
3167c478bdstevel@tonic-gate	 */
3171e2e7a7huah	set	SFMMU_PGCNT_MASK, %g4
3181e2e7a7huah	and	%g4, %g2, %g3			/* g3 = pgcnt - 1 */
3191e2e7a7huah	add	%g3, 1, %g3			/* g3 = pgcnt */
3211e2e7a7huah	andn	%g2, SFMMU_PGCNT_MASK, %g2	/* g2 = sfmmup */
3227c478bdstevel@tonic-gate	srln	%g1, MMU_PAGESHIFT, %g1
3241e2e7a7huah	sethi   %hi(ksfmmup), %g4
3251e2e7a7huah        ldx     [%g4 + %lo(ksfmmup)], %g4
3261e2e7a7huah        cmp     %g4, %g2
3271e2e7a7huah        bne,pn   %xcc, 1f			/* if not kernel as, go to 1 */
3287c478bdstevel@tonic-gate	  slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
3307c478bdstevel@tonic-gate	/* We need to demap in the kernel context */
3317c478bdstevel@tonic-gate	or	DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1
3327c478bdstevel@tonic-gate	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
3331e2e7a7huah	sethi   %hi(FLUSH_ADDR), %g5
3357c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_DTLB_DEMAP
3367c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_ITLB_DEMAP
3371e2e7a7huah	flush	%g5				! flush required by immu
3397c478bdstevel@tonic-gate	deccc	%g3				/* decr pgcnt */
3407c478bdstevel@tonic-gate	bnz,pt	%icc,4b
3417c478bdstevel@tonic-gate	  add	%g1, %g2, %g1			/* next page */
3427c478bdstevel@tonic-gate	retry
3441e2e7a7huah	/*
3451e2e7a7huah	 * We need to demap in a user context
3461e2e7a7huah	 *
3471e2e7a7huah	 * g2 = sfmmup
3481e2e7a7huah	 * g3 = pgcnt
3491e2e7a7huah	 */
3501e2e7a7huah	SFMMU_CPU_CNUM(%g2, %g5, %g6)		! %g5 = sfmmu cnum on this CPU
3521e2e7a7huah	or	DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1
3541e2e7a7huah	ldub	[%g2 + SFMMU_CEXT], %g4		! %g4 = sfmmup->cext
3557c478bdstevel@tonic-gate	sll	%g4, CTXREG_EXT_SHIFT, %g4
3561e2e7a7huah	or	%g5, %g4, %g5
3587c478bdstevel@tonic-gate	set	MMU_PCONTEXT, %g4
3591e2e7a7huah	ldxa	[%g4]ASI_DMMU, %g6		/* rd old ctxnum */
360febcc4ajimand	srlx	%g6, CTXREG_NEXT_SHIFT, %g2	/* %g2 = nucleus pgsz */
361febcc4ajimand	sllx	%g2, CTXREG_NEXT_SHIFT, %g2	/* preserve nucleus pgsz */
362febcc4ajimand	or	%g5, %g2, %g5			/* %g5 = nucleus pgsz | primary pgsz | cnum */
3631e2e7a7huah	stxa	%g5, [%g4]ASI_DMMU		/* wr new ctxum */
3657c478bdstevel@tonic-gate	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
3661e2e7a7huah	sethi   %hi(FLUSH_ADDR), %g5
3687c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_DTLB_DEMAP
3697c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_ITLB_DEMAP
3701e2e7a7huah	flush	%g5				! flush required by immu
3727c478bdstevel@tonic-gate	deccc	%g3				/* decr pgcnt */
3737c478bdstevel@tonic-gate	bnz,pt	%icc,3b
3747c478bdstevel@tonic-gate	  add	%g1, %g2, %g1			/* next page */
3761e2e7a7huah	stxa	%g6, [%g4]ASI_DMMU		/* restore old ctxnum */
3777c478bdstevel@tonic-gate	retry
3787c478bdstevel@tonic-gate	SET_SIZE(vtag_flush_pgcnt_tl1)
3807c478bdstevel@tonic-gate	ENTRY_NP(vtag_flushall_tl1)
3817c478bdstevel@tonic-gate	/*
3827c478bdstevel@tonic-gate	 * x-trap to flush tlb
3837c478bdstevel@tonic-gate	 */
3847c478bdstevel@tonic-gate	set	DEMAP_ALL_TYPE, %g4
3857c478bdstevel@tonic-gate	stxa	%g0, [%g4]ASI_DTLB_DEMAP
3867c478bdstevel@tonic-gate	stxa	%g0, [%g4]ASI_ITLB_DEMAP
3877c478bdstevel@tonic-gate	retry
3887c478bdstevel@tonic-gate	SET_SIZE(vtag_flushall_tl1)
3927c478bdstevel@tonic-gate * vac_flushpage(pfnum, color)
3937c478bdstevel@tonic-gate *	Flush 1 8k page of the D-$ with physical page = pfnum
3947c478bdstevel@tonic-gate *	Algorithm:
3957c478bdstevel@tonic-gate *		The cheetah dcache is a 64k psuedo 4 way accaociative cache.
3967c478bdstevel@tonic-gate *		It is virtual indexed, physically tagged cache.
3977c478bdstevel@tonic-gate */
3987c478bdstevel@tonic-gate	.seg	".data"
3997c478bdstevel@tonic-gate	.align	8
4007c478bdstevel@tonic-gate	.global	dflush_type
4027c478bdstevel@tonic-gate	.word	FLUSHPAGE_TYPE
4047c478bdstevel@tonic-gate	ENTRY(vac_flushpage)
4057c478bdstevel@tonic-gate	/*
4067c478bdstevel@tonic-gate	 * flush page from the d$
4077c478bdstevel@tonic-gate	 *
4087c478bdstevel@tonic-gate	 * %o0 = pfnum, %o1 = color
4097c478bdstevel@tonic-gate	 */
4107c478bdstevel@tonic-gate	DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4)
4117c478bdstevel@tonic-gate	retl
4127c478bdstevel@tonic-gate	  nop
4137c478bdstevel@tonic-gate	SET_SIZE(vac_flushpage)
4167c478bdstevel@tonic-gate	ENTRY_NP(vac_flushpage_tl1)
4177c478bdstevel@tonic-gate	/*
4187c478bdstevel@tonic-gate	 * x-trap to flush page from the d$
4197c478bdstevel@tonic-gate	 *
4207c478bdstevel@tonic-gate	 * %g1 = pfnum, %g2 = color
4217c478bdstevel@tonic-gate	 */
4227c478bdstevel@tonic-gate	DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5)
4237c478bdstevel@tonic-gate	retry
4247c478bdstevel@tonic-gate	SET_SIZE(vac_flushpage_tl1)
4277c478bdstevel@tonic-gate	ENTRY(vac_flushcolor)
4287c478bdstevel@tonic-gate	/*
429444ce08Donghai Qiao	 * %o0 = vcolor
4307c478bdstevel@tonic-gate	 */
431444ce08Donghai Qiao	DCACHE_FLUSHCOLOR(%o0, 0, %o1, %o2, %o3)
432444ce08Donghai Qiao	DCACHE_FLUSHCOLOR(%o0, 1, %o1, %o2, %o3)
433444ce08Donghai Qiao	DCACHE_FLUSHCOLOR(%o0, 2, %o1, %o2, %o3)
434444ce08Donghai Qiao	DCACHE_FLUSHCOLOR(%o0, 3, %o1, %o2, %o3)
4357c478bdstevel@tonic-gate	retl
4367c478bdstevel@tonic-gate	  nop
4377c478bdstevel@tonic-gate	SET_SIZE(vac_flushcolor)
4407c478bdstevel@tonic-gate	ENTRY(vac_flushcolor_tl1)
4417c478bdstevel@tonic-gate	/*
4427c478bdstevel@tonic-gate	 * %g1 = vcolor
4437c478bdstevel@tonic-gate	 */
444444ce08Donghai Qiao	DCACHE_FLUSHCOLOR(%g1, 0, %g2, %g3, %g4)
445444ce08Donghai Qiao	DCACHE_FLUSHCOLOR(%g1, 1, %g2, %g3, %g4)
446444ce08Donghai Qiao	DCACHE_FLUSHCOLOR(%g1, 2, %g2, %g3, %g4)
447444ce08Donghai Qiao	DCACHE_FLUSHCOLOR(%g1, 3, %g2, %g3, %g4)
4487c478bdstevel@tonic-gate	retry
4497c478bdstevel@tonic-gate	SET_SIZE(vac_flushcolor_tl1)
4527c478bdstevel@tonic-gate * Determine whether or not the IDSR is busy.
4537c478bdstevel@tonic-gate * Entry: no arguments
4547c478bdstevel@tonic-gate * Returns: 1 if busy, 0 otherwise
4557c478bdstevel@tonic-gate */
4567c478bdstevel@tonic-gate	ENTRY(idsr_busy)
4577c478bdstevel@tonic-gate	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
4587c478bdstevel@tonic-gate	clr	%o0
4597c478bdstevel@tonic-gate	btst	IDSR_BUSY, %g1
4607c478bdstevel@tonic-gate	bz,a,pt	%xcc, 1f
4617c478bdstevel@tonic-gate	mov	1, %o0
4637c478bdstevel@tonic-gate	retl
4647c478bdstevel@tonic-gate	nop
4657c478bdstevel@tonic-gate	SET_SIZE(idsr_busy)
4677c478bdstevel@tonic-gate	.global _dispatch_status_busy
4697c478bdstevel@tonic-gate	.asciz	"ASI_INTR_DISPATCH_STATUS error: busy"
4707c478bdstevel@tonic-gate	.align	4
4737c478bdstevel@tonic-gate * Setup interrupt dispatch data registers
4747c478bdstevel@tonic-gate * Entry:
4757c478bdstevel@tonic-gate *	%o0 - function or inumber to call
4767c478bdstevel@tonic-gate *	%o1, %o2 - arguments (2 uint64_t's)
4777c478bdstevel@tonic-gate */
4787c478bdstevel@tonic-gate	.seg "text"
4807c478bdstevel@tonic-gate	ENTRY(init_mondo)
4817c478bdstevel@tonic-gate#ifdef DEBUG
4827c478bdstevel@tonic-gate	!
4837c478bdstevel@tonic-gate	! IDSR should not be busy at the moment
4847c478bdstevel@tonic-gate	!
4857c478bdstevel@tonic-gate	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
4867c478bdstevel@tonic-gate	btst	IDSR_BUSY, %g1
4877c478bdstevel@tonic-gate	bz,pt	%xcc, 1f
4887c478bdstevel@tonic-gate	nop
4897c478bdstevel@tonic-gate	sethi	%hi(_dispatch_status_busy), %o0
4907c478bdstevel@tonic-gate	call	panic
4917c478bdstevel@tonic-gate	or	%o0, %lo(_dispatch_status_busy), %o0
4927c478bdstevel@tonic-gate#endif /* DEBUG */
4947c478bdstevel@tonic-gate	ALTENTRY(init_mondo_nocheck)
4957c478bdstevel@tonic-gate	!
4967c478bdstevel@tonic-gate	! interrupt vector dispatch data reg 0
4977c478bdstevel@tonic-gate	!
4997c478bdstevel@tonic-gate	mov	IDDR_0, %g1
5007c478bdstevel@tonic-gate	mov	IDDR_1, %g2
5017c478bdstevel@tonic-gate	mov	IDDR_2, %g3
5027c478bdstevel@tonic-gate	stxa	%o0, [%g1]ASI_INTR_DISPATCH
5047c478bdstevel@tonic-gate	!
5057c478bdstevel@tonic-gate	! interrupt vector dispatch data reg 1
5067c478bdstevel@tonic-gate	!
5077c478bdstevel@tonic-gate	stxa	%o1, [%g2]ASI_INTR_DISPATCH
5097c478bdstevel@tonic-gate	!
5107c478bdstevel@tonic-gate	! interrupt vector dispatch data reg 2
5117c478bdstevel@tonic-gate	!
5127c478bdstevel@tonic-gate	stxa	%o2, [%g3]ASI_INTR_DISPATCH
5147c478bdstevel@tonic-gate	membar	#Sync
5157c478bdstevel@tonic-gate	retl
5167c478bdstevel@tonic-gate	nop
5177c478bdstevel@tonic-gate	SET_SIZE(init_mondo_nocheck)
5187c478bdstevel@tonic-gate	SET_SIZE(init_mondo)
5217c478bdstevel@tonic-gate#if !(defined(JALAPENO) || defined(SERRANO))
5247c478bdstevel@tonic-gate * Ship mondo to aid using busy/nack pair bn
5257c478bdstevel@tonic-gate */
5267c478bdstevel@tonic-gate	ENTRY_NP(shipit)
5277c478bdstevel@tonic-gate	sll	%o0, IDCR_PID_SHIFT, %g1	! IDCR<18:14> = agent id
5287c478bdstevel@tonic-gate	sll	%o1, IDCR_BN_SHIFT, %g2		! IDCR<28:24> = b/n pair
5297c478bdstevel@tonic-gate	or	%g1, IDCR_OFFSET, %g1		! IDCR<13:0> = 0x70
5307c478bdstevel@tonic-gate	or	%g1, %g2, %g1
5317c478bdstevel@tonic-gate	stxa	%g0, [%g1]ASI_INTR_DISPATCH	! interrupt vector dispatch
5327c478bdstevel@tonic-gate	membar	#Sync
5337c478bdstevel@tonic-gate	retl
5347c478bdstevel@tonic-gate	nop
5357c478bdstevel@tonic-gate	SET_SIZE(shipit)
5377c478bdstevel@tonic-gate#endif	/* !(JALAPENO || SERRANO) */
5417c478bdstevel@tonic-gate * flush_instr_mem:
5427c478bdstevel@tonic-gate *	Flush 1 page of the I-$ starting at vaddr
5437c478bdstevel@tonic-gate * 	%o0 vaddr
5447c478bdstevel@tonic-gate *	%o1 bytes to be flushed
5457c478bdstevel@tonic-gate * UltraSPARC-III maintains consistency of the on-chip Instruction Cache with
5467c478bdstevel@tonic-gate * the stores from all processors so that a FLUSH instruction is only needed
5477c478bdstevel@tonic-gate * to ensure pipeline is consistent. This means a single flush is sufficient at
5487c478bdstevel@tonic-gate * the end of a sequence of stores that updates the instruction stream to
5497c478bdstevel@tonic-gate * ensure correct operation.
5507c478bdstevel@tonic-gate */
5527c478bdstevel@tonic-gate	ENTRY(flush_instr_mem)
5536336095mp	flush	%o0			! address irrelevant
5547c478bdstevel@tonic-gate	retl
5557c478bdstevel@tonic-gate	nop
5567c478bdstevel@tonic-gate	SET_SIZE(flush_instr_mem)
5597c478bdstevel@tonic-gate#if defined(CPU_IMP_ECACHE_ASSOC)
5617c478bdstevel@tonic-gate	ENTRY(get_ecache_ctrl)
5627c478bdstevel@tonic-gate	GET_CPU_IMPL(%o0)
5637c478bdstevel@tonic-gate	cmp	%o0, JAGUAR_IMPL
5647c478bdstevel@tonic-gate	!
5657c478bdstevel@tonic-gate	! Putting an ASI access in the delay slot may
5667c478bdstevel@tonic-gate	! cause it to be accessed, even when annulled.
5677c478bdstevel@tonic-gate	!
5687c478bdstevel@tonic-gate	bne	1f
5697c478bdstevel@tonic-gate	  nop
5707c478bdstevel@tonic-gate	ldxa	[%g0]ASI_EC_CFG_TIMING, %o0	! read Jaguar shared E$ ctrl reg
5717c478bdstevel@tonic-gate	b	2f
5727c478bdstevel@tonic-gate	  nop
5747c478bdstevel@tonic-gate	ldxa	[%g0]ASI_EC_CTRL, %o0		! read Ch/Ch+ E$ control reg
5767c478bdstevel@tonic-gate	retl
5777c478bdstevel@tonic-gate	  nop
5787c478bdstevel@tonic-gate	SET_SIZE(get_ecache_ctrl)
5807c478bdstevel@tonic-gate#endif	/* CPU_IMP_ECACHE_ASSOC */
5837c478bdstevel@tonic-gate#if !(defined(JALAPENO) || defined(SERRANO))
5867c478bdstevel@tonic-gate * flush_ecache:
5877c478bdstevel@tonic-gate *	%o0 - 64 bit physical address
5887c478bdstevel@tonic-gate *	%o1 - ecache size
5897c478bdstevel@tonic-gate *	%o2 - ecache linesize
5907c478bdstevel@tonic-gate */
5927c478bdstevel@tonic-gate	ENTRY(flush_ecache)
5947c478bdstevel@tonic-gate	/*
5957c478bdstevel@tonic-gate	 * For certain CPU implementations, we have to flush the L2 cache
5967c478bdstevel@tonic-gate	 * before flushing the ecache.
5977c478bdstevel@tonic-gate	 */
5987c478bdstevel@tonic-gate	PN_L2_FLUSHALL(%g3, %g4, %g5)
6007c478bdstevel@tonic-gate	/*
6017c478bdstevel@tonic-gate	 * Flush the entire Ecache using displacement flush.
6027c478bdstevel@tonic-gate	 */
6037c478bdstevel@tonic-gate	ECACHE_FLUSHALL(%o1, %o2, %o0, %o4)
6057c478bdstevel@tonic-gate	retl
6067c478bdstevel@tonic-gate	nop
6077c478bdstevel@tonic-gate	SET_SIZE(flush_ecache)
6097c478bdstevel@tonic-gate#endif	/* !(JALAPENO || SERRANO) */
6127c478bdstevel@tonic-gate	ENTRY(flush_dcache)
6137c478bdstevel@tonic-gate	ASM_LD(%o0, dcache_size)
6147c478bdstevel@tonic-gate	ASM_LD(%o1, dcache_linesize)
6157c478bdstevel@tonic-gate	CH_DCACHE_FLUSHALL(%o0, %o1, %o2)
6167c478bdstevel@tonic-gate	retl
6177c478bdstevel@tonic-gate	nop
6187c478bdstevel@tonic-gate	SET_SIZE(flush_dcache)
6217c478bdstevel@tonic-gate	ENTRY(flush_icache)
6227c478bdstevel@tonic-gate	GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, flush_icache_1);
6237c478bdstevel@tonic-gate	ld	[%o0 + CHPR_ICACHE_LINESIZE], %o1
6247c478bdstevel@tonic-gate	ba,pt	%icc, 2f
6257c478bdstevel@tonic-gate	  ld	[%o0 + CHPR_ICACHE_SIZE], %o0
6277c478bdstevel@tonic-gate	ASM_LD(%o0, icache_size)
6287c478bdstevel@tonic-gate	ASM_LD(%o1, icache_linesize)
6307c478bdstevel@tonic-gate	CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4)
6317c478bdstevel@tonic-gate	retl
6327c478bdstevel@tonic-gate	nop
6337c478bdstevel@tonic-gate	SET_SIZE(flush_icache)
6357c478bdstevel@tonic-gate	ENTRY(kdi_flush_idcache)
6367c478bdstevel@tonic-gate	CH_DCACHE_FLUSHALL(%o0, %o1, %g1)
6377c478bdstevel@tonic-gate	CH_ICACHE_FLUSHALL(%o2, %o3, %g1, %g2)
6387c478bdstevel@tonic-gate	membar	#Sync
6397c478bdstevel@tonic-gate	retl
6407c478bdstevel@tonic-gate	nop
6417c478bdstevel@tonic-gate	SET_SIZE(kdi_flush_idcache)
6437c478bdstevel@tonic-gate	ENTRY(flush_pcache)
6447c478bdstevel@tonic-gate	PCACHE_FLUSHALL(%o0, %o1, %o2)
6457c478bdstevel@tonic-gate	retl
6467c478bdstevel@tonic-gate	nop
6477c478bdstevel@tonic-gate	SET_SIZE(flush_pcache)
6507c478bdstevel@tonic-gate#if defined(CPU_IMP_L1_CACHE_PARITY)
6537c478bdstevel@tonic-gate * Get dcache data and tag.  The Dcache data is a pointer to a ch_dc_data_t
6547c478bdstevel@tonic-gate * structure (see cheetahregs.h):
6557c478bdstevel@tonic-gate * The Dcache *should* be turned off when this code is executed.
6567c478bdstevel@tonic-gate */
6577c478bdstevel@tonic-gate	.align	128
6587c478bdstevel@tonic-gate	ENTRY(get_dcache_dtag)
6597c478bdstevel@tonic-gate	rdpr	%pstate, %o5
6607c478bdstevel@tonic-gate	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
6617c478bdstevel@tonic-gate	wrpr	%g0, %o3, %pstate
6627c478bdstevel@tonic-gate	b	1f
6637c478bdstevel@tonic-gate	  stx	%o0, [%o1 + CH_DC_IDX]
6657c478bdstevel@tonic-gate	.align	128
6677c478bdstevel@tonic-gate	ldxa	[%o0]ASI_DC_TAG, %o2
6687c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_DC_TAG]
6697c478bdstevel@tonic-gate	membar	#Sync
6707c478bdstevel@tonic-gate	ldxa	[%o0]ASI_DC_UTAG, %o2
6717c478bdstevel@tonic-gate	membar	#Sync
6727c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_DC_UTAG]
6737c478bdstevel@tonic-gate	ldxa	[%o0]ASI_DC_SNP_TAG, %o2
6747c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_DC_SNTAG]
6757c478bdstevel@tonic-gate	add	%o1, CH_DC_DATA, %o1
6767c478bdstevel@tonic-gate	clr	%o3
6787c478bdstevel@tonic-gate	membar	#Sync				! required before ASI_DC_DATA
6797c478bdstevel@tonic-gate	ldxa	[%o0 + %o3]ASI_DC_DATA, %o2
6807c478bdstevel@tonic-gate	membar	#Sync				! required after ASI_DC_DATA
6817c478bdstevel@tonic-gate	stx	%o2, [%o1 + %o3]
6827c478bdstevel@tonic-gate	cmp	%o3, CH_DC_DATA_REG_SIZE - 8
6837c478bdstevel@tonic-gate	blt	2b
6847c478bdstevel@tonic-gate	  add	%o3, 8, %o3
6867c478bdstevel@tonic-gate	/*
6877c478bdstevel@tonic-gate	 * Unlike other CPUs in the family, D$ data parity bits for Panther
6887c478bdstevel@tonic-gate	 * do not reside in the microtag. Instead, we have to read them
6897c478bdstevel@tonic-gate	 * using the DC_data_parity bit of ASI_DCACHE_DATA. Also, instead
6907c478bdstevel@tonic-gate	 * of just having 8 parity bits to protect all 32 bytes of data
6917c478bdstevel@tonic-gate	 * per line, we now have 32 bits of parity.
6927c478bdstevel@tonic-gate	 */
6937c478bdstevel@tonic-gate	GET_CPU_IMPL(%o3)
6947c478bdstevel@tonic-gate	cmp	%o3, PANTHER_IMPL
6957c478bdstevel@tonic-gate	bne	4f
6967c478bdstevel@tonic-gate	  clr	%o3
6987c478bdstevel@tonic-gate	/*
6997c478bdstevel@tonic-gate	 * move our pointer to the next field where we store parity bits
7007c478bdstevel@tonic-gate	 * and add the offset of the last parity byte since we will be
7017c478bdstevel@tonic-gate	 * storing all 4 parity bytes within one 64 bit field like this:
7027c478bdstevel@tonic-gate	 *
7037c478bdstevel@tonic-gate	 * +------+------------+------------+------------+------------+
7047c478bdstevel@tonic-gate	 * |  -   | DC_parity  | DC_parity  | DC_parity  | DC_parity  |
7057c478bdstevel@tonic-gate	 * |  -   | for word 3 | for word 2 | for word 1 | for word 0 |
7067c478bdstevel@tonic-gate	 * +------+------------+------------+------------+------------+
7077c478bdstevel@tonic-gate	 *  63:32     31:24        23:16         15:8          7:0
7087c478bdstevel@tonic-gate	 */
7097c478bdstevel@tonic-gate	add	%o1, CH_DC_PN_DATA_PARITY - CH_DC_DATA + 7, %o1
7117c478bdstevel@tonic-gate	/* add the DC_data_parity bit into our working index */
7127c478bdstevel@tonic-gate	mov	1, %o2
7137c478bdstevel@tonic-gate	sll	%o2, PN_DC_DATA_PARITY_BIT_SHIFT, %o2
7147c478bdstevel@tonic-gate	or	%o0, %o2, %o0
7167c478bdstevel@tonic-gate	membar	#Sync				! required before ASI_DC_DATA
7177c478bdstevel@tonic-gate	ldxa	[%o0 + %o3]ASI_DC_DATA, %o2
7187c478bdstevel@tonic-gate	membar	#Sync				! required after ASI_DC_DATA
7197c478bdstevel@tonic-gate	stb	%o2, [%o1]
7207c478bdstevel@tonic-gate	dec	%o1
7217c478bdstevel@tonic-gate	cmp	%o3, CH_DC_DATA_REG_SIZE - 8
7227c478bdstevel@tonic-gate	blt	3b
7237c478bdstevel@tonic-gate	  add	%o3, 8, %o3
7257c478bdstevel@tonic-gate	retl
7267c478bdstevel@tonic-gate	  wrpr	%g0, %o5, %pstate
7277c478bdstevel@tonic-gate	SET_SIZE(get_dcache_dtag)
7317c478bdstevel@tonic-gate * Get icache data and tag.  The data argument is a pointer to a ch_ic_data_t
7327c478bdstevel@tonic-gate * structure (see cheetahregs.h):
7337c478bdstevel@tonic-gate * The Icache *Must* be turned off when this function is called.
7347c478bdstevel@tonic-gate * This is because diagnostic accesses to the Icache interfere with cache
7357c478bdstevel@tonic-gate * consistency.
7367c478bdstevel@tonic-gate */
7377c478bdstevel@tonic-gate	.align	128
7387c478bdstevel@tonic-gate	ENTRY(get_icache_dtag)
7397c478bdstevel@tonic-gate	rdpr	%pstate, %o5
7407c478bdstevel@tonic-gate	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
7417c478bdstevel@tonic-gate	wrpr	%g0, %o3, %pstate
7437c478bdstevel@tonic-gate	stx	%o0, [%o1 + CH_IC_IDX]
7447c478bdstevel@tonic-gate	ldxa	[%o0]ASI_IC_TAG, %o2
7457c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_IC_PATAG]
7467c478bdstevel@tonic-gate	add	%o0, CH_ICTAG_UTAG, %o0
7477c478bdstevel@tonic-gate	ldxa	[%o0]ASI_IC_TAG, %o2
7487c478bdstevel@tonic-gate	add	%o0, (CH_ICTAG_UPPER - CH_ICTAG_UTAG), %o0
7497c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_IC_UTAG]
7507c478bdstevel@tonic-gate	ldxa	[%o0]ASI_IC_TAG, %o2
7517c478bdstevel@tonic-gate	add	%o0, (CH_ICTAG_LOWER - CH_ICTAG_UPPER), %o0
7527c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_IC_UPPER]
7537c478bdstevel@tonic-gate	ldxa	[%o0]ASI_IC_TAG, %o2
7547c478bdstevel@tonic-gate	andn	%o0, CH_ICTAG_TMASK, %o0
7557c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_IC_LOWER]
7567c478bdstevel@tonic-gate	ldxa	[%o0]ASI_IC_SNP_TAG, %o2
7577c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_IC_SNTAG]
7587c478bdstevel@tonic-gate	add	%o1, CH_IC_DATA, %o1
7597c478bdstevel@tonic-gate	clr	%o3
7617c478bdstevel@tonic-gate	ldxa	[%o0 + %o3]ASI_IC_DATA, %o2
7627c478bdstevel@tonic-gate	stx	%o2, [%o1 + %o3]
7637c478bdstevel@tonic-gate	cmp	%o3, PN_IC_DATA_REG_SIZE - 8
7647c478bdstevel@tonic-gate	blt	2b
7657c478bdstevel@tonic-gate	  add	%o3, 8, %o3
7677c478bdstevel@tonic-gate	retl
7687c478bdstevel@tonic-gate	  wrpr	%g0, %o5, %pstate
7697c478bdstevel@tonic-gate	SET_SIZE(get_icache_dtag)
7727c478bdstevel@tonic-gate * Get pcache data and tags.
7737c478bdstevel@tonic-gate * inputs:
7747c478bdstevel@tonic-gate *   pcache_idx	- fully constructed VA for for accessing P$ diagnostic
7757c478bdstevel@tonic-gate *		  registers. Contains PC_way and PC_addr shifted into
7767c478bdstevel@tonic-gate *		  the correct bit positions. See the PRM for more details.
7777c478bdstevel@tonic-gate *   data	- pointer to a ch_pc_data_t
7787c478bdstevel@tonic-gate * structure (see cheetahregs.h):
7797c478bdstevel@tonic-gate */
7807c478bdstevel@tonic-gate	.align	128
7817c478bdstevel@tonic-gate	ENTRY(get_pcache_dtag)
7827c478bdstevel@tonic-gate	rdpr	%pstate, %o5
7837c478bdstevel@tonic-gate	andn    %o5, PSTATE_IE | PSTATE_AM, %o3
7847c478bdstevel@tonic-gate	wrpr	%g0, %o3, %pstate
7867c478bdstevel@tonic-gate	stx	%o0, [%o1 + CH_PC_IDX]
7877c478bdstevel@tonic-gate	ldxa	[%o0]ASI_PC_STATUS_DATA, %o2
7887c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_PC_STATUS]
7897c478bdstevel@tonic-gate	ldxa	[%o0]ASI_PC_TAG, %o2
7907c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_PC_TAG]
7917c478bdstevel@tonic-gate	ldxa	[%o0]ASI_PC_SNP_TAG, %o2
7927c478bdstevel@tonic-gate	stx	%o2, [%o1 + CH_PC_SNTAG]
7937c478bdstevel@tonic-gate	add	%o1, CH_PC_DATA, %o1
7947c478bdstevel@tonic-gate	clr	%o3
7967c478bdstevel@tonic-gate	ldxa	[%o0 + %o3]ASI_PC_DATA, %o2
7977c478bdstevel@tonic-gate	stx	%o2, [%o1 + %o3]
7987c478bdstevel@tonic-gate	cmp	%o3, CH_PC_DATA_REG_SIZE - 8
7997c478bdstevel@tonic-gate	blt	2b
8007c478bdstevel@tonic-gate	  add	%o3, 8, %o3
8027c478bdstevel@tonic-gate	retl
8037c478bdstevel@tonic-gate	  wrpr	%g0, %o5, %pstate
8047c478bdstevel@tonic-gate	SET_SIZE(get_pcache_dtag)
8067c478bdstevel@tonic-gate#endif	/* CPU_IMP_L1_CACHE_PARITY */
8097c478bdstevel@tonic-gate * re-enable the i$, d$, w$, and p$ according to bootup cache state.
8107c478bdstevel@tonic-gate * Turn on WE, HPE, SPE, PE, IC, and DC bits defined as DCU_CACHE.
8117c478bdstevel@tonic-gate *   %o0 - 64 bit constant
8127c478bdstevel@tonic-gate */
8137c478bdstevel@tonic-gate	ENTRY(set_dcu)
8147c478bdstevel@tonic-gate	stxa	%o0, [%g0]ASI_DCU	! Store to DCU
8157c478bdstevel@tonic-gate	flush	%g0	/* flush required after changing the IC bit */
8167c478bdstevel@tonic-gate	retl
8177c478bdstevel@tonic-gate	nop
8187c478bdstevel@tonic-gate	SET_SIZE(set_dcu)
8227c478bdstevel@tonic-gate * Return DCU register.
8237c478bdstevel@tonic-gate */
8247c478bdstevel@tonic-gate	ENTRY(get_dcu)
8257c478bdstevel@tonic-gate	ldxa	[%g0]ASI_DCU, %o0		/* DCU control register */
8267c478bdstevel@tonic-gate	retl
8277c478bdstevel@tonic-gate	nop
8287c478bdstevel@tonic-gate	SET_SIZE(get_dcu)
8317c478bdstevel@tonic-gate * Cheetah/Cheetah+ level 15 interrupt handler trap table entry.
8327c478bdstevel@tonic-gate *
8337c478bdstevel@tonic-gate * This handler is used to check for softints generated by error trap
8347c478bdstevel@tonic-gate * handlers to report errors.  On Cheetah, this mechanism is used by the
8357c478bdstevel@tonic-gate * Fast ECC at TL>0 error trap handler and, on Cheetah+, by both the Fast
8367c478bdstevel@tonic-gate * ECC at TL>0 error and the I$/D$ parity error at TL>0 trap handlers.
8377c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
8387c478bdstevel@tonic-gate *     be relocatable.
8397c478bdstevel@tonic-gate */
8417c478bdstevel@tonic-gate	ENTRY_NP(ch_pil15_interrupt_instr)
8427c478bdstevel@tonic-gate	ASM_JMP(%g1, ch_pil15_interrupt)
8437c478bdstevel@tonic-gate	SET_SIZE(ch_pil15_interrupt_instr)
8467c478bdstevel@tonic-gate	ENTRY_NP(ch_pil15_interrupt)
8487c478bdstevel@tonic-gate	/*
8497c478bdstevel@tonic-gate	 * Since pil_interrupt is hacked to assume that every level 15
8507c478bdstevel@tonic-gate	 * interrupt is generated by the CPU to indicate a performance
8517c478bdstevel@tonic-gate	 * counter overflow this gets ugly.  Before calling pil_interrupt
8527c478bdstevel@tonic-gate	 * the Error at TL>0 pending status is inspected.  If it is
8537c478bdstevel@tonic-gate	 * non-zero, then an error has occurred and it is handled.
8547c478bdstevel@tonic-gate	 * Otherwise control is transfered to pil_interrupt.  Note that if
8557c478bdstevel@tonic-gate	 * an error is detected pil_interrupt will not be called and
8567c478bdstevel@tonic-gate	 * overflow interrupts may be lost causing erroneous performance
8577c478bdstevel@tonic-gate	 * measurements.  However, error-recovery will have a detrimental
8587c478bdstevel@tonic-gate	 * effect on performance anyway.
8597c478bdstevel@tonic-gate	 */
8607c478bdstevel@tonic-gate	CPU_INDEX(%g1, %g4)
8617c478bdstevel@tonic-gate	set	ch_err_tl1_pending, %g4
8627c478bdstevel@tonic-gate	ldub	[%g1 + %g4], %g2
8637c478bdstevel@tonic-gate	brz	%g2, 1f
8647c478bdstevel@tonic-gate	  nop
8667c478bdstevel@tonic-gate	/*
8677c478bdstevel@tonic-gate	 * We have a pending TL>0 error, clear the TL>0 pending status.
8687c478bdstevel@tonic-gate	 */
8697c478bdstevel@tonic-gate	stb	%g0, [%g1 + %g4]
8717c478bdstevel@tonic-gate	/*
8727c478bdstevel@tonic-gate	 * Clear the softint.
8737c478bdstevel@tonic-gate	 */
8747c478bdstevel@tonic-gate	mov	1, %g5
8757c478bdstevel@tonic-gate	sll	%g5, PIL_15, %g5
8767c478bdstevel@tonic-gate	wr	%g5, CLEAR_SOFTINT
8787c478bdstevel@tonic-gate	/*
8797c478bdstevel@tonic-gate	 * For Cheetah*, call cpu_tl1_error via systrap at PIL 15
8807c478bdstevel@tonic-gate	 * to process the Fast ECC/Cache Parity at TL>0 error.  Clear
8817c478bdstevel@tonic-gate	 * panic flag (%g2).
8827c478bdstevel@tonic-gate	 */
8837c478bdstevel@tonic-gate	set	cpu_tl1_error, %g1
8847c478bdstevel@tonic-gate	clr	%g2
8857c478bdstevel@tonic-gate	ba	sys_trap
8867c478bdstevel@tonic-gate	  mov	PIL_15, %g4
8897c478bdstevel@tonic-gate	/*
8907c478bdstevel@tonic-gate	 * The logout is invalid.
8917c478bdstevel@tonic-gate	 *
8927c478bdstevel@tonic-gate	 * Call the default interrupt handler.
8937c478bdstevel@tonic-gate	 */
8947c478bdstevel@tonic-gate	sethi	%hi(pil_interrupt), %g1
8957c478bdstevel@tonic-gate	jmp	%g1 + %lo(pil_interrupt)
8967c478bdstevel@tonic-gate	  mov	PIL_15, %g4
8987c478bdstevel@tonic-gate	SET_SIZE(ch_pil15_interrupt)
9027c478bdstevel@tonic-gate * Error Handling
9037c478bdstevel@tonic-gate *
9047c478bdstevel@tonic-gate * Cheetah provides error checking for all memory access paths between
9057c478bdstevel@tonic-gate * the CPU, External Cache, Cheetah Data Switch and system bus. Error
9067c478bdstevel@tonic-gate * information is logged in the AFSR, (also AFSR_EXT for Panther) and
9077c478bdstevel@tonic-gate * AFAR and one of the following traps is generated (provided that it
9087c478bdstevel@tonic-gate * is enabled in External Cache Error Enable Register) to handle that
9097c478bdstevel@tonic-gate * error:
9107c478bdstevel@tonic-gate * 1. trap 0x70: Precise trap
9117c478bdstevel@tonic-gate *    tt0_fecc for errors at trap level(TL)>=0
9127c478bdstevel@tonic-gate * 2. trap 0x0A and 0x32: Deferred trap
9137c478bdstevel@tonic-gate *    async_err for errors at TL>=0
9147c478bdstevel@tonic-gate * 3. trap 0x63: Disrupting trap
9157c478bdstevel@tonic-gate *    ce_err for errors at TL=0
9167c478bdstevel@tonic-gate *    (Note that trap 0x63 cannot happen at trap level > 0)
9177c478bdstevel@tonic-gate *
9187c478bdstevel@tonic-gate * Trap level one handlers panic the system except for the fast ecc
9197c478bdstevel@tonic-gate * error handler which tries to recover from certain errors.
9207c478bdstevel@tonic-gate */
9237c478bdstevel@tonic-gate * FAST ECC TRAP STRATEGY:
9247c478bdstevel@tonic-gate *
9257c478bdstevel@tonic-gate * Software must handle single and multi bit errors which occur due to data
9267c478bdstevel@tonic-gate * or instruction cache reads from the external cache. A single or multi bit
9277c478bdstevel@tonic-gate * error occuring in one of these situations results in a precise trap.
9287c478bdstevel@tonic-gate *
9297c478bdstevel@tonic-gate * The basic flow of this trap handler is as follows:
9307c478bdstevel@tonic-gate *
9317c478bdstevel@tonic-gate * 1) Record the state and then turn off the Dcache and Icache.  The Dcache
9327c478bdstevel@tonic-gate *    is disabled because bad data could have been installed.  The Icache is
9337c478bdstevel@tonic-gate *    turned off because we want to capture the Icache line related to the
9347c478bdstevel@tonic-gate *    AFAR.
9357c478bdstevel@tonic-gate * 2) Disable trapping on CEEN/NCCEN errors during TL=0 processing.
9367c478bdstevel@tonic-gate * 3) Park sibling core if caches are shared (to avoid race condition while
9377c478bdstevel@tonic-gate *    accessing shared resources such as L3 data staging register during
9387c478bdstevel@tonic-gate *    CPU logout.
9397c478bdstevel@tonic-gate * 4) Read the AFAR and AFSR.
9407c478bdstevel@tonic-gate * 5) If CPU logout structure is not being used, then:
9417c478bdstevel@tonic-gate *    6) Clear all errors from the AFSR.
9427c478bdstevel@tonic-gate *    7) Capture Ecache, Dcache and Icache lines in "CPU log out" structure.
9437c478bdstevel@tonic-gate *    8) Flush Ecache then Flush Dcache and Icache and restore to previous
9447c478bdstevel@tonic-gate *       state.
9457c478bdstevel@tonic-gate *    9) Unpark sibling core if we parked it earlier.
9467c478bdstevel@tonic-gate *    10) call cpu_fast_ecc_error via systrap at PIL 14 unless we're already
9477c478bdstevel@tonic-gate *        running at PIL 15.
9487c478bdstevel@tonic-gate * 6) Otherwise, if CPU logout structure is being used:
9497c478bdstevel@tonic-gate *    7) Incriment the "logout busy count".
9507c478bdstevel@tonic-gate *    8) Flush Ecache then Flush Dcache and Icache and restore to previous
9517c478bdstevel@tonic-gate *       state.
9527c478bdstevel@tonic-gate *    9) Unpark sibling core if we parked it earlier.
9537c478bdstevel@tonic-gate *    10) Issue a retry since the other CPU error logging code will end up
9547c478bdstevel@tonic-gate *       finding this error bit and logging information about it later.
9557c478bdstevel@tonic-gate * 7) Alternatively (to 5 and 6 above), if the cpu_private struct is not
9567c478bdstevel@tonic-gate *    yet initialized such that we can't even check the logout struct, then
9577c478bdstevel@tonic-gate *    we place the clo_flags data into %g2 (sys_trap->have_win arg #1) and
9587c478bdstevel@tonic-gate *    call cpu_fast_ecc_error via systrap. The clo_flags parameter is used
9594fd7ecadilpreet *    to determine information such as TL, TT, CEEN and NCEEN settings, etc
9604fd7ecadilpreet *    in the high level trap handler since we don't have access to detailed
9614fd7ecadilpreet *    logout information in cases where the cpu_private struct is not yet
9627c478bdstevel@tonic-gate *    initialized.
9637c478bdstevel@tonic-gate *
9647c478bdstevel@tonic-gate * We flush the E$ and D$ here on TL=1 code to prevent getting nested
9657c478bdstevel@tonic-gate * Fast ECC traps in the TL=0 code.  If we get a Fast ECC event here in
9667c478bdstevel@tonic-gate * the TL=1 code, we will go to the Fast ECC at TL>0 handler which,
9677c478bdstevel@tonic-gate * since it is uses different code/data from this handler, has a better
9687c478bdstevel@tonic-gate * chance of fixing things up than simply recursing through this code
9697c478bdstevel@tonic-gate * again (this would probably cause an eventual kernel stack overflow).
9707c478bdstevel@tonic-gate * If the Fast ECC at TL>0 handler encounters a Fast ECC error before it
9717c478bdstevel@tonic-gate * can flush the E$ (or the error is a stuck-at bit), we will recurse in
9727c478bdstevel@tonic-gate * the Fast ECC at TL>0 handler and eventually Red Mode.
9737c478bdstevel@tonic-gate *
9747c478bdstevel@tonic-gate * Note that for Cheetah (and only Cheetah), we use alias addresses for
9757c478bdstevel@tonic-gate * flushing rather than ASI accesses (which don't exist on Cheetah).
9767c478bdstevel@tonic-gate * Should we encounter a Fast ECC error within this handler on Cheetah,
9777c478bdstevel@tonic-gate * there's a good chance it's within the ecache_flushaddr buffer (since
9787c478bdstevel@tonic-gate * it's the largest piece of memory we touch in the handler and it is
9797c478bdstevel@tonic-gate * usually kernel text/data).  For that reason the Fast ECC at TL>0
9807c478bdstevel@tonic-gate * handler for Cheetah uses an alternate buffer: ecache_tl1_flushaddr.
9817c478bdstevel@tonic-gate */
9847c478bdstevel@tonic-gate * Cheetah ecc-protected E$ trap (Trap 70) at TL=0
9857c478bdstevel@tonic-gate * tt0_fecc is replaced by fecc_err_instr in cpu_init_trap of the various
9867c478bdstevel@tonic-gate * architecture-specific files.
9877c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
9887c478bdstevel@tonic-gate *     be relocatable.
9897c478bdstevel@tonic-gate */
9917c478bdstevel@tonic-gate	ENTRY_NP(fecc_err_instr)
9927c478bdstevel@tonic-gate	membar	#Sync			! Cheetah requires membar #Sync
9947c478bdstevel@tonic-gate	/*
9957c478bdstevel@tonic-gate	 * Save current DCU state.  Turn off the Dcache and Icache.
9967c478bdstevel@tonic-gate	 */
9977c478bdstevel@tonic-gate	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
9987c478bdstevel@tonic-gate	andn	%g1, DCU_DC + DCU_IC, %g4
9997c478bdstevel@tonic-gate	stxa	%g4, [%g0]ASI_DCU
10007c478bdstevel@tonic-gate	flush	%g0	/* flush required after changing the IC bit */
10027c478bdstevel@tonic-gate	ASM_JMP(%g4, fast_ecc_err)
10037c478bdstevel@tonic-gate	SET_SIZE(fecc_err_instr)
10067c478bdstevel@tonic-gate#if !(defined(JALAPENO) || defined(SERRANO))
10087c478bdstevel@tonic-gate	.section ".text"
10097c478bdstevel@tonic-gate	.align	64
10107c478bdstevel@tonic-gate	ENTRY_NP(fast_ecc_err)
10127c478bdstevel@tonic-gate	/*
10137c478bdstevel@tonic-gate	 * Turn off CEEN and NCEEN.
10147c478bdstevel@tonic-gate	 */
10157c478bdstevel@tonic-gate	ldxa	[%g0]ASI_ESTATE_ERR, %g3
10167c478bdstevel@tonic-gate	andn	%g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
10177c478bdstevel@tonic-gate	stxa	%g4, [%g0]ASI_ESTATE_ERR
10187c478bdstevel@tonic-gate	membar	#Sync			! membar sync required
10207c478bdstevel@tonic-gate	/*
10217c478bdstevel@tonic-gate	 * Check to see whether we need to park our sibling core
10227c478bdstevel@tonic-gate	 * before recording diagnostic information from caches
10237c478bdstevel@tonic-gate	 * which may be shared by both cores.
10247c478bdstevel@tonic-gate	 * We use %g1 to store information about whether or not
10257c478bdstevel@tonic-gate	 * we had to park the core (%g1 holds our DCUCR value and
10267c478bdstevel@tonic-gate	 * we only use bits from that register which are "reserved"
10277c478bdstevel@tonic-gate	 * to keep track of core parking) so that we know whether
10287c478bdstevel@tonic-gate	 * or not to unpark later. %g5 and %g4 are scratch registers.
10297c478bdstevel@tonic-gate	 */
10307c478bdstevel@tonic-gate	PARK_SIBLING_CORE(%g1, %g5, %g4)
10327c478bdstevel@tonic-gate	/*
10337c478bdstevel@tonic-gate	 * Do the CPU log out capture.
10347c478bdstevel@tonic-gate	 *   %g3 = "failed?" return value.
10357c478bdstevel@tonic-gate	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
10367c478bdstevel@tonic-gate	 *         into this macro via %g4. Output only valid if cpu_private
10377c478bdstevel@tonic-gate	 *         struct has not been initialized.
10387c478bdstevel@tonic-gate	 *   CHPR_FECCTL0_LOGOUT = cpu logout structure offset input
10397c478bdstevel@tonic-gate	 *   %g4 = Trap information stored in the cpu logout flags field
10407c478bdstevel@tonic-gate	 *   %g5 = scr1
10417c478bdstevel@tonic-gate	 *   %g6 = scr2
10427c478bdstevel@tonic-gate	 *   %g3 = scr3
10437c478bdstevel@tonic-gate	 *   %g4 = scr4
10447c478bdstevel@tonic-gate	 */
10454fd7ecadilpreet	 /* store the CEEN and NCEEN values, TL=0 */
10464fd7ecadilpreet	and	%g3, EN_REG_CEEN + EN_REG_NCEEN, %g4
10477c478bdstevel@tonic-gate	set	CHPR_FECCTL0_LOGOUT, %g6
10487c478bdstevel@tonic-gate	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
10507c478bdstevel@tonic-gate	/*
10517c478bdstevel@tonic-gate	 * Flush the Ecache (and L2 cache for Panther) to get the error out
10527c478bdstevel@tonic-gate	 * of the Ecache.  If the UCC or UCU is on a dirty line, then the
10537c478bdstevel@tonic-gate	 * following flush will turn that into a WDC or WDU, respectively.
10547c478bdstevel@tonic-gate	 */
10557c478bdstevel@tonic-gate	PN_L2_FLUSHALL(%g4, %g5, %g6)
10577c478bdstevel@tonic-gate	CPU_INDEX(%g4, %g5)
10587c478bdstevel@tonic-gate	mulx	%g4, CPU_NODE_SIZE, %g4
10597c478bdstevel@tonic-gate	set	cpunodes, %g5
10607c478bdstevel@tonic-gate	add	%g4, %g5, %g4
10617c478bdstevel@tonic-gate	ld	[%g4 + ECACHE_LINESIZE], %g5
10627c478bdstevel@tonic-gate	ld	[%g4 + ECACHE_SIZE], %g4
10647c478bdstevel@tonic-gate	ASM_LDX(%g6, ecache_flushaddr)
10657c478bdstevel@tonic-gate	ECACHE_FLUSHALL(%g4, %g5, %g6, %g7)
10677c478bdstevel@tonic-gate	/*
10687c478bdstevel@tonic-gate	 * Flush the Dcache.  Since bad data could have been installed in
10697c478bdstevel@tonic-gate	 * the Dcache we must flush it before re-enabling it.
10707c478bdstevel@tonic-gate	 */
10717c478bdstevel@tonic-gate	ASM_LD(%g5, dcache_size)
10727c478bdstevel@tonic-gate	ASM_LD(%g6, dcache_linesize)
10737c478bdstevel@tonic-gate	CH_DCACHE_FLUSHALL(%g5, %g6, %g7)
10757c478bdstevel@tonic-gate	/*
10767c478bdstevel@tonic-gate	 * Flush the Icache.  Since we turned off the Icache to capture the
10777c478bdstevel@tonic-gate	 * Icache line it is now stale or corrupted and we must flush it
10787c478bdstevel@tonic-gate	 * before re-enabling it.
10797c478bdstevel@tonic-gate	 */
10807c478bdstevel@tonic-gate	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_5);
10817c478bdstevel@tonic-gate	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
10827c478bdstevel@tonic-gate	ba,pt	%icc, 6f
10837c478bdstevel@tonic-gate	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
10857c478bdstevel@tonic-gate	ASM_LD(%g5, icache_size)
10867c478bdstevel@tonic-gate	ASM_LD(%g6, icache_linesize)
10887c478bdstevel@tonic-gate	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
10907c478bdstevel@tonic-gate	/*
10917c478bdstevel@tonic-gate	 * check to see whether we parked our sibling core at the start
10927c478bdstevel@tonic-gate	 * of this handler. If so, we need to unpark it here.
10937c478bdstevel@tonic-gate	 * We use DCUCR reserved bits (stored in %g1) to keep track of
10947c478bdstevel@tonic-gate	 * whether or not we need to unpark. %g5 and %g4 are scratch registers.
10957c478bdstevel@tonic-gate	 */
10967c478bdstevel@tonic-gate	UNPARK_SIBLING_CORE(%g1, %g5, %g4)
10987c478bdstevel@tonic-gate	/*
10997c478bdstevel@tonic-gate	 * Restore the Dcache and Icache to the previous state.
11007c478bdstevel@tonic-gate	 */
11017c478bdstevel@tonic-gate	stxa	%g1, [%g0]ASI_DCU
11027c478bdstevel@tonic-gate	flush	%g0	/* flush required after changing the IC bit */
11047c478bdstevel@tonic-gate	/*
11057c478bdstevel@tonic-gate	 * Make sure our CPU logout operation was successful.
11067c478bdstevel@tonic-gate	 */
11077c478bdstevel@tonic-gate	cmp	%g3, %g0
11087c478bdstevel@tonic-gate	be	8f
11097c478bdstevel@tonic-gate	  nop
11117c478bdstevel@tonic-gate	/*
11127c478bdstevel@tonic-gate	 * If the logout structure had been busy, how many times have
11137c478bdstevel@tonic-gate	 * we tried to use it and failed (nesting count)? If we have
11147c478bdstevel@tonic-gate	 * already recursed a substantial number of times, then we can
11157c478bdstevel@tonic-gate	 * assume things are not going to get better by themselves and
11167c478bdstevel@tonic-gate	 * so it would be best to panic.
11177c478bdstevel@tonic-gate	 */
11187c478bdstevel@tonic-gate	cmp	%g3, CLO_NESTING_MAX
11197c478bdstevel@tonic-gate	blt	7f
11207c478bdstevel@tonic-gate	  nop
11227c478bdstevel@tonic-gate        call ptl1_panic
11237c478bdstevel@tonic-gate          mov   PTL1_BAD_ECC, %g1
11267c478bdstevel@tonic-gate	/*
11277c478bdstevel@tonic-gate	 * Otherwise, if the logout structure was busy but we have not
11287c478bdstevel@tonic-gate	 * nested more times than our maximum value, then we simply
11297c478bdstevel@tonic-gate	 * issue a retry. Our TL=0 trap handler code will check and
11307c478bdstevel@tonic-gate	 * clear the AFSR after it is done logging what is currently
11317c478bdstevel@tonic-gate	 * in the logout struct and handle this event at that time.
11327c478bdstevel@tonic-gate	 */
11337c478bdstevel@tonic-gate	retry
11357c478bdstevel@tonic-gate	/*
11367c478bdstevel@tonic-gate	 * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're
11377c478bdstevel@tonic-gate	 * already at PIL 15.
11387c478bdstevel@tonic-gate	 */
11397c478bdstevel@tonic-gate	set	cpu_fast_ecc_error, %g1
11407c478bdstevel@tonic-gate	rdpr	%pil, %g4
11417c478bdstevel@tonic-gate	cmp	%g4, PIL_14
11427c478bdstevel@tonic-gate	ba	sys_trap
11437c478bdstevel@tonic-gate	  movl	%icc, PIL_14, %g4
11457c478bdstevel@tonic-gate	SET_SIZE(fast_ecc_err)
11477c478bdstevel@tonic-gate#endif	/* !(JALAPENO || SERRANO) */
11517c478bdstevel@tonic-gate * Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy:
11527c478bdstevel@tonic-gate *
11537c478bdstevel@tonic-gate * The basic flow of this trap handler is as follows:
11547c478bdstevel@tonic-gate *
11557c478bdstevel@tonic-gate * 1) In the "trap 70" trap table code (fecc_err_tl1_instr), generate a
11567c478bdstevel@tonic-gate *    software trap 0 ("ta 0") to buy an extra set of %tpc, etc. which we
11577c478bdstevel@tonic-gate *    will use to save %g1 and %g2.
11587c478bdstevel@tonic-gate * 2) At the software trap 0 at TL>0 trap table code (fecc_err_tl1_cont_instr),
11597c478bdstevel@tonic-gate *    we save %g1+%g2 using %tpc, %tnpc + %tstate and jump to the fast ecc
11607c478bdstevel@tonic-gate *    handler (using the just saved %g1).
11617c478bdstevel@tonic-gate * 3) Turn off the Dcache if it was on and save the state of the Dcache
11627c478bdstevel@tonic-gate *    (whether on or off) in Bit2 (CH_ERR_TSTATE_DC_ON) of %tstate.
11637c478bdstevel@tonic-gate *    NB: we don't turn off the Icache because bad data is not installed nor
11647c478bdstevel@tonic-gate *        will we be doing any diagnostic accesses.
11657c478bdstevel@tonic-gate * 4) compute physical address of the per-cpu/per-tl save area using %g1+%g2
11667c478bdstevel@tonic-gate * 5) Save %g1-%g7 into the per-cpu/per-tl save area (%g1 + %g2 from the
11677c478bdstevel@tonic-gate *    %tpc, %tnpc, %tstate values previously saved).
11687c478bdstevel@tonic-gate * 6) set %tl to %tl - 1.
11697c478bdstevel@tonic-gate * 7) Save the appropriate flags and TPC in the ch_err_tl1_data structure.
11707c478bdstevel@tonic-gate * 8) Save the value of CH_ERR_TSTATE_DC_ON in the ch_err_tl1_tmp field.
11717c478bdstevel@tonic-gate * 9) For Cheetah and Jalapeno, read the AFAR and AFSR and clear.  For
11727c478bdstevel@tonic-gate *    Cheetah+ (and later), read the shadow AFAR and AFSR but don't clear.
11737c478bdstevel@tonic-gate *    Save the values in ch_err_tl1_data.  For Panther, read the shadow
11747c478bdstevel@tonic-gate *    AFSR_EXT and save the value in ch_err_tl1_data.
11757c478bdstevel@tonic-gate * 10) Disable CEEN/NCEEN to prevent any disrupting/deferred errors from
11767c478bdstevel@tonic-gate *    being queued.  We'll report them via the AFSR/AFAR capture in step 13.
11777c478bdstevel@tonic-gate * 11) Flush the Ecache.
11787c478bdstevel@tonic-gate *    NB: the Ecache is flushed assuming the largest possible size with
11797c478bdstevel@tonic-gate *        the smallest possible line size since access to the cpu_nodes may
11807c478bdstevel@tonic-gate *        cause an unrecoverable DTLB miss.
11817c478bdstevel@tonic-gate * 12) Reenable CEEN/NCEEN with the value saved from step 10.
11827c478bdstevel@tonic-gate * 13) For Cheetah and Jalapeno, read the AFAR and AFSR and clear again.
11837c478bdstevel@tonic-gate *    For Cheetah+ (and later), read the primary AFAR and AFSR and now clear.
11847c478bdstevel@tonic-gate *    Save the read AFSR/AFAR values in ch_err_tl1_data.  For Panther,
11857c478bdstevel@tonic-gate *    read and clear the primary AFSR_EXT and save it in ch_err_tl1_data.
11867c478bdstevel@tonic-gate * 14) Flush and re-enable the Dcache if it was on at step 3.
11877c478bdstevel@tonic-gate * 15) Do TRAPTRACE if enabled.
11887c478bdstevel@tonic-gate * 16) Check if a UCU->WDU (or L3_UCU->WDU for Panther) happened, panic if so.
11897c478bdstevel@tonic-gate * 17) Set the event pending flag in ch_err_tl1_pending[CPU]
11907c478bdstevel@tonic-gate * 18) Cause a softint 15.  The pil15_interrupt handler will inspect the
11917c478bdstevel@tonic-gate *    event pending flag and call cpu_tl1_error via systrap if set.
11927c478bdstevel@tonic-gate * 19) Restore the registers from step 5 and issue retry.
11937c478bdstevel@tonic-gate */
11967c478bdstevel@tonic-gate * Cheetah ecc-protected E$ trap (Trap 70) at TL>0
11977c478bdstevel@tonic-gate * tt1_fecc is replaced by fecc_err_tl1_instr in cpu_init_trap of the various
11987c478bdstevel@tonic-gate * architecture-specific files.  This generates a "Software Trap 0" at TL>0,
11997c478bdstevel@tonic-gate * which goes to fecc_err_tl1_cont_instr, and we continue the handling there.
12007c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
12017c478bdstevel@tonic-gate *     be relocatable.
12027c478bdstevel@tonic-gate */
12047c478bdstevel@tonic-gate	ENTRY_NP(fecc_err_tl1_instr)
12057c478bdstevel@tonic-gate	CH_ERR_TL1_TRAPENTRY(SWTRAP_0);
12067c478bdstevel@tonic-gate	SET_SIZE(fecc_err_tl1_instr)
12097c478bdstevel@tonic-gate * Software trap 0 at TL>0.
12107c478bdstevel@tonic-gate * tt1_swtrap0 is replaced by fecc_err_tl1_cont_instr in cpu_init_trap of
12117c478bdstevel@tonic-gate * the various architecture-specific files.  This is used as a continuation
12127c478bdstevel@tonic-gate * of the fast ecc handling where we've bought an extra TL level, so we can
12137c478bdstevel@tonic-gate * use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
12147c478bdstevel@tonic-gate * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
12157c478bdstevel@tonic-gate * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
12167c478bdstevel@tonic-gate * order two bits from %g1 and %g2 respectively).
12177c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
12187c478bdstevel@tonic-gate *     be relocatable.
12197c478bdstevel@tonic-gate */
12217c478bdstevel@tonic-gate	ENTRY_NP(fecc_err_tl1_cont_instr)
12227c478bdstevel@tonic-gate	CH_ERR_TL1_SWTRAPENTRY(fast_ecc_tl1_err)
12237c478bdstevel@tonic-gate	SET_SIZE(fecc_err_tl1_cont_instr)
12277c478bdstevel@tonic-gate * The ce_err function handles disrupting trap type 0x63 at TL=0.
12287c478bdstevel@tonic-gate *
12297c478bdstevel@tonic-gate * AFSR errors bits which cause this trap are:
12307c478bdstevel@tonic-gate *	CE, EMC, EDU:ST, EDC, WDU, WDC, CPU, CPC, IVU, IVC
12317c478bdstevel@tonic-gate *
12327c478bdstevel@tonic-gate * NCEEN Bit of Cheetah External Cache Error Enable Register enables
12337c478bdstevel@tonic-gate * the following AFSR disrupting traps: EDU:ST, WDU, CPU, IVU
12347c478bdstevel@tonic-gate *
12357c478bdstevel@tonic-gate * CEEN Bit of Cheetah External Cache Error Enable Register enables
12367c478bdstevel@tonic-gate * the following AFSR disrupting traps: CE, EMC, EDC, WDC, CPC, IVC
12377c478bdstevel@tonic-gate *
12387c478bdstevel@tonic-gate * Cheetah+ also handles (No additional processing required):
12397c478bdstevel@tonic-gate *    DUE, DTO, DBERR	(NCEEN controlled)
12407c478bdstevel@tonic-gate *    THCE		(CEEN and ET_ECC_en controlled)
12417c478bdstevel@tonic-gate *    TUE		(ET_ECC_en controlled)
12427c478bdstevel@tonic-gate *
12437c478bdstevel@tonic-gate * Panther further adds:
12447c478bdstevel@tonic-gate *    IMU, L3_EDU, L3_WDU, L3_CPU		(NCEEN controlled)
12457c478bdstevel@tonic-gate *    IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE	(CEEN controlled)
12467c478bdstevel@tonic-gate *    TUE_SH, TUE		(NCEEN and L2_tag_ECC_en controlled)
12477c478bdstevel@tonic-gate *    L3_TUE, L3_TUE_SH		(NCEEN and ET_ECC_en controlled)
12487c478bdstevel@tonic-gate *    THCE			(CEEN and L2_tag_ECC_en controlled)
12497c478bdstevel@tonic-gate *    L3_THCE			(CEEN and ET_ECC_en controlled)
12507c478bdstevel@tonic-gate *
12517c478bdstevel@tonic-gate * Steps:
12527c478bdstevel@tonic-gate *	1. Disable hardware corrected disrupting errors only (CEEN)
12537c478bdstevel@tonic-gate *	2. Park sibling core if caches are shared (to avoid race
12547c478bdstevel@tonic-gate *	   condition while accessing shared resources such as L3
12557c478bdstevel@tonic-gate *	   data staging register during CPU logout.
12567c478bdstevel@tonic-gate *	3. If the CPU logout structure is not currently being used:
12577c478bdstevel@tonic-gate *		4. Clear AFSR error bits
12587c478bdstevel@tonic-gate *		5. Capture Ecache, Dcache and Icache lines associated
12597c478bdstevel@tonic-gate *		   with AFAR.
12607c478bdstevel@tonic-gate *		6. Unpark sibling core if we parked it earlier.
12617c478bdstevel@tonic-gate *		7. call cpu_disrupting_error via sys_trap at PIL 14
12627c478bdstevel@tonic-gate *		   unless we're already running at PIL 15.
12637c478bdstevel@tonic-gate *	4. Otherwise, if the CPU logout structure is busy:
12647c478bdstevel@tonic-gate *		5. Incriment "logout busy count" and place into %g3
12657c478bdstevel@tonic-gate *		6. Unpark sibling core if we parked it earlier.
12667c478bdstevel@tonic-gate *		7. Issue a retry since the other CPU error logging
12677c478bdstevel@tonic-gate *		   code will end up finding this error bit and logging
12687c478bdstevel@tonic-gate *		   information about it later.
12697c478bdstevel@tonic-gate *	5. Alternatively (to 3 and 4 above), if the cpu_private struct is
12707c478bdstevel@tonic-gate *         not yet initialized such that we can't even check the logout
12717c478bdstevel@tonic-gate *         struct, then we place the clo_flags data into %g2
12727c478bdstevel@tonic-gate *         (sys_trap->have_win arg #1) and call cpu_disrupting_error via
12737c478bdstevel@tonic-gate *         systrap. The clo_flags parameter is used to determine information
12747c478bdstevel@tonic-gate *         such as TL, TT, CEEN settings, etc in the high level trap
12757c478bdstevel@tonic-gate *         handler since we don't have access to detailed logout information
12767c478bdstevel@tonic-gate *         in cases where the cpu_private struct is not yet initialized.
12777c478bdstevel@tonic-gate *
12787c478bdstevel@tonic-gate * %g3: [ logout busy count ] - arg #2
12797c478bdstevel@tonic-gate * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1
12807c478bdstevel@tonic-gate */
12827c478bdstevel@tonic-gate	.align	128
12837c478bdstevel@tonic-gate	ENTRY_NP(ce_err)
12847c478bdstevel@tonic-gate	membar	#Sync			! Cheetah requires membar #Sync
12867c478bdstevel@tonic-gate	/*
12877c478bdstevel@tonic-gate	 * Disable trap on hardware corrected errors (CEEN) while at TL=0
12887c478bdstevel@tonic-gate	 * to prevent recursion.
12897c478bdstevel@tonic-gate	 */
12907c478bdstevel@tonic-gate	ldxa	[%g0]ASI_ESTATE_ERR, %g1
12917c478bdstevel@tonic-gate	bclr	EN_REG_CEEN, %g1
12927c478bdstevel@tonic-gate	stxa	%g1, [%g0]ASI_ESTATE_ERR
12937c478bdstevel@tonic-gate	membar	#Sync			! membar sync required
12957c478bdstevel@tonic-gate	/*
12967c478bdstevel@tonic-gate	 * Save current DCU state.  Turn off Icache to allow capture of
12977c478bdstevel@tonic-gate	 * Icache data by DO_CPU_LOGOUT.
12987c478bdstevel@tonic-gate	 */
12997c478bdstevel@tonic-gate	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
13007c478bdstevel@tonic-gate	andn	%g1, DCU_IC, %g4
13017c478bdstevel@tonic-gate	stxa	%g4, [%g0]ASI_DCU
13027c478bdstevel@tonic-gate	flush	%g0	/* flush required after changing the IC bit */
13047c478bdstevel@tonic-gate	/*
13057c478bdstevel@tonic-gate	 * Check to see whether we need to park our sibling core
13067c478bdstevel@tonic-gate	 * before recording diagnostic information from caches
13077c478bdstevel@tonic-gate	 * which may be shared by both cores.
13087c478bdstevel@tonic-gate	 * We use %g1 to store information about whether or not
13097c478bdstevel@tonic-gate	 * we had to park the core (%g1 holds our DCUCR value and
13107c478bdstevel@tonic-gate	 * we only use bits from that register which are "reserved"
13117c478bdstevel@tonic-gate	 * to keep track of core parking) so that we know whether
13127c478bdstevel@tonic-gate	 * or not to unpark later. %g5 and %g4 are scratch registers.
13137c478bdstevel@tonic-gate	 */
13147c478bdstevel@tonic-gate	PARK_SIBLING_CORE(%g1, %g5, %g4)
13167c478bdstevel@tonic-gate	/*
13177c478bdstevel@tonic-gate	 * Do the CPU log out capture.
13187c478bdstevel@tonic-gate	 *   %g3 = "failed?" return value.
13197c478bdstevel@tonic-gate	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
13207c478bdstevel@tonic-gate	 *         into this macro via %g4. Output only valid if cpu_private
13217c478bdstevel@tonic-gate	 *         struct has not been initialized.
13227c478bdstevel@tonic-gate	 *   CHPR_CECC_LOGOUT = cpu logout structure offset input
13237c478bdstevel@tonic-gate	 *   %g4 = Trap information stored in the cpu logout flags field
13247c478bdstevel@tonic-gate	 *   %g5 = scr1
13257c478bdstevel@tonic-gate	 *   %g6 = scr2
13267c478bdstevel@tonic-gate	 *   %g3 = scr3
13277c478bdstevel@tonic-gate	 *   %g4 = scr4
13287c478bdstevel@tonic-gate	 */
13297c478bdstevel@tonic-gate	clr	%g4			! TL=0 bit in afsr
13307c478bdstevel@tonic-gate	set	CHPR_CECC_LOGOUT, %g6
13317c478bdstevel@tonic-gate	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
13337c478bdstevel@tonic-gate	/*
13347c478bdstevel@tonic-gate	 * Flush the Icache.  Since we turned off the Icache to capture the
13357c478bdstevel@tonic-gate	 * Icache line it is now stale or corrupted and we must flush it
13367c478bdstevel@tonic-gate	 * before re-enabling it.
13377c478bdstevel@tonic-gate	 */
13387c478bdstevel@tonic-gate	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, ce_err_1);
13397c478bdstevel@tonic-gate	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
13407c478bdstevel@tonic-gate	ba,pt	%icc, 2f
13417c478bdstevel@tonic-gate	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
13437c478bdstevel@tonic-gate	ASM_LD(%g5, icache_size)
13447c478bdstevel@tonic-gate	ASM_LD(%g6, icache_linesize)
13467c478bdstevel@tonic-gate	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
13487c478bdstevel@tonic-gate	/*
13497c478bdstevel@tonic-gate	 * check to see whether we parked our sibling core at the start
13507c478bdstevel@tonic-gate	 * of this handler. If so, we need to unpark it here.
13517c478bdstevel@tonic-gate	 * We use DCUCR reserved bits (stored in %g1) to keep track of
13527c478bdstevel@tonic-gate	 * whether or not we need to unpark. %g5 and %g4 are scratch registers.
13537c478bdstevel@tonic-gate	 */
13547c478bdstevel@tonic-gate	UNPARK_SIBLING_CORE(%g1, %g5, %g4)
13567c478bdstevel@tonic-gate	/*
13577c478bdstevel@tonic-gate	 * Restore Icache to previous state.
13587c478bdstevel@tonic-gate	 */
13597c478bdstevel@tonic-gate	stxa	%g1, [%g0]ASI_DCU
13607c478bdstevel@tonic-gate	flush	%g0	/* flush required after changing the IC bit */
13627c478bdstevel@tonic-gate	/*
13637c478bdstevel@tonic-gate	 * Make sure our CPU logout operation was successful.
13647c478bdstevel@tonic-gate	 */
13657c478bdstevel@tonic-gate	cmp	%g3, %g0
13667c478bdstevel@tonic-gate	be	4f
13677c478bdstevel@tonic-gate	  nop
13697c478bdstevel@tonic-gate	/*
13707c478bdstevel@tonic-gate	 * If the logout structure had been busy, how many times have
13717c478bdstevel@tonic-gate	 * we tried to use it and failed (nesting count)? If we have
13727c478bdstevel@tonic-gate	 * already recursed a substantial number of times, then we can
13737c478bdstevel@tonic-gate	 * assume things are not going to get better by themselves and
13747c478bdstevel@tonic-gate	 * so it would be best to panic.
13757c478bdstevel@tonic-gate	 */
13767c478bdstevel@tonic-gate	cmp	%g3, CLO_NESTING_MAX
13777c478bdstevel@tonic-gate	blt	3f
13787c478bdstevel@tonic-gate	  nop
13807c478bdstevel@tonic-gate        call ptl1_panic
13817c478bdstevel@tonic-gate          mov   PTL1_BAD_ECC, %g1
13847c478bdstevel@tonic-gate	/*
13857c478bdstevel@tonic-gate	 * Otherwise, if the logout structure was busy but we have not
13867c478bdstevel@tonic-gate	 * nested more times than our maximum value, then we simply
13877c478bdstevel@tonic-gate	 * issue a retry. Our TL=0 trap handler code will check and
13887c478bdstevel@tonic-gate	 * clear the AFSR after it is done logging what is currently
13897c478bdstevel@tonic-gate	 * in the logout struct and handle this event at that time.
13907c478bdstevel@tonic-gate	 */
13917c478bdstevel@tonic-gate	retry
13937c478bdstevel@tonic-gate	/*
13947c478bdstevel@tonic-gate	 * Call cpu_disrupting_error via systrap at PIL 14 unless we're
13957c478bdstevel@tonic-gate	 * already at PIL 15.
13967c478bdstevel@tonic-gate	 */
13977c478bdstevel@tonic-gate	set	cpu_disrupting_error, %g1
13987c478bdstevel@tonic-gate	rdpr	%pil, %g4
13997c478bdstevel@tonic-gate	cmp	%g4, PIL_14
14007c478bdstevel@tonic-gate	ba	sys_trap
14017c478bdstevel@tonic-gate	  movl	%icc, PIL_14, %g4
14027c478bdstevel@tonic-gate	SET_SIZE(ce_err)
14067c478bdstevel@tonic-gate * This trap cannot happen at TL>0 which means this routine will never
14077c478bdstevel@tonic-gate * actually be called and so we treat this like a BAD TRAP panic.
14087c478bdstevel@tonic-gate */
14097c478bdstevel@tonic-gate	.align	64
14107c478bdstevel@tonic-gate	ENTRY_NP(ce_err_tl1)
14127c478bdstevel@tonic-gate        call ptl1_panic
14137c478bdstevel@tonic-gate          mov   PTL1_BAD_TRAP, %g1
14157c478bdstevel@tonic-gate	SET_SIZE(ce_err_tl1)
14197c478bdstevel@tonic-gate * The async_err function handles deferred trap types 0xA
14207c478bdstevel@tonic-gate * (instruction_access_error) and 0x32 (data_access_error) at TL>=0.
14217c478bdstevel@tonic-gate *
14227c478bdstevel@tonic-gate * AFSR errors bits which cause this trap are:
14237c478bdstevel@tonic-gate *	UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR
14247c478bdstevel@tonic-gate * On some platforms, EMU may causes cheetah to pull the error pin
14257c478bdstevel@tonic-gate * never giving Solaris a chance to take a trap.
14267c478bdstevel@tonic-gate *
14277c478bdstevel@tonic-gate * NCEEN Bit of Cheetah External Cache Error Enable Register enables
14287c478bdstevel@tonic-gate * the following AFSR deferred traps: UE, EMU, EDU:BLD, TO, BERR
14297c478bdstevel@tonic-gate *
14307c478bdstevel@tonic-gate * Steps:
14317c478bdstevel@tonic-gate *	1. Disable CEEN and NCEEN errors to prevent recursive errors.
14327c478bdstevel@tonic-gate *	2. Turn D$ off per Cheetah PRM P.5 Note 6, turn I$ off to capture
14337c478bdstevel@tonic-gate *         I$ line in DO_CPU_LOGOUT.
14347c478bdstevel@tonic-gate *	3. Park sibling core if caches are shared (to avoid race
14357c478bdstevel@tonic-gate *	   condition while accessing shared resources such as L3
14367c478bdstevel@tonic-gate *	   data staging register during CPU logout.
14377c478bdstevel@tonic-gate *	4. If the CPU logout structure is not currently being used:
14387c478bdstevel@tonic-gate *		5. Clear AFSR error bits
14397c478bdstevel@tonic-gate *		6. Capture Ecache, Dcache and Icache lines associated
14407c478bdstevel@tonic-gate *		   with AFAR.
14417c478bdstevel@tonic-gate *		7. Unpark sibling core if we parked it earlier.
14427c478bdstevel@tonic-gate *		8. call cpu_deferred_error via sys_trap.
14437c478bdstevel@tonic-gate *	5. Otherwise, if the CPU logout structure is busy:
14447c478bdstevel@tonic-gate *		6. Incriment "logout busy count"
14457c478bdstevel@tonic-gate *		7. Unpark sibling core if we parked it earlier.
14467c478bdstevel@tonic-gate *		8) Issue a retry since the other CPU error logging
14477c478bdstevel@tonic-gate *		   code will end up finding this error bit and logging
14487c478bdstevel@tonic-gate *		   information about it later.
14497c478bdstevel@tonic-gate *      6. Alternatively (to 4 and 5 above), if the cpu_private struct is
14507c478bdstevel@tonic-gate *         not yet initialized such that we can't even check the logout
14517c478bdstevel@tonic-gate *         struct, then we place the clo_flags data into %g2
14527c478bdstevel@tonic-gate *         (sys_trap->have_win arg #1) and call cpu_deferred_error via
14537c478bdstevel@tonic-gate *         systrap. The clo_flags parameter is used to determine information
14547c478bdstevel@tonic-gate *         such as TL, TT, CEEN settings, etc in the high level trap handler
14557c478bdstevel@tonic-gate *         since we don't have access to detailed logout information in cases
14567c478bdstevel@tonic-gate *         where the cpu_private struct is not yet initialized.
14577c478bdstevel@tonic-gate *
14587c478bdstevel@tonic-gate * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1
14597c478bdstevel@tonic-gate * %g3: [ logout busy count ] - arg #2
14607c478bdstevel@tonic-gate */
14627c478bdstevel@tonic-gate	ENTRY_NP(async_err)
14637c478bdstevel@tonic-gate	membar	#Sync			! Cheetah requires membar #Sync
14657c478bdstevel@tonic-gate	/*
14667c478bdstevel@tonic-gate	 * Disable CEEN and NCEEN.
14677c478bdstevel@tonic-gate	 */
14687c478bdstevel@tonic-gate	ldxa	[%g0]ASI_ESTATE_ERR, %g3
14697c478bdstevel@tonic-gate	andn	%g3, EN_REG_NCEEN + EN_REG_CEEN, %g4
14707c478bdstevel@tonic-gate	stxa	%g4, [%g0]ASI_ESTATE_ERR
14717c478bdstevel@tonic-gate	membar	#Sync			! membar sync required
14737c478bdstevel@tonic-gate	/*
14747c478bdstevel@tonic-gate	 * Save current DCU state.
14757c478bdstevel@tonic-gate	 * Disable Icache to allow capture of Icache data by DO_CPU_LOGOUT.
14767c478bdstevel@tonic-gate	 * Do this regardless of whether this is a Data Access Error or
14777c478bdstevel@tonic-gate	 * Instruction Access Error Trap.
14787c478bdstevel@tonic-gate	 * Disable Dcache for both Data Access Error and Instruction Access
14797c478bdstevel@tonic-gate	 * Error per Cheetah PRM P.5 Note 6.
14807c478bdstevel@tonic-gate	 */
14817c478bdstevel@tonic-gate	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
14827c478bdstevel@tonic-gate	andn	%g1, DCU_IC + DCU_DC, %g4
14837c478bdstevel@tonic-gate	stxa	%g4, [%g0]ASI_DCU
14847c478bdstevel@tonic-gate	flush	%g0	/* flush required after changing the IC bit */
14867c478bdstevel@tonic-gate	/*
14877c478bdstevel@tonic-gate	 * Check to see whether we need to park our sibling core
14887c478bdstevel@tonic-gate	 * before recording diagnostic information from caches
14897c478bdstevel@tonic-gate	 * which may be shared by both cores.
14907c478bdstevel@tonic-gate	 * We use %g1 to store information about whether or not
14917c478bdstevel@tonic-gate	 * we had to park the core (%g1 holds our DCUCR value and
14927c478bdstevel@tonic-gate	 * we only use bits from that register which are "reserved"
14937c478bdstevel@tonic-gate	 * to keep track of core parking) so that we know whether
14947c478bdstevel@tonic-gate	 * or not to unpark later. %g6 and %g4 are scratch registers.
14957c478bdstevel@tonic-gate	 */
14967c478bdstevel@tonic-gate	PARK_SIBLING_CORE(%g1, %g6, %g4)
14987c478bdstevel@tonic-gate	/*
14997c478bdstevel@tonic-gate	 * Do the CPU logout capture.
15007c478bdstevel@tonic-gate	 *
15017c478bdstevel@tonic-gate	 *   %g3 = "failed?" return value.
15027c478bdstevel@tonic-gate	 *   %g2 = Input = AFAR. Output the clo_flags info which is passed
15037c478bdstevel@tonic-gate	 *         into this macro via %g4. Output only valid if cpu_private
15047c478bdstevel@tonic-gate	 *         struct has not been initialized.
15057c478bdstevel@tonic-gate	 *   CHPR_ASYNC_LOGOUT = cpu logout structure offset input
15067c478bdstevel@tonic-gate	 *   %g4 = Trap information stored in the cpu logout flags field
15077c478bdstevel@tonic-gate	 *   %g5 = scr1
15087c478bdstevel@tonic-gate	 *   %g6 = scr2
15097c478bdstevel@tonic-gate	 *   %g3 = scr3
15107c478bdstevel@tonic-gate	 *   %g4 = scr4
15117c478bdstevel@tonic-gate	 */
15127c478bdstevel@tonic-gate	andcc	%g5, T_TL1, %g0
15137c478bdstevel@tonic-gate	clr	%g6
15147c478bdstevel@tonic-gate	movnz	%xcc, 1, %g6			! set %g6 if T_TL1 set
15157c478bdstevel@tonic-gate	sllx	%g6, CLO_FLAGS_TL_SHIFT, %g6
15167c478bdstevel@tonic-gate	sllx	%g5, CLO_FLAGS_TT_SHIFT, %g4
15177c478bdstevel@tonic-gate	set	CLO_FLAGS_TT_MASK, %g2
15187c478bdstevel@tonic-gate	and	%g4, %g2, %g4			! ttype
15197c478bdstevel@tonic-gate	or	%g6, %g4, %g4			! TT and TL
15207c478bdstevel@tonic-gate	and	%g3, EN_REG_CEEN, %g3		! CEEN value
15217c478bdstevel@tonic-gate	or	%g3, %g4, %g4			! TT and TL and CEEN
15227c478bdstevel@tonic-gate	set	CHPR_ASYNC_LOGOUT, %g6
15237c478bdstevel@tonic-gate	DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4)
15257c478bdstevel@tonic-gate	/*
15267c478bdstevel@tonic-gate	 * If the logout struct was busy, we may need to pass the
15277c478bdstevel@tonic-gate	 * TT, TL, and CEEN information to the TL=0 handler via
15287c478bdstevel@tonic-gate	 * systrap parameter so save it off here.
15297c478bdstevel@tonic-gate	 */
15307c478bdstevel@tonic-gate	cmp	%g3, %g0
15317c478bdstevel@tonic-gate	be	1f
15327c478bdstevel@tonic-gate	  nop
15337c478bdstevel@tonic-gate	sllx	%g4, 32, %g4
15347c478bdstevel@tonic-gate	or	%g4, %g3, %g3
15367c478bdstevel@tonic-gate	/*
15377c478bdstevel@tonic-gate	 * Flush the Icache.  Since we turned off the Icache to capture the
15387c478bdstevel@tonic-gate	 * Icache line it is now stale or corrupted and we must flush it
15397c478bdstevel@tonic-gate	 * before re-enabling it.
15407c478bdstevel@tonic-gate	 */
15417c478bdstevel@tonic-gate	GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, async_err_1);
15427c478bdstevel@tonic-gate	ld	[%g5 + CHPR_ICACHE_LINESIZE], %g6
15437c478bdstevel@tonic-gate	ba,pt	%icc, 2f
15447c478bdstevel@tonic-gate	  ld	[%g5 + CHPR_ICACHE_SIZE], %g5
15467c478bdstevel@tonic-gate	ASM_LD(%g5, icache_size)
15477c478bdstevel@tonic-gate	ASM_LD(%g6, icache_linesize)
15497c478bdstevel@tonic-gate	CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4)
15517c478bdstevel@tonic-gate	/*
15527c478bdstevel@tonic-gate	 * XXX - Don't we need to flush the Dcache before turning it back
15537c478bdstevel@tonic-gate	 *       on to avoid stale or corrupt data? Was this broken?
15547c478bdstevel@tonic-gate	 */
15557c478bdstevel@tonic-gate	/*
15567c478bdstevel@tonic-gate	 * Flush the Dcache before turning it back on since it may now
15577c478bdstevel@tonic-gate	 * contain stale or corrupt data.
15587c478bdstevel@tonic-gate	 */
15597c478bdstevel@tonic-gate	ASM_LD(%g5, dcache_size)
15607c478bdstevel@tonic-gate	ASM_LD(%g6, dcache_linesize)
15617c478bdstevel@tonic-gate	CH_DCACHE_FLUSHALL(%g5, %g6, %g7)
15637c478bdstevel@tonic-gate	/*
15647c478bdstevel@tonic-gate	 * check to see whether we parked our sibling core at the start
15657c478bdstevel@tonic-gate	 * of this handler. If so, we need to unpark it here.
15667c478bdstevel@tonic-gate	 * We use DCUCR reserved bits (stored in %g1) to keep track of
15677c478bdstevel@tonic-gate	 * whether or not we need to unpark. %g5 and %g7 are scratch registers.
15687c478bdstevel@tonic-gate	 */
15697c478bdstevel@tonic-gate	UNPARK_SIBLING_CORE(%g1, %g5, %g7)
15717c478bdstevel@tonic-gate	/*
15727c478bdstevel@tonic-gate	 * Restore Icache and Dcache to previous state.
15737c478bdstevel@tonic-gate	 */
15747c478bdstevel@tonic-gate	stxa	%g1, [%g0]ASI_DCU
15757c478bdstevel@tonic-gate	flush	%g0	/* flush required after changing the IC bit */
15777c478bdstevel@tonic-gate	/*
15787c478bdstevel@tonic-gate	 * Make sure our CPU logout operation was successful.
15797c478bdstevel@tonic-gate	 */
15807c478bdstevel@tonic-gate	cmp	%g3, %g0
15817c478bdstevel@tonic-gate	be	4f
15827c478bdstevel@tonic-gate	  nop
15847c478bdstevel@tonic-gate	/*
15857c478bdstevel@tonic-gate	 * If the logout structure had been busy, how many times have
15867c478bdstevel@tonic-gate	 * we tried to use it and failed (nesting count)? If we have
15877c478bdstevel@tonic-gate	 * already recursed a substantial number of times, then we can
15887c478bdstevel@tonic-gate	 * assume things are not going to get better by themselves and
15897c478bdstevel@tonic-gate	 * so it would be best to panic.
15907c478bdstevel@tonic-gate	 */
15917c478bdstevel@tonic-gate	cmp	%g3, CLO_NESTING_MAX
15927c478bdstevel@tonic-gate	blt	3f
15937c478bdstevel@tonic-gate	  nop
15957c478bdstevel@tonic-gate        call ptl1_panic
15967c478bdstevel@tonic-gate          mov   PTL1_BAD_ECC, %g1
15997c478bdstevel@tonic-gate	/*
16007c478bdstevel@tonic-gate	 * Otherwise, if the logout structure was busy but we have not
16017c478bdstevel@tonic-gate	 * nested more times than our maximum value, then we simply
16027c478bdstevel@tonic-gate	 * issue a retry. Our TL=0 trap handler code will check and
16037c478bdstevel@tonic-gate	 * clear the AFSR after it is done logging what is currently
16047c478bdstevel@tonic-gate	 * in the logout struct and handle this event at that time.
16057c478bdstevel@tonic-gate	 */
16067c478bdstevel@tonic-gate	retry
16082dd3029jimand	RESET_USER_RTT_REGS(%g4, %g5, async_err_resetskip)
16107c478bdstevel@tonic-gate	set	cpu_deferred_error, %g1
16117c478bdstevel@tonic-gate	ba	sys_trap
16127c478bdstevel@tonic-gate	  mov	PIL_15, %g4		! run at pil 15
16137c478bdstevel@tonic-gate	SET_SIZE(async_err)
16157c478bdstevel@tonic-gate#if defined(CPU_IMP_L1_CACHE_PARITY)
16187c478bdstevel@tonic-gate * D$ parity error trap (trap 71) at TL=0.
16197c478bdstevel@tonic-gate * tt0_dperr is replaced by dcache_parity_instr in cpu_init_trap of
16207c478bdstevel@tonic-gate * the various architecture-specific files.  This merely sets up the
16217c478bdstevel@tonic-gate * arguments for cpu_parity_error and calls it via sys_trap.
16227c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
16237c478bdstevel@tonic-gate *     be relocatable.
16247c478bdstevel@tonic-gate */
16257c478bdstevel@tonic-gate	ENTRY_NP(dcache_parity_instr)
16267c478bdstevel@tonic-gate	membar	#Sync			! Cheetah+ requires membar #Sync
16277c478bdstevel@tonic-gate	set	cpu_parity_error, %g1
16287c478bdstevel@tonic-gate	or	%g0, CH_ERR_DPE, %g2
16297c478bdstevel@tonic-gate	rdpr	%tpc, %g3
16307c478bdstevel@tonic-gate	sethi	%hi(sys_trap), %g7
16317c478bdstevel@tonic-gate	jmp	%g7 + %lo(sys_trap)
16327c478bdstevel@tonic-gate	  mov	PIL_15, %g4		! run at pil 15
16337c478bdstevel@tonic-gate	SET_SIZE(dcache_parity_instr)
16377c478bdstevel@tonic-gate * D$ parity error trap (trap 71) at TL>0.
16387c478bdstevel@tonic-gate * tt1_dperr is replaced by dcache_parity_tl1_instr in cpu_init_trap of
16397c478bdstevel@tonic-gate * the various architecture-specific files.  This generates a "Software
16407c478bdstevel@tonic-gate * Trap 1" at TL>0, which goes to dcache_parity_tl1_cont_instr, and we
16417c478bdstevel@tonic-gate * continue the handling there.
16427c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
16437c478bdstevel@tonic-gate *     be relocatable.
16447c478bdstevel@tonic-gate */
16457c478bdstevel@tonic-gate	ENTRY_NP(dcache_parity_tl1_instr)
16467c478bdstevel@tonic-gate	CH_ERR_TL1_TRAPENTRY(SWTRAP_1);
16477c478bdstevel@tonic-gate	SET_SIZE(dcache_parity_tl1_instr)
16517c478bdstevel@tonic-gate * Software trap 1 at TL>0.
16527c478bdstevel@tonic-gate * tt1_swtrap1 is replaced by dcache_parity_tl1_cont_instr in cpu_init_trap
16537c478bdstevel@tonic-gate * of the various architecture-specific files.  This is used as a continuation
16547c478bdstevel@tonic-gate * of the dcache parity handling where we've bought an extra TL level, so we
16557c478bdstevel@tonic-gate * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
16567c478bdstevel@tonic-gate * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
16577c478bdstevel@tonic-gate * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
16587c478bdstevel@tonic-gate * order two bits from %g1 and %g2 respectively).
16597c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
16607c478bdstevel@tonic-gate *     be relocatable.
16617c478bdstevel@tonic-gate */
16627c478bdstevel@tonic-gate	ENTRY_NP(dcache_parity_tl1_cont_instr)
16637c478bdstevel@tonic-gate	CH_ERR_TL1_SWTRAPENTRY(dcache_parity_tl1_err);
16647c478bdstevel@tonic-gate	SET_SIZE(dcache_parity_tl1_cont_instr)
16677c478bdstevel@tonic-gate * D$ parity error at TL>0 handler
16687c478bdstevel@tonic-gate * We get here via trap 71 at TL>0->Software trap 1 at TL>0.  We enter
16697c478bdstevel@tonic-gate * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
16707c478bdstevel@tonic-gate */
16727c478bdstevel@tonic-gate	ENTRY_NP(dcache_parity_tl1_err)
16747c478bdstevel@tonic-gate	/*
16757c478bdstevel@tonic-gate	 * This macro saves all the %g registers in the ch_err_tl1_data
16767c478bdstevel@tonic-gate	 * structure, updates the ch_err_tl1_flags and saves the %tpc in
16777c478bdstevel@tonic-gate	 * ch_err_tl1_tpc.  At the end of this macro, %g1 will point to
16787c478bdstevel@tonic-gate	 * the ch_err_tl1_data structure and %g2 will have the original
16797c478bdstevel@tonic-gate	 * flags in the ch_err_tl1_data structure.  All %g registers
16807c478bdstevel@tonic-gate	 * except for %g1 and %g2 will be available.
16817c478bdstevel@tonic-gate	 */
16827c478bdstevel@tonic-gate	CH_ERR_TL1_ENTER(CH_ERR_DPE);
16847c478bdstevel@tonic-gate#ifdef TRAPTRACE
16857c478bdstevel@tonic-gate	/*
16867c478bdstevel@tonic-gate	 * Get current trap trace entry physical pointer.
16877c478bdstevel@tonic-gate	 */
16887c478bdstevel@tonic-gate	CPU_INDEX(%g6, %g5)
16897c478bdstevel@tonic-gate	sll	%g6, TRAPTR_SIZE_SHIFT, %g6
16907c478bdstevel@tonic-gate	set	trap_trace_ctl, %g5
16917c478bdstevel@tonic-gate	add	%g6, %g5, %g6
16927c478bdstevel@tonic-gate	ld	[%g6 + TRAPTR_LIMIT], %g5
16937c478bdstevel@tonic-gate	tst	%g5
16947c478bdstevel@tonic-gate	be	%icc, dpe_tl1_skip_tt
16957c478bdstevel@tonic-gate	  nop
16967c478bdstevel@tonic-gate	ldx	[%g6 + TRAPTR_PBASE], %g5
16977c478bdstevel@tonic-gate	ld	[%g6 + TRAPTR_OFFSET], %g4
16987c478bdstevel@tonic-gate	add	%g5, %g4, %g5
17007c478bdstevel@tonic-gate	/*
17017c478bdstevel@tonic-gate	 * Create trap trace entry.
17027c478bdstevel@tonic-gate	 */
17037c478bdstevel@tonic-gate	rd	%asi, %g7
17047c478bdstevel@tonic-gate	wr	%g0, TRAPTR_ASI, %asi
17057c478bdstevel@tonic-gate	rd	STICK, %g4
17067c478bdstevel@tonic-gate	stxa	%g4, [%g5 + TRAP_ENT_TICK]%asi
17077c478bdstevel@tonic-gate	rdpr	%tl, %g4
17087c478bdstevel@tonic-gate	stha	%g4, [%g5 + TRAP_ENT_TL]%asi
17097c478bdstevel@tonic-gate	rdpr	%tt, %g4
17107c478bdstevel@tonic-gate	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
17117c478bdstevel@tonic-gate	rdpr	%tpc, %g4
17127c478bdstevel@tonic-gate	stna	%g4, [%g5 + TRAP_ENT_TPC]%asi
17137c478bdstevel@tonic-gate	rdpr	%tstate, %g4
17147c478bdstevel@tonic-gate	stxa	%g4, [%g5 + TRAP_ENT_TSTATE]%asi
17157c478bdstevel@tonic-gate	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
17167c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
17177c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_F1]%asi
17187c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_F2]%asi
17197c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_F3]%asi
17207c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_F4]%asi
17217c478bdstevel@tonic-gate	wr	%g0, %g7, %asi
17237c478bdstevel@tonic-gate	/*
17247c478bdstevel@tonic-gate	 * Advance trap trace pointer.
17257c478bdstevel@tonic-gate	 */
17267c478bdstevel@tonic-gate	ld	[%g6 + TRAPTR_OFFSET], %g5
17277c478bdstevel@tonic-gate	ld	[%g6 + TRAPTR_LIMIT], %g4
17287c478bdstevel@tonic-gate	st	%g5, [%g6 + TRAPTR_LAST_OFFSET]
17297c478bdstevel@tonic-gate	add	%g5, TRAP_ENT_SIZE, %g5
17307c478bdstevel@tonic-gate	sub	%g4, TRAP_ENT_SIZE, %g4
17317c478bdstevel@tonic-gate	cmp	%g5, %g4
17327c478bdstevel@tonic-gate	movge	%icc, 0, %g5
17337c478bdstevel@tonic-gate	st	%g5, [%g6 + TRAPTR_OFFSET]
17357c478bdstevel@tonic-gate#endif	/* TRAPTRACE */
17377c478bdstevel@tonic-gate	/*
17387c478bdstevel@tonic-gate	 * I$ and D$ are automatically turned off by HW when the CPU hits
17397c478bdstevel@tonic-gate	 * a dcache or icache parity error so we will just leave those two
17407c478bdstevel@tonic-gate	 * off for now to avoid repeating this trap.
17417c478bdstevel@tonic-gate	 * For Panther, however, since we trap on P$ data parity errors
17427c478bdstevel@tonic-gate	 * and HW does not automatically disable P$, we need to disable it
17437c478bdstevel@tonic-gate	 * here so that we don't encounter any recursive traps when we
17447c478bdstevel@tonic-gate	 * issue the retry.
17457c478bdstevel@tonic-gate	 */
17467c478bdstevel@tonic-gate	ldxa	[%g0]ASI_DCU, %g3
17477c478bdstevel@tonic-gate	mov	1, %g4
17487c478bdstevel@tonic-gate	sllx	%g4, DCU_PE_SHIFT, %g4
17497c478bdstevel@tonic-gate	andn	%g3, %g4, %g3
17507c478bdstevel@tonic-gate	stxa	%g3, [%g0]ASI_DCU
17517c478bdstevel@tonic-gate	membar	#Sync
17537c478bdstevel@tonic-gate	/*
17547c478bdstevel@tonic-gate	 * We fall into this macro if we've successfully logged the error in
17557c478bdstevel@tonic-gate	 * the ch_err_tl1_data structure and want the PIL15 softint to pick
17567c478bdstevel@tonic-gate	 * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
17577c478bdstevel@tonic-gate	 * Restores the %g registers and issues retry.
17587c478bdstevel@tonic-gate	 */
17597c478bdstevel@tonic-gate	CH_ERR_TL1_EXIT;
17607c478bdstevel@tonic-gate	SET_SIZE(dcache_parity_tl1_err)
17637c478bdstevel@tonic-gate * I$ parity error trap (trap 72) at TL=0.
17647c478bdstevel@tonic-gate * tt0_iperr is replaced by icache_parity_instr in cpu_init_trap of
17657c478bdstevel@tonic-gate * the various architecture-specific files.  This merely sets up the
17667c478bdstevel@tonic-gate * arguments for cpu_parity_error and calls it via sys_trap.
17677c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
17687c478bdstevel@tonic-gate *     be relocatable.
17697c478bdstevel@tonic-gate */
17717c478bdstevel@tonic-gate	ENTRY_NP(icache_parity_instr)
17727c478bdstevel@tonic-gate	membar	#Sync			! Cheetah+ requires membar #Sync
17737c478bdstevel@tonic-gate	set	cpu_parity_error, %g1
17747c478bdstevel@tonic-gate	or	%g0, CH_ERR_IPE, %g2
17757c478bdstevel@tonic-gate	rdpr	%tpc, %g3
17767c478bdstevel@tonic-gate	sethi	%hi(sys_trap), %g7
17777c478bdstevel@tonic-gate	jmp	%g7 + %lo(sys_trap)
17787c478bdstevel@tonic-gate	  mov	PIL_15, %g4		! run at pil 15
17797c478bdstevel@tonic-gate	SET_SIZE(icache_parity_instr)
17827c478bdstevel@tonic-gate * I$ parity error trap (trap 72) at TL>0.
17837c478bdstevel@tonic-gate * tt1_iperr is replaced by icache_parity_tl1_instr in cpu_init_trap of
17847c478bdstevel@tonic-gate * the various architecture-specific files.  This generates a "Software
17857c478bdstevel@tonic-gate * Trap 2" at TL>0, which goes to icache_parity_tl1_cont_instr, and we
17867c478bdstevel@tonic-gate * continue the handling there.
17877c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
17887c478bdstevel@tonic-gate *     be relocatable.
17897c478bdstevel@tonic-gate */
17907c478bdstevel@tonic-gate	ENTRY_NP(icache_parity_tl1_instr)
17917c478bdstevel@tonic-gate	CH_ERR_TL1_TRAPENTRY(SWTRAP_2);
17927c478bdstevel@tonic-gate	SET_SIZE(icache_parity_tl1_instr)
17957c478bdstevel@tonic-gate * Software trap 2 at TL>0.
17967c478bdstevel@tonic-gate * tt1_swtrap2 is replaced by icache_parity_tl1_cont_instr in cpu_init_trap
17977c478bdstevel@tonic-gate * of the various architecture-specific files.  This is used as a continuation
17987c478bdstevel@tonic-gate * of the icache parity handling where we've bought an extra TL level, so we
17997c478bdstevel@tonic-gate * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1
18007c478bdstevel@tonic-gate * and %g2.  Note that %tstate has bits 0-2 and then bits 8-19 as r/w,
18017c478bdstevel@tonic-gate * there's a reserved hole from 3-7.  We only use bits 0-1 and 8-9 (the low
18027c478bdstevel@tonic-gate * order two bits from %g1 and %g2 respectively).
18037c478bdstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must
18047c478bdstevel@tonic-gate *     be relocatable.
18057c478bdstevel@tonic-gate */
18067c478bdstevel@tonic-gate	ENTRY_NP(icache_parity_tl1_cont_instr)
18077c478bdstevel@tonic-gate	CH_ERR_TL1_SWTRAPENTRY(icache_parity_tl1_err);
18087c478bdstevel@tonic-gate	SET_SIZE(icache_parity_tl1_cont_instr)
18127c478bdstevel@tonic-gate * I$ parity error at TL>0 handler
18137c478bdstevel@tonic-gate * We get here via trap 72 at TL>0->Software trap 2 at TL>0.  We enter
18147c478bdstevel@tonic-gate * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate.
18157c478bdstevel@tonic-gate */
18177c478bdstevel@tonic-gate	ENTRY_NP(icache_parity_tl1_err)
18197c478bdstevel@tonic-gate	/*
18207c478bdstevel@tonic-gate	 * This macro saves all the %g registers in the ch_err_tl1_data
18217c478bdstevel@tonic-gate	 * structure, updates the ch_err_tl1_flags and saves the %tpc in
18227c478bdstevel@tonic-gate	 * ch_err_tl1_tpc.  At the end of this macro, %g1 will point to
18237c478bdstevel@tonic-gate	 * the ch_err_tl1_data structure and %g2 will have the original
18247c478bdstevel@tonic-gate	 * flags in the ch_err_tl1_data structure.  All %g registers
18257c478bdstevel@tonic-gate	 * except for %g1 and %g2 will be available.
18267c478bdstevel@tonic-gate	 */
18277c478bdstevel@tonic-gate	CH_ERR_TL1_ENTER(CH_ERR_IPE);
18297c478bdstevel@tonic-gate#ifdef TRAPTRACE
18307c478bdstevel@tonic-gate	/*
18317c478bdstevel@tonic-gate	 * Get current trap trace entry physical pointer.
18327c478bdstevel@tonic-gate	 */
18337c478bdstevel@tonic-gate	CPU_INDEX(%g6, %g5)
18347c478bdstevel@tonic-gate	sll	%g6, TRAPTR_SIZE_SHIFT, %g6
18357c478bdstevel@tonic-gate	set	trap_trace_ctl, %g5
18367c478bdstevel@tonic-gate	add	%g6, %g5, %g6
18377c478bdstevel@tonic-gate	ld	[%g6 + TRAPTR_LIMIT], %g5
18387c478bdstevel@tonic-gate	tst	%g5
18397c478bdstevel@tonic-gate	be	%icc, ipe_tl1_skip_tt
18407c478bdstevel@tonic-gate	  nop
18417c478bdstevel@tonic-gate	ldx	[%g6 + TRAPTR_PBASE], %g5
18427c478bdstevel@tonic-gate	ld	[%g6 + TRAPTR_OFFSET], %g4
18437c478bdstevel@tonic-gate	add	%g5, %g4, %g5
18457c478bdstevel@tonic-gate	/*
18467c478bdstevel@tonic-gate	 * Create trap trace entry.
18477c478bdstevel@tonic-gate	 */
18487c478bdstevel@tonic-gate	rd	%asi, %g7
18497c478bdstevel@tonic-gate	wr	%g0, TRAPTR_ASI, %asi
18507c478bdstevel@tonic-gate	rd	STICK, %g4
18517c478bdstevel@tonic-gate	stxa	%g4, [%g5 + TRAP_ENT_TICK]%asi
18527c478bdstevel@tonic-gate	rdpr	%tl, %g4
18537c478bdstevel@tonic-gate	stha	%g4, [%g5 + TRAP_ENT_TL]%asi
18547c478bdstevel@tonic-gate	rdpr	%tt, %g4
18557c478bdstevel@tonic-gate	stha	%g4, [%g5 + TRAP_ENT_TT]%asi
18567c478bdstevel@tonic-gate	rdpr	%tpc, %g4
18577c478bdstevel@tonic-gate	stna	%g4, [%g5 + TRAP_ENT_TPC]%asi
18587c478bdstevel@tonic-gate	rdpr	%tstate, %g4
18597c478bdstevel@tonic-gate	stxa	%g4, [%g5 + TRAP_ENT_TSTATE]%asi
18607c478bdstevel@tonic-gate	stna	%sp, [%g5 + TRAP_ENT_SP]%asi
18617c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_TR]%asi
18627c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_F1]%asi
18637c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_F2]%asi
18647c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_F3]%asi
18657c478bdstevel@tonic-gate	stna	%g0, [%g5 + TRAP_ENT_F4]%asi
18667c478bdstevel@tonic-gate	wr	%g0, %g7, %asi
18687c478bdstevel@tonic-gate	/*
18697c478bdstevel@tonic-gate	 * Advance trap trace pointer.
18707c478bdstevel@tonic-gate	 */
18717c478bdstevel@tonic-gate	ld	[%g6 + TRAPTR_OFFSET], %g5
18727c478bdstevel@tonic-gate	ld	[%g6 + TRAPTR_LIMIT], %g4
18737c478bdstevel@tonic-gate	st	%g5, [%g6 + TRAPTR_LAST_OFFSET]
18747c478bdstevel@tonic-gate	add	%g5, TRAP_ENT_SIZE, %g5
18757c478bdstevel@tonic-gate	sub	%g4, TRAP_ENT_SIZE, %g4
18767c478bdstevel@tonic-gate	cmp	%g5, %g4
18777c478bdstevel@tonic-gate	movge	%icc, 0, %g5
18787c478bdstevel@tonic-gate	st	%g5, [%g6 + TRAPTR_OFFSET]
18807c478bdstevel@tonic-gate#endif	/* TRAPTRACE */
18827c478bdstevel@tonic-gate	/*
18837c478bdstevel@tonic-gate	 * We fall into this macro if we've successfully logged the error in
18847c478bdstevel@tonic-gate	 * the ch_err_tl1_data structure and want the PIL15 softint to pick
18857c478bdstevel@tonic-gate	 * it up and log it.  %g1 must point to the ch_err_tl1_data structure.
18867c478bdstevel@tonic-gate	 * Restores the %g registers and issues retry.
18877c478bdstevel@tonic-gate	 */
18887c478bdstevel@tonic-gate	CH_ERR_TL1_EXIT;
18907c478bdstevel@tonic-gate	SET_SIZE(icache_parity_tl1_err)
18927c478bdstevel@tonic-gate#endif	/* CPU_IMP_L1_CACHE_PARITY */
18967c478bdstevel@tonic-gate * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the
18977c478bdstevel@tonic-gate * tte, the virtual address, and the ctxnum of the specified tlb entry.  They
18987c478bdstevel@tonic-gate * should only be used in places where you have no choice but to look at the
18997c478bdstevel@tonic-gate * tlb itself.
19007c478bdstevel@tonic-gate *
19017c478bdstevel@tonic-gate * Note: These two routines are required by the Estar "cpr" loadable module.
19027c478bdstevel@tonic-gate */
19047c478bdstevel@tonic-gate	ENTRY_NP(itlb_rd_entry)
19057c478bdstevel@tonic-gate	sllx	%o0, 3, %o0
19067c478bdstevel@tonic-gate	ldxa	[%o0]ASI_ITLB_ACCESS, %g1
19077c478bdstevel@tonic-gate	stx	%g1, [%o1]
19087c478bdstevel@tonic-gate	ldxa	[%o0]ASI_ITLB_TAGREAD, %g2
19097c478bdstevel@tonic-gate	set	TAGREAD_CTX_MASK, %o4
19107c478bdstevel@tonic-gate	andn	%g2, %o4, %o5
19117c478bdstevel@tonic-gate	retl
19127c478bdstevel@tonic-gate	  stx	%o5, [%o2]
19137c478bdstevel@tonic-gate	SET_SIZE(itlb_rd_entry)
19167c478bdstevel@tonic-gate	ENTRY_NP(dtlb_rd_entry)
19177c478bdstevel@tonic-gate	sllx	%o0, 3, %o0
19187c478bdstevel@tonic-gate	ldxa	[%o0]ASI_DTLB_ACCESS, %g1
19197c478bdstevel@tonic-gate	stx	%g1, [%o1]
19207c478bdstevel@tonic-gate	ldxa	[%o0]ASI_DTLB_TAGREAD, %g2
19217c478bdstevel@tonic-gate	set	TAGREAD_CTX_MASK, %o4
19227c478bdstevel@tonic-gate	andn	%g2, %o4, %o5
19237c478bdstevel@tonic-gate	retl
19247c478bdstevel@tonic-gate	  stx	%o5, [%o2]
19257c478bdstevel@tonic-gate	SET_SIZE(dtlb_rd_entry)
19287c478bdstevel@tonic-gate#if !(defined(JALAPENO) || defined(SERRANO))
19307c478bdstevel@tonic-gate	ENTRY(get_safari_config)
19317c478bdstevel@tonic-gate	ldxa	[%g0]ASI_SAFARI_CONFIG, %o0
19327c478bdstevel@tonic-gate	retl
19337c478bdstevel@tonic-gate	nop
19347c478bdstevel@tonic-gate	SET_SIZE(get_safari_config)
19377c478bdstevel@tonic-gate	ENTRY(set_safari_config)
19387c478bdstevel@tonic-gate	stxa	%o0, [%g0]ASI_SAFARI_CONFIG
19397c478bdstevel@tonic-gate	membar	#Sync
19407c478bdstevel@tonic-gate	retl
19417c478bdstevel@tonic-gate	nop
19427c478bdstevel@tonic-gate	SET_SIZE(set_safari_config)
19447c478bdstevel@tonic-gate#endif	/* !(JALAPENO || SERRANO) */
19477c478bdstevel@tonic-gate	/*
19487c478bdstevel@tonic-gate	 * Clear the NPT (non-privileged trap) bit in the %tick/%stick
19497c478bdstevel@tonic-gate	 * registers. In an effort to make the change in the
19507c478bdstevel@tonic-gate	 * tick/stick counter as consistent as possible, we disable
19517c478bdstevel@tonic-gate	 * all interrupts while we're changing the registers. We also
19527c478bdstevel@tonic-gate	 * ensure that the read and write instructions are in the same
19537c478bdstevel@tonic-gate	 * line in the instruction cache.
19547c478bdstevel@tonic-gate	 */
19557c478bdstevel@tonic-gate	ENTRY_NP(cpu_clearticknpt)
19567c478bdstevel@tonic-gate	rdpr	%pstate, %g1		/* save processor state */
19577c478bdstevel@tonic-gate	andn	%g1, PSTATE_IE, %g3	/* turn off */
19587c478bdstevel@tonic-gate	wrpr	%g0, %g3, %pstate	/*   interrupts */
19597c478bdstevel@tonic-gate	rdpr	%tick, %g2		/* get tick register */
19607c478bdstevel@tonic-gate	brgez,pn %g2, 1f		/* if NPT bit off, we're done */
19617c478bdstevel@tonic-gate	mov	1, %g3			/* create mask */
19627c478bdstevel@tonic-gate	sllx	%g3, 63, %g3		/*   for NPT bit */
19637c478bdstevel@tonic-gate	ba,a,pt	%xcc, 2f
19647c478bdstevel@tonic-gate	.align	8			/* Ensure rd/wr in same i$ line */
19667c478bdstevel@tonic-gate	rdpr	%tick, %g2		/* get tick register */
19677c478bdstevel@tonic-gate	wrpr	%g3, %g2, %tick		/* write tick register, */
19687c478bdstevel@tonic-gate					/*   clearing NPT bit   */
19707c478bdstevel@tonic-gate	rd	STICK, %g2		/* get stick register */
19717c478bdstevel@tonic-gate	brgez,pn %g2, 3f		/* if NPT bit off, we're done */
19727c478bdstevel@tonic-gate	mov	1, %g3			/* create mask */
19737c478bdstevel@tonic-gate	sllx	%g3, 63, %g3		/*   for NPT bit */
19747c478bdstevel@tonic-gate	ba,a,pt	%xcc, 4f
19757c478bdstevel@tonic-gate	.align	8			/* Ensure rd/wr in same i$ line */
19777c478bdstevel@tonic-gate	rd	STICK, %g2		/* get stick register */
19787c478bdstevel@tonic-gate	wr	%g3, %g2, STICK		/* write stick register, */
19797c478bdstevel@tonic-gate					/*   clearing NPT bit   */
19817c478bdstevel@tonic-gate	jmp	%g4 + 4
19827c478bdstevel@tonic-gate	wrpr	%g0, %g1, %pstate	/* restore processor state */
19847c478bdstevel@tonic-gate	SET_SIZE(cpu_clearticknpt)
19877c478bdstevel@tonic-gate#if defined(CPU_IMP_L1_CACHE_PARITY)
19907c478bdstevel@tonic-gate * correct_dcache_parity(size_t size, size_t linesize)
19917c478bdstevel@tonic-gate *
19927c478bdstevel@tonic-gate * Correct D$ data parity by zeroing the data and initializing microtag
19937c478bdstevel@tonic-gate * for all indexes and all ways of the D$.
19949b0bb79John Levon *
19957c478bdstevel@tonic-gate */
19967c478bdstevel@tonic-gate	ENTRY(correct_dcache_parity)
19977c478bdstevel@tonic-gate	/*
19987c478bdstevel@tonic-gate	 * Register Usage:
19997c478bdstevel@tonic-gate	 *
20007c478bdstevel@tonic-gate	 * %o0 = input D$ size
20017c478bdstevel@tonic-gate	 * %o1 = input D$ line size
20027c478bdstevel@tonic-gate	 * %o2 = scratch
20037c478bdstevel@tonic-gate	 * %o3 = scratch
20047c478bdstevel@tonic-gate	 * %o4 = scratch
20057c478bdstevel@tonic-gate	 */
20077c478bdstevel@tonic-gate	sub	%o0, %o1, %o0			! init cache line address
20097c478bdstevel@tonic-gate	/*
20107c478bdstevel@tonic-gate	 * For Panther CPUs, we also need to clear the data parity bits
20117c478bdstevel@tonic-gate	 * using DC_data_parity bit of the ASI_DCACHE_DATA register.
20127c478bdstevel@tonic-gate	 */
20137c478bdstevel@tonic-gate	GET_CPU_IMPL(%o3)
20147c478bdstevel@tonic-gate	cmp	%o3, PANTHER_IMPL
20157c478bdstevel@tonic-gate	bne	1f
20167c478bdstevel@tonic-gate	  clr	%o3				! zero for non-Panther
20177c478bdstevel@tonic-gate	mov	1, %o3
20187c478bdstevel@tonic-gate	sll	%o3, PN_DC_DATA_PARITY_BIT_SHIFT, %o3
20217c478bdstevel@tonic-gate	/*
20227c478bdstevel@tonic-gate	 * Set utag = way since it must be unique within an index.
20237c478bdstevel@tonic-gate	 */
20247c478bdstevel@tonic-gate	srl	%o0, 14, %o2			! get cache way (DC_way)
20257c478bdstevel@tonic-gate	membar	#Sync				! required before ASI_DC_UTAG
20267c478bdstevel@tonic-gate	stxa	%o2, [%o0]ASI_DC_UTAG		! set D$ utag = cache way
20277c478bdstevel@tonic-gate	membar	#Sync				! required after ASI_DC_UTAG
20297c478bdstevel@tonic-gate	/*
20307c478bdstevel@tonic-gate	 * Zero line of D$ data (and data parity bits for Panther)
20317c478bdstevel@tonic-gate	 */
20327c478bdstevel@tonic-gate	sub	%o1, 8, %o2
20337c478bdstevel@tonic-gate	or	%o0, %o3, %o4			! same address + DC_data_parity
20357c478bdstevel@tonic-gate	membar	#Sync				! required before ASI_DC_DATA
20367c478bdstevel@tonic-gate	stxa	%g0, [%o0 + %o2]ASI_DC_DATA	! zero 8 bytes of D$ data
20377c478bdstevel@tonic-gate	membar	#Sync				! required after ASI_DC_DATA
20387c478bdstevel@tonic-gate	/*
20397c478bdstevel@tonic-gate	 * We also clear the parity bits if this is a panther. For non-Panther
20407c478bdstevel@tonic-gate	 * CPUs, we simply end up clearing the $data register twice.
20417c478bdstevel@tonic-gate	 */
20427c478bdstevel@tonic-gate	stxa	%g0, [%o4 + %o2]ASI_DC_DATA
20437c478bdstevel@tonic-gate	membar	#Sync
20457c478bdstevel@tonic-gate	subcc	%o2, 8, %o2
20467c478bdstevel@tonic-gate	bge	2b
20477c478bdstevel@tonic-gate	nop
20497c478bdstevel@tonic-gate	subcc	%o0, %o1, %o0
20507c478bdstevel@tonic-gate	bge	1b
20517c478bdstevel@tonic-gate	nop
20537c478bdstevel@tonic-gate	retl
20547c478bdstevel@tonic-gate	  nop
20557c478bdstevel@tonic-gate	SET_SIZE(correct_dcache_parity)
20577c478bdstevel@tonic-gate#endif	/* CPU_IMP_L1_CACHE_PARITY */
20607c478bdstevel@tonic-gate	ENTRY_NP(stick_timestamp)
20617c478bdstevel@tonic-gate	rd	STICK, %g1	! read stick reg
20627c478bdstevel@tonic-gate	sllx	%g1, 1, %g1
20637c478bdstevel@tonic-gate	srlx	%g1, 1, %g1	! clear npt bit
20657c478bdstevel@tonic-gate	retl
20667c478bdstevel@tonic-gate	stx     %g1, [%o0]	! store the timestamp
20677c478bdstevel@tonic-gate	SET_SIZE(stick_timestamp)
20707c478bdstevel@tonic-gate	ENTRY_NP(stick_adj)
20717c478bdstevel@tonic-gate	rdpr	%pstate, %g1		! save processor state
20727c478bdstevel@tonic-gate	andn	%g1, PSTATE_IE, %g3
20737c478bdstevel@tonic-gate	ba	1f			! cache align stick adj
20747c478bdstevel@tonic-gate	wrpr	%g0, %g3, %pstate	! turn off interrupts
20767c478bdstevel@tonic-gate	.align	16
20777c478bdstevel@tonic-gate1:	nop
20797c478bdstevel@tonic-gate	rd	STICK, %g4		! read stick reg
20807c478bdstevel@tonic-gate	add	%g4, %o0, %o1		! adjust stick with skew
20817c478bdstevel@tonic-gate	wr	%o1, %g0, STICK		! write stick reg
20837c478bdstevel@tonic-gate	retl
20847c478bdstevel@tonic-gate	wrpr	%g1, %pstate		! restore processor state
20857c478bdstevel@tonic-gate	SET_SIZE(stick_adj)
20877c478bdstevel@tonic-gate	ENTRY_NP(kdi_get_stick)
20887c478bdstevel@tonic-gate	rd	STICK, %g1
20897c478bdstevel@tonic-gate	stx	%g1, [%o0]
20907c478bdstevel@tonic-gate	retl
20917c478bdstevel@tonic-gate	mov	%g0, %o0
20927c478bdstevel@tonic-gate	SET_SIZE(kdi_get_stick)
20957c478bdstevel@tonic-gate * Invalidate the specified line from the D$.
20967c478bdstevel@tonic-gate *
20977c478bdstevel@tonic-gate * Register usage:
20987c478bdstevel@tonic-gate *	%o0 - index for the invalidation, specifies DC_way and DC_addr
20997c478bdstevel@tonic-gate *
21007c478bdstevel@tonic-gate * ASI_DC_TAG, 0x47, is used in the following manner. A 64-bit value is
21017c478bdstevel@tonic-gate * stored to a particular DC_way and DC_addr in ASI_DC_TAG.
21027c478bdstevel@tonic-gate *
21037c478bdstevel@tonic-gate * The format of the stored 64-bit value is:
21047c478bdstevel@tonic-gate *
21057c478bdstevel@tonic-gate *	+----------+--------+----------+
21067c478bdstevel@tonic-gate *	| Reserved | DC_tag | DC_valid |
21077c478bdstevel@tonic-gate *	+----------+--------+----------+
21087c478bdstevel@tonic-gate *       63      31 30     1	      0
21097c478bdstevel@tonic-gate *
21107c478bdstevel@tonic-gate * DC_tag is the 30-bit physical tag of the associated line.
21117c478bdstevel@tonic-gate * DC_valid is the 1-bit valid field for both the physical and snoop tags.
21127c478bdstevel@tonic-gate *
21137c478bdstevel@tonic-gate * The format of the 64-bit DC_way and DC_addr into ASI_DC_TAG is:
21147c478bdstevel@tonic-gate *
21157c478bdstevel@tonic-gate *	+----------+--------+----------+----------+
21167c478bdstevel@tonic-gate *	| Reserved | DC_way | DC_addr  | Reserved |
21177c478bdstevel@tonic-gate *	+----------+--------+----------+----------+
21187c478bdstevel@tonic-gate *       63      16 15    14 13       5 4        0
21197c478bdstevel@tonic-gate *
21207c478bdstevel@tonic-gate * DC_way is a 2-bit index that selects one of the 4 ways.
21217c478bdstevel@tonic-gate * DC_addr is a 9-bit index that selects one of 512 tag/valid fields.
21227c478bdstevel@tonic-gate *
21237c478bdstevel@tonic-gate * Setting the DC_valid bit to zero for the specified DC_way and
21247c478bdstevel@tonic-gate * DC_addr index into the D$ results in an invalidation of a D$ line.
21257c478bdstevel@tonic-gate */
21267c478bdstevel@tonic-gate	ENTRY(dcache_inval_line)
21277c478bdstevel@tonic-gate	sll	%o0, 5, %o0		! shift index into DC_way and DC_addr
21287c478bdstevel@tonic-gate	stxa	%g0, [%o0]ASI_DC_TAG	! zero the DC_valid and DC_tag bits
21297c478bdstevel@tonic-gate	membar	#Sync
21307c478bdstevel@tonic-gate	retl
21317c478bdstevel@tonic-gate	nop
21327c478bdstevel@tonic-gate	SET_SIZE(dcache_inval_line)
21357c478bdstevel@tonic-gate * Invalidate the entire I$
21367c478bdstevel@tonic-gate *
21377c478bdstevel@tonic-gate * Register usage:
21387c478bdstevel@tonic-gate *	%o0 - specifies IC_way, IC_addr, IC_tag
21397c478bdstevel@tonic-gate *	%o1 - scratch
21407c478bdstevel@tonic-gate *	%o2 - used to save and restore DCU value
21417c478bdstevel@tonic-gate *	%o3 - scratch
21427c478bdstevel@tonic-gate *	%o5 - used to save and restore PSTATE
21437c478bdstevel@tonic-gate *
21447c478bdstevel@tonic-gate * Due to the behavior of the I$ control logic when accessing ASI_IC_TAG,
21457c478bdstevel@tonic-gate * the I$ should be turned off. Accesses to ASI_IC_TAG may collide and
21467c478bdstevel@tonic-gate * block out snoops and invalidates to the I$, causing I$ consistency
21477c478bdstevel@tonic-gate * to be broken. Before turning on the I$, all I$ lines must be invalidated.
21487c478bdstevel@tonic-gate *
21497c478bdstevel@tonic-gate * ASI_IC_TAG, 0x67, is used in the following manner. A 64-bit value is
21507c478bdstevel@tonic-gate * stored to a particular IC_way, IC_addr, IC_tag in ASI_IC_TAG. The
21517c478bdstevel@tonic-gate * info below describes store (write) use of ASI_IC_TAG. Note that read
21527c478bdstevel@tonic-gate * use of ASI_IC_TAG behaves differently.
21537c478bdstevel@tonic-gate *
21547c478bdstevel@tonic-gate * The format of the stored 64-bit value is:
21557c478bdstevel@tonic-gate *
21567c478bdstevel@tonic-gate *	+----------+--------+---------------+-----------+
21577c478bdstevel@tonic-gate *	| Reserved | Valid  | IC_vpred<7:0> | Undefined |
21587c478bdstevel@tonic-gate *	+----------+--------+---------------+-----------+
21597c478bdstevel@tonic-gate *       63      55    54    53           46 45        0
21607c478bdstevel@tonic-gate *
21617c478bdstevel@tonic-gate * Valid is the 1-bit valid field for both the physical and snoop tags.
21627c478bdstevel@tonic-gate * IC_vpred is the 8-bit LPB bits for 8 instructions starting at
21637c478bdstevel@tonic-gate *	the 32-byte boundary aligned address specified by IC_addr.
21647c478bdstevel@tonic-gate *
21657c478bdstevel@tonic-gate * The format of the 64-bit IC_way, IC_addr, IC_tag into ASI_IC_TAG is:
21667c478bdstevel@tonic-gate *
21677c478bdstevel@tonic-gate *	+----------+--------+---------+--------+---------+
21687c478bdstevel@tonic-gate *	| Reserved | IC_way | IC_addr | IC_tag |Reserved |
21697c478bdstevel@tonic-gate *	+----------+--------+---------+--------+---------+
21707c478bdstevel@tonic-gate *       63      16 15    14 13      5 4      3 2       0
21717c478bdstevel@tonic-gate *
21727c478bdstevel@tonic-gate * IC_way is a 2-bit index that selects one of the 4 ways.
21737c478bdstevel@tonic-gate * IC_addr[13:6] is an 8-bit index that selects one of 256 valid fields.
21747c478bdstevel@tonic-gate * IC_addr[5] is a "don't care" for a store.
21757c478bdstevel@tonic-gate * IC_tag set to 2 specifies that the stored value is to be interpreted
21767c478bdstevel@tonic-gate *	as containing Valid and IC_vpred as described above.
21777c478bdstevel@tonic-gate *
21787c478bdstevel@tonic-gate * Setting the Valid bit to zero for the specified IC_way and
21797c478bdstevel@tonic-gate * IC_addr index into the I$ results in an invalidation of an I$ line.
21807c478bdstevel@tonic-gate */
21817c478bdstevel@tonic-gate	ENTRY(icache_inval_all)
21827c478bdstevel@tonic-gate	rdpr	%pstate, %o5
21837c478bdstevel@tonic-gate	andn	%o5, PSTATE_IE, %o3
21847c478bdstevel@tonic-gate	wrpr	%g0, %o3, %pstate	! clear IE bit
21867c478bdstevel@tonic-gate	GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, icache_inval_all_1);
21877c478bdstevel@tonic-gate	ld	[%o0 + CHPR_ICACHE_LINESIZE], %o1
21887c478bdstevel@tonic-gate	ba,pt	%icc, 2f
21897c478bdstevel@tonic-gate	  ld	[%o0 + CHPR_ICACHE_SIZE], %o0
21917c478bdstevel@tonic-gate	ASM_LD(%o0, icache_size)
21927c478bdstevel@tonic-gate	ASM_LD(%o1, icache_linesize)
21947c478bdstevel@tonic-gate	CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4)
21967c478bdstevel@tonic-gate	retl
21977c478bdstevel@tonic-gate	wrpr	%g0, %o5, %pstate	! restore earlier pstate
21987c478bdstevel@tonic-gate	SET_SIZE(icache_inval_all)
22027c478bdstevel@tonic-gate * cache_scrubreq_tl1 is the crosstrap handler called on offlined cpus via a
22037c478bdstevel@tonic-gate * crosstrap.  It atomically increments the outstanding request counter and,
22047c478bdstevel@tonic-gate * if there was not already an outstanding request, branches to setsoftint_tl1
2205b0fc0e7govinda * to enqueue an intr_vec for the given inum.
22067c478bdstevel@tonic-gate */
22087c478bdstevel@tonic-gate	! Register usage:
22097c478bdstevel@tonic-gate	!
22107c478bdstevel@tonic-gate	! Arguments:
22117c478bdstevel@tonic-gate	! %g1 - inum
22127c478bdstevel@tonic-gate	! %g2 - index into chsm_outstanding array
22137c478bdstevel@tonic-gate	!
22147c478bdstevel@tonic-gate	! Internal:
22157c478bdstevel@tonic-gate	! %g2, %g3, %g5 - scratch
22167c478bdstevel@tonic-gate	! %g4 - ptr. to scrub_misc chsm_outstanding[index].
22177c478bdstevel@tonic-gate	! %g6 - setsoftint_tl1 address
22197c478bdstevel@tonic-gate	ENTRY_NP(cache_scrubreq_tl1)
22207c478bdstevel@tonic-gate	mulx	%g2, CHSM_OUTSTANDING_INCR, %g2
22217c478bdstevel@tonic-gate	set	CHPR_SCRUB_MISC + CHSM_OUTSTANDING, %g3
22227c478bdstevel@tonic-gate	add	%g2, %g3, %g2
22237c478bdstevel@tonic-gate	GET_CPU_PRIVATE_PTR(%g2, %g4, %g5, 1f);
22247c478bdstevel@tonic-gate	ld	[%g4], %g2		! cpu's chsm_outstanding[index]
22257c478bdstevel@tonic-gate	!
22267c478bdstevel@tonic-gate	! no need to use atomic instructions for the following
22277c478bdstevel@tonic-gate	! increment - we're at tl1
22287c478bdstevel@tonic-gate	!
22297c478bdstevel@tonic-gate	add	%g2, 0x1, %g3
2230b0fc0e7govinda	brnz,pn	%g2, 1f			! no need to enqueue more intr_vec
22317c478bdstevel@tonic-gate	  st	%g3, [%g4]		! delay - store incremented counter
22327c478bdstevel@tonic-gate	ASM_JMP(%g6, setsoftint_tl1)
22337c478bdstevel@tonic-gate	! not reached
22357c478bdstevel@tonic-gate	retry
22367c478bdstevel@tonic-gate	SET_SIZE(cache_scrubreq_tl1)
22407c478bdstevel@tonic-gate * Get the error state for the processor.
22417c478bdstevel@tonic-gate * Note that this must not be used at TL>0
22427c478bdstevel@tonic-gate */
22437c478bdstevel@tonic-gate	ENTRY(get_cpu_error_state)
22447c478bdstevel@tonic-gate#if defined(CHEETAH_PLUS)
22457c478bdstevel@tonic-gate	set	ASI_SHADOW_REG_VA, %o2
22467c478bdstevel@tonic-gate	ldxa	[%o2]ASI_AFSR, %o1		! shadow afsr reg
22477c478bdstevel@tonic-gate	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR]
22487c478bdstevel@tonic-gate	ldxa	[%o2]ASI_AFAR, %o1		! shadow afar reg
22497c478bdstevel@tonic-gate	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR]
22507c478bdstevel@tonic-gate	GET_CPU_IMPL(%o3)	! Only panther has AFSR_EXT registers
22517c478bdstevel@tonic-gate	cmp	%o3, PANTHER_IMPL
22527c478bdstevel@tonic-gate	bne,a	1f
22537c478bdstevel@tonic-gate	  stx	%g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT]	! zero for non-PN
22547c478bdstevel@tonic-gate	set	ASI_AFSR_EXT_VA, %o2
22557c478bdstevel@tonic-gate	ldxa	[%o2]ASI_AFSR, %o1		! afsr_ext reg
22567c478bdstevel@tonic-gate	stx	%o1, [%o0 + CH_CPU_ERRORS_AFSR_EXT]
22577c478bdstevel@tonic-gate	set	ASI_SHADOW_AFSR_EXT_VA, %o2
22587c478bdstevel@tonic-gate	ldxa	[%o2]ASI_AFSR, %o1		! shadow afsr_ext reg
22597c478bdstevel@tonic-gate	stx	%o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT]
22607c478bdstevel@tonic-gate	b	2f
22617c478bdstevel@tonic-gate	  nop
22637c478bdstevel@tonic-gate	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] ! zero for non-PN
22657c478bdstevel@tonic-gate#else	/* CHEETAH_PLUS */
22667c478bdstevel@tonic-gate	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR]
22677c478bdstevel@tonic-gate	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR]
22687c478bdstevel@tonic-gate	stx	%g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT]
22697c478bdstevel@tonic-gate	stx	%g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT]
22707c478bdstevel@tonic-gate#endif	/* CHEETAH_PLUS */
22717c478bdstevel@tonic-gate#if defined(SERRANO)
22727c478bdstevel@tonic-gate	/*
22737c478bdstevel@tonic-gate	 * Serrano has an afar2 which captures the address on FRC/FRU errors.
22747c478bdstevel@tonic-gate	 * We save this in the afar2 of the register save area.
22757c478bdstevel@tonic-gate	 */
22767c478bdstevel@tonic-gate	set	ASI_MCU_AFAR2_VA, %o2
22777c478bdstevel@tonic-gate	ldxa	[%o2]ASI_MCU_CTRL, %o1
22787c478bdstevel@tonic-gate	stx	%o1, [%o0 + CH_CPU_ERRORS_AFAR2]
22797c478bdstevel@tonic-gate#endif	/* SERRANO */
22807c478bdstevel@tonic-gate	ldxa	[%g0]ASI_AFSR, %o1		! primary afsr reg
22817c478bdstevel@tonic-gate	stx	%o1, [%o0 + CH_CPU_ERRORS_AFSR]
22827c478bdstevel@tonic-gate	ldxa	[%g0]ASI_AFAR, %o1		! primary afar reg
22837c478bdstevel@tonic-gate	retl
22847c478bdstevel@tonic-gate	stx	%o1, [%o0 + CH_CPU_ERRORS_AFAR]
22857c478bdstevel@tonic-gate	SET_SIZE(get_cpu_error_state)
22887c478bdstevel@tonic-gate * Check a page of memory for errors.
22897c478bdstevel@tonic-gate *
22907c478bdstevel@tonic-gate * Load each 64 byte block from physical memory.
22917c478bdstevel@tonic-gate * Check AFSR after each load to see if an error
22927c478bdstevel@tonic-gate * was caused. If so, log/scrub that error.
22937c478bdstevel@tonic-gate *
22947c478bdstevel@tonic-gate * Used to determine if a page contains
22957c478bdstevel@tonic-gate * CEs when CEEN is disabled.
22967c478bdstevel@tonic-gate */
22977c478bdstevel@tonic-gate	ENTRY(cpu_check_block)
22987c478bdstevel@tonic-gate	!
22997c478bdstevel@tonic-gate	! get a new window with room for the error regs
23007c478bdstevel@tonic-gate	!
23017c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME + CH_CPU_ERROR_SIZE), %sp
23027c478bdstevel@tonic-gate	srl	%i1, 6, %l4		! clear top bits of psz
23037c478bdstevel@tonic-gate					! and divide by 64
23047c478bdstevel@tonic-gate	rd	%fprs, %l2		! store FP
23057c478bdstevel@tonic-gate	wr	%g0, FPRS_FEF, %fprs	! enable FP
23077c478bdstevel@tonic-gate	ldda	[%i0]ASI_BLK_P, %d0	! load a block
23087c478bdstevel@tonic-gate	membar	#Sync
23097c478bdstevel@tonic-gate	ldxa    [%g0]ASI_AFSR, %l3	! read afsr reg
23107c478bdstevel@tonic-gate	brz,a,pt %l3, 2f		! check for error
23117c478bdstevel@tonic-gate	nop
23137c478bdstevel@tonic-gate	!
23147c478bdstevel@tonic-gate	! if error, read the error regs and log it
23157c478bdstevel@tonic-gate	!
23167c478bdstevel@tonic-gate	call	get_cpu_error_state
23177c478bdstevel@tonic-gate	add	%fp, STACK_BIAS - CH_CPU_ERROR_SIZE, %o0
23197c478bdstevel@tonic-gate	!
23207c478bdstevel@tonic-gate	! cpu_ce_detected(ch_cpu_errors_t *, flag)
23217c478bdstevel@tonic-gate	!
23227c478bdstevel@tonic-gate	call	cpu_ce_detected		! log the error
23237c478bdstevel@tonic-gate	mov	CE_CEEN_TIMEOUT, %o1
23257c478bdstevel@tonic-gate	dec	%l4			! next 64-byte block
23267c478bdstevel@tonic-gate	brnz,a,pt  %l4, 1b
23277c478bdstevel@tonic-gate	add	%i0, 64, %i0		! increment block addr
23297c478bdstevel@tonic-gate	wr	%l2, %g0, %fprs		! restore FP
23307c478bdstevel@tonic-gate	ret
23317c478bdstevel@tonic-gate	restore
23337c478bdstevel@tonic-gate	SET_SIZE(cpu_check_block)
23367c478bdstevel@tonic-gate * Perform a cpu logout called from C.  This is used where we did not trap
23377c478bdstevel@tonic-gate * for the error but still want to gather "what we can".  Caller must make
23387c478bdstevel@tonic-gate * sure cpu private area exists and that the indicated logout area is free
23397c478bdstevel@tonic-gate * for use, and that we are unable to migrate cpus.
23407c478bdstevel@tonic-gate */
23417c478bdstevel@tonic-gate	ENTRY(cpu_delayed_logout)
23427c478bdstevel@tonic-gate	rdpr	%pstate, %o2
23437c478bdstevel@tonic-gate	andn	%o2, PSTATE_IE, %o2
23447c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pstate		! disable interrupts
23457c478bdstevel@tonic-gate	PARK_SIBLING_CORE(%o2, %o3, %o4)	! %o2 has DCU value
23467c478bdstevel@tonic-gate	add	%o1, CH_CLO_DATA + CH_CHD_EC_DATA, %o1
23477c478bdstevel@tonic-gate	rd	%asi, %g1
23487c478bdstevel@tonic-gate	wr	%g0, ASI_P, %asi
23497c478bdstevel@tonic-gate	GET_ECACHE_DTAGS(%o0, %o1, %o3, %o4, %o5)
23507c478bdstevel@tonic-gate	wr	%g1, %asi
23517c478bdstevel@tonic-gate	UNPARK_SIBLING_CORE(%o2, %o3, %o4)	! can use %o2 again
23527c478bdstevel@tonic-gate	rdpr	%pstate, %o2
23537c478bdstevel@tonic-gate	or	%o2, PSTATE_IE, %o2
23547c478bdstevel@tonic-gate	wrpr	%g0, %o2, %pstate
23557c478bdstevel@tonic-gate	retl
23567c478bdstevel@tonic-gate	  nop
23577c478bdstevel@tonic-gate	SET_SIZE(cpu_delayed_logout)
23597c478bdstevel@tonic-gate	ENTRY(dtrace_blksuword32)
23607c478bdstevel@tonic-gate	save	%sp, -SA(MINFRAME + 4), %sp
23627c478bdstevel@tonic-gate	rdpr	%pstate, %l1
23637c478bdstevel@tonic-gate	andn	%l1, PSTATE_IE, %l2		! disable interrupts to
23647c478bdstevel@tonic-gate	wrpr	%g0, %l2, %pstate		! protect our FPU diddling
23667c478bdstevel@tonic-gate	rd	%fprs, %l0
23677c478bdstevel@tonic-gate	andcc	%l0, FPRS_FEF, %g0
23687c478bdstevel@tonic-gate	bz,a,pt	%xcc, 1f			! if the fpu is disabled
23697c478bdstevel@tonic-gate	wr	%g0, FPRS_FEF, %fprs		! ... enable the fpu
23717c478bdstevel@tonic-gate	st	%f0, [%fp + STACK_BIAS - 4]	! save %f0 to the stack
23737c478bdstevel@tonic-gate	set	0f, %l5
23747c478bdstevel@tonic-gate        /*
23757c478bdstevel@tonic-gate         * We're about to write a block full or either total garbage
23767c478bdstevel@tonic-gate         * (not kernel data, don't worry) or user floating-point data
23777c478bdstevel@tonic-gate         * (so it only _looks_ like garbage).
23787c478bdstevel@tonic-gate         */
23797c478bdstevel@tonic-gate	ld	[%i1], %f0			! modify the block
23807c478bdstevel@tonic-gate	membar	#Sync
23817c478bdstevel@tonic-gate	stn	%l5, [THREAD_REG + T_LOFAULT]	! set up the lofault handler
23827c478bdstevel@tonic-gate	stda	%d0, [%i0]ASI_BLK_COMMIT_S	! store the modified block
23837c478bdstevel@tonic-gate	membar	#Sync
23847c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler
23867c478bdstevel@tonic-gate	bz,a,pt	%xcc, 1f
23877c478bdstevel@tonic-gate	wr	%g0, %l0, %fprs			! restore %fprs
23897c478bdstevel@tonic-gate	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
23927c478bdstevel@tonic-gate	wrpr	%g0, %l1, %pstate		! restore interrupts
23947c478bdstevel@tonic-gate	ret
23957c478bdstevel@tonic-gate	restore	%g0, %g0, %o0
23987c478bdstevel@tonic-gate	membar	#Sync
23997c478bdstevel@tonic-gate	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler
24017c478bdstevel@tonic-gate	bz,a,pt	%xcc, 1f
24027c478bdstevel@tonic-gate	wr	%g0, %l0, %fprs			! restore %fprs
24047c478bdstevel@tonic-gate	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
24077c478bdstevel@tonic-gate	wrpr	%g0, %l1, %pstate		! restore interrupts
24097c478bdstevel@tonic-gate	/*
24107c478bdstevel@tonic-gate	 * If tryagain is set (%i2) we tail-call dtrace_blksuword32_err()
24117c478bdstevel@tonic-gate	 * which deals with watchpoints. Otherwise, just return -1.
24127c478bdstevel@tonic-gate	 */
24137c478bdstevel@tonic-gate	brnz,pt	%i2, 1f
24147c478bdstevel@tonic-gate	nop
24157c478bdstevel@tonic-gate	ret
24167c478bdstevel@tonic-gate	restore	%g0, -1, %o0
24187c478bdstevel@tonic-gate	call	dtrace_blksuword32_err
24197c478bdstevel@tonic-gate	restore
24217c478bdstevel@tonic-gate	SET_SIZE(dtrace_blksuword32)
24237c478bdstevel@tonic-gate#ifdef	CHEETAHPLUS_ERRATUM_25
24259b0bb79John Levon	/* Claim a chunk of physical address space. */
24267c478bdstevel@tonic-gate	ENTRY(claimlines)
24287c478bdstevel@tonic-gate	subcc	%o1, %o2, %o1
24297c478bdstevel@tonic-gate	add	%o0, %o1, %o3
24307c478bdstevel@tonic-gate	bgeu,a,pt	%xcc, 1b
24317c478bdstevel@tonic-gate	casxa	[%o3]ASI_MEM, %g0, %g0