17c478bd9Sstevel@tonic-gate/*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
51e2e7a75Shuah * Common Development and Distribution License (the "License").
61e2e7a75Shuah * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate/*
222dd3029aSjimand * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate */
257c478bd9Sstevel@tonic-gate
267c478bd9Sstevel@tonic-gate#include "assym.h"
277c478bd9Sstevel@tonic-gate
287c478bd9Sstevel@tonic-gate#include <sys/asm_linkage.h>
297c478bd9Sstevel@tonic-gate#include <sys/mmu.h>
307c478bd9Sstevel@tonic-gate#include <vm/hat_sfmmu.h>
317c478bd9Sstevel@tonic-gate#include <sys/machparam.h>
327c478bd9Sstevel@tonic-gate#include <sys/machcpuvar.h>
337c478bd9Sstevel@tonic-gate#include <sys/machthread.h>
347c478bd9Sstevel@tonic-gate#include <sys/privregs.h>
357c478bd9Sstevel@tonic-gate#include <sys/asm_linkage.h>
367c478bd9Sstevel@tonic-gate#include <sys/machasi.h>
377c478bd9Sstevel@tonic-gate#include <sys/trap.h>
387c478bd9Sstevel@tonic-gate#include <sys/spitregs.h>
397c478bd9Sstevel@tonic-gate#include <sys/xc_impl.h>
407c478bd9Sstevel@tonic-gate#include <sys/intreg.h>
417c478bd9Sstevel@tonic-gate#include <sys/async.h>
427c478bd9Sstevel@tonic-gate
437c478bd9Sstevel@tonic-gate#ifdef TRAPTRACE
447c478bd9Sstevel@tonic-gate#include <sys/traptrace.h>
457c478bd9Sstevel@tonic-gate#endif /* TRAPTRACE */
467c478bd9Sstevel@tonic-gate
477c478bd9Sstevel@tonic-gate/* BEGIN CSTYLED */
487c478bd9Sstevel@tonic-gate#define	DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3)			\
497c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_LSU, tmp1					;\
507c478bd9Sstevel@tonic-gate	btst	LSU_DC, tmp1		/* is dcache enabled? */	;\
517c478bd9Sstevel@tonic-gate	bz,pn	%icc, 1f						;\
527c478bd9Sstevel@tonic-gate	sethi	%hi(dcache_linesize), tmp1				;\
537c478bd9Sstevel@tonic-gate	ld	[tmp1 + %lo(dcache_linesize)], tmp1			;\
547c478bd9Sstevel@tonic-gate	sethi	%hi(dflush_type), tmp2					;\
557c478bd9Sstevel@tonic-gate	ld	[tmp2 + %lo(dflush_type)], tmp2				;\
567c478bd9Sstevel@tonic-gate	cmp	tmp2, FLUSHPAGE_TYPE					;\
577c478bd9Sstevel@tonic-gate	be,pt	%icc, 2f						;\
587c478bd9Sstevel@tonic-gate	sllx	arg1, SF_DC_VBIT_SHIFT, arg1	/* tag to compare */	;\
597c478bd9Sstevel@tonic-gate	sethi	%hi(dcache_size), tmp3					;\
607c478bd9Sstevel@tonic-gate	ld	[tmp3 + %lo(dcache_size)], tmp3				;\
617c478bd9Sstevel@tonic-gate	cmp	tmp2, FLUSHMATCH_TYPE					;\
627c478bd9Sstevel@tonic-gate	be,pt	%icc, 3f						;\
637c478bd9Sstevel@tonic-gate	nop								;\
647c478bd9Sstevel@tonic-gate	/*								\
657c478bd9Sstevel@tonic-gate	 * flushtype = FLUSHALL_TYPE, flush the whole thing		\
667c478bd9Sstevel@tonic-gate	 * tmp3 = cache size						\
677c478bd9Sstevel@tonic-gate	 * tmp1 = cache line size					\
687c478bd9Sstevel@tonic-gate	 */								\
697c478bd9Sstevel@tonic-gate	sub	tmp3, tmp1, tmp2					;\
707c478bd9Sstevel@tonic-gate4:									\
717c478bd9Sstevel@tonic-gate	stxa	%g0, [tmp2]ASI_DC_TAG					;\
727c478bd9Sstevel@tonic-gate	membar	#Sync							;\
737c478bd9Sstevel@tonic-gate	cmp	%g0, tmp2						;\
747c478bd9Sstevel@tonic-gate	bne,pt	%icc, 4b						;\
757c478bd9Sstevel@tonic-gate	sub	tmp2, tmp1, tmp2					;\
767c478bd9Sstevel@tonic-gate	ba,pt	%icc, 1f						;\
777c478bd9Sstevel@tonic-gate	nop								;\
787c478bd9Sstevel@tonic-gate	/*								\
797c478bd9Sstevel@tonic-gate	 * flushtype = FLUSHPAGE_TYPE					\
807c478bd9Sstevel@tonic-gate	 * arg1 = tag to compare against				\
817c478bd9Sstevel@tonic-gate	 * arg2 = virtual color						\
827c478bd9Sstevel@tonic-gate	 * tmp1 = cache line size					\
837c478bd9Sstevel@tonic-gate	 * tmp2 = tag from cache					\
847c478bd9Sstevel@tonic-gate	 * tmp3 = counter						\
857c478bd9Sstevel@tonic-gate	 */								\
867c478bd9Sstevel@tonic-gate2:									\
877c478bd9Sstevel@tonic-gate	set	MMU_PAGESIZE, tmp3					;\
887c478bd9Sstevel@tonic-gate	sllx	arg2, MMU_PAGESHIFT, arg2  /* color to dcache page */	;\
897c478bd9Sstevel@tonic-gate	sub	tmp3, tmp1, tmp3					;\
907c478bd9Sstevel@tonic-gate4:									\
917c478bd9Sstevel@tonic-gate	ldxa	[arg2 + tmp3]ASI_DC_TAG, tmp2	/* read tag */		;\
927c478bd9Sstevel@tonic-gate	btst	SF_DC_VBIT_MASK, tmp2					;\
937c478bd9Sstevel@tonic-gate	bz,pn	%icc, 5f	  /* branch if no valid sub-blocks */	;\
947c478bd9Sstevel@tonic-gate	andn	tmp2, SF_DC_VBIT_MASK, tmp2	/* clear out v bits */	;\
957c478bd9Sstevel@tonic-gate	cmp	tmp2, arg1						;\
967c478bd9Sstevel@tonic-gate	bne,pn	%icc, 5f			/* br if tag miss */	;\
977c478bd9Sstevel@tonic-gate	nop								;\
987c478bd9Sstevel@tonic-gate	stxa	%g0, [arg2 + tmp3]ASI_DC_TAG				;\
997c478bd9Sstevel@tonic-gate	membar	#Sync							;\
1007c478bd9Sstevel@tonic-gate5:									\
1017c478bd9Sstevel@tonic-gate	cmp	%g0, tmp3						;\
1027c478bd9Sstevel@tonic-gate	bnz,pt	%icc, 4b		/* branch if not done */	;\
1037c478bd9Sstevel@tonic-gate	sub	tmp3, tmp1, tmp3					;\
1047c478bd9Sstevel@tonic-gate	ba,pt	%icc, 1f						;\
1057c478bd9Sstevel@tonic-gate	nop								;\
1067c478bd9Sstevel@tonic-gate	/*								\
1077c478bd9Sstevel@tonic-gate	 * flushtype = FLUSHMATCH_TYPE					\
1087c478bd9Sstevel@tonic-gate	 * arg1 = tag to compare against				\
1097c478bd9Sstevel@tonic-gate	 * tmp1 = cache line size					\
1107c478bd9Sstevel@tonic-gate	 * tmp3 = cache size						\
1117c478bd9Sstevel@tonic-gate	 * arg2 = counter						\
1127c478bd9Sstevel@tonic-gate	 * tmp2 = cache tag						\
1137c478bd9Sstevel@tonic-gate	 */								\
1147c478bd9Sstevel@tonic-gate3:									\
1157c478bd9Sstevel@tonic-gate	sub	tmp3, tmp1, arg2					;\
1167c478bd9Sstevel@tonic-gate4:									\
1177c478bd9Sstevel@tonic-gate	ldxa	[arg2]ASI_DC_TAG, tmp2		/* read tag */		;\
1187c478bd9Sstevel@tonic-gate	btst	SF_DC_VBIT_MASK, tmp2					;\
1197c478bd9Sstevel@tonic-gate	bz,pn	%icc, 5f		/* br if no valid sub-blocks */	;\
1207c478bd9Sstevel@tonic-gate	andn	tmp2, SF_DC_VBIT_MASK, tmp2	/* clear out v bits */	;\
1217c478bd9Sstevel@tonic-gate	cmp	tmp2, arg1						;\
1227c478bd9Sstevel@tonic-gate	bne,pn	%icc, 5f		/* branch if tag miss */	;\
1237c478bd9Sstevel@tonic-gate	nop								;\
1247c478bd9Sstevel@tonic-gate	stxa	%g0, [arg2]ASI_DC_TAG					;\
1257c478bd9Sstevel@tonic-gate	membar	#Sync							;\
1267c478bd9Sstevel@tonic-gate5:									\
1277c478bd9Sstevel@tonic-gate	cmp	%g0, arg2						;\
1287c478bd9Sstevel@tonic-gate	bne,pt	%icc, 4b		/* branch if not done */	;\
1297c478bd9Sstevel@tonic-gate	sub	arg2, tmp1, arg2					;\
1307c478bd9Sstevel@tonic-gate1:
1317c478bd9Sstevel@tonic-gate
1327c478bd9Sstevel@tonic-gate/*
1337c478bd9Sstevel@tonic-gate * macro that flushes the entire dcache color
1347c478bd9Sstevel@tonic-gate */
1357c478bd9Sstevel@tonic-gate#define	DCACHE_FLUSHCOLOR(arg, tmp1, tmp2)				\
1367c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_LSU, tmp1;					\
1377c478bd9Sstevel@tonic-gate	btst	LSU_DC, tmp1;		/* is dcache enabled? */	\
1387c478bd9Sstevel@tonic-gate	bz,pn	%icc, 1f;						\
1397c478bd9Sstevel@tonic-gate	sethi	%hi(dcache_linesize), tmp1;				\
1407c478bd9Sstevel@tonic-gate	ld	[tmp1 + %lo(dcache_linesize)], tmp1;			\
1417c478bd9Sstevel@tonic-gate	set	MMU_PAGESIZE, tmp2;					\
1427c478bd9Sstevel@tonic-gate	/*								\
1437c478bd9Sstevel@tonic-gate	 * arg = virtual color						\
1447c478bd9Sstevel@tonic-gate	 * tmp2 = page size						\
1457c478bd9Sstevel@tonic-gate	 * tmp1 = cache line size					\
1467c478bd9Sstevel@tonic-gate	 */								\
1477c478bd9Sstevel@tonic-gate	sllx	arg, MMU_PAGESHIFT, arg; /* color to dcache page */	\
1487c478bd9Sstevel@tonic-gate	sub	tmp2, tmp1, tmp2;					\
1497c478bd9Sstevel@tonic-gate2:									\
1507c478bd9Sstevel@tonic-gate	stxa	%g0, [arg + tmp2]ASI_DC_TAG;				\
1517c478bd9Sstevel@tonic-gate	membar	#Sync;							\
1527c478bd9Sstevel@tonic-gate	cmp	%g0, tmp2;						\
1537c478bd9Sstevel@tonic-gate	bne,pt	%icc, 2b;						\
1547c478bd9Sstevel@tonic-gate	sub	tmp2, tmp1, tmp2;					\
1557c478bd9Sstevel@tonic-gate1:
1567c478bd9Sstevel@tonic-gate
1577c478bd9Sstevel@tonic-gate/*
1587c478bd9Sstevel@tonic-gate * macro that flushes the entire dcache
1597c478bd9Sstevel@tonic-gate */
1607c478bd9Sstevel@tonic-gate#define	DCACHE_FLUSHALL(size, linesize, tmp)				\
1617c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_LSU, tmp;					\
1627c478bd9Sstevel@tonic-gate	btst	LSU_DC, tmp;		/* is dcache enabled? */	\
1637c478bd9Sstevel@tonic-gate	bz,pn	%icc, 1f;						\
1647c478bd9Sstevel@tonic-gate									\
1657c478bd9Sstevel@tonic-gate	sub	size, linesize, tmp;					\
1667c478bd9Sstevel@tonic-gate2:									\
1677c478bd9Sstevel@tonic-gate	stxa	%g0, [tmp]ASI_DC_TAG;					\
1687c478bd9Sstevel@tonic-gate	membar	#Sync;							\
1697c478bd9Sstevel@tonic-gate	cmp	%g0, tmp;						\
1707c478bd9Sstevel@tonic-gate	bne,pt	%icc, 2b;						\
1717c478bd9Sstevel@tonic-gate	sub	tmp, linesize, tmp;					\
1727c478bd9Sstevel@tonic-gate1:
1737c478bd9Sstevel@tonic-gate
1747c478bd9Sstevel@tonic-gate/*
1757c478bd9Sstevel@tonic-gate * macro that flushes the entire icache
1767c478bd9Sstevel@tonic-gate */
1777c478bd9Sstevel@tonic-gate#define	ICACHE_FLUSHALL(size, linesize, tmp)				\
1787c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_LSU, tmp;					\
1797c478bd9Sstevel@tonic-gate	btst	LSU_IC, tmp;						\
1807c478bd9Sstevel@tonic-gate	bz,pn	%icc, 1f;						\
1817c478bd9Sstevel@tonic-gate									\
1827c478bd9Sstevel@tonic-gate	sub	size, linesize, tmp;					\
1837c478bd9Sstevel@tonic-gate2:									\
1847c478bd9Sstevel@tonic-gate	stxa	%g0, [tmp]ASI_IC_TAG;					\
1857c478bd9Sstevel@tonic-gate	membar	#Sync;							\
1867c478bd9Sstevel@tonic-gate	cmp	%g0, tmp;						\
1877c478bd9Sstevel@tonic-gate	bne,pt	%icc, 2b;						\
1887c478bd9Sstevel@tonic-gate	sub	tmp, linesize, tmp;					\
1897c478bd9Sstevel@tonic-gate1:
1907c478bd9Sstevel@tonic-gate
1911e2e7a75Shuah#ifdef SF_ERRATA_32
1921e2e7a75Shuah#define SF_WORKAROUND(tmp1, tmp2)                               \
1931e2e7a75Shuah        sethi   %hi(FLUSH_ADDR), tmp2                           ;\
1941e2e7a75Shuah        set     MMU_PCONTEXT, tmp1                              ;\
1951e2e7a75Shuah        stxa    %g0, [tmp1]ASI_DMMU                             ;\
1961e2e7a75Shuah        flush   tmp2                                            ;
1971e2e7a75Shuah#else
1981e2e7a75Shuah#define SF_WORKAROUND(tmp1, tmp2)
1991e2e7a75Shuah#endif /* SF_ERRATA_32 */
2001e2e7a75Shuah
2011e2e7a75Shuah/*
2021e2e7a75Shuah * arg1 = vaddr
2031e2e7a75Shuah * arg2 = ctxnum
2041e2e7a75Shuah *      - disable interrupts and clear address mask
2051e2e7a75Shuah *        to access 64 bit physaddr
2061e2e7a75Shuah *      - Blow out the TLB, flush user page.
2071e2e7a75Shuah *        . use secondary context.
2081e2e7a75Shuah */
2091e2e7a75Shuah#define VTAG_FLUSHUPAGE(lbl, arg1, arg2, tmp1, tmp2, tmp3, tmp4) \
2101e2e7a75Shuah        rdpr    %pstate, tmp1                                   ;\
2111e2e7a75Shuah        andn    tmp1, PSTATE_IE, tmp2				;\
2121e2e7a75Shuah        wrpr    tmp2, 0, %pstate                                ;\
2131e2e7a75Shuah        sethi   %hi(FLUSH_ADDR), tmp2                           ;\
2141e2e7a75Shuah        set     MMU_SCONTEXT, tmp3                              ;\
2151e2e7a75Shuah        ldxa    [tmp3]ASI_DMMU, tmp4                            ;\
2161e2e7a75Shuah        or      DEMAP_SECOND | DEMAP_PAGE_TYPE, arg1, arg1      ;\
2171e2e7a75Shuah        cmp     tmp4, arg2                                      ;\
2181e2e7a75Shuah        be,a,pt %icc, lbl/**/4                                  ;\
2191e2e7a75Shuah          nop                                                   ;\
2201e2e7a75Shuah        stxa    arg2, [tmp3]ASI_DMMU                            ;\
2211e2e7a75Shuahlbl/**/4:                                                       ;\
2221e2e7a75Shuah        stxa    %g0, [arg1]ASI_DTLB_DEMAP                       ;\
2231e2e7a75Shuah        stxa    %g0, [arg1]ASI_ITLB_DEMAP                       ;\
2241e2e7a75Shuah        flush   tmp2                                            ;\
2251e2e7a75Shuah        be,a,pt %icc, lbl/**/5                                  ;\
2261e2e7a75Shuah          nop                                                   ;\
2271e2e7a75Shuah        stxa    tmp4, [tmp3]ASI_DMMU                            ;\
2281e2e7a75Shuah        flush   tmp2                                            ;\
2291e2e7a75Shuahlbl/**/5:                                                       ;\
2301e2e7a75Shuah        wrpr    %g0, tmp1, %pstate
2311e2e7a75Shuah
2321e2e7a75Shuah
2331e2e7a75Shuah/*
2341e2e7a75Shuah * macro that flushes all the user entries in dtlb
2351e2e7a75Shuah * arg1 = dtlb entries
2361e2e7a75Shuah *	- Before first compare:
2371e2e7a75Shuah *              tmp4 = tte
2381e2e7a75Shuah *              tmp5 = vaddr
2391e2e7a75Shuah *              tmp6 = cntxnum
2401e2e7a75Shuah */
2411e2e7a75Shuah#define DTLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \
2421e2e7a75Shuah                                tmp4, tmp5, tmp6) \
2431e2e7a75Shuahlbl/**/0:                                                       ;\
2441e2e7a75Shuah        sllx    arg1, 3, tmp3                                   ;\
2451e2e7a75Shuah        SF_WORKAROUND(tmp1, tmp2)                               ;\
2461e2e7a75Shuah        ldxa    [tmp3]ASI_DTLB_ACCESS, tmp4                     ;\
2471e2e7a75Shuah        srlx    tmp4, 6, tmp4                                   ;\
2481e2e7a75Shuah        andcc   tmp4, 1, %g0                                    ;\
2491e2e7a75Shuah        bnz,pn  %xcc, lbl/**/1                                  ;\
2501e2e7a75Shuah        srlx    tmp4, 57, tmp4                                  ;\
2511e2e7a75Shuah        andcc   tmp4, 1, %g0                                    ;\
2521e2e7a75Shuah        beq,pn  %xcc, lbl/**/1                                  ;\
2531e2e7a75Shuah          nop                                                   ;\
2541e2e7a75Shuah        set     TAGREAD_CTX_MASK, tmp1                          ;\
2551e2e7a75Shuah        ldxa    [tmp3]ASI_DTLB_TAGREAD, tmp2                    ;\
2561e2e7a75Shuah        and     tmp2, tmp1, tmp6                                ;\
2571e2e7a75Shuah        andn    tmp2, tmp1, tmp5                                ;\
2581e2e7a75Shuah	set	KCONTEXT, tmp4					;\
2591e2e7a75Shuah	cmp	tmp6, tmp4					;\
2601e2e7a75Shuah	be	lbl/**/1					;\
2611e2e7a75Shuah	  nop							;\
2621e2e7a75Shuah        VTAG_FLUSHUPAGE(VD/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\
2631e2e7a75Shuahlbl/**/1:                                                       ;\
2641e2e7a75Shuah        brgz,pt arg1, lbl/**/0                                  ;\
2651e2e7a75Shuah          sub     arg1, 1, arg1
2661e2e7a75Shuah
2671e2e7a75Shuah
2681e2e7a75Shuah/*
2691e2e7a75Shuah * macro that flushes all the user entries in itlb
2701e2e7a75Shuah * arg1 = itlb entries
2711e2e7a75Shuah *      - Before first compare:
2721e2e7a75Shuah *              tmp4 = tte
2731e2e7a75Shuah *              tmp5 = vaddr
2741e2e7a75Shuah *              tmp6 = cntxnum
2751e2e7a75Shuah */
2761e2e7a75Shuah#define ITLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \
2771e2e7a75Shuah                                tmp4, tmp5, tmp6) \
2781e2e7a75Shuahlbl/**/0:                                                       ;\
2791e2e7a75Shuah        sllx    arg1, 3, tmp3                                   ;\
2801e2e7a75Shuah        SF_WORKAROUND(tmp1, tmp2)                               ;\
2811e2e7a75Shuah        ldxa    [tmp3]ASI_ITLB_ACCESS, tmp4                     ;\
2821e2e7a75Shuah        srlx    tmp4, 6, tmp4                                   ;\
2831e2e7a75Shuah        andcc   tmp4, 1, %g0                                    ;\
2841e2e7a75Shuah        bnz,pn  %xcc, lbl/**/1                                  ;\
2851e2e7a75Shuah        srlx    tmp4, 57, tmp4                                  ;\
2861e2e7a75Shuah        andcc   tmp4, 1, %g0                                    ;\
2871e2e7a75Shuah        beq,pn  %xcc, lbl/**/1                                  ;\
2881e2e7a75Shuah          nop                                                   ;\
2891e2e7a75Shuah        set     TAGREAD_CTX_MASK, tmp1                          ;\
2901e2e7a75Shuah        ldxa    [tmp3]ASI_ITLB_TAGREAD, tmp2                    ;\
2911e2e7a75Shuah        and     tmp2, tmp1, tmp6                                ;\
2921e2e7a75Shuah        andn    tmp2, tmp1, tmp5                                ;\
2931e2e7a75Shuah	set	KCONTEXT, tmp4					;\
2941e2e7a75Shuah	cmp	tmp6, tmp4					;\
2951e2e7a75Shuah	be	lbl/**/1					;\
2961e2e7a75Shuah	  nop							;\
2971e2e7a75Shuah        VTAG_FLUSHUPAGE(VI/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\
2981e2e7a75Shuahlbl/**/1:                                                       ;\
2991e2e7a75Shuah        brgz,pt arg1, lbl/**/0                                  ;\
3001e2e7a75Shuah        sub     arg1, 1, arg1
3011e2e7a75Shuah
3021e2e7a75Shuah
3031e2e7a75Shuah
3047c478bd9Sstevel@tonic-gate/*
3057c478bd9Sstevel@tonic-gate * Macro for getting to offset from 'cpu_private' ptr. The 'cpu_private'
3067c478bd9Sstevel@tonic-gate * ptr is in the machcpu structure.
3077c478bd9Sstevel@tonic-gate * r_or_s:	Register or symbol off offset from 'cpu_private' ptr.
3087c478bd9Sstevel@tonic-gate * scr1:	Scratch, ptr is returned in this register.
3097c478bd9Sstevel@tonic-gate * scr2:	Scratch
3107c478bd9Sstevel@tonic-gate */
3117c478bd9Sstevel@tonic-gate#define GET_CPU_PRIVATE_PTR(r_or_s, scr1, scr2, label)		\
3127c478bd9Sstevel@tonic-gate	CPU_ADDR(scr1, scr2);						\
3137c478bd9Sstevel@tonic-gate	ldn	[scr1 + CPU_PRIVATE], scr1; 				\
3147c478bd9Sstevel@tonic-gate	cmp	scr1, 0; 						\
3157c478bd9Sstevel@tonic-gate	be	label;							\
3167c478bd9Sstevel@tonic-gate	 nop; 								\
3177c478bd9Sstevel@tonic-gate	add	scr1, r_or_s, scr1;  					\
3187c478bd9Sstevel@tonic-gate
3197c478bd9Sstevel@tonic-gate#ifdef HUMMINGBIRD
3207c478bd9Sstevel@tonic-gate/*
3217c478bd9Sstevel@tonic-gate * UltraSPARC-IIe processor supports both 4-way set associative and
3227c478bd9Sstevel@tonic-gate * direct map E$. For performance reasons, we flush E$ by placing it
3237c478bd9Sstevel@tonic-gate * in direct map mode for data load/store and restore the state after
3247c478bd9Sstevel@tonic-gate * we are done flushing it. Keep interrupts off while flushing in this
3257c478bd9Sstevel@tonic-gate * manner.
3267c478bd9Sstevel@tonic-gate *
3277c478bd9Sstevel@tonic-gate * We flush the entire ecache by starting at one end and loading each
3287c478bd9Sstevel@tonic-gate * successive ecache line for the 2*ecache-size range. We have to repeat
3297c478bd9Sstevel@tonic-gate * the flush operation to guarantee that the entire ecache has been
3307c478bd9Sstevel@tonic-gate * flushed.
3317c478bd9Sstevel@tonic-gate *
3327c478bd9Sstevel@tonic-gate * For flushing a specific physical address, we start at the aliased
3337c478bd9Sstevel@tonic-gate * address and load at set-size stride, wrapping around at 2*ecache-size
3347c478bd9Sstevel@tonic-gate * boundary and skipping the physical address being flushed. It takes
3357c478bd9Sstevel@tonic-gate * 10 loads to guarantee that the physical address has been flushed.
3367c478bd9Sstevel@tonic-gate */
3377c478bd9Sstevel@tonic-gate
3387c478bd9Sstevel@tonic-gate#define	HB_ECACHE_FLUSH_CNT	2
3397c478bd9Sstevel@tonic-gate#define	HB_PHYS_FLUSH_CNT	10	/* #loads to flush specific paddr */
3407c478bd9Sstevel@tonic-gate#endif /* HUMMINGBIRD */
3417c478bd9Sstevel@tonic-gate
3427c478bd9Sstevel@tonic-gate/* END CSTYLED */
3437c478bd9Sstevel@tonic-gate
3447c478bd9Sstevel@tonic-gate/*
3457c478bd9Sstevel@tonic-gate * Spitfire MMU and Cache operations.
3467c478bd9Sstevel@tonic-gate */
3477c478bd9Sstevel@tonic-gate
3487c478bd9Sstevel@tonic-gate	ENTRY_NP(vtag_flushpage)
3497c478bd9Sstevel@tonic-gate	/*
3507c478bd9Sstevel@tonic-gate	 * flush page from the tlb
3517c478bd9Sstevel@tonic-gate	 *
3527c478bd9Sstevel@tonic-gate	 * %o0 = vaddr
3531e2e7a75Shuah	 * %o1 = sfmmup
3547c478bd9Sstevel@tonic-gate	 */
3557c478bd9Sstevel@tonic-gate	rdpr	%pstate, %o5
3567c478bd9Sstevel@tonic-gate#ifdef DEBUG
3571e2e7a75Shuah	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfdi_label1, %g1)
3587c478bd9Sstevel@tonic-gate#endif /* DEBUG */
3597c478bd9Sstevel@tonic-gate	/*
3607c478bd9Sstevel@tonic-gate	 * disable ints
3617c478bd9Sstevel@tonic-gate	 */
3627c478bd9Sstevel@tonic-gate	andn	%o5, PSTATE_IE, %o4
3637c478bd9Sstevel@tonic-gate	wrpr	%o4, 0, %pstate
3647c478bd9Sstevel@tonic-gate
3657c478bd9Sstevel@tonic-gate	/*
3667c478bd9Sstevel@tonic-gate	 * Then, blow out the tlb
3677c478bd9Sstevel@tonic-gate	 * Interrupts are disabled to prevent the secondary ctx register
3687c478bd9Sstevel@tonic-gate	 * from changing underneath us.
3697c478bd9Sstevel@tonic-gate	 */
3701e2e7a75Shuah	sethi   %hi(ksfmmup), %o3
3711e2e7a75Shuah        ldx     [%o3 + %lo(ksfmmup)], %o3
3721e2e7a75Shuah        cmp     %o3, %o1
3731e2e7a75Shuah        bne,pt   %xcc, 1f			! if not kernel as, go to 1
3741e2e7a75Shuah	  sethi	%hi(FLUSH_ADDR), %o3
3757c478bd9Sstevel@tonic-gate	/*
3767c478bd9Sstevel@tonic-gate	 * For KCONTEXT demaps use primary. type = page implicitly
3777c478bd9Sstevel@tonic-gate	 */
3787c478bd9Sstevel@tonic-gate	stxa	%g0, [%o0]ASI_DTLB_DEMAP	/* dmmu flush for KCONTEXT */
3797c478bd9Sstevel@tonic-gate	stxa	%g0, [%o0]ASI_ITLB_DEMAP	/* immu flush for KCONTEXT */
3801e2e7a75Shuah	flush	%o3
3817c478bd9Sstevel@tonic-gate	b	5f
3821e2e7a75Shuah	  nop
3837c478bd9Sstevel@tonic-gate1:
3847c478bd9Sstevel@tonic-gate	/*
3857c478bd9Sstevel@tonic-gate	 * User demap.  We need to set the secondary context properly.
3867c478bd9Sstevel@tonic-gate	 * %o0 = vaddr
3871e2e7a75Shuah	 * %o1 = sfmmup
3887c478bd9Sstevel@tonic-gate	 * %o3 = FLUSH_ADDR
3897c478bd9Sstevel@tonic-gate	 */
3901e2e7a75Shuah	SFMMU_CPU_CNUM(%o1, %g1, %g2)	/* %g1 = sfmmu cnum on this CPU */
3911e2e7a75Shuah
3927c478bd9Sstevel@tonic-gate	set	MMU_SCONTEXT, %o4
3937c478bd9Sstevel@tonic-gate	ldxa	[%o4]ASI_DMMU, %o2		/* rd old ctxnum */
3947c478bd9Sstevel@tonic-gate	or	DEMAP_SECOND | DEMAP_PAGE_TYPE, %o0, %o0
3951e2e7a75Shuah	cmp	%o2, %g1
3961e2e7a75Shuah	be,pt	%icc, 4f
3977c478bd9Sstevel@tonic-gate	  nop
3981e2e7a75Shuah	stxa	%g1, [%o4]ASI_DMMU		/* wr new ctxum */
3997c478bd9Sstevel@tonic-gate4:
4007c478bd9Sstevel@tonic-gate	stxa	%g0, [%o0]ASI_DTLB_DEMAP
4017c478bd9Sstevel@tonic-gate	stxa	%g0, [%o0]ASI_ITLB_DEMAP
4027c478bd9Sstevel@tonic-gate	flush	%o3
4031e2e7a75Shuah	be,pt	%icc, 5f
4047c478bd9Sstevel@tonic-gate	  nop
4057c478bd9Sstevel@tonic-gate	stxa	%o2, [%o4]ASI_DMMU		/* restore old ctxnum */
4067c478bd9Sstevel@tonic-gate	flush	%o3
4077c478bd9Sstevel@tonic-gate5:
4087c478bd9Sstevel@tonic-gate	retl
4097c478bd9Sstevel@tonic-gate	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
4107c478bd9Sstevel@tonic-gate	SET_SIZE(vtag_flushpage)
4111e2e7a75Shuah
4121e2e7a75Shuah        .seg    ".text"
4131e2e7a75Shuah.flushallmsg:
4141e2e7a75Shuah        .asciz  "sfmmu_asm: unimplemented flush operation"
4151e2e7a75Shuah
4161e2e7a75Shuah        ENTRY_NP(vtag_flushall)
4171e2e7a75Shuah        sethi   %hi(.flushallmsg), %o0
4181e2e7a75Shuah        call    panic
4191e2e7a75Shuah          or    %o0, %lo(.flushallmsg), %o0
4201e2e7a75Shuah        SET_SIZE(vtag_flushall)
4217c478bd9Sstevel@tonic-gate
4221e2e7a75Shuah	ENTRY_NP(vtag_flushall_uctxs)
4237c478bd9Sstevel@tonic-gate	/*
4241e2e7a75Shuah	 * flush entire DTLB/ITLB.
4257c478bd9Sstevel@tonic-gate	 */
4261e2e7a75Shuah	CPU_INDEX(%g1, %g2)
4271e2e7a75Shuah	mulx	%g1, CPU_NODE_SIZE, %g1
4281e2e7a75Shuah	set	cpunodes, %g2
4291e2e7a75Shuah	add	%g1, %g2, %g1
4301e2e7a75Shuah	lduh	[%g1 + ITLB_SIZE], %g2		! %g2 = # entries in ITLB
4311e2e7a75Shuah	lduh	[%g1 + DTLB_SIZE], %g1		! %g1 = # entries in DTLB
4321e2e7a75Shuah	sub	%g2, 1, %g2			! %g2 = # entries in ITLB - 1
4331e2e7a75Shuah	sub	%g1, 1, %g1			! %g1 = # entries in DTLB - 1
4341e2e7a75Shuah
4351e2e7a75Shuah        !
4361e2e7a75Shuah        ! Flush itlb's
4371e2e7a75Shuah        !
4381e2e7a75Shuah        ITLB_FLUSH_UNLOCKED_UCTXS(I, %g2, %g3, %g4, %o2, %o3, %o4, %o5)
4391e2e7a75Shuah
4401e2e7a75Shuah	!
4411e2e7a75Shuah        ! Flush dtlb's
4421e2e7a75Shuah        !
4431e2e7a75Shuah        DTLB_FLUSH_UNLOCKED_UCTXS(D, %g1, %g3, %g4, %o2, %o3, %o4, %o5)
4441e2e7a75Shuah
4451e2e7a75Shuah	membar  #Sync
4467c478bd9Sstevel@tonic-gate	retl
4471e2e7a75Shuah	  nop
4481e2e7a75Shuah
4491e2e7a75Shuah	SET_SIZE(vtag_flushall_uctxs)
4507c478bd9Sstevel@tonic-gate
4517c478bd9Sstevel@tonic-gate	ENTRY_NP(vtag_flushpage_tl1)
4527c478bd9Sstevel@tonic-gate	/*
4537c478bd9Sstevel@tonic-gate	 * x-trap to flush page from tlb and tsb
4547c478bd9Sstevel@tonic-gate	 *
4557c478bd9Sstevel@tonic-gate	 * %g1 = vaddr, zero-extended on 32-bit kernel
4561e2e7a75Shuah	 * %g2 = sfmmup
4577c478bd9Sstevel@tonic-gate	 *
4587c478bd9Sstevel@tonic-gate	 * assumes TSBE_TAG = 0
4597c478bd9Sstevel@tonic-gate	 */
4607c478bd9Sstevel@tonic-gate	srln	%g1, MMU_PAGESHIFT, %g1
4617c478bd9Sstevel@tonic-gate	slln	%g1, MMU_PAGESHIFT, %g1			/* g1 = vaddr */
4621e2e7a75Shuah
4631e2e7a75Shuah	SFMMU_CPU_CNUM(%g2, %g3, %g4)   /* %g3 = sfmmu cnum on this CPU */
4641e2e7a75Shuah
4657c478bd9Sstevel@tonic-gate	/* We need to set the secondary context properly. */
4667c478bd9Sstevel@tonic-gate	set	MMU_SCONTEXT, %g4
4677c478bd9Sstevel@tonic-gate	ldxa	[%g4]ASI_DMMU, %g5		/* rd old ctxnum */
4687c478bd9Sstevel@tonic-gate	or	DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1
4691e2e7a75Shuah	stxa	%g3, [%g4]ASI_DMMU		/* wr new ctxum */
4707c478bd9Sstevel@tonic-gate	stxa	%g0, [%g1]ASI_DTLB_DEMAP
4717c478bd9Sstevel@tonic-gate	stxa	%g0, [%g1]ASI_ITLB_DEMAP
4727c478bd9Sstevel@tonic-gate	stxa	%g5, [%g4]ASI_DMMU		/* restore old ctxnum */
4737c478bd9Sstevel@tonic-gate	membar #Sync
4747c478bd9Sstevel@tonic-gate	retry
4757c478bd9Sstevel@tonic-gate	SET_SIZE(vtag_flushpage_tl1)
4767c478bd9Sstevel@tonic-gate
4777c478bd9Sstevel@tonic-gate	ENTRY_NP(vtag_flush_pgcnt_tl1)
4787c478bd9Sstevel@tonic-gate	/*
4797c478bd9Sstevel@tonic-gate	 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb
4807c478bd9Sstevel@tonic-gate	 *
4817c478bd9Sstevel@tonic-gate	 * %g1 = vaddr, zero-extended on 32-bit kernel
4821e2e7a75Shuah	 * %g2 = <sfmmup58 | pgcnt6>
4837c478bd9Sstevel@tonic-gate	 *
4847c478bd9Sstevel@tonic-gate	 * NOTE: this handler relies on the fact that no
4857c478bd9Sstevel@tonic-gate	 *	interrupts or traps can occur during the loop
4867c478bd9Sstevel@tonic-gate	 *	issuing the TLB_DEMAP operations. It is assumed
4877c478bd9Sstevel@tonic-gate	 *	that interrupts are disabled and this code is
4887c478bd9Sstevel@tonic-gate	 *	fetching from the kernel locked text address.
4897c478bd9Sstevel@tonic-gate	 *
4907c478bd9Sstevel@tonic-gate	 * assumes TSBE_TAG = 0
4917c478bd9Sstevel@tonic-gate	 */
4927c478bd9Sstevel@tonic-gate	srln	%g1, MMU_PAGESHIFT, %g1
4937c478bd9Sstevel@tonic-gate	slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
4947c478bd9Sstevel@tonic-gate	or	DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1
4951e2e7a75Shuah
4961e2e7a75Shuah	set	SFMMU_PGCNT_MASK, %g4
4971e2e7a75Shuah	and	%g4, %g2, %g3			/* g3 = pgcnt - 1 */
4981e2e7a75Shuah	add	%g3, 1, %g3			/* g3 = pgcnt */
4991e2e7a75Shuah
5001e2e7a75Shuah	andn	%g2, SFMMU_PGCNT_MASK, %g2	/* g2 = sfmmup */
5011e2e7a75Shuah
5021e2e7a75Shuah	SFMMU_CPU_CNUM(%g2, %g5, %g6)   ! %g5 = sfmmu cnum on this CPU
5031e2e7a75Shuah
5047c478bd9Sstevel@tonic-gate	/* We need to set the secondary context properly. */
5057c478bd9Sstevel@tonic-gate	set	MMU_SCONTEXT, %g4
5061e2e7a75Shuah	ldxa	[%g4]ASI_DMMU, %g6		/* read old ctxnum */
5071e2e7a75Shuah	stxa	%g5, [%g4]ASI_DMMU		/* write new ctxum */
5087c478bd9Sstevel@tonic-gate
5097c478bd9Sstevel@tonic-gate	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
5101e2e7a75Shuah	sethi	 %hi(FLUSH_ADDR), %g5
5117c478bd9Sstevel@tonic-gate1:
5127c478bd9Sstevel@tonic-gate	stxa	%g0, [%g1]ASI_DTLB_DEMAP
5137c478bd9Sstevel@tonic-gate	stxa	%g0, [%g1]ASI_ITLB_DEMAP
5141e2e7a75Shuah	flush	%g5
5157c478bd9Sstevel@tonic-gate	deccc	%g3				/* decr pgcnt */
5167c478bd9Sstevel@tonic-gate	bnz,pt	%icc,1b
5171e2e7a75Shuah	  add	%g1, %g2, %g1			/* go to nextpage */
5187c478bd9Sstevel@tonic-gate
5191e2e7a75Shuah	stxa	%g6, [%g4]ASI_DMMU		/* restore old ctxnum */
5207c478bd9Sstevel@tonic-gate	membar #Sync
5217c478bd9Sstevel@tonic-gate	retry
5227c478bd9Sstevel@tonic-gate	SET_SIZE(vtag_flush_pgcnt_tl1)
5237c478bd9Sstevel@tonic-gate
5247c478bd9Sstevel@tonic-gate	! Not implemented on US1/US2
5257c478bd9Sstevel@tonic-gate	ENTRY_NP(vtag_flushall_tl1)
5267c478bd9Sstevel@tonic-gate	retry
5277c478bd9Sstevel@tonic-gate	SET_SIZE(vtag_flushall_tl1)
5287c478bd9Sstevel@tonic-gate
5297c478bd9Sstevel@tonic-gate/*
5307c478bd9Sstevel@tonic-gate * vac_flushpage(pfnum, color)
5317c478bd9Sstevel@tonic-gate *	Flush 1 8k page of the D-$ with physical page = pfnum
5327c478bd9Sstevel@tonic-gate *	Algorithm:
5337c478bd9Sstevel@tonic-gate *		The spitfire dcache is a 16k direct mapped virtual indexed,
5347c478bd9Sstevel@tonic-gate *		physically tagged cache.  Given the pfnum we read all cache
5357c478bd9Sstevel@tonic-gate *		lines for the corresponding page in the cache (determined by
5367c478bd9Sstevel@tonic-gate *		the color).  Each cache line is compared with
5377c478bd9Sstevel@tonic-gate *		the tag created from the pfnum. If the tags match we flush
5387c478bd9Sstevel@tonic-gate *		the line.
5397c478bd9Sstevel@tonic-gate */
5407c478bd9Sstevel@tonic-gate	.seg	".data"
5417c478bd9Sstevel@tonic-gate	.align	8
5427c478bd9Sstevel@tonic-gate	.global	dflush_type
5437c478bd9Sstevel@tonic-gatedflush_type:
5447c478bd9Sstevel@tonic-gate	.word	FLUSHPAGE_TYPE
5457c478bd9Sstevel@tonic-gate	.seg	".text"
5467c478bd9Sstevel@tonic-gate
5477c478bd9Sstevel@tonic-gate	ENTRY(vac_flushpage)
5487c478bd9Sstevel@tonic-gate	/*
5497c478bd9Sstevel@tonic-gate	 * flush page from the d$
5507c478bd9Sstevel@tonic-gate	 *
5517c478bd9Sstevel@tonic-gate	 * %o0 = pfnum, %o1 = color
5527c478bd9Sstevel@tonic-gate	 */
5537c478bd9Sstevel@tonic-gate	DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4)
5547c478bd9Sstevel@tonic-gate	retl
5557c478bd9Sstevel@tonic-gate	nop
5567c478bd9Sstevel@tonic-gate	SET_SIZE(vac_flushpage)
5577c478bd9Sstevel@tonic-gate
5587c478bd9Sstevel@tonic-gate	ENTRY_NP(vac_flushpage_tl1)
5597c478bd9Sstevel@tonic-gate	/*
5607c478bd9Sstevel@tonic-gate	 * x-trap to flush page from the d$
5617c478bd9Sstevel@tonic-gate	 *
5627c478bd9Sstevel@tonic-gate	 * %g1 = pfnum, %g2 = color
5637c478bd9Sstevel@tonic-gate	 */
5647c478bd9Sstevel@tonic-gate	DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5)
5657c478bd9Sstevel@tonic-gate	retry
5667c478bd9Sstevel@tonic-gate	SET_SIZE(vac_flushpage_tl1)
5677c478bd9Sstevel@tonic-gate
5687c478bd9Sstevel@tonic-gate	ENTRY(vac_flushcolor)
5697c478bd9Sstevel@tonic-gate	/*
5707c478bd9Sstevel@tonic-gate	 * %o0 = vcolor
5717c478bd9Sstevel@tonic-gate	 */
5727c478bd9Sstevel@tonic-gate	DCACHE_FLUSHCOLOR(%o0, %o1, %o2)
5737c478bd9Sstevel@tonic-gate	retl
5747c478bd9Sstevel@tonic-gate	  nop
5757c478bd9Sstevel@tonic-gate	SET_SIZE(vac_flushcolor)
5767c478bd9Sstevel@tonic-gate
5777c478bd9Sstevel@tonic-gate	ENTRY(vac_flushcolor_tl1)
5787c478bd9Sstevel@tonic-gate	/*
5797c478bd9Sstevel@tonic-gate	 * %g1 = vcolor
5807c478bd9Sstevel@tonic-gate	 */
5817c478bd9Sstevel@tonic-gate	DCACHE_FLUSHCOLOR(%g1, %g2, %g3)
5827c478bd9Sstevel@tonic-gate	retry
5837c478bd9Sstevel@tonic-gate	SET_SIZE(vac_flushcolor_tl1)
5847c478bd9Sstevel@tonic-gate
5857c478bd9Sstevel@tonic-gate
5867c478bd9Sstevel@tonic-gate	.global _dispatch_status_busy
5877c478bd9Sstevel@tonic-gate_dispatch_status_busy:
5887c478bd9Sstevel@tonic-gate	.asciz	"ASI_INTR_DISPATCH_STATUS error: busy"
5897c478bd9Sstevel@tonic-gate	.align	4
5907c478bd9Sstevel@tonic-gate
5917c478bd9Sstevel@tonic-gate/*
5927c478bd9Sstevel@tonic-gate * Determine whether or not the IDSR is busy.
5937c478bd9Sstevel@tonic-gate * Entry: no arguments
5947c478bd9Sstevel@tonic-gate * Returns: 1 if busy, 0 otherwise
5957c478bd9Sstevel@tonic-gate */
5967c478bd9Sstevel@tonic-gate	ENTRY(idsr_busy)
5977c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
5987c478bd9Sstevel@tonic-gate	clr	%o0
5997c478bd9Sstevel@tonic-gate	btst	IDSR_BUSY, %g1
6007c478bd9Sstevel@tonic-gate	bz,a,pt	%xcc, 1f
6017c478bd9Sstevel@tonic-gate	mov	1, %o0
6027c478bd9Sstevel@tonic-gate1:
6037c478bd9Sstevel@tonic-gate	retl
6047c478bd9Sstevel@tonic-gate	nop
6057c478bd9Sstevel@tonic-gate	SET_SIZE(idsr_busy)
6067c478bd9Sstevel@tonic-gate
6077c478bd9Sstevel@tonic-gate/*
6087c478bd9Sstevel@tonic-gate * Setup interrupt dispatch data registers
6097c478bd9Sstevel@tonic-gate * Entry:
6107c478bd9Sstevel@tonic-gate *	%o0 - function or inumber to call
6117c478bd9Sstevel@tonic-gate *	%o1, %o2 - arguments (2 uint64_t's)
6127c478bd9Sstevel@tonic-gate */
6137c478bd9Sstevel@tonic-gate	.seg "text"
6147c478bd9Sstevel@tonic-gate
6157c478bd9Sstevel@tonic-gate	ENTRY(init_mondo)
6167c478bd9Sstevel@tonic-gate#ifdef DEBUG
6177c478bd9Sstevel@tonic-gate	!
6187c478bd9Sstevel@tonic-gate	! IDSR should not be busy at the moment
6197c478bd9Sstevel@tonic-gate	!
6207c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
6217c478bd9Sstevel@tonic-gate	btst	IDSR_BUSY, %g1
6227c478bd9Sstevel@tonic-gate	bz,pt	%xcc, 1f
6237c478bd9Sstevel@tonic-gate	nop
6247c478bd9Sstevel@tonic-gate
6257c478bd9Sstevel@tonic-gate	sethi	%hi(_dispatch_status_busy), %o0
6267c478bd9Sstevel@tonic-gate	call	panic
6277c478bd9Sstevel@tonic-gate	or	%o0, %lo(_dispatch_status_busy), %o0
6287c478bd9Sstevel@tonic-gate#endif /* DEBUG */
6297c478bd9Sstevel@tonic-gate
6307c478bd9Sstevel@tonic-gate	ALTENTRY(init_mondo_nocheck)
6317c478bd9Sstevel@tonic-gate	!
6327c478bd9Sstevel@tonic-gate	! interrupt vector dispach data reg 0
6337c478bd9Sstevel@tonic-gate	!
6347c478bd9Sstevel@tonic-gate1:
6357c478bd9Sstevel@tonic-gate	mov	IDDR_0, %g1
6367c478bd9Sstevel@tonic-gate	mov	IDDR_1, %g2
6377c478bd9Sstevel@tonic-gate	mov	IDDR_2, %g3
6387c478bd9Sstevel@tonic-gate	stxa	%o0, [%g1]ASI_INTR_DISPATCH
6397c478bd9Sstevel@tonic-gate
6407c478bd9Sstevel@tonic-gate	!
6417c478bd9Sstevel@tonic-gate	! interrupt vector dispach data reg 1
6427c478bd9Sstevel@tonic-gate	!
6437c478bd9Sstevel@tonic-gate	stxa	%o1, [%g2]ASI_INTR_DISPATCH
6447c478bd9Sstevel@tonic-gate
6457c478bd9Sstevel@tonic-gate	!
6467c478bd9Sstevel@tonic-gate	! interrupt vector dispach data reg 2
6477c478bd9Sstevel@tonic-gate	!
6487c478bd9Sstevel@tonic-gate	stxa	%o2, [%g3]ASI_INTR_DISPATCH
6497c478bd9Sstevel@tonic-gate
6507c478bd9Sstevel@tonic-gate	retl
6517c478bd9Sstevel@tonic-gate	membar	#Sync			! allowed to be in the delay slot
6527c478bd9Sstevel@tonic-gate	SET_SIZE(init_mondo)
6537c478bd9Sstevel@tonic-gate
6547c478bd9Sstevel@tonic-gate/*
6557c478bd9Sstevel@tonic-gate * Ship mondo to upaid
6567c478bd9Sstevel@tonic-gate */
6577c478bd9Sstevel@tonic-gate	ENTRY_NP(shipit)
6587c478bd9Sstevel@tonic-gate	sll	%o0, IDCR_PID_SHIFT, %g1	! IDCR<18:14> = upa id
6597c478bd9Sstevel@tonic-gate	or	%g1, IDCR_OFFSET, %g1		! IDCR<13:0> = 0x70
6607c478bd9Sstevel@tonic-gate	stxa	%g0, [%g1]ASI_INTR_DISPATCH	! interrupt vector dispatch
6617c478bd9Sstevel@tonic-gate#if defined(SF_ERRATA_54)
6627c478bd9Sstevel@tonic-gate	membar	#Sync				! store must occur before load
6637c478bd9Sstevel@tonic-gate	mov	0x20, %g3			! UDBH Control Register Read
6647c478bd9Sstevel@tonic-gate	ldxa	[%g3]ASI_SDB_INTR_R, %g0
6657c478bd9Sstevel@tonic-gate#endif
6667c478bd9Sstevel@tonic-gate	retl
6677c478bd9Sstevel@tonic-gate	membar	#Sync
6687c478bd9Sstevel@tonic-gate	SET_SIZE(shipit)
6697c478bd9Sstevel@tonic-gate
6707c478bd9Sstevel@tonic-gate
6717c478bd9Sstevel@tonic-gate/*
6727c478bd9Sstevel@tonic-gate * flush_instr_mem:
6737c478bd9Sstevel@tonic-gate *	Flush a portion of the I-$ starting at vaddr
6747c478bd9Sstevel@tonic-gate * 	%o0 vaddr
6757c478bd9Sstevel@tonic-gate *	%o1 bytes to be flushed
6767c478bd9Sstevel@tonic-gate */
6777c478bd9Sstevel@tonic-gate
6787c478bd9Sstevel@tonic-gate	ENTRY(flush_instr_mem)
6797c478bd9Sstevel@tonic-gate	membar	#StoreStore				! Ensure the stores
6807c478bd9Sstevel@tonic-gate							! are globally visible
6817c478bd9Sstevel@tonic-gate1:
6827c478bd9Sstevel@tonic-gate	flush	%o0
6837c478bd9Sstevel@tonic-gate	subcc	%o1, ICACHE_FLUSHSZ, %o1		! bytes = bytes-0x20
6847c478bd9Sstevel@tonic-gate	bgu,pt	%ncc, 1b
6857c478bd9Sstevel@tonic-gate	add	%o0, ICACHE_FLUSHSZ, %o0		! vaddr = vaddr+0x20
6867c478bd9Sstevel@tonic-gate
6877c478bd9Sstevel@tonic-gate	retl
6887c478bd9Sstevel@tonic-gate	nop
6897c478bd9Sstevel@tonic-gate	SET_SIZE(flush_instr_mem)
6907c478bd9Sstevel@tonic-gate
6917c478bd9Sstevel@tonic-gate/*
6927c478bd9Sstevel@tonic-gate * flush_ecache:
6937c478bd9Sstevel@tonic-gate * Flush the entire e$ using displacement flush by reading through a
6947c478bd9Sstevel@tonic-gate * physically contiguous area. We use mmu bypass asi (ASI_MEM) while
6957c478bd9Sstevel@tonic-gate * reading this physical address range so that data doesn't go to d$.
6967c478bd9Sstevel@tonic-gate * incoming arguments:
6977c478bd9Sstevel@tonic-gate *	%o0 - 64 bit physical address
6987c478bd9Sstevel@tonic-gate *	%o1 - size of address range to read
6997c478bd9Sstevel@tonic-gate *	%o2 - ecache linesize
7007c478bd9Sstevel@tonic-gate */
7017c478bd9Sstevel@tonic-gate	ENTRY(flush_ecache)
7027c478bd9Sstevel@tonic-gate#ifndef HUMMINGBIRD
7037c478bd9Sstevel@tonic-gate	b	2f
7047c478bd9Sstevel@tonic-gate	  nop
7057c478bd9Sstevel@tonic-gate1:
7067c478bd9Sstevel@tonic-gate	ldxa	[%o0 + %o1]ASI_MEM, %g0	! start reading from physaddr + size
7077c478bd9Sstevel@tonic-gate2:
7087c478bd9Sstevel@tonic-gate	subcc	%o1, %o2, %o1
7097c478bd9Sstevel@tonic-gate	bcc,a,pt %ncc, 1b
7107c478bd9Sstevel@tonic-gate	  nop
7117c478bd9Sstevel@tonic-gate
7127c478bd9Sstevel@tonic-gate#else /* HUMMINGBIRD */
7137c478bd9Sstevel@tonic-gate	/*
7147c478bd9Sstevel@tonic-gate	 * UltraSPARC-IIe processor supports both 4-way set associative
7157c478bd9Sstevel@tonic-gate	 * and direct map E$. For performance reasons, we flush E$ by
7167c478bd9Sstevel@tonic-gate	 * placing it in direct map mode for data load/store and restore
7177c478bd9Sstevel@tonic-gate	 * the state after we are done flushing it. It takes 2 iterations
7187c478bd9Sstevel@tonic-gate	 * to guarantee that the entire ecache has been flushed.
7197c478bd9Sstevel@tonic-gate	 *
7207c478bd9Sstevel@tonic-gate	 * Keep the interrupts disabled while flushing E$ in this manner.
7217c478bd9Sstevel@tonic-gate	 */
7227c478bd9Sstevel@tonic-gate	rdpr	%pstate, %g4		! current pstate (restored later)
7237c478bd9Sstevel@tonic-gate	andn	%g4, PSTATE_IE, %g5
7247c478bd9Sstevel@tonic-gate	wrpr	%g0, %g5, %pstate	! disable interrupts
7257c478bd9Sstevel@tonic-gate
7267c478bd9Sstevel@tonic-gate	! Place E$ in direct map mode for data access
7277c478bd9Sstevel@tonic-gate	or	%g0, 1, %g5
7287c478bd9Sstevel@tonic-gate	sllx	%g5, HB_UPA_DMAP_DATA_BIT, %g5
7297c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later)
7307c478bd9Sstevel@tonic-gate	or	%g1, %g5, %g5
7317c478bd9Sstevel@tonic-gate	membar	#Sync
7327c478bd9Sstevel@tonic-gate	stxa	%g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access
7337c478bd9Sstevel@tonic-gate	membar	#Sync
7347c478bd9Sstevel@tonic-gate
7357c478bd9Sstevel@tonic-gate	! flush entire ecache HB_ECACHE_FLUSH_CNT times
7367c478bd9Sstevel@tonic-gate	mov	HB_ECACHE_FLUSH_CNT-1, %g5
7377c478bd9Sstevel@tonic-gate2:
7387c478bd9Sstevel@tonic-gate	sub	%o1, %o2, %g3		! start from last entry
7397c478bd9Sstevel@tonic-gate1:
7407c478bd9Sstevel@tonic-gate	ldxa	[%o0 + %g3]ASI_MEM, %g0	! start reading from physaddr + size
7417c478bd9Sstevel@tonic-gate	subcc	%g3, %o2, %g3
7427c478bd9Sstevel@tonic-gate	bgeu,a,pt %ncc, 1b
7437c478bd9Sstevel@tonic-gate	  nop
7447c478bd9Sstevel@tonic-gate	brgz,a,pt %g5, 2b
7457c478bd9Sstevel@tonic-gate	  dec	%g5
7467c478bd9Sstevel@tonic-gate
7477c478bd9Sstevel@tonic-gate	membar	#Sync
7487c478bd9Sstevel@tonic-gate	stxa	%g1, [%g0]ASI_UPA_CONFIG ! restore UPA config reg
7497c478bd9Sstevel@tonic-gate	membar	#Sync
7507c478bd9Sstevel@tonic-gate	wrpr	%g0, %g4, %pstate	! restore earlier pstate
7517c478bd9Sstevel@tonic-gate#endif /* HUMMINGBIRD */
7527c478bd9Sstevel@tonic-gate
7537c478bd9Sstevel@tonic-gate	retl
7547c478bd9Sstevel@tonic-gate	nop
7557c478bd9Sstevel@tonic-gate	SET_SIZE(flush_ecache)
7567c478bd9Sstevel@tonic-gate
7577c478bd9Sstevel@tonic-gate/*
7587c478bd9Sstevel@tonic-gate * void kdi_flush_idcache(int dcache_size, int dcache_linesize,
7597c478bd9Sstevel@tonic-gate *			int icache_size, int icache_linesize)
7607c478bd9Sstevel@tonic-gate */
7617c478bd9Sstevel@tonic-gate	ENTRY(kdi_flush_idcache)
7627c478bd9Sstevel@tonic-gate	DCACHE_FLUSHALL(%o0, %o1, %g1)
7637c478bd9Sstevel@tonic-gate	ICACHE_FLUSHALL(%o2, %o3, %g1)
7647c478bd9Sstevel@tonic-gate	membar	#Sync
7657c478bd9Sstevel@tonic-gate	retl
7667c478bd9Sstevel@tonic-gate	nop
7677c478bd9Sstevel@tonic-gate	SET_SIZE(kdi_flush_idcache)
7687c478bd9Sstevel@tonic-gate
7697c478bd9Sstevel@tonic-gate
7707c478bd9Sstevel@tonic-gate/*
7717c478bd9Sstevel@tonic-gate * void get_ecache_dtag(uint32_t ecache_idx, uint64_t *data, uint64_t *tag,
7727c478bd9Sstevel@tonic-gate * 			uint64_t *oafsr, uint64_t *acc_afsr)
7737c478bd9Sstevel@tonic-gate *
7747c478bd9Sstevel@tonic-gate * Get ecache data and tag.  The ecache_idx argument is assumed to be aligned
7757c478bd9Sstevel@tonic-gate * on a 64-byte boundary.  The corresponding AFSR value is also read for each
7767c478bd9Sstevel@tonic-gate * 8 byte ecache data obtained. The ecache data is assumed to be a pointer
7777c478bd9Sstevel@tonic-gate * to an array of 16 uint64_t's (e$data & afsr value).  The action to read the
7787c478bd9Sstevel@tonic-gate * data and tag should be atomic to make sense.  We will be executing at PIL15
7797c478bd9Sstevel@tonic-gate * and will disable IE, so nothing can occur between the two reads.  We also
7807c478bd9Sstevel@tonic-gate * assume that the execution of this code does not interfere with what we are
7817c478bd9Sstevel@tonic-gate * reading - not really possible, but we'll live with it for now.
7827c478bd9Sstevel@tonic-gate * We also pass the old AFSR value before clearing it, and caller will take
7837c478bd9Sstevel@tonic-gate * appropriate actions if the important bits are non-zero.
7847c478bd9Sstevel@tonic-gate *
7857c478bd9Sstevel@tonic-gate * If the caller wishes to track the AFSR in cases where the CP bit is
7867c478bd9Sstevel@tonic-gate * set, an address should be passed in for acc_afsr.  Otherwise, this
7877c478bd9Sstevel@tonic-gate * argument may be null.
7887c478bd9Sstevel@tonic-gate *
7897c478bd9Sstevel@tonic-gate * Register Usage:
7907c478bd9Sstevel@tonic-gate * i0: In: 32-bit e$ index
7917c478bd9Sstevel@tonic-gate * i1: In: addr of e$ data
7927c478bd9Sstevel@tonic-gate * i2: In: addr of e$ tag
7937c478bd9Sstevel@tonic-gate * i3: In: addr of old afsr
7947c478bd9Sstevel@tonic-gate * i4: In: addr of accumulated afsr - may be null
7957c478bd9Sstevel@tonic-gate */
7967c478bd9Sstevel@tonic-gate	ENTRY(get_ecache_dtag)
7977c478bd9Sstevel@tonic-gate	save	%sp, -SA(MINFRAME), %sp
7987c478bd9Sstevel@tonic-gate	or	%g0, 1, %l4
7997c478bd9Sstevel@tonic-gate	sllx	%l4, 39, %l4	! set bit 39 for e$ data access
8007c478bd9Sstevel@tonic-gate	or	%i0, %l4, %g6	! %g6 = e$ addr for data read
8017c478bd9Sstevel@tonic-gate	sllx	%l4, 1, %l4	! set bit 40 for e$ tag access
8027c478bd9Sstevel@tonic-gate	or	%i0, %l4, %l4	! %l4 = e$ addr for tag read
8037c478bd9Sstevel@tonic-gate
8047c478bd9Sstevel@tonic-gate	rdpr    %pstate, %i5
8057c478bd9Sstevel@tonic-gate	andn    %i5, PSTATE_IE | PSTATE_AM, %i0
8067c478bd9Sstevel@tonic-gate	wrpr    %i0, %g0, %pstate       ! clear IE, AM bits
8077c478bd9Sstevel@tonic-gate
8087c478bd9Sstevel@tonic-gate	ldxa    [%g0]ASI_ESTATE_ERR, %g1
8097c478bd9Sstevel@tonic-gate	stxa    %g0, [%g0]ASI_ESTATE_ERR        ! disable errors
8107c478bd9Sstevel@tonic-gate	membar  #Sync
8117c478bd9Sstevel@tonic-gate
8127c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_AFSR, %i0      ! grab the old-afsr before tag read
8137c478bd9Sstevel@tonic-gate	stx     %i0, [%i3]		! write back the old-afsr
8147c478bd9Sstevel@tonic-gate
8157c478bd9Sstevel@tonic-gate	ldxa    [%l4]ASI_EC_R, %g0      ! read tag into E$ tag reg
8167c478bd9Sstevel@tonic-gate	ldxa    [%g0]ASI_EC_DIAG, %i0   ! read tag from E$ tag reg
8177c478bd9Sstevel@tonic-gate	stx     %i0, [%i2]              ! write back tag result
8187c478bd9Sstevel@tonic-gate
8197c478bd9Sstevel@tonic-gate	clr	%i2			! loop count
8207c478bd9Sstevel@tonic-gate
8217c478bd9Sstevel@tonic-gate	brz	%i4, 1f			! acc_afsr == NULL?
8227c478bd9Sstevel@tonic-gate	  ldxa	[%g0]ASI_AFSR, %i0      ! grab the old-afsr before clearing
8237c478bd9Sstevel@tonic-gate	srlx	%i0, P_AFSR_CP_SHIFT, %l0
8247c478bd9Sstevel@tonic-gate	btst	1, %l0
8257c478bd9Sstevel@tonic-gate	bz	1f
8267c478bd9Sstevel@tonic-gate	  nop
8277c478bd9Sstevel@tonic-gate	ldx	[%i4], %g4
8287c478bd9Sstevel@tonic-gate	or	%g4, %i0, %g4		! aggregate AFSR in cpu private
8297c478bd9Sstevel@tonic-gate	stx	%g4, [%i4]
8307c478bd9Sstevel@tonic-gate1:
8317c478bd9Sstevel@tonic-gate	stxa    %i0, [%g0]ASI_AFSR	! clear AFSR
8327c478bd9Sstevel@tonic-gate	membar  #Sync
8337c478bd9Sstevel@tonic-gate	ldxa    [%g6]ASI_EC_R, %i0      ! read the 8byte E$data
8347c478bd9Sstevel@tonic-gate	stx     %i0, [%i1]              ! save the E$data
8357c478bd9Sstevel@tonic-gate	add     %g6, 8, %g6
8367c478bd9Sstevel@tonic-gate	add     %i1, 8, %i1
8377c478bd9Sstevel@tonic-gate	ldxa    [%g0]ASI_AFSR, %i0      ! read AFSR for this 16byte read
8387c478bd9Sstevel@tonic-gate	srlx	%i0, P_AFSR_CP_SHIFT, %l0
8397c478bd9Sstevel@tonic-gate	btst	1, %l0
8407c478bd9Sstevel@tonic-gate	bz	2f
8417c478bd9Sstevel@tonic-gate	  stx     %i0, [%i1]		! save the AFSR
8427c478bd9Sstevel@tonic-gate
8437c478bd9Sstevel@tonic-gate	brz	%i4, 2f			! acc_afsr == NULL?
8447c478bd9Sstevel@tonic-gate	  nop
8457c478bd9Sstevel@tonic-gate	ldx	[%i4], %g4
8467c478bd9Sstevel@tonic-gate	or	%g4, %i0, %g4		! aggregate AFSR in cpu private
8477c478bd9Sstevel@tonic-gate	stx	%g4, [%i4]
8487c478bd9Sstevel@tonic-gate2:
8497c478bd9Sstevel@tonic-gate	add     %i2, 8, %i2
8507c478bd9Sstevel@tonic-gate	cmp     %i2, 64
8517c478bd9Sstevel@tonic-gate	bl,a    1b
8527c478bd9Sstevel@tonic-gate	  add     %i1, 8, %i1
8537c478bd9Sstevel@tonic-gate	stxa    %i0, [%g0]ASI_AFSR              ! clear AFSR
8547c478bd9Sstevel@tonic-gate	membar  #Sync
8557c478bd9Sstevel@tonic-gate	stxa    %g1, [%g0]ASI_ESTATE_ERR        ! restore error enable
8567c478bd9Sstevel@tonic-gate	membar  #Sync
8577c478bd9Sstevel@tonic-gate	wrpr    %g0, %i5, %pstate
8587c478bd9Sstevel@tonic-gate	ret
8597c478bd9Sstevel@tonic-gate	  restore
8607c478bd9Sstevel@tonic-gate	SET_SIZE(get_ecache_dtag)
8617c478bd9Sstevel@tonic-gate
8627c478bd9Sstevel@tonic-gate/*
8637c478bd9Sstevel@tonic-gate * The ce_err function handles trap type 0x63 (corrected_ECC_error) at tl=0.
8647c478bd9Sstevel@tonic-gate * Steps: 1. GET AFSR  2. Get AFAR <40:4> 3. Get datapath error status
8657c478bd9Sstevel@tonic-gate *	  4. Clear datapath error bit(s) 5. Clear AFSR error bit
8667c478bd9Sstevel@tonic-gate *	  6. package data in %g2 and %g3 7. call cpu_ce_error vis sys_trap
8677c478bd9Sstevel@tonic-gate * %g2: [ 52:43 UDB lower | 42:33 UDB upper | 32:0 afsr ] - arg #3/arg #1
8687c478bd9Sstevel@tonic-gate * %g3: [ 40:4 afar ] - sys_trap->have_win: arg #4/arg #2
8697c478bd9Sstevel@tonic-gate */
8707c478bd9Sstevel@tonic-gate	ENTRY_NP(ce_err)
8717c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_AFSR, %g3	! save afsr in g3
8727c478bd9Sstevel@tonic-gate
8737c478bd9Sstevel@tonic-gate	!
8747c478bd9Sstevel@tonic-gate	! Check for a UE... From Kevin.Normoyle:
8757c478bd9Sstevel@tonic-gate	! We try to switch to the trap for the UE, but since that's
8767c478bd9Sstevel@tonic-gate	! a hardware pipeline, we might get to the CE trap before we
8777c478bd9Sstevel@tonic-gate	! can switch. The UDB and AFSR registers will have both the
8787c478bd9Sstevel@tonic-gate	! UE and CE bits set but the UDB syndrome and the AFAR will be
8797c478bd9Sstevel@tonic-gate	! for the UE.
8807c478bd9Sstevel@tonic-gate	!
8817c478bd9Sstevel@tonic-gate	or	%g0, 1, %g1		! put 1 in g1
8827c478bd9Sstevel@tonic-gate	sllx	%g1, 21, %g1		! shift left to <21> afsr UE
8837c478bd9Sstevel@tonic-gate	andcc	%g1, %g3, %g0		! check for UE in afsr
8847c478bd9Sstevel@tonic-gate	bnz	async_err		! handle the UE, not the CE
8857c478bd9Sstevel@tonic-gate	  or	%g0, 0x63, %g5		! pass along the CE ttype
8867c478bd9Sstevel@tonic-gate	!
8877c478bd9Sstevel@tonic-gate	! Disable further CE traps to avoid recursion (stack overflow)
8887c478bd9Sstevel@tonic-gate	! and staying above XCALL_PIL for extended periods.
8897c478bd9Sstevel@tonic-gate	!
8907c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_ESTATE_ERR, %g2
8917c478bd9Sstevel@tonic-gate	andn	%g2, 0x1, %g2		! clear bit 0 - CEEN
8927c478bd9Sstevel@tonic-gate	stxa	%g2, [%g0]ASI_ESTATE_ERR
8937c478bd9Sstevel@tonic-gate	membar	#Sync			! required
8947c478bd9Sstevel@tonic-gate	!
8957c478bd9Sstevel@tonic-gate	! handle the CE
8967c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_AFAR, %g2	! save afar in g2
8977c478bd9Sstevel@tonic-gate
8987c478bd9Sstevel@tonic-gate	set	P_DER_H, %g4		! put P_DER_H in g4
8997c478bd9Sstevel@tonic-gate	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into g5
9007c478bd9Sstevel@tonic-gate	or	%g0, 1, %g6		! put 1 in g6
9017c478bd9Sstevel@tonic-gate	sllx	%g6, 8, %g6		! shift g6 to <8> sdb CE
9027c478bd9Sstevel@tonic-gate	andcc	%g5, %g6, %g1		! check for CE in upper half
9037c478bd9Sstevel@tonic-gate	sllx	%g5, 33, %g5		! shift upper bits to <42:33>
9047c478bd9Sstevel@tonic-gate	or	%g3, %g5, %g3		! or with afsr bits
9057c478bd9Sstevel@tonic-gate	bz,a	1f			! no error, goto 1f
9067c478bd9Sstevel@tonic-gate	  nop
9077c478bd9Sstevel@tonic-gate	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg error bit
9087c478bd9Sstevel@tonic-gate	membar	#Sync			! membar sync required
9097c478bd9Sstevel@tonic-gate1:
9107c478bd9Sstevel@tonic-gate	set	P_DER_L, %g4		! put P_DER_L in g4
9117c478bd9Sstevel@tonic-gate	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g6
9127c478bd9Sstevel@tonic-gate	andcc	%g5, %g6, %g1		! check for CE in lower half
9137c478bd9Sstevel@tonic-gate	sllx	%g5, 43, %g5		! shift upper bits to <52:43>
9147c478bd9Sstevel@tonic-gate	or	%g3, %g5, %g3		! or with afsr bits
9157c478bd9Sstevel@tonic-gate	bz,a	2f			! no error, goto 2f
9167c478bd9Sstevel@tonic-gate	  nop
9177c478bd9Sstevel@tonic-gate	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg error bit
9187c478bd9Sstevel@tonic-gate	membar	#Sync			! membar sync required
9197c478bd9Sstevel@tonic-gate2:
9207c478bd9Sstevel@tonic-gate	or	%g0, 1, %g4		! put 1 in g4
9217c478bd9Sstevel@tonic-gate	sllx	%g4, 20, %g4		! shift left to <20> afsr CE
9227c478bd9Sstevel@tonic-gate	stxa	%g4, [%g0]ASI_AFSR	! use g4 to clear afsr CE error
9237c478bd9Sstevel@tonic-gate	membar	#Sync			! membar sync required
9247c478bd9Sstevel@tonic-gate
9257c478bd9Sstevel@tonic-gate	set	cpu_ce_error, %g1	! put *cpu_ce_error() in g1
9267c478bd9Sstevel@tonic-gate	rdpr	%pil, %g6		! read pil into %g6
9277c478bd9Sstevel@tonic-gate	subcc	%g6, PIL_15, %g0
9287c478bd9Sstevel@tonic-gate	  movneg	%icc, PIL_14, %g4 ! run at pil 14 unless already at 15
9297c478bd9Sstevel@tonic-gate	sethi	%hi(sys_trap), %g5
9307c478bd9Sstevel@tonic-gate	jmp	%g5 + %lo(sys_trap)	! goto sys_trap
9317c478bd9Sstevel@tonic-gate	  movge	%icc, PIL_15, %g4	! already at pil 15
9327c478bd9Sstevel@tonic-gate	SET_SIZE(ce_err)
9337c478bd9Sstevel@tonic-gate
9347c478bd9Sstevel@tonic-gate	ENTRY_NP(ce_err_tl1)
9357c478bd9Sstevel@tonic-gate#ifndef	TRAPTRACE
9367c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_AFSR, %g7
9377c478bd9Sstevel@tonic-gate	stxa	%g7, [%g0]ASI_AFSR
9387c478bd9Sstevel@tonic-gate	membar	#Sync
9397c478bd9Sstevel@tonic-gate	retry
9407c478bd9Sstevel@tonic-gate#else
9417c478bd9Sstevel@tonic-gate	set	ce_trap_tl1, %g1
9427c478bd9Sstevel@tonic-gate	sethi	%hi(dis_err_panic1), %g4
9437c478bd9Sstevel@tonic-gate	jmp	%g4 + %lo(dis_err_panic1)
9447c478bd9Sstevel@tonic-gate	nop
9457c478bd9Sstevel@tonic-gate#endif
9467c478bd9Sstevel@tonic-gate	SET_SIZE(ce_err_tl1)
9477c478bd9Sstevel@tonic-gate
9487c478bd9Sstevel@tonic-gate#ifdef	TRAPTRACE
9497c478bd9Sstevel@tonic-gate.celevel1msg:
9507c478bd9Sstevel@tonic-gate	.asciz	"Softerror with trap tracing at tl1: AFAR 0x%08x.%08x AFSR 0x%08x.%08x";
9517c478bd9Sstevel@tonic-gate
9527c478bd9Sstevel@tonic-gate	ENTRY_NP(ce_trap_tl1)
9537c478bd9Sstevel@tonic-gate	! upper 32 bits of AFSR already in o3
9547c478bd9Sstevel@tonic-gate	mov	%o4, %o0		! save AFAR upper 32 bits
9557c478bd9Sstevel@tonic-gate	mov	%o2, %o4		! lower 32 bits of AFSR
9567c478bd9Sstevel@tonic-gate	mov	%o1, %o2		! lower 32 bits of AFAR
9577c478bd9Sstevel@tonic-gate	mov	%o0, %o1		! upper 32 bits of AFAR
9587c478bd9Sstevel@tonic-gate	set	.celevel1msg, %o0
9597c478bd9Sstevel@tonic-gate	call	panic
9607c478bd9Sstevel@tonic-gate	nop
9617c478bd9Sstevel@tonic-gate	SET_SIZE(ce_trap_tl1)
9627c478bd9Sstevel@tonic-gate#endif
9637c478bd9Sstevel@tonic-gate
964*9b0bb795SJohn Levon/*
965*9b0bb795SJohn Levon * The async_err function handles trap types 0x0A (instruction_access_error)
966*9b0bb795SJohn Levon * and 0x32 (data_access_error) at TL = 0 and TL > 0.  When we branch here,
967*9b0bb795SJohn Levon * %g5 will have the trap type (with 0x200 set if we're at TL > 0).
968*9b0bb795SJohn Levon *
969*9b0bb795SJohn Levon * Steps: 1. Get AFSR 2. Get AFAR <40:4> 3. If not UE error skip UDP registers.
970*9b0bb795SJohn Levon *	  4. Else get and clear datapath error bit(s) 4. Clear AFSR error bits
971*9b0bb795SJohn Levon *	  6. package data in %g2 and %g3 7. disable all cpu errors, because
972*9b0bb795SJohn Levon *	  trap is likely to be fatal 8. call cpu_async_error vis sys_trap
973*9b0bb795SJohn Levon *
974*9b0bb795SJohn Levon * %g3: [ 63:53 tt | 52:43 UDB_L | 42:33 UDB_U | 32:0 afsr ] - arg #3/arg #1
975*9b0bb795SJohn Levon * %g2: [ 40:4 afar ] - sys_trap->have_win: arg #4/arg #2
976*9b0bb795SJohn Levon *
977*9b0bb795SJohn Levon * async_err is the assembly glue code to get us from the actual trap
978*9b0bb795SJohn Levon * into the CPU module's C error handler.  Note that we also branch
979*9b0bb795SJohn Levon * here from ce_err() above.
980*9b0bb795SJohn Levon */
9817c478bd9Sstevel@tonic-gate	ENTRY_NP(async_err)
9827c478bd9Sstevel@tonic-gate	stxa	%g0, [%g0]ASI_ESTATE_ERR ! disable ecc and other cpu errors
9837c478bd9Sstevel@tonic-gate	membar	#Sync			! membar sync required
9847c478bd9Sstevel@tonic-gate
9857c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_AFSR, %g3	! save afsr in g3
9867c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_AFAR, %g2	! save afar in g2
9877c478bd9Sstevel@tonic-gate
9887c478bd9Sstevel@tonic-gate	sllx	%g5, 53, %g5		! move ttype to <63:53>
9897c478bd9Sstevel@tonic-gate	or	%g3, %g5, %g3		! or to afsr in g3
9907c478bd9Sstevel@tonic-gate
9917c478bd9Sstevel@tonic-gate	or	%g0, 1, %g1		! put 1 in g1
9927c478bd9Sstevel@tonic-gate	sllx	%g1, 21, %g1		! shift left to <21> afsr UE
9937c478bd9Sstevel@tonic-gate	andcc	%g1, %g3, %g0		! check for UE in afsr
9947c478bd9Sstevel@tonic-gate	bz,a,pn %icc, 2f		! if !UE skip sdb read/clear
9957c478bd9Sstevel@tonic-gate	  nop
9967c478bd9Sstevel@tonic-gate
9977c478bd9Sstevel@tonic-gate	set	P_DER_H, %g4		! put P_DER_H in g4
9987c478bd9Sstevel@tonic-gate	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into 56
9997c478bd9Sstevel@tonic-gate	or	%g0, 1, %g6		! put 1 in g6
10007c478bd9Sstevel@tonic-gate	sllx	%g6, 9, %g6		! shift g6 to <9> sdb UE
10017c478bd9Sstevel@tonic-gate	andcc	%g5, %g6, %g1		! check for UE in upper half
10027c478bd9Sstevel@tonic-gate	sllx	%g5, 33, %g5		! shift upper bits to <42:33>
10037c478bd9Sstevel@tonic-gate	or	%g3, %g5, %g3		! or with afsr bits
10047c478bd9Sstevel@tonic-gate	bz,a	1f			! no error, goto 1f
10057c478bd9Sstevel@tonic-gate	  nop
10067c478bd9Sstevel@tonic-gate	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg UE error bit
10077c478bd9Sstevel@tonic-gate	membar	#Sync			! membar sync required
10087c478bd9Sstevel@tonic-gate1:
10097c478bd9Sstevel@tonic-gate	set	P_DER_L, %g4		! put P_DER_L in g4
10107c478bd9Sstevel@tonic-gate	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g5
10117c478bd9Sstevel@tonic-gate	andcc	%g5, %g6, %g1		! check for UE in lower half
10127c478bd9Sstevel@tonic-gate	sllx	%g5, 43, %g5		! shift upper bits to <52:43>
10137c478bd9Sstevel@tonic-gate	or	%g3, %g5, %g3		! or with afsr bits
10147c478bd9Sstevel@tonic-gate	bz,a	2f			! no error, goto 2f
10157c478bd9Sstevel@tonic-gate	  nop
10167c478bd9Sstevel@tonic-gate	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg UE error bit
10177c478bd9Sstevel@tonic-gate	membar	#Sync			! membar sync required
10187c478bd9Sstevel@tonic-gate2:
10197c478bd9Sstevel@tonic-gate	stxa	%g3, [%g0]ASI_AFSR	! clear all the sticky bits
10207c478bd9Sstevel@tonic-gate	membar	#Sync			! membar sync required
10217c478bd9Sstevel@tonic-gate
10222dd3029aSjimand	RESET_USER_RTT_REGS(%g4, %g5, async_err_resetskip)
10232dd3029aSjimandasync_err_resetskip:
10248f230a59Sbs
10257c478bd9Sstevel@tonic-gate	set	cpu_async_error, %g1	! put cpu_async_error in g1
10267c478bd9Sstevel@tonic-gate	sethi	%hi(sys_trap), %g5
10277c478bd9Sstevel@tonic-gate	jmp	%g5 + %lo(sys_trap)	! goto sys_trap
10287c478bd9Sstevel@tonic-gate	  or	%g0, PIL_15, %g4	! run at pil 15
10297c478bd9Sstevel@tonic-gate	SET_SIZE(async_err)
10307c478bd9Sstevel@tonic-gate
10317c478bd9Sstevel@tonic-gate	ENTRY_NP(dis_err_panic1)
10327c478bd9Sstevel@tonic-gate	stxa	%g0, [%g0]ASI_ESTATE_ERR ! disable all error traps
10337c478bd9Sstevel@tonic-gate	membar	#Sync
10347c478bd9Sstevel@tonic-gate	! save destination routine is in g1
10357c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_AFAR, %g2	! read afar
10367c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_AFSR, %g3	! read afsr
10377c478bd9Sstevel@tonic-gate	set	P_DER_H, %g4		! put P_DER_H in g4
10387c478bd9Sstevel@tonic-gate	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into g5
10397c478bd9Sstevel@tonic-gate	sllx	%g5, 33, %g5		! shift upper bits to <42:33>
10407c478bd9Sstevel@tonic-gate	or	%g3, %g5, %g3		! or with afsr bits
10417c478bd9Sstevel@tonic-gate	set	P_DER_L, %g4		! put P_DER_L in g4
10427c478bd9Sstevel@tonic-gate	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g5
10437c478bd9Sstevel@tonic-gate	sllx	%g5, 43, %g5		! shift upper bits to <52:43>
10447c478bd9Sstevel@tonic-gate	or	%g3, %g5, %g3		! or with afsr bits
10458f230a59Sbs
10462dd3029aSjimand	RESET_USER_RTT_REGS(%g4, %g5, dis_err_panic1_resetskip)
10472dd3029aSjimanddis_err_panic1_resetskip:
10488f230a59Sbs
10497c478bd9Sstevel@tonic-gate	sethi	%hi(sys_trap), %g5
10507c478bd9Sstevel@tonic-gate	jmp	%g5 + %lo(sys_trap)	! goto sys_trap
10517c478bd9Sstevel@tonic-gate	  sub	%g0, 1, %g4
10527c478bd9Sstevel@tonic-gate	SET_SIZE(dis_err_panic1)
10537c478bd9Sstevel@tonic-gate
1054*9b0bb795SJohn Levon/*
1055*9b0bb795SJohn Levon * The clr_datapath function clears any error bits set in the UDB regs.
1056*9b0bb795SJohn Levon */
10577c478bd9Sstevel@tonic-gate	ENTRY(clr_datapath)
10587c478bd9Sstevel@tonic-gate	set	P_DER_H, %o4			! put P_DER_H in o4
10597c478bd9Sstevel@tonic-gate	ldxa	[%o4]ASI_SDB_INTR_R, %o5	! read sdb upper half into o3
10607c478bd9Sstevel@tonic-gate	or	%g0, 0x3, %o2			! put 0x3 in o2
10617c478bd9Sstevel@tonic-gate	sllx	%o2, 8, %o2			! shift o2 to <9:8> sdb
10627c478bd9Sstevel@tonic-gate	andcc	%o5, %o2, %o1			! check for UE,CE in upper half
10637c478bd9Sstevel@tonic-gate	bz,a	1f				! no error, goto 1f
10647c478bd9Sstevel@tonic-gate	  nop
10657c478bd9Sstevel@tonic-gate	stxa	%o1, [%o4]ASI_SDB_INTR_W	! clear sdb reg UE,CE error bits
10667c478bd9Sstevel@tonic-gate	membar	#Sync				! membar sync required
10677c478bd9Sstevel@tonic-gate1:
10687c478bd9Sstevel@tonic-gate	set	P_DER_L, %o4			! put P_DER_L in o4
10697c478bd9Sstevel@tonic-gate	ldxa	[%o4]ASI_SDB_INTR_R, %o5	! read sdb lower half into o5
10707c478bd9Sstevel@tonic-gate	andcc	%o5, %o2, %o1			! check for UE,CE in lower half
10717c478bd9Sstevel@tonic-gate	bz,a	2f				! no error, goto 2f
10727c478bd9Sstevel@tonic-gate	  nop
10737c478bd9Sstevel@tonic-gate	stxa	%o1, [%o4]ASI_SDB_INTR_W	! clear sdb reg UE,CE error bits
10747c478bd9Sstevel@tonic-gate	membar	#Sync
10757c478bd9Sstevel@tonic-gate2:
10767c478bd9Sstevel@tonic-gate	retl
10777c478bd9Sstevel@tonic-gate	  nop
10787c478bd9Sstevel@tonic-gate	SET_SIZE(clr_datapath)
10797c478bd9Sstevel@tonic-gate
1080*9b0bb795SJohn Levon/*
1081*9b0bb795SJohn Levon * The get_udb_errors() function gets the current value of the
1082*9b0bb795SJohn Levon * Datapath Error Registers.
1083*9b0bb795SJohn Levon */
10847c478bd9Sstevel@tonic-gate	ENTRY(get_udb_errors)
10857c478bd9Sstevel@tonic-gate	set	P_DER_H, %o3
10867c478bd9Sstevel@tonic-gate	ldxa	[%o3]ASI_SDB_INTR_R, %o2
10877c478bd9Sstevel@tonic-gate	stx	%o2, [%o0]
10887c478bd9Sstevel@tonic-gate	set	P_DER_L, %o3
10897c478bd9Sstevel@tonic-gate	ldxa	[%o3]ASI_SDB_INTR_R, %o2
10907c478bd9Sstevel@tonic-gate	retl
10917c478bd9Sstevel@tonic-gate	  stx	%o2, [%o1]
10927c478bd9Sstevel@tonic-gate	SET_SIZE(get_udb_errors)
10937c478bd9Sstevel@tonic-gate
10947c478bd9Sstevel@tonic-gate/*
10957c478bd9Sstevel@tonic-gate * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the
10967c478bd9Sstevel@tonic-gate * tte, the virtual address, and the ctxnum of the specified tlb entry.  They
10977c478bd9Sstevel@tonic-gate * should only be used in places where you have no choice but to look at the
10987c478bd9Sstevel@tonic-gate * tlb itself.
10997c478bd9Sstevel@tonic-gate *
11007c478bd9Sstevel@tonic-gate * Note: These two routines are required by the Estar "cpr" loadable module.
11017c478bd9Sstevel@tonic-gate */
11027c478bd9Sstevel@tonic-gate/*
11037c478bd9Sstevel@tonic-gate * NB - In Spitfire cpus, when reading a tte from the hardware, we
11047c478bd9Sstevel@tonic-gate * need to clear [42-41] because the general definitions in pte.h
11057c478bd9Sstevel@tonic-gate * define the PA to be [42-13] whereas Spitfire really uses [40-13].
11067c478bd9Sstevel@tonic-gate * When cloning these routines for other cpus the "andn" below is not
11077c478bd9Sstevel@tonic-gate * necessary.
11087c478bd9Sstevel@tonic-gate */
11097c478bd9Sstevel@tonic-gate	ENTRY_NP(itlb_rd_entry)
11107c478bd9Sstevel@tonic-gate	sllx	%o0, 3, %o0
11117c478bd9Sstevel@tonic-gate#if defined(SF_ERRATA_32)
11127c478bd9Sstevel@tonic-gate	sethi	%hi(FLUSH_ADDR), %g2
11137c478bd9Sstevel@tonic-gate	set	MMU_PCONTEXT, %g1
11147c478bd9Sstevel@tonic-gate	stxa	%g0, [%g1]ASI_DMMU			! KCONTEXT
11157c478bd9Sstevel@tonic-gate	flush	%g2
11167c478bd9Sstevel@tonic-gate#endif
11177c478bd9Sstevel@tonic-gate	ldxa	[%o0]ASI_ITLB_ACCESS, %g1
11187c478bd9Sstevel@tonic-gate	set	TTE_SPITFIRE_PFNHI_CLEAR, %g2		! spitfire only
11197c478bd9Sstevel@tonic-gate	sllx	%g2, TTE_SPITFIRE_PFNHI_SHIFT, %g2	! see comment above
11207c478bd9Sstevel@tonic-gate	andn	%g1, %g2, %g1				! for details
11217c478bd9Sstevel@tonic-gate	stx	%g1, [%o1]
11227c478bd9Sstevel@tonic-gate	ldxa	[%o0]ASI_ITLB_TAGREAD, %g2
11237c478bd9Sstevel@tonic-gate	set	TAGREAD_CTX_MASK, %o4
11247c478bd9Sstevel@tonic-gate	andn	%g2, %o4, %o5
11257c478bd9Sstevel@tonic-gate	retl
11267c478bd9Sstevel@tonic-gate	  stx	%o5, [%o2]
11277c478bd9Sstevel@tonic-gate	SET_SIZE(itlb_rd_entry)
11287c478bd9Sstevel@tonic-gate
11297c478bd9Sstevel@tonic-gate	ENTRY_NP(dtlb_rd_entry)
11307c478bd9Sstevel@tonic-gate	sllx	%o0, 3, %o0
11317c478bd9Sstevel@tonic-gate#if defined(SF_ERRATA_32)
11327c478bd9Sstevel@tonic-gate	sethi	%hi(FLUSH_ADDR), %g2
11337c478bd9Sstevel@tonic-gate	set	MMU_PCONTEXT, %g1
11347c478bd9Sstevel@tonic-gate	stxa	%g0, [%g1]ASI_DMMU			! KCONTEXT
11357c478bd9Sstevel@tonic-gate	flush	%g2
11367c478bd9Sstevel@tonic-gate#endif
11377c478bd9Sstevel@tonic-gate	ldxa	[%o0]ASI_DTLB_ACCESS, %g1
11387c478bd9Sstevel@tonic-gate	set	TTE_SPITFIRE_PFNHI_CLEAR, %g2		! spitfire only
11397c478bd9Sstevel@tonic-gate	sllx	%g2, TTE_SPITFIRE_PFNHI_SHIFT, %g2	! see comment above
11407c478bd9Sstevel@tonic-gate	andn	%g1, %g2, %g1				! itlb_rd_entry
11417c478bd9Sstevel@tonic-gate	stx	%g1, [%o1]
11427c478bd9Sstevel@tonic-gate	ldxa	[%o0]ASI_DTLB_TAGREAD, %g2
11437c478bd9Sstevel@tonic-gate	set	TAGREAD_CTX_MASK, %o4
11447c478bd9Sstevel@tonic-gate	andn	%g2, %o4, %o5
11457c478bd9Sstevel@tonic-gate	retl
11467c478bd9Sstevel@tonic-gate	  stx	%o5, [%o2]
11477c478bd9Sstevel@tonic-gate	SET_SIZE(dtlb_rd_entry)
11487c478bd9Sstevel@tonic-gate
11497c478bd9Sstevel@tonic-gate	ENTRY(set_lsu)
11507c478bd9Sstevel@tonic-gate	stxa	%o0, [%g0]ASI_LSU		! store to LSU
11517c478bd9Sstevel@tonic-gate	retl
11527c478bd9Sstevel@tonic-gate	membar	#Sync
11537c478bd9Sstevel@tonic-gate	SET_SIZE(set_lsu)
11547c478bd9Sstevel@tonic-gate
11557c478bd9Sstevel@tonic-gate	ENTRY(get_lsu)
11567c478bd9Sstevel@tonic-gate	retl
11577c478bd9Sstevel@tonic-gate	ldxa	[%g0]ASI_LSU, %o0		! load LSU
11587c478bd9Sstevel@tonic-gate	SET_SIZE(get_lsu)
11597c478bd9Sstevel@tonic-gate
11607c478bd9Sstevel@tonic-gate	/*
11617c478bd9Sstevel@tonic-gate	 * Clear the NPT (non-privileged trap) bit in the %tick
11627c478bd9Sstevel@tonic-gate	 * registers. In an effort to make the change in the
11637c478bd9Sstevel@tonic-gate	 * tick counter as consistent as possible, we disable
11647c478bd9Sstevel@tonic-gate	 * all interrupts while we're changing the registers. We also
11657c478bd9Sstevel@tonic-gate	 * ensure that the read and write instructions are in the same
11667c478bd9Sstevel@tonic-gate	 * line in the instruction cache.
11677c478bd9Sstevel@tonic-gate	 */
11687c478bd9Sstevel@tonic-gate	ENTRY_NP(cpu_clearticknpt)
11697c478bd9Sstevel@tonic-gate	rdpr	%pstate, %g1		/* save processor state */
11707c478bd9Sstevel@tonic-gate	andn	%g1, PSTATE_IE, %g3	/* turn off */
11717c478bd9Sstevel@tonic-gate	wrpr	%g0, %g3, %pstate	/*   interrupts */
11727c478bd9Sstevel@tonic-gate	rdpr	%tick, %g2		/* get tick register */
11737c478bd9Sstevel@tonic-gate