xref: /illumos-gate/usr/src/uts/sun4u/sys/cheetahasm.h (revision 55fea89d)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5ed05dc57Svb  * Common Development and Distribution License (the "License").
6ed05dc57Svb  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22ed05dc57Svb  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #ifndef	_CHEETAHASM_H
277c478bd9Sstevel@tonic-gate #define	_CHEETAHASM_H
287c478bd9Sstevel@tonic-gate 
297c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
307c478bd9Sstevel@tonic-gate 
317c478bd9Sstevel@tonic-gate #ifdef	__cplusplus
327c478bd9Sstevel@tonic-gate extern "C" {
337c478bd9Sstevel@tonic-gate #endif
347c478bd9Sstevel@tonic-gate 
357c478bd9Sstevel@tonic-gate #ifdef _ASM
367c478bd9Sstevel@tonic-gate /* BEGIN CSTYLED */
377c478bd9Sstevel@tonic-gate 
387c478bd9Sstevel@tonic-gate #define	ASM_LD(reg, symbol)						\
397c478bd9Sstevel@tonic-gate 	sethi	%hi(symbol), reg;					\
407c478bd9Sstevel@tonic-gate 	ld	[reg + %lo(symbol)], reg;				\
417c478bd9Sstevel@tonic-gate 
427c478bd9Sstevel@tonic-gate #define	ASM_LDX(reg, symbol)						\
437c478bd9Sstevel@tonic-gate 	sethi	%hi(symbol), reg;					\
447c478bd9Sstevel@tonic-gate 	ldx	[reg + %lo(symbol)], reg;				\
457c478bd9Sstevel@tonic-gate 
467c478bd9Sstevel@tonic-gate #define	ASM_JMP(reg, symbol)						\
477c478bd9Sstevel@tonic-gate 	sethi	%hi(symbol), reg;					\
487c478bd9Sstevel@tonic-gate 	jmp	reg + %lo(symbol);					\
497c478bd9Sstevel@tonic-gate 	nop
507c478bd9Sstevel@tonic-gate 
517c478bd9Sstevel@tonic-gate /*
527c478bd9Sstevel@tonic-gate  * Macro for getting to offset from 'cpu_private' ptr.  The 'cpu_private'
537c478bd9Sstevel@tonic-gate  * ptr is in the machcpu structure.
547c478bd9Sstevel@tonic-gate  *  off_reg:  Register offset from 'cpu_private' ptr.
557c478bd9Sstevel@tonic-gate  *  scr1:    Scratch, ptr is returned in this register.
567c478bd9Sstevel@tonic-gate  *  scr2:    Scratch
577c478bd9Sstevel@tonic-gate  *  label:   Label to branch to if cpu_private ptr is null/zero.
587c478bd9Sstevel@tonic-gate  */
597c478bd9Sstevel@tonic-gate #define	GET_CPU_PRIVATE_PTR(off_reg, scr1, scr2, label)			\
607c478bd9Sstevel@tonic-gate 	CPU_ADDR(scr1, scr2);						\
617c478bd9Sstevel@tonic-gate 	ldn	[scr1 + CPU_PRIVATE], scr1;				\
627c478bd9Sstevel@tonic-gate 	cmp	scr1, 0;						\
637c478bd9Sstevel@tonic-gate 	be	label;							\
647c478bd9Sstevel@tonic-gate 	  nop;								\
657c478bd9Sstevel@tonic-gate 	add	scr1, off_reg, scr1
667c478bd9Sstevel@tonic-gate 
677c478bd9Sstevel@tonic-gate /*
687c478bd9Sstevel@tonic-gate  * Macro version of get_dcache_dtag.  We use this macro in the
697c478bd9Sstevel@tonic-gate  * CPU logout code. Since the Dcache is virtually indexed, only
707c478bd9Sstevel@tonic-gate  * bits [12:5] of the AFAR can be used so we need to search through
717c478bd9Sstevel@tonic-gate  * 8 indexes (4 ways + bit 13) in order to find the tag we want.
727c478bd9Sstevel@tonic-gate  *   afar:  input AFAR, not modified.
737c478bd9Sstevel@tonic-gate  *   datap: input ptr to ch_dc_data_t, at end pts to end of ch_dc_data_t.
747c478bd9Sstevel@tonic-gate  *   scr1:  scratch.
757c478bd9Sstevel@tonic-gate  *   scr2:  scratch, will hold tag to look for.
767c478bd9Sstevel@tonic-gate  *   scr3:  used for Dcache index, loops through 4 ways.
777c478bd9Sstevel@tonic-gate  */
787c478bd9Sstevel@tonic-gate #define	GET_DCACHE_DTAG(afar, datap, scr1, scr2, scr3)			\
797c478bd9Sstevel@tonic-gate 	set	CH_DCACHE_IDX_MASK, scr3;				\
807c478bd9Sstevel@tonic-gate 	and	afar, scr3, scr3;					\
817c478bd9Sstevel@tonic-gate 	srlx	afar, CH_DCTAG_PA_SHIFT, scr2;				\
827c478bd9Sstevel@tonic-gate 	b	1f;							\
837c478bd9Sstevel@tonic-gate 	  or	scr2, CH_DCTAG_VALID_BIT, scr2; /* tag we want */	\
847c478bd9Sstevel@tonic-gate 	.align	128;							\
857c478bd9Sstevel@tonic-gate 1:									\
867c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_DC_TAG, scr1;		/* read tag */		\
877c478bd9Sstevel@tonic-gate 	cmp	scr1, scr2;						\
887c478bd9Sstevel@tonic-gate 	bne	4f;				/* not found? */	\
897c478bd9Sstevel@tonic-gate 	  nop;								\
907c478bd9Sstevel@tonic-gate 	stxa	scr3, [datap + CH_DC_IDX]%asi;	/* store index */	\
917c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_DC_TAG]%asi;	/* store tag */		\
927c478bd9Sstevel@tonic-gate 	membar	#Sync;			/* Cheetah PRM 10.6.3 */	\
937c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_DC_UTAG, scr1;	/* read utag */		\
947c478bd9Sstevel@tonic-gate 	membar	#Sync;			/* Cheetah PRM 10.6.3 */	\
957c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_DC_UTAG]%asi;				\
967c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_DC_SNP_TAG, scr1;	/* read snoop tag */	\
977c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_DC_SNTAG]%asi;			\
987c478bd9Sstevel@tonic-gate 	add	datap, CH_DC_DATA, datap;				\
997c478bd9Sstevel@tonic-gate 	clr	scr2;							\
1007c478bd9Sstevel@tonic-gate 2:									\
1017c478bd9Sstevel@tonic-gate 	membar	#Sync;			/* Cheetah PRM 10.6.1 */	\
1027c478bd9Sstevel@tonic-gate 	ldxa	[scr3 + scr2]ASI_DC_DATA, scr1;	/* read data */		\
1037c478bd9Sstevel@tonic-gate 	membar	#Sync;			/* Cheetah PRM 10.6.1 */	\
1047c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap]%asi;					\
1057c478bd9Sstevel@tonic-gate 	add	datap, 8, datap;					\
1067c478bd9Sstevel@tonic-gate 	cmp	scr2, CH_DC_DATA_REG_SIZE - 8;				\
1077c478bd9Sstevel@tonic-gate 	blt	2b;							\
1087c478bd9Sstevel@tonic-gate 	  add	scr2, 8, scr2;						\
1097c478bd9Sstevel@tonic-gate 									\
1107c478bd9Sstevel@tonic-gate 	GET_CPU_IMPL(scr2);	/* Parity bits are elsewhere for */	\
1117c478bd9Sstevel@tonic-gate 	cmp	scr2, PANTHER_IMPL;	/* panther processors. */	\
1127c478bd9Sstevel@tonic-gate 	bne,a	5f;			/* Done if not panther. */	\
1137c478bd9Sstevel@tonic-gate 	  add	datap, 8, datap; /* Skip to the end of the struct. */	\
1147c478bd9Sstevel@tonic-gate 	clr	scr2;							\
1157c478bd9Sstevel@tonic-gate 	add	datap, 7, datap; /* offset of the last parity byte */	\
1167c478bd9Sstevel@tonic-gate 	mov	1, scr1;						\
1177c478bd9Sstevel@tonic-gate 	sll	scr1, PN_DC_DATA_PARITY_BIT_SHIFT, scr1;		\
1187c478bd9Sstevel@tonic-gate 	or	scr3, scr1, scr3; /* add DC_data_parity bit to index */	\
1197c478bd9Sstevel@tonic-gate 3:									\
1207c478bd9Sstevel@tonic-gate 	membar	#Sync;			/* Cheetah PRM 10.6.1 */	\
1217c478bd9Sstevel@tonic-gate 	ldxa	[scr3 + scr2]ASI_DC_DATA, scr1;	/* read parity bits */	\
1227c478bd9Sstevel@tonic-gate 	membar	#Sync;			/* Cheetah PRM 10.6.1 */	\
1237c478bd9Sstevel@tonic-gate 	stba	scr1, [datap]%asi;					\
1247c478bd9Sstevel@tonic-gate 	dec	datap;							\
1257c478bd9Sstevel@tonic-gate 	cmp	scr2, CH_DC_DATA_REG_SIZE - 8;				\
1267c478bd9Sstevel@tonic-gate 	blt	3b;							\
1277c478bd9Sstevel@tonic-gate 	  add	scr2, 8, scr2;						\
1287c478bd9Sstevel@tonic-gate 	b	5f;							\
1297c478bd9Sstevel@tonic-gate 	  add	datap, 5, datap; /* set pointer to end of our struct */	\
1307c478bd9Sstevel@tonic-gate 4:									\
1317c478bd9Sstevel@tonic-gate 	set	CH_DCACHE_IDX_INCR, scr1;	/* incr. idx (scr3) */	\
1327c478bd9Sstevel@tonic-gate 	add	scr3, scr1, scr3;					\
1337c478bd9Sstevel@tonic-gate 	set	CH_DCACHE_IDX_LIMIT, scr1;	/* done? */		\
1347c478bd9Sstevel@tonic-gate 	cmp	scr3, scr1;						\
1357c478bd9Sstevel@tonic-gate 	blt	1b;							\
1367c478bd9Sstevel@tonic-gate 	  nop;								\
1377c478bd9Sstevel@tonic-gate 	add	datap, CH_DC_DATA_SIZE, datap;				\
1387c478bd9Sstevel@tonic-gate 5:
1397c478bd9Sstevel@tonic-gate 
1407c478bd9Sstevel@tonic-gate /*
1417c478bd9Sstevel@tonic-gate  * Macro version of get_icache_dtag.  We use this macro in the CPU
1427c478bd9Sstevel@tonic-gate  * logout code. If the Icache is on, we don't want to capture the data.
1437c478bd9Sstevel@tonic-gate  *   afar:  input AFAR, not modified.
1447c478bd9Sstevel@tonic-gate  *   datap: input ptr to ch_ic_data_t, at end pts to end of ch_ic_data_t.
1457c478bd9Sstevel@tonic-gate  *   scr1:  scratch.
1467c478bd9Sstevel@tonic-gate  *   scr2:  scratch, will hold tag to look for.
1477c478bd9Sstevel@tonic-gate  *   scr3:  used for Icache index, loops through 4 ways.
1487c478bd9Sstevel@tonic-gate  * Note: For Panther, the Icache is virtually indexed and increases in
1497c478bd9Sstevel@tonic-gate  * size to 64KB (instead of 32KB) with a line size of 64 bytes (instead
1507c478bd9Sstevel@tonic-gate  * of 32). This means the IC_addr index bits[14:7] for Panther now
1517c478bd9Sstevel@tonic-gate  * correspond to VA bits[13:6]. But since it is virtually indexed, we
1527c478bd9Sstevel@tonic-gate  * still mask out only bits[12:5] from the AFAR (we have to manually
1537c478bd9Sstevel@tonic-gate  * check bit 13). In order to make this code work for all processors,
1547c478bd9Sstevel@tonic-gate  * we end up checking twice as many indexes (8 instead of 4) as required
1557c478bd9Sstevel@tonic-gate  * for non-Panther CPUs and saving off twice as much data (16 instructions
1567c478bd9Sstevel@tonic-gate  * instead of just 8).
1577c478bd9Sstevel@tonic-gate  */
1587c478bd9Sstevel@tonic-gate #define	GET_ICACHE_DTAG(afar, datap, scr1, scr2, scr3)			\
1597c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_DCU, scr1;					\
1607c478bd9Sstevel@tonic-gate 	btst	DCU_IC, scr1;		/* is Icache enabled? */	\
1617c478bd9Sstevel@tonic-gate 	bne,a	6f;			/* yes, don't capture */	\
1627c478bd9Sstevel@tonic-gate 	  add	datap, CH_IC_DATA_SIZE, datap;	/* anul if no branch */	\
1637c478bd9Sstevel@tonic-gate 	GET_CPU_IMPL(scr2);	/* Panther only uses VA[13:6] */	\
1647c478bd9Sstevel@tonic-gate 	cmp	scr2, PANTHER_IMPL;	/* and we also want to mask */	\
1657c478bd9Sstevel@tonic-gate 	be	1f;			/* out bit 13 since the */	\
1667c478bd9Sstevel@tonic-gate 	  nop;				/* Panther I$ is VIPT. */	\
1677c478bd9Sstevel@tonic-gate 	set	CH_ICACHE_IDX_MASK, scr3;				\
1687c478bd9Sstevel@tonic-gate 	b	2f;							\
1697c478bd9Sstevel@tonic-gate 	  nop;								\
1707c478bd9Sstevel@tonic-gate 1:									\
1717c478bd9Sstevel@tonic-gate 	set	PN_ICACHE_VA_IDX_MASK, scr3;				\
1727c478bd9Sstevel@tonic-gate 2:									\
1737c478bd9Sstevel@tonic-gate 	and	afar, scr3, scr3;					\
1747c478bd9Sstevel@tonic-gate 	sllx	scr3, CH_ICACHE_IDX_SHIFT, scr3;			\
1757c478bd9Sstevel@tonic-gate 	srlx	afar, CH_ICPATAG_SHIFT, scr2;	/* pa tag we want */	\
1767c478bd9Sstevel@tonic-gate 	andn	scr2, CH_ICPATAG_LBITS, scr2;	/* mask off lower */	\
1777c478bd9Sstevel@tonic-gate 	b	3f;							\
1787c478bd9Sstevel@tonic-gate 	  nop;								\
1797c478bd9Sstevel@tonic-gate 	.align	128;							\
1807c478bd9Sstevel@tonic-gate 3:									\
1817c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_IC_TAG, scr1;		/* read pa tag */	\
1827c478bd9Sstevel@tonic-gate 	andn	scr1, CH_ICPATAG_LBITS, scr1;	/* mask off lower */	\
1837c478bd9Sstevel@tonic-gate 	cmp	scr1, scr2;						\
1847c478bd9Sstevel@tonic-gate 	bne	5f;				/* not found? */	\
1857c478bd9Sstevel@tonic-gate 	  nop;								\
1867c478bd9Sstevel@tonic-gate 	stxa	scr3, [datap + CH_IC_IDX]%asi;	/* store index */	\
1877c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_IC_PATAG]%asi; /* store pa tag */	\
1887c478bd9Sstevel@tonic-gate 	add	scr3, CH_ICTAG_UTAG, scr3;	/* read utag */		\
1897c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_IC_TAG, scr1;					\
1907c478bd9Sstevel@tonic-gate 	add	scr3, (CH_ICTAG_UPPER - CH_ICTAG_UTAG), scr3;		\
1917c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_IC_UTAG]%asi;				\
1927c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_IC_TAG, scr1;		/* read upper tag */	\
1937c478bd9Sstevel@tonic-gate 	add	scr3, (CH_ICTAG_LOWER - CH_ICTAG_UPPER), scr3;		\
1947c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_IC_UPPER]%asi;			\
1957c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_IC_TAG, scr1;		/* read lower tag */	\
1967c478bd9Sstevel@tonic-gate 	andn	scr3, CH_ICTAG_TMASK, scr3;				\
1977c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_IC_LOWER]%asi;			\
1987c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_IC_SNP_TAG, scr1;	/* read snoop tag */	\
1997c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_IC_SNTAG]%asi;			\
2007c478bd9Sstevel@tonic-gate 	add	datap, CH_IC_DATA, datap;				\
2017c478bd9Sstevel@tonic-gate 	clr	scr2;							\
2027c478bd9Sstevel@tonic-gate 4:									\
2037c478bd9Sstevel@tonic-gate 	ldxa	[scr3 + scr2]ASI_IC_DATA, scr1;	/* read ins. data */	\
2047c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap]%asi;					\
2057c478bd9Sstevel@tonic-gate 	add	datap, 8, datap;					\
2067c478bd9Sstevel@tonic-gate 	cmp	scr2, PN_IC_DATA_REG_SIZE - 8;				\
2077c478bd9Sstevel@tonic-gate 	blt	4b;							\
2087c478bd9Sstevel@tonic-gate 	  add	scr2, 8, scr2;						\
2097c478bd9Sstevel@tonic-gate 	b	6f;							\
2107c478bd9Sstevel@tonic-gate 	  nop;								\
2117c478bd9Sstevel@tonic-gate 5:									\
2127c478bd9Sstevel@tonic-gate 	set	CH_ICACHE_IDX_INCR, scr1;	/* incr. idx (scr3) */	\
2137c478bd9Sstevel@tonic-gate 	add	scr3, scr1, scr3;					\
2147c478bd9Sstevel@tonic-gate 	set	PN_ICACHE_IDX_LIMIT, scr1;	/* done? */		\
2157c478bd9Sstevel@tonic-gate 	cmp	scr3, scr1;						\
2167c478bd9Sstevel@tonic-gate 	blt	3b;							\
2177c478bd9Sstevel@tonic-gate 	  nop;								\
2187c478bd9Sstevel@tonic-gate 	add	datap, CH_IC_DATA_SIZE, datap;				\
2197c478bd9Sstevel@tonic-gate 6:
2207c478bd9Sstevel@tonic-gate 
2217c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO)
2227c478bd9Sstevel@tonic-gate /*
2237c478bd9Sstevel@tonic-gate  * Macro version of get_ecache_dtag.  We use this macro in the
2247c478bd9Sstevel@tonic-gate  * CPU logout code.
2257c478bd9Sstevel@tonic-gate  *   afar:	input AFAR, not modified
2267c478bd9Sstevel@tonic-gate  *   datap:	Ptr to ch_ec_data_t, at end pts just past ch_ec_data_t.
227*55fea89dSDan Cross  *   ec_way:	Constant value (way number)
2287c478bd9Sstevel@tonic-gate  *   scr1:      Scratch
2297c478bd9Sstevel@tonic-gate  *   scr2:	Scratch.
2307c478bd9Sstevel@tonic-gate  *   scr3:	Scratch.
2317c478bd9Sstevel@tonic-gate  */
2327c478bd9Sstevel@tonic-gate #define	GET_ECACHE_DTAG(afar, datap, ec_way, scr1, scr2, scr3)		\
2337c478bd9Sstevel@tonic-gate 	mov	ec_way, scr1;						\
2347c478bd9Sstevel@tonic-gate 	and	scr1, JP_ECACHE_NWAY - 1, scr1;	/* mask E$ way bits */	\
2357c478bd9Sstevel@tonic-gate 	sllx	scr1, JP_EC_TAG_DATA_WAY_SHIFT, scr1;			\
2367c478bd9Sstevel@tonic-gate 	set	((JP_ECACHE_MAX_SIZE / JP_ECACHE_NWAY) - 1), scr2;	\
2377c478bd9Sstevel@tonic-gate 	and	afar, scr2, scr3;		/* get set offset */	\
2387c478bd9Sstevel@tonic-gate 	andn	scr3, (JP_ECACHE_MAX_LSIZE - 1), scr3; /* VA<5:0>=0 */	\
2397c478bd9Sstevel@tonic-gate 	or	scr3, scr1, scr3;		/* or WAY bits */	\
2407c478bd9Sstevel@tonic-gate 	b	1f;							\
2417c478bd9Sstevel@tonic-gate 	  stxa	scr3, [datap + CH_EC_IDX]%asi;	/* store E$ index */	\
2427c478bd9Sstevel@tonic-gate 	.align	64;							\
2437c478bd9Sstevel@tonic-gate 1:									\
2447c478bd9Sstevel@tonic-gate 	JP_EC_DIAG_ACCESS_MEMBAR;					\
2457c478bd9Sstevel@tonic-gate 	ldxa    [scr3]ASI_EC_DIAG, scr1;	/* get E$ tag */	\
2467c478bd9Sstevel@tonic-gate 	JP_EC_DIAG_ACCESS_MEMBAR;					\
2477c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_EC_TAG]%asi;				\
2487c478bd9Sstevel@tonic-gate 	add	datap, CH_EC_DATA, datap;				\
2497c478bd9Sstevel@tonic-gate 2:									\
2507c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_EC_R, %g0;		/* ld E$ stging regs */	\
2517c478bd9Sstevel@tonic-gate 	clr	scr1;							\
2527c478bd9Sstevel@tonic-gate 3:						/* loop thru 5 regs */	\
2537c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_EC_DATA, scr2;				\
2547c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap]%asi;					\
2557c478bd9Sstevel@tonic-gate 	add	datap, 8, datap;					\
2567c478bd9Sstevel@tonic-gate 	cmp	scr1, CH_ECACHE_STGREG_TOTALSIZE - 8;			\
2577c478bd9Sstevel@tonic-gate 	bne	3b;							\
2587c478bd9Sstevel@tonic-gate 	   add	scr1, 8, scr1;						\
2597c478bd9Sstevel@tonic-gate 	btst	CH_ECACHE_STGREG_SIZE, scr3;	/* done? */		\
2607c478bd9Sstevel@tonic-gate 	beq	2b;							\
2617c478bd9Sstevel@tonic-gate 	   add	scr3, CH_ECACHE_STGREG_SIZE, scr3
2627c478bd9Sstevel@tonic-gate 
2637c478bd9Sstevel@tonic-gate #define	GET_ECACHE_DTAGS(afar, datap, scr1, scr2, scr3)			\
2647c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 0, scr1, scr2, scr3);		\
2657c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 1, scr1, scr2, scr3);		\
2667c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 2, scr1, scr2, scr3);		\
2677c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 3, scr1, scr2, scr3);		\
2687c478bd9Sstevel@tonic-gate 	add	datap, (CHD_EC_DATA_SETS-4)*CH_EC_DATA_SIZE, datap;	\
2697c478bd9Sstevel@tonic-gate 	add	datap, CH_EC_DATA_SIZE * PN_L2_NWAYS, datap;		\
2707c478bd9Sstevel@tonic-gate 
2717c478bd9Sstevel@tonic-gate /*
2727c478bd9Sstevel@tonic-gate  * Jalapeno does not have cores so these macros are null.
2737c478bd9Sstevel@tonic-gate  */
2747c478bd9Sstevel@tonic-gate #define	PARK_SIBLING_CORE(dcucr_reg, scr1, scr2)
2757c478bd9Sstevel@tonic-gate #define	UNPARK_SIBLING_CORE(dcucr_reg, scr1, scr2)
2767c478bd9Sstevel@tonic-gate 
2777c478bd9Sstevel@tonic-gate #if defined(JALAPENO)
2787c478bd9Sstevel@tonic-gate /*
2797c478bd9Sstevel@tonic-gate  * Jalapeno gets primary AFSR and AFAR.  All bits in the AFSR except
2807c478bd9Sstevel@tonic-gate  * the fatal error bits are cleared.
2817c478bd9Sstevel@tonic-gate  *	datap:		pointer to cpu logout structure.
2827c478bd9Sstevel@tonic-gate  *	afar:		returned primary AFAR value.
2837c478bd9Sstevel@tonic-gate  *	scr1:		scratch
2847c478bd9Sstevel@tonic-gate  *	scr2:		scratch
2857c478bd9Sstevel@tonic-gate  */
2867c478bd9Sstevel@tonic-gate #define	GET_AFSR_AFAR(datap, afar, scr1, scr2)				\
2877c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_AFAR, afar;					\
2887c478bd9Sstevel@tonic-gate 	stxa	afar, [datap + (CH_CLO_DATA + CH_CHD_AFAR)]%asi;	\
2897c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_AFSR, scr2;					\
2907c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap + (CH_CLO_DATA + CH_CHD_AFSR)]%asi;	\
2917c478bd9Sstevel@tonic-gate 	sethi	%hh(C_AFSR_FATAL_ERRS), scr1;				\
2927c478bd9Sstevel@tonic-gate 	sllx	scr1, 32, scr1;						\
2937c478bd9Sstevel@tonic-gate 	bclr	scr1, scr2;	/* Clear fatal error bits here, so */	\
2947c478bd9Sstevel@tonic-gate 	stxa	scr2, [%g0]ASI_AFSR; /* they're left as is in AFSR */	\
2957c478bd9Sstevel@tonic-gate 	membar	#Sync
2967c478bd9Sstevel@tonic-gate 
2977c478bd9Sstevel@tonic-gate /*
2987c478bd9Sstevel@tonic-gate  * Jalapeno has no shadow AFAR, null operation.
2997c478bd9Sstevel@tonic-gate  */
3007c478bd9Sstevel@tonic-gate #define	GET_SHADOW_DATA(afar, datap, scr1, scr2, scr3)
3017c478bd9Sstevel@tonic-gate 
3027c478bd9Sstevel@tonic-gate #elif defined(SERRANO)
3037c478bd9Sstevel@tonic-gate /*
3047c478bd9Sstevel@tonic-gate  * Serrano gets primary AFSR and AFAR.  All bits in the AFSR except
3057c478bd9Sstevel@tonic-gate  * the fatal error bits are cleared.  For Serrano, we also save the
306*55fea89dSDan Cross  * AFAR2 register.
3077c478bd9Sstevel@tonic-gate  *	datap:	pointer to cpu logout structure.
3087c478bd9Sstevel@tonic-gate  *	afar:	returned primary AFAR value.
3097c478bd9Sstevel@tonic-gate  *	scr1:	scratch
3107c478bd9Sstevel@tonic-gate  *	scr2:	scratch
3117c478bd9Sstevel@tonic-gate  */
3127c478bd9Sstevel@tonic-gate #define GET_AFSR_AFAR(datap, afar, scr1, scr2)				\
3137c478bd9Sstevel@tonic-gate 	set	ASI_MCU_AFAR2_VA, scr1;					\
3147c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_MCU_CTRL, afar;				\
3157c478bd9Sstevel@tonic-gate 	stxa	afar, [datap + (CH_CLO_DATA + CH_CHD_AFAR2)]%asi;	\
3167c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_AFAR, afar;					\
3177c478bd9Sstevel@tonic-gate 	stxa	afar, [datap + (CH_CLO_DATA + CH_CHD_AFAR)]%asi;	\
3187c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_AFSR, scr2;					\
3197c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap + (CH_CLO_DATA + CH_CHD_AFSR)]%asi;	\
3207c478bd9Sstevel@tonic-gate 	sethi	%hh(C_AFSR_FATAL_ERRS), scr1;				\
3217c478bd9Sstevel@tonic-gate 	sllx	scr1, 32, scr1;						\
3227c478bd9Sstevel@tonic-gate 	bclr	scr1, scr2;	/* Clear fatal error bits here, so */	\
3237c478bd9Sstevel@tonic-gate 	stxa	scr2, [%g0]ASI_AFSR; /* they're left as is in AFSR */ 	\
3247c478bd9Sstevel@tonic-gate 	membar	#Sync
3257c478bd9Sstevel@tonic-gate 
3267c478bd9Sstevel@tonic-gate /*
3277c478bd9Sstevel@tonic-gate  * Serrano needs to capture E$, D$ and I$ lines associated with afar2.
3287c478bd9Sstevel@tonic-gate  *      afar:   scratch, holds afar2.
3297c478bd9Sstevel@tonic-gate  *      datap:  pointer to cpu logout structure
3307c478bd9Sstevel@tonic-gate  *      scr1:   scratch
3317c478bd9Sstevel@tonic-gate  *      scr2:   scratch
3327c478bd9Sstevel@tonic-gate  *      scr3:   scratch
3337c478bd9Sstevel@tonic-gate  */
3347c478bd9Sstevel@tonic-gate #define	GET_SHADOW_DATA(afar, datap, scr1, scr2, scr3)		\
3357c478bd9Sstevel@tonic-gate 	ldxa	[datap + (CH_CLO_DATA + CH_CHD_AFAR2)]%asi, afar;	\
3367c478bd9Sstevel@tonic-gate 	add	datap, CH_CLO_SDW_DATA + CH_CHD_EC_DATA, datap;		\
3377c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAGS(afar, datap, scr1, scr2, scr3);		\
3387c478bd9Sstevel@tonic-gate 	GET_DCACHE_DTAG(afar, datap, scr1, scr2, scr3);			\
3397c478bd9Sstevel@tonic-gate 	GET_ICACHE_DTAG(afar, datap, scr1, scr2, scr3);			\
3407c478bd9Sstevel@tonic-gate 	sub	datap, CH_CPU_LOGOUT_SIZE, datap
3417c478bd9Sstevel@tonic-gate #endif /* SERRANO */
3427c478bd9Sstevel@tonic-gate 
3437c478bd9Sstevel@tonic-gate #elif defined(CHEETAH_PLUS)
3447c478bd9Sstevel@tonic-gate /*
3457c478bd9Sstevel@tonic-gate  * Macro version of get_ecache_dtag.  We use this macro in the
3467c478bd9Sstevel@tonic-gate  * CPU logout code.
3477c478bd9Sstevel@tonic-gate  *   afar:	input AFAR, not modified.
3487c478bd9Sstevel@tonic-gate  *   datap:	Ptr to ch_ec_data_t, at end pts just past ch_ec_data_t.
3497c478bd9Sstevel@tonic-gate  *   pn_way:	ecache way for panther (value = 0-3). For non-panther
3507c478bd9Sstevel@tonic-gate  *		cpus, this macro will be called with pn_way = 0.
3517c478bd9Sstevel@tonic-gate  *   scr1:	Scratch.
3527c478bd9Sstevel@tonic-gate  *   scr2:	Scratch.
3537c478bd9Sstevel@tonic-gate  *   scr3:	Scratch.
3547c478bd9Sstevel@tonic-gate  */
3557c478bd9Sstevel@tonic-gate #define	GET_ECACHE_DTAG(afar, datap, pn_way, scr1, scr2, scr3)		\
3567c478bd9Sstevel@tonic-gate 	mov	afar, scr3;						\
3577c478bd9Sstevel@tonic-gate 	andn	scr3, (CH_ECACHE_SUBBLK_SIZE - 1), scr3; /* VA<5:0>=0 */\
3587c478bd9Sstevel@tonic-gate 	set	(CH_ECACHE_8M_SIZE - 1), scr2;				\
3597c478bd9Sstevel@tonic-gate 	and	scr3, scr2, scr3;		/* VA<63:23>=0 */	\
3607c478bd9Sstevel@tonic-gate 	mov	pn_way, scr1;	/* panther L3$ is 4-way so we ...    */	\
3617c478bd9Sstevel@tonic-gate 	sllx	scr1, PN_L3_WAY_SHIFT, scr1;	/* need to mask...   */	\
3627c478bd9Sstevel@tonic-gate 	or	scr3, scr1, scr3;	/* in the way bits <24:23>.  */	\
3637c478bd9Sstevel@tonic-gate 	b	1f;							\
3647c478bd9Sstevel@tonic-gate 	   stxa	scr3, [datap + CH_EC_IDX]%asi;	/* store E$ index */	\
3657c478bd9Sstevel@tonic-gate 	.align	64;							\
3667c478bd9Sstevel@tonic-gate 1:									\
3677c478bd9Sstevel@tonic-gate 	ldxa    [scr3]ASI_EC_DIAG, scr1;	/* get E$ tag */	\
3687c478bd9Sstevel@tonic-gate 	stxa     scr1, [datap + CH_EC_TAG]%asi;				\
3697c478bd9Sstevel@tonic-gate 	set	CHP_ECACHE_IDX_TAG_ECC, scr1;				\
3707c478bd9Sstevel@tonic-gate 	or	scr3, scr1, scr1;					\
3717c478bd9Sstevel@tonic-gate 	ldxa    [scr1]ASI_EC_DIAG, scr1;	/* get E$ tag ECC */	\
3727c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_EC_TAG_ECC]%asi;			\
3737c478bd9Sstevel@tonic-gate 	add	datap, CH_EC_DATA, datap;				\
3747c478bd9Sstevel@tonic-gate 2:									\
3757c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_EC_R, %g0;		/* ld E$ stging regs */	\
3767c478bd9Sstevel@tonic-gate 	clr	scr1;							\
3777c478bd9Sstevel@tonic-gate 3:						/* loop thru 5 regs */	\
3787c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_EC_DATA, scr2;				\
3797c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap]%asi;					\
3807c478bd9Sstevel@tonic-gate 	add	datap, 8, datap;					\
3817c478bd9Sstevel@tonic-gate 	cmp	scr1, CH_ECACHE_STGREG_TOTALSIZE - 8;			\
3827c478bd9Sstevel@tonic-gate 	bne	3b;							\
3837c478bd9Sstevel@tonic-gate 	   add	scr1, 8, scr1;						\
3847c478bd9Sstevel@tonic-gate 	btst	CH_ECACHE_STGREG_SIZE, scr3;	/* done? */		\
3857c478bd9Sstevel@tonic-gate 	beq	2b;							\
3867c478bd9Sstevel@tonic-gate 	   add	scr3, CH_ECACHE_STGREG_SIZE, scr3
3877c478bd9Sstevel@tonic-gate 
3887c478bd9Sstevel@tonic-gate /*
3897c478bd9Sstevel@tonic-gate  * If this is a panther, we need to make sure the sibling core is
3907c478bd9Sstevel@tonic-gate  * parked so that we avoid any race conditions during diagnostic
3917c478bd9Sstevel@tonic-gate  * accesses to the shared L2 and L3 caches.
3927c478bd9Sstevel@tonic-gate  * dcucr_reg:	This register will be used to keep track of whether
3937c478bd9Sstevel@tonic-gate  *		or not we need to unpark the core later.
3947c478bd9Sstevel@tonic-gate  *		It just so happens that we also use this same register
3957c478bd9Sstevel@tonic-gate  *		to keep track of our saved DCUCR value so we only touch
3967c478bd9Sstevel@tonic-gate  *		bit 4 of the register (which is a "reserved" bit in the
3977c478bd9Sstevel@tonic-gate  *		DCUCR) for keeping track of core parking.
3987c478bd9Sstevel@tonic-gate  * scr1:	Scratch register.
3997c478bd9Sstevel@tonic-gate  * scr2:	Scratch register.
4007c478bd9Sstevel@tonic-gate  */
4017c478bd9Sstevel@tonic-gate #define	PARK_SIBLING_CORE(dcucr_reg, scr1, scr2)			\
4027c478bd9Sstevel@tonic-gate 	GET_CPU_IMPL(scr1);						\
4037c478bd9Sstevel@tonic-gate 	cmp	scr1, PANTHER_IMPL;	/* only park for panthers */	\
4047c478bd9Sstevel@tonic-gate 	bne,a	%xcc, 2f;						\
4057c478bd9Sstevel@tonic-gate 	  andn	dcucr_reg, PN_PARKED_OTHER_CORE, dcucr_reg;		\
4067c478bd9Sstevel@tonic-gate 	set	ASI_CORE_RUNNING_STATUS, scr1;	/* check other core */	\
4077c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_CMP_SHARED, scr2;	/* is it running?   */	\
4087c478bd9Sstevel@tonic-gate 	cmp	scr2, PN_BOTH_CORES_RUNNING;				\
4097c478bd9Sstevel@tonic-gate 	bne,a	%xcc, 2f;	/* if not running, we are done */	\
4107c478bd9Sstevel@tonic-gate 	  andn	dcucr_reg, PN_PARKED_OTHER_CORE, dcucr_reg;		\
4117c478bd9Sstevel@tonic-gate 	or	dcucr_reg, PN_PARKED_OTHER_CORE, dcucr_reg;		\
4127c478bd9Sstevel@tonic-gate 	set	ASI_CORE_ID, scr1;					\
4137c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_CMP_PER_CORE, scr2;				\
4147c478bd9Sstevel@tonic-gate 	and	scr2, COREID_MASK, scr2;				\
4157c478bd9Sstevel@tonic-gate 	or	%g0, 1, scr1;		/* find out which core... */	\
4167c478bd9Sstevel@tonic-gate 	sll	scr1, scr2, scr2;	/* ... we need to park... */	\
4177c478bd9Sstevel@tonic-gate 1:									\
4187c478bd9Sstevel@tonic-gate 	set	ASI_CORE_RUNNING_RW, scr1;				\
419ed05dc57Svb 	ldxa    [scr1]ASI_CMP_SHARED, scr1;	/* ...but are we? */	\
420ed05dc57Svb 	btst    scr1, scr2;        /* check our own parked status */	\
421ed05dc57Svb 	bz      %xcc, 1b;        /* if we are then go round again */	\
422ed05dc57Svb 	nop;								\
423ed05dc57Svb 	set	ASI_CORE_RUNNING_RW, scr1;	/* else proceed... */	\
4247c478bd9Sstevel@tonic-gate 	stxa	scr2, [scr1]ASI_CMP_SHARED;	/* ... and park it. */	\
42599d4e821Scv 	membar	#Sync;							\
42699d4e821Scv 	set	ASI_CORE_RUNNING_STATUS, scr1;	/* spin until... */	\
4277c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_CMP_SHARED, scr1;	/* ... the other...  */	\
4287c478bd9Sstevel@tonic-gate 	cmp	scr1, scr2;	/* ...core is parked according to... */	\
4297c478bd9Sstevel@tonic-gate 	bne,a	%xcc, 1b;	/* ...the core running status reg.  */	\
4307c478bd9Sstevel@tonic-gate 	  nop;								\
4317c478bd9Sstevel@tonic-gate 2:
4327c478bd9Sstevel@tonic-gate 
4337c478bd9Sstevel@tonic-gate /*
4347c478bd9Sstevel@tonic-gate  * The core running this code will unpark its sibling core if the
4357c478bd9Sstevel@tonic-gate  * sibling core had been parked by the current core earlier in this
4367c478bd9Sstevel@tonic-gate  * trap handler.
4377c478bd9Sstevel@tonic-gate  * dcucr_reg:	This register is used to keep track of whether or not
4387c478bd9Sstevel@tonic-gate  *		we need to unpark our sibling core.
4397c478bd9Sstevel@tonic-gate  *		It just so happens that we also use this same register
4407c478bd9Sstevel@tonic-gate  *		to keep track of our saved DCUCR value so we only touch
4417c478bd9Sstevel@tonic-gate  *		bit 4 of the register (which is a "reserved" bit in the
4427c478bd9Sstevel@tonic-gate  *		DCUCR) for keeping track of core parking.
4437c478bd9Sstevel@tonic-gate  * scr1:	Scratch register.
4447c478bd9Sstevel@tonic-gate  * scr2:	Scratch register.
4457c478bd9Sstevel@tonic-gate  */
4467c478bd9Sstevel@tonic-gate #define	UNPARK_SIBLING_CORE(dcucr_reg, scr1, scr2)			\
4477c478bd9Sstevel@tonic-gate 	btst	PN_PARKED_OTHER_CORE, dcucr_reg;			\
4487c478bd9Sstevel@tonic-gate 	bz,pt	%xcc, 1f;	/* if nothing to unpark, we are done */	\
4497c478bd9Sstevel@tonic-gate 	  andn	dcucr_reg, PN_PARKED_OTHER_CORE, dcucr_reg;		\
4507c478bd9Sstevel@tonic-gate 	set	ASI_CORE_RUNNING_RW, scr1;				\
4517c478bd9Sstevel@tonic-gate 	set	PN_BOTH_CORES_RUNNING, scr2;	/* we want both...   */	\
4527c478bd9Sstevel@tonic-gate 	stxa	scr2, [scr1]ASI_CMP_SHARED;	/* ...cores running. */	\
4537c478bd9Sstevel@tonic-gate 	membar	#Sync;							\
4547c478bd9Sstevel@tonic-gate 1:
4557c478bd9Sstevel@tonic-gate 
4567c478bd9Sstevel@tonic-gate /*
4577c478bd9Sstevel@tonic-gate  * Cheetah+ and Jaguar get both primary and secondary AFSR/AFAR.  All bits
4587c478bd9Sstevel@tonic-gate  * in the primary AFSR are cleared except the fatal error bits.  For Panther,
4597c478bd9Sstevel@tonic-gate  * we also have to read and clear the AFSR_EXT, again leaving the fatal
4607c478bd9Sstevel@tonic-gate  * error bits alone.
4617c478bd9Sstevel@tonic-gate  *	datap:		pointer to cpu logout structure.
4627c478bd9Sstevel@tonic-gate  *	afar:		returned primary AFAR value.
4637c478bd9Sstevel@tonic-gate  *	scr1:		scratch
4647c478bd9Sstevel@tonic-gate  *	scr2:		scratch
4657c478bd9Sstevel@tonic-gate  */
4667c478bd9Sstevel@tonic-gate #define	GET_AFSR_AFAR(datap, afar, scr1, scr2)				\
4677c478bd9Sstevel@tonic-gate 	set	ASI_SHADOW_REG_VA, scr1;				\
4687c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_AFAR, scr2;					\
4697c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap + (CH_CLO_SDW_DATA + CH_CHD_AFAR)]%asi;	\
4707c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_AFSR, scr2;					\
4717c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap + (CH_CLO_SDW_DATA + CH_CHD_AFSR)]%asi;	\
4727c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_AFAR, afar;					\
4737c478bd9Sstevel@tonic-gate 	stxa	afar, [datap + (CH_CLO_DATA + CH_CHD_AFAR)]%asi;	\
4747c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_AFSR, scr2;					\
4757c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap + (CH_CLO_DATA + CH_CHD_AFSR)]%asi;	\
4767c478bd9Sstevel@tonic-gate 	sethi	%hh(C_AFSR_FATAL_ERRS), scr1;				\
4777c478bd9Sstevel@tonic-gate 	sllx	scr1, 32, scr1;						\
4787c478bd9Sstevel@tonic-gate 	bclr	scr1, scr2;	/* Clear fatal error bits here, so */ 	\
4797c478bd9Sstevel@tonic-gate 	stxa	scr2, [%g0]ASI_AFSR; /* they're left as is in AFSR */	\
4807c478bd9Sstevel@tonic-gate 	membar	#Sync;							\
4817c478bd9Sstevel@tonic-gate 	GET_CPU_IMPL(scr1);						\
4827c478bd9Sstevel@tonic-gate 	cmp	scr1, PANTHER_IMPL;					\
4837c478bd9Sstevel@tonic-gate 	bne	%xcc, 1f;						\
4847c478bd9Sstevel@tonic-gate 	   nop;								\
4857c478bd9Sstevel@tonic-gate 	set	ASI_SHADOW_AFSR_EXT_VA, scr1;	/* shadow AFSR_EXT */	\
4867c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_AFSR, scr2;					\
4877c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap + (CH_CLO_SDW_DATA + CH_CHD_AFSR_EXT)]%asi; \
4887c478bd9Sstevel@tonic-gate 	set	ASI_AFSR_EXT_VA, scr1;		/* primary AFSR_EXT */	\
4897c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_AFSR, scr2;					\
4907c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap + (CH_CLO_DATA + CH_CHD_AFSR_EXT)]%asi;	\
4917c478bd9Sstevel@tonic-gate 	set	C_AFSR_EXT_FATAL_ERRS, scr1;				\
4927c478bd9Sstevel@tonic-gate 	bclr	scr1, scr2;	/* Clear fatal error bits here, */	\
4937c478bd9Sstevel@tonic-gate 	set	ASI_AFSR_EXT_VA, scr1;	/* so they're left */		\
4947c478bd9Sstevel@tonic-gate 	stxa	scr2, [scr1]ASI_AFSR;	/* as is in AFSR_EXT */		\
4957c478bd9Sstevel@tonic-gate 	membar	#Sync;							\
4967c478bd9Sstevel@tonic-gate 1:
4977c478bd9Sstevel@tonic-gate 
4987c478bd9Sstevel@tonic-gate /*
4997c478bd9Sstevel@tonic-gate  * This macro is used in the CPU logout code to capture diagnostic
5007c478bd9Sstevel@tonic-gate  * information from the L2 cache on panther processors.
5017c478bd9Sstevel@tonic-gate  *   afar:	input AFAR, not modified.
5027c478bd9Sstevel@tonic-gate  *   datap:	Ptr to pn_l2_data_t, at end pts just past pn_l2_data_t.
5037c478bd9Sstevel@tonic-gate  *   scr1:	Scratch.
5047c478bd9Sstevel@tonic-gate  *   scr2:	Scratch.
5057c478bd9Sstevel@tonic-gate  *   scr3:	Scratch.
5067c478bd9Sstevel@tonic-gate  */
5077c478bd9Sstevel@tonic-gate #define	GET_PN_L2_CACHE_DTAGS(afar, datap, scr1, scr2, scr3)		\
5087c478bd9Sstevel@tonic-gate 	mov	afar, scr3;						\
5097c478bd9Sstevel@tonic-gate 	set	PN_L2_INDEX_MASK, scr1;					\
5107c478bd9Sstevel@tonic-gate 	and	scr3, scr1, scr3;					\
5117c478bd9Sstevel@tonic-gate 	b	1f;	/* code to read tags and data should be ...  */	\
5127c478bd9Sstevel@tonic-gate 	   nop;		/* ...on the same cache line if possible.    */	\
5137c478bd9Sstevel@tonic-gate 	.align	128;	/* update this line if you add lines below. */	\
5147c478bd9Sstevel@tonic-gate 1:									\
5157c478bd9Sstevel@tonic-gate 	stxa	scr3, [datap + CH_EC_IDX]%asi;	/* store L2$ index  */	\
5167c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_L2_TAG, scr1;		/* read the L2$ tag */	\
5177c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_EC_TAG]%asi;				\
5187c478bd9Sstevel@tonic-gate 	add	datap, CH_EC_DATA, datap;				\
5197c478bd9Sstevel@tonic-gate 	clr	scr1;							\
5207c478bd9Sstevel@tonic-gate 2:									\
5217c478bd9Sstevel@tonic-gate 	ldxa	[scr3 + scr1]ASI_L2_DATA, scr2;	/* loop through     */	\
5227c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap]%asi;		/* <511:256> of L2  */	\
5237c478bd9Sstevel@tonic-gate 	add	datap, 8, datap;		/* data and record  */	\
5247c478bd9Sstevel@tonic-gate 	cmp	scr1, (PN_L2_LINESIZE / 2) - 8;	/* it in the cpu    */	\
5257c478bd9Sstevel@tonic-gate 	bne	2b;				/* logout struct.   */	\
5267c478bd9Sstevel@tonic-gate 	  add	scr1, 8, scr1;						\
5277c478bd9Sstevel@tonic-gate 	set	PN_L2_DATA_ECC_SEL, scr2;	/* ECC_sel bit.     */	\
5287c478bd9Sstevel@tonic-gate 	ldxa	[scr3 + scr2]ASI_L2_DATA, scr2;	/* Read and record  */	\
5297c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap]%asi;		/* ecc of <511:256> */	\
5307c478bd9Sstevel@tonic-gate 	add	datap, 8, datap;					\
5317c478bd9Sstevel@tonic-gate 3:									\
5327c478bd9Sstevel@tonic-gate 	ldxa	[scr3 + scr1]ASI_L2_DATA, scr2;	/* loop through     */	\
5337c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap]%asi;		/* <255:0> of L2    */	\
5347c478bd9Sstevel@tonic-gate 	add	datap, 8, datap;		/* data and record  */	\
5357c478bd9Sstevel@tonic-gate 	cmp	scr1, PN_L2_LINESIZE - 8;	/* it in the cpu    */	\
5367c478bd9Sstevel@tonic-gate 	bne	3b;				/* logout struct.   */	\
5377c478bd9Sstevel@tonic-gate 	  add	scr1, 8, scr1;						\
5387c478bd9Sstevel@tonic-gate 	set	PN_L2_DATA_ECC_SEL, scr2;	/* ECC_sel bit.     */	\
5397c478bd9Sstevel@tonic-gate 	add	scr2, PN_L2_ECC_LO_REG, scr2;				\
5407c478bd9Sstevel@tonic-gate 	ldxa	[scr3 + scr2]ASI_L2_DATA, scr2;	/* Read and record  */	\
5417c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap]%asi;		/* ecc of <255:0>.  */	\
5427c478bd9Sstevel@tonic-gate 	add	datap, 8, datap;		/* Advance pointer  */	\
5437c478bd9Sstevel@tonic-gate 	set	PN_L2_SET_SIZE, scr2;					\
5447c478bd9Sstevel@tonic-gate 	set	PN_L2_MAX_SET, scr1;					\
5457c478bd9Sstevel@tonic-gate 	cmp	scr1, scr3;	/* more ways to try for this line? */	\
5467c478bd9Sstevel@tonic-gate 	bg,a	%xcc, 1b;	/* if so, start over with next way */	\
5477c478bd9Sstevel@tonic-gate 	  add	scr3, scr2, scr3
5487c478bd9Sstevel@tonic-gate 
5497c478bd9Sstevel@tonic-gate /*
5507c478bd9Sstevel@tonic-gate  * Cheetah+ assumes E$ is 2-way and grabs both E$ lines associated with afar.
5517c478bd9Sstevel@tonic-gate  *	afar:	AFAR from access.
5527c478bd9Sstevel@tonic-gate  *	datap:	pointer to cpu logout structure.
5537c478bd9Sstevel@tonic-gate  *	scr1:	scratch
5547c478bd9Sstevel@tonic-gate  *	scr2:	scratch
5557c478bd9Sstevel@tonic-gate  *	scr3:	scratch
5567c478bd9Sstevel@tonic-gate  */
5577c478bd9Sstevel@tonic-gate #define	GET_ECACHE_DTAGS(afar, datap, scr1, scr2, scr3)			\
5587c478bd9Sstevel@tonic-gate 	GET_CPU_IMPL(scr1);						\
5597c478bd9Sstevel@tonic-gate 	cmp	scr1, PANTHER_IMPL;					\
5607c478bd9Sstevel@tonic-gate 	bne	%xcc, 4f;						\
5617c478bd9Sstevel@tonic-gate 	  nop;								\
5627c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 0, scr1, scr2, scr3);		\
5637c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 1, scr1, scr2, scr3);		\
5647c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 2, scr1, scr2, scr3);		\
5657c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 3, scr1, scr2, scr3);		\
5667c478bd9Sstevel@tonic-gate 	add	datap, (CHD_EC_DATA_SETS-4)*CH_EC_DATA_SIZE, datap;	\
5677c478bd9Sstevel@tonic-gate 	GET_PN_L2_CACHE_DTAGS(afar, datap, scr1, scr2, scr3);		\
5687c478bd9Sstevel@tonic-gate 	b	5f;							\
5697c478bd9Sstevel@tonic-gate 	  nop;								\
5707c478bd9Sstevel@tonic-gate 4:									\
5717c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 0, scr1, scr2, scr3);		\
5727c478bd9Sstevel@tonic-gate 	GET_ECACHE_WAY_BIT(scr1, scr2);					\
5737c478bd9Sstevel@tonic-gate 	xor	afar, scr1, afar;					\
5747c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, 0, scr1, scr2, scr3);		\
5757c478bd9Sstevel@tonic-gate 	GET_ECACHE_WAY_BIT(scr1, scr2);		/* restore AFAR */	\
5767c478bd9Sstevel@tonic-gate 	xor	afar, scr1, afar;					\
5777c478bd9Sstevel@tonic-gate 	add	datap, (CHD_EC_DATA_SETS-2)*CH_EC_DATA_SIZE, datap;	\
5787c478bd9Sstevel@tonic-gate 	add	datap, CH_EC_DATA_SIZE * PN_L2_NWAYS, datap;		\
5797c478bd9Sstevel@tonic-gate 5:
5807c478bd9Sstevel@tonic-gate 
5817c478bd9Sstevel@tonic-gate /*
5827c478bd9Sstevel@tonic-gate  * Cheetah+ needs to capture E$, D$ and I$ lines associated with
5837c478bd9Sstevel@tonic-gate  * shadow afar.
5847c478bd9Sstevel@tonic-gate  *	afar:	scratch, holds shadow afar.
5857c478bd9Sstevel@tonic-gate  *	datap:	pointer to cpu logout structure
5867c478bd9Sstevel@tonic-gate  *	scr1:	scratch
5877c478bd9Sstevel@tonic-gate  *	scr2:	scratch
5887c478bd9Sstevel@tonic-gate  *	scr3:	scratch
5897c478bd9Sstevel@tonic-gate  */
5907c478bd9Sstevel@tonic-gate #define	GET_SHADOW_DATA(afar, datap, scr1, scr2, scr3)		\
5917c478bd9Sstevel@tonic-gate 	ldxa	[datap + (CH_CLO_SDW_DATA + CH_CHD_AFAR)]%asi, afar;	\
5927c478bd9Sstevel@tonic-gate 	add	datap, CH_CLO_SDW_DATA + CH_CHD_EC_DATA, datap;	\
5937c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAGS(afar, datap, scr1, scr2, scr3);		\
5947c478bd9Sstevel@tonic-gate 	GET_DCACHE_DTAG(afar, datap, scr1, scr2, scr3);			\
5957c478bd9Sstevel@tonic-gate 	GET_ICACHE_DTAG(afar, datap, scr1, scr2, scr3);			\
5967c478bd9Sstevel@tonic-gate 	sub	datap, CH_CPU_LOGOUT_SIZE, datap
5977c478bd9Sstevel@tonic-gate 
5987c478bd9Sstevel@tonic-gate /*
5997c478bd9Sstevel@tonic-gate  * Compute the "Way" bit for 2-way Ecache for Cheetah+.
6007c478bd9Sstevel@tonic-gate  */
6017c478bd9Sstevel@tonic-gate #define	GET_ECACHE_WAY_BIT(scr1, scr2)					\
6027c478bd9Sstevel@tonic-gate 	CPU_INDEX(scr1, scr2);						\
6037c478bd9Sstevel@tonic-gate 	mulx	scr1, CPU_NODE_SIZE, scr1;				\
6047c478bd9Sstevel@tonic-gate 	add	scr1, ECACHE_SIZE, scr1;				\
6057c478bd9Sstevel@tonic-gate 	set	cpunodes, scr2;						\
6067c478bd9Sstevel@tonic-gate 	ld	[scr1 + scr2], scr1;					\
6077c478bd9Sstevel@tonic-gate 	srlx	scr1, 1, scr1
6087c478bd9Sstevel@tonic-gate 
6097c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */
6107c478bd9Sstevel@tonic-gate /*
6117c478bd9Sstevel@tonic-gate  * Macro version of get_ecache_dtag.  We use this macro in the
6127c478bd9Sstevel@tonic-gate  * CPU logout code.
6137c478bd9Sstevel@tonic-gate  *   afar:	input AFAR, not modified.
6147c478bd9Sstevel@tonic-gate  *   datap:	Ptr to ch_ec_data_t, at end pts just past ch_ec_data_t.
6157c478bd9Sstevel@tonic-gate  *   scr1:      Scratch.
6167c478bd9Sstevel@tonic-gate  *   scr2:	Scratch.
6177c478bd9Sstevel@tonic-gate  *   scr3:	Scratch.
6187c478bd9Sstevel@tonic-gate  */
6197c478bd9Sstevel@tonic-gate #define	GET_ECACHE_DTAG(afar, datap, scr1, scr2, scr3)			\
6207c478bd9Sstevel@tonic-gate 	mov	afar, scr3;						\
6217c478bd9Sstevel@tonic-gate 	andn	scr3, (CH_ECACHE_SUBBLK_SIZE - 1), scr3; /* VA<5:0>=0 */\
6227c478bd9Sstevel@tonic-gate 	set	(CH_ECACHE_8M_SIZE - 1), scr2;				\
6237c478bd9Sstevel@tonic-gate 	and	scr3, scr2, scr3;		/* VA<63:23>=0 */	\
6247c478bd9Sstevel@tonic-gate 	b	1f;							\
6257c478bd9Sstevel@tonic-gate 	   stxa	scr3, [datap + CH_EC_IDX]%asi;	/* store E$ index */	\
6267c478bd9Sstevel@tonic-gate 	.align	64;							\
6277c478bd9Sstevel@tonic-gate 1:									\
6287c478bd9Sstevel@tonic-gate 	ldxa    [scr3]ASI_EC_DIAG, scr1;	/* get E$ tag */	\
6297c478bd9Sstevel@tonic-gate 	stxa	scr1, [datap + CH_EC_TAG]%asi;				\
6307c478bd9Sstevel@tonic-gate 	add	datap, CH_EC_DATA, datap;				\
6317c478bd9Sstevel@tonic-gate 2:									\
6327c478bd9Sstevel@tonic-gate 	ldxa	[scr3]ASI_EC_R, %g0;		/* ld E$ stging regs */	\
6337c478bd9Sstevel@tonic-gate 	clr	scr1;							\
6347c478bd9Sstevel@tonic-gate 3:						/* loop thru 5 regs */	\
6357c478bd9Sstevel@tonic-gate 	ldxa	[scr1]ASI_EC_DATA, scr2;				\
6367c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap]%asi;					\
6377c478bd9Sstevel@tonic-gate 	add	datap, 8, datap;					\
6387c478bd9Sstevel@tonic-gate 	cmp	scr1, CH_ECACHE_STGREG_TOTALSIZE - 8;			\
6397c478bd9Sstevel@tonic-gate 	bne	3b;							\
6407c478bd9Sstevel@tonic-gate 	   add	scr1, 8, scr1;						\
6417c478bd9Sstevel@tonic-gate 	btst	CH_ECACHE_STGREG_SIZE, scr3;	/* done? */		\
6427c478bd9Sstevel@tonic-gate 	beq	2b;							\
6437c478bd9Sstevel@tonic-gate 	   add	scr3, CH_ECACHE_STGREG_SIZE, scr3
6447c478bd9Sstevel@tonic-gate 
6457c478bd9Sstevel@tonic-gate /*
6467c478bd9Sstevel@tonic-gate  * Cheetah does not have cores so these macros are null.
6477c478bd9Sstevel@tonic-gate  */
6487c478bd9Sstevel@tonic-gate #define	PARK_SIBLING_CORE(dcucr_reg, scr1, scr2)
6497c478bd9Sstevel@tonic-gate #define	UNPARK_SIBLING_CORE(dcucr_reg, scr1, scr2)
6507c478bd9Sstevel@tonic-gate 
6517c478bd9Sstevel@tonic-gate /*
6527c478bd9Sstevel@tonic-gate  * Cheetah gets primary AFSR and AFAR and clears the AFSR, except for the
6537c478bd9Sstevel@tonic-gate  * fatal error bits.
6547c478bd9Sstevel@tonic-gate  *	datap:		pointer to cpu logout structure.
6557c478bd9Sstevel@tonic-gate  *	afar:		returned primary AFAR value.
6567c478bd9Sstevel@tonic-gate  *	scr1:		scratch
6577c478bd9Sstevel@tonic-gate  *	scr2:		scratch
6587c478bd9Sstevel@tonic-gate  */
6597c478bd9Sstevel@tonic-gate #define	GET_AFSR_AFAR(datap, afar, scr1, scr2)	\
6607c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_AFAR, afar;					\
6617c478bd9Sstevel@tonic-gate 	stxa	afar, [datap + (CH_CLO_DATA + CH_CHD_AFAR)]%asi;	\
6627c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_AFSR, scr2;					\
6637c478bd9Sstevel@tonic-gate 	stxa	scr2, [datap + (CH_CLO_DATA + CH_CHD_AFSR)]%asi;	\
6647c478bd9Sstevel@tonic-gate 	sethi	%hh(C_AFSR_FATAL_ERRS), scr1;				\
6657c478bd9Sstevel@tonic-gate 	sllx	scr1, 32, scr1;						\
6667c478bd9Sstevel@tonic-gate 	bclr	scr1, scr2;	/* Clear fatal error bits here, so */	\
6677c478bd9Sstevel@tonic-gate 	stxa	scr2, [%g0]ASI_AFSR; /* they're left as is in AFSR */	\
6687c478bd9Sstevel@tonic-gate 	membar	#Sync
6697c478bd9Sstevel@tonic-gate 
6707c478bd9Sstevel@tonic-gate /*
6717c478bd9Sstevel@tonic-gate  * Cheetah E$ is direct-mapped, so we grab line data and skip second line.
6727c478bd9Sstevel@tonic-gate  *	afar:	AFAR from access.
6737c478bd9Sstevel@tonic-gate  *	datap:	pointer to cpu logout structure.
6747c478bd9Sstevel@tonic-gate  *	scr1:	scratch
6757c478bd9Sstevel@tonic-gate  *	scr2:	scratch
6767c478bd9Sstevel@tonic-gate  *	scr3:	scratch
6777c478bd9Sstevel@tonic-gate  */
6787c478bd9Sstevel@tonic-gate #define	GET_ECACHE_DTAGS(afar, datap, scr1, scr2, scr3)			\
6797c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAG(afar, datap, scr1, scr2, scr3);			\
6807c478bd9Sstevel@tonic-gate 	add	datap, (CHD_EC_DATA_SETS-1)*CH_EC_DATA_SIZE, datap;	\
6817c478bd9Sstevel@tonic-gate 	add	datap, CH_EC_DATA_SIZE * PN_L2_NWAYS, datap;		\
6827c478bd9Sstevel@tonic-gate 
6837c478bd9Sstevel@tonic-gate /*
6847c478bd9Sstevel@tonic-gate  * Cheetah has no shadow AFAR, null operation.
6857c478bd9Sstevel@tonic-gate  */
6867c478bd9Sstevel@tonic-gate #define	GET_SHADOW_DATA(afar, datap, scr1, scr2, scr3)
6877c478bd9Sstevel@tonic-gate 
6887c478bd9Sstevel@tonic-gate #endif	/* CHEETAH_PLUS */
6897c478bd9Sstevel@tonic-gate 
6907c478bd9Sstevel@tonic-gate /*
6917c478bd9Sstevel@tonic-gate  * Cheetah/(Cheetah+ Jaguar Panther)/Jalapeno Macro for capturing CPU
6927c478bd9Sstevel@tonic-gate  * logout data at TL>0. r_val is a register that returns the "failure count"
6937c478bd9Sstevel@tonic-gate  * to the caller, and may be used as a scratch register until the end of
6947c478bd9Sstevel@tonic-gate  * the macro.  afar is used to return the primary AFAR value to the caller
6957c478bd9Sstevel@tonic-gate  * and it too can be used as a scratch register until the end. r_or_s is
6967c478bd9Sstevel@tonic-gate  * a reg or symbol that has the offset within the "cpu_private" data area
6977c478bd9Sstevel@tonic-gate  * to deposit the logout data.  t_flags is a register that has the
6987c478bd9Sstevel@tonic-gate  * trap-type/trap-level/CEEN info. This t_flags register may be used after
6997c478bd9Sstevel@tonic-gate  * the GET_AFSR_AFAR macro.
7007c478bd9Sstevel@tonic-gate  *
7017c478bd9Sstevel@tonic-gate  * The CPU logout operation will fail (r_val > 0) if the logout
7027c478bd9Sstevel@tonic-gate  * structure in question is already being used. Otherwise, the CPU
7037c478bd9Sstevel@tonic-gate  * logout operation will succeed (r_val = 0). For failures, r_val
7047c478bd9Sstevel@tonic-gate  * returns the busy count (# of times we tried using this CPU logout
7057c478bd9Sstevel@tonic-gate  * structure when it was busy.)
7067c478bd9Sstevel@tonic-gate  *
7077c478bd9Sstevel@tonic-gate  *   Register usage:
7087c478bd9Sstevel@tonic-gate  *	%asi:   Must be set to either ASI_MEM if the address in datap
7097c478bd9Sstevel@tonic-gate  *		is a physical address or to ASI_N if the address in
7107c478bd9Sstevel@tonic-gate  *		datap is a virtual address.
7117c478bd9Sstevel@tonic-gate  *	r_val:	This register is the return value which tells the
7127c478bd9Sstevel@tonic-gate  *		caller whether or not the LOGOUT operation was successful.
7137c478bd9Sstevel@tonic-gate  *		For failures, r_val returns the fail count (i.e. number of
7147c478bd9Sstevel@tonic-gate  *		times we have tried to use this logout structure when it was
7157c478bd9Sstevel@tonic-gate  *		already being used.
7167c478bd9Sstevel@tonic-gate  *	afar:	output: contains AFAR on exit
7177c478bd9Sstevel@tonic-gate  *	t_flags: input trap type info, may be used as scratch after stored
7187c478bd9Sstevel@tonic-gate  *		to cpu log out structure.
7197c478bd9Sstevel@tonic-gate  *	datap:	Points to log out data area.
7207c478bd9Sstevel@tonic-gate  *	scr1:	Scratch
7217c478bd9Sstevel@tonic-gate  *	scr2:	Scratch (may be r_val)
7227c478bd9Sstevel@tonic-gate  *	scr3:   Scratch (may be t_flags)
7237c478bd9Sstevel@tonic-gate  */
7247c478bd9Sstevel@tonic-gate #define	DO_TL1_CPU_LOGOUT(r_val, afar, t_flags, datap, scr1, scr2, scr3) \
7257c478bd9Sstevel@tonic-gate 	setx	LOGOUT_INVALID, scr2, scr1;				\
7267c478bd9Sstevel@tonic-gate 	ldxa	[datap + (CH_CLO_DATA + CH_CHD_AFAR)]%asi, scr2;	\
7277c478bd9Sstevel@tonic-gate 	cmp	scr2, scr1;						\
7287c478bd9Sstevel@tonic-gate 	bne	8f;							\
7297c478bd9Sstevel@tonic-gate 	  nop;								\
7307c478bd9Sstevel@tonic-gate 	stxa	t_flags, [datap + CH_CLO_FLAGS]%asi;			\
7317c478bd9Sstevel@tonic-gate 	GET_AFSR_AFAR(datap, afar, scr1, scr2);				\
7327c478bd9Sstevel@tonic-gate 	add	datap, CH_CLO_DATA + CH_CHD_EC_DATA, datap;		\
7337c478bd9Sstevel@tonic-gate 	GET_ECACHE_DTAGS(afar, datap, scr1, scr2, scr3);		\
7347c478bd9Sstevel@tonic-gate 	GET_DCACHE_DTAG(afar, datap, scr1, scr2, scr3);			\
7357c478bd9Sstevel@tonic-gate 	GET_ICACHE_DTAG(afar, datap, scr1, scr2, scr3);			\
7367c478bd9Sstevel@tonic-gate 	sub	datap, CH_CLO_DATA + CH_DIAG_DATA_SIZE, datap;		\
7377c478bd9Sstevel@tonic-gate 	GET_SHADOW_DATA(afar, datap, scr1, scr2, scr3);			\
7387c478bd9Sstevel@tonic-gate 	ldxa	[datap + (CH_CLO_DATA + CH_CHD_AFAR)]%asi, afar;	\
7397c478bd9Sstevel@tonic-gate 	set	0, r_val;	/* return value for success */		\
7407c478bd9Sstevel@tonic-gate 	ba	9f;							\
7417c478bd9Sstevel@tonic-gate 	  nop;								\
7427c478bd9Sstevel@tonic-gate 8:									\
7437c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_AFAR, afar;					\
7447c478bd9Sstevel@tonic-gate 	ldxa	[datap + CH_CLO_NEST_CNT]%asi, r_val;			\
7457c478bd9Sstevel@tonic-gate 	inc	r_val;		/* return value for failure */		\
7467c478bd9Sstevel@tonic-gate 	stxa	r_val, [datap + CH_CLO_NEST_CNT]%asi;			\
7477c478bd9Sstevel@tonic-gate 	membar	#Sync;							\
7487c478bd9Sstevel@tonic-gate 9:
7497c478bd9Sstevel@tonic-gate 
7507c478bd9Sstevel@tonic-gate /*
7517c478bd9Sstevel@tonic-gate  * Cheetah/(Cheetah+ Jaguar Panther)/Jalapeno Macro for capturing CPU
7527c478bd9Sstevel@tonic-gate  * logout data.  Uses DO_TL1_CPU_LOGOUT macro defined above, and sets
7537c478bd9Sstevel@tonic-gate  * up the expected data pointer in the scr1 register and sets the %asi
7547c478bd9Sstevel@tonic-gate  * register to ASI_N for kernel virtual addresses instead of ASI_MEM as
7557c478bd9Sstevel@tonic-gate  * is used at TL>0.
7567c478bd9Sstevel@tonic-gate  *
7577c478bd9Sstevel@tonic-gate  * The CPU logout operation will fail (r_val > 0) if the logout
7587c478bd9Sstevel@tonic-gate  * structure in question is already being used. Otherwise, the CPU
7597c478bd9Sstevel@tonic-gate  * logout operation will succeed (r_val = 0). For failures, r_val
7607c478bd9Sstevel@tonic-gate  * returns the busy count (# of times we tried using this CPU logout
7617c478bd9Sstevel@tonic-gate  * structure when it was busy.)
7627c478bd9Sstevel@tonic-gate  *
7637c478bd9Sstevel@tonic-gate  *   Register usage:
7647c478bd9Sstevel@tonic-gate  *	r_val:	This register is the return value which tells the
7657c478bd9Sstevel@tonic-gate  *		caller whether or not the LOGOUT operation was successful.
7667c478bd9Sstevel@tonic-gate  *		For failures, r_val returns the fail count (i.e. number of
7677c478bd9Sstevel@tonic-gate  *		times we have tried to use this logout structure when it was
7687c478bd9Sstevel@tonic-gate  *		already being used.
7697c478bd9Sstevel@tonic-gate  *	afar:	returns AFAR, used internally as afar value.
7707c478bd9Sstevel@tonic-gate  *		output: if the cpu_private struct has not been initialized,
7717c478bd9Sstevel@tonic-gate  *		        then we return the t_flags value listed below.
7727c478bd9Sstevel@tonic-gate  *	r_or_s:	input offset, either register or constant (symbol).  It's
7737c478bd9Sstevel@tonic-gate  *		OK for r_or_s to be a register as long as it's not scr1 or
7747c478bd9Sstevel@tonic-gate  *		scr3.
7757c478bd9Sstevel@tonic-gate  *	t_flags: input trap type info, may be used as scratch after stored
7767c478bd9Sstevel@tonic-gate  *		to cpu log out structure.
7777c478bd9Sstevel@tonic-gate  *	scr1:	Scratch, points to log out data area.
7787c478bd9Sstevel@tonic-gate  *	scr2:	Scratch (may be r_or_s)
7797c478bd9Sstevel@tonic-gate  *	scr3:	Scratch (may be r_val)
7807c478bd9Sstevel@tonic-gate  *	scr4:   Scratch (may be t_flags)
7817c478bd9Sstevel@tonic-gate  */
7827c478bd9Sstevel@tonic-gate #define	DO_CPU_LOGOUT(r_val, afar, r_or_s, t_flags, scr1, scr2, scr3, scr4) \
7837c478bd9Sstevel@tonic-gate 	GET_CPU_PRIVATE_PTR(r_or_s, scr1, scr3, 7f); /* can't use scr2/4 */ \
7847c478bd9Sstevel@tonic-gate 	wr	%g0, ASI_N, %asi;					\
7857c478bd9Sstevel@tonic-gate 	DO_TL1_CPU_LOGOUT(r_val, afar, t_flags, scr1, scr2, scr3, scr4)	\
7867c478bd9Sstevel@tonic-gate 	ba	6f;							\
7877c478bd9Sstevel@tonic-gate 	  nop;								\
7887c478bd9Sstevel@tonic-gate 7:									\
7897c478bd9Sstevel@tonic-gate 	mov	t_flags, afar;		/* depends on afar = %g2  */	\
7907c478bd9Sstevel@tonic-gate 	set	0, r_val;		/* success in this case.  */	\
7917c478bd9Sstevel@tonic-gate 6:
7927c478bd9Sstevel@tonic-gate 
7937c478bd9Sstevel@tonic-gate /*
7947c478bd9Sstevel@tonic-gate  * The P$ is flushed as a side effect of writing to the Primary
7957c478bd9Sstevel@tonic-gate  * or Secondary Context Register. After writing to a context
7967c478bd9Sstevel@tonic-gate  * register, every line of the P$ in the Valid state is invalidated,
7977c478bd9Sstevel@tonic-gate  * regardless of which context it belongs to.
7987c478bd9Sstevel@tonic-gate  * This routine simply touches the Primary context register by
7997c478bd9Sstevel@tonic-gate  * reading the current value and writing it back. The Primary
8007c478bd9Sstevel@tonic-gate  * context is not changed.
8017c478bd9Sstevel@tonic-gate  */
8027c478bd9Sstevel@tonic-gate #define	PCACHE_FLUSHALL(tmp1, tmp2, tmp3)				\
8037c478bd9Sstevel@tonic-gate 	sethi	%hi(FLUSH_ADDR), tmp1					;\
8047c478bd9Sstevel@tonic-gate 	set	MMU_PCONTEXT, tmp2					;\
8057c478bd9Sstevel@tonic-gate 	ldxa	[tmp2]ASI_DMMU, tmp3					;\
8067c478bd9Sstevel@tonic-gate 	stxa	tmp3, [tmp2]ASI_DMMU					;\
8077c478bd9Sstevel@tonic-gate 	flush	tmp1	/* See Cheetah PRM 8.10.2 */
8087c478bd9Sstevel@tonic-gate 
8097c478bd9Sstevel@tonic-gate /*
8107c478bd9Sstevel@tonic-gate  * Macro that flushes the entire Dcache.
8117c478bd9Sstevel@tonic-gate  *
8127c478bd9Sstevel@tonic-gate  * arg1 = dcache size
8137c478bd9Sstevel@tonic-gate  * arg2 = dcache linesize
8147c478bd9Sstevel@tonic-gate  */
8157c478bd9Sstevel@tonic-gate #define	CH_DCACHE_FLUSHALL(arg1, arg2, tmp1)				\
8167c478bd9Sstevel@tonic-gate 	sub	arg1, arg2, tmp1;					\
8177c478bd9Sstevel@tonic-gate 1:									\
8187c478bd9Sstevel@tonic-gate 	stxa	%g0, [tmp1]ASI_DC_TAG;					\
8197c478bd9Sstevel@tonic-gate 	membar	#Sync;							\
8207c478bd9Sstevel@tonic-gate 	cmp	%g0, tmp1;						\
8217c478bd9Sstevel@tonic-gate 	bne,pt	%icc, 1b;						\
8227c478bd9Sstevel@tonic-gate 	  sub	tmp1, arg2, tmp1;
8237c478bd9Sstevel@tonic-gate 
8247c478bd9Sstevel@tonic-gate /*
8257c478bd9Sstevel@tonic-gate  * Macro that flushes the entire Icache.
8267c478bd9Sstevel@tonic-gate  *
8277c478bd9Sstevel@tonic-gate  * Note that we cannot access ASI 0x67 (ASI_IC_TAG) with the Icache on,
8287c478bd9Sstevel@tonic-gate  * because accesses to ASI 0x67 interfere with Icache coherency.  We
8297c478bd9Sstevel@tonic-gate  * must make sure the Icache is off, then turn it back on after the entire
8307c478bd9Sstevel@tonic-gate  * cache has been invalidated.  If the Icache is originally off, we'll just
8317c478bd9Sstevel@tonic-gate  * clear the tags but not turn the Icache on.
8327c478bd9Sstevel@tonic-gate  *
8337c478bd9Sstevel@tonic-gate  * arg1 = icache size
8347c478bd9Sstevel@tonic-gate  * arg2 = icache linesize
8357c478bd9Sstevel@tonic-gate  */
8367c478bd9Sstevel@tonic-gate #define	CH_ICACHE_FLUSHALL(arg1, arg2, tmp1, tmp2)			\
8377c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_DCU, tmp2;					\
8387c478bd9Sstevel@tonic-gate 	andn	tmp2, DCU_IC, tmp1;					\
8397c478bd9Sstevel@tonic-gate 	stxa	tmp1, [%g0]ASI_DCU;					\
8407c478bd9Sstevel@tonic-gate 	flush	%g0;	/* flush required after changing the IC bit */	\
8417c478bd9Sstevel@tonic-gate 	sllx	arg2, 1, arg2;		/* arg2 = linesize * 2 */	\
8427c478bd9Sstevel@tonic-gate 	sllx	arg1, 1, arg1;		/* arg1 = size * 2 */		\
8437c478bd9Sstevel@tonic-gate 	sub	arg1, arg2, arg1;					\
8447c478bd9Sstevel@tonic-gate 	or	arg1, CH_ICTAG_LOWER, arg1;	/* "write" tag */	\
8457c478bd9Sstevel@tonic-gate 1:									\
8467c478bd9Sstevel@tonic-gate 	stxa	%g0, [arg1]ASI_IC_TAG;					\
8477c478bd9Sstevel@tonic-gate 	membar	#Sync;				/* Cheetah PRM 8.9.3 */	\
8487c478bd9Sstevel@tonic-gate 	cmp	arg1, CH_ICTAG_LOWER;					\
8497c478bd9Sstevel@tonic-gate 	bne,pt	%icc, 1b;						\
8507c478bd9Sstevel@tonic-gate 	  sub	arg1, arg2, arg1;					\
8517c478bd9Sstevel@tonic-gate 	stxa	tmp2, [%g0]ASI_DCU;					\
8527c478bd9Sstevel@tonic-gate 	flush	%g0;	/* flush required after changing the IC bit */
8537c478bd9Sstevel@tonic-gate 
8547c478bd9Sstevel@tonic-gate 
8557c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO)
8567c478bd9Sstevel@tonic-gate 
8577c478bd9Sstevel@tonic-gate /*
858*55fea89dSDan Cross  * ASI access to the L2 tag or L2 flush can hang the cpu when interacting
8597c478bd9Sstevel@tonic-gate  * with combinations of L2 snoops, victims and stores.
8607c478bd9Sstevel@tonic-gate  *
8617c478bd9Sstevel@tonic-gate  * A possible workaround is to surround each L2 ASI access with membars
8627c478bd9Sstevel@tonic-gate  * and make sure that the code is hitting in the Icache.  This requires
8637c478bd9Sstevel@tonic-gate  * aligning code sequence at E$ boundary and forcing I$ fetch by
8647c478bd9Sstevel@tonic-gate  * jumping to selected offsets so that we don't take any I$ misses
8657c478bd9Sstevel@tonic-gate  * during ASI access to the L2 tag or L2 flush.  This also requires
8667c478bd9Sstevel@tonic-gate  * making sure that we don't take any interrupts or traps (such as
8677c478bd9Sstevel@tonic-gate  * fast ECC trap, I$/D$ tag parity error) which can result in eviction
8687c478bd9Sstevel@tonic-gate  * of this code sequence from I$, thus causing a miss.
8697c478bd9Sstevel@tonic-gate  *
8707c478bd9Sstevel@tonic-gate  * Because of the complexity/risk, we have decided to do a partial fix
8717c478bd9Sstevel@tonic-gate  * of adding membar around each ASI access to the L2 tag or L2 flush.
8727c478bd9Sstevel@tonic-gate  */
8737c478bd9Sstevel@tonic-gate 
8747c478bd9Sstevel@tonic-gate #define	JP_EC_DIAG_ACCESS_MEMBAR	\
8757c478bd9Sstevel@tonic-gate 	membar	#Sync
8767c478bd9Sstevel@tonic-gate 
8777c478bd9Sstevel@tonic-gate /*
8787c478bd9Sstevel@tonic-gate  * Jalapeno version of macro that flushes the entire Ecache.
8797c478bd9Sstevel@tonic-gate  *
8807c478bd9Sstevel@tonic-gate  * Uses Jalapeno displacement flush feature of ASI_EC_DIAG.
8817c478bd9Sstevel@tonic-gate  *
8827c478bd9Sstevel@tonic-gate  * arg1 = ecache size
8837c478bd9Sstevel@tonic-gate  * arg2 = ecache linesize - not modified; can be an immediate constant.
8847c478bd9Sstevel@tonic-gate  */
8857c478bd9Sstevel@tonic-gate #define	ECACHE_FLUSHALL(arg1, arg2, tmp1, tmp2)	\
8867c478bd9Sstevel@tonic-gate 	CPU_INDEX(tmp1, tmp2);						\
8877c478bd9Sstevel@tonic-gate 	set	JP_ECACHE_IDX_DISP_FLUSH, tmp2;				\
8887c478bd9Sstevel@tonic-gate 	sllx	tmp1, JP_ECFLUSH_PORTID_SHIFT, tmp1;			\
8897c478bd9Sstevel@tonic-gate 	or	tmp1, tmp2, tmp1;					\
8907c478bd9Sstevel@tonic-gate 	srlx	arg1, JP_EC_TO_SET_SIZE_SHIFT, tmp2;			\
8917c478bd9Sstevel@tonic-gate 1:									\
8927c478bd9Sstevel@tonic-gate 	subcc	tmp2, arg2, tmp2;					\
8937c478bd9Sstevel@tonic-gate 	JP_EC_DIAG_ACCESS_MEMBAR;					\
8947c478bd9Sstevel@tonic-gate 	ldxa	[tmp1 + tmp2]ASI_EC_DIAG, %g0;				\
8957c478bd9Sstevel@tonic-gate 	JP_EC_DIAG_ACCESS_MEMBAR;					\
8967c478bd9Sstevel@tonic-gate 	bg,pt	%xcc, 1b;						\
8977c478bd9Sstevel@tonic-gate 	  nop;								\
8987c478bd9Sstevel@tonic-gate 	mov	1, tmp2;						\
8997c478bd9Sstevel@tonic-gate 	sllx	tmp2, JP_ECFLUSH_EC_WAY_SHIFT, tmp2;			\
9007c478bd9Sstevel@tonic-gate 	add	tmp1, tmp2, tmp1;					\
9017c478bd9Sstevel@tonic-gate 	mov	(JP_ECACHE_NWAY-1), tmp2;				\
9027c478bd9Sstevel@tonic-gate 	sllx	tmp2, JP_ECFLUSH_EC_WAY_SHIFT, tmp2;			\
9037c478bd9Sstevel@tonic-gate 	andcc	tmp1, tmp2, tmp2;					\
9047c478bd9Sstevel@tonic-gate 	bnz,pt	%xcc, 1b;						\
9057c478bd9Sstevel@tonic-gate 	  srlx	arg1, JP_EC_TO_SET_SIZE_SHIFT, tmp2
9067c478bd9Sstevel@tonic-gate 
9077c478bd9Sstevel@tonic-gate #else	/* JALAPENO || SERRANO */
9087c478bd9Sstevel@tonic-gate 
9097c478bd9Sstevel@tonic-gate /*
9107c478bd9Sstevel@tonic-gate  * Cheetah version of macro that flushes the entire Ecache.
9117c478bd9Sstevel@tonic-gate  *
9127c478bd9Sstevel@tonic-gate  *  Need to displacement flush 2x ecache size from Ecache flush area.
9137c478bd9Sstevel@tonic-gate  *
9147c478bd9Sstevel@tonic-gate  * arg1 = ecache size
9157c478bd9Sstevel@tonic-gate  * arg2 = ecache linesize
9167c478bd9Sstevel@tonic-gate  * arg3 = ecache flush address - for cheetah only
9177c478bd9Sstevel@tonic-gate  */
9187c478bd9Sstevel@tonic-gate #define	CH_ECACHE_FLUSHALL(arg1, arg2, arg3)				\
9197c478bd9Sstevel@tonic-gate 	sllx	arg1, 1, arg1;						\
9207c478bd9Sstevel@tonic-gate 1:									\
9217c478bd9Sstevel@tonic-gate 	subcc	arg1, arg2, arg1;					\
9227c478bd9Sstevel@tonic-gate 	bg,pt	%xcc, 1b;						\
9237c478bd9Sstevel@tonic-gate 	  ldxa	[arg1 + arg3]ASI_MEM, %g0;
9247c478bd9Sstevel@tonic-gate 
9257c478bd9Sstevel@tonic-gate /*
9267c478bd9Sstevel@tonic-gate  * Cheetah+ version of macro that flushes the entire Ecache.
9277c478bd9Sstevel@tonic-gate  *
9287c478bd9Sstevel@tonic-gate  * Uses the displacement flush feature.
9297c478bd9Sstevel@tonic-gate  *
9307c478bd9Sstevel@tonic-gate  * arg1 = ecache size
9317c478bd9Sstevel@tonic-gate  * arg2 = ecache linesize
9327c478bd9Sstevel@tonic-gate  * impl = CPU implementation as returned from GET_CPU_IMPL()
9337c478bd9Sstevel@tonic-gate  *        The value in this register is destroyed during execution
9347c478bd9Sstevel@tonic-gate  *        of the macro.
9357c478bd9Sstevel@tonic-gate  */
9367c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS)
9377c478bd9Sstevel@tonic-gate #define	CHP_ECACHE_FLUSHALL(arg1, arg2, impl)				\
9387c478bd9Sstevel@tonic-gate 	cmp	impl, PANTHER_IMPL;					\
9397c478bd9Sstevel@tonic-gate 	bne	%xcc, 1f;						\
9407c478bd9Sstevel@tonic-gate 	  nop;								\
9417c478bd9Sstevel@tonic-gate 	set	PN_L3_IDX_DISP_FLUSH, impl;				\
9427c478bd9Sstevel@tonic-gate 	b	2f;							\
9437c478bd9Sstevel@tonic-gate 	  nop;								\
9447c478bd9Sstevel@tonic-gate 1:									\
9457c478bd9Sstevel@tonic-gate 	set	CHP_ECACHE_IDX_DISP_FLUSH, impl;			\
9467c478bd9Sstevel@tonic-gate 2:									\
9477c478bd9Sstevel@tonic-gate 	subcc	arg1, arg2, arg1;					\
9487c478bd9Sstevel@tonic-gate 	bg,pt	%xcc, 2b;						\
9497c478bd9Sstevel@tonic-gate 	  ldxa	[arg1 + impl]ASI_EC_DIAG, %g0;
9507c478bd9Sstevel@tonic-gate #else	/* CHEETAH_PLUS */
9517c478bd9Sstevel@tonic-gate #define	CHP_ECACHE_FLUSHALL(arg1, arg2, impl)
9527c478bd9Sstevel@tonic-gate #endif	/* CHEETAH_PLUS */
9537c478bd9Sstevel@tonic-gate 
9547c478bd9Sstevel@tonic-gate /*
9557c478bd9Sstevel@tonic-gate  * Macro that flushes the entire Ecache.
9567c478bd9Sstevel@tonic-gate  *
9577c478bd9Sstevel@tonic-gate  * arg1 = ecache size
9587c478bd9Sstevel@tonic-gate  * arg2 = ecache linesize
9597c478bd9Sstevel@tonic-gate  * arg3 = ecache flush address - for cheetah only
9607c478bd9Sstevel@tonic-gate  */
9617c478bd9Sstevel@tonic-gate #define	ECACHE_FLUSHALL(arg1, arg2, arg3, tmp1)				\
9627c478bd9Sstevel@tonic-gate 	GET_CPU_IMPL(tmp1);						\
9637c478bd9Sstevel@tonic-gate 	cmp	tmp1, CHEETAH_IMPL;					\
9647c478bd9Sstevel@tonic-gate 	bne	%xcc, 2f;						\
9657c478bd9Sstevel@tonic-gate 	  nop;								\
9667c478bd9Sstevel@tonic-gate 	CH_ECACHE_FLUSHALL(arg1, arg2, arg3);				\
9677c478bd9Sstevel@tonic-gate 	ba	3f;							\
9687c478bd9Sstevel@tonic-gate 	  nop;								\
9697c478bd9Sstevel@tonic-gate 2:									\
9707c478bd9Sstevel@tonic-gate 	CHP_ECACHE_FLUSHALL(arg1, arg2, tmp1);				\
9717c478bd9Sstevel@tonic-gate 3:
9727c478bd9Sstevel@tonic-gate 
9737c478bd9Sstevel@tonic-gate #endif	/* JALAPENO || SERRANO */
9747c478bd9Sstevel@tonic-gate 
9757c478bd9Sstevel@tonic-gate /*
9767c478bd9Sstevel@tonic-gate  * Macro that flushes the Panther L2 cache.
9777c478bd9Sstevel@tonic-gate  */
9787c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS)
9797c478bd9Sstevel@tonic-gate #define	PN_L2_FLUSHALL(scr1, scr2, scr3)				\
9807c478bd9Sstevel@tonic-gate 	GET_CPU_IMPL(scr3);						\
9817c478bd9Sstevel@tonic-gate 	cmp	scr3, PANTHER_IMPL;					\
9827c478bd9Sstevel@tonic-gate 	bne	%xcc, 2f;						\
9837c478bd9Sstevel@tonic-gate 	  nop;								\
9847c478bd9Sstevel@tonic-gate 	set	PN_L2_SIZE, scr1;					\
9857c478bd9Sstevel@tonic-gate 	set	PN_L2_LINESIZE, scr2;					\
9867c478bd9Sstevel@tonic-gate 	set	PN_L2_IDX_DISP_FLUSH, scr3;				\
9877c478bd9Sstevel@tonic-gate 1:									\
9887c478bd9Sstevel@tonic-gate 	subcc	scr1, scr2, scr1;					\
9897c478bd9Sstevel@tonic-gate 	bg,pt	%xcc, 1b;						\
9907c478bd9Sstevel@tonic-gate 	  ldxa	[scr1 + scr3]ASI_L2_TAG, %g0;				\
9917c478bd9Sstevel@tonic-gate 2:
9927c478bd9Sstevel@tonic-gate #else	/* CHEETAH_PLUS */
9937c478bd9Sstevel@tonic-gate #define	PN_L2_FLUSHALL(scr1, scr2, scr3)
9947c478bd9Sstevel@tonic-gate #endif	/* CHEETAH_PLUS */
9957c478bd9Sstevel@tonic-gate 
9967c478bd9Sstevel@tonic-gate /*
9977c478bd9Sstevel@tonic-gate  * Given a VA and page size (page size as encoded in ASI_MMU_TAG_ACCESS_EXT),
9987c478bd9Sstevel@tonic-gate  * this macro returns the TLB index for that mapping based on a 512 entry
9997c478bd9Sstevel@tonic-gate  * (2-way set associative) TLB. Aaside from the 16 entry fully associative
10007c478bd9Sstevel@tonic-gate  * TLBs, all TLBs in Panther are 512 entry, 2-way set associative.
1001*55fea89dSDan Cross  *
10027c478bd9Sstevel@tonic-gate  * To find the index, we shift the VA right by 13 + (3 * pg_sz) and then
10037c478bd9Sstevel@tonic-gate  * mask out all but the lower 8 bits because:
10047c478bd9Sstevel@tonic-gate  *
10057c478bd9Sstevel@tonic-gate  *    ASI_[D|I]MMU_TAG_ACCESS_EXT.PgSz = 0 for   8K
10067c478bd9Sstevel@tonic-gate  *    ASI_[D|I]MMU_TAG_ACCESS_EXT.PgSz = 1 for  64K
10077c478bd9Sstevel@tonic-gate  *    ASI_[D|I]MMU_TAG_ACCESS_EXT.PgSz = 2 for 512K
10087c478bd9Sstevel@tonic-gate  *    ASI_[D|I]MMU_TAG_ACCESS_EXT.PgSz = 3 for   4M
10097c478bd9Sstevel@tonic-gate  *    ASI_[D|I]MMU_TAG_ACCESS_EXT.PgSz = 4 for  32M
10107c478bd9Sstevel@tonic-gate  *    ASI_[D|I]MMU_TAG_ACCESS_EXT.PgSz = 5 for 256M
10117c478bd9Sstevel@tonic-gate  *
10127c478bd9Sstevel@tonic-gate  * and
10137c478bd9Sstevel@tonic-gate  *
10147c478bd9Sstevel@tonic-gate  *    array index for   8K pages = VA[20:13]
10157c478bd9Sstevel@tonic-gate  *    array index for  64K pages = VA[23:16]
10167c478bd9Sstevel@tonic-gate  *    array index for 512K pages = VA[26:19]
10177c478bd9Sstevel@tonic-gate  *    array index for   4M pages = VA[29:22]
10187c478bd9Sstevel@tonic-gate  *    array index for  32M pages = VA[32:25]
10197c478bd9Sstevel@tonic-gate  *    array index for 256M pages = VA[35:28]
10207c478bd9Sstevel@tonic-gate  *
10217c478bd9Sstevel@tonic-gate  * Inputs:
10227c478bd9Sstevel@tonic-gate  *
10237c478bd9Sstevel@tonic-gate  *    va	- Register.
10247c478bd9Sstevel@tonic-gate  *		  Input: Virtual address in which we are interested.
10257c478bd9Sstevel@tonic-gate  *		  Output: TLB index value.
10267c478bd9Sstevel@tonic-gate  *    pg_sz	- Register. Page Size of the TLB in question as encoded
10277c478bd9Sstevel@tonic-gate  *		  in the ASI_[D|I]MMU_TAG_ACCESS_EXT register.
10287c478bd9Sstevel@tonic-gate  */
10297c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS)
10307c478bd9Sstevel@tonic-gate #define	PN_GET_TLB_INDEX(va, pg_sz)					\
10317c478bd9Sstevel@tonic-gate 	srlx	va, 13, va;	/* first shift the 13 bits and then */	\
10327c478bd9Sstevel@tonic-gate 	srlx	va, pg_sz, va;	/* shift by pg_sz three times. */	\
10337c478bd9Sstevel@tonic-gate 	srlx	va, pg_sz, va;						\
10347c478bd9Sstevel@tonic-gate 	srlx	va, pg_sz, va;						\
10357c478bd9Sstevel@tonic-gate 	and	va, 0xff, va;	/* mask out all but the lower 8 bits */
10367c478bd9Sstevel@tonic-gate #endif	/* CHEETAH_PLUS */
10377c478bd9Sstevel@tonic-gate 
10387c478bd9Sstevel@tonic-gate /*
10397c478bd9Sstevel@tonic-gate  * The following macros are for error traps at TL>0.
10407c478bd9Sstevel@tonic-gate  * The issue with error traps at TL>0 is that there are no safely
10417c478bd9Sstevel@tonic-gate  * available global registers.  So we use the trick of generating a
10427c478bd9Sstevel@tonic-gate  * software trap, then using the %tpc, %tnpc and %tstate registers to
10437c478bd9Sstevel@tonic-gate  * temporarily save the values of %g1 and %g2.
10447c478bd9Sstevel@tonic-gate  */
10457c478bd9Sstevel@tonic-gate 
10467c478bd9Sstevel@tonic-gate /*
10477c478bd9Sstevel@tonic-gate  * Macro to generate 8-instruction trap table entry for TL>0 trap handlers.
10487c478bd9Sstevel@tonic-gate  * Does the following steps:
10497c478bd9Sstevel@tonic-gate  *	1. membar #Sync - required for USIII family errors.
10507c478bd9Sstevel@tonic-gate  *	2. Specified software trap.
10517c478bd9Sstevel@tonic-gate  * NB: Must be 8 instructions or less to fit in trap table and code must
10527c478bd9Sstevel@tonic-gate  *     be relocatable.
10537c478bd9Sstevel@tonic-gate  */
10547c478bd9Sstevel@tonic-gate #define	CH_ERR_TL1_TRAPENTRY(trapno)		\
10557c478bd9Sstevel@tonic-gate 	membar	#Sync;				\
10567c478bd9Sstevel@tonic-gate 	ta	trapno;				\
10577c478bd9Sstevel@tonic-gate 	nop; nop; nop; nop; nop; nop
10587c478bd9Sstevel@tonic-gate 
10597c478bd9Sstevel@tonic-gate /*
10607c478bd9Sstevel@tonic-gate  * Macro to generate 8-instruction trap table entry for TL>0 software trap.
10617c478bd9Sstevel@tonic-gate  * We save the values of %g1 and %g2 in %tpc, %tnpc and %tstate (since
10627c478bd9Sstevel@tonic-gate  * the low-order two bits of %tpc/%tnpc are reserved and read as zero,
10637c478bd9Sstevel@tonic-gate  * we need to put the low-order two bits of %g1 and %g2 in %tstate).
10647c478bd9Sstevel@tonic-gate  * Note that %tstate has a reserved hole from bits 3-7, so we put the
10657c478bd9Sstevel@tonic-gate  * low-order two bits of %g1 in bits 0-1 and the low-order two bits of
10667c478bd9Sstevel@tonic-gate  * %g2 in bits 10-11 (insuring bits 8-9 are zero for use by the D$/I$
10677c478bd9Sstevel@tonic-gate  * state bits).  Note that we must do a jmp instruction, since this
10687c478bd9Sstevel@tonic-gate  * is moved into the trap table entry.
10697c478bd9Sstevel@tonic-gate  * NB: Must be 8 instructions or less to fit in trap table and code must
10707c478bd9Sstevel@tonic-gate  *     be relocatable.
10717c478bd9Sstevel@tonic-gate  */
10727c478bd9Sstevel@tonic-gate #define	CH_ERR_TL1_SWTRAPENTRY(label)		\
10737c478bd9Sstevel@tonic-gate 	wrpr	%g1, %tpc;			\
10747c478bd9Sstevel@tonic-gate 	and	%g1, 3, %g1;			\
10757c478bd9Sstevel@tonic-gate 	wrpr	%g2, %tnpc;			\
10767c478bd9Sstevel@tonic-gate 	sllx	%g2, CH_ERR_G2_TO_TSTATE_SHFT, %g2; \
10777c478bd9Sstevel@tonic-gate 	or	%g1, %g2, %g2;			\
10787c478bd9Sstevel@tonic-gate 	sethi	%hi(label), %g1;		\
10797c478bd9Sstevel@tonic-gate 	jmp	%g1+%lo(label);			\
10807c478bd9Sstevel@tonic-gate 	  wrpr	%g2, %tstate
10817c478bd9Sstevel@tonic-gate 
10827c478bd9Sstevel@tonic-gate /*
10837c478bd9Sstevel@tonic-gate  * Macro to get ptr to ch_err_tl1_data.
10847c478bd9Sstevel@tonic-gate  * reg1 will either point to a physaddr with ASI_MEM in %asi OR it
10857c478bd9Sstevel@tonic-gate  * will point to a kernel nucleus virtual address with ASI_N in %asi.
10867c478bd9Sstevel@tonic-gate  * This allows us to:
10877c478bd9Sstevel@tonic-gate  *   1. Avoid getting MMU misses.  We may have gotten the original
10887c478bd9Sstevel@tonic-gate  *	Fast ECC error in an MMU handler and if we get an MMU trap
10897c478bd9Sstevel@tonic-gate  *	in the TL>0 handlers, we'll scribble on the MMU regs.
10907c478bd9Sstevel@tonic-gate  *   2. Allows us to use the same code in the TL>0 handlers whether
10917c478bd9Sstevel@tonic-gate  *	we're accessing kernel nucleus virtual addresses or physical
10927c478bd9Sstevel@tonic-gate  *	addresses.
10937c478bd9Sstevel@tonic-gate  * pseudo-code:
10947c478bd9Sstevel@tonic-gate  *	reg1 <- ch_err_tl1_paddrs[CPUID];
10957c478bd9Sstevel@tonic-gate  *	if (reg1 == NULL) {
10967c478bd9Sstevel@tonic-gate  *		reg1 <- &ch_err_tl1_data
10977c478bd9Sstevel@tonic-gate  *		%asi <- ASI_N
10987c478bd9Sstevel@tonic-gate  *	} else {
10997c478bd9Sstevel@tonic-gate  *		reg1 <- reg1 + offset +
11007c478bd9Sstevel@tonic-gate  *		    sizeof (ch_err_tl1_data) * (%tl - 3)
11017c478bd9Sstevel@tonic-gate  *		%asi <- ASI_MEM
11027c478bd9Sstevel@tonic-gate  *	}
11037c478bd9Sstevel@tonic-gate  */
11047c478bd9Sstevel@tonic-gate #define	GET_CH_ERR_TL1_PTR(reg1, reg2, offset)	\
11057c478bd9Sstevel@tonic-gate 	CPU_INDEX(reg1, reg2);			\
11067c478bd9Sstevel@tonic-gate 	sllx	reg1, 3, reg1;			\
11077c478bd9Sstevel@tonic-gate 	set	ch_err_tl1_paddrs, reg2;	\
11087c478bd9Sstevel@tonic-gate 	ldx	[reg1+reg2], reg1;		\
11097c478bd9Sstevel@tonic-gate 	brnz	reg1, 1f;			\
11107c478bd9Sstevel@tonic-gate 	add	reg1, offset, reg1;		\
11117c478bd9Sstevel@tonic-gate 	set	ch_err_tl1_data, reg1;		\
11127c478bd9Sstevel@tonic-gate 	ba	2f;				\
11137c478bd9Sstevel@tonic-gate 	wr	%g0, ASI_N, %asi;		\
11147c478bd9Sstevel@tonic-gate 1:	rdpr	%tl, reg2;			\
11157c478bd9Sstevel@tonic-gate 	sub	reg2, 3, reg2;			\
11167c478bd9Sstevel@tonic-gate 	mulx	reg2, CH_ERR_TL1_DATA_SIZE, reg2;	\
11177c478bd9Sstevel@tonic-gate 	add	reg1, reg2, reg1;		\
11187c478bd9Sstevel@tonic-gate 	wr	%g0, ASI_MEM, %asi;		\
11197c478bd9Sstevel@tonic-gate 2:
11207c478bd9Sstevel@tonic-gate 
11217c478bd9Sstevel@tonic-gate /*
11227c478bd9Sstevel@tonic-gate  * Macro to generate entry code for TL>0 error handlers.
11237c478bd9Sstevel@tonic-gate  * At the end of this macro, %g1 will point to the ch_err_tl1_data
11247c478bd9Sstevel@tonic-gate  * structure and %g2 will have the original flags in the ch_err_tl1_data
11257c478bd9Sstevel@tonic-gate  * structure and %g5 will have the value of %tstate where the Fast ECC
11267c478bd9Sstevel@tonic-gate  * routines will save the state of the D$ in Bit2 CH_ERR_TSTATE_DC_ON.
11277c478bd9Sstevel@tonic-gate  * All %g registers except for %g1, %g2 and %g5 will be available after
11287c478bd9Sstevel@tonic-gate  * this macro.
11297c478bd9Sstevel@tonic-gate  * Does the following steps:
11307c478bd9Sstevel@tonic-gate  *   1. Compute physical address of per-cpu/per-tl save area using
11317c478bd9Sstevel@tonic-gate  *	only %g1+%g2 (which we've saved in %tpc, %tnpc, %tstate)
11327c478bd9Sstevel@tonic-gate  *	leaving address in %g1 and updating the %asi register.
11337c478bd9Sstevel@tonic-gate  *	If there is no data area available, we branch to label.
11347c478bd9Sstevel@tonic-gate  *   2. Save %g3-%g7 in save area.
11357c478bd9Sstevel@tonic-gate  *   3. Save %tpc->%g3, %tnpc->%g4, %tstate->%g5, which contain
11367c478bd9Sstevel@tonic-gate  *	original %g1+%g2 values (because we're going to change %tl).
11377c478bd9Sstevel@tonic-gate  *   4. set %tl <- %tl - 1.  We do this ASAP to make window of
11387c478bd9Sstevel@tonic-gate  *	running at %tl+1 as small as possible.
11397c478bd9Sstevel@tonic-gate  *   5. Reconstitute %g1+%g2 from %tpc (%g3), %tnpc (%g4),
11407c478bd9Sstevel@tonic-gate  *	%tstate (%g5) and save in save area, carefully preserving %g5
11417c478bd9Sstevel@tonic-gate  *	because it has the CH_ERR_TSTATE_DC_ON value.
11427c478bd9Sstevel@tonic-gate  *   6. Load existing ch_err_tl1_data flags in %g2
11437c478bd9Sstevel@tonic-gate  *   7. Compute the new flags
11447c478bd9Sstevel@tonic-gate  *   8. If %g2 is non-zero (the structure was busy), shift the new
11457c478bd9Sstevel@tonic-gate  *	flags by CH_ERR_ME_SHIFT and or them with the old flags.
11467c478bd9Sstevel@tonic-gate  *   9. Store the updated flags into ch_err_tl1_data flags.
11477c478bd9Sstevel@tonic-gate  *   10. If %g2 is non-zero, read the %tpc and store it in
11487c478bd9Sstevel@tonic-gate  *	ch_err_tl1_data.
11497c478bd9Sstevel@tonic-gate  */
11507c478bd9Sstevel@tonic-gate #define	CH_ERR_TL1_ENTER(flags)			\
11517c478bd9Sstevel@tonic-gate 	GET_CH_ERR_TL1_PTR(%g1, %g2, CHPR_TL1_ERR_DATA);	\
11527c478bd9Sstevel@tonic-gate 	stxa	%g3, [%g1 + CH_ERR_TL1_G3]%asi;	\
11537c478bd9Sstevel@tonic-gate 	stxa	%g4, [%g1 + CH_ERR_TL1_G4]%asi;	\
11547c478bd9Sstevel@tonic-gate 	stxa	%g5, [%g1 + CH_ERR_TL1_G5]%asi;	\
11557c478bd9Sstevel@tonic-gate 	stxa	%g6, [%g1 + CH_ERR_TL1_G6]%asi;	\
11567c478bd9Sstevel@tonic-gate 	stxa	%g7, [%g1 + CH_ERR_TL1_G7]%asi;	\
11577c478bd9Sstevel@tonic-gate 	rdpr	%tpc, %g3;			\
11587c478bd9Sstevel@tonic-gate 	rdpr	%tnpc, %g4;			\
11597c478bd9Sstevel@tonic-gate 	rdpr	%tstate, %g5;			\
11607c478bd9Sstevel@tonic-gate 	rdpr	%tl, %g6;			\
11617c478bd9Sstevel@tonic-gate 	sub	%g6, 1, %g6;			\
11627c478bd9Sstevel@tonic-gate 	wrpr	%g6, %tl;			\
11637c478bd9Sstevel@tonic-gate 	and	%g5, 3, %g6;			\
11647c478bd9Sstevel@tonic-gate 	andn	%g3, 3, %g3;			\
11657c478bd9Sstevel@tonic-gate 	or	%g3, %g6, %g3;			\
11667c478bd9Sstevel@tonic-gate 	stxa	%g3, [%g1 + CH_ERR_TL1_G1]%asi;	\
11677c478bd9Sstevel@tonic-gate 	srlx	%g5, CH_ERR_G2_TO_TSTATE_SHFT, %g6;	\
11687c478bd9Sstevel@tonic-gate 	and	%g6, 3, %g6;			\
11697c478bd9Sstevel@tonic-gate 	andn	%g4, 3, %g4;			\
11707c478bd9Sstevel@tonic-gate 	or	%g6, %g4, %g4;			\
11717c478bd9Sstevel@tonic-gate 	stxa	%g4, [%g1 + CH_ERR_TL1_G2]%asi;	\
11727c478bd9Sstevel@tonic-gate 	ldxa	[%g1 + CH_ERR_TL1_FLAGS]%asi, %g2;	\
11737c478bd9Sstevel@tonic-gate 	set	flags | CH_ERR_TL, %g3;		\
11747c478bd9Sstevel@tonic-gate 	brz	%g2, 9f;			\
11757c478bd9Sstevel@tonic-gate 	sllx	%g3, CH_ERR_ME_SHIFT, %g4;	\
11767c478bd9Sstevel@tonic-gate 	or	%g2, %g4, %g3;			\
11777c478bd9Sstevel@tonic-gate 9:	stxa	%g3, [%g1 + CH_ERR_TL1_FLAGS]%asi;	\
11787c478bd9Sstevel@tonic-gate 	brnz	%g2, 8f;			\
11797c478bd9Sstevel@tonic-gate 	rdpr	%tpc, %g4;			\
11807c478bd9Sstevel@tonic-gate 	stxa	%g4, [%g1 + CH_ERR_TL1_TPC]%asi;	\
11817c478bd9Sstevel@tonic-gate 8:
11827c478bd9Sstevel@tonic-gate 
11837c478bd9Sstevel@tonic-gate /*
11847c478bd9Sstevel@tonic-gate  * Turns off D$/I$ and saves the state of DCU_DC+DCU_IC in %tstate Bits 8+9
11857c478bd9Sstevel@tonic-gate  * (CH_ERR_TSTATE_DC_ON/CH_ERR_TSTATE_IC_ON).  This is invoked on Fast ECC
11867c478bd9Sstevel@tonic-gate  * at TL>0 handlers because the D$ may have corrupted data and we need to
11877c478bd9Sstevel@tonic-gate  * turn off the I$ to allow for diagnostic accesses.  We then invoke
11887c478bd9Sstevel@tonic-gate  * the normal entry macro and after it is done we save the values of
11897c478bd9Sstevel@tonic-gate  * the original D$/I$ state, which is in %g5 bits CH_ERR_TSTATE_DC_ON/
11907c478bd9Sstevel@tonic-gate  * CH_ERR_TSTATE_IC_ON in ch_err_tl1_tmp.
11917c478bd9Sstevel@tonic-gate  */
11927c478bd9Sstevel@tonic-gate #define	CH_ERR_TL1_FECC_ENTER			\
11937c478bd9Sstevel@tonic-gate 	ldxa	[%g0]ASI_DCU, %g1;		\
11947c478bd9Sstevel@tonic-gate 	andn	%g1, DCU_DC + DCU_IC, %g2;	\
11957c478bd9Sstevel@tonic-gate 	stxa	%g2, [%g0]ASI_DCU;		\
11967c478bd9Sstevel@tonic-gate 	flush	%g0;	/* DCU_IC need flush */	\
11977c478bd9Sstevel@tonic-gate 	rdpr	%tstate, %g2;			\
11987c478bd9Sstevel@tonic-gate 	and	%g1, DCU_DC + DCU_IC, %g1;	\
11997c478bd9Sstevel@tonic-gate 	sllx	%g1, CH_ERR_DCU_TO_TSTATE_SHFT, %g1;	\
12007c478bd9Sstevel@tonic-gate 	or	%g1, %g2, %g2;			\
12017c478bd9Sstevel@tonic-gate 	wrpr	%g2, %tstate;			\
12027c478bd9Sstevel@tonic-gate 	CH_ERR_TL1_ENTER(CH_ERR_FECC);		\
12037c478bd9Sstevel@tonic-gate 	and	%g5, CH_ERR_TSTATE_DC_ON + CH_ERR_TSTATE_IC_ON, %g5;	\
12047c478bd9Sstevel@tonic-gate 	stxa	%g5, [%g1 + CH_ERR_TL1_TMP]%asi
12057c478bd9Sstevel@tonic-gate 
12067c478bd9Sstevel@tonic-gate /*
12077c478bd9Sstevel@tonic-gate  * Macro to generate exit code for TL>0 error handlers.
12087c478bd9Sstevel@tonic-gate  * We fall into this macro if we've successfully logged the error in
12097c478bd9Sstevel@tonic-gate  * the ch_err_tl1_data structure and want the PIL15 softint to pick
12107c478bd9Sstevel@tonic-gate  * it up and log it.
12117c478bd9Sstevel@tonic-gate  * Does the following steps:
12127c478bd9Sstevel@tonic-gate  *   1.	Set pending flag for this cpu in ch_err_tl1_pending.
12137c478bd9Sstevel@tonic-gate  *   2.	Write %set_softint with (1<<pil) to cause a pil level trap
12147c478bd9Sstevel@tonic-gate  *   3.	Restore registers from ch_err_tl1_data, which is pointed to
12157c478bd9Sstevel@tonic-gate  *	by %g1, last register to restore is %g1 since it's pointing
12167c478bd9Sstevel@tonic-gate  *	to the save area.
12177c478bd9Sstevel@tonic-gate  *   4. Execute retry
12187c478bd9Sstevel@tonic-gate  */
12197c478bd9Sstevel@tonic-gate #define	CH_ERR_TL1_EXIT				\
12207c478bd9Sstevel@tonic-gate 	CPU_INDEX(%g2, %g3);			\
12217c478bd9Sstevel@tonic-gate 	set	ch_err_tl1_pending, %g3;	\
12227c478bd9Sstevel@tonic-gate 	set	-1, %g4;			\
12237c478bd9Sstevel@tonic-gate 	stb	%g4, [%g2 + %g3];		\
12247c478bd9Sstevel@tonic-gate 	mov	1, %g2;				\
12257c478bd9Sstevel@tonic-gate 	sll	%g2, PIL_15, %g2;		\
12267c478bd9Sstevel@tonic-gate 	wr	%g2, SET_SOFTINT;		\
12277c478bd9Sstevel@tonic-gate 	ldxa	[%g1 + CH_ERR_TL1_G7]%asi, %g7;	\
12287c478bd9Sstevel@tonic-gate 	ldxa	[%g1 + CH_ERR_TL1_G6]%asi, %g6;	\
12297c478bd9Sstevel@tonic-gate 	ldxa	[%g1 + CH_ERR_TL1_G5]%asi, %g5;	\
12307c478bd9Sstevel@tonic-gate 	ldxa	[%g1 + CH_ERR_TL1_G4]%asi, %g4;	\
12317c478bd9Sstevel@tonic-gate 	ldxa	[%g1 + CH_ERR_TL1_G3]%asi, %g3;	\
12327c478bd9Sstevel@tonic-gate 	ldxa	[%g1 + CH_ERR_TL1_G2]%asi, %g2;	\
12337c478bd9Sstevel@tonic-gate 	ldxa	[%g1 + CH_ERR_TL1_G1]%asi, %g1;	\
12347c478bd9Sstevel@tonic-gate 	retry
12357c478bd9Sstevel@tonic-gate 
12367c478bd9Sstevel@tonic-gate /*
12377c478bd9Sstevel@tonic-gate  * Generates unrecoverable error label for TL>0 handlers.
12387c478bd9Sstevel@tonic-gate  * At label (Unrecoverable error routine)
12397c478bd9Sstevel@tonic-gate  *   1. Sets flags in ch_err_tl1_data and leaves in %g2 (first
12407c478bd9Sstevel@tonic-gate  *	argument to cpu_tl1_err_panic).
12417c478bd9Sstevel@tonic-gate  *   2.	Call cpu_tl1_err_panic via systrap at PIL 15
12427c478bd9Sstevel@tonic-gate  */
12437c478bd9Sstevel@tonic-gate #define	CH_ERR_TL1_PANIC_EXIT(label)		\
12447c478bd9Sstevel@tonic-gate label:	ldxa	[%g1 + CH_ERR_TL1_FLAGS]%asi, %g2;	\
12457c478bd9Sstevel@tonic-gate 	or	%g2, CH_ERR_TL | CH_ERR_PANIC, %g2;	\
12467c478bd9Sstevel@tonic-gate 	stxa	%g2, [%g1 + CH_ERR_TL1_FLAGS]%asi;	\
12477c478bd9Sstevel@tonic-gate 	set	cpu_tl1_err_panic, %g1;		\
12487c478bd9Sstevel@tonic-gate 	ba	sys_trap;			\
12497c478bd9Sstevel@tonic-gate 	  mov	PIL_15, %g4
12507c478bd9Sstevel@tonic-gate 
12517c478bd9Sstevel@tonic-gate 
12527c478bd9Sstevel@tonic-gate 
12537c478bd9Sstevel@tonic-gate /* END CSTYLED */
12547c478bd9Sstevel@tonic-gate #endif	/* _ASM */
12557c478bd9Sstevel@tonic-gate 
12567c478bd9Sstevel@tonic-gate #ifdef	__cplusplus
12577c478bd9Sstevel@tonic-gate }
12587c478bd9Sstevel@tonic-gate #endif
12597c478bd9Sstevel@tonic-gate 
12607c478bd9Sstevel@tonic-gate #endif /* _CHEETAHASM_H */
1261