1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#include "assym.h"
27
28#include <sys/asm_linkage.h>
29#include <sys/mmu.h>
30#include <vm/hat_sfmmu.h>
31#include <sys/machparam.h>
32#include <sys/machcpuvar.h>
33#include <sys/machthread.h>
34#include <sys/privregs.h>
35#include <sys/asm_linkage.h>
36#include <sys/machasi.h>
37#include <sys/trap.h>
38#include <sys/spitregs.h>
39#include <sys/xc_impl.h>
40#include <sys/intreg.h>
41#include <sys/async.h>
42
43#ifdef TRAPTRACE
44#include <sys/traptrace.h>
45#endif /* TRAPTRACE */
46
47/* BEGIN CSTYLED */
48#define	DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3)			\
49	ldxa	[%g0]ASI_LSU, tmp1					;\
50	btst	LSU_DC, tmp1		/* is dcache enabled? */	;\
51	bz,pn	%icc, 1f						;\
52	sethi	%hi(dcache_linesize), tmp1				;\
53	ld	[tmp1 + %lo(dcache_linesize)], tmp1			;\
54	sethi	%hi(dflush_type), tmp2					;\
55	ld	[tmp2 + %lo(dflush_type)], tmp2				;\
56	cmp	tmp2, FLUSHPAGE_TYPE					;\
57	be,pt	%icc, 2f						;\
58	sllx	arg1, SF_DC_VBIT_SHIFT, arg1	/* tag to compare */	;\
59	sethi	%hi(dcache_size), tmp3					;\
60	ld	[tmp3 + %lo(dcache_size)], tmp3				;\
61	cmp	tmp2, FLUSHMATCH_TYPE					;\
62	be,pt	%icc, 3f						;\
63	nop								;\
64	/*								\
65	 * flushtype = FLUSHALL_TYPE, flush the whole thing		\
66	 * tmp3 = cache size						\
67	 * tmp1 = cache line size					\
68	 */								\
69	sub	tmp3, tmp1, tmp2					;\
704:									\
71	stxa	%g0, [tmp2]ASI_DC_TAG					;\
72	membar	#Sync							;\
73	cmp	%g0, tmp2						;\
74	bne,pt	%icc, 4b						;\
75	sub	tmp2, tmp1, tmp2					;\
76	ba,pt	%icc, 1f						;\
77	nop								;\
78	/*								\
79	 * flushtype = FLUSHPAGE_TYPE					\
80	 * arg1 = tag to compare against				\
81	 * arg2 = virtual color						\
82	 * tmp1 = cache line size					\
83	 * tmp2 = tag from cache					\
84	 * tmp3 = counter						\
85	 */								\
862:									\
87	set	MMU_PAGESIZE, tmp3					;\
88	sllx	arg2, MMU_PAGESHIFT, arg2  /* color to dcache page */	;\
89	sub	tmp3, tmp1, tmp3					;\
904:									\
91	ldxa	[arg2 + tmp3]ASI_DC_TAG, tmp2	/* read tag */		;\
92	btst	SF_DC_VBIT_MASK, tmp2					;\
93	bz,pn	%icc, 5f	  /* branch if no valid sub-blocks */	;\
94	andn	tmp2, SF_DC_VBIT_MASK, tmp2	/* clear out v bits */	;\
95	cmp	tmp2, arg1						;\
96	bne,pn	%icc, 5f			/* br if tag miss */	;\
97	nop								;\
98	stxa	%g0, [arg2 + tmp3]ASI_DC_TAG				;\
99	membar	#Sync							;\
1005:									\
101	cmp	%g0, tmp3						;\
102	bnz,pt	%icc, 4b		/* branch if not done */	;\
103	sub	tmp3, tmp1, tmp3					;\
104	ba,pt	%icc, 1f						;\
105	nop								;\
106	/*								\
107	 * flushtype = FLUSHMATCH_TYPE					\
108	 * arg1 = tag to compare against				\
109	 * tmp1 = cache line size					\
110	 * tmp3 = cache size						\
111	 * arg2 = counter						\
112	 * tmp2 = cache tag						\
113	 */								\
1143:									\
115	sub	tmp3, tmp1, arg2					;\
1164:									\
117	ldxa	[arg2]ASI_DC_TAG, tmp2		/* read tag */		;\
118	btst	SF_DC_VBIT_MASK, tmp2					;\
119	bz,pn	%icc, 5f		/* br if no valid sub-blocks */	;\
120	andn	tmp2, SF_DC_VBIT_MASK, tmp2	/* clear out v bits */	;\
121	cmp	tmp2, arg1						;\
122	bne,pn	%icc, 5f		/* branch if tag miss */	;\
123	nop								;\
124	stxa	%g0, [arg2]ASI_DC_TAG					;\
125	membar	#Sync							;\
1265:									\
127	cmp	%g0, arg2						;\
128	bne,pt	%icc, 4b		/* branch if not done */	;\
129	sub	arg2, tmp1, arg2					;\
1301:
131
132/*
133 * macro that flushes the entire dcache color
134 */
135#define	DCACHE_FLUSHCOLOR(arg, tmp1, tmp2)				\
136	ldxa	[%g0]ASI_LSU, tmp1;					\
137	btst	LSU_DC, tmp1;		/* is dcache enabled? */	\
138	bz,pn	%icc, 1f;						\
139	sethi	%hi(dcache_linesize), tmp1;				\
140	ld	[tmp1 + %lo(dcache_linesize)], tmp1;			\
141	set	MMU_PAGESIZE, tmp2;					\
142	/*								\
143	 * arg = virtual color						\
144	 * tmp2 = page size						\
145	 * tmp1 = cache line size					\
146	 */								\
147	sllx	arg, MMU_PAGESHIFT, arg; /* color to dcache page */	\
148	sub	tmp2, tmp1, tmp2;					\
1492:									\
150	stxa	%g0, [arg + tmp2]ASI_DC_TAG;				\
151	membar	#Sync;							\
152	cmp	%g0, tmp2;						\
153	bne,pt	%icc, 2b;						\
154	sub	tmp2, tmp1, tmp2;					\
1551:
156
157/*
158 * macro that flushes the entire dcache
159 */
160#define	DCACHE_FLUSHALL(size, linesize, tmp)				\
161	ldxa	[%g0]ASI_LSU, tmp;					\
162	btst	LSU_DC, tmp;		/* is dcache enabled? */	\
163	bz,pn	%icc, 1f;						\
164									\
165	sub	size, linesize, tmp;					\
1662:									\
167	stxa	%g0, [tmp]ASI_DC_TAG;					\
168	membar	#Sync;							\
169	cmp	%g0, tmp;						\
170	bne,pt	%icc, 2b;						\
171	sub	tmp, linesize, tmp;					\
1721:
173
174/*
175 * macro that flushes the entire icache
176 */
177#define	ICACHE_FLUSHALL(size, linesize, tmp)				\
178	ldxa	[%g0]ASI_LSU, tmp;					\
179	btst	LSU_IC, tmp;						\
180	bz,pn	%icc, 1f;						\
181									\
182	sub	size, linesize, tmp;					\
1832:									\
184	stxa	%g0, [tmp]ASI_IC_TAG;					\
185	membar	#Sync;							\
186	cmp	%g0, tmp;						\
187	bne,pt	%icc, 2b;						\
188	sub	tmp, linesize, tmp;					\
1891:
190
191#ifdef SF_ERRATA_32
192#define SF_WORKAROUND(tmp1, tmp2)                               \
193        sethi   %hi(FLUSH_ADDR), tmp2                           ;\
194        set     MMU_PCONTEXT, tmp1                              ;\
195        stxa    %g0, [tmp1]ASI_DMMU                             ;\
196        flush   tmp2                                            ;
197#else
198#define SF_WORKAROUND(tmp1, tmp2)
199#endif /* SF_ERRATA_32 */
200
201/*
202 * arg1 = vaddr
203 * arg2 = ctxnum
204 *      - disable interrupts and clear address mask
205 *        to access 64 bit physaddr
206 *      - Blow out the TLB, flush user page.
207 *        . use secondary context.
208 */
209#define VTAG_FLUSHUPAGE(lbl, arg1, arg2, tmp1, tmp2, tmp3, tmp4) \
210        rdpr    %pstate, tmp1                                   ;\
211        andn    tmp1, PSTATE_IE, tmp2				;\
212        wrpr    tmp2, 0, %pstate                                ;\
213        sethi   %hi(FLUSH_ADDR), tmp2                           ;\
214        set     MMU_SCONTEXT, tmp3                              ;\
215        ldxa    [tmp3]ASI_DMMU, tmp4                            ;\
216        or      DEMAP_SECOND | DEMAP_PAGE_TYPE, arg1, arg1      ;\
217        cmp     tmp4, arg2                                      ;\
218        be,a,pt %icc, lbl/**/4                                  ;\
219          nop                                                   ;\
220        stxa    arg2, [tmp3]ASI_DMMU                            ;\
221lbl/**/4:                                                       ;\
222        stxa    %g0, [arg1]ASI_DTLB_DEMAP                       ;\
223        stxa    %g0, [arg1]ASI_ITLB_DEMAP                       ;\
224        flush   tmp2                                            ;\
225        be,a,pt %icc, lbl/**/5                                  ;\
226          nop                                                   ;\
227        stxa    tmp4, [tmp3]ASI_DMMU                            ;\
228        flush   tmp2                                            ;\
229lbl/**/5:                                                       ;\
230        wrpr    %g0, tmp1, %pstate
231
232
233/*
234 * macro that flushes all the user entries in dtlb
235 * arg1 = dtlb entries
236 *	- Before first compare:
237 *              tmp4 = tte
238 *              tmp5 = vaddr
239 *              tmp6 = cntxnum
240 */
241#define DTLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \
242                                tmp4, tmp5, tmp6) \
243lbl/**/0:                                                       ;\
244        sllx    arg1, 3, tmp3                                   ;\
245        SF_WORKAROUND(tmp1, tmp2)                               ;\
246        ldxa    [tmp3]ASI_DTLB_ACCESS, tmp4                     ;\
247        srlx    tmp4, 6, tmp4                                   ;\
248        andcc   tmp4, 1, %g0                                    ;\
249        bnz,pn  %xcc, lbl/**/1                                  ;\
250        srlx    tmp4, 57, tmp4                                  ;\
251        andcc   tmp4, 1, %g0                                    ;\
252        beq,pn  %xcc, lbl/**/1                                  ;\
253          nop                                                   ;\
254        set     TAGREAD_CTX_MASK, tmp1                          ;\
255        ldxa    [tmp3]ASI_DTLB_TAGREAD, tmp2                    ;\
256        and     tmp2, tmp1, tmp6                                ;\
257        andn    tmp2, tmp1, tmp5                                ;\
258	set	KCONTEXT, tmp4					;\
259	cmp	tmp6, tmp4					;\
260	be	lbl/**/1					;\
261	  nop							;\
262        VTAG_FLUSHUPAGE(VD/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\
263lbl/**/1:                                                       ;\
264        brgz,pt arg1, lbl/**/0                                  ;\
265          sub     arg1, 1, arg1
266
267
268/*
269 * macro that flushes all the user entries in itlb
270 * arg1 = itlb entries
271 *      - Before first compare:
272 *              tmp4 = tte
273 *              tmp5 = vaddr
274 *              tmp6 = cntxnum
275 */
276#define ITLB_FLUSH_UNLOCKED_UCTXS(lbl, arg1, tmp1, tmp2, tmp3, \
277                                tmp4, tmp5, tmp6) \
278lbl/**/0:                                                       ;\
279        sllx    arg1, 3, tmp3                                   ;\
280        SF_WORKAROUND(tmp1, tmp2)                               ;\
281        ldxa    [tmp3]ASI_ITLB_ACCESS, tmp4                     ;\
282        srlx    tmp4, 6, tmp4                                   ;\
283        andcc   tmp4, 1, %g0                                    ;\
284        bnz,pn  %xcc, lbl/**/1                                  ;\
285        srlx    tmp4, 57, tmp4                                  ;\
286        andcc   tmp4, 1, %g0                                    ;\
287        beq,pn  %xcc, lbl/**/1                                  ;\
288          nop                                                   ;\
289        set     TAGREAD_CTX_MASK, tmp1                          ;\
290        ldxa    [tmp3]ASI_ITLB_TAGREAD, tmp2                    ;\
291        and     tmp2, tmp1, tmp6                                ;\
292        andn    tmp2, tmp1, tmp5                                ;\
293	set	KCONTEXT, tmp4					;\
294	cmp	tmp6, tmp4					;\
295	be	lbl/**/1					;\
296	  nop							;\
297        VTAG_FLUSHUPAGE(VI/**/lbl, tmp5, tmp6, tmp1, tmp2, tmp3, tmp4) ;\
298lbl/**/1:                                                       ;\
299        brgz,pt arg1, lbl/**/0                                  ;\
300        sub     arg1, 1, arg1
301
302
303
304/*
305 * Macro for getting to offset from 'cpu_private' ptr. The 'cpu_private'
306 * ptr is in the machcpu structure.
307 * r_or_s:	Register or symbol off offset from 'cpu_private' ptr.
308 * scr1:	Scratch, ptr is returned in this register.
309 * scr2:	Scratch
310 */
311#define GET_CPU_PRIVATE_PTR(r_or_s, scr1, scr2, label)		\
312	CPU_ADDR(scr1, scr2);						\
313	ldn	[scr1 + CPU_PRIVATE], scr1; 				\
314	cmp	scr1, 0; 						\
315	be	label;							\
316	 nop; 								\
317	add	scr1, r_or_s, scr1;  					\
318
319#ifdef HUMMINGBIRD
320/*
321 * UltraSPARC-IIe processor supports both 4-way set associative and
322 * direct map E$. For performance reasons, we flush E$ by placing it
323 * in direct map mode for data load/store and restore the state after
324 * we are done flushing it. Keep interrupts off while flushing in this
325 * manner.
326 *
327 * We flush the entire ecache by starting at one end and loading each
328 * successive ecache line for the 2*ecache-size range. We have to repeat
329 * the flush operation to guarantee that the entire ecache has been
330 * flushed.
331 *
332 * For flushing a specific physical address, we start at the aliased
333 * address and load at set-size stride, wrapping around at 2*ecache-size
334 * boundary and skipping the physical address being flushed. It takes
335 * 10 loads to guarantee that the physical address has been flushed.
336 */
337
338#define	HB_ECACHE_FLUSH_CNT	2
339#define	HB_PHYS_FLUSH_CNT	10	/* #loads to flush specific paddr */
340#endif /* HUMMINGBIRD */
341
342/* END CSTYLED */
343
344/*
345 * Spitfire MMU and Cache operations.
346 */
347
348	ENTRY_NP(vtag_flushpage)
349	/*
350	 * flush page from the tlb
351	 *
352	 * %o0 = vaddr
353	 * %o1 = sfmmup
354	 */
355	rdpr	%pstate, %o5
356#ifdef DEBUG
357	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfdi_label1, %g1)
358#endif /* DEBUG */
359	/*
360	 * disable ints
361	 */
362	andn	%o5, PSTATE_IE, %o4
363	wrpr	%o4, 0, %pstate
364
365	/*
366	 * Then, blow out the tlb
367	 * Interrupts are disabled to prevent the secondary ctx register
368	 * from changing underneath us.
369	 */
370	sethi   %hi(ksfmmup), %o3
371        ldx     [%o3 + %lo(ksfmmup)], %o3
372        cmp     %o3, %o1
373        bne,pt   %xcc, 1f			! if not kernel as, go to 1
374	  sethi	%hi(FLUSH_ADDR), %o3
375	/*
376	 * For KCONTEXT demaps use primary. type = page implicitly
377	 */
378	stxa	%g0, [%o0]ASI_DTLB_DEMAP	/* dmmu flush for KCONTEXT */
379	stxa	%g0, [%o0]ASI_ITLB_DEMAP	/* immu flush for KCONTEXT */
380	flush	%o3
381	b	5f
382	  nop
3831:
384	/*
385	 * User demap.  We need to set the secondary context properly.
386	 * %o0 = vaddr
387	 * %o1 = sfmmup
388	 * %o3 = FLUSH_ADDR
389	 */
390	SFMMU_CPU_CNUM(%o1, %g1, %g2)	/* %g1 = sfmmu cnum on this CPU */
391
392	set	MMU_SCONTEXT, %o4
393	ldxa	[%o4]ASI_DMMU, %o2		/* rd old ctxnum */
394	or	DEMAP_SECOND | DEMAP_PAGE_TYPE, %o0, %o0
395	cmp	%o2, %g1
396	be,pt	%icc, 4f
397	  nop
398	stxa	%g1, [%o4]ASI_DMMU		/* wr new ctxum */
3994:
400	stxa	%g0, [%o0]ASI_DTLB_DEMAP
401	stxa	%g0, [%o0]ASI_ITLB_DEMAP
402	flush	%o3
403	be,pt	%icc, 5f
404	  nop
405	stxa	%o2, [%o4]ASI_DMMU		/* restore old ctxnum */
406	flush	%o3
4075:
408	retl
409	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
410	SET_SIZE(vtag_flushpage)
411
412        .seg    ".text"
413.flushallmsg:
414        .asciz  "sfmmu_asm: unimplemented flush operation"
415
416        ENTRY_NP(vtag_flushall)
417        sethi   %hi(.flushallmsg), %o0
418        call    panic
419          or    %o0, %lo(.flushallmsg), %o0
420        SET_SIZE(vtag_flushall)
421
422	ENTRY_NP(vtag_flushall_uctxs)
423	/*
424	 * flush entire DTLB/ITLB.
425	 */
426	CPU_INDEX(%g1, %g2)
427	mulx	%g1, CPU_NODE_SIZE, %g1
428	set	cpunodes, %g2
429	add	%g1, %g2, %g1
430	lduh	[%g1 + ITLB_SIZE], %g2		! %g2 = # entries in ITLB
431	lduh	[%g1 + DTLB_SIZE], %g1		! %g1 = # entries in DTLB
432	sub	%g2, 1, %g2			! %g2 = # entries in ITLB - 1
433	sub	%g1, 1, %g1			! %g1 = # entries in DTLB - 1
434
435        !
436        ! Flush itlb's
437        !
438        ITLB_FLUSH_UNLOCKED_UCTXS(I, %g2, %g3, %g4, %o2, %o3, %o4, %o5)
439
440	!
441        ! Flush dtlb's
442        !
443        DTLB_FLUSH_UNLOCKED_UCTXS(D, %g1, %g3, %g4, %o2, %o3, %o4, %o5)
444
445	membar  #Sync
446	retl
447	  nop
448
449	SET_SIZE(vtag_flushall_uctxs)
450
451	ENTRY_NP(vtag_flushpage_tl1)
452	/*
453	 * x-trap to flush page from tlb and tsb
454	 *
455	 * %g1 = vaddr, zero-extended on 32-bit kernel
456	 * %g2 = sfmmup
457	 *
458	 * assumes TSBE_TAG = 0
459	 */
460	srln	%g1, MMU_PAGESHIFT, %g1
461	slln	%g1, MMU_PAGESHIFT, %g1			/* g1 = vaddr */
462
463	SFMMU_CPU_CNUM(%g2, %g3, %g4)   /* %g3 = sfmmu cnum on this CPU */
464
465	/* We need to set the secondary context properly. */
466	set	MMU_SCONTEXT, %g4
467	ldxa	[%g4]ASI_DMMU, %g5		/* rd old ctxnum */
468	or	DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1
469	stxa	%g3, [%g4]ASI_DMMU		/* wr new ctxum */
470	stxa	%g0, [%g1]ASI_DTLB_DEMAP
471	stxa	%g0, [%g1]ASI_ITLB_DEMAP
472	stxa	%g5, [%g4]ASI_DMMU		/* restore old ctxnum */
473	membar #Sync
474	retry
475	SET_SIZE(vtag_flushpage_tl1)
476
477	ENTRY_NP(vtag_flush_pgcnt_tl1)
478	/*
479	 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb
480	 *
481	 * %g1 = vaddr, zero-extended on 32-bit kernel
482	 * %g2 = <sfmmup58 | pgcnt6>
483	 *
484	 * NOTE: this handler relies on the fact that no
485	 *	interrupts or traps can occur during the loop
486	 *	issuing the TLB_DEMAP operations. It is assumed
487	 *	that interrupts are disabled and this code is
488	 *	fetching from the kernel locked text address.
489	 *
490	 * assumes TSBE_TAG = 0
491	 */
492	srln	%g1, MMU_PAGESHIFT, %g1
493	slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
494	or	DEMAP_SECOND | DEMAP_PAGE_TYPE, %g1, %g1
495
496	set	SFMMU_PGCNT_MASK, %g4
497	and	%g4, %g2, %g3			/* g3 = pgcnt - 1 */
498	add	%g3, 1, %g3			/* g3 = pgcnt */
499
500	andn	%g2, SFMMU_PGCNT_MASK, %g2	/* g2 = sfmmup */
501
502	SFMMU_CPU_CNUM(%g2, %g5, %g6)   ! %g5 = sfmmu cnum on this CPU
503
504	/* We need to set the secondary context properly. */
505	set	MMU_SCONTEXT, %g4
506	ldxa	[%g4]ASI_DMMU, %g6		/* read old ctxnum */
507	stxa	%g5, [%g4]ASI_DMMU		/* write new ctxum */
508
509	set	MMU_PAGESIZE, %g2		/* g2 = pgsize */
510	sethi	 %hi(FLUSH_ADDR), %g5
5111:
512	stxa	%g0, [%g1]ASI_DTLB_DEMAP
513	stxa	%g0, [%g1]ASI_ITLB_DEMAP
514	flush	%g5
515	deccc	%g3				/* decr pgcnt */
516	bnz,pt	%icc,1b
517	  add	%g1, %g2, %g1			/* go to nextpage */
518
519	stxa	%g6, [%g4]ASI_DMMU		/* restore old ctxnum */
520	membar #Sync
521	retry
522	SET_SIZE(vtag_flush_pgcnt_tl1)
523
524	! Not implemented on US1/US2
525	ENTRY_NP(vtag_flushall_tl1)
526	retry
527	SET_SIZE(vtag_flushall_tl1)
528
529/*
530 * vac_flushpage(pfnum, color)
531 *	Flush 1 8k page of the D-$ with physical page = pfnum
532 *	Algorithm:
533 *		The spitfire dcache is a 16k direct mapped virtual indexed,
534 *		physically tagged cache.  Given the pfnum we read all cache
535 *		lines for the corresponding page in the cache (determined by
536 *		the color).  Each cache line is compared with
537 *		the tag created from the pfnum. If the tags match we flush
538 *		the line.
539 */
540	.seg	".data"
541	.align	8
542	.global	dflush_type
543dflush_type:
544	.word	FLUSHPAGE_TYPE
545	.seg	".text"
546
547	ENTRY(vac_flushpage)
548	/*
549	 * flush page from the d$
550	 *
551	 * %o0 = pfnum, %o1 = color
552	 */
553	DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4)
554	retl
555	nop
556	SET_SIZE(vac_flushpage)
557
558	ENTRY_NP(vac_flushpage_tl1)
559	/*
560	 * x-trap to flush page from the d$
561	 *
562	 * %g1 = pfnum, %g2 = color
563	 */
564	DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5)
565	retry
566	SET_SIZE(vac_flushpage_tl1)
567
568	ENTRY(vac_flushcolor)
569	/*
570	 * %o0 = vcolor
571	 */
572	DCACHE_FLUSHCOLOR(%o0, %o1, %o2)
573	retl
574	  nop
575	SET_SIZE(vac_flushcolor)
576
577	ENTRY(vac_flushcolor_tl1)
578	/*
579	 * %g1 = vcolor
580	 */
581	DCACHE_FLUSHCOLOR(%g1, %g2, %g3)
582	retry
583	SET_SIZE(vac_flushcolor_tl1)
584
585
586	.global _dispatch_status_busy
587_dispatch_status_busy:
588	.asciz	"ASI_INTR_DISPATCH_STATUS error: busy"
589	.align	4
590
591/*
592 * Determine whether or not the IDSR is busy.
593 * Entry: no arguments
594 * Returns: 1 if busy, 0 otherwise
595 */
596	ENTRY(idsr_busy)
597	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
598	clr	%o0
599	btst	IDSR_BUSY, %g1
600	bz,a,pt	%xcc, 1f
601	mov	1, %o0
6021:
603	retl
604	nop
605	SET_SIZE(idsr_busy)
606
607/*
608 * Setup interrupt dispatch data registers
609 * Entry:
610 *	%o0 - function or inumber to call
611 *	%o1, %o2 - arguments (2 uint64_t's)
612 */
613	.seg "text"
614
615	ENTRY(init_mondo)
616#ifdef DEBUG
617	!
618	! IDSR should not be busy at the moment
619	!
620	ldxa	[%g0]ASI_INTR_DISPATCH_STATUS, %g1
621	btst	IDSR_BUSY, %g1
622	bz,pt	%xcc, 1f
623	nop
624
625	sethi	%hi(_dispatch_status_busy), %o0
626	call	panic
627	or	%o0, %lo(_dispatch_status_busy), %o0
628#endif /* DEBUG */
629
630	ALTENTRY(init_mondo_nocheck)
631	!
632	! interrupt vector dispach data reg 0
633	!
6341:
635	mov	IDDR_0, %g1
636	mov	IDDR_1, %g2
637	mov	IDDR_2, %g3
638	stxa	%o0, [%g1]ASI_INTR_DISPATCH
639
640	!
641	! interrupt vector dispach data reg 1
642	!
643	stxa	%o1, [%g2]ASI_INTR_DISPATCH
644
645	!
646	! interrupt vector dispach data reg 2
647	!
648	stxa	%o2, [%g3]ASI_INTR_DISPATCH
649
650	retl
651	membar	#Sync			! allowed to be in the delay slot
652	SET_SIZE(init_mondo)
653
654/*
655 * Ship mondo to upaid
656 */
657	ENTRY_NP(shipit)
658	sll	%o0, IDCR_PID_SHIFT, %g1	! IDCR<18:14> = upa id
659	or	%g1, IDCR_OFFSET, %g1		! IDCR<13:0> = 0x70
660	stxa	%g0, [%g1]ASI_INTR_DISPATCH	! interrupt vector dispatch
661#if defined(SF_ERRATA_54)
662	membar	#Sync				! store must occur before load
663	mov	0x20, %g3			! UDBH Control Register Read
664	ldxa	[%g3]ASI_SDB_INTR_R, %g0
665#endif
666	retl
667	membar	#Sync
668	SET_SIZE(shipit)
669
670
671/*
672 * flush_instr_mem:
673 *	Flush a portion of the I-$ starting at vaddr
674 * 	%o0 vaddr
675 *	%o1 bytes to be flushed
676 */
677
678	ENTRY(flush_instr_mem)
679	membar	#StoreStore				! Ensure the stores
680							! are globally visible
6811:
682	flush	%o0
683	subcc	%o1, ICACHE_FLUSHSZ, %o1		! bytes = bytes-0x20
684	bgu,pt	%ncc, 1b
685	add	%o0, ICACHE_FLUSHSZ, %o0		! vaddr = vaddr+0x20
686
687	retl
688	nop
689	SET_SIZE(flush_instr_mem)
690
691/*
692 * flush_ecache:
693 * Flush the entire e$ using displacement flush by reading through a
694 * physically contiguous area. We use mmu bypass asi (ASI_MEM) while
695 * reading this physical address range so that data doesn't go to d$.
696 * incoming arguments:
697 *	%o0 - 64 bit physical address
698 *	%o1 - size of address range to read
699 *	%o2 - ecache linesize
700 */
701	ENTRY(flush_ecache)
702#ifndef HUMMINGBIRD
703	b	2f
704	  nop
7051:
706	ldxa	[%o0 + %o1]ASI_MEM, %g0	! start reading from physaddr + size
7072:
708	subcc	%o1, %o2, %o1
709	bcc,a,pt %ncc, 1b
710	  nop
711
712#else /* HUMMINGBIRD */
713	/*
714	 * UltraSPARC-IIe processor supports both 4-way set associative
715	 * and direct map E$. For performance reasons, we flush E$ by
716	 * placing it in direct map mode for data load/store and restore
717	 * the state after we are done flushing it. It takes 2 iterations
718	 * to guarantee that the entire ecache has been flushed.
719	 *
720	 * Keep the interrupts disabled while flushing E$ in this manner.
721	 */
722	rdpr	%pstate, %g4		! current pstate (restored later)
723	andn	%g4, PSTATE_IE, %g5
724	wrpr	%g0, %g5, %pstate	! disable interrupts
725
726	! Place E$ in direct map mode for data access
727	or	%g0, 1, %g5
728	sllx	%g5, HB_UPA_DMAP_DATA_BIT, %g5
729	ldxa	[%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later)
730	or	%g1, %g5, %g5
731	membar	#Sync
732	stxa	%g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access
733	membar	#Sync
734
735	! flush entire ecache HB_ECACHE_FLUSH_CNT times
736	mov	HB_ECACHE_FLUSH_CNT-1, %g5
7372:
738	sub	%o1, %o2, %g3		! start from last entry
7391:
740	ldxa	[%o0 + %g3]ASI_MEM, %g0	! start reading from physaddr + size
741	subcc	%g3, %o2, %g3
742	bgeu,a,pt %ncc, 1b
743	  nop
744	brgz,a,pt %g5, 2b
745	  dec	%g5
746
747	membar	#Sync
748	stxa	%g1, [%g0]ASI_UPA_CONFIG ! restore UPA config reg
749	membar	#Sync
750	wrpr	%g0, %g4, %pstate	! restore earlier pstate
751#endif /* HUMMINGBIRD */
752
753	retl
754	nop
755	SET_SIZE(flush_ecache)
756
757/*
758 * void kdi_flush_idcache(int dcache_size, int dcache_linesize,
759 *			int icache_size, int icache_linesize)
760 */
761	ENTRY(kdi_flush_idcache)
762	DCACHE_FLUSHALL(%o0, %o1, %g1)
763	ICACHE_FLUSHALL(%o2, %o3, %g1)
764	membar	#Sync
765	retl
766	nop
767	SET_SIZE(kdi_flush_idcache)
768
769
770/*
771 * void get_ecache_dtag(uint32_t ecache_idx, uint64_t *data, uint64_t *tag,
772 * 			uint64_t *oafsr, uint64_t *acc_afsr)
773 *
774 * Get ecache data and tag.  The ecache_idx argument is assumed to be aligned
775 * on a 64-byte boundary.  The corresponding AFSR value is also read for each
776 * 8 byte ecache data obtained. The ecache data is assumed to be a pointer
777 * to an array of 16 uint64_t's (e$data & afsr value).  The action to read the
778 * data and tag should be atomic to make sense.  We will be executing at PIL15
779 * and will disable IE, so nothing can occur between the two reads.  We also
780 * assume that the execution of this code does not interfere with what we are
781 * reading - not really possible, but we'll live with it for now.
782 * We also pass the old AFSR value before clearing it, and caller will take
783 * appropriate actions if the important bits are non-zero.
784 *
785 * If the caller wishes to track the AFSR in cases where the CP bit is
786 * set, an address should be passed in for acc_afsr.  Otherwise, this
787 * argument may be null.
788 *
789 * Register Usage:
790 * i0: In: 32-bit e$ index
791 * i1: In: addr of e$ data
792 * i2: In: addr of e$ tag
793 * i3: In: addr of old afsr
794 * i4: In: addr of accumulated afsr - may be null
795 */
796	ENTRY(get_ecache_dtag)
797	save	%sp, -SA(MINFRAME), %sp
798	or	%g0, 1, %l4
799	sllx	%l4, 39, %l4	! set bit 39 for e$ data access
800	or	%i0, %l4, %g6	! %g6 = e$ addr for data read
801	sllx	%l4, 1, %l4	! set bit 40 for e$ tag access
802	or	%i0, %l4, %l4	! %l4 = e$ addr for tag read
803
804	rdpr    %pstate, %i5
805	andn    %i5, PSTATE_IE | PSTATE_AM, %i0
806	wrpr    %i0, %g0, %pstate       ! clear IE, AM bits
807
808	ldxa    [%g0]ASI_ESTATE_ERR, %g1
809	stxa    %g0, [%g0]ASI_ESTATE_ERR        ! disable errors
810	membar  #Sync
811
812	ldxa	[%g0]ASI_AFSR, %i0      ! grab the old-afsr before tag read
813	stx     %i0, [%i3]		! write back the old-afsr
814
815	ldxa    [%l4]ASI_EC_R, %g0      ! read tag into E$ tag reg
816	ldxa    [%g0]ASI_EC_DIAG, %i0   ! read tag from E$ tag reg
817	stx     %i0, [%i2]              ! write back tag result
818
819	clr	%i2			! loop count
820
821	brz	%i4, 1f			! acc_afsr == NULL?
822	  ldxa	[%g0]ASI_AFSR, %i0      ! grab the old-afsr before clearing
823	srlx	%i0, P_AFSR_CP_SHIFT, %l0
824	btst	1, %l0
825	bz	1f
826	  nop
827	ldx	[%i4], %g4
828	or	%g4, %i0, %g4		! aggregate AFSR in cpu private
829	stx	%g4, [%i4]
8301:
831	stxa    %i0, [%g0]ASI_AFSR	! clear AFSR
832	membar  #Sync
833	ldxa    [%g6]ASI_EC_R, %i0      ! read the 8byte E$data
834	stx     %i0, [%i1]              ! save the E$data
835	add     %g6, 8, %g6
836	add     %i1, 8, %i1
837	ldxa    [%g0]ASI_AFSR, %i0      ! read AFSR for this 16byte read
838	srlx	%i0, P_AFSR_CP_SHIFT, %l0
839	btst	1, %l0
840	bz	2f
841	  stx     %i0, [%i1]		! save the AFSR
842
843	brz	%i4, 2f			! acc_afsr == NULL?
844	  nop
845	ldx	[%i4], %g4
846	or	%g4, %i0, %g4		! aggregate AFSR in cpu private
847	stx	%g4, [%i4]
8482:
849	add     %i2, 8, %i2
850	cmp     %i2, 64
851	bl,a    1b
852	  add     %i1, 8, %i1
853	stxa    %i0, [%g0]ASI_AFSR              ! clear AFSR
854	membar  #Sync
855	stxa    %g1, [%g0]ASI_ESTATE_ERR        ! restore error enable
856	membar  #Sync
857	wrpr    %g0, %i5, %pstate
858	ret
859	  restore
860	SET_SIZE(get_ecache_dtag)
861
862/*
863 * The ce_err function handles trap type 0x63 (corrected_ECC_error) at tl=0.
864 * Steps: 1. GET AFSR  2. Get AFAR <40:4> 3. Get datapath error status
865 *	  4. Clear datapath error bit(s) 5. Clear AFSR error bit
866 *	  6. package data in %g2 and %g3 7. call cpu_ce_error vis sys_trap
867 * %g2: [ 52:43 UDB lower | 42:33 UDB upper | 32:0 afsr ] - arg #3/arg #1
868 * %g3: [ 40:4 afar ] - sys_trap->have_win: arg #4/arg #2
869 */
870	ENTRY_NP(ce_err)
871	ldxa	[%g0]ASI_AFSR, %g3	! save afsr in g3
872
873	!
874	! Check for a UE... From Kevin.Normoyle:
875	! We try to switch to the trap for the UE, but since that's
876	! a hardware pipeline, we might get to the CE trap before we
877	! can switch. The UDB and AFSR registers will have both the
878	! UE and CE bits set but the UDB syndrome and the AFAR will be
879	! for the UE.
880	!
881	or	%g0, 1, %g1		! put 1 in g1
882	sllx	%g1, 21, %g1		! shift left to <21> afsr UE
883	andcc	%g1, %g3, %g0		! check for UE in afsr
884	bnz	async_err		! handle the UE, not the CE
885	  or	%g0, 0x63, %g5		! pass along the CE ttype
886	!
887	! Disable further CE traps to avoid recursion (stack overflow)
888	! and staying above XCALL_PIL for extended periods.
889	!
890	ldxa	[%g0]ASI_ESTATE_ERR, %g2
891	andn	%g2, 0x1, %g2		! clear bit 0 - CEEN
892	stxa	%g2, [%g0]ASI_ESTATE_ERR
893	membar	#Sync			! required
894	!
895	! handle the CE
896	ldxa	[%g0]ASI_AFAR, %g2	! save afar in g2
897
898	set	P_DER_H, %g4		! put P_DER_H in g4
899	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into g5
900	or	%g0, 1, %g6		! put 1 in g6
901	sllx	%g6, 8, %g6		! shift g6 to <8> sdb CE
902	andcc	%g5, %g6, %g1		! check for CE in upper half
903	sllx	%g5, 33, %g5		! shift upper bits to <42:33>
904	or	%g3, %g5, %g3		! or with afsr bits
905	bz,a	1f			! no error, goto 1f
906	  nop
907	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg error bit
908	membar	#Sync			! membar sync required
9091:
910	set	P_DER_L, %g4		! put P_DER_L in g4
911	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g6
912	andcc	%g5, %g6, %g1		! check for CE in lower half
913	sllx	%g5, 43, %g5		! shift upper bits to <52:43>
914	or	%g3, %g5, %g3		! or with afsr bits
915	bz,a	2f			! no error, goto 2f
916	  nop
917	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg error bit
918	membar	#Sync			! membar sync required
9192:
920	or	%g0, 1, %g4		! put 1 in g4
921	sllx	%g4, 20, %g4		! shift left to <20> afsr CE
922	stxa	%g4, [%g0]ASI_AFSR	! use g4 to clear afsr CE error
923	membar	#Sync			! membar sync required
924
925	set	cpu_ce_error, %g1	! put *cpu_ce_error() in g1
926	rdpr	%pil, %g6		! read pil into %g6
927	subcc	%g6, PIL_15, %g0
928	  movneg	%icc, PIL_14, %g4 ! run at pil 14 unless already at 15
929	sethi	%hi(sys_trap), %g5
930	jmp	%g5 + %lo(sys_trap)	! goto sys_trap
931	  movge	%icc, PIL_15, %g4	! already at pil 15
932	SET_SIZE(ce_err)
933
934	ENTRY_NP(ce_err_tl1)
935#ifndef	TRAPTRACE
936	ldxa	[%g0]ASI_AFSR, %g7
937	stxa	%g7, [%g0]ASI_AFSR
938	membar	#Sync
939	retry
940#else
941	set	ce_trap_tl1, %g1
942	sethi	%hi(dis_err_panic1), %g4
943	jmp	%g4 + %lo(dis_err_panic1)
944	nop
945#endif
946	SET_SIZE(ce_err_tl1)
947
948#ifdef	TRAPTRACE
949.celevel1msg:
950	.asciz	"Softerror with trap tracing at tl1: AFAR 0x%08x.%08x AFSR 0x%08x.%08x";
951
952	ENTRY_NP(ce_trap_tl1)
953	! upper 32 bits of AFSR already in o3
954	mov	%o4, %o0		! save AFAR upper 32 bits
955	mov	%o2, %o4		! lower 32 bits of AFSR
956	mov	%o1, %o2		! lower 32 bits of AFAR
957	mov	%o0, %o1		! upper 32 bits of AFAR
958	set	.celevel1msg, %o0
959	call	panic
960	nop
961	SET_SIZE(ce_trap_tl1)
962#endif
963
964/*
965 * The async_err function handles trap types 0x0A (instruction_access_error)
966 * and 0x32 (data_access_error) at TL = 0 and TL > 0.  When we branch here,
967 * %g5 will have the trap type (with 0x200 set if we're at TL > 0).
968 *
969 * Steps: 1. Get AFSR 2. Get AFAR <40:4> 3. If not UE error skip UDP registers.
970 *	  4. Else get and clear datapath error bit(s) 4. Clear AFSR error bits
971 *	  6. package data in %g2 and %g3 7. disable all cpu errors, because
972 *	  trap is likely to be fatal 8. call cpu_async_error vis sys_trap
973 *
974 * %g3: [ 63:53 tt | 52:43 UDB_L | 42:33 UDB_U | 32:0 afsr ] - arg #3/arg #1
975 * %g2: [ 40:4 afar ] - sys_trap->have_win: arg #4/arg #2
976 *
977 * async_err is the assembly glue code to get us from the actual trap
978 * into the CPU module's C error handler.  Note that we also branch
979 * here from ce_err() above.
980 */
981	ENTRY_NP(async_err)
982	stxa	%g0, [%g0]ASI_ESTATE_ERR ! disable ecc and other cpu errors
983	membar	#Sync			! membar sync required
984
985	ldxa	[%g0]ASI_AFSR, %g3	! save afsr in g3
986	ldxa	[%g0]ASI_AFAR, %g2	! save afar in g2
987
988	sllx	%g5, 53, %g5		! move ttype to <63:53>
989	or	%g3, %g5, %g3		! or to afsr in g3
990
991	or	%g0, 1, %g1		! put 1 in g1
992	sllx	%g1, 21, %g1		! shift left to <21> afsr UE
993	andcc	%g1, %g3, %g0		! check for UE in afsr
994	bz,a,pn %icc, 2f		! if !UE skip sdb read/clear
995	  nop
996
997	set	P_DER_H, %g4		! put P_DER_H in g4
998	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into 56
999	or	%g0, 1, %g6		! put 1 in g6
1000	sllx	%g6, 9, %g6		! shift g6 to <9> sdb UE
1001	andcc	%g5, %g6, %g1		! check for UE in upper half
1002	sllx	%g5, 33, %g5		! shift upper bits to <42:33>
1003	or	%g3, %g5, %g3		! or with afsr bits
1004	bz,a	1f			! no error, goto 1f
1005	  nop
1006	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg UE error bit
1007	membar	#Sync			! membar sync required
10081:
1009	set	P_DER_L, %g4		! put P_DER_L in g4
1010	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g5
1011	andcc	%g5, %g6, %g1		! check for UE in lower half
1012	sllx	%g5, 43, %g5		! shift upper bits to <52:43>
1013	or	%g3, %g5, %g3		! or with afsr bits
1014	bz,a	2f			! no error, goto 2f
1015	  nop
1016	stxa	%g1, [%g4]ASI_SDB_INTR_W ! clear sdb reg UE error bit
1017	membar	#Sync			! membar sync required
10182:
1019	stxa	%g3, [%g0]ASI_AFSR	! clear all the sticky bits
1020	membar	#Sync			! membar sync required
1021
1022	RESET_USER_RTT_REGS(%g4, %g5, async_err_resetskip)
1023async_err_resetskip:
1024
1025	set	cpu_async_error, %g1	! put cpu_async_error in g1
1026	sethi	%hi(sys_trap), %g5
1027	jmp	%g5 + %lo(sys_trap)	! goto sys_trap
1028	  or	%g0, PIL_15, %g4	! run at pil 15
1029	SET_SIZE(async_err)
1030
1031	ENTRY_NP(dis_err_panic1)
1032	stxa	%g0, [%g0]ASI_ESTATE_ERR ! disable all error traps
1033	membar	#Sync
1034	! save destination routine is in g1
1035	ldxa	[%g0]ASI_AFAR, %g2	! read afar
1036	ldxa	[%g0]ASI_AFSR, %g3	! read afsr
1037	set	P_DER_H, %g4		! put P_DER_H in g4
1038	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb upper half into g5
1039	sllx	%g5, 33, %g5		! shift upper bits to <42:33>
1040	or	%g3, %g5, %g3		! or with afsr bits
1041	set	P_DER_L, %g4		! put P_DER_L in g4
1042	ldxa	[%g4]ASI_SDB_INTR_R, %g5 ! read sdb lower half into g5
1043	sllx	%g5, 43, %g5		! shift upper bits to <52:43>
1044	or	%g3, %g5, %g3		! or with afsr bits
1045
1046	RESET_USER_RTT_REGS(%g4, %g5, dis_err_panic1_resetskip)
1047dis_err_panic1_resetskip:
1048
1049	sethi	%hi(sys_trap), %g5
1050	jmp	%g5 + %lo(sys_trap)	! goto sys_trap
1051	  sub	%g0, 1, %g4
1052	SET_SIZE(dis_err_panic1)
1053
1054/*
1055 * The clr_datapath function clears any error bits set in the UDB regs.
1056 */
1057	ENTRY(clr_datapath)
1058	set	P_DER_H, %o4			! put P_DER_H in o4
1059	ldxa	[%o4]ASI_SDB_INTR_R, %o5	! read sdb upper half into o3
1060	or	%g0, 0x3, %o2			! put 0x3 in o2
1061	sllx	%o2, 8, %o2			! shift o2 to <9:8> sdb
1062	andcc	%o5, %o2, %o1			! check for UE,CE in upper half
1063	bz,a	1f				! no error, goto 1f
1064	  nop
1065	stxa	%o1, [%o4]ASI_SDB_INTR_W	! clear sdb reg UE,CE error bits
1066	membar	#Sync				! membar sync required
10671:
1068	set	P_DER_L, %o4			! put P_DER_L in o4
1069	ldxa	[%o4]ASI_SDB_INTR_R, %o5	! read sdb lower half into o5
1070	andcc	%o5, %o2, %o1			! check for UE,CE in lower half
1071	bz,a	2f				! no error, goto 2f
1072	  nop
1073	stxa	%o1, [%o4]ASI_SDB_INTR_W	! clear sdb reg UE,CE error bits
1074	membar	#Sync
10752:
1076	retl
1077	  nop
1078	SET_SIZE(clr_datapath)
1079
1080/*
1081 * The get_udb_errors() function gets the current value of the
1082 * Datapath Error Registers.
1083 */
1084	ENTRY(get_udb_errors)
1085	set	P_DER_H, %o3
1086	ldxa	[%o3]ASI_SDB_INTR_R, %o2
1087	stx	%o2, [%o0]
1088	set	P_DER_L, %o3
1089	ldxa	[%o3]ASI_SDB_INTR_R, %o2
1090	retl
1091	  stx	%o2, [%o1]
1092	SET_SIZE(get_udb_errors)
1093
1094/*
1095 * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the
1096 * tte, the virtual address, and the ctxnum of the specified tlb entry.  They
1097 * should only be used in places where you have no choice but to look at the
1098 * tlb itself.
1099 *
1100 * Note: These two routines are required by the Estar "cpr" loadable module.
1101 */
1102/*
1103 * NB - In Spitfire cpus, when reading a tte from the hardware, we
1104 * need to clear [42-41] because the general definitions in pte.h
1105 * define the PA to be [42-13] whereas Spitfire really uses [40-13].
1106 * When cloning these routines for other cpus the "andn" below is not
1107 * necessary.
1108 */
1109	ENTRY_NP(itlb_rd_entry)
1110	sllx	%o0, 3, %o0
1111#if defined(SF_ERRATA_32)
1112	sethi	%hi(FLUSH_ADDR), %g2
1113	set	MMU_PCONTEXT, %g1
1114	stxa	%g0, [%g1]ASI_DMMU			! KCONTEXT
1115	flush	%g2
1116#endif
1117	ldxa	[%o0]ASI_ITLB_ACCESS, %g1
1118	set	TTE_SPITFIRE_PFNHI_CLEAR, %g2		! spitfire only
1119	sllx	%g2, TTE_SPITFIRE_PFNHI_SHIFT, %g2	! see comment above
1120	andn	%g1, %g2, %g1				! for details
1121	stx	%g1, [%o1]
1122	ldxa	[%o0]ASI_ITLB_TAGREAD, %g2
1123	set	TAGREAD_CTX_MASK, %o4
1124	andn	%g2, %o4, %o5
1125	retl
1126	  stx	%o5, [%o2]
1127	SET_SIZE(itlb_rd_entry)
1128
1129	ENTRY_NP(dtlb_rd_entry)
1130	sllx	%o0, 3, %o0
1131#if defined(SF_ERRATA_32)
1132	sethi	%hi(FLUSH_ADDR), %g2
1133	set	MMU_PCONTEXT, %g1
1134	stxa	%g0, [%g1]ASI_DMMU			! KCONTEXT
1135	flush	%g2
1136#endif
1137	ldxa	[%o0]ASI_DTLB_ACCESS, %g1
1138	set	TTE_SPITFIRE_PFNHI_CLEAR, %g2		! spitfire only
1139	sllx	%g2, TTE_SPITFIRE_PFNHI_SHIFT, %g2	! see comment above
1140	andn	%g1, %g2, %g1				! itlb_rd_entry
1141	stx	%g1, [%o1]
1142	ldxa	[%o0]ASI_DTLB_TAGREAD, %g2
1143	set	TAGREAD_CTX_MASK, %o4
1144	andn	%g2, %o4, %o5
1145	retl
1146	  stx	%o5, [%o2]
1147	SET_SIZE(dtlb_rd_entry)
1148
1149	ENTRY(set_lsu)
1150	stxa	%o0, [%g0]ASI_LSU		! store to LSU
1151	retl
1152	membar	#Sync
1153	SET_SIZE(set_lsu)
1154
1155	ENTRY(get_lsu)
1156	retl
1157	ldxa	[%g0]ASI_LSU, %o0		! load LSU
1158	SET_SIZE(get_lsu)
1159
1160	/*
1161	 * Clear the NPT (non-privileged trap) bit in the %tick
1162	 * registers. In an effort to make the change in the
1163	 * tick counter as consistent as possible, we disable
1164	 * all interrupts while we're changing the registers. We also
1165	 * ensure that the read and write instructions are in the same
1166	 * line in the instruction cache.
1167	 */
1168	ENTRY_NP(cpu_clearticknpt)
1169	rdpr	%pstate, %g1		/* save processor state */
1170	andn	%g1, PSTATE_IE, %g3	/* turn off */
1171	wrpr	%g0, %g3, %pstate	/*   interrupts */
1172	rdpr	%tick, %g2		/* get tick register */
1173	brgez,pn %g2, 1f		/* if NPT bit off, we're done */
1174	mov	1, %g3			/* create mask */
1175	sllx	%g3, 63, %g3		/*   for NPT bit */
1176	ba,a,pt	%xcc, 2f
1177	.align	64			/* Align to I$ boundary */
11782:
1179	rdpr	%tick, %g2		/* get tick register */
1180	wrpr	%g3, %g2, %tick		/* write tick register, */
1181					/*   clearing NPT bit   */
1182#if defined(BB_ERRATA_1)
1183	rdpr	%tick, %g0		/* read (s)tick (BB_ERRATA_1) */
1184#endif
11851:
1186	jmp	%g4 + 4
1187	wrpr	%g0, %g1, %pstate	/* restore processor state */
1188	SET_SIZE(cpu_clearticknpt)
1189
1190	/*
1191	 * get_ecache_tag()
1192	 * Register Usage:
1193	 * %o0: In: 32-bit E$ index
1194	 *      Out: 64-bit E$ tag value
1195	 * %o1: In: 64-bit AFSR value after clearing sticky bits
1196	 * %o2: In: address of cpu private afsr storage
1197	 */
1198	ENTRY(get_ecache_tag)
1199	or	%g0, 1, %o4
1200	sllx	%o4, 40, %o4			! set bit 40 for e$ tag access
1201	or	%o0, %o4, %o4			! %o4 = e$ addr for tag read
1202	rdpr	%pstate, %o5
1203	andn	%o5, PSTATE_IE | PSTATE_AM, %o0
1204	wrpr	%o0, %g0, %pstate		! clear IE, AM bits
1205
1206	ldxa	[%g0]ASI_ESTATE_ERR, %g1
1207	stxa	%g0, [%g0]ASI_ESTATE_ERR	! Turn off Error enable
1208	membar	#Sync
1209
1210	ldxa	[%g0]ASI_AFSR, %o0
1211	srlx	%o0, P_AFSR_CP_SHIFT, %o3
1212	btst	1, %o3
1213	bz	1f
1214	  nop
1215	ldx	[%o2], %g4
1216	or	%g4, %o0, %g4			! aggregate AFSR in cpu private
1217	stx	%g4, [%o2]
12181:
1219	stxa	%o0, [%g0]ASI_AFSR		! clear AFSR
1220	membar  #Sync
1221
1222	ldxa	[%o4]ASI_EC_R, %g0
1223	ldxa	[%g0]ASI_EC_DIAG, %o0		! read tag from e$ tag reg
1224
1225	ldxa	[%g0]ASI_AFSR, %o3
1226	srlx	%o3, P_AFSR_CP_SHIFT, %o4
1227	btst	1, %o4
1228	bz	2f
1229	  stx	%o3, [%o1]			! AFSR after sticky clear
1230	ldx	[%o2], %g4
1231	or	%g4, %o3, %g4			! aggregate AFSR in cpu private
1232	stx	%g4, [%o2]
12332:
1234	membar	#Sync
1235
1236	stxa	%g1, [%g0]ASI_ESTATE_ERR	! Turn error enable back on
1237	membar	#Sync
1238	retl
1239	wrpr	%g0, %o5, %pstate
1240	SET_SIZE(get_ecache_tag)
1241
1242	/*
1243	 * check_ecache_line()
1244	 * Register Usage:
1245	 * %o0: In: 32-bit E$ index
1246	 *      Out: 64-bit accumulated AFSR
1247	 * %o1: In: address of cpu private afsr storage
1248	 */
1249	ENTRY(check_ecache_line)
1250	or	%g0, 1, %o4
1251	sllx	%o4, 39, %o4			! set bit 39 for e$ data access
1252	or	%o0, %o4, %o4		 	! %o4 = e$ addr for data read
1253
1254	rdpr	%pstate, %o5
1255	andn	%o5, PSTATE_IE | PSTATE_AM, %o0
1256	wrpr	%o0, %g0, %pstate		! clear IE, AM bits
1257
1258	ldxa	[%g0]ASI_ESTATE_ERR, %g1
1259	stxa	%g0, [%g0]ASI_ESTATE_ERR 	! Turn off Error enable
1260	membar	#Sync
1261
1262	ldxa 	[%g0]ASI_AFSR, %o0
1263	srlx	%o0, P_AFSR_CP_SHIFT, %o2
1264	btst	1, %o2
1265	bz	1f
1266	  clr	%o2				! loop count
1267	ldx	[%o1], %o3
1268	or	%o3, %o0, %o3			! aggregate AFSR in cpu private
1269	stx	%o3, [%o1]
12701:
1271	stxa    %o0, [%g0]ASI_AFSR              ! clear AFSR
1272	membar	#Sync
1273
12742:
1275	ldxa	[%o4]ASI_EC_R, %g0		! Read the E$ data 8bytes each
1276	add	%o2, 1, %o2
1277	cmp	%o2, 8
1278	bl,a 	2b
1279	  add	%o4, 8, %o4
1280
1281	membar	#Sync
1282	ldxa	[%g0]ASI_AFSR, %o0		! read accumulated AFSR
1283	srlx	%o0, P_AFSR_CP_SHIFT, %o2
1284	btst	1, %o2
1285	bz	3f
1286	  nop
1287	ldx	[%o1], %o3
1288	or	%o3, %o0, %o3			! aggregate AFSR in cpu private
1289	stx	%o3, [%o1]
12903:
1291	stxa	%o0, [%g0]ASI_AFSR		! clear AFSR
1292	membar	#Sync
1293	stxa	%g1, [%g0]ASI_ESTATE_ERR	! Turn error enable back on
1294	membar	#Sync
1295	retl
1296	wrpr	%g0, %o5, %pstate
1297	SET_SIZE(check_ecache_line)
1298
1299	ENTRY(read_and_clear_afsr)
1300	ldxa	[%g0]ASI_AFSR, %o0
1301	retl
1302	  stxa	%o0, [%g0]ASI_AFSR		! clear AFSR
1303	SET_SIZE(read_and_clear_afsr)
1304
1305/*
1306 * scrubphys - Pass in the aligned physical memory address that you want
1307 * to scrub, along with the ecache size.
1308 *
1309 *	1) Displacement flush the E$ line corresponding to %addr.
1310 *	   The first ldxa guarantees that the %addr is no longer in
1311 *	   M, O, or E (goes to I or S (if instruction fetch also happens).
1312 *	2) "Write" the data using a CAS %addr,%g0,%g0.
1313 *	   The casxa guarantees a transition from I to M or S to M.
1314 *	3) Displacement flush the E$ line corresponding to %addr.
1315 *	   The second ldxa pushes the M line out of the ecache, into the
1316 *	   writeback buffers, on the way to memory.
1317 *	4) The "membar #Sync" pushes the cache line out of the writeback
1318 *	   buffers onto the bus, on the way to dram finally.
1319 *
1320 * This is a modified version of the algorithm suggested by Gary Lauterbach.
1321 * In theory the CAS %addr,%g0,%g0 is supposed to mark the addr's cache line
1322 * as modified, but then we found out that for spitfire, if it misses in the
1323 * E$ it will probably install as an M, but if it hits in the E$, then it
1324 * will stay E, if the store doesn't happen. So the first displacement flush
1325 * should ensure that the CAS will miss in the E$.  Arrgh.
1326 */
1327
1328	ENTRY(scrubphys)
1329	or	%o1, %g0, %o2	! put ecache size in %o2
1330#ifndef HUMMINGBIRD
1331	xor	%o0, %o2, %o1	! calculate alias address
1332	add	%o2, %o2, %o3	! 2 * ecachesize in case
1333				! addr == ecache_flushaddr
1334	sub	%o3, 1, %o3	! -1 == mask
1335	and	%o1, %o3, %o1	! and with xor'd address
1336	set	ecache_flushaddr, %o3
1337	ldx	[%o3], %o3
1338
1339	rdpr	%pstate, %o4
1340	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
1341	wrpr	%o5, %g0, %pstate	! clear IE, AM bits
1342
1343	ldxa	[%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias
1344	casxa	[%o0]ASI_MEM, %g0, %g0
1345	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
1346
1347#else /* HUMMINGBIRD */
1348	/*
1349	 * UltraSPARC-IIe processor supports both 4-way set associative
1350	 * and direct map E$. We need to reconfigure E$ to direct map
1351	 * mode for data load/store before displacement flush. Also, we
1352	 * need to flush all 4 sets of the E$ to ensure that the physaddr
1353	 * has been flushed. Keep the interrupts disabled while flushing
1354	 * E$ in this manner.
1355	 *
1356	 * For flushing a specific physical address, we start at the
1357	 * aliased address and load at set-size stride, wrapping around
1358	 * at 2*ecache-size boundary and skipping fault physical address.
1359	 * It takes 10 loads to guarantee that the physical address has
1360	 * been flushed.
1361	 *
1362	 * Usage:
1363	 *	%o0	physaddr
1364	 *	%o5	physaddr - ecache_flushaddr
1365	 *	%g1	UPA config (restored later)
1366	 *	%g2	E$ set size
1367	 *	%g3	E$ flush address range mask (i.e. 2 * E$ -1)
1368	 *	%g4	#loads to flush phys address
1369	 *	%g5	temp
1370	 */
1371
1372	sethi	%hi(ecache_associativity), %g5
1373	ld	[%g5 + %lo(ecache_associativity)], %g5
1374	udivx	%o2, %g5, %g2	! set size (i.e. ecache_size/#sets)
1375	xor	%o0, %o2, %o1	! calculate alias address
1376	add	%o2, %o2, %g3	! 2 * ecachesize in case
1377				! addr == ecache_flushaddr
1378	sub	%g3, 1, %g3	! 2 * ecachesize -1 == mask
1379	and	%o1, %g3, %o1	! and with xor'd address
1380	sethi	%hi(ecache_flushaddr), %o3
1381	ldx	[%o3 + %lo(ecache_flushaddr)], %o3
1382
1383	rdpr	%pstate, %o4
1384	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
1385	wrpr	%o5, %g0, %pstate	! clear IE, AM bits
1386
1387	! Place E$ in direct map mode for data access
1388	or	%g0, 1, %g5
1389	sllx	%g5, HB_UPA_DMAP_DATA_BIT, %g5
1390	ldxa	[%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later)
1391	or	%g1, %g5, %g5
1392	membar	#Sync
1393	stxa	%g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access
1394	membar	#Sync
1395
1396	! Displace cache line from each set of E$ starting at the
1397	! aliased address. at set-size stride, wrapping at 2*ecache_size
1398	! and skipping load from physaddr. We need 10 loads to flush the
1399	! physaddr from E$.
1400	mov	HB_PHYS_FLUSH_CNT-1, %g4 ! #loads to flush phys addr
1401	sub	%o0, %o3, %o5		! physaddr - ecache_flushaddr
1402	or	%o1, %g0, %g5		! starting aliased offset
14032:
1404	ldxa	[%g5 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
14051:
1406	add	%g5, %g2, %g5		! calculate offset in next set
1407	and	%g5, %g3, %g5		! force offset within aliased range
1408	cmp	%g5, %o5		! skip loads from physaddr
1409	be,pn %ncc, 1b
1410	  nop
1411	brgz,pt	%g4, 2b
1412	  dec	%g4
1413
1414	casxa	[%o0]ASI_MEM, %g0, %g0
1415
1416	! Flush %o0 from ecahe again.
1417	! Need single displacement flush at offset %o1 this time as
1418	! the E$ is already in direct map mode.
1419	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
1420
1421	membar	#Sync
1422	stxa	%g1, [%g0]ASI_UPA_CONFIG ! restore UPA config (DM bits)
1423	membar	#Sync
1424#endif /* HUMMINGBIRD */
1425	wrpr	%g0, %o4, %pstate	! restore earlier pstate register value
1426
1427	retl
1428	membar	#Sync			! move the data out of the load buffer
1429	SET_SIZE(scrubphys)
1430
1431/*
1432 * clearphys - Pass in the aligned physical memory address that you want
1433 * to push out, as a 64 byte block of zeros, from the ecache zero-filled.
1434 * Since this routine does not bypass the ecache, it is possible that
1435 * it could generate a UE error while trying to clear the a bad line.
1436 * This routine clears and restores the error enable flag.
1437 * TBD - Hummingbird may need similar protection
1438 */
1439	ENTRY(clearphys)
1440	or	%o2, %g0, %o3	! ecache linesize
1441	or	%o1, %g0, %o2	! ecache size
1442#ifndef HUMMINGBIRD
1443	or	%o3, %g0, %o4	! save ecache linesize
1444	xor	%o0, %o2, %o1	! calculate alias address
1445	add	%o2, %o2, %o3	! 2 * ecachesize
1446	sub	%o3, 1, %o3	! -1 == mask
1447	and	%o1, %o3, %o1	! and with xor'd address
1448	set	ecache_flushaddr, %o3
1449	ldx	[%o3], %o3
1450	or	%o4, %g0, %o2	! saved ecache linesize
1451
1452	rdpr	%pstate, %o4
1453	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
1454	wrpr	%o5, %g0, %pstate	! clear IE, AM bits
1455
1456	ldxa	[%g0]ASI_ESTATE_ERR, %g1
1457	stxa	%g0, [%g0]ASI_ESTATE_ERR	! disable errors
1458	membar	#Sync
1459
1460	! need to put zeros in the cache line before displacing it
1461
1462	sub	%o2, 8, %o2	! get offset of last double word in ecache line
14631:
1464	stxa	%g0, [%o0 + %o2]ASI_MEM	! put zeros in the ecache line
1465	sub	%o2, 8, %o2
1466	brgez,a,pt %o2, 1b
1467	nop
1468	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
1469	casxa	[%o0]ASI_MEM, %g0, %g0
1470	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
1471
1472	stxa	%g1, [%g0]ASI_ESTATE_ERR	! restore error enable
1473	membar	#Sync
1474
1475#else /* HUMMINGBIRD... */
1476	/*
1477	 * UltraSPARC-IIe processor supports both 4-way set associative
1478	 * and direct map E$. We need to reconfigure E$ to direct map
1479	 * mode for data load/store before displacement flush. Also, we
1480	 * need to flush all 4 sets of the E$ to ensure that the physaddr
1481	 * has been flushed. Keep the interrupts disabled while flushing
1482	 * E$ in this manner.
1483	 *
1484	 * For flushing a specific physical address, we start at the
1485	 * aliased address and load at set-size stride, wrapping around
1486	 * at 2*ecache-size boundary and skipping fault physical address.
1487	 * It takes 10 loads to guarantee that the physical address has
1488	 * been flushed.
1489	 *
1490	 * Usage:
1491	 *	%o0	physaddr
1492	 *	%o5	physaddr - ecache_flushaddr
1493	 *	%g1	UPA config (restored later)
1494	 *	%g2	E$ set size
1495	 *	%g3	E$ flush address range mask (i.e. 2 * E$ -1)
1496	 *	%g4	#loads to flush phys address
1497	 *	%g5	temp
1498	 */
1499
1500	or	%o3, %g0, %o4	! save ecache linesize
1501	sethi	%hi(ecache_associativity), %g5
1502	ld	[%g5 + %lo(ecache_associativity)], %g5
1503	udivx	%o2, %g5, %g2	! set size (i.e. ecache_size/#sets)
1504
1505	xor	%o0, %o2, %o1	! calculate alias address
1506	add	%o2, %o2, %g3	! 2 * ecachesize
1507	sub	%g3, 1, %g3	! 2 * ecachesize -1 == mask
1508	and	%o1, %g3, %o1	! and with xor'd address
1509	sethi	%hi(ecache_flushaddr), %o3
1510	ldx	[%o3 +%lo(ecache_flushaddr)], %o3
1511	or	%o4, %g0, %o2	! saved ecache linesize
1512
1513	rdpr	%pstate, %o4
1514	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
1515	wrpr	%o5, %g0, %pstate	! clear IE, AM bits
1516
1517	! Place E$ in direct map mode for data access
1518	or	%g0, 1, %g5
1519	sllx	%g5, HB_UPA_DMAP_DATA_BIT, %g5
1520	ldxa	[%g0]ASI_UPA_CONFIG, %g1 ! current UPA config (restored later)
1521	or	%g1, %g5, %g5
1522	membar	#Sync
1523	stxa	%g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access
1524	membar	#Sync
1525
1526	! need to put zeros in the cache line before displacing it
1527
1528	sub	%o2, 8, %o2	! get offset of last double word in ecache line
15291:
1530	stxa	%g0, [%o0 + %o2]ASI_MEM	! put zeros in the ecache line
1531	sub	%o2, 8, %o2
1532	brgez,a,pt %o2, 1b
1533	nop
1534
1535	! Displace cache line from each set of E$ starting at the
1536	! aliased address. at set-size stride, wrapping at 2*ecache_size
1537	! and skipping load from physaddr. We need 10 loads to flush the
1538	! physaddr from E$.
1539	mov	HB_PHYS_FLUSH_CNT-1, %g4 ! #loads to flush phys addr
1540	sub	%o0, %o3, %o5		! physaddr - ecache_flushaddr
1541	or	%o1, %g0, %g5		! starting offset
15422:
1543	ldxa	[%g5 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
15443:
1545	add	%g5, %g2, %g5		! calculate offset in next set
1546	and	%g5, %g3, %g5		! force offset within aliased range
1547	cmp	%g5, %o5		! skip loads from physaddr
1548	be,pn %ncc, 3b
1549	  nop
1550	brgz,pt	%g4, 2b
1551	  dec	%g4
1552
1553	casxa	[%o0]ASI_MEM, %g0, %g0
1554
1555	! Flush %o0 from ecahe again.
1556	! Need single displacement flush at offset %o1 this time as
1557	! the E$ is already in direct map mode.
1558	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
1559
1560	membar	#Sync
1561	stxa	%g1, [%g0]ASI_UPA_CONFIG ! restore UPA config (DM bits)
1562	membar	#Sync
1563#endif /* HUMMINGBIRD... */
1564
1565	retl
1566	wrpr	%g0, %o4, %pstate	! restore earlier pstate register value
1567	SET_SIZE(clearphys)
1568
1569/*
1570 * flushecacheline - This is a simpler version of scrubphys
1571 * which simply does a displacement flush of the line in
1572 * question. This routine is mainly used in handling async
1573 * errors where we want to get rid of a bad line in ecache.
1574 * Note that if the line is modified and it has suffered
1575 * data corruption - we are guarantee that the hw will write
1576 * a UE back to mark the page poisoned.
1577 */
1578        ENTRY(flushecacheline)
1579        or      %o1, %g0, %o2   ! put ecache size in %o2
1580#ifndef HUMMINGBIRD
1581        xor     %o0, %o2, %o1   ! calculate alias address
1582        add     %o2, %o2, %o3   ! 2 * ecachesize in case
1583                                ! addr == ecache_flushaddr
1584        sub     %o3, 1, %o3     ! -1 == mask
1585        and     %o1, %o3, %o1   ! and with xor'd address
1586        set     ecache_flushaddr, %o3
1587        ldx     [%o3], %o3
1588
1589        rdpr    %pstate, %o4
1590        andn    %o4, PSTATE_IE | PSTATE_AM, %o5
1591        wrpr    %o5, %g0, %pstate       ! clear IE, AM bits
1592
1593	ldxa	[%g0]ASI_ESTATE_ERR, %g1
1594	stxa	%g0, [%g0]ASI_ESTATE_ERR	! disable errors
1595	membar	#Sync
1596
1597        ldxa    [%o1 + %o3]ASI_MEM, %g0 ! load ecache_flushaddr + alias
1598	membar	#Sync
1599	stxa	%g1, [%g0]ASI_ESTATE_ERR	! restore error enable
1600        membar  #Sync
1601#else /* HUMMINGBIRD */
1602	/*
1603	 * UltraSPARC-IIe processor supports both 4-way set associative
1604	 * and direct map E$. We need to reconfigure E$ to direct map
1605	 * mode for data load/store before displacement flush. Also, we
1606	 * need to flush all 4 sets of the E$ to ensure that the physaddr
1607	 * has been flushed. Keep the interrupts disabled while flushing
1608	 * E$ in this manner.
1609	 *
1610	 * For flushing a specific physical address, we start at the
1611	 * aliased address and load at set-size stride, wrapping around
1612	 * at 2*ecache-size boundary and skipping fault physical address.
1613	 * It takes 10 loads to guarantee that the physical address has
1614	 * been flushed.
1615	 *
1616	 * Usage:
1617	 *	%o0	physaddr
1618	 *	%o5	physaddr - ecache_flushaddr
1619	 *	%g1	error enable register
1620	 *	%g2	E$ set size
1621	 *	%g3	E$ flush address range mask (i.e. 2 * E$ -1)
1622	 *	%g4	UPA config (restored later)
1623	 *	%g5	temp
1624	 */
1625
1626	sethi	%hi(ecache_associativity), %g5
1627	ld	[%g5 + %lo(ecache_associativity)], %g5
1628	udivx	%o2, %g5, %g2	! set size (i.e. ecache_size/#sets)
1629	xor	%o0, %o2, %o1	! calculate alias address
1630	add	%o2, %o2, %g3	! 2 * ecachesize in case
1631				! addr == ecache_flushaddr
1632	sub	%g3, 1, %g3	! 2 * ecachesize -1 == mask
1633	and	%o1, %g3, %o1	! and with xor'd address
1634	sethi	%hi(ecache_flushaddr), %o3
1635	ldx	[%o3 + %lo(ecache_flushaddr)], %o3
1636
1637	rdpr	%pstate, %o4
1638	andn	%o4, PSTATE_IE | PSTATE_AM, %o5
1639	wrpr	%o5, %g0, %pstate	! clear IE, AM bits
1640
1641	! Place E$ in direct map mode for data access
1642	or	%g0, 1, %g5
1643	sllx	%g5, HB_UPA_DMAP_DATA_BIT, %g5
1644	ldxa	[%g0]ASI_UPA_CONFIG, %g4 ! current UPA config (restored later)
1645	or	%g4, %g5, %g5
1646	membar	#Sync
1647	stxa	%g5, [%g0]ASI_UPA_CONFIG ! enable direct map for data access
1648	membar	#Sync
1649
1650	ldxa	[%g0]ASI_ESTATE_ERR, %g1
1651	stxa	%g0, [%g0]ASI_ESTATE_ERR	! disable errors
1652	membar	#Sync
1653
1654	! Displace cache line from each set of E$ starting at the
1655	! aliased address. at set-size stride, wrapping at 2*ecache_size
1656	! and skipping load from physaddr. We need 10 loads to flush the
1657	! physaddr from E$.
1658	mov	HB_PHYS_FLUSH_CNT-1, %g5 ! #loads to flush physaddr
1659	sub	%o0, %o3, %o5		! physaddr - ecache_flushaddr
16602:
1661	ldxa	[%o1 + %o3]ASI_MEM, %g0	! load ecache_flushaddr + alias
16623:
1663	add	%o1, %g2, %o1		! calculate offset in next set
1664	and	%o1, %g3, %o1		! force offset within aliased range
1665	cmp	%o1, %o5		! skip loads from physaddr
1666	be,pn %ncc, 3b
1667	  nop
1668	brgz,pt	%g5, 2b
1669	  dec	%g5
1670
1671	membar	#Sync
1672	stxa	%g1, [%g0]ASI_ESTATE_ERR	! restore error enable
1673        membar  #Sync
1674
1675	stxa	%g4, [%g0]ASI_UPA_CONFIG ! restore UPA config (DM bits)
1676	membar	#Sync
1677#endif /* HUMMINGBIRD */
1678        retl
1679        wrpr    %g0, %o4, %pstate
1680        SET_SIZE(flushecacheline)
1681
1682/*
1683 * ecache_scrubreq_tl1 is the crosstrap handler called at ecache_calls_a_sec Hz
1684 * from the clock CPU.  It atomically increments the outstanding request
1685 * counter and, if there was not already an outstanding request,
1686 * branches to setsoftint_tl1 to enqueue an intr_vec for the given inum.
1687 */
1688
1689	! Register usage:
1690	!
1691	! Arguments:
1692	! %g1 - inum
1693	!
1694	! Internal:
1695	! %g2, %g3, %g5 - scratch
1696	! %g4 - ptr. to spitfire_scrub_misc ec_scrub_outstanding.
1697	! %g6 - setsoftint_tl1 address
1698
1699	ENTRY_NP(ecache_scrubreq_tl1)
1700	set	SFPR_SCRUB_MISC + EC_SCRUB_OUTSTANDING, %g2
1701	GET_CPU_PRIVATE_PTR(%g2, %g4, %g5, 1f);
1702	ld	[%g4], %g2		! cpu's ec_scrub_outstanding.
1703	set	setsoftint_tl1, %g6
1704	!
1705	! no need to use atomic instructions for the following
1706	! increment - we're at tl1
1707	!
1708	add	%g2, 0x1, %g3
1709	brnz,pn	%g2, 1f			! no need to enqueue more intr_vec
1710	  st	%g3, [%g4]		! delay - store incremented counter
1711	jmp	%g6			! setsoftint_tl1(%g1) - queue intr_vec
1712	  nop
1713	! not reached
17141:
1715	retry
1716	SET_SIZE(ecache_scrubreq_tl1)
1717
1718	/*
1719         * write_ec_tag_parity(), which zero's the ecache tag,
1720         * marks the state as invalid and writes good parity to the tag.
1721         * Input %o1= 32 bit E$ index
1722         */
1723        ENTRY(write_ec_tag_parity)
1724        or      %g0, 1, %o4
1725        sllx    %o4, 39, %o4                    ! set bit 40 for e$ tag access
1726        or      %o0, %o4, %o4                 ! %o4 = ecache addr for tag write
1727
1728        rdpr    %pstate, %o5
1729        andn    %o5, PSTATE_IE | PSTATE_AM, %o1
1730        wrpr    %o1, %g0, %pstate               ! clear IE, AM bits
1731
1732        ldxa    [%g0]ASI_ESTATE_ERR, %g1
1733        stxa    %g0, [%g0]ASI_ESTATE_ERR        ! Turn off Error enable
1734        membar  #Sync
1735
1736        ba      1f
1737         nop
1738	/*
1739         * Align on the ecache boundary in order to force
1740         * ciritical code section onto the same ecache line.
1741         */
1742         .align 64
1743
17441:
1745        set     S_EC_PARITY, %o3         	! clear tag, state invalid
1746        sllx    %o3, S_ECPAR_SHIFT, %o3   	! and with good tag parity
1747        stxa    %o3, [%g0]ASI_EC_DIAG           ! update with the above info
1748        stxa    %g0, [%o4]ASI_EC_W
1749        membar  #Sync
1750
1751        stxa    %g1, [%g0]ASI_ESTATE_ERR        ! Turn error enable back on
1752        membar  #Sync
1753        retl
1754        wrpr    %g0, %o5, %pstate
1755        SET_SIZE(write_ec_tag_parity)
1756
1757	/*
1758         * write_hb_ec_tag_parity(), which zero's the ecache tag,
1759         * marks the state as invalid and writes good parity to the tag.
1760         * Input %o1= 32 bit E$ index
1761         */
1762        ENTRY(write_hb_ec_tag_parity)
1763        or      %g0, 1, %o4
1764        sllx    %o4, 39, %o4                    ! set bit 40 for e$ tag access
1765        or      %o0, %o4, %o4               ! %o4 = ecache addr for tag write
1766
1767        rdpr    %pstate, %o5
1768        andn    %o5, PSTATE_IE | PSTATE_AM, %o1
1769        wrpr    %o1, %g0, %pstate               ! clear IE, AM bits
1770
1771        ldxa    [%g0]ASI_ESTATE_ERR, %g1
1772        stxa    %g0, [%g0]ASI_ESTATE_ERR        ! Turn off Error enable
1773        membar  #Sync
1774
1775        ba      1f
1776         nop
1777	/*
1778         * Align on the ecache boundary in order to force
1779         * ciritical code section onto the same ecache line.
1780         */
1781         .align 64
17821:
1783#ifdef HUMMINGBIRD
1784        set     HB_EC_PARITY, %o3         	! clear tag, state invalid
1785        sllx    %o3, HB_ECPAR_SHIFT, %o3   	! and with good tag parity
1786#else /* !HUMMINGBIRD */
1787        set     SB_EC_PARITY, %o3         	! clear tag, state invalid
1788        sllx    %o3, SB_ECPAR_SHIFT, %o3   	! and with good tag parity
1789#endif /* !HUMMINGBIRD */
1790
1791        stxa    %o3, [%g0]ASI_EC_DIAG           ! update with the above info
1792        stxa    %g0, [%o4]ASI_EC_W
1793        membar  #Sync
1794
1795        stxa    %g1, [%g0]ASI_ESTATE_ERR        ! Turn error enable back on
1796        membar  #Sync
1797        retl
1798        wrpr    %g0, %o5, %pstate
1799        SET_SIZE(write_hb_ec_tag_parity)
1800
1801#define	VIS_BLOCKSIZE		64
1802
1803	ENTRY(dtrace_blksuword32)
1804	save	%sp, -SA(MINFRAME + 4), %sp
1805
1806	rdpr	%pstate, %l1
1807	andn	%l1, PSTATE_IE, %l2		! disable interrupts to
1808	wrpr	%g0, %l2, %pstate		! protect our FPU diddling
1809
1810	rd	%fprs, %l0
1811	andcc	%l0, FPRS_FEF, %g0
1812	bz,a,pt	%xcc, 1f			! if the fpu is disabled
1813	wr	%g0, FPRS_FEF, %fprs		! ... enable the fpu
1814
1815	st	%f0, [%fp + STACK_BIAS - 4]	! save %f0 to the stack
18161:
1817	set	0f, %l5
1818        /*
1819         * We're about to write a block full or either total garbage
1820         * (not kernel data, don't worry) or user floating-point data
1821         * (so it only _looks_ like garbage).
1822         */
1823	ld	[%i1], %f0			! modify the block
1824	membar	#Sync
1825	stn	%l5, [THREAD_REG + T_LOFAULT]	! set up the lofault handler
1826	stda	%d0, [%i0]ASI_BLK_COMMIT_S	! store the modified block
1827	membar	#Sync
1828	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler
1829
1830	bz,a,pt	%xcc, 1f
1831	wr	%g0, %l0, %fprs			! restore %fprs
1832
1833	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
18341:
1835
1836	wrpr	%g0, %l1, %pstate		! restore interrupts
1837
1838	ret
1839	restore	%g0, %g0, %o0
1840
18410:
1842	membar	#Sync
1843	stn	%g0, [THREAD_REG + T_LOFAULT]	! remove the lofault handler
1844
1845	bz,a,pt	%xcc, 1f
1846	wr	%g0, %l0, %fprs			! restore %fprs
1847
1848	ld	[%fp + STACK_BIAS - 4], %f0	! restore %f0
18491:
1850
1851	wrpr	%g0, %l1, %pstate		! restore interrupts
1852
1853	/*
1854	 * If tryagain is set (%i2) we tail-call dtrace_blksuword32_err()
1855	 * which deals with watchpoints. Otherwise, just return -1.
1856	 */
1857	brnz,pt	%i2, 1f
1858	nop
1859	ret
1860	restore	%g0, -1, %o0
18611:
1862	call	dtrace_blksuword32_err
1863	restore
1864
1865	SET_SIZE(dtrace_blksuword32)
1866
1867