sfmmu_asm.s revision 1bd453f385f392a0415fad0b14efc9f5a545320f
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#pragma ident	"%Z%%M%	%I%	%E% SMI"
28
29/*
30 * SFMMU primitives.  These primitives should only be used by sfmmu
31 * routines.
32 */
33
34#if defined(lint)
35#include <sys/types.h>
36#else	/* lint */
37#include "assym.h"
38#endif	/* lint */
39
40#include <sys/asm_linkage.h>
41#include <sys/machtrap.h>
42#include <sys/machasi.h>
43#include <sys/sun4asi.h>
44#include <sys/pte.h>
45#include <sys/mmu.h>
46#include <vm/hat_sfmmu.h>
47#include <vm/seg_spt.h>
48#include <sys/machparam.h>
49#include <sys/privregs.h>
50#include <sys/scb.h>
51#include <sys/intreg.h>
52#include <sys/machthread.h>
53#include <sys/intr.h>
54#include <sys/clock.h>
55#include <sys/trapstat.h>
56
57#ifdef TRAPTRACE
58#include <sys/traptrace.h>
59
60/*
61 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
62 */
63#define	TT_TRACE(label)		\
64	ba	label		;\
65	rd	%pc, %g7
66#else
67
68#define	TT_TRACE(label)
69
70#endif /* TRAPTRACE */
71
72#ifndef	lint
73
74#if (TTE_SUSPEND_SHIFT > 0)
75#define	TTE_SUSPEND_INT_SHIFT(reg)				\
76	sllx	reg, TTE_SUSPEND_SHIFT, reg
77#else
78#define	TTE_SUSPEND_INT_SHIFT(reg)
79#endif
80
81#endif /* lint */
82
83#ifndef	lint
84
85/*
86 * Assumes TSBE_TAG is 0
87 * Assumes TSBE_INTHI is 0
88 * Assumes TSBREG.split is 0
89 */
90
91#if TSBE_TAG != 0
92#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
93#endif
94
95#if TSBTAG_INTHI != 0
96#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
97#endif
98
99/*
100 * The following code assumes the tsb is not split.
101 *
102 * With TSBs no longer shared between processes, it's no longer
103 * necessary to hash the context bits into the tsb index to get
104 * tsb coloring; the new implementation treats the TSB as a
105 * direct-mapped, virtually-addressed cache.
106 *
107 * In:
108 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
109 *    tsbbase = base address of TSB (clobbered)
110 *    tagacc = tag access register (clobbered)
111 *    szc = size code of TSB (ro)
112 *    tmp = scratch reg
113 * Out:
114 *    tsbbase = pointer to entry in TSB
115 */
116#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
117	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
118	srlx	tagacc, vpshift, tagacc 				;\
119	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
120	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
121	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
122	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
123	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */
124
125/*
126 * When the kpm TSB is used it is assumed that it is direct mapped
127 * using (vaddr>>vpshift)%tsbsz as the index.
128 *
129 * Note that, for now, the kpm TSB and kernel TSB are the same for
130 * each mapping size.  However that need not always be the case.  If
131 * the trap handlers are updated to search a different TSB for kpm
132 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
133 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
134 *
135 * In:
136 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
137 *    vaddr = virtual address (clobbered)
138 *    tsbp, szc, tmp = scratch
139 * Out:
140 *    tsbp = pointer to entry in TSB
141 */
142#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
143	cmp	vpshift, MMU_PAGESHIFT					;\
144	bne,pn	%icc, 1f		/* branch if large case */	;\
145	  sethi	%hi(kpmsm_tsbsz), szc					;\
146	sethi	%hi(kpmsm_tsbbase), tsbp				;\
147	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
148	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
149	ba,pt	%icc, 2f						;\
150	  nop								;\
1511:	sethi	%hi(kpm_tsbsz), szc					;\
152	sethi	%hi(kpm_tsbbase), tsbp					;\
153	ld	[szc + %lo(kpm_tsbsz)], szc				;\
154	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
1552:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
156
157/*
158 * Lock the TSBE at virtual address tsbep.
159 *
160 * tsbep = TSBE va (ro)
161 * tmp1, tmp2 = scratch registers (clobbered)
162 * label = label to use for branches (text)
163 * %asi = ASI to use for TSB access
164 *
165 * NOTE that we flush the TSB using fast VIS instructions that
166 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
167 * not be treated as a locked entry or we'll get stuck spinning on
168 * an entry that isn't locked but really invalid.
169 */
170
171#if defined(UTSB_PHYS)
172
173#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
174	lda	[tsbep]ASI_MEM, tmp1					;\
175label:									;\
176	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
177	cmp	tmp1, tmp2 						;\
178	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
179	  lda	[tsbep]ASI_MEM, tmp1					;\
180	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
181	cmp	tmp1, tmp2 						;\
182	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
183	  lda	[tsbep]ASI_MEM, tmp1					;\
184	/* tsbe lock acquired */					;\
185	membar #StoreStore
186
187#else /* UTSB_PHYS */
188
189#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
190	lda	[tsbep]%asi, tmp1					;\
191label:									;\
192	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
193	cmp	tmp1, tmp2 						;\
194	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
195	  lda	[tsbep]%asi, tmp1					;\
196	casa	[tsbep]%asi, tmp1, tmp2					;\
197	cmp	tmp1, tmp2 						;\
198	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
199	  lda	[tsbep]%asi, tmp1					;\
200	/* tsbe lock acquired */					;\
201	membar #StoreStore
202
203#endif /* UTSB_PHYS */
204
205/*
206 * Atomically write TSBE at virtual address tsbep.
207 *
208 * tsbep = TSBE va (ro)
209 * tte = TSBE TTE (ro)
210 * tagtarget = TSBE tag (ro)
211 * %asi = ASI to use for TSB access
212 */
213
214#if defined(UTSB_PHYS)
215
216#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
217	add	tsbep, TSBE_TTE, tmp1					;\
218	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
219	membar #StoreStore						;\
220	add	tsbep, TSBE_TAG, tmp1					;\
221	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */
222
223#else /* UTSB_PHYS */
224
225#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
226	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
227	membar #StoreStore						;\
228	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
229
230#endif /* UTSB_PHYS */
231
232/*
233 * Load an entry into the TSB at TL > 0.
234 *
235 * tsbep = pointer to the TSBE to load as va (ro)
236 * tte = value of the TTE retrieved and loaded (wo)
237 * tagtarget = tag target register.  To get TSBE tag to load,
238 *   we need to mask off the context and leave only the va (clobbered)
239 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
240 * tmp1, tmp2 = scratch registers
241 * label = label to use for branches (text)
242 * %asi = ASI to use for TSB access
243 */
244
245#if defined(UTSB_PHYS)
246
247#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
248	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
249	/*								;\
250	 * I don't need to update the TSB then check for the valid tte.	;\
251	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
252	 * we always invalidate the hash table before we unload the TSB.;\
253	 */								;\
254	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
255	ldxa	[ttepa]ASI_MEM, tte					;\
256	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
257	sethi	%hi(TSBTAG_INVALID), tmp2				;\
258	add	tsbep, TSBE_TAG, tmp1					;\
259	brgez,a,pn tte, label/**/f					;\
260	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
261	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
262label:
263
264#else /* UTSB_PHYS */
265
266#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
267	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
268	/*								;\
269	 * I don't need to update the TSB then check for the valid tte.	;\
270	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
271	 * we always invalidate the hash table before we unload the TSB.;\
272	 */								;\
273	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
274	ldxa	[ttepa]ASI_MEM, tte					;\
275	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
276	sethi	%hi(TSBTAG_INVALID), tmp2				;\
277	brgez,a,pn tte, label/**/f					;\
278	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
279	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
280label:
281
282/*
283 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
284 *   for ITLB synthesis.
285 *
286 * tsbep = pointer to the TSBE to load as va (ro)
287 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
288 *   with exec_perm turned off and exec_synth turned on
289 * tagtarget = tag target register.  To get TSBE tag to load,
290 *   we need to mask off the context and leave only the va (clobbered)
291 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
292 * tmp1, tmp2 = scratch registers
293 * label = label to use for branch (text)
294 * %asi = ASI to use for TSB access
295 */
296
297#define	TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
298	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
299	/*								;\
300	 * I don't need to update the TSB then check for the valid tte.	;\
301	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
302	 * we always invalidate the hash table before we unload the TSB.;\
303	 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0	;\
304	 * and exec_synth bit to 1.					;\
305	 */								;\
306	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
307	mov	tte, tmp1						;\
308	ldxa	[ttepa]ASI_MEM, tte					;\
309	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
310	sethi	%hi(TSBTAG_INVALID), tmp2				;\
311	brgez,a,pn tte, label/**/f					;\
312	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
313	or	tte, tmp1, tte						;\
314	andn	tte, TTE_EXECPRM_INT, tte				;\
315	or	tte, TTE_E_SYNTH_INT, tte				;\
316	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
317label:
318
319/*
320 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
321 *
322 * tte = value of the TTE, used to get tte_size bits (ro)
323 * tagaccess = tag access register, used to get 4M pfn bits (ro)
324 * pfn = 4M pfn bits shifted to offset for tte (out)
325 * tmp1 = scratch register
326 * label = label to use for branch (text)
327 */
328
329#define	GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)			\
330	/*								;\
331	 * Get 4M bits from tagaccess for 32M, 256M pagesizes.		;\
332	 * Return them, shifted, in pfn.				;\
333	 */								;\
334	srlx	tagaccess, MMU_PAGESHIFT4M, tagaccess			;\
335	srlx	tte, TTE_SZ_SHFT, tmp		/* isolate the */	;\
336	andcc	tmp, TTE_SZ_BITS, %g0		/* tte_size bits */	;\
337	bz,a,pt %icc, label/**/f		/* if 0, is */		;\
338	  and	tagaccess, 0x7, tagaccess	/* 32M page size */	;\
339	and	tagaccess, 0x3f, tagaccess /* else 256M page size */	;\
340label:									;\
341	sllx	tagaccess, MMU_PAGESHIFT4M, pfn
342
343/*
344 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
345 * for ITLB synthesis.
346 *
347 * tte = value of the TTE, used to get tte_size bits (rw)
348 * tmp1 = scratch register
349 */
350
351#define	SET_TTE4M_PN(tte, tmp)						\
352	/*								;\
353	 * Set 4M pagesize tte bits. 					;\
354	 */								;\
355	set	TTE4M, tmp						;\
356	sllx	tmp, TTE_SZ_SHFT, tmp					;\
357	or	tte, tmp, tte
358
359#endif /* UTSB_PHYS */
360
361/*
362 * Load an entry into the TSB at TL=0.
363 *
364 * tsbep = pointer to the TSBE to load as va (ro)
365 * tteva = pointer to the TTE to load as va (ro)
366 * tagtarget = TSBE tag to load (which contains no context), synthesized
367 * to match va of MMU tag target register only (ro)
368 * tmp1, tmp2 = scratch registers (clobbered)
369 * label = label to use for branches (text)
370 * %asi = ASI to use for TSB access
371 */
372
373#if defined(UTSB_PHYS)
374
375#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
376	/* can't rd tteva after locking tsb because it can tlb miss */	;\
377	ldx	[tteva], tteva			/* load tte */		;\
378	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
379	sethi	%hi(TSBTAG_INVALID), tmp2				;\
380	add	tsbep, TSBE_TAG, tmp1					;\
381	brgez,a,pn tteva, label/**/f					;\
382	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
383	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
384label:
385
386#else /* UTSB_PHYS */
387
388#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
389	/* can't rd tteva after locking tsb because it can tlb miss */	;\
390	ldx	[tteva], tteva			/* load tte */		;\
391	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
392	sethi	%hi(TSBTAG_INVALID), tmp2				;\
393	brgez,a,pn tteva, label/**/f					;\
394	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
395	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
396label:
397
398#endif /* UTSB_PHYS */
399
400/*
401 * Invalidate a TSB entry in the TSB.
402 *
403 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
404 *	 about this earlier to ensure this is true.  Thus when we are
405 *	 directly referencing tsbep below, we are referencing the tte_tag
406 *	 field of the TSBE.  If this  offset ever changes, the code below
407 *	 will need to be modified.
408 *
409 * tsbep = pointer to TSBE as va (ro)
410 * tag = invalidation is done if this matches the TSBE tag (ro)
411 * tmp1 - tmp3 = scratch registers (clobbered)
412 * label = label name to use for branches (text)
413 * %asi = ASI to use for TSB access
414 */
415
416#if defined(UTSB_PHYS)
417
418#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
419	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
420	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
421label/**/1:								;\
422	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
423	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
424	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
425	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
426	cmp	tag, tmp3		/* compare tags */		;\
427	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
428	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
429	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
430	cmp	tmp1, tmp3		/* if not successful */		;\
431	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
432	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
433label/**/2:
434
435#else /* UTSB_PHYS */
436
437#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
438	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
439	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
440label/**/1:								;\
441	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
442	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
443	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
444	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
445	cmp	tag, tmp3		/* compare tags */		;\
446	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
447	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
448	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
449	cmp	tmp1, tmp3		/* if not successful */		;\
450	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
451	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
452label/**/2:
453
454#endif /* UTSB_PHYS */
455
456#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
457#error	- TSB_SOFTSZ_MASK too small
458#endif
459
460
461/*
462 * An implementation of setx which will be hot patched at run time.
463 * since it is being hot patched, there is no value passed in.
464 * Thus, essentially we are implementing
465 *	setx value, tmp, dest
466 * where value is RUNTIME_PATCH (aka 0) in this case.
467 */
468#define	RUNTIME_PATCH_SETX(dest, tmp)					\
469	sethi	%hh(RUNTIME_PATCH), tmp					;\
470	sethi	%lm(RUNTIME_PATCH), dest				;\
471	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
472	or	dest, %lo(RUNTIME_PATCH), dest				;\
473	sllx	tmp, 32, tmp						;\
474	nop				/* for perf reasons */		;\
475	or	tmp, dest, dest		/* contents of patched value */
476
477
478#endif (lint)
479
480
481#if defined (lint)
482
483/*
484 * sfmmu related subroutines
485 */
486
487/*
488 * Use cas, if tte has changed underneath us then reread and try again.
489 * In the case of a retry, it will update sttep with the new original.
490 */
491/* ARGSUSED */
492int
493sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
494{ return(0); }
495
496/*
497 * Use cas, if tte has changed underneath us then return 1, else return 0
498 */
499/* ARGSUSED */
500int
501sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
502{ return(0); }
503
504/* ARGSUSED */
505void
506sfmmu_copytte(tte_t *sttep, tte_t *dttep)
507{}
508
509/*ARGSUSED*/
510struct tsbe *
511sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
512{ return(0); }
513
514/*ARGSUSED*/
515uint64_t
516sfmmu_make_tsbtag(caddr_t va)
517{ return(0); }
518
519#else	/* lint */
520
521	.seg	".data"
522	.global	sfmmu_panic1
523sfmmu_panic1:
524	.asciz	"sfmmu_asm: interrupts already disabled"
525
526	.global	sfmmu_panic3
527sfmmu_panic3:
528	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"
529
530	.global	sfmmu_panic4
531sfmmu_panic4:
532	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"
533
534	.global	sfmmu_panic5
535sfmmu_panic5:
536	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"
537
538
539	ENTRY_NP(sfmmu_modifytte)
540	ldx	[%o2], %g3			/* current */
541	ldx	[%o0], %g1			/* original */
5422:
543	ldx	[%o1], %g2			/* modified */
544	cmp	%g2, %g3			/* is modified = current? */
545	be,a,pt	%xcc,1f				/* yes, don't write */
546	stx	%g3, [%o0]			/* update new original */
547	casx	[%o2], %g1, %g2
548	cmp	%g1, %g2
549	be,pt	%xcc, 1f			/* cas succeeded - return */
550	  nop
551	ldx	[%o2], %g3			/* new current */
552	stx	%g3, [%o0]			/* save as new original */
553	ba,pt	%xcc, 2b
554	  mov	%g3, %g1
5551:	retl
556	membar	#StoreLoad
557	SET_SIZE(sfmmu_modifytte)
558
559	ENTRY_NP(sfmmu_modifytte_try)
560	ldx	[%o1], %g2			/* modified */
561	ldx	[%o2], %g3			/* current */
562	ldx	[%o0], %g1			/* original */
563	cmp	%g3, %g2			/* is modified = current? */
564	be,a,pn %xcc,1f				/* yes, don't write */
565	mov	0, %o1				/* as if cas failed. */
566
567	casx	[%o2], %g1, %g2
568	membar	#StoreLoad
569	cmp	%g1, %g2
570	movne	%xcc, -1, %o1			/* cas failed. */
571	move	%xcc, 1, %o1			/* cas succeeded. */
5721:
573	stx	%g2, [%o0]			/* report "current" value */
574	retl
575	mov	%o1, %o0
576	SET_SIZE(sfmmu_modifytte_try)
577
578	ENTRY_NP(sfmmu_copytte)
579	ldx	[%o0], %g1
580	retl
581	stx	%g1, [%o1]
582	SET_SIZE(sfmmu_copytte)
583
584
585	/*
586	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
587	 * %o0 = TSB base address (in), pointer to TSB entry (out)
588	 * %o1 = vaddr (in)
589	 * %o2 = vpshift (in)
590	 * %o3 = tsb size code (in)
591	 * %o4 = scratch register
592	 */
593	ENTRY_NP(sfmmu_get_tsbe)
594	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
595	retl
596	nop
597	SET_SIZE(sfmmu_get_tsbe)
598
599	/*
600	 * Return a TSB tag for the given va.
601	 * %o0 = va (in/clobbered)
602	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
603	 */
604	ENTRY_NP(sfmmu_make_tsbtag)
605	retl
606	srln	%o0, TTARGET_VA_SHIFT, %o0
607	SET_SIZE(sfmmu_make_tsbtag)
608
609#endif /* lint */
610
611/*
612 * Other sfmmu primitives
613 */
614
615
616#if defined (lint)
617void
618sfmmu_patch_ktsb(void)
619{
620}
621
622void
623sfmmu_kpm_patch_tlbm(void)
624{
625}
626
627void
628sfmmu_kpm_patch_tsbm(void)
629{
630}
631
632/* ARGSUSED */
633void
634sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
635{
636}
637
638/* ARGSUSED */
639void
640sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
641{
642}
643
644/* ARGSUSED */
645void
646sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
647{
648}
649
650/* ARGSUSED */
651void
652sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
653{
654}
655
656#else /* lint */
657
658#define	I_SIZE		4
659
660	ENTRY_NP(sfmmu_fix_ktlb_traptable)
661	/*
662	 * %o0 = start of patch area
663	 * %o1 = size code of TSB to patch
664	 * %o3 = scratch
665	 */
666	/* fix sll */
667	ld	[%o0], %o3			/* get sll */
668	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
669	st	%o3, [%o0]			/* write sll */
670	flush	%o0
671	/* fix srl */
672	add	%o0, I_SIZE, %o0		/* goto next instr. */
673	ld	[%o0], %o3			/* get srl */
674	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
675	st	%o3, [%o0]			/* write srl */
676	retl
677	flush	%o0
678	SET_SIZE(sfmmu_fix_ktlb_traptable)
679
680	ENTRY_NP(sfmmu_fixup_ktsbbase)
681	/*
682	 * %o0 = start of patch area
683	 * %o5 = kernel virtual or physical tsb base address
684	 * %o2, %o3 are used as scratch registers.
685	 */
686	/* fixup sethi instruction */
687	ld	[%o0], %o3
688	srl	%o5, 10, %o2			! offset is bits 32:10
689	or	%o3, %o2, %o3			! set imm22
690	st	%o3, [%o0]
691	/* fixup offset of lduw/ldx */
692	add	%o0, I_SIZE, %o0		! next instr
693	ld	[%o0], %o3
694	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
695	or	%o3, %o2, %o3
696	st	%o3, [%o0]
697	retl
698	flush	%o0
699	SET_SIZE(sfmmu_fixup_ktsbbase)
700
701	ENTRY_NP(sfmmu_fixup_setx)
702	/*
703	 * %o0 = start of patch area
704	 * %o4 = 64 bit value to patch
705	 * %o2, %o3 are used as scratch registers.
706	 *
707	 * Note: Assuming that all parts of the instructions which need to be
708	 *	 patched correspond to RUNTIME_PATCH (aka 0)
709	 *
710	 * Note the implementation of setx which is being patched is as follows:
711	 *
712	 * sethi   %hh(RUNTIME_PATCH), tmp
713	 * sethi   %lm(RUNTIME_PATCH), dest
714	 * or      tmp, %hm(RUNTIME_PATCH), tmp
715	 * or      dest, %lo(RUNTIME_PATCH), dest
716	 * sllx    tmp, 32, tmp
717	 * nop
718	 * or      tmp, dest, dest
719	 *
720	 * which differs from the implementation in the
721	 * "SPARC Architecture Manual"
722	 */
723	/* fixup sethi instruction */
724	ld	[%o0], %o3
725	srlx	%o4, 42, %o2			! bits [63:42]
726	or	%o3, %o2, %o3			! set imm22
727	st	%o3, [%o0]
728	/* fixup sethi instruction */
729	add	%o0, I_SIZE, %o0		! next instr
730	ld	[%o0], %o3
731	sllx	%o4, 32, %o2			! clear upper bits
732	srlx	%o2, 42, %o2			! bits [31:10]
733	or	%o3, %o2, %o3			! set imm22
734	st	%o3, [%o0]
735	/* fixup or instruction */
736	add	%o0, I_SIZE, %o0		! next instr
737	ld	[%o0], %o3
738	srlx	%o4, 32, %o2			! bits [63:32]
739	and	%o2, 0x3ff, %o2			! bits [41:32]
740	or	%o3, %o2, %o3			! set imm
741	st	%o3, [%o0]
742	/* fixup or instruction */
743	add	%o0, I_SIZE, %o0		! next instr
744	ld	[%o0], %o3
745	and	%o4, 0x3ff, %o2			! bits [9:0]
746	or	%o3, %o2, %o3			! set imm
747	st	%o3, [%o0]
748	retl
749	flush	%o0
750	SET_SIZE(sfmmu_fixup_setx)
751
752	ENTRY_NP(sfmmu_fixup_or)
753	/*
754	 * %o0 = start of patch area
755	 * %o4 = 32 bit value to patch
756	 * %o2, %o3 are used as scratch registers.
757	 * Note: Assuming that all parts of the instructions which need to be
758	 *	 patched correspond to RUNTIME_PATCH (aka 0)
759	 */
760	ld	[%o0], %o3
761	and	%o4, 0x3ff, %o2			! bits [9:0]
762	or	%o3, %o2, %o3			! set imm
763	st	%o3, [%o0]
764	retl
765	flush	%o0
766	SET_SIZE(sfmmu_fixup_or)
767
768	ENTRY_NP(sfmmu_fixup_shiftx)
769	/*
770	 * %o0 = start of patch area
771	 * %o4 = signed int immediate value to add to sllx/srlx imm field
772	 * %o2, %o3 are used as scratch registers.
773	 *
774	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
775	 * so we do a simple add.  The caller must be careful to prevent
776	 * overflow, which could easily occur if the initial value is nonzero!
777	 */
778	ld	[%o0], %o3			! %o3 = instruction to patch
779	and	%o3, 0x3f, %o2			! %o2 = existing imm value
780	add	%o2, %o4, %o2			! %o2 = new imm value
781	andn	%o3, 0x3f, %o3			! clear old imm value
782	and	%o2, 0x3f, %o2			! truncate new imm value
783	or	%o3, %o2, %o3			! set new imm value
784	st	%o3, [%o0]			! store updated instruction
785	retl
786	flush	%o0
787	SET_SIZE(sfmmu_fixup_shiftx)
788
789	ENTRY_NP(sfmmu_fixup_mmu_asi)
790	/*
791	 * Patch imm_asi of all ldda instructions in the MMU
792	 * trap handlers.  We search MMU_PATCH_INSTR instructions
793	 * starting from the itlb miss handler (trap 0x64).
794	 * %o0 = address of tt[0,1]_itlbmiss
795	 * %o1 = imm_asi to setup, shifted by appropriate offset.
796	 * %o3 = number of instructions to search
797	 * %o4 = reserved by caller: called from leaf routine
798	 */
7991:	ldsw	[%o0], %o2			! load instruction to %o2
800	brgez,pt %o2, 2f
801	  srl	%o2, 30, %o5
802	btst	1, %o5				! test bit 30; skip if not set
803	bz,pt	%icc, 2f
804	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
805	srlx	%o5, 58, %o5			! isolate op3 part of opcode
806	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
807	brnz,pt	%o5, 2f				! skip if not a match
808	  or	%o2, %o1, %o2			! or in imm_asi
809	st	%o2, [%o0]			! write patched instruction
8102:	dec	%o3
811	brnz,a,pt %o3, 1b			! loop until we're done
812	  add	%o0, I_SIZE, %o0
813	retl
814	flush	%o0
815	SET_SIZE(sfmmu_fixup_mmu_asi)
816
817	/*
818	 * Patch immediate ASI used to access the TSB in the
819	 * trap table.
820	 * inputs: %o0 = value of ktsb_phys
821	 */
822	ENTRY_NP(sfmmu_patch_mmu_asi)
823	mov	%o7, %o4			! save return pc in %o4
824	movrnz	%o0, ASI_QUAD_LDD_PHYS, %o3
825	movrz	%o0, ASI_NQUAD_LD, %o3
826	sll	%o3, 5, %o1			! imm_asi offset
827	mov	6, %o3				! number of instructions
828	sethi	%hi(dktsb), %o0			! to search
829	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
830	  or	%o0, %lo(dktsb), %o0
831	mov	6, %o3				! number of instructions
832	sethi	%hi(dktsb4m), %o0		! to search
833	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
834	  or	%o0, %lo(dktsb4m), %o0
835	mov	6, %o3				! number of instructions
836	sethi	%hi(iktsb), %o0			! to search
837	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
838	  or	%o0, %lo(iktsb), %o0
839	mov	%o4, %o7			! retore return pc -- leaf
840	retl
841	nop
842	SET_SIZE(sfmmu_patch_mmu_asi)
843
844	ENTRY_NP(sfmmu_patch_ktsb)
845	/*
846	 * We need to fix iktsb, dktsb, et. al.
847	 */
848	save	%sp, -SA(MINFRAME), %sp
849	set	ktsb_phys, %o1
850	ld	[%o1], %o4
851	set	ktsb_base, %o5
852	set	ktsb4m_base, %l1
853	brz,pt	%o4, 1f
854	  nop
855	set	ktsb_pbase, %o5
856	set	ktsb4m_pbase, %l1
8571:
858	sethi	%hi(ktsb_szcode), %o1
859	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */
860
861	sethi	%hi(iktsb), %o0
862	call	sfmmu_fix_ktlb_traptable
863	  or	%o0, %lo(iktsb), %o0
864
865	sethi	%hi(dktsb), %o0
866	call	sfmmu_fix_ktlb_traptable
867	  or	%o0, %lo(dktsb), %o0
868
869	sethi	%hi(ktsb4m_szcode), %o1
870	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */
871
872	sethi	%hi(dktsb4m), %o0
873	call	sfmmu_fix_ktlb_traptable
874	  or	%o0, %lo(dktsb4m), %o0
875
876#ifndef sun4v
877	mov	ASI_N, %o2
878	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
879	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
880	sethi	%hi(tsb_kernel_patch_asi), %o0
881	call	sfmmu_fixup_or
882	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
883#endif
884
885	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)
886
887	sethi	%hi(dktsbbase), %o0
888	call	sfmmu_fixup_setx	! patch value of ktsb base addr
889	  or	%o0, %lo(dktsbbase), %o0
890
891	sethi	%hi(iktsbbase), %o0
892	call	sfmmu_fixup_setx	! patch value of ktsb base addr
893	  or	%o0, %lo(iktsbbase), %o0
894
895	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
896	call	sfmmu_fixup_setx	! patch value of ktsb base addr
897	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
898
899#ifdef sun4v
900	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
901	call	sfmmu_fixup_setx	! patch value of ktsb base addr
902	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
903#endif /* sun4v */
904
905	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)
906
907	sethi	%hi(dktsb4mbase), %o0
908	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
909	  or	%o0, %lo(dktsb4mbase), %o0
910
911	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
912	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
913	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
914
915#ifdef sun4v
916	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
917	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
918	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
919#endif /* sun4v */
920
921	set	ktsb_szcode, %o4
922	ld	[%o4], %o4
923	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
924	call	sfmmu_fixup_or		! patch value of ktsb_szcode
925	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
926
927#ifdef sun4v
928	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
929	call	sfmmu_fixup_or		! patch value of ktsb_szcode
930	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
931#endif /* sun4v */
932
933	set	ktsb4m_szcode, %o4
934	ld	[%o4], %o4
935	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
936	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
937	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
938
939#ifdef sun4v
940	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
941	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
942	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
943#endif /* sun4v */
944
945	ret
946	restore
947	SET_SIZE(sfmmu_patch_ktsb)
948
949	ENTRY_NP(sfmmu_kpm_patch_tlbm)
950	/*
951	 * Fixup trap handlers in common segkpm case.  This is reserved
952	 * for future use should kpm TSB be changed to be other than the
953	 * kernel TSB.
954	 */
955	retl
956	nop
957	SET_SIZE(sfmmu_kpm_patch_tlbm)
958
959	ENTRY_NP(sfmmu_kpm_patch_tsbm)
960	/*
961	 * nop the branch to sfmmu_kpm_dtsb_miss_small
962	 * in the case where we are using large pages for
963	 * seg_kpm (and hence must probe the second TSB for
964	 * seg_kpm VAs)
965	 */
966	set	dktsb4m_kpmcheck_small, %o0
967	MAKE_NOP_INSTR(%o1)
968	st	%o1, [%o0]
969	flush	%o0
970	retl
971	nop
972	SET_SIZE(sfmmu_kpm_patch_tsbm)
973
974	ENTRY_NP(sfmmu_patch_utsb)
975#ifdef sun4v
976	retl
977	nop
978#else /* sun4v */
979	/*
980	 * We need to hot patch utsb_vabase and utsb4m_vabase
981	 */
982	save	%sp, -SA(MINFRAME), %sp
983
984	/* patch value of utsb_vabase */
985	set	utsb_vabase, %o1
986	ldx	[%o1], %o4
987	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
988	call	sfmmu_fixup_setx
989	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
990	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
991	call	sfmmu_fixup_setx
992	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
993	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
994	call	sfmmu_fixup_setx
995	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
996
997	/* patch value of utsb4m_vabase */
998	set	utsb4m_vabase, %o1
999	ldx	[%o1], %o4
1000	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1001	call	sfmmu_fixup_setx
1002	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1003	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1004	call	sfmmu_fixup_setx
1005	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1006	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1007	call	sfmmu_fixup_setx
1008	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1009
1010	/*
1011	 * Patch TSB base register masks and shifts if needed.
1012	 * By default the TSB base register contents are set up for 4M slab.
1013	 * If we're using a smaller slab size and reserved VA range we need
1014	 * to patch up those values here.
1015	 */
1016	set	tsb_slab_shift, %o1
1017	set	MMU_PAGESHIFT4M, %o4
1018	ldsw	[%o1], %o3
1019	subcc	%o4, %o3, %o4
1020	bz,pt	%icc, 1f
1021	  /* delay slot safe */
1022
1023	/* patch reserved VA range size if needed. */
1024	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
1025	call	sfmmu_fixup_shiftx
1026	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1027	call	sfmmu_fixup_shiftx
1028	  add	%o0, I_SIZE, %o0
1029	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
1030	call	sfmmu_fixup_shiftx
1031	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1032	call	sfmmu_fixup_shiftx
1033	  add	%o0, I_SIZE, %o0
10341:
1035	/* patch TSBREG_VAMASK used to set up TSB base register */
1036	set	tsb_slab_mask, %o1
1037	lduw	[%o1], %o4
1038	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1039	call	sfmmu_fixup_or
1040	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1041	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1042	call	sfmmu_fixup_or
1043	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1044
1045	ret
1046	restore
1047#endif /* sun4v */
1048	SET_SIZE(sfmmu_patch_utsb)
1049
1050
1051	/*
1052	 * Routine that loads an entry into a tsb using virtual addresses.
1053	 * Locking is required since all cpus can use the same TSB.
1054	 * Note that it is no longer required to have a valid context
1055	 * when calling this function.
1056	 */
1057	ENTRY_NP(sfmmu_load_tsbe)
1058	/*
1059	 * %o0 = pointer to tsbe to load
1060	 * %o1 = tsb tag
1061	 * %o2 = virtual pointer to TTE
1062	 * %o3 = 1 if physical address in %o0 else 0
1063	 */
1064	rdpr	%pstate, %o5
1065#ifdef DEBUG
1066	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1067	bnz,pt 	%icc, 1f			/* disabled, panic	 */
1068	  nop
1069
1070	sethi	%hi(panicstr), %g1
1071	ldx	[%g1 + %lo(panicstr)], %g1
1072	tst	%g1
1073	bnz,pt	%icc, 1f
1074	  nop
1075
1076	save	%sp, -SA(MINFRAME), %sp
1077	sethi	%hi(sfmmu_panic1), %o0
1078	call	panic
1079	 or	%o0, %lo(sfmmu_panic1), %o0
10801:
1081#endif /* DEBUG */
1082
1083	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1084
1085	SETUP_TSB_ASI(%o3, %g3)
1086	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1)
1087
1088	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1089
1090	retl
1091	membar	#StoreStore|#StoreLoad
1092	SET_SIZE(sfmmu_load_tsbe)
1093
1094	/*
1095	 * Flush TSB of a given entry if the tag matches.
1096	 */
1097	ENTRY(sfmmu_unload_tsbe)
1098	/*
1099	 * %o0 = pointer to tsbe to be flushed
1100	 * %o1 = tag to match
1101	 * %o2 = 1 if physical address in %o0 else 0
1102	 */
1103	SETUP_TSB_ASI(%o2, %g1)
1104	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1105	retl
1106	membar	#StoreStore|#StoreLoad
1107	SET_SIZE(sfmmu_unload_tsbe)
1108
1109	/*
1110	 * Routine that loads a TTE into the kpm TSB from C code.
1111	 * Locking is required since kpm TSB is shared among all CPUs.
1112	 */
1113	ENTRY_NP(sfmmu_kpm_load_tsb)
1114	/*
1115	 * %o0 = vaddr
1116	 * %o1 = ttep
1117	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1118	 */
1119	rdpr	%pstate, %o5			! %o5 = saved pstate
1120#ifdef DEBUG
1121	andcc	%o5, PSTATE_IE, %g0		! if interrupts already
1122	bnz,pt	%icc, 1f			! disabled, panic
1123	  nop
1124
1125	sethi	%hi(panicstr), %g1
1126	ldx	[%g1 + %lo(panicstr)], %g1
1127	tst	%g1
1128	bnz,pt	%icc, 1f
1129	  nop
1130
1131	save	%sp, -SA(MINFRAME), %sp
1132	sethi	%hi(sfmmu_panic1), %o0
1133	call	panic
1134	  or	%o0, %lo(sfmmu_panic1), %o0
11351:
1136#endif /* DEBUG */
1137	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts
1138
1139#ifndef sun4v
1140	sethi	%hi(ktsb_phys), %o4
1141	mov	ASI_N, %o3
1142	ld	[%o4 + %lo(ktsb_phys)], %o4
1143	movrnz	%o4, ASI_MEM, %o3
1144	mov	%o3, %asi
1145#endif
1146	mov	%o0, %g1			! %g1 = vaddr
1147
1148	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1149	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1150	/* %g2 = tsbep, %g1 clobbered */
1151
1152	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1153	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1154	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1)
1155
1156	wrpr	%g0, %o5, %pstate		! enable interrupts
1157	retl
1158	  membar #StoreStore|#StoreLoad
1159	SET_SIZE(sfmmu_kpm_load_tsb)
1160
1161	/*
1162	 * Routine that shoots down a TTE in the kpm TSB or in the
1163	 * kernel TSB depending on virtpg. Locking is required since
1164	 * kpm/kernel TSB is shared among all CPUs.
1165	 */
1166	ENTRY_NP(sfmmu_kpm_unload_tsb)
1167	/*
1168	 * %o0 = vaddr
1169	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1170	 */
1171#ifndef sun4v
1172	sethi	%hi(ktsb_phys), %o4
1173	mov	ASI_N, %o3
1174	ld	[%o4 + %lo(ktsb_phys)], %o4
1175	movrnz	%o4, ASI_MEM, %o3
1176	mov	%o3, %asi
1177#endif
1178	mov	%o0, %g1			! %g1 = vaddr
1179
1180	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1181	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1182	/* %g2 = tsbep, %g1 clobbered */
1183
1184	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1185	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1186	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1187
1188	retl
1189	  membar	#StoreStore|#StoreLoad
1190	SET_SIZE(sfmmu_kpm_unload_tsb)
1191
1192#endif /* lint */
1193
1194
1195#if defined (lint)
1196
1197/*ARGSUSED*/
1198pfn_t
1199sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
1200{ return(0); }
1201
1202#else /* lint */
1203
1204	ENTRY_NP(sfmmu_ttetopfn)
1205	ldx	[%o0], %g1			/* read tte */
1206	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1207	/*
1208	 * g1 = pfn
1209	 */
1210	retl
1211	mov	%g1, %o0
1212	SET_SIZE(sfmmu_ttetopfn)
1213
1214#endif /* !lint */
1215
1216
1217#if defined (lint)
1218/*
1219 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
1220 * the hash list.
1221 */
1222/* ARGSUSED */
1223void
1224sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1225	uint64_t hblkpa)
1226{
1227}
1228
1229/*
1230 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
1231 * hash list.
1232 */
1233/* ARGSUSED */
1234void
1235sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1236	uint64_t hblkpa, struct hme_blk *prev_hblkp)
1237{
1238}
1239#else /* lint */
1240
1241/*
1242 * Functions to grab/release hme bucket list lock.  I only use a byte
1243 * instead of the whole int because eventually we might want to
1244 * put some counters on the other bytes (of course, these routines would
1245 * have to change).  The code that grab this lock should execute
1246 * with interrupts disabled and hold the lock for the least amount of time
1247 * possible.
1248 */
1249
1250/*
1251 * Even though hmeh_listlock is updated using pa there's no need to flush
1252 * dcache since hmeh_listlock will be restored to the original value (0)
1253 * before interrupts are reenabled.
1254 */
1255
1256/*
1257 * For sparcv9 hme hash buckets may not be in the nucleus.  hme hash update
1258 * routines still use virtual addresses to update the bucket fields. But they
1259 * must not cause a TLB miss after grabbing the low level bucket lock. To
1260 * achieve this we must make sure the bucket structure is completely within an
1261 * 8K page.
1262 */
1263
1264#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1))
1265#error - the size of hmehash_bucket structure is not power of 2
1266#endif
1267
1268#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi)           \
1269	mov     0xff, tmp2                                      ;\
1270	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1271label1:                                                         ;\
1272	casa    [tmp1]asi, %g0, tmp2                            ;\
1273	brnz,pn tmp2, label1                                    ;\
1274	mov     0xff, tmp2                                      ;\
1275	membar  #LoadLoad
1276
1277#define HMELOCK_EXIT(hmebp, tmp1, asi)                          \
1278	membar  #LoadStore|#StoreStore                          ;\
1279	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1280	sta     %g0, [tmp1]asi
1281
1282	.seg	".data"
1283hblk_add_panic1:
1284	.ascii	"sfmmu_hblk_hash_add: interrupts disabled"
1285	.byte	0
1286hblk_add_panic2:
1287	.ascii	"sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
1288	.byte	0
1289	.align	4
1290	.seg	".text"
1291
1292	ENTRY_NP(sfmmu_hblk_hash_add)
1293	/*
1294	 * %o0 = hmebp
1295	 * %o1 = hmeblkp
1296	 * %o2 = hblkpa
1297	 */
1298	rdpr	%pstate, %o5
1299#ifdef DEBUG
1300	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1301	bnz,pt %icc, 3f				/* disabled, panic	 */
1302	  nop
1303	save	%sp, -SA(MINFRAME), %sp
1304	sethi	%hi(hblk_add_panic1), %o0
1305	call	panic
1306	 or	%o0, %lo(hblk_add_panic1), %o0
1307	ret
1308	restore
1309
13103:
1311#endif /* DEBUG */
1312	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1313	mov	%o2, %g1
1314
1315	/*
1316	 * g1 = hblkpa
1317	 */
1318	ldn	[%o0 + HMEBUCK_HBLK], %o4	/* next hmeblk */
1319	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = next hblkpa */
1320#ifdef	DEBUG
1321	cmp	%o4, %g0
1322	bne,pt %xcc, 1f
1323	 nop
1324	brz,pt %g2, 1f
1325	 nop
1326	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1327	save	%sp, -SA(MINFRAME), %sp
1328	sethi	%hi(hblk_add_panic2), %o0
1329	call	panic
1330	  or	%o0, %lo(hblk_add_panic2), %o0
1331	ret
1332	restore
13331:
1334#endif /* DEBUG */
1335	/*
1336	 * We update hmeblks entries before grabbing lock because the stores
1337	 * could take a tlb miss and require the hash lock.  The buckets
1338	 * are part of the nucleus so we are cool with those stores.
1339	 *
1340	 * if buckets are not part of the nucleus our game is to
1341	 * not touch any other page via va until we drop the lock.
1342	 * This guarantees we won't get a tlb miss before the lock release
1343	 * since interrupts are disabled.
1344	 */
1345	stn	%o4, [%o1 + HMEBLK_NEXT]	/* update hmeblk's next */
1346	stx	%g2, [%o1 + HMEBLK_NEXTPA]	/* update hmeblk's next pa */
1347	HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N)
1348	stn	%o1, [%o0 + HMEBUCK_HBLK]	/* update bucket hblk next */
1349	stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* add hmeblk to list */
1350	HMELOCK_EXIT(%o0, %g2, ASI_N)
1351	retl
1352	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1353	SET_SIZE(sfmmu_hblk_hash_add)
1354
1355	ENTRY_NP(sfmmu_hblk_hash_rm)
1356	/*
1357	 * This function removes an hmeblk from the hash chain.
1358	 * It is written to guarantee we don't take a tlb miss
1359	 * by using physical addresses to update the list.
1360	 *
1361	 * %o0 = hmebp
1362	 * %o1 = hmeblkp
1363	 * %o2 = hmeblkp previous pa
1364	 * %o3 = hmeblkp previous
1365	 */
1366
1367	mov	%o3, %o4			/* o4 = hmeblkp previous */
1368
1369	rdpr	%pstate, %o5
1370#ifdef DEBUG
1371	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1372	bnz,pt 	%icc, 3f			/* disabled, panic	 */
1373	  nop
1374
1375	sethi	%hi(panicstr), %g1
1376	ldx	[%g1 + %lo(panicstr)], %g1
1377	tst	%g1
1378	bnz,pt	%icc, 3f
1379	  nop
1380
1381	sethi	%hi(sfmmu_panic1), %o0
1382	call	panic
1383	 or	%o0, %lo(sfmmu_panic1), %o0
13843:
1385#endif /* DEBUG */
1386	/*
1387	 * disable interrupts, clear Address Mask to access 64 bit physaddr
1388	 */
1389	andn    %o5, PSTATE_IE, %g1
1390	wrpr    %g1, 0, %pstate
1391
1392#ifndef sun4v
1393	sethi   %hi(dcache_line_mask), %g4
1394	ld      [%g4 + %lo(dcache_line_mask)], %g4
1395#endif /* sun4v */
1396
1397	/*
1398	 * if buckets are not part of the nucleus our game is to
1399	 * not touch any other page via va until we drop the lock.
1400	 * This guarantees we won't get a tlb miss before the lock release
1401	 * since interrupts are disabled.
1402	 */
1403	HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N)
1404	ldn	[%o0 + HMEBUCK_HBLK], %g2	/* first hmeblk in list */
1405	cmp	%g2, %o1
1406	bne,pt	%ncc,1f
1407	 mov	ASI_MEM, %asi
1408	/*
1409	 * hmeblk is first on list
1410	 */
1411	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = hmeblk pa */
1412	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1413	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1414	stn	%o3, [%o0 + HMEBUCK_HBLK]	/* write va */
1415	ba,pt	%xcc, 2f
1416	  stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* write pa */
14171:
1418	/* hmeblk is not first on list */
1419
1420	mov	%o2, %g3
1421#ifndef sun4v
1422	GET_CPU_IMPL(%g2)
1423	cmp %g2, CHEETAH_IMPL
1424	bge %icc, hblk_hash_rm_1
1425	and	%o4, %g4, %g2
1426	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev pa from dcache */
1427	add	%o4, HMEBLK_NEXT, %o4
1428	and	%o4, %g4, %g2
1429	ba	hblk_hash_rm_2
1430	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev va from dcache */
1431hblk_hash_rm_1:
1432
1433	stxa	%g0, [%g3]ASI_DC_INVAL		/* flush prev pa from dcache */
1434	membar	#Sync
1435	add     %g3, HMEBLK_NEXT, %g2
1436	stxa	%g0, [%g2]ASI_DC_INVAL		/* flush prev va from dcache */
1437hblk_hash_rm_2:
1438	membar	#Sync
1439#endif /* sun4v */
1440	ldxa	[%g3 + HMEBLK_NEXTPA] %asi, %g2	/* g2 = hmeblk pa */
1441	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1442	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1443	stna	%o3, [%g3 + HMEBLK_NEXT] %asi	/* write va */
1444	stxa	%g1, [%g3 + HMEBLK_NEXTPA] %asi	/* write pa */
14452:
1446	HMELOCK_EXIT(%o0, %g2, ASI_N)
1447	retl
1448	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1449	SET_SIZE(sfmmu_hblk_hash_rm)
1450
1451#endif /* lint */
1452
1453/*
1454 * These macros are used to update global sfmmu hme hash statistics
1455 * in perf critical paths. It is only enabled in debug kernels or
1456 * if SFMMU_STAT_GATHER is defined
1457 */
1458#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1459#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1460	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1461	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
1462	cmp	tmp1, hatid						;\
1463	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
1464	set	sfmmu_global_stat, tmp1					;\
1465	add	tmp1, tmp2, tmp1					;\
1466	ld	[tmp1], tmp2						;\
1467	inc	tmp2							;\
1468	st	tmp2, [tmp1]
1469
1470#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1471	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1472	mov	HATSTAT_KHASH_LINKS, tmp2				;\
1473	cmp	tmp1, hatid						;\
1474	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
1475	set	sfmmu_global_stat, tmp1					;\
1476	add	tmp1, tmp2, tmp1					;\
1477	ld	[tmp1], tmp2						;\
1478	inc	tmp2							;\
1479	st	tmp2, [tmp1]
1480
1481
1482#else /* DEBUG || SFMMU_STAT_GATHER */
1483
1484#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1485
1486#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1487
1488#endif  /* DEBUG || SFMMU_STAT_GATHER */
1489
1490/*
1491 * This macro is used to update global sfmmu kstas in non
1492 * perf critical areas so they are enabled all the time
1493 */
1494#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
1495	sethi	%hi(sfmmu_global_stat), tmp1				;\
1496	add	tmp1, statname, tmp1					;\
1497	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
1498	inc	tmp2							;\
1499	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1500
1501/*
1502 * These macros are used to update per cpu stats in non perf
1503 * critical areas so they are enabled all the time
1504 */
1505#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
1506	ld	[tsbarea + stat], tmp1					;\
1507	inc	tmp1							;\
1508	st	tmp1, [tsbarea + stat]
1509
1510/*
1511 * These macros are used to update per cpu stats in non perf
1512 * critical areas so they are enabled all the time
1513 */
1514#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
1515	lduh	[tsbarea + stat], tmp1					;\
1516	inc	tmp1							;\
1517	stuh	tmp1, [tsbarea + stat]
1518
1519#if defined(KPM_TLBMISS_STATS_GATHER)
1520	/*
1521	 * Count kpm dtlb misses separately to allow a different
1522	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1523	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1524	 */
1525#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
1526	brgez	tagacc, label	/* KPM VA? */				;\
1527	nop								;\
1528	CPU_INDEX(tmp1, tsbma)						;\
1529	sethi	%hi(kpmtsbm_area), tsbma				;\
1530	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
1531	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
1532	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
1533	/* VA range check */						;\
1534	ldx	[tsbma + KPMTSBM_VBASE], val				;\
1535	cmp	tagacc, val						;\
1536	blu,pn	%xcc, label						;\
1537	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
1538	cmp	tagacc, tmp1						;\
1539	bgeu,pn	%xcc, label						;\
1540	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
1541	inc	val							;\
1542	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
1543label:
1544#else
1545#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1546#endif	/* KPM_TLBMISS_STATS_GATHER */
1547
1548#if defined (lint)
1549/*
1550 * The following routines are jumped to from the mmu trap handlers to do
1551 * the setting up to call systrap.  They are separate routines instead of
1552 * being part of the handlers because the handlers would exceed 32
1553 * instructions and since this is part of the slow path the jump
1554 * cost is irrelevant.
1555 */
1556void
1557sfmmu_pagefault(void)
1558{
1559}
1560
1561void
1562sfmmu_mmu_trap(void)
1563{
1564}
1565
1566void
1567sfmmu_window_trap(void)
1568{
1569}
1570
1571void
1572sfmmu_kpm_exception(void)
1573{
1574}
1575
1576#else /* lint */
1577
1578#ifdef	PTL1_PANIC_DEBUG
1579	.seg	".data"
1580	.global	test_ptl1_panic
1581test_ptl1_panic:
1582	.word	0
1583	.align	8
1584
1585	.seg	".text"
1586	.align	4
1587#endif	/* PTL1_PANIC_DEBUG */
1588
1589
1590	ENTRY_NP(sfmmu_pagefault)
1591	USE_ALTERNATE_GLOBALS(%g5)
1592	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1593	rdpr	%tt, %g6
1594	cmp	%g6, FAST_IMMU_MISS_TT
1595	be,a,pn	%icc, 1f
1596	  mov	T_INSTR_MMU_MISS, %g3
1597	cmp	%g6, T_INSTR_MMU_MISS
1598	be,a,pn	%icc, 1f
1599	  mov	T_INSTR_MMU_MISS, %g3
1600	mov	%g5, %g2
1601	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1602	cmp	%g6, FAST_DMMU_MISS_TT
1603	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1604	cmp	%g6, T_DATA_MMU_MISS
1605	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1606
1607#ifdef  PTL1_PANIC_DEBUG
1608	/* check if we want to test the tl1 panic */
1609	sethi	%hi(test_ptl1_panic), %g4
1610	ld	[%g4 + %lo(test_ptl1_panic)], %g1
1611	st	%g0, [%g4 + %lo(test_ptl1_panic)]
1612	cmp	%g1, %g0
1613	bne,a,pn %icc, ptl1_panic
1614	  or	%g0, PTL1_BAD_DEBUG, %g1
1615#endif	/* PTL1_PANIC_DEBUG */
16161:
1617	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
1618	/*
1619	 * g2 = tag access reg
1620	 * g3.l = type
1621	 * g3.h = 0
1622	 */
1623	sethi	%hi(trap), %g1
1624	or	%g1, %lo(trap), %g1
16252:
1626	ba,pt	%xcc, sys_trap
1627	  mov	-1, %g4
1628	SET_SIZE(sfmmu_pagefault)
1629
1630	ENTRY_NP(sfmmu_mmu_trap)
1631	USE_ALTERNATE_GLOBALS(%g5)
1632	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
1633	rdpr	%tt, %g6
1634	cmp	%g6, FAST_IMMU_MISS_TT
1635	be,a,pn	%icc, 1f
1636	  mov	T_INSTR_MMU_MISS, %g3
1637	cmp	%g6, T_INSTR_MMU_MISS
1638	be,a,pn	%icc, 1f
1639	  mov	T_INSTR_MMU_MISS, %g3
1640	mov	%g5, %g2
1641	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1642	cmp	%g6, FAST_DMMU_MISS_TT
1643	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1644	cmp	%g6, T_DATA_MMU_MISS
1645	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
16461:
1647	/*
1648	 * g2 = tag access reg
1649	 * g3 = type
1650	 */
1651	sethi	%hi(sfmmu_tsbmiss_exception), %g1
1652	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
1653	ba,pt	%xcc, sys_trap
1654	  mov	-1, %g4
1655	/*NOTREACHED*/
1656	SET_SIZE(sfmmu_mmu_trap)
1657
1658	ENTRY_NP(sfmmu_suspend_tl)
1659	USE_ALTERNATE_GLOBALS(%g5)
1660	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
1661	rdpr	%tt, %g6
1662	cmp	%g6, FAST_IMMU_MISS_TT
1663	be,a,pn	%icc, 1f
1664	  mov	T_INSTR_MMU_MISS, %g3
1665	mov	%g5, %g2
1666	cmp	%g6, FAST_DMMU_MISS_TT
1667	move	%icc, T_DATA_MMU_MISS, %g3
1668	movne	%icc, T_DATA_PROT, %g3
16691:
1670	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
1671	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
1672	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
1673	ba,pt	%xcc, sys_trap
1674	  mov	PIL_15, %g4
1675	/*NOTREACHED*/
1676	SET_SIZE(sfmmu_suspend_tl)
1677
1678	/*
1679	 * No %g registers in use at this point.
1680	 */
1681	ENTRY_NP(sfmmu_window_trap)
1682	rdpr	%tpc, %g1
1683#ifdef sun4v
1684#ifdef DEBUG
1685	/* We assume previous %gl was 1 */
1686	rdpr	%tstate, %g4
1687	srlx	%g4, TSTATE_GL_SHIFT, %g4
1688	and	%g4, TSTATE_GL_MASK, %g4
1689	cmp	%g4, 1
1690	bne,a,pn %icc, ptl1_panic
1691	  mov	PTL1_BAD_WTRAP, %g1
1692#endif /* DEBUG */
1693	/* user miss at tl>1. better be the window handler or user_rtt */
1694	/* in user_rtt? */
1695	set	rtt_fill_start, %g4
1696	cmp	%g1, %g4
1697	blu,pn %xcc, 6f
1698	 .empty
1699	set	rtt_fill_end, %g4
1700	cmp	%g1, %g4
1701	bgeu,pn %xcc, 6f
1702	 nop
1703	set	fault_rtt_fn1, %g1
1704	wrpr	%g0, %g1, %tnpc
1705	ba,a	7f
17066:
1707	! must save this trap level before descending trap stack
1708	! no need to save %tnpc, either overwritten or discarded
1709	! already got it: rdpr	%tpc, %g1
1710	rdpr	%tstate, %g6
1711	rdpr	%tt, %g7
1712	! trap level saved, go get underlying trap type
1713	rdpr	%tl, %g5
1714	sub	%g5, 1, %g3
1715	wrpr	%g3, %tl
1716	rdpr	%tt, %g2
1717	wrpr	%g5, %tl
1718	! restore saved trap level
1719	wrpr	%g1, %tpc
1720	wrpr	%g6, %tstate
1721	wrpr	%g7, %tt
1722#else /* sun4v */
1723	/* user miss at tl>1. better be the window handler */
1724	rdpr	%tl, %g5
1725	sub	%g5, 1, %g3
1726	wrpr	%g3, %tl
1727	rdpr	%tt, %g2
1728	wrpr	%g5, %tl
1729#endif /* sun4v */
1730	and	%g2, WTRAP_TTMASK, %g4
1731	cmp	%g4, WTRAP_TYPE
1732	bne,pn	%xcc, 1f
1733	 nop
1734	/* tpc should be in the trap table */
1735	set	trap_table, %g4
1736	cmp	%g1, %g4
1737	blt,pn %xcc, 1f
1738	 .empty
1739	set	etrap_table, %g4
1740	cmp	%g1, %g4
1741	bge,pn %xcc, 1f
1742	 .empty
1743	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
1744	add	%g1, WTRAP_FAULTOFF, %g1
1745	wrpr	%g0, %g1, %tnpc
17467:
1747	/*
1748	 * some wbuf handlers will call systrap to resolve the fault
1749	 * we pass the trap type so they figure out the correct parameters.
1750	 * g5 = trap type, g6 = tag access reg
1751	 */
1752
1753	/*
1754	 * only use g5, g6, g7 registers after we have switched to alternate
1755	 * globals.
1756	 */
1757	SET_GL_REG(1)
1758	USE_ALTERNATE_GLOBALS(%g5)
1759	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
1760	rdpr	%tt, %g7
1761	cmp	%g7, FAST_IMMU_MISS_TT
1762	be,a,pn	%icc, ptl1_panic
1763	  mov	PTL1_BAD_WTRAP, %g1
1764	cmp	%g7, T_INSTR_MMU_MISS
1765	be,a,pn	%icc, ptl1_panic
1766	  mov	PTL1_BAD_WTRAP, %g1
1767	mov	T_DATA_PROT, %g5
1768	cmp	%g7, FAST_DMMU_MISS_TT
1769	move	%icc, T_DATA_MMU_MISS, %g5
1770	cmp	%g7, T_DATA_MMU_MISS
1771	move	%icc, T_DATA_MMU_MISS, %g5
1772	! XXXQ AGS re-check out this one
1773	done
17741:
1775	CPU_ADDR(%g1, %g4)
1776	ld	[%g1 + CPU_TL1_HDLR], %g4
1777	brnz,a,pt %g4, sfmmu_mmu_trap
1778	  st	%g0, [%g1 + CPU_TL1_HDLR]
1779	ba,pt	%icc, ptl1_panic
1780	  mov	PTL1_BAD_TRAP, %g1
1781	SET_SIZE(sfmmu_window_trap)
1782
1783	ENTRY_NP(sfmmu_kpm_exception)
1784	/*
1785	 * We have accessed an unmapped segkpm address or a legal segkpm
1786	 * address which is involved in a VAC alias conflict prevention.
1787	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
1788	 * set. If it is, we will instead note that a fault has occurred
1789	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
1790	 * a "retry"). This will step over the faulting instruction.
1791	 * Note that this means that a legal segkpm address involved in
1792	 * a VAC alias conflict prevention (a rare case to begin with)
1793	 * cannot be used in DTrace.
1794	 */
1795	CPU_INDEX(%g1, %g2)
1796	set	cpu_core, %g2
1797	sllx	%g1, CPU_CORE_SHIFT, %g1
1798	add	%g1, %g2, %g1
1799	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
1800	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
1801	bz	0f
1802	or	%g2, CPU_DTRACE_BADADDR, %g2
1803	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
1804	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
1805	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
1806	done
18070:
1808	TSTAT_CHECK_TL1(1f, %g1, %g2)
18091:
1810	USE_ALTERNATE_GLOBALS(%g5)
1811	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
1812	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1813	/*
1814	 * g2=tagacc g3.l=type g3.h=0
1815	 */
1816	sethi	%hi(trap), %g1
1817	or	%g1, %lo(trap), %g1
1818	ba,pt	%xcc, sys_trap
1819	mov	-1, %g4
1820	SET_SIZE(sfmmu_kpm_exception)
1821
1822#endif /* lint */
1823
1824#if defined (lint)
1825
1826void
1827sfmmu_tsb_miss(void)
1828{
1829}
1830
1831void
1832sfmmu_kpm_dtsb_miss(void)
1833{
1834}
1835
1836void
1837sfmmu_kpm_dtsb_miss_small(void)
1838{
1839}
1840
1841#else /* lint */
1842
1843
1844#if (CTX_SIZE != (1 << CTX_SZ_SHIFT))
1845#error - size of context struct does not match with CTX_SZ_SHIFT
1846#endif
1847
1848#if (IMAP_SEG != 0)
1849#error - ism_map->ism_seg offset is not zero
1850#endif
1851
1852/*
1853 * Copies ism mapping for this ctx in param "ism" if this is a ISM
1854 * tlb miss and branches to label "ismhit". If this is not an ISM
1855 * process or an ISM tlb miss it falls thru.
1856 *
1857 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
1858 * this process.
1859 * If so, it will branch to label "ismhit".  If not, it will fall through.
1860 *
1861 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
1862 * so that any other threads of this process will not try and walk the ism
1863 * maps while they are being changed.
1864 *
1865 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
1866 *       will make sure of that. This means we can terminate our search on
1867 *       the first zero mapping we find.
1868 *
1869 * Parameters:
1870 * tagacc	= tag access register (vaddr + ctx) (in)
1871 * tsbmiss	= address of tsb miss area (in)
1872 * ismseg	= contents of ism_seg for this ism map (out)
1873 * ismhat	= physical address of imap_ismhat for this ism map (out)
1874 * tmp1		= scratch reg (CLOBBERED)
1875 * tmp2		= scratch reg (CLOBBERED)
1876 * tmp3		= scratch reg (CLOBBERED)
1877 * label:    temporary labels
1878 * ismhit:   label where to jump to if an ism dtlb miss
1879 * exitlabel:label where to jump if hat is busy due to hat_unshare.
1880 */
1881#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
1882	label, ismhit)							\
1883	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
1884	brlz,pt  tmp1, label/**/3		/* exit if -1 */	;\
1885	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
1886label/**/1:								;\
1887	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
1888	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
1889label/**/2:								;\
1890	brz,pt  ismseg, label/**/3		/* no mapping */	;\
1891	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
1892	lduha	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
1893	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
1894	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
1895	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
1896	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
1897	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
1898	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
1899	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
1900	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
1901	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
1902									;\
1903	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
1904	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
1905	cmp	ismhat, tmp1						;\
1906	bl,pt	%xcc, label/**/2		/* keep looking  */	;\
1907	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
1908									;\
1909	add	tmp3, IBLK_NEXTPA, tmp1					;\
1910	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
1911	brgez,pt tmp1, label/**/1		/* continue if not -1*/	;\
1912	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
1913label/**/3:
1914
1915/*
1916 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
1917 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
1918 * Parameters:
1919 * vaddr = reg containing virtual address
1920 * hatid = reg containing sfmmu pointer
1921 * hmeshift = constant/register to shift vaddr to obtain vapg
1922 * hmebp = register where bucket pointer will be stored
1923 * vapg = register where virtual page will be stored
1924 * tmp1, tmp2 = tmp registers
1925 */
1926
1927
1928#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
1929	vapg, label, tmp1, tmp2)					\
1930	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
1931	brnz,a,pt tmp1, label/**/1					;\
1932	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
1933	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
1934	ba,pt	%xcc, label/**/2					;\
1935	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
1936label/**/1:								;\
1937	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
1938label/**/2:								;\
1939	srlx	tagacc, hmeshift, vapg					;\
1940	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
1941	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
1942	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
1943	add	hmebp, tmp1, hmebp
1944
1945/*
1946 * hashtag includes bspage + hashno (64 bits).
1947 */
1948
1949#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
1950	sllx	vapg, hmeshift, vapg					;\
1951	or	vapg, hashno, hblktag
1952
1953/*
1954 * Function to traverse hmeblk hash link list and find corresponding match.
1955 * The search is done using physical pointers. It returns the physical address
1956 * and virtual address pointers to the hmeblk that matches with the tag
1957 * provided.
1958 * Parameters:
1959 * hmebp	= register that points to hme hash bucket, also used as
1960 *		  tmp reg (clobbered)
1961 * hmeblktag	= register with hmeblk tag match
1962 * hatid	= register with hatid
1963 * hmeblkpa	= register where physical ptr will be stored
1964 * hmeblkva	= register where virtual ptr will be stored
1965 * tmp1		= tmp reg
1966 * label: temporary label
1967 */
1968
1969#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva,	\
1970	tsbarea, tmp1, label)					 	\
1971	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
1972	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
1973	add     hmebp, HMEBUCK_HBLK, hmeblkva				;\
1974	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
1975	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1976label/**/1:								;\
1977	brz,pn	hmeblkva, label/**/2					;\
1978	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1979	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
1980	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
1981	add	hmebp, CLONGSIZE, hmebp					;\
1982	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
1983	xor	tmp1, hmeblktag, tmp1					;\
1984	xor	hmebp, hatid, hmebp					;\
1985	or	hmebp, tmp1, hmebp					;\
1986	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
1987	  add	hmeblkpa, HMEBLK_NEXT, hmebp				;\
1988	ldna	[hmebp]ASI_MEM, hmeblkva	/* hmeblk ptr va */	;\
1989	add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
1990	ba,pt	%xcc, label/**/1					;\
1991	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
1992label/**/2:
1993
1994
1995#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
1996#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
1997#endif
1998
1999/*
2000 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2001 * he offset for the corresponding hment.
2002 * Parameters:
2003 * vaddr = register with virtual address
2004 * hmeblkpa = physical pointer to hme_blk
2005 * hment = register where address of hment will be stored
2006 * hmentoff = register where hment offset will be stored
2007 * label1 = temporary label
2008 */
2009#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, tmp1, label1)	\
2010	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
2011	lda	[hmentoff]ASI_MEM, tmp1 				;\
2012	andcc	tmp1, HBLK_SZMASK, %g0	 /* tmp1 = get_hblk_sz(%g5) */	;\
2013	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
2014	  or	%g0, HMEBLK_HME1, hmentoff				;\
2015	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
2016	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
2017	sllx	tmp1, SFHME_SHIFT, tmp1					;\
2018	add	tmp1, HMEBLK_HME1, hmentoff				;\
2019label1:
2020
2021/*
2022 * GET_TTE is a macro that returns a TTE given a tag and hatid.
2023 *
2024 * tagacc	= tag access register (vaddr + ctx) (in)
2025 * hatid	= sfmmu pointer for TSB miss (in)
2026 * tte		= tte for TLB miss if found, otherwise clobbered (out)
2027 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
2028 * hmeblkva	= VA of hment if found, otherwise clobbered (out)
2029 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
2030 * hmentoff	= temporarily stores hment offset (clobbered)
2031 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
2032 *		  for this page size.
2033 * hashno	= constant/register hash number
2034 * label	= temporary label for branching within macro.
2035 * foundlabel	= label to jump to when tte is found.
2036 * suspendlabel= label to jump to when tte is suspended.
2037 * exitlabel	= label to jump to when tte is not found.  The hmebp lock
2038 *		  is still held at this time.
2039 *
2040 * The caller should set up the tsbmiss->scratch[2] field correctly before
2041 * calling this funciton  (aka TSBMISS_SCRATCH + TSBMISS_HATID)
2042 */
2043#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmentoff, \
2044		hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \
2045									;\
2046	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2047	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2048	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2049		hmeblkpa, label/**/5, hmentoff, hmeblkva)		;\
2050									;\
2051	/*								;\
2052	 * tagacc = tagacc						;\
2053	 * hatid = hatid						;\
2054	 * tsbarea = tsbarea						;\
2055	 * tte   = hmebp (hme bucket pointer)				;\
2056	 * hmeblkpa  = vapg  (virtual page)				;\
2057	 * hmentoff, hmeblkva = scratch					;\
2058	 */								;\
2059	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmentoff)	;\
2060									;\
2061	/*								;\
2062	 * tagacc = tagacc						;\
2063	 * hatid = hatid						;\
2064	 * tte   = hmebp						;\
2065	 * hmeblkpa  = CLOBBERED					;\
2066	 * hmentoff  = htag_bspage & hashno				;\
2067	 * hmeblkva  = scratch						;\
2068	 */								;\
2069	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2070	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
2071	HMEHASH_SEARCH(tte, hmentoff, hatid, hmeblkpa, hmeblkva, 	\
2072		tsbarea, tagacc, label/**/1)				;\
2073	/*								;\
2074	 * tagacc = CLOBBERED						;\
2075	 * tte = CLOBBERED						;\
2076	 * hmeblkpa = hmeblkpa						;\
2077	 * hmeblkva = hmeblkva						;\
2078	 */								;\
2079	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2080	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2081	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2082	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2083	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2084	  nop								;\
2085label/**/4:								;\
2086	/*								;\
2087	 * We have found the hmeblk containing the hment.		;\
2088	 * Now we calculate the corresponding tte.			;\
2089	 *								;\
2090	 * tagacc = tagacc						;\
2091	 * hatid = clobbered						;\
2092	 * tte   = hmebp						;\
2093	 * hmeblkpa  = hmeblkpa						;\
2094	 * hmentoff  = hblktag						;\
2095	 * hmeblkva  = hmeblkva 					;\
2096	 */								;\
2097	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hmentoff, hatid, label/**/2)	;\
2098									;\
2099	add	hmentoff, SFHME_TTE, hmentoff				;\
2100	add	hmeblkpa, hmentoff, hmeblkpa				;\
2101	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2102	add	hmeblkva, hmentoff, hmeblkva				;\
2103	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2104	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmentoff ;\
2105	HMELOCK_EXIT(hmentoff, hmentoff, ASI_MEM)	/* drop lock */	;\
2106	set	TTE_SUSPEND, hmentoff					;\
2107	TTE_SUSPEND_INT_SHIFT(hmentoff)					;\
2108	btst	tte, hmentoff						;\
2109	bz,pt	%xcc, foundlabel					;\
2110	 nop								;\
2111									;\
2112	/*								;\
2113	 * Mapping is suspended, so goto suspend label.			;\
2114	 */								;\
2115	ba,pt	%xcc, suspendlabel					;\
2116	  nop
2117
2118	/*
2119	 * KERNEL PROTECTION HANDLER
2120	 *
2121	 * g1 = tsb8k pointer register (clobbered)
2122	 * g2 = tag access register (ro)
2123	 * g3 - g7 = scratch registers
2124	 *
2125	 * Note: This function is patched at runtime for performance reasons.
2126	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
2127	 */
2128	ENTRY_NP(sfmmu_kprot_trap)
2129	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2130sfmmu_kprot_patch_ktsb_base:
2131	RUNTIME_PATCH_SETX(%g1, %g6)
2132	/* %g1 = contents of ktsb_base or ktsb_pbase */
2133sfmmu_kprot_patch_ktsb_szcode:
2134	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
2135
2136	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2137	! %g1 = First TSB entry pointer, as TSB miss handler expects
2138
2139	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2140sfmmu_kprot_patch_ktsb4m_base:
2141	RUNTIME_PATCH_SETX(%g3, %g6)
2142	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2143sfmmu_kprot_patch_ktsb4m_szcode:
2144	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
2145
2146	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2147	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2148
2149	CPU_TSBMISS_AREA(%g6, %g7)
2150	HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2151	ba,pt	%xcc, sfmmu_tsb_miss_tt
2152	  nop
2153
2154	/*
2155	 * USER PROTECTION HANDLER
2156	 *
2157	 * g1 = tsb8k pointer register (ro)
2158	 * g2 = tag access register (ro)
2159	 * g3 = faulting context (clobbered, currently not used)
2160	 * g4 - g7 = scratch registers
2161	 */
2162	ALTENTRY(sfmmu_uprot_trap)
2163#ifdef sun4v
2164	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2165	/* %g1 = first TSB entry ptr now, %g2 preserved */
2166
2167	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
2168	brlz,pt %g3, 9f			/* check for 2nd TSB */
2169	  mov	%g0, %g3		/* clear second tsbe ptr */
2170
2171	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2172	/* %g3 = second TSB entry ptr now, %g2 preserved */
2173
2174#else /* sun4v */
2175
2176	brgez,pt %g1, 9f		/* check for 2nd TSB */
2177	  mov	%g0, %g3		/* clear second tsbe ptr */
2178
2179	mov	%g2, %g7
2180	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2181	/* %g3 = second TSB entry ptr now, %g7 clobbered */
2182	mov	%g1, %g7
2183	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2184
2185#endif /* sun4v */
21869:
2187	CPU_TSBMISS_AREA(%g6, %g7)
2188	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2189	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
2190	  nop
2191
2192	/*
2193	 * Kernel 8K page iTLB miss.  We also get here if we took a
2194	 * fast instruction access mmu miss trap while running in
2195	 * invalid context.
2196	 *
2197	 * %g1 = 8K TSB pointer register (not used, clobbered)
2198	 * %g2 = tag access register (used)
2199	 * %g3 = faulting context id (used)
2200	 * %g7 = 4M virtual page number for tag matching  (used)
2201	 */
2202	.align	64
2203	ALTENTRY(sfmmu_kitlb_miss)
2204	brnz,pn %g3, tsb_tl0_noctxt
2205	  nop
2206
2207	/* kernel miss */
2208	/* get kernel tsb pointer */
2209	/* we patch the next set of instructions at run time */
2210	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2211iktsbbase:
2212	RUNTIME_PATCH_SETX(%g4, %g5)
2213	/* %g4 = contents of ktsb_base or ktsb_pbase */
2214
2215iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2216	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2217	or	%g4, %g1, %g1			! form tsb ptr
2218	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2219	cmp	%g4, %g7
2220	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
2221	  andcc %g5, TTE_EXECPRM_INT, %g0	! check exec bit
2222	bz,pn	%icc, exec_fault
2223	  nop
2224	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2225	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2226	retry
2227
2228	/*
2229	 * Kernel dTLB miss.  We also get here if we took a fast data
2230	 * access mmu miss trap while running in invalid context.
2231	 *
2232	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2233	 *	We select the TSB miss handler to branch to depending on
2234	 *	the virtual address of the access.  In the future it may
2235	 *	be desirable to separate kpm TTEs into their own TSB,
2236	 *	in which case all that needs to be done is to set
2237	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2238	 *	early in the miss if we detect a kpm VA to a new handler.
2239	 *
2240	 * %g1 = 8K TSB pointer register (not used, clobbered)
2241	 * %g2 = tag access register (used)
2242	 * %g3 = faulting context id (used)
2243	 */
2244	.align	64
2245	ALTENTRY(sfmmu_kdtlb_miss)
2246	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
2247	  nop
2248
2249	/* Gather some stats for kpm misses in the TLB. */
2250	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2251	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2252
2253	/*
2254	 * Get first TSB offset and look for 8K/64K/512K mapping
2255	 * using the 8K virtual page as the index.
2256	 *
2257	 * We patch the next set of instructions at run time;
2258	 * any changes here require sfmmu_patch_ktsb changes too.
2259	 */
2260dktsbbase:
2261	RUNTIME_PATCH_SETX(%g7, %g6)
2262	/* %g7 = contents of ktsb_base or ktsb_pbase */
2263
2264dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2265	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2266
2267	/*
2268	 * At this point %g1 is our index into the TSB.
2269	 * We just masked off enough bits of the VA depending
2270	 * on our TSB size code.
2271	 */
2272	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2273	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2274	cmp	%g6, %g4			! compare tag
2275	bne,pn	%xcc, dktsb4m_kpmcheck_small
2276	  add	%g7, %g1, %g1			/* form tsb ptr */
2277	TT_TRACE(trace_tsbhit)
2278	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2279	/* trapstat expects tte in %g5 */
2280	retry
2281
2282	/*
2283	 * If kpm is using large pages, the following instruction needs
2284	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2285	 * so that we will probe the 4M TSB regardless of the VA.  In
2286	 * the case kpm is using small pages, we know no large kernel
2287	 * mappings are located above 0x80000000.00000000 so we skip the
2288	 * probe as an optimization.
2289	 */
2290dktsb4m_kpmcheck_small:
2291	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2292	  /* delay slot safe, below */
2293
2294	/*
2295	 * Get second TSB offset and look for 4M mapping
2296	 * using 4M virtual page as the TSB index.
2297	 *
2298	 * Here:
2299	 * %g1 = 8K TSB pointer.  Don't squash it.
2300	 * %g2 = tag access register (we still need it)
2301	 */
2302	srlx	%g2, MMU_PAGESHIFT4M, %g3
2303
2304	/*
2305	 * We patch the next set of instructions at run time;
2306	 * any changes here require sfmmu_patch_ktsb changes too.
2307	 */
2308dktsb4mbase:
2309	RUNTIME_PATCH_SETX(%g7, %g6)
2310	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2311dktsb4m:
2312	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2313	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2314
2315	/*
2316	 * At this point %g3 is our index into the TSB.
2317	 * We just masked off enough bits of the VA depending
2318	 * on our TSB size code.
2319	 */
2320	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2321	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2322	cmp	%g6, %g4			! compare tag
2323
2324dktsb4m_tsbmiss:
2325	bne,pn	%xcc, dktsb4m_kpmcheck
2326	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
2327	TT_TRACE(trace_tsbhit)
2328	/* we don't check TTE size here since we assume 4M TSB is separate */
2329	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2330	/* trapstat expects tte in %g5 */
2331	retry
2332
2333	/*
2334	 * So, we failed to find a valid TTE to match the faulting
2335	 * address in either TSB.  There are a few cases that could land
2336	 * us here:
2337	 *
2338	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2339	 *    to sfmmu_tsb_miss_tt to handle the miss.
2340	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2341	 *    4M TSB.  Let segkpm handle it.
2342	 *
2343	 * Note that we shouldn't land here in the case of a kpm VA when
2344	 * kpm_smallpages is active -- we handled that case earlier at
2345	 * dktsb4m_kpmcheck_small.
2346	 *
2347	 * At this point:
2348	 *  g1 = 8K-indexed primary TSB pointer
2349	 *  g2 = tag access register
2350	 *  g3 = 4M-indexed secondary TSB pointer
2351	 */
2352dktsb4m_kpmcheck:
2353	cmp	%g2, %g0
2354	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
2355	  nop
2356	ba,a,pt	%icc, sfmmu_tsb_miss_tt
2357	  nop
2358
2359#ifdef sun4v
2360	/*
2361	 * User instruction miss w/ single TSB.
2362	 * The first probe covers 8K, 64K, and 512K page sizes,
2363	 * because 64K and 512K mappings are replicated off 8K
2364	 * pointer.
2365	 *
2366	 * g1 = tsb8k pointer register
2367	 * g2 = tag access register
2368	 * g3 - g6 = scratch registers
2369	 * g7 = TSB tag to match
2370	 */
2371	.align	64
2372	ALTENTRY(sfmmu_uitlb_fastpath)
2373
2374	SETUP_UTSB_ATOMIC_ASI(%g4, %g5)
2375	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2376	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2377	ba,pn	%xcc, sfmmu_tsb_miss_tt
2378	  mov	%g0, %g3
2379
2380	/*
2381	 * User data miss w/ single TSB.
2382	 * The first probe covers 8K, 64K, and 512K page sizes,
2383	 * because 64K and 512K mappings are replicated off 8K
2384	 * pointer.
2385	 *
2386	 * g1 = tsb8k pointer register
2387	 * g2 = tag access register
2388	 * g3 - g6 = scratch registers
2389	 * g7 = TSB tag to match
2390	 */
2391	.align 64
2392	ALTENTRY(sfmmu_udtlb_fastpath)
2393
2394	SETUP_UTSB_ATOMIC_ASI(%g4, %g6)
2395	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2396	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2397	ba,pn	%xcc, sfmmu_tsb_miss_tt
2398	  mov	%g0, %g3
2399
2400#endif /* sun4v */
2401
2402	/*
2403	 * User instruction miss w/ multiple TSBs.
2404	 * The first probe covers 8K, 64K, and 512K page sizes,
2405	 * because 64K and 512K mappings are replicated off 8K
2406	 * pointer.  Second probe covers 4M page size only.
2407	 *
2408	 * Just like sfmmu_udtlb_slowpath, except:
2409	 *   o Uses ASI_ITLB_IN
2410	 *   o checks for execute permission
2411	 *   o No ISM prediction.
2412	 *
2413	 * g1 = tsb8k pointer register
2414	 * g2 = tag access register
2415	 * g3 - g6 = scratch registers
2416	 * g7 = TSB tag to match
2417	 */
2418	.align	64
2419	ALTENTRY(sfmmu_uitlb_slowpath)
2420
2421#ifdef sun4v
2422	SETUP_UTSB_ATOMIC_ASI(%g4, %g5)
2423	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2424
2425	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2426	/* g4 - g5 = clobbered here */
2427
2428	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2429	/* g1 = first TSB pointer, g3 = second TSB pointer */
2430	srlx	%g2, TAG_VALO_SHIFT, %g7
2431	PROBE_2ND_ITSB(%g3, %g7)
2432	/* NOT REACHED */
2433#else /* sun4v */
2434	mov	%g1, %g3	/* save tsb8k reg in %g3 */
2435	SETUP_UTSB_ATOMIC_ASI(%g4, %g5)
2436	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
2437
2438	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2439	/* g4 - g5 = clobbered here */
2440
2441	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
2442	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
2443	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
2444	/* g1 = first TSB pointer, g3 = second TSB pointer */
2445	srlx	%g2, TAG_VALO_SHIFT, %g7
2446	PROBE_2ND_ITSB(%g3, %g7, isynth)
2447	/* NOT REACHED */
2448#endif /* sun4v */
2449
2450	/*
2451	 * User data miss w/ multiple TSBs.
2452	 * The first probe covers 8K, 64K, and 512K page sizes,
2453	 * because 64K and 512K mappings are replicated off 8K
2454	 * pointer.  Second probe covers 4M page size only.
2455	 *
2456	 * We consider probing for 4M pages first if the VA falls
2457	 * in a range that's likely to be ISM.
2458	 *
2459	 * g1 = tsb8k pointer register
2460	 * g2 = tag access register
2461	 * g3 - g6 = scratch registers
2462	 * g7 = TSB tag to match
2463	 */
2464	.align 64
2465	ALTENTRY(sfmmu_udtlb_slowpath)
2466
2467	SETUP_UTSB_ATOMIC_ASI(%g4, %g6)
2468
2469	/*
2470	 * Check for ISM.  If it exists, look for 4M mappings in the second TSB
2471	 * first, then probe for other mappings in the first TSB if that fails.
2472	 */
2473	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
2474	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
2475	  mov	%g1, %g3
2476
2477udtlb_miss_probefirst:
2478	/*
2479	 * g1 = 8K TSB pointer register
2480	 * g2 = tag access register
2481	 * g3 = (potentially) second TSB entry ptr
2482	 * g6 = ism pred.
2483	 * g7 = vpg_4m
2484	 */
2485#ifdef sun4v
2486	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2487	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2488
2489	/*
2490	 * Here:
2491	 *   g1 = first TSB pointer
2492	 *   g2 = tag access reg
2493	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2494	 */
2495	brgz,pn	%g6, sfmmu_tsb_miss_tt
2496	  nop
2497#else /* sun4v */
2498	mov	%g1, %g4
2499	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
2500	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2501
2502	/*
2503	 * Here:
2504	 *   g1 = first TSB pointer
2505	 *   g2 = tag access reg
2506	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2507	 */
2508	brgz,pn	%g6, sfmmu_tsb_miss_tt
2509	  nop
2510	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
2511	/* fall through in 8K->4M probe order */
2512#endif /* sun4v */
2513
2514udtlb_miss_probesecond:
2515	/*
2516	 * Look in the second TSB for the TTE
2517	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
2518	 * g2 = tag access reg
2519	 * g3 = 8K TSB pointer register
2520	 * g6 = ism pred.
2521	 * g7 = vpg_4m
2522	 */
2523#ifdef sun4v
2524	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
2525	/* tagacc (%g2) not destroyed */
2526	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2527	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
2528#else
2529	mov	%g3, %g7
2530	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
2531	/* %g2 clobbered, %g3 =second tsbe ptr */
2532	mov	MMU_TAG_ACCESS, %g2
2533	ldxa	[%g2]ASI_DMMU, %g2
2534#endif
2535
2536	srlx	%g2, TAG_VALO_SHIFT, %g7
2537	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2538	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
2539	brgz,pn	%g6, udtlb_miss_probefirst
2540	  nop
2541
2542	/* fall through to sfmmu_tsb_miss_tt */
2543
2544	ALTENTRY(sfmmu_tsb_miss_tt)
2545	TT_TRACE(trace_tsbmiss)
2546	/*
2547	 * We get here if there is a TSB miss OR a write protect trap.
2548	 *
2549	 * g1 = First TSB entry pointer
2550	 * g2 = tag access register
2551	 * g3 = 4M TSB entry pointer; NULL if no 2nd TSB
2552	 * g4 - g7 = scratch registers
2553	 */
2554
2555	ALTENTRY(sfmmu_tsb_miss)
2556
2557	/*
2558	 * If trapstat is running, we need to shift the %tpc and %tnpc to
2559	 * point to trapstat's TSB miss return code (note that trapstat
2560	 * itself will patch the correct offset to add).
2561	 */
2562	rdpr	%tl, %g7
2563	cmp	%g7, 1
2564	ble,pt	%xcc, 0f
2565	  sethi	%hi(KERNELBASE), %g6
2566	rdpr	%tpc, %g7
2567	or	%g6, %lo(KERNELBASE), %g6
2568	cmp	%g7, %g6
2569	bgeu,pt	%xcc, 0f
2570	/* delay slot safe */
2571
2572	ALTENTRY(tsbmiss_trapstat_patch_point)
2573	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
2574	wrpr	%g7, %tpc
2575	add	%g7, 4, %g7
2576	wrpr	%g7, %tnpc
25770:
2578	CPU_TSBMISS_AREA(%g6, %g7)
2579
2580	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save first tsb pointer */
2581	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save second tsb pointer */
2582
2583	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
2584	brz,a,pn %g3, 1f			/* skip ahead if kernel */
2585	  ldn	[%g6 + TSBMISS_KHATID], %g7
2586	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
2587	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
2588
2589	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
2590
2591	cmp	%g3, INVALID_CONTEXT
2592	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
2593	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
2594
2595	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
2596	/*
2597	 * The miss wasn't in an ISM segment.
2598	 *
2599	 * %g1 %g3, %g4, %g5, %g7 all clobbered
2600	 * %g2 = tag access (vaddr + ctx)
2601	 */
2602
2603	ba,pt	%icc, 2f
2604	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
2605
26061:
2607	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
2608	/*
2609	 * 8K and 64K hash.
2610	 */
26112:
2612
2613	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2614		MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte,
2615		sfmmu_suspend_tl, tsb_512K)
2616	/* NOT REACHED */
2617
2618tsb_512K:
2619	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2620	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2621	brz,pn	%g5, 3f
2622	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2623	and	%g4, HAT_512K_FLAG, %g5
2624
2625	/*
2626	 * Note that there is a small window here where we may have
2627	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
2628	 * flag yet, so we will skip searching the 512k hash list.
2629	 * In this case we will end up in pagefault which will find
2630	 * the mapping and return.  So, in this instance we will end up
2631	 * spending a bit more time resolving this TSB miss, but it can
2632	 * only happen once per process and even then, the chances of that
2633	 * are very small, so it's not worth the extra overhead it would
2634	 * take to close this window.
2635	 */
2636	brz,pn	%g5, tsb_4M
2637	  nop
26383:
2639	/*
2640	 * 512K hash
2641	 */
2642
2643	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2644		MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte,
2645		sfmmu_suspend_tl, tsb_4M)
2646	/* NOT REACHED */
2647
2648tsb_4M:
2649	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2650	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2651	brz,pn	%g5, 4f
2652	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2653	and	%g4, HAT_4M_FLAG, %g5
2654	brz,pn	%g5, tsb_32M
2655	  nop
26564:
2657	/*
2658	 * 4M hash
2659	 */
2660
2661	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2662		MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte,
2663		sfmmu_suspend_tl, tsb_32M)
2664	/* NOT REACHED */
2665
2666tsb_32M:
2667#ifndef sun4v
2668	GET_CPU_IMPL(%g5)
2669	cmp	%g5, PANTHER_IMPL
2670	bne,pt	%xcc, tsb_pagefault
2671	  nop
2672#endif
2673
2674	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2675	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2676#ifdef sun4v
2677	brz,pn	%g5, 6f
2678#else
2679	brz,pn	%g5, tsb_pagefault
2680#endif
2681	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2682	and	%g4, HAT_32M_FLAG, %g5
2683	brz,pn	%g5, tsb_256M
2684	  nop
26855:
2686	/*
2687	 * 32M hash
2688	 */
2689
2690	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2691		MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte,
2692		sfmmu_suspend_tl, tsb_256M)
2693	/* NOT REACHED */
2694
2695tsb_256M:
2696	lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2697	and	%g4, HAT_256M_FLAG, %g5
2698	brz,pn	%g5, tsb_pagefault
2699	  nop
27006:
2701	/*
2702	 * 256M hash
2703	 */
2704
2705	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2706	    MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte,
2707	    sfmmu_suspend_tl, tsb_pagefault)
2708	/* NOT REACHED */
2709
2710tsb_checktte:
2711	/*
2712	 * g3 = tte
2713	 * g4 = tte pa
2714	 * g5 = tte va
2715	 * g6 = tsbmiss area
2716	 */
2717	brgez,pn %g3, tsb_pagefault	/* if tte invalid branch */
2718	  nop
2719
2720tsb_validtte:
2721	/*
2722	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
2723	 */
2724	rdpr	%tt, %g7
2725	cmp	%g7, FAST_PROT_TT
2726	bne,pt	%icc, 4f
2727	  nop
2728
2729	TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod,
2730	    tsb_protfault)
2731
2732	rdpr	%tt, %g5
2733	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
2734	ba,pt	%xcc, tsb_update_tl1
2735	  nop
2736
27374:
2738	/*
2739	 * If ITLB miss check exec bit.
2740	 * If not set treat as invalid TTE.
2741	 */
2742	cmp     %g7, T_INSTR_MMU_MISS
2743	be,pn	%icc, 5f
2744	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
2745	cmp     %g7, FAST_IMMU_MISS_TT
2746	bne,pt %icc, 3f
2747	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
27485:
2749	bz,pn %icc, tsb_protfault
2750	  nop
2751
27523:
2753	/*
2754	 * Set reference bit if not already set
2755	 */
2756	TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref)
2757
2758	/*
2759	 * Now, load into TSB/TLB.  At this point:
2760	 * g3 = tte
2761	 * g4 = patte
2762	 * g6 = tsbmiss area
2763	 */
2764	rdpr	%tt, %g5
2765#ifdef sun4v
2766	MMU_FAULT_STATUS_AREA(%g2)
2767	cmp	%g5, T_INSTR_MMU_MISS
2768	be,a,pt	%icc, 9f
2769	  nop
2770	cmp	%g5, FAST_IMMU_MISS_TT
2771	be,a,pt	%icc, 9f
2772	  nop
2773	add	%g2, MMFSA_D_, %g2
27749:
2775	ldx	[%g2 + MMFSA_CTX_], %g7
2776	sllx	%g7, TTARGET_CTX_SHIFT, %g7
2777	ldx	[%g2 + MMFSA_ADDR_], %g2
2778	srlx	%g2, TTARGET_VA_SHIFT, %g2
2779	or	%g2, %g7, %g2
2780#else
2781	cmp	%g5, FAST_IMMU_MISS_TT
2782	be,a,pt	%icc, tsb_update_tl1
2783	  ldxa	[%g0]ASI_IMMU, %g2
2784	ldxa	[%g0]ASI_DMMU, %g2
2785#endif
2786tsb_update_tl1:
2787	srlx	%g2, TTARGET_CTX_SHIFT, %g7
2788	brz,pn	%g7, tsb_kernel
2789#ifdef sun4v
2790	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
2791#else
2792	  srlx	%g3, TTE_SZ_SHFT, %g7
2793#endif
2794
2795tsb_user:
2796#ifdef sun4v
2797	cmp	%g7, TTE4M
2798	bge,pn	%icc, tsb_user4m
2799	  nop
2800#else
2801	cmp	%g7, TTESZ_VALID | TTE4M
2802	be,pn	%icc, tsb_user4m
2803	  srlx	%g3, TTE_SZ2_SHFT, %g7
2804	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
2805	bnz,a,pn %icc, tsb_user_pn_synth
2806	  cmp	%g5, FAST_IMMU_MISS_TT
2807#endif
2808
2809tsb_user8k:
2810	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = first TSB ptr
2811
2812#ifndef sun4v
2813	mov	ASI_N, %g7	! user TSBs always accessed by VA
2814	mov	%g7, %asi
2815#endif /* sun4v */
2816
2817	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5)
2818
2819#ifdef sun4v
2820	cmp	%g5, T_INSTR_MMU_MISS
2821	be,a,pn	%xcc, 9f
2822	  mov	%g3, %g5
2823#endif /* sun4v */
2824	cmp	%g5, FAST_IMMU_MISS_TT
2825	be,pn	%xcc, 9f
2826	  mov	%g3, %g5
2827
2828	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2829	! trapstat wants TTE in %g5
2830	retry
28319:
2832	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2833	! trapstat wants TTE in %g5
2834	retry
2835
2836tsb_user4m:
2837	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */
28384:
2839	brz,pn	%g1, 5f	/* Check to see if we have 2nd TSB programmed */
2840	  nop
2841
2842#ifndef sun4v
2843        mov     ASI_N, %g7      ! user TSBs always accessed by VA
2844        mov     %g7, %asi
2845#endif
2846
2847        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6)
2848
28495:
2850#ifdef sun4v
2851        cmp     %g5, T_INSTR_MMU_MISS
2852        be,a,pn %xcc, 9f
2853          mov   %g3, %g5
2854#endif /* sun4v */
2855        cmp     %g5, FAST_IMMU_MISS_TT
2856        be,pn   %xcc, 9f
2857        mov     %g3, %g5
2858
2859        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2860        ! trapstat wants TTE in %g5
2861        retry
28629:
2863        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2864        ! trapstat wants TTE in %g5
2865        retry
2866
2867#ifndef sun4v
2868	/*
2869	 * Panther ITLB synthesis.
2870	 * The Panther 32M and 256M ITLB code simulates these two large page
2871	 * sizes with 4M pages, to provide support for programs, for example
2872	 * Java, that may copy instructions into a 32M or 256M data page and
2873	 * then execute them. The code below generates the 4M pfn bits and
2874	 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
2875	 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
2876	 * are ignored by the hardware.
2877	 *
2878	 * Now, load into TSB/TLB.  At this point:
2879	 * g2 = tagtarget
2880	 * g3 = tte
2881	 * g4 = patte
2882	 * g5 = tt
2883	 * g6 = tsbmiss area
2884	 */
2885tsb_user_pn_synth:
2886	be,pt	%xcc, tsb_user_itlb_synth	/* ITLB miss */
2887	  andcc %g3, TTE_EXECPRM_INT, %g0	/* is execprm bit set */
2888	bz,pn %icc, 4b				/* if not, been here before */
2889	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	/* g1 = tsbp */
2890	brz,a,pn %g1, 5f			/* no 2nd tsb */
2891	  mov	%g3, %g5
2892
2893	mov	MMU_TAG_ACCESS, %g7
2894	ldxa	[%g7]ASI_DMMU, %g6		/* get tag access va */
2895	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)	/* make 4M pfn offset */
2896
2897	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
2898	mov	%g7, %asi
2899	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */
29005:
2901        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2902        retry
2903
2904tsb_user_itlb_synth:
2905	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */
2906
2907	mov	MMU_TAG_ACCESS, %g7
2908	ldxa	[%g7]ASI_IMMU, %g6		/* get tag access va */
2909	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)	/* make 4M pfn offset */
2910	brz,a,pn %g1, 7f	/* Check to see if we have 2nd TSB programmed */
2911	  or	%g5, %g3, %g5			/* add 4M bits to TTE */
2912
2913	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
2914	mov	%g7, %asi
2915	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */
29167:
2917	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
2918        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2919        retry
2920#endif
2921
2922tsb_kernel:					! no 32M or 256M support
2923#ifdef sun4v
2924	cmp	%g7, TTE4M
2925#else
2926	cmp	%g7, TTESZ_VALID | TTE4M
2927#endif
2928	be,pn	%icc, 5f
2929	  nop
2930	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8k tsbptr
2931	ba,pt	%xcc, 6f
2932	  nop
29335:
2934	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4m tsbptr
2935	brz,pn	%g1, 3f		/* skip programming if 4m TSB ptr is NULL */
2936	  nop
29376:
2938#ifndef sun4v
2939tsb_kernel_patch_asi:
2940	or	%g0, RUNTIME_PATCH, %g6
2941	mov	%g6, %asi	! XXX avoid writing to %asi !!
2942#endif
2943	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7)
29443:
2945#ifdef sun4v
2946	cmp	%g5, T_INSTR_MMU_MISS
2947	be,a,pn	%icc, 1f
2948	  mov	%g3, %g5			! trapstat wants TTE in %g5
2949#endif /* sun4v */
2950	cmp	%g5, FAST_IMMU_MISS_TT
2951	be,pn	%icc, 1f
2952	  mov	%g3, %g5			! trapstat wants TTE in %g5
2953	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2954	! trapstat wants TTE in %g5
2955	retry
29561:
2957	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2958	! trapstat wants TTE in %g5
2959	retry
2960
2961tsb_ism:
2962	/*
2963	 * This is an ISM [i|d]tlb miss.  We optimize for largest
2964	 * page size down to smallest.
2965	 *
2966	 * g2 = vaddr + ctx	aka tag access register
2967	 * g3 = ismmap->ism_seg
2968	 * g4 = physical address of ismmap->ism_sfmmu
2969	 * g6 = tsbmiss area
2970	 */
2971	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
2972	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
2973	  mov	PTL1_BAD_ISM, %g1
2974						/* g5 = pa of imap_vb_shift */
2975	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
2976	lduha	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
2977	srlx	%g3, %g4, %g3			/* clr size field */
2978	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
2979	sllx	%g3, %g4, %g3			/* g3 = ism vbase */
2980	and	%g2, %g1, %g4			/* g4 = ctx number */
2981	andn	%g2, %g1, %g1			/* g1 = tlb miss vaddr */
2982	sub	%g1, %g3, %g2			/* g2 = offset in ISM seg */
2983	or	%g2, %g4, %g2			/* g2 = tagacc (vaddr + ctx) */
2984
2985	/*
2986	 * ISM pages are always locked down.
2987	 * If we can't find the tte then pagefault
2988	 * and let the spt segment driver resovle it.
2989	 *
2990	 * g2 = ISM vaddr (offset in ISM seg)
2991	 * g6 = tsb miss area
2992	 * g7 = ISM hatid
2993	 */
2994	sub	%g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
2995	lduha	[%g5]ASI_MEM, %g4		/* g5 = pa of imap_hatflags */
2996	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
2997	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
2998	  nop
2999
3000tsb_ism_32M:
3001	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
3002	brz,pn	%g5, tsb_ism_256M
3003	  nop
3004
3005	/*
3006	 * 32M hash.
3007	 */
3008
3009	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M,
3010	    TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3011	    tsb_ism_4M)
3012	/* NOT REACHED */
3013
3014tsb_ism_32M_found:
3015	brlz,pt %g3, tsb_validtte
3016	  nop
3017	ba,pt	%xcc, tsb_ism_4M
3018	  nop
3019
3020tsb_ism_256M:
3021	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
3022	brz,a,pn %g5, ptl1_panic
3023	  mov	PTL1_BAD_ISM, %g1
3024
3025	/*
3026	 * 256M hash.
3027	 */
3028	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M,
3029	    TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3030	    tsb_ism_4M)
3031
3032tsb_ism_256M_found:
3033	brlz,pt %g3, tsb_validtte
3034	  nop
3035
3036tsb_ism_4M:
3037	/*
3038	 * 4M hash.
3039	 */
3040	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M,
3041	    TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3042	    tsb_ism_8K)
3043	/* NOT REACHED */
3044
3045tsb_ism_4M_found:
3046	brlz,pt %g3, tsb_validtte
3047	  nop
3048
3049tsb_ism_8K:
3050	/*
3051	 * 8K and 64K hash.
3052	 */
3053
3054	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K,
3055	    TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3056	    tsb_pagefault)
3057	/* NOT REACHED */
3058
3059tsb_ism_8K_found:
3060	brlz,pt	%g3, tsb_validtte
3061	  nop
3062
3063tsb_pagefault:
3064	rdpr	%tt, %g7
3065	cmp	%g7, FAST_PROT_TT
3066	be,a,pn	%icc, tsb_protfault
3067	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt
3068
3069tsb_protfault:
3070	/*
3071	 * we get here if we couldn't find a valid tte in the hash.
3072	 *
3073	 * If user and we are at tl>1 we go to window handling code.
3074	 *
3075	 * If kernel and the fault is on the same page as our stack
3076	 * pointer, then we know the stack is bad and the trap handler
3077	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3078	 *
3079	 * If this is a kernel trap and tl>1, panic.
3080	 *
3081	 * Otherwise we call pagefault.
3082	 */
3083	cmp	%g7, FAST_IMMU_MISS_TT
3084#ifdef sun4v
3085	MMU_FAULT_STATUS_AREA(%g4)
3086	ldx	[%g4 + MMFSA_I_CTX], %g5
3087	ldx	[%g4 + MMFSA_D_CTX], %g4
3088	move	%icc, %g5, %g4
3089	cmp	%g7, T_INSTR_MMU_MISS
3090	move	%icc, %g5, %g4
3091#else
3092	mov	MMU_TAG_ACCESS, %g4
3093	ldxa	[%g4]ASI_DMMU, %g2
3094	ldxa	[%g4]ASI_IMMU, %g5
3095	move	%icc, %g5, %g2
3096	cmp	%g7, T_INSTR_MMU_MISS
3097	move	%icc, %g5, %g2
3098	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
3099#endif
3100	brnz,pn	%g4, 3f				/* skip if not kernel */
3101	  rdpr	%tl, %g5
3102
3103	add	%sp, STACK_BIAS, %g3
3104	srlx	%g3, MMU_PAGESHIFT, %g3
3105	srlx	%g2, MMU_PAGESHIFT, %g4
3106	cmp	%g3, %g4
3107	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
3108	  mov	PTL1_BAD_STACK, %g1
3109
3110	cmp	%g5, 1
3111	ble,pt	%icc, 2f
3112	  nop
3113	TSTAT_CHECK_TL1(2f, %g1, %g2)
3114	rdpr	%tt, %g2
3115	cmp	%g2, FAST_PROT_TT
3116	mov	PTL1_BAD_KPROT_FAULT, %g1
3117	movne	%icc, PTL1_BAD_KMISS, %g1
3118	ba,pt	%icc, ptl1_panic
3119	  nop
3120
31212:
3122	/*
3123	 * We are taking a pagefault in the kernel on a kernel address.  If
3124	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
3125	 * want to call sfmmu_pagefault -- we will instead note that a fault
3126	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
3127	 * (instead of a "retry").  This will step over the faulting
3128	 * instruction.
3129	 */
3130	CPU_INDEX(%g1, %g2)
3131	set	cpu_core, %g2
3132	sllx	%g1, CPU_CORE_SHIFT, %g1
3133	add	%g1, %g2, %g1
3134	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3135	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3136	bz	sfmmu_pagefault
3137	or	%g2, CPU_DTRACE_BADADDR, %g2
3138	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3139	GET_MMU_D_ADDR(%g3, %g4)
3140	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3141	done
3142
31433:
3144	cmp	%g5, 1
3145	ble,pt	%icc, 4f
3146	  nop
3147	TSTAT_CHECK_TL1(4f, %g1, %g2)
3148	ba,pt	%icc, sfmmu_window_trap
3149	  nop
3150
31514:
3152	/*
3153	 * We are taking a pagefault on a non-kernel address.  If we are in
3154	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
3155	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
3156	 */
3157	CPU_INDEX(%g1, %g2)
3158	set	cpu_core, %g2
3159	sllx	%g1, CPU_CORE_SHIFT, %g1
3160	add	%g1, %g2, %g1
3161	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3162	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3163	bz	sfmmu_pagefault
3164	or	%g2, CPU_DTRACE_BADADDR, %g2
3165	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3166	GET_MMU_D_ADDR(%g3, %g4)
3167	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3168
3169	/*
3170	 * Be sure that we're actually taking this miss from the kernel --
3171	 * otherwise we have managed to return to user-level with
3172	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3173	 */
3174	rdpr	%tstate, %g2
3175	btst	TSTATE_PRIV, %g2
3176	bz,a	ptl1_panic
3177	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3178	done
3179
3180	ALTENTRY(tsb_tl0_noctxt)
3181	/*
3182	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
3183	 * if it is, indicated that we have faulted and issue a done.
3184	 */
3185	CPU_INDEX(%g5, %g6)
3186	set	cpu_core, %g6
3187	sllx	%g5, CPU_CORE_SHIFT, %g5
3188	add	%g5, %g6, %g5
3189	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
3190	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
3191	bz	1f
3192	or	%g6, CPU_DTRACE_BADADDR, %g6
3193	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
3194	GET_MMU_D_ADDR(%g3, %g4)
3195	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]
3196
3197	/*
3198	 * Be sure that we're actually taking this miss from the kernel --
3199	 * otherwise we have managed to return to user-level with
3200	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3201	 */
3202	rdpr	%tstate, %g5
3203	btst	TSTATE_PRIV, %g5
3204	bz,a	ptl1_panic
3205	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3206	done
3207
32081:
3209	rdpr	%tt, %g5
3210	cmp	%g5, FAST_IMMU_MISS_TT
3211#ifdef sun4v
3212	MMU_FAULT_STATUS_AREA(%g2)
3213	be,a,pt	%icc, 2f
3214	  ldx	[%g2 + MMFSA_I_CTX], %g3
3215	cmp	%g5, T_INSTR_MMU_MISS
3216	be,a,pt	%icc, 2f
3217	  ldx	[%g2 + MMFSA_I_CTX], %g3
3218	ldx	[%g2 + MMFSA_D_CTX], %g3
32192:
3220#else
3221	mov	MMU_TAG_ACCESS, %g2
3222	be,a,pt	%icc, 2f
3223	  ldxa	[%g2]ASI_IMMU, %g3
3224	ldxa	[%g2]ASI_DMMU, %g3
32252:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
3226#endif
3227	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
3228	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
3229	rdpr	%tl, %g5
3230	cmp	%g5, 1
3231	ble,pt	%icc, sfmmu_mmu_trap
3232	  nop
3233	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
3234	ba,pt	%icc, sfmmu_window_trap
3235	  nop
3236	SET_SIZE(sfmmu_tsb_miss)
3237
3238#if (1<< TSBMISS_SHIFT) != TSBMISS_SIZE
3239#error - TSBMISS_SHIFT does not correspond to size of tsbmiss struct
3240#endif
3241
3242#endif /* lint */
3243
3244#if defined (lint)
3245/*
3246 * This routine will look for a user or kernel vaddr in the hash
3247 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't
3248 * grab any locks.  It should only be used by other sfmmu routines.
3249 */
3250/* ARGSUSED */
3251pfn_t
3252sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
3253{
3254	return(0);
3255}
3256
3257#else /* lint */
3258
3259	ENTRY_NP(sfmmu_vatopfn)
3260 	/*
3261 	 * disable interrupts
3262 	 */
3263 	rdpr	%pstate, %o3
3264#ifdef DEBUG
3265	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
3266	bnz,pt	%icc, 1f			/* disabled, panic	 */
3267	  nop
3268
3269	sethi	%hi(panicstr), %g1
3270	ldx	[%g1 + %lo(panicstr)], %g1
3271	tst	%g1
3272	bnz,pt	%icc, 1f
3273	  nop
3274
3275	save	%sp, -SA(MINFRAME), %sp
3276	sethi	%hi(sfmmu_panic1), %o0
3277	call	panic
3278	 or	%o0, %lo(sfmmu_panic1), %o0
32791:
3280#endif
3281	/*
3282	 * disable interrupts to protect the TSBMISS area
3283	 */
3284	andn    %o3, PSTATE_IE, %o5
3285	wrpr    %o5, 0, %pstate
3286
3287	/*
3288	 * o0 = vaddr
3289	 * o1 = sfmmup
3290	 * o2 = ttep
3291	 */
3292	CPU_TSBMISS_AREA(%g1, %o5)
3293	ldn	[%g1 + TSBMISS_KHATID], %o4
3294	cmp	%o4, %o1
3295	bne,pn	%ncc, vatopfn_nokernel
3296	  mov	TTE64K, %g5			/* g5 = rehash # */
3297	mov %g1,%o5				/* o5 = tsbmiss_area */
3298	/*
3299	 * o0 = vaddr
3300	 * o1 & o4 = hatid
3301	 * o2 = ttep
3302	 * o5 = tsbmiss area
3303	 */
3304	mov	HBLK_RANGE_SHIFT, %g6
33051:
3306
3307	/*
3308	 * o0 = vaddr
3309	 * o1 = sfmmup
3310	 * o2 = ttep
3311	 * o3 = old %pstate
3312	 * o4 = hatid
3313	 * o5 = tsbmiss
3314	 * g5 = rehash #
3315	 * g6 = hmeshift
3316	 *
3317	 * The first arg to GET_TTE is actually tagaccess register
3318	 * not just vaddr. Since this call is for kernel we need to clear
3319	 * any lower vaddr bits that would be interpreted as ctx bits.
3320	 */
3321	set     TAGACC_CTX_MASK, %g1
3322	andn    %o0, %g1, %o0
3323	GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5,
3324		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
3325
3326kvtop_hblk_found:
3327	/*
3328	 * o0 = vaddr
3329	 * o1 = sfmmup
3330	 * o2 = ttep
3331	 * g1 = tte
3332	 * g2 = tte pa
3333	 * g3 = tte va
3334	 * o2 = tsbmiss area
3335	 * o1 = hat id
3336	 */
3337	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
3338	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3339	stx %g1,[%o2]				/* put tte into *ttep */
3340	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
3341	/*
3342	 * o0 = vaddr
3343	 * o1 = sfmmup
3344	 * o2 = ttep
3345	 * g1 = pfn
3346	 */
3347	ba,pt	%xcc, 6f
3348	  mov	%g1, %o0
3349
3350kvtop_nohblk:
3351	/*
3352	 * we get here if we couldn't find valid hblk in hash.  We rehash
3353	 * if neccesary.
3354	 */
3355	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
3356#ifdef sun4v
3357	cmp	%g5, MAX_HASHCNT
3358#else
3359	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
3360#endif
3361	be,a,pn	%icc, 6f
3362	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3363	mov	%o1, %o4			/* restore hatid */
3364#ifdef sun4v
3365        add	%g5, 2, %g5
3366	cmp	%g5, 3
3367	move	%icc, MMU_PAGESHIFT4M, %g6
3368	ba,pt	%icc, 1b
3369	movne	%icc, MMU_PAGESHIFT256M, %g6
3370#else
3371        inc	%g5
3372	cmp	%g5, 2
3373	move	%icc, MMU_PAGESHIFT512K, %g6
3374	ba,pt	%icc, 1b
3375	movne	%icc, MMU_PAGESHIFT4M, %g6
3376#endif
33776:
3378	retl
3379 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3380
3381tsb_suspend:
3382	/*
3383	 * o0 = vaddr
3384	 * o1 = sfmmup
3385	 * o2 = ttep
3386	 * g1 = tte
3387	 * g2 = tte pa
3388	 * g3 = tte va
3389	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
3390	 */
3391	stx %g1,[%o2]				/* put tte into *ttep */
3392	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
3393	  sub	%g0, 1, %o0			/* output = -1 (PFN_INVALID) */
3394	TTETOPFN(%g1, %o0, vatopfn_l3, %g2, %g3, %g4)
3395	/*
3396	 * o0 = PFN return value PFN_INVALID, PFN_SUSPENDED, or pfn#
3397	 * o1 = sfmmup
3398	 * o2 = ttep
3399	 * g1 = pfn
3400	 */
3401	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
34028:
3403	retl
3404	 wrpr	%g0, %o3, %pstate		/* enable interrupts */
3405
3406vatopfn_nokernel:
3407	/*
3408	 * This routine does NOT support user addresses
3409	 * There is a routine in C that supports this.
3410	 * The only reason why we don't have the C routine
3411	 * support kernel addresses as well is because
3412	 * we do va_to_pa while holding the hashlock.
3413	 */
3414 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3415	save	%sp, -SA(MINFRAME), %sp
3416	sethi	%hi(sfmmu_panic3), %o0
3417	call	panic
3418	 or	%o0, %lo(sfmmu_panic3), %o0
3419
3420	SET_SIZE(sfmmu_vatopfn)
3421#endif /* lint */
3422
3423
3424
3425#if !defined(lint)
3426
3427/*
3428 * kpm lock used between trap level tsbmiss handler and kpm C level.
3429 */
3430#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
3431	mov     0xff, tmp1						;\
3432label1:									;\
3433	casa    [kpmlckp]asi, %g0, tmp1					;\
3434	brnz,pn tmp1, label1						;\
3435	mov     0xff, tmp1						;\
3436	membar  #LoadLoad
3437
3438#define KPMLOCK_EXIT(kpmlckp, asi)					\
3439	membar  #LoadStore|#StoreStore					;\
3440	sta     %g0, [kpmlckp]asi
3441
3442/*
3443 * Lookup a memseg for a given pfn and if found, return the physical
3444 * address of the corresponding struct memseg in mseg, otherwise
3445 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
3446 * tsbmp, %asi is assumed to be ASI_MEM.
3447 * This lookup is done by strictly traversing only the physical memseg
3448 * linkage. The more generic approach, to check the virtual linkage
3449 * before using the physical (used e.g. with hmehash buckets), cannot
3450 * be used here. Memory DR operations can run in parallel to this
3451 * lookup w/o any locks and updates of the physical and virtual linkage
3452 * cannot be done atomically wrt. to each other. Because physical
3453 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
3454 * as "physical NULL" pointer.
3455 */
3456#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
3457	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
3458	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
3459	udivx	pfn, mseg, mseg						;\
3460	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
3461	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
3462	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
3463	add	tmp1, mseg, tmp1					;\
3464	ldxa	[tmp1]%asi, mseg					;\
3465	cmp	mseg, MSEG_NULLPTR_PA					;\
3466	be,pn	%xcc, label/**/1		/* if not found */	;\
3467	  nop								;\
3468	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3469	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3470	blu,pn	%xcc, label/**/1					;\
3471	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3472	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3473	bgeu,pn	%xcc, label/**/1					;\
3474	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
3475	mulx	tmp1, PAGE_SIZE, tmp1					;\
3476	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
3477	add	tmp2, tmp1, tmp1			/* pp */	;\
3478	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
3479	cmp	tmp2, pfn						;\
3480	be,pt	%xcc, label/**/_ok			/* found */	;\
3481label/**/1:								;\
3482	/* brute force lookup */					;\
3483	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
3484	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
3485label/**/2:								;\
3486	cmp	mseg, MSEG_NULLPTR_PA					;\
3487	be,pn	%xcc, label/**/_ok		/* if not found */	;\
3488	  nop								;\
3489	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3490	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3491	blu,a,pt %xcc, label/**/2					;\
3492	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3493	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3494	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3495	bgeu,a,pt %xcc, label/**/2					;\
3496	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3497label/**/_ok:
3498
3499	/*
3500	 * kpm tsb miss handler large pages
3501	 * g1 = 8K kpm TSB entry pointer
3502	 * g2 = tag access register
3503	 * g3 = 4M kpm TSB entry pointer
3504	 */
3505	ALTENTRY(sfmmu_kpm_dtsb_miss)
3506	TT_TRACE(trace_tsbmiss)
3507
3508	CPU_INDEX(%g7, %g6)
3509	sethi	%hi(kpmtsbm_area), %g6
3510	sllx	%g7, KPMTSBM_SHIFT, %g7
3511	or	%g6, %lo(kpmtsbm_area), %g6
3512	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
3513
3514	/* check enable flag */
3515	ldub	[%g6 + KPMTSBM_FLAGS], %g4
3516	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
3517	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
3518	  nop
3519
3520	/* VA range check */
3521	ldx	[%g6 + KPMTSBM_VBASE], %g7
3522	cmp	%g2, %g7
3523	blu,pn	%xcc, sfmmu_tsb_miss
3524	  ldx	[%g6 + KPMTSBM_VEND], %g5
3525	cmp	%g2, %g5
3526	bgeu,pn	%xcc, sfmmu_tsb_miss
3527	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]
3528
3529	/*
3530	 * check TL tsbmiss handling flag
3531	 * bump tsbmiss counter
3532	 */
3533	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
3534#ifdef	DEBUG
3535	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
3536	inc	%g5
3537	brz,pn	%g3, sfmmu_kpm_exception
3538	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
3539#else
3540	inc	%g5
3541	st	%g5, [%g6 + KPMTSBM_TSBMISS]
3542#endif
3543	/*
3544	 * At this point:
3545	 *  g1 = 8K kpm TSB pointer (not used)
3546	 *  g2 = tag access register
3547	 *  g3 = clobbered
3548	 *  g6 = per-CPU kpm tsbmiss area
3549	 *  g7 = kpm_vbase
3550	 */
3551
3552	/* vaddr2pfn */
3553	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
3554	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
3555	srax    %g4, %g3, %g2			/* which alias range (r) */
3556	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
3557	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
3558
3559	/*
3560	 * Setup %asi
3561	 * mseg_pa = page_numtomemseg_nolock(pfn)
3562	 * if (mseg_pa == NULL) sfmmu_kpm_exception
3563	 * g2=pfn
3564	 */
3565	mov	ASI_MEM, %asi
3566	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
3567	cmp	%g3, MSEG_NULLPTR_PA
3568	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
3569	  nop
3570
3571	/*
3572	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
3573	 * g2=pfn g3=mseg_pa
3574	 */
3575	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
3576	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
3577	srlx	%g2, %g5, %g4
3578	sllx	%g4, %g5, %g4
3579	sub	%g4, %g7, %g4
3580	srlx	%g4, %g5, %g4
3581
3582	/*
3583	 * Validate inx value
3584	 * g2=pfn g3=mseg_pa g4=inx
3585	 */
3586#ifdef	DEBUG
3587	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
3588	cmp	%g4, %g5			/* inx - nkpmpgs */
3589	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
3590	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3591#else
3592	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3593#endif
3594	/*
3595	 * kp = &mseg_pa->kpm_pages[inx]
3596	 */
3597	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
3598	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
3599	add	%g5, %g4, %g5			/* kp */
3600
3601	/*
3602	 * KPMP_HASH(kp)
3603	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
3604	 */
3605	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
3606	sub	%g7, 1, %g7			/* mask */
3607	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
3608	add	%g5, %g1, %g5			/* y = ksp + x */
3609	and 	%g5, %g7, %g5			/* hashinx = y & mask */
3610
3611	/*
3612	 * Calculate physical kpm_page pointer
3613	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
3614	 */
3615	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
3616	add	%g1, %g4, %g1			/* kp_pa */
3617
3618	/*
3619	 * Calculate physical hash lock address
3620	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
3621	 */
3622	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
3623	sllx	%g5, KPMHLK_SHIFT, %g5
3624	add	%g4, %g5, %g3
3625	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */
3626
3627	/*
3628	 * Assemble tte
3629	 * g1=kp_pa g2=pfn g3=hlck_pa
3630	 */
3631#ifdef sun4v
3632	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
3633	sllx	%g5, 32, %g5
3634	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3635	or	%g4, TTE4M, %g4
3636	or	%g5, %g4, %g5
3637#else
3638	sethi	%hi(TTE_VALID_INT), %g4
3639	mov	TTE4M, %g5
3640	sllx	%g5, TTE_SZ_SHFT_INT, %g5
3641	or	%g5, %g4, %g5			/* upper part */
3642	sllx	%g5, 32, %g5
3643	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3644	or	%g5, %g4, %g5
3645#endif
3646	sllx	%g2, MMU_PAGESHIFT, %g4
3647	or	%g5, %g4, %g5			/* tte */
3648	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
3649	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3650
3651	/*
3652	 * tsb dropin
3653	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
3654	 */
3655
3656	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
3657	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
3658
3659	/* use C-handler if there's no go for dropin */
3660	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
3661	cmp	%g7, -1
3662	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
3663	  nop
3664
3665#ifdef	DEBUG
3666	/* double check refcnt */
3667	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
3668	brz,pn	%g7, 5f			/* let C-handler deal with this */
3669	  nop
3670#endif
3671
3672#ifndef sun4v
3673	ldub	[%g6 + KPMTSBM_FLAGS], %g7
3674	mov	ASI_N, %g1
3675	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
3676	movnz	%icc, ASI_MEM, %g1
3677	mov	%g1, %asi
3678#endif
3679
3680	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
3681	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
3682
3683	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
3684	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
3685
3686	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
3687
3688	/* KPMLOCK_EXIT(kpmlckp, asi) */
3689	KPMLOCK_EXIT(%g3, ASI_MEM)
3690
3691	/*
3692	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3693	 * point to trapstat's TSB miss return code (note that trapstat
3694	 * itself will patch the correct offset to add).
3695	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
3696	 */
3697	rdpr	%tl, %g7
3698	cmp	%g7, 1
3699	ble	%icc, 0f
3700	sethi	%hi(KERNELBASE), %g6
3701	rdpr	%tpc, %g7
3702	or	%g6, %lo(KERNELBASE), %g6
3703	cmp	%g7, %g6
3704	bgeu	%xcc, 0f
3705	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
3706	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3707	wrpr	%g7, %tpc
3708	add	%g7, 4, %g7
3709	wrpr	%g7, %tnpc
37100:
3711	retry
37125:
3713	/* g3=hlck_pa */
3714	KPMLOCK_EXIT(%g3, ASI_MEM)
3715	ba,pt	%icc, sfmmu_kpm_exception
3716	  nop
3717	SET_SIZE(sfmmu_kpm_dtsb_miss)
3718
3719	/*
3720	 * kpm tsbmiss handler for smallpages
3721	 * g1 = 8K kpm TSB pointer
3722	 * g2 = tag access register
3723	 * g3 = 4M kpm TSB pointer
3724	 */
3725	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
3726	TT_TRACE(trace_tsbmiss)
3727	CPU_INDEX(%g7, %g6)
3728	sethi	%hi(kpmtsbm_area), %g6
3729	sllx	%g7, KPMTSBM_SHIFT, %g7
3730	or	%g6, %lo(kpmtsbm_area), %g6
3731	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
3732
3733	/* check enable flag */
3734	ldub	[%g6 + KPMTSBM_FLAGS], %g4
3735	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
3736	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
3737	  nop
3738
3739	/*
3740	 * VA range check
3741	 * On fail: goto sfmmu_tsb_miss
3742	 */
3743	ldx	[%g6 + KPMTSBM_VBASE], %g7
3744	cmp	%g2, %g7
3745	blu,pn	%xcc, sfmmu_tsb_miss
3746	  ldx	[%g6 + KPMTSBM_VEND], %g5
3747	cmp	%g2, %g5
3748	bgeu,pn	%xcc, sfmmu_tsb_miss
3749	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */
3750
3751	/*
3752	 * check TL tsbmiss handling flag
3753	 * bump tsbmiss counter
3754	 */
3755	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
3756#ifdef	DEBUG
3757	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
3758	inc	%g5
3759	brz,pn	%g1, sfmmu_kpm_exception
3760	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
3761#else
3762	inc	%g5
3763	st	%g5, [%g6 + KPMTSBM_TSBMISS]
3764#endif
3765	/*
3766	 * At this point:
3767	 *  g1 = clobbered
3768	 *  g2 = tag access register
3769	 *  g3 = 4M kpm TSB pointer (not used)
3770	 *  g6 = per-CPU kpm tsbmiss area
3771	 *  g7 = kpm_vbase
3772	 */
3773
3774	/* vaddr2pfn */
3775	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
3776	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
3777	srax    %g4, %g3, %g2			/* which alias range (r) */
3778	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
3779	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
3780
3781	/*
3782	 * Setup %asi
3783	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
3784	 * if (mseg not found) sfmmu_kpm_exception
3785	 * g2=pfn
3786	 */
3787	mov	ASI_MEM, %asi
3788	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
3789	cmp	%g3, MSEG_NULLPTR_PA
3790	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
3791	  nop
3792
3793	/*
3794	 * inx = pfn - mseg_pa->kpm_pbase
3795	 * g2=pfn g3=mseg_pa
3796	 */
3797	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
3798	sub	%g2, %g7, %g4
3799
3800#ifdef	DEBUG
3801	/*
3802	 * Validate inx value
3803	 * g2=pfn g3=mseg_pa g4=inx
3804	 */
3805	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
3806	cmp	%g4, %g5			/* inx - nkpmpgs */
3807	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
3808	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3809#else
3810	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3811#endif
3812	/* ksp = &mseg_pa->kpm_spages[inx] */
3813	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
3814	add	%g5, %g4, %g5			/* ksp */
3815
3816	/*
3817	 * KPMP_SHASH(kp)
3818	 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz
3819	 */
3820	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
3821	sub	%g7, 1, %g7			/* mask */
3822	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
3823	add	%g5, %g1, %g5			/* y = ksp + x */
3824	and 	%g5, %g7, %g5			/* hashinx = y & mask */
3825
3826	/*
3827	 * Calculate physical kpm_spage pointer
3828	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
3829	 */
3830	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
3831	add	%g1, %g4, %g1			/* ksp_pa */
3832
3833	/*
3834	 * Calculate physical hash lock address.
3835	 * Note: Changes in kpm_shlk_t must be reflected here.
3836	 * g1=ksp_pa g2=pfn g5=hashinx
3837	 */
3838	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
3839	sllx	%g5, KPMSHLK_SHIFT, %g5
3840	add	%g4, %g5, %g3			/* hlck_pa */
3841
3842	/*
3843	 * Assemble tte
3844	 * g1=ksp_pa g2=pfn g3=hlck_pa
3845	 */
3846	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
3847	sllx	%g5, 32, %g5
3848	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3849	or	%g5, %g4, %g5
3850	sllx	%g2, MMU_PAGESHIFT, %g4
3851	or	%g5, %g4, %g5			/* tte */
3852	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
3853	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3854
3855	/*
3856	 * tsb dropin
3857	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte
3858	 */
3859
3860	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
3861	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
3862
3863	/* use C-handler if there's no go for dropin */
3864	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */
3865	cmp	%g7, -1
3866	bne,pn	%xcc, 5f
3867	  nop
3868
3869#ifndef sun4v
3870	ldub	[%g6 + KPMTSBM_FLAGS], %g7
3871	mov	ASI_N, %g1
3872	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
3873	movnz	%icc, ASI_MEM, %g1
3874	mov	%g1, %asi
3875#endif
3876
3877	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
3878	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
3879
3880	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
3881	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
3882
3883	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
3884
3885	/* KPMLOCK_EXIT(kpmlckp, asi) */
3886	KPMLOCK_EXIT(%g3, ASI_MEM)
3887
3888	/*
3889	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3890	 * point to trapstat's TSB miss return code (note that trapstat
3891	 * itself will patch the correct offset to add).
3892	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
3893	 */
3894	rdpr	%tl, %g7
3895	cmp	%g7, 1
3896	ble	%icc, 0f
3897	sethi	%hi(KERNELBASE), %g6
3898	rdpr	%tpc, %g7
3899	or	%g6, %lo(KERNELBASE), %g6
3900	cmp	%g7, %g6
3901	bgeu	%xcc, 0f
3902	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
3903	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3904	wrpr	%g7, %tpc
3905	add	%g7, 4, %g7
3906	wrpr	%g7, %tnpc
39070:
3908	retry
39095:
3910	/* g3=hlck_pa */
3911	KPMLOCK_EXIT(%g3, ASI_MEM)
3912	ba,pt	%icc, sfmmu_kpm_exception
3913	  nop
3914	SET_SIZE(sfmmu_kpm_dtsb_miss_small)
3915
3916#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
3917#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
3918#endif
3919
3920#endif /* lint */
3921
3922#ifdef	lint
3923/*
3924 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
3925 * Called from C-level, sets/clears "go" indication for trap level handler.
3926 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
3927 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
3928 * Assumes khl_mutex is held when called from C-level.
3929 */
3930/* ARGSUSED */
3931void
3932sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
3933{
3934}
3935
3936/*
3937 * kpm_smallpages: stores val to byte at address mapped within
3938 * low level lock brackets. The old value is returned.
3939 * Called from C-level.
3940 */
3941/* ARGSUSED */
3942int
3943sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val)
3944{
3945	return (0);
3946}
3947
3948#else /* lint */
3949
3950	.seg	".data"
3951sfmmu_kpm_tsbmtl_panic:
3952	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
3953	.byte	0
3954sfmmu_kpm_stsbmtl_panic:
3955	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
3956	.byte	0
3957	.align	4
3958	.seg	".text"
3959
3960	ENTRY_NP(sfmmu_kpm_tsbmtl)
3961	rdpr	%pstate, %o3
3962	/*
3963	 * %o0 = &kp_refcntc
3964	 * %o1 = &khl_lock
3965	 * %o2 = 0/1 (off/on)
3966	 * %o3 = pstate save
3967	 */
3968#ifdef DEBUG
3969	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
3970	bnz,pt %icc, 1f				/* disabled, panic	 */
3971	  nop
3972	save	%sp, -SA(MINFRAME), %sp
3973	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
3974	call	panic
3975	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
3976	ret
3977	restore
39781:
3979#endif /* DEBUG */
3980	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
3981
3982	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
3983	mov	-1, %o5
3984	brz,a	%o2, 2f
3985	  mov	0, %o5
39862:
3987	sth	%o5, [%o0]
3988	KPMLOCK_EXIT(%o1, ASI_N)
3989
3990	retl
3991	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
3992	SET_SIZE(sfmmu_kpm_tsbmtl)
3993
3994	ENTRY_NP(sfmmu_kpm_stsbmtl)
3995	rdpr	%pstate, %o3
3996	/*
3997	 * %o0 = &mapped
3998	 * %o1 = &kshl_lock
3999	 * %o2 = val
4000	 * %o3 = pstate save
4001	 */
4002#ifdef DEBUG
4003	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4004	bnz,pt %icc, 1f				/* disabled, panic	 */
4005	  nop
4006	save	%sp, -SA(MINFRAME), %sp
4007	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
4008	call	panic
4009	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
4010	ret
4011	restore
40121:
4013#endif /* DEBUG */
4014	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4015
4016	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
4017	ldsb	[%o0], %o5
4018	stb	%o2, [%o0]
4019	KPMLOCK_EXIT(%o1, ASI_N)
4020
4021	mov	%o5, %o0			/* return old val */
4022	retl
4023	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4024	SET_SIZE(sfmmu_kpm_stsbmtl)
4025
4026#endif /* lint */
4027
4028#ifndef lint
4029#ifdef sun4v
4030	/*
4031	 * User/kernel data miss w// multiple TSBs
4032	 * The first probe covers 8K, 64K, and 512K page sizes,
4033	 * because 64K and 512K mappings are replicated off 8K
4034	 * pointer.  Second probe covers 4M page size only.
4035	 *
4036	 * MMU fault area contains miss address and context.
4037	 */
4038	ALTENTRY(sfmmu_slow_dmmu_miss)
4039	GET_MMU_D_TAGACC_CTX(%g2, %g3)	! %g2 = tagacc, %g3 = ctx
4040
4041slow_miss_common:
4042	/*
4043	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
4044	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
4045	 */
4046	brnz,pt	%g3, 8f			! check for user context
4047	  nop
4048
4049	/*
4050	 * Kernel miss
4051	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
4052	 * branch to sfmmu_tsb_miss_tt to handle it.
4053	 */
4054	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4055sfmmu_dslow_patch_ktsb_base:
4056	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
4057sfmmu_dslow_patch_ktsb_szcode:
4058	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
4059
4060	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
4061	! %g1 = First TSB entry pointer, as TSB miss handler expects
4062
4063	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4064sfmmu_dslow_patch_ktsb4m_base:
4065	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
4066sfmmu_dslow_patch_ktsb4m_szcode:
4067	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
4068
4069	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
4070	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
4071	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4072	.empty
4073
40748:
4075	/*
4076	 * User miss
4077	 * Get first TSB pointer in %g1
4078	 * Get second TSB pointer (or NULL if no second TSB) in %g3
4079	 * Branch to sfmmu_tsb_miss_tt to handle it
4080	 */
4081	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
4082	/* %g1 = first TSB entry ptr now, %g2 preserved */
4083
4084	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
4085	brlz,a,pt %g3, sfmmu_tsb_miss_tt	/* done if no 2nd TSB */
4086	  mov	%g0, %g3
4087
4088	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
4089	/* %g3 = second TSB entry ptr now, %g2 preserved */
40909:
4091	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4092	.empty
4093	SET_SIZE(sfmmu_slow_dmmu_miss)
4094
4095
4096	/*
4097	 * User/kernel instruction miss w/ multiple TSBs
4098	 * The first probe covers 8K, 64K, and 512K page sizes,
4099	 * because 64K and 512K mappings are replicated off 8K
4100	 * pointer.  Second probe covers 4M page size only.
4101	 *
4102	 * MMU fault area contains miss address and context.
4103	 */
4104	ALTENTRY(sfmmu_slow_immu_miss)
4105	MMU_FAULT_STATUS_AREA(%g2)
4106	ldx	[%g2 + MMFSA_I_CTX], %g3
4107	ldx	[%g2 + MMFSA_I_ADDR], %g2
4108	srlx	%g2, MMU_PAGESHIFT, %g2	! align address to page boundry
4109	sllx	%g2, MMU_PAGESHIFT, %g2
4110	ba,pt	%xcc, slow_miss_common
4111	or	%g2, %g3, %g2
4112	SET_SIZE(sfmmu_slow_immu_miss)
4113
4114#endif /* sun4v */
4115#endif	/* lint */
4116
4117#ifndef lint
4118
4119/*
4120 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
4121 */
4122	.seg	".data"
4123	.align	64
4124	.global tsbmiss_area
4125tsbmiss_area:
4126	.skip	(TSBMISS_SIZE * NCPU)
4127
4128	.align	64
4129	.global kpmtsbm_area
4130kpmtsbm_area:
4131	.skip	(KPMTSBM_SIZE * NCPU)
4132#endif	/* lint */
4133