sfmmu_asm.s revision 12b2e3e73495663d8524de577375cab2b57dedf1
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#pragma ident	"%Z%%M%	%I%	%E% SMI"
28
29/*
30 * SFMMU primitives.  These primitives should only be used by sfmmu
31 * routines.
32 */
33
34#if defined(lint)
35#include <sys/types.h>
36#else	/* lint */
37#include "assym.h"
38#endif	/* lint */
39
40#include <sys/asm_linkage.h>
41#include <sys/machtrap.h>
42#include <sys/machasi.h>
43#include <sys/sun4asi.h>
44#include <sys/pte.h>
45#include <sys/mmu.h>
46#include <vm/hat_sfmmu.h>
47#include <vm/seg_spt.h>
48#include <sys/machparam.h>
49#include <sys/privregs.h>
50#include <sys/scb.h>
51#include <sys/intreg.h>
52#include <sys/machthread.h>
53#include <sys/intr.h>
54#include <sys/clock.h>
55#include <sys/trapstat.h>
56
57#ifdef TRAPTRACE
58#include <sys/traptrace.h>
59
60/*
61 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
62 */
63#define	TT_TRACE(label)		\
64	ba	label		;\
65	rd	%pc, %g7
66#else
67
68#define	TT_TRACE(label)
69
70#endif /* TRAPTRACE */
71
72#ifndef	lint
73
74#if (TTE_SUSPEND_SHIFT > 0)
75#define	TTE_SUSPEND_INT_SHIFT(reg)				\
76	sllx	reg, TTE_SUSPEND_SHIFT, reg
77#else
78#define	TTE_SUSPEND_INT_SHIFT(reg)
79#endif
80
81#endif /* lint */
82
83#ifndef	lint
84
85/*
86 * Assumes TSBE_TAG is 0
87 * Assumes TSBE_INTHI is 0
88 * Assumes TSBREG.split is 0
89 */
90
91#if TSBE_TAG != 0
92#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
93#endif
94
95#if TSBTAG_INTHI != 0
96#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
97#endif
98
99/*
100 * The following code assumes the tsb is not split.
101 *
102 * With TSBs no longer shared between processes, it's no longer
103 * necessary to hash the context bits into the tsb index to get
104 * tsb coloring; the new implementation treats the TSB as a
105 * direct-mapped, virtually-addressed cache.
106 *
107 * In:
108 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
109 *    tsbbase = base address of TSB (clobbered)
110 *    tagacc = tag access register (clobbered)
111 *    szc = size code of TSB (ro)
112 *    tmp = scratch reg
113 * Out:
114 *    tsbbase = pointer to entry in TSB
115 */
116#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
117	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
118	srlx	tagacc, vpshift, tagacc 				;\
119	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
120	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
121	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
122	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
123	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */
124
125/*
126 * When the kpm TSB is used it is assumed that it is direct mapped
127 * using (vaddr>>vpshift)%tsbsz as the index.
128 *
129 * Note that, for now, the kpm TSB and kernel TSB are the same for
130 * each mapping size.  However that need not always be the case.  If
131 * the trap handlers are updated to search a different TSB for kpm
132 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
133 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
134 *
135 * In:
136 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
137 *    vaddr = virtual address (clobbered)
138 *    tsbp, szc, tmp = scratch
139 * Out:
140 *    tsbp = pointer to entry in TSB
141 */
142#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
143	cmp	vpshift, MMU_PAGESHIFT					;\
144	bne,pn	%icc, 1f		/* branch if large case */	;\
145	  sethi	%hi(kpmsm_tsbsz), szc					;\
146	sethi	%hi(kpmsm_tsbbase), tsbp				;\
147	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
148	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
149	ba,pt	%icc, 2f						;\
150	  nop								;\
1511:	sethi	%hi(kpm_tsbsz), szc					;\
152	sethi	%hi(kpm_tsbbase), tsbp					;\
153	ld	[szc + %lo(kpm_tsbsz)], szc				;\
154	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
1552:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
156
157/*
158 * Lock the TSBE at virtual address tsbep.
159 *
160 * tsbep = TSBE va (ro)
161 * tmp1, tmp2 = scratch registers (clobbered)
162 * label = label to use for branches (text)
163 * %asi = ASI to use for TSB access
164 *
165 * NOTE that we flush the TSB using fast VIS instructions that
166 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
167 * not be treated as a locked entry or we'll get stuck spinning on
168 * an entry that isn't locked but really invalid.
169 */
170
171#if defined(UTSB_PHYS)
172
173#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
174	lda	[tsbep]ASI_MEM, tmp1					;\
175label:									;\
176	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
177	cmp	tmp1, tmp2 						;\
178	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
179	  lda	[tsbep]ASI_MEM, tmp1					;\
180	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
181	cmp	tmp1, tmp2 						;\
182	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
183	  lda	[tsbep]ASI_MEM, tmp1					;\
184	/* tsbe lock acquired */					;\
185	membar #StoreStore
186
187#else /* UTSB_PHYS */
188
189#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
190	lda	[tsbep]%asi, tmp1					;\
191label:									;\
192	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
193	cmp	tmp1, tmp2 						;\
194	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
195	  lda	[tsbep]%asi, tmp1					;\
196	casa	[tsbep]%asi, tmp1, tmp2					;\
197	cmp	tmp1, tmp2 						;\
198	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
199	  lda	[tsbep]%asi, tmp1					;\
200	/* tsbe lock acquired */					;\
201	membar #StoreStore
202
203#endif /* UTSB_PHYS */
204
205/*
206 * Atomically write TSBE at virtual address tsbep.
207 *
208 * tsbep = TSBE va (ro)
209 * tte = TSBE TTE (ro)
210 * tagtarget = TSBE tag (ro)
211 * %asi = ASI to use for TSB access
212 */
213
214#if defined(UTSB_PHYS)
215
216#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
217	add	tsbep, TSBE_TTE, tmp1					;\
218	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
219	membar #StoreStore						;\
220	add	tsbep, TSBE_TAG, tmp1					;\
221	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */
222
223#else /* UTSB_PHYS */
224
225#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
226	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
227	membar #StoreStore						;\
228	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
229
230#endif /* UTSB_PHYS */
231
232/*
233 * Load an entry into the TSB at TL > 0.
234 *
235 * tsbep = pointer to the TSBE to load as va (ro)
236 * tte = value of the TTE retrieved and loaded (wo)
237 * tagtarget = tag target register.  To get TSBE tag to load,
238 *   we need to mask off the context and leave only the va (clobbered)
239 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
240 * tmp1, tmp2 = scratch registers
241 * label = label to use for branches (text)
242 * %asi = ASI to use for TSB access
243 */
244
245#if defined(UTSB_PHYS)
246
247#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
248	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
249	/*								;\
250	 * I don't need to update the TSB then check for the valid tte.	;\
251	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
252	 * we always invalidate the hash table before we unload the TSB.;\
253	 */								;\
254	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
255	ldxa	[ttepa]ASI_MEM, tte					;\
256	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
257	sethi	%hi(TSBTAG_INVALID), tmp2				;\
258	add	tsbep, TSBE_TAG, tmp1					;\
259	brgez,a,pn tte, label/**/f					;\
260	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
261	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
262label:
263
264#else /* UTSB_PHYS */
265
266#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
267	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
268	/*								;\
269	 * I don't need to update the TSB then check for the valid tte.	;\
270	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
271	 * we always invalidate the hash table before we unload the TSB.;\
272	 */								;\
273	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
274	ldxa	[ttepa]ASI_MEM, tte					;\
275	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
276	sethi	%hi(TSBTAG_INVALID), tmp2				;\
277	brgez,a,pn tte, label/**/f					;\
278	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
279	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
280label:
281
282/*
283 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
284 *   for ITLB synthesis.
285 *
286 * tsbep = pointer to the TSBE to load as va (ro)
287 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
288 *   with exec_perm turned off and exec_synth turned on
289 * tagtarget = tag target register.  To get TSBE tag to load,
290 *   we need to mask off the context and leave only the va (clobbered)
291 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
292 * tmp1, tmp2 = scratch registers
293 * label = label to use for branch (text)
294 * %asi = ASI to use for TSB access
295 */
296
297#define	TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
298	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
299	/*								;\
300	 * I don't need to update the TSB then check for the valid tte.	;\
301	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
302	 * we always invalidate the hash table before we unload the TSB.;\
303	 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0	;\
304	 * and exec_synth bit to 1.					;\
305	 */								;\
306	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
307	mov	tte, tmp1						;\
308	ldxa	[ttepa]ASI_MEM, tte					;\
309	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
310	sethi	%hi(TSBTAG_INVALID), tmp2				;\
311	brgez,a,pn tte, label/**/f					;\
312	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
313	or	tte, tmp1, tte						;\
314	andn	tte, TTE_EXECPRM_INT, tte				;\
315	or	tte, TTE_E_SYNTH_INT, tte				;\
316	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
317label:
318
319/*
320 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
321 *
322 * tte = value of the TTE, used to get tte_size bits (ro)
323 * tagaccess = tag access register, used to get 4M pfn bits (ro)
324 * pfn = 4M pfn bits shifted to offset for tte (out)
325 * tmp1 = scratch register
326 * label = label to use for branch (text)
327 */
328
329#define	GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)			\
330	/*								;\
331	 * Get 4M bits from tagaccess for 32M, 256M pagesizes.		;\
332	 * Return them, shifted, in pfn.				;\
333	 */								;\
334	srlx	tagaccess, MMU_PAGESHIFT4M, tagaccess			;\
335	srlx	tte, TTE_SZ_SHFT, tmp		/* isolate the */	;\
336	andcc	tmp, TTE_SZ_BITS, %g0		/* tte_size bits */	;\
337	bz,a,pt %icc, label/**/f		/* if 0, is */		;\
338	  and	tagaccess, 0x7, tagaccess	/* 32M page size */	;\
339	and	tagaccess, 0x3f, tagaccess /* else 256M page size */	;\
340label:									;\
341	sllx	tagaccess, MMU_PAGESHIFT4M, pfn
342
343/*
344 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
345 * for ITLB synthesis.
346 *
347 * tte = value of the TTE, used to get tte_size bits (rw)
348 * tmp1 = scratch register
349 */
350
351#define	SET_TTE4M_PN(tte, tmp)						\
352	/*								;\
353	 * Set 4M pagesize tte bits. 					;\
354	 */								;\
355	set	TTE4M, tmp						;\
356	sllx	tmp, TTE_SZ_SHFT, tmp					;\
357	or	tte, tmp, tte
358
359#endif /* UTSB_PHYS */
360
361/*
362 * Load an entry into the TSB at TL=0.
363 *
364 * tsbep = pointer to the TSBE to load as va (ro)
365 * tteva = pointer to the TTE to load as va (ro)
366 * tagtarget = TSBE tag to load (which contains no context), synthesized
367 * to match va of MMU tag target register only (ro)
368 * tmp1, tmp2 = scratch registers (clobbered)
369 * label = label to use for branches (text)
370 * %asi = ASI to use for TSB access
371 */
372
373#if defined(UTSB_PHYS)
374
375#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
376	/* can't rd tteva after locking tsb because it can tlb miss */	;\
377	ldx	[tteva], tteva			/* load tte */		;\
378	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
379	sethi	%hi(TSBTAG_INVALID), tmp2				;\
380	add	tsbep, TSBE_TAG, tmp1					;\
381	brgez,a,pn tteva, label/**/f					;\
382	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
383	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
384label:
385
386#else /* UTSB_PHYS */
387
388#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
389	/* can't rd tteva after locking tsb because it can tlb miss */	;\
390	ldx	[tteva], tteva			/* load tte */		;\
391	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
392	sethi	%hi(TSBTAG_INVALID), tmp2				;\
393	brgez,a,pn tteva, label/**/f					;\
394	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
395	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
396label:
397
398#endif /* UTSB_PHYS */
399
400/*
401 * Invalidate a TSB entry in the TSB.
402 *
403 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
404 *	 about this earlier to ensure this is true.  Thus when we are
405 *	 directly referencing tsbep below, we are referencing the tte_tag
406 *	 field of the TSBE.  If this  offset ever changes, the code below
407 *	 will need to be modified.
408 *
409 * tsbep = pointer to TSBE as va (ro)
410 * tag = invalidation is done if this matches the TSBE tag (ro)
411 * tmp1 - tmp3 = scratch registers (clobbered)
412 * label = label name to use for branches (text)
413 * %asi = ASI to use for TSB access
414 */
415
416#if defined(UTSB_PHYS)
417
418#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
419	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
420	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
421label/**/1:								;\
422	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
423	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
424	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
425	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
426	cmp	tag, tmp3		/* compare tags */		;\
427	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
428	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
429	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
430	cmp	tmp1, tmp3		/* if not successful */		;\
431	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
432	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
433label/**/2:
434
435#else /* UTSB_PHYS */
436
437#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
438	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
439	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
440label/**/1:								;\
441	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
442	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
443	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
444	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
445	cmp	tag, tmp3		/* compare tags */		;\
446	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
447	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
448	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
449	cmp	tmp1, tmp3		/* if not successful */		;\
450	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
451	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
452label/**/2:
453
454#endif /* UTSB_PHYS */
455
456#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
457#error	- TSB_SOFTSZ_MASK too small
458#endif
459
460
461/*
462 * An implementation of setx which will be hot patched at run time.
463 * since it is being hot patched, there is no value passed in.
464 * Thus, essentially we are implementing
465 *	setx value, tmp, dest
466 * where value is RUNTIME_PATCH (aka 0) in this case.
467 */
468#define	RUNTIME_PATCH_SETX(dest, tmp)					\
469	sethi	%hh(RUNTIME_PATCH), tmp					;\
470	sethi	%lm(RUNTIME_PATCH), dest				;\
471	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
472	or	dest, %lo(RUNTIME_PATCH), dest				;\
473	sllx	tmp, 32, tmp						;\
474	nop				/* for perf reasons */		;\
475	or	tmp, dest, dest		/* contents of patched value */
476
477
478#endif (lint)
479
480
481#if defined (lint)
482
483/*
484 * sfmmu related subroutines
485 */
486
487/*
488 * Use cas, if tte has changed underneath us then reread and try again.
489 * In the case of a retry, it will update sttep with the new original.
490 */
491/* ARGSUSED */
492int
493sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
494{ return(0); }
495
496/*
497 * Use cas, if tte has changed underneath us then return 1, else return 0
498 */
499/* ARGSUSED */
500int
501sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
502{ return(0); }
503
504/* ARGSUSED */
505void
506sfmmu_copytte(tte_t *sttep, tte_t *dttep)
507{}
508
509/*ARGSUSED*/
510struct tsbe *
511sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
512{ return(0); }
513
514/*ARGSUSED*/
515uint64_t
516sfmmu_make_tsbtag(caddr_t va)
517{ return(0); }
518
519#else	/* lint */
520
521	.seg	".data"
522	.global	sfmmu_panic1
523sfmmu_panic1:
524	.asciz	"sfmmu_asm: interrupts already disabled"
525
526	.global	sfmmu_panic3
527sfmmu_panic3:
528	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"
529
530	.global	sfmmu_panic4
531sfmmu_panic4:
532	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"
533
534	.global	sfmmu_panic5
535sfmmu_panic5:
536	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"
537
538
539	ENTRY_NP(sfmmu_modifytte)
540	ldx	[%o2], %g3			/* current */
541	ldx	[%o0], %g1			/* original */
5422:
543	ldx	[%o1], %g2			/* modified */
544	cmp	%g2, %g3			/* is modified = current? */
545	be,a,pt	%xcc,1f				/* yes, don't write */
546	stx	%g3, [%o0]			/* update new original */
547	casx	[%o2], %g1, %g2
548	cmp	%g1, %g2
549	be,pt	%xcc, 1f			/* cas succeeded - return */
550	  nop
551	ldx	[%o2], %g3			/* new current */
552	stx	%g3, [%o0]			/* save as new original */
553	ba,pt	%xcc, 2b
554	  mov	%g3, %g1
5551:	retl
556	membar	#StoreLoad
557	SET_SIZE(sfmmu_modifytte)
558
559	ENTRY_NP(sfmmu_modifytte_try)
560	ldx	[%o1], %g2			/* modified */
561	ldx	[%o2], %g3			/* current */
562	ldx	[%o0], %g1			/* original */
563	cmp	%g3, %g2			/* is modified = current? */
564	be,a,pn %xcc,1f				/* yes, don't write */
565	mov	0, %o1				/* as if cas failed. */
566
567	casx	[%o2], %g1, %g2
568	membar	#StoreLoad
569	cmp	%g1, %g2
570	movne	%xcc, -1, %o1			/* cas failed. */
571	move	%xcc, 1, %o1			/* cas succeeded. */
5721:
573	stx	%g2, [%o0]			/* report "current" value */
574	retl
575	mov	%o1, %o0
576	SET_SIZE(sfmmu_modifytte_try)
577
578	ENTRY_NP(sfmmu_copytte)
579	ldx	[%o0], %g1
580	retl
581	stx	%g1, [%o1]
582	SET_SIZE(sfmmu_copytte)
583
584
585	/*
586	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
587	 * %o0 = TSB base address (in), pointer to TSB entry (out)
588	 * %o1 = vaddr (in)
589	 * %o2 = vpshift (in)
590	 * %o3 = tsb size code (in)
591	 * %o4 = scratch register
592	 */
593	ENTRY_NP(sfmmu_get_tsbe)
594	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
595	retl
596	nop
597	SET_SIZE(sfmmu_get_tsbe)
598
599	/*
600	 * Return a TSB tag for the given va.
601	 * %o0 = va (in/clobbered)
602	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
603	 */
604	ENTRY_NP(sfmmu_make_tsbtag)
605	retl
606	srln	%o0, TTARGET_VA_SHIFT, %o0
607	SET_SIZE(sfmmu_make_tsbtag)
608
609#endif /* lint */
610
611/*
612 * Other sfmmu primitives
613 */
614
615
616#if defined (lint)
617void
618sfmmu_patch_ktsb(void)
619{
620}
621
622void
623sfmmu_kpm_patch_tlbm(void)
624{
625}
626
627void
628sfmmu_kpm_patch_tsbm(void)
629{
630}
631
632/* ARGSUSED */
633void
634sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
635{
636}
637
638/* ARGSUSED */
639void
640sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
641{
642}
643
644/* ARGSUSED */
645void
646sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
647{
648}
649
650/* ARGSUSED */
651void
652sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
653{
654}
655
656#else /* lint */
657
658#define	I_SIZE		4
659
660	ENTRY_NP(sfmmu_fix_ktlb_traptable)
661	/*
662	 * %o0 = start of patch area
663	 * %o1 = size code of TSB to patch
664	 * %o3 = scratch
665	 */
666	/* fix sll */
667	ld	[%o0], %o3			/* get sll */
668	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
669	st	%o3, [%o0]			/* write sll */
670	flush	%o0
671	/* fix srl */
672	add	%o0, I_SIZE, %o0		/* goto next instr. */
673	ld	[%o0], %o3			/* get srl */
674	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
675	st	%o3, [%o0]			/* write srl */
676	retl
677	flush	%o0
678	SET_SIZE(sfmmu_fix_ktlb_traptable)
679
680	ENTRY_NP(sfmmu_fixup_ktsbbase)
681	/*
682	 * %o0 = start of patch area
683	 * %o5 = kernel virtual or physical tsb base address
684	 * %o2, %o3 are used as scratch registers.
685	 */
686	/* fixup sethi instruction */
687	ld	[%o0], %o3
688	srl	%o5, 10, %o2			! offset is bits 32:10
689	or	%o3, %o2, %o3			! set imm22
690	st	%o3, [%o0]
691	/* fixup offset of lduw/ldx */
692	add	%o0, I_SIZE, %o0		! next instr
693	ld	[%o0], %o3
694	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
695	or	%o3, %o2, %o3
696	st	%o3, [%o0]
697	retl
698	flush	%o0
699	SET_SIZE(sfmmu_fixup_ktsbbase)
700
701	ENTRY_NP(sfmmu_fixup_setx)
702	/*
703	 * %o0 = start of patch area
704	 * %o4 = 64 bit value to patch
705	 * %o2, %o3 are used as scratch registers.
706	 *
707	 * Note: Assuming that all parts of the instructions which need to be
708	 *	 patched correspond to RUNTIME_PATCH (aka 0)
709	 *
710	 * Note the implementation of setx which is being patched is as follows:
711	 *
712	 * sethi   %hh(RUNTIME_PATCH), tmp
713	 * sethi   %lm(RUNTIME_PATCH), dest
714	 * or      tmp, %hm(RUNTIME_PATCH), tmp
715	 * or      dest, %lo(RUNTIME_PATCH), dest
716	 * sllx    tmp, 32, tmp
717	 * nop
718	 * or      tmp, dest, dest
719	 *
720	 * which differs from the implementation in the
721	 * "SPARC Architecture Manual"
722	 */
723	/* fixup sethi instruction */
724	ld	[%o0], %o3
725	srlx	%o4, 42, %o2			! bits [63:42]
726	or	%o3, %o2, %o3			! set imm22
727	st	%o3, [%o0]
728	/* fixup sethi instruction */
729	add	%o0, I_SIZE, %o0		! next instr
730	ld	[%o0], %o3
731	sllx	%o4, 32, %o2			! clear upper bits
732	srlx	%o2, 42, %o2			! bits [31:10]
733	or	%o3, %o2, %o3			! set imm22
734	st	%o3, [%o0]
735	/* fixup or instruction */
736	add	%o0, I_SIZE, %o0		! next instr
737	ld	[%o0], %o3
738	srlx	%o4, 32, %o2			! bits [63:32]
739	and	%o2, 0x3ff, %o2			! bits [41:32]
740	or	%o3, %o2, %o3			! set imm
741	st	%o3, [%o0]
742	/* fixup or instruction */
743	add	%o0, I_SIZE, %o0		! next instr
744	ld	[%o0], %o3
745	and	%o4, 0x3ff, %o2			! bits [9:0]
746	or	%o3, %o2, %o3			! set imm
747	st	%o3, [%o0]
748	retl
749	flush	%o0
750	SET_SIZE(sfmmu_fixup_setx)
751
752	ENTRY_NP(sfmmu_fixup_or)
753	/*
754	 * %o0 = start of patch area
755	 * %o4 = 32 bit value to patch
756	 * %o2, %o3 are used as scratch registers.
757	 * Note: Assuming that all parts of the instructions which need to be
758	 *	 patched correspond to RUNTIME_PATCH (aka 0)
759	 */
760	ld	[%o0], %o3
761	and	%o4, 0x3ff, %o2			! bits [9:0]
762	or	%o3, %o2, %o3			! set imm
763	st	%o3, [%o0]
764	retl
765	flush	%o0
766	SET_SIZE(sfmmu_fixup_or)
767
768	ENTRY_NP(sfmmu_fixup_shiftx)
769	/*
770	 * %o0 = start of patch area
771	 * %o4 = signed int immediate value to add to sllx/srlx imm field
772	 * %o2, %o3 are used as scratch registers.
773	 *
774	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
775	 * so we do a simple add.  The caller must be careful to prevent
776	 * overflow, which could easily occur if the initial value is nonzero!
777	 */
778	ld	[%o0], %o3			! %o3 = instruction to patch
779	and	%o3, 0x3f, %o2			! %o2 = existing imm value
780	add	%o2, %o4, %o2			! %o2 = new imm value
781	andn	%o3, 0x3f, %o3			! clear old imm value
782	and	%o2, 0x3f, %o2			! truncate new imm value
783	or	%o3, %o2, %o3			! set new imm value
784	st	%o3, [%o0]			! store updated instruction
785	retl
786	flush	%o0
787	SET_SIZE(sfmmu_fixup_shiftx)
788
789	ENTRY_NP(sfmmu_fixup_mmu_asi)
790	/*
791	 * Patch imm_asi of all ldda instructions in the MMU
792	 * trap handlers.  We search MMU_PATCH_INSTR instructions
793	 * starting from the itlb miss handler (trap 0x64).
794	 * %o0 = address of tt[0,1]_itlbmiss
795	 * %o1 = imm_asi to setup, shifted by appropriate offset.
796	 * %o3 = number of instructions to search
797	 * %o4 = reserved by caller: called from leaf routine
798	 */
7991:	ldsw	[%o0], %o2			! load instruction to %o2
800	brgez,pt %o2, 2f
801	  srl	%o2, 30, %o5
802	btst	1, %o5				! test bit 30; skip if not set
803	bz,pt	%icc, 2f
804	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
805	srlx	%o5, 58, %o5			! isolate op3 part of opcode
806	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
807	brnz,pt	%o5, 2f				! skip if not a match
808	  or	%o2, %o1, %o2			! or in imm_asi
809	st	%o2, [%o0]			! write patched instruction
8102:	dec	%o3
811	brnz,a,pt %o3, 1b			! loop until we're done
812	  add	%o0, I_SIZE, %o0
813	retl
814	flush	%o0
815	SET_SIZE(sfmmu_fixup_mmu_asi)
816
817	/*
818	 * Patch immediate ASI used to access the TSB in the
819	 * trap table.
820	 * inputs: %o0 = value of ktsb_phys
821	 */
822	ENTRY_NP(sfmmu_patch_mmu_asi)
823	mov	%o7, %o4			! save return pc in %o4
824	movrnz	%o0, ASI_QUAD_LDD_PHYS, %o3
825	movrz	%o0, ASI_NQUAD_LD, %o3
826	sll	%o3, 5, %o1			! imm_asi offset
827	mov	6, %o3				! number of instructions
828	sethi	%hi(dktsb), %o0			! to search
829	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
830	  or	%o0, %lo(dktsb), %o0
831	mov	6, %o3				! number of instructions
832	sethi	%hi(dktsb4m), %o0		! to search
833	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
834	  or	%o0, %lo(dktsb4m), %o0
835	mov	6, %o3				! number of instructions
836	sethi	%hi(iktsb), %o0			! to search
837	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
838	  or	%o0, %lo(iktsb), %o0
839	mov	%o4, %o7			! retore return pc -- leaf
840	retl
841	nop
842	SET_SIZE(sfmmu_patch_mmu_asi)
843
844	ENTRY_NP(sfmmu_patch_ktsb)
845	/*
846	 * We need to fix iktsb, dktsb, et. al.
847	 */
848	save	%sp, -SA(MINFRAME), %sp
849	set	ktsb_phys, %o1
850	ld	[%o1], %o4
851	set	ktsb_base, %o5
852	set	ktsb4m_base, %l1
853	brz,pt	%o4, 1f
854	  nop
855	set	ktsb_pbase, %o5
856	set	ktsb4m_pbase, %l1
8571:
858	sethi	%hi(ktsb_szcode), %o1
859	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */
860
861	sethi	%hi(iktsb), %o0
862	call	sfmmu_fix_ktlb_traptable
863	  or	%o0, %lo(iktsb), %o0
864
865	sethi	%hi(dktsb), %o0
866	call	sfmmu_fix_ktlb_traptable
867	  or	%o0, %lo(dktsb), %o0
868
869	sethi	%hi(ktsb4m_szcode), %o1
870	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */
871
872	sethi	%hi(dktsb4m), %o0
873	call	sfmmu_fix_ktlb_traptable
874	  or	%o0, %lo(dktsb4m), %o0
875
876#ifndef sun4v
877	mov	ASI_N, %o2
878	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
879	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
880	sethi	%hi(tsb_kernel_patch_asi), %o0
881	call	sfmmu_fixup_or
882	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
883#endif
884
885	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)
886
887	sethi	%hi(dktsbbase), %o0
888	call	sfmmu_fixup_setx	! patch value of ktsb base addr
889	  or	%o0, %lo(dktsbbase), %o0
890
891	sethi	%hi(iktsbbase), %o0
892	call	sfmmu_fixup_setx	! patch value of ktsb base addr
893	  or	%o0, %lo(iktsbbase), %o0
894
895	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
896	call	sfmmu_fixup_setx	! patch value of ktsb base addr
897	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
898
899#ifdef sun4v
900	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
901	call	sfmmu_fixup_setx	! patch value of ktsb base addr
902	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
903#endif /* sun4v */
904
905	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)
906
907	sethi	%hi(dktsb4mbase), %o0
908	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
909	  or	%o0, %lo(dktsb4mbase), %o0
910
911	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
912	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
913	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
914
915#ifdef sun4v
916	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
917	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
918	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
919#endif /* sun4v */
920
921	set	ktsb_szcode, %o4
922	ld	[%o4], %o4
923	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
924	call	sfmmu_fixup_or		! patch value of ktsb_szcode
925	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
926
927#ifdef sun4v
928	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
929	call	sfmmu_fixup_or		! patch value of ktsb_szcode
930	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
931#endif /* sun4v */
932
933	set	ktsb4m_szcode, %o4
934	ld	[%o4], %o4
935	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
936	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
937	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
938
939#ifdef sun4v
940	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
941	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
942	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
943#endif /* sun4v */
944
945	ret
946	restore
947	SET_SIZE(sfmmu_patch_ktsb)
948
949	ENTRY_NP(sfmmu_kpm_patch_tlbm)
950	/*
951	 * Fixup trap handlers in common segkpm case.  This is reserved
952	 * for future use should kpm TSB be changed to be other than the
953	 * kernel TSB.
954	 */
955	retl
956	nop
957	SET_SIZE(sfmmu_kpm_patch_tlbm)
958
959	ENTRY_NP(sfmmu_kpm_patch_tsbm)
960	/*
961	 * nop the branch to sfmmu_kpm_dtsb_miss_small
962	 * in the case where we are using large pages for
963	 * seg_kpm (and hence must probe the second TSB for
964	 * seg_kpm VAs)
965	 */
966	set	dktsb4m_kpmcheck_small, %o0
967	MAKE_NOP_INSTR(%o1)
968	st	%o1, [%o0]
969	flush	%o0
970	retl
971	nop
972	SET_SIZE(sfmmu_kpm_patch_tsbm)
973
974	ENTRY_NP(sfmmu_patch_utsb)
975#ifdef sun4v
976	retl
977	nop
978#else /* sun4v */
979	/*
980	 * We need to hot patch utsb_vabase and utsb4m_vabase
981	 */
982	save	%sp, -SA(MINFRAME), %sp
983
984	/* patch value of utsb_vabase */
985	set	utsb_vabase, %o1
986	ldx	[%o1], %o4
987	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
988	call	sfmmu_fixup_setx
989	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
990	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
991	call	sfmmu_fixup_setx
992	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
993	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
994	call	sfmmu_fixup_setx
995	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
996
997	/* patch value of utsb4m_vabase */
998	set	utsb4m_vabase, %o1
999	ldx	[%o1], %o4
1000	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1001	call	sfmmu_fixup_setx
1002	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1003	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1004	call	sfmmu_fixup_setx
1005	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1006	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1007	call	sfmmu_fixup_setx
1008	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1009
1010	/*
1011	 * Patch TSB base register masks and shifts if needed.
1012	 * By default the TSB base register contents are set up for 4M slab.
1013	 * If we're using a smaller slab size and reserved VA range we need
1014	 * to patch up those values here.
1015	 */
1016	set	tsb_slab_shift, %o1
1017	set	MMU_PAGESHIFT4M, %o4
1018	ldsw	[%o1], %o3
1019	subcc	%o4, %o3, %o4
1020	bz,pt	%icc, 1f
1021	  /* delay slot safe */
1022
1023	/* patch reserved VA range size if needed. */
1024	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
1025	call	sfmmu_fixup_shiftx
1026	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1027	call	sfmmu_fixup_shiftx
1028	  add	%o0, I_SIZE, %o0
1029	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
1030	call	sfmmu_fixup_shiftx
1031	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1032	call	sfmmu_fixup_shiftx
1033	  add	%o0, I_SIZE, %o0
10341:
1035	/* patch TSBREG_VAMASK used to set up TSB base register */
1036	set	tsb_slab_mask, %o1
1037	lduw	[%o1], %o4
1038	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1039	call	sfmmu_fixup_or
1040	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1041	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1042	call	sfmmu_fixup_or
1043	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1044
1045	ret
1046	restore
1047#endif /* sun4v */
1048	SET_SIZE(sfmmu_patch_utsb)
1049
1050
1051	/*
1052	 * Routine that loads an entry into a tsb using virtual addresses.
1053	 * Locking is required since all cpus can use the same TSB.
1054	 * Note that it is no longer required to have a valid context
1055	 * when calling this function.
1056	 */
1057	ENTRY_NP(sfmmu_load_tsbe)
1058	/*
1059	 * %o0 = pointer to tsbe to load
1060	 * %o1 = tsb tag
1061	 * %o2 = virtual pointer to TTE
1062	 * %o3 = 1 if physical address in %o0 else 0
1063	 */
1064	rdpr	%pstate, %o5
1065#ifdef DEBUG
1066	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1067	bnz,pt 	%icc, 1f			/* disabled, panic	 */
1068	  nop
1069
1070	sethi	%hi(panicstr), %g1
1071	ldx	[%g1 + %lo(panicstr)], %g1
1072	tst	%g1
1073	bnz,pt	%icc, 1f
1074	  nop
1075
1076	save	%sp, -SA(MINFRAME), %sp
1077	sethi	%hi(sfmmu_panic1), %o0
1078	call	panic
1079	 or	%o0, %lo(sfmmu_panic1), %o0
10801:
1081#endif /* DEBUG */
1082
1083	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1084
1085	SETUP_TSB_ASI(%o3, %g3)
1086	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1)
1087
1088	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1089
1090	retl
1091	membar	#StoreStore|#StoreLoad
1092	SET_SIZE(sfmmu_load_tsbe)
1093
1094	/*
1095	 * Flush TSB of a given entry if the tag matches.
1096	 */
1097	ENTRY(sfmmu_unload_tsbe)
1098	/*
1099	 * %o0 = pointer to tsbe to be flushed
1100	 * %o1 = tag to match
1101	 * %o2 = 1 if physical address in %o0 else 0
1102	 */
1103	SETUP_TSB_ASI(%o2, %g1)
1104	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1105	retl
1106	membar	#StoreStore|#StoreLoad
1107	SET_SIZE(sfmmu_unload_tsbe)
1108
1109	/*
1110	 * Routine that loads a TTE into the kpm TSB from C code.
1111	 * Locking is required since kpm TSB is shared among all CPUs.
1112	 */
1113	ENTRY_NP(sfmmu_kpm_load_tsb)
1114	/*
1115	 * %o0 = vaddr
1116	 * %o1 = ttep
1117	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1118	 */
1119	rdpr	%pstate, %o5			! %o5 = saved pstate
1120#ifdef DEBUG
1121	andcc	%o5, PSTATE_IE, %g0		! if interrupts already
1122	bnz,pt	%icc, 1f			! disabled, panic
1123	  nop
1124
1125	sethi	%hi(panicstr), %g1
1126	ldx	[%g1 + %lo(panicstr)], %g1
1127	tst	%g1
1128	bnz,pt	%icc, 1f
1129	  nop
1130
1131	save	%sp, -SA(MINFRAME), %sp
1132	sethi	%hi(sfmmu_panic1), %o0
1133	call	panic
1134	  or	%o0, %lo(sfmmu_panic1), %o0
11351:
1136#endif /* DEBUG */
1137	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts
1138
1139#ifndef sun4v
1140	sethi	%hi(ktsb_phys), %o4
1141	mov	ASI_N, %o3
1142	ld	[%o4 + %lo(ktsb_phys)], %o4
1143	movrnz	%o4, ASI_MEM, %o3
1144	mov	%o3, %asi
1145#endif
1146	mov	%o0, %g1			! %g1 = vaddr
1147
1148	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1149	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1150	/* %g2 = tsbep, %g1 clobbered */
1151
1152	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1153	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1154	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1)
1155
1156	wrpr	%g0, %o5, %pstate		! enable interrupts
1157	retl
1158	  membar #StoreStore|#StoreLoad
1159	SET_SIZE(sfmmu_kpm_load_tsb)
1160
1161	/*
1162	 * Routine that shoots down a TTE in the kpm TSB or in the
1163	 * kernel TSB depending on virtpg. Locking is required since
1164	 * kpm/kernel TSB is shared among all CPUs.
1165	 */
1166	ENTRY_NP(sfmmu_kpm_unload_tsb)
1167	/*
1168	 * %o0 = vaddr
1169	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1170	 */
1171#ifndef sun4v
1172	sethi	%hi(ktsb_phys), %o4
1173	mov	ASI_N, %o3
1174	ld	[%o4 + %lo(ktsb_phys)], %o4
1175	movrnz	%o4, ASI_MEM, %o3
1176	mov	%o3, %asi
1177#endif
1178	mov	%o0, %g1			! %g1 = vaddr
1179
1180	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1181	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1182	/* %g2 = tsbep, %g1 clobbered */
1183
1184	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1185	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1186	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1187
1188	retl
1189	  membar	#StoreStore|#StoreLoad
1190	SET_SIZE(sfmmu_kpm_unload_tsb)
1191
1192#endif /* lint */
1193
1194
1195#if defined (lint)
1196
1197/*ARGSUSED*/
1198pfn_t
1199sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
1200{ return(0); }
1201
1202#else /* lint */
1203
1204	ENTRY_NP(sfmmu_ttetopfn)
1205	ldx	[%o0], %g1			/* read tte */
1206	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1207	/*
1208	 * g1 = pfn
1209	 */
1210	retl
1211	mov	%g1, %o0
1212	SET_SIZE(sfmmu_ttetopfn)
1213
1214#endif /* !lint */
1215
1216
1217#if defined (lint)
1218/*
1219 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
1220 * the hash list.
1221 */
1222/* ARGSUSED */
1223void
1224sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1225	uint64_t hblkpa)
1226{
1227}
1228
1229/*
1230 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
1231 * hash list.
1232 */
1233/* ARGSUSED */
1234void
1235sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1236	uint64_t hblkpa, struct hme_blk *prev_hblkp)
1237{
1238}
1239#else /* lint */
1240
1241/*
1242 * Functions to grab/release hme bucket list lock.  I only use a byte
1243 * instead of the whole int because eventually we might want to
1244 * put some counters on the other bytes (of course, these routines would
1245 * have to change).  The code that grab this lock should execute
1246 * with interrupts disabled and hold the lock for the least amount of time
1247 * possible.
1248 */
1249
1250/*
1251 * Even though hmeh_listlock is updated using pa there's no need to flush
1252 * dcache since hmeh_listlock will be restored to the original value (0)
1253 * before interrupts are reenabled.
1254 */
1255
1256/*
1257 * For sparcv9 hme hash buckets may not be in the nucleus.  hme hash update
1258 * routines still use virtual addresses to update the bucket fields. But they
1259 * must not cause a TLB miss after grabbing the low level bucket lock. To
1260 * achieve this we must make sure the bucket structure is completely within an
1261 * 8K page.
1262 */
1263
1264#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1))
1265#error - the size of hmehash_bucket structure is not power of 2
1266#endif
1267
1268#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi)           \
1269	mov     0xff, tmp2                                      ;\
1270	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1271label1:                                                         ;\
1272	casa    [tmp1]asi, %g0, tmp2                            ;\
1273	brnz,pn tmp2, label1                                    ;\
1274	mov     0xff, tmp2                                      ;\
1275	membar  #LoadLoad
1276
1277#define HMELOCK_EXIT(hmebp, tmp1, asi)                          \
1278	membar  #LoadStore|#StoreStore                          ;\
1279	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1280	sta     %g0, [tmp1]asi
1281
1282	.seg	".data"
1283hblk_add_panic1:
1284	.ascii	"sfmmu_hblk_hash_add: interrupts disabled"
1285	.byte	0
1286hblk_add_panic2:
1287	.ascii	"sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
1288	.byte	0
1289	.align	4
1290	.seg	".text"
1291
1292	ENTRY_NP(sfmmu_hblk_hash_add)
1293	/*
1294	 * %o0 = hmebp
1295	 * %o1 = hmeblkp
1296	 * %o2 = hblkpa
1297	 */
1298	rdpr	%pstate, %o5
1299#ifdef DEBUG
1300	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1301	bnz,pt %icc, 3f				/* disabled, panic	 */
1302	  nop
1303	save	%sp, -SA(MINFRAME), %sp
1304	sethi	%hi(hblk_add_panic1), %o0
1305	call	panic
1306	 or	%o0, %lo(hblk_add_panic1), %o0
1307	ret
1308	restore
1309
13103:
1311#endif /* DEBUG */
1312	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1313	mov	%o2, %g1
1314
1315	/*
1316	 * g1 = hblkpa
1317	 */
1318	ldn	[%o0 + HMEBUCK_HBLK], %o4	/* next hmeblk */
1319	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = next hblkpa */
1320#ifdef	DEBUG
1321	cmp	%o4, %g0
1322	bne,pt %xcc, 1f
1323	 nop
1324	brz,pt %g2, 1f
1325	 nop
1326	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1327	save	%sp, -SA(MINFRAME), %sp
1328	sethi	%hi(hblk_add_panic2), %o0
1329	call	panic
1330	  or	%o0, %lo(hblk_add_panic2), %o0
1331	ret
1332	restore
13331:
1334#endif /* DEBUG */
1335	/*
1336	 * We update hmeblks entries before grabbing lock because the stores
1337	 * could take a tlb miss and require the hash lock.  The buckets
1338	 * are part of the nucleus so we are cool with those stores.
1339	 *
1340	 * if buckets are not part of the nucleus our game is to
1341	 * not touch any other page via va until we drop the lock.
1342	 * This guarantees we won't get a tlb miss before the lock release
1343	 * since interrupts are disabled.
1344	 */
1345	stn	%o4, [%o1 + HMEBLK_NEXT]	/* update hmeblk's next */
1346	stx	%g2, [%o1 + HMEBLK_NEXTPA]	/* update hmeblk's next pa */
1347	HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N)
1348	stn	%o1, [%o0 + HMEBUCK_HBLK]	/* update bucket hblk next */
1349	stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* add hmeblk to list */
1350	HMELOCK_EXIT(%o0, %g2, ASI_N)
1351	retl
1352	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1353	SET_SIZE(sfmmu_hblk_hash_add)
1354
1355	ENTRY_NP(sfmmu_hblk_hash_rm)
1356	/*
1357	 * This function removes an hmeblk from the hash chain.
1358	 * It is written to guarantee we don't take a tlb miss
1359	 * by using physical addresses to update the list.
1360	 *
1361	 * %o0 = hmebp
1362	 * %o1 = hmeblkp
1363	 * %o2 = hmeblkp previous pa
1364	 * %o3 = hmeblkp previous
1365	 */
1366
1367	mov	%o3, %o4			/* o4 = hmeblkp previous */
1368
1369	rdpr	%pstate, %o5
1370#ifdef DEBUG
1371	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1372	bnz,pt 	%icc, 3f			/* disabled, panic	 */
1373	  nop
1374
1375	sethi	%hi(panicstr), %g1
1376	ldx	[%g1 + %lo(panicstr)], %g1
1377	tst	%g1
1378	bnz,pt	%icc, 3f
1379	  nop
1380
1381	sethi	%hi(sfmmu_panic1), %o0
1382	call	panic
1383	 or	%o0, %lo(sfmmu_panic1), %o0
13843:
1385#endif /* DEBUG */
1386	/*
1387	 * disable interrupts, clear Address Mask to access 64 bit physaddr
1388	 */
1389	andn    %o5, PSTATE_IE, %g1
1390	wrpr    %g1, 0, %pstate
1391
1392#ifndef sun4v
1393	sethi   %hi(dcache_line_mask), %g4
1394	ld      [%g4 + %lo(dcache_line_mask)], %g4
1395#endif /* sun4v */
1396
1397	/*
1398	 * if buckets are not part of the nucleus our game is to
1399	 * not touch any other page via va until we drop the lock.
1400	 * This guarantees we won't get a tlb miss before the lock release
1401	 * since interrupts are disabled.
1402	 */
1403	HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N)
1404	ldn	[%o0 + HMEBUCK_HBLK], %g2	/* first hmeblk in list */
1405	cmp	%g2, %o1
1406	bne,pt	%ncc,1f
1407	 mov	ASI_MEM, %asi
1408	/*
1409	 * hmeblk is first on list
1410	 */
1411	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = hmeblk pa */
1412	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1413	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1414	stn	%o3, [%o0 + HMEBUCK_HBLK]	/* write va */
1415	ba,pt	%xcc, 2f
1416	  stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* write pa */
14171:
1418	/* hmeblk is not first on list */
1419
1420	mov	%o2, %g3
1421#ifndef sun4v
1422	GET_CPU_IMPL(%g2)
1423	cmp %g2, CHEETAH_IMPL
1424	bge %icc, hblk_hash_rm_1
1425	and	%o4, %g4, %g2
1426	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev pa from dcache */
1427	add	%o4, HMEBLK_NEXT, %o4
1428	and	%o4, %g4, %g2
1429	ba	hblk_hash_rm_2
1430	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev va from dcache */
1431hblk_hash_rm_1:
1432
1433	stxa	%g0, [%g3]ASI_DC_INVAL		/* flush prev pa from dcache */
1434	membar	#Sync
1435	add     %g3, HMEBLK_NEXT, %g2
1436	stxa	%g0, [%g2]ASI_DC_INVAL		/* flush prev va from dcache */
1437hblk_hash_rm_2:
1438	membar	#Sync
1439#endif /* sun4v */
1440	ldxa	[%g3 + HMEBLK_NEXTPA] %asi, %g2	/* g2 = hmeblk pa */
1441	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1442	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1443	stna	%o3, [%g3 + HMEBLK_NEXT] %asi	/* write va */
1444	stxa	%g1, [%g3 + HMEBLK_NEXTPA] %asi	/* write pa */
14452:
1446	HMELOCK_EXIT(%o0, %g2, ASI_N)
1447	retl
1448	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1449	SET_SIZE(sfmmu_hblk_hash_rm)
1450
1451#endif /* lint */
1452
1453/*
1454 * These macros are used to update global sfmmu hme hash statistics
1455 * in perf critical paths. It is only enabled in debug kernels or
1456 * if SFMMU_STAT_GATHER is defined
1457 */
1458#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1459#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1460	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1461	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
1462	cmp	tmp1, hatid						;\
1463	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
1464	set	sfmmu_global_stat, tmp1					;\
1465	add	tmp1, tmp2, tmp1					;\
1466	ld	[tmp1], tmp2						;\
1467	inc	tmp2							;\
1468	st	tmp2, [tmp1]
1469
1470#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1471	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1472	mov	HATSTAT_KHASH_LINKS, tmp2				;\
1473	cmp	tmp1, hatid						;\
1474	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
1475	set	sfmmu_global_stat, tmp1					;\
1476	add	tmp1, tmp2, tmp1					;\
1477	ld	[tmp1], tmp2						;\
1478	inc	tmp2							;\
1479	st	tmp2, [tmp1]
1480
1481
1482#else /* DEBUG || SFMMU_STAT_GATHER */
1483
1484#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1485
1486#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1487
1488#endif  /* DEBUG || SFMMU_STAT_GATHER */
1489
1490/*
1491 * This macro is used to update global sfmmu kstas in non
1492 * perf critical areas so they are enabled all the time
1493 */
1494#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
1495	sethi	%hi(sfmmu_global_stat), tmp1				;\
1496	add	tmp1, statname, tmp1					;\
1497	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
1498	inc	tmp2							;\
1499	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1500
1501/*
1502 * These macros are used to update per cpu stats in non perf
1503 * critical areas so they are enabled all the time
1504 */
1505#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
1506	ld	[tsbarea + stat], tmp1					;\
1507	inc	tmp1							;\
1508	st	tmp1, [tsbarea + stat]
1509
1510/*
1511 * These macros are used to update per cpu stats in non perf
1512 * critical areas so they are enabled all the time
1513 */
1514#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
1515	lduh	[tsbarea + stat], tmp1					;\
1516	inc	tmp1							;\
1517	stuh	tmp1, [tsbarea + stat]
1518
1519#if defined(KPM_TLBMISS_STATS_GATHER)
1520	/*
1521	 * Count kpm dtlb misses separately to allow a different
1522	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1523	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1524	 */
1525#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
1526	brgez	tagacc, label	/* KPM VA? */				;\
1527	nop								;\
1528	CPU_INDEX(tmp1, tsbma)						;\
1529	sethi	%hi(kpmtsbm_area), tsbma				;\
1530	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
1531	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
1532	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
1533	/* VA range check */						;\
1534	ldx	[tsbma + KPMTSBM_VBASE], val				;\
1535	cmp	tagacc, val						;\
1536	blu,pn	%xcc, label						;\
1537	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
1538	cmp	tagacc, tmp1						;\
1539	bgeu,pn	%xcc, label						;\
1540	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
1541	inc	val							;\
1542	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
1543label:
1544#else
1545#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1546#endif	/* KPM_TLBMISS_STATS_GATHER */
1547
1548#if defined (lint)
1549/*
1550 * The following routines are jumped to from the mmu trap handlers to do
1551 * the setting up to call systrap.  They are separate routines instead of
1552 * being part of the handlers because the handlers would exceed 32
1553 * instructions and since this is part of the slow path the jump
1554 * cost is irrelevant.
1555 */
1556void
1557sfmmu_pagefault(void)
1558{
1559}
1560
1561void
1562sfmmu_mmu_trap(void)
1563{
1564}
1565
1566void
1567sfmmu_window_trap(void)
1568{
1569}
1570
1571void
1572sfmmu_kpm_exception(void)
1573{
1574}
1575
1576#else /* lint */
1577
1578#ifdef	PTL1_PANIC_DEBUG
1579	.seg	".data"
1580	.global	test_ptl1_panic
1581test_ptl1_panic:
1582	.word	0
1583	.align	8
1584
1585	.seg	".text"
1586	.align	4
1587#endif	/* PTL1_PANIC_DEBUG */
1588
1589
1590	ENTRY_NP(sfmmu_pagefault)
1591	SET_GL_REG(1)
1592	USE_ALTERNATE_GLOBALS(%g5)
1593	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1594	rdpr	%tt, %g6
1595	cmp	%g6, FAST_IMMU_MISS_TT
1596	be,a,pn	%icc, 1f
1597	  mov	T_INSTR_MMU_MISS, %g3
1598	cmp	%g6, T_INSTR_MMU_MISS
1599	be,a,pn	%icc, 1f
1600	  mov	T_INSTR_MMU_MISS, %g3
1601	mov	%g5, %g2
1602	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1603	cmp	%g6, FAST_DMMU_MISS_TT
1604	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1605	cmp	%g6, T_DATA_MMU_MISS
1606	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1607
1608#ifdef  PTL1_PANIC_DEBUG
1609	/* check if we want to test the tl1 panic */
1610	sethi	%hi(test_ptl1_panic), %g4
1611	ld	[%g4 + %lo(test_ptl1_panic)], %g1
1612	st	%g0, [%g4 + %lo(test_ptl1_panic)]
1613	cmp	%g1, %g0
1614	bne,a,pn %icc, ptl1_panic
1615	  or	%g0, PTL1_BAD_DEBUG, %g1
1616#endif	/* PTL1_PANIC_DEBUG */
16171:
1618	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
1619	/*
1620	 * g2 = tag access reg
1621	 * g3.l = type
1622	 * g3.h = 0
1623	 */
1624	sethi	%hi(trap), %g1
1625	or	%g1, %lo(trap), %g1
16262:
1627	ba,pt	%xcc, sys_trap
1628	  mov	-1, %g4
1629	SET_SIZE(sfmmu_pagefault)
1630
1631	ENTRY_NP(sfmmu_mmu_trap)
1632	SET_GL_REG(1)
1633	USE_ALTERNATE_GLOBALS(%g5)
1634	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
1635	rdpr	%tt, %g6
1636	cmp	%g6, FAST_IMMU_MISS_TT
1637	be,a,pn	%icc, 1f
1638	  mov	T_INSTR_MMU_MISS, %g3
1639	cmp	%g6, T_INSTR_MMU_MISS
1640	be,a,pn	%icc, 1f
1641	  mov	T_INSTR_MMU_MISS, %g3
1642	mov	%g5, %g2
1643	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1644	cmp	%g6, FAST_DMMU_MISS_TT
1645	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1646	cmp	%g6, T_DATA_MMU_MISS
1647	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
16481:
1649	/*
1650	 * g2 = tag access reg
1651	 * g3 = type
1652	 */
1653	sethi	%hi(sfmmu_tsbmiss_exception), %g1
1654	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
1655	ba,pt	%xcc, sys_trap
1656	  mov	-1, %g4
1657	/*NOTREACHED*/
1658	SET_SIZE(sfmmu_mmu_trap)
1659
1660	ENTRY_NP(sfmmu_suspend_tl)
1661	SET_GL_REG(1)
1662	USE_ALTERNATE_GLOBALS(%g5)
1663	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
1664	rdpr	%tt, %g6
1665	cmp	%g6, FAST_IMMU_MISS_TT
1666	be,a,pn	%icc, 1f
1667	  mov	T_INSTR_MMU_MISS, %g3
1668	mov	%g5, %g2
1669	cmp	%g6, FAST_DMMU_MISS_TT
1670	move	%icc, T_DATA_MMU_MISS, %g3
1671	movne	%icc, T_DATA_PROT, %g3
16721:
1673	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
1674	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
1675	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
1676	ba,pt	%xcc, sys_trap
1677	  mov	PIL_15, %g4
1678	/*NOTREACHED*/
1679	SET_SIZE(sfmmu_suspend_tl)
1680
1681	/*
1682	 * No %g registers in use at this point.
1683	 */
1684	ENTRY_NP(sfmmu_window_trap)
1685	rdpr	%tpc, %g1
1686#ifdef sun4v
1687#ifdef DEBUG
1688	/* We assume previous %gl was 1 */
1689	rdpr	%tstate, %g4
1690	srlx	%g4, TSTATE_GL_SHIFT, %g4
1691	and	%g4, TSTATE_GL_MASK, %g4
1692	cmp	%g4, 1
1693	bne,a,pn %icc, ptl1_panic
1694	  mov	PTL1_BAD_WTRAP, %g1
1695#endif /* DEBUG */
1696	/* user miss at tl>1. better be the window handler or user_rtt */
1697	/* in user_rtt? */
1698	set	rtt_fill_start, %g4
1699	cmp	%g1, %g4
1700	blu,pn %xcc, 6f
1701	 .empty
1702	set	rtt_fill_end, %g4
1703	cmp	%g1, %g4
1704	bgeu,pn %xcc, 6f
1705	 nop
1706	set	fault_rtt_fn1, %g1
1707	wrpr	%g0, %g1, %tnpc
1708	ba,a	7f
17096:
1710	! must save this trap level before descending trap stack
1711	! no need to save %tnpc, either overwritten or discarded
1712	! already got it: rdpr	%tpc, %g1
1713	rdpr	%tstate, %g6
1714	rdpr	%tt, %g7
1715	! trap level saved, go get underlying trap type
1716	rdpr	%tl, %g5
1717	sub	%g5, 1, %g3
1718	wrpr	%g3, %tl
1719	rdpr	%tt, %g2
1720	wrpr	%g5, %tl
1721	! restore saved trap level
1722	wrpr	%g1, %tpc
1723	wrpr	%g6, %tstate
1724	wrpr	%g7, %tt
1725#else /* sun4v */
1726	/* user miss at tl>1. better be the window handler */
1727	rdpr	%tl, %g5
1728	sub	%g5, 1, %g3
1729	wrpr	%g3, %tl
1730	rdpr	%tt, %g2
1731	wrpr	%g5, %tl
1732#endif /* sun4v */
1733	and	%g2, WTRAP_TTMASK, %g4
1734	cmp	%g4, WTRAP_TYPE
1735	bne,pn	%xcc, 1f
1736	 nop
1737	/* tpc should be in the trap table */
1738	set	trap_table, %g4
1739	cmp	%g1, %g4
1740	blt,pn %xcc, 1f
1741	 .empty
1742	set	etrap_table, %g4
1743	cmp	%g1, %g4
1744	bge,pn %xcc, 1f
1745	 .empty
1746	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
1747	add	%g1, WTRAP_FAULTOFF, %g1
1748	wrpr	%g0, %g1, %tnpc
17497:
1750	/*
1751	 * some wbuf handlers will call systrap to resolve the fault
1752	 * we pass the trap type so they figure out the correct parameters.
1753	 * g5 = trap type, g6 = tag access reg
1754	 */
1755
1756	/*
1757	 * only use g5, g6, g7 registers after we have switched to alternate
1758	 * globals.
1759	 */
1760	SET_GL_REG(1)
1761	USE_ALTERNATE_GLOBALS(%g5)
1762	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
1763	rdpr	%tt, %g7
1764	cmp	%g7, FAST_IMMU_MISS_TT
1765	be,a,pn	%icc, ptl1_panic
1766	  mov	PTL1_BAD_WTRAP, %g1
1767	cmp	%g7, T_INSTR_MMU_MISS
1768	be,a,pn	%icc, ptl1_panic
1769	  mov	PTL1_BAD_WTRAP, %g1
1770	mov	T_DATA_PROT, %g5
1771	cmp	%g7, FAST_DMMU_MISS_TT
1772	move	%icc, T_DATA_MMU_MISS, %g5
1773	cmp	%g7, T_DATA_MMU_MISS
1774	move	%icc, T_DATA_MMU_MISS, %g5
1775	! XXXQ AGS re-check out this one
1776	done
17771:
1778	CPU_ADDR(%g1, %g4)
1779	ld	[%g1 + CPU_TL1_HDLR], %g4
1780	brnz,a,pt %g4, sfmmu_mmu_trap
1781	  st	%g0, [%g1 + CPU_TL1_HDLR]
1782	ba,pt	%icc, ptl1_panic
1783	  mov	PTL1_BAD_TRAP, %g1
1784	SET_SIZE(sfmmu_window_trap)
1785
1786	ENTRY_NP(sfmmu_kpm_exception)
1787	/*
1788	 * We have accessed an unmapped segkpm address or a legal segkpm
1789	 * address which is involved in a VAC alias conflict prevention.
1790	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
1791	 * set. If it is, we will instead note that a fault has occurred
1792	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
1793	 * a "retry"). This will step over the faulting instruction.
1794	 * Note that this means that a legal segkpm address involved in
1795	 * a VAC alias conflict prevention (a rare case to begin with)
1796	 * cannot be used in DTrace.
1797	 */
1798	CPU_INDEX(%g1, %g2)
1799	set	cpu_core, %g2
1800	sllx	%g1, CPU_CORE_SHIFT, %g1
1801	add	%g1, %g2, %g1
1802	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
1803	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
1804	bz	0f
1805	or	%g2, CPU_DTRACE_BADADDR, %g2
1806	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
1807	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
1808	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
1809	done
18100:
1811	TSTAT_CHECK_TL1(1f, %g1, %g2)
18121:
1813	SET_GL_REG(1)
1814	USE_ALTERNATE_GLOBALS(%g5)
1815	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
1816	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1817	/*
1818	 * g2=tagacc g3.l=type g3.h=0
1819	 */
1820	sethi	%hi(trap), %g1
1821	or	%g1, %lo(trap), %g1
1822	ba,pt	%xcc, sys_trap
1823	mov	-1, %g4
1824	SET_SIZE(sfmmu_kpm_exception)
1825
1826#endif /* lint */
1827
1828#if defined (lint)
1829
1830void
1831sfmmu_tsb_miss(void)
1832{
1833}
1834
1835void
1836sfmmu_kpm_dtsb_miss(void)
1837{
1838}
1839
1840void
1841sfmmu_kpm_dtsb_miss_small(void)
1842{
1843}
1844
1845#else /* lint */
1846
1847
1848#if (CTX_SIZE != (1 << CTX_SZ_SHIFT))
1849#error - size of context struct does not match with CTX_SZ_SHIFT
1850#endif
1851
1852#if (IMAP_SEG != 0)
1853#error - ism_map->ism_seg offset is not zero
1854#endif
1855
1856/*
1857 * Copies ism mapping for this ctx in param "ism" if this is a ISM
1858 * tlb miss and branches to label "ismhit". If this is not an ISM
1859 * process or an ISM tlb miss it falls thru.
1860 *
1861 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
1862 * this process.
1863 * If so, it will branch to label "ismhit".  If not, it will fall through.
1864 *
1865 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
1866 * so that any other threads of this process will not try and walk the ism
1867 * maps while they are being changed.
1868 *
1869 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
1870 *       will make sure of that. This means we can terminate our search on
1871 *       the first zero mapping we find.
1872 *
1873 * Parameters:
1874 * tagacc	= tag access register (vaddr + ctx) (in)
1875 * tsbmiss	= address of tsb miss area (in)
1876 * ismseg	= contents of ism_seg for this ism map (out)
1877 * ismhat	= physical address of imap_ismhat for this ism map (out)
1878 * tmp1		= scratch reg (CLOBBERED)
1879 * tmp2		= scratch reg (CLOBBERED)
1880 * tmp3		= scratch reg (CLOBBERED)
1881 * label:    temporary labels
1882 * ismhit:   label where to jump to if an ism dtlb miss
1883 * exitlabel:label where to jump if hat is busy due to hat_unshare.
1884 */
1885#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
1886	label, ismhit)							\
1887	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
1888	brlz,pt  tmp1, label/**/3		/* exit if -1 */	;\
1889	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
1890label/**/1:								;\
1891	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
1892	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
1893label/**/2:								;\
1894	brz,pt  ismseg, label/**/3		/* no mapping */	;\
1895	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
1896	lduha	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
1897	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
1898	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
1899	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
1900	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
1901	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
1902	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
1903	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
1904	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
1905	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
1906									;\
1907	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
1908	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
1909	cmp	ismhat, tmp1						;\
1910	bl,pt	%xcc, label/**/2		/* keep looking  */	;\
1911	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
1912									;\
1913	add	tmp3, IBLK_NEXTPA, tmp1					;\
1914	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
1915	brgez,pt tmp1, label/**/1		/* continue if not -1*/	;\
1916	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
1917label/**/3:
1918
1919/*
1920 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
1921 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
1922 * Parameters:
1923 * vaddr = reg containing virtual address
1924 * hatid = reg containing sfmmu pointer
1925 * hmeshift = constant/register to shift vaddr to obtain vapg
1926 * hmebp = register where bucket pointer will be stored
1927 * vapg = register where virtual page will be stored
1928 * tmp1, tmp2 = tmp registers
1929 */
1930
1931
1932#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
1933	vapg, label, tmp1, tmp2)					\
1934	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
1935	brnz,a,pt tmp1, label/**/1					;\
1936	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
1937	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
1938	ba,pt	%xcc, label/**/2					;\
1939	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
1940label/**/1:								;\
1941	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
1942label/**/2:								;\
1943	srlx	tagacc, hmeshift, vapg					;\
1944	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
1945	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
1946	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
1947	add	hmebp, tmp1, hmebp
1948
1949/*
1950 * hashtag includes bspage + hashno (64 bits).
1951 */
1952
1953#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
1954	sllx	vapg, hmeshift, vapg					;\
1955	or	vapg, hashno, hblktag
1956
1957/*
1958 * Function to traverse hmeblk hash link list and find corresponding match.
1959 * The search is done using physical pointers. It returns the physical address
1960 * and virtual address pointers to the hmeblk that matches with the tag
1961 * provided.
1962 * Parameters:
1963 * hmebp	= register that points to hme hash bucket, also used as
1964 *		  tmp reg (clobbered)
1965 * hmeblktag	= register with hmeblk tag match
1966 * hatid	= register with hatid
1967 * hmeblkpa	= register where physical ptr will be stored
1968 * hmeblkva	= register where virtual ptr will be stored
1969 * tmp1		= tmp reg
1970 * label: temporary label
1971 */
1972
1973#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva,	\
1974	tsbarea, tmp1, label)					 	\
1975	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
1976	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
1977	add     hmebp, HMEBUCK_HBLK, hmeblkva				;\
1978	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
1979	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1980label/**/1:								;\
1981	brz,pn	hmeblkva, label/**/2					;\
1982	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1983	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
1984	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
1985	add	hmebp, CLONGSIZE, hmebp					;\
1986	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
1987	xor	tmp1, hmeblktag, tmp1					;\
1988	xor	hmebp, hatid, hmebp					;\
1989	or	hmebp, tmp1, hmebp					;\
1990	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
1991	  add	hmeblkpa, HMEBLK_NEXT, hmebp				;\
1992	ldna	[hmebp]ASI_MEM, hmeblkva	/* hmeblk ptr va */	;\
1993	add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
1994	ba,pt	%xcc, label/**/1					;\
1995	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
1996label/**/2:
1997
1998
1999#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
2000#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
2001#endif
2002
2003/*
2004 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2005 * he offset for the corresponding hment.
2006 * Parameters:
2007 * vaddr = register with virtual address
2008 * hmeblkpa = physical pointer to hme_blk
2009 * hment = register where address of hment will be stored
2010 * hmentoff = register where hment offset will be stored
2011 * label1 = temporary label
2012 */
2013#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, tmp1, label1)	\
2014	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
2015	lda	[hmentoff]ASI_MEM, tmp1 				;\
2016	andcc	tmp1, HBLK_SZMASK, %g0	 /* tmp1 = get_hblk_sz(%g5) */	;\
2017	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
2018	  or	%g0, HMEBLK_HME1, hmentoff				;\
2019	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
2020	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
2021	sllx	tmp1, SFHME_SHIFT, tmp1					;\
2022	add	tmp1, HMEBLK_HME1, hmentoff				;\
2023label1:
2024
2025/*
2026 * GET_TTE is a macro that returns a TTE given a tag and hatid.
2027 *
2028 * tagacc	= tag access register (vaddr + ctx) (in)
2029 * hatid	= sfmmu pointer for TSB miss (in)
2030 * tte		= tte for TLB miss if found, otherwise clobbered (out)
2031 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
2032 * hmeblkva	= VA of hment if found, otherwise clobbered (out)
2033 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
2034 * hmentoff	= temporarily stores hment offset (clobbered)
2035 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
2036 *		  for this page size.
2037 * hashno	= constant/register hash number
2038 * label	= temporary label for branching within macro.
2039 * foundlabel	= label to jump to when tte is found.
2040 * suspendlabel= label to jump to when tte is suspended.
2041 * exitlabel	= label to jump to when tte is not found.  The hmebp lock
2042 *		  is still held at this time.
2043 *
2044 * The caller should set up the tsbmiss->scratch[2] field correctly before
2045 * calling this funciton  (aka TSBMISS_SCRATCH + TSBMISS_HATID)
2046 */
2047#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmentoff, \
2048		hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \
2049									;\
2050	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2051	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2052	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2053		hmeblkpa, label/**/5, hmentoff, hmeblkva)		;\
2054									;\
2055	/*								;\
2056	 * tagacc = tagacc						;\
2057	 * hatid = hatid						;\
2058	 * tsbarea = tsbarea						;\
2059	 * tte   = hmebp (hme bucket pointer)				;\
2060	 * hmeblkpa  = vapg  (virtual page)				;\
2061	 * hmentoff, hmeblkva = scratch					;\
2062	 */								;\
2063	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmentoff)	;\
2064									;\
2065	/*								;\
2066	 * tagacc = tagacc						;\
2067	 * hatid = hatid						;\
2068	 * tte   = hmebp						;\
2069	 * hmeblkpa  = CLOBBERED					;\
2070	 * hmentoff  = htag_bspage & hashno				;\
2071	 * hmeblkva  = scratch						;\
2072	 */								;\
2073	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2074	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
2075	HMEHASH_SEARCH(tte, hmentoff, hatid, hmeblkpa, hmeblkva, 	\
2076		tsbarea, tagacc, label/**/1)				;\
2077	/*								;\
2078	 * tagacc = CLOBBERED						;\
2079	 * tte = CLOBBERED						;\
2080	 * hmeblkpa = hmeblkpa						;\
2081	 * hmeblkva = hmeblkva						;\
2082	 */								;\
2083	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2084	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2085	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2086	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2087	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2088	  nop								;\
2089label/**/4:								;\
2090	/*								;\
2091	 * We have found the hmeblk containing the hment.		;\
2092	 * Now we calculate the corresponding tte.			;\
2093	 *								;\
2094	 * tagacc = tagacc						;\
2095	 * hatid = clobbered						;\
2096	 * tte   = hmebp						;\
2097	 * hmeblkpa  = hmeblkpa						;\
2098	 * hmentoff  = hblktag						;\
2099	 * hmeblkva  = hmeblkva 					;\
2100	 */								;\
2101	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hmentoff, hatid, label/**/2)	;\
2102									;\
2103	add	hmentoff, SFHME_TTE, hmentoff				;\
2104	add	hmeblkpa, hmentoff, hmeblkpa				;\
2105	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2106	add	hmeblkva, hmentoff, hmeblkva				;\
2107	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2108	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmentoff ;\
2109	HMELOCK_EXIT(hmentoff, hmentoff, ASI_MEM)	/* drop lock */	;\
2110	set	TTE_SUSPEND, hmentoff					;\
2111	TTE_SUSPEND_INT_SHIFT(hmentoff)					;\
2112	btst	tte, hmentoff						;\
2113	bz,pt	%xcc, foundlabel					;\
2114	 nop								;\
2115									;\
2116	/*								;\
2117	 * Mapping is suspended, so goto suspend label.			;\
2118	 */								;\
2119	ba,pt	%xcc, suspendlabel					;\
2120	  nop
2121
2122	/*
2123	 * KERNEL PROTECTION HANDLER
2124	 *
2125	 * g1 = tsb8k pointer register (clobbered)
2126	 * g2 = tag access register (ro)
2127	 * g3 - g7 = scratch registers
2128	 *
2129	 * Note: This function is patched at runtime for performance reasons.
2130	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
2131	 */
2132	ENTRY_NP(sfmmu_kprot_trap)
2133	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2134sfmmu_kprot_patch_ktsb_base:
2135	RUNTIME_PATCH_SETX(%g1, %g6)
2136	/* %g1 = contents of ktsb_base or ktsb_pbase */
2137sfmmu_kprot_patch_ktsb_szcode:
2138	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
2139
2140	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2141	! %g1 = First TSB entry pointer, as TSB miss handler expects
2142
2143	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2144sfmmu_kprot_patch_ktsb4m_base:
2145	RUNTIME_PATCH_SETX(%g3, %g6)
2146	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2147sfmmu_kprot_patch_ktsb4m_szcode:
2148	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
2149
2150	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2151	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2152
2153	CPU_TSBMISS_AREA(%g6, %g7)
2154	HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2155	ba,pt	%xcc, sfmmu_tsb_miss_tt
2156	  nop
2157
2158	/*
2159	 * USER PROTECTION HANDLER
2160	 *
2161	 * g1 = tsb8k pointer register (ro)
2162	 * g2 = tag access register (ro)
2163	 * g3 = faulting context (clobbered, currently not used)
2164	 * g4 - g7 = scratch registers
2165	 */
2166	ALTENTRY(sfmmu_uprot_trap)
2167#ifdef sun4v
2168	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2169	/* %g1 = first TSB entry ptr now, %g2 preserved */
2170
2171	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
2172	brlz,pt %g3, 9f			/* check for 2nd TSB */
2173	  mov	%g0, %g3		/* clear second tsbe ptr */
2174
2175	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2176	/* %g3 = second TSB entry ptr now, %g2 preserved */
2177
2178#else /* sun4v */
2179
2180	brgez,pt %g1, 9f		/* check for 2nd TSB */
2181	  mov	%g0, %g3		/* clear second tsbe ptr */
2182
2183	mov	%g2, %g7
2184	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2185	/* %g3 = second TSB entry ptr now, %g7 clobbered */
2186	mov	%g1, %g7
2187	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2188
2189#endif /* sun4v */
21909:
2191	CPU_TSBMISS_AREA(%g6, %g7)
2192	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2193	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
2194	  nop
2195
2196	/*
2197	 * Kernel 8K page iTLB miss.  We also get here if we took a
2198	 * fast instruction access mmu miss trap while running in
2199	 * invalid context.
2200	 *
2201	 * %g1 = 8K TSB pointer register (not used, clobbered)
2202	 * %g2 = tag access register (used)
2203	 * %g3 = faulting context id (used)
2204	 * %g7 = 4M virtual page number for tag matching  (used)
2205	 */
2206	.align	64
2207	ALTENTRY(sfmmu_kitlb_miss)
2208	brnz,pn %g3, tsb_tl0_noctxt
2209	  nop
2210
2211	/* kernel miss */
2212	/* get kernel tsb pointer */
2213	/* we patch the next set of instructions at run time */
2214	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2215iktsbbase:
2216	RUNTIME_PATCH_SETX(%g4, %g5)
2217	/* %g4 = contents of ktsb_base or ktsb_pbase */
2218
2219iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2220	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2221	or	%g4, %g1, %g1			! form tsb ptr
2222	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2223	cmp	%g4, %g7
2224	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
2225	  andcc %g5, TTE_EXECPRM_INT, %g0	! check exec bit
2226	bz,pn	%icc, exec_fault
2227	  nop
2228	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2229	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2230	retry
2231
2232	/*
2233	 * Kernel dTLB miss.  We also get here if we took a fast data
2234	 * access mmu miss trap while running in invalid context.
2235	 *
2236	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2237	 *	We select the TSB miss handler to branch to depending on
2238	 *	the virtual address of the access.  In the future it may
2239	 *	be desirable to separate kpm TTEs into their own TSB,
2240	 *	in which case all that needs to be done is to set
2241	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2242	 *	early in the miss if we detect a kpm VA to a new handler.
2243	 *
2244	 * %g1 = 8K TSB pointer register (not used, clobbered)
2245	 * %g2 = tag access register (used)
2246	 * %g3 = faulting context id (used)
2247	 */
2248	.align	64
2249	ALTENTRY(sfmmu_kdtlb_miss)
2250	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
2251	  nop
2252
2253	/* Gather some stats for kpm misses in the TLB. */
2254	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2255	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2256
2257	/*
2258	 * Get first TSB offset and look for 8K/64K/512K mapping
2259	 * using the 8K virtual page as the index.
2260	 *
2261	 * We patch the next set of instructions at run time;
2262	 * any changes here require sfmmu_patch_ktsb changes too.
2263	 */
2264dktsbbase:
2265	RUNTIME_PATCH_SETX(%g7, %g6)
2266	/* %g7 = contents of ktsb_base or ktsb_pbase */
2267
2268dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2269	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2270
2271	/*
2272	 * At this point %g1 is our index into the TSB.
2273	 * We just masked off enough bits of the VA depending
2274	 * on our TSB size code.
2275	 */
2276	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2277	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2278	cmp	%g6, %g4			! compare tag
2279	bne,pn	%xcc, dktsb4m_kpmcheck_small
2280	  add	%g7, %g1, %g1			/* form tsb ptr */
2281	TT_TRACE(trace_tsbhit)
2282	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2283	/* trapstat expects tte in %g5 */
2284	retry
2285
2286	/*
2287	 * If kpm is using large pages, the following instruction needs
2288	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2289	 * so that we will probe the 4M TSB regardless of the VA.  In
2290	 * the case kpm is using small pages, we know no large kernel
2291	 * mappings are located above 0x80000000.00000000 so we skip the
2292	 * probe as an optimization.
2293	 */
2294dktsb4m_kpmcheck_small:
2295	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2296	  /* delay slot safe, below */
2297
2298	/*
2299	 * Get second TSB offset and look for 4M mapping
2300	 * using 4M virtual page as the TSB index.
2301	 *
2302	 * Here:
2303	 * %g1 = 8K TSB pointer.  Don't squash it.
2304	 * %g2 = tag access register (we still need it)
2305	 */
2306	srlx	%g2, MMU_PAGESHIFT4M, %g3
2307
2308	/*
2309	 * We patch the next set of instructions at run time;
2310	 * any changes here require sfmmu_patch_ktsb changes too.
2311	 */
2312dktsb4mbase:
2313	RUNTIME_PATCH_SETX(%g7, %g6)
2314	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2315dktsb4m:
2316	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2317	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2318
2319	/*
2320	 * At this point %g3 is our index into the TSB.
2321	 * We just masked off enough bits of the VA depending
2322	 * on our TSB size code.
2323	 */
2324	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2325	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2326	cmp	%g6, %g4			! compare tag
2327
2328dktsb4m_tsbmiss:
2329	bne,pn	%xcc, dktsb4m_kpmcheck
2330	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
2331	TT_TRACE(trace_tsbhit)
2332	/* we don't check TTE size here since we assume 4M TSB is separate */
2333	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2334	/* trapstat expects tte in %g5 */
2335	retry
2336
2337	/*
2338	 * So, we failed to find a valid TTE to match the faulting
2339	 * address in either TSB.  There are a few cases that could land
2340	 * us here:
2341	 *
2342	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2343	 *    to sfmmu_tsb_miss_tt to handle the miss.
2344	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2345	 *    4M TSB.  Let segkpm handle it.
2346	 *
2347	 * Note that we shouldn't land here in the case of a kpm VA when
2348	 * kpm_smallpages is active -- we handled that case earlier at
2349	 * dktsb4m_kpmcheck_small.
2350	 *
2351	 * At this point:
2352	 *  g1 = 8K-indexed primary TSB pointer
2353	 *  g2 = tag access register
2354	 *  g3 = 4M-indexed secondary TSB pointer
2355	 */
2356dktsb4m_kpmcheck:
2357	cmp	%g2, %g0
2358	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
2359	  nop
2360	ba,a,pt	%icc, sfmmu_tsb_miss_tt
2361	  nop
2362
2363#ifdef sun4v
2364	/*
2365	 * User instruction miss w/ single TSB.
2366	 * The first probe covers 8K, 64K, and 512K page sizes,
2367	 * because 64K and 512K mappings are replicated off 8K
2368	 * pointer.
2369	 *
2370	 * g1 = tsb8k pointer register
2371	 * g2 = tag access register
2372	 * g3 - g6 = scratch registers
2373	 * g7 = TSB tag to match
2374	 */
2375	.align	64
2376	ALTENTRY(sfmmu_uitlb_fastpath)
2377
2378	SETUP_UTSB_ATOMIC_ASI(%g4, %g5)
2379	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2380	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2381	ba,pn	%xcc, sfmmu_tsb_miss_tt
2382	  mov	%g0, %g3
2383
2384	/*
2385	 * User data miss w/ single TSB.
2386	 * The first probe covers 8K, 64K, and 512K page sizes,
2387	 * because 64K and 512K mappings are replicated off 8K
2388	 * pointer.
2389	 *
2390	 * g1 = tsb8k pointer register
2391	 * g2 = tag access register
2392	 * g3 - g6 = scratch registers
2393	 * g7 = TSB tag to match
2394	 */
2395	.align 64
2396	ALTENTRY(sfmmu_udtlb_fastpath)
2397
2398	SETUP_UTSB_ATOMIC_ASI(%g4, %g6)
2399	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2400	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2401	ba,pn	%xcc, sfmmu_tsb_miss_tt
2402	  mov	%g0, %g3
2403
2404#endif /* sun4v */
2405
2406	/*
2407	 * User instruction miss w/ multiple TSBs.
2408	 * The first probe covers 8K, 64K, and 512K page sizes,
2409	 * because 64K and 512K mappings are replicated off 8K
2410	 * pointer.  Second probe covers 4M page size only.
2411	 *
2412	 * Just like sfmmu_udtlb_slowpath, except:
2413	 *   o Uses ASI_ITLB_IN
2414	 *   o checks for execute permission
2415	 *   o No ISM prediction.
2416	 *
2417	 * g1 = tsb8k pointer register
2418	 * g2 = tag access register
2419	 * g3 - g6 = scratch registers
2420	 * g7 = TSB tag to match
2421	 */
2422	.align	64
2423	ALTENTRY(sfmmu_uitlb_slowpath)
2424
2425#ifdef sun4v
2426	SETUP_UTSB_ATOMIC_ASI(%g4, %g5)
2427	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2428
2429	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2430	/* g4 - g5 = clobbered here */
2431
2432	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2433	/* g1 = first TSB pointer, g3 = second TSB pointer */
2434	srlx	%g2, TAG_VALO_SHIFT, %g7
2435	PROBE_2ND_ITSB(%g3, %g7)
2436	/* NOT REACHED */
2437#else /* sun4v */
2438	mov	%g1, %g3	/* save tsb8k reg in %g3 */
2439	SETUP_UTSB_ATOMIC_ASI(%g4, %g5)
2440	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
2441
2442	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2443	/* g4 - g5 = clobbered here */
2444
2445	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
2446	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
2447	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
2448	/* g1 = first TSB pointer, g3 = second TSB pointer */
2449	srlx	%g2, TAG_VALO_SHIFT, %g7
2450	PROBE_2ND_ITSB(%g3, %g7, isynth)
2451	/* NOT REACHED */
2452#endif /* sun4v */
2453
2454	/*
2455	 * User data miss w/ multiple TSBs.
2456	 * The first probe covers 8K, 64K, and 512K page sizes,
2457	 * because 64K and 512K mappings are replicated off 8K
2458	 * pointer.  Second probe covers 4M page size only.
2459	 *
2460	 * We consider probing for 4M pages first if the VA falls
2461	 * in a range that's likely to be ISM.
2462	 *
2463	 * g1 = tsb8k pointer register
2464	 * g2 = tag access register
2465	 * g3 - g6 = scratch registers
2466	 * g7 = TSB tag to match
2467	 */
2468	.align 64
2469	ALTENTRY(sfmmu_udtlb_slowpath)
2470
2471	SETUP_UTSB_ATOMIC_ASI(%g4, %g6)
2472
2473	/*
2474	 * Check for ISM.  If it exists, look for 4M mappings in the second TSB
2475	 * first, then probe for other mappings in the first TSB if that fails.
2476	 */
2477	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
2478	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
2479	  mov	%g1, %g3
2480
2481udtlb_miss_probefirst:
2482	/*
2483	 * g1 = 8K TSB pointer register
2484	 * g2 = tag access register
2485	 * g3 = (potentially) second TSB entry ptr
2486	 * g6 = ism pred.
2487	 * g7 = vpg_4m
2488	 */
2489#ifdef sun4v
2490	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2491	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2492
2493	/*
2494	 * Here:
2495	 *   g1 = first TSB pointer
2496	 *   g2 = tag access reg
2497	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2498	 */
2499	brgz,pn	%g6, sfmmu_tsb_miss_tt
2500	  nop
2501#else /* sun4v */
2502	mov	%g1, %g4
2503	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
2504	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2505
2506	/*
2507	 * Here:
2508	 *   g1 = first TSB pointer
2509	 *   g2 = tag access reg
2510	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2511	 */
2512	brgz,pn	%g6, sfmmu_tsb_miss_tt
2513	  nop
2514	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
2515	/* fall through in 8K->4M probe order */
2516#endif /* sun4v */
2517
2518udtlb_miss_probesecond:
2519	/*
2520	 * Look in the second TSB for the TTE
2521	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
2522	 * g2 = tag access reg
2523	 * g3 = 8K TSB pointer register
2524	 * g6 = ism pred.
2525	 * g7 = vpg_4m
2526	 */
2527#ifdef sun4v
2528	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
2529	/* tagacc (%g2) not destroyed */
2530	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2531	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
2532#else
2533	mov	%g3, %g7
2534	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
2535	/* %g2 clobbered, %g3 =second tsbe ptr */
2536	mov	MMU_TAG_ACCESS, %g2
2537	ldxa	[%g2]ASI_DMMU, %g2
2538#endif
2539
2540	srlx	%g2, TAG_VALO_SHIFT, %g7
2541	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2542	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
2543	brgz,pn	%g6, udtlb_miss_probefirst
2544	  nop
2545
2546	/* fall through to sfmmu_tsb_miss_tt */
2547
2548	ALTENTRY(sfmmu_tsb_miss_tt)
2549	TT_TRACE(trace_tsbmiss)
2550	/*
2551	 * We get here if there is a TSB miss OR a write protect trap.
2552	 *
2553	 * g1 = First TSB entry pointer
2554	 * g2 = tag access register
2555	 * g3 = 4M TSB entry pointer; NULL if no 2nd TSB
2556	 * g4 - g7 = scratch registers
2557	 */
2558
2559	ALTENTRY(sfmmu_tsb_miss)
2560
2561	/*
2562	 * If trapstat is running, we need to shift the %tpc and %tnpc to
2563	 * point to trapstat's TSB miss return code (note that trapstat
2564	 * itself will patch the correct offset to add).
2565	 */
2566	rdpr	%tl, %g7
2567	cmp	%g7, 1
2568	ble,pt	%xcc, 0f
2569	  sethi	%hi(KERNELBASE), %g6
2570	rdpr	%tpc, %g7
2571	or	%g6, %lo(KERNELBASE), %g6
2572	cmp	%g7, %g6
2573	bgeu,pt	%xcc, 0f
2574	/* delay slot safe */
2575
2576	ALTENTRY(tsbmiss_trapstat_patch_point)
2577	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
2578	wrpr	%g7, %tpc
2579	add	%g7, 4, %g7
2580	wrpr	%g7, %tnpc
25810:
2582	CPU_TSBMISS_AREA(%g6, %g7)
2583
2584	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save first tsb pointer */
2585	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save second tsb pointer */
2586
2587	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
2588	brz,a,pn %g3, 1f			/* skip ahead if kernel */
2589	  ldn	[%g6 + TSBMISS_KHATID], %g7
2590	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
2591	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
2592
2593	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
2594
2595	cmp	%g3, INVALID_CONTEXT
2596	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
2597	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
2598
2599	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
2600	/*
2601	 * The miss wasn't in an ISM segment.
2602	 *
2603	 * %g1 %g3, %g4, %g5, %g7 all clobbered
2604	 * %g2 = tag access (vaddr + ctx)
2605	 */
2606
2607	ba,pt	%icc, 2f
2608	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
2609
26101:
2611	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
2612	/*
2613	 * 8K and 64K hash.
2614	 */
26152:
2616
2617	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2618		MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte,
2619		sfmmu_suspend_tl, tsb_512K)
2620	/* NOT REACHED */
2621
2622tsb_512K:
2623	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2624	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2625	brz,pn	%g5, 3f
2626	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2627	and	%g4, HAT_512K_FLAG, %g5
2628
2629	/*
2630	 * Note that there is a small window here where we may have
2631	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
2632	 * flag yet, so we will skip searching the 512k hash list.
2633	 * In this case we will end up in pagefault which will find
2634	 * the mapping and return.  So, in this instance we will end up
2635	 * spending a bit more time resolving this TSB miss, but it can
2636	 * only happen once per process and even then, the chances of that
2637	 * are very small, so it's not worth the extra overhead it would
2638	 * take to close this window.
2639	 */
2640	brz,pn	%g5, tsb_4M
2641	  nop
26423:
2643	/*
2644	 * 512K hash
2645	 */
2646
2647	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2648		MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte,
2649		sfmmu_suspend_tl, tsb_4M)
2650	/* NOT REACHED */
2651
2652tsb_4M:
2653	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2654	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2655	brz,pn	%g5, 4f
2656	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2657	and	%g4, HAT_4M_FLAG, %g5
2658	brz,pn	%g5, tsb_32M
2659	  nop
26604:
2661	/*
2662	 * 4M hash
2663	 */
2664
2665	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2666		MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte,
2667		sfmmu_suspend_tl, tsb_32M)
2668	/* NOT REACHED */
2669
2670tsb_32M:
2671#ifndef sun4v
2672	GET_CPU_IMPL(%g5)
2673	cmp	%g5, PANTHER_IMPL
2674	bne,pt	%xcc, tsb_pagefault
2675	  nop
2676#endif
2677
2678	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2679	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2680#ifdef sun4v
2681	brz,pn	%g5, 6f
2682#else
2683	brz,pn	%g5, tsb_pagefault
2684#endif
2685	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2686	and	%g4, HAT_32M_FLAG, %g5
2687	brz,pn	%g5, tsb_256M
2688	  nop
26895:
2690	/*
2691	 * 32M hash
2692	 */
2693
2694	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2695		MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte,
2696		sfmmu_suspend_tl, tsb_256M)
2697	/* NOT REACHED */
2698
2699tsb_256M:
2700	lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2701	and	%g4, HAT_256M_FLAG, %g5
2702	brz,pn	%g5, tsb_pagefault
2703	  nop
27046:
2705	/*
2706	 * 256M hash
2707	 */
2708
2709	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2710	    MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte,
2711	    sfmmu_suspend_tl, tsb_pagefault)
2712	/* NOT REACHED */
2713
2714tsb_checktte:
2715	/*
2716	 * g3 = tte
2717	 * g4 = tte pa
2718	 * g5 = tte va
2719	 * g6 = tsbmiss area
2720	 */
2721	brgez,pn %g3, tsb_pagefault	/* if tte invalid branch */
2722	  nop
2723
2724tsb_validtte:
2725	/*
2726	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
2727	 */
2728	rdpr	%tt, %g7
2729	cmp	%g7, FAST_PROT_TT
2730	bne,pt	%icc, 4f
2731	  nop
2732
2733	TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod,
2734	    tsb_protfault)
2735
2736	rdpr	%tt, %g5
2737	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
2738	ba,pt	%xcc, tsb_update_tl1
2739	  nop
2740
27414:
2742	/*
2743	 * If ITLB miss check exec bit.
2744	 * If not set treat as invalid TTE.
2745	 */
2746	cmp     %g7, T_INSTR_MMU_MISS
2747	be,pn	%icc, 5f
2748	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
2749	cmp     %g7, FAST_IMMU_MISS_TT
2750	bne,pt %icc, 3f
2751	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
27525:
2753	bz,pn %icc, tsb_protfault
2754	  nop
2755
27563:
2757	/*
2758	 * Set reference bit if not already set
2759	 */
2760	TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref)
2761
2762	/*
2763	 * Now, load into TSB/TLB.  At this point:
2764	 * g3 = tte
2765	 * g4 = patte
2766	 * g6 = tsbmiss area
2767	 */
2768	rdpr	%tt, %g5
2769#ifdef sun4v
2770	MMU_FAULT_STATUS_AREA(%g2)
2771	cmp	%g5, T_INSTR_MMU_MISS
2772	be,a,pt	%icc, 9f
2773	  nop
2774	cmp	%g5, FAST_IMMU_MISS_TT
2775	be,a,pt	%icc, 9f
2776	  nop
2777	add	%g2, MMFSA_D_, %g2
27789:
2779	ldx	[%g2 + MMFSA_CTX_], %g7
2780	sllx	%g7, TTARGET_CTX_SHIFT, %g7
2781	ldx	[%g2 + MMFSA_ADDR_], %g2
2782	srlx	%g2, TTARGET_VA_SHIFT, %g2
2783	or	%g2, %g7, %g2
2784#else
2785	cmp	%g5, FAST_IMMU_MISS_TT
2786	be,a,pt	%icc, tsb_update_tl1
2787	  ldxa	[%g0]ASI_IMMU, %g2
2788	ldxa	[%g0]ASI_DMMU, %g2
2789#endif
2790tsb_update_tl1:
2791	srlx	%g2, TTARGET_CTX_SHIFT, %g7
2792	brz,pn	%g7, tsb_kernel
2793#ifdef sun4v
2794	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
2795#else
2796	  srlx	%g3, TTE_SZ_SHFT, %g7
2797#endif
2798
2799tsb_user:
2800#ifdef sun4v
2801	cmp	%g7, TTE4M
2802	bge,pn	%icc, tsb_user4m
2803	  nop
2804#else
2805	cmp	%g7, TTESZ_VALID | TTE4M
2806	be,pn	%icc, tsb_user4m
2807	  srlx	%g3, TTE_SZ2_SHFT, %g7
2808	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
2809	bnz,a,pn %icc, tsb_user_pn_synth
2810	  cmp	%g5, FAST_IMMU_MISS_TT
2811#endif
2812
2813tsb_user8k:
2814	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = first TSB ptr
2815
2816#ifndef sun4v
2817	mov	ASI_N, %g7	! user TSBs always accessed by VA
2818	mov	%g7, %asi
2819#endif /* sun4v */
2820
2821	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5)
2822
2823#ifdef sun4v
2824	cmp	%g5, T_INSTR_MMU_MISS
2825	be,a,pn	%xcc, 9f
2826	  mov	%g3, %g5
2827#endif /* sun4v */
2828	cmp	%g5, FAST_IMMU_MISS_TT
2829	be,pn	%xcc, 9f
2830	  mov	%g3, %g5
2831
2832	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2833	! trapstat wants TTE in %g5
2834	retry
28359:
2836	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2837	! trapstat wants TTE in %g5
2838	retry
2839
2840tsb_user4m:
2841	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */
28424:
2843	brz,pn	%g1, 5f	/* Check to see if we have 2nd TSB programmed */
2844	  nop
2845
2846#ifndef sun4v
2847        mov     ASI_N, %g7      ! user TSBs always accessed by VA
2848        mov     %g7, %asi
2849#endif
2850
2851        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6)
2852
28535:
2854#ifdef sun4v
2855        cmp     %g5, T_INSTR_MMU_MISS
2856        be,a,pn %xcc, 9f
2857          mov   %g3, %g5
2858#endif /* sun4v */
2859        cmp     %g5, FAST_IMMU_MISS_TT
2860        be,pn   %xcc, 9f
2861        mov     %g3, %g5
2862
2863        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2864        ! trapstat wants TTE in %g5
2865        retry
28669:
2867        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2868        ! trapstat wants TTE in %g5
2869        retry
2870
2871#ifndef sun4v
2872	/*
2873	 * Panther ITLB synthesis.
2874	 * The Panther 32M and 256M ITLB code simulates these two large page
2875	 * sizes with 4M pages, to provide support for programs, for example
2876	 * Java, that may copy instructions into a 32M or 256M data page and
2877	 * then execute them. The code below generates the 4M pfn bits and
2878	 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
2879	 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
2880	 * are ignored by the hardware.
2881	 *
2882	 * Now, load into TSB/TLB.  At this point:
2883	 * g2 = tagtarget
2884	 * g3 = tte
2885	 * g4 = patte
2886	 * g5 = tt
2887	 * g6 = tsbmiss area
2888	 */
2889tsb_user_pn_synth:
2890	be,pt	%xcc, tsb_user_itlb_synth	/* ITLB miss */
2891	  andcc %g3, TTE_EXECPRM_INT, %g0	/* is execprm bit set */
2892	bz,pn %icc, 4b				/* if not, been here before */
2893	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	/* g1 = tsbp */
2894	brz,a,pn %g1, 5f			/* no 2nd tsb */
2895	  mov	%g3, %g5
2896
2897	mov	MMU_TAG_ACCESS, %g7
2898	ldxa	[%g7]ASI_DMMU, %g6		/* get tag access va */
2899	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)	/* make 4M pfn offset */
2900
2901	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
2902	mov	%g7, %asi
2903	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */
29045:
2905        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2906        retry
2907
2908tsb_user_itlb_synth:
2909	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */
2910
2911	mov	MMU_TAG_ACCESS, %g7
2912	ldxa	[%g7]ASI_IMMU, %g6		/* get tag access va */
2913	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)	/* make 4M pfn offset */
2914	brz,a,pn %g1, 7f	/* Check to see if we have 2nd TSB programmed */
2915	  or	%g5, %g3, %g5			/* add 4M bits to TTE */
2916
2917	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
2918	mov	%g7, %asi
2919	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */
29207:
2921	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
2922        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2923        retry
2924#endif
2925
2926tsb_kernel:
2927#ifdef sun4v
2928	cmp	%g7, TTE4M
2929	bge,pn	%icc, 5f
2930#else
2931	cmp	%g7, TTESZ_VALID | TTE4M	! no 32M or 256M support
2932	be,pn	%icc, 5f
2933#endif
2934	  nop
2935	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8k tsbptr
2936	ba,pt	%xcc, 6f
2937	  nop
29385:
2939	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4m tsbptr
2940	brz,pn	%g1, 3f		/* skip programming if 4m TSB ptr is NULL */
2941	  nop
29426:
2943#ifndef sun4v
2944tsb_kernel_patch_asi:
2945	or	%g0, RUNTIME_PATCH, %g6
2946	mov	%g6, %asi	! XXX avoid writing to %asi !!
2947#endif
2948	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7)
29493:
2950#ifdef sun4v
2951	cmp	%g5, T_INSTR_MMU_MISS
2952	be,a,pn	%icc, 1f
2953	  mov	%g3, %g5			! trapstat wants TTE in %g5
2954#endif /* sun4v */
2955	cmp	%g5, FAST_IMMU_MISS_TT
2956	be,pn	%icc, 1f
2957	  mov	%g3, %g5			! trapstat wants TTE in %g5
2958	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2959	! trapstat wants TTE in %g5
2960	retry
29611:
2962	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2963	! trapstat wants TTE in %g5
2964	retry
2965
2966tsb_ism:
2967	/*
2968	 * This is an ISM [i|d]tlb miss.  We optimize for largest
2969	 * page size down to smallest.
2970	 *
2971	 * g2 = vaddr + ctx	aka tag access register
2972	 * g3 = ismmap->ism_seg
2973	 * g4 = physical address of ismmap->ism_sfmmu
2974	 * g6 = tsbmiss area
2975	 */
2976	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
2977	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
2978	  mov	PTL1_BAD_ISM, %g1
2979						/* g5 = pa of imap_vb_shift */
2980	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
2981	lduha	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
2982	srlx	%g3, %g4, %g3			/* clr size field */
2983	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
2984	sllx	%g3, %g4, %g3			/* g3 = ism vbase */
2985	and	%g2, %g1, %g4			/* g4 = ctx number */
2986	andn	%g2, %g1, %g1			/* g1 = tlb miss vaddr */
2987	sub	%g1, %g3, %g2			/* g2 = offset in ISM seg */
2988	or	%g2, %g4, %g2			/* g2 = tagacc (vaddr + ctx) */
2989
2990	/*
2991	 * ISM pages are always locked down.
2992	 * If we can't find the tte then pagefault
2993	 * and let the spt segment driver resovle it.
2994	 *
2995	 * g2 = ISM vaddr (offset in ISM seg)
2996	 * g6 = tsb miss area
2997	 * g7 = ISM hatid
2998	 */
2999	sub	%g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
3000	lduha	[%g5]ASI_MEM, %g4		/* g5 = pa of imap_hatflags */
3001	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
3002	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
3003	  nop
3004
3005tsb_ism_32M:
3006	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
3007	brz,pn	%g5, tsb_ism_256M
3008	  nop
3009
3010	/*
3011	 * 32M hash.
3012	 */
3013
3014	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M,
3015	    TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3016	    tsb_ism_4M)
3017	/* NOT REACHED */
3018
3019tsb_ism_32M_found:
3020	brlz,pt %g3, tsb_validtte
3021	  nop
3022	ba,pt	%xcc, tsb_ism_4M
3023	  nop
3024
3025tsb_ism_256M:
3026	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
3027	brz,a,pn %g5, ptl1_panic
3028	  mov	PTL1_BAD_ISM, %g1
3029
3030	/*
3031	 * 256M hash.
3032	 */
3033	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M,
3034	    TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3035	    tsb_ism_4M)
3036
3037tsb_ism_256M_found:
3038	brlz,pt %g3, tsb_validtte
3039	  nop
3040
3041tsb_ism_4M:
3042	/*
3043	 * 4M hash.
3044	 */
3045	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M,
3046	    TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3047	    tsb_ism_8K)
3048	/* NOT REACHED */
3049
3050tsb_ism_4M_found:
3051	brlz,pt %g3, tsb_validtte
3052	  nop
3053
3054tsb_ism_8K:
3055	/*
3056	 * 8K and 64K hash.
3057	 */
3058
3059	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K,
3060	    TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3061	    tsb_pagefault)
3062	/* NOT REACHED */
3063
3064tsb_ism_8K_found:
3065	brlz,pt	%g3, tsb_validtte
3066	  nop
3067
3068tsb_pagefault:
3069	rdpr	%tt, %g7
3070	cmp	%g7, FAST_PROT_TT
3071	be,a,pn	%icc, tsb_protfault
3072	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt
3073
3074tsb_protfault:
3075	/*
3076	 * we get here if we couldn't find a valid tte in the hash.
3077	 *
3078	 * If user and we are at tl>1 we go to window handling code.
3079	 *
3080	 * If kernel and the fault is on the same page as our stack
3081	 * pointer, then we know the stack is bad and the trap handler
3082	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3083	 *
3084	 * If this is a kernel trap and tl>1, panic.
3085	 *
3086	 * Otherwise we call pagefault.
3087	 */
3088	cmp	%g7, FAST_IMMU_MISS_TT
3089#ifdef sun4v
3090	MMU_FAULT_STATUS_AREA(%g4)
3091	ldx	[%g4 + MMFSA_I_CTX], %g5
3092	ldx	[%g4 + MMFSA_D_CTX], %g4
3093	move	%icc, %g5, %g4
3094	cmp	%g7, T_INSTR_MMU_MISS
3095	move	%icc, %g5, %g4
3096#else
3097	mov	MMU_TAG_ACCESS, %g4
3098	ldxa	[%g4]ASI_DMMU, %g2
3099	ldxa	[%g4]ASI_IMMU, %g5
3100	move	%icc, %g5, %g2
3101	cmp	%g7, T_INSTR_MMU_MISS
3102	move	%icc, %g5, %g2
3103	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
3104#endif
3105	brnz,pn	%g4, 3f				/* skip if not kernel */
3106	  rdpr	%tl, %g5
3107
3108	add	%sp, STACK_BIAS, %g3
3109	srlx	%g3, MMU_PAGESHIFT, %g3
3110	srlx	%g2, MMU_PAGESHIFT, %g4
3111	cmp	%g3, %g4
3112	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
3113	  mov	PTL1_BAD_STACK, %g1
3114
3115	cmp	%g5, 1
3116	ble,pt	%icc, 2f
3117	  nop
3118	TSTAT_CHECK_TL1(2f, %g1, %g2)
3119	rdpr	%tt, %g2
3120	cmp	%g2, FAST_PROT_TT
3121	mov	PTL1_BAD_KPROT_FAULT, %g1
3122	movne	%icc, PTL1_BAD_KMISS, %g1
3123	ba,pt	%icc, ptl1_panic
3124	  nop
3125
31262:
3127	/*
3128	 * We are taking a pagefault in the kernel on a kernel address.  If
3129	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
3130	 * want to call sfmmu_pagefault -- we will instead note that a fault
3131	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
3132	 * (instead of a "retry").  This will step over the faulting
3133	 * instruction.
3134	 */
3135	CPU_INDEX(%g1, %g2)
3136	set	cpu_core, %g2
3137	sllx	%g1, CPU_CORE_SHIFT, %g1
3138	add	%g1, %g2, %g1
3139	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3140	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3141	bz	sfmmu_pagefault
3142	or	%g2, CPU_DTRACE_BADADDR, %g2
3143	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3144	GET_MMU_D_ADDR(%g3, %g4)
3145	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3146	done
3147
31483:
3149	cmp	%g5, 1
3150	ble,pt	%icc, 4f
3151	  nop
3152	TSTAT_CHECK_TL1(4f, %g1, %g2)
3153	ba,pt	%icc, sfmmu_window_trap
3154	  nop
3155
31564:
3157	/*
3158	 * We are taking a pagefault on a non-kernel address.  If we are in
3159	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
3160	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
3161	 */
3162	CPU_INDEX(%g1, %g2)
3163	set	cpu_core, %g2
3164	sllx	%g1, CPU_CORE_SHIFT, %g1
3165	add	%g1, %g2, %g1
3166	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3167	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3168	bz	sfmmu_pagefault
3169	or	%g2, CPU_DTRACE_BADADDR, %g2
3170	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3171	GET_MMU_D_ADDR(%g3, %g4)
3172	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3173
3174	/*
3175	 * Be sure that we're actually taking this miss from the kernel --
3176	 * otherwise we have managed to return to user-level with
3177	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3178	 */
3179	rdpr	%tstate, %g2
3180	btst	TSTATE_PRIV, %g2
3181	bz,a	ptl1_panic
3182	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3183	done
3184
3185	ALTENTRY(tsb_tl0_noctxt)
3186	/*
3187	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
3188	 * if it is, indicated that we have faulted and issue a done.
3189	 */
3190	CPU_INDEX(%g5, %g6)
3191	set	cpu_core, %g6
3192	sllx	%g5, CPU_CORE_SHIFT, %g5
3193	add	%g5, %g6, %g5
3194	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
3195	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
3196	bz	1f
3197	or	%g6, CPU_DTRACE_BADADDR, %g6
3198	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
3199	GET_MMU_D_ADDR(%g3, %g4)
3200	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]
3201
3202	/*
3203	 * Be sure that we're actually taking this miss from the kernel --
3204	 * otherwise we have managed to return to user-level with
3205	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3206	 */
3207	rdpr	%tstate, %g5
3208	btst	TSTATE_PRIV, %g5
3209	bz,a	ptl1_panic
3210	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3211	done
3212
32131:
3214	rdpr	%tt, %g5
3215	cmp	%g5, FAST_IMMU_MISS_TT
3216#ifdef sun4v
3217	MMU_FAULT_STATUS_AREA(%g2)
3218	be,a,pt	%icc, 2f
3219	  ldx	[%g2 + MMFSA_I_CTX], %g3
3220	cmp	%g5, T_INSTR_MMU_MISS
3221	be,a,pt	%icc, 2f
3222	  ldx	[%g2 + MMFSA_I_CTX], %g3
3223	ldx	[%g2 + MMFSA_D_CTX], %g3
32242:
3225#else
3226	mov	MMU_TAG_ACCESS, %g2
3227	be,a,pt	%icc, 2f
3228	  ldxa	[%g2]ASI_IMMU, %g3
3229	ldxa	[%g2]ASI_DMMU, %g3
32302:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
3231#endif
3232	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
3233	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
3234	rdpr	%tl, %g5
3235	cmp	%g5, 1
3236	ble,pt	%icc, sfmmu_mmu_trap
3237	  nop
3238	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
3239	ba,pt	%icc, sfmmu_window_trap
3240	  nop
3241	SET_SIZE(sfmmu_tsb_miss)
3242
3243#if (1<< TSBMISS_SHIFT) != TSBMISS_SIZE
3244#error - TSBMISS_SHIFT does not correspond to size of tsbmiss struct
3245#endif
3246
3247#endif /* lint */
3248
3249#if defined (lint)
3250/*
3251 * This routine will look for a user or kernel vaddr in the hash
3252 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't
3253 * grab any locks.  It should only be used by other sfmmu routines.
3254 */
3255/* ARGSUSED */
3256pfn_t
3257sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
3258{
3259	return(0);
3260}
3261
3262#else /* lint */
3263
3264	ENTRY_NP(sfmmu_vatopfn)
3265 	/*
3266 	 * disable interrupts
3267 	 */
3268 	rdpr	%pstate, %o3
3269#ifdef DEBUG
3270	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
3271	bnz,pt	%icc, 1f			/* disabled, panic	 */
3272	  nop
3273
3274	sethi	%hi(panicstr), %g1
3275	ldx	[%g1 + %lo(panicstr)], %g1
3276	tst	%g1
3277	bnz,pt	%icc, 1f
3278	  nop
3279
3280	save	%sp, -SA(MINFRAME), %sp
3281	sethi	%hi(sfmmu_panic1), %o0
3282	call	panic
3283	 or	%o0, %lo(sfmmu_panic1), %o0
32841:
3285#endif
3286	/*
3287	 * disable interrupts to protect the TSBMISS area
3288	 */
3289	andn    %o3, PSTATE_IE, %o5
3290	wrpr    %o5, 0, %pstate
3291
3292	/*
3293	 * o0 = vaddr
3294	 * o1 = sfmmup
3295	 * o2 = ttep
3296	 */
3297	CPU_TSBMISS_AREA(%g1, %o5)
3298	ldn	[%g1 + TSBMISS_KHATID], %o4
3299	cmp	%o4, %o1
3300	bne,pn	%ncc, vatopfn_nokernel
3301	  mov	TTE64K, %g5			/* g5 = rehash # */
3302	mov %g1,%o5				/* o5 = tsbmiss_area */
3303	/*
3304	 * o0 = vaddr
3305	 * o1 & o4 = hatid
3306	 * o2 = ttep
3307	 * o5 = tsbmiss area
3308	 */
3309	mov	HBLK_RANGE_SHIFT, %g6
33101:
3311
3312	/*
3313	 * o0 = vaddr
3314	 * o1 = sfmmup
3315	 * o2 = ttep
3316	 * o3 = old %pstate
3317	 * o4 = hatid
3318	 * o5 = tsbmiss
3319	 * g5 = rehash #
3320	 * g6 = hmeshift
3321	 *
3322	 * The first arg to GET_TTE is actually tagaccess register
3323	 * not just vaddr. Since this call is for kernel we need to clear
3324	 * any lower vaddr bits that would be interpreted as ctx bits.
3325	 */
3326	set     TAGACC_CTX_MASK, %g1
3327	andn    %o0, %g1, %o0
3328	GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5,
3329		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
3330
3331kvtop_hblk_found:
3332	/*
3333	 * o0 = vaddr
3334	 * o1 = sfmmup
3335	 * o2 = ttep
3336	 * g1 = tte
3337	 * g2 = tte pa
3338	 * g3 = tte va
3339	 * o2 = tsbmiss area
3340	 * o1 = hat id
3341	 */
3342	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
3343	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3344	stx %g1,[%o2]				/* put tte into *ttep */
3345	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
3346	/*
3347	 * o0 = vaddr
3348	 * o1 = sfmmup
3349	 * o2 = ttep
3350	 * g1 = pfn
3351	 */
3352	ba,pt	%xcc, 6f
3353	  mov	%g1, %o0
3354
3355kvtop_nohblk:
3356	/*
3357	 * we get here if we couldn't find valid hblk in hash.  We rehash
3358	 * if neccesary.
3359	 */
3360	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
3361#ifdef sun4v
3362	cmp	%g5, MAX_HASHCNT
3363#else
3364	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
3365#endif
3366	be,a,pn	%icc, 6f
3367	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3368	mov	%o1, %o4			/* restore hatid */
3369#ifdef sun4v
3370        add	%g5, 2, %g5
3371	cmp	%g5, 3
3372	move	%icc, MMU_PAGESHIFT4M, %g6
3373	ba,pt	%icc, 1b
3374	movne	%icc, MMU_PAGESHIFT256M, %g6
3375#else
3376        inc	%g5
3377	cmp	%g5, 2
3378	move	%icc, MMU_PAGESHIFT512K, %g6
3379	ba,pt	%icc, 1b
3380	movne	%icc, MMU_PAGESHIFT4M, %g6
3381#endif
33826:
3383	retl
3384 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3385
3386tsb_suspend:
3387	/*
3388	 * o0 = vaddr
3389	 * o1 = sfmmup
3390	 * o2 = ttep
3391	 * g1 = tte
3392	 * g2 = tte pa
3393	 * g3 = tte va
3394	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
3395	 */
3396	stx %g1,[%o2]				/* put tte into *ttep */
3397	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
3398	  sub	%g0, 1, %o0			/* output = -1 (PFN_INVALID) */
3399	TTETOPFN(%g1, %o0, vatopfn_l3, %g2, %g3, %g4)
3400	/*
3401	 * o0 = PFN return value PFN_INVALID, PFN_SUSPENDED, or pfn#
3402	 * o1 = sfmmup
3403	 * o2 = ttep
3404	 * g1 = pfn
3405	 */
3406	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
34078:
3408	retl
3409	 wrpr	%g0, %o3, %pstate		/* enable interrupts */
3410
3411vatopfn_nokernel:
3412	/*
3413	 * This routine does NOT support user addresses
3414	 * There is a routine in C that supports this.
3415	 * The only reason why we don't have the C routine
3416	 * support kernel addresses as well is because
3417	 * we do va_to_pa while holding the hashlock.
3418	 */
3419 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3420	save	%sp, -SA(MINFRAME), %sp
3421	sethi	%hi(sfmmu_panic3), %o0
3422	call	panic
3423	 or	%o0, %lo(sfmmu_panic3), %o0
3424
3425	SET_SIZE(sfmmu_vatopfn)
3426#endif /* lint */
3427
3428
3429
3430#if !defined(lint)
3431
3432/*
3433 * kpm lock used between trap level tsbmiss handler and kpm C level.
3434 */
3435#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
3436	mov     0xff, tmp1						;\
3437label1:									;\
3438	casa    [kpmlckp]asi, %g0, tmp1					;\
3439	brnz,pn tmp1, label1						;\
3440	mov     0xff, tmp1						;\
3441	membar  #LoadLoad
3442
3443#define KPMLOCK_EXIT(kpmlckp, asi)					\
3444	membar  #LoadStore|#StoreStore					;\
3445	sta     %g0, [kpmlckp]asi
3446
3447/*
3448 * Lookup a memseg for a given pfn and if found, return the physical
3449 * address of the corresponding struct memseg in mseg, otherwise
3450 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
3451 * tsbmp, %asi is assumed to be ASI_MEM.
3452 * This lookup is done by strictly traversing only the physical memseg
3453 * linkage. The more generic approach, to check the virtual linkage
3454 * before using the physical (used e.g. with hmehash buckets), cannot
3455 * be used here. Memory DR operations can run in parallel to this
3456 * lookup w/o any locks and updates of the physical and virtual linkage
3457 * cannot be done atomically wrt. to each other. Because physical
3458 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
3459 * as "physical NULL" pointer.
3460 */
3461#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
3462	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
3463	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
3464	udivx	pfn, mseg, mseg						;\
3465	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
3466	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
3467	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
3468	add	tmp1, mseg, tmp1					;\
3469	ldxa	[tmp1]%asi, mseg					;\
3470	cmp	mseg, MSEG_NULLPTR_PA					;\
3471	be,pn	%xcc, label/**/1		/* if not found */	;\
3472	  nop								;\
3473	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3474	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3475	blu,pn	%xcc, label/**/1					;\
3476	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3477	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3478	bgeu,pn	%xcc, label/**/1					;\
3479	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
3480	mulx	tmp1, PAGE_SIZE, tmp1					;\
3481	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
3482	add	tmp2, tmp1, tmp1			/* pp */	;\
3483	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
3484	cmp	tmp2, pfn						;\
3485	be,pt	%xcc, label/**/_ok			/* found */	;\
3486label/**/1:								;\
3487	/* brute force lookup */					;\
3488	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
3489	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
3490label/**/2:								;\
3491	cmp	mseg, MSEG_NULLPTR_PA					;\
3492	be,pn	%xcc, label/**/_ok		/* if not found */	;\
3493	  nop								;\
3494	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3495	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3496	blu,a,pt %xcc, label/**/2					;\
3497	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3498	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3499	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3500	bgeu,a,pt %xcc, label/**/2					;\
3501	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3502label/**/_ok:
3503
3504	/*
3505	 * kpm tsb miss handler large pages
3506	 * g1 = 8K kpm TSB entry pointer
3507	 * g2 = tag access register
3508	 * g3 = 4M kpm TSB entry pointer
3509	 */
3510	ALTENTRY(sfmmu_kpm_dtsb_miss)
3511	TT_TRACE(trace_tsbmiss)
3512
3513	CPU_INDEX(%g7, %g6)
3514	sethi	%hi(kpmtsbm_area), %g6
3515	sllx	%g7, KPMTSBM_SHIFT, %g7
3516	or	%g6, %lo(kpmtsbm_area), %g6
3517	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
3518
3519	/* check enable flag */
3520	ldub	[%g6 + KPMTSBM_FLAGS], %g4
3521	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
3522	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
3523	  nop
3524
3525	/* VA range check */
3526	ldx	[%g6 + KPMTSBM_VBASE], %g7
3527	cmp	%g2, %g7
3528	blu,pn	%xcc, sfmmu_tsb_miss
3529	  ldx	[%g6 + KPMTSBM_VEND], %g5
3530	cmp	%g2, %g5
3531	bgeu,pn	%xcc, sfmmu_tsb_miss
3532	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]
3533
3534	/*
3535	 * check TL tsbmiss handling flag
3536	 * bump tsbmiss counter
3537	 */
3538	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
3539#ifdef	DEBUG
3540	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
3541	inc	%g5
3542	brz,pn	%g3, sfmmu_kpm_exception
3543	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
3544#else
3545	inc	%g5
3546	st	%g5, [%g6 + KPMTSBM_TSBMISS]
3547#endif
3548	/*
3549	 * At this point:
3550	 *  g1 = 8K kpm TSB pointer (not used)
3551	 *  g2 = tag access register
3552	 *  g3 = clobbered
3553	 *  g6 = per-CPU kpm tsbmiss area
3554	 *  g7 = kpm_vbase
3555	 */
3556
3557	/* vaddr2pfn */
3558	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
3559	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
3560	srax    %g4, %g3, %g2			/* which alias range (r) */
3561	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
3562	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
3563
3564	/*
3565	 * Setup %asi
3566	 * mseg_pa = page_numtomemseg_nolock(pfn)
3567	 * if (mseg_pa == NULL) sfmmu_kpm_exception
3568	 * g2=pfn
3569	 */
3570	mov	ASI_MEM, %asi
3571	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
3572	cmp	%g3, MSEG_NULLPTR_PA
3573	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
3574	  nop
3575
3576	/*
3577	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
3578	 * g2=pfn g3=mseg_pa
3579	 */
3580	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
3581	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
3582	srlx	%g2, %g5, %g4
3583	sllx	%g4, %g5, %g4
3584	sub	%g4, %g7, %g4
3585	srlx	%g4, %g5, %g4
3586
3587	/*
3588	 * Validate inx value
3589	 * g2=pfn g3=mseg_pa g4=inx
3590	 */
3591#ifdef	DEBUG
3592	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
3593	cmp	%g4, %g5			/* inx - nkpmpgs */
3594	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
3595	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3596#else
3597	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3598#endif
3599	/*
3600	 * kp = &mseg_pa->kpm_pages[inx]
3601	 */
3602	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
3603	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
3604	add	%g5, %g4, %g5			/* kp */
3605
3606	/*
3607	 * KPMP_HASH(kp)
3608	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
3609	 */
3610	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
3611	sub	%g7, 1, %g7			/* mask */
3612	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
3613	add	%g5, %g1, %g5			/* y = ksp + x */
3614	and 	%g5, %g7, %g5			/* hashinx = y & mask */
3615
3616	/*
3617	 * Calculate physical kpm_page pointer
3618	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
3619	 */
3620	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
3621	add	%g1, %g4, %g1			/* kp_pa */
3622
3623	/*
3624	 * Calculate physical hash lock address
3625	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
3626	 */
3627	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
3628	sllx	%g5, KPMHLK_SHIFT, %g5
3629	add	%g4, %g5, %g3
3630	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */
3631
3632	/*
3633	 * Assemble tte
3634	 * g1=kp_pa g2=pfn g3=hlck_pa
3635	 */
3636#ifdef sun4v
3637	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
3638	sllx	%g5, 32, %g5
3639	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3640	or	%g4, TTE4M, %g4
3641	or	%g5, %g4, %g5
3642#else
3643	sethi	%hi(TTE_VALID_INT), %g4
3644	mov	TTE4M, %g5
3645	sllx	%g5, TTE_SZ_SHFT_INT, %g5
3646	or	%g5, %g4, %g5			/* upper part */
3647	sllx	%g5, 32, %g5
3648	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3649	or	%g5, %g4, %g5
3650#endif
3651	sllx	%g2, MMU_PAGESHIFT, %g4
3652	or	%g5, %g4, %g5			/* tte */
3653	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
3654	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3655
3656	/*
3657	 * tsb dropin
3658	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
3659	 */
3660
3661	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
3662	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
3663
3664	/* use C-handler if there's no go for dropin */
3665	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
3666	cmp	%g7, -1
3667	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
3668	  nop
3669
3670#ifdef	DEBUG
3671	/* double check refcnt */
3672	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
3673	brz,pn	%g7, 5f			/* let C-handler deal with this */
3674	  nop
3675#endif
3676
3677#ifndef sun4v
3678	ldub	[%g6 + KPMTSBM_FLAGS], %g7
3679	mov	ASI_N, %g1
3680	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
3681	movnz	%icc, ASI_MEM, %g1
3682	mov	%g1, %asi
3683#endif
3684
3685	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
3686	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
3687
3688	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
3689	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
3690
3691	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
3692
3693	/* KPMLOCK_EXIT(kpmlckp, asi) */
3694	KPMLOCK_EXIT(%g3, ASI_MEM)
3695
3696	/*
3697	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3698	 * point to trapstat's TSB miss return code (note that trapstat
3699	 * itself will patch the correct offset to add).
3700	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
3701	 */
3702	rdpr	%tl, %g7
3703	cmp	%g7, 1
3704	ble	%icc, 0f
3705	sethi	%hi(KERNELBASE), %g6
3706	rdpr	%tpc, %g7
3707	or	%g6, %lo(KERNELBASE), %g6
3708	cmp	%g7, %g6
3709	bgeu	%xcc, 0f
3710	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
3711	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3712	wrpr	%g7, %tpc
3713	add	%g7, 4, %g7
3714	wrpr	%g7, %tnpc
37150:
3716	retry
37175:
3718	/* g3=hlck_pa */
3719	KPMLOCK_EXIT(%g3, ASI_MEM)
3720	ba,pt	%icc, sfmmu_kpm_exception
3721	  nop
3722	SET_SIZE(sfmmu_kpm_dtsb_miss)
3723
3724	/*
3725	 * kpm tsbmiss handler for smallpages
3726	 * g1 = 8K kpm TSB pointer
3727	 * g2 = tag access register
3728	 * g3 = 4M kpm TSB pointer
3729	 */
3730	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
3731	TT_TRACE(trace_tsbmiss)
3732	CPU_INDEX(%g7, %g6)
3733	sethi	%hi(kpmtsbm_area), %g6
3734	sllx	%g7, KPMTSBM_SHIFT, %g7
3735	or	%g6, %lo(kpmtsbm_area), %g6
3736	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
3737
3738	/* check enable flag */
3739	ldub	[%g6 + KPMTSBM_FLAGS], %g4
3740	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
3741	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
3742	  nop
3743
3744	/*
3745	 * VA range check
3746	 * On fail: goto sfmmu_tsb_miss
3747	 */
3748	ldx	[%g6 + KPMTSBM_VBASE], %g7
3749	cmp	%g2, %g7
3750	blu,pn	%xcc, sfmmu_tsb_miss
3751	  ldx	[%g6 + KPMTSBM_VEND], %g5
3752	cmp	%g2, %g5
3753	bgeu,pn	%xcc, sfmmu_tsb_miss
3754	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */
3755
3756	/*
3757	 * check TL tsbmiss handling flag
3758	 * bump tsbmiss counter
3759	 */
3760	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
3761#ifdef	DEBUG
3762	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
3763	inc	%g5
3764	brz,pn	%g1, sfmmu_kpm_exception
3765	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
3766#else
3767	inc	%g5
3768	st	%g5, [%g6 + KPMTSBM_TSBMISS]
3769#endif
3770	/*
3771	 * At this point:
3772	 *  g1 = clobbered
3773	 *  g2 = tag access register
3774	 *  g3 = 4M kpm TSB pointer (not used)
3775	 *  g6 = per-CPU kpm tsbmiss area
3776	 *  g7 = kpm_vbase
3777	 */
3778
3779	/* vaddr2pfn */
3780	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
3781	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
3782	srax    %g4, %g3, %g2			/* which alias range (r) */
3783	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
3784	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
3785
3786	/*
3787	 * Setup %asi
3788	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
3789	 * if (mseg not found) sfmmu_kpm_exception
3790	 * g2=pfn
3791	 */
3792	mov	ASI_MEM, %asi
3793	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
3794	cmp	%g3, MSEG_NULLPTR_PA
3795	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
3796	  nop
3797
3798	/*
3799	 * inx = pfn - mseg_pa->kpm_pbase
3800	 * g2=pfn g3=mseg_pa
3801	 */
3802	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
3803	sub	%g2, %g7, %g4
3804
3805#ifdef	DEBUG
3806	/*
3807	 * Validate inx value
3808	 * g2=pfn g3=mseg_pa g4=inx
3809	 */
3810	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
3811	cmp	%g4, %g5			/* inx - nkpmpgs */
3812	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
3813	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3814#else
3815	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3816#endif
3817	/* ksp = &mseg_pa->kpm_spages[inx] */
3818	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
3819	add	%g5, %g4, %g5			/* ksp */
3820
3821	/*
3822	 * KPMP_SHASH(kp)
3823	 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz
3824	 */
3825	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
3826	sub	%g7, 1, %g7			/* mask */
3827	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
3828	add	%g5, %g1, %g5			/* y = ksp + x */
3829	and 	%g5, %g7, %g5			/* hashinx = y & mask */
3830
3831	/*
3832	 * Calculate physical kpm_spage pointer
3833	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
3834	 */
3835	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
3836	add	%g1, %g4, %g1			/* ksp_pa */
3837
3838	/*
3839	 * Calculate physical hash lock address.
3840	 * Note: Changes in kpm_shlk_t must be reflected here.
3841	 * g1=ksp_pa g2=pfn g5=hashinx
3842	 */
3843	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
3844	sllx	%g5, KPMSHLK_SHIFT, %g5
3845	add	%g4, %g5, %g3			/* hlck_pa */
3846
3847	/*
3848	 * Assemble tte
3849	 * g1=ksp_pa g2=pfn g3=hlck_pa
3850	 */
3851	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
3852	sllx	%g5, 32, %g5
3853	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3854	or	%g5, %g4, %g5
3855	sllx	%g2, MMU_PAGESHIFT, %g4
3856	or	%g5, %g4, %g5			/* tte */
3857	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
3858	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3859
3860	/*
3861	 * tsb dropin
3862	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte
3863	 */
3864
3865	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
3866	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
3867
3868	/* use C-handler if there's no go for dropin */
3869	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */
3870	cmp	%g7, -1
3871	bne,pn	%xcc, 5f
3872	  nop
3873
3874#ifndef sun4v
3875	ldub	[%g6 + KPMTSBM_FLAGS], %g7
3876	mov	ASI_N, %g1
3877	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
3878	movnz	%icc, ASI_MEM, %g1
3879	mov	%g1, %asi
3880#endif
3881
3882	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
3883	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
3884
3885	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
3886	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
3887
3888	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
3889
3890	/* KPMLOCK_EXIT(kpmlckp, asi) */
3891	KPMLOCK_EXIT(%g3, ASI_MEM)
3892
3893	/*
3894	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3895	 * point to trapstat's TSB miss return code (note that trapstat
3896	 * itself will patch the correct offset to add).
3897	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
3898	 */
3899	rdpr	%tl, %g7
3900	cmp	%g7, 1
3901	ble	%icc, 0f
3902	sethi	%hi(KERNELBASE), %g6
3903	rdpr	%tpc, %g7
3904	or	%g6, %lo(KERNELBASE), %g6
3905	cmp	%g7, %g6
3906	bgeu	%xcc, 0f
3907	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
3908	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3909	wrpr	%g7, %tpc
3910	add	%g7, 4, %g7
3911	wrpr	%g7, %tnpc
39120:
3913	retry
39145:
3915	/* g3=hlck_pa */
3916	KPMLOCK_EXIT(%g3, ASI_MEM)
3917	ba,pt	%icc, sfmmu_kpm_exception
3918	  nop
3919	SET_SIZE(sfmmu_kpm_dtsb_miss_small)
3920
3921#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
3922#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
3923#endif
3924
3925#endif /* lint */
3926
3927#ifdef	lint
3928/*
3929 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
3930 * Called from C-level, sets/clears "go" indication for trap level handler.
3931 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
3932 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
3933 * Assumes khl_mutex is held when called from C-level.
3934 */
3935/* ARGSUSED */
3936void
3937sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
3938{
3939}
3940
3941/*
3942 * kpm_smallpages: stores val to byte at address mapped within
3943 * low level lock brackets. The old value is returned.
3944 * Called from C-level.
3945 */
3946/* ARGSUSED */
3947int
3948sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val)
3949{
3950	return (0);
3951}
3952
3953#else /* lint */
3954
3955	.seg	".data"
3956sfmmu_kpm_tsbmtl_panic:
3957	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
3958	.byte	0
3959sfmmu_kpm_stsbmtl_panic:
3960	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
3961	.byte	0
3962	.align	4
3963	.seg	".text"
3964
3965	ENTRY_NP(sfmmu_kpm_tsbmtl)
3966	rdpr	%pstate, %o3
3967	/*
3968	 * %o0 = &kp_refcntc
3969	 * %o1 = &khl_lock
3970	 * %o2 = 0/1 (off/on)
3971	 * %o3 = pstate save
3972	 */
3973#ifdef DEBUG
3974	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
3975	bnz,pt %icc, 1f				/* disabled, panic	 */
3976	  nop
3977	save	%sp, -SA(MINFRAME), %sp
3978	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
3979	call	panic
3980	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
3981	ret
3982	restore
39831:
3984#endif /* DEBUG */
3985	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
3986
3987	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
3988	mov	-1, %o5
3989	brz,a	%o2, 2f
3990	  mov	0, %o5
39912:
3992	sth	%o5, [%o0]
3993	KPMLOCK_EXIT(%o1, ASI_N)
3994
3995	retl
3996	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
3997	SET_SIZE(sfmmu_kpm_tsbmtl)
3998
3999	ENTRY_NP(sfmmu_kpm_stsbmtl)
4000	rdpr	%pstate, %o3
4001	/*
4002	 * %o0 = &mapped
4003	 * %o1 = &kshl_lock
4004	 * %o2 = val
4005	 * %o3 = pstate save
4006	 */
4007#ifdef DEBUG
4008	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4009	bnz,pt %icc, 1f				/* disabled, panic	 */
4010	  nop
4011	save	%sp, -SA(MINFRAME), %sp
4012	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
4013	call	panic
4014	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
4015	ret
4016	restore
40171:
4018#endif /* DEBUG */
4019	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4020
4021	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
4022	ldsb	[%o0], %o5
4023	stb	%o2, [%o0]
4024	KPMLOCK_EXIT(%o1, ASI_N)
4025
4026	mov	%o5, %o0			/* return old val */
4027	retl
4028	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4029	SET_SIZE(sfmmu_kpm_stsbmtl)
4030
4031#endif /* lint */
4032
4033#ifndef lint
4034#ifdef sun4v
4035	/*
4036	 * User/kernel data miss w// multiple TSBs
4037	 * The first probe covers 8K, 64K, and 512K page sizes,
4038	 * because 64K and 512K mappings are replicated off 8K
4039	 * pointer.  Second probe covers 4M page size only.
4040	 *
4041	 * MMU fault area contains miss address and context.
4042	 */
4043	ALTENTRY(sfmmu_slow_dmmu_miss)
4044	GET_MMU_D_TAGACC_CTX(%g2, %g3)	! %g2 = tagacc, %g3 = ctx
4045
4046slow_miss_common:
4047	/*
4048	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
4049	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
4050	 */
4051	brnz,pt	%g3, 8f			! check for user context
4052	  nop
4053
4054	/*
4055	 * Kernel miss
4056	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
4057	 * branch to sfmmu_tsb_miss_tt to handle it.
4058	 */
4059	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4060sfmmu_dslow_patch_ktsb_base:
4061	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
4062sfmmu_dslow_patch_ktsb_szcode:
4063	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
4064
4065	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
4066	! %g1 = First TSB entry pointer, as TSB miss handler expects
4067
4068	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4069sfmmu_dslow_patch_ktsb4m_base:
4070	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
4071sfmmu_dslow_patch_ktsb4m_szcode:
4072	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
4073
4074	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
4075	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
4076	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4077	.empty
4078
40798:
4080	/*
4081	 * User miss
4082	 * Get first TSB pointer in %g1
4083	 * Get second TSB pointer (or NULL if no second TSB) in %g3
4084	 * Branch to sfmmu_tsb_miss_tt to handle it
4085	 */
4086	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
4087	/* %g1 = first TSB entry ptr now, %g2 preserved */
4088
4089	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
4090	brlz,a,pt %g3, sfmmu_tsb_miss_tt	/* done if no 2nd TSB */
4091	  mov	%g0, %g3
4092
4093	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
4094	/* %g3 = second TSB entry ptr now, %g2 preserved */
40959:
4096	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4097	.empty
4098	SET_SIZE(sfmmu_slow_dmmu_miss)
4099
4100
4101	/*
4102	 * User/kernel instruction miss w/ multiple TSBs
4103	 * The first probe covers 8K, 64K, and 512K page sizes,
4104	 * because 64K and 512K mappings are replicated off 8K
4105	 * pointer.  Second probe covers 4M page size only.
4106	 *
4107	 * MMU fault area contains miss address and context.
4108	 */
4109	ALTENTRY(sfmmu_slow_immu_miss)
4110	MMU_FAULT_STATUS_AREA(%g2)
4111	ldx	[%g2 + MMFSA_I_CTX], %g3
4112	ldx	[%g2 + MMFSA_I_ADDR], %g2
4113	srlx	%g2, MMU_PAGESHIFT, %g2	! align address to page boundry
4114	sllx	%g2, MMU_PAGESHIFT, %g2
4115	ba,pt	%xcc, slow_miss_common
4116	or	%g2, %g3, %g2
4117	SET_SIZE(sfmmu_slow_immu_miss)
4118
4119#endif /* sun4v */
4120#endif	/* lint */
4121
4122#ifndef lint
4123
4124/*
4125 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
4126 */
4127	.seg	".data"
4128	.align	64
4129	.global tsbmiss_area
4130tsbmiss_area:
4131	.skip	(TSBMISS_SIZE * NCPU)
4132
4133	.align	64
4134	.global kpmtsbm_area
4135kpmtsbm_area:
4136	.skip	(KPMTSBM_SIZE * NCPU)
4137#endif	/* lint */
4138