sfmmu_asm.s revision 0066ea643632362631f3bb653a0fcf6ed1f07cd3
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#pragma ident	"%Z%%M%	%I%	%E% SMI"
27
28/*
29 * SFMMU primitives.  These primitives should only be used by sfmmu
30 * routines.
31 */
32
33#if defined(lint)
34#include <sys/types.h>
35#else	/* lint */
36#include "assym.h"
37#endif	/* lint */
38
39#include <sys/asm_linkage.h>
40#include <sys/machtrap.h>
41#include <sys/machasi.h>
42#include <sys/sun4asi.h>
43#include <sys/pte.h>
44#include <sys/mmu.h>
45#include <vm/hat_sfmmu.h>
46#include <vm/seg_spt.h>
47#include <sys/machparam.h>
48#include <sys/privregs.h>
49#include <sys/scb.h>
50#include <sys/intreg.h>
51#include <sys/machthread.h>
52#include <sys/intr.h>
53#include <sys/clock.h>
54#include <sys/trapstat.h>
55
56#ifdef TRAPTRACE
57#include <sys/traptrace.h>
58
59/*
60 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
61 */
62#define	TT_TRACE(label)		\
63	ba	label		;\
64	rd	%pc, %g7
65#else
66
67#define	TT_TRACE(label)
68
69#endif /* TRAPTRACE */
70
71#ifndef	lint
72
73#if (TTE_SUSPEND_SHIFT > 0)
74#define	TTE_SUSPEND_INT_SHIFT(reg)				\
75	sllx	reg, TTE_SUSPEND_SHIFT, reg
76#else
77#define	TTE_SUSPEND_INT_SHIFT(reg)
78#endif
79
80#endif /* lint */
81
82#ifndef	lint
83
84/*
85 * Assumes TSBE_TAG is 0
86 * Assumes TSBE_INTHI is 0
87 * Assumes TSBREG.split is 0
88 */
89
90#if TSBE_TAG != 0
91#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
92#endif
93
94#if TSBTAG_INTHI != 0
95#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
96#endif
97
98/*
99 * The following code assumes the tsb is not split.
100 *
101 * With TSBs no longer shared between processes, it's no longer
102 * necessary to hash the context bits into the tsb index to get
103 * tsb coloring; the new implementation treats the TSB as a
104 * direct-mapped, virtually-addressed cache.
105 *
106 * In:
107 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
108 *    tsbbase = base address of TSB (clobbered)
109 *    tagacc = tag access register (clobbered)
110 *    szc = size code of TSB (ro)
111 *    tmp = scratch reg
112 * Out:
113 *    tsbbase = pointer to entry in TSB
114 */
115#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
116	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
117	srlx	tagacc, vpshift, tagacc 				;\
118	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
119	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
120	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
121	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
122	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */
123
124/*
125 * When the kpm TSB is used it is assumed that it is direct mapped
126 * using (vaddr>>vpshift)%tsbsz as the index.
127 *
128 * Note that, for now, the kpm TSB and kernel TSB are the same for
129 * each mapping size.  However that need not always be the case.  If
130 * the trap handlers are updated to search a different TSB for kpm
131 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
132 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
133 *
134 * In:
135 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
136 *    vaddr = virtual address (clobbered)
137 *    tsbp, szc, tmp = scratch
138 * Out:
139 *    tsbp = pointer to entry in TSB
140 */
141#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
142	cmp	vpshift, MMU_PAGESHIFT					;\
143	bne,pn	%icc, 1f		/* branch if large case */	;\
144	  sethi	%hi(kpmsm_tsbsz), szc					;\
145	sethi	%hi(kpmsm_tsbbase), tsbp				;\
146	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
147	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
148	ba,pt	%icc, 2f						;\
149	  nop								;\
1501:	sethi	%hi(kpm_tsbsz), szc					;\
151	sethi	%hi(kpm_tsbbase), tsbp					;\
152	ld	[szc + %lo(kpm_tsbsz)], szc				;\
153	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
1542:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
155
156/*
157 * Lock the TSBE at virtual address tsbep.
158 *
159 * tsbep = TSBE va (ro)
160 * tmp1, tmp2 = scratch registers (clobbered)
161 * label = label to use for branches (text)
162 * %asi = ASI to use for TSB access
163 *
164 * NOTE that we flush the TSB using fast VIS instructions that
165 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
166 * not be treated as a locked entry or we'll get stuck spinning on
167 * an entry that isn't locked but really invalid.
168 */
169
170#if defined(UTSB_PHYS)
171
172#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
173	lda	[tsbep]ASI_MEM, tmp1					;\
174label:									;\
175	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
176	cmp	tmp1, tmp2 						;\
177	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
178	  lda	[tsbep]ASI_MEM, tmp1					;\
179	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
180	cmp	tmp1, tmp2 						;\
181	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
182	  lda	[tsbep]ASI_MEM, tmp1					;\
183	/* tsbe lock acquired */					;\
184	membar #StoreStore
185
186#else /* UTSB_PHYS */
187
188#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
189	lda	[tsbep]%asi, tmp1					;\
190label:									;\
191	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
192	cmp	tmp1, tmp2 						;\
193	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
194	  lda	[tsbep]%asi, tmp1					;\
195	casa	[tsbep]%asi, tmp1, tmp2					;\
196	cmp	tmp1, tmp2 						;\
197	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
198	  lda	[tsbep]%asi, tmp1					;\
199	/* tsbe lock acquired */					;\
200	membar #StoreStore
201
202#endif /* UTSB_PHYS */
203
204/*
205 * Atomically write TSBE at virtual address tsbep.
206 *
207 * tsbep = TSBE va (ro)
208 * tte = TSBE TTE (ro)
209 * tagtarget = TSBE tag (ro)
210 * %asi = ASI to use for TSB access
211 */
212
213#if defined(UTSB_PHYS)
214
215#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
216	add	tsbep, TSBE_TTE, tmp1					;\
217	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
218	membar #StoreStore						;\
219	add	tsbep, TSBE_TAG, tmp1					;\
220	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */
221
222#else /* UTSB_PHYS */
223
224#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
225	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
226	membar #StoreStore						;\
227	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
228
229#endif /* UTSB_PHYS */
230
231/*
232 * Load an entry into the TSB at TL > 0.
233 *
234 * tsbep = pointer to the TSBE to load as va (ro)
235 * tte = value of the TTE retrieved and loaded (wo)
236 * tagtarget = tag target register.  To get TSBE tag to load,
237 *   we need to mask off the context and leave only the va (clobbered)
238 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
239 * tmp1, tmp2 = scratch registers
240 * label = label to use for branches (text)
241 * %asi = ASI to use for TSB access
242 */
243
244#if defined(UTSB_PHYS)
245
246#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
247	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
248	/*								;\
249	 * I don't need to update the TSB then check for the valid tte.	;\
250	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
251	 * we always invalidate the hash table before we unload the TSB.;\
252	 */								;\
253	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
254	ldxa	[ttepa]ASI_MEM, tte					;\
255	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
256	sethi	%hi(TSBTAG_INVALID), tmp2				;\
257	add	tsbep, TSBE_TAG, tmp1					;\
258	brgez,a,pn tte, label/**/f					;\
259	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
260	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
261label:
262
263#else /* UTSB_PHYS */
264
265#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
266	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
267	/*								;\
268	 * I don't need to update the TSB then check for the valid tte.	;\
269	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
270	 * we always invalidate the hash table before we unload the TSB.;\
271	 */								;\
272	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
273	ldxa	[ttepa]ASI_MEM, tte					;\
274	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
275	sethi	%hi(TSBTAG_INVALID), tmp2				;\
276	brgez,a,pn tte, label/**/f					;\
277	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
278	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
279label:
280
281#endif /* UTSB_PHYS */
282
283/*
284 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
285 *   for ITLB synthesis.
286 *
287 * tsbep = pointer to the TSBE to load as va (ro)
288 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
289 *   with exec_perm turned off and exec_synth turned on
290 * tagtarget = tag target register.  To get TSBE tag to load,
291 *   we need to mask off the context and leave only the va (clobbered)
292 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
293 * tmp1, tmp2 = scratch registers
294 * label = label to use for branch (text)
295 * %asi = ASI to use for TSB access
296 */
297
298#define	TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
299	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
300	/*								;\
301	 * I don't need to update the TSB then check for the valid tte.	;\
302	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
303	 * we always invalidate the hash table before we unload the TSB.;\
304	 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0	;\
305	 * and exec_synth bit to 1.					;\
306	 */								;\
307	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
308	mov	tte, tmp1						;\
309	ldxa	[ttepa]ASI_MEM, tte					;\
310	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
311	sethi	%hi(TSBTAG_INVALID), tmp2				;\
312	brgez,a,pn tte, label/**/f					;\
313	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
314	or	tte, tmp1, tte						;\
315	andn	tte, TTE_EXECPRM_INT, tte				;\
316	or	tte, TTE_E_SYNTH_INT, tte				;\
317	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
318label:
319
320/*
321 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
322 *
323 * tte = value of the TTE, used to get tte_size bits (ro)
324 * tagaccess = tag access register, used to get 4M pfn bits (ro)
325 * pfn = 4M pfn bits shifted to offset for tte (out)
326 * tmp1 = scratch register
327 * label = label to use for branch (text)
328 */
329
330#define	GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)			\
331	/*								;\
332	 * Get 4M bits from tagaccess for 32M, 256M pagesizes.		;\
333	 * Return them, shifted, in pfn.				;\
334	 */								;\
335	srlx	tagaccess, MMU_PAGESHIFT4M, tagaccess			;\
336	srlx	tte, TTE_SZ_SHFT, tmp		/* isolate the */	;\
337	andcc	tmp, TTE_SZ_BITS, %g0		/* tte_size bits */	;\
338	bz,a,pt %icc, label/**/f		/* if 0, is */		;\
339	  and	tagaccess, 0x7, tagaccess	/* 32M page size */	;\
340	and	tagaccess, 0x3f, tagaccess /* else 256M page size */	;\
341label:									;\
342	sllx	tagaccess, MMU_PAGESHIFT4M, pfn
343
344/*
345 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
346 * for ITLB synthesis.
347 *
348 * tte = value of the TTE, used to get tte_size bits (rw)
349 * tmp1 = scratch register
350 */
351
352#define	SET_TTE4M_PN(tte, tmp)						\
353	/*								;\
354	 * Set 4M pagesize tte bits. 					;\
355	 */								;\
356	set	TTE4M, tmp						;\
357	sllx	tmp, TTE_SZ_SHFT, tmp					;\
358	or	tte, tmp, tte
359
360/*
361 * Load an entry into the TSB at TL=0.
362 *
363 * tsbep = pointer to the TSBE to load as va (ro)
364 * tteva = pointer to the TTE to load as va (ro)
365 * tagtarget = TSBE tag to load (which contains no context), synthesized
366 * to match va of MMU tag target register only (ro)
367 * tmp1, tmp2 = scratch registers (clobbered)
368 * label = label to use for branches (text)
369 * %asi = ASI to use for TSB access
370 */
371
372#if defined(UTSB_PHYS)
373
374#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
375	/* can't rd tteva after locking tsb because it can tlb miss */	;\
376	ldx	[tteva], tteva			/* load tte */		;\
377	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
378	sethi	%hi(TSBTAG_INVALID), tmp2				;\
379	add	tsbep, TSBE_TAG, tmp1					;\
380	brgez,a,pn tteva, label/**/f					;\
381	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
382	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
383label:
384
385#else /* UTSB_PHYS */
386
387#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
388	/* can't rd tteva after locking tsb because it can tlb miss */	;\
389	ldx	[tteva], tteva			/* load tte */		;\
390	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
391	sethi	%hi(TSBTAG_INVALID), tmp2				;\
392	brgez,a,pn tteva, label/**/f					;\
393	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
394	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
395label:
396
397#endif /* UTSB_PHYS */
398
399/*
400 * Invalidate a TSB entry in the TSB.
401 *
402 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
403 *	 about this earlier to ensure this is true.  Thus when we are
404 *	 directly referencing tsbep below, we are referencing the tte_tag
405 *	 field of the TSBE.  If this  offset ever changes, the code below
406 *	 will need to be modified.
407 *
408 * tsbep = pointer to TSBE as va (ro)
409 * tag = invalidation is done if this matches the TSBE tag (ro)
410 * tmp1 - tmp3 = scratch registers (clobbered)
411 * label = label name to use for branches (text)
412 * %asi = ASI to use for TSB access
413 */
414
415#if defined(UTSB_PHYS)
416
417#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
418	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
419	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
420label/**/1:								;\
421	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
422	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
423	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
424	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
425	cmp	tag, tmp3		/* compare tags */		;\
426	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
427	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
428	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
429	cmp	tmp1, tmp3		/* if not successful */		;\
430	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
431	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
432label/**/2:
433
434#else /* UTSB_PHYS */
435
436#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
437	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
438	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
439label/**/1:								;\
440	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
441	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
442	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
443	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
444	cmp	tag, tmp3		/* compare tags */		;\
445	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
446	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
447	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
448	cmp	tmp1, tmp3		/* if not successful */		;\
449	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
450	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
451label/**/2:
452
453#endif /* UTSB_PHYS */
454
455#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
456#error	- TSB_SOFTSZ_MASK too small
457#endif
458
459
460/*
461 * An implementation of setx which will be hot patched at run time.
462 * since it is being hot patched, there is no value passed in.
463 * Thus, essentially we are implementing
464 *	setx value, tmp, dest
465 * where value is RUNTIME_PATCH (aka 0) in this case.
466 */
467#define	RUNTIME_PATCH_SETX(dest, tmp)					\
468	sethi	%hh(RUNTIME_PATCH), tmp					;\
469	sethi	%lm(RUNTIME_PATCH), dest				;\
470	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
471	or	dest, %lo(RUNTIME_PATCH), dest				;\
472	sllx	tmp, 32, tmp						;\
473	nop				/* for perf reasons */		;\
474	or	tmp, dest, dest		/* contents of patched value */
475
476
477#endif (lint)
478
479
480#if defined (lint)
481
482/*
483 * sfmmu related subroutines
484 */
485
486/*
487 * Use cas, if tte has changed underneath us then reread and try again.
488 * In the case of a retry, it will update sttep with the new original.
489 */
490/* ARGSUSED */
491int
492sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
493{ return(0); }
494
495/*
496 * Use cas, if tte has changed underneath us then return 1, else return 0
497 */
498/* ARGSUSED */
499int
500sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
501{ return(0); }
502
503/* ARGSUSED */
504void
505sfmmu_copytte(tte_t *sttep, tte_t *dttep)
506{}
507
508/*ARGSUSED*/
509struct tsbe *
510sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
511{ return(0); }
512
513/*ARGSUSED*/
514uint64_t
515sfmmu_make_tsbtag(caddr_t va)
516{ return(0); }
517
518#else	/* lint */
519
520	.seg	".data"
521	.global	sfmmu_panic1
522sfmmu_panic1:
523	.asciz	"sfmmu_asm: interrupts already disabled"
524
525	.global	sfmmu_panic3
526sfmmu_panic3:
527	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"
528
529	.global	sfmmu_panic4
530sfmmu_panic4:
531	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"
532
533	.global	sfmmu_panic5
534sfmmu_panic5:
535	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"
536
537
538	ENTRY_NP(sfmmu_modifytte)
539	ldx	[%o2], %g3			/* current */
540	ldx	[%o0], %g1			/* original */
5412:
542	ldx	[%o1], %g2			/* modified */
543	cmp	%g2, %g3			/* is modified = current? */
544	be,a,pt	%xcc,1f				/* yes, don't write */
545	stx	%g3, [%o0]			/* update new original */
546	casx	[%o2], %g1, %g2
547	cmp	%g1, %g2
548	be,pt	%xcc, 1f			/* cas succeeded - return */
549	  nop
550	ldx	[%o2], %g3			/* new current */
551	stx	%g3, [%o0]			/* save as new original */
552	ba,pt	%xcc, 2b
553	  mov	%g3, %g1
5541:	retl
555	membar	#StoreLoad
556	SET_SIZE(sfmmu_modifytte)
557
558	ENTRY_NP(sfmmu_modifytte_try)
559	ldx	[%o1], %g2			/* modified */
560	ldx	[%o2], %g3			/* current */
561	ldx	[%o0], %g1			/* original */
562	cmp	%g3, %g2			/* is modified = current? */
563	be,a,pn %xcc,1f				/* yes, don't write */
564	mov	0, %o1				/* as if cas failed. */
565
566	casx	[%o2], %g1, %g2
567	membar	#StoreLoad
568	cmp	%g1, %g2
569	movne	%xcc, -1, %o1			/* cas failed. */
570	move	%xcc, 1, %o1			/* cas succeeded. */
5711:
572	stx	%g2, [%o0]			/* report "current" value */
573	retl
574	mov	%o1, %o0
575	SET_SIZE(sfmmu_modifytte_try)
576
577	ENTRY_NP(sfmmu_copytte)
578	ldx	[%o0], %g1
579	retl
580	stx	%g1, [%o1]
581	SET_SIZE(sfmmu_copytte)
582
583
584	/*
585	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
586	 * %o0 = TSB base address (in), pointer to TSB entry (out)
587	 * %o1 = vaddr (in)
588	 * %o2 = vpshift (in)
589	 * %o3 = tsb size code (in)
590	 * %o4 = scratch register
591	 */
592	ENTRY_NP(sfmmu_get_tsbe)
593	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
594	retl
595	nop
596	SET_SIZE(sfmmu_get_tsbe)
597
598	/*
599	 * Return a TSB tag for the given va.
600	 * %o0 = va (in/clobbered)
601	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
602	 */
603	ENTRY_NP(sfmmu_make_tsbtag)
604	retl
605	srln	%o0, TTARGET_VA_SHIFT, %o0
606	SET_SIZE(sfmmu_make_tsbtag)
607
608#endif /* lint */
609
610/*
611 * Other sfmmu primitives
612 */
613
614
615#if defined (lint)
616void
617sfmmu_patch_ktsb(void)
618{
619}
620
621void
622sfmmu_kpm_patch_tlbm(void)
623{
624}
625
626void
627sfmmu_kpm_patch_tsbm(void)
628{
629}
630
631/* ARGSUSED */
632void
633sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
634{
635}
636
637/* ARGSUSED */
638void
639sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
640{
641}
642
643/* ARGSUSED */
644void
645sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
646{
647}
648
649/* ARGSUSED */
650void
651sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
652{
653}
654
655#else /* lint */
656
657#define	I_SIZE		4
658
659	ENTRY_NP(sfmmu_fix_ktlb_traptable)
660	/*
661	 * %o0 = start of patch area
662	 * %o1 = size code of TSB to patch
663	 * %o3 = scratch
664	 */
665	/* fix sll */
666	ld	[%o0], %o3			/* get sll */
667	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
668	st	%o3, [%o0]			/* write sll */
669	flush	%o0
670	/* fix srl */
671	add	%o0, I_SIZE, %o0		/* goto next instr. */
672	ld	[%o0], %o3			/* get srl */
673	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
674	st	%o3, [%o0]			/* write srl */
675	retl
676	flush	%o0
677	SET_SIZE(sfmmu_fix_ktlb_traptable)
678
679	ENTRY_NP(sfmmu_fixup_ktsbbase)
680	/*
681	 * %o0 = start of patch area
682	 * %o5 = kernel virtual or physical tsb base address
683	 * %o2, %o3 are used as scratch registers.
684	 */
685	/* fixup sethi instruction */
686	ld	[%o0], %o3
687	srl	%o5, 10, %o2			! offset is bits 32:10
688	or	%o3, %o2, %o3			! set imm22
689	st	%o3, [%o0]
690	/* fixup offset of lduw/ldx */
691	add	%o0, I_SIZE, %o0		! next instr
692	ld	[%o0], %o3
693	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
694	or	%o3, %o2, %o3
695	st	%o3, [%o0]
696	retl
697	flush	%o0
698	SET_SIZE(sfmmu_fixup_ktsbbase)
699
700	ENTRY_NP(sfmmu_fixup_setx)
701	/*
702	 * %o0 = start of patch area
703	 * %o4 = 64 bit value to patch
704	 * %o2, %o3 are used as scratch registers.
705	 *
706	 * Note: Assuming that all parts of the instructions which need to be
707	 *	 patched correspond to RUNTIME_PATCH (aka 0)
708	 *
709	 * Note the implementation of setx which is being patched is as follows:
710	 *
711	 * sethi   %hh(RUNTIME_PATCH), tmp
712	 * sethi   %lm(RUNTIME_PATCH), dest
713	 * or      tmp, %hm(RUNTIME_PATCH), tmp
714	 * or      dest, %lo(RUNTIME_PATCH), dest
715	 * sllx    tmp, 32, tmp
716	 * nop
717	 * or      tmp, dest, dest
718	 *
719	 * which differs from the implementation in the
720	 * "SPARC Architecture Manual"
721	 */
722	/* fixup sethi instruction */
723	ld	[%o0], %o3
724	srlx	%o4, 42, %o2			! bits [63:42]
725	or	%o3, %o2, %o3			! set imm22
726	st	%o3, [%o0]
727	/* fixup sethi instruction */
728	add	%o0, I_SIZE, %o0		! next instr
729	ld	[%o0], %o3
730	sllx	%o4, 32, %o2			! clear upper bits
731	srlx	%o2, 42, %o2			! bits [31:10]
732	or	%o3, %o2, %o3			! set imm22
733	st	%o3, [%o0]
734	/* fixup or instruction */
735	add	%o0, I_SIZE, %o0		! next instr
736	ld	[%o0], %o3
737	srlx	%o4, 32, %o2			! bits [63:32]
738	and	%o2, 0x3ff, %o2			! bits [41:32]
739	or	%o3, %o2, %o3			! set imm
740	st	%o3, [%o0]
741	/* fixup or instruction */
742	add	%o0, I_SIZE, %o0		! next instr
743	ld	[%o0], %o3
744	and	%o4, 0x3ff, %o2			! bits [9:0]
745	or	%o3, %o2, %o3			! set imm
746	st	%o3, [%o0]
747	retl
748	flush	%o0
749	SET_SIZE(sfmmu_fixup_setx)
750
751	ENTRY_NP(sfmmu_fixup_or)
752	/*
753	 * %o0 = start of patch area
754	 * %o4 = 32 bit value to patch
755	 * %o2, %o3 are used as scratch registers.
756	 * Note: Assuming that all parts of the instructions which need to be
757	 *	 patched correspond to RUNTIME_PATCH (aka 0)
758	 */
759	ld	[%o0], %o3
760	and	%o4, 0x3ff, %o2			! bits [9:0]
761	or	%o3, %o2, %o3			! set imm
762	st	%o3, [%o0]
763	retl
764	flush	%o0
765	SET_SIZE(sfmmu_fixup_or)
766
767	ENTRY_NP(sfmmu_fixup_shiftx)
768	/*
769	 * %o0 = start of patch area
770	 * %o4 = signed int immediate value to add to sllx/srlx imm field
771	 * %o2, %o3 are used as scratch registers.
772	 *
773	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
774	 * so we do a simple add.  The caller must be careful to prevent
775	 * overflow, which could easily occur if the initial value is nonzero!
776	 */
777	ld	[%o0], %o3			! %o3 = instruction to patch
778	and	%o3, 0x3f, %o2			! %o2 = existing imm value
779	add	%o2, %o4, %o2			! %o2 = new imm value
780	andn	%o3, 0x3f, %o3			! clear old imm value
781	and	%o2, 0x3f, %o2			! truncate new imm value
782	or	%o3, %o2, %o3			! set new imm value
783	st	%o3, [%o0]			! store updated instruction
784	retl
785	flush	%o0
786	SET_SIZE(sfmmu_fixup_shiftx)
787
788	ENTRY_NP(sfmmu_fixup_mmu_asi)
789	/*
790	 * Patch imm_asi of all ldda instructions in the MMU
791	 * trap handlers.  We search MMU_PATCH_INSTR instructions
792	 * starting from the itlb miss handler (trap 0x64).
793	 * %o0 = address of tt[0,1]_itlbmiss
794	 * %o1 = imm_asi to setup, shifted by appropriate offset.
795	 * %o3 = number of instructions to search
796	 * %o4 = reserved by caller: called from leaf routine
797	 */
7981:	ldsw	[%o0], %o2			! load instruction to %o2
799	brgez,pt %o2, 2f
800	  srl	%o2, 30, %o5
801	btst	1, %o5				! test bit 30; skip if not set
802	bz,pt	%icc, 2f
803	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
804	srlx	%o5, 58, %o5			! isolate op3 part of opcode
805	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
806	brnz,pt	%o5, 2f				! skip if not a match
807	  or	%o2, %o1, %o2			! or in imm_asi
808	st	%o2, [%o0]			! write patched instruction
8092:	dec	%o3
810	brnz,a,pt %o3, 1b			! loop until we're done
811	  add	%o0, I_SIZE, %o0
812	retl
813	flush	%o0
814	SET_SIZE(sfmmu_fixup_mmu_asi)
815
816	/*
817	 * Patch immediate ASI used to access the TSB in the
818	 * trap table.
819	 * inputs: %o0 = value of ktsb_phys
820	 */
821	ENTRY_NP(sfmmu_patch_mmu_asi)
822	mov	%o7, %o4			! save return pc in %o4
823	movrnz	%o0, ASI_QUAD_LDD_PHYS, %o3
824	movrz	%o0, ASI_NQUAD_LD, %o3
825	sll	%o3, 5, %o1			! imm_asi offset
826	mov	6, %o3				! number of instructions
827	sethi	%hi(dktsb), %o0			! to search
828	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
829	  or	%o0, %lo(dktsb), %o0
830	mov	6, %o3				! number of instructions
831	sethi	%hi(dktsb4m), %o0		! to search
832	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
833	  or	%o0, %lo(dktsb4m), %o0
834	mov	6, %o3				! number of instructions
835	sethi	%hi(iktsb), %o0			! to search
836	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
837	  or	%o0, %lo(iktsb), %o0
838	mov	%o4, %o7			! retore return pc -- leaf
839	retl
840	nop
841	SET_SIZE(sfmmu_patch_mmu_asi)
842
843	ENTRY_NP(sfmmu_patch_ktsb)
844	/*
845	 * We need to fix iktsb, dktsb, et. al.
846	 */
847	save	%sp, -SA(MINFRAME), %sp
848	set	ktsb_phys, %o1
849	ld	[%o1], %o4
850	set	ktsb_base, %o5
851	set	ktsb4m_base, %l1
852	brz,pt	%o4, 1f
853	  nop
854	set	ktsb_pbase, %o5
855	set	ktsb4m_pbase, %l1
8561:
857	sethi	%hi(ktsb_szcode), %o1
858	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */
859
860	sethi	%hi(iktsb), %o0
861	call	sfmmu_fix_ktlb_traptable
862	  or	%o0, %lo(iktsb), %o0
863
864	sethi	%hi(dktsb), %o0
865	call	sfmmu_fix_ktlb_traptable
866	  or	%o0, %lo(dktsb), %o0
867
868	sethi	%hi(ktsb4m_szcode), %o1
869	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */
870
871	sethi	%hi(dktsb4m), %o0
872	call	sfmmu_fix_ktlb_traptable
873	  or	%o0, %lo(dktsb4m), %o0
874
875#ifndef sun4v
876	mov	ASI_N, %o2
877	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
878	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
879	sethi	%hi(tsb_kernel_patch_asi), %o0
880	call	sfmmu_fixup_or
881	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
882#endif
883
884	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)
885
886	sethi	%hi(dktsbbase), %o0
887	call	sfmmu_fixup_setx	! patch value of ktsb base addr
888	  or	%o0, %lo(dktsbbase), %o0
889
890	sethi	%hi(iktsbbase), %o0
891	call	sfmmu_fixup_setx	! patch value of ktsb base addr
892	  or	%o0, %lo(iktsbbase), %o0
893
894	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
895	call	sfmmu_fixup_setx	! patch value of ktsb base addr
896	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
897
898#ifdef sun4v
899	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
900	call	sfmmu_fixup_setx	! patch value of ktsb base addr
901	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
902#endif /* sun4v */
903
904	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)
905
906	sethi	%hi(dktsb4mbase), %o0
907	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
908	  or	%o0, %lo(dktsb4mbase), %o0
909
910	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
911	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
912	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
913
914#ifdef sun4v
915	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
916	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
917	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
918#endif /* sun4v */
919
920	set	ktsb_szcode, %o4
921	ld	[%o4], %o4
922	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
923	call	sfmmu_fixup_or		! patch value of ktsb_szcode
924	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
925
926#ifdef sun4v
927	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
928	call	sfmmu_fixup_or		! patch value of ktsb_szcode
929	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
930#endif /* sun4v */
931
932	set	ktsb4m_szcode, %o4
933	ld	[%o4], %o4
934	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
935	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
936	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
937
938#ifdef sun4v
939	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
940	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
941	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
942#endif /* sun4v */
943
944	ret
945	restore
946	SET_SIZE(sfmmu_patch_ktsb)
947
948	ENTRY_NP(sfmmu_kpm_patch_tlbm)
949	/*
950	 * Fixup trap handlers in common segkpm case.  This is reserved
951	 * for future use should kpm TSB be changed to be other than the
952	 * kernel TSB.
953	 */
954	retl
955	nop
956	SET_SIZE(sfmmu_kpm_patch_tlbm)
957
958	ENTRY_NP(sfmmu_kpm_patch_tsbm)
959	/*
960	 * nop the branch to sfmmu_kpm_dtsb_miss_small
961	 * in the case where we are using large pages for
962	 * seg_kpm (and hence must probe the second TSB for
963	 * seg_kpm VAs)
964	 */
965	set	dktsb4m_kpmcheck_small, %o0
966	MAKE_NOP_INSTR(%o1)
967	st	%o1, [%o0]
968	flush	%o0
969	retl
970	nop
971	SET_SIZE(sfmmu_kpm_patch_tsbm)
972
973	ENTRY_NP(sfmmu_patch_utsb)
974#ifdef UTSB_PHYS
975	retl
976	nop
977#else /* UTSB_PHYS */
978	/*
979	 * We need to hot patch utsb_vabase and utsb4m_vabase
980	 */
981	save	%sp, -SA(MINFRAME), %sp
982
983	/* patch value of utsb_vabase */
984	set	utsb_vabase, %o1
985	ldx	[%o1], %o4
986	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
987	call	sfmmu_fixup_setx
988	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
989	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
990	call	sfmmu_fixup_setx
991	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
992	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
993	call	sfmmu_fixup_setx
994	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
995
996	/* patch value of utsb4m_vabase */
997	set	utsb4m_vabase, %o1
998	ldx	[%o1], %o4
999	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1000	call	sfmmu_fixup_setx
1001	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1002	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1003	call	sfmmu_fixup_setx
1004	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1005	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1006	call	sfmmu_fixup_setx
1007	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1008
1009	/*
1010	 * Patch TSB base register masks and shifts if needed.
1011	 * By default the TSB base register contents are set up for 4M slab.
1012	 * If we're using a smaller slab size and reserved VA range we need
1013	 * to patch up those values here.
1014	 */
1015	set	tsb_slab_shift, %o1
1016	set	MMU_PAGESHIFT4M, %o4
1017	ldsw	[%o1], %o3
1018	subcc	%o4, %o3, %o4
1019	bz,pt	%icc, 1f
1020	  /* delay slot safe */
1021
1022	/* patch reserved VA range size if needed. */
1023	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
1024	call	sfmmu_fixup_shiftx
1025	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1026	call	sfmmu_fixup_shiftx
1027	  add	%o0, I_SIZE, %o0
1028	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
1029	call	sfmmu_fixup_shiftx
1030	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1031	call	sfmmu_fixup_shiftx
1032	  add	%o0, I_SIZE, %o0
10331:
1034	/* patch TSBREG_VAMASK used to set up TSB base register */
1035	set	tsb_slab_mask, %o1
1036	lduw	[%o1], %o4
1037	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1038	call	sfmmu_fixup_or
1039	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1040	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1041	call	sfmmu_fixup_or
1042	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1043
1044	ret
1045	restore
1046#endif /* UTSB_PHYS */
1047	SET_SIZE(sfmmu_patch_utsb)
1048
1049
1050	/*
1051	 * Routine that loads an entry into a tsb using virtual addresses.
1052	 * Locking is required since all cpus can use the same TSB.
1053	 * Note that it is no longer required to have a valid context
1054	 * when calling this function.
1055	 */
1056	ENTRY_NP(sfmmu_load_tsbe)
1057	/*
1058	 * %o0 = pointer to tsbe to load
1059	 * %o1 = tsb tag
1060	 * %o2 = virtual pointer to TTE
1061	 * %o3 = 1 if physical address in %o0 else 0
1062	 */
1063	rdpr	%pstate, %o5
1064#ifdef DEBUG
1065	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1066	bnz,pt 	%icc, 1f			/* disabled, panic	 */
1067	  nop
1068
1069	sethi	%hi(panicstr), %g1
1070	ldx	[%g1 + %lo(panicstr)], %g1
1071	tst	%g1
1072	bnz,pt	%icc, 1f
1073	  nop
1074
1075	save	%sp, -SA(MINFRAME), %sp
1076	sethi	%hi(sfmmu_panic1), %o0
1077	call	panic
1078	 or	%o0, %lo(sfmmu_panic1), %o0
10791:
1080#endif /* DEBUG */
1081
1082	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1083
1084	SETUP_TSB_ASI(%o3, %g3)
1085	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1)
1086
1087	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1088
1089	retl
1090	membar	#StoreStore|#StoreLoad
1091	SET_SIZE(sfmmu_load_tsbe)
1092
1093	/*
1094	 * Flush TSB of a given entry if the tag matches.
1095	 */
1096	ENTRY(sfmmu_unload_tsbe)
1097	/*
1098	 * %o0 = pointer to tsbe to be flushed
1099	 * %o1 = tag to match
1100	 * %o2 = 1 if physical address in %o0 else 0
1101	 */
1102	SETUP_TSB_ASI(%o2, %g1)
1103	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1104	retl
1105	membar	#StoreStore|#StoreLoad
1106	SET_SIZE(sfmmu_unload_tsbe)
1107
1108	/*
1109	 * Routine that loads a TTE into the kpm TSB from C code.
1110	 * Locking is required since kpm TSB is shared among all CPUs.
1111	 */
1112	ENTRY_NP(sfmmu_kpm_load_tsb)
1113	/*
1114	 * %o0 = vaddr
1115	 * %o1 = ttep
1116	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1117	 */
1118	rdpr	%pstate, %o5			! %o5 = saved pstate
1119#ifdef DEBUG
1120	andcc	%o5, PSTATE_IE, %g0		! if interrupts already
1121	bnz,pt	%icc, 1f			! disabled, panic
1122	  nop
1123
1124	sethi	%hi(panicstr), %g1
1125	ldx	[%g1 + %lo(panicstr)], %g1
1126	tst	%g1
1127	bnz,pt	%icc, 1f
1128	  nop
1129
1130	save	%sp, -SA(MINFRAME), %sp
1131	sethi	%hi(sfmmu_panic1), %o0
1132	call	panic
1133	  or	%o0, %lo(sfmmu_panic1), %o0
11341:
1135#endif /* DEBUG */
1136	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts
1137
1138#ifndef sun4v
1139	sethi	%hi(ktsb_phys), %o4
1140	mov	ASI_N, %o3
1141	ld	[%o4 + %lo(ktsb_phys)], %o4
1142	movrnz	%o4, ASI_MEM, %o3
1143	mov	%o3, %asi
1144#endif
1145	mov	%o0, %g1			! %g1 = vaddr
1146
1147	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1148	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1149	/* %g2 = tsbep, %g1 clobbered */
1150
1151	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1152	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1153	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1)
1154
1155	wrpr	%g0, %o5, %pstate		! enable interrupts
1156	retl
1157	  membar #StoreStore|#StoreLoad
1158	SET_SIZE(sfmmu_kpm_load_tsb)
1159
1160	/*
1161	 * Routine that shoots down a TTE in the kpm TSB or in the
1162	 * kernel TSB depending on virtpg. Locking is required since
1163	 * kpm/kernel TSB is shared among all CPUs.
1164	 */
1165	ENTRY_NP(sfmmu_kpm_unload_tsb)
1166	/*
1167	 * %o0 = vaddr
1168	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1169	 */
1170#ifndef sun4v
1171	sethi	%hi(ktsb_phys), %o4
1172	mov	ASI_N, %o3
1173	ld	[%o4 + %lo(ktsb_phys)], %o4
1174	movrnz	%o4, ASI_MEM, %o3
1175	mov	%o3, %asi
1176#endif
1177	mov	%o0, %g1			! %g1 = vaddr
1178
1179	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1180	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1181	/* %g2 = tsbep, %g1 clobbered */
1182
1183	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1184	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1185	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1186
1187	retl
1188	  membar	#StoreStore|#StoreLoad
1189	SET_SIZE(sfmmu_kpm_unload_tsb)
1190
1191#endif /* lint */
1192
1193
1194#if defined (lint)
1195
1196/*ARGSUSED*/
1197pfn_t
1198sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
1199{ return(0); }
1200
1201#else /* lint */
1202
1203	ENTRY_NP(sfmmu_ttetopfn)
1204	ldx	[%o0], %g1			/* read tte */
1205	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1206	/*
1207	 * g1 = pfn
1208	 */
1209	retl
1210	mov	%g1, %o0
1211	SET_SIZE(sfmmu_ttetopfn)
1212
1213#endif /* !lint */
1214
1215
1216#if defined (lint)
1217/*
1218 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
1219 * the hash list.
1220 */
1221/* ARGSUSED */
1222void
1223sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1224	uint64_t hblkpa)
1225{
1226}
1227
1228/*
1229 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
1230 * hash list.
1231 */
1232/* ARGSUSED */
1233void
1234sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1235	uint64_t hblkpa, struct hme_blk *prev_hblkp)
1236{
1237}
1238#else /* lint */
1239
1240/*
1241 * Functions to grab/release hme bucket list lock.  I only use a byte
1242 * instead of the whole int because eventually we might want to
1243 * put some counters on the other bytes (of course, these routines would
1244 * have to change).  The code that grab this lock should execute
1245 * with interrupts disabled and hold the lock for the least amount of time
1246 * possible.
1247 */
1248
1249/*
1250 * Even though hmeh_listlock is updated using pa there's no need to flush
1251 * dcache since hmeh_listlock will be restored to the original value (0)
1252 * before interrupts are reenabled.
1253 */
1254
1255/*
1256 * For sparcv9 hme hash buckets may not be in the nucleus.  hme hash update
1257 * routines still use virtual addresses to update the bucket fields. But they
1258 * must not cause a TLB miss after grabbing the low level bucket lock. To
1259 * achieve this we must make sure the bucket structure is completely within an
1260 * 8K page.
1261 */
1262
1263#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1))
1264#error - the size of hmehash_bucket structure is not power of 2
1265#endif
1266
1267#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi)           \
1268	mov     0xff, tmp2                                      ;\
1269	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1270label1:                                                         ;\
1271	casa    [tmp1]asi, %g0, tmp2                            ;\
1272	brnz,pn tmp2, label1                                    ;\
1273	mov     0xff, tmp2                                      ;\
1274	membar  #LoadLoad
1275
1276#define HMELOCK_EXIT(hmebp, tmp1, asi)                          \
1277	membar  #LoadStore|#StoreStore                          ;\
1278	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1279	sta     %g0, [tmp1]asi
1280
1281	.seg	".data"
1282hblk_add_panic1:
1283	.ascii	"sfmmu_hblk_hash_add: interrupts disabled"
1284	.byte	0
1285hblk_add_panic2:
1286	.ascii	"sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
1287	.byte	0
1288	.align	4
1289	.seg	".text"
1290
1291	ENTRY_NP(sfmmu_hblk_hash_add)
1292	/*
1293	 * %o0 = hmebp
1294	 * %o1 = hmeblkp
1295	 * %o2 = hblkpa
1296	 */
1297	rdpr	%pstate, %o5
1298#ifdef DEBUG
1299	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1300	bnz,pt %icc, 3f				/* disabled, panic	 */
1301	  nop
1302	save	%sp, -SA(MINFRAME), %sp
1303	sethi	%hi(hblk_add_panic1), %o0
1304	call	panic
1305	 or	%o0, %lo(hblk_add_panic1), %o0
1306	ret
1307	restore
1308
13093:
1310#endif /* DEBUG */
1311	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1312	mov	%o2, %g1
1313
1314	/*
1315	 * g1 = hblkpa
1316	 */
1317	ldn	[%o0 + HMEBUCK_HBLK], %o4	/* next hmeblk */
1318	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = next hblkpa */
1319#ifdef	DEBUG
1320	cmp	%o4, %g0
1321	bne,pt %xcc, 1f
1322	 nop
1323	brz,pt %g2, 1f
1324	 nop
1325	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1326	save	%sp, -SA(MINFRAME), %sp
1327	sethi	%hi(hblk_add_panic2), %o0
1328	call	panic
1329	  or	%o0, %lo(hblk_add_panic2), %o0
1330	ret
1331	restore
13321:
1333#endif /* DEBUG */
1334	/*
1335	 * We update hmeblks entries before grabbing lock because the stores
1336	 * could take a tlb miss and require the hash lock.  The buckets
1337	 * are part of the nucleus so we are cool with those stores.
1338	 *
1339	 * if buckets are not part of the nucleus our game is to
1340	 * not touch any other page via va until we drop the lock.
1341	 * This guarantees we won't get a tlb miss before the lock release
1342	 * since interrupts are disabled.
1343	 */
1344	stn	%o4, [%o1 + HMEBLK_NEXT]	/* update hmeblk's next */
1345	stx	%g2, [%o1 + HMEBLK_NEXTPA]	/* update hmeblk's next pa */
1346	HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N)
1347	stn	%o1, [%o0 + HMEBUCK_HBLK]	/* update bucket hblk next */
1348	stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* add hmeblk to list */
1349	HMELOCK_EXIT(%o0, %g2, ASI_N)
1350	retl
1351	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1352	SET_SIZE(sfmmu_hblk_hash_add)
1353
1354	ENTRY_NP(sfmmu_hblk_hash_rm)
1355	/*
1356	 * This function removes an hmeblk from the hash chain.
1357	 * It is written to guarantee we don't take a tlb miss
1358	 * by using physical addresses to update the list.
1359	 *
1360	 * %o0 = hmebp
1361	 * %o1 = hmeblkp
1362	 * %o2 = hmeblkp previous pa
1363	 * %o3 = hmeblkp previous
1364	 */
1365
1366	mov	%o3, %o4			/* o4 = hmeblkp previous */
1367
1368	rdpr	%pstate, %o5
1369#ifdef DEBUG
1370	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1371	bnz,pt 	%icc, 3f			/* disabled, panic	 */
1372	  nop
1373
1374	sethi	%hi(panicstr), %g1
1375	ldx	[%g1 + %lo(panicstr)], %g1
1376	tst	%g1
1377	bnz,pt	%icc, 3f
1378	  nop
1379
1380	sethi	%hi(sfmmu_panic1), %o0
1381	call	panic
1382	 or	%o0, %lo(sfmmu_panic1), %o0
13833:
1384#endif /* DEBUG */
1385	/*
1386	 * disable interrupts, clear Address Mask to access 64 bit physaddr
1387	 */
1388	andn    %o5, PSTATE_IE, %g1
1389	wrpr    %g1, 0, %pstate
1390
1391#ifndef sun4v
1392	sethi   %hi(dcache_line_mask), %g4
1393	ld      [%g4 + %lo(dcache_line_mask)], %g4
1394#endif /* sun4v */
1395
1396	/*
1397	 * if buckets are not part of the nucleus our game is to
1398	 * not touch any other page via va until we drop the lock.
1399	 * This guarantees we won't get a tlb miss before the lock release
1400	 * since interrupts are disabled.
1401	 */
1402	HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N)
1403	ldn	[%o0 + HMEBUCK_HBLK], %g2	/* first hmeblk in list */
1404	cmp	%g2, %o1
1405	bne,pt	%ncc,1f
1406	 mov	ASI_MEM, %asi
1407	/*
1408	 * hmeblk is first on list
1409	 */
1410	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = hmeblk pa */
1411	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1412	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1413	stn	%o3, [%o0 + HMEBUCK_HBLK]	/* write va */
1414	ba,pt	%xcc, 2f
1415	  stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* write pa */
14161:
1417	/* hmeblk is not first on list */
1418
1419	mov	%o2, %g3
1420#ifndef sun4v
1421	GET_CPU_IMPL(%g2)
1422	cmp 	%g2, CHEETAH_IMPL
1423	bge,a,pt %icc, hblk_hash_rm_1
1424	  and	%o4, %g4, %g2
1425	cmp	%g2, SPITFIRE_IMPL
1426	blt	%icc, hblk_hash_rm_2		/* no flushing needed for OPL */
1427	  and	%o4, %g4, %g2
1428	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev pa from dcache */
1429	add	%o4, HMEBLK_NEXT, %o4
1430	and	%o4, %g4, %g2
1431	ba	hblk_hash_rm_2
1432	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev va from dcache */
1433hblk_hash_rm_1:
1434
1435	stxa	%g0, [%g3]ASI_DC_INVAL		/* flush prev pa from dcache */
1436	membar	#Sync
1437	add     %g3, HMEBLK_NEXT, %g2
1438	stxa	%g0, [%g2]ASI_DC_INVAL		/* flush prev va from dcache */
1439hblk_hash_rm_2:
1440	membar	#Sync
1441#endif /* sun4v */
1442	ldxa	[%g3 + HMEBLK_NEXTPA] %asi, %g2	/* g2 = hmeblk pa */
1443	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1444	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1445	stna	%o3, [%g3 + HMEBLK_NEXT] %asi	/* write va */
1446	stxa	%g1, [%g3 + HMEBLK_NEXTPA] %asi	/* write pa */
14472:
1448	HMELOCK_EXIT(%o0, %g2, ASI_N)
1449	retl
1450	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1451	SET_SIZE(sfmmu_hblk_hash_rm)
1452
1453#endif /* lint */
1454
1455/*
1456 * These macros are used to update global sfmmu hme hash statistics
1457 * in perf critical paths. It is only enabled in debug kernels or
1458 * if SFMMU_STAT_GATHER is defined
1459 */
1460#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1461#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1462	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1463	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
1464	cmp	tmp1, hatid						;\
1465	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
1466	set	sfmmu_global_stat, tmp1					;\
1467	add	tmp1, tmp2, tmp1					;\
1468	ld	[tmp1], tmp2						;\
1469	inc	tmp2							;\
1470	st	tmp2, [tmp1]
1471
1472#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1473	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1474	mov	HATSTAT_KHASH_LINKS, tmp2				;\
1475	cmp	tmp1, hatid						;\
1476	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
1477	set	sfmmu_global_stat, tmp1					;\
1478	add	tmp1, tmp2, tmp1					;\
1479	ld	[tmp1], tmp2						;\
1480	inc	tmp2							;\
1481	st	tmp2, [tmp1]
1482
1483
1484#else /* DEBUG || SFMMU_STAT_GATHER */
1485
1486#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1487
1488#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1489
1490#endif  /* DEBUG || SFMMU_STAT_GATHER */
1491
1492/*
1493 * This macro is used to update global sfmmu kstas in non
1494 * perf critical areas so they are enabled all the time
1495 */
1496#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
1497	sethi	%hi(sfmmu_global_stat), tmp1				;\
1498	add	tmp1, statname, tmp1					;\
1499	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
1500	inc	tmp2							;\
1501	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1502
1503/*
1504 * These macros are used to update per cpu stats in non perf
1505 * critical areas so they are enabled all the time
1506 */
1507#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
1508	ld	[tsbarea + stat], tmp1					;\
1509	inc	tmp1							;\
1510	st	tmp1, [tsbarea + stat]
1511
1512/*
1513 * These macros are used to update per cpu stats in non perf
1514 * critical areas so they are enabled all the time
1515 */
1516#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
1517	lduh	[tsbarea + stat], tmp1					;\
1518	inc	tmp1							;\
1519	stuh	tmp1, [tsbarea + stat]
1520
1521#if defined(KPM_TLBMISS_STATS_GATHER)
1522	/*
1523	 * Count kpm dtlb misses separately to allow a different
1524	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1525	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1526	 */
1527#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
1528	brgez	tagacc, label	/* KPM VA? */				;\
1529	nop								;\
1530	CPU_INDEX(tmp1, tsbma)						;\
1531	sethi	%hi(kpmtsbm_area), tsbma				;\
1532	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
1533	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
1534	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
1535	/* VA range check */						;\
1536	ldx	[tsbma + KPMTSBM_VBASE], val				;\
1537	cmp	tagacc, val						;\
1538	blu,pn	%xcc, label						;\
1539	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
1540	cmp	tagacc, tmp1						;\
1541	bgeu,pn	%xcc, label						;\
1542	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
1543	inc	val							;\
1544	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
1545label:
1546#else
1547#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1548#endif	/* KPM_TLBMISS_STATS_GATHER */
1549
1550#if defined (lint)
1551/*
1552 * The following routines are jumped to from the mmu trap handlers to do
1553 * the setting up to call systrap.  They are separate routines instead of
1554 * being part of the handlers because the handlers would exceed 32
1555 * instructions and since this is part of the slow path the jump
1556 * cost is irrelevant.
1557 */
1558void
1559sfmmu_pagefault(void)
1560{
1561}
1562
1563void
1564sfmmu_mmu_trap(void)
1565{
1566}
1567
1568void
1569sfmmu_window_trap(void)
1570{
1571}
1572
1573void
1574sfmmu_kpm_exception(void)
1575{
1576}
1577
1578#else /* lint */
1579
1580#ifdef	PTL1_PANIC_DEBUG
1581	.seg	".data"
1582	.global	test_ptl1_panic
1583test_ptl1_panic:
1584	.word	0
1585	.align	8
1586
1587	.seg	".text"
1588	.align	4
1589#endif	/* PTL1_PANIC_DEBUG */
1590
1591
1592	ENTRY_NP(sfmmu_pagefault)
1593	SET_GL_REG(1)
1594	USE_ALTERNATE_GLOBALS(%g5)
1595	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1596	rdpr	%tt, %g6
1597	cmp	%g6, FAST_IMMU_MISS_TT
1598	be,a,pn	%icc, 1f
1599	  mov	T_INSTR_MMU_MISS, %g3
1600	cmp	%g6, T_INSTR_MMU_MISS
1601	be,a,pn	%icc, 1f
1602	  mov	T_INSTR_MMU_MISS, %g3
1603	mov	%g5, %g2
1604	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1605	cmp	%g6, FAST_DMMU_MISS_TT
1606	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1607	cmp	%g6, T_DATA_MMU_MISS
1608	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1609
1610#ifdef  PTL1_PANIC_DEBUG
1611	/* check if we want to test the tl1 panic */
1612	sethi	%hi(test_ptl1_panic), %g4
1613	ld	[%g4 + %lo(test_ptl1_panic)], %g1
1614	st	%g0, [%g4 + %lo(test_ptl1_panic)]
1615	cmp	%g1, %g0
1616	bne,a,pn %icc, ptl1_panic
1617	  or	%g0, PTL1_BAD_DEBUG, %g1
1618#endif	/* PTL1_PANIC_DEBUG */
16191:
1620	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
1621	/*
1622	 * g2 = tag access reg
1623	 * g3.l = type
1624	 * g3.h = 0
1625	 */
1626	sethi	%hi(trap), %g1
1627	or	%g1, %lo(trap), %g1
16282:
1629	ba,pt	%xcc, sys_trap
1630	  mov	-1, %g4
1631	SET_SIZE(sfmmu_pagefault)
1632
1633	ENTRY_NP(sfmmu_mmu_trap)
1634	SET_GL_REG(1)
1635	USE_ALTERNATE_GLOBALS(%g5)
1636	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
1637	rdpr	%tt, %g6
1638	cmp	%g6, FAST_IMMU_MISS_TT
1639	be,a,pn	%icc, 1f
1640	  mov	T_INSTR_MMU_MISS, %g3
1641	cmp	%g6, T_INSTR_MMU_MISS
1642	be,a,pn	%icc, 1f
1643	  mov	T_INSTR_MMU_MISS, %g3
1644	mov	%g5, %g2
1645	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1646	cmp	%g6, FAST_DMMU_MISS_TT
1647	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1648	cmp	%g6, T_DATA_MMU_MISS
1649	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
16501:
1651	/*
1652	 * g2 = tag access reg
1653	 * g3 = type
1654	 */
1655	sethi	%hi(sfmmu_tsbmiss_exception), %g1
1656	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
1657	ba,pt	%xcc, sys_trap
1658	  mov	-1, %g4
1659	/*NOTREACHED*/
1660	SET_SIZE(sfmmu_mmu_trap)
1661
1662	ENTRY_NP(sfmmu_suspend_tl)
1663	SET_GL_REG(1)
1664	USE_ALTERNATE_GLOBALS(%g5)
1665	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
1666	rdpr	%tt, %g6
1667	cmp	%g6, FAST_IMMU_MISS_TT
1668	be,a,pn	%icc, 1f
1669	  mov	T_INSTR_MMU_MISS, %g3
1670	mov	%g5, %g2
1671	cmp	%g6, FAST_DMMU_MISS_TT
1672	move	%icc, T_DATA_MMU_MISS, %g3
1673	movne	%icc, T_DATA_PROT, %g3
16741:
1675	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
1676	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
1677	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
1678	ba,pt	%xcc, sys_trap
1679	  mov	PIL_15, %g4
1680	/*NOTREACHED*/
1681	SET_SIZE(sfmmu_suspend_tl)
1682
1683	/*
1684	 * No %g registers in use at this point.
1685	 */
1686	ENTRY_NP(sfmmu_window_trap)
1687	rdpr	%tpc, %g1
1688#ifdef sun4v
1689#ifdef DEBUG
1690	/* We assume previous %gl was 1 */
1691	rdpr	%tstate, %g4
1692	srlx	%g4, TSTATE_GL_SHIFT, %g4
1693	and	%g4, TSTATE_GL_MASK, %g4
1694	cmp	%g4, 1
1695	bne,a,pn %icc, ptl1_panic
1696	  mov	PTL1_BAD_WTRAP, %g1
1697#endif /* DEBUG */
1698	/* user miss at tl>1. better be the window handler or user_rtt */
1699	/* in user_rtt? */
1700	set	rtt_fill_start, %g4
1701	cmp	%g1, %g4
1702	blu,pn %xcc, 6f
1703	 .empty
1704	set	rtt_fill_end, %g4
1705	cmp	%g1, %g4
1706	bgeu,pn %xcc, 6f
1707	 nop
1708	set	fault_rtt_fn1, %g1
1709	wrpr	%g0, %g1, %tnpc
1710	ba,a	7f
17116:
1712	! must save this trap level before descending trap stack
1713	! no need to save %tnpc, either overwritten or discarded
1714	! already got it: rdpr	%tpc, %g1
1715	rdpr	%tstate, %g6
1716	rdpr	%tt, %g7
1717	! trap level saved, go get underlying trap type
1718	rdpr	%tl, %g5
1719	sub	%g5, 1, %g3
1720	wrpr	%g3, %tl
1721	rdpr	%tt, %g2
1722	wrpr	%g5, %tl
1723	! restore saved trap level
1724	wrpr	%g1, %tpc
1725	wrpr	%g6, %tstate
1726	wrpr	%g7, %tt
1727#else /* sun4v */
1728	/* user miss at tl>1. better be the window handler */
1729	rdpr	%tl, %g5
1730	sub	%g5, 1, %g3
1731	wrpr	%g3, %tl
1732	rdpr	%tt, %g2
1733	wrpr	%g5, %tl
1734#endif /* sun4v */
1735	and	%g2, WTRAP_TTMASK, %g4
1736	cmp	%g4, WTRAP_TYPE
1737	bne,pn	%xcc, 1f
1738	 nop
1739	/* tpc should be in the trap table */
1740	set	trap_table, %g4
1741	cmp	%g1, %g4
1742	blt,pn %xcc, 1f
1743	 .empty
1744	set	etrap_table, %g4
1745	cmp	%g1, %g4
1746	bge,pn %xcc, 1f
1747	 .empty
1748	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
1749	add	%g1, WTRAP_FAULTOFF, %g1
1750	wrpr	%g0, %g1, %tnpc
17517:
1752	/*
1753	 * some wbuf handlers will call systrap to resolve the fault
1754	 * we pass the trap type so they figure out the correct parameters.
1755	 * g5 = trap type, g6 = tag access reg
1756	 */
1757
1758	/*
1759	 * only use g5, g6, g7 registers after we have switched to alternate
1760	 * globals.
1761	 */
1762	SET_GL_REG(1)
1763	USE_ALTERNATE_GLOBALS(%g5)
1764	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
1765	rdpr	%tt, %g7
1766	cmp	%g7, FAST_IMMU_MISS_TT
1767	be,a,pn	%icc, ptl1_panic
1768	  mov	PTL1_BAD_WTRAP, %g1
1769	cmp	%g7, T_INSTR_MMU_MISS
1770	be,a,pn	%icc, ptl1_panic
1771	  mov	PTL1_BAD_WTRAP, %g1
1772	mov	T_DATA_PROT, %g5
1773	cmp	%g7, FAST_DMMU_MISS_TT
1774	move	%icc, T_DATA_MMU_MISS, %g5
1775	cmp	%g7, T_DATA_MMU_MISS
1776	move	%icc, T_DATA_MMU_MISS, %g5
1777	! XXXQ AGS re-check out this one
1778	done
17791:
1780	CPU_ADDR(%g1, %g4)
1781	ld	[%g1 + CPU_TL1_HDLR], %g4
1782	brnz,a,pt %g4, sfmmu_mmu_trap
1783	  st	%g0, [%g1 + CPU_TL1_HDLR]
1784	ba,pt	%icc, ptl1_panic
1785	  mov	PTL1_BAD_TRAP, %g1
1786	SET_SIZE(sfmmu_window_trap)
1787
1788	ENTRY_NP(sfmmu_kpm_exception)
1789	/*
1790	 * We have accessed an unmapped segkpm address or a legal segkpm
1791	 * address which is involved in a VAC alias conflict prevention.
1792	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
1793	 * set. If it is, we will instead note that a fault has occurred
1794	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
1795	 * a "retry"). This will step over the faulting instruction.
1796	 * Note that this means that a legal segkpm address involved in
1797	 * a VAC alias conflict prevention (a rare case to begin with)
1798	 * cannot be used in DTrace.
1799	 */
1800	CPU_INDEX(%g1, %g2)
1801	set	cpu_core, %g2
1802	sllx	%g1, CPU_CORE_SHIFT, %g1
1803	add	%g1, %g2, %g1
1804	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
1805	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
1806	bz	0f
1807	or	%g2, CPU_DTRACE_BADADDR, %g2
1808	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
1809	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
1810	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
1811	done
18120:
1813	TSTAT_CHECK_TL1(1f, %g1, %g2)
18141:
1815	SET_GL_REG(1)
1816	USE_ALTERNATE_GLOBALS(%g5)
1817	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
1818	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1819	/*
1820	 * g2=tagacc g3.l=type g3.h=0
1821	 */
1822	sethi	%hi(trap), %g1
1823	or	%g1, %lo(trap), %g1
1824	ba,pt	%xcc, sys_trap
1825	mov	-1, %g4
1826	SET_SIZE(sfmmu_kpm_exception)
1827
1828#endif /* lint */
1829
1830#if defined (lint)
1831
1832void
1833sfmmu_tsb_miss(void)
1834{
1835}
1836
1837void
1838sfmmu_kpm_dtsb_miss(void)
1839{
1840}
1841
1842void
1843sfmmu_kpm_dtsb_miss_small(void)
1844{
1845}
1846
1847#else /* lint */
1848
1849
1850#if (CTX_SIZE != (1 << CTX_SZ_SHIFT))
1851#error - size of context struct does not match with CTX_SZ_SHIFT
1852#endif
1853
1854#if (IMAP_SEG != 0)
1855#error - ism_map->ism_seg offset is not zero
1856#endif
1857
1858/*
1859 * Copies ism mapping for this ctx in param "ism" if this is a ISM
1860 * tlb miss and branches to label "ismhit". If this is not an ISM
1861 * process or an ISM tlb miss it falls thru.
1862 *
1863 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
1864 * this process.
1865 * If so, it will branch to label "ismhit".  If not, it will fall through.
1866 *
1867 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
1868 * so that any other threads of this process will not try and walk the ism
1869 * maps while they are being changed.
1870 *
1871 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
1872 *       will make sure of that. This means we can terminate our search on
1873 *       the first zero mapping we find.
1874 *
1875 * Parameters:
1876 * tagacc	= tag access register (vaddr + ctx) (in)
1877 * tsbmiss	= address of tsb miss area (in)
1878 * ismseg	= contents of ism_seg for this ism map (out)
1879 * ismhat	= physical address of imap_ismhat for this ism map (out)
1880 * tmp1		= scratch reg (CLOBBERED)
1881 * tmp2		= scratch reg (CLOBBERED)
1882 * tmp3		= scratch reg (CLOBBERED)
1883 * label:    temporary labels
1884 * ismhit:   label where to jump to if an ism dtlb miss
1885 * exitlabel:label where to jump if hat is busy due to hat_unshare.
1886 */
1887#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
1888	label, ismhit)							\
1889	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
1890	brlz,pt  tmp1, label/**/3		/* exit if -1 */	;\
1891	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
1892label/**/1:								;\
1893	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
1894	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
1895label/**/2:								;\
1896	brz,pt  ismseg, label/**/3		/* no mapping */	;\
1897	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
1898	lduha	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
1899	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
1900	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
1901	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
1902	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
1903	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
1904	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
1905	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
1906	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
1907	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
1908									;\
1909	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
1910	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
1911	cmp	ismhat, tmp1						;\
1912	bl,pt	%xcc, label/**/2		/* keep looking  */	;\
1913	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
1914									;\
1915	add	tmp3, IBLK_NEXTPA, tmp1					;\
1916	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
1917	brgez,pt tmp1, label/**/1		/* continue if not -1*/	;\
1918	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
1919label/**/3:
1920
1921/*
1922 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
1923 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
1924 * Parameters:
1925 * vaddr = reg containing virtual address
1926 * hatid = reg containing sfmmu pointer
1927 * hmeshift = constant/register to shift vaddr to obtain vapg
1928 * hmebp = register where bucket pointer will be stored
1929 * vapg = register where virtual page will be stored
1930 * tmp1, tmp2 = tmp registers
1931 */
1932
1933
1934#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
1935	vapg, label, tmp1, tmp2)					\
1936	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
1937	brnz,a,pt tmp1, label/**/1					;\
1938	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
1939	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
1940	ba,pt	%xcc, label/**/2					;\
1941	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
1942label/**/1:								;\
1943	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
1944label/**/2:								;\
1945	srlx	tagacc, hmeshift, vapg					;\
1946	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
1947	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
1948	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
1949	add	hmebp, tmp1, hmebp
1950
1951/*
1952 * hashtag includes bspage + hashno (64 bits).
1953 */
1954
1955#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
1956	sllx	vapg, hmeshift, vapg					;\
1957	or	vapg, hashno, hblktag
1958
1959/*
1960 * Function to traverse hmeblk hash link list and find corresponding match.
1961 * The search is done using physical pointers. It returns the physical address
1962 * and virtual address pointers to the hmeblk that matches with the tag
1963 * provided.
1964 * Parameters:
1965 * hmebp	= register that points to hme hash bucket, also used as
1966 *		  tmp reg (clobbered)
1967 * hmeblktag	= register with hmeblk tag match
1968 * hatid	= register with hatid
1969 * hmeblkpa	= register where physical ptr will be stored
1970 * hmeblkva	= register where virtual ptr will be stored
1971 * tmp1		= tmp reg
1972 * label: temporary label
1973 */
1974
1975#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva,	\
1976	tsbarea, tmp1, label)					 	\
1977	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
1978	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
1979	add     hmebp, HMEBUCK_HBLK, hmeblkva				;\
1980	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
1981	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1982label/**/1:								;\
1983	brz,pn	hmeblkva, label/**/2					;\
1984	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1985	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
1986	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
1987	add	hmebp, CLONGSIZE, hmebp					;\
1988	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
1989	xor	tmp1, hmeblktag, tmp1					;\
1990	xor	hmebp, hatid, hmebp					;\
1991	or	hmebp, tmp1, hmebp					;\
1992	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
1993	  add	hmeblkpa, HMEBLK_NEXT, hmebp				;\
1994	ldna	[hmebp]ASI_MEM, hmeblkva	/* hmeblk ptr va */	;\
1995	add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
1996	ba,pt	%xcc, label/**/1					;\
1997	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
1998label/**/2:
1999
2000
2001#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
2002#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
2003#endif
2004
2005/*
2006 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2007 * he offset for the corresponding hment.
2008 * Parameters:
2009 * vaddr = register with virtual address
2010 * hmeblkpa = physical pointer to hme_blk
2011 * hment = register where address of hment will be stored
2012 * hmentoff = register where hment offset will be stored
2013 * label1 = temporary label
2014 */
2015#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, tmp1, label1)	\
2016	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
2017	lda	[hmentoff]ASI_MEM, tmp1 				;\
2018	andcc	tmp1, HBLK_SZMASK, %g0	 /* tmp1 = get_hblk_sz(%g5) */	;\
2019	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
2020	  or	%g0, HMEBLK_HME1, hmentoff				;\
2021	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
2022	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
2023	sllx	tmp1, SFHME_SHIFT, tmp1					;\
2024	add	tmp1, HMEBLK_HME1, hmentoff				;\
2025label1:
2026
2027/*
2028 * GET_TTE is a macro that returns a TTE given a tag and hatid.
2029 *
2030 * tagacc	= tag access register (vaddr + ctx) (in)
2031 * hatid	= sfmmu pointer for TSB miss (in)
2032 * tte		= tte for TLB miss if found, otherwise clobbered (out)
2033 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
2034 * hmeblkva	= VA of hment if found, otherwise clobbered (out)
2035 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
2036 * hmentoff	= temporarily stores hment offset (clobbered)
2037 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
2038 *		  for this page size.
2039 * hashno	= constant/register hash number
2040 * label	= temporary label for branching within macro.
2041 * foundlabel	= label to jump to when tte is found.
2042 * suspendlabel= label to jump to when tte is suspended.
2043 * exitlabel	= label to jump to when tte is not found.  The hmebp lock
2044 *		  is still held at this time.
2045 *
2046 * The caller should set up the tsbmiss->scratch[2] field correctly before
2047 * calling this funciton  (aka TSBMISS_SCRATCH + TSBMISS_HATID)
2048 */
2049#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmentoff, \
2050		hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \
2051									;\
2052	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2053	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2054	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2055		hmeblkpa, label/**/5, hmentoff, hmeblkva)		;\
2056									;\
2057	/*								;\
2058	 * tagacc = tagacc						;\
2059	 * hatid = hatid						;\
2060	 * tsbarea = tsbarea						;\
2061	 * tte   = hmebp (hme bucket pointer)				;\
2062	 * hmeblkpa  = vapg  (virtual page)				;\
2063	 * hmentoff, hmeblkva = scratch					;\
2064	 */								;\
2065	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmentoff)	;\
2066									;\
2067	/*								;\
2068	 * tagacc = tagacc						;\
2069	 * hatid = hatid						;\
2070	 * tte   = hmebp						;\
2071	 * hmeblkpa  = CLOBBERED					;\
2072	 * hmentoff  = htag_bspage & hashno				;\
2073	 * hmeblkva  = scratch						;\
2074	 */								;\
2075	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2076	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
2077	HMEHASH_SEARCH(tte, hmentoff, hatid, hmeblkpa, hmeblkva, 	\
2078		tsbarea, tagacc, label/**/1)				;\
2079	/*								;\
2080	 * tagacc = CLOBBERED						;\
2081	 * tte = CLOBBERED						;\
2082	 * hmeblkpa = hmeblkpa						;\
2083	 * hmeblkva = hmeblkva						;\
2084	 */								;\
2085	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2086	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2087	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2088	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2089	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2090	  nop								;\
2091label/**/4:								;\
2092	/*								;\
2093	 * We have found the hmeblk containing the hment.		;\
2094	 * Now we calculate the corresponding tte.			;\
2095	 *								;\
2096	 * tagacc = tagacc						;\
2097	 * hatid = clobbered						;\
2098	 * tte   = hmebp						;\
2099	 * hmeblkpa  = hmeblkpa						;\
2100	 * hmentoff  = hblktag						;\
2101	 * hmeblkva  = hmeblkva 					;\
2102	 */								;\
2103	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hmentoff, hatid, label/**/2)	;\
2104									;\
2105	add	hmentoff, SFHME_TTE, hmentoff				;\
2106	add	hmeblkpa, hmentoff, hmeblkpa				;\
2107	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2108	add	hmeblkva, hmentoff, hmeblkva				;\
2109	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2110	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmentoff ;\
2111	HMELOCK_EXIT(hmentoff, hmentoff, ASI_MEM)	/* drop lock */	;\
2112	set	TTE_SUSPEND, hmentoff					;\
2113	TTE_SUSPEND_INT_SHIFT(hmentoff)					;\
2114	btst	tte, hmentoff						;\
2115	bz,pt	%xcc, foundlabel					;\
2116	  nop								;\
2117									;\
2118	/*								;\
2119	 * Mapping is suspended, so goto suspend label.			;\
2120	 */								;\
2121	ba,pt	%xcc, suspendlabel					;\
2122	  nop
2123
2124	/*
2125	 * KERNEL PROTECTION HANDLER
2126	 *
2127	 * g1 = tsb8k pointer register (clobbered)
2128	 * g2 = tag access register (ro)
2129	 * g3 - g7 = scratch registers
2130	 *
2131	 * Note: This function is patched at runtime for performance reasons.
2132	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
2133	 */
2134	ENTRY_NP(sfmmu_kprot_trap)
2135	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2136sfmmu_kprot_patch_ktsb_base:
2137	RUNTIME_PATCH_SETX(%g1, %g6)
2138	/* %g1 = contents of ktsb_base or ktsb_pbase */
2139sfmmu_kprot_patch_ktsb_szcode:
2140	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
2141
2142	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2143	! %g1 = First TSB entry pointer, as TSB miss handler expects
2144
2145	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2146sfmmu_kprot_patch_ktsb4m_base:
2147	RUNTIME_PATCH_SETX(%g3, %g6)
2148	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2149sfmmu_kprot_patch_ktsb4m_szcode:
2150	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
2151
2152	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2153	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2154
2155	CPU_TSBMISS_AREA(%g6, %g7)
2156	HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2157	ba,pt	%xcc, sfmmu_tsb_miss_tt
2158	  nop
2159
2160	/*
2161	 * USER PROTECTION HANDLER
2162	 *
2163	 * g1 = tsb8k pointer register (ro)
2164	 * g2 = tag access register (ro)
2165	 * g3 = faulting context (clobbered, currently not used)
2166	 * g4 - g7 = scratch registers
2167	 */
2168	ALTENTRY(sfmmu_uprot_trap)
2169#ifdef sun4v
2170	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2171	/* %g1 = first TSB entry ptr now, %g2 preserved */
2172
2173	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
2174	brlz,pt %g3, 9f			/* check for 2nd TSB */
2175	  mov	%g0, %g3		/* clear second tsbe ptr */
2176
2177	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2178	/* %g3 = second TSB entry ptr now, %g2 preserved */
2179
2180#else /* sun4v */
2181#ifdef UTSB_PHYS
2182	/* g1 = first TSB entry ptr */
2183	GET_2ND_TSBREG(%g3)
2184	brlz,a,pt %g3, 9f		/* check for 2nd TSB */
2185	  mov	%g0, %g3		/* clear second tsbe ptr */
2186
2187	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2188	/* %g3 = second TSB entry ptr now, %g2 preserved */
2189#else /* UTSB_PHYS */
2190	brgez,pt %g1, 9f		/* check for 2nd TSB */
2191	  mov	%g0, %g3		/* clear second tsbe ptr */
2192
2193	mov	%g2, %g7
2194	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2195	/* %g3 = second TSB entry ptr now, %g7 clobbered */
2196	mov	%g1, %g7
2197	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2198#endif /* UTSB_PHYS */
2199#endif /* sun4v */
22009:
2201	CPU_TSBMISS_AREA(%g6, %g7)
2202	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2203	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
2204	  nop
2205
2206	/*
2207	 * Kernel 8K page iTLB miss.  We also get here if we took a
2208	 * fast instruction access mmu miss trap while running in
2209	 * invalid context.
2210	 *
2211	 * %g1 = 8K TSB pointer register (not used, clobbered)
2212	 * %g2 = tag access register (used)
2213	 * %g3 = faulting context id (used)
2214	 * %g7 = 4M virtual page number for tag matching  (used)
2215	 */
2216	.align	64
2217	ALTENTRY(sfmmu_kitlb_miss)
2218	brnz,pn %g3, tsb_tl0_noctxt
2219	  nop
2220
2221	/* kernel miss */
2222	/* get kernel tsb pointer */
2223	/* we patch the next set of instructions at run time */
2224	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2225iktsbbase:
2226	RUNTIME_PATCH_SETX(%g4, %g5)
2227	/* %g4 = contents of ktsb_base or ktsb_pbase */
2228
2229iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2230	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2231	or	%g4, %g1, %g1			! form tsb ptr
2232	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2233	cmp	%g4, %g7
2234	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
2235	  andcc %g5, TTE_EXECPRM_INT, %g0	! check exec bit
2236	bz,pn	%icc, exec_fault
2237	  nop
2238	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2239	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2240	retry
2241
2242	/*
2243	 * Kernel dTLB miss.  We also get here if we took a fast data
2244	 * access mmu miss trap while running in invalid context.
2245	 *
2246	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2247	 *	We select the TSB miss handler to branch to depending on
2248	 *	the virtual address of the access.  In the future it may
2249	 *	be desirable to separate kpm TTEs into their own TSB,
2250	 *	in which case all that needs to be done is to set
2251	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2252	 *	early in the miss if we detect a kpm VA to a new handler.
2253	 *
2254	 * %g1 = 8K TSB pointer register (not used, clobbered)
2255	 * %g2 = tag access register (used)
2256	 * %g3 = faulting context id (used)
2257	 */
2258	.align	64
2259	ALTENTRY(sfmmu_kdtlb_miss)
2260	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
2261	  nop
2262
2263	/* Gather some stats for kpm misses in the TLB. */
2264	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2265	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2266
2267	/*
2268	 * Get first TSB offset and look for 8K/64K/512K mapping
2269	 * using the 8K virtual page as the index.
2270	 *
2271	 * We patch the next set of instructions at run time;
2272	 * any changes here require sfmmu_patch_ktsb changes too.
2273	 */
2274dktsbbase:
2275	RUNTIME_PATCH_SETX(%g7, %g6)
2276	/* %g7 = contents of ktsb_base or ktsb_pbase */
2277
2278dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2279	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2280
2281	/*
2282	 * At this point %g1 is our index into the TSB.
2283	 * We just masked off enough bits of the VA depending
2284	 * on our TSB size code.
2285	 */
2286	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2287	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2288	cmp	%g6, %g4			! compare tag
2289	bne,pn	%xcc, dktsb4m_kpmcheck_small
2290	  add	%g7, %g1, %g1			/* form tsb ptr */
2291	TT_TRACE(trace_tsbhit)
2292	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2293	/* trapstat expects tte in %g5 */
2294	retry
2295
2296	/*
2297	 * If kpm is using large pages, the following instruction needs
2298	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2299	 * so that we will probe the 4M TSB regardless of the VA.  In
2300	 * the case kpm is using small pages, we know no large kernel
2301	 * mappings are located above 0x80000000.00000000 so we skip the
2302	 * probe as an optimization.
2303	 */
2304dktsb4m_kpmcheck_small:
2305	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2306	  /* delay slot safe, below */
2307
2308	/*
2309	 * Get second TSB offset and look for 4M mapping
2310	 * using 4M virtual page as the TSB index.
2311	 *
2312	 * Here:
2313	 * %g1 = 8K TSB pointer.  Don't squash it.
2314	 * %g2 = tag access register (we still need it)
2315	 */
2316	srlx	%g2, MMU_PAGESHIFT4M, %g3
2317
2318	/*
2319	 * We patch the next set of instructions at run time;
2320	 * any changes here require sfmmu_patch_ktsb changes too.
2321	 */
2322dktsb4mbase:
2323	RUNTIME_PATCH_SETX(%g7, %g6)
2324	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2325dktsb4m:
2326	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2327	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2328
2329	/*
2330	 * At this point %g3 is our index into the TSB.
2331	 * We just masked off enough bits of the VA depending
2332	 * on our TSB size code.
2333	 */
2334	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2335	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2336	cmp	%g6, %g4			! compare tag
2337
2338dktsb4m_tsbmiss:
2339	bne,pn	%xcc, dktsb4m_kpmcheck
2340	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
2341	TT_TRACE(trace_tsbhit)
2342	/* we don't check TTE size here since we assume 4M TSB is separate */
2343	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2344	/* trapstat expects tte in %g5 */
2345	retry
2346
2347	/*
2348	 * So, we failed to find a valid TTE to match the faulting
2349	 * address in either TSB.  There are a few cases that could land
2350	 * us here:
2351	 *
2352	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2353	 *    to sfmmu_tsb_miss_tt to handle the miss.
2354	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2355	 *    4M TSB.  Let segkpm handle it.
2356	 *
2357	 * Note that we shouldn't land here in the case of a kpm VA when
2358	 * kpm_smallpages is active -- we handled that case earlier at
2359	 * dktsb4m_kpmcheck_small.
2360	 *
2361	 * At this point:
2362	 *  g1 = 8K-indexed primary TSB pointer
2363	 *  g2 = tag access register
2364	 *  g3 = 4M-indexed secondary TSB pointer
2365	 */
2366dktsb4m_kpmcheck:
2367	cmp	%g2, %g0
2368	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
2369	  nop
2370	ba,a,pt	%icc, sfmmu_tsb_miss_tt
2371	  nop
2372
2373#ifdef sun4v
2374	/*
2375	 * User instruction miss w/ single TSB.
2376	 * The first probe covers 8K, 64K, and 512K page sizes,
2377	 * because 64K and 512K mappings are replicated off 8K
2378	 * pointer.
2379	 *
2380	 * g1 = tsb8k pointer register
2381	 * g2 = tag access register
2382	 * g3 - g6 = scratch registers
2383	 * g7 = TSB tag to match
2384	 */
2385	.align	64
2386	ALTENTRY(sfmmu_uitlb_fastpath)
2387
2388	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2389	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2390	ba,pn	%xcc, sfmmu_tsb_miss_tt
2391	  mov	%g0, %g3
2392
2393	/*
2394	 * User data miss w/ single TSB.
2395	 * The first probe covers 8K, 64K, and 512K page sizes,
2396	 * because 64K and 512K mappings are replicated off 8K
2397	 * pointer.
2398	 *
2399	 * g1 = tsb8k pointer register
2400	 * g2 = tag access register
2401	 * g3 - g6 = scratch registers
2402	 * g7 = TSB tag to match
2403	 */
2404	.align 64
2405	ALTENTRY(sfmmu_udtlb_fastpath)
2406
2407	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2408	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2409	ba,pn	%xcc, sfmmu_tsb_miss_tt
2410	  mov	%g0, %g3
2411
2412	/*
2413	 * User instruction miss w/ multiple TSBs (sun4v).
2414	 * The first probe covers 8K, 64K, and 512K page sizes,
2415	 * because 64K and 512K mappings are replicated off 8K
2416	 * pointer.  Second probe covers 4M page size only.
2417	 *
2418	 * Just like sfmmu_udtlb_slowpath, except:
2419	 *   o Uses ASI_ITLB_IN
2420	 *   o checks for execute permission
2421	 *   o No ISM prediction.
2422	 *
2423	 * g1 = tsb8k pointer register
2424	 * g2 = tag access register
2425	 * g3 - g6 = scratch registers
2426	 * g7 = TSB tag to match
2427	 */
2428	.align	64
2429	ALTENTRY(sfmmu_uitlb_slowpath)
2430
2431	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2432	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2433	/* g4 - g5 = clobbered here */
2434
2435	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2436	/* g1 = first TSB pointer, g3 = second TSB pointer */
2437	srlx	%g2, TAG_VALO_SHIFT, %g7
2438	PROBE_2ND_ITSB(%g3, %g7)
2439	/* NOT REACHED */
2440
2441#else /* sun4v */
2442
2443	/*
2444	 * User instruction miss w/ multiple TSBs (sun4u).
2445	 * The first probe covers 8K, 64K, and 512K page sizes,
2446	 * because 64K and 512K mappings are replicated off 8K
2447	 * pointer.  Second probe covers 4M page size only.
2448	 *
2449	 * Just like sfmmu_udtlb_slowpath, except:
2450	 *   o Uses ASI_ITLB_IN
2451	 *   o checks for execute permission
2452	 *   o No ISM prediction.
2453	 *
2454	 * g1 = tsb8k pointer register
2455	 * g2 = tag access register
2456	 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch
2457	 * g4 - g6 = scratch registers
2458	 * g7 = TSB tag to match
2459	 */
2460	.align	64
2461	ALTENTRY(sfmmu_uitlb_slowpath)
2462
2463#ifdef UTSB_PHYS
2464	/*
2465	 * g1 = 1st TSB entry pointer
2466	 * g3 = 2nd TSB base register
2467	 * Need 2nd TSB entry pointer for 2nd probe.
2468	 */
2469	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2470
2471	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2472#else /* UTSB_PHYS */
2473	mov	%g1, %g3	/* save tsb8k reg in %g3 */
2474	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
2475	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2476
2477	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
2478	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
2479	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
2480#endif /* UTSB_PHYS */
2481	/* g1 = first TSB pointer, g3 = second TSB pointer */
2482	srlx	%g2, TAG_VALO_SHIFT, %g7
2483	PROBE_2ND_ITSB(%g3, %g7, isynth)
2484	/* NOT REACHED */
2485#endif /* sun4v */
2486
2487	/*
2488	 * User data miss w/ multiple TSBs.
2489	 * The first probe covers 8K, 64K, and 512K page sizes,
2490	 * because 64K and 512K mappings are replicated off 8K
2491	 * pointer.  Second probe covers 4M page size only.
2492	 *
2493	 * We consider probing for 4M pages first if the VA falls
2494	 * in a range that's likely to be ISM.
2495	 *
2496	 * g1 = tsb8k pointer register
2497	 * g2 = tag access register
2498	 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch
2499	 * g4 - g6 = scratch registers
2500	 * g7 = TSB tag to match
2501	 */
2502	.align 64
2503	ALTENTRY(sfmmu_udtlb_slowpath)
2504
2505	/*
2506	 * Check for ISM.  If it exists, look for 4M mappings in the second TSB
2507	 * first, then probe for other mappings in the first TSB if that fails.
2508	 */
2509	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
2510	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
2511	  mov	%g1, %g3
2512
2513udtlb_miss_probefirst:
2514	/*
2515	 * g1 = 8K TSB pointer register
2516	 * g2 = tag access register
2517	 * g3 = (potentially) second TSB entry ptr
2518	 * g6 = ism pred.
2519	 * g7 = vpg_4m
2520	 */
2521#ifdef sun4v
2522	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2523	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2524
2525	/*
2526	 * Here:
2527	 *   g1 = first TSB pointer
2528	 *   g2 = tag access reg
2529	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2530	 */
2531	brgz,pn	%g6, sfmmu_tsb_miss_tt
2532	  nop
2533#else /* sun4v */
2534#ifndef UTSB_PHYS
2535	mov	%g1, %g4
2536	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
2537#endif UTSB_PHYS
2538	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2539
2540	/*
2541	 * Here:
2542	 *   g1 = first TSB pointer
2543	 *   g2 = tag access reg
2544	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2545	 */
2546	brgz,pn	%g6, sfmmu_tsb_miss_tt
2547	  nop
2548#ifndef UTSB_PHYS
2549	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
2550#endif UTSB_PHYS
2551	/* fall through in 8K->4M probe order */
2552#endif /* sun4v */
2553
2554udtlb_miss_probesecond:
2555	/*
2556	 * Look in the second TSB for the TTE
2557	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
2558	 * g2 = tag access reg
2559	 * g3 = 8K TSB pointer register
2560	 * g6 = ism pred.
2561	 * g7 = vpg_4m
2562	 */
2563#ifdef sun4v
2564	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
2565	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2566	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
2567#else /* sun4v */
2568#ifdef UTSB_PHYS
2569	GET_2ND_TSBREG(%g3)
2570	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2571	/* tagacc (%g2) is okay, no need to reload, %g3 = second tsbe ptr */
2572#else /* UTSB_PHYS */
2573	mov	%g3, %g7
2574	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
2575	/* %g2 clobbered, %g3 =second tsbe ptr */
2576	mov	MMU_TAG_ACCESS, %g2
2577	ldxa	[%g2]ASI_DMMU, %g2
2578#endif /* UTSB_PHYS */
2579#endif /* sun4v */
2580
2581	srlx	%g2, TAG_VALO_SHIFT, %g7
2582	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2583	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
2584	brgz,pn	%g6, udtlb_miss_probefirst
2585	  nop
2586
2587	/* fall through to sfmmu_tsb_miss_tt */
2588
2589	ALTENTRY(sfmmu_tsb_miss_tt)
2590	TT_TRACE(trace_tsbmiss)
2591	/*
2592	 * We get here if there is a TSB miss OR a write protect trap.
2593	 *
2594	 * g1 = First TSB entry pointer
2595	 * g2 = tag access register
2596	 * g3 = 4M TSB entry pointer; NULL if no 2nd TSB
2597	 * g4 - g7 = scratch registers
2598	 */
2599
2600	ALTENTRY(sfmmu_tsb_miss)
2601
2602	/*
2603	 * If trapstat is running, we need to shift the %tpc and %tnpc to
2604	 * point to trapstat's TSB miss return code (note that trapstat
2605	 * itself will patch the correct offset to add).
2606	 */
2607	rdpr	%tl, %g7
2608	cmp	%g7, 1
2609	ble,pt	%xcc, 0f
2610	  sethi	%hi(KERNELBASE), %g6
2611	rdpr	%tpc, %g7
2612	or	%g6, %lo(KERNELBASE), %g6
2613	cmp	%g7, %g6
2614	bgeu,pt	%xcc, 0f
2615	/* delay slot safe */
2616
2617	ALTENTRY(tsbmiss_trapstat_patch_point)
2618	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
2619	wrpr	%g7, %tpc
2620	add	%g7, 4, %g7
2621	wrpr	%g7, %tnpc
26220:
2623	CPU_TSBMISS_AREA(%g6, %g7)
2624
2625	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save first tsb pointer */
2626	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save second tsb pointer */
2627
2628	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
2629	brz,a,pn %g3, 1f			/* skip ahead if kernel */
2630	  ldn	[%g6 + TSBMISS_KHATID], %g7
2631	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
2632	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
2633
2634	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
2635
2636	cmp	%g3, INVALID_CONTEXT
2637	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
2638	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
2639
2640	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
2641	/*
2642	 * The miss wasn't in an ISM segment.
2643	 *
2644	 * %g1 %g3, %g4, %g5, %g7 all clobbered
2645	 * %g2 = tag access (vaddr + ctx)
2646	 */
2647
2648	ba,pt	%icc, 2f
2649	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
2650
26511:
2652	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
2653	/*
2654	 * 8K and 64K hash.
2655	 */
26562:
2657
2658	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2659		MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte,
2660		sfmmu_suspend_tl, tsb_512K)
2661	/* NOT REACHED */
2662
2663tsb_512K:
2664	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2665	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2666	brz,pn	%g5, 3f
2667	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2668	and	%g4, HAT_512K_FLAG, %g5
2669
2670	/*
2671	 * Note that there is a small window here where we may have
2672	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
2673	 * flag yet, so we will skip searching the 512k hash list.
2674	 * In this case we will end up in pagefault which will find
2675	 * the mapping and return.  So, in this instance we will end up
2676	 * spending a bit more time resolving this TSB miss, but it can
2677	 * only happen once per process and even then, the chances of that
2678	 * are very small, so it's not worth the extra overhead it would
2679	 * take to close this window.
2680	 */
2681	brz,pn	%g5, tsb_4M
2682	  nop
26833:
2684	/*
2685	 * 512K hash
2686	 */
2687
2688	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2689		MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte,
2690		sfmmu_suspend_tl, tsb_4M)
2691	/* NOT REACHED */
2692
2693tsb_4M:
2694	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2695	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2696	brz,pn	%g5, 4f
2697	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2698	and	%g4, HAT_4M_FLAG, %g5
2699	brz,pn	%g5, tsb_32M
2700	  nop
27014:
2702	/*
2703	 * 4M hash
2704	 */
2705
2706	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2707		MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte,
2708		sfmmu_suspend_tl, tsb_32M)
2709	/* NOT REACHED */
2710
2711tsb_32M:
2712#ifndef sun4v
2713	GET_CPU_IMPL(%g5)
2714	cmp	%g5, OLYMPUS_C_IMPL
2715	be,pn	%xcc, 0f
2716	  nop
2717	cmp	%g5, PANTHER_IMPL
2718	bne,pt	%xcc, tsb_pagefault
2719	  nop
2720#endif
2721
27220:
2723	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2724	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2725#ifdef sun4v
2726        brz,pn	%g5, 6f
2727#else
2728	brz,pn	%g5, tsb_pagefault
2729#endif
2730	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2731	and	%g4, HAT_32M_FLAG, %g5
2732	brz,pn	%g5, tsb_256M
2733	  nop
27345:
2735	/*
2736	 * 32M hash
2737	 */
2738
2739	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2740		MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte,
2741		sfmmu_suspend_tl, tsb_256M)
2742	/* NOT REACHED */
2743
2744tsb_256M:
2745	lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2746	and	%g4, HAT_256M_FLAG, %g5
2747	brz,pn	%g5, tsb_pagefault
2748	  nop
27496:
2750	/*
2751	 * 256M hash
2752	 */
2753
2754	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2755	    MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte,
2756	    sfmmu_suspend_tl, tsb_pagefault)
2757	/* NOT REACHED */
2758
2759tsb_checktte:
2760	/*
2761	 * g3 = tte
2762	 * g4 = tte pa
2763	 * g5 = tte va
2764	 * g6 = tsbmiss area
2765	 */
2766	brgez,pn %g3, tsb_pagefault	/* if tte invalid branch */
2767	  nop
2768
2769tsb_validtte:
2770	/*
2771	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
2772	 */
2773	rdpr	%tt, %g7
2774	cmp	%g7, FAST_PROT_TT
2775	bne,pt	%icc, 4f
2776	  nop
2777
2778	TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod,
2779	    tsb_protfault)
2780
2781	rdpr	%tt, %g5
2782	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
2783	ba,pt	%xcc, tsb_update_tl1
2784	  nop
2785
27864:
2787	/*
2788	 * If ITLB miss check exec bit.
2789	 * If not set treat as invalid TTE.
2790	 */
2791	cmp     %g7, T_INSTR_MMU_MISS
2792	be,pn	%icc, 5f
2793	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
2794	cmp     %g7, FAST_IMMU_MISS_TT
2795	bne,pt %icc, 3f
2796	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
27975:
2798	bz,pn %icc, tsb_protfault
2799	  nop
2800
28013:
2802	/*
2803	 * Set reference bit if not already set
2804	 */
2805	TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref)
2806
2807	/*
2808	 * Now, load into TSB/TLB.  At this point:
2809	 * g3 = tte
2810	 * g4 = patte
2811	 * g6 = tsbmiss area
2812	 */
2813	rdpr	%tt, %g5
2814#ifdef sun4v
2815	MMU_FAULT_STATUS_AREA(%g2)
2816	cmp	%g5, T_INSTR_MMU_MISS
2817	be,a,pt	%icc, 9f
2818	  nop
2819	cmp	%g5, FAST_IMMU_MISS_TT
2820	be,a,pt	%icc, 9f
2821	  nop
2822	add	%g2, MMFSA_D_, %g2
28239:
2824	ldx	[%g2 + MMFSA_CTX_], %g7
2825	sllx	%g7, TTARGET_CTX_SHIFT, %g7
2826	ldx	[%g2 + MMFSA_ADDR_], %g2
2827	srlx	%g2, TTARGET_VA_SHIFT, %g2
2828	or	%g2, %g7, %g2
2829#else
2830	cmp	%g5, FAST_IMMU_MISS_TT
2831	be,a,pt	%icc, tsb_update_tl1
2832	  ldxa	[%g0]ASI_IMMU, %g2
2833	ldxa	[%g0]ASI_DMMU, %g2
2834#endif
2835tsb_update_tl1:
2836	srlx	%g2, TTARGET_CTX_SHIFT, %g7
2837	brz,pn	%g7, tsb_kernel
2838#ifdef sun4v
2839	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
2840#else
2841	  srlx	%g3, TTE_SZ_SHFT, %g7
2842#endif
2843
2844tsb_user:
2845#ifdef sun4v
2846	cmp	%g7, TTE4M
2847	bge,pn	%icc, tsb_user4m
2848	  nop
2849#else /* sun4v */
2850	cmp	%g7, TTESZ_VALID | TTE4M
2851	be,pn	%icc, tsb_user4m
2852	  srlx	%g3, TTE_SZ2_SHFT, %g7
2853	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
2854#ifdef ITLB_32M_256M_SUPPORT
2855	bnz,pn	%icc, tsb_user4m
2856	  nop
2857#else /* ITLB_32M_256M_SUPPORT */
2858	bnz,a,pn %icc, tsb_user_pn_synth
2859	 cmp	%g5, FAST_IMMU_MISS_TT
2860#endif /* ITLB_32M_256M_SUPPORT */
2861#endif /* sun4v */
2862
2863tsb_user8k:
2864	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = first TSB ptr
2865
2866#ifndef UTSB_PHYS
2867	mov	ASI_N, %g7	! user TSBs accessed by VA
2868	mov	%g7, %asi
2869#endif /* UTSB_PHYS */
2870
2871	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5)
2872
2873#ifdef sun4v
2874	cmp	%g5, T_INSTR_MMU_MISS
2875	be,a,pn	%xcc, 9f
2876	  mov	%g3, %g5
2877#endif /* sun4v */
2878	cmp	%g5, FAST_IMMU_MISS_TT
2879	be,pn	%xcc, 9f
2880	  mov	%g3, %g5
2881
2882	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2883	! trapstat wants TTE in %g5
2884	retry
28859:
2886	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2887	! trapstat wants TTE in %g5
2888	retry
2889
2890tsb_user4m:
2891	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */
28924:
2893	brz,pn	%g1, 5f	/* Check to see if we have 2nd TSB programmed */
2894	  nop
2895
2896#ifndef UTSB_PHYS
2897	mov	ASI_N, %g7	! user TSBs accessed by VA
2898	mov	%g7, %asi
2899#endif /* UTSB_PHYS */
2900
2901        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6)
2902
29035:
2904#ifdef sun4v
2905        cmp     %g5, T_INSTR_MMU_MISS
2906        be,a,pn %xcc, 9f
2907          mov   %g3, %g5
2908#endif /* sun4v */
2909        cmp     %g5, FAST_IMMU_MISS_TT
2910        be,pn   %xcc, 9f
2911        mov     %g3, %g5
2912
2913        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2914        ! trapstat wants TTE in %g5
2915        retry
29169:
2917        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2918        ! trapstat wants TTE in %g5
2919        retry
2920
2921#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT)
2922	/*
2923	 * Panther ITLB synthesis.
2924	 * The Panther 32M and 256M ITLB code simulates these two large page
2925	 * sizes with 4M pages, to provide support for programs, for example
2926	 * Java, that may copy instructions into a 32M or 256M data page and
2927	 * then execute them. The code below generates the 4M pfn bits and
2928	 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
2929	 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
2930	 * are ignored by the hardware.
2931	 *
2932	 * Now, load into TSB/TLB.  At this point:
2933	 * g2 = tagtarget
2934	 * g3 = tte
2935	 * g4 = patte
2936	 * g5 = tt
2937	 * g6 = tsbmiss area
2938	 */
2939tsb_user_pn_synth:
2940	be,pt	%xcc, tsb_user_itlb_synth	/* ITLB miss */
2941	  andcc %g3, TTE_EXECPRM_INT, %g0	/* is execprm bit set */
2942	bz,pn %icc, 4b				/* if not, been here before */
2943	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	/* g1 = tsbp */
2944	brz,a,pn %g1, 5f			/* no 2nd tsb */
2945	  mov	%g3, %g5
2946
2947	mov	MMU_TAG_ACCESS, %g7
2948	ldxa	[%g7]ASI_DMMU, %g6		/* get tag access va */
2949	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)	/* make 4M pfn offset */
2950
2951	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
2952	mov	%g7, %asi
2953	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */
29545:
2955        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2956        retry
2957
2958tsb_user_itlb_synth:
2959	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */
2960
2961	mov	MMU_TAG_ACCESS, %g7
2962	ldxa	[%g7]ASI_IMMU, %g6		/* get tag access va */
2963	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)	/* make 4M pfn offset */
2964	brz,a,pn %g1, 7f	/* Check to see if we have 2nd TSB programmed */
2965	  or	%g5, %g3, %g5			/* add 4M bits to TTE */
2966
2967	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
2968	mov	%g7, %asi
2969	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */
29707:
2971	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
2972        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2973        retry
2974#endif /* sun4v && ITLB_32M_256M_SUPPORT */
2975
2976tsb_kernel:
2977#ifdef sun4v
2978	cmp	%g7, TTE4M
2979	bge,pn	%icc, 5f
2980#else
2981	cmp	%g7, TTESZ_VALID | TTE4M	! no 32M or 256M support
2982	be,pn	%icc, 5f
2983#endif
2984	  nop
2985	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8k tsbptr
2986	ba,pt	%xcc, 6f
2987	  nop
29885:
2989	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4m tsbptr
2990	brz,pn	%g1, 3f		/* skip programming if 4m TSB ptr is NULL */
2991	  nop
29926:
2993#ifndef sun4v
2994tsb_kernel_patch_asi:
2995	or	%g0, RUNTIME_PATCH, %g6
2996	mov	%g6, %asi	! XXX avoid writing to %asi !!
2997#endif
2998	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7)
29993:
3000#ifdef sun4v
3001	cmp	%g5, T_INSTR_MMU_MISS
3002	be,a,pn	%icc, 1f
3003	  mov	%g3, %g5			! trapstat wants TTE in %g5
3004#endif /* sun4v */
3005	cmp	%g5, FAST_IMMU_MISS_TT
3006	be,pn	%icc, 1f
3007	  mov	%g3, %g5			! trapstat wants TTE in %g5
3008	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3009	! trapstat wants TTE in %g5
3010	retry
30111:
3012	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3013	! trapstat wants TTE in %g5
3014	retry
3015
3016tsb_ism:
3017	/*
3018	 * This is an ISM [i|d]tlb miss.  We optimize for largest
3019	 * page size down to smallest.
3020	 *
3021	 * g2 = vaddr + ctx	aka tag access register
3022	 * g3 = ismmap->ism_seg
3023	 * g4 = physical address of ismmap->ism_sfmmu
3024	 * g6 = tsbmiss area
3025	 */
3026	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
3027	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
3028	  mov	PTL1_BAD_ISM, %g1
3029						/* g5 = pa of imap_vb_shift */
3030	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
3031	lduha	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
3032	srlx	%g3, %g4, %g3			/* clr size field */
3033	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
3034	sllx	%g3, %g4, %g3			/* g3 = ism vbase */
3035	and	%g2, %g1, %g4			/* g4 = ctx number */
3036	andn	%g2, %g1, %g1			/* g1 = tlb miss vaddr */
3037	sub	%g1, %g3, %g2			/* g2 = offset in ISM seg */
3038	or	%g2, %g4, %g2			/* g2 = tagacc (vaddr + ctx) */
3039
3040	/*
3041	 * ISM pages are always locked down.
3042	 * If we can't find the tte then pagefault
3043	 * and let the spt segment driver resovle it.
3044	 *
3045	 * g2 = ISM vaddr (offset in ISM seg)
3046	 * g6 = tsb miss area
3047	 * g7 = ISM hatid
3048	 */
3049	sub	%g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
3050	lduha	[%g5]ASI_MEM, %g4		/* g5 = pa of imap_hatflags */
3051	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
3052	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
3053	  nop
3054
3055tsb_ism_32M:
3056	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
3057	brz,pn	%g5, tsb_ism_256M
3058	  nop
3059
3060	/*
3061	 * 32M hash.
3062	 */
3063
3064	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M,
3065	    TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3066	    tsb_ism_4M)
3067	/* NOT REACHED */
3068
3069tsb_ism_32M_found:
3070	brlz,pt %g3, tsb_validtte
3071	  nop
3072	ba,pt	%xcc, tsb_ism_4M
3073	  nop
3074
3075tsb_ism_256M:
3076	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
3077	brz,a,pn %g5, ptl1_panic
3078	  mov	PTL1_BAD_ISM, %g1
3079
3080	/*
3081	 * 256M hash.
3082	 */
3083	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M,
3084	    TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3085	    tsb_ism_4M)
3086
3087tsb_ism_256M_found:
3088	brlz,pt %g3, tsb_validtte
3089	  nop
3090
3091tsb_ism_4M:
3092	/*
3093	 * 4M hash.
3094	 */
3095	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M,
3096	    TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3097	    tsb_ism_8K)
3098	/* NOT REACHED */
3099
3100tsb_ism_4M_found:
3101	brlz,pt %g3, tsb_validtte
3102	  nop
3103
3104tsb_ism_8K:
3105	/*
3106	 * 8K and 64K hash.
3107	 */
3108
3109	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K,
3110	    TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3111	    tsb_pagefault)
3112	/* NOT REACHED */
3113
3114tsb_ism_8K_found:
3115	brlz,pt	%g3, tsb_validtte
3116	  nop
3117
3118tsb_pagefault:
3119	rdpr	%tt, %g7
3120	cmp	%g7, FAST_PROT_TT
3121	be,a,pn	%icc, tsb_protfault
3122	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt
3123
3124tsb_protfault:
3125	/*
3126	 * we get here if we couldn't find a valid tte in the hash.
3127	 *
3128	 * If user and we are at tl>1 we go to window handling code.
3129	 *
3130	 * If kernel and the fault is on the same page as our stack
3131	 * pointer, then we know the stack is bad and the trap handler
3132	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3133	 *
3134	 * If this is a kernel trap and tl>1, panic.
3135	 *
3136	 * Otherwise we call pagefault.
3137	 */
3138	cmp	%g7, FAST_IMMU_MISS_TT
3139#ifdef sun4v
3140	MMU_FAULT_STATUS_AREA(%g4)
3141	ldx	[%g4 + MMFSA_I_CTX], %g5
3142	ldx	[%g4 + MMFSA_D_CTX], %g4
3143	move	%icc, %g5, %g4
3144	cmp	%g7, T_INSTR_MMU_MISS
3145	move	%icc, %g5, %g4
3146#else
3147	mov	MMU_TAG_ACCESS, %g4
3148	ldxa	[%g4]ASI_DMMU, %g2
3149	ldxa	[%g4]ASI_IMMU, %g5
3150	move	%icc, %g5, %g2
3151	cmp	%g7, T_INSTR_MMU_MISS
3152	move	%icc, %g5, %g2
3153	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
3154#endif
3155	brnz,pn	%g4, 3f				/* skip if not kernel */
3156	  rdpr	%tl, %g5
3157
3158	add	%sp, STACK_BIAS, %g3
3159	srlx	%g3, MMU_PAGESHIFT, %g3
3160	srlx	%g2, MMU_PAGESHIFT, %g4
3161	cmp	%g3, %g4
3162	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
3163	  mov	PTL1_BAD_STACK, %g1
3164
3165	cmp	%g5, 1
3166	ble,pt	%icc, 2f
3167	  nop
3168	TSTAT_CHECK_TL1(2f, %g1, %g2)
3169	rdpr	%tt, %g2
3170	cmp	%g2, FAST_PROT_TT
3171	mov	PTL1_BAD_KPROT_FAULT, %g1
3172	movne	%icc, PTL1_BAD_KMISS, %g1
3173	ba,pt	%icc, ptl1_panic
3174	  nop
3175
31762:
3177	/*
3178	 * We are taking a pagefault in the kernel on a kernel address.  If
3179	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
3180	 * want to call sfmmu_pagefault -- we will instead note that a fault
3181	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
3182	 * (instead of a "retry").  This will step over the faulting
3183	 * instruction.
3184	 */
3185	CPU_INDEX(%g1, %g2)
3186	set	cpu_core, %g2
3187	sllx	%g1, CPU_CORE_SHIFT, %g1
3188	add	%g1, %g2, %g1
3189	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3190	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3191	bz	sfmmu_pagefault
3192	or	%g2, CPU_DTRACE_BADADDR, %g2
3193	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3194	GET_MMU_D_ADDR(%g3, %g4)
3195	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3196	done
3197
31983:
3199	cmp	%g5, 1
3200	ble,pt	%icc, 4f
3201	  nop
3202	TSTAT_CHECK_TL1(4f, %g1, %g2)
3203	ba,pt	%icc, sfmmu_window_trap
3204	  nop
3205
32064:
3207	/*
3208	 * We are taking a pagefault on a non-kernel address.  If we are in
3209	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
3210	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
3211	 */
3212	CPU_INDEX(%g1, %g2)
3213	set	cpu_core, %g2
3214	sllx	%g1, CPU_CORE_SHIFT, %g1
3215	add	%g1, %g2, %g1
3216	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3217	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3218	bz	sfmmu_pagefault
3219	or	%g2, CPU_DTRACE_BADADDR, %g2
3220	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3221	GET_MMU_D_ADDR(%g3, %g4)
3222	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3223
3224	/*
3225	 * Be sure that we're actually taking this miss from the kernel --
3226	 * otherwise we have managed to return to user-level with
3227	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3228	 */
3229	rdpr	%tstate, %g2
3230	btst	TSTATE_PRIV, %g2
3231	bz,a	ptl1_panic
3232	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3233	done
3234
3235	ALTENTRY(tsb_tl0_noctxt)
3236	/*
3237	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
3238	 * if it is, indicated that we have faulted and issue a done.
3239	 */
3240	CPU_INDEX(%g5, %g6)
3241	set	cpu_core, %g6
3242	sllx	%g5, CPU_CORE_SHIFT, %g5
3243	add	%g5, %g6, %g5
3244	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
3245	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
3246	bz	1f
3247	or	%g6, CPU_DTRACE_BADADDR, %g6
3248	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
3249	GET_MMU_D_ADDR(%g3, %g4)
3250	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]
3251
3252	/*
3253	 * Be sure that we're actually taking this miss from the kernel --
3254	 * otherwise we have managed to return to user-level with
3255	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3256	 */
3257	rdpr	%tstate, %g5
3258	btst	TSTATE_PRIV, %g5
3259	bz,a	ptl1_panic
3260	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3261	done
3262
32631:
3264	rdpr	%tt, %g5
3265	cmp	%g5, FAST_IMMU_MISS_TT
3266#ifdef sun4v
3267	MMU_FAULT_STATUS_AREA(%g2)
3268	be,a,pt	%icc, 2f
3269	  ldx	[%g2 + MMFSA_I_CTX], %g3
3270	cmp	%g5, T_INSTR_MMU_MISS
3271	be,a,pt	%icc, 2f
3272	  ldx	[%g2 + MMFSA_I_CTX], %g3
3273	ldx	[%g2 + MMFSA_D_CTX], %g3
32742:
3275#else
3276	mov	MMU_TAG_ACCESS, %g2
3277	be,a,pt	%icc, 2f
3278	  ldxa	[%g2]ASI_IMMU, %g3
3279	ldxa	[%g2]ASI_DMMU, %g3
32802:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
3281#endif
3282	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
3283	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
3284	rdpr	%tl, %g5
3285	cmp	%g5, 1
3286	ble,pt	%icc, sfmmu_mmu_trap
3287	  nop
3288	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
3289	ba,pt	%icc, sfmmu_window_trap
3290	  nop
3291	SET_SIZE(sfmmu_tsb_miss)
3292
3293#if (1<< TSBMISS_SHIFT) != TSBMISS_SIZE
3294#error - TSBMISS_SHIFT does not correspond to size of tsbmiss struct
3295#endif
3296
3297#endif /* lint */
3298
3299#if defined (lint)
3300/*
3301 * This routine will look for a user or kernel vaddr in the hash
3302 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't
3303 * grab any locks.  It should only be used by other sfmmu routines.
3304 */
3305/* ARGSUSED */
3306pfn_t
3307sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
3308{
3309	return(0);
3310}
3311
3312#else /* lint */
3313
3314	ENTRY_NP(sfmmu_vatopfn)
3315 	/*
3316 	 * disable interrupts
3317 	 */
3318 	rdpr	%pstate, %o3
3319#ifdef DEBUG
3320	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
3321	bnz,pt	%icc, 1f			/* disabled, panic	 */
3322	  nop
3323
3324	sethi	%hi(panicstr), %g1
3325	ldx	[%g1 + %lo(panicstr)], %g1
3326	tst	%g1
3327	bnz,pt	%icc, 1f
3328	  nop
3329
3330	save	%sp, -SA(MINFRAME), %sp
3331	sethi	%hi(sfmmu_panic1), %o0
3332	call	panic
3333	 or	%o0, %lo(sfmmu_panic1), %o0
33341:
3335#endif
3336	/*
3337	 * disable interrupts to protect the TSBMISS area
3338	 */
3339	andn    %o3, PSTATE_IE, %o5
3340	wrpr    %o5, 0, %pstate
3341
3342	/*
3343	 * o0 = vaddr
3344	 * o1 = sfmmup
3345	 * o2 = ttep
3346	 */
3347	CPU_TSBMISS_AREA(%g1, %o5)
3348	ldn	[%g1 + TSBMISS_KHATID], %o4
3349	cmp	%o4, %o1
3350	bne,pn	%ncc, vatopfn_nokernel
3351	  mov	TTE64K, %g5			/* g5 = rehash # */
3352	mov %g1,%o5				/* o5 = tsbmiss_area */
3353	/*
3354	 * o0 = vaddr
3355	 * o1 & o4 = hatid
3356	 * o2 = ttep
3357	 * o5 = tsbmiss area
3358	 */
3359	mov	HBLK_RANGE_SHIFT, %g6
33601:
3361
3362	/*
3363	 * o0 = vaddr
3364	 * o1 = sfmmup
3365	 * o2 = ttep
3366	 * o3 = old %pstate
3367	 * o4 = hatid
3368	 * o5 = tsbmiss
3369	 * g5 = rehash #
3370	 * g6 = hmeshift
3371	 *
3372	 * The first arg to GET_TTE is actually tagaccess register
3373	 * not just vaddr. Since this call is for kernel we need to clear
3374	 * any lower vaddr bits that would be interpreted as ctx bits.
3375	 */
3376	set     TAGACC_CTX_MASK, %g1
3377	andn    %o0, %g1, %o0
3378	GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5,
3379		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
3380
3381kvtop_hblk_found:
3382	/*
3383	 * o0 = vaddr
3384	 * o1 = sfmmup
3385	 * o2 = ttep
3386	 * g1 = tte
3387	 * g2 = tte pa
3388	 * g3 = tte va
3389	 * o2 = tsbmiss area
3390	 * o1 = hat id
3391	 */
3392	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
3393	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3394	stx %g1,[%o2]				/* put tte into *ttep */
3395	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
3396	/*
3397	 * o0 = vaddr
3398	 * o1 = sfmmup
3399	 * o2 = ttep
3400	 * g1 = pfn
3401	 */
3402	ba,pt	%xcc, 6f
3403	  mov	%g1, %o0
3404
3405kvtop_nohblk:
3406	/*
3407	 * we get here if we couldn't find valid hblk in hash.  We rehash
3408	 * if neccesary.
3409	 */
3410	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
3411#ifdef sun4v
3412	cmp	%g5, MAX_HASHCNT
3413#else
3414	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
3415#endif
3416	be,a,pn	%icc, 6f
3417	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3418	mov	%o1, %o4			/* restore hatid */
3419#ifdef sun4v
3420        add	%g5, 2, %g5
3421	cmp	%g5, 3
3422	move	%icc, MMU_PAGESHIFT4M, %g6
3423	ba,pt	%icc, 1b
3424	movne	%icc, MMU_PAGESHIFT256M, %g6
3425#else
3426        inc	%g5
3427	cmp	%g5, 2
3428	move	%icc, MMU_PAGESHIFT512K, %g6
3429	ba,pt	%icc, 1b
3430	movne	%icc, MMU_PAGESHIFT4M, %g6
3431#endif
34326:
3433	retl
3434 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3435
3436tsb_suspend:
3437	/*
3438	 * o0 = vaddr
3439	 * o1 = sfmmup
3440	 * o2 = ttep
3441	 * g1 = tte
3442	 * g2 = tte pa
3443	 * g3 = tte va
3444	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
3445	 */
3446	stx %g1,[%o2]				/* put tte into *ttep */
3447	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
3448	  sub	%g0, 1, %o0			/* output = -1 (PFN_INVALID) */
3449	TTETOPFN(%g1, %o0, vatopfn_l3, %g2, %g3, %g4)
3450	/*
3451	 * o0 = PFN return value PFN_INVALID, PFN_SUSPENDED, or pfn#
3452	 * o1 = sfmmup
3453	 * o2 = ttep
3454	 * g1 = pfn
3455	 */
3456	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
34578:
3458	retl
3459	 wrpr	%g0, %o3, %pstate		/* enable interrupts */
3460
3461vatopfn_nokernel:
3462	/*
3463	 * This routine does NOT support user addresses
3464	 * There is a routine in C that supports this.
3465	 * The only reason why we don't have the C routine
3466	 * support kernel addresses as well is because
3467	 * we do va_to_pa while holding the hashlock.
3468	 */
3469 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3470	save	%sp, -SA(MINFRAME), %sp
3471	sethi	%hi(sfmmu_panic3), %o0
3472	call	panic
3473	 or	%o0, %lo(sfmmu_panic3), %o0
3474
3475	SET_SIZE(sfmmu_vatopfn)
3476#endif /* lint */
3477
3478
3479
3480#if !defined(lint)
3481
3482/*
3483 * kpm lock used between trap level tsbmiss handler and kpm C level.
3484 */
3485#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
3486	mov     0xff, tmp1						;\
3487label1:									;\
3488	casa    [kpmlckp]asi, %g0, tmp1					;\
3489	brnz,pn tmp1, label1						;\
3490	mov     0xff, tmp1						;\
3491	membar  #LoadLoad
3492
3493#define KPMLOCK_EXIT(kpmlckp, asi)					\
3494	membar  #LoadStore|#StoreStore					;\
3495	sta     %g0, [kpmlckp]asi
3496
3497/*
3498 * Lookup a memseg for a given pfn and if found, return the physical
3499 * address of the corresponding struct memseg in mseg, otherwise
3500 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
3501 * tsbmp, %asi is assumed to be ASI_MEM.
3502 * This lookup is done by strictly traversing only the physical memseg
3503 * linkage. The more generic approach, to check the virtual linkage
3504 * before using the physical (used e.g. with hmehash buckets), cannot
3505 * be used here. Memory DR operations can run in parallel to this
3506 * lookup w/o any locks and updates of the physical and virtual linkage
3507 * cannot be done atomically wrt. to each other. Because physical
3508 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
3509 * as "physical NULL" pointer.
3510 */
3511#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
3512	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
3513	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
3514	udivx	pfn, mseg, mseg						;\
3515	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
3516	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
3517	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
3518	add	tmp1, mseg, tmp1					;\
3519	ldxa	[tmp1]%asi, mseg					;\
3520	cmp	mseg, MSEG_NULLPTR_PA					;\
3521	be,pn	%xcc, label/**/1		/* if not found */	;\
3522	  nop								;\
3523	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3524	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3525	blu,pn	%xcc, label/**/1					;\
3526	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3527	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3528	bgeu,pn	%xcc, label/**/1					;\
3529	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
3530	mulx	tmp1, PAGE_SIZE, tmp1					;\
3531	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
3532	add	tmp2, tmp1, tmp1			/* pp */	;\
3533	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
3534	cmp	tmp2, pfn						;\
3535	be,pt	%xcc, label/**/_ok			/* found */	;\
3536label/**/1:								;\
3537	/* brute force lookup */					;\
3538	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
3539	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
3540label/**/2:								;\
3541	cmp	mseg, MSEG_NULLPTR_PA					;\
3542	be,pn	%xcc, label/**/_ok		/* if not found */	;\
3543	  nop								;\
3544	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3545	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3546	blu,a,pt %xcc, label/**/2					;\
3547	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3548	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3549	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3550	bgeu,a,pt %xcc, label/**/2					;\
3551	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3552label/**/_ok:
3553
3554	/*
3555	 * kpm tsb miss handler large pages
3556	 * g1 = 8K kpm TSB entry pointer
3557	 * g2 = tag access register
3558	 * g3 = 4M kpm TSB entry pointer
3559	 */
3560	ALTENTRY(sfmmu_kpm_dtsb_miss)
3561	TT_TRACE(trace_tsbmiss)
3562
3563	CPU_INDEX(%g7, %g6)
3564	sethi	%hi(kpmtsbm_area), %g6
3565	sllx	%g7, KPMTSBM_SHIFT, %g7
3566	or	%g6, %lo(kpmtsbm_area), %g6
3567	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
3568
3569	/* check enable flag */
3570	ldub	[%g6 + KPMTSBM_FLAGS], %g4
3571	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
3572	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
3573	  nop
3574
3575	/* VA range check */
3576	ldx	[%g6 + KPMTSBM_VBASE], %g7
3577	cmp	%g2, %g7
3578	blu,pn	%xcc, sfmmu_tsb_miss
3579	  ldx	[%g6 + KPMTSBM_VEND], %g5
3580	cmp	%g2, %g5
3581	bgeu,pn	%xcc, sfmmu_tsb_miss
3582	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]
3583
3584	/*
3585	 * check TL tsbmiss handling flag
3586	 * bump tsbmiss counter
3587	 */
3588	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
3589#ifdef	DEBUG
3590	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
3591	inc	%g5
3592	brz,pn	%g3, sfmmu_kpm_exception
3593	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
3594#else
3595	inc	%g5
3596	st	%g5, [%g6 + KPMTSBM_TSBMISS]
3597#endif
3598	/*
3599	 * At this point:
3600	 *  g1 = 8K kpm TSB pointer (not used)
3601	 *  g2 = tag access register
3602	 *  g3 = clobbered
3603	 *  g6 = per-CPU kpm tsbmiss area
3604	 *  g7 = kpm_vbase
3605	 */
3606
3607	/* vaddr2pfn */
3608	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
3609	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
3610	srax    %g4, %g3, %g2			/* which alias range (r) */
3611	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
3612	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
3613
3614	/*
3615	 * Setup %asi
3616	 * mseg_pa = page_numtomemseg_nolock(pfn)
3617	 * if (mseg_pa == NULL) sfmmu_kpm_exception
3618	 * g2=pfn
3619	 */
3620	mov	ASI_MEM, %asi
3621	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
3622	cmp	%g3, MSEG_NULLPTR_PA
3623	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
3624	  nop
3625
3626	/*
3627	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
3628	 * g2=pfn g3=mseg_pa
3629	 */
3630	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
3631	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
3632	srlx	%g2, %g5, %g4
3633	sllx	%g4, %g5, %g4
3634	sub	%g4, %g7, %g4
3635	srlx	%g4, %g5, %g4
3636
3637	/*
3638	 * Validate inx value
3639	 * g2=pfn g3=mseg_pa g4=inx
3640	 */
3641#ifdef	DEBUG
3642	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
3643	cmp	%g4, %g5			/* inx - nkpmpgs */
3644	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
3645	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3646#else
3647	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3648#endif
3649	/*
3650	 * kp = &mseg_pa->kpm_pages[inx]
3651	 */
3652	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
3653	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
3654	add	%g5, %g4, %g5			/* kp */
3655
3656	/*
3657	 * KPMP_HASH(kp)
3658	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
3659	 */
3660	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
3661	sub	%g7, 1, %g7			/* mask */
3662	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
3663	add	%g5, %g1, %g5			/* y = ksp + x */
3664	and 	%g5, %g7, %g5			/* hashinx = y & mask */
3665
3666	/*
3667	 * Calculate physical kpm_page pointer
3668	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
3669	 */
3670	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
3671	add	%g1, %g4, %g1			/* kp_pa */
3672
3673	/*
3674	 * Calculate physical hash lock address
3675	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
3676	 */
3677	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
3678	sllx	%g5, KPMHLK_SHIFT, %g5
3679	add	%g4, %g5, %g3
3680	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */
3681
3682	/*
3683	 * Assemble tte
3684	 * g1=kp_pa g2=pfn g3=hlck_pa
3685	 */
3686#ifdef sun4v
3687	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
3688	sllx	%g5, 32, %g5
3689	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3690	or	%g4, TTE4M, %g4
3691	or	%g5, %g4, %g5
3692#else
3693	sethi	%hi(TTE_VALID_INT), %g4
3694	mov	TTE4M, %g5
3695	sllx	%g5, TTE_SZ_SHFT_INT, %g5
3696	or	%g5, %g4, %g5			/* upper part */
3697	sllx	%g5, 32, %g5
3698	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3699	or	%g5, %g4, %g5
3700#endif
3701	sllx	%g2, MMU_PAGESHIFT, %g4
3702	or	%g5, %g4, %g5			/* tte */
3703	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
3704	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3705
3706	/*
3707	 * tsb dropin
3708	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
3709	 */
3710
3711	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
3712	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
3713
3714	/* use C-handler if there's no go for dropin */
3715	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
3716	cmp	%g7, -1
3717	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
3718	  nop
3719
3720#ifdef	DEBUG
3721	/* double check refcnt */
3722	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
3723	brz,pn	%g7, 5f			/* let C-handler deal with this */
3724	  nop
3725#endif
3726
3727#ifndef sun4v
3728	ldub	[%g6 + KPMTSBM_FLAGS], %g7
3729	mov	ASI_N, %g1
3730	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
3731	movnz	%icc, ASI_MEM, %g1
3732	mov	%g1, %asi
3733#endif
3734
3735	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
3736	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
3737
3738	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
3739	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
3740
3741	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
3742
3743	/* KPMLOCK_EXIT(kpmlckp, asi) */
3744	KPMLOCK_EXIT(%g3, ASI_MEM)
3745
3746	/*
3747	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3748	 * point to trapstat's TSB miss return code (note that trapstat
3749	 * itself will patch the correct offset to add).
3750	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
3751	 */
3752	rdpr	%tl, %g7
3753	cmp	%g7, 1
3754	ble	%icc, 0f
3755	sethi	%hi(KERNELBASE), %g6
3756	rdpr	%tpc, %g7
3757	or	%g6, %lo(KERNELBASE), %g6
3758	cmp	%g7, %g6
3759	bgeu	%xcc, 0f
3760	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
3761	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3762	wrpr	%g7, %tpc
3763	add	%g7, 4, %g7
3764	wrpr	%g7, %tnpc
37650:
3766	retry
37675:
3768	/* g3=hlck_pa */
3769	KPMLOCK_EXIT(%g3, ASI_MEM)
3770	ba,pt	%icc, sfmmu_kpm_exception
3771	  nop
3772	SET_SIZE(sfmmu_kpm_dtsb_miss)
3773
3774	/*
3775	 * kpm tsbmiss handler for smallpages
3776	 * g1 = 8K kpm TSB pointer
3777	 * g2 = tag access register
3778	 * g3 = 4M kpm TSB pointer
3779	 */
3780	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
3781	TT_TRACE(trace_tsbmiss)
3782	CPU_INDEX(%g7, %g6)
3783	sethi	%hi(kpmtsbm_area), %g6
3784	sllx	%g7, KPMTSBM_SHIFT, %g7
3785	or	%g6, %lo(kpmtsbm_area), %g6
3786	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
3787
3788	/* check enable flag */
3789	ldub	[%g6 + KPMTSBM_FLAGS], %g4
3790	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
3791	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
3792	  nop
3793
3794	/*
3795	 * VA range check
3796	 * On fail: goto sfmmu_tsb_miss
3797	 */
3798	ldx	[%g6 + KPMTSBM_VBASE], %g7
3799	cmp	%g2, %g7
3800	blu,pn	%xcc, sfmmu_tsb_miss
3801	  ldx	[%g6 + KPMTSBM_VEND], %g5
3802	cmp	%g2, %g5
3803	bgeu,pn	%xcc, sfmmu_tsb_miss
3804	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */
3805
3806	/*
3807	 * check TL tsbmiss handling flag
3808	 * bump tsbmiss counter
3809	 */
3810	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
3811#ifdef	DEBUG
3812	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
3813	inc	%g5
3814	brz,pn	%g1, sfmmu_kpm_exception
3815	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
3816#else
3817	inc	%g5
3818	st	%g5, [%g6 + KPMTSBM_TSBMISS]
3819#endif
3820	/*
3821	 * At this point:
3822	 *  g1 = clobbered
3823	 *  g2 = tag access register
3824	 *  g3 = 4M kpm TSB pointer (not used)
3825	 *  g6 = per-CPU kpm tsbmiss area
3826	 *  g7 = kpm_vbase
3827	 */
3828
3829	/* vaddr2pfn */
3830	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
3831	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
3832	srax    %g4, %g3, %g2			/* which alias range (r) */
3833	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
3834	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
3835
3836	/*
3837	 * Setup %asi
3838	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
3839	 * if (mseg not found) sfmmu_kpm_exception
3840	 * g2=pfn
3841	 */
3842	mov	ASI_MEM, %asi
3843	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
3844	cmp	%g3, MSEG_NULLPTR_PA
3845	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
3846	  nop
3847
3848	/*
3849	 * inx = pfn - mseg_pa->kpm_pbase
3850	 * g2=pfn g3=mseg_pa
3851	 */
3852	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
3853	sub	%g2, %g7, %g4
3854
3855#ifdef	DEBUG
3856	/*
3857	 * Validate inx value
3858	 * g2=pfn g3=mseg_pa g4=inx
3859	 */
3860	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
3861	cmp	%g4, %g5			/* inx - nkpmpgs */
3862	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
3863	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3864#else
3865	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3866#endif
3867	/* ksp = &mseg_pa->kpm_spages[inx] */
3868	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
3869	add	%g5, %g4, %g5			/* ksp */
3870
3871	/*
3872	 * KPMP_SHASH(kp)
3873	 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz
3874	 */
3875	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
3876	sub	%g7, 1, %g7			/* mask */
3877	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
3878	add	%g5, %g1, %g5			/* y = ksp + x */
3879	and 	%g5, %g7, %g5			/* hashinx = y & mask */
3880
3881	/*
3882	 * Calculate physical kpm_spage pointer
3883	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
3884	 */
3885	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
3886	add	%g1, %g4, %g1			/* ksp_pa */
3887
3888	/*
3889	 * Calculate physical hash lock address.
3890	 * Note: Changes in kpm_shlk_t must be reflected here.
3891	 * g1=ksp_pa g2=pfn g5=hashinx
3892	 */
3893	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
3894	sllx	%g5, KPMSHLK_SHIFT, %g5
3895	add	%g4, %g5, %g3			/* hlck_pa */
3896
3897	/*
3898	 * Assemble tte
3899	 * g1=ksp_pa g2=pfn g3=hlck_pa
3900	 */
3901	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
3902	sllx	%g5, 32, %g5
3903	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3904	or	%g5, %g4, %g5
3905	sllx	%g2, MMU_PAGESHIFT, %g4
3906	or	%g5, %g4, %g5			/* tte */
3907	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
3908	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3909
3910	/*
3911	 * tsb dropin
3912	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte
3913	 */
3914
3915	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
3916	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
3917
3918	/* use C-handler if there's no go for dropin */
3919	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */
3920	cmp	%g7, -1
3921	bne,pn	%xcc, 5f
3922	  nop
3923
3924#ifndef sun4v
3925	ldub	[%g6 + KPMTSBM_FLAGS], %g7
3926	mov	ASI_N, %g1
3927	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
3928	movnz	%icc, ASI_MEM, %g1
3929	mov	%g1, %asi
3930#endif
3931
3932	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
3933	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
3934
3935	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
3936	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
3937
3938	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
3939
3940	/* KPMLOCK_EXIT(kpmlckp, asi) */
3941	KPMLOCK_EXIT(%g3, ASI_MEM)
3942
3943	/*
3944	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3945	 * point to trapstat's TSB miss return code (note that trapstat
3946	 * itself will patch the correct offset to add).
3947	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
3948	 */
3949	rdpr	%tl, %g7
3950	cmp	%g7, 1
3951	ble	%icc, 0f
3952	sethi	%hi(KERNELBASE), %g6
3953	rdpr	%tpc, %g7
3954	or	%g6, %lo(KERNELBASE), %g6
3955	cmp	%g7, %g6
3956	bgeu	%xcc, 0f
3957	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
3958	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3959	wrpr	%g7, %tpc
3960	add	%g7, 4, %g7
3961	wrpr	%g7, %tnpc
39620:
3963	retry
39645:
3965	/* g3=hlck_pa */
3966	KPMLOCK_EXIT(%g3, ASI_MEM)
3967	ba,pt	%icc, sfmmu_kpm_exception
3968	  nop
3969	SET_SIZE(sfmmu_kpm_dtsb_miss_small)
3970
3971#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
3972#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
3973#endif
3974
3975#endif /* lint */
3976
3977#ifdef	lint
3978/*
3979 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
3980 * Called from C-level, sets/clears "go" indication for trap level handler.
3981 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
3982 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
3983 * Assumes khl_mutex is held when called from C-level.
3984 */
3985/* ARGSUSED */
3986void
3987sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
3988{
3989}
3990
3991/*
3992 * kpm_smallpages: stores val to byte at address mapped within
3993 * low level lock brackets. The old value is returned.
3994 * Called from C-level.
3995 */
3996/* ARGSUSED */
3997int
3998sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val)
3999{
4000	return (0);
4001}
4002
4003#else /* lint */
4004
4005	.seg	".data"
4006sfmmu_kpm_tsbmtl_panic:
4007	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
4008	.byte	0
4009sfmmu_kpm_stsbmtl_panic:
4010	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
4011	.byte	0
4012	.align	4
4013	.seg	".text"
4014
4015	ENTRY_NP(sfmmu_kpm_tsbmtl)
4016	rdpr	%pstate, %o3
4017	/*
4018	 * %o0 = &kp_refcntc
4019	 * %o1 = &khl_lock
4020	 * %o2 = 0/1 (off/on)
4021	 * %o3 = pstate save
4022	 */
4023#ifdef DEBUG
4024	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4025	bnz,pt %icc, 1f				/* disabled, panic	 */
4026	  nop
4027	save	%sp, -SA(MINFRAME), %sp
4028	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
4029	call	panic
4030	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
4031	ret
4032	restore
40331:
4034#endif /* DEBUG */
4035	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4036
4037	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
4038	mov	-1, %o5
4039	brz,a	%o2, 2f
4040	  mov	0, %o5
40412:
4042	sth	%o5, [%o0]
4043	KPMLOCK_EXIT(%o1, ASI_N)
4044
4045	retl
4046	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4047	SET_SIZE(sfmmu_kpm_tsbmtl)
4048
4049	ENTRY_NP(sfmmu_kpm_stsbmtl)
4050	rdpr	%pstate, %o3
4051	/*
4052	 * %o0 = &mapped
4053	 * %o1 = &kshl_lock
4054	 * %o2 = val
4055	 * %o3 = pstate save
4056	 */
4057#ifdef DEBUG
4058	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4059	bnz,pt %icc, 1f				/* disabled, panic	 */
4060	  nop
4061	save	%sp, -SA(MINFRAME), %sp
4062	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
4063	call	panic
4064	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
4065	ret
4066	restore
40671:
4068#endif /* DEBUG */
4069	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4070
4071	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
4072	ldsb	[%o0], %o5
4073	stb	%o2, [%o0]
4074	KPMLOCK_EXIT(%o1, ASI_N)
4075
4076	mov	%o5, %o0			/* return old val */
4077	retl
4078	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4079	SET_SIZE(sfmmu_kpm_stsbmtl)
4080
4081#endif /* lint */
4082
4083#ifndef lint
4084#ifdef sun4v
4085	/*
4086	 * User/kernel data miss w// multiple TSBs
4087	 * The first probe covers 8K, 64K, and 512K page sizes,
4088	 * because 64K and 512K mappings are replicated off 8K
4089	 * pointer.  Second probe covers 4M page size only.
4090	 *
4091	 * MMU fault area contains miss address and context.
4092	 */
4093	ALTENTRY(sfmmu_slow_dmmu_miss)
4094	GET_MMU_D_TAGACC_CTX(%g2, %g3)	! %g2 = tagacc, %g3 = ctx
4095
4096slow_miss_common:
4097	/*
4098	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
4099	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
4100	 */
4101	brnz,pt	%g3, 8f			! check for user context
4102	  nop
4103
4104	/*
4105	 * Kernel miss
4106	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
4107	 * branch to sfmmu_tsb_miss_tt to handle it.
4108	 */
4109	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4110sfmmu_dslow_patch_ktsb_base:
4111	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
4112sfmmu_dslow_patch_ktsb_szcode:
4113	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
4114
4115	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
4116	! %g1 = First TSB entry pointer, as TSB miss handler expects
4117
4118	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4119sfmmu_dslow_patch_ktsb4m_base:
4120	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
4121sfmmu_dslow_patch_ktsb4m_szcode:
4122	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
4123
4124	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
4125	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
4126	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4127	.empty
4128
41298:
4130	/*
4131	 * User miss
4132	 * Get first TSB pointer in %g1
4133	 * Get second TSB pointer (or NULL if no second TSB) in %g3
4134	 * Branch to sfmmu_tsb_miss_tt to handle it
4135	 */
4136	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
4137	/* %g1 = first TSB entry ptr now, %g2 preserved */
4138
4139	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
4140	brlz,a,pt %g3, sfmmu_tsb_miss_tt	/* done if no 2nd TSB */
4141	  mov	%g0, %g3
4142
4143	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
4144	/* %g3 = second TSB entry ptr now, %g2 preserved */
41459:
4146	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4147	.empty
4148	SET_SIZE(sfmmu_slow_dmmu_miss)
4149
4150
4151	/*
4152	 * User/kernel instruction miss w/ multiple TSBs
4153	 * The first probe covers 8K, 64K, and 512K page sizes,
4154	 * because 64K and 512K mappings are replicated off 8K
4155	 * pointer.  Second probe covers 4M page size only.
4156	 *
4157	 * MMU fault area contains miss address and context.
4158	 */
4159	ALTENTRY(sfmmu_slow_immu_miss)
4160	MMU_FAULT_STATUS_AREA(%g2)
4161	ldx	[%g2 + MMFSA_I_CTX], %g3
4162	ldx	[%g2 + MMFSA_I_ADDR], %g2
4163	srlx	%g2, MMU_PAGESHIFT, %g2	! align address to page boundry
4164	sllx	%g2, MMU_PAGESHIFT, %g2
4165	ba,pt	%xcc, slow_miss_common
4166	or	%g2, %g3, %g2
4167	SET_SIZE(sfmmu_slow_immu_miss)
4168
4169#endif /* sun4v */
4170#endif	/* lint */
4171
4172#ifndef lint
4173
4174/*
4175 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
4176 */
4177	.seg	".data"
4178	.align	64
4179	.global tsbmiss_area
4180tsbmiss_area:
4181	.skip	(TSBMISS_SIZE * NCPU)
4182
4183	.align	64
4184	.global kpmtsbm_area
4185kpmtsbm_area:
4186	.skip	(KPMTSBM_SIZE * NCPU)
4187#endif	/* lint */
4188