1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27/*
28 * SFMMU primitives.  These primitives should only be used by sfmmu
29 * routines.
30 */
31
32#if defined(lint)
33#include <sys/types.h>
34#else	/* lint */
35#include "assym.h"
36#endif	/* lint */
37
38#include <sys/asm_linkage.h>
39#include <sys/machtrap.h>
40#include <sys/machasi.h>
41#include <sys/sun4asi.h>
42#include <sys/pte.h>
43#include <sys/mmu.h>
44#include <vm/hat_sfmmu.h>
45#include <vm/seg_spt.h>
46#include <sys/machparam.h>
47#include <sys/privregs.h>
48#include <sys/scb.h>
49#include <sys/intreg.h>
50#include <sys/machthread.h>
51#include <sys/intr.h>
52#include <sys/clock.h>
53#include <sys/trapstat.h>
54
55#ifdef TRAPTRACE
56#include <sys/traptrace.h>
57
58/*
59 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
60 */
61#define	TT_TRACE(label)		\
62	ba	label		;\
63	rd	%pc, %g7
64#else
65
66#define	TT_TRACE(label)
67
68#endif /* TRAPTRACE */
69
70#ifndef	lint
71
72#if (TTE_SUSPEND_SHIFT > 0)
73#define	TTE_SUSPEND_INT_SHIFT(reg)				\
74	sllx	reg, TTE_SUSPEND_SHIFT, reg
75#else
76#define	TTE_SUSPEND_INT_SHIFT(reg)
77#endif
78
79#endif /* lint */
80
81#ifndef	lint
82
83/*
84 * Assumes TSBE_TAG is 0
85 * Assumes TSBE_INTHI is 0
86 * Assumes TSBREG.split is 0
87 */
88
89#if TSBE_TAG != 0
90#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
91#endif
92
93#if TSBTAG_INTHI != 0
94#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
95#endif
96
97/*
98 * The following code assumes the tsb is not split.
99 *
100 * With TSBs no longer shared between processes, it's no longer
101 * necessary to hash the context bits into the tsb index to get
102 * tsb coloring; the new implementation treats the TSB as a
103 * direct-mapped, virtually-addressed cache.
104 *
105 * In:
106 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
107 *    tsbbase = base address of TSB (clobbered)
108 *    tagacc = tag access register (clobbered)
109 *    szc = size code of TSB (ro)
110 *    tmp = scratch reg
111 * Out:
112 *    tsbbase = pointer to entry in TSB
113 */
114#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
115	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
116	srlx	tagacc, vpshift, tagacc 				;\
117	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
118	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
119	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
120	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
121	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */
122
123/*
124 * When the kpm TSB is used it is assumed that it is direct mapped
125 * using (vaddr>>vpshift)%tsbsz as the index.
126 *
127 * Note that, for now, the kpm TSB and kernel TSB are the same for
128 * each mapping size.  However that need not always be the case.  If
129 * the trap handlers are updated to search a different TSB for kpm
130 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
131 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
132 *
133 * In:
134 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
135 *    vaddr = virtual address (clobbered)
136 *    tsbp, szc, tmp = scratch
137 * Out:
138 *    tsbp = pointer to entry in TSB
139 */
140#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
141	cmp	vpshift, MMU_PAGESHIFT					;\
142	bne,pn	%icc, 1f		/* branch if large case */	;\
143	  sethi	%hi(kpmsm_tsbsz), szc					;\
144	sethi	%hi(kpmsm_tsbbase), tsbp				;\
145	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
146	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
147	ba,pt	%icc, 2f						;\
148	  nop								;\
1491:	sethi	%hi(kpm_tsbsz), szc					;\
150	sethi	%hi(kpm_tsbbase), tsbp					;\
151	ld	[szc + %lo(kpm_tsbsz)], szc				;\
152	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
1532:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
154
155/*
156 * Lock the TSBE at virtual address tsbep.
157 *
158 * tsbep = TSBE va (ro)
159 * tmp1, tmp2 = scratch registers (clobbered)
160 * label = label to jump to if we fail to lock the tsb entry
161 * %asi = ASI to use for TSB access
162 *
163 * NOTE that we flush the TSB using fast VIS instructions that
164 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
165 * not be treated as a locked entry or we'll get stuck spinning on
166 * an entry that isn't locked but really invalid.
167 */
168
169#if defined(UTSB_PHYS)
170
171#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
172	lda	[tsbep]ASI_MEM, tmp1					;\
173	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
174	cmp	tmp1, tmp2 						;\
175	be,a,pn	%icc, label		/* if locked ignore */		;\
176	  nop								;\
177	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
178	cmp	tmp1, tmp2 						;\
179	bne,a,pn %icc, label		/* didn't lock so ignore */	;\
180	  nop								;\
181	/* tsbe lock acquired */					;\
182	membar #StoreStore
183
184#else /* UTSB_PHYS */
185
186#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
187	lda	[tsbep]%asi, tmp1					;\
188	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
189	cmp	tmp1, tmp2 						;\
190	be,a,pn	%icc, label		/* if locked ignore */		;\
191	  nop								;\
192	casa	[tsbep]%asi, tmp1, tmp2					;\
193	cmp	tmp1, tmp2 						;\
194	bne,a,pn %icc, label		/* didn't lock so ignore */	;\
195	  nop								;\
196	/* tsbe lock acquired */					;\
197	membar #StoreStore
198
199#endif /* UTSB_PHYS */
200
201/*
202 * Atomically write TSBE at virtual address tsbep.
203 *
204 * tsbep = TSBE va (ro)
205 * tte = TSBE TTE (ro)
206 * tagtarget = TSBE tag (ro)
207 * %asi = ASI to use for TSB access
208 */
209
210#if defined(UTSB_PHYS)
211
212#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
213	add	tsbep, TSBE_TTE, tmp1					;\
214	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
215	membar #StoreStore						;\
216	add	tsbep, TSBE_TAG, tmp1					;\
217	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */
218
219#else /* UTSB_PHYS */
220
221#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
222	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
223	membar #StoreStore						;\
224	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
225
226#endif /* UTSB_PHYS */
227
228/*
229 * Load an entry into the TSB at TL > 0.
230 *
231 * tsbep = pointer to the TSBE to load as va (ro)
232 * tte = value of the TTE retrieved and loaded (wo)
233 * tagtarget = tag target register.  To get TSBE tag to load,
234 *   we need to mask off the context and leave only the va (clobbered)
235 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
236 * tmp1, tmp2 = scratch registers
237 * label = label to jump to if we fail to lock the tsb entry
238 * %asi = ASI to use for TSB access
239 */
240
241#if defined(UTSB_PHYS)
242
243#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
244	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
245	/*								;\
246	 * I don't need to update the TSB then check for the valid tte.	;\
247	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
248	 * we always invalidate the hash table before we unload the TSB.;\
249	 */								;\
250	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
251	ldxa	[ttepa]ASI_MEM, tte					;\
252	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
253	sethi	%hi(TSBTAG_INVALID), tmp2				;\
254	add	tsbep, TSBE_TAG, tmp1					;\
255	brgez,a,pn tte, label						;\
256	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
257	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
258label:
259
260#else /* UTSB_PHYS */
261
262#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
263	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
264	/*								;\
265	 * I don't need to update the TSB then check for the valid tte.	;\
266	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
267	 * we always invalidate the hash table before we unload the TSB.;\
268	 */								;\
269	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
270	ldxa	[ttepa]ASI_MEM, tte					;\
271	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
272	sethi	%hi(TSBTAG_INVALID), tmp2				;\
273	brgez,a,pn tte, label						;\
274	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
275	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
276label:
277
278#endif /* UTSB_PHYS */
279
280/*
281 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
282 *   for ITLB synthesis.
283 *
284 * tsbep = pointer to the TSBE to load as va (ro)
285 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
286 *   with exec_perm turned off and exec_synth turned on
287 * tagtarget = tag target register.  To get TSBE tag to load,
288 *   we need to mask off the context and leave only the va (clobbered)
289 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
290 * tmp1, tmp2 = scratch registers
291 * label = label to use for branch (text)
292 * %asi = ASI to use for TSB access
293 */
294
295#define	TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
296	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
297	/*								;\
298	 * I don't need to update the TSB then check for the valid tte.	;\
299	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
300	 * we always invalidate the hash table before we unload the TSB.;\
301	 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0	;\
302	 * and exec_synth bit to 1.					;\
303	 */								;\
304	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
305	mov	tte, tmp1						;\
306	ldxa	[ttepa]ASI_MEM, tte					;\
307	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
308	sethi	%hi(TSBTAG_INVALID), tmp2				;\
309	brgez,a,pn tte, label						;\
310	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
311	or	tte, tmp1, tte						;\
312	andn	tte, TTE_EXECPRM_INT, tte				;\
313	or	tte, TTE_E_SYNTH_INT, tte				;\
314	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
315label:
316
317/*
318 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
319 *
320 * tte = value of the TTE, used to get tte_size bits (ro)
321 * tagaccess = tag access register, used to get 4M pfn bits (ro)
322 * pfn = 4M pfn bits shifted to offset for tte (out)
323 * tmp1 = scratch register
324 * label = label to use for branch (text)
325 */
326
327#define	GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)			\
328	/*								;\
329	 * Get 4M bits from tagaccess for 32M, 256M pagesizes.		;\
330	 * Return them, shifted, in pfn.				;\
331	 */								;\
332	srlx	tagaccess, MMU_PAGESHIFT4M, tagaccess			;\
333	srlx	tte, TTE_SZ_SHFT, tmp		/* isolate the */	;\
334	andcc	tmp, TTE_SZ_BITS, %g0		/* tte_size bits */	;\
335	bz,a,pt %icc, label/**/f		/* if 0, is */		;\
336	  and	tagaccess, 0x7, tagaccess	/* 32M page size */	;\
337	and	tagaccess, 0x3f, tagaccess /* else 256M page size */	;\
338label:									;\
339	sllx	tagaccess, MMU_PAGESHIFT4M, pfn
340
341/*
342 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
343 * for ITLB synthesis.
344 *
345 * tte = value of the TTE, used to get tte_size bits (rw)
346 * tmp1 = scratch register
347 */
348
349#define	SET_TTE4M_PN(tte, tmp)						\
350	/*								;\
351	 * Set 4M pagesize tte bits. 					;\
352	 */								;\
353	set	TTE4M, tmp						;\
354	sllx	tmp, TTE_SZ_SHFT, tmp					;\
355	or	tte, tmp, tte
356
357/*
358 * Load an entry into the TSB at TL=0.
359 *
360 * tsbep = pointer to the TSBE to load as va (ro)
361 * tteva = pointer to the TTE to load as va (ro)
362 * tagtarget = TSBE tag to load (which contains no context), synthesized
363 * to match va of MMU tag target register only (ro)
364 * tmp1, tmp2 = scratch registers (clobbered)
365 * label = label to use for branches (text)
366 * %asi = ASI to use for TSB access
367 */
368
369#if defined(UTSB_PHYS)
370
371#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
372	/* can't rd tteva after locking tsb because it can tlb miss */	;\
373	ldx	[tteva], tteva			/* load tte */		;\
374	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
375	sethi	%hi(TSBTAG_INVALID), tmp2				;\
376	add	tsbep, TSBE_TAG, tmp1					;\
377	brgez,a,pn tteva, label						;\
378	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
379	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
380label:
381
382#else /* UTSB_PHYS */
383
384#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
385	/* can't rd tteva after locking tsb because it can tlb miss */	;\
386	ldx	[tteva], tteva			/* load tte */		;\
387	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
388	sethi	%hi(TSBTAG_INVALID), tmp2				;\
389	brgez,a,pn tteva, label						;\
390	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
391	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
392label:
393
394#endif /* UTSB_PHYS */
395
396/*
397 * Invalidate a TSB entry in the TSB.
398 *
399 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
400 *	 about this earlier to ensure this is true.  Thus when we are
401 *	 directly referencing tsbep below, we are referencing the tte_tag
402 *	 field of the TSBE.  If this  offset ever changes, the code below
403 *	 will need to be modified.
404 *
405 * tsbep = pointer to TSBE as va (ro)
406 * tag = invalidation is done if this matches the TSBE tag (ro)
407 * tmp1 - tmp3 = scratch registers (clobbered)
408 * label = label name to use for branches (text)
409 * %asi = ASI to use for TSB access
410 */
411
412#if defined(UTSB_PHYS)
413
414#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
415	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
416	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
417label/**/1:								;\
418	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
419	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
420	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
421	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
422	cmp	tag, tmp3		/* compare tags */		;\
423	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
424	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
425	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
426	cmp	tmp1, tmp3		/* if not successful */		;\
427	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
428	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
429label/**/2:
430
431#else /* UTSB_PHYS */
432
433#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
434	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
435	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
436label/**/1:								;\
437	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
438	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
439	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
440	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
441	cmp	tag, tmp3		/* compare tags */		;\
442	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
443	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
444	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
445	cmp	tmp1, tmp3		/* if not successful */		;\
446	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
447	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
448label/**/2:
449
450#endif /* UTSB_PHYS */
451
452#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
453#error	- TSB_SOFTSZ_MASK too small
454#endif
455
456
457/*
458 * An implementation of setx which will be hot patched at run time.
459 * since it is being hot patched, there is no value passed in.
460 * Thus, essentially we are implementing
461 *	setx value, tmp, dest
462 * where value is RUNTIME_PATCH (aka 0) in this case.
463 */
464#define	RUNTIME_PATCH_SETX(dest, tmp)					\
465	sethi	%hh(RUNTIME_PATCH), tmp					;\
466	sethi	%lm(RUNTIME_PATCH), dest				;\
467	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
468	or	dest, %lo(RUNTIME_PATCH), dest				;\
469	sllx	tmp, 32, tmp						;\
470	nop				/* for perf reasons */		;\
471	or	tmp, dest, dest		/* contents of patched value */
472
473#endif /* lint */
474
475
476#if defined (lint)
477
478/*
479 * sfmmu related subroutines
480 */
481uint_t
482sfmmu_disable_intrs()
483{ return(0); }
484
485/* ARGSUSED */
486void
487sfmmu_enable_intrs(uint_t pstate_save)
488{}
489
490/* ARGSUSED */
491int
492sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag)
493{ return(0); }
494
495/*
496 * Use cas, if tte has changed underneath us then reread and try again.
497 * In the case of a retry, it will update sttep with the new original.
498 */
499/* ARGSUSED */
500int
501sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
502{ return(0); }
503
504/*
505 * Use cas, if tte has changed underneath us then return 1, else return 0
506 */
507/* ARGSUSED */
508int
509sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
510{ return(0); }
511
512/* ARGSUSED */
513void
514sfmmu_copytte(tte_t *sttep, tte_t *dttep)
515{}
516
517/*ARGSUSED*/
518struct tsbe *
519sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
520{ return(0); }
521
522/*ARGSUSED*/
523uint64_t
524sfmmu_make_tsbtag(caddr_t va)
525{ return(0); }
526
527#else	/* lint */
528
529	.seg	".data"
530	.global	sfmmu_panic1
531sfmmu_panic1:
532	.asciz	"sfmmu_asm: interrupts already disabled"
533
534	.global	sfmmu_panic3
535sfmmu_panic3:
536	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"
537
538	.global	sfmmu_panic4
539sfmmu_panic4:
540	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"
541
542	.global	sfmmu_panic5
543sfmmu_panic5:
544	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"
545
546	.global	sfmmu_panic6
547sfmmu_panic6:
548	.asciz	"sfmmu_asm: interrupts not disabled"
549
550	.global	sfmmu_panic7
551sfmmu_panic7:
552	.asciz	"sfmmu_asm: kernel as"
553
554	.global	sfmmu_panic8
555sfmmu_panic8:
556	.asciz	"sfmmu_asm: gnum is zero"
557
558	.global	sfmmu_panic9
559sfmmu_panic9:
560	.asciz	"sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL"
561
562	.global	sfmmu_panic10
563sfmmu_panic10:
564	.asciz	"sfmmu_asm: valid SCD with no 3rd scd TSB"
565
566	.global	sfmmu_panic11
567sfmmu_panic11:
568	.asciz	"sfmmu_asm: ktsb_phys must not be 0 on a sun4v platform"
569
570        ENTRY(sfmmu_disable_intrs)
571        rdpr    %pstate, %o0
572#ifdef DEBUG
573	PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1)
574#endif /* DEBUG */
575        retl
576          wrpr   %o0, PSTATE_IE, %pstate
577        SET_SIZE(sfmmu_disable_intrs)
578
579	ENTRY(sfmmu_enable_intrs)
580        retl
581          wrpr    %g0, %o0, %pstate
582        SET_SIZE(sfmmu_enable_intrs)
583
584/*
585 * This routine is called both by resume() and sfmmu_get_ctx() to
586 * allocate a new context for the process on a MMU.
587 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID .
588 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which
589 * is the case when sfmmu_alloc_ctx is called from resume().
590 *
591 * The caller must disable interrupts before entering this routine.
592 * To reduce ctx switch overhead, the code contains both 'fast path' and
593 * 'slow path' code. The fast path code covers the common case where only
594 * a quick check is needed and the real ctx allocation is not required.
595 * It can be done without holding the per-process (PP) lock.
596 * The 'slow path' code must be protected by the PP Lock and performs ctx
597 * allocation.
598 * Hardware context register and HAT mmu cnum are updated accordingly.
599 *
600 * %o0 - sfmmup
601 * %o1 - allocflag
602 * %o2 - CPU
603 * %o3 - sfmmu private/shared flag
604 *
605 * ret - 0: no ctx is allocated
606 *       1: a ctx is allocated
607 */
608        ENTRY_NP(sfmmu_alloc_ctx)
609
610#ifdef DEBUG
611	sethi   %hi(ksfmmup), %g1
612	ldx     [%g1 + %lo(ksfmmup)], %g1
613	cmp     %g1, %o0
614	bne,pt   %xcc, 0f
615	  nop
616
617	sethi   %hi(panicstr), %g1		! if kernel as, panic
618        ldx     [%g1 + %lo(panicstr)], %g1
619        tst     %g1
620        bnz,pn  %icc, 7f
621          nop
622
623	sethi	%hi(sfmmu_panic7), %o0
624	call	panic
625	  or	%o0, %lo(sfmmu_panic7), %o0
626
6277:
628	retl
629	  mov	%g0, %o0			! %o0 = ret = 0
630
6310:
632	PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1)
633#endif /* DEBUG */
634
635	mov	%o3, %g1			! save sfmmu pri/sh flag in %g1
636
637	! load global mmu_ctxp info
638	ldx	[%o2 + CPU_MMU_CTXP], %o3		! %o3 = mmu_ctx_t ptr
639
640#ifdef sun4v
641	/* During suspend on sun4v, context domains can be temporary removed */
642	brz,a,pn       %o3, 0f
643	  nop
644#endif
645
646        lduw	[%o2 + CPU_MMU_IDX], %g2		! %g2 = mmu index
647
648	! load global mmu_ctxp gnum
649	ldx	[%o3 + MMU_CTX_GNUM], %o4		! %o4 = mmu_ctxp->gnum
650
651#ifdef DEBUG
652	cmp	%o4, %g0		! mmu_ctxp->gnum should never be 0
653	bne,pt	%xcc, 3f
654	  nop
655
656	sethi   %hi(panicstr), %g1	! test if panicstr is already set
657        ldx     [%g1 + %lo(panicstr)], %g1
658        tst     %g1
659        bnz,pn  %icc, 1f
660          nop
661
662	sethi	%hi(sfmmu_panic8), %o0
663	call	panic
664	  or	%o0, %lo(sfmmu_panic8), %o0
6651:
666	retl
667	  mov	%g0, %o0			! %o0 = ret = 0
6683:
669#endif
670
671	! load HAT sfmmu_ctxs[mmuid] gnum, cnum
672
673	sllx	%g2, SFMMU_MMU_CTX_SHIFT, %g2
674	add	%o0, %g2, %g2		! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
675
676	/*
677	 * %g5 = sfmmu gnum returned
678	 * %g6 = sfmmu cnum returned
679	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
680	 * %g4 = scratch
681	 *
682	 * Fast path code, do a quick check.
683	 */
684	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
685
686	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
687	bne,pt	%icc, 1f			! valid hat cnum, check gnum
688	  nop
689
690	! cnum == INVALID, check allocflag
691	mov	%g0, %g4	! %g4 = ret = 0
692	brz,pt  %o1, 8f		! allocflag == 0, skip ctx allocation, bail
693	  mov	%g6, %o1
694
695	! (invalid HAT cnum) && (allocflag == 1)
696	ba,pt	%icc, 2f
697	  nop
698#ifdef sun4v
6990:
700	set	INVALID_CONTEXT, %o1
701	membar	#LoadStore|#StoreStore
702	ba,pt	%icc, 8f
703	  mov   %g0, %g4                ! %g4 = ret = 0
704#endif
7051:
706	! valid HAT cnum, check gnum
707	cmp	%g5, %o4
708	mov	1, %g4				!%g4 = ret = 1
709	be,a,pt	%icc, 8f			! gnum unchanged, go to done
710	  mov	%g6, %o1
711
7122:
713	/*
714	 * Grab per process (PP) sfmmu_ctx_lock spinlock,
715	 * followed by the 'slow path' code.
716	 */
717	ldstub	[%o0 + SFMMU_CTX_LOCK], %g3	! %g3 = per process (PP) lock
7183:
719	brz	%g3, 5f
720	  nop
7214:
722	brnz,a,pt       %g3, 4b				! spin if lock is 1
723	  ldub	[%o0 + SFMMU_CTX_LOCK], %g3
724	ba	%xcc, 3b				! retry the lock
725	  ldstub	[%o0 + SFMMU_CTX_LOCK], %g3    ! %g3 = PP lock
726
7275:
728	membar  #LoadLoad
729	/*
730	 * %g5 = sfmmu gnum returned
731	 * %g6 = sfmmu cnum returned
732	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
733	 * %g4 = scratch
734	 */
735	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
736
737	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
738	bne,pt	%icc, 1f			! valid hat cnum, check gnum
739	  nop
740
741	! cnum == INVALID, check allocflag
742	mov	%g0, %g4	! %g4 = ret = 0
743	brz,pt	%o1, 2f		! allocflag == 0, called from resume, set hw
744	  mov	%g6, %o1
745
746	! (invalid HAT cnum) && (allocflag == 1)
747	ba,pt	%icc, 6f
748	  nop
7491:
750	! valid HAT cnum, check gnum
751	cmp	%g5, %o4
752	mov	1, %g4				! %g4 = ret  = 1
753	be,a,pt	%icc, 2f			! gnum unchanged, go to done
754	  mov	%g6, %o1
755
756	ba,pt	%icc, 6f
757	  nop
7582:
759	membar  #LoadStore|#StoreStore
760	ba,pt %icc, 8f
761	  clrb  [%o0 + SFMMU_CTX_LOCK]
7626:
763	/*
764	 * We get here if we do not have a valid context, or
765	 * the HAT gnum does not match global gnum. We hold
766	 * sfmmu_ctx_lock spinlock. Allocate that context.
767	 *
768	 * %o3 = mmu_ctxp
769	 */
770	add	%o3, MMU_CTX_CNUM, %g3
771	ld	[%o3 + MMU_CTX_NCTXS], %g4
772
773	/*
774         * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS;
775         * %g3 = mmu cnum address
776	 * %g4 = mmu nctxs
777	 *
778	 * %o0 = sfmmup
779	 * %o1 = mmu current cnum value (used as new cnum)
780	 * %o4 = mmu gnum
781	 *
782	 * %o5 = scratch
783	 */
784	ld	[%g3], %o1
7850:
786	cmp	%o1, %g4
787	bl,a,pt %icc, 1f
788	  add	%o1, 1, %o5		! %o5 = mmu_ctxp->cnum + 1
789
790	/*
791	 * cnum reachs max, bail, so wrap around can be performed later.
792	 */
793	set	INVALID_CONTEXT, %o1
794	mov	%g0, %g4		! %g4 = ret = 0
795
796	membar  #LoadStore|#StoreStore
797	ba,pt	%icc, 8f
798	  clrb	[%o0 + SFMMU_CTX_LOCK]
7991:
800	! %g3 = addr of mmu_ctxp->cnum
801	! %o5 = mmu_ctxp->cnum + 1
802	cas	[%g3], %o1, %o5
803	cmp	%o1, %o5
804	bne,a,pn %xcc, 0b	! cas failed
805	  ld	[%g3], %o1
806
807#ifdef DEBUG
808        set	MAX_SFMMU_CTX_VAL, %o5
809	cmp	%o1, %o5
810	ble,pt %icc, 2f
811	  nop
812
813	sethi	%hi(sfmmu_panic9), %o0
814	call	panic
815	  or	%o0, %lo(sfmmu_panic9), %o0
8162:
817#endif
818	! update hat gnum and cnum
819	sllx	%o4, SFMMU_MMU_GNUM_RSHIFT, %o4
820	or	%o4, %o1, %o4
821	stx	%o4, [%g2 + SFMMU_CTXS]
822
823	membar  #LoadStore|#StoreStore
824	clrb	[%o0 + SFMMU_CTX_LOCK]
825
826	mov	1, %g4			! %g4 = ret = 1
8278:
828	/*
829	 * program the secondary context register
830	 *
831	 * %o1 = cnum
832	 * %g1 = sfmmu private/shared flag (0:private,  1:shared)
833	 */
834
835	/*
836	 * When we come here and context is invalid, we want to set both
837	 * private and shared ctx regs to INVALID. In order to
838	 * do so, we set the sfmmu priv/shared flag to 'private' regardless
839	 * so that private ctx reg will be set to invalid.
840	 * Note that on sun4v values written to private context register are
841	 * automatically written to corresponding shared context register as
842	 * well. On sun4u SET_SECCTX() will invalidate shared context register
843	 * when it sets a private secondary context register.
844	 */
845
846	cmp	%o1, INVALID_CONTEXT
847	be,a,pn	%icc, 9f
848	  clr	%g1
8499:
850
851#ifdef	sun4u
852	ldub	[%o0 + SFMMU_CEXT], %o2
853	sll	%o2, CTXREG_EXT_SHIFT, %o2
854	or	%o1, %o2, %o1
855#endif /* sun4u */
856
857	SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1)
858
859        retl
860          mov   %g4, %o0                        ! %o0 = ret
861
862	SET_SIZE(sfmmu_alloc_ctx)
863
864	ENTRY_NP(sfmmu_modifytte)
865	ldx	[%o2], %g3			/* current */
866	ldx	[%o0], %g1			/* original */
8672:
868	ldx	[%o1], %g2			/* modified */
869	cmp	%g2, %g3			/* is modified = current? */
870	be,a,pt	%xcc,1f				/* yes, don't write */
871	stx	%g3, [%o0]			/* update new original */
872	casx	[%o2], %g1, %g2
873	cmp	%g1, %g2
874	be,pt	%xcc, 1f			/* cas succeeded - return */
875	  nop
876	ldx	[%o2], %g3			/* new current */
877	stx	%g3, [%o0]			/* save as new original */
878	ba,pt	%xcc, 2b
879	  mov	%g3, %g1
8801:	retl
881	membar	#StoreLoad
882	SET_SIZE(sfmmu_modifytte)
883
884	ENTRY_NP(sfmmu_modifytte_try)
885	ldx	[%o1], %g2			/* modified */
886	ldx	[%o2], %g3			/* current */
887	ldx	[%o0], %g1			/* original */
888	cmp	%g3, %g2			/* is modified = current? */
889	be,a,pn %xcc,1f				/* yes, don't write */
890	mov	0, %o1				/* as if cas failed. */
891
892	casx	[%o2], %g1, %g2
893	membar	#StoreLoad
894	cmp	%g1, %g2
895	movne	%xcc, -1, %o1			/* cas failed. */
896	move	%xcc, 1, %o1			/* cas succeeded. */
8971:
898	stx	%g2, [%o0]			/* report "current" value */
899	retl
900	mov	%o1, %o0
901	SET_SIZE(sfmmu_modifytte_try)
902
903	ENTRY_NP(sfmmu_copytte)
904	ldx	[%o0], %g1
905	retl
906	stx	%g1, [%o1]
907	SET_SIZE(sfmmu_copytte)
908
909
910	/*
911	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
912	 * %o0 = TSB base address (in), pointer to TSB entry (out)
913	 * %o1 = vaddr (in)
914	 * %o2 = vpshift (in)
915	 * %o3 = tsb size code (in)
916	 * %o4 = scratch register
917	 */
918	ENTRY_NP(sfmmu_get_tsbe)
919	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
920	retl
921	nop
922	SET_SIZE(sfmmu_get_tsbe)
923
924	/*
925	 * Return a TSB tag for the given va.
926	 * %o0 = va (in/clobbered)
927	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
928	 */
929	ENTRY_NP(sfmmu_make_tsbtag)
930	retl
931	srln	%o0, TTARGET_VA_SHIFT, %o0
932	SET_SIZE(sfmmu_make_tsbtag)
933
934#endif /* lint */
935
936/*
937 * Other sfmmu primitives
938 */
939
940
941#if defined (lint)
942void
943sfmmu_patch_ktsb(void)
944{
945}
946
947void
948sfmmu_kpm_patch_tlbm(void)
949{
950}
951
952void
953sfmmu_kpm_patch_tsbm(void)
954{
955}
956
957void
958sfmmu_patch_shctx(void)
959{
960}
961
962/* ARGSUSED */
963void
964sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
965{
966}
967
968/* ARGSUSED */
969void
970sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
971{
972}
973
974/* ARGSUSED */
975void
976sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
977{
978}
979
980/* ARGSUSED */
981void
982sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
983{
984}
985
986#else /* lint */
987
988#define	I_SIZE		4
989
990	ENTRY_NP(sfmmu_fix_ktlb_traptable)
991	/*
992	 * %o0 = start of patch area
993	 * %o1 = size code of TSB to patch
994	 * %o3 = scratch
995	 */
996	/* fix sll */
997	ld	[%o0], %o3			/* get sll */
998	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
999	st	%o3, [%o0]			/* write sll */
1000	flush	%o0
1001	/* fix srl */
1002	add	%o0, I_SIZE, %o0		/* goto next instr. */
1003	ld	[%o0], %o3			/* get srl */
1004	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
1005	st	%o3, [%o0]			/* write srl */
1006	retl
1007	flush	%o0
1008	SET_SIZE(sfmmu_fix_ktlb_traptable)
1009
1010	ENTRY_NP(sfmmu_fixup_ktsbbase)
1011	/*
1012	 * %o0 = start of patch area
1013	 * %o5 = kernel virtual or physical tsb base address
1014	 * %o2, %o3 are used as scratch registers.
1015	 */
1016	/* fixup sethi instruction */
1017	ld	[%o0], %o3
1018	srl	%o5, 10, %o2			! offset is bits 32:10
1019	or	%o3, %o2, %o3			! set imm22
1020	st	%o3, [%o0]
1021	/* fixup offset of lduw/ldx */
1022	add	%o0, I_SIZE, %o0		! next instr
1023	ld	[%o0], %o3
1024	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
1025	or	%o3, %o2, %o3
1026	st	%o3, [%o0]
1027	retl
1028	flush	%o0
1029	SET_SIZE(sfmmu_fixup_ktsbbase)
1030
1031	ENTRY_NP(sfmmu_fixup_setx)
1032	/*
1033	 * %o0 = start of patch area
1034	 * %o4 = 64 bit value to patch
1035	 * %o2, %o3 are used as scratch registers.
1036	 *
1037	 * Note: Assuming that all parts of the instructions which need to be
1038	 *	 patched correspond to RUNTIME_PATCH (aka 0)
1039	 *
1040	 * Note the implementation of setx which is being patched is as follows:
1041	 *
1042	 * sethi   %hh(RUNTIME_PATCH), tmp
1043	 * sethi   %lm(RUNTIME_PATCH), dest
1044	 * or      tmp, %hm(RUNTIME_PATCH), tmp
1045	 * or      dest, %lo(RUNTIME_PATCH), dest
1046	 * sllx    tmp, 32, tmp
1047	 * nop
1048	 * or      tmp, dest, dest
1049	 *
1050	 * which differs from the implementation in the
1051	 * "SPARC Architecture Manual"
1052	 */
1053	/* fixup sethi instruction */
1054	ld	[%o0], %o3
1055	srlx	%o4, 42, %o2			! bits [63:42]
1056	or	%o3, %o2, %o3			! set imm22
1057	st	%o3, [%o0]
1058	/* fixup sethi instruction */
1059	add	%o0, I_SIZE, %o0		! next instr
1060	ld	[%o0], %o3
1061	sllx	%o4, 32, %o2			! clear upper bits
1062	srlx	%o2, 42, %o2			! bits [31:10]
1063	or	%o3, %o2, %o3			! set imm22
1064	st	%o3, [%o0]
1065	/* fixup or instruction */
1066	add	%o0, I_SIZE, %o0		! next instr
1067	ld	[%o0], %o3
1068	srlx	%o4, 32, %o2			! bits [63:32]
1069	and	%o2, 0x3ff, %o2			! bits [41:32]
1070	or	%o3, %o2, %o3			! set imm
1071	st	%o3, [%o0]
1072	/* fixup or instruction */
1073	add	%o0, I_SIZE, %o0		! next instr
1074	ld	[%o0], %o3
1075	and	%o4, 0x3ff, %o2			! bits [9:0]
1076	or	%o3, %o2, %o3			! set imm
1077	st	%o3, [%o0]
1078	retl
1079	flush	%o0
1080	SET_SIZE(sfmmu_fixup_setx)
1081
1082	ENTRY_NP(sfmmu_fixup_or)
1083	/*
1084	 * %o0 = start of patch area
1085	 * %o4 = 32 bit value to patch
1086	 * %o2, %o3 are used as scratch registers.
1087	 * Note: Assuming that all parts of the instructions which need to be
1088	 *	 patched correspond to RUNTIME_PATCH (aka 0)
1089	 */
1090	ld	[%o0], %o3
1091	and	%o4, 0x3ff, %o2			! bits [9:0]
1092	or	%o3, %o2, %o3			! set imm
1093	st	%o3, [%o0]
1094	retl
1095	flush	%o0
1096	SET_SIZE(sfmmu_fixup_or)
1097
1098	ENTRY_NP(sfmmu_fixup_shiftx)
1099	/*
1100	 * %o0 = start of patch area
1101	 * %o4 = signed int immediate value to add to sllx/srlx imm field
1102	 * %o2, %o3 are used as scratch registers.
1103	 *
1104	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
1105	 * so we do a simple add.  The caller must be careful to prevent
1106	 * overflow, which could easily occur if the initial value is nonzero!
1107	 */
1108	ld	[%o0], %o3			! %o3 = instruction to patch
1109	and	%o3, 0x3f, %o2			! %o2 = existing imm value
1110	add	%o2, %o4, %o2			! %o2 = new imm value
1111	andn	%o3, 0x3f, %o3			! clear old imm value
1112	and	%o2, 0x3f, %o2			! truncate new imm value
1113	or	%o3, %o2, %o3			! set new imm value
1114	st	%o3, [%o0]			! store updated instruction
1115	retl
1116	flush	%o0
1117	SET_SIZE(sfmmu_fixup_shiftx)
1118
1119	ENTRY_NP(sfmmu_fixup_mmu_asi)
1120	/*
1121	 * Patch imm_asi of all ldda instructions in the MMU
1122	 * trap handlers.  We search MMU_PATCH_INSTR instructions
1123	 * starting from the itlb miss handler (trap 0x64).
1124	 * %o0 = address of tt[0,1]_itlbmiss
1125	 * %o1 = imm_asi to setup, shifted by appropriate offset.
1126	 * %o3 = number of instructions to search
1127	 * %o4 = reserved by caller: called from leaf routine
1128	 */
11291:	ldsw	[%o0], %o2			! load instruction to %o2
1130	brgez,pt %o2, 2f
1131	  srl	%o2, 30, %o5
1132	btst	1, %o5				! test bit 30; skip if not set
1133	bz,pt	%icc, 2f
1134	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
1135	srlx	%o5, 58, %o5			! isolate op3 part of opcode
1136	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
1137	brnz,pt	%o5, 2f				! skip if not a match
1138	  or	%o2, %o1, %o2			! or in imm_asi
1139	st	%o2, [%o0]			! write patched instruction
11402:	dec	%o3
1141	brnz,a,pt %o3, 1b			! loop until we're done
1142	  add	%o0, I_SIZE, %o0
1143	retl
1144	flush	%o0
1145	SET_SIZE(sfmmu_fixup_mmu_asi)
1146
1147	/*
1148	 * Patch immediate ASI used to access the TSB in the
1149	 * trap table.
1150	 * inputs: %o0 = value of ktsb_phys
1151	 */
1152	ENTRY_NP(sfmmu_patch_mmu_asi)
1153	mov	%o7, %o4			! save return pc in %o4
1154	mov	ASI_QUAD_LDD_PHYS, %o3		! set QUAD_LDD_PHYS by default
1155
1156#ifdef sun4v
1157
1158	/*
1159	 * Check ktsb_phys. It must be non-zero for sun4v, panic if not.
1160	 */
1161
1162	brnz,pt %o0, do_patch
1163	nop
1164
1165	sethi	%hi(sfmmu_panic11), %o0
1166	call	panic
1167	  or	%o0, %lo(sfmmu_panic11), %o0
1168do_patch:
1169
1170#else /* sun4v */
1171	/*
1172	 * Some non-sun4v platforms deploy virtual ktsb (ktsb_phys==0).
1173	 * Note that ASI_NQUAD_LD is not defined/used for sun4v
1174	 */
1175	movrz	%o0, ASI_NQUAD_LD, %o3
1176
1177#endif /* sun4v */
1178
1179	sll	%o3, 5, %o1			! imm_asi offset
1180	mov	6, %o3				! number of instructions
1181	sethi	%hi(dktsb), %o0			! to search
1182	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
1183	  or	%o0, %lo(dktsb), %o0
1184	mov	6, %o3				! number of instructions
1185	sethi	%hi(dktsb4m), %o0		! to search
1186	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
1187	  or	%o0, %lo(dktsb4m), %o0
1188	mov	6, %o3				! number of instructions
1189	sethi	%hi(iktsb), %o0			! to search
1190	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
1191	  or	%o0, %lo(iktsb), %o0
1192	mov	6, %o3				! number of instructions
1193	sethi	%hi(iktsb4m), %o0		! to search
1194	call	sfmmu_fixup_mmu_asi		! patch kitlb4m miss
1195	  or	%o0, %lo(iktsb4m), %o0
1196	mov	%o4, %o7			! retore return pc -- leaf
1197	retl
1198	nop
1199	SET_SIZE(sfmmu_patch_mmu_asi)
1200
1201
1202	ENTRY_NP(sfmmu_patch_ktsb)
1203	/*
1204	 * We need to fix iktsb, dktsb, et. al.
1205	 */
1206	save	%sp, -SA(MINFRAME), %sp
1207	set	ktsb_phys, %o1
1208	ld	[%o1], %o4
1209	set	ktsb_base, %o5
1210	set	ktsb4m_base, %l1
1211	brz,pt	%o4, 1f
1212	  nop
1213	set	ktsb_pbase, %o5
1214	set	ktsb4m_pbase, %l1
12151:
1216	sethi	%hi(ktsb_szcode), %o1
1217	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */
1218
1219	sethi	%hi(iktsb), %o0
1220	call	sfmmu_fix_ktlb_traptable
1221	  or	%o0, %lo(iktsb), %o0
1222
1223	sethi	%hi(dktsb), %o0
1224	call	sfmmu_fix_ktlb_traptable
1225	  or	%o0, %lo(dktsb), %o0
1226
1227	sethi	%hi(ktsb4m_szcode), %o1
1228	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */
1229
1230	sethi	%hi(iktsb4m), %o0
1231	call	sfmmu_fix_ktlb_traptable
1232	  or	%o0, %lo(iktsb4m), %o0
1233
1234	sethi	%hi(dktsb4m), %o0
1235	call	sfmmu_fix_ktlb_traptable
1236	  or	%o0, %lo(dktsb4m), %o0
1237
1238#ifndef sun4v
1239	mov	ASI_N, %o2
1240	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
1241	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
1242	sethi	%hi(tsb_kernel_patch_asi), %o0
1243	call	sfmmu_fixup_or
1244	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
1245#endif /* !sun4v */
1246
1247	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)
1248
1249	sethi	%hi(dktsbbase), %o0
1250	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1251	  or	%o0, %lo(dktsbbase), %o0
1252
1253	sethi	%hi(iktsbbase), %o0
1254	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1255	  or	%o0, %lo(iktsbbase), %o0
1256
1257	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
1258	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1259	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
1260
1261#ifdef sun4v
1262	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
1263	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1264	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
1265#endif /* sun4v */
1266
1267	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)
1268
1269	sethi	%hi(dktsb4mbase), %o0
1270	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1271	  or	%o0, %lo(dktsb4mbase), %o0
1272
1273	sethi	%hi(iktsb4mbase), %o0
1274	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1275	  or	%o0, %lo(iktsb4mbase), %o0
1276
1277	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
1278	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1279	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
1280
1281#ifdef sun4v
1282	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
1283	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1284	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
1285#endif /* sun4v */
1286
1287	set	ktsb_szcode, %o4
1288	ld	[%o4], %o4
1289	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
1290	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1291	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
1292
1293#ifdef sun4v
1294	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
1295	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1296	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
1297#endif /* sun4v */
1298
1299	set	ktsb4m_szcode, %o4
1300	ld	[%o4], %o4
1301	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1302	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1303	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1304
1305#ifdef sun4v
1306	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1307	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1308	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1309#endif /* sun4v */
1310
1311	ret
1312	restore
1313	SET_SIZE(sfmmu_patch_ktsb)
1314
1315	ENTRY_NP(sfmmu_kpm_patch_tlbm)
1316	/*
1317	 * Fixup trap handlers in common segkpm case.  This is reserved
1318	 * for future use should kpm TSB be changed to be other than the
1319	 * kernel TSB.
1320	 */
1321	retl
1322	nop
1323	SET_SIZE(sfmmu_kpm_patch_tlbm)
1324
1325	ENTRY_NP(sfmmu_kpm_patch_tsbm)
1326	/*
1327	 * nop the branch to sfmmu_kpm_dtsb_miss_small
1328	 * in the case where we are using large pages for
1329	 * seg_kpm (and hence must probe the second TSB for
1330	 * seg_kpm VAs)
1331	 */
1332	set	dktsb4m_kpmcheck_small, %o0
1333	MAKE_NOP_INSTR(%o1)
1334	st	%o1, [%o0]
1335	flush	%o0
1336	retl
1337	nop
1338	SET_SIZE(sfmmu_kpm_patch_tsbm)
1339
1340	ENTRY_NP(sfmmu_patch_utsb)
1341#ifdef UTSB_PHYS
1342	retl
1343	nop
1344#else /* UTSB_PHYS */
1345	/*
1346	 * We need to hot patch utsb_vabase and utsb4m_vabase
1347	 */
1348	save	%sp, -SA(MINFRAME), %sp
1349
1350	/* patch value of utsb_vabase */
1351	set	utsb_vabase, %o1
1352	ldx	[%o1], %o4
1353	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1354	call	sfmmu_fixup_setx
1355	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1356	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1357	call	sfmmu_fixup_setx
1358	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1359	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1360	call	sfmmu_fixup_setx
1361	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1362
1363	/* patch value of utsb4m_vabase */
1364	set	utsb4m_vabase, %o1
1365	ldx	[%o1], %o4
1366	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1367	call	sfmmu_fixup_setx
1368	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1369	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1370	call	sfmmu_fixup_setx
1371	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1372	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1373	call	sfmmu_fixup_setx
1374	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1375
1376	/*
1377	 * Patch TSB base register masks and shifts if needed.
1378	 * By default the TSB base register contents are set up for 4M slab.
1379	 * If we're using a smaller slab size and reserved VA range we need
1380	 * to patch up those values here.
1381	 */
1382	set	tsb_slab_shift, %o1
1383	set	MMU_PAGESHIFT4M, %o4
1384	lduw	[%o1], %o3
1385	subcc	%o4, %o3, %o4
1386	bz,pt	%icc, 1f
1387	  /* delay slot safe */
1388
1389	/* patch reserved VA range size if needed. */
1390	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
1391	call	sfmmu_fixup_shiftx
1392	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1393	call	sfmmu_fixup_shiftx
1394	  add	%o0, I_SIZE, %o0
1395	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
1396	call	sfmmu_fixup_shiftx
1397	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1398	call	sfmmu_fixup_shiftx
1399	  add	%o0, I_SIZE, %o0
14001:
1401	/* patch TSBREG_VAMASK used to set up TSB base register */
1402	set	tsb_slab_mask, %o1
1403	ldx	[%o1], %o4
1404	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1405	call	sfmmu_fixup_or
1406	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1407	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1408	call	sfmmu_fixup_or
1409	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1410
1411	ret
1412	restore
1413#endif /* UTSB_PHYS */
1414	SET_SIZE(sfmmu_patch_utsb)
1415
1416	ENTRY_NP(sfmmu_patch_shctx)
1417#ifdef sun4u
1418	retl
1419	  nop
1420#else /* sun4u */
1421	set	sfmmu_shctx_cpu_mondo_patch, %o0
1422	MAKE_JMP_INSTR(5, %o1, %o2)	! jmp       %g5
1423	st	%o1, [%o0]
1424	flush	%o0
1425	MAKE_NOP_INSTR(%o1)
1426	add	%o0, I_SIZE, %o0	! next instr
1427	st	%o1, [%o0]
1428	flush	%o0
1429
1430	set	sfmmu_shctx_user_rtt_patch, %o0
1431	st      %o1, [%o0]		! nop 1st instruction
1432	flush	%o0
1433	add     %o0, I_SIZE, %o0
1434	st      %o1, [%o0]		! nop 2nd instruction
1435	flush	%o0
1436	add     %o0, I_SIZE, %o0
1437	st      %o1, [%o0]		! nop 3rd instruction
1438	flush	%o0
1439	add     %o0, I_SIZE, %o0
1440	st      %o1, [%o0]		! nop 4th instruction
1441	flush	%o0
1442	add     %o0, I_SIZE, %o0
1443	st      %o1, [%o0]		! nop 5th instruction
1444	flush	%o0
1445	add     %o0, I_SIZE, %o0
1446	st      %o1, [%o0]		! nop 6th instruction
1447	retl
1448	flush	%o0
1449#endif /* sun4u */
1450	SET_SIZE(sfmmu_patch_shctx)
1451
1452	/*
1453	 * Routine that loads an entry into a tsb using virtual addresses.
1454	 * Locking is required since all cpus can use the same TSB.
1455	 * Note that it is no longer required to have a valid context
1456	 * when calling this function.
1457	 */
1458	ENTRY_NP(sfmmu_load_tsbe)
1459	/*
1460	 * %o0 = pointer to tsbe to load
1461	 * %o1 = tsb tag
1462	 * %o2 = virtual pointer to TTE
1463	 * %o3 = 1 if physical address in %o0 else 0
1464	 */
1465	rdpr	%pstate, %o5
1466#ifdef DEBUG
1467	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1)
1468#endif /* DEBUG */
1469
1470	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1471
1472	SETUP_TSB_ASI(%o3, %g3)
1473	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8)
1474
1475	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1476
1477	retl
1478	membar	#StoreStore|#StoreLoad
1479	SET_SIZE(sfmmu_load_tsbe)
1480
1481	/*
1482	 * Flush TSB of a given entry if the tag matches.
1483	 */
1484	ENTRY(sfmmu_unload_tsbe)
1485	/*
1486	 * %o0 = pointer to tsbe to be flushed
1487	 * %o1 = tag to match
1488	 * %o2 = 1 if physical address in %o0 else 0
1489	 */
1490	SETUP_TSB_ASI(%o2, %g1)
1491	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1492	retl
1493	membar	#StoreStore|#StoreLoad
1494	SET_SIZE(sfmmu_unload_tsbe)
1495
1496	/*
1497	 * Routine that loads a TTE into the kpm TSB from C code.
1498	 * Locking is required since kpm TSB is shared among all CPUs.
1499	 */
1500	ENTRY_NP(sfmmu_kpm_load_tsb)
1501	/*
1502	 * %o0 = vaddr
1503	 * %o1 = ttep
1504	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1505	 */
1506	rdpr	%pstate, %o5			! %o5 = saved pstate
1507#ifdef DEBUG
1508	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1)
1509#endif /* DEBUG */
1510	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts
1511
1512#ifndef sun4v
1513	sethi	%hi(ktsb_phys), %o4
1514	mov	ASI_N, %o3
1515	ld	[%o4 + %lo(ktsb_phys)], %o4
1516	movrnz	%o4, ASI_MEM, %o3
1517	mov	%o3, %asi
1518#endif /* !sun4v */
1519	mov	%o0, %g1			! %g1 = vaddr
1520
1521	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1522	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1523	/* %g2 = tsbep, %g1 clobbered */
1524
1525	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1526	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1527	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9)
1528
1529	wrpr	%g0, %o5, %pstate		! enable interrupts
1530	retl
1531	  membar #StoreStore|#StoreLoad
1532	SET_SIZE(sfmmu_kpm_load_tsb)
1533
1534	/*
1535	 * Routine that shoots down a TTE in the kpm TSB or in the
1536	 * kernel TSB depending on virtpg. Locking is required since
1537	 * kpm/kernel TSB is shared among all CPUs.
1538	 */
1539	ENTRY_NP(sfmmu_kpm_unload_tsb)
1540	/*
1541	 * %o0 = vaddr
1542	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1543	 */
1544#ifndef sun4v
1545	sethi	%hi(ktsb_phys), %o4
1546	mov	ASI_N, %o3
1547	ld	[%o4 + %lo(ktsb_phys)], %o4
1548	movrnz	%o4, ASI_MEM, %o3
1549	mov	%o3, %asi
1550#endif /* !sun4v */
1551	mov	%o0, %g1			! %g1 = vaddr
1552
1553	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1554	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1555	/* %g2 = tsbep, %g1 clobbered */
1556
1557	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1558	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1559	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1560
1561	retl
1562	  membar	#StoreStore|#StoreLoad
1563	SET_SIZE(sfmmu_kpm_unload_tsb)
1564
1565#endif /* lint */
1566
1567
1568#if defined (lint)
1569
1570/*ARGSUSED*/
1571pfn_t
1572sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
1573{ return(0); }
1574
1575#else /* lint */
1576
1577	ENTRY_NP(sfmmu_ttetopfn)
1578	ldx	[%o0], %g1			/* read tte */
1579	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1580	/*
1581	 * g1 = pfn
1582	 */
1583	retl
1584	mov	%g1, %o0
1585	SET_SIZE(sfmmu_ttetopfn)
1586
1587#endif /* !lint */
1588
1589/*
1590 * These macros are used to update global sfmmu hme hash statistics
1591 * in perf critical paths. It is only enabled in debug kernels or
1592 * if SFMMU_STAT_GATHER is defined
1593 */
1594#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1595#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1596	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1597	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
1598	cmp	tmp1, hatid						;\
1599	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
1600	set	sfmmu_global_stat, tmp1					;\
1601	add	tmp1, tmp2, tmp1					;\
1602	ld	[tmp1], tmp2						;\
1603	inc	tmp2							;\
1604	st	tmp2, [tmp1]
1605
1606#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1607	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1608	mov	HATSTAT_KHASH_LINKS, tmp2				;\
1609	cmp	tmp1, hatid						;\
1610	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
1611	set	sfmmu_global_stat, tmp1					;\
1612	add	tmp1, tmp2, tmp1					;\
1613	ld	[tmp1], tmp2						;\
1614	inc	tmp2							;\
1615	st	tmp2, [tmp1]
1616
1617
1618#else /* DEBUG || SFMMU_STAT_GATHER */
1619
1620#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1621
1622#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1623
1624#endif  /* DEBUG || SFMMU_STAT_GATHER */
1625
1626/*
1627 * This macro is used to update global sfmmu kstas in non
1628 * perf critical areas so they are enabled all the time
1629 */
1630#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
1631	sethi	%hi(sfmmu_global_stat), tmp1				;\
1632	add	tmp1, statname, tmp1					;\
1633	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
1634	inc	tmp2							;\
1635	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1636
1637/*
1638 * These macros are used to update per cpu stats in non perf
1639 * critical areas so they are enabled all the time
1640 */
1641#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
1642	ld	[tsbarea + stat], tmp1					;\
1643	inc	tmp1							;\
1644	st	tmp1, [tsbarea + stat]
1645
1646/*
1647 * These macros are used to update per cpu stats in non perf
1648 * critical areas so they are enabled all the time
1649 */
1650#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
1651	lduh	[tsbarea + stat], tmp1					;\
1652	inc	tmp1							;\
1653	stuh	tmp1, [tsbarea + stat]
1654
1655#if defined(KPM_TLBMISS_STATS_GATHER)
1656	/*
1657	 * Count kpm dtlb misses separately to allow a different
1658	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1659	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1660	 */
1661#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
1662	brgez	tagacc, label	/* KPM VA? */				;\
1663	nop								;\
1664	CPU_INDEX(tmp1, tsbma)						;\
1665	sethi	%hi(kpmtsbm_area), tsbma				;\
1666	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
1667	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
1668	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
1669	/* VA range check */						;\
1670	ldx	[tsbma + KPMTSBM_VBASE], val				;\
1671	cmp	tagacc, val						;\
1672	blu,pn	%xcc, label						;\
1673	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
1674	cmp	tagacc, tmp1						;\
1675	bgeu,pn	%xcc, label						;\
1676	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
1677	inc	val							;\
1678	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
1679label:
1680#else
1681#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1682#endif	/* KPM_TLBMISS_STATS_GATHER */
1683
1684#if defined (lint)
1685/*
1686 * The following routines are jumped to from the mmu trap handlers to do
1687 * the setting up to call systrap.  They are separate routines instead of
1688 * being part of the handlers because the handlers would exceed 32
1689 * instructions and since this is part of the slow path the jump
1690 * cost is irrelevant.
1691 */
1692void
1693sfmmu_pagefault(void)
1694{
1695}
1696
1697void
1698sfmmu_mmu_trap(void)
1699{
1700}
1701
1702void
1703sfmmu_window_trap(void)
1704{
1705}
1706
1707void
1708sfmmu_kpm_exception(void)
1709{
1710}
1711
1712#else /* lint */
1713
1714#ifdef	PTL1_PANIC_DEBUG
1715	.seg	".data"
1716	.global	test_ptl1_panic
1717test_ptl1_panic:
1718	.word	0
1719	.align	8
1720
1721	.seg	".text"
1722	.align	4
1723#endif	/* PTL1_PANIC_DEBUG */
1724
1725
1726	ENTRY_NP(sfmmu_pagefault)
1727	SET_GL_REG(1)
1728	USE_ALTERNATE_GLOBALS(%g5)
1729	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1730	rdpr	%tt, %g6
1731	cmp	%g6, FAST_IMMU_MISS_TT
1732	be,a,pn	%icc, 1f
1733	  mov	T_INSTR_MMU_MISS, %g3
1734	cmp	%g6, T_INSTR_MMU_MISS
1735	be,a,pn	%icc, 1f
1736	  mov	T_INSTR_MMU_MISS, %g3
1737	mov	%g5, %g2
1738	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1739	cmp	%g6, FAST_DMMU_MISS_TT
1740	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1741	cmp	%g6, T_DATA_MMU_MISS
1742	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1743
1744#ifdef  PTL1_PANIC_DEBUG
1745	/* check if we want to test the tl1 panic */
1746	sethi	%hi(test_ptl1_panic), %g4
1747	ld	[%g4 + %lo(test_ptl1_panic)], %g1
1748	st	%g0, [%g4 + %lo(test_ptl1_panic)]
1749	cmp	%g1, %g0
1750	bne,a,pn %icc, ptl1_panic
1751	  or	%g0, PTL1_BAD_DEBUG, %g1
1752#endif	/* PTL1_PANIC_DEBUG */
17531:
1754	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
1755	/*
1756	 * g2 = tag access reg
1757	 * g3.l = type
1758	 * g3.h = 0
1759	 */
1760	sethi	%hi(trap), %g1
1761	or	%g1, %lo(trap), %g1
17622:
1763	ba,pt	%xcc, sys_trap
1764	  mov	-1, %g4
1765	SET_SIZE(sfmmu_pagefault)
1766
1767	ENTRY_NP(sfmmu_mmu_trap)
1768	SET_GL_REG(1)
1769	USE_ALTERNATE_GLOBALS(%g5)
1770	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
1771	rdpr	%tt, %g6
1772	cmp	%g6, FAST_IMMU_MISS_TT
1773	be,a,pn	%icc, 1f
1774	  mov	T_INSTR_MMU_MISS, %g3
1775	cmp	%g6, T_INSTR_MMU_MISS
1776	be,a,pn	%icc, 1f
1777	  mov	T_INSTR_MMU_MISS, %g3
1778	mov	%g5, %g2
1779	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1780	cmp	%g6, FAST_DMMU_MISS_TT
1781	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1782	cmp	%g6, T_DATA_MMU_MISS
1783	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
17841:
1785	/*
1786	 * g2 = tag access reg
1787	 * g3 = type
1788	 */
1789	sethi	%hi(sfmmu_tsbmiss_exception), %g1
1790	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
1791	ba,pt	%xcc, sys_trap
1792	  mov	-1, %g4
1793	/*NOTREACHED*/
1794	SET_SIZE(sfmmu_mmu_trap)
1795
1796	ENTRY_NP(sfmmu_suspend_tl)
1797	SET_GL_REG(1)
1798	USE_ALTERNATE_GLOBALS(%g5)
1799	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
1800	rdpr	%tt, %g6
1801	cmp	%g6, FAST_IMMU_MISS_TT
1802	be,a,pn	%icc, 1f
1803	  mov	T_INSTR_MMU_MISS, %g3
1804	mov	%g5, %g2
1805	cmp	%g6, FAST_DMMU_MISS_TT
1806	move	%icc, T_DATA_MMU_MISS, %g3
1807	movne	%icc, T_DATA_PROT, %g3
18081:
1809	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
1810	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
1811	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
1812	ba,pt	%xcc, sys_trap
1813	  mov	PIL_15, %g4
1814	/*NOTREACHED*/
1815	SET_SIZE(sfmmu_suspend_tl)
1816
1817	/*
1818	 * No %g registers in use at this point.
1819	 */
1820	ENTRY_NP(sfmmu_window_trap)
1821	rdpr	%tpc, %g1
1822#ifdef sun4v
1823#ifdef DEBUG
1824	/* We assume previous %gl was 1 */
1825	rdpr	%tstate, %g4
1826	srlx	%g4, TSTATE_GL_SHIFT, %g4
1827	and	%g4, TSTATE_GL_MASK, %g4
1828	cmp	%g4, 1
1829	bne,a,pn %icc, ptl1_panic
1830	  mov	PTL1_BAD_WTRAP, %g1
1831#endif /* DEBUG */
1832	/* user miss at tl>1. better be the window handler or user_rtt */
1833	/* in user_rtt? */
1834	set	rtt_fill_start, %g4
1835	cmp	%g1, %g4
1836	blu,pn %xcc, 6f
1837	 .empty
1838	set	rtt_fill_end, %g4
1839	cmp	%g1, %g4
1840	bgeu,pn %xcc, 6f
1841	 nop
1842	set	fault_rtt_fn1, %g1
1843	wrpr	%g0, %g1, %tnpc
1844	ba,a	7f
18456:
1846	! must save this trap level before descending trap stack
1847	! no need to save %tnpc, either overwritten or discarded
1848	! already got it: rdpr	%tpc, %g1
1849	rdpr	%tstate, %g6
1850	rdpr	%tt, %g7
1851	! trap level saved, go get underlying trap type
1852	rdpr	%tl, %g5
1853	sub	%g5, 1, %g3
1854	wrpr	%g3, %tl
1855	rdpr	%tt, %g2
1856	wrpr	%g5, %tl
1857	! restore saved trap level
1858	wrpr	%g1, %tpc
1859	wrpr	%g6, %tstate
1860	wrpr	%g7, %tt
1861#else /* sun4v */
1862	/* user miss at tl>1. better be the window handler */
1863	rdpr	%tl, %g5
1864	sub	%g5, 1, %g3
1865	wrpr	%g3, %tl
1866	rdpr	%tt, %g2
1867	wrpr	%g5, %tl
1868#endif /* sun4v */
1869	and	%g2, WTRAP_TTMASK, %g4
1870	cmp	%g4, WTRAP_TYPE
1871	bne,pn	%xcc, 1f
1872	 nop
1873	/* tpc should be in the trap table */
1874	set	trap_table, %g4
1875	cmp	%g1, %g4
1876	blt,pn %xcc, 1f
1877	 .empty
1878	set	etrap_table, %g4
1879	cmp	%g1, %g4
1880	bge,pn %xcc, 1f
1881	 .empty
1882	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
1883	add	%g1, WTRAP_FAULTOFF, %g1
1884	wrpr	%g0, %g1, %tnpc
18857:
1886	/*
1887	 * some wbuf handlers will call systrap to resolve the fault
1888	 * we pass the trap type so they figure out the correct parameters.
1889	 * g5 = trap type, g6 = tag access reg
1890	 */
1891
1892	/*
1893	 * only use g5, g6, g7 registers after we have switched to alternate
1894	 * globals.
1895	 */
1896	SET_GL_REG(1)
1897	USE_ALTERNATE_GLOBALS(%g5)
1898	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
1899	rdpr	%tt, %g7
1900	cmp	%g7, FAST_IMMU_MISS_TT
1901	be,a,pn	%icc, ptl1_panic
1902	  mov	PTL1_BAD_WTRAP, %g1
1903	cmp	%g7, T_INSTR_MMU_MISS
1904	be,a,pn	%icc, ptl1_panic
1905	  mov	PTL1_BAD_WTRAP, %g1
1906	mov	T_DATA_PROT, %g5
1907	cmp	%g7, FAST_DMMU_MISS_TT
1908	move	%icc, T_DATA_MMU_MISS, %g5
1909	cmp	%g7, T_DATA_MMU_MISS
1910	move	%icc, T_DATA_MMU_MISS, %g5
1911	! XXXQ AGS re-check out this one
1912	done
19131:
1914	CPU_PADDR(%g1, %g4)
1915	add	%g1, CPU_TL1_HDLR, %g1
1916	lda	[%g1]ASI_MEM, %g4
1917	brnz,a,pt %g4, sfmmu_mmu_trap
1918	  sta	%g0, [%g1]ASI_MEM
1919	ba,pt	%icc, ptl1_panic
1920	  mov	PTL1_BAD_TRAP, %g1
1921	SET_SIZE(sfmmu_window_trap)
1922
1923	ENTRY_NP(sfmmu_kpm_exception)
1924	/*
1925	 * We have accessed an unmapped segkpm address or a legal segkpm
1926	 * address which is involved in a VAC alias conflict prevention.
1927	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
1928	 * set. If it is, we will instead note that a fault has occurred
1929	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
1930	 * a "retry"). This will step over the faulting instruction.
1931	 * Note that this means that a legal segkpm address involved in
1932	 * a VAC alias conflict prevention (a rare case to begin with)
1933	 * cannot be used in DTrace.
1934	 */
1935	CPU_INDEX(%g1, %g2)
1936	set	cpu_core, %g2
1937	sllx	%g1, CPU_CORE_SHIFT, %g1
1938	add	%g1, %g2, %g1
1939	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
1940	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
1941	bz	0f
1942	or	%g2, CPU_DTRACE_BADADDR, %g2
1943	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
1944	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
1945	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
1946	done
19470:
1948	TSTAT_CHECK_TL1(1f, %g1, %g2)
19491:
1950	SET_GL_REG(1)
1951	USE_ALTERNATE_GLOBALS(%g5)
1952	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
1953	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1954	/*
1955	 * g2=tagacc g3.l=type g3.h=0
1956	 */
1957	sethi	%hi(trap), %g1
1958	or	%g1, %lo(trap), %g1
1959	ba,pt	%xcc, sys_trap
1960	mov	-1, %g4
1961	SET_SIZE(sfmmu_kpm_exception)
1962
1963#endif /* lint */
1964
1965#if defined (lint)
1966
1967void
1968sfmmu_tsb_miss(void)
1969{
1970}
1971
1972void
1973sfmmu_kpm_dtsb_miss(void)
1974{
1975}
1976
1977void
1978sfmmu_kpm_dtsb_miss_small(void)
1979{
1980}
1981
1982#else /* lint */
1983
1984#if (IMAP_SEG != 0)
1985#error - ism_map->ism_seg offset is not zero
1986#endif
1987
1988/*
1989 * Copies ism mapping for this ctx in param "ism" if this is a ISM
1990 * tlb miss and branches to label "ismhit". If this is not an ISM
1991 * process or an ISM tlb miss it falls thru.
1992 *
1993 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
1994 * this process.
1995 * If so, it will branch to label "ismhit".  If not, it will fall through.
1996 *
1997 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
1998 * so that any other threads of this process will not try and walk the ism
1999 * maps while they are being changed.
2000 *
2001 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
2002 *       will make sure of that. This means we can terminate our search on
2003 *       the first zero mapping we find.
2004 *
2005 * Parameters:
2006 * tagacc	= (pseudo-)tag access register (vaddr + ctx) (in)
2007 * tsbmiss	= address of tsb miss area (in)
2008 * ismseg	= contents of ism_seg for this ism map (out)
2009 * ismhat	= physical address of imap_ismhat for this ism map (out)
2010 * tmp1		= scratch reg (CLOBBERED)
2011 * tmp2		= scratch reg (CLOBBERED)
2012 * tmp3		= scratch reg (CLOBBERED)
2013 * label:    temporary labels
2014 * ismhit:   label where to jump to if an ism dtlb miss
2015 * exitlabel:label where to jump if hat is busy due to hat_unshare.
2016 */
2017#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
2018	label, ismhit)							\
2019	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
2020	brlz,pt  tmp1, label/**/3		/* exit if -1 */	;\
2021	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
2022label/**/1:								;\
2023	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
2024	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
2025label/**/2:								;\
2026	brz,pt  ismseg, label/**/3		/* no mapping */	;\
2027	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
2028	lduba	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
2029	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
2030	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
2031	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
2032	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
2033	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
2034	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
2035	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
2036	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
2037	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
2038									;\
2039	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
2040	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
2041	cmp	ismhat, tmp1						;\
2042	bl,pt	%xcc, label/**/2		/* keep looking  */	;\
2043	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
2044									;\
2045	add	tmp3, IBLK_NEXTPA, tmp1					;\
2046	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
2047	brgez,pt tmp1, label/**/1		/* continue if not -1*/	;\
2048	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
2049label/**/3:
2050
2051/*
2052 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
2053 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
2054 * Parameters:
2055 * tagacc = reg containing virtual address
2056 * hatid = reg containing sfmmu pointer
2057 * hmeshift = constant/register to shift vaddr to obtain vapg
2058 * hmebp = register where bucket pointer will be stored
2059 * vapg = register where virtual page will be stored
2060 * tmp1, tmp2 = tmp registers
2061 */
2062
2063
2064#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
2065	vapg, label, tmp1, tmp2)					\
2066	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
2067	brnz,a,pt tmp1, label/**/1					;\
2068	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
2069	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
2070	ba,pt	%xcc, label/**/2					;\
2071	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
2072label/**/1:								;\
2073	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
2074label/**/2:								;\
2075	srlx	tagacc, hmeshift, vapg					;\
2076	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
2077	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
2078	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
2079	add	hmebp, tmp1, hmebp
2080
2081/*
2082 * hashtag includes bspage + hashno (64 bits).
2083 */
2084
2085#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
2086	sllx	vapg, hmeshift, vapg					;\
2087	mov	hashno, hblktag						;\
2088	sllx	hblktag, HTAG_REHASH_SHIFT, hblktag			;\
2089	or	vapg, hblktag, hblktag
2090
2091/*
2092 * Function to traverse hmeblk hash link list and find corresponding match.
2093 * The search is done using physical pointers. It returns the physical address
2094 * pointer to the hmeblk that matches with the tag provided.
2095 * Parameters:
2096 * hmebp	= register that points to hme hash bucket, also used as
2097 *		  tmp reg (clobbered)
2098 * hmeblktag	= register with hmeblk tag match
2099 * hatid	= register with hatid
2100 * hmeblkpa	= register where physical ptr will be stored
2101 * tmp1		= tmp reg
2102 * label: temporary label
2103 */
2104
2105#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, 	\
2106	tmp1, label)							\
2107	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
2108	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
2109	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
2110label/**/1:								;\
2111	cmp	hmeblkpa, HMEBLK_ENDPA					;\
2112	be,pn   %xcc, label/**/2					;\
2113	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
2114	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
2115	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
2116	add	hmebp, CLONGSIZE, hmebp					;\
2117	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
2118	xor	tmp1, hmeblktag, tmp1					;\
2119	xor	hmebp, hatid, hmebp					;\
2120	or	hmebp, tmp1, hmebp					;\
2121	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
2122	  add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
2123	ba,pt	%xcc, label/**/1					;\
2124	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
2125label/**/2:
2126
2127/*
2128 * Function to traverse hmeblk hash link list and find corresponding match.
2129 * The search is done using physical pointers. It returns the physical address
2130 * pointer to the hmeblk that matches with the tag
2131 * provided.
2132 * Parameters:
2133 * hmeblktag	= register with hmeblk tag match (rid field is 0)
2134 * hatid	= register with hatid (pointer to SRD)
2135 * hmeblkpa	= register where physical ptr will be stored
2136 * tmp1		= tmp reg
2137 * tmp2		= tmp reg
2138 * label: temporary label
2139 */
2140
2141#define	HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea,	\
2142	tmp1, tmp2, label)			 			\
2143label/**/1:								;\
2144	cmp	hmeblkpa, HMEBLK_ENDPA					;\
2145	be,pn   %xcc, label/**/4					;\
2146	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			;\
2147	add	hmeblkpa, HMEBLK_TAG, tmp2				;\
2148	ldxa	[tmp2]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
2149	add	tmp2, CLONGSIZE, tmp2					;\
2150	ldxa	[tmp2]ASI_MEM, tmp2 	/* read 2nd part of tag */	;\
2151	xor	tmp1, hmeblktag, tmp1					;\
2152	xor	tmp2, hatid, tmp2					;\
2153	brz,pn	tmp2, label/**/3	/* branch on hit */		;\
2154	  add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
2155label/**/2:								;\
2156	ba,pt	%xcc, label/**/1					;\
2157	  ldxa	[tmp2]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */		;\
2158label/**/3:								;\
2159	cmp	tmp1, SFMMU_MAX_HME_REGIONS				;\
2160	bgeu,pt	%xcc, label/**/2					;\
2161	  add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
2162	and	tmp1, BT_ULMASK, tmp2					;\
2163	srlx	tmp1, BT_ULSHIFT, tmp1					;\
2164	sllx	tmp1, CLONGSHIFT, tmp1					;\
2165	add	tsbarea, tmp1, tmp1					;\
2166	ldx	[tmp1 + TSBMISS_SHMERMAP], tmp1				;\
2167	srlx	tmp1, tmp2, tmp1					;\
2168	btst	0x1, tmp1						;\
2169	bz,pn	%xcc, label/**/2					;\
2170	  add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
2171label/**/4:
2172
2173#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
2174#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
2175#endif
2176
2177/*
2178 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2179 * the offset for the corresponding hment.
2180 * Parameters:
2181 * In:
2182 *	vaddr = register with virtual address
2183 *	hmeblkpa = physical pointer to hme_blk
2184 * Out:
2185 *	hmentoff = register where hment offset will be stored
2186 *	hmemisc = hblk_misc
2187 * Scratch:
2188 *	tmp1
2189 */
2190#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\
2191	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
2192	lda	[hmentoff]ASI_MEM, hmemisc 				;\
2193	andcc	hmemisc, HBLK_SZMASK, %g0				;\
2194	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
2195	  or	%g0, HMEBLK_HME1, hmentoff				;\
2196	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
2197	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
2198	sllx	tmp1, SFHME_SHIFT, tmp1					;\
2199	add	tmp1, HMEBLK_HME1, hmentoff				;\
2200label1:
2201
2202/*
2203 * GET_TTE is a macro that returns a TTE given a tag and hatid.
2204 *
2205 * tagacc	= (pseudo-)tag access register (in)
2206 * hatid	= sfmmu pointer for TSB miss (in)
2207 * tte		= tte for TLB miss if found, otherwise clobbered (out)
2208 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
2209 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
2210 * hmemisc	= hblk_misc if TTE is found (out), otherwise clobbered
2211 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
2212 *		  for this page size.
2213 * hashno	= constant/register hash number
2214 * tmp		= temp value - clobbered
2215 * label	= temporary label for branching within macro.
2216 * foundlabel	= label to jump to when tte is found.
2217 * suspendlabel= label to jump to when tte is suspended.
2218 * exitlabel	= label to jump to when tte is not found.
2219 *
2220 */
2221#define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \
2222		 hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \
2223									;\
2224	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2225	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2226	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2227		hmeblkpa, label/**/5, hmemisc, tmp)			;\
2228									;\
2229	/*								;\
2230	 * tagacc = tagacc						;\
2231	 * hatid = hatid						;\
2232	 * tsbarea = tsbarea						;\
2233	 * tte   = hmebp (hme bucket pointer)				;\
2234	 * hmeblkpa  = vapg  (virtual page)				;\
2235	 * hmemisc, tmp = scratch					;\
2236	 */								;\
2237	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2238	or	hmemisc, SFMMU_INVALID_SHMERID, hmemisc			;\
2239									;\
2240	/*								;\
2241	 * tagacc = tagacc						;\
2242	 * hatid = hatid						;\
2243	 * tte   = hmebp						;\
2244	 * hmeblkpa  = CLOBBERED					;\
2245	 * hmemisc  = htag_bspage+hashno+invalid_rid			;\
2246	 * tmp  = scratch						;\
2247	 */								;\
2248	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2249	HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, 	 		\
2250		tsbarea, tagacc, label/**/1)				;\
2251	/*								;\
2252	 * tagacc = CLOBBERED						;\
2253	 * tte = CLOBBERED						;\
2254	 * hmeblkpa = hmeblkpa						;\
2255	 * tmp = scratch						;\
2256	 */								;\
2257	cmp	hmeblkpa, HMEBLK_ENDPA					;\
2258	bne,pn   %xcc, label/**/4       /* branch if hmeblk found */    ;\
2259	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2260	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2261	  nop								;\
2262label/**/4:								;\
2263	/*								;\
2264	 * We have found the hmeblk containing the hment.		;\
2265	 * Now we calculate the corresponding tte.			;\
2266	 *								;\
2267	 * tagacc = tagacc						;\
2268	 * hatid = hatid						;\
2269	 * tte   = clobbered						;\
2270	 * hmeblkpa  = hmeblkpa						;\
2271	 * hmemisc  = hblktag						;\
2272	 * tmp = scratch						;\
2273	 */								;\
2274	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2275		label/**/2)						;\
2276									;\
2277	/*								;\
2278	 * tagacc = tagacc						;\
2279	 * hatid = hmentoff						;\
2280	 * tte   = clobbered						;\
2281	 * hmeblkpa  = hmeblkpa						;\
2282	 * hmemisc  = hblk_misc						;\
2283	 * tmp = scratch						;\
2284	 */								;\
2285									;\
2286	add	hatid, SFHME_TTE, hatid					;\
2287	add	hmeblkpa, hatid, hmeblkpa				;\
2288	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2289	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2290	set	TTE_SUSPEND, hatid					;\
2291	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2292	btst	tte, hatid						;\
2293	bz,pt	%xcc, foundlabel					;\
2294	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2295									;\
2296	/*								;\
2297	 * Mapping is suspended, so goto suspend label.			;\
2298	 */								;\
2299	ba,pt	%xcc, suspendlabel					;\
2300	  nop
2301
2302/*
2303 * GET_SHME_TTE is similar to GET_TTE() except it searches
2304 * shared hmeblks via HMEHASH_SEARCH_SHME() macro.
2305 * If valid tte is found, hmemisc = shctx flag, i.e., shme is
2306 * either 0 (not part of scd) or 1 (part of scd).
2307 */
2308#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, 	\
2309		hmeshift, hashno, tmp, label, foundlabel,		\
2310		suspendlabel, exitlabel)				\
2311									;\
2312	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2313	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2314	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2315		hmeblkpa, label/**/5, hmemisc, tmp)			;\
2316									;\
2317	/*								;\
2318	 * tagacc = tagacc						;\
2319	 * hatid = hatid						;\
2320	 * tsbarea = tsbarea						;\
2321	 * tte   = hmebp (hme bucket pointer)				;\
2322	 * hmeblkpa  = vapg  (virtual page)				;\
2323	 * hmemisc, tmp = scratch					;\
2324	 */								;\
2325	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2326									;\
2327	/*								;\
2328	 * tagacc = tagacc						;\
2329	 * hatid = hatid						;\
2330	 * tsbarea = tsbarea						;\
2331	 * tte   = hmebp						;\
2332	 * hmemisc  = htag_bspage + hashno + 0 (for rid)		;\
2333	 * hmeblkpa  = CLOBBERED					;\
2334	 * tmp = scratch						;\
2335	 */								;\
2336	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2337									;\
2338	add     tte, HMEBUCK_NEXTPA, hmeblkpa				;\
2339	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
2340	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte)			;\
2341									;\
2342label/**/8:								;\
2343	HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa,			\
2344		tsbarea, tagacc, tte, label/**/1)			;\
2345	/*								;\
2346	 * tagacc = CLOBBERED						;\
2347	 * tte = CLOBBERED						;\
2348	 * hmeblkpa = hmeblkpa						;\
2349	 * tmp = scratch						;\
2350	 */								;\
2351	cmp	hmeblkpa, HMEBLK_ENDPA					;\
2352	bne,pn   %xcc, label/**/4       /* branch if hmeblk found */    ;\
2353	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2354	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2355	  nop								;\
2356label/**/4:								;\
2357	/*								;\
2358	 * We have found the hmeblk containing the hment.		;\
2359	 * Now we calculate the corresponding tte.			;\
2360	 *								;\
2361	 * tagacc = tagacc						;\
2362	 * hatid = hatid						;\
2363	 * tte   = clobbered						;\
2364	 * hmeblkpa  = hmeblkpa						;\
2365	 * hmemisc  = hblktag						;\
2366	 * tsbarea = tsbmiss area					;\
2367	 * tmp = scratch						;\
2368	 */								;\
2369	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2370		label/**/2)						;\
2371									;\
2372	/*								;\
2373	 * tagacc = tagacc						;\
2374	 * hatid = hmentoff						;\
2375	 * tte = clobbered						;\
2376	 * hmeblkpa  = hmeblkpa						;\
2377	 * hmemisc  = hblk_misc						;\
2378	 * tsbarea = tsbmiss area					;\
2379	 * tmp = scratch						;\
2380	 */								;\
2381									;\
2382	add	hatid, SFHME_TTE, hatid					;\
2383	add	hmeblkpa, hatid, hmeblkpa				;\
2384	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2385	brlz,pt tte, label/**/6						;\
2386	  nop								;\
2387	btst	HBLK_SZMASK, hmemisc					;\
2388	bnz,a,pt %icc, label/**/7					;\
2389	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2390									;\
2391	/*								;\
2392 	 * We found an invalid 8K tte in shme.				;\
2393	 * it may not belong to shme's region since			;\
2394	 * region size/alignment granularity is 8K but different	;\
2395	 * regions don't share hmeblks. Continue the search.		;\
2396	 */								;\
2397	sub	hmeblkpa, hatid, hmeblkpa				;\
2398	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2399	srlx	tagacc, hmeshift, tte					;\
2400	add	hmeblkpa, HMEBLK_NEXTPA, hmeblkpa			;\
2401	ldxa	[hmeblkpa]ASI_MEM, hmeblkpa				;\
2402	MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc)		;\
2403	ba,a,pt	%xcc, label/**/8					;\
2404label/**/6:								;\
2405	GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc)		;\
2406	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2407label/**/7:								;\
2408	set	TTE_SUSPEND, hatid					;\
2409	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2410	btst	tte, hatid						;\
2411	bz,pt	%xcc, foundlabel					;\
2412	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2413									;\
2414	/*								;\
2415	 * Mapping is suspended, so goto suspend label.			;\
2416	 */								;\
2417	ba,pt	%xcc, suspendlabel					;\
2418	  nop
2419
2420	/*
2421	 * KERNEL PROTECTION HANDLER
2422	 *
2423	 * g1 = tsb8k pointer register (clobbered)
2424	 * g2 = tag access register (ro)
2425	 * g3 - g7 = scratch registers
2426	 *
2427	 * Note: This function is patched at runtime for performance reasons.
2428	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
2429	 */
2430	ENTRY_NP(sfmmu_kprot_trap)
2431	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2432sfmmu_kprot_patch_ktsb_base:
2433	RUNTIME_PATCH_SETX(%g1, %g6)
2434	/* %g1 = contents of ktsb_base or ktsb_pbase */
2435sfmmu_kprot_patch_ktsb_szcode:
2436	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
2437
2438	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2439	! %g1 = First TSB entry pointer, as TSB miss handler expects
2440
2441	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2442sfmmu_kprot_patch_ktsb4m_base:
2443	RUNTIME_PATCH_SETX(%g3, %g6)
2444	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2445sfmmu_kprot_patch_ktsb4m_szcode:
2446	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
2447
2448	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2449	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2450
2451        CPU_TSBMISS_AREA(%g6, %g7)
2452        HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2453	ba,pt	%xcc, sfmmu_tsb_miss_tt
2454	  nop
2455
2456	/*
2457	 * USER PROTECTION HANDLER
2458	 *
2459	 * g1 = tsb8k pointer register (ro)
2460	 * g2 = tag access register (ro)
2461	 * g3 = faulting context (clobbered, currently not used)
2462	 * g4 - g7 = scratch registers
2463	 */
2464	ALTENTRY(sfmmu_uprot_trap)
2465#ifdef sun4v
2466	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2467	/* %g1 = first TSB entry ptr now, %g2 preserved */
2468
2469	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
2470	brlz,pt %g3, 9f				/* check for 2nd TSB */
2471	  nop
2472
2473	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2474	/* %g3 = second TSB entry ptr now, %g2 preserved */
2475
2476#else /* sun4v */
2477#ifdef UTSB_PHYS
2478	/* g1 = first TSB entry ptr */
2479	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2480	brlz,pt %g3, 9f			/* check for 2nd TSB */
2481	  nop
2482
2483	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2484	/* %g3 = second TSB entry ptr now, %g2 preserved */
2485#else /* UTSB_PHYS */
2486	brgez,pt %g1, 9f		/* check for 2nd TSB */
2487	  mov	-1, %g3			/* set second tsbe ptr to -1 */
2488
2489	mov	%g2, %g7
2490	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2491	/* %g3 = second TSB entry ptr now, %g7 clobbered */
2492	mov	%g1, %g7
2493	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2494#endif /* UTSB_PHYS */
2495#endif /* sun4v */
24969:
2497	CPU_TSBMISS_AREA(%g6, %g7)
2498	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2499	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
2500	  nop
2501
2502	/*
2503	 * Kernel 8K page iTLB miss.  We also get here if we took a
2504	 * fast instruction access mmu miss trap while running in
2505	 * invalid context.
2506	 *
2507	 * %g1 = 8K TSB pointer register (not used, clobbered)
2508	 * %g2 = tag access register (used)
2509	 * %g3 = faulting context id (used)
2510	 * %g7 = TSB tag to match (used)
2511	 */
2512	.align	64
2513	ALTENTRY(sfmmu_kitlb_miss)
2514	brnz,pn %g3, tsb_tl0_noctxt
2515	  nop
2516
2517	/* kernel miss */
2518	/* get kernel tsb pointer */
2519	/* we patch the next set of instructions at run time */
2520	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2521iktsbbase:
2522	RUNTIME_PATCH_SETX(%g4, %g5)
2523	/* %g4 = contents of ktsb_base or ktsb_pbase */
2524
2525iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2526	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2527	or	%g4, %g1, %g1			! form tsb ptr
2528	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2529	cmp	%g4, %g7
2530	bne,pn	%xcc, iktsb4mbase		! check 4m ktsb
2531	  srlx    %g2, MMU_PAGESHIFT4M, %g3	! use 4m virt-page as TSB index
2532
2533	andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2534	bz,pn	%icc, exec_fault
2535	  nop
2536	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2537	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2538	retry
2539
2540iktsb4mbase:
2541        RUNTIME_PATCH_SETX(%g4, %g6)
2542        /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */
2543iktsb4m:
2544	sllx    %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2545        srlx    %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2546	add	%g4, %g3, %g3			! %g3 = 4m tsbe ptr
2547	ldda	[%g3]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2548	cmp	%g4, %g7
2549	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
2550	  andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2551	bz,pn	%icc, exec_fault
2552	  nop
2553	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2554	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2555	retry
2556
2557	/*
2558	 * Kernel dTLB miss.  We also get here if we took a fast data
2559	 * access mmu miss trap while running in invalid context.
2560	 *
2561	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2562	 *	We select the TSB miss handler to branch to depending on
2563	 *	the virtual address of the access.  In the future it may
2564	 *	be desirable to separate kpm TTEs into their own TSB,
2565	 *	in which case all that needs to be done is to set
2566	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2567	 *	early in the miss if we detect a kpm VA to a new handler.
2568	 *
2569	 * %g1 = 8K TSB pointer register (not used, clobbered)
2570	 * %g2 = tag access register (used)
2571	 * %g3 = faulting context id (used)
2572	 */
2573	.align	64
2574	ALTENTRY(sfmmu_kdtlb_miss)
2575	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
2576	  nop
2577
2578	/* Gather some stats for kpm misses in the TLB. */
2579	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2580	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2581
2582	/*
2583	 * Get first TSB offset and look for 8K/64K/512K mapping
2584	 * using the 8K virtual page as the index.
2585	 *
2586	 * We patch the next set of instructions at run time;
2587	 * any changes here require sfmmu_patch_ktsb changes too.
2588	 */
2589dktsbbase:
2590	RUNTIME_PATCH_SETX(%g7, %g6)
2591	/* %g7 = contents of ktsb_base or ktsb_pbase */
2592
2593dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2594	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2595
2596	/*
2597	 * At this point %g1 is our index into the TSB.
2598	 * We just masked off enough bits of the VA depending
2599	 * on our TSB size code.
2600	 */
2601	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2602	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2603	cmp	%g6, %g4			! compare tag
2604	bne,pn	%xcc, dktsb4m_kpmcheck_small
2605	  add	%g7, %g1, %g1			/* form tsb ptr */
2606	TT_TRACE(trace_tsbhit)
2607	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2608	/* trapstat expects tte in %g5 */
2609	retry
2610
2611	/*
2612	 * If kpm is using large pages, the following instruction needs
2613	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2614	 * so that we will probe the 4M TSB regardless of the VA.  In
2615	 * the case kpm is using small pages, we know no large kernel
2616	 * mappings are located above 0x80000000.00000000 so we skip the
2617	 * probe as an optimization.
2618	 */
2619dktsb4m_kpmcheck_small:
2620	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2621	  /* delay slot safe, below */
2622
2623	/*
2624	 * Get second TSB offset and look for 4M mapping
2625	 * using 4M virtual page as the TSB index.
2626	 *
2627	 * Here:
2628	 * %g1 = 8K TSB pointer.  Don't squash it.
2629	 * %g2 = tag access register (we still need it)
2630	 */
2631	srlx	%g2, MMU_PAGESHIFT4M, %g3
2632
2633	/*
2634	 * We patch the next set of instructions at run time;
2635	 * any changes here require sfmmu_patch_ktsb changes too.
2636	 */
2637dktsb4mbase:
2638	RUNTIME_PATCH_SETX(%g7, %g6)
2639	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2640dktsb4m:
2641	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2642	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2643
2644	/*
2645	 * At this point %g3 is our index into the TSB.
2646	 * We just masked off enough bits of the VA depending
2647	 * on our TSB size code.
2648	 */
2649	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2650	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2651	cmp	%g6, %g4			! compare tag
2652
2653dktsb4m_tsbmiss:
2654	bne,pn	%xcc, dktsb4m_kpmcheck
2655	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
2656	TT_TRACE(trace_tsbhit)
2657	/* we don't check TTE size here since we assume 4M TSB is separate */
2658	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2659	/* trapstat expects tte in %g5 */
2660	retry
2661
2662	/*
2663	 * So, we failed to find a valid TTE to match the faulting
2664	 * address in either TSB.  There are a few cases that could land
2665	 * us here:
2666	 *
2667	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2668	 *    to sfmmu_tsb_miss_tt to handle the miss.
2669	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2670	 *    4M TSB.  Let segkpm handle it.
2671	 *
2672	 * Note that we shouldn't land here in the case of a kpm VA when
2673	 * kpm_smallpages is active -- we handled that case earlier at
2674	 * dktsb4m_kpmcheck_small.
2675	 *
2676	 * At this point:
2677	 *  g1 = 8K-indexed primary TSB pointer
2678	 *  g2 = tag access register
2679	 *  g3 = 4M-indexed secondary TSB pointer
2680	 */
2681dktsb4m_kpmcheck:
2682	cmp	%g2, %g0
2683	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
2684	  nop
2685	ba,a,pt	%icc, sfmmu_tsb_miss_tt
2686	  nop
2687
2688#ifdef sun4v
2689	/*
2690	 * User instruction miss w/ single TSB.
2691	 * The first probe covers 8K, 64K, and 512K page sizes,
2692	 * because 64K and 512K mappings are replicated off 8K
2693	 * pointer.
2694	 *
2695	 * g1 = tsb8k pointer register
2696	 * g2 = tag access register
2697	 * g3 - g6 = scratch registers
2698	 * g7 = TSB tag to match
2699	 */
2700	.align	64
2701	ALTENTRY(sfmmu_uitlb_fastpath)
2702
2703	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2704	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2705	ba,pn	%xcc, sfmmu_tsb_miss_tt
2706	  mov	-1, %g3
2707
2708	/*
2709	 * User data miss w/ single TSB.
2710	 * The first probe covers 8K, 64K, and 512K page sizes,
2711	 * because 64K and 512K mappings are replicated off 8K
2712	 * pointer.
2713	 *
2714	 * g1 = tsb8k pointer register
2715	 * g2 = tag access register
2716	 * g3 - g6 = scratch registers
2717	 * g7 = TSB tag to match
2718	 */
2719	.align 64
2720	ALTENTRY(sfmmu_udtlb_fastpath)
2721
2722	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2723	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2724	ba,pn	%xcc, sfmmu_tsb_miss_tt
2725	  mov	-1, %g3
2726
2727	/*
2728	 * User instruction miss w/ multiple TSBs (sun4v).
2729	 * The first probe covers 8K, 64K, and 512K page sizes,
2730	 * because 64K and 512K mappings are replicated off 8K
2731	 * pointer.  Second probe covers 4M page size only.
2732	 *
2733	 * Just like sfmmu_udtlb_slowpath, except:
2734	 *   o Uses ASI_ITLB_IN
2735	 *   o checks for execute permission
2736	 *   o No ISM prediction.
2737	 *
2738	 * g1 = tsb8k pointer register
2739	 * g2 = tag access register
2740	 * g3 - g6 = scratch registers
2741	 * g7 = TSB tag to match
2742	 */
2743	.align	64
2744	ALTENTRY(sfmmu_uitlb_slowpath)
2745
2746	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2747	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2748	/* g4 - g5 = clobbered here */
2749
2750	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2751	/* g1 = first TSB pointer, g3 = second TSB pointer */
2752	srlx	%g2, TAG_VALO_SHIFT, %g7
2753	PROBE_2ND_ITSB(%g3, %g7)
2754	/* NOT REACHED */
2755
2756#else /* sun4v */
2757
2758	/*
2759	 * User instruction miss w/ multiple TSBs (sun4u).
2760	 * The first probe covers 8K, 64K, and 512K page sizes,
2761	 * because 64K and 512K mappings are replicated off 8K
2762	 * pointer.  Probe of 1st TSB has already been done prior to entry
2763	 * into this routine. For the UTSB_PHYS case we probe up to 3
2764	 * valid other TSBs in the following order:
2765	 * 1) shared TSB for 4M-256M pages
2766	 * 2) private TSB for 4M-256M pages
2767	 * 3) shared TSB for 8K-512K pages
2768	 *
2769	 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs
2770	 * 4M-256M pages.
2771	 *
2772	 * Just like sfmmu_udtlb_slowpath, except:
2773	 *   o Uses ASI_ITLB_IN
2774	 *   o checks for execute permission
2775	 *   o No ISM prediction.
2776	 *
2777	 * g1 = tsb8k pointer register
2778	 * g2 = tag access register
2779	 * g4 - g6 = scratch registers
2780	 * g7 = TSB tag to match
2781	 */
2782	.align	64
2783	ALTENTRY(sfmmu_uitlb_slowpath)
2784
2785#ifdef UTSB_PHYS
2786
2787       GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2788        brlz,pt %g6, 1f
2789          nop
2790        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2791        PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail)
27921:
2793        GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2794        brlz,pt %g3, 2f
2795          nop
2796        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2797        PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail)
27982:
2799        GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2800        brlz,pt %g6, sfmmu_tsb_miss_tt
2801          nop
2802        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2803        PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail)
2804        ba,pn   %xcc, sfmmu_tsb_miss_tt
2805          nop
2806
2807#else /* UTSB_PHYS */
2808	mov	%g1, %g3	/* save tsb8k reg in %g3 */
2809	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
2810	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2811	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
2812	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
2813	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
2814       /* g1 = first TSB pointer, g3 = second TSB pointer */
2815        srlx    %g2, TAG_VALO_SHIFT, %g7
2816        PROBE_2ND_ITSB(%g3, %g7, isynth)
2817	ba,pn	%xcc, sfmmu_tsb_miss_tt
2818	  nop
2819
2820#endif /* UTSB_PHYS */
2821#endif /* sun4v */
2822
2823#if defined(sun4u) && defined(UTSB_PHYS)
2824
2825        /*
2826	 * We come here for ism predict DTLB_MISS case or if
2827	 * if probe in first TSB failed.
2828         */
2829
2830        .align 64
2831        ALTENTRY(sfmmu_udtlb_slowpath_noismpred)
2832
2833	/*
2834         * g1 = tsb8k pointer register
2835         * g2 = tag access register
2836         * g4 - %g6 = scratch registers
2837         * g7 = TSB tag to match
2838	 */
2839
2840	/*
2841	 * ISM non-predict probe order
2842         * probe 1ST_TSB (8K index)
2843         * probe 2ND_TSB (4M index)
2844         * probe 4TH_TSB (4M index)
2845         * probe 3RD_TSB (8K index)
2846	 *
2847	 * We already probed first TSB in DTLB_MISS handler.
2848	 */
2849
2850        /*
2851         * Private 2ND TSB 4M-256 pages
2852         */
2853	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2854	brlz,pt %g3, 1f
2855	  nop
2856        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2857        PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2858
2859	/*
2860	 * Shared Context 4TH TSB 4M-256 pages
2861	 */
28621:
2863	GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2864	brlz,pt %g6, 2f
2865	  nop
2866        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2867        PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail)
2868
2869        /*
2870         * Shared Context 3RD TSB 8K-512K pages
2871         */
28722:
2873	GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2874	brlz,pt %g6, sfmmu_tsb_miss_tt
2875	  nop
2876        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2877        PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail)
2878	ba,pn	%xcc, sfmmu_tsb_miss_tt
2879	  nop
2880
2881	.align 64
2882        ALTENTRY(sfmmu_udtlb_slowpath_ismpred)
2883
2884	/*
2885         * g1 = tsb8k pointer register
2886         * g2 = tag access register
2887         * g4 - g6 = scratch registers
2888         * g7 = TSB tag to match
2889	 */
2890
2891	/*
2892	 * ISM predict probe order
2893	 * probe 4TH_TSB (4M index)
2894	 * probe 2ND_TSB (4M index)
2895	 * probe 1ST_TSB (8K index)
2896	 * probe 3RD_TSB (8K index)
2897
2898	/*
2899	 * Shared Context 4TH TSB 4M-256 pages
2900	 */
2901	GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2902	brlz,pt %g6, 4f
2903	  nop
2904        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2905        PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2)
2906
2907        /*
2908         * Private 2ND TSB 4M-256 pages
2909         */
29104:
2911	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2912	brlz,pt %g3, 5f
2913	  nop
2914        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2915        PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2)
2916
29175:
2918        PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2)
2919
2920        /*
2921         * Shared Context 3RD TSB 8K-512K pages
2922         */
2923	GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2924	brlz,pt %g6, 6f
2925	  nop
2926        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2927        PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2)
29286:
2929	ba,pn	%xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */
2930	  nop
2931
2932#else /* sun4u && UTSB_PHYS */
2933
2934       .align 64
2935        ALTENTRY(sfmmu_udtlb_slowpath)
2936
2937	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
2938	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
2939	  mov	%g1, %g3
2940
2941udtlb_miss_probefirst:
2942	/*
2943	 * g1 = 8K TSB pointer register
2944	 * g2 = tag access register
2945	 * g3 = (potentially) second TSB entry ptr
2946	 * g6 = ism pred.
2947	 * g7 = vpg_4m
2948	 */
2949#ifdef sun4v
2950	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2951	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2952
2953	/*
2954	 * Here:
2955	 *   g1 = first TSB pointer
2956	 *   g2 = tag access reg
2957	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2958	 */
2959	brgz,pn	%g6, sfmmu_tsb_miss_tt
2960	  nop
2961#else /* sun4v */
2962	mov	%g1, %g4
2963	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
2964	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2965
2966	/*
2967	 * Here:
2968	 *   g1 = first TSB pointer
2969	 *   g2 = tag access reg
2970	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2971	 */
2972	brgz,pn	%g6, sfmmu_tsb_miss_tt
2973	  nop
2974	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
2975	/* fall through in 8K->4M probe order */
2976#endif /* sun4v */
2977
2978udtlb_miss_probesecond:
2979	/*
2980	 * Look in the second TSB for the TTE
2981	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
2982	 * g2 = tag access reg
2983	 * g3 = 8K TSB pointer register
2984	 * g6 = ism pred.
2985	 * g7 = vpg_4m
2986	 */
2987#ifdef sun4v
2988	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
2989	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2990	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
2991#else /* sun4v */
2992	mov	%g3, %g7
2993	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
2994	/* %g2 clobbered, %g3 =second tsbe ptr */
2995	mov	MMU_TAG_ACCESS, %g2
2996	ldxa	[%g2]ASI_DMMU, %g2
2997#endif /* sun4v */
2998
2999	srlx	%g2, TAG_VALO_SHIFT, %g7
3000	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
3001	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
3002	brgz,pn	%g6, udtlb_miss_probefirst
3003	  nop
3004
3005	/* fall through to sfmmu_tsb_miss_tt */
3006#endif /* sun4u && UTSB_PHYS */
3007
3008
3009	ALTENTRY(sfmmu_tsb_miss_tt)
3010	TT_TRACE(trace_tsbmiss)
3011	/*
3012	 * We get here if there is a TSB miss OR a write protect trap.
3013	 *
3014	 * g1 = First TSB entry pointer
3015	 * g2 = tag access register
3016	 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB
3017	 * g4 - g7 = scratch registers
3018	 */
3019
3020	ALTENTRY(sfmmu_tsb_miss)
3021
3022	/*
3023	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3024	 * point to trapstat's TSB miss return code (note that trapstat
3025	 * itself will patch the correct offset to add).
3026	 */
3027	rdpr	%tl, %g7
3028	cmp	%g7, 1
3029	ble,pt	%xcc, 0f
3030	  sethi	%hi(KERNELBASE), %g6
3031	rdpr	%tpc, %g7
3032	or	%g6, %lo(KERNELBASE), %g6
3033	cmp	%g7, %g6
3034	bgeu,pt	%xcc, 0f
3035	/* delay slot safe */
3036
3037	ALTENTRY(tsbmiss_trapstat_patch_point)
3038	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3039	wrpr	%g7, %tpc
3040	add	%g7, 4, %g7
3041	wrpr	%g7, %tnpc
30420:
3043	CPU_TSBMISS_AREA(%g6, %g7)
3044	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save 1ST tsb pointer */
3045	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save 2ND tsb pointer */
3046
3047	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
3048	brz,a,pn %g3, 1f			/* skip ahead if kernel */
3049	  ldn	[%g6 + TSBMISS_KHATID], %g7
3050	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
3051	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
3052
3053	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
3054
3055	cmp	%g3, INVALID_CONTEXT
3056	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
3057	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
3058
3059#if defined(sun4v) || defined(UTSB_PHYS)
3060        ldub    [%g6 + TSBMISS_URTTEFLAGS], %g7	/* clear ctx1 flag set from */
3061        andn    %g7, HAT_CHKCTX1_FLAG, %g7	/* the previous tsb miss    */
3062        stub    %g7, [%g6 + TSBMISS_URTTEFLAGS]
3063#endif /* sun4v || UTSB_PHYS */
3064
3065	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
3066	/*
3067	 * The miss wasn't in an ISM segment.
3068	 *
3069	 * %g1 %g3, %g4, %g5, %g7 all clobbered
3070	 * %g2 = (pseudo) tag access
3071	 */
3072
3073	ba,pt	%icc, 2f
3074	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
3075
30761:
3077	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
3078	/*
3079	 * 8K and 64K hash.
3080	 */
30812:
3082
3083	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3084		MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte,
3085		sfmmu_suspend_tl, tsb_512K)
3086	/* NOT REACHED */
3087
3088tsb_512K:
3089	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3090	brz,pn	%g5, 3f
3091	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3092	and	%g4, HAT_512K_FLAG, %g5
3093
3094	/*
3095	 * Note that there is a small window here where we may have
3096	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
3097	 * flag yet, so we will skip searching the 512k hash list.
3098	 * In this case we will end up in pagefault which will find
3099	 * the mapping and return.  So, in this instance we will end up
3100	 * spending a bit more time resolving this TSB miss, but it can
3101	 * only happen once per process and even then, the chances of that
3102	 * are very small, so it's not worth the extra overhead it would
3103	 * take to close this window.
3104	 */
3105	brz,pn	%g5, tsb_4M
3106	  nop
31073:
3108	/*
3109	 * 512K hash
3110	 */
3111
3112	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3113		MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte,
3114		sfmmu_suspend_tl, tsb_4M)
3115	/* NOT REACHED */
3116
3117tsb_4M:
3118	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3119	brz,pn	%g5, 4f
3120	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3121	and	%g4, HAT_4M_FLAG, %g5
3122	brz,pn	%g5, tsb_32M
3123	  nop
31244:
3125	/*
3126	 * 4M hash
3127	 */
3128
3129	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3130		MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte,
3131		sfmmu_suspend_tl, tsb_32M)
3132	/* NOT REACHED */
3133
3134tsb_32M:
3135	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3136#ifdef	sun4v
3137        brz,pn	%g5, 6f
3138#else
3139	brz,pn  %g5, tsb_pagefault
3140#endif
3141	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3142	and	%g4, HAT_32M_FLAG, %g5
3143	brz,pn	%g5, tsb_256M
3144	  nop
31455:
3146	/*
3147	 * 32M hash
3148	 */
3149
3150	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3151		MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte,
3152		sfmmu_suspend_tl, tsb_256M)
3153	/* NOT REACHED */
3154
3155#if defined(sun4u) && !defined(UTSB_PHYS)
3156#define tsb_shme        tsb_pagefault
3157#endif
3158tsb_256M:
3159	ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3160	and	%g4, HAT_256M_FLAG, %g5
3161	brz,pn	%g5, tsb_shme
3162	  nop
31636:
3164	/*
3165	 * 256M hash
3166	 */
3167
3168	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3169	    MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte,
3170	    sfmmu_suspend_tl, tsb_shme)
3171	/* NOT REACHED */
3172
3173tsb_checktte:
3174	/*
3175	 * g1 = hblk_misc
3176	 * g2 = tagacc
3177	 * g3 = tte
3178	 * g4 = tte pa
3179	 * g6 = tsbmiss area
3180	 * g7 = hatid
3181	 */
3182	brlz,a,pt %g3, tsb_validtte
3183	  rdpr	%tt, %g7
3184
3185#if defined(sun4u) && !defined(UTSB_PHYS)
3186#undef tsb_shme
3187	ba      tsb_pagefault
3188	  nop
3189#else /* sun4u && !UTSB_PHYS */
3190
3191tsb_shme:
3192	/*
3193	 * g2 = tagacc
3194	 * g6 = tsbmiss area
3195	 */
3196	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3197	brz,pn	%g5, tsb_pagefault
3198	  nop
3199	ldx	[%g6 + TSBMISS_SHARED_UHATID], %g7	/* g7 = srdp */
3200	brz,pn	%g7, tsb_pagefault
3201	  nop
3202
3203	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3204		MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte,
3205		sfmmu_suspend_tl, tsb_shme_512K)
3206	/* NOT REACHED */
3207
3208tsb_shme_512K:
3209	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3210	and	%g4, HAT_512K_FLAG, %g5
3211	brz,pn	%g5, tsb_shme_4M
3212	  nop
3213
3214	/*
3215	 * 512K hash
3216	 */
3217
3218	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3219		MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte,
3220		sfmmu_suspend_tl, tsb_shme_4M)
3221	/* NOT REACHED */
3222
3223tsb_shme_4M:
3224	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3225	and	%g4, HAT_4M_FLAG, %g5
3226	brz,pn	%g5, tsb_shme_32M
3227	  nop
32284:
3229	/*
3230	 * 4M hash
3231	 */
3232	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3233		MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte,
3234		sfmmu_suspend_tl, tsb_shme_32M)
3235	/* NOT REACHED */
3236
3237tsb_shme_32M:
3238	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3239	and	%g4, HAT_32M_FLAG, %g5
3240	brz,pn	%g5, tsb_shme_256M
3241	  nop
3242
3243	/*
3244	 * 32M hash
3245	 */
3246
3247	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3248		MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte,
3249		sfmmu_suspend_tl, tsb_shme_256M)
3250	/* NOT REACHED */
3251
3252tsb_shme_256M:
3253	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3254	and	%g4, HAT_256M_FLAG, %g5
3255	brz,pn	%g5, tsb_pagefault
3256	  nop
3257
3258	/*
3259	 * 256M hash
3260	 */
3261
3262	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3263	    MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte,
3264	    sfmmu_suspend_tl, tsb_pagefault)
3265	/* NOT REACHED */
3266
3267tsb_shme_checktte:
3268
3269	brgez,pn %g3, tsb_pagefault
3270	  rdpr	%tt, %g7
3271	/*
3272	 * g1 = ctx1 flag
3273	 * g3 = tte
3274	 * g4 = tte pa
3275	 * g6 = tsbmiss area
3276	 * g7 = tt
3277	 */
3278
3279	brz,pt  %g1, tsb_validtte
3280	  nop
3281	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g1
3282	  or	%g1, HAT_CHKCTX1_FLAG, %g1
3283	stub    %g1, [%g6 + TSBMISS_URTTEFLAGS]
3284
3285	SAVE_CTX1(%g7, %g2, %g1, tsb_shmel)
3286#endif /* sun4u && !UTSB_PHYS */
3287
3288tsb_validtte:
3289	/*
3290	 * g3 = tte
3291	 * g4 = tte pa
3292	 * g6 = tsbmiss area
3293	 * g7 = tt
3294	 */
3295
3296	/*
3297	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
3298	 */
3299	cmp	%g7, FAST_PROT_TT
3300	bne,pt	%icc, 4f
3301	  nop
3302
3303	TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod,
3304	    tsb_protfault)
3305
3306	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3307#ifdef sun4v
3308	MMU_FAULT_STATUS_AREA(%g7)
3309	ldx	[%g7 + MMFSA_D_ADDR], %g5	/* load fault addr for later */
3310#else /* sun4v */
3311	mov     MMU_TAG_ACCESS, %g5
3312	ldxa    [%g5]ASI_DMMU, %g5
3313#endif /* sun4v */
3314	ba,pt	%xcc, tsb_update_tl1
3315	  nop
33164:
3317	/*
3318	 * If ITLB miss check exec bit.
3319	 * If not set treat as invalid TTE.
3320	 */
3321	cmp     %g7, T_INSTR_MMU_MISS
3322	be,pn	%icc, 5f
3323	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
3324	cmp     %g7, FAST_IMMU_MISS_TT
3325	bne,pt %icc, 3f
3326	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
33275:
3328	bz,pn %icc, tsb_protfault
3329	  nop
3330
33313:
3332	/*
3333	 * Set reference bit if not already set
3334	 */
3335	TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref)
3336
3337	/*
3338	 * Now, load into TSB/TLB.  At this point:
3339	 * g3 = tte
3340	 * g4 = patte
3341	 * g6 = tsbmiss area
3342	 */
3343	rdpr	%tt, %g7
3344#ifdef sun4v
3345	MMU_FAULT_STATUS_AREA(%g2)
3346	cmp	%g7, T_INSTR_MMU_MISS
3347	be,a,pt	%icc, 9f
3348	  nop
3349	cmp	%g7, FAST_IMMU_MISS_TT
3350	be,a,pt	%icc, 9f
3351	  nop
3352	add	%g2, MMFSA_D_, %g2
33539:
3354	ldx	[%g2 + MMFSA_CTX_], %g7
3355	sllx	%g7, TTARGET_CTX_SHIFT, %g7
3356	ldx	[%g2 + MMFSA_ADDR_], %g2
3357	mov	%g2, %g5		! load the fault addr for later use
3358	srlx	%g2, TTARGET_VA_SHIFT, %g2
3359	or	%g2, %g7, %g2
3360#else /* sun4v */
3361	mov     MMU_TAG_ACCESS, %g5
3362	cmp     %g7, FAST_IMMU_MISS_TT
3363	be,a,pt %icc, 9f
3364	   ldxa  [%g0]ASI_IMMU, %g2
3365	ldxa    [%g0]ASI_DMMU, %g2
3366	ba,pt   %icc, tsb_update_tl1
3367	   ldxa  [%g5]ASI_DMMU, %g5
33689:
3369	ldxa    [%g5]ASI_IMMU, %g5
3370#endif /* sun4v */
3371
3372tsb_update_tl1:
3373	srlx	%g2, TTARGET_CTX_SHIFT, %g7
3374	brz,pn	%g7, tsb_kernel
3375#ifdef sun4v
3376	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
3377#else  /* sun4v */
3378	  srlx	%g3, TTE_SZ_SHFT, %g7
3379#endif /* sun4v */
3380
3381tsb_user:
3382#ifdef sun4v
3383	cmp	%g7, TTE4M
3384	bge,pn	%icc, tsb_user4m
3385	  nop
3386#else /* sun4v */
3387	cmp	%g7, TTESZ_VALID | TTE4M
3388	be,pn	%icc, tsb_user4m
3389	  srlx	%g3, TTE_SZ2_SHFT, %g7
3390	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
3391#ifdef ITLB_32M_256M_SUPPORT
3392	bnz,pn	%icc, tsb_user4m
3393	  nop
3394#else /* ITLB_32M_256M_SUPPORT */
3395	bnz,a,pn %icc, tsb_user_pn_synth
3396	 nop
3397#endif /* ITLB_32M_256M_SUPPORT */
3398#endif /* sun4v */
3399
3400tsb_user8k:
3401#if defined(sun4v) || defined(UTSB_PHYS)
3402	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3403	and	%g7, HAT_CHKCTX1_FLAG, %g1
3404	brz,a,pn %g1, 1f
3405	  ldn	[%g6 + TSBMISS_TSBPTR], %g1		! g1 = 1ST TSB ptr
3406	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1)
3407	brlz,a,pn %g1, ptl1_panic			! if no shared 3RD tsb
3408	  mov PTL1_NO_SCDTSB8K, %g1			! panic
3409        GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7)
34101:
3411#else /* defined(sun4v) || defined(UTSB_PHYS) */
3412	ldn   [%g6 + TSBMISS_TSBPTR], %g1             ! g1 = 1ST TSB ptr
3413#endif /* defined(sun4v) || defined(UTSB_PHYS) */
3414
3415#ifndef UTSB_PHYS
3416	mov	ASI_N, %g7	! user TSBs accessed by VA
3417	mov	%g7, %asi
3418#endif /* !UTSB_PHYS */
3419
3420	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3)
3421
3422	rdpr    %tt, %g5
3423#ifdef sun4v
3424	cmp	%g5, T_INSTR_MMU_MISS
3425	be,a,pn	%xcc, 9f
3426	  mov	%g3, %g5
3427#endif /* sun4v */
3428	cmp	%g5, FAST_IMMU_MISS_TT
3429	be,pn	%xcc, 9f
3430	  mov	%g3, %g5
3431
3432	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3433	! trapstat wants TTE in %g5
3434	retry
34359:
3436	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3437	! trapstat wants TTE in %g5
3438	retry
3439
3440tsb_user4m:
3441#if defined(sun4v) || defined(UTSB_PHYS)
3442	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3443	and	%g7, HAT_CHKCTX1_FLAG, %g1
3444	brz,a,pn %g1, 4f
3445	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		! g1 = 2ND TSB ptr
3446	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr
3447	brlz,a,pn %g1, 5f				! if no shared 4TH TSB
3448	  nop
3449        GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7)
3450
3451#else /* defined(sun4v) || defined(UTSB_PHYS) */
3452	ldn   [%g6 + TSBMISS_TSBPTR4M], %g1             ! g1 = 2ND TSB ptr
3453#endif /* defined(sun4v) || defined(UTSB_PHYS) */
34544:
3455	brlz,pn %g1, 5f	/* Check to see if we have 2nd TSB programmed */
3456	  nop
3457
3458#ifndef UTSB_PHYS
3459	mov	ASI_N, %g7	! user TSBs accessed by VA
3460	mov	%g7, %asi
3461#endif /* UTSB_PHYS */
3462
3463        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4)
3464
34655:
3466	rdpr    %tt, %g5
3467#ifdef sun4v
3468        cmp     %g5, T_INSTR_MMU_MISS
3469        be,a,pn %xcc, 9f
3470          mov   %g3, %g5
3471#endif /* sun4v */
3472        cmp     %g5, FAST_IMMU_MISS_TT
3473        be,pn   %xcc, 9f
3474        mov     %g3, %g5
3475
3476        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3477        ! trapstat wants TTE in %g5
3478        retry
34799:
3480        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3481        ! trapstat wants TTE in %g5
3482        retry
3483
3484#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT)
3485	/*
3486	 * Panther ITLB synthesis.
3487	 * The Panther 32M and 256M ITLB code simulates these two large page
3488	 * sizes with 4M pages, to provide support for programs, for example
3489	 * Java, that may copy instructions into a 32M or 256M data page and
3490	 * then execute them. The code below generates the 4M pfn bits and
3491	 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
3492	 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
3493	 * are ignored by the hardware.
3494	 *
3495	 * Now, load into TSB/TLB.  At this point:
3496	 * g2 = tagtarget
3497	 * g3 = tte
3498	 * g4 = patte
3499	 * g5 = tt
3500	 * g6 = tsbmiss area
3501	 */
3502tsb_user_pn_synth:
3503	rdpr %tt, %g5
3504	cmp    %g5, FAST_IMMU_MISS_TT
3505	be,pt	%xcc, tsb_user_itlb_synth	/* ITLB miss */
3506	  andcc %g3, TTE_EXECPRM_INT, %g0	/* is execprm bit set */
3507	bz,pn %icc, 4b				/* if not, been here before */
3508	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	/* g1 = tsbp */
3509	brlz,a,pn %g1, 5f			/* no 2nd tsb */
3510	  mov	%g3, %g5
3511
3512	mov	MMU_TAG_ACCESS, %g7
3513	ldxa	[%g7]ASI_DMMU, %g6		/* get tag access va */
3514	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)	/* make 4M pfn offset */
3515
3516	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3517	mov	%g7, %asi
3518	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */
35195:
3520        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3521        retry
3522
3523tsb_user_itlb_synth:
3524	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 =  2ND TSB */
3525
3526	mov	MMU_TAG_ACCESS, %g7
3527	ldxa	[%g7]ASI_IMMU, %g6		/* get tag access va */
3528	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)	/* make 4M pfn offset */
3529	brlz,a,pn %g1, 7f	/* Check to see if we have 2nd TSB programmed */
3530	  or	%g5, %g3, %g5			/* add 4M bits to TTE */
3531
3532	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3533	mov	%g7, %asi
3534	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */
35357:
3536	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
3537        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3538        retry
3539#endif /* sun4v && ITLB_32M_256M_SUPPORT */
3540
3541tsb_kernel:
3542	rdpr	%tt, %g5
3543#ifdef sun4v
3544	cmp	%g7, TTE4M
3545	bge,pn	%icc, 5f
3546#else
3547	cmp	%g7, TTESZ_VALID | TTE4M	! no 32M or 256M support
3548	be,pn	%icc, 5f
3549#endif /* sun4v */
3550	  nop
3551	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8K TSB ptr
3552	ba,pt	%xcc, 6f
3553	  nop
35545:
3555	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4M TSB ptr
3556	brlz,pn	%g1, 3f		/* skip programming if 4M TSB ptr is -1 */
3557	  nop
35586:
3559#ifndef sun4v
3560tsb_kernel_patch_asi:
3561	or	%g0, RUNTIME_PATCH, %g6
3562	mov	%g6, %asi	! XXX avoid writing to %asi !!
3563#endif
3564	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7)
35653:
3566#ifdef sun4v
3567	cmp	%g5, T_INSTR_MMU_MISS
3568	be,a,pn	%icc, 1f
3569	  mov	%g3, %g5			! trapstat wants TTE in %g5
3570#endif /* sun4v */
3571	cmp	%g5, FAST_IMMU_MISS_TT
3572	be,pn	%icc, 1f
3573	  mov	%g3, %g5			! trapstat wants TTE in %g5
3574	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3575	! trapstat wants TTE in %g5
3576	retry
35771:
3578	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3579	! trapstat wants TTE in %g5
3580	retry
3581
3582tsb_ism:
3583	/*
3584	 * This is an ISM [i|d]tlb miss.  We optimize for largest
3585	 * page size down to smallest.
3586	 *
3587	 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access
3588	 *	register
3589	 * g3 = ismmap->ism_seg
3590	 * g4 = physical address of ismmap->ism_sfmmu
3591	 * g6 = tsbmiss area
3592	 */
3593	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
3594	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
3595	  mov	PTL1_BAD_ISM, %g1
3596						/* g5 = pa of imap_vb_shift */
3597	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
3598	lduba	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
3599	srlx	%g3, %g4, %g3			/* clr size field */
3600	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
3601	sllx    %g3, %g4, %g3                   /* g3 = ism vbase */
3602	and     %g2, %g1, %g4                   /* g4 = ctx number */
3603	andn    %g2, %g1, %g1                   /* g1 = tlb miss vaddr */
3604	sub     %g1, %g3, %g2                   /* g2 = offset in ISM seg */
3605	or      %g2, %g4, %g2                   /* g2 = (pseudo-)tagacc */
3606	sub     %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
3607	lduha   [%g5]ASI_MEM, %g4               /* g5 = pa of imap_hatflags */
3608#if defined(sun4v) || defined(UTSB_PHYS)
3609	and     %g4, HAT_CTX1_FLAG, %g5         /* g5 = imap_hatflags */
3610	brz,pt %g5, tsb_chk4M_ism
3611	  nop
3612	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g5
3613	or      %g5, HAT_CHKCTX1_FLAG, %g5
3614	stub    %g5, [%g6 + TSBMISS_URTTEFLAGS]
3615	rdpr    %tt, %g5
3616	SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl)
3617#endif /* defined(sun4v) || defined(UTSB_PHYS) */
3618
3619	/*
3620	 * ISM pages are always locked down.
3621	 * If we can't find the tte then pagefault
3622	 * and let the spt segment driver resolve it.
3623	 *
3624	 * g2 = tagacc w/ISM vaddr (offset in ISM seg)
3625	 * g4 = imap_hatflags
3626	 * g6 = tsb miss area
3627	 * g7 = ISM hatid
3628	 */
3629
3630tsb_chk4M_ism:
3631	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
3632	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
3633	  nop
3634
3635tsb_ism_32M:
3636	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
3637	brz,pn	%g5, tsb_ism_256M
3638	  nop
3639
3640	/*
3641	 * 32M hash.
3642	 */
3643
3644	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M,
3645	    TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3646	    tsb_ism_4M)
3647	/* NOT REACHED */
3648
3649tsb_ism_32M_found:
3650	brlz,a,pt %g3, tsb_validtte
3651	  rdpr	%tt, %g7
3652	ba,pt	%xcc, tsb_ism_4M
3653	  nop
3654
3655tsb_ism_256M:
3656	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
3657	brz,a,pn %g5, ptl1_panic
3658	  mov	PTL1_BAD_ISM, %g1
3659
3660	/*
3661	 * 256M hash.
3662	 */
3663	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M,
3664	    TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3665	    tsb_ism_4M)
3666
3667tsb_ism_256M_found:
3668	brlz,a,pt %g3, tsb_validtte
3669	  rdpr	%tt, %g7
3670
3671tsb_ism_4M:
3672	/*
3673	 * 4M hash.
3674	 */
3675	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M,
3676	    TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3677	    tsb_ism_8K)
3678	/* NOT REACHED */
3679
3680tsb_ism_4M_found:
3681	brlz,a,pt %g3, tsb_validtte
3682	  rdpr	%tt, %g7
3683
3684tsb_ism_8K:
3685	/*
3686	 * 8K and 64K hash.
3687	 */
3688
3689	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K,
3690	    TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3691	    tsb_pagefault)
3692	/* NOT REACHED */
3693
3694tsb_ism_8K_found:
3695	brlz,a,pt %g3, tsb_validtte
3696	  rdpr	%tt, %g7
3697
3698tsb_pagefault:
3699	rdpr	%tt, %g7
3700	cmp	%g7, FAST_PROT_TT
3701	be,a,pn	%icc, tsb_protfault
3702	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt
3703
3704tsb_protfault:
3705	/*
3706	 * we get here if we couldn't find a valid tte in the hash.
3707	 *
3708	 * If user and we are at tl>1 we go to window handling code.
3709	 *
3710	 * If kernel and the fault is on the same page as our stack
3711	 * pointer, then we know the stack is bad and the trap handler
3712	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3713	 *
3714	 * If this is a kernel trap and tl>1, panic.
3715	 *
3716	 * Otherwise we call pagefault.
3717	 */
3718	cmp	%g7, FAST_IMMU_MISS_TT
3719#ifdef sun4v
3720	MMU_FAULT_STATUS_AREA(%g4)
3721	ldx	[%g4 + MMFSA_I_CTX], %g5
3722	ldx	[%g4 + MMFSA_D_CTX], %g4
3723	move	%icc, %g5, %g4
3724	cmp	%g7, T_INSTR_MMU_MISS
3725	move	%icc, %g5, %g4
3726#else
3727	mov	MMU_TAG_ACCESS, %g4
3728	ldxa	[%g4]ASI_DMMU, %g2
3729	ldxa	[%g4]ASI_IMMU, %g5
3730	move	%icc, %g5, %g2
3731	cmp	%g7, T_INSTR_MMU_MISS
3732	move	%icc, %g5, %g2
3733	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
3734#endif /* sun4v */
3735	brnz,pn	%g4, 3f				/* skip if not kernel */
3736	  rdpr	%tl, %g5
3737
3738	add	%sp, STACK_BIAS, %g3
3739	srlx	%g3, MMU_PAGESHIFT, %g3
3740	srlx	%g2, MMU_PAGESHIFT, %g4
3741	cmp	%g3, %g4
3742	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
3743	  mov	PTL1_BAD_STACK, %g1
3744
3745	cmp	%g5, 1
3746	ble,pt	%icc, 2f
3747	  nop
3748	TSTAT_CHECK_TL1(2f, %g1, %g2)
3749	rdpr	%tt, %g2
3750	cmp	%g2, FAST_PROT_TT
3751	mov	PTL1_BAD_KPROT_FAULT, %g1
3752	movne	%icc, PTL1_BAD_KMISS, %g1
3753	ba,pt	%icc, ptl1_panic
3754	  nop
3755
37562:
3757	/*
3758	 * We are taking a pagefault in the kernel on a kernel address.  If
3759	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
3760	 * want to call sfmmu_pagefault -- we will instead note that a fault
3761	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
3762	 * (instead of a "retry").  This will step over the faulting
3763	 * instruction.
3764	 */
3765	CPU_INDEX(%g1, %g2)
3766	set	cpu_core, %g2
3767	sllx	%g1, CPU_CORE_SHIFT, %g1
3768	add	%g1, %g2, %g1
3769	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3770	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3771	bz	sfmmu_pagefault
3772	or	%g2, CPU_DTRACE_BADADDR, %g2
3773	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3774	GET_MMU_D_ADDR(%g3, %g4)
3775	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3776	done
3777
37783:
3779	cmp	%g5, 1
3780	ble,pt	%icc, 4f
3781	  nop
3782	TSTAT_CHECK_TL1(4f, %g1, %g2)
3783	ba,pt	%icc, sfmmu_window_trap
3784	  nop
3785
37864:
3787	/*
3788	 * We are taking a pagefault on a non-kernel address.  If we are in
3789	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
3790	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
3791	 */
3792	CPU_INDEX(%g1, %g2)
3793	set	cpu_core, %g2
3794	sllx	%g1, CPU_CORE_SHIFT, %g1
3795	add	%g1, %g2, %g1
3796	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3797	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3798	bz	sfmmu_mmu_trap
3799	or	%g2, CPU_DTRACE_BADADDR, %g2
3800	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3801	GET_MMU_D_ADDR(%g3, %g4)
3802	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3803
3804	/*
3805	 * Be sure that we're actually taking this miss from the kernel --
3806	 * otherwise we have managed to return to user-level with
3807	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3808	 */
3809	rdpr	%tstate, %g2
3810	btst	TSTATE_PRIV, %g2
3811	bz,a	ptl1_panic
3812	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3813	done
3814
3815	ALTENTRY(tsb_tl0_noctxt)
3816	/*
3817	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
3818	 * if it is, indicated that we have faulted and issue a done.
3819	 */
3820	CPU_INDEX(%g5, %g6)
3821	set	cpu_core, %g6
3822	sllx	%g5, CPU_CORE_SHIFT, %g5
3823	add	%g5, %g6, %g5
3824	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
3825	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
3826	bz	1f
3827	or	%g6, CPU_DTRACE_BADADDR, %g6
3828	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
3829	GET_MMU_D_ADDR(%g3, %g4)
3830	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]
3831
3832	/*
3833	 * Be sure that we're actually taking this miss from the kernel --
3834	 * otherwise we have managed to return to user-level with
3835	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3836	 */
3837	rdpr	%tstate, %g5
3838	btst	TSTATE_PRIV, %g5
3839	bz,a	ptl1_panic
3840	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3841	TSTAT_CHECK_TL1(2f, %g1, %g2);
38422:
3843	done
3844
38451:
3846	rdpr	%tt, %g5
3847	cmp	%g5, FAST_IMMU_MISS_TT
3848#ifdef sun4v
3849	MMU_FAULT_STATUS_AREA(%g2)
3850	be,a,pt	%icc, 2f
3851	  ldx	[%g2 + MMFSA_I_CTX], %g3
3852	cmp	%g5, T_INSTR_MMU_MISS
3853	be,a,pt	%icc, 2f
3854	  ldx	[%g2 + MMFSA_I_CTX], %g3
3855	ldx	[%g2 + MMFSA_D_CTX], %g3
38562:
3857#else
3858	mov	MMU_TAG_ACCESS, %g2
3859	be,a,pt	%icc, 2f
3860	  ldxa	[%g2]ASI_IMMU, %g3
3861	ldxa	[%g2]ASI_DMMU, %g3
38622:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
3863#endif /* sun4v */
3864	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
3865	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
3866	rdpr	%tl, %g5
3867	cmp	%g5, 1
3868	ble,pt	%icc, sfmmu_mmu_trap
3869	  nop
3870	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
3871	ba,pt	%icc, sfmmu_window_trap
3872	  nop
3873	SET_SIZE(sfmmu_tsb_miss)
3874#endif  /* lint */
3875
3876#if defined (lint)
3877/*
3878 * This routine will look for a user or kernel vaddr in the hash
3879 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't
3880 * grab any locks.  It should only be used by other sfmmu routines.
3881 */
3882/* ARGSUSED */
3883pfn_t
3884sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
3885{
3886	return(0);
3887}
3888
3889/* ARGSUSED */
3890pfn_t
3891sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno)
3892{
3893	return(0);
3894}
3895
3896#else /* lint */
3897
3898	ENTRY_NP(sfmmu_vatopfn)
3899 	/*
3900 	 * disable interrupts
3901 	 */
3902 	rdpr	%pstate, %o3
3903#ifdef DEBUG
3904	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1)
3905#endif
3906	/*
3907	 * disable interrupts to protect the TSBMISS area
3908	 */
3909	andn    %o3, PSTATE_IE, %o5
3910	wrpr    %o5, 0, %pstate
3911
3912	/*
3913	 * o0 = vaddr
3914	 * o1 = sfmmup
3915	 * o2 = ttep
3916	 */
3917	CPU_TSBMISS_AREA(%g1, %o5)
3918	ldn	[%g1 + TSBMISS_KHATID], %o4
3919	cmp	%o4, %o1
3920	bne,pn	%ncc, vatopfn_nokernel
3921	  mov	TTE64K, %g5			/* g5 = rehash # */
3922	mov %g1,%o5				/* o5 = tsbmiss_area */
3923	/*
3924	 * o0 = vaddr
3925	 * o1 & o4 = hatid
3926	 * o2 = ttep
3927	 * o5 = tsbmiss area
3928	 */
3929	mov	HBLK_RANGE_SHIFT, %g6
39301:
3931
3932	/*
3933	 * o0 = vaddr
3934	 * o1 = sfmmup
3935	 * o2 = ttep
3936	 * o3 = old %pstate
3937	 * o4 = hatid
3938	 * o5 = tsbmiss
3939	 * g5 = rehash #
3940	 * g6 = hmeshift
3941	 *
3942	 * The first arg to GET_TTE is actually tagaccess register
3943	 * not just vaddr. Since this call is for kernel we need to clear
3944	 * any lower vaddr bits that would be interpreted as ctx bits.
3945	 */
3946	set     TAGACC_CTX_MASK, %g1
3947	andn    %o0, %g1, %o0
3948	GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3,
3949		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
3950
3951kvtop_hblk_found:
3952	/*
3953	 * o0 = vaddr
3954	 * o1 = sfmmup
3955	 * o2 = ttep
3956	 * g1 = tte
3957	 * g2 = tte pa
3958	 * g3 = scratch
3959	 * o2 = tsbmiss area
3960	 * o1 = hat id
3961	 */
3962	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
3963	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3964	stx %g1,[%o2]				/* put tte into *ttep */
3965	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
3966	/*
3967	 * o0 = vaddr
3968	 * o1 = sfmmup
3969	 * o2 = ttep
3970	 * g1 = pfn
3971	 */
3972	ba,pt	%xcc, 6f
3973	  mov	%g1, %o0
3974
3975kvtop_nohblk:
3976	/*
3977	 * we get here if we couldn't find valid hblk in hash.  We rehash
3978	 * if neccesary.
3979	 */
3980	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
3981#ifdef sun4v
3982	cmp	%g5, MAX_HASHCNT
3983#else
3984	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
3985#endif /* sun4v */
3986	be,a,pn	%icc, 6f
3987	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3988	mov	%o1, %o4			/* restore hatid */
3989#ifdef sun4v
3990        add	%g5, 2, %g5
3991	cmp	%g5, 3
3992	move	%icc, MMU_PAGESHIFT4M, %g6
3993	ba,pt	%icc, 1b
3994	movne	%icc, MMU_PAGESHIFT256M, %g6
3995#else
3996        inc	%g5
3997	cmp	%g5, 2
3998	move	%icc, MMU_PAGESHIFT512K, %g6
3999	ba,pt	%icc, 1b
4000	movne	%icc, MMU_PAGESHIFT4M, %g6
4001#endif /* sun4v */
40026:
4003	retl
4004 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4005
4006tsb_suspend:
4007	/*
4008	 * o0 = vaddr
4009	 * o1 = sfmmup
4010	 * o2 = ttep
4011	 * g1 = tte
4012	 * g2 = tte pa
4013	 * g3 = tte va
4014	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
4015	 */
4016	stx %g1,[%o2]				/* put tte into *ttep */
4017	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
4018	  sub	%g0, 1, %o0			/* output = PFN_INVALID */
4019	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
40208:
4021	retl
4022	 wrpr	%g0, %o3, %pstate		/* enable interrupts */
4023
4024vatopfn_nokernel:
4025	/*
4026	 * This routine does NOT support user addresses
4027	 * There is a routine in C that supports this.
4028	 * The only reason why we don't have the C routine
4029	 * support kernel addresses as well is because
4030	 * we do va_to_pa while holding the hashlock.
4031	 */
4032 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4033	save	%sp, -SA(MINFRAME), %sp
4034	sethi	%hi(sfmmu_panic3), %o0
4035	call	panic
4036	 or	%o0, %lo(sfmmu_panic3), %o0
4037
4038	SET_SIZE(sfmmu_vatopfn)
4039
4040	/*
4041	 * %o0 = vaddr
4042	 * %o1 = hashno (aka szc)
4043	 *
4044	 *
4045	 * This routine is similar to sfmmu_vatopfn() but will only look for
4046	 * a kernel vaddr in the hash structure for the specified rehash value.
4047	 * It's just an optimization for the case when pagesize for a given
4048	 * va range is already known (e.g. large page heap) and we don't want
4049	 * to start the search with rehash value 1 as sfmmu_vatopfn() does.
4050	 *
4051	 * Returns valid pfn or PFN_INVALID if
4052	 * tte for specified rehash # is not found, invalid or suspended.
4053	 */
4054	ENTRY_NP(sfmmu_kvaszc2pfn)
4055 	/*
4056 	 * disable interrupts
4057 	 */
4058 	rdpr	%pstate, %o3
4059#ifdef DEBUG
4060	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1)
4061#endif
4062	/*
4063	 * disable interrupts to protect the TSBMISS area
4064	 */
4065	andn    %o3, PSTATE_IE, %o5
4066	wrpr    %o5, 0, %pstate
4067
4068	CPU_TSBMISS_AREA(%g1, %o5)
4069	ldn	[%g1 + TSBMISS_KHATID], %o4
4070	sll	%o1, 1, %g6
4071	add	%g6, %o1, %g6
4072	add	%g6, MMU_PAGESHIFT, %g6
4073	/*
4074	 * %o0 = vaddr
4075	 * %o1 = hashno
4076	 * %o3 = old %pstate
4077	 * %o4 = ksfmmup
4078	 * %g1 = tsbmiss area
4079	 * %g6 = hmeshift
4080	 */
4081
4082	/*
4083	 * The first arg to GET_TTE is actually tagaccess register
4084	 * not just vaddr. Since this call is for kernel we need to clear
4085	 * any lower vaddr bits that would be interpreted as ctx bits.
4086	 */
4087	srlx	%o0, MMU_PAGESHIFT, %o0
4088	sllx	%o0, MMU_PAGESHIFT, %o0
4089	GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5,
4090		kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk,
4091		kvaszc2pfn_nohblk)
4092
4093kvaszc2pfn_hblk_found:
4094	/*
4095	 * %g3 = tte
4096	 * %o0 = vaddr
4097	 */
4098	brgez,a,pn %g3, 1f			/* check if tte is invalid */
4099	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4100	TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5)
4101	/*
4102	 * g3 = pfn
4103	 */
4104	ba,pt	%xcc, 1f
4105	  mov	%g3, %o0
4106
4107kvaszc2pfn_nohblk:
4108	mov	-1, %o0
4109
41101:
4111	retl
4112 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4113
4114	SET_SIZE(sfmmu_kvaszc2pfn)
4115
4116#endif /* lint */
4117
4118
4119
4120#if !defined(lint)
4121
4122/*
4123 * kpm lock used between trap level tsbmiss handler and kpm C level.
4124 */
4125#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
4126	mov     0xff, tmp1						;\
4127label1:									;\
4128	casa    [kpmlckp]asi, %g0, tmp1					;\
4129	brnz,pn tmp1, label1						;\
4130	mov     0xff, tmp1						;\
4131	membar  #LoadLoad
4132
4133#define KPMLOCK_EXIT(kpmlckp, asi)					\
4134	membar  #LoadStore|#StoreStore					;\
4135	sta     %g0, [kpmlckp]asi
4136
4137/*
4138 * Lookup a memseg for a given pfn and if found, return the physical
4139 * address of the corresponding struct memseg in mseg, otherwise
4140 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
4141 * tsbmp, %asi is assumed to be ASI_MEM.
4142 * This lookup is done by strictly traversing only the physical memseg
4143 * linkage. The more generic approach, to check the virtual linkage
4144 * before using the physical (used e.g. with hmehash buckets), cannot
4145 * be used here. Memory DR operations can run in parallel to this
4146 * lookup w/o any locks and updates of the physical and virtual linkage
4147 * cannot be done atomically wrt. to each other. Because physical
4148 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
4149 * as "physical NULL" pointer.
4150 */
4151#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
4152	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
4153	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
4154	udivx	pfn, mseg, mseg						;\
4155	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
4156	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
4157	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
4158	add	tmp1, mseg, tmp1					;\
4159	ldxa	[tmp1]%asi, mseg					;\
4160	cmp	mseg, MSEG_NULLPTR_PA					;\
4161	be,pn	%xcc, label/**/1		/* if not found */	;\
4162	  nop								;\
4163	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
4164	cmp	pfn, tmp1			/* pfn - pages_base */	;\
4165	blu,pn	%xcc, label/**/1					;\
4166	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
4167	cmp	pfn, tmp2			/* pfn - pages_end */	;\
4168	bgeu,pn	%xcc, label/**/1					;\
4169	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
4170	mulx	tmp1, PAGE_SIZE, tmp1					;\
4171	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
4172	add	tmp2, tmp1, tmp1			/* pp */	;\
4173	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
4174	cmp	tmp2, pfn						;\
4175	be,pt	%xcc, label/**/_ok			/* found */	;\
4176label/**/1:								;\
4177	/* brute force lookup */					;\
4178	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
4179	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
4180label/**/2:								;\
4181	cmp	mseg, MSEG_NULLPTR_PA					;\
4182	be,pn	%xcc, label/**/_ok		/* if not found */	;\
4183	  nop								;\
4184	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
4185	cmp	pfn, tmp1			/* pfn - pages_base */	;\
4186	blu,a,pt %xcc, label/**/2					;\
4187	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
4188	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
4189	cmp	pfn, tmp2			/* pfn - pages_end */	;\
4190	bgeu,a,pt %xcc, label/**/2					;\
4191	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
4192label/**/_ok:
4193
4194	/*
4195	 * kpm tsb miss handler large pages
4196	 * g1 = 8K kpm TSB entry pointer
4197	 * g2 = tag access register
4198	 * g3 = 4M kpm TSB entry pointer
4199	 */
4200	ALTENTRY(sfmmu_kpm_dtsb_miss)
4201	TT_TRACE(trace_tsbmiss)
4202
4203	CPU_INDEX(%g7, %g6)
4204	sethi	%hi(kpmtsbm_area), %g6
4205	sllx	%g7, KPMTSBM_SHIFT, %g7
4206	or	%g6, %lo(kpmtsbm_area), %g6
4207	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4208
4209	/* check enable flag */
4210	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4211	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4212	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4213	  nop
4214
4215	/* VA range check */
4216	ldx	[%g6 + KPMTSBM_VBASE], %g7
4217	cmp	%g2, %g7
4218	blu,pn	%xcc, sfmmu_tsb_miss
4219	  ldx	[%g6 + KPMTSBM_VEND], %g5
4220	cmp	%g2, %g5
4221	bgeu,pn	%xcc, sfmmu_tsb_miss
4222	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]
4223
4224	/*
4225	 * check TL tsbmiss handling flag
4226	 * bump tsbmiss counter
4227	 */
4228	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4229#ifdef	DEBUG
4230	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
4231	inc	%g5
4232	brz,pn	%g3, sfmmu_kpm_exception
4233	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4234#else
4235	inc	%g5
4236	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4237#endif
4238	/*
4239	 * At this point:
4240	 *  g1 = 8K kpm TSB pointer (not used)
4241	 *  g2 = tag access register
4242	 *  g3 = clobbered
4243	 *  g6 = per-CPU kpm tsbmiss area
4244	 *  g7 = kpm_vbase
4245	 */
4246
4247	/* vaddr2pfn */
4248	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
4249	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4250	srax    %g4, %g3, %g2			/* which alias range (r) */
4251	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
4252	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
4253
4254	/*
4255	 * Setup %asi
4256	 * mseg_pa = page_numtomemseg_nolock(pfn)
4257	 * if (mseg_pa == NULL) sfmmu_kpm_exception
4258	 * g2=pfn
4259	 */
4260	mov	ASI_MEM, %asi
4261	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
4262	cmp	%g3, MSEG_NULLPTR_PA
4263	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4264	  nop
4265
4266	/*
4267	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
4268	 * g2=pfn g3=mseg_pa
4269	 */
4270	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
4271	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4272	srlx	%g2, %g5, %g4
4273	sllx	%g4, %g5, %g4
4274	sub	%g4, %g7, %g4
4275	srlx	%g4, %g5, %g4
4276
4277	/*
4278	 * Validate inx value
4279	 * g2=pfn g3=mseg_pa g4=inx
4280	 */
4281#ifdef	DEBUG
4282	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4283	cmp	%g4, %g5			/* inx - nkpmpgs */
4284	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4285	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4286#else
4287	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4288#endif
4289	/*
4290	 * kp = &mseg_pa->kpm_pages[inx]
4291	 */
4292	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
4293	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
4294	add	%g5, %g4, %g5			/* kp */
4295
4296	/*
4297	 * KPMP_HASH(kp)
4298	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
4299	 */
4300	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4301	sub	%g7, 1, %g7			/* mask */
4302	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
4303	add	%g5, %g1, %g5			/* y = ksp + x */
4304	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4305
4306	/*
4307	 * Calculate physical kpm_page pointer
4308	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4309	 */
4310	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
4311	add	%g1, %g4, %g1			/* kp_pa */
4312
4313	/*
4314	 * Calculate physical hash lock address
4315	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
4316	 */
4317	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
4318	sllx	%g5, KPMHLK_SHIFT, %g5
4319	add	%g4, %g5, %g3
4320	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */
4321
4322	/*
4323	 * Assemble tte
4324	 * g1=kp_pa g2=pfn g3=hlck_pa
4325	 */
4326#ifdef sun4v
4327	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4328	sllx	%g5, 32, %g5
4329	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4330	or	%g4, TTE4M, %g4
4331	or	%g5, %g4, %g5
4332#else
4333	sethi	%hi(TTE_VALID_INT), %g4
4334	mov	TTE4M, %g5
4335	sllx	%g5, TTE_SZ_SHFT_INT, %g5
4336	or	%g5, %g4, %g5			/* upper part */
4337	sllx	%g5, 32, %g5
4338	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4339	or	%g5, %g4, %g5
4340#endif
4341	sllx	%g2, MMU_PAGESHIFT, %g4
4342	or	%g5, %g4, %g5			/* tte */
4343	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4344	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4345
4346	/*
4347	 * tsb dropin
4348	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
4349	 */
4350
4351	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4352	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
4353
4354	/* use C-handler if there's no go for dropin */
4355	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
4356	cmp	%g7, -1
4357	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
4358	  nop
4359
4360#ifdef	DEBUG
4361	/* double check refcnt */
4362	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
4363	brz,pn	%g7, 5f			/* let C-handler deal with this */
4364	  nop
4365#endif
4366
4367#ifndef sun4v
4368	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4369	mov	ASI_N, %g1
4370	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4371	movnz	%icc, ASI_MEM, %g1
4372	mov	%g1, %asi
4373#endif
4374
4375	/*
4376	 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
4377	 * If we fail to lock the TSB entry then just load the tte into the
4378	 * TLB.
4379	 */
4380	TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1)
4381
4382	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4383	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4384locked_tsb_l1:
4385	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
4386
4387	/* KPMLOCK_EXIT(kpmlckp, asi) */
4388	KPMLOCK_EXIT(%g3, ASI_MEM)
4389
4390	/*
4391	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4392	 * point to trapstat's TSB miss return code (note that trapstat
4393	 * itself will patch the correct offset to add).
4394	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4395	 */
4396	rdpr	%tl, %g7
4397	cmp	%g7, 1
4398	ble	%icc, 0f
4399	sethi	%hi(KERNELBASE), %g6
4400	rdpr	%tpc, %g7
4401	or	%g6, %lo(KERNELBASE), %g6
4402	cmp	%g7, %g6
4403	bgeu	%xcc, 0f
4404	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
4405	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4406	wrpr	%g7, %tpc
4407	add	%g7, 4, %g7
4408	wrpr	%g7, %tnpc
44090:
4410	retry
44115:
4412	/* g3=hlck_pa */
4413	KPMLOCK_EXIT(%g3, ASI_MEM)
4414	ba,pt	%icc, sfmmu_kpm_exception
4415	  nop
4416	SET_SIZE(sfmmu_kpm_dtsb_miss)
4417
4418	/*
4419	 * kpm tsbmiss handler for smallpages
4420	 * g1 = 8K kpm TSB pointer
4421	 * g2 = tag access register
4422	 * g3 = 4M kpm TSB pointer
4423	 */
4424	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
4425	TT_TRACE(trace_tsbmiss)
4426	CPU_INDEX(%g7, %g6)
4427	sethi	%hi(kpmtsbm_area), %g6
4428	sllx	%g7, KPMTSBM_SHIFT, %g7
4429	or	%g6, %lo(kpmtsbm_area), %g6
4430	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4431
4432	/* check enable flag */
4433	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4434	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4435	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4436	  nop
4437
4438	/*
4439	 * VA range check
4440	 * On fail: goto sfmmu_tsb_miss
4441	 */
4442	ldx	[%g6 + KPMTSBM_VBASE], %g7
4443	cmp	%g2, %g7
4444	blu,pn	%xcc, sfmmu_tsb_miss
4445	  ldx	[%g6 + KPMTSBM_VEND], %g5
4446	cmp	%g2, %g5
4447	bgeu,pn	%xcc, sfmmu_tsb_miss
4448	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */
4449
4450	/*
4451	 * check TL tsbmiss handling flag
4452	 * bump tsbmiss counter
4453	 */
4454	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4455#ifdef	DEBUG
4456	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
4457	inc	%g5
4458	brz,pn	%g1, sfmmu_kpm_exception
4459	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4460#else
4461	inc	%g5
4462	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4463#endif
4464	/*
4465	 * At this point:
4466	 *  g1 = clobbered
4467	 *  g2 = tag access register
4468	 *  g3 = 4M kpm TSB pointer (not used)
4469	 *  g6 = per-CPU kpm tsbmiss area
4470	 *  g7 = kpm_vbase
4471	 */
4472
4473	/*
4474	 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr)
4475	 * which is defined in mach_kpm.h. Any changes in that macro
4476	 * should also be ported back to this assembly code.
4477	 */
4478	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3	/* g3 = kpm_size_shift */
4479	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4480	srax    %g4, %g3, %g7			/* which alias range (r) */
4481	brz,pt	%g7, 2f
4482	  sethi   %hi(vac_colors_mask), %g5
4483	ld	[%g5 + %lo(vac_colors_mask)], %g5
4484
4485	srlx	%g2, MMU_PAGESHIFT, %g1		/* vaddr >> MMU_PAGESHIFT */
4486	and	%g1, %g5, %g1			/* g1 = v */
4487	sllx	%g7, %g3, %g5			/* g5 = r << kpm_size_shift */
4488	cmp	%g7, %g1			/* if (r > v) */
4489	bleu,pn %xcc, 1f
4490	  sub   %g4, %g5, %g4			/* paddr -= r << kpm_size_shift */
4491	sub	%g7, %g1, %g5			/* g5 = r - v */
4492	sllx	%g5, MMU_PAGESHIFT, %g7		/* (r-v) << MMU_PAGESHIFT */
4493	add	%g4, %g7, %g4			/* paddr += (r-v)<<MMU_PAGESHIFT */
4494	ba	2f
4495	  nop
44961:
4497	sllx	%g7, MMU_PAGESHIFT, %g5		/* else */
4498	sub	%g4, %g5, %g4			/* paddr -= r << MMU_PAGESHIFT */
4499
4500	/*
4501	 * paddr2pfn
4502	 *  g1 = vcolor (not used)
4503	 *  g2 = tag access register
4504	 *  g3 = clobbered
4505	 *  g4 = paddr
4506	 *  g5 = clobbered
4507	 *  g6 = per-CPU kpm tsbmiss area
4508	 *  g7 = clobbered
4509	 */
45102:
4511	srlx	%g4, MMU_PAGESHIFT, %g2		/* g2 = pfn */
4512
4513	/*
4514	 * Setup %asi
4515	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
4516	 * if (mseg not found) sfmmu_kpm_exception
4517	 * g2=pfn g6=per-CPU kpm tsbmiss area
4518	 * g4 g5 g7 for scratch use.
4519	 */
4520	mov	ASI_MEM, %asi
4521	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
4522	cmp	%g3, MSEG_NULLPTR_PA
4523	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4524	  nop
4525
4526	/*
4527	 * inx = pfn - mseg_pa->kpm_pbase
4528	 * g2=pfn  g3=mseg_pa  g6=per-CPU kpm tsbmiss area
4529	 */
4530	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4531	sub	%g2, %g7, %g4
4532
4533#ifdef	DEBUG
4534	/*
4535	 * Validate inx value
4536	 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area
4537	 */
4538	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4539	cmp	%g4, %g5			/* inx - nkpmpgs */
4540	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4541	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4542#else
4543	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4544#endif
4545	/* ksp = &mseg_pa->kpm_spages[inx] */
4546	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
4547	add	%g5, %g4, %g5			/* ksp */
4548
4549	/*
4550	 * KPMP_SHASH(kp)
4551	 * g2=pfn g3=mseg_pa g4=inx g5=ksp
4552	 * g6=per-CPU kpm tsbmiss area  g7=kpmp_stable_sz
4553	 */
4554	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4555	sub	%g7, 1, %g7			/* mask */
4556	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
4557	add	%g5, %g1, %g5			/* y = ksp + x */
4558	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4559
4560	/*
4561	 * Calculate physical kpm_spage pointer
4562	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4563	 * g6=per-CPU kpm tsbmiss area
4564	 */
4565	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
4566	add	%g1, %g4, %g1			/* ksp_pa */
4567
4568	/*
4569	 * Calculate physical hash lock address.
4570	 * Note: Changes in kpm_shlk_t must be reflected here.
4571	 * g1=ksp_pa g2=pfn g5=hashinx
4572	 * g6=per-CPU kpm tsbmiss area
4573	 */
4574	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
4575	sllx	%g5, KPMSHLK_SHIFT, %g5
4576	add	%g4, %g5, %g3			/* hlck_pa */
4577
4578	/*
4579	 * Assemble non-cacheable tte initially
4580	 * g1=ksp_pa g2=pfn g3=hlck_pa
4581	 * g6=per-CPU kpm tsbmiss area
4582	 */
4583	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4584	sllx	%g5, 32, %g5
4585	mov	(TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4586	or	%g5, %g4, %g5
4587	sllx	%g2, MMU_PAGESHIFT, %g4
4588	or	%g5, %g4, %g5			/* tte */
4589	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4590	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4591
4592	/*
4593	 * tsb dropin
4594	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable)
4595	 * g6=per-CPU kpm tsbmiss area  g7=scratch register
4596	 */
4597
4598	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4599	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
4600
4601	/* use C-handler if there's no go for dropin */
4602	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7	/* kp_mapped */
4603	andcc	%g7, KPM_MAPPED_GO, %g0			/* go or no go ? */
4604	bz,pt	%icc, 5f				/* no go */
4605	  nop
4606	and	%g7, KPM_MAPPED_MASK, %g7		/* go */
4607	cmp	%g7, KPM_MAPPEDS			/* cacheable ? */
4608	be,a,pn	%xcc, 3f
4609	  or	%g5, TTE_CV_INT, %g5			/* cacheable */
46103:
4611#ifndef sun4v
4612	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4613	mov	ASI_N, %g1
4614	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4615	movnz	%icc, ASI_MEM, %g1
4616	mov	%g1, %asi
4617#endif
4618
4619	/*
4620	 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
4621	 * If we fail to lock the TSB entry then just load the tte into the
4622	 * TLB.
4623	 */
4624	TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2)
4625
4626	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4627	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4628locked_tsb_l2:
4629	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
4630
4631	/* KPMLOCK_EXIT(kpmlckp, asi) */
4632	KPMLOCK_EXIT(%g3, ASI_MEM)
4633
4634	/*
4635	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4636	 * point to trapstat's TSB miss return code (note that trapstat
4637	 * itself will patch the correct offset to add).
4638	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4639	 */
4640	rdpr	%tl, %g7
4641	cmp	%g7, 1
4642	ble	%icc, 0f
4643	sethi	%hi(KERNELBASE), %g6
4644	rdpr	%tpc, %g7
4645	or	%g6, %lo(KERNELBASE), %g6
4646	cmp	%g7, %g6
4647	bgeu	%xcc, 0f
4648	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
4649	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4650	wrpr	%g7, %tpc
4651	add	%g7, 4, %g7
4652	wrpr	%g7, %tnpc
46530:
4654	retry
46555:
4656	/* g3=hlck_pa */
4657	KPMLOCK_EXIT(%g3, ASI_MEM)
4658	ba,pt	%icc, sfmmu_kpm_exception
4659	  nop
4660	SET_SIZE(sfmmu_kpm_dtsb_miss_small)
4661
4662#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
4663#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
4664#endif
4665
4666#endif /* lint */
4667
4668#ifdef	lint
4669/*
4670 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
4671 * Called from C-level, sets/clears "go" indication for trap level handler.
4672 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
4673 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
4674 * Assumes khl_mutex is held when called from C-level.
4675 */
4676/* ARGSUSED */
4677void
4678sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
4679{
4680}
4681
4682/*
4683 * kpm_smallpages: stores val to byte at address mapped within
4684 * low level lock brackets. The old value is returned.
4685 * Called from C-level.
4686 */
4687/* ARGSUSED */
4688int
4689sfmmu_kpm_stsbmtl(uchar_t *mapped, uint_t *kshl_lock, int val)
4690{
4691	return (0);
4692}
4693
4694#else /* lint */
4695
4696	.seg	".data"
4697sfmmu_kpm_tsbmtl_panic:
4698	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
4699	.byte	0
4700sfmmu_kpm_stsbmtl_panic:
4701	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
4702	.byte	0
4703	.align	4
4704	.seg	".text"
4705
4706	ENTRY_NP(sfmmu_kpm_tsbmtl)
4707	rdpr	%pstate, %o3
4708	/*
4709	 * %o0 = &kp_refcntc
4710	 * %o1 = &khl_lock
4711	 * %o2 = 0/1 (off/on)
4712	 * %o3 = pstate save
4713	 */
4714#ifdef DEBUG
4715	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4716	bnz,pt %icc, 1f				/* disabled, panic	 */
4717	  nop
4718	save	%sp, -SA(MINFRAME), %sp
4719	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
4720	call	panic
4721	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
4722	ret
4723	restore
47241:
4725#endif /* DEBUG */
4726	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4727
4728	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
4729	mov	-1, %o5
4730	brz,a	%o2, 2f
4731	  mov	0, %o5
47322:
4733	sth	%o5, [%o0]
4734	KPMLOCK_EXIT(%o1, ASI_N)
4735
4736	retl
4737	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4738	SET_SIZE(sfmmu_kpm_tsbmtl)
4739
4740	ENTRY_NP(sfmmu_kpm_stsbmtl)
4741	rdpr	%pstate, %o3
4742	/*
4743	 * %o0 = &mapped
4744	 * %o1 = &kshl_lock
4745	 * %o2 = val
4746	 * %o3 = pstate save
4747	 */
4748#ifdef DEBUG
4749	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4750	bnz,pt %icc, 1f				/* disabled, panic	 */
4751	  nop
4752	save	%sp, -SA(MINFRAME), %sp
4753	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
4754	call	panic
4755	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
4756	ret
4757	restore
47581:
4759#endif /* DEBUG */
4760	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4761
4762	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
4763	ldsb	[%o0], %o5
4764	stb	%o2, [%o0]
4765	KPMLOCK_EXIT(%o1, ASI_N)
4766
4767	and	%o5, KPM_MAPPED_MASK, %o0	/* return old val */
4768	retl
4769	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4770	SET_SIZE(sfmmu_kpm_stsbmtl)
4771
4772#endif /* lint */
4773
4774#ifndef lint
4775#ifdef sun4v
4776	/*
4777	 * User/kernel data miss w// multiple TSBs
4778	 * The first probe covers 8K, 64K, and 512K page sizes,
4779	 * because 64K and 512K mappings are replicated off 8K
4780	 * pointer.  Second probe covers 4M page size only.
4781	 *
4782	 * MMU fault area contains miss address and context.
4783	 */
4784	ALTENTRY(sfmmu_slow_dmmu_miss)
4785	GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)	! %g2 = ptagacc, %g3 = ctx type
4786
4787slow_miss_common:
4788	/*
4789	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
4790	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
4791	 */
4792	brnz,pt	%g3, 8f			! check for user context
4793	  nop
4794
4795	/*
4796	 * Kernel miss
4797	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
4798	 * branch to sfmmu_tsb_miss_tt to handle it.
4799	 */
4800	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4801sfmmu_dslow_patch_ktsb_base:
4802	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
4803sfmmu_dslow_patch_ktsb_szcode:
4804	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
4805
4806	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
4807	! %g1 = First TSB entry pointer, as TSB miss handler expects
4808
4809	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4810sfmmu_dslow_patch_ktsb4m_base:
4811	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
4812sfmmu_dslow_patch_ktsb4m_szcode:
4813	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
4814
4815	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
4816	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
4817	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4818	.empty
4819
48208:
4821	/*
4822	 * User miss
4823	 * Get first TSB pointer in %g1
4824	 * Get second TSB pointer (or NULL if no second TSB) in %g3
4825	 * Branch to sfmmu_tsb_miss_tt to handle it
4826	 */
4827	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
4828	/* %g1 = first TSB entry ptr now, %g2 preserved */
4829
4830	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
4831	brlz,pt %g3, sfmmu_tsb_miss_tt		/* done if no 2nd TSB */
4832	  nop
4833
4834	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
4835	/* %g3 = second TSB entry ptr now, %g2 preserved */
48369:
4837	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4838	.empty
4839	SET_SIZE(sfmmu_slow_dmmu_miss)
4840
4841
4842	/*
4843	 * User/kernel instruction miss w/ multiple TSBs
4844	 * The first probe covers 8K, 64K, and 512K page sizes,
4845	 * because 64K and 512K mappings are replicated off 8K
4846	 * pointer.  Second probe covers 4M page size only.
4847	 *
4848	 * MMU fault area contains miss address and context.
4849	 */
4850	ALTENTRY(sfmmu_slow_immu_miss)
4851	GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)
4852	ba,a,pt	%xcc, slow_miss_common
4853	SET_SIZE(sfmmu_slow_immu_miss)
4854
4855#endif /* sun4v */
4856#endif	/* lint */
4857
4858#ifndef lint
4859
4860/*
4861 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
4862 */
4863	.seg	".data"
4864	.align	64
4865	.global tsbmiss_area
4866tsbmiss_area:
4867	.skip	(TSBMISS_SIZE * NCPU)
4868
4869	.align	64
4870	.global kpmtsbm_area
4871kpmtsbm_area:
4872	.skip	(KPMTSBM_SIZE * NCPU)
4873#endif	/* lint */
4874