xref: /illumos-gate/usr/src/uts/sfmmu/ml/sfmmu_asm.S (revision 5d9d9091)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26
27/*
28 * SFMMU primitives.  These primitives should only be used by sfmmu
29 * routines.
30 */
31
32#include "assym.h"
33
34#include <sys/asm_linkage.h>
35#include <sys/machtrap.h>
36#include <sys/machasi.h>
37#include <sys/sun4asi.h>
38#include <sys/pte.h>
39#include <sys/mmu.h>
40#include <vm/hat_sfmmu.h>
41#include <vm/seg_spt.h>
42#include <sys/machparam.h>
43#include <sys/privregs.h>
44#include <sys/scb.h>
45#include <sys/intreg.h>
46#include <sys/machthread.h>
47#include <sys/intr.h>
48#include <sys/clock.h>
49#include <sys/trapstat.h>
50
51#ifdef TRAPTRACE
52#include <sys/traptrace.h>
53
54/*
55 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
56 */
57#define	TT_TRACE(label)		\
58	ba	label		;\
59	rd	%pc, %g7
60#else
61
62#define	TT_TRACE(label)
63
64#endif /* TRAPTRACE */
65
66#if (TTE_SUSPEND_SHIFT > 0)
67#define	TTE_SUSPEND_INT_SHIFT(reg)				\
68	sllx	reg, TTE_SUSPEND_SHIFT, reg
69#else
70#define	TTE_SUSPEND_INT_SHIFT(reg)
71#endif
72
73/*
74 * Assumes TSBE_TAG is 0
75 * Assumes TSBE_INTHI is 0
76 * Assumes TSBREG.split is 0
77 */
78
79#if TSBE_TAG != 0
80#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
81#endif
82
83#if TSBTAG_INTHI != 0
84#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
85#endif
86
87/*
88 * The following code assumes the tsb is not split.
89 *
90 * With TSBs no longer shared between processes, it's no longer
91 * necessary to hash the context bits into the tsb index to get
92 * tsb coloring; the new implementation treats the TSB as a
93 * direct-mapped, virtually-addressed cache.
94 *
95 * In:
96 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
97 *    tsbbase = base address of TSB (clobbered)
98 *    tagacc = tag access register (clobbered)
99 *    szc = size code of TSB (ro)
100 *    tmp = scratch reg
101 * Out:
102 *    tsbbase = pointer to entry in TSB
103 */
104#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
105	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
106	srlx	tagacc, vpshift, tagacc 				;\
107	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
108	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
109	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
110	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
111	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */
112
113/*
114 * When the kpm TSB is used it is assumed that it is direct mapped
115 * using (vaddr>>vpshift)%tsbsz as the index.
116 *
117 * Note that, for now, the kpm TSB and kernel TSB are the same for
118 * each mapping size.  However that need not always be the case.  If
119 * the trap handlers are updated to search a different TSB for kpm
120 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
121 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
122 *
123 * In:
124 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
125 *    vaddr = virtual address (clobbered)
126 *    tsbp, szc, tmp = scratch
127 * Out:
128 *    tsbp = pointer to entry in TSB
129 */
130#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
131	cmp	vpshift, MMU_PAGESHIFT					;\
132	bne,pn	%icc, 1f		/* branch if large case */	;\
133	  sethi	%hi(kpmsm_tsbsz), szc					;\
134	sethi	%hi(kpmsm_tsbbase), tsbp				;\
135	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
136	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
137	ba,pt	%icc, 2f						;\
138	  nop								;\
1391:	sethi	%hi(kpm_tsbsz), szc					;\
140	sethi	%hi(kpm_tsbbase), tsbp					;\
141	ld	[szc + %lo(kpm_tsbsz)], szc				;\
142	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
1432:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
144
145/*
146 * Lock the TSBE at virtual address tsbep.
147 *
148 * tsbep = TSBE va (ro)
149 * tmp1, tmp2 = scratch registers (clobbered)
150 * label = label to jump to if we fail to lock the tsb entry
151 * %asi = ASI to use for TSB access
152 *
153 * NOTE that we flush the TSB using fast VIS instructions that
154 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
155 * not be treated as a locked entry or we'll get stuck spinning on
156 * an entry that isn't locked but really invalid.
157 */
158
159#if defined(UTSB_PHYS)
160
161#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
162	lda	[tsbep]ASI_MEM, tmp1					;\
163	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
164	cmp	tmp1, tmp2 						;\
165	be,a,pn	%icc, label		/* if locked ignore */		;\
166	  nop								;\
167	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
168	cmp	tmp1, tmp2 						;\
169	bne,a,pn %icc, label		/* didn't lock so ignore */	;\
170	  nop								;\
171	/* tsbe lock acquired */					;\
172	membar #StoreStore
173
174#else /* UTSB_PHYS */
175
176#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
177	lda	[tsbep]%asi, tmp1					;\
178	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
179	cmp	tmp1, tmp2 						;\
180	be,a,pn	%icc, label		/* if locked ignore */		;\
181	  nop								;\
182	casa	[tsbep]%asi, tmp1, tmp2					;\
183	cmp	tmp1, tmp2 						;\
184	bne,a,pn %icc, label		/* didn't lock so ignore */	;\
185	  nop								;\
186	/* tsbe lock acquired */					;\
187	membar #StoreStore
188
189#endif /* UTSB_PHYS */
190
191/*
192 * Atomically write TSBE at virtual address tsbep.
193 *
194 * tsbep = TSBE va (ro)
195 * tte = TSBE TTE (ro)
196 * tagtarget = TSBE tag (ro)
197 * %asi = ASI to use for TSB access
198 */
199
200#if defined(UTSB_PHYS)
201
202#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
203	add	tsbep, TSBE_TTE, tmp1					;\
204	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
205	membar #StoreStore						;\
206	add	tsbep, TSBE_TAG, tmp1					;\
207	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */
208
209#else /* UTSB_PHYS */
210
211#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
212	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
213	membar #StoreStore						;\
214	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
215
216#endif /* UTSB_PHYS */
217
218/*
219 * Load an entry into the TSB at TL > 0.
220 *
221 * tsbep = pointer to the TSBE to load as va (ro)
222 * tte = value of the TTE retrieved and loaded (wo)
223 * tagtarget = tag target register.  To get TSBE tag to load,
224 *   we need to mask off the context and leave only the va (clobbered)
225 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
226 * tmp1, tmp2 = scratch registers
227 * label = label to jump to if we fail to lock the tsb entry
228 * %asi = ASI to use for TSB access
229 */
230
231#if defined(UTSB_PHYS)
232
233#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
234	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
235	/*								;\
236	 * I don't need to update the TSB then check for the valid tte.	;\
237	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
238	 * we always invalidate the hash table before we unload the TSB.;\
239	 */								;\
240	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
241	ldxa	[ttepa]ASI_MEM, tte					;\
242	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
243	sethi	%hi(TSBTAG_INVALID), tmp2				;\
244	add	tsbep, TSBE_TAG, tmp1					;\
245	brgez,a,pn tte, label						;\
246	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
247	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
248label:
249
250#else /* UTSB_PHYS */
251
252#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
253	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
254	/*								;\
255	 * I don't need to update the TSB then check for the valid tte.	;\
256	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
257	 * we always invalidate the hash table before we unload the TSB.;\
258	 */								;\
259	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
260	ldxa	[ttepa]ASI_MEM, tte					;\
261	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
262	sethi	%hi(TSBTAG_INVALID), tmp2				;\
263	brgez,a,pn tte, label						;\
264	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
265	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
266label:
267
268#endif /* UTSB_PHYS */
269
270/*
271 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
272 *   for ITLB synthesis.
273 *
274 * tsbep = pointer to the TSBE to load as va (ro)
275 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
276 *   with exec_perm turned off and exec_synth turned on
277 * tagtarget = tag target register.  To get TSBE tag to load,
278 *   we need to mask off the context and leave only the va (clobbered)
279 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
280 * tmp1, tmp2 = scratch registers
281 * label = label to use for branch (text)
282 * %asi = ASI to use for TSB access
283 */
284
285#define	TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
286	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
287	/*								;\
288	 * I don't need to update the TSB then check for the valid tte.	;\
289	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
290	 * we always invalidate the hash table before we unload the TSB.;\
291	 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0	;\
292	 * and exec_synth bit to 1.					;\
293	 */								;\
294	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
295	mov	tte, tmp1						;\
296	ldxa	[ttepa]ASI_MEM, tte					;\
297	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
298	sethi	%hi(TSBTAG_INVALID), tmp2				;\
299	brgez,a,pn tte, label						;\
300	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
301	or	tte, tmp1, tte						;\
302	andn	tte, TTE_EXECPRM_INT, tte				;\
303	or	tte, TTE_E_SYNTH_INT, tte				;\
304	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
305label:
306
307/*
308 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
309 *
310 * tte = value of the TTE, used to get tte_size bits (ro)
311 * tagaccess = tag access register, used to get 4M pfn bits (ro)
312 * pfn = 4M pfn bits shifted to offset for tte (out)
313 * tmp1 = scratch register
314 * label = label to use for branch (text)
315 */
316
317#define	GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)			\
318	/*								;\
319	 * Get 4M bits from tagaccess for 32M, 256M pagesizes.		;\
320	 * Return them, shifted, in pfn.				;\
321	 */								;\
322	srlx	tagaccess, MMU_PAGESHIFT4M, tagaccess			;\
323	srlx	tte, TTE_SZ_SHFT, tmp		/* isolate the */	;\
324	andcc	tmp, TTE_SZ_BITS, %g0		/* tte_size bits */	;\
325	bz,a,pt %icc, label##f		/* if 0, is */		;\
326	  and	tagaccess, 0x7, tagaccess	/* 32M page size */	;\
327	and	tagaccess, 0x3f, tagaccess /* else 256M page size */	;\
328label:									;\
329	sllx	tagaccess, MMU_PAGESHIFT4M, pfn
330
331/*
332 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
333 * for ITLB synthesis.
334 *
335 * tte = value of the TTE, used to get tte_size bits (rw)
336 * tmp1 = scratch register
337 */
338
339#define	SET_TTE4M_PN(tte, tmp)						\
340	/*								;\
341	 * Set 4M pagesize tte bits. 					;\
342	 */								;\
343	set	TTE4M, tmp						;\
344	sllx	tmp, TTE_SZ_SHFT, tmp					;\
345	or	tte, tmp, tte
346
347/*
348 * Load an entry into the TSB at TL=0.
349 *
350 * tsbep = pointer to the TSBE to load as va (ro)
351 * tteva = pointer to the TTE to load as va (ro)
352 * tagtarget = TSBE tag to load (which contains no context), synthesized
353 * to match va of MMU tag target register only (ro)
354 * tmp1, tmp2 = scratch registers (clobbered)
355 * label = label to use for branches (text)
356 * %asi = ASI to use for TSB access
357 */
358
359#if defined(UTSB_PHYS)
360
361#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
362	/* can't rd tteva after locking tsb because it can tlb miss */	;\
363	ldx	[tteva], tteva			/* load tte */		;\
364	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
365	sethi	%hi(TSBTAG_INVALID), tmp2				;\
366	add	tsbep, TSBE_TAG, tmp1					;\
367	brgez,a,pn tteva, label						;\
368	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
369	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
370label:
371
372#else /* UTSB_PHYS */
373
374#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
375	/* can't rd tteva after locking tsb because it can tlb miss */	;\
376	ldx	[tteva], tteva			/* load tte */		;\
377	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
378	sethi	%hi(TSBTAG_INVALID), tmp2				;\
379	brgez,a,pn tteva, label						;\
380	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
381	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
382label:
383
384#endif /* UTSB_PHYS */
385
386/*
387 * Invalidate a TSB entry in the TSB.
388 *
389 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
390 *	 about this earlier to ensure this is true.  Thus when we are
391 *	 directly referencing tsbep below, we are referencing the tte_tag
392 *	 field of the TSBE.  If this  offset ever changes, the code below
393 *	 will need to be modified.
394 *
395 * tsbep = pointer to TSBE as va (ro)
396 * tag = invalidation is done if this matches the TSBE tag (ro)
397 * tmp1 - tmp3 = scratch registers (clobbered)
398 * label = label name to use for branches (text)
399 * %asi = ASI to use for TSB access
400 */
401
402#if defined(UTSB_PHYS)
403
404#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
405	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
406	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
407label##1:								;\
408	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
409	be,a,pn	%icc, label##1	/* so, loop until unlocked */	;\
410	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
411	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
412	cmp	tag, tmp3		/* compare tags */		;\
413	bne,pt	%xcc, label##2	/* if different, do nothing */	;\
414	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
415	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
416	cmp	tmp1, tmp3		/* if not successful */		;\
417	bne,a,pn %icc, label##1	/* start over from the top */	;\
418	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
419label##2:
420
421#else /* UTSB_PHYS */
422
423#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
424	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
425	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
426label##1:								;\
427	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
428	be,a,pn	%icc, label##1	/* so, loop until unlocked */	;\
429	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
430	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
431	cmp	tag, tmp3		/* compare tags */		;\
432	bne,pt	%xcc, label##2	/* if different, do nothing */	;\
433	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
434	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
435	cmp	tmp1, tmp3		/* if not successful */		;\
436	bne,a,pn %icc, label##1	/* start over from the top */	;\
437	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
438label##2:
439
440#endif /* UTSB_PHYS */
441
442#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
443#error	- TSB_SOFTSZ_MASK too small
444#endif
445
446
447/*
448 * An implementation of setx which will be hot patched at run time.
449 * since it is being hot patched, there is no value passed in.
450 * Thus, essentially we are implementing
451 *	setx value, tmp, dest
452 * where value is RUNTIME_PATCH (aka 0) in this case.
453 */
454#define	RUNTIME_PATCH_SETX(dest, tmp)					\
455	sethi	%hh(RUNTIME_PATCH), tmp					;\
456	sethi	%lm(RUNTIME_PATCH), dest				;\
457	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
458	or	dest, %lo(RUNTIME_PATCH), dest				;\
459	sllx	tmp, 32, tmp						;\
460	nop				/* for perf reasons */		;\
461	or	tmp, dest, dest		/* contents of patched value */
462
463
464	.seg	".data"
465	.global	sfmmu_panic1
466sfmmu_panic1:
467	.asciz	"sfmmu_asm: interrupts already disabled"
468
469	.global	sfmmu_panic3
470sfmmu_panic3:
471	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"
472
473	.global	sfmmu_panic4
474sfmmu_panic4:
475	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"
476
477	.global	sfmmu_panic5
478sfmmu_panic5:
479	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"
480
481	.global	sfmmu_panic6
482sfmmu_panic6:
483	.asciz	"sfmmu_asm: interrupts not disabled"
484
485	.global	sfmmu_panic7
486sfmmu_panic7:
487	.asciz	"sfmmu_asm: kernel as"
488
489	.global	sfmmu_panic8
490sfmmu_panic8:
491	.asciz	"sfmmu_asm: gnum is zero"
492
493	.global	sfmmu_panic9
494sfmmu_panic9:
495	.asciz	"sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL"
496
497	.global	sfmmu_panic10
498sfmmu_panic10:
499	.asciz	"sfmmu_asm: valid SCD with no 3rd scd TSB"
500
501	.global	sfmmu_panic11
502sfmmu_panic11:
503	.asciz	"sfmmu_asm: ktsb_phys must not be 0 on a sun4v platform"
504
505        ENTRY(sfmmu_disable_intrs)
506        rdpr    %pstate, %o0
507#ifdef DEBUG
508	PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1)
509#endif /* DEBUG */
510        retl
511          wrpr   %o0, PSTATE_IE, %pstate
512        SET_SIZE(sfmmu_disable_intrs)
513
514	ENTRY(sfmmu_enable_intrs)
515        retl
516          wrpr    %g0, %o0, %pstate
517        SET_SIZE(sfmmu_enable_intrs)
518
519/*
520 * This routine is called both by resume() and sfmmu_get_ctx() to
521 * allocate a new context for the process on a MMU.
522 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID .
523 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which
524 * is the case when sfmmu_alloc_ctx is called from resume().
525 *
526 * The caller must disable interrupts before entering this routine.
527 * To reduce ctx switch overhead, the code contains both 'fast path' and
528 * 'slow path' code. The fast path code covers the common case where only
529 * a quick check is needed and the real ctx allocation is not required.
530 * It can be done without holding the per-process (PP) lock.
531 * The 'slow path' code must be protected by the PP Lock and performs ctx
532 * allocation.
533 * Hardware context register and HAT mmu cnum are updated accordingly.
534 *
535 * %o0 - sfmmup
536 * %o1 - allocflag
537 * %o2 - CPU
538 * %o3 - sfmmu private/shared flag
539 *
540 * ret - 0: no ctx is allocated
541 *       1: a ctx is allocated
542 */
543        ENTRY_NP(sfmmu_alloc_ctx)
544
545#ifdef DEBUG
546	sethi   %hi(ksfmmup), %g1
547	ldx     [%g1 + %lo(ksfmmup)], %g1
548	cmp     %g1, %o0
549	bne,pt   %xcc, 0f
550	  nop
551
552	sethi   %hi(panicstr), %g1		! if kernel as, panic
553        ldx     [%g1 + %lo(panicstr)], %g1
554        tst     %g1
555        bnz,pn  %icc, 7f
556          nop
557
558	sethi	%hi(sfmmu_panic7), %o0
559	call	panic
560	  or	%o0, %lo(sfmmu_panic7), %o0
561
5627:
563	retl
564	  mov	%g0, %o0			! %o0 = ret = 0
565
5660:
567	PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1)
568#endif /* DEBUG */
569
570	mov	%o3, %g1			! save sfmmu pri/sh flag in %g1
571
572	! load global mmu_ctxp info
573	ldx	[%o2 + CPU_MMU_CTXP], %o3		! %o3 = mmu_ctx_t ptr
574
575#ifdef sun4v
576	/* During suspend on sun4v, context domains can be temporary removed */
577	brz,a,pn       %o3, 0f
578	  nop
579#endif
580
581        lduw	[%o2 + CPU_MMU_IDX], %g2		! %g2 = mmu index
582
583	! load global mmu_ctxp gnum
584	ldx	[%o3 + MMU_CTX_GNUM], %o4		! %o4 = mmu_ctxp->gnum
585
586#ifdef DEBUG
587	cmp	%o4, %g0		! mmu_ctxp->gnum should never be 0
588	bne,pt	%xcc, 3f
589	  nop
590
591	sethi   %hi(panicstr), %g1	! test if panicstr is already set
592        ldx     [%g1 + %lo(panicstr)], %g1
593        tst     %g1
594        bnz,pn  %icc, 1f
595          nop
596
597	sethi	%hi(sfmmu_panic8), %o0
598	call	panic
599	  or	%o0, %lo(sfmmu_panic8), %o0
6001:
601	retl
602	  mov	%g0, %o0			! %o0 = ret = 0
6033:
604#endif
605
606	! load HAT sfmmu_ctxs[mmuid] gnum, cnum
607
608	sllx	%g2, SFMMU_MMU_CTX_SHIFT, %g2
609	add	%o0, %g2, %g2		! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
610
611	/*
612	 * %g5 = sfmmu gnum returned
613	 * %g6 = sfmmu cnum returned
614	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
615	 * %g4 = scratch
616	 *
617	 * Fast path code, do a quick check.
618	 */
619	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
620
621	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
622	bne,pt	%icc, 1f			! valid hat cnum, check gnum
623	  nop
624
625	! cnum == INVALID, check allocflag
626	mov	%g0, %g4	! %g4 = ret = 0
627	brz,pt  %o1, 8f		! allocflag == 0, skip ctx allocation, bail
628	  mov	%g6, %o1
629
630	! (invalid HAT cnum) && (allocflag == 1)
631	ba,pt	%icc, 2f
632	  nop
633#ifdef sun4v
6340:
635	set	INVALID_CONTEXT, %o1
636	membar	#LoadStore|#StoreStore
637	ba,pt	%icc, 8f
638	  mov   %g0, %g4                ! %g4 = ret = 0
639#endif
6401:
641	! valid HAT cnum, check gnum
642	cmp	%g5, %o4
643	mov	1, %g4				!%g4 = ret = 1
644	be,a,pt	%icc, 8f			! gnum unchanged, go to done
645	  mov	%g6, %o1
646
6472:
648	/*
649	 * Grab per process (PP) sfmmu_ctx_lock spinlock,
650	 * followed by the 'slow path' code.
651	 */
652	ldstub	[%o0 + SFMMU_CTX_LOCK], %g3	! %g3 = per process (PP) lock
6533:
654	brz	%g3, 5f
655	  nop
6564:
657	brnz,a,pt       %g3, 4b				! spin if lock is 1
658	  ldub	[%o0 + SFMMU_CTX_LOCK], %g3
659	ba	%xcc, 3b				! retry the lock
660	  ldstub	[%o0 + SFMMU_CTX_LOCK], %g3    ! %g3 = PP lock
661
6625:
663	membar  #LoadLoad
664	/*
665	 * %g5 = sfmmu gnum returned
666	 * %g6 = sfmmu cnum returned
667	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
668	 * %g4 = scratch
669	 */
670	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
671
672	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
673	bne,pt	%icc, 1f			! valid hat cnum, check gnum
674	  nop
675
676	! cnum == INVALID, check allocflag
677	mov	%g0, %g4	! %g4 = ret = 0
678	brz,pt	%o1, 2f		! allocflag == 0, called from resume, set hw
679	  mov	%g6, %o1
680
681	! (invalid HAT cnum) && (allocflag == 1)
682	ba,pt	%icc, 6f
683	  nop
6841:
685	! valid HAT cnum, check gnum
686	cmp	%g5, %o4
687	mov	1, %g4				! %g4 = ret  = 1
688	be,a,pt	%icc, 2f			! gnum unchanged, go to done
689	  mov	%g6, %o1
690
691	ba,pt	%icc, 6f
692	  nop
6932:
694	membar  #LoadStore|#StoreStore
695	ba,pt %icc, 8f
696	  clrb  [%o0 + SFMMU_CTX_LOCK]
6976:
698	/*
699	 * We get here if we do not have a valid context, or
700	 * the HAT gnum does not match global gnum. We hold
701	 * sfmmu_ctx_lock spinlock. Allocate that context.
702	 *
703	 * %o3 = mmu_ctxp
704	 */
705	add	%o3, MMU_CTX_CNUM, %g3
706	ld	[%o3 + MMU_CTX_NCTXS], %g4
707
708	/*
709         * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS;
710         * %g3 = mmu cnum address
711	 * %g4 = mmu nctxs
712	 *
713	 * %o0 = sfmmup
714	 * %o1 = mmu current cnum value (used as new cnum)
715	 * %o4 = mmu gnum
716	 *
717	 * %o5 = scratch
718	 */
719	ld	[%g3], %o1
7200:
721	cmp	%o1, %g4
722	bl,a,pt %icc, 1f
723	  add	%o1, 1, %o5		! %o5 = mmu_ctxp->cnum + 1
724
725	/*
726	 * cnum reachs max, bail, so wrap around can be performed later.
727	 */
728	set	INVALID_CONTEXT, %o1
729	mov	%g0, %g4		! %g4 = ret = 0
730
731	membar  #LoadStore|#StoreStore
732	ba,pt	%icc, 8f
733	  clrb	[%o0 + SFMMU_CTX_LOCK]
7341:
735	! %g3 = addr of mmu_ctxp->cnum
736	! %o5 = mmu_ctxp->cnum + 1
737	cas	[%g3], %o1, %o5
738	cmp	%o1, %o5
739	bne,a,pn %xcc, 0b	! cas failed
740	  ld	[%g3], %o1
741
742#ifdef DEBUG
743        set	MAX_SFMMU_CTX_VAL, %o5
744	cmp	%o1, %o5
745	ble,pt %icc, 2f
746	  nop
747
748	sethi	%hi(sfmmu_panic9), %o0
749	call	panic
750	  or	%o0, %lo(sfmmu_panic9), %o0
7512:
752#endif
753	! update hat gnum and cnum
754	sllx	%o4, SFMMU_MMU_GNUM_RSHIFT, %o4
755	or	%o4, %o1, %o4
756	stx	%o4, [%g2 + SFMMU_CTXS]
757
758	membar  #LoadStore|#StoreStore
759	clrb	[%o0 + SFMMU_CTX_LOCK]
760
761	mov	1, %g4			! %g4 = ret = 1
7628:
763	/*
764	 * program the secondary context register
765	 *
766	 * %o1 = cnum
767	 * %g1 = sfmmu private/shared flag (0:private,  1:shared)
768	 */
769
770	/*
771	 * When we come here and context is invalid, we want to set both
772	 * private and shared ctx regs to INVALID. In order to
773	 * do so, we set the sfmmu priv/shared flag to 'private' regardless
774	 * so that private ctx reg will be set to invalid.
775	 * Note that on sun4v values written to private context register are
776	 * automatically written to corresponding shared context register as
777	 * well. On sun4u SET_SECCTX() will invalidate shared context register
778	 * when it sets a private secondary context register.
779	 */
780
781	cmp	%o1, INVALID_CONTEXT
782	be,a,pn	%icc, 9f
783	  clr	%g1
7849:
785
786#ifdef	sun4u
787	ldub	[%o0 + SFMMU_CEXT], %o2
788	sll	%o2, CTXREG_EXT_SHIFT, %o2
789	or	%o1, %o2, %o1
790#endif /* sun4u */
791
792	SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1)
793
794        retl
795          mov   %g4, %o0                        ! %o0 = ret
796
797	SET_SIZE(sfmmu_alloc_ctx)
798
799	ENTRY_NP(sfmmu_modifytte)
800	ldx	[%o2], %g3			/* current */
801	ldx	[%o0], %g1			/* original */
8022:
803	ldx	[%o1], %g2			/* modified */
804	cmp	%g2, %g3			/* is modified = current? */
805	be,a,pt	%xcc,1f				/* yes, don't write */
806	stx	%g3, [%o0]			/* update new original */
807	casx	[%o2], %g1, %g2
808	cmp	%g1, %g2
809	be,pt	%xcc, 1f			/* cas succeeded - return */
810	  nop
811	ldx	[%o2], %g3			/* new current */
812	stx	%g3, [%o0]			/* save as new original */
813	ba,pt	%xcc, 2b
814	  mov	%g3, %g1
8151:	retl
816	membar	#StoreLoad
817	SET_SIZE(sfmmu_modifytte)
818
819	ENTRY_NP(sfmmu_modifytte_try)
820	ldx	[%o1], %g2			/* modified */
821	ldx	[%o2], %g3			/* current */
822	ldx	[%o0], %g1			/* original */
823	cmp	%g3, %g2			/* is modified = current? */
824	be,a,pn %xcc,1f				/* yes, don't write */
825	mov	0, %o1				/* as if cas failed. */
826
827	casx	[%o2], %g1, %g2
828	membar	#StoreLoad
829	cmp	%g1, %g2
830	movne	%xcc, -1, %o1			/* cas failed. */
831	move	%xcc, 1, %o1			/* cas succeeded. */
8321:
833	stx	%g2, [%o0]			/* report "current" value */
834	retl
835	mov	%o1, %o0
836	SET_SIZE(sfmmu_modifytte_try)
837
838	ENTRY_NP(sfmmu_copytte)
839	ldx	[%o0], %g1
840	retl
841	stx	%g1, [%o1]
842	SET_SIZE(sfmmu_copytte)
843
844
845	/*
846	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
847	 * %o0 = TSB base address (in), pointer to TSB entry (out)
848	 * %o1 = vaddr (in)
849	 * %o2 = vpshift (in)
850	 * %o3 = tsb size code (in)
851	 * %o4 = scratch register
852	 */
853	ENTRY_NP(sfmmu_get_tsbe)
854	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
855	retl
856	nop
857	SET_SIZE(sfmmu_get_tsbe)
858
859	/*
860	 * Return a TSB tag for the given va.
861	 * %o0 = va (in/clobbered)
862	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
863	 */
864	ENTRY_NP(sfmmu_make_tsbtag)
865	retl
866	srln	%o0, TTARGET_VA_SHIFT, %o0
867	SET_SIZE(sfmmu_make_tsbtag)
868
869/*
870 * Other sfmmu primitives
871 */
872
873
874#define	I_SIZE		4
875
876	ENTRY_NP(sfmmu_fix_ktlb_traptable)
877	/*
878	 * %o0 = start of patch area
879	 * %o1 = size code of TSB to patch
880	 * %o3 = scratch
881	 */
882	/* fix sll */
883	ld	[%o0], %o3			/* get sll */
884	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
885	st	%o3, [%o0]			/* write sll */
886	flush	%o0
887	/* fix srl */
888	add	%o0, I_SIZE, %o0		/* goto next instr. */
889	ld	[%o0], %o3			/* get srl */
890	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
891	st	%o3, [%o0]			/* write srl */
892	retl
893	flush	%o0
894	SET_SIZE(sfmmu_fix_ktlb_traptable)
895
896	ENTRY_NP(sfmmu_fixup_ktsbbase)
897	/*
898	 * %o0 = start of patch area
899	 * %o5 = kernel virtual or physical tsb base address
900	 * %o2, %o3 are used as scratch registers.
901	 */
902	/* fixup sethi instruction */
903	ld	[%o0], %o3
904	srl	%o5, 10, %o2			! offset is bits 32:10
905	or	%o3, %o2, %o3			! set imm22
906	st	%o3, [%o0]
907	/* fixup offset of lduw/ldx */
908	add	%o0, I_SIZE, %o0		! next instr
909	ld	[%o0], %o3
910	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
911	or	%o3, %o2, %o3
912	st	%o3, [%o0]
913	retl
914	flush	%o0
915	SET_SIZE(sfmmu_fixup_ktsbbase)
916
917	ENTRY_NP(sfmmu_fixup_setx)
918	/*
919	 * %o0 = start of patch area
920	 * %o4 = 64 bit value to patch
921	 * %o2, %o3 are used as scratch registers.
922	 *
923	 * Note: Assuming that all parts of the instructions which need to be
924	 *	 patched correspond to RUNTIME_PATCH (aka 0)
925	 *
926	 * Note the implementation of setx which is being patched is as follows:
927	 *
928	 * sethi   %hh(RUNTIME_PATCH), tmp
929	 * sethi   %lm(RUNTIME_PATCH), dest
930	 * or      tmp, %hm(RUNTIME_PATCH), tmp
931	 * or      dest, %lo(RUNTIME_PATCH), dest
932	 * sllx    tmp, 32, tmp
933	 * nop
934	 * or      tmp, dest, dest
935	 *
936	 * which differs from the implementation in the
937	 * "SPARC Architecture Manual"
938	 */
939	/* fixup sethi instruction */
940	ld	[%o0], %o3
941	srlx	%o4, 42, %o2			! bits [63:42]
942	or	%o3, %o2, %o3			! set imm22
943	st	%o3, [%o0]
944	/* fixup sethi instruction */
945	add	%o0, I_SIZE, %o0		! next instr
946	ld	[%o0], %o3
947	sllx	%o4, 32, %o2			! clear upper bits
948	srlx	%o2, 42, %o2			! bits [31:10]
949	or	%o3, %o2, %o3			! set imm22
950	st	%o3, [%o0]
951	/* fixup or instruction */
952	add	%o0, I_SIZE, %o0		! next instr
953	ld	[%o0], %o3
954	srlx	%o4, 32, %o2			! bits [63:32]
955	and	%o2, 0x3ff, %o2			! bits [41:32]
956	or	%o3, %o2, %o3			! set imm
957	st	%o3, [%o0]
958	/* fixup or instruction */
959	add	%o0, I_SIZE, %o0		! next instr
960	ld	[%o0], %o3
961	and	%o4, 0x3ff, %o2			! bits [9:0]
962	or	%o3, %o2, %o3			! set imm
963	st	%o3, [%o0]
964	retl
965	flush	%o0
966	SET_SIZE(sfmmu_fixup_setx)
967
968	ENTRY_NP(sfmmu_fixup_or)
969	/*
970	 * %o0 = start of patch area
971	 * %o4 = 32 bit value to patch
972	 * %o2, %o3 are used as scratch registers.
973	 * Note: Assuming that all parts of the instructions which need to be
974	 *	 patched correspond to RUNTIME_PATCH (aka 0)
975	 */
976	ld	[%o0], %o3
977	and	%o4, 0x3ff, %o2			! bits [9:0]
978	or	%o3, %o2, %o3			! set imm
979	st	%o3, [%o0]
980	retl
981	flush	%o0
982	SET_SIZE(sfmmu_fixup_or)
983
984	ENTRY_NP(sfmmu_fixup_shiftx)
985	/*
986	 * %o0 = start of patch area
987	 * %o4 = signed int immediate value to add to sllx/srlx imm field
988	 * %o2, %o3 are used as scratch registers.
989	 *
990	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
991	 * so we do a simple add.  The caller must be careful to prevent
992	 * overflow, which could easily occur if the initial value is nonzero!
993	 */
994	ld	[%o0], %o3			! %o3 = instruction to patch
995	and	%o3, 0x3f, %o2			! %o2 = existing imm value
996	add	%o2, %o4, %o2			! %o2 = new imm value
997	andn	%o3, 0x3f, %o3			! clear old imm value
998	and	%o2, 0x3f, %o2			! truncate new imm value
999	or	%o3, %o2, %o3			! set new imm value
1000	st	%o3, [%o0]			! store updated instruction
1001	retl
1002	flush	%o0
1003	SET_SIZE(sfmmu_fixup_shiftx)
1004
1005	ENTRY_NP(sfmmu_fixup_mmu_asi)
1006	/*
1007	 * Patch imm_asi of all ldda instructions in the MMU
1008	 * trap handlers.  We search MMU_PATCH_INSTR instructions
1009	 * starting from the itlb miss handler (trap 0x64).
1010	 * %o0 = address of tt[0,1]_itlbmiss
1011	 * %o1 = imm_asi to setup, shifted by appropriate offset.
1012	 * %o3 = number of instructions to search
1013	 * %o4 = reserved by caller: called from leaf routine
1014	 */
10151:	ldsw	[%o0], %o2			! load instruction to %o2
1016	brgez,pt %o2, 2f
1017	  srl	%o2, 30, %o5
1018	btst	1, %o5				! test bit 30; skip if not set
1019	bz,pt	%icc, 2f
1020	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
1021	srlx	%o5, 58, %o5			! isolate op3 part of opcode
1022	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
1023	brnz,pt	%o5, 2f				! skip if not a match
1024	  or	%o2, %o1, %o2			! or in imm_asi
1025	st	%o2, [%o0]			! write patched instruction
10262:	dec	%o3
1027	brnz,a,pt %o3, 1b			! loop until we're done
1028	  add	%o0, I_SIZE, %o0
1029	retl
1030	flush	%o0
1031	SET_SIZE(sfmmu_fixup_mmu_asi)
1032
1033	/*
1034	 * Patch immediate ASI used to access the TSB in the
1035	 * trap table.
1036	 * inputs: %o0 = value of ktsb_phys
1037	 */
1038	ENTRY_NP(sfmmu_patch_mmu_asi)
1039	mov	%o7, %o4			! save return pc in %o4
1040	mov	ASI_QUAD_LDD_PHYS, %o3		! set QUAD_LDD_PHYS by default
1041
1042#ifdef sun4v
1043
1044	/*
1045	 * Check ktsb_phys. It must be non-zero for sun4v, panic if not.
1046	 */
1047
1048	brnz,pt %o0, do_patch
1049	nop
1050
1051	sethi	%hi(sfmmu_panic11), %o0
1052	call	panic
1053	  or	%o0, %lo(sfmmu_panic11), %o0
1054do_patch:
1055
1056#else /* sun4v */
1057	/*
1058	 * Some non-sun4v platforms deploy virtual ktsb (ktsb_phys==0).
1059	 * Note that ASI_NQUAD_LD is not defined/used for sun4v
1060	 */
1061	movrz	%o0, ASI_NQUAD_LD, %o3
1062
1063#endif /* sun4v */
1064
1065	sll	%o3, 5, %o1			! imm_asi offset
1066	mov	6, %o3				! number of instructions
1067	sethi	%hi(dktsb), %o0			! to search
1068	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
1069	  or	%o0, %lo(dktsb), %o0
1070	mov	6, %o3				! number of instructions
1071	sethi	%hi(dktsb4m), %o0		! to search
1072	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
1073	  or	%o0, %lo(dktsb4m), %o0
1074	mov	6, %o3				! number of instructions
1075	sethi	%hi(iktsb), %o0			! to search
1076	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
1077	  or	%o0, %lo(iktsb), %o0
1078	mov	6, %o3				! number of instructions
1079	sethi	%hi(iktsb4m), %o0		! to search
1080	call	sfmmu_fixup_mmu_asi		! patch kitlb4m miss
1081	  or	%o0, %lo(iktsb4m), %o0
1082	mov	%o4, %o7			! retore return pc -- leaf
1083	retl
1084	nop
1085	SET_SIZE(sfmmu_patch_mmu_asi)
1086
1087
1088	ENTRY_NP(sfmmu_patch_ktsb)
1089	/*
1090	 * We need to fix iktsb, dktsb, et. al.
1091	 */
1092	save	%sp, -SA(MINFRAME), %sp
1093	set	ktsb_phys, %o1
1094	ld	[%o1], %o4
1095	set	ktsb_base, %o5
1096	set	ktsb4m_base, %l1
1097	brz,pt	%o4, 1f
1098	  nop
1099	set	ktsb_pbase, %o5
1100	set	ktsb4m_pbase, %l1
11011:
1102	sethi	%hi(ktsb_szcode), %o1
1103	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */
1104
1105	sethi	%hi(iktsb), %o0
1106	call	sfmmu_fix_ktlb_traptable
1107	  or	%o0, %lo(iktsb), %o0
1108
1109	sethi	%hi(dktsb), %o0
1110	call	sfmmu_fix_ktlb_traptable
1111	  or	%o0, %lo(dktsb), %o0
1112
1113	sethi	%hi(ktsb4m_szcode), %o1
1114	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */
1115
1116	sethi	%hi(iktsb4m), %o0
1117	call	sfmmu_fix_ktlb_traptable
1118	  or	%o0, %lo(iktsb4m), %o0
1119
1120	sethi	%hi(dktsb4m), %o0
1121	call	sfmmu_fix_ktlb_traptable
1122	  or	%o0, %lo(dktsb4m), %o0
1123
1124#ifndef sun4v
1125	mov	ASI_N, %o2
1126	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
1127	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
1128	sethi	%hi(tsb_kernel_patch_asi), %o0
1129	call	sfmmu_fixup_or
1130	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
1131#endif /* !sun4v */
1132
1133	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)
1134
1135	sethi	%hi(dktsbbase), %o0
1136	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1137	  or	%o0, %lo(dktsbbase), %o0
1138
1139	sethi	%hi(iktsbbase), %o0
1140	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1141	  or	%o0, %lo(iktsbbase), %o0
1142
1143	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
1144	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1145	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
1146
1147#ifdef sun4v
1148	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
1149	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1150	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
1151#endif /* sun4v */
1152
1153	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)
1154
1155	sethi	%hi(dktsb4mbase), %o0
1156	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1157	  or	%o0, %lo(dktsb4mbase), %o0
1158
1159	sethi	%hi(iktsb4mbase), %o0
1160	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1161	  or	%o0, %lo(iktsb4mbase), %o0
1162
1163	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
1164	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1165	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
1166
1167#ifdef sun4v
1168	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
1169	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1170	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
1171#endif /* sun4v */
1172
1173	set	ktsb_szcode, %o4
1174	ld	[%o4], %o4
1175	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
1176	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1177	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
1178
1179#ifdef sun4v
1180	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
1181	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1182	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
1183#endif /* sun4v */
1184
1185	set	ktsb4m_szcode, %o4
1186	ld	[%o4], %o4
1187	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1188	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1189	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1190
1191#ifdef sun4v
1192	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1193	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1194	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1195#endif /* sun4v */
1196
1197	ret
1198	restore
1199	SET_SIZE(sfmmu_patch_ktsb)
1200
1201	ENTRY_NP(sfmmu_kpm_patch_tlbm)
1202	/*
1203	 * Fixup trap handlers in common segkpm case.  This is reserved
1204	 * for future use should kpm TSB be changed to be other than the
1205	 * kernel TSB.
1206	 */
1207	retl
1208	nop
1209	SET_SIZE(sfmmu_kpm_patch_tlbm)
1210
1211	ENTRY_NP(sfmmu_kpm_patch_tsbm)
1212	/*
1213	 * nop the branch to sfmmu_kpm_dtsb_miss_small
1214	 * in the case where we are using large pages for
1215	 * seg_kpm (and hence must probe the second TSB for
1216	 * seg_kpm VAs)
1217	 */
1218	set	dktsb4m_kpmcheck_small, %o0
1219	MAKE_NOP_INSTR(%o1)
1220	st	%o1, [%o0]
1221	flush	%o0
1222	retl
1223	nop
1224	SET_SIZE(sfmmu_kpm_patch_tsbm)
1225
1226	ENTRY_NP(sfmmu_patch_utsb)
1227#ifdef UTSB_PHYS
1228	retl
1229	nop
1230#else /* UTSB_PHYS */
1231	/*
1232	 * We need to hot patch utsb_vabase and utsb4m_vabase
1233	 */
1234	save	%sp, -SA(MINFRAME), %sp
1235
1236	/* patch value of utsb_vabase */
1237	set	utsb_vabase, %o1
1238	ldx	[%o1], %o4
1239	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1240	call	sfmmu_fixup_setx
1241	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1242	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1243	call	sfmmu_fixup_setx
1244	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1245	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1246	call	sfmmu_fixup_setx
1247	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1248
1249	/* patch value of utsb4m_vabase */
1250	set	utsb4m_vabase, %o1
1251	ldx	[%o1], %o4
1252	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1253	call	sfmmu_fixup_setx
1254	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1255	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1256	call	sfmmu_fixup_setx
1257	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1258	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1259	call	sfmmu_fixup_setx
1260	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1261
1262	/*
1263	 * Patch TSB base register masks and shifts if needed.
1264	 * By default the TSB base register contents are set up for 4M slab.
1265	 * If we're using a smaller slab size and reserved VA range we need
1266	 * to patch up those values here.
1267	 */
1268	set	tsb_slab_shift, %o1
1269	set	MMU_PAGESHIFT4M, %o4
1270	lduw	[%o1], %o3
1271	subcc	%o4, %o3, %o4
1272	bz,pt	%icc, 1f
1273	  /* delay slot safe */
1274
1275	/* patch reserved VA range size if needed. */
1276	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
1277	call	sfmmu_fixup_shiftx
1278	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1279	call	sfmmu_fixup_shiftx
1280	  add	%o0, I_SIZE, %o0
1281	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
1282	call	sfmmu_fixup_shiftx
1283	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1284	call	sfmmu_fixup_shiftx
1285	  add	%o0, I_SIZE, %o0
12861:
1287	/* patch TSBREG_VAMASK used to set up TSB base register */
1288	set	tsb_slab_mask, %o1
1289	ldx	[%o1], %o4
1290	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1291	call	sfmmu_fixup_or
1292	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1293	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1294	call	sfmmu_fixup_or
1295	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1296
1297	ret
1298	restore
1299#endif /* UTSB_PHYS */
1300	SET_SIZE(sfmmu_patch_utsb)
1301
1302	ENTRY_NP(sfmmu_patch_shctx)
1303#ifdef sun4u
1304	retl
1305	  nop
1306#else /* sun4u */
1307	set	sfmmu_shctx_cpu_mondo_patch, %o0
1308	MAKE_JMP_INSTR(5, %o1, %o2)	! jmp       %g5
1309	st	%o1, [%o0]
1310	flush	%o0
1311	MAKE_NOP_INSTR(%o1)
1312	add	%o0, I_SIZE, %o0	! next instr
1313	st	%o1, [%o0]
1314	flush	%o0
1315
1316	set	sfmmu_shctx_user_rtt_patch, %o0
1317	st      %o1, [%o0]		! nop 1st instruction
1318	flush	%o0
1319	add     %o0, I_SIZE, %o0
1320	st      %o1, [%o0]		! nop 2nd instruction
1321	flush	%o0
1322	add     %o0, I_SIZE, %o0
1323	st      %o1, [%o0]		! nop 3rd instruction
1324	flush	%o0
1325	add     %o0, I_SIZE, %o0
1326	st      %o1, [%o0]		! nop 4th instruction
1327	flush	%o0
1328	add     %o0, I_SIZE, %o0
1329	st      %o1, [%o0]		! nop 5th instruction
1330	flush	%o0
1331	add     %o0, I_SIZE, %o0
1332	st      %o1, [%o0]		! nop 6th instruction
1333	retl
1334	flush	%o0
1335#endif /* sun4u */
1336	SET_SIZE(sfmmu_patch_shctx)
1337
1338	/*
1339	 * Routine that loads an entry into a tsb using virtual addresses.
1340	 * Locking is required since all cpus can use the same TSB.
1341	 * Note that it is no longer required to have a valid context
1342	 * when calling this function.
1343	 */
1344	ENTRY_NP(sfmmu_load_tsbe)
1345	/*
1346	 * %o0 = pointer to tsbe to load
1347	 * %o1 = tsb tag
1348	 * %o2 = virtual pointer to TTE
1349	 * %o3 = 1 if physical address in %o0 else 0
1350	 */
1351	rdpr	%pstate, %o5
1352#ifdef DEBUG
1353	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1)
1354#endif /* DEBUG */
1355
1356	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1357
1358	SETUP_TSB_ASI(%o3, %g3)
1359	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, locked_tsb_l8)
1360
1361	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1362
1363	retl
1364	membar	#StoreStore|#StoreLoad
1365	SET_SIZE(sfmmu_load_tsbe)
1366
1367	/*
1368	 * Flush TSB of a given entry if the tag matches.
1369	 */
1370	ENTRY(sfmmu_unload_tsbe)
1371	/*
1372	 * %o0 = pointer to tsbe to be flushed
1373	 * %o1 = tag to match
1374	 * %o2 = 1 if physical address in %o0 else 0
1375	 */
1376	SETUP_TSB_ASI(%o2, %g1)
1377	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1378	retl
1379	membar	#StoreStore|#StoreLoad
1380	SET_SIZE(sfmmu_unload_tsbe)
1381
1382	/*
1383	 * Routine that loads a TTE into the kpm TSB from C code.
1384	 * Locking is required since kpm TSB is shared among all CPUs.
1385	 */
1386	ENTRY_NP(sfmmu_kpm_load_tsb)
1387	/*
1388	 * %o0 = vaddr
1389	 * %o1 = ttep
1390	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1391	 */
1392	rdpr	%pstate, %o5			! %o5 = saved pstate
1393#ifdef DEBUG
1394	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1)
1395#endif /* DEBUG */
1396	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts
1397
1398#ifndef sun4v
1399	sethi	%hi(ktsb_phys), %o4
1400	mov	ASI_N, %o3
1401	ld	[%o4 + %lo(ktsb_phys)], %o4
1402	movrnz	%o4, ASI_MEM, %o3
1403	mov	%o3, %asi
1404#endif /* !sun4v */
1405	mov	%o0, %g1			! %g1 = vaddr
1406
1407	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1408	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1409	/* %g2 = tsbep, %g1 clobbered */
1410
1411	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1412	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1413	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, locked_tsb_l9)
1414
1415	wrpr	%g0, %o5, %pstate		! enable interrupts
1416	retl
1417	  membar #StoreStore|#StoreLoad
1418	SET_SIZE(sfmmu_kpm_load_tsb)
1419
1420	/*
1421	 * Routine that shoots down a TTE in the kpm TSB or in the
1422	 * kernel TSB depending on virtpg. Locking is required since
1423	 * kpm/kernel TSB is shared among all CPUs.
1424	 */
1425	ENTRY_NP(sfmmu_kpm_unload_tsb)
1426	/*
1427	 * %o0 = vaddr
1428	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1429	 */
1430#ifndef sun4v
1431	sethi	%hi(ktsb_phys), %o4
1432	mov	ASI_N, %o3
1433	ld	[%o4 + %lo(ktsb_phys)], %o4
1434	movrnz	%o4, ASI_MEM, %o3
1435	mov	%o3, %asi
1436#endif /* !sun4v */
1437	mov	%o0, %g1			! %g1 = vaddr
1438
1439	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1440	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1441	/* %g2 = tsbep, %g1 clobbered */
1442
1443	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1444	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1445	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1446
1447	retl
1448	  membar	#StoreStore|#StoreLoad
1449	SET_SIZE(sfmmu_kpm_unload_tsb)
1450
1451
1452	ENTRY_NP(sfmmu_ttetopfn)
1453	ldx	[%o0], %g1			/* read tte */
1454	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1455	/*
1456	 * g1 = pfn
1457	 */
1458	retl
1459	mov	%g1, %o0
1460	SET_SIZE(sfmmu_ttetopfn)
1461
1462/*
1463 * These macros are used to update global sfmmu hme hash statistics
1464 * in perf critical paths. It is only enabled in debug kernels or
1465 * if SFMMU_STAT_GATHER is defined
1466 */
1467#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1468#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1469	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1470	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
1471	cmp	tmp1, hatid						;\
1472	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
1473	set	sfmmu_global_stat, tmp1					;\
1474	add	tmp1, tmp2, tmp1					;\
1475	ld	[tmp1], tmp2						;\
1476	inc	tmp2							;\
1477	st	tmp2, [tmp1]
1478
1479#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1480	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1481	mov	HATSTAT_KHASH_LINKS, tmp2				;\
1482	cmp	tmp1, hatid						;\
1483	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
1484	set	sfmmu_global_stat, tmp1					;\
1485	add	tmp1, tmp2, tmp1					;\
1486	ld	[tmp1], tmp2						;\
1487	inc	tmp2							;\
1488	st	tmp2, [tmp1]
1489
1490
1491#else /* DEBUG || SFMMU_STAT_GATHER */
1492
1493#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1494
1495#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1496
1497#endif  /* DEBUG || SFMMU_STAT_GATHER */
1498
1499/*
1500 * This macro is used to update global sfmmu kstas in non
1501 * perf critical areas so they are enabled all the time
1502 */
1503#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
1504	sethi	%hi(sfmmu_global_stat), tmp1				;\
1505	add	tmp1, statname, tmp1					;\
1506	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
1507	inc	tmp2							;\
1508	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1509
1510/*
1511 * These macros are used to update per cpu stats in non perf
1512 * critical areas so they are enabled all the time
1513 */
1514#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
1515	ld	[tsbarea + stat], tmp1					;\
1516	inc	tmp1							;\
1517	st	tmp1, [tsbarea + stat]
1518
1519/*
1520 * These macros are used to update per cpu stats in non perf
1521 * critical areas so they are enabled all the time
1522 */
1523#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
1524	lduh	[tsbarea + stat], tmp1					;\
1525	inc	tmp1							;\
1526	stuh	tmp1, [tsbarea + stat]
1527
1528#if defined(KPM_TLBMISS_STATS_GATHER)
1529	/*
1530	 * Count kpm dtlb misses separately to allow a different
1531	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1532	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1533	 */
1534#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
1535	brgez	tagacc, label	/* KPM VA? */				;\
1536	nop								;\
1537	CPU_INDEX(tmp1, tsbma)						;\
1538	sethi	%hi(kpmtsbm_area), tsbma				;\
1539	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
1540	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
1541	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
1542	/* VA range check */						;\
1543	ldx	[tsbma + KPMTSBM_VBASE], val				;\
1544	cmp	tagacc, val						;\
1545	blu,pn	%xcc, label						;\
1546	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
1547	cmp	tagacc, tmp1						;\
1548	bgeu,pn	%xcc, label						;\
1549	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
1550	inc	val							;\
1551	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
1552label:
1553#else
1554#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1555#endif	/* KPM_TLBMISS_STATS_GATHER */
1556
1557#ifdef	PTL1_PANIC_DEBUG
1558	.seg	".data"
1559	.global	test_ptl1_panic
1560test_ptl1_panic:
1561	.word	0
1562	.align	8
1563
1564	.seg	".text"
1565	.align	4
1566#endif	/* PTL1_PANIC_DEBUG */
1567
1568	/*
1569	 * The following routines are jumped to from the mmu trap handlers to do
1570	 * the setting up to call systrap.  They are separate routines instead
1571	 * of being part of the handlers because the handlers would exceed 32
1572	 * instructions and since this is part of the slow path the jump cost is
1573	 * irrelevant.
1574	 */
1575
1576	ENTRY_NP(sfmmu_pagefault)
1577	SET_GL_REG(1)
1578	USE_ALTERNATE_GLOBALS(%g5)
1579	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1580	rdpr	%tt, %g6
1581	cmp	%g6, FAST_IMMU_MISS_TT
1582	be,a,pn	%icc, 1f
1583	  mov	T_INSTR_MMU_MISS, %g3
1584	cmp	%g6, T_INSTR_MMU_MISS
1585	be,a,pn	%icc, 1f
1586	  mov	T_INSTR_MMU_MISS, %g3
1587	mov	%g5, %g2
1588	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1589	cmp	%g6, FAST_DMMU_MISS_TT
1590	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1591	cmp	%g6, T_DATA_MMU_MISS
1592	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1593
1594#ifdef  PTL1_PANIC_DEBUG
1595	/* check if we want to test the tl1 panic */
1596	sethi	%hi(test_ptl1_panic), %g4
1597	ld	[%g4 + %lo(test_ptl1_panic)], %g1
1598	st	%g0, [%g4 + %lo(test_ptl1_panic)]
1599	cmp	%g1, %g0
1600	bne,a,pn %icc, ptl1_panic
1601	  or	%g0, PTL1_BAD_DEBUG, %g1
1602#endif	/* PTL1_PANIC_DEBUG */
16031:
1604	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
1605	/*
1606	 * g2 = tag access reg
1607	 * g3.l = type
1608	 * g3.h = 0
1609	 */
1610	sethi	%hi(trap), %g1
1611	or	%g1, %lo(trap), %g1
16122:
1613	ba,pt	%xcc, sys_trap
1614	  mov	-1, %g4
1615	SET_SIZE(sfmmu_pagefault)
1616
1617	ENTRY_NP(sfmmu_mmu_trap)
1618	SET_GL_REG(1)
1619	USE_ALTERNATE_GLOBALS(%g5)
1620	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
1621	rdpr	%tt, %g6
1622	cmp	%g6, FAST_IMMU_MISS_TT
1623	be,a,pn	%icc, 1f
1624	  mov	T_INSTR_MMU_MISS, %g3
1625	cmp	%g6, T_INSTR_MMU_MISS
1626	be,a,pn	%icc, 1f
1627	  mov	T_INSTR_MMU_MISS, %g3
1628	mov	%g5, %g2
1629	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1630	cmp	%g6, FAST_DMMU_MISS_TT
1631	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1632	cmp	%g6, T_DATA_MMU_MISS
1633	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
16341:
1635	/*
1636	 * g2 = tag access reg
1637	 * g3 = type
1638	 */
1639	sethi	%hi(sfmmu_tsbmiss_exception), %g1
1640	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
1641	ba,pt	%xcc, sys_trap
1642	  mov	-1, %g4
1643	/*NOTREACHED*/
1644	SET_SIZE(sfmmu_mmu_trap)
1645
1646	ENTRY_NP(sfmmu_suspend_tl)
1647	SET_GL_REG(1)
1648	USE_ALTERNATE_GLOBALS(%g5)
1649	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
1650	rdpr	%tt, %g6
1651	cmp	%g6, FAST_IMMU_MISS_TT
1652	be,a,pn	%icc, 1f
1653	  mov	T_INSTR_MMU_MISS, %g3
1654	mov	%g5, %g2
1655	cmp	%g6, FAST_DMMU_MISS_TT
1656	move	%icc, T_DATA_MMU_MISS, %g3
1657	movne	%icc, T_DATA_PROT, %g3
16581:
1659	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
1660	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
1661	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
1662	ba,pt	%xcc, sys_trap
1663	  mov	PIL_15, %g4
1664	/*NOTREACHED*/
1665	SET_SIZE(sfmmu_suspend_tl)
1666
1667	/*
1668	 * No %g registers in use at this point.
1669	 */
1670	ENTRY_NP(sfmmu_window_trap)
1671	rdpr	%tpc, %g1
1672#ifdef sun4v
1673#ifdef DEBUG
1674	/* We assume previous %gl was 1 */
1675	rdpr	%tstate, %g4
1676	srlx	%g4, TSTATE_GL_SHIFT, %g4
1677	and	%g4, TSTATE_GL_MASK, %g4
1678	cmp	%g4, 1
1679	bne,a,pn %icc, ptl1_panic
1680	  mov	PTL1_BAD_WTRAP, %g1
1681#endif /* DEBUG */
1682	/* user miss at tl>1. better be the window handler or user_rtt */
1683	/* in user_rtt? */
1684	set	rtt_fill_start, %g4
1685	cmp	%g1, %g4
1686	blu,pn %xcc, 6f
1687	 .empty
1688	set	rtt_fill_end, %g4
1689	cmp	%g1, %g4
1690	bgeu,pn %xcc, 6f
1691	 nop
1692	set	fault_rtt_fn1, %g1
1693	wrpr	%g0, %g1, %tnpc
1694	ba,a	7f
16956:
1696	! must save this trap level before descending trap stack
1697	! no need to save %tnpc, either overwritten or discarded
1698	! already got it: rdpr	%tpc, %g1
1699	rdpr	%tstate, %g6
1700	rdpr	%tt, %g7
1701	! trap level saved, go get underlying trap type
1702	rdpr	%tl, %g5
1703	sub	%g5, 1, %g3
1704	wrpr	%g3, %tl
1705	rdpr	%tt, %g2
1706	wrpr	%g5, %tl
1707	! restore saved trap level
1708	wrpr	%g1, %tpc
1709	wrpr	%g6, %tstate
1710	wrpr	%g7, %tt
1711#else /* sun4v */
1712	/* user miss at tl>1. better be the window handler */
1713	rdpr	%tl, %g5
1714	sub	%g5, 1, %g3
1715	wrpr	%g3, %tl
1716	rdpr	%tt, %g2
1717	wrpr	%g5, %tl
1718#endif /* sun4v */
1719	and	%g2, WTRAP_TTMASK, %g4
1720	cmp	%g4, WTRAP_TYPE
1721	bne,pn	%xcc, 1f
1722	 nop
1723	/* tpc should be in the trap table */
1724	set	trap_table, %g4
1725	cmp	%g1, %g4
1726	blt,pn %xcc, 1f
1727	 .empty
1728	set	etrap_table, %g4
1729	cmp	%g1, %g4
1730	bge,pn %xcc, 1f
1731	 .empty
1732	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
1733	add	%g1, WTRAP_FAULTOFF, %g1
1734	wrpr	%g0, %g1, %tnpc
17357:
1736	/*
1737	 * some wbuf handlers will call systrap to resolve the fault
1738	 * we pass the trap type so they figure out the correct parameters.
1739	 * g5 = trap type, g6 = tag access reg
1740	 */
1741
1742	/*
1743	 * only use g5, g6, g7 registers after we have switched to alternate
1744	 * globals.
1745	 */
1746	SET_GL_REG(1)
1747	USE_ALTERNATE_GLOBALS(%g5)
1748	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
1749	rdpr	%tt, %g7
1750	cmp	%g7, FAST_IMMU_MISS_TT
1751	be,a,pn	%icc, ptl1_panic
1752	  mov	PTL1_BAD_WTRAP, %g1
1753	cmp	%g7, T_INSTR_MMU_MISS
1754	be,a,pn	%icc, ptl1_panic
1755	  mov	PTL1_BAD_WTRAP, %g1
1756	mov	T_DATA_PROT, %g5
1757	cmp	%g7, FAST_DMMU_MISS_TT
1758	move	%icc, T_DATA_MMU_MISS, %g5
1759	cmp	%g7, T_DATA_MMU_MISS
1760	move	%icc, T_DATA_MMU_MISS, %g5
1761	! XXXQ AGS re-check out this one
1762	done
17631:
1764	CPU_PADDR(%g1, %g4)
1765	add	%g1, CPU_TL1_HDLR, %g1
1766	lda	[%g1]ASI_MEM, %g4
1767	brnz,a,pt %g4, sfmmu_mmu_trap
1768	  sta	%g0, [%g1]ASI_MEM
1769	ba,pt	%icc, ptl1_panic
1770	  mov	PTL1_BAD_TRAP, %g1
1771	SET_SIZE(sfmmu_window_trap)
1772
1773	ENTRY_NP(sfmmu_kpm_exception)
1774	/*
1775	 * We have accessed an unmapped segkpm address or a legal segkpm
1776	 * address which is involved in a VAC alias conflict prevention.
1777	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
1778	 * set. If it is, we will instead note that a fault has occurred
1779	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
1780	 * a "retry"). This will step over the faulting instruction.
1781	 * Note that this means that a legal segkpm address involved in
1782	 * a VAC alias conflict prevention (a rare case to begin with)
1783	 * cannot be used in DTrace.
1784	 */
1785	CPU_INDEX(%g1, %g2)
1786	set	cpu_core, %g2
1787	sllx	%g1, CPU_CORE_SHIFT, %g1
1788	add	%g1, %g2, %g1
1789	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
1790	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
1791	bz	0f
1792	or	%g2, CPU_DTRACE_BADADDR, %g2
1793	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
1794	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
1795	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
1796	done
17970:
1798	TSTAT_CHECK_TL1(1f, %g1, %g2)
17991:
1800	SET_GL_REG(1)
1801	USE_ALTERNATE_GLOBALS(%g5)
1802	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
1803	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1804	/*
1805	 * g2=tagacc g3.l=type g3.h=0
1806	 */
1807	sethi	%hi(trap), %g1
1808	or	%g1, %lo(trap), %g1
1809	ba,pt	%xcc, sys_trap
1810	mov	-1, %g4
1811	SET_SIZE(sfmmu_kpm_exception)
1812
1813#if (IMAP_SEG != 0)
1814#error - ism_map->ism_seg offset is not zero
1815#endif
1816
1817/*
1818 * Copies ism mapping for this ctx in param "ism" if this is a ISM
1819 * tlb miss and branches to label "ismhit". If this is not an ISM
1820 * process or an ISM tlb miss it falls thru.
1821 *
1822 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
1823 * this process.
1824 * If so, it will branch to label "ismhit".  If not, it will fall through.
1825 *
1826 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
1827 * so that any other threads of this process will not try and walk the ism
1828 * maps while they are being changed.
1829 *
1830 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
1831 *       will make sure of that. This means we can terminate our search on
1832 *       the first zero mapping we find.
1833 *
1834 * Parameters:
1835 * tagacc	= (pseudo-)tag access register (vaddr + ctx) (in)
1836 * tsbmiss	= address of tsb miss area (in)
1837 * ismseg	= contents of ism_seg for this ism map (out)
1838 * ismhat	= physical address of imap_ismhat for this ism map (out)
1839 * tmp1		= scratch reg (CLOBBERED)
1840 * tmp2		= scratch reg (CLOBBERED)
1841 * tmp3		= scratch reg (CLOBBERED)
1842 * label:    temporary labels
1843 * ismhit:   label where to jump to if an ism dtlb miss
1844 * exitlabel:label where to jump if hat is busy due to hat_unshare.
1845 */
1846#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
1847	label, ismhit)							\
1848	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
1849	brlz,pt  tmp1, label##3		/* exit if -1 */	;\
1850	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
1851label##1:								;\
1852	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
1853	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
1854label##2:								;\
1855	brz,pt  ismseg, label##3		/* no mapping */	;\
1856	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
1857	lduba	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
1858	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
1859	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
1860	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
1861	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
1862	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
1863	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
1864	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
1865	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
1866	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
1867									;\
1868	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
1869	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
1870	cmp	ismhat, tmp1						;\
1871	bl,pt	%xcc, label##2		/* keep looking  */	;\
1872	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
1873									;\
1874	add	tmp3, IBLK_NEXTPA, tmp1					;\
1875	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
1876	brgez,pt tmp1, label##1		/* continue if not -1*/	;\
1877	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
1878label##3:
1879
1880/*
1881 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
1882 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
1883 * Parameters:
1884 * tagacc = reg containing virtual address
1885 * hatid = reg containing sfmmu pointer
1886 * hmeshift = constant/register to shift vaddr to obtain vapg
1887 * hmebp = register where bucket pointer will be stored
1888 * vapg = register where virtual page will be stored
1889 * tmp1, tmp2 = tmp registers
1890 */
1891
1892
1893#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
1894	vapg, label, tmp1, tmp2)					\
1895	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
1896	brnz,a,pt tmp1, label##1					;\
1897	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
1898	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
1899	ba,pt	%xcc, label##2					;\
1900	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
1901label##1:								;\
1902	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
1903label##2:								;\
1904	srlx	tagacc, hmeshift, vapg					;\
1905	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
1906	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
1907	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
1908	add	hmebp, tmp1, hmebp
1909
1910/*
1911 * hashtag includes bspage + hashno (64 bits).
1912 */
1913
1914#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
1915	sllx	vapg, hmeshift, vapg					;\
1916	mov	hashno, hblktag						;\
1917	sllx	hblktag, HTAG_REHASH_SHIFT, hblktag			;\
1918	or	vapg, hblktag, hblktag
1919
1920/*
1921 * Function to traverse hmeblk hash link list and find corresponding match.
1922 * The search is done using physical pointers. It returns the physical address
1923 * pointer to the hmeblk that matches with the tag provided.
1924 * Parameters:
1925 * hmebp	= register that points to hme hash bucket, also used as
1926 *		  tmp reg (clobbered)
1927 * hmeblktag	= register with hmeblk tag match
1928 * hatid	= register with hatid
1929 * hmeblkpa	= register where physical ptr will be stored
1930 * tmp1		= tmp reg
1931 * label: temporary label
1932 */
1933
1934#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, tsbarea, 	\
1935	tmp1, label)							\
1936	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
1937	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
1938	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1939label##1:								;\
1940	cmp	hmeblkpa, HMEBLK_ENDPA					;\
1941	be,pn   %xcc, label##2					;\
1942	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1943	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
1944	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
1945	add	hmebp, CLONGSIZE, hmebp					;\
1946	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
1947	xor	tmp1, hmeblktag, tmp1					;\
1948	xor	hmebp, hatid, hmebp					;\
1949	or	hmebp, tmp1, hmebp					;\
1950	brz,pn	hmebp, label##2	/* branch on hit */		;\
1951	  add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
1952	ba,pt	%xcc, label##1					;\
1953	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
1954label##2:
1955
1956/*
1957 * Function to traverse hmeblk hash link list and find corresponding match.
1958 * The search is done using physical pointers. It returns the physical address
1959 * pointer to the hmeblk that matches with the tag
1960 * provided.
1961 * Parameters:
1962 * hmeblktag	= register with hmeblk tag match (rid field is 0)
1963 * hatid	= register with hatid (pointer to SRD)
1964 * hmeblkpa	= register where physical ptr will be stored
1965 * tmp1		= tmp reg
1966 * tmp2		= tmp reg
1967 * label: temporary label
1968 */
1969
1970#define	HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, tsbarea,	\
1971	tmp1, tmp2, label)			 			\
1972label##1:								;\
1973	cmp	hmeblkpa, HMEBLK_ENDPA					;\
1974	be,pn   %xcc, label##4					;\
1975	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			;\
1976	add	hmeblkpa, HMEBLK_TAG, tmp2				;\
1977	ldxa	[tmp2]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
1978	add	tmp2, CLONGSIZE, tmp2					;\
1979	ldxa	[tmp2]ASI_MEM, tmp2 	/* read 2nd part of tag */	;\
1980	xor	tmp1, hmeblktag, tmp1					;\
1981	xor	tmp2, hatid, tmp2					;\
1982	brz,pn	tmp2, label##3	/* branch on hit */		;\
1983	  add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
1984label##2:								;\
1985	ba,pt	%xcc, label##1					;\
1986	  ldxa	[tmp2]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */		;\
1987label##3:								;\
1988	cmp	tmp1, SFMMU_MAX_HME_REGIONS				;\
1989	bgeu,pt	%xcc, label##2					;\
1990	  add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
1991	and	tmp1, BT_ULMASK, tmp2					;\
1992	srlx	tmp1, BT_ULSHIFT, tmp1					;\
1993	sllx	tmp1, CLONGSHIFT, tmp1					;\
1994	add	tsbarea, tmp1, tmp1					;\
1995	ldx	[tmp1 + TSBMISS_SHMERMAP], tmp1				;\
1996	srlx	tmp1, tmp2, tmp1					;\
1997	btst	0x1, tmp1						;\
1998	bz,pn	%xcc, label##2					;\
1999	  add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
2000label##4:
2001
2002#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
2003#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
2004#endif
2005
2006/*
2007 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2008 * the offset for the corresponding hment.
2009 * Parameters:
2010 * In:
2011 *	vaddr = register with virtual address
2012 *	hmeblkpa = physical pointer to hme_blk
2013 * Out:
2014 *	hmentoff = register where hment offset will be stored
2015 *	hmemisc = hblk_misc
2016 * Scratch:
2017 *	tmp1
2018 */
2019#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\
2020	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
2021	lda	[hmentoff]ASI_MEM, hmemisc 				;\
2022	andcc	hmemisc, HBLK_SZMASK, %g0				;\
2023	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
2024	  or	%g0, HMEBLK_HME1, hmentoff				;\
2025	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
2026	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
2027	sllx	tmp1, SFHME_SHIFT, tmp1					;\
2028	add	tmp1, HMEBLK_HME1, hmentoff				;\
2029label1:
2030
2031/*
2032 * GET_TTE is a macro that returns a TTE given a tag and hatid.
2033 *
2034 * tagacc	= (pseudo-)tag access register (in)
2035 * hatid	= sfmmu pointer for TSB miss (in)
2036 * tte		= tte for TLB miss if found, otherwise clobbered (out)
2037 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
2038 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
2039 * hmemisc	= hblk_misc if TTE is found (out), otherwise clobbered
2040 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
2041 *		  for this page size.
2042 * hashno	= constant/register hash number
2043 * tmp		= temp value - clobbered
2044 * label	= temporary label for branching within macro.
2045 * foundlabel	= label to jump to when tte is found.
2046 * suspendlabel= label to jump to when tte is suspended.
2047 * exitlabel	= label to jump to when tte is not found.
2048 *
2049 */
2050#define GET_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, hmeshift, \
2051		 hashno, tmp, label, foundlabel, suspendlabel, exitlabel) \
2052									;\
2053	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2054	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2055	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2056		hmeblkpa, label##5, hmemisc, tmp)			;\
2057									;\
2058	/*								;\
2059	 * tagacc = tagacc						;\
2060	 * hatid = hatid						;\
2061	 * tsbarea = tsbarea						;\
2062	 * tte   = hmebp (hme bucket pointer)				;\
2063	 * hmeblkpa  = vapg  (virtual page)				;\
2064	 * hmemisc, tmp = scratch					;\
2065	 */								;\
2066	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2067	or	hmemisc, SFMMU_INVALID_SHMERID, hmemisc			;\
2068									;\
2069	/*								;\
2070	 * tagacc = tagacc						;\
2071	 * hatid = hatid						;\
2072	 * tte   = hmebp						;\
2073	 * hmeblkpa  = CLOBBERED					;\
2074	 * hmemisc  = htag_bspage+hashno+invalid_rid			;\
2075	 * tmp  = scratch						;\
2076	 */								;\
2077	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2078	HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, 	 		\
2079		tsbarea, tagacc, label##1)				;\
2080	/*								;\
2081	 * tagacc = CLOBBERED						;\
2082	 * tte = CLOBBERED						;\
2083	 * hmeblkpa = hmeblkpa						;\
2084	 * tmp = scratch						;\
2085	 */								;\
2086	cmp	hmeblkpa, HMEBLK_ENDPA					;\
2087	bne,pn   %xcc, label##4       /* branch if hmeblk found */    ;\
2088	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2089	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2090	  nop								;\
2091label##4:								;\
2092	/*								;\
2093	 * We have found the hmeblk containing the hment.		;\
2094	 * Now we calculate the corresponding tte.			;\
2095	 *								;\
2096	 * tagacc = tagacc						;\
2097	 * hatid = hatid						;\
2098	 * tte   = clobbered						;\
2099	 * hmeblkpa  = hmeblkpa						;\
2100	 * hmemisc  = hblktag						;\
2101	 * tmp = scratch						;\
2102	 */								;\
2103	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2104		label##2)						;\
2105									;\
2106	/*								;\
2107	 * tagacc = tagacc						;\
2108	 * hatid = hmentoff						;\
2109	 * tte   = clobbered						;\
2110	 * hmeblkpa  = hmeblkpa						;\
2111	 * hmemisc  = hblk_misc						;\
2112	 * tmp = scratch						;\
2113	 */								;\
2114									;\
2115	add	hatid, SFHME_TTE, hatid					;\
2116	add	hmeblkpa, hatid, hmeblkpa				;\
2117	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2118	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2119	set	TTE_SUSPEND, hatid					;\
2120	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2121	btst	tte, hatid						;\
2122	bz,pt	%xcc, foundlabel					;\
2123	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2124									;\
2125	/*								;\
2126	 * Mapping is suspended, so goto suspend label.			;\
2127	 */								;\
2128	ba,pt	%xcc, suspendlabel					;\
2129	  nop
2130
2131/*
2132 * GET_SHME_TTE is similar to GET_TTE() except it searches
2133 * shared hmeblks via HMEHASH_SEARCH_SHME() macro.
2134 * If valid tte is found, hmemisc = shctx flag, i.e., shme is
2135 * either 0 (not part of scd) or 1 (part of scd).
2136 */
2137#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, tsbarea, hmemisc, 	\
2138		hmeshift, hashno, tmp, label, foundlabel,		\
2139		suspendlabel, exitlabel)				\
2140									;\
2141	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2142	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2143	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2144		hmeblkpa, label##5, hmemisc, tmp)			;\
2145									;\
2146	/*								;\
2147	 * tagacc = tagacc						;\
2148	 * hatid = hatid						;\
2149	 * tsbarea = tsbarea						;\
2150	 * tte   = hmebp (hme bucket pointer)				;\
2151	 * hmeblkpa  = vapg  (virtual page)				;\
2152	 * hmemisc, tmp = scratch					;\
2153	 */								;\
2154	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2155									;\
2156	/*								;\
2157	 * tagacc = tagacc						;\
2158	 * hatid = hatid						;\
2159	 * tsbarea = tsbarea						;\
2160	 * tte   = hmebp						;\
2161	 * hmemisc  = htag_bspage + hashno + 0 (for rid)		;\
2162	 * hmeblkpa  = CLOBBERED					;\
2163	 * tmp = scratch						;\
2164	 */								;\
2165	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2166									;\
2167	add     tte, HMEBUCK_NEXTPA, hmeblkpa				;\
2168	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
2169	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte)			;\
2170									;\
2171label##8:								;\
2172	HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa,			\
2173		tsbarea, tagacc, tte, label##1)			;\
2174	/*								;\
2175	 * tagacc = CLOBBERED						;\
2176	 * tte = CLOBBERED						;\
2177	 * hmeblkpa = hmeblkpa						;\
2178	 * tmp = scratch						;\
2179	 */								;\
2180	cmp	hmeblkpa, HMEBLK_ENDPA					;\
2181	bne,pn   %xcc, label##4       /* branch if hmeblk found */    ;\
2182	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2183	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2184	  nop								;\
2185label##4:								;\
2186	/*								;\
2187	 * We have found the hmeblk containing the hment.		;\
2188	 * Now we calculate the corresponding tte.			;\
2189	 *								;\
2190	 * tagacc = tagacc						;\
2191	 * hatid = hatid						;\
2192	 * tte   = clobbered						;\
2193	 * hmeblkpa  = hmeblkpa						;\
2194	 * hmemisc  = hblktag						;\
2195	 * tsbarea = tsbmiss area					;\
2196	 * tmp = scratch						;\
2197	 */								;\
2198	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2199		label##2)						;\
2200									;\
2201	/*								;\
2202	 * tagacc = tagacc						;\
2203	 * hatid = hmentoff						;\
2204	 * tte = clobbered						;\
2205	 * hmeblkpa  = hmeblkpa						;\
2206	 * hmemisc  = hblk_misc						;\
2207	 * tsbarea = tsbmiss area					;\
2208	 * tmp = scratch						;\
2209	 */								;\
2210									;\
2211	add	hatid, SFHME_TTE, hatid					;\
2212	add	hmeblkpa, hatid, hmeblkpa				;\
2213	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2214	brlz,pt tte, label##6						;\
2215	  nop								;\
2216	btst	HBLK_SZMASK, hmemisc					;\
2217	bnz,a,pt %icc, label##7					;\
2218	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2219									;\
2220	/*								;\
2221 	 * We found an invalid 8K tte in shme.				;\
2222	 * it may not belong to shme's region since			;\
2223	 * region size/alignment granularity is 8K but different	;\
2224	 * regions don't share hmeblks. Continue the search.		;\
2225	 */								;\
2226	sub	hmeblkpa, hatid, hmeblkpa				;\
2227	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2228	srlx	tagacc, hmeshift, tte					;\
2229	add	hmeblkpa, HMEBLK_NEXTPA, hmeblkpa			;\
2230	ldxa	[hmeblkpa]ASI_MEM, hmeblkpa				;\
2231	MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc)		;\
2232	ba,a,pt	%xcc, label##8					;\
2233label##6:								;\
2234	GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc)		;\
2235	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2236label##7:								;\
2237	set	TTE_SUSPEND, hatid					;\
2238	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2239	btst	tte, hatid						;\
2240	bz,pt	%xcc, foundlabel					;\
2241	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2242									;\
2243	/*								;\
2244	 * Mapping is suspended, so goto suspend label.			;\
2245	 */								;\
2246	ba,pt	%xcc, suspendlabel					;\
2247	  nop
2248
2249	/*
2250	 * KERNEL PROTECTION HANDLER
2251	 *
2252	 * g1 = tsb8k pointer register (clobbered)
2253	 * g2 = tag access register (ro)
2254	 * g3 - g7 = scratch registers
2255	 *
2256	 * Note: This function is patched at runtime for performance reasons.
2257	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
2258	 */
2259	ENTRY_NP(sfmmu_kprot_trap)
2260	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2261sfmmu_kprot_patch_ktsb_base:
2262	RUNTIME_PATCH_SETX(%g1, %g6)
2263	/* %g1 = contents of ktsb_base or ktsb_pbase */
2264sfmmu_kprot_patch_ktsb_szcode:
2265	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
2266
2267	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2268	! %g1 = First TSB entry pointer, as TSB miss handler expects
2269
2270	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2271sfmmu_kprot_patch_ktsb4m_base:
2272	RUNTIME_PATCH_SETX(%g3, %g6)
2273	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2274sfmmu_kprot_patch_ktsb4m_szcode:
2275	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
2276
2277	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2278	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2279
2280        CPU_TSBMISS_AREA(%g6, %g7)
2281        HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2282	ba,pt	%xcc, sfmmu_tsb_miss_tt
2283	  nop
2284
2285	/*
2286	 * USER PROTECTION HANDLER
2287	 *
2288	 * g1 = tsb8k pointer register (ro)
2289	 * g2 = tag access register (ro)
2290	 * g3 = faulting context (clobbered, currently not used)
2291	 * g4 - g7 = scratch registers
2292	 */
2293	ALTENTRY(sfmmu_uprot_trap)
2294#ifdef sun4v
2295	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2296	/* %g1 = first TSB entry ptr now, %g2 preserved */
2297
2298	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
2299	brlz,pt %g3, 9f				/* check for 2nd TSB */
2300	  nop
2301
2302	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2303	/* %g3 = second TSB entry ptr now, %g2 preserved */
2304
2305#else /* sun4v */
2306#ifdef UTSB_PHYS
2307	/* g1 = first TSB entry ptr */
2308	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2309	brlz,pt %g3, 9f			/* check for 2nd TSB */
2310	  nop
2311
2312	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2313	/* %g3 = second TSB entry ptr now, %g2 preserved */
2314#else /* UTSB_PHYS */
2315	brgez,pt %g1, 9f		/* check for 2nd TSB */
2316	  mov	-1, %g3			/* set second tsbe ptr to -1 */
2317
2318	mov	%g2, %g7
2319	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2320	/* %g3 = second TSB entry ptr now, %g7 clobbered */
2321	mov	%g1, %g7
2322	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2323#endif /* UTSB_PHYS */
2324#endif /* sun4v */
23259:
2326	CPU_TSBMISS_AREA(%g6, %g7)
2327	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2328	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
2329	  nop
2330
2331	/*
2332	 * Kernel 8K page iTLB miss.  We also get here if we took a
2333	 * fast instruction access mmu miss trap while running in
2334	 * invalid context.
2335	 *
2336	 * %g1 = 8K TSB pointer register (not used, clobbered)
2337	 * %g2 = tag access register (used)
2338	 * %g3 = faulting context id (used)
2339	 * %g7 = TSB tag to match (used)
2340	 */
2341	.align	64
2342	ALTENTRY(sfmmu_kitlb_miss)
2343	brnz,pn %g3, tsb_tl0_noctxt
2344	  nop
2345
2346	/* kernel miss */
2347	/* get kernel tsb pointer */
2348	/* we patch the next set of instructions at run time */
2349	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2350iktsbbase:
2351	RUNTIME_PATCH_SETX(%g4, %g5)
2352	/* %g4 = contents of ktsb_base or ktsb_pbase */
2353
2354iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2355	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2356	or	%g4, %g1, %g1			! form tsb ptr
2357	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2358	cmp	%g4, %g7
2359	bne,pn	%xcc, iktsb4mbase		! check 4m ktsb
2360	  srlx    %g2, MMU_PAGESHIFT4M, %g3	! use 4m virt-page as TSB index
2361
2362	andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2363	bz,pn	%icc, exec_fault
2364	  nop
2365	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2366	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2367	retry
2368
2369iktsb4mbase:
2370        RUNTIME_PATCH_SETX(%g4, %g6)
2371        /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */
2372iktsb4m:
2373	sllx    %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2374        srlx    %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2375	add	%g4, %g3, %g3			! %g3 = 4m tsbe ptr
2376	ldda	[%g3]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2377	cmp	%g4, %g7
2378	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
2379	  andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2380	bz,pn	%icc, exec_fault
2381	  nop
2382	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2383	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2384	retry
2385
2386	/*
2387	 * Kernel dTLB miss.  We also get here if we took a fast data
2388	 * access mmu miss trap while running in invalid context.
2389	 *
2390	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2391	 *	We select the TSB miss handler to branch to depending on
2392	 *	the virtual address of the access.  In the future it may
2393	 *	be desirable to separate kpm TTEs into their own TSB,
2394	 *	in which case all that needs to be done is to set
2395	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2396	 *	early in the miss if we detect a kpm VA to a new handler.
2397	 *
2398	 * %g1 = 8K TSB pointer register (not used, clobbered)
2399	 * %g2 = tag access register (used)
2400	 * %g3 = faulting context id (used)
2401	 */
2402	.align	64
2403	ALTENTRY(sfmmu_kdtlb_miss)
2404	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
2405	  nop
2406
2407	/* Gather some stats for kpm misses in the TLB. */
2408	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2409	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2410
2411	/*
2412	 * Get first TSB offset and look for 8K/64K/512K mapping
2413	 * using the 8K virtual page as the index.
2414	 *
2415	 * We patch the next set of instructions at run time;
2416	 * any changes here require sfmmu_patch_ktsb changes too.
2417	 */
2418dktsbbase:
2419	RUNTIME_PATCH_SETX(%g7, %g6)
2420	/* %g7 = contents of ktsb_base or ktsb_pbase */
2421
2422dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2423	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2424
2425	/*
2426	 * At this point %g1 is our index into the TSB.
2427	 * We just masked off enough bits of the VA depending
2428	 * on our TSB size code.
2429	 */
2430	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2431	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2432	cmp	%g6, %g4			! compare tag
2433	bne,pn	%xcc, dktsb4m_kpmcheck_small
2434	  add	%g7, %g1, %g1			/* form tsb ptr */
2435	TT_TRACE(trace_tsbhit)
2436	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2437	/* trapstat expects tte in %g5 */
2438	retry
2439
2440	/*
2441	 * If kpm is using large pages, the following instruction needs
2442	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2443	 * so that we will probe the 4M TSB regardless of the VA.  In
2444	 * the case kpm is using small pages, we know no large kernel
2445	 * mappings are located above 0x80000000.00000000 so we skip the
2446	 * probe as an optimization.
2447	 */
2448dktsb4m_kpmcheck_small:
2449	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2450	  /* delay slot safe, below */
2451
2452	/*
2453	 * Get second TSB offset and look for 4M mapping
2454	 * using 4M virtual page as the TSB index.
2455	 *
2456	 * Here:
2457	 * %g1 = 8K TSB pointer.  Don't squash it.
2458	 * %g2 = tag access register (we still need it)
2459	 */
2460	srlx	%g2, MMU_PAGESHIFT4M, %g3
2461
2462	/*
2463	 * We patch the next set of instructions at run time;
2464	 * any changes here require sfmmu_patch_ktsb changes too.
2465	 */
2466dktsb4mbase:
2467	RUNTIME_PATCH_SETX(%g7, %g6)
2468	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2469dktsb4m:
2470	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2471	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2472
2473	/*
2474	 * At this point %g3 is our index into the TSB.
2475	 * We just masked off enough bits of the VA depending
2476	 * on our TSB size code.
2477	 */
2478	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2479	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2480	cmp	%g6, %g4			! compare tag
2481
2482dktsb4m_tsbmiss:
2483	bne,pn	%xcc, dktsb4m_kpmcheck
2484	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
2485	TT_TRACE(trace_tsbhit)
2486	/* we don't check TTE size here since we assume 4M TSB is separate */
2487	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2488	/* trapstat expects tte in %g5 */
2489	retry
2490
2491	/*
2492	 * So, we failed to find a valid TTE to match the faulting
2493	 * address in either TSB.  There are a few cases that could land
2494	 * us here:
2495	 *
2496	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2497	 *    to sfmmu_tsb_miss_tt to handle the miss.
2498	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2499	 *    4M TSB.  Let segkpm handle it.
2500	 *
2501	 * Note that we shouldn't land here in the case of a kpm VA when
2502	 * kpm_smallpages is active -- we handled that case earlier at
2503	 * dktsb4m_kpmcheck_small.
2504	 *
2505	 * At this point:
2506	 *  g1 = 8K-indexed primary TSB pointer
2507	 *  g2 = tag access register
2508	 *  g3 = 4M-indexed secondary TSB pointer
2509	 */
2510dktsb4m_kpmcheck:
2511	cmp	%g2, %g0
2512	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
2513	  nop
2514	ba,a,pt	%icc, sfmmu_tsb_miss_tt
2515	  nop
2516
2517#ifdef sun4v
2518	/*
2519	 * User instruction miss w/ single TSB.
2520	 * The first probe covers 8K, 64K, and 512K page sizes,
2521	 * because 64K and 512K mappings are replicated off 8K
2522	 * pointer.
2523	 *
2524	 * g1 = tsb8k pointer register
2525	 * g2 = tag access register
2526	 * g3 - g6 = scratch registers
2527	 * g7 = TSB tag to match
2528	 */
2529	.align	64
2530	ALTENTRY(sfmmu_uitlb_fastpath)
2531
2532	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2533	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2534	ba,pn	%xcc, sfmmu_tsb_miss_tt
2535	  mov	-1, %g3
2536
2537	/*
2538	 * User data miss w/ single TSB.
2539	 * The first probe covers 8K, 64K, and 512K page sizes,
2540	 * because 64K and 512K mappings are replicated off 8K
2541	 * pointer.
2542	 *
2543	 * g1 = tsb8k pointer register
2544	 * g2 = tag access register
2545	 * g3 - g6 = scratch registers
2546	 * g7 = TSB tag to match
2547	 */
2548	.align 64
2549	ALTENTRY(sfmmu_udtlb_fastpath)
2550
2551	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2552	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2553	ba,pn	%xcc, sfmmu_tsb_miss_tt
2554	  mov	-1, %g3
2555
2556	/*
2557	 * User instruction miss w/ multiple TSBs (sun4v).
2558	 * The first probe covers 8K, 64K, and 512K page sizes,
2559	 * because 64K and 512K mappings are replicated off 8K
2560	 * pointer.  Second probe covers 4M page size only.
2561	 *
2562	 * Just like sfmmu_udtlb_slowpath, except:
2563	 *   o Uses ASI_ITLB_IN
2564	 *   o checks for execute permission
2565	 *   o No ISM prediction.
2566	 *
2567	 * g1 = tsb8k pointer register
2568	 * g2 = tag access register
2569	 * g3 - g6 = scratch registers
2570	 * g7 = TSB tag to match
2571	 */
2572	.align	64
2573	ALTENTRY(sfmmu_uitlb_slowpath)
2574
2575	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2576	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2577	/* g4 - g5 = clobbered here */
2578
2579	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2580	/* g1 = first TSB pointer, g3 = second TSB pointer */
2581	srlx	%g2, TAG_VALO_SHIFT, %g7
2582	PROBE_2ND_ITSB(%g3, %g7)
2583	/* NOT REACHED */
2584
2585#else /* sun4v */
2586
2587	/*
2588	 * User instruction miss w/ multiple TSBs (sun4u).
2589	 * The first probe covers 8K, 64K, and 512K page sizes,
2590	 * because 64K and 512K mappings are replicated off 8K
2591	 * pointer.  Probe of 1st TSB has already been done prior to entry
2592	 * into this routine. For the UTSB_PHYS case we probe up to 3
2593	 * valid other TSBs in the following order:
2594	 * 1) shared TSB for 4M-256M pages
2595	 * 2) private TSB for 4M-256M pages
2596	 * 3) shared TSB for 8K-512K pages
2597	 *
2598	 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs
2599	 * 4M-256M pages.
2600	 *
2601	 * Just like sfmmu_udtlb_slowpath, except:
2602	 *   o Uses ASI_ITLB_IN
2603	 *   o checks for execute permission
2604	 *   o No ISM prediction.
2605	 *
2606	 * g1 = tsb8k pointer register
2607	 * g2 = tag access register
2608	 * g4 - g6 = scratch registers
2609	 * g7 = TSB tag to match
2610	 */
2611	.align	64
2612	ALTENTRY(sfmmu_uitlb_slowpath)
2613
2614#ifdef UTSB_PHYS
2615
2616       GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2617        brlz,pt %g6, 1f
2618          nop
2619        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2620        PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail)
26211:
2622        GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2623        brlz,pt %g3, 2f
2624          nop
2625        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2626        PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail)
26272:
2628        GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2629        brlz,pt %g6, sfmmu_tsb_miss_tt
2630          nop
2631        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2632        PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail)
2633        ba,pn   %xcc, sfmmu_tsb_miss_tt
2634          nop
2635
2636#else /* UTSB_PHYS */
2637	mov	%g1, %g3	/* save tsb8k reg in %g3 */
2638	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
2639	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2640	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
2641	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
2642	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
2643       /* g1 = first TSB pointer, g3 = second TSB pointer */
2644        srlx    %g2, TAG_VALO_SHIFT, %g7
2645        PROBE_2ND_ITSB(%g3, %g7, isynth)
2646	ba,pn	%xcc, sfmmu_tsb_miss_tt
2647	  nop
2648
2649#endif /* UTSB_PHYS */
2650#endif /* sun4v */
2651
2652#if defined(sun4u) && defined(UTSB_PHYS)
2653
2654        /*
2655	 * We come here for ism predict DTLB_MISS case or if
2656	 * if probe in first TSB failed.
2657         */
2658
2659        .align 64
2660        ALTENTRY(sfmmu_udtlb_slowpath_noismpred)
2661
2662	/*
2663         * g1 = tsb8k pointer register
2664         * g2 = tag access register
2665         * g4 - %g6 = scratch registers
2666         * g7 = TSB tag to match
2667	 */
2668
2669	/*
2670	 * ISM non-predict probe order
2671         * probe 1ST_TSB (8K index)
2672         * probe 2ND_TSB (4M index)
2673         * probe 4TH_TSB (4M index)
2674         * probe 3RD_TSB (8K index)
2675	 *
2676	 * We already probed first TSB in DTLB_MISS handler.
2677	 */
2678
2679        /*
2680         * Private 2ND TSB 4M-256 pages
2681         */
2682	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2683	brlz,pt %g3, 1f
2684	  nop
2685        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2686        PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2687
2688	/*
2689	 * Shared Context 4TH TSB 4M-256 pages
2690	 */
26911:
2692	GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2693	brlz,pt %g6, 2f
2694	  nop
2695        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2696        PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail)
2697
2698        /*
2699         * Shared Context 3RD TSB 8K-512K pages
2700         */
27012:
2702	GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2703	brlz,pt %g6, sfmmu_tsb_miss_tt
2704	  nop
2705        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2706        PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail)
2707	ba,pn	%xcc, sfmmu_tsb_miss_tt
2708	  nop
2709
2710	.align 64
2711        ALTENTRY(sfmmu_udtlb_slowpath_ismpred)
2712
2713	/*
2714         * g1 = tsb8k pointer register
2715         * g2 = tag access register
2716         * g4 - g6 = scratch registers
2717         * g7 = TSB tag to match
2718	 */
2719
2720	/*
2721	 * ISM predict probe order
2722	 * probe 4TH_TSB (4M index)
2723	 * probe 2ND_TSB (4M index)
2724	 * probe 1ST_TSB (8K index)
2725	 * probe 3RD_TSB (8K index)
2726
2727	/*
2728	 * Shared Context 4TH TSB 4M-256 pages
2729	 */
2730	GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
2731	brlz,pt %g6, 4f
2732	  nop
2733        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
2734        PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2)
2735
2736        /*
2737         * Private 2ND TSB 4M-256 pages
2738         */
27394:
2740	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2741	brlz,pt %g3, 5f
2742	  nop
2743        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2744        PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2)
2745
27465:
2747        PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2)
2748
2749        /*
2750         * Shared Context 3RD TSB 8K-512K pages
2751         */
2752	GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
2753	brlz,pt %g6, 6f
2754	  nop
2755        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
2756        PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2)
27576:
2758	ba,pn	%xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */
2759	  nop
2760
2761#else /* sun4u && UTSB_PHYS */
2762
2763       .align 64
2764        ALTENTRY(sfmmu_udtlb_slowpath)
2765
2766	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
2767	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
2768	  mov	%g1, %g3
2769
2770udtlb_miss_probefirst:
2771	/*
2772	 * g1 = 8K TSB pointer register
2773	 * g2 = tag access register
2774	 * g3 = (potentially) second TSB entry ptr
2775	 * g6 = ism pred.
2776	 * g7 = vpg_4m
2777	 */
2778#ifdef sun4v
2779	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2780	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2781
2782	/*
2783	 * Here:
2784	 *   g1 = first TSB pointer
2785	 *   g2 = tag access reg
2786	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2787	 */
2788	brgz,pn	%g6, sfmmu_tsb_miss_tt
2789	  nop
2790#else /* sun4v */
2791	mov	%g1, %g4
2792	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
2793	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2794
2795	/*
2796	 * Here:
2797	 *   g1 = first TSB pointer
2798	 *   g2 = tag access reg
2799	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2800	 */
2801	brgz,pn	%g6, sfmmu_tsb_miss_tt
2802	  nop
2803	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
2804	/* fall through in 8K->4M probe order */
2805#endif /* sun4v */
2806
2807udtlb_miss_probesecond:
2808	/*
2809	 * Look in the second TSB for the TTE
2810	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
2811	 * g2 = tag access reg
2812	 * g3 = 8K TSB pointer register
2813	 * g6 = ism pred.
2814	 * g7 = vpg_4m
2815	 */
2816#ifdef sun4v
2817	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
2818	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2819	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
2820#else /* sun4v */
2821	mov	%g3, %g7
2822	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
2823	/* %g2 clobbered, %g3 =second tsbe ptr */
2824	mov	MMU_TAG_ACCESS, %g2
2825	ldxa	[%g2]ASI_DMMU, %g2
2826#endif /* sun4v */
2827
2828	srlx	%g2, TAG_VALO_SHIFT, %g7
2829	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2830	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
2831	brgz,pn	%g6, udtlb_miss_probefirst
2832	  nop
2833
2834	/* fall through to sfmmu_tsb_miss_tt */
2835#endif /* sun4u && UTSB_PHYS */
2836
2837
2838	ALTENTRY(sfmmu_tsb_miss_tt)
2839	TT_TRACE(trace_tsbmiss)
2840	/*
2841	 * We get here if there is a TSB miss OR a write protect trap.
2842	 *
2843	 * g1 = First TSB entry pointer
2844	 * g2 = tag access register
2845	 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB
2846	 * g4 - g7 = scratch registers
2847	 */
2848
2849	ALTENTRY(sfmmu_tsb_miss)
2850
2851	/*
2852	 * If trapstat is running, we need to shift the %tpc and %tnpc to
2853	 * point to trapstat's TSB miss return code (note that trapstat
2854	 * itself will patch the correct offset to add).
2855	 */
2856	rdpr	%tl, %g7
2857	cmp	%g7, 1
2858	ble,pt	%xcc, 0f
2859	  sethi	%hi(KERNELBASE), %g6
2860	rdpr	%tpc, %g7
2861	or	%g6, %lo(KERNELBASE), %g6
2862	cmp	%g7, %g6
2863	bgeu,pt	%xcc, 0f
2864	/* delay slot safe */
2865
2866	ALTENTRY(tsbmiss_trapstat_patch_point)
2867	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
2868	wrpr	%g7, %tpc
2869	add	%g7, 4, %g7
2870	wrpr	%g7, %tnpc
28710:
2872	CPU_TSBMISS_AREA(%g6, %g7)
2873	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save 1ST tsb pointer */
2874	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save 2ND tsb pointer */
2875
2876	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
2877	brz,a,pn %g3, 1f			/* skip ahead if kernel */
2878	  ldn	[%g6 + TSBMISS_KHATID], %g7
2879	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
2880	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
2881
2882	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
2883
2884	cmp	%g3, INVALID_CONTEXT
2885	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
2886	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
2887
2888#if defined(sun4v) || defined(UTSB_PHYS)
2889        ldub    [%g6 + TSBMISS_URTTEFLAGS], %g7	/* clear ctx1 flag set from */
2890        andn    %g7, HAT_CHKCTX1_FLAG, %g7	/* the previous tsb miss    */
2891        stub    %g7, [%g6 + TSBMISS_URTTEFLAGS]
2892#endif /* sun4v || UTSB_PHYS */
2893
2894	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
2895	/*
2896	 * The miss wasn't in an ISM segment.
2897	 *
2898	 * %g1 %g3, %g4, %g5, %g7 all clobbered
2899	 * %g2 = (pseudo) tag access
2900	 */
2901
2902	ba,pt	%icc, 2f
2903	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
2904
29051:
2906	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
2907	/*
2908	 * 8K and 64K hash.
2909	 */
29102:
2911
2912	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2913		MMU_PAGESHIFT64K, TTE64K, %g5, tsb_l8K, tsb_checktte,
2914		sfmmu_suspend_tl, tsb_512K)
2915	/* NOT REACHED */
2916
2917tsb_512K:
2918	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
2919	brz,pn	%g5, 3f
2920	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
2921	and	%g4, HAT_512K_FLAG, %g5
2922
2923	/*
2924	 * Note that there is a small window here where we may have
2925	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
2926	 * flag yet, so we will skip searching the 512k hash list.
2927	 * In this case we will end up in pagefault which will find
2928	 * the mapping and return.  So, in this instance we will end up
2929	 * spending a bit more time resolving this TSB miss, but it can
2930	 * only happen once per process and even then, the chances of that
2931	 * are very small, so it's not worth the extra overhead it would
2932	 * take to close this window.
2933	 */
2934	brz,pn	%g5, tsb_4M
2935	  nop
29363:
2937	/*
2938	 * 512K hash
2939	 */
2940
2941	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2942		MMU_PAGESHIFT512K, TTE512K, %g5, tsb_l512K, tsb_checktte,
2943		sfmmu_suspend_tl, tsb_4M)
2944	/* NOT REACHED */
2945
2946tsb_4M:
2947	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
2948	brz,pn	%g5, 4f
2949	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
2950	and	%g4, HAT_4M_FLAG, %g5
2951	brz,pn	%g5, tsb_32M
2952	  nop
29534:
2954	/*
2955	 * 4M hash
2956	 */
2957
2958	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2959		MMU_PAGESHIFT4M, TTE4M, %g5, tsb_l4M, tsb_checktte,
2960		sfmmu_suspend_tl, tsb_32M)
2961	/* NOT REACHED */
2962
2963tsb_32M:
2964	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
2965#ifdef	sun4v
2966        brz,pn	%g5, 6f
2967#else
2968	brz,pn  %g5, tsb_pagefault
2969#endif
2970	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
2971	and	%g4, HAT_32M_FLAG, %g5
2972	brz,pn	%g5, tsb_256M
2973	  nop
29745:
2975	/*
2976	 * 32M hash
2977	 */
2978
2979	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2980		MMU_PAGESHIFT32M, TTE32M, %g5, tsb_l32M, tsb_checktte,
2981		sfmmu_suspend_tl, tsb_256M)
2982	/* NOT REACHED */
2983
2984#if defined(sun4u) && !defined(UTSB_PHYS)
2985#define tsb_shme        tsb_pagefault
2986#endif
2987tsb_256M:
2988	ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
2989	and	%g4, HAT_256M_FLAG, %g5
2990	brz,pn	%g5, tsb_shme
2991	  nop
29926:
2993	/*
2994	 * 256M hash
2995	 */
2996
2997	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
2998	    MMU_PAGESHIFT256M, TTE256M, %g5, tsb_l256M, tsb_checktte,
2999	    sfmmu_suspend_tl, tsb_shme)
3000	/* NOT REACHED */
3001
3002tsb_checktte:
3003	/*
3004	 * g1 = hblk_misc
3005	 * g2 = tagacc
3006	 * g3 = tte
3007	 * g4 = tte pa
3008	 * g6 = tsbmiss area
3009	 * g7 = hatid
3010	 */
3011	brlz,a,pt %g3, tsb_validtte
3012	  rdpr	%tt, %g7
3013
3014#if defined(sun4u) && !defined(UTSB_PHYS)
3015#undef tsb_shme
3016	ba      tsb_pagefault
3017	  nop
3018#else /* sun4u && !UTSB_PHYS */
3019
3020tsb_shme:
3021	/*
3022	 * g2 = tagacc
3023	 * g6 = tsbmiss area
3024	 */
3025	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3026	brz,pn	%g5, tsb_pagefault
3027	  nop
3028	ldx	[%g6 + TSBMISS_SHARED_UHATID], %g7	/* g7 = srdp */
3029	brz,pn	%g7, tsb_pagefault
3030	  nop
3031
3032	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3033		MMU_PAGESHIFT64K, TTE64K, %g5, tsb_shme_l8K, tsb_shme_checktte,
3034		sfmmu_suspend_tl, tsb_shme_512K)
3035	/* NOT REACHED */
3036
3037tsb_shme_512K:
3038	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3039	and	%g4, HAT_512K_FLAG, %g5
3040	brz,pn	%g5, tsb_shme_4M
3041	  nop
3042
3043	/*
3044	 * 512K hash
3045	 */
3046
3047	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3048		MMU_PAGESHIFT512K, TTE512K, %g5, tsb_shme_l512K, tsb_shme_checktte,
3049		sfmmu_suspend_tl, tsb_shme_4M)
3050	/* NOT REACHED */
3051
3052tsb_shme_4M:
3053	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3054	and	%g4, HAT_4M_FLAG, %g5
3055	brz,pn	%g5, tsb_shme_32M
3056	  nop
30574:
3058	/*
3059	 * 4M hash
3060	 */
3061	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3062		MMU_PAGESHIFT4M, TTE4M, %g5, tsb_shme_l4M, tsb_shme_checktte,
3063		sfmmu_suspend_tl, tsb_shme_32M)
3064	/* NOT REACHED */
3065
3066tsb_shme_32M:
3067	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3068	and	%g4, HAT_32M_FLAG, %g5
3069	brz,pn	%g5, tsb_shme_256M
3070	  nop
3071
3072	/*
3073	 * 32M hash
3074	 */
3075
3076	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3077		MMU_PAGESHIFT32M, TTE32M, %g5, tsb_shme_l32M, tsb_shme_checktte,
3078		sfmmu_suspend_tl, tsb_shme_256M)
3079	/* NOT REACHED */
3080
3081tsb_shme_256M:
3082	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3083	and	%g4, HAT_256M_FLAG, %g5
3084	brz,pn	%g5, tsb_pagefault
3085	  nop
3086
3087	/*
3088	 * 256M hash
3089	 */
3090
3091	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g6, %g1,
3092	    MMU_PAGESHIFT256M, TTE256M, %g5, tsb_shme_l256M, tsb_shme_checktte,
3093	    sfmmu_suspend_tl, tsb_pagefault)
3094	/* NOT REACHED */
3095
3096tsb_shme_checktte:
3097
3098	brgez,pn %g3, tsb_pagefault
3099	  rdpr	%tt, %g7
3100	/*
3101	 * g1 = ctx1 flag
3102	 * g3 = tte
3103	 * g4 = tte pa
3104	 * g6 = tsbmiss area
3105	 * g7 = tt
3106	 */
3107
3108	brz,pt  %g1, tsb_validtte
3109	  nop
3110	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g1
3111	  or	%g1, HAT_CHKCTX1_FLAG, %g1
3112	stub    %g1, [%g6 + TSBMISS_URTTEFLAGS]
3113
3114	SAVE_CTX1(%g7, %g2, %g1, tsb_shmel)
3115#endif /* sun4u && !UTSB_PHYS */
3116
3117tsb_validtte:
3118	/*
3119	 * g3 = tte
3120	 * g4 = tte pa
3121	 * g6 = tsbmiss area
3122	 * g7 = tt
3123	 */
3124
3125	/*
3126	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
3127	 */
3128	cmp	%g7, FAST_PROT_TT
3129	bne,pt	%icc, 4f
3130	  nop
3131
3132	TTE_SET_REFMOD_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_refmod,
3133	    tsb_protfault)
3134
3135	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3136#ifdef sun4v
3137	MMU_FAULT_STATUS_AREA(%g7)
3138	ldx	[%g7 + MMFSA_D_ADDR], %g5	/* load fault addr for later */
3139#else /* sun4v */
3140	mov     MMU_TAG_ACCESS, %g5
3141	ldxa    [%g5]ASI_DMMU, %g5
3142#endif /* sun4v */
3143	ba,pt	%xcc, tsb_update_tl1
3144	  nop
31454:
3146	/*
3147	 * If ITLB miss check exec bit.
3148	 * If not set treat as invalid TTE.
3149	 */
3150	cmp     %g7, T_INSTR_MMU_MISS
3151	be,pn	%icc, 5f
3152	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
3153	cmp     %g7, FAST_IMMU_MISS_TT
3154	bne,pt %icc, 3f
3155	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
31565:
3157	bz,pn %icc, tsb_protfault
3158	  nop
3159
31603:
3161	/*
3162	 * Set reference bit if not already set
3163	 */
3164	TTE_SET_REF_ML(%g3, %g4, %g6, %g7, %g5, tsb_lset_ref)
3165
3166	/*
3167	 * Now, load into TSB/TLB.  At this point:
3168	 * g3 = tte
3169	 * g4 = patte
3170	 * g6 = tsbmiss area
3171	 */
3172	rdpr	%tt, %g7
3173#ifdef sun4v
3174	MMU_FAULT_STATUS_AREA(%g2)
3175	cmp	%g7, T_INSTR_MMU_MISS
3176	be,a,pt	%icc, 9f
3177	  nop
3178	cmp	%g7, FAST_IMMU_MISS_TT
3179	be,a,pt	%icc, 9f
3180	  nop
3181	add	%g2, MMFSA_D_, %g2
31829:
3183	ldx	[%g2 + MMFSA_CTX_], %g7
3184	sllx	%g7, TTARGET_CTX_SHIFT, %g7
3185	ldx	[%g2 + MMFSA_ADDR_], %g2
3186	mov	%g2, %g5		! load the fault addr for later use
3187	srlx	%g2, TTARGET_VA_SHIFT, %g2
3188	or	%g2, %g7, %g2
3189#else /* sun4v */
3190	mov     MMU_TAG_ACCESS, %g5
3191	cmp     %g7, FAST_IMMU_MISS_TT
3192	be,a,pt %icc, 9f
3193	   ldxa  [%g0]ASI_IMMU, %g2
3194	ldxa    [%g0]ASI_DMMU, %g2
3195	ba,pt   %icc, tsb_update_tl1
3196	   ldxa  [%g5]ASI_DMMU, %g5
31979:
3198	ldxa    [%g5]ASI_IMMU, %g5
3199#endif /* sun4v */
3200
3201tsb_update_tl1:
3202	srlx	%g2, TTARGET_CTX_SHIFT, %g7
3203	brz,pn	%g7, tsb_kernel
3204#ifdef sun4v
3205	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
3206#else  /* sun4v */
3207	  srlx	%g3, TTE_SZ_SHFT, %g7
3208#endif /* sun4v */
3209
3210tsb_user:
3211#ifdef sun4v
3212	cmp	%g7, TTE4M
3213	bge,pn	%icc, tsb_user4m
3214	  nop
3215#else /* sun4v */
3216	cmp	%g7, TTESZ_VALID | TTE4M
3217	be,pn	%icc, tsb_user4m
3218	  srlx	%g3, TTE_SZ2_SHFT, %g7
3219	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
3220#ifdef ITLB_32M_256M_SUPPORT
3221	bnz,pn	%icc, tsb_user4m
3222	  nop
3223#else /* ITLB_32M_256M_SUPPORT */
3224	bnz,a,pn %icc, tsb_user_pn_synth
3225	 nop
3226#endif /* ITLB_32M_256M_SUPPORT */
3227#endif /* sun4v */
3228
3229tsb_user8k:
3230#if defined(sun4v) || defined(UTSB_PHYS)
3231	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3232	and	%g7, HAT_CHKCTX1_FLAG, %g1
3233	brz,a,pn %g1, 1f
3234	  ldn	[%g6 + TSBMISS_TSBPTR], %g1		! g1 = 1ST TSB ptr
3235	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1)
3236	brlz,a,pn %g1, ptl1_panic			! if no shared 3RD tsb
3237	  mov PTL1_NO_SCDTSB8K, %g1			! panic
3238        GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7)
32391:
3240#else /* defined(sun4v) || defined(UTSB_PHYS) */
3241	ldn   [%g6 + TSBMISS_TSBPTR], %g1             ! g1 = 1ST TSB ptr
3242#endif /* defined(sun4v) || defined(UTSB_PHYS) */
3243
3244#ifndef UTSB_PHYS
3245	mov	ASI_N, %g7	! user TSBs accessed by VA
3246	mov	%g7, %asi
3247#endif /* !UTSB_PHYS */
3248
3249	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l3)
3250
3251	rdpr    %tt, %g5
3252#ifdef sun4v
3253	cmp	%g5, T_INSTR_MMU_MISS
3254	be,a,pn	%xcc, 9f
3255	  mov	%g3, %g5
3256#endif /* sun4v */
3257	cmp	%g5, FAST_IMMU_MISS_TT
3258	be,pn	%xcc, 9f
3259	  mov	%g3, %g5
3260
3261	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3262	! trapstat wants TTE in %g5
3263	retry
32649:
3265	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3266	! trapstat wants TTE in %g5
3267	retry
3268
3269tsb_user4m:
3270#if defined(sun4v) || defined(UTSB_PHYS)
3271	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3272	and	%g7, HAT_CHKCTX1_FLAG, %g1
3273	brz,a,pn %g1, 4f
3274	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		! g1 = 2ND TSB ptr
3275	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr
3276	brlz,a,pn %g1, 5f				! if no shared 4TH TSB
3277	  nop
3278        GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7)
3279
3280#else /* defined(sun4v) || defined(UTSB_PHYS) */
3281	ldn   [%g6 + TSBMISS_TSBPTR4M], %g1             ! g1 = 2ND TSB ptr
3282#endif /* defined(sun4v) || defined(UTSB_PHYS) */
32834:
3284	brlz,pn %g1, 5f	/* Check to see if we have 2nd TSB programmed */
3285	  nop
3286
3287#ifndef UTSB_PHYS
3288	mov	ASI_N, %g7	! user TSBs accessed by VA
3289	mov	%g7, %asi
3290#endif /* UTSB_PHYS */
3291
3292        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l4)
3293
32945:
3295	rdpr    %tt, %g5
3296#ifdef sun4v
3297        cmp     %g5, T_INSTR_MMU_MISS
3298        be,a,pn %xcc, 9f
3299          mov   %g3, %g5
3300#endif /* sun4v */
3301        cmp     %g5, FAST_IMMU_MISS_TT
3302        be,pn   %xcc, 9f
3303        mov     %g3, %g5
3304
3305        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3306        ! trapstat wants TTE in %g5
3307        retry
33089:
3309        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3310        ! trapstat wants TTE in %g5
3311        retry
3312
3313#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT)
3314	/*
3315	 * Panther ITLB synthesis.
3316	 * The Panther 32M and 256M ITLB code simulates these two large page
3317	 * sizes with 4M pages, to provide support for programs, for example
3318	 * Java, that may copy instructions into a 32M or 256M data page and
3319	 * then execute them. The code below generates the 4M pfn bits and
3320	 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
3321	 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
3322	 * are ignored by the hardware.
3323	 *
3324	 * Now, load into TSB/TLB.  At this point:
3325	 * g2 = tagtarget
3326	 * g3 = tte
3327	 * g4 = patte
3328	 * g5 = tt
3329	 * g6 = tsbmiss area
3330	 */
3331tsb_user_pn_synth:
3332	rdpr %tt, %g5
3333	cmp    %g5, FAST_IMMU_MISS_TT
3334	be,pt	%xcc, tsb_user_itlb_synth	/* ITLB miss */
3335	  andcc %g3, TTE_EXECPRM_INT, %g0	/* is execprm bit set */
3336	bz,pn %icc, 4b				/* if not, been here before */
3337	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	/* g1 = tsbp */
3338	brlz,a,pn %g1, 5f			/* no 2nd tsb */
3339	  mov	%g3, %g5
3340
3341	mov	MMU_TAG_ACCESS, %g7
3342	ldxa	[%g7]ASI_DMMU, %g6		/* get tag access va */
3343	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)	/* make 4M pfn offset */
3344
3345	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3346	mov	%g7, %asi
3347	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l5) /* update TSB */
33485:
3349        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3350        retry
3351
3352tsb_user_itlb_synth:
3353	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 =  2ND TSB */
3354
3355	mov	MMU_TAG_ACCESS, %g7
3356	ldxa	[%g7]ASI_IMMU, %g6		/* get tag access va */
3357	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)	/* make 4M pfn offset */
3358	brlz,a,pn %g1, 7f	/* Check to see if we have 2nd TSB programmed */
3359	  or	%g5, %g3, %g5			/* add 4M bits to TTE */
3360
3361	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3362	mov	%g7, %asi
3363	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, locked_tsb_l6) /* update TSB */
33647:
3365	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
3366        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3367        retry
3368#endif /* sun4v && ITLB_32M_256M_SUPPORT */
3369
3370tsb_kernel:
3371	rdpr	%tt, %g5
3372#ifdef sun4v
3373	cmp	%g7, TTE4M
3374	bge,pn	%icc, 5f
3375#else
3376	cmp	%g7, TTESZ_VALID | TTE4M	! no 32M or 256M support
3377	be,pn	%icc, 5f
3378#endif /* sun4v */
3379	  nop
3380	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8K TSB ptr
3381	ba,pt	%xcc, 6f
3382	  nop
33835:
3384	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4M TSB ptr
3385	brlz,pn	%g1, 3f		/* skip programming if 4M TSB ptr is -1 */
3386	  nop
33876:
3388#ifndef sun4v
3389tsb_kernel_patch_asi:
3390	or	%g0, RUNTIME_PATCH, %g6
3391	mov	%g6, %asi	! XXX avoid writing to %asi !!
3392#endif
3393	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, locked_tsb_l7)
33943:
3395#ifdef sun4v
3396	cmp	%g5, T_INSTR_MMU_MISS
3397	be,a,pn	%icc, 1f
3398	  mov	%g3, %g5			! trapstat wants TTE in %g5
3399#endif /* sun4v */
3400	cmp	%g5, FAST_IMMU_MISS_TT
3401	be,pn	%icc, 1f
3402	  mov	%g3, %g5			! trapstat wants TTE in %g5
3403	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3404	! trapstat wants TTE in %g5
3405	retry
34061:
3407	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3408	! trapstat wants TTE in %g5
3409	retry
3410
3411tsb_ism:
3412	/*
3413	 * This is an ISM [i|d]tlb miss.  We optimize for largest
3414	 * page size down to smallest.
3415	 *
3416	 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access
3417	 *	register
3418	 * g3 = ismmap->ism_seg
3419	 * g4 = physical address of ismmap->ism_sfmmu
3420	 * g6 = tsbmiss area
3421	 */
3422	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
3423	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
3424	  mov	PTL1_BAD_ISM, %g1
3425						/* g5 = pa of imap_vb_shift */
3426	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
3427	lduba	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
3428	srlx	%g3, %g4, %g3			/* clr size field */
3429	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
3430	sllx    %g3, %g4, %g3                   /* g3 = ism vbase */
3431	and     %g2, %g1, %g4                   /* g4 = ctx number */
3432	andn    %g2, %g1, %g1                   /* g1 = tlb miss vaddr */
3433	sub     %g1, %g3, %g2                   /* g2 = offset in ISM seg */
3434	or      %g2, %g4, %g2                   /* g2 = (pseudo-)tagacc */
3435	sub     %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
3436	lduha   [%g5]ASI_MEM, %g4               /* g5 = pa of imap_hatflags */
3437#if defined(sun4v) || defined(UTSB_PHYS)
3438	and     %g4, HAT_CTX1_FLAG, %g5         /* g5 = imap_hatflags */
3439	brz,pt %g5, tsb_chk4M_ism
3440	  nop
3441	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g5
3442	or      %g5, HAT_CHKCTX1_FLAG, %g5
3443	stub    %g5, [%g6 + TSBMISS_URTTEFLAGS]
3444	rdpr    %tt, %g5
3445	SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl)
3446#endif /* defined(sun4v) || defined(UTSB_PHYS) */
3447
3448	/*
3449	 * ISM pages are always locked down.
3450	 * If we can't find the tte then pagefault
3451	 * and let the spt segment driver resolve it.
3452	 *
3453	 * g2 = tagacc w/ISM vaddr (offset in ISM seg)
3454	 * g4 = imap_hatflags
3455	 * g6 = tsb miss area
3456	 * g7 = ISM hatid
3457	 */
3458
3459tsb_chk4M_ism:
3460	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
3461	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
3462	  nop
3463
3464tsb_ism_32M:
3465	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
3466	brz,pn	%g5, tsb_ism_256M
3467	  nop
3468
3469	/*
3470	 * 32M hash.
3471	 */
3472
3473	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT32M,
3474	    TTE32M, %g5, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3475	    tsb_ism_4M)
3476	/* NOT REACHED */
3477
3478tsb_ism_32M_found:
3479	brlz,a,pt %g3, tsb_validtte
3480	  rdpr	%tt, %g7
3481	ba,pt	%xcc, tsb_ism_4M
3482	  nop
3483
3484tsb_ism_256M:
3485	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
3486	brz,a,pn %g5, ptl1_panic
3487	  mov	PTL1_BAD_ISM, %g1
3488
3489	/*
3490	 * 256M hash.
3491	 */
3492	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT256M,
3493	    TTE256M, %g5, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3494	    tsb_ism_4M)
3495
3496tsb_ism_256M_found:
3497	brlz,a,pt %g3, tsb_validtte
3498	  rdpr	%tt, %g7
3499
3500tsb_ism_4M:
3501	/*
3502	 * 4M hash.
3503	 */
3504	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT4M,
3505	    TTE4M, %g5, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3506	    tsb_ism_8K)
3507	/* NOT REACHED */
3508
3509tsb_ism_4M_found:
3510	brlz,a,pt %g3, tsb_validtte
3511	  rdpr	%tt, %g7
3512
3513tsb_ism_8K:
3514	/*
3515	 * 8K and 64K hash.
3516	 */
3517
3518	GET_TTE(%g2, %g7, %g3, %g4, %g6, %g1, MMU_PAGESHIFT64K,
3519	    TTE64K, %g5, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3520	    tsb_pagefault)
3521	/* NOT REACHED */
3522
3523tsb_ism_8K_found:
3524	brlz,a,pt %g3, tsb_validtte
3525	  rdpr	%tt, %g7
3526
3527tsb_pagefault:
3528	rdpr	%tt, %g7
3529	cmp	%g7, FAST_PROT_TT
3530	be,a,pn	%icc, tsb_protfault
3531	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt
3532
3533tsb_protfault:
3534	/*
3535	 * we get here if we couldn't find a valid tte in the hash.
3536	 *
3537	 * If user and we are at tl>1 we go to window handling code.
3538	 *
3539	 * If kernel and the fault is on the same page as our stack
3540	 * pointer, then we know the stack is bad and the trap handler
3541	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3542	 *
3543	 * If this is a kernel trap and tl>1, panic.
3544	 *
3545	 * Otherwise we call pagefault.
3546	 */
3547	cmp	%g7, FAST_IMMU_MISS_TT
3548#ifdef sun4v
3549	MMU_FAULT_STATUS_AREA(%g4)
3550	ldx	[%g4 + MMFSA_I_CTX], %g5
3551	ldx	[%g4 + MMFSA_D_CTX], %g4
3552	move	%icc, %g5, %g4
3553	cmp	%g7, T_INSTR_MMU_MISS
3554	move	%icc, %g5, %g4
3555#else
3556	mov	MMU_TAG_ACCESS, %g4
3557	ldxa	[%g4]ASI_DMMU, %g2
3558	ldxa	[%g4]ASI_IMMU, %g5
3559	move	%icc, %g5, %g2
3560	cmp	%g7, T_INSTR_MMU_MISS
3561	move	%icc, %g5, %g2
3562	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
3563#endif /* sun4v */
3564	brnz,pn	%g4, 3f				/* skip if not kernel */
3565	  rdpr	%tl, %g5
3566
3567	add	%sp, STACK_BIAS, %g3
3568	srlx	%g3, MMU_PAGESHIFT, %g3
3569	srlx	%g2, MMU_PAGESHIFT, %g4
3570	cmp	%g3, %g4
3571	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
3572	  mov	PTL1_BAD_STACK, %g1
3573
3574	cmp	%g5, 1
3575	ble,pt	%icc, 2f
3576	  nop
3577	TSTAT_CHECK_TL1(2f, %g1, %g2)
3578	rdpr	%tt, %g2
3579	cmp	%g2, FAST_PROT_TT
3580	mov	PTL1_BAD_KPROT_FAULT, %g1
3581	movne	%icc, PTL1_BAD_KMISS, %g1
3582	ba,pt	%icc, ptl1_panic
3583	  nop
3584
35852:
3586	/*
3587	 * We are taking a pagefault in the kernel on a kernel address.  If
3588	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
3589	 * want to call sfmmu_pagefault -- we will instead note that a fault
3590	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
3591	 * (instead of a "retry").  This will step over the faulting
3592	 * instruction.
3593	 */
3594	CPU_INDEX(%g1, %g2)
3595	set	cpu_core, %g2
3596	sllx	%g1, CPU_CORE_SHIFT, %g1
3597	add	%g1, %g2, %g1
3598	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3599	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3600	bz	sfmmu_pagefault
3601	or	%g2, CPU_DTRACE_BADADDR, %g2
3602	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3603	GET_MMU_D_ADDR(%g3, %g4)
3604	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3605	done
3606
36073:
3608	cmp	%g5, 1
3609	ble,pt	%icc, 4f
3610	  nop
3611	TSTAT_CHECK_TL1(4f, %g1, %g2)
3612	ba,pt	%icc, sfmmu_window_trap
3613	  nop
3614
36154:
3616	/*
3617	 * We are taking a pagefault on a non-kernel address.  If we are in
3618	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
3619	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
3620	 */
3621	CPU_INDEX(%g1, %g2)
3622	set	cpu_core, %g2
3623	sllx	%g1, CPU_CORE_SHIFT, %g1
3624	add	%g1, %g2, %g1
3625	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3626	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3627	bz	sfmmu_mmu_trap
3628	or	%g2, CPU_DTRACE_BADADDR, %g2
3629	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3630	GET_MMU_D_ADDR(%g3, %g4)
3631	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3632
3633	/*
3634	 * Be sure that we're actually taking this miss from the kernel --
3635	 * otherwise we have managed to return to user-level with
3636	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3637	 */
3638	rdpr	%tstate, %g2
3639	btst	TSTATE_PRIV, %g2
3640	bz,a	ptl1_panic
3641	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3642	done
3643
3644	ALTENTRY(tsb_tl0_noctxt)
3645	/*
3646	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
3647	 * if it is, indicated that we have faulted and issue a done.
3648	 */
3649	CPU_INDEX(%g5, %g6)
3650	set	cpu_core, %g6
3651	sllx	%g5, CPU_CORE_SHIFT, %g5
3652	add	%g5, %g6, %g5
3653	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
3654	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
3655	bz	1f
3656	or	%g6, CPU_DTRACE_BADADDR, %g6
3657	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
3658	GET_MMU_D_ADDR(%g3, %g4)
3659	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]
3660
3661	/*
3662	 * Be sure that we're actually taking this miss from the kernel --
3663	 * otherwise we have managed to return to user-level with
3664	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3665	 */
3666	rdpr	%tstate, %g5
3667	btst	TSTATE_PRIV, %g5
3668	bz,a	ptl1_panic
3669	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3670	TSTAT_CHECK_TL1(2f, %g1, %g2);
36712:
3672	done
3673
36741:
3675	rdpr	%tt, %g5
3676	cmp	%g5, FAST_IMMU_MISS_TT
3677#ifdef sun4v
3678	MMU_FAULT_STATUS_AREA(%g2)
3679	be,a,pt	%icc, 2f
3680	  ldx	[%g2 + MMFSA_I_CTX], %g3
3681	cmp	%g5, T_INSTR_MMU_MISS
3682	be,a,pt	%icc, 2f
3683	  ldx	[%g2 + MMFSA_I_CTX], %g3
3684	ldx	[%g2 + MMFSA_D_CTX], %g3
36852:
3686#else
3687	mov	MMU_TAG_ACCESS, %g2
3688	be,a,pt	%icc, 2f
3689	  ldxa	[%g2]ASI_IMMU, %g3
3690	ldxa	[%g2]ASI_DMMU, %g3
36912:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
3692#endif /* sun4v */
3693	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
3694	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
3695	rdpr	%tl, %g5
3696	cmp	%g5, 1
3697	ble,pt	%icc, sfmmu_mmu_trap
3698	  nop
3699	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
3700	ba,pt	%icc, sfmmu_window_trap
3701	  nop
3702	SET_SIZE(sfmmu_tsb_miss)
3703
3704	/*
3705	 * This routine will look for a user or kernel vaddr in the hash
3706	 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't grab
3707	 * any locks.  It should only be used by other sfmmu routines.
3708	 */
3709	ENTRY_NP(sfmmu_vatopfn)
3710 	/*
3711 	 * disable interrupts
3712 	 */
3713 	rdpr	%pstate, %o3
3714#ifdef DEBUG
3715	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1)
3716#endif
3717	/*
3718	 * disable interrupts to protect the TSBMISS area
3719	 */
3720	andn    %o3, PSTATE_IE, %o5
3721	wrpr    %o5, 0, %pstate
3722
3723	/*
3724	 * o0 = vaddr
3725	 * o1 = sfmmup
3726	 * o2 = ttep
3727	 */
3728	CPU_TSBMISS_AREA(%g1, %o5)
3729	ldn	[%g1 + TSBMISS_KHATID], %o4
3730	cmp	%o4, %o1
3731	bne,pn	%ncc, vatopfn_nokernel
3732	  mov	TTE64K, %g5			/* g5 = rehash # */
3733	mov %g1,%o5				/* o5 = tsbmiss_area */
3734	/*
3735	 * o0 = vaddr
3736	 * o1 & o4 = hatid
3737	 * o2 = ttep
3738	 * o5 = tsbmiss area
3739	 */
3740	mov	HBLK_RANGE_SHIFT, %g6
37411:
3742
3743	/*
3744	 * o0 = vaddr
3745	 * o1 = sfmmup
3746	 * o2 = ttep
3747	 * o3 = old %pstate
3748	 * o4 = hatid
3749	 * o5 = tsbmiss
3750	 * g5 = rehash #
3751	 * g6 = hmeshift
3752	 *
3753	 * The first arg to GET_TTE is actually tagaccess register
3754	 * not just vaddr. Since this call is for kernel we need to clear
3755	 * any lower vaddr bits that would be interpreted as ctx bits.
3756	 */
3757	set     TAGACC_CTX_MASK, %g1
3758	andn    %o0, %g1, %o0
3759	GET_TTE(%o0, %o4, %g1, %g2, %o5, %g4, %g6, %g5, %g3,
3760		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
3761
3762kvtop_hblk_found:
3763	/*
3764	 * o0 = vaddr
3765	 * o1 = sfmmup
3766	 * o2 = ttep
3767	 * g1 = tte
3768	 * g2 = tte pa
3769	 * g3 = scratch
3770	 * o2 = tsbmiss area
3771	 * o1 = hat id
3772	 */
3773	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
3774	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3775	stx %g1,[%o2]				/* put tte into *ttep */
3776	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
3777	/*
3778	 * o0 = vaddr
3779	 * o1 = sfmmup
3780	 * o2 = ttep
3781	 * g1 = pfn
3782	 */
3783	ba,pt	%xcc, 6f
3784	  mov	%g1, %o0
3785
3786kvtop_nohblk:
3787	/*
3788	 * we get here if we couldn't find valid hblk in hash.  We rehash
3789	 * if neccesary.
3790	 */
3791	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
3792#ifdef sun4v
3793	cmp	%g5, MAX_HASHCNT
3794#else
3795	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
3796#endif /* sun4v */
3797	be,a,pn	%icc, 6f
3798	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3799	mov	%o1, %o4			/* restore hatid */
3800#ifdef sun4v
3801        add	%g5, 2, %g5
3802	cmp	%g5, 3
3803	move	%icc, MMU_PAGESHIFT4M, %g6
3804	ba,pt	%icc, 1b
3805	movne	%icc, MMU_PAGESHIFT256M, %g6
3806#else
3807        inc	%g5
3808	cmp	%g5, 2
3809	move	%icc, MMU_PAGESHIFT512K, %g6
3810	ba,pt	%icc, 1b
3811	movne	%icc, MMU_PAGESHIFT4M, %g6
3812#endif /* sun4v */
38136:
3814	retl
3815 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3816
3817tsb_suspend:
3818	/*
3819	 * o0 = vaddr
3820	 * o1 = sfmmup
3821	 * o2 = ttep
3822	 * g1 = tte
3823	 * g2 = tte pa
3824	 * g3 = tte va
3825	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
3826	 */
3827	stx %g1,[%o2]				/* put tte into *ttep */
3828	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
3829	  sub	%g0, 1, %o0			/* output = PFN_INVALID */
3830	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
38318:
3832	retl
3833	 wrpr	%g0, %o3, %pstate		/* enable interrupts */
3834
3835vatopfn_nokernel:
3836	/*
3837	 * This routine does NOT support user addresses
3838	 * There is a routine in C that supports this.
3839	 * The only reason why we don't have the C routine
3840	 * support kernel addresses as well is because
3841	 * we do va_to_pa while holding the hashlock.
3842	 */
3843 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3844	save	%sp, -SA(MINFRAME), %sp
3845	sethi	%hi(sfmmu_panic3), %o0
3846	call	panic
3847	 or	%o0, %lo(sfmmu_panic3), %o0
3848
3849	SET_SIZE(sfmmu_vatopfn)
3850
3851	/*
3852	 * %o0 = vaddr
3853	 * %o1 = hashno (aka szc)
3854	 *
3855	 *
3856	 * This routine is similar to sfmmu_vatopfn() but will only look for
3857	 * a kernel vaddr in the hash structure for the specified rehash value.
3858	 * It's just an optimization for the case when pagesize for a given
3859	 * va range is already known (e.g. large page heap) and we don't want
3860	 * to start the search with rehash value 1 as sfmmu_vatopfn() does.
3861	 *
3862	 * Returns valid pfn or PFN_INVALID if
3863	 * tte for specified rehash # is not found, invalid or suspended.
3864	 */
3865	ENTRY_NP(sfmmu_kvaszc2pfn)
3866 	/*
3867 	 * disable interrupts
3868 	 */
3869 	rdpr	%pstate, %o3
3870#ifdef DEBUG
3871	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1)
3872#endif
3873	/*
3874	 * disable interrupts to protect the TSBMISS area
3875	 */
3876	andn    %o3, PSTATE_IE, %o5
3877	wrpr    %o5, 0, %pstate
3878
3879	CPU_TSBMISS_AREA(%g1, %o5)
3880	ldn	[%g1 + TSBMISS_KHATID], %o4
3881	sll	%o1, 1, %g6
3882	add	%g6, %o1, %g6
3883	add	%g6, MMU_PAGESHIFT, %g6
3884	/*
3885	 * %o0 = vaddr
3886	 * %o1 = hashno
3887	 * %o3 = old %pstate
3888	 * %o4 = ksfmmup
3889	 * %g1 = tsbmiss area
3890	 * %g6 = hmeshift
3891	 */
3892
3893	/*
3894	 * The first arg to GET_TTE is actually tagaccess register
3895	 * not just vaddr. Since this call is for kernel we need to clear
3896	 * any lower vaddr bits that would be interpreted as ctx bits.
3897	 */
3898	srlx	%o0, MMU_PAGESHIFT, %o0
3899	sllx	%o0, MMU_PAGESHIFT, %o0
3900	GET_TTE(%o0, %o4, %g3, %g4, %g1, %o5, %g6, %o1, %g5,
3901		kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk,
3902		kvaszc2pfn_nohblk)
3903
3904kvaszc2pfn_hblk_found:
3905	/*
3906	 * %g3 = tte
3907	 * %o0 = vaddr
3908	 */
3909	brgez,a,pn %g3, 1f			/* check if tte is invalid */
3910	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3911	TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5)
3912	/*
3913	 * g3 = pfn
3914	 */
3915	ba,pt	%xcc, 1f
3916	  mov	%g3, %o0
3917
3918kvaszc2pfn_nohblk:
3919	mov	-1, %o0
3920
39211:
3922	retl
3923 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3924
3925	SET_SIZE(sfmmu_kvaszc2pfn)
3926
3927
3928
3929/*
3930 * kpm lock used between trap level tsbmiss handler and kpm C level.
3931 */
3932#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
3933	mov     0xff, tmp1						;\
3934label1:									;\
3935	casa    [kpmlckp]asi, %g0, tmp1					;\
3936	brnz,pn tmp1, label1						;\
3937	mov     0xff, tmp1						;\
3938	membar  #LoadLoad
3939
3940#define KPMLOCK_EXIT(kpmlckp, asi)					\
3941	membar  #LoadStore|#StoreStore					;\
3942	sta     %g0, [kpmlckp]asi
3943
3944/*
3945 * Lookup a memseg for a given pfn and if found, return the physical
3946 * address of the corresponding struct memseg in mseg, otherwise
3947 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
3948 * tsbmp, %asi is assumed to be ASI_MEM.
3949 * This lookup is done by strictly traversing only the physical memseg
3950 * linkage. The more generic approach, to check the virtual linkage
3951 * before using the physical (used e.g. with hmehash buckets), cannot
3952 * be used here. Memory DR operations can run in parallel to this
3953 * lookup w/o any locks and updates of the physical and virtual linkage
3954 * cannot be done atomically wrt. to each other. Because physical
3955 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
3956 * as "physical NULL" pointer.
3957 */
3958#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
3959	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
3960	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
3961	udivx	pfn, mseg, mseg						;\
3962	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
3963	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
3964	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
3965	add	tmp1, mseg, tmp1					;\
3966	ldxa	[tmp1]%asi, mseg					;\
3967	cmp	mseg, MSEG_NULLPTR_PA					;\
3968	be,pn	%xcc, label##1		/* if not found */	;\
3969	  nop								;\
3970	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3971	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3972	blu,pn	%xcc, label##1					;\
3973	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3974	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3975	bgeu,pn	%xcc, label##1					;\
3976	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
3977	mulx	tmp1, PAGE_SIZE, tmp1					;\
3978	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
3979	add	tmp2, tmp1, tmp1			/* pp */	;\
3980	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
3981	cmp	tmp2, pfn						;\
3982	be,pt	%xcc, label##_ok			/* found */	;\
3983label##1:								;\
3984	/* brute force lookup */					;\
3985	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
3986	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
3987label##2:								;\
3988	cmp	mseg, MSEG_NULLPTR_PA					;\
3989	be,pn	%xcc, label##_ok		/* if not found */	;\
3990	  nop								;\
3991	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3992	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3993	blu,a,pt %xcc, label##2					;\
3994	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3995	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3996	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3997	bgeu,a,pt %xcc, label##2					;\
3998	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3999label##_ok:
4000
4001	/*
4002	 * kpm tsb miss handler large pages
4003	 * g1 = 8K kpm TSB entry pointer
4004	 * g2 = tag access register
4005	 * g3 = 4M kpm TSB entry pointer
4006	 */
4007	ALTENTRY(sfmmu_kpm_dtsb_miss)
4008	TT_TRACE(trace_tsbmiss)
4009
4010	CPU_INDEX(%g7, %g6)
4011	sethi	%hi(kpmtsbm_area), %g6
4012	sllx	%g7, KPMTSBM_SHIFT, %g7
4013	or	%g6, %lo(kpmtsbm_area), %g6
4014	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4015
4016	/* check enable flag */
4017	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4018	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4019	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4020	  nop
4021
4022	/* VA range check */
4023	ldx	[%g6 + KPMTSBM_VBASE], %g7
4024	cmp	%g2, %g7
4025	blu,pn	%xcc, sfmmu_tsb_miss
4026	  ldx	[%g6 + KPMTSBM_VEND], %g5
4027	cmp	%g2, %g5
4028	bgeu,pn	%xcc, sfmmu_tsb_miss
4029	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]
4030
4031	/*
4032	 * check TL tsbmiss handling flag
4033	 * bump tsbmiss counter
4034	 */
4035	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4036#ifdef	DEBUG
4037	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
4038	inc	%g5
4039	brz,pn	%g3, sfmmu_kpm_exception
4040	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4041#else
4042	inc	%g5
4043	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4044#endif
4045	/*
4046	 * At this point:
4047	 *  g1 = 8K kpm TSB pointer (not used)
4048	 *  g2 = tag access register
4049	 *  g3 = clobbered
4050	 *  g6 = per-CPU kpm tsbmiss area
4051	 *  g7 = kpm_vbase
4052	 */
4053
4054	/* vaddr2pfn */
4055	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
4056	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4057	srax    %g4, %g3, %g2			/* which alias range (r) */
4058	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
4059	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
4060
4061	/*
4062	 * Setup %asi
4063	 * mseg_pa = page_numtomemseg_nolock(pfn)
4064	 * if (mseg_pa == NULL) sfmmu_kpm_exception
4065	 * g2=pfn
4066	 */
4067	mov	ASI_MEM, %asi
4068	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
4069	cmp	%g3, MSEG_NULLPTR_PA
4070	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4071	  nop
4072
4073	/*
4074	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
4075	 * g2=pfn g3=mseg_pa
4076	 */
4077	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
4078	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4079	srlx	%g2, %g5, %g4
4080	sllx	%g4, %g5, %g4
4081	sub	%g4, %g7, %g4
4082	srlx	%g4, %g5, %g4
4083
4084	/*
4085	 * Validate inx value
4086	 * g2=pfn g3=mseg_pa g4=inx
4087	 */
4088#ifdef	DEBUG
4089	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4090	cmp	%g4, %g5			/* inx - nkpmpgs */
4091	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4092	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4093#else
4094	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4095#endif
4096	/*
4097	 * kp = &mseg_pa->kpm_pages[inx]
4098	 */
4099	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
4100	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
4101	add	%g5, %g4, %g5			/* kp */
4102
4103	/*
4104	 * KPMP_HASH(kp)
4105	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
4106	 */
4107	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4108	sub	%g7, 1, %g7			/* mask */
4109	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
4110	add	%g5, %g1, %g5			/* y = ksp + x */
4111	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4112
4113	/*
4114	 * Calculate physical kpm_page pointer
4115	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4116	 */
4117	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
4118	add	%g1, %g4, %g1			/* kp_pa */
4119
4120	/*
4121	 * Calculate physical hash lock address
4122	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
4123	 */
4124	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
4125	sllx	%g5, KPMHLK_SHIFT, %g5
4126	add	%g4, %g5, %g3
4127	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */
4128
4129	/*
4130	 * Assemble tte
4131	 * g1=kp_pa g2=pfn g3=hlck_pa
4132	 */
4133#ifdef sun4v
4134	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4135	sllx	%g5, 32, %g5
4136	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4137	or	%g4, TTE4M, %g4
4138	or	%g5, %g4, %g5
4139#else
4140	sethi	%hi(TTE_VALID_INT), %g4
4141	mov	TTE4M, %g5
4142	sllx	%g5, TTE_SZ_SHFT_INT, %g5
4143	or	%g5, %g4, %g5			/* upper part */
4144	sllx	%g5, 32, %g5
4145	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4146	or	%g5, %g4, %g5
4147#endif
4148	sllx	%g2, MMU_PAGESHIFT, %g4
4149	or	%g5, %g4, %g5			/* tte */
4150	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4151	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4152
4153	/*
4154	 * tsb dropin
4155	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
4156	 */
4157
4158	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4159	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
4160
4161	/* use C-handler if there's no go for dropin */
4162	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
4163	cmp	%g7, -1
4164	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
4165	  nop
4166
4167#ifdef	DEBUG
4168	/* double check refcnt */
4169	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
4170	brz,pn	%g7, 5f			/* let C-handler deal with this */
4171	  nop
4172#endif
4173
4174#ifndef sun4v
4175	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4176	mov	ASI_N, %g1
4177	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4178	movnz	%icc, ASI_MEM, %g1
4179	mov	%g1, %asi
4180#endif
4181
4182	/*
4183	 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
4184	 * If we fail to lock the TSB entry then just load the tte into the
4185	 * TLB.
4186	 */
4187	TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l1)
4188
4189	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4190	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4191locked_tsb_l1:
4192	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
4193
4194	/* KPMLOCK_EXIT(kpmlckp, asi) */
4195	KPMLOCK_EXIT(%g3, ASI_MEM)
4196
4197	/*
4198	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4199	 * point to trapstat's TSB miss return code (note that trapstat
4200	 * itself will patch the correct offset to add).
4201	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4202	 */
4203	rdpr	%tl, %g7
4204	cmp	%g7, 1
4205	ble	%icc, 0f
4206	sethi	%hi(KERNELBASE), %g6
4207	rdpr	%tpc, %g7
4208	or	%g6, %lo(KERNELBASE), %g6
4209	cmp	%g7, %g6
4210	bgeu	%xcc, 0f
4211	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
4212	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4213	wrpr	%g7, %tpc
4214	add	%g7, 4, %g7
4215	wrpr	%g7, %tnpc
42160:
4217	retry
42185:
4219	/* g3=hlck_pa */
4220	KPMLOCK_EXIT(%g3, ASI_MEM)
4221	ba,pt	%icc, sfmmu_kpm_exception
4222	  nop
4223	SET_SIZE(sfmmu_kpm_dtsb_miss)
4224
4225	/*
4226	 * kpm tsbmiss handler for smallpages
4227	 * g1 = 8K kpm TSB pointer
4228	 * g2 = tag access register
4229	 * g3 = 4M kpm TSB pointer
4230	 */
4231	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
4232	TT_TRACE(trace_tsbmiss)
4233	CPU_INDEX(%g7, %g6)
4234	sethi	%hi(kpmtsbm_area), %g6
4235	sllx	%g7, KPMTSBM_SHIFT, %g7
4236	or	%g6, %lo(kpmtsbm_area), %g6
4237	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4238
4239	/* check enable flag */
4240	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4241	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4242	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4243	  nop
4244
4245	/*
4246	 * VA range check
4247	 * On fail: goto sfmmu_tsb_miss
4248	 */
4249	ldx	[%g6 + KPMTSBM_VBASE], %g7
4250	cmp	%g2, %g7
4251	blu,pn	%xcc, sfmmu_tsb_miss
4252	  ldx	[%g6 + KPMTSBM_VEND], %g5
4253	cmp	%g2, %g5
4254	bgeu,pn	%xcc, sfmmu_tsb_miss
4255	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */
4256
4257	/*
4258	 * check TL tsbmiss handling flag
4259	 * bump tsbmiss counter
4260	 */
4261	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4262#ifdef	DEBUG
4263	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
4264	inc	%g5
4265	brz,pn	%g1, sfmmu_kpm_exception
4266	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4267#else
4268	inc	%g5
4269	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4270#endif
4271	/*
4272	 * At this point:
4273	 *  g1 = clobbered
4274	 *  g2 = tag access register
4275	 *  g3 = 4M kpm TSB pointer (not used)
4276	 *  g6 = per-CPU kpm tsbmiss area
4277	 *  g7 = kpm_vbase
4278	 */
4279
4280	/*
4281	 * Assembly implementation of SFMMU_KPM_VTOP(vaddr, paddr)
4282	 * which is defined in mach_kpm.h. Any changes in that macro
4283	 * should also be ported back to this assembly code.
4284	 */
4285	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3	/* g3 = kpm_size_shift */
4286	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4287	srax    %g4, %g3, %g7			/* which alias range (r) */
4288	brz,pt	%g7, 2f
4289	  sethi   %hi(vac_colors_mask), %g5
4290	ld	[%g5 + %lo(vac_colors_mask)], %g5
4291
4292	srlx	%g2, MMU_PAGESHIFT, %g1		/* vaddr >> MMU_PAGESHIFT */
4293	and	%g1, %g5, %g1			/* g1 = v */
4294	sllx	%g7, %g3, %g5			/* g5 = r << kpm_size_shift */
4295	cmp	%g7, %g1			/* if (r > v) */
4296	bleu,pn %xcc, 1f
4297	  sub   %g4, %g5, %g4			/* paddr -= r << kpm_size_shift */
4298	sub	%g7, %g1, %g5			/* g5 = r - v */
4299	sllx	%g5, MMU_PAGESHIFT, %g7		/* (r-v) << MMU_PAGESHIFT */
4300	add	%g4, %g7, %g4			/* paddr += (r-v)<<MMU_PAGESHIFT */
4301	ba	2f
4302	  nop
43031:
4304	sllx	%g7, MMU_PAGESHIFT, %g5		/* else */
4305	sub	%g4, %g5, %g4			/* paddr -= r << MMU_PAGESHIFT */
4306
4307	/*
4308	 * paddr2pfn
4309	 *  g1 = vcolor (not used)
4310	 *  g2 = tag access register
4311	 *  g3 = clobbered
4312	 *  g4 = paddr
4313	 *  g5 = clobbered
4314	 *  g6 = per-CPU kpm tsbmiss area
4315	 *  g7 = clobbered
4316	 */
43172:
4318	srlx	%g4, MMU_PAGESHIFT, %g2		/* g2 = pfn */
4319
4320	/*
4321	 * Setup %asi
4322	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
4323	 * if (mseg not found) sfmmu_kpm_exception
4324	 * g2=pfn g6=per-CPU kpm tsbmiss area
4325	 * g4 g5 g7 for scratch use.
4326	 */
4327	mov	ASI_MEM, %asi
4328	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
4329	cmp	%g3, MSEG_NULLPTR_PA
4330	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4331	  nop
4332
4333	/*
4334	 * inx = pfn - mseg_pa->kpm_pbase
4335	 * g2=pfn  g3=mseg_pa  g6=per-CPU kpm tsbmiss area
4336	 */
4337	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4338	sub	%g2, %g7, %g4
4339
4340#ifdef	DEBUG
4341	/*
4342	 * Validate inx value
4343	 * g2=pfn g3=mseg_pa g4=inx g6=per-CPU tsbmiss area
4344	 */
4345	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4346	cmp	%g4, %g5			/* inx - nkpmpgs */
4347	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4348	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4349#else
4350	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4351#endif
4352	/* ksp = &mseg_pa->kpm_spages[inx] */
4353	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
4354	add	%g5, %g4, %g5			/* ksp */
4355
4356	/*
4357	 * KPMP_SHASH(kp)
4358	 * g2=pfn g3=mseg_pa g4=inx g5=ksp
4359	 * g6=per-CPU kpm tsbmiss area  g7=kpmp_stable_sz
4360	 */
4361	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4362	sub	%g7, 1, %g7			/* mask */
4363	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
4364	add	%g5, %g1, %g5			/* y = ksp + x */
4365	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4366
4367	/*
4368	 * Calculate physical kpm_spage pointer
4369	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4370	 * g6=per-CPU kpm tsbmiss area
4371	 */
4372	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
4373	add	%g1, %g4, %g1			/* ksp_pa */
4374
4375	/*
4376	 * Calculate physical hash lock address.
4377	 * Note: Changes in kpm_shlk_t must be reflected here.
4378	 * g1=ksp_pa g2=pfn g5=hashinx
4379	 * g6=per-CPU kpm tsbmiss area
4380	 */
4381	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
4382	sllx	%g5, KPMSHLK_SHIFT, %g5
4383	add	%g4, %g5, %g3			/* hlck_pa */
4384
4385	/*
4386	 * Assemble non-cacheable tte initially
4387	 * g1=ksp_pa g2=pfn g3=hlck_pa
4388	 * g6=per-CPU kpm tsbmiss area
4389	 */
4390	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4391	sllx	%g5, 32, %g5
4392	mov	(TTE_CP_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4393	or	%g5, %g4, %g5
4394	sllx	%g2, MMU_PAGESHIFT, %g4
4395	or	%g5, %g4, %g5			/* tte */
4396	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4397	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4398
4399	/*
4400	 * tsb dropin
4401	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte (non-cacheable)
4402	 * g6=per-CPU kpm tsbmiss area  g7=scratch register
4403	 */
4404
4405	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4406	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
4407
4408	/* use C-handler if there's no go for dropin */
4409	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7	/* kp_mapped */
4410	andcc	%g7, KPM_MAPPED_GO, %g0			/* go or no go ? */
4411	bz,pt	%icc, 5f				/* no go */
4412	  nop
4413	and	%g7, KPM_MAPPED_MASK, %g7		/* go */
4414	cmp	%g7, KPM_MAPPEDS			/* cacheable ? */
4415	be,a,pn	%xcc, 3f
4416	  or	%g5, TTE_CV_INT, %g5			/* cacheable */
44173:
4418#ifndef sun4v
4419	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4420	mov	ASI_N, %g1
4421	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4422	movnz	%icc, ASI_MEM, %g1
4423	mov	%g1, %asi
4424#endif
4425
4426	/*
4427	 * TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set)
4428	 * If we fail to lock the TSB entry then just load the tte into the
4429	 * TLB.
4430	 */
4431	TSB_LOCK_ENTRY(%g4, %g1, %g7, locked_tsb_l2)
4432
4433	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4434	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4435locked_tsb_l2:
4436	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
4437
4438	/* KPMLOCK_EXIT(kpmlckp, asi) */
4439	KPMLOCK_EXIT(%g3, ASI_MEM)
4440
4441	/*
4442	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4443	 * point to trapstat's TSB miss return code (note that trapstat
4444	 * itself will patch the correct offset to add).
4445	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4446	 */
4447	rdpr	%tl, %g7
4448	cmp	%g7, 1
4449	ble	%icc, 0f
4450	sethi	%hi(KERNELBASE), %g6
4451	rdpr	%tpc, %g7
4452	or	%g6, %lo(KERNELBASE), %g6
4453	cmp	%g7, %g6
4454	bgeu	%xcc, 0f
4455	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
4456	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4457	wrpr	%g7, %tpc
4458	add	%g7, 4, %g7
4459	wrpr	%g7, %tnpc
44600:
4461	retry
44625:
4463	/* g3=hlck_pa */
4464	KPMLOCK_EXIT(%g3, ASI_MEM)
4465	ba,pt	%icc, sfmmu_kpm_exception
4466	  nop
4467	SET_SIZE(sfmmu_kpm_dtsb_miss_small)
4468
4469#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
4470#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
4471#endif
4472
4473	.seg	".data"
4474sfmmu_kpm_tsbmtl_panic:
4475	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
4476	.byte	0
4477sfmmu_kpm_stsbmtl_panic:
4478	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
4479	.byte	0
4480	.align	4
4481	.seg	".text"
4482
4483	/*
4484	 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
4485	 * Called from C-level, sets/clears "go" indication for trap level
4486	 * handler.  khl_lock is a low level spin lock to protect the kp_tsbmtl
4487	 * field.  Assumed that &kp->kp_refcntc is checked for zero or -1 at
4488	 * C-level.  Assumes khl_mutex is held when called from C-level.
4489	 */
4490	ENTRY_NP(sfmmu_kpm_tsbmtl)
4491	rdpr	%pstate, %o3
4492	/*
4493	 * %o0 = &kp_refcntc
4494	 * %o1 = &khl_lock
4495	 * %o2 = 0/1 (off/on)
4496	 * %o3 = pstate save
4497	 */
4498#ifdef DEBUG
4499	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4500	bnz,pt %icc, 1f				/* disabled, panic	 */
4501	  nop
4502	save	%sp, -SA(MINFRAME), %sp
4503	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
4504	call	panic
4505	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
4506	ret
4507	restore
45081:
4509#endif /* DEBUG */
4510	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4511
4512	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
4513	mov	-1, %o5
4514	brz,a	%o2, 2f
4515	  mov	0, %o5
45162:
4517	sth	%o5, [%o0]
4518	KPMLOCK_EXIT(%o1, ASI_N)
4519
4520	retl
4521	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4522	SET_SIZE(sfmmu_kpm_tsbmtl)
4523
4524	/*
4525	 * kpm_smallpages: stores val to byte at address mapped within low level
4526	 * lock brackets. The old value is returned.  Called from C-level.
4527	 */
4528	ENTRY_NP(sfmmu_kpm_stsbmtl)
4529	rdpr	%pstate, %o3
4530	/*
4531	 * %o0 = &mapped
4532	 * %o1 = &kshl_lock
4533	 * %o2 = val
4534	 * %o3 = pstate save
4535	 */
4536#ifdef DEBUG
4537	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4538	bnz,pt %icc, 1f				/* disabled, panic	 */
4539	  nop
4540	save	%sp, -SA(MINFRAME), %sp
4541	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
4542	call	panic
4543	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
4544	ret
4545	restore
45461:
4547#endif /* DEBUG */
4548	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4549
4550	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
4551	ldsb	[%o0], %o5
4552	stb	%o2, [%o0]
4553	KPMLOCK_EXIT(%o1, ASI_N)
4554
4555	and	%o5, KPM_MAPPED_MASK, %o0	/* return old val */
4556	retl
4557	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4558	SET_SIZE(sfmmu_kpm_stsbmtl)
4559
4560#ifdef sun4v
4561	/*
4562	 * User/kernel data miss w// multiple TSBs
4563	 * The first probe covers 8K, 64K, and 512K page sizes,
4564	 * because 64K and 512K mappings are replicated off 8K
4565	 * pointer.  Second probe covers 4M page size only.
4566	 *
4567	 * MMU fault area contains miss address and context.
4568	 */
4569	ALTENTRY(sfmmu_slow_dmmu_miss)
4570	GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)	! %g2 = ptagacc, %g3 = ctx type
4571
4572slow_miss_common:
4573	/*
4574	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
4575	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
4576	 */
4577	brnz,pt	%g3, 8f			! check for user context
4578	  nop
4579
4580	/*
4581	 * Kernel miss
4582	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
4583	 * branch to sfmmu_tsb_miss_tt to handle it.
4584	 */
4585	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4586sfmmu_dslow_patch_ktsb_base:
4587	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
4588sfmmu_dslow_patch_ktsb_szcode:
4589	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
4590
4591	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
4592	! %g1 = First TSB entry pointer, as TSB miss handler expects
4593
4594	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4595sfmmu_dslow_patch_ktsb4m_base:
4596	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
4597sfmmu_dslow_patch_ktsb4m_szcode:
4598	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
4599
4600	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
4601	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
4602	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4603	.empty
4604
46058:
4606	/*
4607	 * User miss
4608	 * Get first TSB pointer in %g1
4609	 * Get second TSB pointer (or NULL if no second TSB) in %g3
4610	 * Branch to sfmmu_tsb_miss_tt to handle it
4611	 */
4612	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
4613	/* %g1 = first TSB entry ptr now, %g2 preserved */
4614
4615	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
4616	brlz,pt %g3, sfmmu_tsb_miss_tt		/* done if no 2nd TSB */
4617	  nop
4618
4619	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
4620	/* %g3 = second TSB entry ptr now, %g2 preserved */
46219:
4622	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4623	.empty
4624	SET_SIZE(sfmmu_slow_dmmu_miss)
4625
4626
4627	/*
4628	 * User/kernel instruction miss w/ multiple TSBs
4629	 * The first probe covers 8K, 64K, and 512K page sizes,
4630	 * because 64K and 512K mappings are replicated off 8K
4631	 * pointer.  Second probe covers 4M page size only.
4632	 *
4633	 * MMU fault area contains miss address and context.
4634	 */
4635	ALTENTRY(sfmmu_slow_immu_miss)
4636	GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)
4637	ba,a,pt	%xcc, slow_miss_common
4638	SET_SIZE(sfmmu_slow_immu_miss)
4639
4640#endif /* sun4v */
4641
4642/*
4643 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
4644 */
4645	.seg	".data"
4646	.align	64
4647	.global tsbmiss_area
4648tsbmiss_area:
4649	.skip	(TSBMISS_SIZE * NCPU)
4650
4651	.align	64
4652	.global kpmtsbm_area
4653kpmtsbm_area:
4654	.skip	(KPMTSBM_SIZE * NCPU)
4655