sfmmu_asm.s revision f0856d05e8558eaf3f26c3f975cf951f09101477
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#pragma ident	"%Z%%M%	%I%	%E% SMI"
27
28/*
29 * SFMMU primitives.  These primitives should only be used by sfmmu
30 * routines.
31 */
32
33#if defined(lint)
34#include <sys/types.h>
35#else	/* lint */
36#include "assym.h"
37#endif	/* lint */
38
39#include <sys/asm_linkage.h>
40#include <sys/machtrap.h>
41#include <sys/machasi.h>
42#include <sys/sun4asi.h>
43#include <sys/pte.h>
44#include <sys/mmu.h>
45#include <vm/hat_sfmmu.h>
46#include <vm/seg_spt.h>
47#include <sys/machparam.h>
48#include <sys/privregs.h>
49#include <sys/scb.h>
50#include <sys/intreg.h>
51#include <sys/machthread.h>
52#include <sys/intr.h>
53#include <sys/clock.h>
54#include <sys/trapstat.h>
55
56#ifdef TRAPTRACE
57#include <sys/traptrace.h>
58
59/*
60 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
61 */
62#define	TT_TRACE(label)		\
63	ba	label		;\
64	rd	%pc, %g7
65#else
66
67#define	TT_TRACE(label)
68
69#endif /* TRAPTRACE */
70
71#ifndef	lint
72
73#if (TTE_SUSPEND_SHIFT > 0)
74#define	TTE_SUSPEND_INT_SHIFT(reg)				\
75	sllx	reg, TTE_SUSPEND_SHIFT, reg
76#else
77#define	TTE_SUSPEND_INT_SHIFT(reg)
78#endif
79
80#endif /* lint */
81
82#ifndef	lint
83
84/*
85 * Assumes TSBE_TAG is 0
86 * Assumes TSBE_INTHI is 0
87 * Assumes TSBREG.split is 0
88 */
89
90#if TSBE_TAG != 0
91#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
92#endif
93
94#if TSBTAG_INTHI != 0
95#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
96#endif
97
98/*
99 * The following code assumes the tsb is not split.
100 *
101 * With TSBs no longer shared between processes, it's no longer
102 * necessary to hash the context bits into the tsb index to get
103 * tsb coloring; the new implementation treats the TSB as a
104 * direct-mapped, virtually-addressed cache.
105 *
106 * In:
107 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
108 *    tsbbase = base address of TSB (clobbered)
109 *    tagacc = tag access register (clobbered)
110 *    szc = size code of TSB (ro)
111 *    tmp = scratch reg
112 * Out:
113 *    tsbbase = pointer to entry in TSB
114 */
115#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
116	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
117	srlx	tagacc, vpshift, tagacc 				;\
118	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
119	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
120	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
121	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
122	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */
123
124/*
125 * When the kpm TSB is used it is assumed that it is direct mapped
126 * using (vaddr>>vpshift)%tsbsz as the index.
127 *
128 * Note that, for now, the kpm TSB and kernel TSB are the same for
129 * each mapping size.  However that need not always be the case.  If
130 * the trap handlers are updated to search a different TSB for kpm
131 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
132 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
133 *
134 * In:
135 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
136 *    vaddr = virtual address (clobbered)
137 *    tsbp, szc, tmp = scratch
138 * Out:
139 *    tsbp = pointer to entry in TSB
140 */
141#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
142	cmp	vpshift, MMU_PAGESHIFT					;\
143	bne,pn	%icc, 1f		/* branch if large case */	;\
144	  sethi	%hi(kpmsm_tsbsz), szc					;\
145	sethi	%hi(kpmsm_tsbbase), tsbp				;\
146	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
147	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
148	ba,pt	%icc, 2f						;\
149	  nop								;\
1501:	sethi	%hi(kpm_tsbsz), szc					;\
151	sethi	%hi(kpm_tsbbase), tsbp					;\
152	ld	[szc + %lo(kpm_tsbsz)], szc				;\
153	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
1542:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
155
156/*
157 * Lock the TSBE at virtual address tsbep.
158 *
159 * tsbep = TSBE va (ro)
160 * tmp1, tmp2 = scratch registers (clobbered)
161 * label = label to use for branches (text)
162 * %asi = ASI to use for TSB access
163 *
164 * NOTE that we flush the TSB using fast VIS instructions that
165 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
166 * not be treated as a locked entry or we'll get stuck spinning on
167 * an entry that isn't locked but really invalid.
168 */
169
170#if defined(UTSB_PHYS)
171
172#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
173	lda	[tsbep]ASI_MEM, tmp1					;\
174label:									;\
175	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
176	cmp	tmp1, tmp2 						;\
177	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
178	  lda	[tsbep]ASI_MEM, tmp1					;\
179	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
180	cmp	tmp1, tmp2 						;\
181	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
182	  lda	[tsbep]ASI_MEM, tmp1					;\
183	/* tsbe lock acquired */					;\
184	membar #StoreStore
185
186#else /* UTSB_PHYS */
187
188#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
189	lda	[tsbep]%asi, tmp1					;\
190label:									;\
191	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
192	cmp	tmp1, tmp2 						;\
193	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
194	  lda	[tsbep]%asi, tmp1					;\
195	casa	[tsbep]%asi, tmp1, tmp2					;\
196	cmp	tmp1, tmp2 						;\
197	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
198	  lda	[tsbep]%asi, tmp1					;\
199	/* tsbe lock acquired */					;\
200	membar #StoreStore
201
202#endif /* UTSB_PHYS */
203
204/*
205 * Atomically write TSBE at virtual address tsbep.
206 *
207 * tsbep = TSBE va (ro)
208 * tte = TSBE TTE (ro)
209 * tagtarget = TSBE tag (ro)
210 * %asi = ASI to use for TSB access
211 */
212
213#if defined(UTSB_PHYS)
214
215#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
216	add	tsbep, TSBE_TTE, tmp1					;\
217	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
218	membar #StoreStore						;\
219	add	tsbep, TSBE_TAG, tmp1					;\
220	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */
221
222#else /* UTSB_PHYS */
223
224#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
225	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
226	membar #StoreStore						;\
227	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
228
229#endif /* UTSB_PHYS */
230
231/*
232 * Load an entry into the TSB at TL > 0.
233 *
234 * tsbep = pointer to the TSBE to load as va (ro)
235 * tte = value of the TTE retrieved and loaded (wo)
236 * tagtarget = tag target register.  To get TSBE tag to load,
237 *   we need to mask off the context and leave only the va (clobbered)
238 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
239 * tmp1, tmp2 = scratch registers
240 * label = label to use for branches (text)
241 * %asi = ASI to use for TSB access
242 */
243
244#if defined(UTSB_PHYS)
245
246#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
247	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
248	/*								;\
249	 * I don't need to update the TSB then check for the valid tte.	;\
250	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
251	 * we always invalidate the hash table before we unload the TSB.;\
252	 */								;\
253	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
254	ldxa	[ttepa]ASI_MEM, tte					;\
255	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
256	sethi	%hi(TSBTAG_INVALID), tmp2				;\
257	add	tsbep, TSBE_TAG, tmp1					;\
258	brgez,a,pn tte, label/**/f					;\
259	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
260	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
261label:
262
263#else /* UTSB_PHYS */
264
265#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
266	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
267	/*								;\
268	 * I don't need to update the TSB then check for the valid tte.	;\
269	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
270	 * we always invalidate the hash table before we unload the TSB.;\
271	 */								;\
272	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
273	ldxa	[ttepa]ASI_MEM, tte					;\
274	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
275	sethi	%hi(TSBTAG_INVALID), tmp2				;\
276	brgez,a,pn tte, label/**/f					;\
277	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
278	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
279label:
280
281#endif /* UTSB_PHYS */
282
283/*
284 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
285 *   for ITLB synthesis.
286 *
287 * tsbep = pointer to the TSBE to load as va (ro)
288 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
289 *   with exec_perm turned off and exec_synth turned on
290 * tagtarget = tag target register.  To get TSBE tag to load,
291 *   we need to mask off the context and leave only the va (clobbered)
292 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
293 * tmp1, tmp2 = scratch registers
294 * label = label to use for branch (text)
295 * %asi = ASI to use for TSB access
296 */
297
298#define	TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
299	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
300	/*								;\
301	 * I don't need to update the TSB then check for the valid tte.	;\
302	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
303	 * we always invalidate the hash table before we unload the TSB.;\
304	 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0	;\
305	 * and exec_synth bit to 1.					;\
306	 */								;\
307	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
308	mov	tte, tmp1						;\
309	ldxa	[ttepa]ASI_MEM, tte					;\
310	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
311	sethi	%hi(TSBTAG_INVALID), tmp2				;\
312	brgez,a,pn tte, label/**/f					;\
313	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
314	or	tte, tmp1, tte						;\
315	andn	tte, TTE_EXECPRM_INT, tte				;\
316	or	tte, TTE_E_SYNTH_INT, tte				;\
317	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
318label:
319
320/*
321 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
322 *
323 * tte = value of the TTE, used to get tte_size bits (ro)
324 * tagaccess = tag access register, used to get 4M pfn bits (ro)
325 * pfn = 4M pfn bits shifted to offset for tte (out)
326 * tmp1 = scratch register
327 * label = label to use for branch (text)
328 */
329
330#define	GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)			\
331	/*								;\
332	 * Get 4M bits from tagaccess for 32M, 256M pagesizes.		;\
333	 * Return them, shifted, in pfn.				;\
334	 */								;\
335	srlx	tagaccess, MMU_PAGESHIFT4M, tagaccess			;\
336	srlx	tte, TTE_SZ_SHFT, tmp		/* isolate the */	;\
337	andcc	tmp, TTE_SZ_BITS, %g0		/* tte_size bits */	;\
338	bz,a,pt %icc, label/**/f		/* if 0, is */		;\
339	  and	tagaccess, 0x7, tagaccess	/* 32M page size */	;\
340	and	tagaccess, 0x3f, tagaccess /* else 256M page size */	;\
341label:									;\
342	sllx	tagaccess, MMU_PAGESHIFT4M, pfn
343
344/*
345 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
346 * for ITLB synthesis.
347 *
348 * tte = value of the TTE, used to get tte_size bits (rw)
349 * tmp1 = scratch register
350 */
351
352#define	SET_TTE4M_PN(tte, tmp)						\
353	/*								;\
354	 * Set 4M pagesize tte bits. 					;\
355	 */								;\
356	set	TTE4M, tmp						;\
357	sllx	tmp, TTE_SZ_SHFT, tmp					;\
358	or	tte, tmp, tte
359
360/*
361 * Load an entry into the TSB at TL=0.
362 *
363 * tsbep = pointer to the TSBE to load as va (ro)
364 * tteva = pointer to the TTE to load as va (ro)
365 * tagtarget = TSBE tag to load (which contains no context), synthesized
366 * to match va of MMU tag target register only (ro)
367 * tmp1, tmp2 = scratch registers (clobbered)
368 * label = label to use for branches (text)
369 * %asi = ASI to use for TSB access
370 */
371
372#if defined(UTSB_PHYS)
373
374#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
375	/* can't rd tteva after locking tsb because it can tlb miss */	;\
376	ldx	[tteva], tteva			/* load tte */		;\
377	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
378	sethi	%hi(TSBTAG_INVALID), tmp2				;\
379	add	tsbep, TSBE_TAG, tmp1					;\
380	brgez,a,pn tteva, label/**/f					;\
381	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
382	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
383label:
384
385#else /* UTSB_PHYS */
386
387#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
388	/* can't rd tteva after locking tsb because it can tlb miss */	;\
389	ldx	[tteva], tteva			/* load tte */		;\
390	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
391	sethi	%hi(TSBTAG_INVALID), tmp2				;\
392	brgez,a,pn tteva, label/**/f					;\
393	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
394	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
395label:
396
397#endif /* UTSB_PHYS */
398
399/*
400 * Invalidate a TSB entry in the TSB.
401 *
402 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
403 *	 about this earlier to ensure this is true.  Thus when we are
404 *	 directly referencing tsbep below, we are referencing the tte_tag
405 *	 field of the TSBE.  If this  offset ever changes, the code below
406 *	 will need to be modified.
407 *
408 * tsbep = pointer to TSBE as va (ro)
409 * tag = invalidation is done if this matches the TSBE tag (ro)
410 * tmp1 - tmp3 = scratch registers (clobbered)
411 * label = label name to use for branches (text)
412 * %asi = ASI to use for TSB access
413 */
414
415#if defined(UTSB_PHYS)
416
417#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
418	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
419	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
420label/**/1:								;\
421	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
422	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
423	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
424	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
425	cmp	tag, tmp3		/* compare tags */		;\
426	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
427	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
428	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
429	cmp	tmp1, tmp3		/* if not successful */		;\
430	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
431	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
432label/**/2:
433
434#else /* UTSB_PHYS */
435
436#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
437	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
438	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
439label/**/1:								;\
440	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
441	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
442	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
443	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
444	cmp	tag, tmp3		/* compare tags */		;\
445	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
446	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
447	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
448	cmp	tmp1, tmp3		/* if not successful */		;\
449	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
450	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
451label/**/2:
452
453#endif /* UTSB_PHYS */
454
455#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
456#error	- TSB_SOFTSZ_MASK too small
457#endif
458
459
460/*
461 * An implementation of setx which will be hot patched at run time.
462 * since it is being hot patched, there is no value passed in.
463 * Thus, essentially we are implementing
464 *	setx value, tmp, dest
465 * where value is RUNTIME_PATCH (aka 0) in this case.
466 */
467#define	RUNTIME_PATCH_SETX(dest, tmp)					\
468	sethi	%hh(RUNTIME_PATCH), tmp					;\
469	sethi	%lm(RUNTIME_PATCH), dest				;\
470	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
471	or	dest, %lo(RUNTIME_PATCH), dest				;\
472	sllx	tmp, 32, tmp						;\
473	nop				/* for perf reasons */		;\
474	or	tmp, dest, dest		/* contents of patched value */
475
476
477#endif (lint)
478
479
480#if defined (lint)
481
482/*
483 * sfmmu related subroutines
484 */
485uint_t
486sfmmu_disable_intrs()
487{ return(0); }
488
489/* ARGSUSED */
490void
491sfmmu_enable_intrs(uint_t pstate_save)
492{}
493
494/* ARGSUSED */
495int
496sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag)
497{ return(0); }
498
499/*
500 * Use cas, if tte has changed underneath us then reread and try again.
501 * In the case of a retry, it will update sttep with the new original.
502 */
503/* ARGSUSED */
504int
505sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
506{ return(0); }
507
508/*
509 * Use cas, if tte has changed underneath us then return 1, else return 0
510 */
511/* ARGSUSED */
512int
513sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
514{ return(0); }
515
516/* ARGSUSED */
517void
518sfmmu_copytte(tte_t *sttep, tte_t *dttep)
519{}
520
521/*ARGSUSED*/
522struct tsbe *
523sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
524{ return(0); }
525
526/*ARGSUSED*/
527uint64_t
528sfmmu_make_tsbtag(caddr_t va)
529{ return(0); }
530
531#else	/* lint */
532
533	.seg	".data"
534	.global	sfmmu_panic1
535sfmmu_panic1:
536	.asciz	"sfmmu_asm: interrupts already disabled"
537
538	.global	sfmmu_panic3
539sfmmu_panic3:
540	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"
541
542	.global	sfmmu_panic4
543sfmmu_panic4:
544	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"
545
546	.global	sfmmu_panic5
547sfmmu_panic5:
548	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"
549
550	.global	sfmmu_panic6
551sfmmu_panic6:
552	.asciz	"sfmmu_asm: interrupts not disabled"
553
554	.global	sfmmu_panic7
555sfmmu_panic7:
556	.asciz	"sfmmu_asm: kernel as"
557
558	.global	sfmmu_panic8
559sfmmu_panic8:
560	.asciz	"sfmmu_asm: gnum is zero"
561
562	.global	sfmmu_panic9
563sfmmu_panic9:
564	.asciz	"sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL"
565
566	.global	sfmmu_panic10
567sfmmu_panic10:
568	.asciz	"sfmmu_asm: valid SCD with no 3rd scd TSB"
569
570        ENTRY(sfmmu_disable_intrs)
571        rdpr    %pstate, %o0
572#ifdef DEBUG
573	PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1)
574#endif /* DEBUG */
575        retl
576          wrpr   %o0, PSTATE_IE, %pstate
577        SET_SIZE(sfmmu_disable_intrs)
578
579	ENTRY(sfmmu_enable_intrs)
580        retl
581          wrpr    %g0, %o0, %pstate
582        SET_SIZE(sfmmu_enable_intrs)
583
584/*
585 * This routine is called both by resume() and sfmmu_get_ctx() to
586 * allocate a new context for the process on a MMU.
587 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID .
588 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which
589 * is the case when sfmmu_alloc_ctx is called from resume().
590 *
591 * The caller must disable interrupts before entering this routine.
592 * To reduce ctx switch overhead, the code contains both 'fast path' and
593 * 'slow path' code. The fast path code covers the common case where only
594 * a quick check is needed and the real ctx allocation is not required.
595 * It can be done without holding the per-process (PP) lock.
596 * The 'slow path' code must be protected by the PP Lock and performs ctx
597 * allocation.
598 * Hardware context register and HAT mmu cnum are updated accordingly.
599 *
600 * %o0 - sfmmup
601 * %o1 - allocflag
602 * %o2 - CPU
603 * %o3 - sfmmu private/shared flag
604 *
605 * ret - 0: no ctx is allocated
606 *       1: a ctx is allocated
607 */
608        ENTRY_NP(sfmmu_alloc_ctx)
609
610#ifdef DEBUG
611	sethi   %hi(ksfmmup), %g1
612	ldx     [%g1 + %lo(ksfmmup)], %g1
613	cmp     %g1, %o0
614	bne,pt   %xcc, 0f
615	  nop
616
617	sethi   %hi(panicstr), %g1		! if kernel as, panic
618        ldx     [%g1 + %lo(panicstr)], %g1
619        tst     %g1
620        bnz,pn  %icc, 7f
621          nop
622
623	sethi	%hi(sfmmu_panic7), %o0
624	call	panic
625	  or	%o0, %lo(sfmmu_panic7), %o0
626
6277:
628	retl
629	  mov	%g0, %o0			! %o0 = ret = 0
630
6310:
632	PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1)
633#endif /* DEBUG */
634
635	mov	%o3, %g1			! save sfmmu pri/sh flag in %g1
636
637	! load global mmu_ctxp info
638	ldx	[%o2 + CPU_MMU_CTXP], %o3		! %o3 = mmu_ctx_t ptr
639        lduw	[%o2 + CPU_MMU_IDX], %g2		! %g2 = mmu index
640
641	! load global mmu_ctxp gnum
642	ldx	[%o3 + MMU_CTX_GNUM], %o4		! %o4 = mmu_ctxp->gnum
643
644#ifdef DEBUG
645	cmp	%o4, %g0		! mmu_ctxp->gnum should never be 0
646	bne,pt	%xcc, 3f
647	  nop
648
649	sethi   %hi(panicstr), %g1	! test if panicstr is already set
650        ldx     [%g1 + %lo(panicstr)], %g1
651        tst     %g1
652        bnz,pn  %icc, 1f
653          nop
654
655	sethi	%hi(sfmmu_panic8), %o0
656	call	panic
657	  or	%o0, %lo(sfmmu_panic8), %o0
6581:
659	retl
660	  mov	%g0, %o0			! %o0 = ret = 0
6613:
662#endif
663
664	! load HAT sfmmu_ctxs[mmuid] gnum, cnum
665
666	sllx	%g2, SFMMU_MMU_CTX_SHIFT, %g2
667	add	%o0, %g2, %g2		! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
668
669	/*
670	 * %g5 = sfmmu gnum returned
671	 * %g6 = sfmmu cnum returned
672	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
673	 * %g4 = scratch
674	 *
675	 * Fast path code, do a quick check.
676	 */
677	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
678
679	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
680	bne,pt	%icc, 1f			! valid hat cnum, check gnum
681	  nop
682
683	! cnum == INVALID, check allocflag
684	mov	%g0, %g4	! %g4 = ret = 0
685	brz,pt  %o1, 8f		! allocflag == 0, skip ctx allocation, bail
686	  mov	%g6, %o1
687
688	! (invalid HAT cnum) && (allocflag == 1)
689	ba,pt	%icc, 2f
690	  nop
6911:
692	! valid HAT cnum, check gnum
693	cmp	%g5, %o4
694	mov	1, %g4				!%g4 = ret = 1
695	be,a,pt	%icc, 8f			! gnum unchanged, go to done
696	  mov	%g6, %o1
697
6982:
699	/*
700	 * Grab per process (PP) sfmmu_ctx_lock spinlock,
701	 * followed by the 'slow path' code.
702	 */
703	ldstub	[%o0 + SFMMU_CTX_LOCK], %g3	! %g3 = per process (PP) lock
7043:
705	brz	%g3, 5f
706	  nop
7074:
708	brnz,a,pt       %g3, 4b				! spin if lock is 1
709	  ldub	[%o0 + SFMMU_CTX_LOCK], %g3
710	ba	%xcc, 3b				! retry the lock
711	  ldstub	[%o0 + SFMMU_CTX_LOCK], %g3    ! %g3 = PP lock
712
7135:
714	membar  #LoadLoad
715	/*
716	 * %g5 = sfmmu gnum returned
717	 * %g6 = sfmmu cnum returned
718	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
719	 * %g4 = scratch
720	 */
721	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
722
723	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
724	bne,pt	%icc, 1f			! valid hat cnum, check gnum
725	  nop
726
727	! cnum == INVALID, check allocflag
728	mov	%g0, %g4	! %g4 = ret = 0
729	brz,pt	%o1, 2f		! allocflag == 0, called from resume, set hw
730	  mov	%g6, %o1
731
732	! (invalid HAT cnum) && (allocflag == 1)
733	ba,pt	%icc, 6f
734	  nop
7351:
736	! valid HAT cnum, check gnum
737	cmp	%g5, %o4
738	mov	1, %g4				! %g4 = ret  = 1
739	be,a,pt	%icc, 2f			! gnum unchanged, go to done
740	  mov	%g6, %o1
741
742	ba,pt	%icc, 6f
743	  nop
7442:
745	membar  #LoadStore|#StoreStore
746	ba,pt %icc, 8f
747	  clrb  [%o0 + SFMMU_CTX_LOCK]
7486:
749	/*
750	 * We get here if we do not have a valid context, or
751	 * the HAT gnum does not match global gnum. We hold
752	 * sfmmu_ctx_lock spinlock. Allocate that context.
753	 *
754	 * %o3 = mmu_ctxp
755	 */
756	add	%o3, MMU_CTX_CNUM, %g3
757	ld	[%o3 + MMU_CTX_NCTXS], %g4
758
759	/*
760         * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS;
761         * %g3 = mmu cnum address
762	 * %g4 = mmu nctxs
763	 *
764	 * %o0 = sfmmup
765	 * %o1 = mmu current cnum value (used as new cnum)
766	 * %o4 = mmu gnum
767	 *
768	 * %o5 = scratch
769	 */
770	ld	[%g3], %o1
7710:
772	cmp	%o1, %g4
773	bl,a,pt %icc, 1f
774	  add	%o1, 1, %o5		! %o5 = mmu_ctxp->cnum + 1
775
776	/*
777	 * cnum reachs max, bail, so wrap around can be performed later.
778	 */
779	set	INVALID_CONTEXT, %o1
780	mov	%g0, %g4		! %g4 = ret = 0
781
782	membar  #LoadStore|#StoreStore
783	ba,pt	%icc, 8f
784	  clrb	[%o0 + SFMMU_CTX_LOCK]
7851:
786	! %g3 = addr of mmu_ctxp->cnum
787	! %o5 = mmu_ctxp->cnum + 1
788	cas	[%g3], %o1, %o5
789	cmp	%o1, %o5
790	bne,a,pn %xcc, 0b	! cas failed
791	  ld	[%g3], %o1
792
793#ifdef DEBUG
794        set	MAX_SFMMU_CTX_VAL, %o5
795	cmp	%o1, %o5
796	ble,pt %icc, 2f
797	  nop
798
799	sethi	%hi(sfmmu_panic9), %o0
800	call	panic
801	  or	%o0, %lo(sfmmu_panic9), %o0
8022:
803#endif
804	! update hat gnum and cnum
805	sllx	%o4, SFMMU_MMU_GNUM_RSHIFT, %o4
806	or	%o4, %o1, %o4
807	stx	%o4, [%g2 + SFMMU_CTXS]
808
809	membar  #LoadStore|#StoreStore
810	clrb	[%o0 + SFMMU_CTX_LOCK]
811
812	mov	1, %g4			! %g4 = ret = 1
8138:
814	/*
815	 * program the secondary context register
816	 *
817	 * %o1 = cnum
818	 * %g1 = sfmmu private/shared flag (0:private,  1:shared)
819	 */
820
821	/*
822	 * When we come here and context is invalid, we want to set both
823	 * private and shared ctx regs to INVALID. In order to
824	 * do so, we set the sfmmu priv/shared flag to 'private' regardless
825	 * so that private ctx reg will be set to invalid.
826	 * Note that on sun4v values written to private context register are
827	 * automatically written to corresponding shared context register as
828	 * well. On sun4u SET_SECCTX() will invalidate shared context register
829	 * when it sets a private secondary context register.
830	 */
831
832	cmp	%o1, INVALID_CONTEXT
833	be,a,pn	%icc, 9f
834	  clr	%g1
8359:
836
837#ifdef	sun4u
838	ldub	[%o0 + SFMMU_CEXT], %o2
839	sll	%o2, CTXREG_EXT_SHIFT, %o2
840	or	%o1, %o2, %o1
841#endif /* sun4u */
842
843	SET_SECCTX(%o1, %g1, %o4, %o5, alloc_ctx_lbl1)
844
845        retl
846          mov   %g4, %o0                        ! %o0 = ret
847
848	SET_SIZE(sfmmu_alloc_ctx)
849
850	ENTRY_NP(sfmmu_modifytte)
851	ldx	[%o2], %g3			/* current */
852	ldx	[%o0], %g1			/* original */
8532:
854	ldx	[%o1], %g2			/* modified */
855	cmp	%g2, %g3			/* is modified = current? */
856	be,a,pt	%xcc,1f				/* yes, don't write */
857	stx	%g3, [%o0]			/* update new original */
858	casx	[%o2], %g1, %g2
859	cmp	%g1, %g2
860	be,pt	%xcc, 1f			/* cas succeeded - return */
861	  nop
862	ldx	[%o2], %g3			/* new current */
863	stx	%g3, [%o0]			/* save as new original */
864	ba,pt	%xcc, 2b
865	  mov	%g3, %g1
8661:	retl
867	membar	#StoreLoad
868	SET_SIZE(sfmmu_modifytte)
869
870	ENTRY_NP(sfmmu_modifytte_try)
871	ldx	[%o1], %g2			/* modified */
872	ldx	[%o2], %g3			/* current */
873	ldx	[%o0], %g1			/* original */
874	cmp	%g3, %g2			/* is modified = current? */
875	be,a,pn %xcc,1f				/* yes, don't write */
876	mov	0, %o1				/* as if cas failed. */
877
878	casx	[%o2], %g1, %g2
879	membar	#StoreLoad
880	cmp	%g1, %g2
881	movne	%xcc, -1, %o1			/* cas failed. */
882	move	%xcc, 1, %o1			/* cas succeeded. */
8831:
884	stx	%g2, [%o0]			/* report "current" value */
885	retl
886	mov	%o1, %o0
887	SET_SIZE(sfmmu_modifytte_try)
888
889	ENTRY_NP(sfmmu_copytte)
890	ldx	[%o0], %g1
891	retl
892	stx	%g1, [%o1]
893	SET_SIZE(sfmmu_copytte)
894
895
896	/*
897	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
898	 * %o0 = TSB base address (in), pointer to TSB entry (out)
899	 * %o1 = vaddr (in)
900	 * %o2 = vpshift (in)
901	 * %o3 = tsb size code (in)
902	 * %o4 = scratch register
903	 */
904	ENTRY_NP(sfmmu_get_tsbe)
905	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
906	retl
907	nop
908	SET_SIZE(sfmmu_get_tsbe)
909
910	/*
911	 * Return a TSB tag for the given va.
912	 * %o0 = va (in/clobbered)
913	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
914	 */
915	ENTRY_NP(sfmmu_make_tsbtag)
916	retl
917	srln	%o0, TTARGET_VA_SHIFT, %o0
918	SET_SIZE(sfmmu_make_tsbtag)
919
920#endif /* lint */
921
922/*
923 * Other sfmmu primitives
924 */
925
926
927#if defined (lint)
928void
929sfmmu_patch_ktsb(void)
930{
931}
932
933void
934sfmmu_kpm_patch_tlbm(void)
935{
936}
937
938void
939sfmmu_kpm_patch_tsbm(void)
940{
941}
942
943void
944sfmmu_patch_shctx(void)
945{
946}
947
948/* ARGSUSED */
949void
950sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
951{
952}
953
954/* ARGSUSED */
955void
956sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
957{
958}
959
960/* ARGSUSED */
961void
962sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
963{
964}
965
966/* ARGSUSED */
967void
968sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
969{
970}
971
972#else /* lint */
973
974#define	I_SIZE		4
975
976	ENTRY_NP(sfmmu_fix_ktlb_traptable)
977	/*
978	 * %o0 = start of patch area
979	 * %o1 = size code of TSB to patch
980	 * %o3 = scratch
981	 */
982	/* fix sll */
983	ld	[%o0], %o3			/* get sll */
984	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
985	st	%o3, [%o0]			/* write sll */
986	flush	%o0
987	/* fix srl */
988	add	%o0, I_SIZE, %o0		/* goto next instr. */
989	ld	[%o0], %o3			/* get srl */
990	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
991	st	%o3, [%o0]			/* write srl */
992	retl
993	flush	%o0
994	SET_SIZE(sfmmu_fix_ktlb_traptable)
995
996	ENTRY_NP(sfmmu_fixup_ktsbbase)
997	/*
998	 * %o0 = start of patch area
999	 * %o5 = kernel virtual or physical tsb base address
1000	 * %o2, %o3 are used as scratch registers.
1001	 */
1002	/* fixup sethi instruction */
1003	ld	[%o0], %o3
1004	srl	%o5, 10, %o2			! offset is bits 32:10
1005	or	%o3, %o2, %o3			! set imm22
1006	st	%o3, [%o0]
1007	/* fixup offset of lduw/ldx */
1008	add	%o0, I_SIZE, %o0		! next instr
1009	ld	[%o0], %o3
1010	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
1011	or	%o3, %o2, %o3
1012	st	%o3, [%o0]
1013	retl
1014	flush	%o0
1015	SET_SIZE(sfmmu_fixup_ktsbbase)
1016
1017	ENTRY_NP(sfmmu_fixup_setx)
1018	/*
1019	 * %o0 = start of patch area
1020	 * %o4 = 64 bit value to patch
1021	 * %o2, %o3 are used as scratch registers.
1022	 *
1023	 * Note: Assuming that all parts of the instructions which need to be
1024	 *	 patched correspond to RUNTIME_PATCH (aka 0)
1025	 *
1026	 * Note the implementation of setx which is being patched is as follows:
1027	 *
1028	 * sethi   %hh(RUNTIME_PATCH), tmp
1029	 * sethi   %lm(RUNTIME_PATCH), dest
1030	 * or      tmp, %hm(RUNTIME_PATCH), tmp
1031	 * or      dest, %lo(RUNTIME_PATCH), dest
1032	 * sllx    tmp, 32, tmp
1033	 * nop
1034	 * or      tmp, dest, dest
1035	 *
1036	 * which differs from the implementation in the
1037	 * "SPARC Architecture Manual"
1038	 */
1039	/* fixup sethi instruction */
1040	ld	[%o0], %o3
1041	srlx	%o4, 42, %o2			! bits [63:42]
1042	or	%o3, %o2, %o3			! set imm22
1043	st	%o3, [%o0]
1044	/* fixup sethi instruction */
1045	add	%o0, I_SIZE, %o0		! next instr
1046	ld	[%o0], %o3
1047	sllx	%o4, 32, %o2			! clear upper bits
1048	srlx	%o2, 42, %o2			! bits [31:10]
1049	or	%o3, %o2, %o3			! set imm22
1050	st	%o3, [%o0]
1051	/* fixup or instruction */
1052	add	%o0, I_SIZE, %o0		! next instr
1053	ld	[%o0], %o3
1054	srlx	%o4, 32, %o2			! bits [63:32]
1055	and	%o2, 0x3ff, %o2			! bits [41:32]
1056	or	%o3, %o2, %o3			! set imm
1057	st	%o3, [%o0]
1058	/* fixup or instruction */
1059	add	%o0, I_SIZE, %o0		! next instr
1060	ld	[%o0], %o3
1061	and	%o4, 0x3ff, %o2			! bits [9:0]
1062	or	%o3, %o2, %o3			! set imm
1063	st	%o3, [%o0]
1064	retl
1065	flush	%o0
1066	SET_SIZE(sfmmu_fixup_setx)
1067
1068	ENTRY_NP(sfmmu_fixup_or)
1069	/*
1070	 * %o0 = start of patch area
1071	 * %o4 = 32 bit value to patch
1072	 * %o2, %o3 are used as scratch registers.
1073	 * Note: Assuming that all parts of the instructions which need to be
1074	 *	 patched correspond to RUNTIME_PATCH (aka 0)
1075	 */
1076	ld	[%o0], %o3
1077	and	%o4, 0x3ff, %o2			! bits [9:0]
1078	or	%o3, %o2, %o3			! set imm
1079	st	%o3, [%o0]
1080	retl
1081	flush	%o0
1082	SET_SIZE(sfmmu_fixup_or)
1083
1084	ENTRY_NP(sfmmu_fixup_shiftx)
1085	/*
1086	 * %o0 = start of patch area
1087	 * %o4 = signed int immediate value to add to sllx/srlx imm field
1088	 * %o2, %o3 are used as scratch registers.
1089	 *
1090	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
1091	 * so we do a simple add.  The caller must be careful to prevent
1092	 * overflow, which could easily occur if the initial value is nonzero!
1093	 */
1094	ld	[%o0], %o3			! %o3 = instruction to patch
1095	and	%o3, 0x3f, %o2			! %o2 = existing imm value
1096	add	%o2, %o4, %o2			! %o2 = new imm value
1097	andn	%o3, 0x3f, %o3			! clear old imm value
1098	and	%o2, 0x3f, %o2			! truncate new imm value
1099	or	%o3, %o2, %o3			! set new imm value
1100	st	%o3, [%o0]			! store updated instruction
1101	retl
1102	flush	%o0
1103	SET_SIZE(sfmmu_fixup_shiftx)
1104
1105	ENTRY_NP(sfmmu_fixup_mmu_asi)
1106	/*
1107	 * Patch imm_asi of all ldda instructions in the MMU
1108	 * trap handlers.  We search MMU_PATCH_INSTR instructions
1109	 * starting from the itlb miss handler (trap 0x64).
1110	 * %o0 = address of tt[0,1]_itlbmiss
1111	 * %o1 = imm_asi to setup, shifted by appropriate offset.
1112	 * %o3 = number of instructions to search
1113	 * %o4 = reserved by caller: called from leaf routine
1114	 */
11151:	ldsw	[%o0], %o2			! load instruction to %o2
1116	brgez,pt %o2, 2f
1117	  srl	%o2, 30, %o5
1118	btst	1, %o5				! test bit 30; skip if not set
1119	bz,pt	%icc, 2f
1120	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
1121	srlx	%o5, 58, %o5			! isolate op3 part of opcode
1122	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
1123	brnz,pt	%o5, 2f				! skip if not a match
1124	  or	%o2, %o1, %o2			! or in imm_asi
1125	st	%o2, [%o0]			! write patched instruction
11262:	dec	%o3
1127	brnz,a,pt %o3, 1b			! loop until we're done
1128	  add	%o0, I_SIZE, %o0
1129	retl
1130	flush	%o0
1131	SET_SIZE(sfmmu_fixup_mmu_asi)
1132
1133	/*
1134	 * Patch immediate ASI used to access the TSB in the
1135	 * trap table.
1136	 * inputs: %o0 = value of ktsb_phys
1137	 */
1138	ENTRY_NP(sfmmu_patch_mmu_asi)
1139	mov	%o7, %o4			! save return pc in %o4
1140	movrnz	%o0, ASI_QUAD_LDD_PHYS, %o3
1141	movrz	%o0, ASI_NQUAD_LD, %o3
1142	sll	%o3, 5, %o1			! imm_asi offset
1143	mov	6, %o3				! number of instructions
1144	sethi	%hi(dktsb), %o0			! to search
1145	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
1146	  or	%o0, %lo(dktsb), %o0
1147	mov	6, %o3				! number of instructions
1148	sethi	%hi(dktsb4m), %o0		! to search
1149	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
1150	  or	%o0, %lo(dktsb4m), %o0
1151	mov	6, %o3				! number of instructions
1152	sethi	%hi(iktsb), %o0			! to search
1153	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
1154	  or	%o0, %lo(iktsb), %o0
1155	mov	6, %o3				! number of instructions
1156	sethi	%hi(iktsb4m), %o0		! to search
1157	call	sfmmu_fixup_mmu_asi		! patch kitlb4m miss
1158	  or	%o0, %lo(iktsb4m), %o0
1159	mov	%o4, %o7			! retore return pc -- leaf
1160	retl
1161	nop
1162	SET_SIZE(sfmmu_patch_mmu_asi)
1163
1164	ENTRY_NP(sfmmu_patch_ktsb)
1165	/*
1166	 * We need to fix iktsb, dktsb, et. al.
1167	 */
1168	save	%sp, -SA(MINFRAME), %sp
1169	set	ktsb_phys, %o1
1170	ld	[%o1], %o4
1171	set	ktsb_base, %o5
1172	set	ktsb4m_base, %l1
1173	brz,pt	%o4, 1f
1174	  nop
1175	set	ktsb_pbase, %o5
1176	set	ktsb4m_pbase, %l1
11771:
1178	sethi	%hi(ktsb_szcode), %o1
1179	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */
1180
1181	sethi	%hi(iktsb), %o0
1182	call	sfmmu_fix_ktlb_traptable
1183	  or	%o0, %lo(iktsb), %o0
1184
1185	sethi	%hi(dktsb), %o0
1186	call	sfmmu_fix_ktlb_traptable
1187	  or	%o0, %lo(dktsb), %o0
1188
1189	sethi	%hi(ktsb4m_szcode), %o1
1190	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */
1191
1192	sethi	%hi(iktsb4m), %o0
1193	call	sfmmu_fix_ktlb_traptable
1194	  or	%o0, %lo(iktsb4m), %o0
1195
1196	sethi	%hi(dktsb4m), %o0
1197	call	sfmmu_fix_ktlb_traptable
1198	  or	%o0, %lo(dktsb4m), %o0
1199
1200#ifndef sun4v
1201	mov	ASI_N, %o2
1202	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
1203	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
1204	sethi	%hi(tsb_kernel_patch_asi), %o0
1205	call	sfmmu_fixup_or
1206	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
1207#endif /* !sun4v */
1208
1209	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)
1210
1211	sethi	%hi(dktsbbase), %o0
1212	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1213	  or	%o0, %lo(dktsbbase), %o0
1214
1215	sethi	%hi(iktsbbase), %o0
1216	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1217	  or	%o0, %lo(iktsbbase), %o0
1218
1219	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
1220	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1221	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
1222
1223#ifdef sun4v
1224	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
1225	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1226	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
1227#endif /* sun4v */
1228
1229	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)
1230
1231	sethi	%hi(dktsb4mbase), %o0
1232	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1233	  or	%o0, %lo(dktsb4mbase), %o0
1234
1235	sethi	%hi(iktsb4mbase), %o0
1236	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1237	  or	%o0, %lo(iktsb4mbase), %o0
1238
1239	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
1240	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1241	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
1242
1243#ifdef sun4v
1244	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
1245	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1246	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
1247#endif /* sun4v */
1248
1249	set	ktsb_szcode, %o4
1250	ld	[%o4], %o4
1251	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
1252	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1253	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
1254
1255#ifdef sun4v
1256	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
1257	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1258	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
1259#endif /* sun4v */
1260
1261	set	ktsb4m_szcode, %o4
1262	ld	[%o4], %o4
1263	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1264	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1265	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1266
1267#ifdef sun4v
1268	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1269	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1270	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1271#endif /* sun4v */
1272
1273	ret
1274	restore
1275	SET_SIZE(sfmmu_patch_ktsb)
1276
1277	ENTRY_NP(sfmmu_kpm_patch_tlbm)
1278	/*
1279	 * Fixup trap handlers in common segkpm case.  This is reserved
1280	 * for future use should kpm TSB be changed to be other than the
1281	 * kernel TSB.
1282	 */
1283	retl
1284	nop
1285	SET_SIZE(sfmmu_kpm_patch_tlbm)
1286
1287	ENTRY_NP(sfmmu_kpm_patch_tsbm)
1288	/*
1289	 * nop the branch to sfmmu_kpm_dtsb_miss_small
1290	 * in the case where we are using large pages for
1291	 * seg_kpm (and hence must probe the second TSB for
1292	 * seg_kpm VAs)
1293	 */
1294	set	dktsb4m_kpmcheck_small, %o0
1295	MAKE_NOP_INSTR(%o1)
1296	st	%o1, [%o0]
1297	flush	%o0
1298	retl
1299	nop
1300	SET_SIZE(sfmmu_kpm_patch_tsbm)
1301
1302	ENTRY_NP(sfmmu_patch_utsb)
1303#ifdef UTSB_PHYS
1304	retl
1305	nop
1306#else /* UTSB_PHYS */
1307	/*
1308	 * We need to hot patch utsb_vabase and utsb4m_vabase
1309	 */
1310	save	%sp, -SA(MINFRAME), %sp
1311
1312	/* patch value of utsb_vabase */
1313	set	utsb_vabase, %o1
1314	ldx	[%o1], %o4
1315	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1316	call	sfmmu_fixup_setx
1317	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1318	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1319	call	sfmmu_fixup_setx
1320	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1321	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1322	call	sfmmu_fixup_setx
1323	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1324
1325	/* patch value of utsb4m_vabase */
1326	set	utsb4m_vabase, %o1
1327	ldx	[%o1], %o4
1328	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1329	call	sfmmu_fixup_setx
1330	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1331	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1332	call	sfmmu_fixup_setx
1333	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1334	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1335	call	sfmmu_fixup_setx
1336	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1337
1338	/*
1339	 * Patch TSB base register masks and shifts if needed.
1340	 * By default the TSB base register contents are set up for 4M slab.
1341	 * If we're using a smaller slab size and reserved VA range we need
1342	 * to patch up those values here.
1343	 */
1344	set	tsb_slab_shift, %o1
1345	set	MMU_PAGESHIFT4M, %o4
1346	lduw	[%o1], %o3
1347	subcc	%o4, %o3, %o4
1348	bz,pt	%icc, 1f
1349	  /* delay slot safe */
1350
1351	/* patch reserved VA range size if needed. */
1352	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
1353	call	sfmmu_fixup_shiftx
1354	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1355	call	sfmmu_fixup_shiftx
1356	  add	%o0, I_SIZE, %o0
1357	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
1358	call	sfmmu_fixup_shiftx
1359	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1360	call	sfmmu_fixup_shiftx
1361	  add	%o0, I_SIZE, %o0
13621:
1363	/* patch TSBREG_VAMASK used to set up TSB base register */
1364	set	tsb_slab_mask, %o1
1365	ldx	[%o1], %o4
1366	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1367	call	sfmmu_fixup_or
1368	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1369	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1370	call	sfmmu_fixup_or
1371	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1372
1373	ret
1374	restore
1375#endif /* UTSB_PHYS */
1376	SET_SIZE(sfmmu_patch_utsb)
1377
1378	ENTRY_NP(sfmmu_patch_shctx)
1379#ifdef sun4u
1380	retl
1381	  nop
1382#else /* sun4u */
1383	set	sfmmu_shctx_cpu_mondo_patch, %o0
1384	MAKE_JMP_INSTR(5, %o1, %o2)	! jmp       %g5
1385	st	%o1, [%o0]
1386	flush	%o0
1387	MAKE_NOP_INSTR(%o1)
1388	add	%o0, I_SIZE, %o0	! next instr
1389	st	%o1, [%o0]
1390	flush	%o0
1391
1392	set	sfmmu_shctx_user_rtt_patch, %o0
1393	st      %o1, [%o0]		! nop 1st instruction
1394	flush	%o0
1395	add     %o0, I_SIZE, %o0
1396	st      %o1, [%o0]		! nop 2nd instruction
1397	flush	%o0
1398	add     %o0, I_SIZE, %o0
1399	st      %o1, [%o0]		! nop 3rd instruction
1400	flush	%o0
1401	add     %o0, I_SIZE, %o0
1402	st      %o1, [%o0]		! nop 4th instruction
1403	flush	%o0
1404	add     %o0, I_SIZE, %o0
1405	st      %o1, [%o0]		! nop 5th instruction
1406	retl
1407	  flush	%o0
1408#endif /* sun4u */
1409	SET_SIZE(sfmmu_patch_shctx)
1410
1411	/*
1412	 * Routine that loads an entry into a tsb using virtual addresses.
1413	 * Locking is required since all cpus can use the same TSB.
1414	 * Note that it is no longer required to have a valid context
1415	 * when calling this function.
1416	 */
1417	ENTRY_NP(sfmmu_load_tsbe)
1418	/*
1419	 * %o0 = pointer to tsbe to load
1420	 * %o1 = tsb tag
1421	 * %o2 = virtual pointer to TTE
1422	 * %o3 = 1 if physical address in %o0 else 0
1423	 */
1424	rdpr	%pstate, %o5
1425#ifdef DEBUG
1426	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1)
1427#endif /* DEBUG */
1428
1429	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1430
1431	SETUP_TSB_ASI(%o3, %g3)
1432	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1)
1433
1434	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1435
1436	retl
1437	membar	#StoreStore|#StoreLoad
1438	SET_SIZE(sfmmu_load_tsbe)
1439
1440	/*
1441	 * Flush TSB of a given entry if the tag matches.
1442	 */
1443	ENTRY(sfmmu_unload_tsbe)
1444	/*
1445	 * %o0 = pointer to tsbe to be flushed
1446	 * %o1 = tag to match
1447	 * %o2 = 1 if physical address in %o0 else 0
1448	 */
1449	SETUP_TSB_ASI(%o2, %g1)
1450	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1451	retl
1452	membar	#StoreStore|#StoreLoad
1453	SET_SIZE(sfmmu_unload_tsbe)
1454
1455	/*
1456	 * Routine that loads a TTE into the kpm TSB from C code.
1457	 * Locking is required since kpm TSB is shared among all CPUs.
1458	 */
1459	ENTRY_NP(sfmmu_kpm_load_tsb)
1460	/*
1461	 * %o0 = vaddr
1462	 * %o1 = ttep
1463	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1464	 */
1465	rdpr	%pstate, %o5			! %o5 = saved pstate
1466#ifdef DEBUG
1467	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1)
1468#endif /* DEBUG */
1469	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts
1470
1471#ifndef sun4v
1472	sethi	%hi(ktsb_phys), %o4
1473	mov	ASI_N, %o3
1474	ld	[%o4 + %lo(ktsb_phys)], %o4
1475	movrnz	%o4, ASI_MEM, %o3
1476	mov	%o3, %asi
1477#endif /* !sun4v */
1478	mov	%o0, %g1			! %g1 = vaddr
1479
1480	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1481	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1482	/* %g2 = tsbep, %g1 clobbered */
1483
1484	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1485	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1486	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1)
1487
1488	wrpr	%g0, %o5, %pstate		! enable interrupts
1489	retl
1490	  membar #StoreStore|#StoreLoad
1491	SET_SIZE(sfmmu_kpm_load_tsb)
1492
1493	/*
1494	 * Routine that shoots down a TTE in the kpm TSB or in the
1495	 * kernel TSB depending on virtpg. Locking is required since
1496	 * kpm/kernel TSB is shared among all CPUs.
1497	 */
1498	ENTRY_NP(sfmmu_kpm_unload_tsb)
1499	/*
1500	 * %o0 = vaddr
1501	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1502	 */
1503#ifndef sun4v
1504	sethi	%hi(ktsb_phys), %o4
1505	mov	ASI_N, %o3
1506	ld	[%o4 + %lo(ktsb_phys)], %o4
1507	movrnz	%o4, ASI_MEM, %o3
1508	mov	%o3, %asi
1509#endif /* !sun4v */
1510	mov	%o0, %g1			! %g1 = vaddr
1511
1512	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1513	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1514	/* %g2 = tsbep, %g1 clobbered */
1515
1516	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1517	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1518	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1519
1520	retl
1521	  membar	#StoreStore|#StoreLoad
1522	SET_SIZE(sfmmu_kpm_unload_tsb)
1523
1524#endif /* lint */
1525
1526
1527#if defined (lint)
1528
1529/*ARGSUSED*/
1530pfn_t
1531sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
1532{ return(0); }
1533
1534#else /* lint */
1535
1536	ENTRY_NP(sfmmu_ttetopfn)
1537	ldx	[%o0], %g1			/* read tte */
1538	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1539	/*
1540	 * g1 = pfn
1541	 */
1542	retl
1543	mov	%g1, %o0
1544	SET_SIZE(sfmmu_ttetopfn)
1545
1546#endif /* !lint */
1547
1548
1549#if defined (lint)
1550/*
1551 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
1552 * the hash list.
1553 */
1554/* ARGSUSED */
1555void
1556sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1557	uint64_t hblkpa)
1558{
1559}
1560
1561/*
1562 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
1563 * hash list.
1564 */
1565/* ARGSUSED */
1566void
1567sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1568	uint64_t hblkpa, struct hme_blk *prev_hblkp)
1569{
1570}
1571#else /* lint */
1572
1573/*
1574 * Functions to grab/release hme bucket list lock.  I only use a byte
1575 * instead of the whole int because eventually we might want to
1576 * put some counters on the other bytes (of course, these routines would
1577 * have to change).  The code that grab this lock should execute
1578 * with interrupts disabled and hold the lock for the least amount of time
1579 * possible.
1580 */
1581
1582/*
1583 * Even though hmeh_listlock is updated using pa there's no need to flush
1584 * dcache since hmeh_listlock will be restored to the original value (0)
1585 * before interrupts are reenabled.
1586 */
1587
1588/*
1589 * For sparcv9 hme hash buckets may not be in the nucleus.  hme hash update
1590 * routines still use virtual addresses to update the bucket fields. But they
1591 * must not cause a TLB miss after grabbing the low level bucket lock. To
1592 * achieve this we must make sure the bucket structure is completely within an
1593 * 8K page.
1594 */
1595
1596#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1))
1597#error - the size of hmehash_bucket structure is not power of 2
1598#endif
1599
1600/*
1601 * Enable backoff to significantly reduce locking overhead and reduce a chance
1602 * of xcall timeout. This is only enabled for sun4v as a Makefile compile-
1603 * time option.
1604 * The rd %ccr is better for performance compared to a non pipeline releasing
1605 * tight spin on N2/VF.
1606 * Backoff based fix is a temporary solution and doesn't allow scaling above
1607 * lock saturation point. The final fix is to eliminate HMELOCK_ENTER()
1608 * to avoid xcall timeouts and improve GET_TTE() performance.
1609 */
1610
1611#ifdef HMELOCK_BACKOFF_ENABLE
1612
1613#define HMELOCK_BACKOFF(reg, val)                               \
1614	set     val, reg                                        ;\
1615	rd	%ccr, %g0                                       ;\
1616	brnz	reg, .-4                                        ;\
1617	dec	reg
1618
1619#define CAS_HME(tmp1, tmp2, exitlabel, asi)                     \
1620	mov     0xff, tmp2                                      ;\
1621	casa    [tmp1]asi, %g0, tmp2                            ;\
1622	brz,a,pt tmp2, exitlabel                                ;\
1623	membar  #LoadLoad
1624
1625#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label, asi)            \
1626	mov     0xff, tmp2                                      ;\
1627	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1628	casa    [tmp1]asi, %g0, tmp2                            ;\
1629	brz,a,pt tmp2, label/**/2                               ;\
1630	membar  #LoadLoad                                       ;\
1631	HMELOCK_BACKOFF(tmp2,0x8)                               ;\
1632	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1633	HMELOCK_BACKOFF(tmp2,0x10)                              ;\
1634	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1635	HMELOCK_BACKOFF(tmp2,0x20)                              ;\
1636	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1637	HMELOCK_BACKOFF(tmp2,0x40)                              ;\
1638	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1639	HMELOCK_BACKOFF(tmp2,0x80)                              ;\
1640	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1641label/**/1:                                                     ;\
1642	HMELOCK_BACKOFF(tmp2,0x100)                             ;\
1643	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1644	HMELOCK_BACKOFF(tmp2,0x200)                             ;\
1645	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1646	HMELOCK_BACKOFF(tmp2,0x400)                             ;\
1647	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1648	HMELOCK_BACKOFF(tmp2,0x800)                             ;\
1649	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1650	HMELOCK_BACKOFF(tmp2,0x1000)                            ;\
1651	CAS_HME(tmp1, tmp2, label/**/2, asi)                    ;\
1652	HMELOCK_BACKOFF(tmp2,0x2000)                            ;\
1653	mov     0xff, tmp2                                      ;\
1654	casa    [tmp1]asi, %g0, tmp2                            ;\
1655	brnz,pn tmp2, label/**/1     /* reset backoff */        ;\
1656	membar  #LoadLoad                                       ;\
1657label/**/2:
1658
1659#else /* HMELOCK_BACKOFF_ENABLE */
1660
1661#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi)           \
1662	mov     0xff, tmp2                                      ;\
1663	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1664label1:                                                         ;\
1665	casa    [tmp1]asi, %g0, tmp2                            ;\
1666	brnz,pn tmp2, label1                                    ;\
1667	mov     0xff, tmp2                                      ;\
1668	membar  #LoadLoad
1669
1670#endif /* HMELOCK_BACKOFF_ENABLE */
1671
1672#define HMELOCK_EXIT(hmebp, tmp1, asi)                          \
1673	membar  #LoadStore|#StoreStore                          ;\
1674	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1675	sta     %g0, [tmp1]asi
1676
1677	.seg	".data"
1678hblk_add_panic1:
1679	.ascii	"sfmmu_hblk_hash_add: interrupts disabled"
1680	.byte	0
1681hblk_add_panic2:
1682	.ascii	"sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
1683	.byte	0
1684	.align	4
1685	.seg	".text"
1686
1687	ENTRY_NP(sfmmu_hblk_hash_add)
1688	/*
1689	 * %o0 = hmebp
1690	 * %o1 = hmeblkp
1691	 * %o2 = hblkpa
1692	 */
1693	rdpr	%pstate, %o5
1694#ifdef DEBUG
1695	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1696	bnz,pt %icc, 3f				/* disabled, panic	 */
1697	  nop
1698	save	%sp, -SA(MINFRAME), %sp
1699	sethi	%hi(hblk_add_panic1), %o0
1700	call	panic
1701	 or	%o0, %lo(hblk_add_panic1), %o0
1702	ret
1703	restore
1704
17053:
1706#endif /* DEBUG */
1707	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1708	mov	%o2, %g1
1709
1710	/*
1711	 * g1 = hblkpa
1712	 */
1713	ldn	[%o0 + HMEBUCK_HBLK], %o4	/* next hmeblk */
1714	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = next hblkpa */
1715#ifdef	DEBUG
1716	cmp	%o4, %g0
1717	bne,pt %xcc, 1f
1718	 nop
1719	brz,pt %g2, 1f
1720	 nop
1721	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1722	save	%sp, -SA(MINFRAME), %sp
1723	sethi	%hi(hblk_add_panic2), %o0
1724	call	panic
1725	  or	%o0, %lo(hblk_add_panic2), %o0
1726	ret
1727	restore
17281:
1729#endif /* DEBUG */
1730	/*
1731	 * We update hmeblks entries before grabbing lock because the stores
1732	 * could take a tlb miss and require the hash lock.  The buckets
1733	 * are part of the nucleus so we are cool with those stores.
1734	 *
1735	 * if buckets are not part of the nucleus our game is to
1736	 * not touch any other page via va until we drop the lock.
1737	 * This guarantees we won't get a tlb miss before the lock release
1738	 * since interrupts are disabled.
1739	 */
1740	stn	%o4, [%o1 + HMEBLK_NEXT]	/* update hmeblk's next */
1741	stx	%g2, [%o1 + HMEBLK_NEXTPA]	/* update hmeblk's next pa */
1742	HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N)
1743	stn	%o1, [%o0 + HMEBUCK_HBLK]	/* update bucket hblk next */
1744	stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* add hmeblk to list */
1745	HMELOCK_EXIT(%o0, %g2, ASI_N)
1746	retl
1747	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1748	SET_SIZE(sfmmu_hblk_hash_add)
1749
1750	ENTRY_NP(sfmmu_hblk_hash_rm)
1751	/*
1752	 * This function removes an hmeblk from the hash chain.
1753	 * It is written to guarantee we don't take a tlb miss
1754	 * by using physical addresses to update the list.
1755	 *
1756	 * %o0 = hmebp
1757	 * %o1 = hmeblkp
1758	 * %o2 = hmeblkp previous pa
1759	 * %o3 = hmeblkp previous
1760	 */
1761
1762	mov	%o3, %o4			/* o4 = hmeblkp previous */
1763
1764	rdpr	%pstate, %o5
1765#ifdef DEBUG
1766	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1)
1767#endif /* DEBUG */
1768	/*
1769	 * disable interrupts, clear Address Mask to access 64 bit physaddr
1770	 */
1771	andn    %o5, PSTATE_IE, %g1
1772	wrpr    %g1, 0, %pstate
1773
1774#ifndef sun4v
1775	sethi   %hi(dcache_line_mask), %g4
1776	ld      [%g4 + %lo(dcache_line_mask)], %g4
1777#endif /* sun4v */
1778
1779	/*
1780	 * if buckets are not part of the nucleus our game is to
1781	 * not touch any other page via va until we drop the lock.
1782	 * This guarantees we won't get a tlb miss before the lock release
1783	 * since interrupts are disabled.
1784	 */
1785	HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N)
1786	ldn	[%o0 + HMEBUCK_HBLK], %g2	/* first hmeblk in list */
1787	cmp	%g2, %o1
1788	bne,pt	%ncc,1f
1789	 mov	ASI_MEM, %asi
1790	/*
1791	 * hmeblk is first on list
1792	 */
1793	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = hmeblk pa */
1794	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1795	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1796	stn	%o3, [%o0 + HMEBUCK_HBLK]	/* write va */
1797	ba,pt	%xcc, 2f
1798	  stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* write pa */
17991:
1800	/* hmeblk is not first on list */
1801
1802	mov	%o2, %g3
1803#ifndef sun4v
1804	GET_CPU_IMPL(%g2)
1805	cmp 	%g2, CHEETAH_IMPL
1806	bge,a,pt %icc, hblk_hash_rm_1
1807	  and	%o4, %g4, %g2
1808	cmp	%g2, SPITFIRE_IMPL
1809	blt	%icc, hblk_hash_rm_2		/* no flushing needed for OPL */
1810	  and	%o4, %g4, %g2
1811	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev pa from dcache */
1812	add	%o4, HMEBLK_NEXT, %o4
1813	and	%o4, %g4, %g2
1814	ba	hblk_hash_rm_2
1815	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev va from dcache */
1816hblk_hash_rm_1:
1817
1818	stxa	%g0, [%g3]ASI_DC_INVAL		/* flush prev pa from dcache */
1819	membar	#Sync
1820	add     %g3, HMEBLK_NEXT, %g2
1821	stxa	%g0, [%g2]ASI_DC_INVAL		/* flush prev va from dcache */
1822hblk_hash_rm_2:
1823	membar	#Sync
1824#endif /* sun4v */
1825	ldxa	[%g3 + HMEBLK_NEXTPA] %asi, %g2	/* g2 = hmeblk pa */
1826	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1827	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1828	stna	%o3, [%g3 + HMEBLK_NEXT] %asi	/* write va */
1829	stxa	%g1, [%g3 + HMEBLK_NEXTPA] %asi	/* write pa */
18302:
1831	HMELOCK_EXIT(%o0, %g2, ASI_N)
1832	retl
1833	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1834	SET_SIZE(sfmmu_hblk_hash_rm)
1835
1836#endif /* lint */
1837
1838/*
1839 * These macros are used to update global sfmmu hme hash statistics
1840 * in perf critical paths. It is only enabled in debug kernels or
1841 * if SFMMU_STAT_GATHER is defined
1842 */
1843#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1844#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1845	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1846	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
1847	cmp	tmp1, hatid						;\
1848	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
1849	set	sfmmu_global_stat, tmp1					;\
1850	add	tmp1, tmp2, tmp1					;\
1851	ld	[tmp1], tmp2						;\
1852	inc	tmp2							;\
1853	st	tmp2, [tmp1]
1854
1855#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1856	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1857	mov	HATSTAT_KHASH_LINKS, tmp2				;\
1858	cmp	tmp1, hatid						;\
1859	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
1860	set	sfmmu_global_stat, tmp1					;\
1861	add	tmp1, tmp2, tmp1					;\
1862	ld	[tmp1], tmp2						;\
1863	inc	tmp2							;\
1864	st	tmp2, [tmp1]
1865
1866
1867#else /* DEBUG || SFMMU_STAT_GATHER */
1868
1869#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1870
1871#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1872
1873#endif  /* DEBUG || SFMMU_STAT_GATHER */
1874
1875/*
1876 * This macro is used to update global sfmmu kstas in non
1877 * perf critical areas so they are enabled all the time
1878 */
1879#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
1880	sethi	%hi(sfmmu_global_stat), tmp1				;\
1881	add	tmp1, statname, tmp1					;\
1882	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
1883	inc	tmp2							;\
1884	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1885
1886/*
1887 * These macros are used to update per cpu stats in non perf
1888 * critical areas so they are enabled all the time
1889 */
1890#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
1891	ld	[tsbarea + stat], tmp1					;\
1892	inc	tmp1							;\
1893	st	tmp1, [tsbarea + stat]
1894
1895/*
1896 * These macros are used to update per cpu stats in non perf
1897 * critical areas so they are enabled all the time
1898 */
1899#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
1900	lduh	[tsbarea + stat], tmp1					;\
1901	inc	tmp1							;\
1902	stuh	tmp1, [tsbarea + stat]
1903
1904#if defined(KPM_TLBMISS_STATS_GATHER)
1905	/*
1906	 * Count kpm dtlb misses separately to allow a different
1907	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1908	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1909	 */
1910#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
1911	brgez	tagacc, label	/* KPM VA? */				;\
1912	nop								;\
1913	CPU_INDEX(tmp1, tsbma)						;\
1914	sethi	%hi(kpmtsbm_area), tsbma				;\
1915	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
1916	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
1917	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
1918	/* VA range check */						;\
1919	ldx	[tsbma + KPMTSBM_VBASE], val				;\
1920	cmp	tagacc, val						;\
1921	blu,pn	%xcc, label						;\
1922	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
1923	cmp	tagacc, tmp1						;\
1924	bgeu,pn	%xcc, label						;\
1925	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
1926	inc	val							;\
1927	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
1928label:
1929#else
1930#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1931#endif	/* KPM_TLBMISS_STATS_GATHER */
1932
1933#if defined (lint)
1934/*
1935 * The following routines are jumped to from the mmu trap handlers to do
1936 * the setting up to call systrap.  They are separate routines instead of
1937 * being part of the handlers because the handlers would exceed 32
1938 * instructions and since this is part of the slow path the jump
1939 * cost is irrelevant.
1940 */
1941void
1942sfmmu_pagefault(void)
1943{
1944}
1945
1946void
1947sfmmu_mmu_trap(void)
1948{
1949}
1950
1951void
1952sfmmu_window_trap(void)
1953{
1954}
1955
1956void
1957sfmmu_kpm_exception(void)
1958{
1959}
1960
1961#else /* lint */
1962
1963#ifdef	PTL1_PANIC_DEBUG
1964	.seg	".data"
1965	.global	test_ptl1_panic
1966test_ptl1_panic:
1967	.word	0
1968	.align	8
1969
1970	.seg	".text"
1971	.align	4
1972#endif	/* PTL1_PANIC_DEBUG */
1973
1974
1975	ENTRY_NP(sfmmu_pagefault)
1976	SET_GL_REG(1)
1977	USE_ALTERNATE_GLOBALS(%g5)
1978	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1979	rdpr	%tt, %g6
1980	cmp	%g6, FAST_IMMU_MISS_TT
1981	be,a,pn	%icc, 1f
1982	  mov	T_INSTR_MMU_MISS, %g3
1983	cmp	%g6, T_INSTR_MMU_MISS
1984	be,a,pn	%icc, 1f
1985	  mov	T_INSTR_MMU_MISS, %g3
1986	mov	%g5, %g2
1987	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1988	cmp	%g6, FAST_DMMU_MISS_TT
1989	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1990	cmp	%g6, T_DATA_MMU_MISS
1991	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1992
1993#ifdef  PTL1_PANIC_DEBUG
1994	/* check if we want to test the tl1 panic */
1995	sethi	%hi(test_ptl1_panic), %g4
1996	ld	[%g4 + %lo(test_ptl1_panic)], %g1
1997	st	%g0, [%g4 + %lo(test_ptl1_panic)]
1998	cmp	%g1, %g0
1999	bne,a,pn %icc, ptl1_panic
2000	  or	%g0, PTL1_BAD_DEBUG, %g1
2001#endif	/* PTL1_PANIC_DEBUG */
20021:
2003	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
2004	/*
2005	 * g2 = tag access reg
2006	 * g3.l = type
2007	 * g3.h = 0
2008	 */
2009	sethi	%hi(trap), %g1
2010	or	%g1, %lo(trap), %g1
20112:
2012	ba,pt	%xcc, sys_trap
2013	  mov	-1, %g4
2014	SET_SIZE(sfmmu_pagefault)
2015
2016	ENTRY_NP(sfmmu_mmu_trap)
2017	SET_GL_REG(1)
2018	USE_ALTERNATE_GLOBALS(%g5)
2019	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
2020	rdpr	%tt, %g6
2021	cmp	%g6, FAST_IMMU_MISS_TT
2022	be,a,pn	%icc, 1f
2023	  mov	T_INSTR_MMU_MISS, %g3
2024	cmp	%g6, T_INSTR_MMU_MISS
2025	be,a,pn	%icc, 1f
2026	  mov	T_INSTR_MMU_MISS, %g3
2027	mov	%g5, %g2
2028	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
2029	cmp	%g6, FAST_DMMU_MISS_TT
2030	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
2031	cmp	%g6, T_DATA_MMU_MISS
2032	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
20331:
2034	/*
2035	 * g2 = tag access reg
2036	 * g3 = type
2037	 */
2038	sethi	%hi(sfmmu_tsbmiss_exception), %g1
2039	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
2040	ba,pt	%xcc, sys_trap
2041	  mov	-1, %g4
2042	/*NOTREACHED*/
2043	SET_SIZE(sfmmu_mmu_trap)
2044
2045	ENTRY_NP(sfmmu_suspend_tl)
2046	SET_GL_REG(1)
2047	USE_ALTERNATE_GLOBALS(%g5)
2048	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
2049	rdpr	%tt, %g6
2050	cmp	%g6, FAST_IMMU_MISS_TT
2051	be,a,pn	%icc, 1f
2052	  mov	T_INSTR_MMU_MISS, %g3
2053	mov	%g5, %g2
2054	cmp	%g6, FAST_DMMU_MISS_TT
2055	move	%icc, T_DATA_MMU_MISS, %g3
2056	movne	%icc, T_DATA_PROT, %g3
20571:
2058	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
2059	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
2060	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
2061	ba,pt	%xcc, sys_trap
2062	  mov	PIL_15, %g4
2063	/*NOTREACHED*/
2064	SET_SIZE(sfmmu_suspend_tl)
2065
2066	/*
2067	 * No %g registers in use at this point.
2068	 */
2069	ENTRY_NP(sfmmu_window_trap)
2070	rdpr	%tpc, %g1
2071#ifdef sun4v
2072#ifdef DEBUG
2073	/* We assume previous %gl was 1 */
2074	rdpr	%tstate, %g4
2075	srlx	%g4, TSTATE_GL_SHIFT, %g4
2076	and	%g4, TSTATE_GL_MASK, %g4
2077	cmp	%g4, 1
2078	bne,a,pn %icc, ptl1_panic
2079	  mov	PTL1_BAD_WTRAP, %g1
2080#endif /* DEBUG */
2081	/* user miss at tl>1. better be the window handler or user_rtt */
2082	/* in user_rtt? */
2083	set	rtt_fill_start, %g4
2084	cmp	%g1, %g4
2085	blu,pn %xcc, 6f
2086	 .empty
2087	set	rtt_fill_end, %g4
2088	cmp	%g1, %g4
2089	bgeu,pn %xcc, 6f
2090	 nop
2091	set	fault_rtt_fn1, %g1
2092	wrpr	%g0, %g1, %tnpc
2093	ba,a	7f
20946:
2095	! must save this trap level before descending trap stack
2096	! no need to save %tnpc, either overwritten or discarded
2097	! already got it: rdpr	%tpc, %g1
2098	rdpr	%tstate, %g6
2099	rdpr	%tt, %g7
2100	! trap level saved, go get underlying trap type
2101	rdpr	%tl, %g5
2102	sub	%g5, 1, %g3
2103	wrpr	%g3, %tl
2104	rdpr	%tt, %g2
2105	wrpr	%g5, %tl
2106	! restore saved trap level
2107	wrpr	%g1, %tpc
2108	wrpr	%g6, %tstate
2109	wrpr	%g7, %tt
2110#else /* sun4v */
2111	/* user miss at tl>1. better be the window handler */
2112	rdpr	%tl, %g5
2113	sub	%g5, 1, %g3
2114	wrpr	%g3, %tl
2115	rdpr	%tt, %g2
2116	wrpr	%g5, %tl
2117#endif /* sun4v */
2118	and	%g2, WTRAP_TTMASK, %g4
2119	cmp	%g4, WTRAP_TYPE
2120	bne,pn	%xcc, 1f
2121	 nop
2122	/* tpc should be in the trap table */
2123	set	trap_table, %g4
2124	cmp	%g1, %g4
2125	blt,pn %xcc, 1f
2126	 .empty
2127	set	etrap_table, %g4
2128	cmp	%g1, %g4
2129	bge,pn %xcc, 1f
2130	 .empty
2131	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
2132	add	%g1, WTRAP_FAULTOFF, %g1
2133	wrpr	%g0, %g1, %tnpc
21347:
2135	/*
2136	 * some wbuf handlers will call systrap to resolve the fault
2137	 * we pass the trap type so they figure out the correct parameters.
2138	 * g5 = trap type, g6 = tag access reg
2139	 */
2140
2141	/*
2142	 * only use g5, g6, g7 registers after we have switched to alternate
2143	 * globals.
2144	 */
2145	SET_GL_REG(1)
2146	USE_ALTERNATE_GLOBALS(%g5)
2147	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
2148	rdpr	%tt, %g7
2149	cmp	%g7, FAST_IMMU_MISS_TT
2150	be,a,pn	%icc, ptl1_panic
2151	  mov	PTL1_BAD_WTRAP, %g1
2152	cmp	%g7, T_INSTR_MMU_MISS
2153	be,a,pn	%icc, ptl1_panic
2154	  mov	PTL1_BAD_WTRAP, %g1
2155	mov	T_DATA_PROT, %g5
2156	cmp	%g7, FAST_DMMU_MISS_TT
2157	move	%icc, T_DATA_MMU_MISS, %g5
2158	cmp	%g7, T_DATA_MMU_MISS
2159	move	%icc, T_DATA_MMU_MISS, %g5
2160	! XXXQ AGS re-check out this one
2161	done
21621:
2163	CPU_PADDR(%g1, %g4)
2164	add	%g1, CPU_TL1_HDLR, %g1
2165	lda	[%g1]ASI_MEM, %g4
2166	brnz,a,pt %g4, sfmmu_mmu_trap
2167	  sta	%g0, [%g1]ASI_MEM
2168	ba,pt	%icc, ptl1_panic
2169	  mov	PTL1_BAD_TRAP, %g1
2170	SET_SIZE(sfmmu_window_trap)
2171
2172	ENTRY_NP(sfmmu_kpm_exception)
2173	/*
2174	 * We have accessed an unmapped segkpm address or a legal segkpm
2175	 * address which is involved in a VAC alias conflict prevention.
2176	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
2177	 * set. If it is, we will instead note that a fault has occurred
2178	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
2179	 * a "retry"). This will step over the faulting instruction.
2180	 * Note that this means that a legal segkpm address involved in
2181	 * a VAC alias conflict prevention (a rare case to begin with)
2182	 * cannot be used in DTrace.
2183	 */
2184	CPU_INDEX(%g1, %g2)
2185	set	cpu_core, %g2
2186	sllx	%g1, CPU_CORE_SHIFT, %g1
2187	add	%g1, %g2, %g1
2188	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
2189	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
2190	bz	0f
2191	or	%g2, CPU_DTRACE_BADADDR, %g2
2192	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
2193	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
2194	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
2195	done
21960:
2197	TSTAT_CHECK_TL1(1f, %g1, %g2)
21981:
2199	SET_GL_REG(1)
2200	USE_ALTERNATE_GLOBALS(%g5)
2201	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
2202	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
2203	/*
2204	 * g2=tagacc g3.l=type g3.h=0
2205	 */
2206	sethi	%hi(trap), %g1
2207	or	%g1, %lo(trap), %g1
2208	ba,pt	%xcc, sys_trap
2209	mov	-1, %g4
2210	SET_SIZE(sfmmu_kpm_exception)
2211
2212#endif /* lint */
2213
2214#if defined (lint)
2215
2216void
2217sfmmu_tsb_miss(void)
2218{
2219}
2220
2221void
2222sfmmu_kpm_dtsb_miss(void)
2223{
2224}
2225
2226void
2227sfmmu_kpm_dtsb_miss_small(void)
2228{
2229}
2230
2231#else /* lint */
2232
2233#if (IMAP_SEG != 0)
2234#error - ism_map->ism_seg offset is not zero
2235#endif
2236
2237/*
2238 * Copies ism mapping for this ctx in param "ism" if this is a ISM
2239 * tlb miss and branches to label "ismhit". If this is not an ISM
2240 * process or an ISM tlb miss it falls thru.
2241 *
2242 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
2243 * this process.
2244 * If so, it will branch to label "ismhit".  If not, it will fall through.
2245 *
2246 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
2247 * so that any other threads of this process will not try and walk the ism
2248 * maps while they are being changed.
2249 *
2250 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
2251 *       will make sure of that. This means we can terminate our search on
2252 *       the first zero mapping we find.
2253 *
2254 * Parameters:
2255 * tagacc	= (pseudo-)tag access register (vaddr + ctx) (in)
2256 * tsbmiss	= address of tsb miss area (in)
2257 * ismseg	= contents of ism_seg for this ism map (out)
2258 * ismhat	= physical address of imap_ismhat for this ism map (out)
2259 * tmp1		= scratch reg (CLOBBERED)
2260 * tmp2		= scratch reg (CLOBBERED)
2261 * tmp3		= scratch reg (CLOBBERED)
2262 * label:    temporary labels
2263 * ismhit:   label where to jump to if an ism dtlb miss
2264 * exitlabel:label where to jump if hat is busy due to hat_unshare.
2265 */
2266#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
2267	label, ismhit)							\
2268	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
2269	brlz,pt  tmp1, label/**/3		/* exit if -1 */	;\
2270	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
2271label/**/1:								;\
2272	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
2273	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
2274label/**/2:								;\
2275	brz,pt  ismseg, label/**/3		/* no mapping */	;\
2276	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
2277	lduba	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
2278	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
2279	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
2280	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
2281	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
2282	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
2283	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
2284	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
2285	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
2286	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
2287									;\
2288	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
2289	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
2290	cmp	ismhat, tmp1						;\
2291	bl,pt	%xcc, label/**/2		/* keep looking  */	;\
2292	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
2293									;\
2294	add	tmp3, IBLK_NEXTPA, tmp1					;\
2295	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
2296	brgez,pt tmp1, label/**/1		/* continue if not -1*/	;\
2297	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
2298label/**/3:
2299
2300/*
2301 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
2302 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
2303 * Parameters:
2304 * tagacc = reg containing virtual address
2305 * hatid = reg containing sfmmu pointer
2306 * hmeshift = constant/register to shift vaddr to obtain vapg
2307 * hmebp = register where bucket pointer will be stored
2308 * vapg = register where virtual page will be stored
2309 * tmp1, tmp2 = tmp registers
2310 */
2311
2312
2313#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
2314	vapg, label, tmp1, tmp2)					\
2315	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
2316	brnz,a,pt tmp1, label/**/1					;\
2317	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
2318	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
2319	ba,pt	%xcc, label/**/2					;\
2320	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
2321label/**/1:								;\
2322	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
2323label/**/2:								;\
2324	srlx	tagacc, hmeshift, vapg					;\
2325	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
2326	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
2327	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
2328	add	hmebp, tmp1, hmebp
2329
2330/*
2331 * hashtag includes bspage + hashno (64 bits).
2332 */
2333
2334#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
2335	sllx	vapg, hmeshift, vapg					;\
2336	mov	hashno, hblktag						;\
2337	sllx	hblktag, HTAG_REHASH_SHIFT, hblktag			;\
2338	or	vapg, hblktag, hblktag
2339
2340/*
2341 * Function to traverse hmeblk hash link list and find corresponding match.
2342 * The search is done using physical pointers. It returns the physical address
2343 * and virtual address pointers to the hmeblk that matches with the tag
2344 * provided.
2345 * Parameters:
2346 * hmebp	= register that points to hme hash bucket, also used as
2347 *		  tmp reg (clobbered)
2348 * hmeblktag	= register with hmeblk tag match
2349 * hatid	= register with hatid
2350 * hmeblkpa	= register where physical ptr will be stored
2351 * hmeblkva	= register where virtual ptr will be stored
2352 * tmp1		= tmp reg
2353 * label: temporary label
2354 */
2355
2356#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva,	\
2357	tsbarea, tmp1, label)					 	\
2358	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
2359	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
2360	add     hmebp, HMEBUCK_HBLK, hmeblkva				;\
2361	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
2362	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
2363label/**/1:								;\
2364	brz,pn	hmeblkva, label/**/2					;\
2365	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
2366	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
2367	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
2368	add	hmebp, CLONGSIZE, hmebp					;\
2369	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
2370	xor	tmp1, hmeblktag, tmp1					;\
2371	xor	hmebp, hatid, hmebp					;\
2372	or	hmebp, tmp1, hmebp					;\
2373	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
2374	  add	hmeblkpa, HMEBLK_NEXT, hmebp				;\
2375	ldna	[hmebp]ASI_MEM, hmeblkva	/* hmeblk ptr va */	;\
2376	add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
2377	ba,pt	%xcc, label/**/1					;\
2378	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
2379label/**/2:
2380
2381/*
2382 * Function to traverse hmeblk hash link list and find corresponding match.
2383 * The search is done using physical pointers. It returns the physical address
2384 * and virtual address pointers to the hmeblk that matches with the tag
2385 * provided.
2386 * Parameters:
2387 * hmeblktag	= register with hmeblk tag match (rid field is 0)
2388 * hatid	= register with hatid (pointer to SRD)
2389 * hmeblkpa	= register where physical ptr will be stored
2390 * hmeblkva	= register where virtual ptr will be stored
2391 * tmp1		= tmp reg
2392 * tmp2		= tmp reg
2393 * label: temporary label
2394 */
2395
2396#define	HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva,	\
2397	tsbarea, tmp1, tmp2, label)			 		\
2398label/**/1:								;\
2399	brz,pn	hmeblkva, label/**/4					;\
2400	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			;\
2401	add	hmeblkpa, HMEBLK_TAG, tmp2				;\
2402	ldxa	[tmp2]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
2403	add	tmp2, CLONGSIZE, tmp2					;\
2404	ldxa	[tmp2]ASI_MEM, tmp2 	/* read 2nd part of tag */	;\
2405	xor	tmp1, hmeblktag, tmp1					;\
2406	xor	tmp2, hatid, tmp2					;\
2407	brz,pn	tmp2, label/**/3	/* branch on hit */		;\
2408	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
2409label/**/2:								;\
2410	ldna	[tmp2]ASI_MEM, hmeblkva	/* hmeblk ptr va */		;\
2411	add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
2412	ba,pt	%xcc, label/**/1					;\
2413	  ldxa	[tmp2]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */		;\
2414label/**/3:								;\
2415	cmp	tmp1, SFMMU_MAX_HME_REGIONS				;\
2416	bgeu,pt	%xcc, label/**/2					;\
2417	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
2418	and	tmp1, BT_ULMASK, tmp2					;\
2419	srlx	tmp1, BT_ULSHIFT, tmp1					;\
2420	sllx	tmp1, CLONGSHIFT, tmp1					;\
2421	add	tsbarea, tmp1, tmp1					;\
2422	ldx	[tmp1 + TSBMISS_SHMERMAP], tmp1				;\
2423	srlx	tmp1, tmp2, tmp1					;\
2424	btst	0x1, tmp1						;\
2425	bz,pn	%xcc, label/**/2					;\
2426	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
2427label/**/4:
2428
2429#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
2430#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
2431#endif
2432
2433/*
2434 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2435 * he offset for the corresponding hment.
2436 * Parameters:
2437 * In:
2438 *	vaddr = register with virtual address
2439 *	hmeblkpa = physical pointer to hme_blk
2440 * Out:
2441 *	hmentoff = register where hment offset will be stored
2442 *	hmemisc = hblk_misc
2443 * Scratch:
2444 *	tmp1
2445 */
2446#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\
2447	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
2448	lda	[hmentoff]ASI_MEM, hmemisc 				;\
2449	andcc	hmemisc, HBLK_SZMASK, %g0				;\
2450	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
2451	  or	%g0, HMEBLK_HME1, hmentoff				;\
2452	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
2453	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
2454	sllx	tmp1, SFHME_SHIFT, tmp1					;\
2455	add	tmp1, HMEBLK_HME1, hmentoff				;\
2456label1:
2457
2458/*
2459 * GET_TTE is a macro that returns a TTE given a tag and hatid.
2460 *
2461 * tagacc	= (pseudo-)tag access register (in)
2462 * hatid	= sfmmu pointer for TSB miss (in)
2463 * tte		= tte for TLB miss if found, otherwise clobbered (out)
2464 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
2465 * hmeblkva	= VA of hment if found, otherwise clobbered (out)
2466 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
2467 * hmemisc	= hblk_misc if TTE is found (out), otherwise clobbered
2468 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
2469 *		  for this page size.
2470 * hashno	= constant/register hash number
2471 * label	= temporary label for branching within macro.
2472 * foundlabel	= label to jump to when tte is found.
2473 * suspendlabel= label to jump to when tte is suspended.
2474 * exitlabel	= label to jump to when tte is not found.
2475 *
2476 */
2477#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \
2478		hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \
2479									;\
2480	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2481	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2482	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2483		hmeblkpa, label/**/5, hmemisc, hmeblkva)		;\
2484									;\
2485	/*								;\
2486	 * tagacc = tagacc						;\
2487	 * hatid = hatid						;\
2488	 * tsbarea = tsbarea						;\
2489	 * tte   = hmebp (hme bucket pointer)				;\
2490	 * hmeblkpa  = vapg  (virtual page)				;\
2491	 * hmemisc, hmeblkva = scratch					;\
2492	 */								;\
2493	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2494	or	hmemisc, SFMMU_INVALID_SHMERID, hmemisc			;\
2495									;\
2496	/*								;\
2497	 * tagacc = tagacc						;\
2498	 * hatid = hatid						;\
2499	 * tte   = hmebp						;\
2500	 * hmeblkpa  = CLOBBERED					;\
2501	 * hmemisc  = htag_bspage+hashno+invalid_rid			;\
2502	 * hmeblkva  = scratch						;\
2503	 */								;\
2504	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2505	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
2506	HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, 	\
2507		tsbarea, tagacc, label/**/1)				;\
2508	/*								;\
2509	 * tagacc = CLOBBERED						;\
2510	 * tte = CLOBBERED						;\
2511	 * hmeblkpa = hmeblkpa						;\
2512	 * hmeblkva = hmeblkva						;\
2513	 */								;\
2514	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2515	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2516	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2517	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2518	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2519	  nop								;\
2520label/**/4:								;\
2521	/*								;\
2522	 * We have found the hmeblk containing the hment.		;\
2523	 * Now we calculate the corresponding tte.			;\
2524	 *								;\
2525	 * tagacc = tagacc						;\
2526	 * hatid = hatid						;\
2527	 * tte   = clobbered						;\
2528	 * hmeblkpa  = hmeblkpa						;\
2529	 * hmemisc  = hblktag						;\
2530	 * hmeblkva  = hmeblkva 					;\
2531	 */								;\
2532	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2533		label/**/2)						;\
2534									;\
2535	/*								;\
2536	 * tagacc = tagacc						;\
2537	 * hatid = hmentoff						;\
2538	 * tte   = clobbered						;\
2539	 * hmeblkpa  = hmeblkpa						;\
2540	 * hmemisc  = hblk_misc						;\
2541	 * hmeblkva  = hmeblkva 					;\
2542	 */								;\
2543									;\
2544	add	hatid, SFHME_TTE, hatid					;\
2545	add	hmeblkpa, hatid, hmeblkpa				;\
2546	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2547	add	hmeblkva, hatid, hmeblkva				;\
2548	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2549	HMELOCK_EXIT(hatid, hatid, ASI_MEM)	/* drop lock */		;\
2550	set	TTE_SUSPEND, hatid					;\
2551	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2552	btst	tte, hatid						;\
2553	bz,pt	%xcc, foundlabel					;\
2554	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2555									;\
2556	/*								;\
2557	 * Mapping is suspended, so goto suspend label.			;\
2558	 */								;\
2559	ba,pt	%xcc, suspendlabel					;\
2560	  nop
2561
2562/*
2563 * GET_SHME_TTE is similar to GET_TTE() except it searches
2564 * shared hmeblks via HMEHASH_SEARCH_SHME() macro.
2565 * If valid tte is found, hmemisc = shctx flag, i.e., shme is
2566 * either 0 (not part of scd) or 1 (part of scd).
2567 */
2568#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea,	\
2569		hmemisc, hmeshift, hashno, label, foundlabel,		\
2570		suspendlabel, exitlabel)				\
2571									;\
2572	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2573	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2574	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2575		hmeblkpa, label/**/5, hmemisc, hmeblkva)		;\
2576									;\
2577	/*								;\
2578	 * tagacc = tagacc						;\
2579	 * hatid = hatid						;\
2580	 * tsbarea = tsbarea						;\
2581	 * tte   = hmebp (hme bucket pointer)				;\
2582	 * hmeblkpa  = vapg  (virtual page)				;\
2583	 * hmemisc, hmeblkva = scratch					;\
2584	 */								;\
2585	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2586									;\
2587	/*								;\
2588	 * tagacc = tagacc						;\
2589	 * hatid = hatid						;\
2590	 * tsbarea = tsbarea						;\
2591	 * tte   = hmebp						;\
2592	 * hmemisc  = htag_bspage + hashno + 0 (for rid)		;\
2593	 * hmeblkpa  = CLOBBERED					;\
2594	 * hmeblkva  = scratch						;\
2595	 */								;\
2596	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2597	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
2598									;\
2599	add     tte, HMEBUCK_NEXTPA, hmeblkpa				;\
2600	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
2601	add     tte, HMEBUCK_HBLK, hmeblkva				;\
2602	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
2603	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte)			;\
2604									;\
2605label/**/8:								;\
2606	HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, 	\
2607		tsbarea, tagacc, tte, label/**/1)			;\
2608	/*								;\
2609	 * tagacc = CLOBBERED						;\
2610	 * tte = CLOBBERED						;\
2611	 * hmeblkpa = hmeblkpa						;\
2612	 * hmeblkva = hmeblkva						;\
2613	 */								;\
2614	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2615	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2616	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2617	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2618	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2619	  nop								;\
2620label/**/4:								;\
2621	/*								;\
2622	 * We have found the hmeblk containing the hment.		;\
2623	 * Now we calculate the corresponding tte.			;\
2624	 *								;\
2625	 * tagacc = tagacc						;\
2626	 * hatid = hatid						;\
2627	 * tte   = clobbered						;\
2628	 * hmeblkpa  = hmeblkpa						;\
2629	 * hmemisc  = hblktag						;\
2630	 * hmeblkva  = hmeblkva 					;\
2631	 * tsbarea = tsbmiss area					;\
2632	 */								;\
2633	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2634		label/**/2)						;\
2635									;\
2636	/*								;\
2637	 * tagacc = tagacc						;\
2638	 * hatid = hmentoff						;\
2639	 * tte = clobbered						;\
2640	 * hmeblkpa  = hmeblkpa						;\
2641	 * hmemisc  = hblk_misc						;\
2642	 * hmeblkva  = hmeblkva						;\
2643	 * tsbarea = tsbmiss area					;\
2644	 */								;\
2645									;\
2646	add	hatid, SFHME_TTE, hatid					;\
2647	add	hmeblkpa, hatid, hmeblkpa				;\
2648	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2649	brlz,pt tte, label/**/6						;\
2650	  add	hmeblkva, hatid, hmeblkva				;\
2651	btst	HBLK_SZMASK, hmemisc					;\
2652	bnz,a,pt %icc, label/**/7					;\
2653	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2654									;\
2655	/*								;\
2656 	 * We found an invalid 8K tte in shme.				;\
2657	 * it may not belong to shme's region since			;\
2658	 * region size/alignment granularity is 8K but different	;\
2659	 * regions don't share hmeblks. Continue the search.		;\
2660	 */								;\
2661	sub	hmeblkpa, hatid, hmeblkpa				;\
2662	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2663	srlx	tagacc, hmeshift, tte					;\
2664	add	hmeblkpa, HMEBLK_NEXT, hmeblkva				;\
2665	ldxa	[hmeblkva]ASI_MEM, hmeblkva				;\
2666	add	hmeblkpa, HMEBLK_NEXTPA, hmeblkpa			;\
2667	ldxa	[hmeblkpa]ASI_MEM, hmeblkpa				;\
2668	MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc)		;\
2669	ba,a,pt	%xcc, label/**/8					;\
2670label/**/6:								;\
2671	GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc)		;\
2672	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2673label/**/7:								;\
2674	HMELOCK_EXIT(hatid, hatid, ASI_MEM)	/* drop lock */		;\
2675	set	TTE_SUSPEND, hatid					;\
2676	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2677	btst	tte, hatid						;\
2678	bz,pt	%xcc, foundlabel					;\
2679	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2680									;\
2681	/*								;\
2682	 * Mapping is suspended, so goto suspend label.			;\
2683	 */								;\
2684	ba,pt	%xcc, suspendlabel					;\
2685	  nop
2686
2687	/*
2688	 * KERNEL PROTECTION HANDLER
2689	 *
2690	 * g1 = tsb8k pointer register (clobbered)
2691	 * g2 = tag access register (ro)
2692	 * g3 - g7 = scratch registers
2693	 *
2694	 * Note: This function is patched at runtime for performance reasons.
2695	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
2696	 */
2697	ENTRY_NP(sfmmu_kprot_trap)
2698	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2699sfmmu_kprot_patch_ktsb_base:
2700	RUNTIME_PATCH_SETX(%g1, %g6)
2701	/* %g1 = contents of ktsb_base or ktsb_pbase */
2702sfmmu_kprot_patch_ktsb_szcode:
2703	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
2704
2705	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2706	! %g1 = First TSB entry pointer, as TSB miss handler expects
2707
2708	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2709sfmmu_kprot_patch_ktsb4m_base:
2710	RUNTIME_PATCH_SETX(%g3, %g6)
2711	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2712sfmmu_kprot_patch_ktsb4m_szcode:
2713	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
2714
2715	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2716	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2717
2718        CPU_TSBMISS_AREA(%g6, %g7)
2719        HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2720	ba,pt	%xcc, sfmmu_tsb_miss_tt
2721	  nop
2722
2723	/*
2724	 * USER PROTECTION HANDLER
2725	 *
2726	 * g1 = tsb8k pointer register (ro)
2727	 * g2 = tag access register (ro)
2728	 * g3 = faulting context (clobbered, currently not used)
2729	 * g4 - g7 = scratch registers
2730	 */
2731	ALTENTRY(sfmmu_uprot_trap)
2732#ifdef sun4v
2733	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2734	/* %g1 = first TSB entry ptr now, %g2 preserved */
2735
2736	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
2737	brlz,pt %g3, 9f				/* check for 2nd TSB */
2738	  nop
2739
2740	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2741	/* %g3 = second TSB entry ptr now, %g2 preserved */
2742
2743#else /* sun4v */
2744#ifdef UTSB_PHYS
2745	/* g1 = first TSB entry ptr */
2746	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
2747	brlz,pt %g3, 9f			/* check for 2nd TSB */
2748	  nop
2749
2750	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2751	/* %g3 = second TSB entry ptr now, %g2 preserved */
2752#else /* UTSB_PHYS */
2753	brgez,pt %g1, 9f		/* check for 2nd TSB */
2754	  mov	-1, %g3			/* set second tsbe ptr to -1 */
2755
2756	mov	%g2, %g7
2757	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2758	/* %g3 = second TSB entry ptr now, %g7 clobbered */
2759	mov	%g1, %g7
2760	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2761#endif /* UTSB_PHYS */
2762#endif /* sun4v */
27639:
2764	CPU_TSBMISS_AREA(%g6, %g7)
2765	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2766	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
2767	  nop
2768
2769	/*
2770	 * Kernel 8K page iTLB miss.  We also get here if we took a
2771	 * fast instruction access mmu miss trap while running in
2772	 * invalid context.
2773	 *
2774	 * %g1 = 8K TSB pointer register (not used, clobbered)
2775	 * %g2 = tag access register (used)
2776	 * %g3 = faulting context id (used)
2777	 * %g7 = TSB tag to match (used)
2778	 */
2779	.align	64
2780	ALTENTRY(sfmmu_kitlb_miss)
2781	brnz,pn %g3, tsb_tl0_noctxt
2782	  nop
2783
2784	/* kernel miss */
2785	/* get kernel tsb pointer */
2786	/* we patch the next set of instructions at run time */
2787	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2788iktsbbase:
2789	RUNTIME_PATCH_SETX(%g4, %g5)
2790	/* %g4 = contents of ktsb_base or ktsb_pbase */
2791
2792iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2793	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2794	or	%g4, %g1, %g1			! form tsb ptr
2795	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2796	cmp	%g4, %g7
2797	bne,pn	%xcc, iktsb4mbase		! check 4m ktsb
2798	  srlx    %g2, MMU_PAGESHIFT4M, %g3	! use 4m virt-page as TSB index
2799
2800	andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2801	bz,pn	%icc, exec_fault
2802	  nop
2803	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2804	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2805	retry
2806
2807iktsb4mbase:
2808        RUNTIME_PATCH_SETX(%g4, %g6)
2809        /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */
2810iktsb4m:
2811	sllx    %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2812        srlx    %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2813	add	%g4, %g3, %g3			! %g3 = 4m tsbe ptr
2814	ldda	[%g3]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2815	cmp	%g4, %g7
2816	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
2817	  andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2818	bz,pn	%icc, exec_fault
2819	  nop
2820	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2821	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2822	retry
2823
2824	/*
2825	 * Kernel dTLB miss.  We also get here if we took a fast data
2826	 * access mmu miss trap while running in invalid context.
2827	 *
2828	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2829	 *	We select the TSB miss handler to branch to depending on
2830	 *	the virtual address of the access.  In the future it may
2831	 *	be desirable to separate kpm TTEs into their own TSB,
2832	 *	in which case all that needs to be done is to set
2833	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2834	 *	early in the miss if we detect a kpm VA to a new handler.
2835	 *
2836	 * %g1 = 8K TSB pointer register (not used, clobbered)
2837	 * %g2 = tag access register (used)
2838	 * %g3 = faulting context id (used)
2839	 */
2840	.align	64
2841	ALTENTRY(sfmmu_kdtlb_miss)
2842	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
2843	  nop
2844
2845	/* Gather some stats for kpm misses in the TLB. */
2846	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2847	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2848
2849	/*
2850	 * Get first TSB offset and look for 8K/64K/512K mapping
2851	 * using the 8K virtual page as the index.
2852	 *
2853	 * We patch the next set of instructions at run time;
2854	 * any changes here require sfmmu_patch_ktsb changes too.
2855	 */
2856dktsbbase:
2857	RUNTIME_PATCH_SETX(%g7, %g6)
2858	/* %g7 = contents of ktsb_base or ktsb_pbase */
2859
2860dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2861	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2862
2863	/*
2864	 * At this point %g1 is our index into the TSB.
2865	 * We just masked off enough bits of the VA depending
2866	 * on our TSB size code.
2867	 */
2868	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2869	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2870	cmp	%g6, %g4			! compare tag
2871	bne,pn	%xcc, dktsb4m_kpmcheck_small
2872	  add	%g7, %g1, %g1			/* form tsb ptr */
2873	TT_TRACE(trace_tsbhit)
2874	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2875	/* trapstat expects tte in %g5 */
2876	retry
2877
2878	/*
2879	 * If kpm is using large pages, the following instruction needs
2880	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2881	 * so that we will probe the 4M TSB regardless of the VA.  In
2882	 * the case kpm is using small pages, we know no large kernel
2883	 * mappings are located above 0x80000000.00000000 so we skip the
2884	 * probe as an optimization.
2885	 */
2886dktsb4m_kpmcheck_small:
2887	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2888	  /* delay slot safe, below */
2889
2890	/*
2891	 * Get second TSB offset and look for 4M mapping
2892	 * using 4M virtual page as the TSB index.
2893	 *
2894	 * Here:
2895	 * %g1 = 8K TSB pointer.  Don't squash it.
2896	 * %g2 = tag access register (we still need it)
2897	 */
2898	srlx	%g2, MMU_PAGESHIFT4M, %g3
2899
2900	/*
2901	 * We patch the next set of instructions at run time;
2902	 * any changes here require sfmmu_patch_ktsb changes too.
2903	 */
2904dktsb4mbase:
2905	RUNTIME_PATCH_SETX(%g7, %g6)
2906	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2907dktsb4m:
2908	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2909	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2910
2911	/*
2912	 * At this point %g3 is our index into the TSB.
2913	 * We just masked off enough bits of the VA depending
2914	 * on our TSB size code.
2915	 */
2916	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2917	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2918	cmp	%g6, %g4			! compare tag
2919
2920dktsb4m_tsbmiss:
2921	bne,pn	%xcc, dktsb4m_kpmcheck
2922	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
2923	TT_TRACE(trace_tsbhit)
2924	/* we don't check TTE size here since we assume 4M TSB is separate */
2925	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2926	/* trapstat expects tte in %g5 */
2927	retry
2928
2929	/*
2930	 * So, we failed to find a valid TTE to match the faulting
2931	 * address in either TSB.  There are a few cases that could land
2932	 * us here:
2933	 *
2934	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2935	 *    to sfmmu_tsb_miss_tt to handle the miss.
2936	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2937	 *    4M TSB.  Let segkpm handle it.
2938	 *
2939	 * Note that we shouldn't land here in the case of a kpm VA when
2940	 * kpm_smallpages is active -- we handled that case earlier at
2941	 * dktsb4m_kpmcheck_small.
2942	 *
2943	 * At this point:
2944	 *  g1 = 8K-indexed primary TSB pointer
2945	 *  g2 = tag access register
2946	 *  g3 = 4M-indexed secondary TSB pointer
2947	 */
2948dktsb4m_kpmcheck:
2949	cmp	%g2, %g0
2950	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
2951	  nop
2952	ba,a,pt	%icc, sfmmu_tsb_miss_tt
2953	  nop
2954
2955#ifdef sun4v
2956	/*
2957	 * User instruction miss w/ single TSB.
2958	 * The first probe covers 8K, 64K, and 512K page sizes,
2959	 * because 64K and 512K mappings are replicated off 8K
2960	 * pointer.
2961	 *
2962	 * g1 = tsb8k pointer register
2963	 * g2 = tag access register
2964	 * g3 - g6 = scratch registers
2965	 * g7 = TSB tag to match
2966	 */
2967	.align	64
2968	ALTENTRY(sfmmu_uitlb_fastpath)
2969
2970	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2971	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2972	ba,pn	%xcc, sfmmu_tsb_miss_tt
2973	  mov	-1, %g3
2974
2975	/*
2976	 * User data miss w/ single TSB.
2977	 * The first probe covers 8K, 64K, and 512K page sizes,
2978	 * because 64K and 512K mappings are replicated off 8K
2979	 * pointer.
2980	 *
2981	 * g1 = tsb8k pointer register
2982	 * g2 = tag access register
2983	 * g3 - g6 = scratch registers
2984	 * g7 = TSB tag to match
2985	 */
2986	.align 64
2987	ALTENTRY(sfmmu_udtlb_fastpath)
2988
2989	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2990	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2991	ba,pn	%xcc, sfmmu_tsb_miss_tt
2992	  mov	-1, %g3
2993
2994	/*
2995	 * User instruction miss w/ multiple TSBs (sun4v).
2996	 * The first probe covers 8K, 64K, and 512K page sizes,
2997	 * because 64K and 512K mappings are replicated off 8K
2998	 * pointer.  Second probe covers 4M page size only.
2999	 *
3000	 * Just like sfmmu_udtlb_slowpath, except:
3001	 *   o Uses ASI_ITLB_IN
3002	 *   o checks for execute permission
3003	 *   o No ISM prediction.
3004	 *
3005	 * g1 = tsb8k pointer register
3006	 * g2 = tag access register
3007	 * g3 - g6 = scratch registers
3008	 * g7 = TSB tag to match
3009	 */
3010	.align	64
3011	ALTENTRY(sfmmu_uitlb_slowpath)
3012
3013	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
3014	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
3015	/* g4 - g5 = clobbered here */
3016
3017	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3018	/* g1 = first TSB pointer, g3 = second TSB pointer */
3019	srlx	%g2, TAG_VALO_SHIFT, %g7
3020	PROBE_2ND_ITSB(%g3, %g7)
3021	/* NOT REACHED */
3022
3023#else /* sun4v */
3024
3025	/*
3026	 * User instruction miss w/ multiple TSBs (sun4u).
3027	 * The first probe covers 8K, 64K, and 512K page sizes,
3028	 * because 64K and 512K mappings are replicated off 8K
3029	 * pointer.  Probe of 1st TSB has already been done prior to entry
3030	 * into this routine. For the UTSB_PHYS case we probe up to 3
3031	 * valid other TSBs in the following order:
3032	 * 1) shared TSB for 4M-256M pages
3033	 * 2) private TSB for 4M-256M pages
3034	 * 3) shared TSB for 8K-512K pages
3035	 *
3036	 * For the non UTSB_PHYS case we probe the 2nd TSB here that backs
3037	 * 4M-256M pages.
3038	 *
3039	 * Just like sfmmu_udtlb_slowpath, except:
3040	 *   o Uses ASI_ITLB_IN
3041	 *   o checks for execute permission
3042	 *   o No ISM prediction.
3043	 *
3044	 * g1 = tsb8k pointer register
3045	 * g2 = tag access register
3046	 * g4 - g6 = scratch registers
3047	 * g7 = TSB tag to match
3048	 */
3049	.align	64
3050	ALTENTRY(sfmmu_uitlb_slowpath)
3051
3052#ifdef UTSB_PHYS
3053
3054       GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
3055        brlz,pt %g6, 1f
3056          nop
3057        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
3058        PROBE_4TH_ITSB(%g6, %g7, uitlb_4m_scd_probefail)
30591:
3060        GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
3061        brlz,pt %g3, 2f
3062          nop
3063        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3064        PROBE_2ND_ITSB(%g3, %g7, uitlb_4m_probefail)
30652:
3066        GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
3067        brlz,pt %g6, sfmmu_tsb_miss_tt
3068          nop
3069        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
3070        PROBE_3RD_ITSB(%g6, %g7, uitlb_8K_scd_probefail)
3071        ba,pn   %xcc, sfmmu_tsb_miss_tt
3072          nop
3073
3074#else /* UTSB_PHYS */
3075	mov	%g1, %g3	/* save tsb8k reg in %g3 */
3076	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
3077	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
3078	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
3079	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
3080	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
3081       /* g1 = first TSB pointer, g3 = second TSB pointer */
3082        srlx    %g2, TAG_VALO_SHIFT, %g7
3083        PROBE_2ND_ITSB(%g3, %g7, isynth)
3084	ba,pn	%xcc, sfmmu_tsb_miss_tt
3085	  nop
3086
3087#endif /* UTSB_PHYS */
3088#endif /* sun4v */
3089
3090#if defined(sun4u) && defined(UTSB_PHYS)
3091
3092        /*
3093	 * We come here for ism predict DTLB_MISS case or if
3094	 * if probe in first TSB failed.
3095         */
3096
3097        .align 64
3098        ALTENTRY(sfmmu_udtlb_slowpath_noismpred)
3099
3100	/*
3101         * g1 = tsb8k pointer register
3102         * g2 = tag access register
3103         * g4 - %g6 = scratch registers
3104         * g7 = TSB tag to match
3105	 */
3106
3107	/*
3108	 * ISM non-predict probe order
3109         * probe 1ST_TSB (8K index)
3110         * probe 2ND_TSB (4M index)
3111         * probe 4TH_TSB (4M index)
3112         * probe 3RD_TSB (8K index)
3113	 *
3114	 * We already probed first TSB in DTLB_MISS handler.
3115	 */
3116
3117        /*
3118         * Private 2ND TSB 4M-256 pages
3119         */
3120	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
3121	brlz,pt %g3, 1f
3122	  nop
3123        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3124        PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
3125
3126	/*
3127	 * Shared Context 4TH TSB 4M-256 pages
3128	 */
31291:
3130	GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
3131	brlz,pt %g6, 2f
3132	  nop
3133        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
3134        PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail)
3135
3136        /*
3137         * Shared Context 3RD TSB 8K-512K pages
3138         */
31392:
3140	GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
3141	brlz,pt %g6, sfmmu_tsb_miss_tt
3142	  nop
3143        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
3144        PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail)
3145	ba,pn	%xcc, sfmmu_tsb_miss_tt
3146	  nop
3147
3148	.align 64
3149        ALTENTRY(sfmmu_udtlb_slowpath_ismpred)
3150
3151	/*
3152         * g1 = tsb8k pointer register
3153         * g2 = tag access register
3154         * g4 - g6 = scratch registers
3155         * g7 = TSB tag to match
3156	 */
3157
3158	/*
3159	 * ISM predict probe order
3160	 * probe 4TH_TSB (4M index)
3161	 * probe 2ND_TSB (4M index)
3162	 * probe 1ST_TSB (8K index)
3163	 * probe 3RD_TSB (8K index)
3164
3165	/*
3166	 * Shared Context 4TH TSB 4M-256 pages
3167	 */
3168	GET_UTSBREG(SCRATCHPAD_UTSBREG4, %g6)
3169	brlz,pt %g6, 4f
3170	  nop
3171        GET_4TH_TSBE_PTR(%g2, %g6, %g4, %g5)
3172        PROBE_4TH_DTSB(%g6, %g7, udtlb_4m_shctx_probefail2)
3173
3174        /*
3175         * Private 2ND TSB 4M-256 pages
3176         */
31774:
3178	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)
3179	brlz,pt %g3, 5f
3180	  nop
3181        GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3182        PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail2)
3183
31845:
3185        PROBE_1ST_DTSB(%g1, %g7, udtlb_8k_first_probefail2)
3186
3187        /*
3188         * Shared Context 3RD TSB 8K-512K pages
3189         */
3190	GET_UTSBREG(SCRATCHPAD_UTSBREG3, %g6)
3191	brlz,pt %g6, 6f
3192	  nop
3193        GET_3RD_TSBE_PTR(%g2, %g6, %g4, %g5)
3194        PROBE_3RD_DTSB(%g6, %g7, udtlb_8k_shctx_probefail2)
31956:
3196	ba,pn	%xcc, sfmmu_tsb_miss_tt /* ISM Predict and ISM non-predict path */
3197	  nop
3198
3199#else /* sun4u && UTSB_PHYS */
3200
3201       .align 64
3202        ALTENTRY(sfmmu_udtlb_slowpath)
3203
3204	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
3205	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
3206	  mov	%g1, %g3
3207
3208udtlb_miss_probefirst:
3209	/*
3210	 * g1 = 8K TSB pointer register
3211	 * g2 = tag access register
3212	 * g3 = (potentially) second TSB entry ptr
3213	 * g6 = ism pred.
3214	 * g7 = vpg_4m
3215	 */
3216#ifdef sun4v
3217	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
3218	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
3219
3220	/*
3221	 * Here:
3222	 *   g1 = first TSB pointer
3223	 *   g2 = tag access reg
3224	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
3225	 */
3226	brgz,pn	%g6, sfmmu_tsb_miss_tt
3227	  nop
3228#else /* sun4v */
3229	mov	%g1, %g4
3230	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
3231	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
3232
3233	/*
3234	 * Here:
3235	 *   g1 = first TSB pointer
3236	 *   g2 = tag access reg
3237	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
3238	 */
3239	brgz,pn	%g6, sfmmu_tsb_miss_tt
3240	  nop
3241	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
3242	/* fall through in 8K->4M probe order */
3243#endif /* sun4v */
3244
3245udtlb_miss_probesecond:
3246	/*
3247	 * Look in the second TSB for the TTE
3248	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
3249	 * g2 = tag access reg
3250	 * g3 = 8K TSB pointer register
3251	 * g6 = ism pred.
3252	 * g7 = vpg_4m
3253	 */
3254#ifdef sun4v
3255	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
3256	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3257	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
3258#else /* sun4v */
3259	mov	%g3, %g7
3260	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
3261	/* %g2 clobbered, %g3 =second tsbe ptr */
3262	mov	MMU_TAG_ACCESS, %g2
3263	ldxa	[%g2]ASI_DMMU, %g2
3264#endif /* sun4v */
3265
3266	srlx	%g2, TAG_VALO_SHIFT, %g7
3267	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
3268	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
3269	brgz,pn	%g6, udtlb_miss_probefirst
3270	  nop
3271
3272	/* fall through to sfmmu_tsb_miss_tt */
3273#endif /* sun4u && UTSB_PHYS */
3274
3275
3276	ALTENTRY(sfmmu_tsb_miss_tt)
3277	TT_TRACE(trace_tsbmiss)
3278	/*
3279	 * We get here if there is a TSB miss OR a write protect trap.
3280	 *
3281	 * g1 = First TSB entry pointer
3282	 * g2 = tag access register
3283	 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB
3284	 * g4 - g7 = scratch registers
3285	 */
3286
3287	ALTENTRY(sfmmu_tsb_miss)
3288
3289	/*
3290	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3291	 * point to trapstat's TSB miss return code (note that trapstat
3292	 * itself will patch the correct offset to add).
3293	 */
3294	rdpr	%tl, %g7
3295	cmp	%g7, 1
3296	ble,pt	%xcc, 0f
3297	  sethi	%hi(KERNELBASE), %g6
3298	rdpr	%tpc, %g7
3299	or	%g6, %lo(KERNELBASE), %g6
3300	cmp	%g7, %g6
3301	bgeu,pt	%xcc, 0f
3302	/* delay slot safe */
3303
3304	ALTENTRY(tsbmiss_trapstat_patch_point)
3305	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3306	wrpr	%g7, %tpc
3307	add	%g7, 4, %g7
3308	wrpr	%g7, %tnpc
33090:
3310	CPU_TSBMISS_AREA(%g6, %g7)
3311	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save 1ST tsb pointer */
3312	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save 2ND tsb pointer */
3313
3314	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
3315	brz,a,pn %g3, 1f			/* skip ahead if kernel */
3316	  ldn	[%g6 + TSBMISS_KHATID], %g7
3317	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
3318	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
3319
3320	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
3321
3322	cmp	%g3, INVALID_CONTEXT
3323	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
3324	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
3325
3326#if defined(sun4v) || defined(UTSB_PHYS)
3327        ldub    [%g6 + TSBMISS_URTTEFLAGS], %g7	/* clear ctx1 flag set from */
3328        andn    %g7, HAT_CHKCTX1_FLAG, %g7	/* the previous tsb miss    */
3329        stub    %g7, [%g6 + TSBMISS_URTTEFLAGS]
3330#endif /* sun4v || UTSB_PHYS */
3331
3332	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
3333	/*
3334	 * The miss wasn't in an ISM segment.
3335	 *
3336	 * %g1 %g3, %g4, %g5, %g7 all clobbered
3337	 * %g2 = (pseudo) tag access
3338	 */
3339
3340	ba,pt	%icc, 2f
3341	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
3342
33431:
3344	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
3345	/*
3346	 * 8K and 64K hash.
3347	 */
33482:
3349
3350	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3351		MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte,
3352		sfmmu_suspend_tl, tsb_512K)
3353	/* NOT REACHED */
3354
3355tsb_512K:
3356	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3357	brz,pn	%g5, 3f
3358	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3359	and	%g4, HAT_512K_FLAG, %g5
3360
3361	/*
3362	 * Note that there is a small window here where we may have
3363	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
3364	 * flag yet, so we will skip searching the 512k hash list.
3365	 * In this case we will end up in pagefault which will find
3366	 * the mapping and return.  So, in this instance we will end up
3367	 * spending a bit more time resolving this TSB miss, but it can
3368	 * only happen once per process and even then, the chances of that
3369	 * are very small, so it's not worth the extra overhead it would
3370	 * take to close this window.
3371	 */
3372	brz,pn	%g5, tsb_4M
3373	  nop
33743:
3375	/*
3376	 * 512K hash
3377	 */
3378
3379	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3380		MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte,
3381		sfmmu_suspend_tl, tsb_4M)
3382	/* NOT REACHED */
3383
3384tsb_4M:
3385	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3386	brz,pn	%g5, 4f
3387	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3388	and	%g4, HAT_4M_FLAG, %g5
3389	brz,pn	%g5, tsb_32M
3390	  nop
33914:
3392	/*
3393	 * 4M hash
3394	 */
3395
3396	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3397		MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte,
3398		sfmmu_suspend_tl, tsb_32M)
3399	/* NOT REACHED */
3400
3401tsb_32M:
3402	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3403#ifdef	sun4v
3404        brz,pn	%g5, 6f
3405#else
3406	brz,pn  %g5, tsb_pagefault
3407#endif
3408	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3409	and	%g4, HAT_32M_FLAG, %g5
3410	brz,pn	%g5, tsb_256M
3411	  nop
34125:
3413	/*
3414	 * 32M hash
3415	 */
3416
3417	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3418		MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte,
3419		sfmmu_suspend_tl, tsb_256M)
3420	/* NOT REACHED */
3421
3422#if defined(sun4u) && !defined(UTSB_PHYS)
3423#define tsb_shme        tsb_pagefault
3424#endif
3425tsb_256M:
3426	ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3427	and	%g4, HAT_256M_FLAG, %g5
3428	brz,pn	%g5, tsb_shme
3429	  nop
34306:
3431	/*
3432	 * 256M hash
3433	 */
3434
3435	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3436	    MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte,
3437	    sfmmu_suspend_tl, tsb_shme)
3438	/* NOT REACHED */
3439
3440tsb_checktte:
3441	/*
3442	 * g1 = hblk_misc
3443	 * g2 = tagacc
3444	 * g3 = tte
3445	 * g4 = tte pa
3446	 * g5 = tte va
3447	 * g6 = tsbmiss area
3448	 * g7 = hatid
3449	 */
3450	brlz,a,pt %g3, tsb_validtte
3451	  rdpr	%tt, %g7
3452
3453#if defined(sun4u) && !defined(UTSB_PHYS)
3454#undef tsb_shme
3455	ba      tsb_pagefault
3456	  nop
3457#else /* sun4u && !UTSB_PHYS */
3458
3459tsb_shme:
3460	/*
3461	 * g2 = tagacc
3462	 * g6 = tsbmiss area
3463	 */
3464	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3465	brz,pn	%g5, tsb_pagefault
3466	  nop
3467	ldx	[%g6 + TSBMISS_SHARED_UHATID], %g7	/* g7 = srdp */
3468	brz,pn	%g7, tsb_pagefault
3469	  nop
3470
3471	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3472		MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte,
3473		sfmmu_suspend_tl, tsb_shme_512K)
3474	/* NOT REACHED */
3475
3476tsb_shme_512K:
3477	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3478	and	%g4, HAT_512K_FLAG, %g5
3479	brz,pn	%g5, tsb_shme_4M
3480	  nop
3481
3482	/*
3483	 * 512K hash
3484	 */
3485
3486	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3487		MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte,
3488		sfmmu_suspend_tl, tsb_shme_4M)
3489	/* NOT REACHED */
3490
3491tsb_shme_4M:
3492	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3493	and	%g4, HAT_4M_FLAG, %g5
3494	brz,pn	%g5, tsb_shme_32M
3495	  nop
34964:
3497	/*
3498	 * 4M hash
3499	 */
3500	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3501		MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte,
3502		sfmmu_suspend_tl, tsb_shme_32M)
3503	/* NOT REACHED */
3504
3505tsb_shme_32M:
3506	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3507	and	%g4, HAT_32M_FLAG, %g5
3508	brz,pn	%g5, tsb_shme_256M
3509	  nop
3510
3511	/*
3512	 * 32M hash
3513	 */
3514
3515	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3516		MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte,
3517		sfmmu_suspend_tl, tsb_shme_256M)
3518	/* NOT REACHED */
3519
3520tsb_shme_256M:
3521	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3522	and	%g4, HAT_256M_FLAG, %g5
3523	brz,pn	%g5, tsb_pagefault
3524	  nop
3525
3526	/*
3527	 * 256M hash
3528	 */
3529
3530	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3531	    MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte,
3532	    sfmmu_suspend_tl, tsb_pagefault)
3533	/* NOT REACHED */
3534
3535tsb_shme_checktte:
3536
3537	brgez,pn %g3, tsb_pagefault
3538	  rdpr	%tt, %g7
3539	/*
3540	 * g1 = ctx1 flag
3541	 * g3 = tte
3542	 * g4 = tte pa
3543	 * g5 = tte va
3544	 * g6 = tsbmiss area
3545	 * g7 = tt
3546	 */
3547
3548	brz,pt  %g1, tsb_validtte
3549	  nop
3550	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g1
3551	  or	%g1, HAT_CHKCTX1_FLAG, %g1
3552	stub    %g1, [%g6 + TSBMISS_URTTEFLAGS]
3553
3554	SAVE_CTX1(%g7, %g2, %g1, tsb_shmel)
3555#endif /* sun4u && !UTSB_PHYS */
3556
3557tsb_validtte:
3558	/*
3559	 * g3 = tte
3560	 * g4 = tte pa
3561	 * g5 = tte va
3562	 * g6 = tsbmiss area
3563	 * g7 = tt
3564	 */
3565
3566	/*
3567	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
3568	 */
3569	cmp	%g7, FAST_PROT_TT
3570	bne,pt	%icc, 4f
3571	  nop
3572
3573	TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod,
3574	    tsb_protfault)
3575
3576	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3577#ifdef sun4v
3578	MMU_FAULT_STATUS_AREA(%g7)
3579	ldx	[%g7 + MMFSA_D_ADDR], %g5	/* load fault addr for later */
3580#else /* sun4v */
3581	mov     MMU_TAG_ACCESS, %g5
3582	ldxa    [%g5]ASI_DMMU, %g5
3583#endif /* sun4v */
3584	ba,pt	%xcc, tsb_update_tl1
3585	  nop
35864:
3587	/*
3588	 * If ITLB miss check exec bit.
3589	 * If not set treat as invalid TTE.
3590	 */
3591	cmp     %g7, T_INSTR_MMU_MISS
3592	be,pn	%icc, 5f
3593	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
3594	cmp     %g7, FAST_IMMU_MISS_TT
3595	bne,pt %icc, 3f
3596	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
35975:
3598	bz,pn %icc, tsb_protfault
3599	  nop
3600
36013:
3602	/*
3603	 * Set reference bit if not already set
3604	 */
3605	TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref)
3606
3607	/*
3608	 * Now, load into TSB/TLB.  At this point:
3609	 * g3 = tte
3610	 * g4 = patte
3611	 * g6 = tsbmiss area
3612	 */
3613	rdpr	%tt, %g7
3614#ifdef sun4v
3615	MMU_FAULT_STATUS_AREA(%g2)
3616	cmp	%g7, T_INSTR_MMU_MISS
3617	be,a,pt	%icc, 9f
3618	  nop
3619	cmp	%g7, FAST_IMMU_MISS_TT
3620	be,a,pt	%icc, 9f
3621	  nop
3622	add	%g2, MMFSA_D_, %g2
36239:
3624	ldx	[%g2 + MMFSA_CTX_], %g7
3625	sllx	%g7, TTARGET_CTX_SHIFT, %g7
3626	ldx	[%g2 + MMFSA_ADDR_], %g2
3627	mov	%g2, %g5		! load the fault addr for later use
3628	srlx	%g2, TTARGET_VA_SHIFT, %g2
3629	or	%g2, %g7, %g2
3630#else /* sun4v */
3631	mov     MMU_TAG_ACCESS, %g5
3632	cmp     %g7, FAST_IMMU_MISS_TT
3633	be,a,pt %icc, 9f
3634	   ldxa  [%g0]ASI_IMMU, %g2
3635	ldxa    [%g0]ASI_DMMU, %g2
3636	ba,pt   %icc, tsb_update_tl1
3637	   ldxa  [%g5]ASI_DMMU, %g5
36389:
3639	ldxa    [%g5]ASI_IMMU, %g5
3640#endif /* sun4v */
3641
3642tsb_update_tl1:
3643	srlx	%g2, TTARGET_CTX_SHIFT, %g7
3644	brz,pn	%g7, tsb_kernel
3645#ifdef sun4v
3646	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
3647#else  /* sun4v */
3648	  srlx	%g3, TTE_SZ_SHFT, %g7
3649#endif /* sun4v */
3650
3651tsb_user:
3652#ifdef sun4v
3653	cmp	%g7, TTE4M
3654	bge,pn	%icc, tsb_user4m
3655	  nop
3656#else /* sun4v */
3657	cmp	%g7, TTESZ_VALID | TTE4M
3658	be,pn	%icc, tsb_user4m
3659	  srlx	%g3, TTE_SZ2_SHFT, %g7
3660	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
3661#ifdef ITLB_32M_256M_SUPPORT
3662	bnz,pn	%icc, tsb_user4m
3663	  nop
3664#else /* ITLB_32M_256M_SUPPORT */
3665	bnz,a,pn %icc, tsb_user_pn_synth
3666	 nop
3667#endif /* ITLB_32M_256M_SUPPORT */
3668#endif /* sun4v */
3669
3670tsb_user8k:
3671#if defined(sun4v) || defined(UTSB_PHYS)
3672	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3673	and	%g7, HAT_CHKCTX1_FLAG, %g1
3674	brz,a,pn %g1, 1f
3675	  ldn	[%g6 + TSBMISS_TSBPTR], %g1		! g1 = 1ST TSB ptr
3676	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1)
3677	brlz,a,pn %g1, ptl1_panic			! if no shared 3RD tsb
3678	  mov PTL1_NO_SCDTSB8K, %g1			! panic
3679        GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7)
36801:
3681#else /* defined(sun4v) || defined(UTSB_PHYS) */
3682	ldn   [%g6 + TSBMISS_TSBPTR], %g1             ! g1 = 1ST TSB ptr
3683#endif /* defined(sun4v) || defined(UTSB_PHYS) */
3684
3685#ifndef UTSB_PHYS
3686	mov	ASI_N, %g7	! user TSBs accessed by VA
3687	mov	%g7, %asi
3688#endif /* !UTSB_PHYS */
3689
3690	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5)
3691
3692	rdpr    %tt, %g5
3693#ifdef sun4v
3694	cmp	%g5, T_INSTR_MMU_MISS
3695	be,a,pn	%xcc, 9f
3696	  mov	%g3, %g5
3697#endif /* sun4v */
3698	cmp	%g5, FAST_IMMU_MISS_TT
3699	be,pn	%xcc, 9f
3700	  mov	%g3, %g5
3701
3702	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3703	! trapstat wants TTE in %g5
3704	retry
37059:
3706	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3707	! trapstat wants TTE in %g5
3708	retry
3709
3710tsb_user4m:
3711#if defined(sun4v) || defined(UTSB_PHYS)
3712	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3713	and	%g7, HAT_CHKCTX1_FLAG, %g1
3714	brz,a,pn %g1, 4f
3715	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		! g1 = 2ND TSB ptr
3716	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)! g1 = 4TH TSB ptr
3717	brlz,a,pn %g1, 5f				! if no shared 4TH TSB
3718	  nop
3719        GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7)
3720
3721#else /* defined(sun4v) || defined(UTSB_PHYS) */
3722	ldn   [%g6 + TSBMISS_TSBPTR4M], %g1             ! g1 = 2ND TSB ptr
3723#endif /* defined(sun4v) || defined(UTSB_PHYS) */
37244:
3725	brlz,pn %g1, 5f	/* Check to see if we have 2nd TSB programmed */
3726	  nop
3727
3728#ifndef UTSB_PHYS
3729	mov	ASI_N, %g7	! user TSBs accessed by VA
3730	mov	%g7, %asi
3731#endif /* UTSB_PHYS */
3732
3733        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6)
3734
37355:
3736	rdpr    %tt, %g5
3737#ifdef sun4v
3738        cmp     %g5, T_INSTR_MMU_MISS
3739        be,a,pn %xcc, 9f
3740          mov   %g3, %g5
3741#endif /* sun4v */
3742        cmp     %g5, FAST_IMMU_MISS_TT
3743        be,pn   %xcc, 9f
3744        mov     %g3, %g5
3745
3746        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3747        ! trapstat wants TTE in %g5
3748        retry
37499:
3750        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3751        ! trapstat wants TTE in %g5
3752        retry
3753
3754#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT)
3755	/*
3756	 * Panther ITLB synthesis.
3757	 * The Panther 32M and 256M ITLB code simulates these two large page
3758	 * sizes with 4M pages, to provide support for programs, for example
3759	 * Java, that may copy instructions into a 32M or 256M data page and
3760	 * then execute them. The code below generates the 4M pfn bits and
3761	 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
3762	 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
3763	 * are ignored by the hardware.
3764	 *
3765	 * Now, load into TSB/TLB.  At this point:
3766	 * g2 = tagtarget
3767	 * g3 = tte
3768	 * g4 = patte
3769	 * g5 = tt
3770	 * g6 = tsbmiss area
3771	 */
3772tsb_user_pn_synth:
3773	rdpr %tt, %g5
3774	cmp    %g5, FAST_IMMU_MISS_TT
3775	be,pt	%xcc, tsb_user_itlb_synth	/* ITLB miss */
3776	  andcc %g3, TTE_EXECPRM_INT, %g0	/* is execprm bit set */
3777	bz,pn %icc, 4b				/* if not, been here before */
3778	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	/* g1 = tsbp */
3779	brlz,a,pn %g1, 5f			/* no 2nd tsb */
3780	  mov	%g3, %g5
3781
3782	mov	MMU_TAG_ACCESS, %g7
3783	ldxa	[%g7]ASI_DMMU, %g6		/* get tag access va */
3784	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)	/* make 4M pfn offset */
3785
3786	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3787	mov	%g7, %asi
3788	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */
37895:
3790        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3791        retry
3792
3793tsb_user_itlb_synth:
3794	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 =  2ND TSB */
3795
3796	mov	MMU_TAG_ACCESS, %g7
3797	ldxa	[%g7]ASI_IMMU, %g6		/* get tag access va */
3798	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)	/* make 4M pfn offset */
3799	brlz,a,pn %g1, 7f	/* Check to see if we have 2nd TSB programmed */
3800	  or	%g5, %g3, %g5			/* add 4M bits to TTE */
3801
3802	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3803	mov	%g7, %asi
3804	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */
38057:
3806	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
3807        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3808        retry
3809#endif /* sun4v && ITLB_32M_256M_SUPPORT */
3810
3811tsb_kernel:
3812	rdpr	%tt, %g5
3813#ifdef sun4v
3814	cmp	%g7, TTE4M
3815	bge,pn	%icc, 5f
3816#else
3817	cmp	%g7, TTESZ_VALID | TTE4M	! no 32M or 256M support
3818	be,pn	%icc, 5f
3819#endif /* sun4v */
3820	  nop
3821	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8K TSB ptr
3822	ba,pt	%xcc, 6f
3823	  nop
38245:
3825	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4M TSB ptr
3826	brlz,pn	%g1, 3f		/* skip programming if 4M TSB ptr is -1 */
3827	  nop
38286:
3829#ifndef sun4v
3830tsb_kernel_patch_asi:
3831	or	%g0, RUNTIME_PATCH, %g6
3832	mov	%g6, %asi	! XXX avoid writing to %asi !!
3833#endif
3834	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7)
38353:
3836#ifdef sun4v
3837	cmp	%g5, T_INSTR_MMU_MISS
3838	be,a,pn	%icc, 1f
3839	  mov	%g3, %g5			! trapstat wants TTE in %g5
3840#endif /* sun4v */
3841	cmp	%g5, FAST_IMMU_MISS_TT
3842	be,pn	%icc, 1f
3843	  mov	%g3, %g5			! trapstat wants TTE in %g5
3844	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3845	! trapstat wants TTE in %g5
3846	retry
38471:
3848	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3849	! trapstat wants TTE in %g5
3850	retry
3851
3852tsb_ism:
3853	/*
3854	 * This is an ISM [i|d]tlb miss.  We optimize for largest
3855	 * page size down to smallest.
3856	 *
3857	 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access
3858	 *	register
3859	 * g3 = ismmap->ism_seg
3860	 * g4 = physical address of ismmap->ism_sfmmu
3861	 * g6 = tsbmiss area
3862	 */
3863	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
3864	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
3865	  mov	PTL1_BAD_ISM, %g1
3866						/* g5 = pa of imap_vb_shift */
3867	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
3868	lduba	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
3869	srlx	%g3, %g4, %g3			/* clr size field */
3870	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
3871	sllx    %g3, %g4, %g3                   /* g3 = ism vbase */
3872	and     %g2, %g1, %g4                   /* g4 = ctx number */
3873	andn    %g2, %g1, %g1                   /* g1 = tlb miss vaddr */
3874	sub     %g1, %g3, %g2                   /* g2 = offset in ISM seg */
3875	or      %g2, %g4, %g2                   /* g2 = (pseudo-)tagacc */
3876	sub     %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
3877	lduha   [%g5]ASI_MEM, %g4               /* g5 = pa of imap_hatflags */
3878#if defined(sun4v) || defined(UTSB_PHYS)
3879	and     %g4, HAT_CTX1_FLAG, %g5         /* g5 = imap_hatflags */
3880	brz,pt %g5, tsb_chk4M_ism
3881	  nop
3882	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g5
3883	or      %g5, HAT_CHKCTX1_FLAG, %g5
3884	stub    %g5, [%g6 + TSBMISS_URTTEFLAGS]
3885	rdpr    %tt, %g5
3886	SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl)
3887#endif /* defined(sun4v) || defined(UTSB_PHYS) */
3888
3889	/*
3890	 * ISM pages are always locked down.
3891	 * If we can't find the tte then pagefault
3892	 * and let the spt segment driver resolve it.
3893	 *
3894	 * g2 = tagacc w/ISM vaddr (offset in ISM seg)
3895	 * g4 = imap_hatflags
3896	 * g6 = tsb miss area
3897	 * g7 = ISM hatid
3898	 */
3899
3900tsb_chk4M_ism:
3901	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
3902	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
3903	  nop
3904
3905tsb_ism_32M:
3906	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
3907	brz,pn	%g5, tsb_ism_256M
3908	  nop
3909
3910	/*
3911	 * 32M hash.
3912	 */
3913
3914	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M,
3915	    TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3916	    tsb_ism_4M)
3917	/* NOT REACHED */
3918
3919tsb_ism_32M_found:
3920	brlz,a,pt %g3, tsb_validtte
3921	  rdpr	%tt, %g7
3922	ba,pt	%xcc, tsb_ism_4M
3923	  nop
3924
3925tsb_ism_256M:
3926	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
3927	brz,a,pn %g5, ptl1_panic
3928	  mov	PTL1_BAD_ISM, %g1
3929
3930	/*
3931	 * 256M hash.
3932	 */
3933	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M,
3934	    TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3935	    tsb_ism_4M)
3936
3937tsb_ism_256M_found:
3938	brlz,a,pt %g3, tsb_validtte
3939	  rdpr	%tt, %g7
3940
3941tsb_ism_4M:
3942	/*
3943	 * 4M hash.
3944	 */
3945	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M,
3946	    TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3947	    tsb_ism_8K)
3948	/* NOT REACHED */
3949
3950tsb_ism_4M_found:
3951	brlz,a,pt %g3, tsb_validtte
3952	  rdpr	%tt, %g7
3953
3954tsb_ism_8K:
3955	/*
3956	 * 8K and 64K hash.
3957	 */
3958
3959	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K,
3960	    TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3961	    tsb_pagefault)
3962	/* NOT REACHED */
3963
3964tsb_ism_8K_found:
3965	brlz,a,pt %g3, tsb_validtte
3966	  rdpr	%tt, %g7
3967
3968tsb_pagefault:
3969	rdpr	%tt, %g7
3970	cmp	%g7, FAST_PROT_TT
3971	be,a,pn	%icc, tsb_protfault
3972	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt
3973
3974tsb_protfault:
3975	/*
3976	 * we get here if we couldn't find a valid tte in the hash.
3977	 *
3978	 * If user and we are at tl>1 we go to window handling code.
3979	 *
3980	 * If kernel and the fault is on the same page as our stack
3981	 * pointer, then we know the stack is bad and the trap handler
3982	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3983	 *
3984	 * If this is a kernel trap and tl>1, panic.
3985	 *
3986	 * Otherwise we call pagefault.
3987	 */
3988	cmp	%g7, FAST_IMMU_MISS_TT
3989#ifdef sun4v
3990	MMU_FAULT_STATUS_AREA(%g4)
3991	ldx	[%g4 + MMFSA_I_CTX], %g5
3992	ldx	[%g4 + MMFSA_D_CTX], %g4
3993	move	%icc, %g5, %g4
3994	cmp	%g7, T_INSTR_MMU_MISS
3995	move	%icc, %g5, %g4
3996#else
3997	mov	MMU_TAG_ACCESS, %g4
3998	ldxa	[%g4]ASI_DMMU, %g2
3999	ldxa	[%g4]ASI_IMMU, %g5
4000	move	%icc, %g5, %g2
4001	cmp	%g7, T_INSTR_MMU_MISS
4002	move	%icc, %g5, %g2
4003	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
4004#endif /* sun4v */
4005	brnz,pn	%g4, 3f				/* skip if not kernel */
4006	  rdpr	%tl, %g5
4007
4008	add	%sp, STACK_BIAS, %g3
4009	srlx	%g3, MMU_PAGESHIFT, %g3
4010	srlx	%g2, MMU_PAGESHIFT, %g4
4011	cmp	%g3, %g4
4012	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
4013	  mov	PTL1_BAD_STACK, %g1
4014
4015	cmp	%g5, 1
4016	ble,pt	%icc, 2f
4017	  nop
4018	TSTAT_CHECK_TL1(2f, %g1, %g2)
4019	rdpr	%tt, %g2
4020	cmp	%g2, FAST_PROT_TT
4021	mov	PTL1_BAD_KPROT_FAULT, %g1
4022	movne	%icc, PTL1_BAD_KMISS, %g1
4023	ba,pt	%icc, ptl1_panic
4024	  nop
4025
40262:
4027	/*
4028	 * We are taking a pagefault in the kernel on a kernel address.  If
4029	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
4030	 * want to call sfmmu_pagefault -- we will instead note that a fault
4031	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
4032	 * (instead of a "retry").  This will step over the faulting
4033	 * instruction.
4034	 */
4035	CPU_INDEX(%g1, %g2)
4036	set	cpu_core, %g2
4037	sllx	%g1, CPU_CORE_SHIFT, %g1
4038	add	%g1, %g2, %g1
4039	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
4040	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
4041	bz	sfmmu_pagefault
4042	or	%g2, CPU_DTRACE_BADADDR, %g2
4043	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
4044	GET_MMU_D_ADDR(%g3, %g4)
4045	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
4046	done
4047
40483:
4049	cmp	%g5, 1
4050	ble,pt	%icc, 4f
4051	  nop
4052	TSTAT_CHECK_TL1(4f, %g1, %g2)
4053	ba,pt	%icc, sfmmu_window_trap
4054	  nop
4055
40564:
4057	/*
4058	 * We are taking a pagefault on a non-kernel address.  If we are in
4059	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
4060	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
4061	 */
4062	CPU_INDEX(%g1, %g2)
4063	set	cpu_core, %g2
4064	sllx	%g1, CPU_CORE_SHIFT, %g1
4065	add	%g1, %g2, %g1
4066	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
4067	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
4068	bz	sfmmu_mmu_trap
4069	or	%g2, CPU_DTRACE_BADADDR, %g2
4070	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
4071	GET_MMU_D_ADDR(%g3, %g4)
4072	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
4073
4074	/*
4075	 * Be sure that we're actually taking this miss from the kernel --
4076	 * otherwise we have managed to return to user-level with
4077	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
4078	 */
4079	rdpr	%tstate, %g2
4080	btst	TSTATE_PRIV, %g2
4081	bz,a	ptl1_panic
4082	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
4083	done
4084
4085	ALTENTRY(tsb_tl0_noctxt)
4086	/*
4087	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
4088	 * if it is, indicated that we have faulted and issue a done.
4089	 */
4090	CPU_INDEX(%g5, %g6)
4091	set	cpu_core, %g6
4092	sllx	%g5, CPU_CORE_SHIFT, %g5
4093	add	%g5, %g6, %g5
4094	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
4095	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
4096	bz	1f
4097	or	%g6, CPU_DTRACE_BADADDR, %g6
4098	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
4099	GET_MMU_D_ADDR(%g3, %g4)
4100	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]
4101
4102	/*
4103	 * Be sure that we're actually taking this miss from the kernel --
4104	 * otherwise we have managed to return to user-level with
4105	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
4106	 */
4107	rdpr	%tstate, %g5
4108	btst	TSTATE_PRIV, %g5
4109	bz,a	ptl1_panic
4110	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
4111	TSTAT_CHECK_TL1(2f, %g1, %g2);
41122:
4113	done
4114
41151:
4116	rdpr	%tt, %g5
4117	cmp	%g5, FAST_IMMU_MISS_TT
4118#ifdef sun4v
4119	MMU_FAULT_STATUS_AREA(%g2)
4120	be,a,pt	%icc, 2f
4121	  ldx	[%g2 + MMFSA_I_CTX], %g3
4122	cmp	%g5, T_INSTR_MMU_MISS
4123	be,a,pt	%icc, 2f
4124	  ldx	[%g2 + MMFSA_I_CTX], %g3
4125	ldx	[%g2 + MMFSA_D_CTX], %g3
41262:
4127#else
4128	mov	MMU_TAG_ACCESS, %g2
4129	be,a,pt	%icc, 2f
4130	  ldxa	[%g2]ASI_IMMU, %g3
4131	ldxa	[%g2]ASI_DMMU, %g3
41322:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
4133#endif /* sun4v */
4134	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
4135	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
4136	rdpr	%tl, %g5
4137	cmp	%g5, 1
4138	ble,pt	%icc, sfmmu_mmu_trap
4139	  nop
4140	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
4141	ba,pt	%icc, sfmmu_window_trap
4142	  nop
4143	SET_SIZE(sfmmu_tsb_miss)
4144#endif  /* lint */
4145
4146#if defined (lint)
4147/*
4148 * This routine will look for a user or kernel vaddr in the hash
4149 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't
4150 * grab any locks.  It should only be used by other sfmmu routines.
4151 */
4152/* ARGSUSED */
4153pfn_t
4154sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
4155{
4156	return(0);
4157}
4158
4159/* ARGSUSED */
4160pfn_t
4161sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno)
4162{
4163	return(0);
4164}
4165
4166#else /* lint */
4167
4168	ENTRY_NP(sfmmu_vatopfn)
4169 	/*
4170 	 * disable interrupts
4171 	 */
4172 	rdpr	%pstate, %o3
4173#ifdef DEBUG
4174	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1)
4175#endif
4176	/*
4177	 * disable interrupts to protect the TSBMISS area
4178	 */
4179	andn    %o3, PSTATE_IE, %o5
4180	wrpr    %o5, 0, %pstate
4181
4182	/*
4183	 * o0 = vaddr
4184	 * o1 = sfmmup
4185	 * o2 = ttep
4186	 */
4187	CPU_TSBMISS_AREA(%g1, %o5)
4188	ldn	[%g1 + TSBMISS_KHATID], %o4
4189	cmp	%o4, %o1
4190	bne,pn	%ncc, vatopfn_nokernel
4191	  mov	TTE64K, %g5			/* g5 = rehash # */
4192	mov %g1,%o5				/* o5 = tsbmiss_area */
4193	/*
4194	 * o0 = vaddr
4195	 * o1 & o4 = hatid
4196	 * o2 = ttep
4197	 * o5 = tsbmiss area
4198	 */
4199	mov	HBLK_RANGE_SHIFT, %g6
42001:
4201
4202	/*
4203	 * o0 = vaddr
4204	 * o1 = sfmmup
4205	 * o2 = ttep
4206	 * o3 = old %pstate
4207	 * o4 = hatid
4208	 * o5 = tsbmiss
4209	 * g5 = rehash #
4210	 * g6 = hmeshift
4211	 *
4212	 * The first arg to GET_TTE is actually tagaccess register
4213	 * not just vaddr. Since this call is for kernel we need to clear
4214	 * any lower vaddr bits that would be interpreted as ctx bits.
4215	 */
4216	set     TAGACC_CTX_MASK, %g1
4217	andn    %o0, %g1, %o0
4218	GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5,
4219		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
4220
4221kvtop_hblk_found:
4222	/*
4223	 * o0 = vaddr
4224	 * o1 = sfmmup
4225	 * o2 = ttep
4226	 * g1 = tte
4227	 * g2 = tte pa
4228	 * g3 = tte va
4229	 * o2 = tsbmiss area
4230	 * o1 = hat id
4231	 */
4232	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
4233	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4234	stx %g1,[%o2]				/* put tte into *ttep */
4235	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
4236	/*
4237	 * o0 = vaddr
4238	 * o1 = sfmmup
4239	 * o2 = ttep
4240	 * g1 = pfn
4241	 */
4242	ba,pt	%xcc, 6f
4243	  mov	%g1, %o0
4244
4245kvtop_nohblk:
4246	/*
4247	 * we get here if we couldn't find valid hblk in hash.  We rehash
4248	 * if neccesary.
4249	 */
4250	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
4251#ifdef sun4v
4252	cmp	%g5, MAX_HASHCNT
4253#else
4254	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
4255#endif /* sun4v */
4256	be,a,pn	%icc, 6f
4257	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4258	mov	%o1, %o4			/* restore hatid */
4259#ifdef sun4v
4260        add	%g5, 2, %g5
4261	cmp	%g5, 3
4262	move	%icc, MMU_PAGESHIFT4M, %g6
4263	ba,pt	%icc, 1b
4264	movne	%icc, MMU_PAGESHIFT256M, %g6
4265#else
4266        inc	%g5
4267	cmp	%g5, 2
4268	move	%icc, MMU_PAGESHIFT512K, %g6
4269	ba,pt	%icc, 1b
4270	movne	%icc, MMU_PAGESHIFT4M, %g6
4271#endif /* sun4v */
42726:
4273	retl
4274 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4275
4276tsb_suspend:
4277	/*
4278	 * o0 = vaddr
4279	 * o1 = sfmmup
4280	 * o2 = ttep
4281	 * g1 = tte
4282	 * g2 = tte pa
4283	 * g3 = tte va
4284	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
4285	 */
4286	stx %g1,[%o2]				/* put tte into *ttep */
4287	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
4288	  sub	%g0, 1, %o0			/* output = PFN_INVALID */
4289	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
42908:
4291	retl
4292	 wrpr	%g0, %o3, %pstate		/* enable interrupts */
4293
4294vatopfn_nokernel:
4295	/*
4296	 * This routine does NOT support user addresses
4297	 * There is a routine in C that supports this.
4298	 * The only reason why we don't have the C routine
4299	 * support kernel addresses as well is because
4300	 * we do va_to_pa while holding the hashlock.
4301	 */
4302 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4303	save	%sp, -SA(MINFRAME), %sp
4304	sethi	%hi(sfmmu_panic3), %o0
4305	call	panic
4306	 or	%o0, %lo(sfmmu_panic3), %o0
4307
4308	SET_SIZE(sfmmu_vatopfn)
4309
4310	/*
4311	 * %o0 = vaddr
4312	 * %o1 = hashno (aka szc)
4313	 *
4314	 *
4315	 * This routine is similar to sfmmu_vatopfn() but will only look for
4316	 * a kernel vaddr in the hash structure for the specified rehash value.
4317	 * It's just an optimization for the case when pagesize for a given
4318	 * va range is already known (e.g. large page heap) and we don't want
4319	 * to start the search with rehash value 1 as sfmmu_vatopfn() does.
4320	 *
4321	 * Returns valid pfn or PFN_INVALID if
4322	 * tte for specified rehash # is not found, invalid or suspended.
4323	 */
4324	ENTRY_NP(sfmmu_kvaszc2pfn)
4325 	/*
4326 	 * disable interrupts
4327 	 */
4328 	rdpr	%pstate, %o3
4329#ifdef DEBUG
4330	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1)
4331#endif
4332	/*
4333	 * disable interrupts to protect the TSBMISS area
4334	 */
4335	andn    %o3, PSTATE_IE, %o5
4336	wrpr    %o5, 0, %pstate
4337
4338	CPU_TSBMISS_AREA(%g1, %o5)
4339	ldn	[%g1 + TSBMISS_KHATID], %o4
4340	sll	%o1, 1, %g6
4341	add	%g6, %o1, %g6
4342	add	%g6, MMU_PAGESHIFT, %g6
4343	/*
4344	 * %o0 = vaddr
4345	 * %o1 = hashno
4346	 * %o3 = old %pstate
4347	 * %o4 = ksfmmup
4348	 * %g1 = tsbmiss area
4349	 * %g6 = hmeshift
4350	 */
4351
4352	/*
4353	 * The first arg to GET_TTE is actually tagaccess register
4354	 * not just vaddr. Since this call is for kernel we need to clear
4355	 * any lower vaddr bits that would be interpreted as ctx bits.
4356	 */
4357	srlx	%o0, MMU_PAGESHIFT, %o0
4358	sllx	%o0, MMU_PAGESHIFT, %o0
4359	GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1,
4360		kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk,
4361		kvaszc2pfn_nohblk)
4362
4363kvaszc2pfn_hblk_found:
4364	/*
4365	 * %g3 = tte
4366	 * %o0 = vaddr
4367	 */
4368	brgez,a,pn %g3, 1f			/* check if tte is invalid */
4369	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4370	TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5)
4371	/*
4372	 * g3 = pfn
4373	 */
4374	ba,pt	%xcc, 1f
4375	  mov	%g3, %o0
4376
4377kvaszc2pfn_nohblk:
4378	mov	-1, %o0
4379
43801:
4381	retl
4382 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4383
4384	SET_SIZE(sfmmu_kvaszc2pfn)
4385
4386#endif /* lint */
4387
4388
4389
4390#if !defined(lint)
4391
4392/*
4393 * kpm lock used between trap level tsbmiss handler and kpm C level.
4394 */
4395#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
4396	mov     0xff, tmp1						;\
4397label1:									;\
4398	casa    [kpmlckp]asi, %g0, tmp1					;\
4399	brnz,pn tmp1, label1						;\
4400	mov     0xff, tmp1						;\
4401	membar  #LoadLoad
4402
4403#define KPMLOCK_EXIT(kpmlckp, asi)					\
4404	membar  #LoadStore|#StoreStore					;\
4405	sta     %g0, [kpmlckp]asi
4406
4407/*
4408 * Lookup a memseg for a given pfn and if found, return the physical
4409 * address of the corresponding struct memseg in mseg, otherwise
4410 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
4411 * tsbmp, %asi is assumed to be ASI_MEM.
4412 * This lookup is done by strictly traversing only the physical memseg
4413 * linkage. The more generic approach, to check the virtual linkage
4414 * before using the physical (used e.g. with hmehash buckets), cannot
4415 * be used here. Memory DR operations can run in parallel to this
4416 * lookup w/o any locks and updates of the physical and virtual linkage
4417 * cannot be done atomically wrt. to each other. Because physical
4418 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
4419 * as "physical NULL" pointer.
4420 */
4421#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
4422	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
4423	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
4424	udivx	pfn, mseg, mseg						;\
4425	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
4426	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
4427	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
4428	add	tmp1, mseg, tmp1					;\
4429	ldxa	[tmp1]%asi, mseg					;\
4430	cmp	mseg, MSEG_NULLPTR_PA					;\
4431	be,pn	%xcc, label/**/1		/* if not found */	;\
4432	  nop								;\
4433	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
4434	cmp	pfn, tmp1			/* pfn - pages_base */	;\
4435	blu,pn	%xcc, label/**/1					;\
4436	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
4437	cmp	pfn, tmp2			/* pfn - pages_end */	;\
4438	bgeu,pn	%xcc, label/**/1					;\
4439	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
4440	mulx	tmp1, PAGE_SIZE, tmp1					;\
4441	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
4442	add	tmp2, tmp1, tmp1			/* pp */	;\
4443	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
4444	cmp	tmp2, pfn						;\
4445	be,pt	%xcc, label/**/_ok			/* found */	;\
4446label/**/1:								;\
4447	/* brute force lookup */					;\
4448	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
4449	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
4450label/**/2:								;\
4451	cmp	mseg, MSEG_NULLPTR_PA					;\
4452	be,pn	%xcc, label/**/_ok		/* if not found */	;\
4453	  nop								;\
4454	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
4455	cmp	pfn, tmp1			/* pfn - pages_base */	;\
4456	blu,a,pt %xcc, label/**/2					;\
4457	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
4458	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
4459	cmp	pfn, tmp2			/* pfn - pages_end */	;\
4460	bgeu,a,pt %xcc, label/**/2					;\
4461	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
4462label/**/_ok:
4463
4464	/*
4465	 * kpm tsb miss handler large pages
4466	 * g1 = 8K kpm TSB entry pointer
4467	 * g2 = tag access register
4468	 * g3 = 4M kpm TSB entry pointer
4469	 */
4470	ALTENTRY(sfmmu_kpm_dtsb_miss)
4471	TT_TRACE(trace_tsbmiss)
4472
4473	CPU_INDEX(%g7, %g6)
4474	sethi	%hi(kpmtsbm_area), %g6
4475	sllx	%g7, KPMTSBM_SHIFT, %g7
4476	or	%g6, %lo(kpmtsbm_area), %g6
4477	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4478
4479	/* check enable flag */
4480	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4481	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4482	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4483	  nop
4484
4485	/* VA range check */
4486	ldx	[%g6 + KPMTSBM_VBASE], %g7
4487	cmp	%g2, %g7
4488	blu,pn	%xcc, sfmmu_tsb_miss
4489	  ldx	[%g6 + KPMTSBM_VEND], %g5
4490	cmp	%g2, %g5
4491	bgeu,pn	%xcc, sfmmu_tsb_miss
4492	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]
4493
4494	/*
4495	 * check TL tsbmiss handling flag
4496	 * bump tsbmiss counter
4497	 */
4498	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4499#ifdef	DEBUG
4500	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
4501	inc	%g5
4502	brz,pn	%g3, sfmmu_kpm_exception
4503	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4504#else
4505	inc	%g5
4506	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4507#endif
4508	/*
4509	 * At this point:
4510	 *  g1 = 8K kpm TSB pointer (not used)
4511	 *  g2 = tag access register
4512	 *  g3 = clobbered
4513	 *  g6 = per-CPU kpm tsbmiss area
4514	 *  g7 = kpm_vbase
4515	 */
4516
4517	/* vaddr2pfn */
4518	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
4519	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4520	srax    %g4, %g3, %g2			/* which alias range (r) */
4521	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
4522	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
4523
4524	/*
4525	 * Setup %asi
4526	 * mseg_pa = page_numtomemseg_nolock(pfn)
4527	 * if (mseg_pa == NULL) sfmmu_kpm_exception
4528	 * g2=pfn
4529	 */
4530	mov	ASI_MEM, %asi
4531	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
4532	cmp	%g3, MSEG_NULLPTR_PA
4533	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4534	  nop
4535
4536	/*
4537	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
4538	 * g2=pfn g3=mseg_pa
4539	 */
4540	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
4541	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4542	srlx	%g2, %g5, %g4
4543	sllx	%g4, %g5, %g4
4544	sub	%g4, %g7, %g4
4545	srlx	%g4, %g5, %g4
4546
4547	/*
4548	 * Validate inx value
4549	 * g2=pfn g3=mseg_pa g4=inx
4550	 */
4551#ifdef	DEBUG
4552	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4553	cmp	%g4, %g5			/* inx - nkpmpgs */
4554	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4555	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4556#else
4557	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4558#endif
4559	/*
4560	 * kp = &mseg_pa->kpm_pages[inx]
4561	 */
4562	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
4563	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
4564	add	%g5, %g4, %g5			/* kp */
4565
4566	/*
4567	 * KPMP_HASH(kp)
4568	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
4569	 */
4570	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4571	sub	%g7, 1, %g7			/* mask */
4572	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
4573	add	%g5, %g1, %g5			/* y = ksp + x */
4574	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4575
4576	/*
4577	 * Calculate physical kpm_page pointer
4578	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4579	 */
4580	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
4581	add	%g1, %g4, %g1			/* kp_pa */
4582
4583	/*
4584	 * Calculate physical hash lock address
4585	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
4586	 */
4587	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
4588	sllx	%g5, KPMHLK_SHIFT, %g5
4589	add	%g4, %g5, %g3
4590	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */
4591
4592	/*
4593	 * Assemble tte
4594	 * g1=kp_pa g2=pfn g3=hlck_pa
4595	 */
4596#ifdef sun4v
4597	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4598	sllx	%g5, 32, %g5
4599	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4600	or	%g4, TTE4M, %g4
4601	or	%g5, %g4, %g5
4602#else
4603	sethi	%hi(TTE_VALID_INT), %g4
4604	mov	TTE4M, %g5
4605	sllx	%g5, TTE_SZ_SHFT_INT, %g5
4606	or	%g5, %g4, %g5			/* upper part */
4607	sllx	%g5, 32, %g5
4608	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4609	or	%g5, %g4, %g5
4610#endif
4611	sllx	%g2, MMU_PAGESHIFT, %g4
4612	or	%g5, %g4, %g5			/* tte */
4613	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4614	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4615
4616	/*
4617	 * tsb dropin
4618	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
4619	 */
4620
4621	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4622	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
4623
4624	/* use C-handler if there's no go for dropin */
4625	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
4626	cmp	%g7, -1
4627	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
4628	  nop
4629
4630#ifdef	DEBUG
4631	/* double check refcnt */
4632	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
4633	brz,pn	%g7, 5f			/* let C-handler deal with this */
4634	  nop
4635#endif
4636
4637#ifndef sun4v
4638	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4639	mov	ASI_N, %g1
4640	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4641	movnz	%icc, ASI_MEM, %g1
4642	mov	%g1, %asi
4643#endif
4644
4645	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
4646	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
4647
4648	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4649	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4650
4651	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
4652
4653	/* KPMLOCK_EXIT(kpmlckp, asi) */
4654	KPMLOCK_EXIT(%g3, ASI_MEM)
4655
4656	/*
4657	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4658	 * point to trapstat's TSB miss return code (note that trapstat
4659	 * itself will patch the correct offset to add).
4660	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4661	 */
4662	rdpr	%tl, %g7
4663	cmp	%g7, 1
4664	ble	%icc, 0f
4665	sethi	%hi(KERNELBASE), %g6
4666	rdpr	%tpc, %g7
4667	or	%g6, %lo(KERNELBASE), %g6
4668	cmp	%g7, %g6
4669	bgeu	%xcc, 0f
4670	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
4671	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4672	wrpr	%g7, %tpc
4673	add	%g7, 4, %g7
4674	wrpr	%g7, %tnpc
46750:
4676	retry
46775:
4678	/* g3=hlck_pa */
4679	KPMLOCK_EXIT(%g3, ASI_MEM)
4680	ba,pt	%icc, sfmmu_kpm_exception
4681	  nop
4682	SET_SIZE(sfmmu_kpm_dtsb_miss)
4683
4684	/*
4685	 * kpm tsbmiss handler for smallpages
4686	 * g1 = 8K kpm TSB pointer
4687	 * g2 = tag access register
4688	 * g3 = 4M kpm TSB pointer
4689	 */
4690	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
4691	TT_TRACE(trace_tsbmiss)
4692	CPU_INDEX(%g7, %g6)
4693	sethi	%hi(kpmtsbm_area), %g6
4694	sllx	%g7, KPMTSBM_SHIFT, %g7
4695	or	%g6, %lo(kpmtsbm_area), %g6
4696	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4697
4698	/* check enable flag */
4699	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4700	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4701	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4702	  nop
4703
4704	/*
4705	 * VA range check
4706	 * On fail: goto sfmmu_tsb_miss
4707	 */
4708	ldx	[%g6 + KPMTSBM_VBASE], %g7
4709	cmp	%g2, %g7
4710	blu,pn	%xcc, sfmmu_tsb_miss
4711	  ldx	[%g6 + KPMTSBM_VEND], %g5
4712	cmp	%g2, %g5
4713	bgeu,pn	%xcc, sfmmu_tsb_miss
4714	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */
4715
4716	/*
4717	 * check TL tsbmiss handling flag
4718	 * bump tsbmiss counter
4719	 */
4720	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4721#ifdef	DEBUG
4722	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
4723	inc	%g5
4724	brz,pn	%g1, sfmmu_kpm_exception
4725	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4726#else
4727	inc	%g5
4728	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4729#endif
4730	/*
4731	 * At this point:
4732	 *  g1 = clobbered
4733	 *  g2 = tag access register
4734	 *  g3 = 4M kpm TSB pointer (not used)
4735	 *  g6 = per-CPU kpm tsbmiss area
4736	 *  g7 = kpm_vbase
4737	 */
4738
4739	/* vaddr2pfn */
4740	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
4741	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4742	srax    %g4, %g3, %g2			/* which alias range (r) */
4743	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
4744	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
4745
4746	/*
4747	 * Setup %asi
4748	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
4749	 * if (mseg not found) sfmmu_kpm_exception
4750	 * g2=pfn
4751	 */
4752	mov	ASI_MEM, %asi
4753	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
4754	cmp	%g3, MSEG_NULLPTR_PA
4755	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4756	  nop
4757
4758	/*
4759	 * inx = pfn - mseg_pa->kpm_pbase
4760	 * g2=pfn g3=mseg_pa
4761	 */
4762	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4763	sub	%g2, %g7, %g4
4764
4765#ifdef	DEBUG
4766	/*
4767	 * Validate inx value
4768	 * g2=pfn g3=mseg_pa g4=inx
4769	 */
4770	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4771	cmp	%g4, %g5			/* inx - nkpmpgs */
4772	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4773	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4774#else
4775	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4776#endif
4777	/* ksp = &mseg_pa->kpm_spages[inx] */
4778	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
4779	add	%g5, %g4, %g5			/* ksp */
4780
4781	/*
4782	 * KPMP_SHASH(kp)
4783	 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz
4784	 */
4785	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4786	sub	%g7, 1, %g7			/* mask */
4787	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
4788	add	%g5, %g1, %g5			/* y = ksp + x */
4789	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4790
4791	/*
4792	 * Calculate physical kpm_spage pointer
4793	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4794	 */
4795	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
4796	add	%g1, %g4, %g1			/* ksp_pa */
4797
4798	/*
4799	 * Calculate physical hash lock address.
4800	 * Note: Changes in kpm_shlk_t must be reflected here.
4801	 * g1=ksp_pa g2=pfn g5=hashinx
4802	 */
4803	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
4804	sllx	%g5, KPMSHLK_SHIFT, %g5
4805	add	%g4, %g5, %g3			/* hlck_pa */
4806
4807	/*
4808	 * Assemble tte
4809	 * g1=ksp_pa g2=pfn g3=hlck_pa
4810	 */
4811	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4812	sllx	%g5, 32, %g5
4813	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4814	or	%g5, %g4, %g5
4815	sllx	%g2, MMU_PAGESHIFT, %g4
4816	or	%g5, %g4, %g5			/* tte */
4817	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4818	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4819
4820	/*
4821	 * tsb dropin
4822	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte
4823	 */
4824
4825	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4826	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
4827
4828	/* use C-handler if there's no go for dropin */
4829	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */
4830	cmp	%g7, -1
4831	bne,pn	%xcc, 5f
4832	  nop
4833
4834#ifndef sun4v
4835	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4836	mov	ASI_N, %g1
4837	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4838	movnz	%icc, ASI_MEM, %g1
4839	mov	%g1, %asi
4840#endif
4841
4842	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
4843	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
4844
4845	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4846	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4847
4848	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
4849
4850	/* KPMLOCK_EXIT(kpmlckp, asi) */
4851	KPMLOCK_EXIT(%g3, ASI_MEM)
4852
4853	/*
4854	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4855	 * point to trapstat's TSB miss return code (note that trapstat
4856	 * itself will patch the correct offset to add).
4857	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4858	 */
4859	rdpr	%tl, %g7
4860	cmp	%g7, 1
4861	ble	%icc, 0f
4862	sethi	%hi(KERNELBASE), %g6
4863	rdpr	%tpc, %g7
4864	or	%g6, %lo(KERNELBASE), %g6
4865	cmp	%g7, %g6
4866	bgeu	%xcc, 0f
4867	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
4868	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4869	wrpr	%g7, %tpc
4870	add	%g7, 4, %g7
4871	wrpr	%g7, %tnpc
48720:
4873	retry
48745:
4875	/* g3=hlck_pa */
4876	KPMLOCK_EXIT(%g3, ASI_MEM)
4877	ba,pt	%icc, sfmmu_kpm_exception
4878	  nop
4879	SET_SIZE(sfmmu_kpm_dtsb_miss_small)
4880
4881#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
4882#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
4883#endif
4884
4885#endif /* lint */
4886
4887#ifdef	lint
4888/*
4889 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
4890 * Called from C-level, sets/clears "go" indication for trap level handler.
4891 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
4892 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
4893 * Assumes khl_mutex is held when called from C-level.
4894 */
4895/* ARGSUSED */
4896void
4897sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
4898{
4899}
4900
4901/*
4902 * kpm_smallpages: stores val to byte at address mapped within
4903 * low level lock brackets. The old value is returned.
4904 * Called from C-level.
4905 */
4906/* ARGSUSED */
4907int
4908sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val)
4909{
4910	return (0);
4911}
4912
4913#else /* lint */
4914
4915	.seg	".data"
4916sfmmu_kpm_tsbmtl_panic:
4917	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
4918	.byte	0
4919sfmmu_kpm_stsbmtl_panic:
4920	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
4921	.byte	0
4922	.align	4
4923	.seg	".text"
4924
4925	ENTRY_NP(sfmmu_kpm_tsbmtl)
4926	rdpr	%pstate, %o3
4927	/*
4928	 * %o0 = &kp_refcntc
4929	 * %o1 = &khl_lock
4930	 * %o2 = 0/1 (off/on)
4931	 * %o3 = pstate save
4932	 */
4933#ifdef DEBUG
4934	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4935	bnz,pt %icc, 1f				/* disabled, panic	 */
4936	  nop
4937	save	%sp, -SA(MINFRAME), %sp
4938	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
4939	call	panic
4940	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
4941	ret
4942	restore
49431:
4944#endif /* DEBUG */
4945	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4946
4947	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
4948	mov	-1, %o5
4949	brz,a	%o2, 2f
4950	  mov	0, %o5
49512:
4952	sth	%o5, [%o0]
4953	KPMLOCK_EXIT(%o1, ASI_N)
4954
4955	retl
4956	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4957	SET_SIZE(sfmmu_kpm_tsbmtl)
4958
4959	ENTRY_NP(sfmmu_kpm_stsbmtl)
4960	rdpr	%pstate, %o3
4961	/*
4962	 * %o0 = &mapped
4963	 * %o1 = &kshl_lock
4964	 * %o2 = val
4965	 * %o3 = pstate save
4966	 */
4967#ifdef DEBUG
4968	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4969	bnz,pt %icc, 1f				/* disabled, panic	 */
4970	  nop
4971	save	%sp, -SA(MINFRAME), %sp
4972	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
4973	call	panic
4974	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
4975	ret
4976	restore
49771:
4978#endif /* DEBUG */
4979	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4980
4981	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
4982	ldsb	[%o0], %o5
4983	stb	%o2, [%o0]
4984	KPMLOCK_EXIT(%o1, ASI_N)
4985
4986	mov	%o5, %o0			/* return old val */
4987	retl
4988	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4989	SET_SIZE(sfmmu_kpm_stsbmtl)
4990
4991#endif /* lint */
4992
4993#ifndef lint
4994#ifdef sun4v
4995	/*
4996	 * User/kernel data miss w// multiple TSBs
4997	 * The first probe covers 8K, 64K, and 512K page sizes,
4998	 * because 64K and 512K mappings are replicated off 8K
4999	 * pointer.  Second probe covers 4M page size only.
5000	 *
5001	 * MMU fault area contains miss address and context.
5002	 */
5003	ALTENTRY(sfmmu_slow_dmmu_miss)
5004	GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)	! %g2 = ptagacc, %g3 = ctx type
5005
5006slow_miss_common:
5007	/*
5008	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
5009	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
5010	 */
5011	brnz,pt	%g3, 8f			! check for user context
5012	  nop
5013
5014	/*
5015	 * Kernel miss
5016	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
5017	 * branch to sfmmu_tsb_miss_tt to handle it.
5018	 */
5019	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
5020sfmmu_dslow_patch_ktsb_base:
5021	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
5022sfmmu_dslow_patch_ktsb_szcode:
5023	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
5024
5025	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
5026	! %g1 = First TSB entry pointer, as TSB miss handler expects
5027
5028	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
5029sfmmu_dslow_patch_ktsb4m_base:
5030	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
5031sfmmu_dslow_patch_ktsb4m_szcode:
5032	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
5033
5034	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
5035	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
5036	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
5037	.empty
5038
50398:
5040	/*
5041	 * User miss
5042	 * Get first TSB pointer in %g1
5043	 * Get second TSB pointer (or NULL if no second TSB) in %g3
5044	 * Branch to sfmmu_tsb_miss_tt to handle it
5045	 */
5046	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
5047	/* %g1 = first TSB entry ptr now, %g2 preserved */
5048
5049	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
5050	brlz,pt %g3, sfmmu_tsb_miss_tt		/* done if no 2nd TSB */
5051	  nop
5052
5053	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
5054	/* %g3 = second TSB entry ptr now, %g2 preserved */
50559:
5056	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
5057	.empty
5058	SET_SIZE(sfmmu_slow_dmmu_miss)
5059
5060
5061	/*
5062	 * User/kernel instruction miss w/ multiple TSBs
5063	 * The first probe covers 8K, 64K, and 512K page sizes,
5064	 * because 64K and 512K mappings are replicated off 8K
5065	 * pointer.  Second probe covers 4M page size only.
5066	 *
5067	 * MMU fault area contains miss address and context.
5068	 */
5069	ALTENTRY(sfmmu_slow_immu_miss)
5070	GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)
5071	ba,a,pt	%xcc, slow_miss_common
5072	SET_SIZE(sfmmu_slow_immu_miss)
5073
5074#endif /* sun4v */
5075#endif	/* lint */
5076
5077#ifndef lint
5078
5079/*
5080 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
5081 */
5082	.seg	".data"
5083	.align	64
5084	.global tsbmiss_area
5085tsbmiss_area:
5086	.skip	(TSBMISS_SIZE * NCPU)
5087
5088	.align	64
5089	.global kpmtsbm_area
5090kpmtsbm_area:
5091	.skip	(KPMTSBM_SIZE * NCPU)
5092#endif	/* lint */
5093