sfmmu_asm.s revision ac20c57d6652cecf7859e3346336b9a48e5d5f82
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#pragma ident	"%Z%%M%	%I%	%E% SMI"
27
28/*
29 * SFMMU primitives.  These primitives should only be used by sfmmu
30 * routines.
31 */
32
33#if defined(lint)
34#include <sys/types.h>
35#else	/* lint */
36#include "assym.h"
37#endif	/* lint */
38
39#include <sys/asm_linkage.h>
40#include <sys/machtrap.h>
41#include <sys/machasi.h>
42#include <sys/sun4asi.h>
43#include <sys/pte.h>
44#include <sys/mmu.h>
45#include <vm/hat_sfmmu.h>
46#include <vm/seg_spt.h>
47#include <sys/machparam.h>
48#include <sys/privregs.h>
49#include <sys/scb.h>
50#include <sys/intreg.h>
51#include <sys/machthread.h>
52#include <sys/intr.h>
53#include <sys/clock.h>
54#include <sys/trapstat.h>
55
56#ifdef TRAPTRACE
57#include <sys/traptrace.h>
58
59/*
60 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
61 */
62#define	TT_TRACE(label)		\
63	ba	label		;\
64	rd	%pc, %g7
65#else
66
67#define	TT_TRACE(label)
68
69#endif /* TRAPTRACE */
70
71#ifndef	lint
72
73#if (TTE_SUSPEND_SHIFT > 0)
74#define	TTE_SUSPEND_INT_SHIFT(reg)				\
75	sllx	reg, TTE_SUSPEND_SHIFT, reg
76#else
77#define	TTE_SUSPEND_INT_SHIFT(reg)
78#endif
79
80#endif /* lint */
81
82#ifndef	lint
83
84/*
85 * Assumes TSBE_TAG is 0
86 * Assumes TSBE_INTHI is 0
87 * Assumes TSBREG.split is 0
88 */
89
90#if TSBE_TAG != 0
91#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
92#endif
93
94#if TSBTAG_INTHI != 0
95#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
96#endif
97
98/*
99 * The following code assumes the tsb is not split.
100 *
101 * With TSBs no longer shared between processes, it's no longer
102 * necessary to hash the context bits into the tsb index to get
103 * tsb coloring; the new implementation treats the TSB as a
104 * direct-mapped, virtually-addressed cache.
105 *
106 * In:
107 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
108 *    tsbbase = base address of TSB (clobbered)
109 *    tagacc = tag access register (clobbered)
110 *    szc = size code of TSB (ro)
111 *    tmp = scratch reg
112 * Out:
113 *    tsbbase = pointer to entry in TSB
114 */
115#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
116	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
117	srlx	tagacc, vpshift, tagacc 				;\
118	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
119	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
120	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
121	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
122	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */
123
124/*
125 * When the kpm TSB is used it is assumed that it is direct mapped
126 * using (vaddr>>vpshift)%tsbsz as the index.
127 *
128 * Note that, for now, the kpm TSB and kernel TSB are the same for
129 * each mapping size.  However that need not always be the case.  If
130 * the trap handlers are updated to search a different TSB for kpm
131 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
132 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
133 *
134 * In:
135 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
136 *    vaddr = virtual address (clobbered)
137 *    tsbp, szc, tmp = scratch
138 * Out:
139 *    tsbp = pointer to entry in TSB
140 */
141#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
142	cmp	vpshift, MMU_PAGESHIFT					;\
143	bne,pn	%icc, 1f		/* branch if large case */	;\
144	  sethi	%hi(kpmsm_tsbsz), szc					;\
145	sethi	%hi(kpmsm_tsbbase), tsbp				;\
146	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
147	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
148	ba,pt	%icc, 2f						;\
149	  nop								;\
1501:	sethi	%hi(kpm_tsbsz), szc					;\
151	sethi	%hi(kpm_tsbbase), tsbp					;\
152	ld	[szc + %lo(kpm_tsbsz)], szc				;\
153	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
1542:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
155
156/*
157 * Lock the TSBE at virtual address tsbep.
158 *
159 * tsbep = TSBE va (ro)
160 * tmp1, tmp2 = scratch registers (clobbered)
161 * label = label to use for branches (text)
162 * %asi = ASI to use for TSB access
163 *
164 * NOTE that we flush the TSB using fast VIS instructions that
165 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
166 * not be treated as a locked entry or we'll get stuck spinning on
167 * an entry that isn't locked but really invalid.
168 */
169
170#if defined(UTSB_PHYS)
171
172#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
173	lda	[tsbep]ASI_MEM, tmp1					;\
174label:									;\
175	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
176	cmp	tmp1, tmp2 						;\
177	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
178	  lda	[tsbep]ASI_MEM, tmp1					;\
179	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
180	cmp	tmp1, tmp2 						;\
181	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
182	  lda	[tsbep]ASI_MEM, tmp1					;\
183	/* tsbe lock acquired */					;\
184	membar #StoreStore
185
186#else /* UTSB_PHYS */
187
188#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
189	lda	[tsbep]%asi, tmp1					;\
190label:									;\
191	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
192	cmp	tmp1, tmp2 						;\
193	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
194	  lda	[tsbep]%asi, tmp1					;\
195	casa	[tsbep]%asi, tmp1, tmp2					;\
196	cmp	tmp1, tmp2 						;\
197	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
198	  lda	[tsbep]%asi, tmp1					;\
199	/* tsbe lock acquired */					;\
200	membar #StoreStore
201
202#endif /* UTSB_PHYS */
203
204/*
205 * Atomically write TSBE at virtual address tsbep.
206 *
207 * tsbep = TSBE va (ro)
208 * tte = TSBE TTE (ro)
209 * tagtarget = TSBE tag (ro)
210 * %asi = ASI to use for TSB access
211 */
212
213#if defined(UTSB_PHYS)
214
215#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
216	add	tsbep, TSBE_TTE, tmp1					;\
217	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
218	membar #StoreStore						;\
219	add	tsbep, TSBE_TAG, tmp1					;\
220	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */
221
222#else /* UTSB_PHYS */
223
224#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
225	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
226	membar #StoreStore						;\
227	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
228
229#endif /* UTSB_PHYS */
230
231/*
232 * Load an entry into the TSB at TL > 0.
233 *
234 * tsbep = pointer to the TSBE to load as va (ro)
235 * tte = value of the TTE retrieved and loaded (wo)
236 * tagtarget = tag target register.  To get TSBE tag to load,
237 *   we need to mask off the context and leave only the va (clobbered)
238 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
239 * tmp1, tmp2 = scratch registers
240 * label = label to use for branches (text)
241 * %asi = ASI to use for TSB access
242 */
243
244#if defined(UTSB_PHYS)
245
246#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
247	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
248	/*								;\
249	 * I don't need to update the TSB then check for the valid tte.	;\
250	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
251	 * we always invalidate the hash table before we unload the TSB.;\
252	 */								;\
253	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
254	ldxa	[ttepa]ASI_MEM, tte					;\
255	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
256	sethi	%hi(TSBTAG_INVALID), tmp2				;\
257	add	tsbep, TSBE_TAG, tmp1					;\
258	brgez,a,pn tte, label/**/f					;\
259	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
260	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
261label:
262
263#else /* UTSB_PHYS */
264
265#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
266	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
267	/*								;\
268	 * I don't need to update the TSB then check for the valid tte.	;\
269	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
270	 * we always invalidate the hash table before we unload the TSB.;\
271	 */								;\
272	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
273	ldxa	[ttepa]ASI_MEM, tte					;\
274	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
275	sethi	%hi(TSBTAG_INVALID), tmp2				;\
276	brgez,a,pn tte, label/**/f					;\
277	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
278	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
279label:
280
281#endif /* UTSB_PHYS */
282
283/*
284 * Load a 32M/256M Panther TSB entry into the TSB at TL > 0,
285 *   for ITLB synthesis.
286 *
287 * tsbep = pointer to the TSBE to load as va (ro)
288 * tte = 4M pfn offset (in), value of the TTE retrieved and loaded (out)
289 *   with exec_perm turned off and exec_synth turned on
290 * tagtarget = tag target register.  To get TSBE tag to load,
291 *   we need to mask off the context and leave only the va (clobbered)
292 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
293 * tmp1, tmp2 = scratch registers
294 * label = label to use for branch (text)
295 * %asi = ASI to use for TSB access
296 */
297
298#define	TSB_UPDATE_TL_PN(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
299	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
300	/*								;\
301	 * I don't need to update the TSB then check for the valid tte.	;\
302	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
303	 * we always invalidate the hash table before we unload the TSB.;\
304	 * Or in 4M pfn offset to TTE and set the exec_perm bit to 0	;\
305	 * and exec_synth bit to 1.					;\
306	 */								;\
307	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
308	mov	tte, tmp1						;\
309	ldxa	[ttepa]ASI_MEM, tte					;\
310	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
311	sethi	%hi(TSBTAG_INVALID), tmp2				;\
312	brgez,a,pn tte, label/**/f					;\
313	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
314	or	tte, tmp1, tte						;\
315	andn	tte, TTE_EXECPRM_INT, tte				;\
316	or	tte, TTE_E_SYNTH_INT, tte				;\
317	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
318label:
319
320/*
321 * Build a 4M pfn offset for a Panther 32M/256M page, for ITLB synthesis.
322 *
323 * tte = value of the TTE, used to get tte_size bits (ro)
324 * tagaccess = tag access register, used to get 4M pfn bits (ro)
325 * pfn = 4M pfn bits shifted to offset for tte (out)
326 * tmp1 = scratch register
327 * label = label to use for branch (text)
328 */
329
330#define	GET_4M_PFN_OFF(tte, tagaccess, pfn, tmp, label)			\
331	/*								;\
332	 * Get 4M bits from tagaccess for 32M, 256M pagesizes.		;\
333	 * Return them, shifted, in pfn.				;\
334	 */								;\
335	srlx	tagaccess, MMU_PAGESHIFT4M, tagaccess			;\
336	srlx	tte, TTE_SZ_SHFT, tmp		/* isolate the */	;\
337	andcc	tmp, TTE_SZ_BITS, %g0		/* tte_size bits */	;\
338	bz,a,pt %icc, label/**/f		/* if 0, is */		;\
339	  and	tagaccess, 0x7, tagaccess	/* 32M page size */	;\
340	and	tagaccess, 0x3f, tagaccess /* else 256M page size */	;\
341label:									;\
342	sllx	tagaccess, MMU_PAGESHIFT4M, pfn
343
344/*
345 * Add 4M TTE size code to a tte for a Panther 32M/256M page,
346 * for ITLB synthesis.
347 *
348 * tte = value of the TTE, used to get tte_size bits (rw)
349 * tmp1 = scratch register
350 */
351
352#define	SET_TTE4M_PN(tte, tmp)						\
353	/*								;\
354	 * Set 4M pagesize tte bits. 					;\
355	 */								;\
356	set	TTE4M, tmp						;\
357	sllx	tmp, TTE_SZ_SHFT, tmp					;\
358	or	tte, tmp, tte
359
360/*
361 * Load an entry into the TSB at TL=0.
362 *
363 * tsbep = pointer to the TSBE to load as va (ro)
364 * tteva = pointer to the TTE to load as va (ro)
365 * tagtarget = TSBE tag to load (which contains no context), synthesized
366 * to match va of MMU tag target register only (ro)
367 * tmp1, tmp2 = scratch registers (clobbered)
368 * label = label to use for branches (text)
369 * %asi = ASI to use for TSB access
370 */
371
372#if defined(UTSB_PHYS)
373
374#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
375	/* can't rd tteva after locking tsb because it can tlb miss */	;\
376	ldx	[tteva], tteva			/* load tte */		;\
377	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
378	sethi	%hi(TSBTAG_INVALID), tmp2				;\
379	add	tsbep, TSBE_TAG, tmp1					;\
380	brgez,a,pn tteva, label/**/f					;\
381	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
382	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
383label:
384
385#else /* UTSB_PHYS */
386
387#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
388	/* can't rd tteva after locking tsb because it can tlb miss */	;\
389	ldx	[tteva], tteva			/* load tte */		;\
390	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
391	sethi	%hi(TSBTAG_INVALID), tmp2				;\
392	brgez,a,pn tteva, label/**/f					;\
393	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
394	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
395label:
396
397#endif /* UTSB_PHYS */
398
399/*
400 * Invalidate a TSB entry in the TSB.
401 *
402 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
403 *	 about this earlier to ensure this is true.  Thus when we are
404 *	 directly referencing tsbep below, we are referencing the tte_tag
405 *	 field of the TSBE.  If this  offset ever changes, the code below
406 *	 will need to be modified.
407 *
408 * tsbep = pointer to TSBE as va (ro)
409 * tag = invalidation is done if this matches the TSBE tag (ro)
410 * tmp1 - tmp3 = scratch registers (clobbered)
411 * label = label name to use for branches (text)
412 * %asi = ASI to use for TSB access
413 */
414
415#if defined(UTSB_PHYS)
416
417#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
418	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
419	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
420label/**/1:								;\
421	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
422	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
423	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
424	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
425	cmp	tag, tmp3		/* compare tags */		;\
426	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
427	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
428	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
429	cmp	tmp1, tmp3		/* if not successful */		;\
430	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
431	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
432label/**/2:
433
434#else /* UTSB_PHYS */
435
436#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
437	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
438	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
439label/**/1:								;\
440	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
441	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
442	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
443	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
444	cmp	tag, tmp3		/* compare tags */		;\
445	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
446	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
447	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
448	cmp	tmp1, tmp3		/* if not successful */		;\
449	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
450	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
451label/**/2:
452
453#endif /* UTSB_PHYS */
454
455#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
456#error	- TSB_SOFTSZ_MASK too small
457#endif
458
459
460/*
461 * An implementation of setx which will be hot patched at run time.
462 * since it is being hot patched, there is no value passed in.
463 * Thus, essentially we are implementing
464 *	setx value, tmp, dest
465 * where value is RUNTIME_PATCH (aka 0) in this case.
466 */
467#define	RUNTIME_PATCH_SETX(dest, tmp)					\
468	sethi	%hh(RUNTIME_PATCH), tmp					;\
469	sethi	%lm(RUNTIME_PATCH), dest				;\
470	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
471	or	dest, %lo(RUNTIME_PATCH), dest				;\
472	sllx	tmp, 32, tmp						;\
473	nop				/* for perf reasons */		;\
474	or	tmp, dest, dest		/* contents of patched value */
475
476
477#endif (lint)
478
479
480#if defined (lint)
481
482/*
483 * sfmmu related subroutines
484 */
485uint_t
486sfmmu_disable_intrs()
487{ return(0); }
488
489/* ARGSUSED */
490void
491sfmmu_enable_intrs(uint_t pstate_save)
492{}
493
494/* ARGSUSED */
495int
496sfmmu_alloc_ctx(sfmmu_t *sfmmup, int allocflag, struct cpu *cp, int shflag)
497{ return(0); }
498
499/*
500 * Use cas, if tte has changed underneath us then reread and try again.
501 * In the case of a retry, it will update sttep with the new original.
502 */
503/* ARGSUSED */
504int
505sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
506{ return(0); }
507
508/*
509 * Use cas, if tte has changed underneath us then return 1, else return 0
510 */
511/* ARGSUSED */
512int
513sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
514{ return(0); }
515
516/* ARGSUSED */
517void
518sfmmu_copytte(tte_t *sttep, tte_t *dttep)
519{}
520
521/*ARGSUSED*/
522struct tsbe *
523sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
524{ return(0); }
525
526/*ARGSUSED*/
527uint64_t
528sfmmu_make_tsbtag(caddr_t va)
529{ return(0); }
530
531#else	/* lint */
532
533	.seg	".data"
534	.global	sfmmu_panic1
535sfmmu_panic1:
536	.asciz	"sfmmu_asm: interrupts already disabled"
537
538	.global	sfmmu_panic3
539sfmmu_panic3:
540	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"
541
542	.global	sfmmu_panic4
543sfmmu_panic4:
544	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"
545
546	.global	sfmmu_panic5
547sfmmu_panic5:
548	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"
549
550	.global	sfmmu_panic6
551sfmmu_panic6:
552	.asciz	"sfmmu_asm: interrupts not disabled"
553
554	.global	sfmmu_panic7
555sfmmu_panic7:
556	.asciz	"sfmmu_asm: kernel as"
557
558	.global	sfmmu_panic8
559sfmmu_panic8:
560	.asciz	"sfmmu_asm: gnum is zero"
561
562	.global	sfmmu_panic9
563sfmmu_panic9:
564	.asciz	"sfmmu_asm: cnum is greater than MAX_SFMMU_CTX_VAL"
565
566	.global	sfmmu_panic10
567sfmmu_panic10:
568	.asciz	"sfmmu_asm: valid SCD with no 3rd scd TSB"
569
570        ENTRY(sfmmu_disable_intrs)
571        rdpr    %pstate, %o0
572#ifdef DEBUG
573	PANIC_IF_INTR_DISABLED_PSTR(%o0, sfmmu_di_l0, %g1)
574#endif /* DEBUG */
575        retl
576          wrpr   %o0, PSTATE_IE, %pstate
577        SET_SIZE(sfmmu_disable_intrs)
578
579	ENTRY(sfmmu_enable_intrs)
580        retl
581          wrpr    %g0, %o0, %pstate
582        SET_SIZE(sfmmu_enable_intrs)
583
584/*
585 * This routine is called both by resume() and sfmmu_get_ctx() to
586 * allocate a new context for the process on a MMU.
587 * if allocflag == 1, then alloc ctx when HAT mmu cnum == INVALID .
588 * if allocflag == 0, then do not alloc ctx if HAT mmu cnum == INVALID, which
589 * is the case when sfmmu_alloc_ctx is called from resume().
590 *
591 * The caller must disable interrupts before entering this routine.
592 * To reduce ctx switch overhead, the code contains both 'fast path' and
593 * 'slow path' code. The fast path code covers the common case where only
594 * a quick check is needed and the real ctx allocation is not required.
595 * It can be done without holding the per-process (PP) lock.
596 * The 'slow path' code must be protected by the PP Lock and performs ctx
597 * allocation.
598 * Hardware context register and HAT mmu cnum are updated accordingly.
599 *
600 * %o0 - sfmmup
601 * %o1 - allocflag
602 * %o2 - CPU
603 * %o3 - sfmmu private/shared flag
604 *
605 * ret - 0: no ctx is allocated
606 *       1: a ctx is allocated
607 */
608        ENTRY_NP(sfmmu_alloc_ctx)
609
610#ifdef DEBUG
611	sethi   %hi(ksfmmup), %g1
612	ldx     [%g1 + %lo(ksfmmup)], %g1
613	cmp     %g1, %o0
614	bne,pt   %xcc, 0f
615	  nop
616
617	sethi   %hi(panicstr), %g1		! if kernel as, panic
618        ldx     [%g1 + %lo(panicstr)], %g1
619        tst     %g1
620        bnz,pn  %icc, 7f
621          nop
622
623	sethi	%hi(sfmmu_panic7), %o0
624	call	panic
625	  or	%o0, %lo(sfmmu_panic7), %o0
626
6277:
628	retl
629	  mov	%g0, %o0			! %o0 = ret = 0
630
6310:
632	PANIC_IF_INTR_ENABLED_PSTR(sfmmu_ei_l1, %g1)
633#endif /* DEBUG */
634
635	mov	%o3, %g1			! save sfmmu pri/sh flag in %g1
636
637	! load global mmu_ctxp info
638	ldx	[%o2 + CPU_MMU_CTXP], %o3		! %o3 = mmu_ctx_t ptr
639        lduw	[%o2 + CPU_MMU_IDX], %g2		! %g2 = mmu index
640
641	! load global mmu_ctxp gnum
642	ldx	[%o3 + MMU_CTX_GNUM], %o4		! %o4 = mmu_ctxp->gnum
643
644#ifdef DEBUG
645	cmp	%o4, %g0		! mmu_ctxp->gnum should never be 0
646	bne,pt	%xcc, 3f
647	  nop
648
649	sethi   %hi(panicstr), %g1	! test if panicstr is already set
650        ldx     [%g1 + %lo(panicstr)], %g1
651        tst     %g1
652        bnz,pn  %icc, 1f
653          nop
654
655	sethi	%hi(sfmmu_panic8), %o0
656	call	panic
657	  or	%o0, %lo(sfmmu_panic8), %o0
6581:
659	retl
660	  mov	%g0, %o0			! %o0 = ret = 0
6613:
662#endif
663
664	! load HAT sfmmu_ctxs[mmuid] gnum, cnum
665
666	sllx	%g2, SFMMU_MMU_CTX_SHIFT, %g2
667	add	%o0, %g2, %g2		! %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
668
669	/*
670	 * %g5 = sfmmu gnum returned
671	 * %g6 = sfmmu cnum returned
672	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
673	 * %g4 = scratch
674	 *
675	 * Fast path code, do a quick check.
676	 */
677	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
678
679	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
680	bne,pt	%icc, 1f			! valid hat cnum, check gnum
681	  nop
682
683	! cnum == INVALID, check allocflag
684	mov	%g0, %g4	! %g4 = ret = 0
685	brz,pt  %o1, 8f		! allocflag == 0, skip ctx allocation, bail
686	  mov	%g6, %o1
687
688	! (invalid HAT cnum) && (allocflag == 1)
689	ba,pt	%icc, 2f
690	  nop
6911:
692	! valid HAT cnum, check gnum
693	cmp	%g5, %o4
694	mov	1, %g4				!%g4 = ret = 1
695	be,a,pt	%icc, 8f			! gnum unchanged, go to done
696	  mov	%g6, %o1
697
6982:
699	/*
700	 * Grab per process (PP) sfmmu_ctx_lock spinlock,
701	 * followed by the 'slow path' code.
702	 */
703	ldstub	[%o0 + SFMMU_CTX_LOCK], %g3	! %g3 = per process (PP) lock
7043:
705	brz	%g3, 5f
706	  nop
7074:
708	brnz,a,pt       %g3, 4b				! spin if lock is 1
709	  ldub	[%o0 + SFMMU_CTX_LOCK], %g3
710	ba	%xcc, 3b				! retry the lock
711	  ldstub	[%o0 + SFMMU_CTX_LOCK], %g3    ! %g3 = PP lock
712
7135:
714	membar  #LoadLoad
715	/*
716	 * %g5 = sfmmu gnum returned
717	 * %g6 = sfmmu cnum returned
718	 * %g2 = &sfmmu_ctxs[mmuid] - SFMMU_CTXS
719	 * %g4 = scratch
720	 */
721	SFMMU_MMUID_GNUM_CNUM(%g2, %g5, %g6, %g4)
722
723	cmp	%g6, INVALID_CONTEXT		! hat cnum == INVALID ??
724	bne,pt	%icc, 1f			! valid hat cnum, check gnum
725	  nop
726
727	! cnum == INVALID, check allocflag
728	mov	%g0, %g4	! %g4 = ret = 0
729	brz,pt	%o1, 2f		! allocflag == 0, called from resume, set hw
730	  mov	%g6, %o1
731
732	! (invalid HAT cnum) && (allocflag == 1)
733	ba,pt	%icc, 6f
734	  nop
7351:
736	! valid HAT cnum, check gnum
737	cmp	%g5, %o4
738	mov	1, %g4				! %g4 = ret  = 1
739	be,a,pt	%icc, 2f			! gnum unchanged, go to done
740	  mov	%g6, %o1
741
742	ba,pt	%icc, 6f
743	  nop
7442:
745	membar  #LoadStore|#StoreStore
746	ba,pt %icc, 8f
747	  clrb  [%o0 + SFMMU_CTX_LOCK]
7486:
749	/*
750	 * We get here if we do not have a valid context, or
751	 * the HAT gnum does not match global gnum. We hold
752	 * sfmmu_ctx_lock spinlock. Allocate that context.
753	 *
754	 * %o3 = mmu_ctxp
755	 */
756	add	%o3, MMU_CTX_CNUM, %g3
757	ld	[%o3 + MMU_CTX_NCTXS], %g4
758
759	/*
760         * %g2 = &sfmmu_ctx_t[mmuid] - SFMMU_CTXS;
761         * %g3 = mmu cnum address
762	 * %g4 = mmu nctxs
763	 *
764	 * %o0 = sfmmup
765	 * %o1 = mmu current cnum value (used as new cnum)
766	 * %o4 = mmu gnum
767	 *
768	 * %o5 = scratch
769	 */
770	ld	[%g3], %o1
7710:
772	cmp	%o1, %g4
773	bl,a,pt %icc, 1f
774	  add	%o1, 1, %o5		! %o5 = mmu_ctxp->cnum + 1
775
776	/*
777	 * cnum reachs max, bail, so wrap around can be performed later.
778	 */
779	set	INVALID_CONTEXT, %o1
780	/*
781	 * When the routine is called by shared ctx, we want to set
782	 * both private and shared ctx regs to INVALID. In order to
783	 * do so, we set the sfmmu priv/shared flag to 'private' regardless.
784	 * so that private ctx reg will be set to invalid.
785	 * Note that values written to private context register are
786	 * automatically written to shared context register as well.
787	 */
788	mov	%g0, %g1		! %g1 = sfmmu private/shared flag
789	mov	%g0, %g4		! %g4 = ret = 0
790
791	membar  #LoadStore|#StoreStore
792	ba,pt	%icc, 8f
793	  clrb	[%o0 + SFMMU_CTX_LOCK]
7941:
795	! %g3 = addr of mmu_ctxp->cnum
796	! %o5 = mmu_ctxp->cnum + 1
797	cas	[%g3], %o1, %o5
798	cmp	%o1, %o5
799	bne,a,pn %xcc, 0b	! cas failed
800	  ld	[%g3], %o1
801
802#ifdef DEBUG
803        set	MAX_SFMMU_CTX_VAL, %o5
804	cmp	%o1, %o5
805	ble,pt %icc, 2f
806	  nop
807
808	sethi	%hi(sfmmu_panic9), %o0
809	call	panic
810	  or	%o0, %lo(sfmmu_panic9), %o0
8112:
812#endif
813	! update hat gnum and cnum
814	sllx	%o4, SFMMU_MMU_GNUM_RSHIFT, %o4
815	or	%o4, %o1, %o4
816	stx	%o4, [%g2 + SFMMU_CTXS]
817
818	membar  #LoadStore|#StoreStore
819	clrb	[%o0 + SFMMU_CTX_LOCK]
820
821	mov	1, %g4			! %g4 = ret = 1
8228:
823	/*
824	 * program the secondary context register
825	 *
826	 * %o1 = cnum
827	 * %g1 = sfmmu private/shared flag (0:private,  1:shared)
828	 */
829
830#ifdef	sun4u
831	ldub	[%o0 + SFMMU_CEXT], %o2
832	sll	%o2, CTXREG_EXT_SHIFT, %o2
833	or	%o1, %o2, %o1
834#endif
835	SET_SECCTX(%o1, %g1, %o4, %o5)
836
837	retl
838	  mov	%g4, %o0			! %o0 = ret
839
840	SET_SIZE(sfmmu_alloc_ctx)
841
842	ENTRY_NP(sfmmu_modifytte)
843	ldx	[%o2], %g3			/* current */
844	ldx	[%o0], %g1			/* original */
8452:
846	ldx	[%o1], %g2			/* modified */
847	cmp	%g2, %g3			/* is modified = current? */
848	be,a,pt	%xcc,1f				/* yes, don't write */
849	stx	%g3, [%o0]			/* update new original */
850	casx	[%o2], %g1, %g2
851	cmp	%g1, %g2
852	be,pt	%xcc, 1f			/* cas succeeded - return */
853	  nop
854	ldx	[%o2], %g3			/* new current */
855	stx	%g3, [%o0]			/* save as new original */
856	ba,pt	%xcc, 2b
857	  mov	%g3, %g1
8581:	retl
859	membar	#StoreLoad
860	SET_SIZE(sfmmu_modifytte)
861
862	ENTRY_NP(sfmmu_modifytte_try)
863	ldx	[%o1], %g2			/* modified */
864	ldx	[%o2], %g3			/* current */
865	ldx	[%o0], %g1			/* original */
866	cmp	%g3, %g2			/* is modified = current? */
867	be,a,pn %xcc,1f				/* yes, don't write */
868	mov	0, %o1				/* as if cas failed. */
869
870	casx	[%o2], %g1, %g2
871	membar	#StoreLoad
872	cmp	%g1, %g2
873	movne	%xcc, -1, %o1			/* cas failed. */
874	move	%xcc, 1, %o1			/* cas succeeded. */
8751:
876	stx	%g2, [%o0]			/* report "current" value */
877	retl
878	mov	%o1, %o0
879	SET_SIZE(sfmmu_modifytte_try)
880
881	ENTRY_NP(sfmmu_copytte)
882	ldx	[%o0], %g1
883	retl
884	stx	%g1, [%o1]
885	SET_SIZE(sfmmu_copytte)
886
887
888	/*
889	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
890	 * %o0 = TSB base address (in), pointer to TSB entry (out)
891	 * %o1 = vaddr (in)
892	 * %o2 = vpshift (in)
893	 * %o3 = tsb size code (in)
894	 * %o4 = scratch register
895	 */
896	ENTRY_NP(sfmmu_get_tsbe)
897	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
898	retl
899	nop
900	SET_SIZE(sfmmu_get_tsbe)
901
902	/*
903	 * Return a TSB tag for the given va.
904	 * %o0 = va (in/clobbered)
905	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
906	 */
907	ENTRY_NP(sfmmu_make_tsbtag)
908	retl
909	srln	%o0, TTARGET_VA_SHIFT, %o0
910	SET_SIZE(sfmmu_make_tsbtag)
911
912#endif /* lint */
913
914/*
915 * Other sfmmu primitives
916 */
917
918
919#if defined (lint)
920void
921sfmmu_patch_ktsb(void)
922{
923}
924
925void
926sfmmu_kpm_patch_tlbm(void)
927{
928}
929
930void
931sfmmu_kpm_patch_tsbm(void)
932{
933}
934
935void
936sfmmu_patch_shctx(void)
937{
938}
939
940/* ARGSUSED */
941void
942sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
943{
944}
945
946/* ARGSUSED */
947void
948sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
949{
950}
951
952/* ARGSUSED */
953void
954sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
955{
956}
957
958/* ARGSUSED */
959void
960sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
961{
962}
963
964#else /* lint */
965
966#define	I_SIZE		4
967
968	ENTRY_NP(sfmmu_fix_ktlb_traptable)
969	/*
970	 * %o0 = start of patch area
971	 * %o1 = size code of TSB to patch
972	 * %o3 = scratch
973	 */
974	/* fix sll */
975	ld	[%o0], %o3			/* get sll */
976	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
977	st	%o3, [%o0]			/* write sll */
978	flush	%o0
979	/* fix srl */
980	add	%o0, I_SIZE, %o0		/* goto next instr. */
981	ld	[%o0], %o3			/* get srl */
982	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
983	st	%o3, [%o0]			/* write srl */
984	retl
985	flush	%o0
986	SET_SIZE(sfmmu_fix_ktlb_traptable)
987
988	ENTRY_NP(sfmmu_fixup_ktsbbase)
989	/*
990	 * %o0 = start of patch area
991	 * %o5 = kernel virtual or physical tsb base address
992	 * %o2, %o3 are used as scratch registers.
993	 */
994	/* fixup sethi instruction */
995	ld	[%o0], %o3
996	srl	%o5, 10, %o2			! offset is bits 32:10
997	or	%o3, %o2, %o3			! set imm22
998	st	%o3, [%o0]
999	/* fixup offset of lduw/ldx */
1000	add	%o0, I_SIZE, %o0		! next instr
1001	ld	[%o0], %o3
1002	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
1003	or	%o3, %o2, %o3
1004	st	%o3, [%o0]
1005	retl
1006	flush	%o0
1007	SET_SIZE(sfmmu_fixup_ktsbbase)
1008
1009	ENTRY_NP(sfmmu_fixup_setx)
1010	/*
1011	 * %o0 = start of patch area
1012	 * %o4 = 64 bit value to patch
1013	 * %o2, %o3 are used as scratch registers.
1014	 *
1015	 * Note: Assuming that all parts of the instructions which need to be
1016	 *	 patched correspond to RUNTIME_PATCH (aka 0)
1017	 *
1018	 * Note the implementation of setx which is being patched is as follows:
1019	 *
1020	 * sethi   %hh(RUNTIME_PATCH), tmp
1021	 * sethi   %lm(RUNTIME_PATCH), dest
1022	 * or      tmp, %hm(RUNTIME_PATCH), tmp
1023	 * or      dest, %lo(RUNTIME_PATCH), dest
1024	 * sllx    tmp, 32, tmp
1025	 * nop
1026	 * or      tmp, dest, dest
1027	 *
1028	 * which differs from the implementation in the
1029	 * "SPARC Architecture Manual"
1030	 */
1031	/* fixup sethi instruction */
1032	ld	[%o0], %o3
1033	srlx	%o4, 42, %o2			! bits [63:42]
1034	or	%o3, %o2, %o3			! set imm22
1035	st	%o3, [%o0]
1036	/* fixup sethi instruction */
1037	add	%o0, I_SIZE, %o0		! next instr
1038	ld	[%o0], %o3
1039	sllx	%o4, 32, %o2			! clear upper bits
1040	srlx	%o2, 42, %o2			! bits [31:10]
1041	or	%o3, %o2, %o3			! set imm22
1042	st	%o3, [%o0]
1043	/* fixup or instruction */
1044	add	%o0, I_SIZE, %o0		! next instr
1045	ld	[%o0], %o3
1046	srlx	%o4, 32, %o2			! bits [63:32]
1047	and	%o2, 0x3ff, %o2			! bits [41:32]
1048	or	%o3, %o2, %o3			! set imm
1049	st	%o3, [%o0]
1050	/* fixup or instruction */
1051	add	%o0, I_SIZE, %o0		! next instr
1052	ld	[%o0], %o3
1053	and	%o4, 0x3ff, %o2			! bits [9:0]
1054	or	%o3, %o2, %o3			! set imm
1055	st	%o3, [%o0]
1056	retl
1057	flush	%o0
1058	SET_SIZE(sfmmu_fixup_setx)
1059
1060	ENTRY_NP(sfmmu_fixup_or)
1061	/*
1062	 * %o0 = start of patch area
1063	 * %o4 = 32 bit value to patch
1064	 * %o2, %o3 are used as scratch registers.
1065	 * Note: Assuming that all parts of the instructions which need to be
1066	 *	 patched correspond to RUNTIME_PATCH (aka 0)
1067	 */
1068	ld	[%o0], %o3
1069	and	%o4, 0x3ff, %o2			! bits [9:0]
1070	or	%o3, %o2, %o3			! set imm
1071	st	%o3, [%o0]
1072	retl
1073	flush	%o0
1074	SET_SIZE(sfmmu_fixup_or)
1075
1076	ENTRY_NP(sfmmu_fixup_shiftx)
1077	/*
1078	 * %o0 = start of patch area
1079	 * %o4 = signed int immediate value to add to sllx/srlx imm field
1080	 * %o2, %o3 are used as scratch registers.
1081	 *
1082	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
1083	 * so we do a simple add.  The caller must be careful to prevent
1084	 * overflow, which could easily occur if the initial value is nonzero!
1085	 */
1086	ld	[%o0], %o3			! %o3 = instruction to patch
1087	and	%o3, 0x3f, %o2			! %o2 = existing imm value
1088	add	%o2, %o4, %o2			! %o2 = new imm value
1089	andn	%o3, 0x3f, %o3			! clear old imm value
1090	and	%o2, 0x3f, %o2			! truncate new imm value
1091	or	%o3, %o2, %o3			! set new imm value
1092	st	%o3, [%o0]			! store updated instruction
1093	retl
1094	flush	%o0
1095	SET_SIZE(sfmmu_fixup_shiftx)
1096
1097	ENTRY_NP(sfmmu_fixup_mmu_asi)
1098	/*
1099	 * Patch imm_asi of all ldda instructions in the MMU
1100	 * trap handlers.  We search MMU_PATCH_INSTR instructions
1101	 * starting from the itlb miss handler (trap 0x64).
1102	 * %o0 = address of tt[0,1]_itlbmiss
1103	 * %o1 = imm_asi to setup, shifted by appropriate offset.
1104	 * %o3 = number of instructions to search
1105	 * %o4 = reserved by caller: called from leaf routine
1106	 */
11071:	ldsw	[%o0], %o2			! load instruction to %o2
1108	brgez,pt %o2, 2f
1109	  srl	%o2, 30, %o5
1110	btst	1, %o5				! test bit 30; skip if not set
1111	bz,pt	%icc, 2f
1112	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
1113	srlx	%o5, 58, %o5			! isolate op3 part of opcode
1114	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
1115	brnz,pt	%o5, 2f				! skip if not a match
1116	  or	%o2, %o1, %o2			! or in imm_asi
1117	st	%o2, [%o0]			! write patched instruction
11182:	dec	%o3
1119	brnz,a,pt %o3, 1b			! loop until we're done
1120	  add	%o0, I_SIZE, %o0
1121	retl
1122	flush	%o0
1123	SET_SIZE(sfmmu_fixup_mmu_asi)
1124
1125	/*
1126	 * Patch immediate ASI used to access the TSB in the
1127	 * trap table.
1128	 * inputs: %o0 = value of ktsb_phys
1129	 */
1130	ENTRY_NP(sfmmu_patch_mmu_asi)
1131	mov	%o7, %o4			! save return pc in %o4
1132	movrnz	%o0, ASI_QUAD_LDD_PHYS, %o3
1133	movrz	%o0, ASI_NQUAD_LD, %o3
1134	sll	%o3, 5, %o1			! imm_asi offset
1135	mov	6, %o3				! number of instructions
1136	sethi	%hi(dktsb), %o0			! to search
1137	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
1138	  or	%o0, %lo(dktsb), %o0
1139	mov	6, %o3				! number of instructions
1140	sethi	%hi(dktsb4m), %o0		! to search
1141	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
1142	  or	%o0, %lo(dktsb4m), %o0
1143	mov	6, %o3				! number of instructions
1144	sethi	%hi(iktsb), %o0			! to search
1145	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
1146	  or	%o0, %lo(iktsb), %o0
1147	mov	6, %o3				! number of instructions
1148	sethi	%hi(iktsb4m), %o0		! to search
1149	call	sfmmu_fixup_mmu_asi		! patch kitlb4m miss
1150	  or	%o0, %lo(iktsb4m), %o0
1151	mov	%o4, %o7			! retore return pc -- leaf
1152	retl
1153	nop
1154	SET_SIZE(sfmmu_patch_mmu_asi)
1155
1156	ENTRY_NP(sfmmu_patch_ktsb)
1157	/*
1158	 * We need to fix iktsb, dktsb, et. al.
1159	 */
1160	save	%sp, -SA(MINFRAME), %sp
1161	set	ktsb_phys, %o1
1162	ld	[%o1], %o4
1163	set	ktsb_base, %o5
1164	set	ktsb4m_base, %l1
1165	brz,pt	%o4, 1f
1166	  nop
1167	set	ktsb_pbase, %o5
1168	set	ktsb4m_pbase, %l1
11691:
1170	sethi	%hi(ktsb_szcode), %o1
1171	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */
1172
1173	sethi	%hi(iktsb), %o0
1174	call	sfmmu_fix_ktlb_traptable
1175	  or	%o0, %lo(iktsb), %o0
1176
1177	sethi	%hi(dktsb), %o0
1178	call	sfmmu_fix_ktlb_traptable
1179	  or	%o0, %lo(dktsb), %o0
1180
1181	sethi	%hi(ktsb4m_szcode), %o1
1182	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */
1183
1184	sethi	%hi(iktsb4m), %o0
1185	call	sfmmu_fix_ktlb_traptable
1186	  or	%o0, %lo(iktsb4m), %o0
1187
1188	sethi	%hi(dktsb4m), %o0
1189	call	sfmmu_fix_ktlb_traptable
1190	  or	%o0, %lo(dktsb4m), %o0
1191
1192#ifndef sun4v
1193	mov	ASI_N, %o2
1194	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
1195	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
1196	sethi	%hi(tsb_kernel_patch_asi), %o0
1197	call	sfmmu_fixup_or
1198	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
1199#endif
1200
1201	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)
1202
1203	sethi	%hi(dktsbbase), %o0
1204	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1205	  or	%o0, %lo(dktsbbase), %o0
1206
1207	sethi	%hi(iktsbbase), %o0
1208	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1209	  or	%o0, %lo(iktsbbase), %o0
1210
1211	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
1212	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1213	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
1214
1215#ifdef sun4v
1216	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
1217	call	sfmmu_fixup_setx	! patch value of ktsb base addr
1218	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
1219#endif /* sun4v */
1220
1221	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)
1222
1223	sethi	%hi(dktsb4mbase), %o0
1224	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1225	  or	%o0, %lo(dktsb4mbase), %o0
1226
1227	sethi	%hi(iktsb4mbase), %o0
1228	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1229	  or	%o0, %lo(iktsb4mbase), %o0
1230
1231	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
1232	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1233	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
1234
1235#ifdef sun4v
1236	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
1237	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
1238	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
1239#endif /* sun4v */
1240
1241	set	ktsb_szcode, %o4
1242	ld	[%o4], %o4
1243	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
1244	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1245	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
1246
1247#ifdef sun4v
1248	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
1249	call	sfmmu_fixup_or		! patch value of ktsb_szcode
1250	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
1251#endif /* sun4v */
1252
1253	set	ktsb4m_szcode, %o4
1254	ld	[%o4], %o4
1255	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1256	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1257	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
1258
1259#ifdef sun4v
1260	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1261	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
1262	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
1263#endif /* sun4v */
1264
1265	ret
1266	restore
1267	SET_SIZE(sfmmu_patch_ktsb)
1268
1269	ENTRY_NP(sfmmu_kpm_patch_tlbm)
1270	/*
1271	 * Fixup trap handlers in common segkpm case.  This is reserved
1272	 * for future use should kpm TSB be changed to be other than the
1273	 * kernel TSB.
1274	 */
1275	retl
1276	nop
1277	SET_SIZE(sfmmu_kpm_patch_tlbm)
1278
1279	ENTRY_NP(sfmmu_kpm_patch_tsbm)
1280	/*
1281	 * nop the branch to sfmmu_kpm_dtsb_miss_small
1282	 * in the case where we are using large pages for
1283	 * seg_kpm (and hence must probe the second TSB for
1284	 * seg_kpm VAs)
1285	 */
1286	set	dktsb4m_kpmcheck_small, %o0
1287	MAKE_NOP_INSTR(%o1)
1288	st	%o1, [%o0]
1289	flush	%o0
1290	retl
1291	nop
1292	SET_SIZE(sfmmu_kpm_patch_tsbm)
1293
1294	ENTRY_NP(sfmmu_patch_utsb)
1295#ifdef UTSB_PHYS
1296	retl
1297	nop
1298#else /* UTSB_PHYS */
1299	/*
1300	 * We need to hot patch utsb_vabase and utsb4m_vabase
1301	 */
1302	save	%sp, -SA(MINFRAME), %sp
1303
1304	/* patch value of utsb_vabase */
1305	set	utsb_vabase, %o1
1306	ldx	[%o1], %o4
1307	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1308	call	sfmmu_fixup_setx
1309	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
1310	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1311	call	sfmmu_fixup_setx
1312	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
1313	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1314	call	sfmmu_fixup_setx
1315	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
1316
1317	/* patch value of utsb4m_vabase */
1318	set	utsb4m_vabase, %o1
1319	ldx	[%o1], %o4
1320	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
1321	call	sfmmu_fixup_setx
1322	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
1323	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
1324	call	sfmmu_fixup_setx
1325	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
1326	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
1327	call	sfmmu_fixup_setx
1328	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
1329
1330	/*
1331	 * Patch TSB base register masks and shifts if needed.
1332	 * By default the TSB base register contents are set up for 4M slab.
1333	 * If we're using a smaller slab size and reserved VA range we need
1334	 * to patch up those values here.
1335	 */
1336	set	tsb_slab_shift, %o1
1337	set	MMU_PAGESHIFT4M, %o4
1338	lduw	[%o1], %o3
1339	subcc	%o4, %o3, %o4
1340	bz,pt	%icc, 1f
1341	  /* delay slot safe */
1342
1343	/* patch reserved VA range size if needed. */
1344	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
1345	call	sfmmu_fixup_shiftx
1346	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
1347	call	sfmmu_fixup_shiftx
1348	  add	%o0, I_SIZE, %o0
1349	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
1350	call	sfmmu_fixup_shiftx
1351	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
1352	call	sfmmu_fixup_shiftx
1353	  add	%o0, I_SIZE, %o0
13541:
1355	/* patch TSBREG_VAMASK used to set up TSB base register */
1356	set	tsb_slab_mask, %o1
1357	ldx	[%o1], %o4
1358	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
1359	call	sfmmu_fixup_or
1360	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
1361	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1362	call	sfmmu_fixup_or
1363	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
1364
1365	ret
1366	restore
1367#endif /* UTSB_PHYS */
1368	SET_SIZE(sfmmu_patch_utsb)
1369
1370	ENTRY_NP(sfmmu_patch_shctx)
1371#ifdef sun4u
1372	retl
1373	  nop
1374#else /* sun4u */
1375	set	sfmmu_shctx_cpu_mondo_patch, %o0
1376	MAKE_JMP_INSTR(5, %o1, %o2)	! jmp       %g5
1377	st	%o1, [%o0]
1378	flush	%o0
1379	MAKE_NOP_INSTR(%o1)
1380	add	%o0, I_SIZE, %o0	! next instr
1381	st	%o1, [%o0]
1382	flush	%o0
1383
1384	set	sfmmu_shctx_user_rtt_patch, %o0
1385	st      %o1, [%o0]		! nop 1st instruction
1386	flush	%o0
1387	add     %o0, I_SIZE, %o0
1388	st      %o1, [%o0]		! nop 2nd instruction
1389	flush	%o0
1390	add     %o0, I_SIZE, %o0
1391	st      %o1, [%o0]		! nop 3rd instruction
1392	flush	%o0
1393	add     %o0, I_SIZE, %o0
1394	st      %o1, [%o0]		! nop 4th instruction
1395	flush	%o0
1396	add     %o0, I_SIZE, %o0
1397	st      %o1, [%o0]		! nop 5th instruction
1398	retl
1399	  flush	%o0
1400#endif /* sun4u */
1401	SET_SIZE(sfmmu_patch_shctx)
1402
1403	/*
1404	 * Routine that loads an entry into a tsb using virtual addresses.
1405	 * Locking is required since all cpus can use the same TSB.
1406	 * Note that it is no longer required to have a valid context
1407	 * when calling this function.
1408	 */
1409	ENTRY_NP(sfmmu_load_tsbe)
1410	/*
1411	 * %o0 = pointer to tsbe to load
1412	 * %o1 = tsb tag
1413	 * %o2 = virtual pointer to TTE
1414	 * %o3 = 1 if physical address in %o0 else 0
1415	 */
1416	rdpr	%pstate, %o5
1417#ifdef DEBUG
1418	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l2, %g1)
1419#endif /* DEBUG */
1420
1421	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1422
1423	SETUP_TSB_ASI(%o3, %g3)
1424	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1)
1425
1426	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1427
1428	retl
1429	membar	#StoreStore|#StoreLoad
1430	SET_SIZE(sfmmu_load_tsbe)
1431
1432	/*
1433	 * Flush TSB of a given entry if the tag matches.
1434	 */
1435	ENTRY(sfmmu_unload_tsbe)
1436	/*
1437	 * %o0 = pointer to tsbe to be flushed
1438	 * %o1 = tag to match
1439	 * %o2 = 1 if physical address in %o0 else 0
1440	 */
1441	SETUP_TSB_ASI(%o2, %g1)
1442	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1443	retl
1444	membar	#StoreStore|#StoreLoad
1445	SET_SIZE(sfmmu_unload_tsbe)
1446
1447	/*
1448	 * Routine that loads a TTE into the kpm TSB from C code.
1449	 * Locking is required since kpm TSB is shared among all CPUs.
1450	 */
1451	ENTRY_NP(sfmmu_kpm_load_tsb)
1452	/*
1453	 * %o0 = vaddr
1454	 * %o1 = ttep
1455	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1456	 */
1457	rdpr	%pstate, %o5			! %o5 = saved pstate
1458#ifdef DEBUG
1459	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l3, %g1)
1460#endif /* DEBUG */
1461	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts
1462
1463#ifndef sun4v
1464	sethi	%hi(ktsb_phys), %o4
1465	mov	ASI_N, %o3
1466	ld	[%o4 + %lo(ktsb_phys)], %o4
1467	movrnz	%o4, ASI_MEM, %o3
1468	mov	%o3, %asi
1469#endif
1470	mov	%o0, %g1			! %g1 = vaddr
1471
1472	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1473	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1474	/* %g2 = tsbep, %g1 clobbered */
1475
1476	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1477	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1478	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1)
1479
1480	wrpr	%g0, %o5, %pstate		! enable interrupts
1481	retl
1482	  membar #StoreStore|#StoreLoad
1483	SET_SIZE(sfmmu_kpm_load_tsb)
1484
1485	/*
1486	 * Routine that shoots down a TTE in the kpm TSB or in the
1487	 * kernel TSB depending on virtpg. Locking is required since
1488	 * kpm/kernel TSB is shared among all CPUs.
1489	 */
1490	ENTRY_NP(sfmmu_kpm_unload_tsb)
1491	/*
1492	 * %o0 = vaddr
1493	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1494	 */
1495#ifndef sun4v
1496	sethi	%hi(ktsb_phys), %o4
1497	mov	ASI_N, %o3
1498	ld	[%o4 + %lo(ktsb_phys)], %o4
1499	movrnz	%o4, ASI_MEM, %o3
1500	mov	%o3, %asi
1501#endif
1502	mov	%o0, %g1			! %g1 = vaddr
1503
1504	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1505	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1506	/* %g2 = tsbep, %g1 clobbered */
1507
1508	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1509	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1510	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1511
1512	retl
1513	  membar	#StoreStore|#StoreLoad
1514	SET_SIZE(sfmmu_kpm_unload_tsb)
1515
1516#endif /* lint */
1517
1518
1519#if defined (lint)
1520
1521/*ARGSUSED*/
1522pfn_t
1523sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
1524{ return(0); }
1525
1526#else /* lint */
1527
1528	ENTRY_NP(sfmmu_ttetopfn)
1529	ldx	[%o0], %g1			/* read tte */
1530	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1531	/*
1532	 * g1 = pfn
1533	 */
1534	retl
1535	mov	%g1, %o0
1536	SET_SIZE(sfmmu_ttetopfn)
1537
1538#endif /* !lint */
1539
1540
1541#if defined (lint)
1542/*
1543 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
1544 * the hash list.
1545 */
1546/* ARGSUSED */
1547void
1548sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1549	uint64_t hblkpa)
1550{
1551}
1552
1553/*
1554 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
1555 * hash list.
1556 */
1557/* ARGSUSED */
1558void
1559sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1560	uint64_t hblkpa, struct hme_blk *prev_hblkp)
1561{
1562}
1563#else /* lint */
1564
1565/*
1566 * Functions to grab/release hme bucket list lock.  I only use a byte
1567 * instead of the whole int because eventually we might want to
1568 * put some counters on the other bytes (of course, these routines would
1569 * have to change).  The code that grab this lock should execute
1570 * with interrupts disabled and hold the lock for the least amount of time
1571 * possible.
1572 */
1573
1574/*
1575 * Even though hmeh_listlock is updated using pa there's no need to flush
1576 * dcache since hmeh_listlock will be restored to the original value (0)
1577 * before interrupts are reenabled.
1578 */
1579
1580/*
1581 * For sparcv9 hme hash buckets may not be in the nucleus.  hme hash update
1582 * routines still use virtual addresses to update the bucket fields. But they
1583 * must not cause a TLB miss after grabbing the low level bucket lock. To
1584 * achieve this we must make sure the bucket structure is completely within an
1585 * 8K page.
1586 */
1587
1588#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1))
1589#error - the size of hmehash_bucket structure is not power of 2
1590#endif
1591
1592#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi)           \
1593	mov     0xff, tmp2                                      ;\
1594	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1595label1:                                                         ;\
1596	casa    [tmp1]asi, %g0, tmp2                            ;\
1597	brnz,pn tmp2, label1                                    ;\
1598	mov     0xff, tmp2                                      ;\
1599	membar  #LoadLoad
1600
1601#define HMELOCK_EXIT(hmebp, tmp1, asi)                          \
1602	membar  #LoadStore|#StoreStore                          ;\
1603	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1604	sta     %g0, [tmp1]asi
1605
1606	.seg	".data"
1607hblk_add_panic1:
1608	.ascii	"sfmmu_hblk_hash_add: interrupts disabled"
1609	.byte	0
1610hblk_add_panic2:
1611	.ascii	"sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
1612	.byte	0
1613	.align	4
1614	.seg	".text"
1615
1616	ENTRY_NP(sfmmu_hblk_hash_add)
1617	/*
1618	 * %o0 = hmebp
1619	 * %o1 = hmeblkp
1620	 * %o2 = hblkpa
1621	 */
1622	rdpr	%pstate, %o5
1623#ifdef DEBUG
1624	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1625	bnz,pt %icc, 3f				/* disabled, panic	 */
1626	  nop
1627	save	%sp, -SA(MINFRAME), %sp
1628	sethi	%hi(hblk_add_panic1), %o0
1629	call	panic
1630	 or	%o0, %lo(hblk_add_panic1), %o0
1631	ret
1632	restore
1633
16343:
1635#endif /* DEBUG */
1636	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1637	mov	%o2, %g1
1638
1639	/*
1640	 * g1 = hblkpa
1641	 */
1642	ldn	[%o0 + HMEBUCK_HBLK], %o4	/* next hmeblk */
1643	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = next hblkpa */
1644#ifdef	DEBUG
1645	cmp	%o4, %g0
1646	bne,pt %xcc, 1f
1647	 nop
1648	brz,pt %g2, 1f
1649	 nop
1650	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1651	save	%sp, -SA(MINFRAME), %sp
1652	sethi	%hi(hblk_add_panic2), %o0
1653	call	panic
1654	  or	%o0, %lo(hblk_add_panic2), %o0
1655	ret
1656	restore
16571:
1658#endif /* DEBUG */
1659	/*
1660	 * We update hmeblks entries before grabbing lock because the stores
1661	 * could take a tlb miss and require the hash lock.  The buckets
1662	 * are part of the nucleus so we are cool with those stores.
1663	 *
1664	 * if buckets are not part of the nucleus our game is to
1665	 * not touch any other page via va until we drop the lock.
1666	 * This guarantees we won't get a tlb miss before the lock release
1667	 * since interrupts are disabled.
1668	 */
1669	stn	%o4, [%o1 + HMEBLK_NEXT]	/* update hmeblk's next */
1670	stx	%g2, [%o1 + HMEBLK_NEXTPA]	/* update hmeblk's next pa */
1671	HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N)
1672	stn	%o1, [%o0 + HMEBUCK_HBLK]	/* update bucket hblk next */
1673	stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* add hmeblk to list */
1674	HMELOCK_EXIT(%o0, %g2, ASI_N)
1675	retl
1676	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1677	SET_SIZE(sfmmu_hblk_hash_add)
1678
1679	ENTRY_NP(sfmmu_hblk_hash_rm)
1680	/*
1681	 * This function removes an hmeblk from the hash chain.
1682	 * It is written to guarantee we don't take a tlb miss
1683	 * by using physical addresses to update the list.
1684	 *
1685	 * %o0 = hmebp
1686	 * %o1 = hmeblkp
1687	 * %o2 = hmeblkp previous pa
1688	 * %o3 = hmeblkp previous
1689	 */
1690
1691	mov	%o3, %o4			/* o4 = hmeblkp previous */
1692
1693	rdpr	%pstate, %o5
1694#ifdef DEBUG
1695	PANIC_IF_INTR_DISABLED_PSTR(%o5, sfmmu_di_l4, %g1)
1696#endif /* DEBUG */
1697	/*
1698	 * disable interrupts, clear Address Mask to access 64 bit physaddr
1699	 */
1700	andn    %o5, PSTATE_IE, %g1
1701	wrpr    %g1, 0, %pstate
1702
1703#ifndef sun4v
1704	sethi   %hi(dcache_line_mask), %g4
1705	ld      [%g4 + %lo(dcache_line_mask)], %g4
1706#endif /* sun4v */
1707
1708	/*
1709	 * if buckets are not part of the nucleus our game is to
1710	 * not touch any other page via va until we drop the lock.
1711	 * This guarantees we won't get a tlb miss before the lock release
1712	 * since interrupts are disabled.
1713	 */
1714	HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N)
1715	ldn	[%o0 + HMEBUCK_HBLK], %g2	/* first hmeblk in list */
1716	cmp	%g2, %o1
1717	bne,pt	%ncc,1f
1718	 mov	ASI_MEM, %asi
1719	/*
1720	 * hmeblk is first on list
1721	 */
1722	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = hmeblk pa */
1723	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1724	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1725	stn	%o3, [%o0 + HMEBUCK_HBLK]	/* write va */
1726	ba,pt	%xcc, 2f
1727	  stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* write pa */
17281:
1729	/* hmeblk is not first on list */
1730
1731	mov	%o2, %g3
1732#ifndef sun4v
1733	GET_CPU_IMPL(%g2)
1734	cmp 	%g2, CHEETAH_IMPL
1735	bge,a,pt %icc, hblk_hash_rm_1
1736	  and	%o4, %g4, %g2
1737	cmp	%g2, SPITFIRE_IMPL
1738	blt	%icc, hblk_hash_rm_2		/* no flushing needed for OPL */
1739	  and	%o4, %g4, %g2
1740	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev pa from dcache */
1741	add	%o4, HMEBLK_NEXT, %o4
1742	and	%o4, %g4, %g2
1743	ba	hblk_hash_rm_2
1744	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev va from dcache */
1745hblk_hash_rm_1:
1746
1747	stxa	%g0, [%g3]ASI_DC_INVAL		/* flush prev pa from dcache */
1748	membar	#Sync
1749	add     %g3, HMEBLK_NEXT, %g2
1750	stxa	%g0, [%g2]ASI_DC_INVAL		/* flush prev va from dcache */
1751hblk_hash_rm_2:
1752	membar	#Sync
1753#endif /* sun4v */
1754	ldxa	[%g3 + HMEBLK_NEXTPA] %asi, %g2	/* g2 = hmeblk pa */
1755	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1756	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1757	stna	%o3, [%g3 + HMEBLK_NEXT] %asi	/* write va */
1758	stxa	%g1, [%g3 + HMEBLK_NEXTPA] %asi	/* write pa */
17592:
1760	HMELOCK_EXIT(%o0, %g2, ASI_N)
1761	retl
1762	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1763	SET_SIZE(sfmmu_hblk_hash_rm)
1764
1765#endif /* lint */
1766
1767/*
1768 * These macros are used to update global sfmmu hme hash statistics
1769 * in perf critical paths. It is only enabled in debug kernels or
1770 * if SFMMU_STAT_GATHER is defined
1771 */
1772#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1773#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1774	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1775	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
1776	cmp	tmp1, hatid						;\
1777	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
1778	set	sfmmu_global_stat, tmp1					;\
1779	add	tmp1, tmp2, tmp1					;\
1780	ld	[tmp1], tmp2						;\
1781	inc	tmp2							;\
1782	st	tmp2, [tmp1]
1783
1784#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1785	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1786	mov	HATSTAT_KHASH_LINKS, tmp2				;\
1787	cmp	tmp1, hatid						;\
1788	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
1789	set	sfmmu_global_stat, tmp1					;\
1790	add	tmp1, tmp2, tmp1					;\
1791	ld	[tmp1], tmp2						;\
1792	inc	tmp2							;\
1793	st	tmp2, [tmp1]
1794
1795
1796#else /* DEBUG || SFMMU_STAT_GATHER */
1797
1798#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1799
1800#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1801
1802#endif  /* DEBUG || SFMMU_STAT_GATHER */
1803
1804/*
1805 * This macro is used to update global sfmmu kstas in non
1806 * perf critical areas so they are enabled all the time
1807 */
1808#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
1809	sethi	%hi(sfmmu_global_stat), tmp1				;\
1810	add	tmp1, statname, tmp1					;\
1811	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
1812	inc	tmp2							;\
1813	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1814
1815/*
1816 * These macros are used to update per cpu stats in non perf
1817 * critical areas so they are enabled all the time
1818 */
1819#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
1820	ld	[tsbarea + stat], tmp1					;\
1821	inc	tmp1							;\
1822	st	tmp1, [tsbarea + stat]
1823
1824/*
1825 * These macros are used to update per cpu stats in non perf
1826 * critical areas so they are enabled all the time
1827 */
1828#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
1829	lduh	[tsbarea + stat], tmp1					;\
1830	inc	tmp1							;\
1831	stuh	tmp1, [tsbarea + stat]
1832
1833#if defined(KPM_TLBMISS_STATS_GATHER)
1834	/*
1835	 * Count kpm dtlb misses separately to allow a different
1836	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1837	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1838	 */
1839#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
1840	brgez	tagacc, label	/* KPM VA? */				;\
1841	nop								;\
1842	CPU_INDEX(tmp1, tsbma)						;\
1843	sethi	%hi(kpmtsbm_area), tsbma				;\
1844	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
1845	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
1846	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
1847	/* VA range check */						;\
1848	ldx	[tsbma + KPMTSBM_VBASE], val				;\
1849	cmp	tagacc, val						;\
1850	blu,pn	%xcc, label						;\
1851	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
1852	cmp	tagacc, tmp1						;\
1853	bgeu,pn	%xcc, label						;\
1854	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
1855	inc	val							;\
1856	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
1857label:
1858#else
1859#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1860#endif	/* KPM_TLBMISS_STATS_GATHER */
1861
1862#if defined (lint)
1863/*
1864 * The following routines are jumped to from the mmu trap handlers to do
1865 * the setting up to call systrap.  They are separate routines instead of
1866 * being part of the handlers because the handlers would exceed 32
1867 * instructions and since this is part of the slow path the jump
1868 * cost is irrelevant.
1869 */
1870void
1871sfmmu_pagefault(void)
1872{
1873}
1874
1875void
1876sfmmu_mmu_trap(void)
1877{
1878}
1879
1880void
1881sfmmu_window_trap(void)
1882{
1883}
1884
1885void
1886sfmmu_kpm_exception(void)
1887{
1888}
1889
1890#else /* lint */
1891
1892#ifdef	PTL1_PANIC_DEBUG
1893	.seg	".data"
1894	.global	test_ptl1_panic
1895test_ptl1_panic:
1896	.word	0
1897	.align	8
1898
1899	.seg	".text"
1900	.align	4
1901#endif	/* PTL1_PANIC_DEBUG */
1902
1903
1904	ENTRY_NP(sfmmu_pagefault)
1905	SET_GL_REG(1)
1906	USE_ALTERNATE_GLOBALS(%g5)
1907	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1908	rdpr	%tt, %g6
1909	cmp	%g6, FAST_IMMU_MISS_TT
1910	be,a,pn	%icc, 1f
1911	  mov	T_INSTR_MMU_MISS, %g3
1912	cmp	%g6, T_INSTR_MMU_MISS
1913	be,a,pn	%icc, 1f
1914	  mov	T_INSTR_MMU_MISS, %g3
1915	mov	%g5, %g2
1916	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1917	cmp	%g6, FAST_DMMU_MISS_TT
1918	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1919	cmp	%g6, T_DATA_MMU_MISS
1920	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1921
1922#ifdef  PTL1_PANIC_DEBUG
1923	/* check if we want to test the tl1 panic */
1924	sethi	%hi(test_ptl1_panic), %g4
1925	ld	[%g4 + %lo(test_ptl1_panic)], %g1
1926	st	%g0, [%g4 + %lo(test_ptl1_panic)]
1927	cmp	%g1, %g0
1928	bne,a,pn %icc, ptl1_panic
1929	  or	%g0, PTL1_BAD_DEBUG, %g1
1930#endif	/* PTL1_PANIC_DEBUG */
19311:
1932	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
1933	/*
1934	 * g2 = tag access reg
1935	 * g3.l = type
1936	 * g3.h = 0
1937	 */
1938	sethi	%hi(trap), %g1
1939	or	%g1, %lo(trap), %g1
19402:
1941	ba,pt	%xcc, sys_trap
1942	  mov	-1, %g4
1943	SET_SIZE(sfmmu_pagefault)
1944
1945	ENTRY_NP(sfmmu_mmu_trap)
1946	SET_GL_REG(1)
1947	USE_ALTERNATE_GLOBALS(%g5)
1948	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
1949	rdpr	%tt, %g6
1950	cmp	%g6, FAST_IMMU_MISS_TT
1951	be,a,pn	%icc, 1f
1952	  mov	T_INSTR_MMU_MISS, %g3
1953	cmp	%g6, T_INSTR_MMU_MISS
1954	be,a,pn	%icc, 1f
1955	  mov	T_INSTR_MMU_MISS, %g3
1956	mov	%g5, %g2
1957	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1958	cmp	%g6, FAST_DMMU_MISS_TT
1959	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1960	cmp	%g6, T_DATA_MMU_MISS
1961	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
19621:
1963	/*
1964	 * g2 = tag access reg
1965	 * g3 = type
1966	 */
1967	sethi	%hi(sfmmu_tsbmiss_exception), %g1
1968	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
1969	ba,pt	%xcc, sys_trap
1970	  mov	-1, %g4
1971	/*NOTREACHED*/
1972	SET_SIZE(sfmmu_mmu_trap)
1973
1974	ENTRY_NP(sfmmu_suspend_tl)
1975	SET_GL_REG(1)
1976	USE_ALTERNATE_GLOBALS(%g5)
1977	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
1978	rdpr	%tt, %g6
1979	cmp	%g6, FAST_IMMU_MISS_TT
1980	be,a,pn	%icc, 1f
1981	  mov	T_INSTR_MMU_MISS, %g3
1982	mov	%g5, %g2
1983	cmp	%g6, FAST_DMMU_MISS_TT
1984	move	%icc, T_DATA_MMU_MISS, %g3
1985	movne	%icc, T_DATA_PROT, %g3
19861:
1987	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
1988	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
1989	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
1990	ba,pt	%xcc, sys_trap
1991	  mov	PIL_15, %g4
1992	/*NOTREACHED*/
1993	SET_SIZE(sfmmu_suspend_tl)
1994
1995	/*
1996	 * No %g registers in use at this point.
1997	 */
1998	ENTRY_NP(sfmmu_window_trap)
1999	rdpr	%tpc, %g1
2000#ifdef sun4v
2001#ifdef DEBUG
2002	/* We assume previous %gl was 1 */
2003	rdpr	%tstate, %g4
2004	srlx	%g4, TSTATE_GL_SHIFT, %g4
2005	and	%g4, TSTATE_GL_MASK, %g4
2006	cmp	%g4, 1
2007	bne,a,pn %icc, ptl1_panic
2008	  mov	PTL1_BAD_WTRAP, %g1
2009#endif /* DEBUG */
2010	/* user miss at tl>1. better be the window handler or user_rtt */
2011	/* in user_rtt? */
2012	set	rtt_fill_start, %g4
2013	cmp	%g1, %g4
2014	blu,pn %xcc, 6f
2015	 .empty
2016	set	rtt_fill_end, %g4
2017	cmp	%g1, %g4
2018	bgeu,pn %xcc, 6f
2019	 nop
2020	set	fault_rtt_fn1, %g1
2021	wrpr	%g0, %g1, %tnpc
2022	ba,a	7f
20236:
2024	! must save this trap level before descending trap stack
2025	! no need to save %tnpc, either overwritten or discarded
2026	! already got it: rdpr	%tpc, %g1
2027	rdpr	%tstate, %g6
2028	rdpr	%tt, %g7
2029	! trap level saved, go get underlying trap type
2030	rdpr	%tl, %g5
2031	sub	%g5, 1, %g3
2032	wrpr	%g3, %tl
2033	rdpr	%tt, %g2
2034	wrpr	%g5, %tl
2035	! restore saved trap level
2036	wrpr	%g1, %tpc
2037	wrpr	%g6, %tstate
2038	wrpr	%g7, %tt
2039#else /* sun4v */
2040	/* user miss at tl>1. better be the window handler */
2041	rdpr	%tl, %g5
2042	sub	%g5, 1, %g3
2043	wrpr	%g3, %tl
2044	rdpr	%tt, %g2
2045	wrpr	%g5, %tl
2046#endif /* sun4v */
2047	and	%g2, WTRAP_TTMASK, %g4
2048	cmp	%g4, WTRAP_TYPE
2049	bne,pn	%xcc, 1f
2050	 nop
2051	/* tpc should be in the trap table */
2052	set	trap_table, %g4
2053	cmp	%g1, %g4
2054	blt,pn %xcc, 1f
2055	 .empty
2056	set	etrap_table, %g4
2057	cmp	%g1, %g4
2058	bge,pn %xcc, 1f
2059	 .empty
2060	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
2061	add	%g1, WTRAP_FAULTOFF, %g1
2062	wrpr	%g0, %g1, %tnpc
20637:
2064	/*
2065	 * some wbuf handlers will call systrap to resolve the fault
2066	 * we pass the trap type so they figure out the correct parameters.
2067	 * g5 = trap type, g6 = tag access reg
2068	 */
2069
2070	/*
2071	 * only use g5, g6, g7 registers after we have switched to alternate
2072	 * globals.
2073	 */
2074	SET_GL_REG(1)
2075	USE_ALTERNATE_GLOBALS(%g5)
2076	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
2077	rdpr	%tt, %g7
2078	cmp	%g7, FAST_IMMU_MISS_TT
2079	be,a,pn	%icc, ptl1_panic
2080	  mov	PTL1_BAD_WTRAP, %g1
2081	cmp	%g7, T_INSTR_MMU_MISS
2082	be,a,pn	%icc, ptl1_panic
2083	  mov	PTL1_BAD_WTRAP, %g1
2084	mov	T_DATA_PROT, %g5
2085	cmp	%g7, FAST_DMMU_MISS_TT
2086	move	%icc, T_DATA_MMU_MISS, %g5
2087	cmp	%g7, T_DATA_MMU_MISS
2088	move	%icc, T_DATA_MMU_MISS, %g5
2089	! XXXQ AGS re-check out this one
2090	done
20911:
2092	CPU_ADDR(%g1, %g4)
2093	ld	[%g1 + CPU_TL1_HDLR], %g4
2094	brnz,a,pt %g4, sfmmu_mmu_trap
2095	  st	%g0, [%g1 + CPU_TL1_HDLR]
2096	ba,pt	%icc, ptl1_panic
2097	  mov	PTL1_BAD_TRAP, %g1
2098	SET_SIZE(sfmmu_window_trap)
2099
2100	ENTRY_NP(sfmmu_kpm_exception)
2101	/*
2102	 * We have accessed an unmapped segkpm address or a legal segkpm
2103	 * address which is involved in a VAC alias conflict prevention.
2104	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
2105	 * set. If it is, we will instead note that a fault has occurred
2106	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
2107	 * a "retry"). This will step over the faulting instruction.
2108	 * Note that this means that a legal segkpm address involved in
2109	 * a VAC alias conflict prevention (a rare case to begin with)
2110	 * cannot be used in DTrace.
2111	 */
2112	CPU_INDEX(%g1, %g2)
2113	set	cpu_core, %g2
2114	sllx	%g1, CPU_CORE_SHIFT, %g1
2115	add	%g1, %g2, %g1
2116	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
2117	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
2118	bz	0f
2119	or	%g2, CPU_DTRACE_BADADDR, %g2
2120	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
2121	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
2122	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
2123	done
21240:
2125	TSTAT_CHECK_TL1(1f, %g1, %g2)
21261:
2127	SET_GL_REG(1)
2128	USE_ALTERNATE_GLOBALS(%g5)
2129	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
2130	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
2131	/*
2132	 * g2=tagacc g3.l=type g3.h=0
2133	 */
2134	sethi	%hi(trap), %g1
2135	or	%g1, %lo(trap), %g1
2136	ba,pt	%xcc, sys_trap
2137	mov	-1, %g4
2138	SET_SIZE(sfmmu_kpm_exception)
2139
2140#endif /* lint */
2141
2142#if defined (lint)
2143
2144void
2145sfmmu_tsb_miss(void)
2146{
2147}
2148
2149void
2150sfmmu_kpm_dtsb_miss(void)
2151{
2152}
2153
2154void
2155sfmmu_kpm_dtsb_miss_small(void)
2156{
2157}
2158
2159#else /* lint */
2160
2161#if (IMAP_SEG != 0)
2162#error - ism_map->ism_seg offset is not zero
2163#endif
2164
2165/*
2166 * Copies ism mapping for this ctx in param "ism" if this is a ISM
2167 * tlb miss and branches to label "ismhit". If this is not an ISM
2168 * process or an ISM tlb miss it falls thru.
2169 *
2170 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
2171 * this process.
2172 * If so, it will branch to label "ismhit".  If not, it will fall through.
2173 *
2174 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
2175 * so that any other threads of this process will not try and walk the ism
2176 * maps while they are being changed.
2177 *
2178 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
2179 *       will make sure of that. This means we can terminate our search on
2180 *       the first zero mapping we find.
2181 *
2182 * Parameters:
2183 * tagacc	= (pseudo-)tag access register (vaddr + ctx) (in)
2184 * tsbmiss	= address of tsb miss area (in)
2185 * ismseg	= contents of ism_seg for this ism map (out)
2186 * ismhat	= physical address of imap_ismhat for this ism map (out)
2187 * tmp1		= scratch reg (CLOBBERED)
2188 * tmp2		= scratch reg (CLOBBERED)
2189 * tmp3		= scratch reg (CLOBBERED)
2190 * label:    temporary labels
2191 * ismhit:   label where to jump to if an ism dtlb miss
2192 * exitlabel:label where to jump if hat is busy due to hat_unshare.
2193 */
2194#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
2195	label, ismhit)							\
2196	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
2197	brlz,pt  tmp1, label/**/3		/* exit if -1 */	;\
2198	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
2199label/**/1:								;\
2200	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
2201	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
2202label/**/2:								;\
2203	brz,pt  ismseg, label/**/3		/* no mapping */	;\
2204	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
2205	lduba	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
2206	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
2207	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
2208	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
2209	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
2210	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
2211	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
2212	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
2213	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
2214	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
2215									;\
2216	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
2217	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
2218	cmp	ismhat, tmp1						;\
2219	bl,pt	%xcc, label/**/2		/* keep looking  */	;\
2220	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
2221									;\
2222	add	tmp3, IBLK_NEXTPA, tmp1					;\
2223	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
2224	brgez,pt tmp1, label/**/1		/* continue if not -1*/	;\
2225	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
2226label/**/3:
2227
2228/*
2229 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
2230 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
2231 * Parameters:
2232 * tagacc = reg containing virtual address
2233 * hatid = reg containing sfmmu pointer
2234 * hmeshift = constant/register to shift vaddr to obtain vapg
2235 * hmebp = register where bucket pointer will be stored
2236 * vapg = register where virtual page will be stored
2237 * tmp1, tmp2 = tmp registers
2238 */
2239
2240
2241#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
2242	vapg, label, tmp1, tmp2)					\
2243	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
2244	brnz,a,pt tmp1, label/**/1					;\
2245	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
2246	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
2247	ba,pt	%xcc, label/**/2					;\
2248	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
2249label/**/1:								;\
2250	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
2251label/**/2:								;\
2252	srlx	tagacc, hmeshift, vapg					;\
2253	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
2254	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
2255	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
2256	add	hmebp, tmp1, hmebp
2257
2258/*
2259 * hashtag includes bspage + hashno (64 bits).
2260 */
2261
2262#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
2263	sllx	vapg, hmeshift, vapg					;\
2264	mov	hashno, hblktag						;\
2265	sllx	hblktag, HTAG_REHASH_SHIFT, hblktag			;\
2266	or	vapg, hblktag, hblktag
2267
2268/*
2269 * Function to traverse hmeblk hash link list and find corresponding match.
2270 * The search is done using physical pointers. It returns the physical address
2271 * and virtual address pointers to the hmeblk that matches with the tag
2272 * provided.
2273 * Parameters:
2274 * hmebp	= register that points to hme hash bucket, also used as
2275 *		  tmp reg (clobbered)
2276 * hmeblktag	= register with hmeblk tag match
2277 * hatid	= register with hatid
2278 * hmeblkpa	= register where physical ptr will be stored
2279 * hmeblkva	= register where virtual ptr will be stored
2280 * tmp1		= tmp reg
2281 * label: temporary label
2282 */
2283
2284#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva,	\
2285	tsbarea, tmp1, label)					 	\
2286	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
2287	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
2288	add     hmebp, HMEBUCK_HBLK, hmeblkva				;\
2289	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
2290	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
2291label/**/1:								;\
2292	brz,pn	hmeblkva, label/**/2					;\
2293	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
2294	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
2295	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
2296	add	hmebp, CLONGSIZE, hmebp					;\
2297	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
2298	xor	tmp1, hmeblktag, tmp1					;\
2299	xor	hmebp, hatid, hmebp					;\
2300	or	hmebp, tmp1, hmebp					;\
2301	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
2302	  add	hmeblkpa, HMEBLK_NEXT, hmebp				;\
2303	ldna	[hmebp]ASI_MEM, hmeblkva	/* hmeblk ptr va */	;\
2304	add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
2305	ba,pt	%xcc, label/**/1					;\
2306	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
2307label/**/2:
2308
2309/*
2310 * Function to traverse hmeblk hash link list and find corresponding match.
2311 * The search is done using physical pointers. It returns the physical address
2312 * and virtual address pointers to the hmeblk that matches with the tag
2313 * provided.
2314 * Parameters:
2315 * hmeblktag	= register with hmeblk tag match (rid field is 0)
2316 * hatid	= register with hatid (pointer to SRD)
2317 * hmeblkpa	= register where physical ptr will be stored
2318 * hmeblkva	= register where virtual ptr will be stored
2319 * tmp1		= tmp reg
2320 * tmp2		= tmp reg
2321 * label: temporary label
2322 */
2323
2324#define	HMEHASH_SEARCH_SHME(hmeblktag, hatid, hmeblkpa, hmeblkva,	\
2325	tsbarea, tmp1, tmp2, label)			 		\
2326label/**/1:								;\
2327	brz,pn	hmeblkva, label/**/4					;\
2328	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			;\
2329	add	hmeblkpa, HMEBLK_TAG, tmp2				;\
2330	ldxa	[tmp2]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
2331	add	tmp2, CLONGSIZE, tmp2					;\
2332	ldxa	[tmp2]ASI_MEM, tmp2 	/* read 2nd part of tag */	;\
2333	xor	tmp1, hmeblktag, tmp1					;\
2334	xor	tmp2, hatid, tmp2					;\
2335	brz,pn	tmp2, label/**/3	/* branch on hit */		;\
2336	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
2337label/**/2:								;\
2338	ldna	[tmp2]ASI_MEM, hmeblkva	/* hmeblk ptr va */		;\
2339	add	hmeblkpa, HMEBLK_NEXTPA, tmp2				;\
2340	ba,pt	%xcc, label/**/1					;\
2341	  ldxa	[tmp2]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */		;\
2342label/**/3:								;\
2343	cmp	tmp1, SFMMU_MAX_HME_REGIONS				;\
2344	bgeu,pt	%xcc, label/**/2					;\
2345	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
2346	and	tmp1, BT_ULMASK, tmp2					;\
2347	srlx	tmp1, BT_ULSHIFT, tmp1					;\
2348	sllx	tmp1, CLONGSHIFT, tmp1					;\
2349	add	tsbarea, tmp1, tmp1					;\
2350	ldx	[tmp1 + TSBMISS_SHMERMAP], tmp1				;\
2351	srlx	tmp1, tmp2, tmp1					;\
2352	btst	0x1, tmp1						;\
2353	bz,pn	%xcc, label/**/2					;\
2354	  add	hmeblkpa, HMEBLK_NEXT, tmp2				;\
2355label/**/4:
2356
2357#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
2358#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
2359#endif
2360
2361/*
2362 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
2363 * he offset for the corresponding hment.
2364 * Parameters:
2365 * In:
2366 *	vaddr = register with virtual address
2367 *	hmeblkpa = physical pointer to hme_blk
2368 * Out:
2369 *	hmentoff = register where hment offset will be stored
2370 *	hmemisc = hblk_misc
2371 * Scratch:
2372 *	tmp1
2373 */
2374#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, hmemisc, tmp1, label1)\
2375	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
2376	lda	[hmentoff]ASI_MEM, hmemisc 				;\
2377	andcc	hmemisc, HBLK_SZMASK, %g0				;\
2378	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
2379	  or	%g0, HMEBLK_HME1, hmentoff				;\
2380	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
2381	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
2382	sllx	tmp1, SFHME_SHIFT, tmp1					;\
2383	add	tmp1, HMEBLK_HME1, hmentoff				;\
2384label1:
2385
2386/*
2387 * GET_TTE is a macro that returns a TTE given a tag and hatid.
2388 *
2389 * tagacc	= (pseudo-)tag access register (in)
2390 * hatid	= sfmmu pointer for TSB miss (in)
2391 * tte		= tte for TLB miss if found, otherwise clobbered (out)
2392 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
2393 * hmeblkva	= VA of hment if found, otherwise clobbered (out)
2394 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
2395 * hmemisc	= hblk_misc if TTE is found (out), otherwise clobbered
2396 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
2397 *		  for this page size.
2398 * hashno	= constant/register hash number
2399 * label	= temporary label for branching within macro.
2400 * foundlabel	= label to jump to when tte is found.
2401 * suspendlabel= label to jump to when tte is suspended.
2402 * exitlabel	= label to jump to when tte is not found.
2403 *
2404 */
2405#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmemisc, \
2406		hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \
2407									;\
2408	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2409	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2410	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2411		hmeblkpa, label/**/5, hmemisc, hmeblkva)		;\
2412									;\
2413	/*								;\
2414	 * tagacc = tagacc						;\
2415	 * hatid = hatid						;\
2416	 * tsbarea = tsbarea						;\
2417	 * tte   = hmebp (hme bucket pointer)				;\
2418	 * hmeblkpa  = vapg  (virtual page)				;\
2419	 * hmemisc, hmeblkva = scratch					;\
2420	 */								;\
2421	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2422	or	hmemisc, SFMMU_INVALID_SHMERID, hmemisc			;\
2423									;\
2424	/*								;\
2425	 * tagacc = tagacc						;\
2426	 * hatid = hatid						;\
2427	 * tte   = hmebp						;\
2428	 * hmeblkpa  = CLOBBERED					;\
2429	 * hmemisc  = htag_bspage+hashno+invalid_rid			;\
2430	 * hmeblkva  = scratch						;\
2431	 */								;\
2432	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2433	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
2434	HMEHASH_SEARCH(tte, hmemisc, hatid, hmeblkpa, hmeblkva, 	\
2435		tsbarea, tagacc, label/**/1)				;\
2436	/*								;\
2437	 * tagacc = CLOBBERED						;\
2438	 * tte = CLOBBERED						;\
2439	 * hmeblkpa = hmeblkpa						;\
2440	 * hmeblkva = hmeblkva						;\
2441	 */								;\
2442	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2443	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2444	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2445	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2446	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2447	  nop								;\
2448label/**/4:								;\
2449	/*								;\
2450	 * We have found the hmeblk containing the hment.		;\
2451	 * Now we calculate the corresponding tte.			;\
2452	 *								;\
2453	 * tagacc = tagacc						;\
2454	 * hatid = hatid						;\
2455	 * tte   = clobbered						;\
2456	 * hmeblkpa  = hmeblkpa						;\
2457	 * hmemisc  = hblktag						;\
2458	 * hmeblkva  = hmeblkva 					;\
2459	 */								;\
2460	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2461		label/**/2)						;\
2462									;\
2463	/*								;\
2464	 * tagacc = tagacc						;\
2465	 * hatid = hmentoff						;\
2466	 * tte   = clobbered						;\
2467	 * hmeblkpa  = hmeblkpa						;\
2468	 * hmemisc  = hblk_misc						;\
2469	 * hmeblkva  = hmeblkva 					;\
2470	 */								;\
2471									;\
2472	add	hatid, SFHME_TTE, hatid					;\
2473	add	hmeblkpa, hatid, hmeblkpa				;\
2474	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2475	add	hmeblkva, hatid, hmeblkva				;\
2476	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2477	HMELOCK_EXIT(hatid, hatid, ASI_MEM)	/* drop lock */		;\
2478	set	TTE_SUSPEND, hatid					;\
2479	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2480	btst	tte, hatid						;\
2481	bz,pt	%xcc, foundlabel					;\
2482	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2483									;\
2484	/*								;\
2485	 * Mapping is suspended, so goto suspend label.			;\
2486	 */								;\
2487	ba,pt	%xcc, suspendlabel					;\
2488	  nop
2489
2490/*
2491 * GET_SHME_TTE is similar to GET_TTE() except it searches
2492 * shared hmeblks via HMEHASH_SEARCH_SHME() macro.
2493 * If valid tte is found, hmemisc = shctx flag, i.e., shme is
2494 * either 0 (not part of scd) or 1 (part of scd).
2495 */
2496#define GET_SHME_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea,	\
2497		hmemisc, hmeshift, hashno, label, foundlabel,		\
2498		suspendlabel, exitlabel)				\
2499									;\
2500	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
2501	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
2502	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
2503		hmeblkpa, label/**/5, hmemisc, hmeblkva)		;\
2504									;\
2505	/*								;\
2506	 * tagacc = tagacc						;\
2507	 * hatid = hatid						;\
2508	 * tsbarea = tsbarea						;\
2509	 * tte   = hmebp (hme bucket pointer)				;\
2510	 * hmeblkpa  = vapg  (virtual page)				;\
2511	 * hmemisc, hmeblkva = scratch					;\
2512	 */								;\
2513	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmemisc)	;\
2514									;\
2515	/*								;\
2516	 * tagacc = tagacc						;\
2517	 * hatid = hatid						;\
2518	 * tsbarea = tsbarea						;\
2519	 * tte   = hmebp						;\
2520	 * hmemisc  = htag_bspage + hashno + 0 (for rid)		;\
2521	 * hmeblkpa  = CLOBBERED					;\
2522	 * hmeblkva  = scratch						;\
2523	 */								;\
2524	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
2525	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
2526									;\
2527	add     tte, HMEBUCK_NEXTPA, hmeblkpa				;\
2528	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
2529	add     tte, HMEBUCK_HBLK, hmeblkva				;\
2530	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
2531	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tagacc, tte)			;\
2532									;\
2533label/**/8:								;\
2534	HMEHASH_SEARCH_SHME(hmemisc, hatid, hmeblkpa, hmeblkva, 	\
2535		tsbarea, tagacc, tte, label/**/1)			;\
2536	/*								;\
2537	 * tagacc = CLOBBERED						;\
2538	 * tte = CLOBBERED						;\
2539	 * hmeblkpa = hmeblkpa						;\
2540	 * hmeblkva = hmeblkva						;\
2541	 */								;\
2542	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2543	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2544	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2545	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2546	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2547	  nop								;\
2548label/**/4:								;\
2549	/*								;\
2550	 * We have found the hmeblk containing the hment.		;\
2551	 * Now we calculate the corresponding tte.			;\
2552	 *								;\
2553	 * tagacc = tagacc						;\
2554	 * hatid = hatid						;\
2555	 * tte   = clobbered						;\
2556	 * hmeblkpa  = hmeblkpa						;\
2557	 * hmemisc  = hblktag						;\
2558	 * hmeblkva  = hmeblkva 					;\
2559	 * tsbarea = tsbmiss area					;\
2560	 */								;\
2561	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hatid, hmemisc, tte,		\
2562		label/**/2)						;\
2563									;\
2564	/*								;\
2565	 * tagacc = tagacc						;\
2566	 * hatid = hmentoff						;\
2567	 * tte = clobbered						;\
2568	 * hmeblkpa  = hmeblkpa						;\
2569	 * hmemisc  = hblk_misc						;\
2570	 * hmeblkva  = hmeblkva						;\
2571	 * tsbarea = tsbmiss area					;\
2572	 */								;\
2573									;\
2574	add	hatid, SFHME_TTE, hatid					;\
2575	add	hmeblkpa, hatid, hmeblkpa				;\
2576	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2577	brlz,pt tte, label/**/6						;\
2578	  add	hmeblkva, hatid, hmeblkva				;\
2579	btst	HBLK_SZMASK, hmemisc					;\
2580	bnz,a,pt %icc, label/**/7					;\
2581	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2582									;\
2583	/*								;\
2584 	 * We found an invalid 8K tte in shme.				;\
2585	 * it may not belong to shme's region since			;\
2586	 * region size/alignment granularity is 8K but different	;\
2587	 * regions don't share hmeblks. Continue the search.		;\
2588	 */								;\
2589	sub	hmeblkpa, hatid, hmeblkpa				;\
2590	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2591	srlx	tagacc, hmeshift, tte					;\
2592	add	hmeblkpa, HMEBLK_NEXT, hmeblkva				;\
2593	ldxa	[hmeblkva]ASI_MEM, hmeblkva				;\
2594	add	hmeblkpa, HMEBLK_NEXTPA, hmeblkpa			;\
2595	ldxa	[hmeblkpa]ASI_MEM, hmeblkpa				;\
2596	MAKE_HASHTAG(tte, hatid, hmeshift, hashno, hmemisc)		;\
2597	ba,a,pt	%xcc, label/**/8					;\
2598label/**/6:								;\
2599	GET_SCDSHMERMAP(tsbarea, hmeblkpa, hatid, hmemisc)		;\
2600	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hatid 	;\
2601label/**/7:								;\
2602	HMELOCK_EXIT(hatid, hatid, ASI_MEM)	/* drop lock */		;\
2603	set	TTE_SUSPEND, hatid					;\
2604	TTE_SUSPEND_INT_SHIFT(hatid)					;\
2605	btst	tte, hatid						;\
2606	bz,pt	%xcc, foundlabel					;\
2607	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2608									;\
2609	/*								;\
2610	 * Mapping is suspended, so goto suspend label.			;\
2611	 */								;\
2612	ba,pt	%xcc, suspendlabel					;\
2613	  nop
2614
2615	/*
2616	 * KERNEL PROTECTION HANDLER
2617	 *
2618	 * g1 = tsb8k pointer register (clobbered)
2619	 * g2 = tag access register (ro)
2620	 * g3 - g7 = scratch registers
2621	 *
2622	 * Note: This function is patched at runtime for performance reasons.
2623	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
2624	 */
2625	ENTRY_NP(sfmmu_kprot_trap)
2626	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2627sfmmu_kprot_patch_ktsb_base:
2628	RUNTIME_PATCH_SETX(%g1, %g6)
2629	/* %g1 = contents of ktsb_base or ktsb_pbase */
2630sfmmu_kprot_patch_ktsb_szcode:
2631	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
2632
2633	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2634	! %g1 = First TSB entry pointer, as TSB miss handler expects
2635
2636	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2637sfmmu_kprot_patch_ktsb4m_base:
2638	RUNTIME_PATCH_SETX(%g3, %g6)
2639	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2640sfmmu_kprot_patch_ktsb4m_szcode:
2641	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
2642
2643	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2644	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2645
2646        CPU_TSBMISS_AREA(%g6, %g7)
2647        HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2648	ba,pt	%xcc, sfmmu_tsb_miss_tt
2649	  nop
2650
2651	/*
2652	 * USER PROTECTION HANDLER
2653	 *
2654	 * g1 = tsb8k pointer register (ro)
2655	 * g2 = tag access register (ro)
2656	 * g3 = faulting context (clobbered, currently not used)
2657	 * g4 - g7 = scratch registers
2658	 */
2659	ALTENTRY(sfmmu_uprot_trap)
2660#ifdef sun4v
2661	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2662	/* %g1 = first TSB entry ptr now, %g2 preserved */
2663
2664	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
2665	brlz,pt %g3, 9f				/* check for 2nd TSB */
2666	  nop
2667
2668	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2669	/* %g3 = second TSB entry ptr now, %g2 preserved */
2670
2671#else /* sun4v */
2672#ifdef UTSB_PHYS
2673	/* g1 = first TSB entry ptr */
2674	GET_2ND_TSBREG(%g3)
2675	brlz,pt %g3, 9f			/* check for 2nd TSB */
2676	  nop
2677
2678	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2679	/* %g3 = second TSB entry ptr now, %g2 preserved */
2680#else /* UTSB_PHYS */
2681	brgez,pt %g1, 9f		/* check for 2nd TSB */
2682	  mov	-1, %g3			/* set second tsbe ptr to -1 */
2683
2684	mov	%g2, %g7
2685	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2686	/* %g3 = second TSB entry ptr now, %g7 clobbered */
2687	mov	%g1, %g7
2688	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2689#endif /* UTSB_PHYS */
2690#endif /* sun4v */
26919:
2692	CPU_TSBMISS_AREA(%g6, %g7)
2693	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2694	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
2695	  nop
2696
2697	/*
2698	 * Kernel 8K page iTLB miss.  We also get here if we took a
2699	 * fast instruction access mmu miss trap while running in
2700	 * invalid context.
2701	 *
2702	 * %g1 = 8K TSB pointer register (not used, clobbered)
2703	 * %g2 = tag access register (used)
2704	 * %g3 = faulting context id (used)
2705	 * %g7 = TSB tag to match (used)
2706	 */
2707	.align	64
2708	ALTENTRY(sfmmu_kitlb_miss)
2709	brnz,pn %g3, tsb_tl0_noctxt
2710	  nop
2711
2712	/* kernel miss */
2713	/* get kernel tsb pointer */
2714	/* we patch the next set of instructions at run time */
2715	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2716iktsbbase:
2717	RUNTIME_PATCH_SETX(%g4, %g5)
2718	/* %g4 = contents of ktsb_base or ktsb_pbase */
2719
2720iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2721	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2722	or	%g4, %g1, %g1			! form tsb ptr
2723	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2724	cmp	%g4, %g7
2725	bne,pn	%xcc, iktsb4mbase		! check 4m ktsb
2726	  srlx    %g2, MMU_PAGESHIFT4M, %g3	! use 4m virt-page as TSB index
2727
2728	andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2729	bz,pn	%icc, exec_fault
2730	  nop
2731	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2732	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2733	retry
2734
2735iktsb4mbase:
2736        RUNTIME_PATCH_SETX(%g4, %g6)
2737        /* %g4 = contents of ktsb4m_base or ktsb4m_pbase */
2738iktsb4m:
2739	sllx    %g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2740        srlx    %g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2741	add	%g4, %g3, %g3			! %g3 = 4m tsbe ptr
2742	ldda	[%g3]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2743	cmp	%g4, %g7
2744	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
2745	  andcc %g5, TTE_EXECPRM_INT, %g0		! check exec bit
2746	bz,pn	%icc, exec_fault
2747	  nop
2748	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2749	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2750	retry
2751
2752	/*
2753	 * Kernel dTLB miss.  We also get here if we took a fast data
2754	 * access mmu miss trap while running in invalid context.
2755	 *
2756	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2757	 *	We select the TSB miss handler to branch to depending on
2758	 *	the virtual address of the access.  In the future it may
2759	 *	be desirable to separate kpm TTEs into their own TSB,
2760	 *	in which case all that needs to be done is to set
2761	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2762	 *	early in the miss if we detect a kpm VA to a new handler.
2763	 *
2764	 * %g1 = 8K TSB pointer register (not used, clobbered)
2765	 * %g2 = tag access register (used)
2766	 * %g3 = faulting context id (used)
2767	 */
2768	.align	64
2769	ALTENTRY(sfmmu_kdtlb_miss)
2770	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
2771	  nop
2772
2773	/* Gather some stats for kpm misses in the TLB. */
2774	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2775	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2776
2777	/*
2778	 * Get first TSB offset and look for 8K/64K/512K mapping
2779	 * using the 8K virtual page as the index.
2780	 *
2781	 * We patch the next set of instructions at run time;
2782	 * any changes here require sfmmu_patch_ktsb changes too.
2783	 */
2784dktsbbase:
2785	RUNTIME_PATCH_SETX(%g7, %g6)
2786	/* %g7 = contents of ktsb_base or ktsb_pbase */
2787
2788dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2789	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2790
2791	/*
2792	 * At this point %g1 is our index into the TSB.
2793	 * We just masked off enough bits of the VA depending
2794	 * on our TSB size code.
2795	 */
2796	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2797	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2798	cmp	%g6, %g4			! compare tag
2799	bne,pn	%xcc, dktsb4m_kpmcheck_small
2800	  add	%g7, %g1, %g1			/* form tsb ptr */
2801	TT_TRACE(trace_tsbhit)
2802	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2803	/* trapstat expects tte in %g5 */
2804	retry
2805
2806	/*
2807	 * If kpm is using large pages, the following instruction needs
2808	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2809	 * so that we will probe the 4M TSB regardless of the VA.  In
2810	 * the case kpm is using small pages, we know no large kernel
2811	 * mappings are located above 0x80000000.00000000 so we skip the
2812	 * probe as an optimization.
2813	 */
2814dktsb4m_kpmcheck_small:
2815	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2816	  /* delay slot safe, below */
2817
2818	/*
2819	 * Get second TSB offset and look for 4M mapping
2820	 * using 4M virtual page as the TSB index.
2821	 *
2822	 * Here:
2823	 * %g1 = 8K TSB pointer.  Don't squash it.
2824	 * %g2 = tag access register (we still need it)
2825	 */
2826	srlx	%g2, MMU_PAGESHIFT4M, %g3
2827
2828	/*
2829	 * We patch the next set of instructions at run time;
2830	 * any changes here require sfmmu_patch_ktsb changes too.
2831	 */
2832dktsb4mbase:
2833	RUNTIME_PATCH_SETX(%g7, %g6)
2834	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2835dktsb4m:
2836	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2837	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2838
2839	/*
2840	 * At this point %g3 is our index into the TSB.
2841	 * We just masked off enough bits of the VA depending
2842	 * on our TSB size code.
2843	 */
2844	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2845	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2846	cmp	%g6, %g4			! compare tag
2847
2848dktsb4m_tsbmiss:
2849	bne,pn	%xcc, dktsb4m_kpmcheck
2850	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
2851	TT_TRACE(trace_tsbhit)
2852	/* we don't check TTE size here since we assume 4M TSB is separate */
2853	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2854	/* trapstat expects tte in %g5 */
2855	retry
2856
2857	/*
2858	 * So, we failed to find a valid TTE to match the faulting
2859	 * address in either TSB.  There are a few cases that could land
2860	 * us here:
2861	 *
2862	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2863	 *    to sfmmu_tsb_miss_tt to handle the miss.
2864	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2865	 *    4M TSB.  Let segkpm handle it.
2866	 *
2867	 * Note that we shouldn't land here in the case of a kpm VA when
2868	 * kpm_smallpages is active -- we handled that case earlier at
2869	 * dktsb4m_kpmcheck_small.
2870	 *
2871	 * At this point:
2872	 *  g1 = 8K-indexed primary TSB pointer
2873	 *  g2 = tag access register
2874	 *  g3 = 4M-indexed secondary TSB pointer
2875	 */
2876dktsb4m_kpmcheck:
2877	cmp	%g2, %g0
2878	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
2879	  nop
2880	ba,a,pt	%icc, sfmmu_tsb_miss_tt
2881	  nop
2882
2883#ifdef sun4v
2884	/*
2885	 * User instruction miss w/ single TSB.
2886	 * The first probe covers 8K, 64K, and 512K page sizes,
2887	 * because 64K and 512K mappings are replicated off 8K
2888	 * pointer.
2889	 *
2890	 * g1 = tsb8k pointer register
2891	 * g2 = tag access register
2892	 * g3 - g6 = scratch registers
2893	 * g7 = TSB tag to match
2894	 */
2895	.align	64
2896	ALTENTRY(sfmmu_uitlb_fastpath)
2897
2898	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2899	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2900	ba,pn	%xcc, sfmmu_tsb_miss_tt
2901	  mov	-1, %g3
2902
2903	/*
2904	 * User data miss w/ single TSB.
2905	 * The first probe covers 8K, 64K, and 512K page sizes,
2906	 * because 64K and 512K mappings are replicated off 8K
2907	 * pointer.
2908	 *
2909	 * g1 = tsb8k pointer register
2910	 * g2 = tag access register
2911	 * g3 - g6 = scratch registers
2912	 * g7 = TSB tag to match
2913	 */
2914	.align 64
2915	ALTENTRY(sfmmu_udtlb_fastpath)
2916
2917	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2918	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2919	ba,pn	%xcc, sfmmu_tsb_miss_tt
2920	  mov	-1, %g3
2921
2922	/*
2923	 * User instruction miss w/ multiple TSBs (sun4v).
2924	 * The first probe covers 8K, 64K, and 512K page sizes,
2925	 * because 64K and 512K mappings are replicated off 8K
2926	 * pointer.  Second probe covers 4M page size only.
2927	 *
2928	 * Just like sfmmu_udtlb_slowpath, except:
2929	 *   o Uses ASI_ITLB_IN
2930	 *   o checks for execute permission
2931	 *   o No ISM prediction.
2932	 *
2933	 * g1 = tsb8k pointer register
2934	 * g2 = tag access register
2935	 * g3 - g6 = scratch registers
2936	 * g7 = TSB tag to match
2937	 */
2938	.align	64
2939	ALTENTRY(sfmmu_uitlb_slowpath)
2940
2941	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2942	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2943	/* g4 - g5 = clobbered here */
2944
2945	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2946	/* g1 = first TSB pointer, g3 = second TSB pointer */
2947	srlx	%g2, TAG_VALO_SHIFT, %g7
2948	PROBE_2ND_ITSB(%g3, %g7)
2949	/* NOT REACHED */
2950
2951#else /* sun4v */
2952
2953	/*
2954	 * User instruction miss w/ multiple TSBs (sun4u).
2955	 * The first probe covers 8K, 64K, and 512K page sizes,
2956	 * because 64K and 512K mappings are replicated off 8K
2957	 * pointer.  Second probe covers 4M page size only.
2958	 *
2959	 * Just like sfmmu_udtlb_slowpath, except:
2960	 *   o Uses ASI_ITLB_IN
2961	 *   o checks for execute permission
2962	 *   o No ISM prediction.
2963	 *
2964	 * g1 = tsb8k pointer register
2965	 * g2 = tag access register
2966	 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch
2967	 * g4 - g6 = scratch registers
2968	 * g7 = TSB tag to match
2969	 */
2970	.align	64
2971	ALTENTRY(sfmmu_uitlb_slowpath)
2972
2973#ifdef UTSB_PHYS
2974	/*
2975	 * g1 = 1st TSB entry pointer
2976	 * g3 = 2nd TSB base register
2977	 * Need 2nd TSB entry pointer for 2nd probe.
2978	 */
2979	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2980
2981	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2982#else /* UTSB_PHYS */
2983	mov	%g1, %g3	/* save tsb8k reg in %g3 */
2984	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
2985	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2986
2987	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
2988	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
2989	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
2990#endif /* UTSB_PHYS */
2991	/* g1 = first TSB pointer, g3 = second TSB pointer */
2992	srlx	%g2, TAG_VALO_SHIFT, %g7
2993	PROBE_2ND_ITSB(%g3, %g7, isynth)
2994	/* NOT REACHED */
2995#endif /* sun4v */
2996
2997	/*
2998	 * User data miss w/ multiple TSBs.
2999	 * The first probe covers 8K, 64K, and 512K page sizes,
3000	 * because 64K and 512K mappings are replicated off 8K
3001	 * pointer.  Second probe covers 4M page size only.
3002	 *
3003	 * We consider probing for 4M pages first if the VA falls
3004	 * in a range that's likely to be ISM.
3005	 *
3006	 * g1 = tsb8k pointer register
3007	 * g2 = tag access register
3008	 * g3 = 2nd tsbreg if defined UTSB_PHYS, else scratch
3009	 * g4 - g6 = scratch registers
3010	 * g7 = TSB tag to match
3011	 */
3012	.align 64
3013	ALTENTRY(sfmmu_udtlb_slowpath)
3014
3015	/*
3016	 * Check for ISM.  If it exists, look for 4M mappings in the second TSB
3017	 * first, then probe for other mappings in the first TSB if that fails.
3018	 */
3019	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
3020	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
3021	  mov	%g1, %g3
3022
3023udtlb_miss_probefirst:
3024	/*
3025	 * g1 = 8K TSB pointer register
3026	 * g2 = tag access register
3027	 * g3 = (potentially) second TSB entry ptr
3028	 * g6 = ism pred.
3029	 * g7 = vpg_4m
3030	 */
3031#ifdef sun4v
3032	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
3033	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
3034
3035	/*
3036	 * Here:
3037	 *   g1 = first TSB pointer
3038	 *   g2 = tag access reg
3039	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
3040	 */
3041	brgz,pn	%g6, sfmmu_tsb_miss_tt
3042	  nop
3043#else /* sun4v */
3044#ifndef UTSB_PHYS
3045	mov	%g1, %g4
3046	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
3047#endif UTSB_PHYS
3048	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
3049
3050	/*
3051	 * Here:
3052	 *   g1 = first TSB pointer
3053	 *   g2 = tag access reg
3054	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
3055	 */
3056	brgz,pn	%g6, sfmmu_tsb_miss_tt
3057	  nop
3058#ifndef UTSB_PHYS
3059	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
3060#endif UTSB_PHYS
3061	/* fall through in 8K->4M probe order */
3062#endif /* sun4v */
3063
3064udtlb_miss_probesecond:
3065	/*
3066	 * Look in the second TSB for the TTE
3067	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
3068	 * g2 = tag access reg
3069	 * g3 = 8K TSB pointer register
3070	 * g6 = ism pred.
3071	 * g7 = vpg_4m
3072	 */
3073#ifdef sun4v
3074	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
3075	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3076	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
3077#else /* sun4v */
3078#ifdef UTSB_PHYS
3079	GET_2ND_TSBREG(%g3)
3080	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3081	/* tagacc (%g2) is okay, no need to reload, %g3 = second tsbe ptr */
3082#else /* UTSB_PHYS */
3083	mov	%g3, %g7
3084	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
3085	/* %g2 clobbered, %g3 =second tsbe ptr */
3086	mov	MMU_TAG_ACCESS, %g2
3087	ldxa	[%g2]ASI_DMMU, %g2
3088#endif /* UTSB_PHYS */
3089#endif /* sun4v */
3090
3091	srlx	%g2, TAG_VALO_SHIFT, %g7
3092	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
3093	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
3094	brgz,pn	%g6, udtlb_miss_probefirst
3095	  nop
3096
3097	/* fall through to sfmmu_tsb_miss_tt */
3098
3099	ALTENTRY(sfmmu_tsb_miss_tt)
3100	TT_TRACE(trace_tsbmiss)
3101	/*
3102	 * We get here if there is a TSB miss OR a write protect trap.
3103	 *
3104	 * g1 = First TSB entry pointer
3105	 * g2 = tag access register
3106	 * g3 = 4M TSB entry pointer; -1 if no 2nd TSB
3107	 * g4 - g7 = scratch registers
3108	 */
3109
3110	ALTENTRY(sfmmu_tsb_miss)
3111
3112	/*
3113	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3114	 * point to trapstat's TSB miss return code (note that trapstat
3115	 * itself will patch the correct offset to add).
3116	 */
3117	rdpr	%tl, %g7
3118	cmp	%g7, 1
3119	ble,pt	%xcc, 0f
3120	  sethi	%hi(KERNELBASE), %g6
3121	rdpr	%tpc, %g7
3122	or	%g6, %lo(KERNELBASE), %g6
3123	cmp	%g7, %g6
3124	bgeu,pt	%xcc, 0f
3125	/* delay slot safe */
3126
3127	ALTENTRY(tsbmiss_trapstat_patch_point)
3128	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3129	wrpr	%g7, %tpc
3130	add	%g7, 4, %g7
3131	wrpr	%g7, %tnpc
31320:
3133	CPU_TSBMISS_AREA(%g6, %g7)
3134
3135	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save first tsb pointer */
3136	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save second tsb pointer */
3137
3138	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
3139	brz,a,pn %g3, 1f			/* skip ahead if kernel */
3140	  ldn	[%g6 + TSBMISS_KHATID], %g7
3141	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
3142	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
3143
3144	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
3145
3146	cmp	%g3, INVALID_CONTEXT
3147	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
3148	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
3149
3150#ifdef sun4v
3151        ldub    [%g6 + TSBMISS_URTTEFLAGS], %g7	/* clear ctx1 flag set from */
3152        andn    %g7, HAT_CHKCTX1_FLAG, %g7	/* the previous tsb miss    */
3153        stub    %g7, [%g6 + TSBMISS_URTTEFLAGS]
3154#endif
3155
3156	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
3157	/*
3158	 * The miss wasn't in an ISM segment.
3159	 *
3160	 * %g1 %g3, %g4, %g5, %g7 all clobbered
3161	 * %g2 = (pseudo) tag access
3162	 */
3163
3164	ba,pt	%icc, 2f
3165	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
3166
31671:
3168	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
3169	/*
3170	 * 8K and 64K hash.
3171	 */
31722:
3173
3174	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3175		MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte,
3176		sfmmu_suspend_tl, tsb_512K)
3177	/* NOT REACHED */
3178
3179tsb_512K:
3180	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3181	brz,pn	%g5, 3f
3182	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3183	and	%g4, HAT_512K_FLAG, %g5
3184
3185	/*
3186	 * Note that there is a small window here where we may have
3187	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
3188	 * flag yet, so we will skip searching the 512k hash list.
3189	 * In this case we will end up in pagefault which will find
3190	 * the mapping and return.  So, in this instance we will end up
3191	 * spending a bit more time resolving this TSB miss, but it can
3192	 * only happen once per process and even then, the chances of that
3193	 * are very small, so it's not worth the extra overhead it would
3194	 * take to close this window.
3195	 */
3196	brz,pn	%g5, tsb_4M
3197	  nop
31983:
3199	/*
3200	 * 512K hash
3201	 */
3202
3203	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3204		MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte,
3205		sfmmu_suspend_tl, tsb_4M)
3206	/* NOT REACHED */
3207
3208tsb_4M:
3209	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3210	brz,pn	%g5, 4f
3211	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3212	and	%g4, HAT_4M_FLAG, %g5
3213	brz,pn	%g5, tsb_32M
3214	  nop
32154:
3216	/*
3217	 * 4M hash
3218	 */
3219
3220	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3221		MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte,
3222		sfmmu_suspend_tl, tsb_32M)
3223	/* NOT REACHED */
3224
3225tsb_32M:
3226	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3227#ifdef sun4v
3228        brz,pn	%g5, 6f
3229#else
3230	brz,pn	%g5, tsb_pagefault
3231#endif
3232	  ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3233	and	%g4, HAT_32M_FLAG, %g5
3234	brz,pn	%g5, tsb_256M
3235	  nop
32365:
3237	/*
3238	 * 32M hash
3239	 */
3240
3241	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3242		MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte,
3243		sfmmu_suspend_tl, tsb_256M)
3244	/* NOT REACHED */
3245
3246#ifdef sun4u
3247#define tsb_shme        tsb_pagefault
3248#endif
3249tsb_256M:
3250	ldub	[%g6 + TSBMISS_UTTEFLAGS], %g4
3251	and	%g4, HAT_256M_FLAG, %g5
3252	brz,pn	%g5, tsb_shme
3253	  nop
32546:
3255	/*
3256	 * 256M hash
3257	 */
3258
3259	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3260	    MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte,
3261	    sfmmu_suspend_tl, tsb_shme)
3262	/* NOT REACHED */
3263
3264tsb_checktte:
3265	/*
3266	 * g1 = hblk_misc
3267	 * g2 = tagacc
3268	 * g3 = tte
3269	 * g4 = tte pa
3270	 * g5 = tte va
3271	 * g6 = tsbmiss area
3272	 * g7 = hatid
3273	 */
3274	brlz,a,pt %g3, tsb_validtte
3275	  rdpr	%tt, %g7
3276
3277#ifdef sun4u
3278#undef tsb_shme
3279	ba	tsb_pagefault
3280	  nop
3281#else
3282
3283tsb_shme:
3284	/*
3285	 * g2 = tagacc
3286	 * g6 = tsbmiss area
3287	 */
3288	sllx	%g2, TAGACC_CTX_LSHIFT, %g5
3289	brz,pn	%g5, tsb_pagefault
3290	  nop
3291	ldx	[%g6 + TSBMISS_SHARED_UHATID], %g7	/* g7 = srdp */
3292	brz,pn	%g7, tsb_pagefault
3293	  nop
3294
3295	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3296		MMU_PAGESHIFT64K, TTE64K, tsb_shme_l8K, tsb_shme_checktte,
3297		sfmmu_suspend_tl, tsb_shme_512K)
3298	/* NOT REACHED */
3299
3300tsb_shme_512K:
3301	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3302	and	%g4, HAT_512K_FLAG, %g5
3303	brz,pn	%g5, tsb_shme_4M
3304	  nop
3305
3306	/*
3307	 * 512K hash
3308	 */
3309
3310	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3311		MMU_PAGESHIFT512K, TTE512K, tsb_shme_l512K, tsb_shme_checktte,
3312		sfmmu_suspend_tl, tsb_shme_4M)
3313	/* NOT REACHED */
3314
3315tsb_shme_4M:
3316	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3317	and	%g4, HAT_4M_FLAG, %g5
3318	brz,pn	%g5, tsb_shme_32M
3319	  nop
33204:
3321	/*
3322	 * 4M hash
3323	 */
3324	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3325		MMU_PAGESHIFT4M, TTE4M, tsb_shme_l4M, tsb_shme_checktte,
3326		sfmmu_suspend_tl, tsb_shme_32M)
3327	/* NOT REACHED */
3328
3329tsb_shme_32M:
3330	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3331	and	%g4, HAT_32M_FLAG, %g5
3332	brz,pn	%g5, tsb_shme_256M
3333	  nop
3334
3335	/*
3336	 * 32M hash
3337	 */
3338
3339	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3340		MMU_PAGESHIFT32M, TTE32M, tsb_shme_l32M, tsb_shme_checktte,
3341		sfmmu_suspend_tl, tsb_shme_256M)
3342	/* NOT REACHED */
3343
3344tsb_shme_256M:
3345	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g4
3346	and	%g4, HAT_256M_FLAG, %g5
3347	brz,pn	%g5, tsb_pagefault
3348	  nop
3349
3350	/*
3351	 * 256M hash
3352	 */
3353
3354	GET_SHME_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
3355	    MMU_PAGESHIFT256M, TTE256M, tsb_shme_l256M, tsb_shme_checktte,
3356	    sfmmu_suspend_tl, tsb_pagefault)
3357	/* NOT REACHED */
3358
3359tsb_shme_checktte:
3360
3361	brgez,pn %g3, tsb_pagefault
3362	  rdpr	%tt, %g7
3363	/*
3364	 * g1 = ctx1 flag
3365	 * g3 = tte
3366	 * g4 = tte pa
3367	 * g5 = tte va
3368	 * g6 = tsbmiss area
3369	 * g7 = tt
3370	 */
3371
3372	brz,pt  %g1, tsb_validtte
3373	  nop
3374	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g1
3375	  or	%g1, HAT_CHKCTX1_FLAG, %g1
3376	stub    %g1, [%g6 + TSBMISS_URTTEFLAGS]
3377
3378	SAVE_CTX1(%g7, %g2, %g1, tsb_shmel)
3379#endif /* sun4u */
3380
3381tsb_validtte:
3382	/*
3383	 * g3 = tte
3384	 * g4 = tte pa
3385	 * g5 = tte va
3386	 * g6 = tsbmiss area
3387	 * g7 = tt
3388	 */
3389
3390	/*
3391	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
3392	 */
3393	cmp	%g7, FAST_PROT_TT
3394	bne,pt	%icc, 4f
3395	  nop
3396
3397	TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod,
3398	    tsb_protfault)
3399
3400	rdpr	%tt, %g5
3401	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3402#ifdef sun4v
3403	MMU_FAULT_STATUS_AREA(%g7)
3404	ldx	[%g7 + MMFSA_D_ADDR], %g5	/* save fault addr for later */
3405#endif
3406	ba,pt	%xcc, tsb_update_tl1
3407	  nop
3408
34094:
3410	/*
3411	 * If ITLB miss check exec bit.
3412	 * If not set treat as invalid TTE.
3413	 */
3414	cmp     %g7, T_INSTR_MMU_MISS
3415	be,pn	%icc, 5f
3416	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
3417	cmp     %g7, FAST_IMMU_MISS_TT
3418	bne,pt %icc, 3f
3419	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
34205:
3421	bz,pn %icc, tsb_protfault
3422	  nop
3423
34243:
3425	/*
3426	 * Set reference bit if not already set
3427	 */
3428	TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref)
3429
3430	/*
3431	 * Now, load into TSB/TLB.  At this point:
3432	 * g3 = tte
3433	 * g4 = patte
3434	 * g6 = tsbmiss area
3435	 */
3436	rdpr	%tt, %g5
3437#ifdef sun4v
3438	MMU_FAULT_STATUS_AREA(%g2)
3439	cmp	%g5, T_INSTR_MMU_MISS
3440	be,a,pt	%icc, 9f
3441	  nop
3442	cmp	%g5, FAST_IMMU_MISS_TT
3443	be,a,pt	%icc, 9f
3444	  nop
3445	add	%g2, MMFSA_D_, %g2
34469:
3447	ldx	[%g2 + MMFSA_CTX_], %g7
3448	sllx	%g7, TTARGET_CTX_SHIFT, %g7
3449	ldx	[%g2 + MMFSA_ADDR_], %g2
3450	mov	%g2, %g5		! save the fault addr for later use
3451	srlx	%g2, TTARGET_VA_SHIFT, %g2
3452	or	%g2, %g7, %g2
3453#else
3454	cmp	%g5, FAST_IMMU_MISS_TT
3455	be,a,pt	%icc, tsb_update_tl1
3456	  ldxa	[%g0]ASI_IMMU, %g2
3457	ldxa	[%g0]ASI_DMMU, %g2
3458#endif
3459tsb_update_tl1:
3460	srlx	%g2, TTARGET_CTX_SHIFT, %g7
3461	brz,pn	%g7, tsb_kernel
3462#ifdef sun4v
3463	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
3464#else
3465	  srlx	%g3, TTE_SZ_SHFT, %g7
3466#endif
3467
3468tsb_user:
3469#ifdef sun4v
3470	cmp	%g7, TTE4M
3471	bge,pn	%icc, tsb_user4m
3472	  nop
3473#else /* sun4v */
3474	cmp	%g7, TTESZ_VALID | TTE4M
3475	be,pn	%icc, tsb_user4m
3476	  srlx	%g3, TTE_SZ2_SHFT, %g7
3477	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
3478#ifdef ITLB_32M_256M_SUPPORT
3479	bnz,pn	%icc, tsb_user4m
3480	  nop
3481#else /* ITLB_32M_256M_SUPPORT */
3482	bnz,a,pn %icc, tsb_user_pn_synth
3483	 cmp	%g5, FAST_IMMU_MISS_TT
3484#endif /* ITLB_32M_256M_SUPPORT */
3485#endif /* sun4v */
3486
3487tsb_user8k:
3488#ifdef sun4v
3489	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3490	and	%g7, HAT_CHKCTX1_FLAG, %g1
3491	brz,a,pn %g1, 1f
3492	  ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = first TSB ptr
3493	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR, %g1)
3494	brlz,a,pn %g1, ptl1_panic			! if no shared tsb
3495	  mov PTL1_NO_SCDTSB8K, %g1			! panic
3496	GET_3RD_TSBE_PTR(%g5, %g1, %g6, %g7)
34971:
3498#else
3499	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = first TSB ptr
3500
3501#ifndef UTSB_PHYS
3502	mov	ASI_N, %g7	! user TSBs accessed by VA
3503	mov	%g7, %asi
3504#endif /* UTSB_PHYS */
3505
3506#endif /* sun4v */
3507
3508	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5)
3509
3510#ifdef sun4v
3511	rdpr    %tt, %g5
3512	cmp	%g5, T_INSTR_MMU_MISS
3513	be,a,pn	%xcc, 9f
3514	  mov	%g3, %g5
3515#endif /* sun4v */
3516	cmp	%g5, FAST_IMMU_MISS_TT
3517	be,pn	%xcc, 9f
3518	  mov	%g3, %g5
3519
3520	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3521	! trapstat wants TTE in %g5
3522	retry
35239:
3524	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3525	! trapstat wants TTE in %g5
3526	retry
3527
3528tsb_user4m:
3529#ifdef sun4v
3530	ldub	[%g6 + TSBMISS_URTTEFLAGS], %g7
3531	and	%g7, HAT_CHKCTX1_FLAG, %g1
3532	brz,a,pn %g1, 4f
3533	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		! g1 = TSB ptr
3534	GET_UTSBREG_SHCTX(%g6, TSBMISS_TSBSCDPTR4M, %g1)
3535	brlz,a,pn %g1, 5f				! if no shared 2nd tsb
3536	  nop
3537	GET_4TH_TSBE_PTR(%g5, %g1, %g6, %g7)
3538#else
3539	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		! g1 = TSB ptr
3540#endif
35414:
3542	brlz,pn %g1, 5f	/* Check to see if we have 2nd TSB programmed */
3543	  nop
3544
3545#ifndef UTSB_PHYS
3546	mov	ASI_N, %g7	! user TSBs accessed by VA
3547	mov	%g7, %asi
3548#endif /* UTSB_PHYS */
3549
3550        TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6)
3551
35525:
3553#ifdef sun4v
3554	rdpr    %tt, %g5
3555        cmp     %g5, T_INSTR_MMU_MISS
3556        be,a,pn %xcc, 9f
3557          mov   %g3, %g5
3558#endif /* sun4v */
3559        cmp     %g5, FAST_IMMU_MISS_TT
3560        be,pn   %xcc, 9f
3561        mov     %g3, %g5
3562
3563        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3564        ! trapstat wants TTE in %g5
3565        retry
35669:
3567        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3568        ! trapstat wants TTE in %g5
3569        retry
3570
3571#if !defined(sun4v) && !defined(ITLB_32M_256M_SUPPORT)
3572	/*
3573	 * Panther ITLB synthesis.
3574	 * The Panther 32M and 256M ITLB code simulates these two large page
3575	 * sizes with 4M pages, to provide support for programs, for example
3576	 * Java, that may copy instructions into a 32M or 256M data page and
3577	 * then execute them. The code below generates the 4M pfn bits and
3578	 * saves them in the modified 32M/256M ttes in the TSB. If the tte is
3579	 * stored in the DTLB to map a 32M/256M page, the 4M pfn offset bits
3580	 * are ignored by the hardware.
3581	 *
3582	 * Now, load into TSB/TLB.  At this point:
3583	 * g2 = tagtarget
3584	 * g3 = tte
3585	 * g4 = patte
3586	 * g5 = tt
3587	 * g6 = tsbmiss area
3588	 */
3589tsb_user_pn_synth:
3590	be,pt	%xcc, tsb_user_itlb_synth	/* ITLB miss */
3591	  andcc %g3, TTE_EXECPRM_INT, %g0	/* is execprm bit set */
3592	bz,pn %icc, 4b				/* if not, been here before */
3593	  ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	/* g1 = tsbp */
3594	brlz,a,pn %g1, 5f			/* no 2nd tsb */
3595	  mov	%g3, %g5
3596
3597	mov	MMU_TAG_ACCESS, %g7
3598	ldxa	[%g7]ASI_DMMU, %g6		/* get tag access va */
3599	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 1)	/* make 4M pfn offset */
3600
3601	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3602	mov	%g7, %asi
3603	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 4) /* update TSB */
36045:
3605        DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3606        retry
3607
3608tsb_user_itlb_synth:
3609	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */
3610
3611	mov	MMU_TAG_ACCESS, %g7
3612	ldxa	[%g7]ASI_IMMU, %g6		/* get tag access va */
3613	GET_4M_PFN_OFF(%g3, %g6, %g5, %g7, 2)	/* make 4M pfn offset */
3614	brlz,a,pn %g1, 7f	/* Check to see if we have 2nd TSB programmed */
3615	  or	%g5, %g3, %g5			/* add 4M bits to TTE */
3616
3617	mov	ASI_N, %g7	/* user TSBs always accessed by VA */
3618	mov	%g7, %asi
3619	TSB_UPDATE_TL_PN(%g1, %g5, %g2, %g4, %g7, %g3, 6) /* update TSB */
36207:
3621	SET_TTE4M_PN(%g5, %g7)			/* add TTE4M pagesize to TTE */
3622        ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3623        retry
3624#endif /* sun4v && ITLB_32M_256M_SUPPORT */
3625
3626tsb_kernel:
3627#ifdef sun4v
3628	rdpr	%tt, %g5
3629	cmp	%g7, TTE4M
3630	bge,pn	%icc, 5f
3631#else
3632	cmp	%g7, TTESZ_VALID | TTE4M	! no 32M or 256M support
3633	be,pn	%icc, 5f
3634#endif
3635	  nop
3636	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8k tsbptr
3637	ba,pt	%xcc, 6f
3638	  nop
36395:
3640	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4m tsbptr
3641	brlz,pn	%g1, 3f		/* skip programming if 4m TSB ptr is -1 */
3642	  nop
36436:
3644#ifndef sun4v
3645tsb_kernel_patch_asi:
3646	or	%g0, RUNTIME_PATCH, %g6
3647	mov	%g6, %asi	! XXX avoid writing to %asi !!
3648#endif
3649	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7)
36503:
3651#ifdef sun4v
3652	cmp	%g5, T_INSTR_MMU_MISS
3653	be,a,pn	%icc, 1f
3654	  mov	%g3, %g5			! trapstat wants TTE in %g5
3655#endif /* sun4v */
3656	cmp	%g5, FAST_IMMU_MISS_TT
3657	be,pn	%icc, 1f
3658	  mov	%g3, %g5			! trapstat wants TTE in %g5
3659	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3660	! trapstat wants TTE in %g5
3661	retry
36621:
3663	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
3664	! trapstat wants TTE in %g5
3665	retry
3666
3667tsb_ism:
3668	/*
3669	 * This is an ISM [i|d]tlb miss.  We optimize for largest
3670	 * page size down to smallest.
3671	 *
3672	 * g2 = vaddr + ctx(or ctxtype (sun4v)) aka (pseudo-)tag access
3673	 *	register
3674	 * g3 = ismmap->ism_seg
3675	 * g4 = physical address of ismmap->ism_sfmmu
3676	 * g6 = tsbmiss area
3677	 */
3678	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
3679	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
3680	  mov	PTL1_BAD_ISM, %g1
3681						/* g5 = pa of imap_vb_shift */
3682	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
3683	lduba	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
3684	srlx	%g3, %g4, %g3			/* clr size field */
3685	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
3686	sllx    %g3, %g4, %g3                   /* g3 = ism vbase */
3687	and     %g2, %g1, %g4                   /* g4 = ctx number */
3688	andn    %g2, %g1, %g1                   /* g1 = tlb miss vaddr */
3689	sub     %g1, %g3, %g2                   /* g2 = offset in ISM seg */
3690	or      %g2, %g4, %g2                   /* g2 = (pseudo-)tagacc */
3691	sub     %g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
3692	lduha   [%g5]ASI_MEM, %g4               /* g5 = pa of imap_hatflags */
3693#ifdef sun4v
3694	and     %g4, HAT_CTX1_FLAG, %g5         /* g5 = imap_hatflags */
3695	brz,pt %g5, tsb_chk4M_ism
3696	  nop
3697	ldub    [%g6 + TSBMISS_URTTEFLAGS], %g5
3698	or      %g5, HAT_CHKCTX1_FLAG, %g5
3699	stub    %g5, [%g6 + TSBMISS_URTTEFLAGS]
3700	rdpr    %tt, %g5
3701	SAVE_CTX1(%g5, %g3, %g1, tsb_shctxl)
3702#endif
3703	/*
3704	 * ISM pages are always locked down.
3705	 * If we can't find the tte then pagefault
3706	 * and let the spt segment driver resolve it.
3707	 *
3708	 * g2 = tagacc w/ISM vaddr (offset in ISM seg)
3709	 * g4 = imap_hatflags
3710	 * g6 = tsb miss area
3711	 * g7 = ISM hatid
3712	 */
3713
3714tsb_chk4M_ism:
3715	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
3716	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
3717	  nop
3718
3719tsb_ism_32M:
3720	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
3721	brz,pn	%g5, tsb_ism_256M
3722	  nop
3723
3724	/*
3725	 * 32M hash.
3726	 */
3727
3728	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M,
3729	    TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
3730	    tsb_ism_4M)
3731	/* NOT REACHED */
3732
3733tsb_ism_32M_found:
3734	brlz,a,pt %g3, tsb_validtte
3735	  rdpr	%tt, %g7
3736	ba,pt	%xcc, tsb_ism_4M
3737	  nop
3738
3739tsb_ism_256M:
3740	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
3741	brz,a,pn %g5, ptl1_panic
3742	  mov	PTL1_BAD_ISM, %g1
3743
3744	/*
3745	 * 256M hash.
3746	 */
3747	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M,
3748	    TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
3749	    tsb_ism_4M)
3750
3751tsb_ism_256M_found:
3752	brlz,a,pt %g3, tsb_validtte
3753	  rdpr	%tt, %g7
3754
3755tsb_ism_4M:
3756	/*
3757	 * 4M hash.
3758	 */
3759	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M,
3760	    TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
3761	    tsb_ism_8K)
3762	/* NOT REACHED */
3763
3764tsb_ism_4M_found:
3765	brlz,a,pt %g3, tsb_validtte
3766	  rdpr	%tt, %g7
3767
3768tsb_ism_8K:
3769	/*
3770	 * 8K and 64K hash.
3771	 */
3772
3773	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K,
3774	    TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
3775	    tsb_pagefault)
3776	/* NOT REACHED */
3777
3778tsb_ism_8K_found:
3779	brlz,a,pt %g3, tsb_validtte
3780	  rdpr	%tt, %g7
3781
3782tsb_pagefault:
3783	rdpr	%tt, %g7
3784	cmp	%g7, FAST_PROT_TT
3785	be,a,pn	%icc, tsb_protfault
3786	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt
3787
3788tsb_protfault:
3789	/*
3790	 * we get here if we couldn't find a valid tte in the hash.
3791	 *
3792	 * If user and we are at tl>1 we go to window handling code.
3793	 *
3794	 * If kernel and the fault is on the same page as our stack
3795	 * pointer, then we know the stack is bad and the trap handler
3796	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
3797	 *
3798	 * If this is a kernel trap and tl>1, panic.
3799	 *
3800	 * Otherwise we call pagefault.
3801	 */
3802	cmp	%g7, FAST_IMMU_MISS_TT
3803#ifdef sun4v
3804	MMU_FAULT_STATUS_AREA(%g4)
3805	ldx	[%g4 + MMFSA_I_CTX], %g5
3806	ldx	[%g4 + MMFSA_D_CTX], %g4
3807	move	%icc, %g5, %g4
3808	cmp	%g7, T_INSTR_MMU_MISS
3809	move	%icc, %g5, %g4
3810#else
3811	mov	MMU_TAG_ACCESS, %g4
3812	ldxa	[%g4]ASI_DMMU, %g2
3813	ldxa	[%g4]ASI_IMMU, %g5
3814	move	%icc, %g5, %g2
3815	cmp	%g7, T_INSTR_MMU_MISS
3816	move	%icc, %g5, %g2
3817	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
3818#endif
3819	brnz,pn	%g4, 3f				/* skip if not kernel */
3820	  rdpr	%tl, %g5
3821
3822	add	%sp, STACK_BIAS, %g3
3823	srlx	%g3, MMU_PAGESHIFT, %g3
3824	srlx	%g2, MMU_PAGESHIFT, %g4
3825	cmp	%g3, %g4
3826	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
3827	  mov	PTL1_BAD_STACK, %g1
3828
3829	cmp	%g5, 1
3830	ble,pt	%icc, 2f
3831	  nop
3832	TSTAT_CHECK_TL1(2f, %g1, %g2)
3833	rdpr	%tt, %g2
3834	cmp	%g2, FAST_PROT_TT
3835	mov	PTL1_BAD_KPROT_FAULT, %g1
3836	movne	%icc, PTL1_BAD_KMISS, %g1
3837	ba,pt	%icc, ptl1_panic
3838	  nop
3839
38402:
3841	/*
3842	 * We are taking a pagefault in the kernel on a kernel address.  If
3843	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
3844	 * want to call sfmmu_pagefault -- we will instead note that a fault
3845	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
3846	 * (instead of a "retry").  This will step over the faulting
3847	 * instruction.
3848	 */
3849	CPU_INDEX(%g1, %g2)
3850	set	cpu_core, %g2
3851	sllx	%g1, CPU_CORE_SHIFT, %g1
3852	add	%g1, %g2, %g1
3853	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3854	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3855	bz	sfmmu_pagefault
3856	or	%g2, CPU_DTRACE_BADADDR, %g2
3857	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3858	GET_MMU_D_ADDR(%g3, %g4)
3859	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3860	done
3861
38623:
3863	cmp	%g5, 1
3864	ble,pt	%icc, 4f
3865	  nop
3866	TSTAT_CHECK_TL1(4f, %g1, %g2)
3867	ba,pt	%icc, sfmmu_window_trap
3868	  nop
3869
38704:
3871	/*
3872	 * We are taking a pagefault on a non-kernel address.  If we are in
3873	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
3874	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
3875	 */
3876	CPU_INDEX(%g1, %g2)
3877	set	cpu_core, %g2
3878	sllx	%g1, CPU_CORE_SHIFT, %g1
3879	add	%g1, %g2, %g1
3880	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3881	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3882	bz	sfmmu_mmu_trap
3883	or	%g2, CPU_DTRACE_BADADDR, %g2
3884	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3885	GET_MMU_D_ADDR(%g3, %g4)
3886	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3887
3888	/*
3889	 * Be sure that we're actually taking this miss from the kernel --
3890	 * otherwise we have managed to return to user-level with
3891	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3892	 */
3893	rdpr	%tstate, %g2
3894	btst	TSTATE_PRIV, %g2
3895	bz,a	ptl1_panic
3896	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3897	done
3898
3899	ALTENTRY(tsb_tl0_noctxt)
3900	/*
3901	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
3902	 * if it is, indicated that we have faulted and issue a done.
3903	 */
3904	CPU_INDEX(%g5, %g6)
3905	set	cpu_core, %g6
3906	sllx	%g5, CPU_CORE_SHIFT, %g5
3907	add	%g5, %g6, %g5
3908	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
3909	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
3910	bz	1f
3911	or	%g6, CPU_DTRACE_BADADDR, %g6
3912	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
3913	GET_MMU_D_ADDR(%g3, %g4)
3914	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]
3915
3916	/*
3917	 * Be sure that we're actually taking this miss from the kernel --
3918	 * otherwise we have managed to return to user-level with
3919	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3920	 */
3921	rdpr	%tstate, %g5
3922	btst	TSTATE_PRIV, %g5
3923	bz,a	ptl1_panic
3924	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3925	TSTAT_CHECK_TL1(2f, %g1, %g2);
39262:
3927	done
3928
39291:
3930	rdpr	%tt, %g5
3931	cmp	%g5, FAST_IMMU_MISS_TT
3932#ifdef sun4v
3933	MMU_FAULT_STATUS_AREA(%g2)
3934	be,a,pt	%icc, 2f
3935	  ldx	[%g2 + MMFSA_I_CTX], %g3
3936	cmp	%g5, T_INSTR_MMU_MISS
3937	be,a,pt	%icc, 2f
3938	  ldx	[%g2 + MMFSA_I_CTX], %g3
3939	ldx	[%g2 + MMFSA_D_CTX], %g3
39402:
3941#else
3942	mov	MMU_TAG_ACCESS, %g2
3943	be,a,pt	%icc, 2f
3944	  ldxa	[%g2]ASI_IMMU, %g3
3945	ldxa	[%g2]ASI_DMMU, %g3
39462:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
3947#endif
3948	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
3949	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
3950	rdpr	%tl, %g5
3951	cmp	%g5, 1
3952	ble,pt	%icc, sfmmu_mmu_trap
3953	  nop
3954	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
3955	ba,pt	%icc, sfmmu_window_trap
3956	  nop
3957	SET_SIZE(sfmmu_tsb_miss)
3958#endif /* lint */
3959
3960#if defined (lint)
3961/*
3962 * This routine will look for a user or kernel vaddr in the hash
3963 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't
3964 * grab any locks.  It should only be used by other sfmmu routines.
3965 */
3966/* ARGSUSED */
3967pfn_t
3968sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
3969{
3970	return(0);
3971}
3972
3973/* ARGSUSED */
3974pfn_t
3975sfmmu_kvaszc2pfn(caddr_t vaddr, int hashno)
3976{
3977	return(0);
3978}
3979
3980#else /* lint */
3981
3982	ENTRY_NP(sfmmu_vatopfn)
3983 	/*
3984 	 * disable interrupts
3985 	 */
3986 	rdpr	%pstate, %o3
3987#ifdef DEBUG
3988	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l5, %g1)
3989#endif
3990	/*
3991	 * disable interrupts to protect the TSBMISS area
3992	 */
3993	andn    %o3, PSTATE_IE, %o5
3994	wrpr    %o5, 0, %pstate
3995
3996	/*
3997	 * o0 = vaddr
3998	 * o1 = sfmmup
3999	 * o2 = ttep
4000	 */
4001	CPU_TSBMISS_AREA(%g1, %o5)
4002	ldn	[%g1 + TSBMISS_KHATID], %o4
4003	cmp	%o4, %o1
4004	bne,pn	%ncc, vatopfn_nokernel
4005	  mov	TTE64K, %g5			/* g5 = rehash # */
4006	mov %g1,%o5				/* o5 = tsbmiss_area */
4007	/*
4008	 * o0 = vaddr
4009	 * o1 & o4 = hatid
4010	 * o2 = ttep
4011	 * o5 = tsbmiss area
4012	 */
4013	mov	HBLK_RANGE_SHIFT, %g6
40141:
4015
4016	/*
4017	 * o0 = vaddr
4018	 * o1 = sfmmup
4019	 * o2 = ttep
4020	 * o3 = old %pstate
4021	 * o4 = hatid
4022	 * o5 = tsbmiss
4023	 * g5 = rehash #
4024	 * g6 = hmeshift
4025	 *
4026	 * The first arg to GET_TTE is actually tagaccess register
4027	 * not just vaddr. Since this call is for kernel we need to clear
4028	 * any lower vaddr bits that would be interpreted as ctx bits.
4029	 */
4030	set     TAGACC_CTX_MASK, %g1
4031	andn    %o0, %g1, %o0
4032	GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5,
4033		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
4034
4035kvtop_hblk_found:
4036	/*
4037	 * o0 = vaddr
4038	 * o1 = sfmmup
4039	 * o2 = ttep
4040	 * g1 = tte
4041	 * g2 = tte pa
4042	 * g3 = tte va
4043	 * o2 = tsbmiss area
4044	 * o1 = hat id
4045	 */
4046	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
4047	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4048	stx %g1,[%o2]				/* put tte into *ttep */
4049	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
4050	/*
4051	 * o0 = vaddr
4052	 * o1 = sfmmup
4053	 * o2 = ttep
4054	 * g1 = pfn
4055	 */
4056	ba,pt	%xcc, 6f
4057	  mov	%g1, %o0
4058
4059kvtop_nohblk:
4060	/*
4061	 * we get here if we couldn't find valid hblk in hash.  We rehash
4062	 * if neccesary.
4063	 */
4064	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
4065#ifdef sun4v
4066	cmp	%g5, MAX_HASHCNT
4067#else
4068	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
4069#endif
4070	be,a,pn	%icc, 6f
4071	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4072	mov	%o1, %o4			/* restore hatid */
4073#ifdef sun4v
4074        add	%g5, 2, %g5
4075	cmp	%g5, 3
4076	move	%icc, MMU_PAGESHIFT4M, %g6
4077	ba,pt	%icc, 1b
4078	movne	%icc, MMU_PAGESHIFT256M, %g6
4079#else
4080        inc	%g5
4081	cmp	%g5, 2
4082	move	%icc, MMU_PAGESHIFT512K, %g6
4083	ba,pt	%icc, 1b
4084	movne	%icc, MMU_PAGESHIFT4M, %g6
4085#endif
40866:
4087	retl
4088 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4089
4090tsb_suspend:
4091	/*
4092	 * o0 = vaddr
4093	 * o1 = sfmmup
4094	 * o2 = ttep
4095	 * g1 = tte
4096	 * g2 = tte pa
4097	 * g3 = tte va
4098	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
4099	 */
4100	stx %g1,[%o2]				/* put tte into *ttep */
4101	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
4102	  sub	%g0, 1, %o0			/* output = PFN_INVALID */
4103	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
41048:
4105	retl
4106	 wrpr	%g0, %o3, %pstate		/* enable interrupts */
4107
4108vatopfn_nokernel:
4109	/*
4110	 * This routine does NOT support user addresses
4111	 * There is a routine in C that supports this.
4112	 * The only reason why we don't have the C routine
4113	 * support kernel addresses as well is because
4114	 * we do va_to_pa while holding the hashlock.
4115	 */
4116 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4117	save	%sp, -SA(MINFRAME), %sp
4118	sethi	%hi(sfmmu_panic3), %o0
4119	call	panic
4120	 or	%o0, %lo(sfmmu_panic3), %o0
4121
4122	SET_SIZE(sfmmu_vatopfn)
4123
4124	/*
4125	 * %o0 = vaddr
4126	 * %o1 = hashno (aka szc)
4127	 *
4128	 *
4129	 * This routine is similar to sfmmu_vatopfn() but will only look for
4130	 * a kernel vaddr in the hash structure for the specified rehash value.
4131	 * It's just an optimization for the case when pagesize for a given
4132	 * va range is already known (e.g. large page heap) and we don't want
4133	 * to start the search with rehash value 1 as sfmmu_vatopfn() does.
4134	 *
4135	 * Returns valid pfn or PFN_INVALID if
4136	 * tte for specified rehash # is not found, invalid or suspended.
4137	 */
4138	ENTRY_NP(sfmmu_kvaszc2pfn)
4139 	/*
4140 	 * disable interrupts
4141 	 */
4142 	rdpr	%pstate, %o3
4143#ifdef DEBUG
4144	PANIC_IF_INTR_DISABLED_PSTR(%o3, sfmmu_di_l6, %g1)
4145#endif
4146	/*
4147	 * disable interrupts to protect the TSBMISS area
4148	 */
4149	andn    %o3, PSTATE_IE, %o5
4150	wrpr    %o5, 0, %pstate
4151
4152	CPU_TSBMISS_AREA(%g1, %o5)
4153	ldn	[%g1 + TSBMISS_KHATID], %o4
4154	sll	%o1, 1, %g6
4155	add	%g6, %o1, %g6
4156	add	%g6, MMU_PAGESHIFT, %g6
4157	/*
4158	 * %o0 = vaddr
4159	 * %o1 = hashno
4160	 * %o3 = old %pstate
4161	 * %o4 = ksfmmup
4162	 * %g1 = tsbmiss area
4163	 * %g6 = hmeshift
4164	 */
4165
4166	/*
4167	 * The first arg to GET_TTE is actually tagaccess register
4168	 * not just vaddr. Since this call is for kernel we need to clear
4169	 * any lower vaddr bits that would be interpreted as ctx bits.
4170	 */
4171	srlx	%o0, MMU_PAGESHIFT, %o0
4172	sllx	%o0, MMU_PAGESHIFT, %o0
4173	GET_TTE(%o0, %o4, %g3, %g4, %g5, %g1, %o5, %g6, %o1,
4174		kvaszc2pfn_l1, kvaszc2pfn_hblk_found, kvaszc2pfn_nohblk,
4175		kvaszc2pfn_nohblk)
4176
4177kvaszc2pfn_hblk_found:
4178	/*
4179	 * %g3 = tte
4180	 * %o0 = vaddr
4181	 */
4182	brgez,a,pn %g3, 1f			/* check if tte is invalid */
4183	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
4184	TTETOPFN(%g3, %o0, kvaszc2pfn_l2, %g2, %g4, %g5)
4185	/*
4186	 * g3 = pfn
4187	 */
4188	ba,pt	%xcc, 1f
4189	  mov	%g3, %o0
4190
4191kvaszc2pfn_nohblk:
4192	mov	-1, %o0
4193
41941:
4195	retl
4196 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
4197
4198	SET_SIZE(sfmmu_kvaszc2pfn)
4199
4200#endif /* lint */
4201
4202
4203
4204#if !defined(lint)
4205
4206/*
4207 * kpm lock used between trap level tsbmiss handler and kpm C level.
4208 */
4209#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
4210	mov     0xff, tmp1						;\
4211label1:									;\
4212	casa    [kpmlckp]asi, %g0, tmp1					;\
4213	brnz,pn tmp1, label1						;\
4214	mov     0xff, tmp1						;\
4215	membar  #LoadLoad
4216
4217#define KPMLOCK_EXIT(kpmlckp, asi)					\
4218	membar  #LoadStore|#StoreStore					;\
4219	sta     %g0, [kpmlckp]asi
4220
4221/*
4222 * Lookup a memseg for a given pfn and if found, return the physical
4223 * address of the corresponding struct memseg in mseg, otherwise
4224 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
4225 * tsbmp, %asi is assumed to be ASI_MEM.
4226 * This lookup is done by strictly traversing only the physical memseg
4227 * linkage. The more generic approach, to check the virtual linkage
4228 * before using the physical (used e.g. with hmehash buckets), cannot
4229 * be used here. Memory DR operations can run in parallel to this
4230 * lookup w/o any locks and updates of the physical and virtual linkage
4231 * cannot be done atomically wrt. to each other. Because physical
4232 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
4233 * as "physical NULL" pointer.
4234 */
4235#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
4236	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
4237	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
4238	udivx	pfn, mseg, mseg						;\
4239	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
4240	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
4241	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
4242	add	tmp1, mseg, tmp1					;\
4243	ldxa	[tmp1]%asi, mseg					;\
4244	cmp	mseg, MSEG_NULLPTR_PA					;\
4245	be,pn	%xcc, label/**/1		/* if not found */	;\
4246	  nop								;\
4247	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
4248	cmp	pfn, tmp1			/* pfn - pages_base */	;\
4249	blu,pn	%xcc, label/**/1					;\
4250	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
4251	cmp	pfn, tmp2			/* pfn - pages_end */	;\
4252	bgeu,pn	%xcc, label/**/1					;\
4253	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
4254	mulx	tmp1, PAGE_SIZE, tmp1					;\
4255	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
4256	add	tmp2, tmp1, tmp1			/* pp */	;\
4257	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
4258	cmp	tmp2, pfn						;\
4259	be,pt	%xcc, label/**/_ok			/* found */	;\
4260label/**/1:								;\
4261	/* brute force lookup */					;\
4262	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
4263	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
4264label/**/2:								;\
4265	cmp	mseg, MSEG_NULLPTR_PA					;\
4266	be,pn	%xcc, label/**/_ok		/* if not found */	;\
4267	  nop								;\
4268	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
4269	cmp	pfn, tmp1			/* pfn - pages_base */	;\
4270	blu,a,pt %xcc, label/**/2					;\
4271	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
4272	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
4273	cmp	pfn, tmp2			/* pfn - pages_end */	;\
4274	bgeu,a,pt %xcc, label/**/2					;\
4275	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
4276label/**/_ok:
4277
4278	/*
4279	 * kpm tsb miss handler large pages
4280	 * g1 = 8K kpm TSB entry pointer
4281	 * g2 = tag access register
4282	 * g3 = 4M kpm TSB entry pointer
4283	 */
4284	ALTENTRY(sfmmu_kpm_dtsb_miss)
4285	TT_TRACE(trace_tsbmiss)
4286
4287	CPU_INDEX(%g7, %g6)
4288	sethi	%hi(kpmtsbm_area), %g6
4289	sllx	%g7, KPMTSBM_SHIFT, %g7
4290	or	%g6, %lo(kpmtsbm_area), %g6
4291	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4292
4293	/* check enable flag */
4294	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4295	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4296	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4297	  nop
4298
4299	/* VA range check */
4300	ldx	[%g6 + KPMTSBM_VBASE], %g7
4301	cmp	%g2, %g7
4302	blu,pn	%xcc, sfmmu_tsb_miss
4303	  ldx	[%g6 + KPMTSBM_VEND], %g5
4304	cmp	%g2, %g5
4305	bgeu,pn	%xcc, sfmmu_tsb_miss
4306	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]
4307
4308	/*
4309	 * check TL tsbmiss handling flag
4310	 * bump tsbmiss counter
4311	 */
4312	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4313#ifdef	DEBUG
4314	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
4315	inc	%g5
4316	brz,pn	%g3, sfmmu_kpm_exception
4317	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4318#else
4319	inc	%g5
4320	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4321#endif
4322	/*
4323	 * At this point:
4324	 *  g1 = 8K kpm TSB pointer (not used)
4325	 *  g2 = tag access register
4326	 *  g3 = clobbered
4327	 *  g6 = per-CPU kpm tsbmiss area
4328	 *  g7 = kpm_vbase
4329	 */
4330
4331	/* vaddr2pfn */
4332	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
4333	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4334	srax    %g4, %g3, %g2			/* which alias range (r) */
4335	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
4336	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
4337
4338	/*
4339	 * Setup %asi
4340	 * mseg_pa = page_numtomemseg_nolock(pfn)
4341	 * if (mseg_pa == NULL) sfmmu_kpm_exception
4342	 * g2=pfn
4343	 */
4344	mov	ASI_MEM, %asi
4345	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
4346	cmp	%g3, MSEG_NULLPTR_PA
4347	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4348	  nop
4349
4350	/*
4351	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
4352	 * g2=pfn g3=mseg_pa
4353	 */
4354	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
4355	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4356	srlx	%g2, %g5, %g4
4357	sllx	%g4, %g5, %g4
4358	sub	%g4, %g7, %g4
4359	srlx	%g4, %g5, %g4
4360
4361	/*
4362	 * Validate inx value
4363	 * g2=pfn g3=mseg_pa g4=inx
4364	 */
4365#ifdef	DEBUG
4366	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4367	cmp	%g4, %g5			/* inx - nkpmpgs */
4368	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4369	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4370#else
4371	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4372#endif
4373	/*
4374	 * kp = &mseg_pa->kpm_pages[inx]
4375	 */
4376	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
4377	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
4378	add	%g5, %g4, %g5			/* kp */
4379
4380	/*
4381	 * KPMP_HASH(kp)
4382	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
4383	 */
4384	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4385	sub	%g7, 1, %g7			/* mask */
4386	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
4387	add	%g5, %g1, %g5			/* y = ksp + x */
4388	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4389
4390	/*
4391	 * Calculate physical kpm_page pointer
4392	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4393	 */
4394	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
4395	add	%g1, %g4, %g1			/* kp_pa */
4396
4397	/*
4398	 * Calculate physical hash lock address
4399	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
4400	 */
4401	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
4402	sllx	%g5, KPMHLK_SHIFT, %g5
4403	add	%g4, %g5, %g3
4404	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */
4405
4406	/*
4407	 * Assemble tte
4408	 * g1=kp_pa g2=pfn g3=hlck_pa
4409	 */
4410#ifdef sun4v
4411	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4412	sllx	%g5, 32, %g5
4413	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4414	or	%g4, TTE4M, %g4
4415	or	%g5, %g4, %g5
4416#else
4417	sethi	%hi(TTE_VALID_INT), %g4
4418	mov	TTE4M, %g5
4419	sllx	%g5, TTE_SZ_SHFT_INT, %g5
4420	or	%g5, %g4, %g5			/* upper part */
4421	sllx	%g5, 32, %g5
4422	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4423	or	%g5, %g4, %g5
4424#endif
4425	sllx	%g2, MMU_PAGESHIFT, %g4
4426	or	%g5, %g4, %g5			/* tte */
4427	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4428	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4429
4430	/*
4431	 * tsb dropin
4432	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
4433	 */
4434
4435	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4436	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
4437
4438	/* use C-handler if there's no go for dropin */
4439	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
4440	cmp	%g7, -1
4441	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
4442	  nop
4443
4444#ifdef	DEBUG
4445	/* double check refcnt */
4446	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
4447	brz,pn	%g7, 5f			/* let C-handler deal with this */
4448	  nop
4449#endif
4450
4451#ifndef sun4v
4452	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4453	mov	ASI_N, %g1
4454	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4455	movnz	%icc, ASI_MEM, %g1
4456	mov	%g1, %asi
4457#endif
4458
4459	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
4460	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
4461
4462	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4463	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4464
4465	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
4466
4467	/* KPMLOCK_EXIT(kpmlckp, asi) */
4468	KPMLOCK_EXIT(%g3, ASI_MEM)
4469
4470	/*
4471	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4472	 * point to trapstat's TSB miss return code (note that trapstat
4473	 * itself will patch the correct offset to add).
4474	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4475	 */
4476	rdpr	%tl, %g7
4477	cmp	%g7, 1
4478	ble	%icc, 0f
4479	sethi	%hi(KERNELBASE), %g6
4480	rdpr	%tpc, %g7
4481	or	%g6, %lo(KERNELBASE), %g6
4482	cmp	%g7, %g6
4483	bgeu	%xcc, 0f
4484	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
4485	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4486	wrpr	%g7, %tpc
4487	add	%g7, 4, %g7
4488	wrpr	%g7, %tnpc
44890:
4490	retry
44915:
4492	/* g3=hlck_pa */
4493	KPMLOCK_EXIT(%g3, ASI_MEM)
4494	ba,pt	%icc, sfmmu_kpm_exception
4495	  nop
4496	SET_SIZE(sfmmu_kpm_dtsb_miss)
4497
4498	/*
4499	 * kpm tsbmiss handler for smallpages
4500	 * g1 = 8K kpm TSB pointer
4501	 * g2 = tag access register
4502	 * g3 = 4M kpm TSB pointer
4503	 */
4504	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
4505	TT_TRACE(trace_tsbmiss)
4506	CPU_INDEX(%g7, %g6)
4507	sethi	%hi(kpmtsbm_area), %g6
4508	sllx	%g7, KPMTSBM_SHIFT, %g7
4509	or	%g6, %lo(kpmtsbm_area), %g6
4510	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
4511
4512	/* check enable flag */
4513	ldub	[%g6 + KPMTSBM_FLAGS], %g4
4514	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
4515	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
4516	  nop
4517
4518	/*
4519	 * VA range check
4520	 * On fail: goto sfmmu_tsb_miss
4521	 */
4522	ldx	[%g6 + KPMTSBM_VBASE], %g7
4523	cmp	%g2, %g7
4524	blu,pn	%xcc, sfmmu_tsb_miss
4525	  ldx	[%g6 + KPMTSBM_VEND], %g5
4526	cmp	%g2, %g5
4527	bgeu,pn	%xcc, sfmmu_tsb_miss
4528	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */
4529
4530	/*
4531	 * check TL tsbmiss handling flag
4532	 * bump tsbmiss counter
4533	 */
4534	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
4535#ifdef	DEBUG
4536	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
4537	inc	%g5
4538	brz,pn	%g1, sfmmu_kpm_exception
4539	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
4540#else
4541	inc	%g5
4542	st	%g5, [%g6 + KPMTSBM_TSBMISS]
4543#endif
4544	/*
4545	 * At this point:
4546	 *  g1 = clobbered
4547	 *  g2 = tag access register
4548	 *  g3 = 4M kpm TSB pointer (not used)
4549	 *  g6 = per-CPU kpm tsbmiss area
4550	 *  g7 = kpm_vbase
4551	 */
4552
4553	/* vaddr2pfn */
4554	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
4555	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
4556	srax    %g4, %g3, %g2			/* which alias range (r) */
4557	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
4558	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
4559
4560	/*
4561	 * Setup %asi
4562	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
4563	 * if (mseg not found) sfmmu_kpm_exception
4564	 * g2=pfn
4565	 */
4566	mov	ASI_MEM, %asi
4567	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
4568	cmp	%g3, MSEG_NULLPTR_PA
4569	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
4570	  nop
4571
4572	/*
4573	 * inx = pfn - mseg_pa->kpm_pbase
4574	 * g2=pfn g3=mseg_pa
4575	 */
4576	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
4577	sub	%g2, %g7, %g4
4578
4579#ifdef	DEBUG
4580	/*
4581	 * Validate inx value
4582	 * g2=pfn g3=mseg_pa g4=inx
4583	 */
4584	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
4585	cmp	%g4, %g5			/* inx - nkpmpgs */
4586	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
4587	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4588#else
4589	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
4590#endif
4591	/* ksp = &mseg_pa->kpm_spages[inx] */
4592	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
4593	add	%g5, %g4, %g5			/* ksp */
4594
4595	/*
4596	 * KPMP_SHASH(kp)
4597	 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz
4598	 */
4599	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
4600	sub	%g7, 1, %g7			/* mask */
4601	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
4602	add	%g5, %g1, %g5			/* y = ksp + x */
4603	and 	%g5, %g7, %g5			/* hashinx = y & mask */
4604
4605	/*
4606	 * Calculate physical kpm_spage pointer
4607	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
4608	 */
4609	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
4610	add	%g1, %g4, %g1			/* ksp_pa */
4611
4612	/*
4613	 * Calculate physical hash lock address.
4614	 * Note: Changes in kpm_shlk_t must be reflected here.
4615	 * g1=ksp_pa g2=pfn g5=hashinx
4616	 */
4617	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
4618	sllx	%g5, KPMSHLK_SHIFT, %g5
4619	add	%g4, %g5, %g3			/* hlck_pa */
4620
4621	/*
4622	 * Assemble tte
4623	 * g1=ksp_pa g2=pfn g3=hlck_pa
4624	 */
4625	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
4626	sllx	%g5, 32, %g5
4627	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
4628	or	%g5, %g4, %g5
4629	sllx	%g2, MMU_PAGESHIFT, %g4
4630	or	%g5, %g4, %g5			/* tte */
4631	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
4632	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
4633
4634	/*
4635	 * tsb dropin
4636	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte
4637	 */
4638
4639	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
4640	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
4641
4642	/* use C-handler if there's no go for dropin */
4643	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */
4644	cmp	%g7, -1
4645	bne,pn	%xcc, 5f
4646	  nop
4647
4648#ifndef sun4v
4649	ldub	[%g6 + KPMTSBM_FLAGS], %g7
4650	mov	ASI_N, %g1
4651	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
4652	movnz	%icc, ASI_MEM, %g1
4653	mov	%g1, %asi
4654#endif
4655
4656	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
4657	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
4658
4659	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
4660	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
4661
4662	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
4663
4664	/* KPMLOCK_EXIT(kpmlckp, asi) */
4665	KPMLOCK_EXIT(%g3, ASI_MEM)
4666
4667	/*
4668	 * If trapstat is running, we need to shift the %tpc and %tnpc to
4669	 * point to trapstat's TSB miss return code (note that trapstat
4670	 * itself will patch the correct offset to add).
4671	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
4672	 */
4673	rdpr	%tl, %g7
4674	cmp	%g7, 1
4675	ble	%icc, 0f
4676	sethi	%hi(KERNELBASE), %g6
4677	rdpr	%tpc, %g7
4678	or	%g6, %lo(KERNELBASE), %g6
4679	cmp	%g7, %g6
4680	bgeu	%xcc, 0f
4681	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
4682	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
4683	wrpr	%g7, %tpc
4684	add	%g7, 4, %g7
4685	wrpr	%g7, %tnpc
46860:
4687	retry
46885:
4689	/* g3=hlck_pa */
4690	KPMLOCK_EXIT(%g3, ASI_MEM)
4691	ba,pt	%icc, sfmmu_kpm_exception
4692	  nop
4693	SET_SIZE(sfmmu_kpm_dtsb_miss_small)
4694
4695#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
4696#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
4697#endif
4698
4699#endif /* lint */
4700
4701#ifdef	lint
4702/*
4703 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
4704 * Called from C-level, sets/clears "go" indication for trap level handler.
4705 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
4706 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
4707 * Assumes khl_mutex is held when called from C-level.
4708 */
4709/* ARGSUSED */
4710void
4711sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
4712{
4713}
4714
4715/*
4716 * kpm_smallpages: stores val to byte at address mapped within
4717 * low level lock brackets. The old value is returned.
4718 * Called from C-level.
4719 */
4720/* ARGSUSED */
4721int
4722sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val)
4723{
4724	return (0);
4725}
4726
4727#else /* lint */
4728
4729	.seg	".data"
4730sfmmu_kpm_tsbmtl_panic:
4731	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
4732	.byte	0
4733sfmmu_kpm_stsbmtl_panic:
4734	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
4735	.byte	0
4736	.align	4
4737	.seg	".text"
4738
4739	ENTRY_NP(sfmmu_kpm_tsbmtl)
4740	rdpr	%pstate, %o3
4741	/*
4742	 * %o0 = &kp_refcntc
4743	 * %o1 = &khl_lock
4744	 * %o2 = 0/1 (off/on)
4745	 * %o3 = pstate save
4746	 */
4747#ifdef DEBUG
4748	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4749	bnz,pt %icc, 1f				/* disabled, panic	 */
4750	  nop
4751	save	%sp, -SA(MINFRAME), %sp
4752	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
4753	call	panic
4754	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
4755	ret
4756	restore
47571:
4758#endif /* DEBUG */
4759	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4760
4761	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
4762	mov	-1, %o5
4763	brz,a	%o2, 2f
4764	  mov	0, %o5
47652:
4766	sth	%o5, [%o0]
4767	KPMLOCK_EXIT(%o1, ASI_N)
4768
4769	retl
4770	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4771	SET_SIZE(sfmmu_kpm_tsbmtl)
4772
4773	ENTRY_NP(sfmmu_kpm_stsbmtl)
4774	rdpr	%pstate, %o3
4775	/*
4776	 * %o0 = &mapped
4777	 * %o1 = &kshl_lock
4778	 * %o2 = val
4779	 * %o3 = pstate save
4780	 */
4781#ifdef DEBUG
4782	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
4783	bnz,pt %icc, 1f				/* disabled, panic	 */
4784	  nop
4785	save	%sp, -SA(MINFRAME), %sp
4786	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
4787	call	panic
4788	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
4789	ret
4790	restore
47911:
4792#endif /* DEBUG */
4793	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
4794
4795	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
4796	ldsb	[%o0], %o5
4797	stb	%o2, [%o0]
4798	KPMLOCK_EXIT(%o1, ASI_N)
4799
4800	mov	%o5, %o0			/* return old val */
4801	retl
4802	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
4803	SET_SIZE(sfmmu_kpm_stsbmtl)
4804
4805#endif /* lint */
4806
4807#ifndef lint
4808#ifdef sun4v
4809	/*
4810	 * User/kernel data miss w// multiple TSBs
4811	 * The first probe covers 8K, 64K, and 512K page sizes,
4812	 * because 64K and 512K mappings are replicated off 8K
4813	 * pointer.  Second probe covers 4M page size only.
4814	 *
4815	 * MMU fault area contains miss address and context.
4816	 */
4817	ALTENTRY(sfmmu_slow_dmmu_miss)
4818	GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)	! %g2 = ptagacc, %g3 = ctx type
4819
4820slow_miss_common:
4821	/*
4822	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
4823	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
4824	 */
4825	brnz,pt	%g3, 8f			! check for user context
4826	  nop
4827
4828	/*
4829	 * Kernel miss
4830	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
4831	 * branch to sfmmu_tsb_miss_tt to handle it.
4832	 */
4833	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4834sfmmu_dslow_patch_ktsb_base:
4835	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
4836sfmmu_dslow_patch_ktsb_szcode:
4837	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
4838
4839	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
4840	! %g1 = First TSB entry pointer, as TSB miss handler expects
4841
4842	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
4843sfmmu_dslow_patch_ktsb4m_base:
4844	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
4845sfmmu_dslow_patch_ktsb4m_szcode:
4846	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
4847
4848	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
4849	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
4850	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4851	.empty
4852
48538:
4854	/*
4855	 * User miss
4856	 * Get first TSB pointer in %g1
4857	 * Get second TSB pointer (or NULL if no second TSB) in %g3
4858	 * Branch to sfmmu_tsb_miss_tt to handle it
4859	 */
4860	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
4861	/* %g1 = first TSB entry ptr now, %g2 preserved */
4862
4863	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
4864	brlz,pt %g3, sfmmu_tsb_miss_tt		/* done if no 2nd TSB */
4865	  nop
4866
4867	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
4868	/* %g3 = second TSB entry ptr now, %g2 preserved */
48699:
4870	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
4871	.empty
4872	SET_SIZE(sfmmu_slow_dmmu_miss)
4873
4874
4875	/*
4876	 * User/kernel instruction miss w/ multiple TSBs
4877	 * The first probe covers 8K, 64K, and 512K page sizes,
4878	 * because 64K and 512K mappings are replicated off 8K
4879	 * pointer.  Second probe covers 4M page size only.
4880	 *
4881	 * MMU fault area contains miss address and context.
4882	 */
4883	ALTENTRY(sfmmu_slow_immu_miss)
4884	GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)
4885	ba,a,pt	%xcc, slow_miss_common
4886	SET_SIZE(sfmmu_slow_immu_miss)
4887
4888#endif /* sun4v */
4889#endif	/* lint */
4890
4891#ifndef lint
4892
4893/*
4894 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
4895 */
4896	.seg	".data"
4897	.align	64
4898	.global tsbmiss_area
4899tsbmiss_area:
4900	.skip	(TSBMISS_SIZE * NCPU)
4901
4902	.align	64
4903	.global kpmtsbm_area
4904kpmtsbm_area:
4905	.skip	(KPMTSBM_SIZE * NCPU)
4906#endif	/* lint */
4907