sfmmu_asm.s revision 7c478bd95313f5f23a4c958a745db2134aa03244
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#pragma ident	"%Z%%M%	%I%	%E% SMI"
28
29/*
30 * SFMMU primitives.  These primitives should only be used by sfmmu
31 * routines.
32 */
33
34#if defined(lint)
35#include <sys/types.h>
36#else	/* lint */
37#include "assym.h"
38#endif	/* lint */
39
40#include <sys/asm_linkage.h>
41#include <sys/machtrap.h>
42#include <sys/machasi.h>
43#include <sys/sun4asi.h>
44#include <sys/pte.h>
45#include <sys/mmu.h>
46#include <vm/hat_sfmmu.h>
47#include <vm/seg_spt.h>
48#include <sys/machparam.h>
49#include <sys/privregs.h>
50#include <sys/scb.h>
51#include <sys/intreg.h>
52#include <sys/machthread.h>
53#include <sys/intr.h>
54#include <sys/clock.h>
55#include <sys/trapstat.h>
56
57#ifdef TRAPTRACE
58#include <sys/traptrace.h>
59
60/*
61 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
62 */
63#define	TT_TRACE(label)		\
64	ba	label		;\
65	rd	%pc, %g7
66#else
67
68#define	TT_TRACE(label)
69
70#endif /* TRAPTRACE */
71
72#ifndef	lint
73
74#if (TTE_SUSPEND_SHIFT > 0)
75#define	TTE_SUSPEND_INT_SHIFT(reg)				\
76	sllx	reg, TTE_SUSPEND_SHIFT, reg
77#else
78#define	TTE_SUSPEND_INT_SHIFT(reg)
79#endif
80
81#endif /* lint */
82
83#ifndef	lint
84
85/*
86 * Assumes TSBE_TAG is 0
87 * Assumes TSBE_INTHI is 0
88 * Assumes TSBREG.split is 0
89 */
90
91#if TSBE_TAG != 0
92#error "TSB_UPDATE and TSB_INVALIDATE assume TSBE_TAG = 0"
93#endif
94
95#if TSBTAG_INTHI != 0
96#error "TSB_UPDATE and TSB_INVALIDATE assume TSBTAG_INTHI = 0"
97#endif
98
99/*
100 * The following code assumes the tsb is not split.
101 *
102 * With TSBs no longer shared between processes, it's no longer
103 * necessary to hash the context bits into the tsb index to get
104 * tsb coloring; the new implementation treats the TSB as a
105 * direct-mapped, virtually-addressed cache.
106 *
107 * In:
108 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
109 *    tsbbase = base address of TSB (clobbered)
110 *    tagacc = tag access register (clobbered)
111 *    szc = size code of TSB (ro)
112 *    tmp = scratch reg
113 * Out:
114 *    tsbbase = pointer to entry in TSB
115 */
116#define	GET_TSBE_POINTER(vpshift, tsbbase, tagacc, szc, tmp)		\
117	mov	TSB_ENTRIES(0), tmp	/* nentries in TSB size 0 */	;\
118	srlx	tagacc, vpshift, tagacc 				;\
119	sllx	tmp, szc, tmp		/* tmp = nentries in TSB */	;\
120	sub	tmp, 1, tmp		/* mask = nentries - 1 */	;\
121	and	tagacc, tmp, tmp	/* tsbent = virtpage & mask */	;\
122	sllx	tmp, TSB_ENTRY_SHIFT, tmp	/* entry num --> ptr */	;\
123	add	tsbbase, tmp, tsbbase	/* add entry offset to TSB base */
124
125/*
126 * When the kpm TSB is used it is assumed that it is direct mapped
127 * using (vaddr>>vpshift)%tsbsz as the index.
128 *
129 * Note that, for now, the kpm TSB and kernel TSB are the same for
130 * each mapping size.  However that need not always be the case.  If
131 * the trap handlers are updated to search a different TSB for kpm
132 * addresses than for kernel addresses then kpm_tsbbase and kpm_tsbsz
133 * (and/or kpmsm_tsbbase/kpmsm_tsbsz) may be entirely independent.
134 *
135 * In:
136 *    vpshift = virtual page shift; e.g. 13 for 8K TTEs (constant or ro)
137 *    vaddr = virtual address (clobbered)
138 *    tsbp, szc, tmp = scratch
139 * Out:
140 *    tsbp = pointer to entry in TSB
141 */
142#define	GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)		\
143	cmp	vpshift, MMU_PAGESHIFT					;\
144	bne,pn	%icc, 1f		/* branch if large case */	;\
145	  sethi	%hi(kpmsm_tsbsz), szc					;\
146	sethi	%hi(kpmsm_tsbbase), tsbp				;\
147	ld	[szc + %lo(kpmsm_tsbsz)], szc				;\
148	ldx	[tsbp + %lo(kpmsm_tsbbase)], tsbp			;\
149	ba,pt	%icc, 2f						;\
150	  nop								;\
1511:	sethi	%hi(kpm_tsbsz), szc					;\
152	sethi	%hi(kpm_tsbbase), tsbp					;\
153	ld	[szc + %lo(kpm_tsbsz)], szc				;\
154	ldx	[tsbp + %lo(kpm_tsbbase)], tsbp				;\
1552:	GET_TSBE_POINTER(vpshift, tsbp, vaddr, szc, tmp)
156
157/*
158 * Lock the TSBE at virtual address tsbep.
159 *
160 * tsbep = TSBE va (ro)
161 * tmp1, tmp2 = scratch registers (clobbered)
162 * label = label to use for branches (text)
163 * %asi = ASI to use for TSB access
164 *
165 * NOTE that we flush the TSB using fast VIS instructions that
166 * set all 1's in the TSB tag, so TSBTAG_LOCKED|TSBTAG_INVALID must
167 * not be treated as a locked entry or we'll get stuck spinning on
168 * an entry that isn't locked but really invalid.
169 */
170
171#if defined(UTSB_PHYS)
172
173#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
174	lda	[tsbep]ASI_MEM, tmp1					;\
175label:									;\
176	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
177	cmp	tmp1, tmp2 						;\
178	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
179	  lda	[tsbep]ASI_MEM, tmp1					;\
180	casa	[tsbep]ASI_MEM, tmp1, tmp2				;\
181	cmp	tmp1, tmp2 						;\
182	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
183	  lda	[tsbep]ASI_MEM, tmp1					;\
184	/* tsbe lock acquired */					;\
185	membar #StoreStore
186
187#else /* UTSB_PHYS */
188
189#define	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			\
190	lda	[tsbep]%asi, tmp1					;\
191label:									;\
192	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
193	cmp	tmp1, tmp2 						;\
194	be,a,pn	%icc, label/**/b	/* if locked spin */		;\
195	  lda	[tsbep]%asi, tmp1					;\
196	casa	[tsbep]%asi, tmp1, tmp2					;\
197	cmp	tmp1, tmp2 						;\
198	bne,a,pn %icc, label/**/b	/* didn't lock so try again */	;\
199	  lda	[tsbep]%asi, tmp1					;\
200	/* tsbe lock acquired */					;\
201	membar #StoreStore
202
203#endif /* UTSB_PHYS */
204
205/*
206 * Atomically write TSBE at virtual address tsbep.
207 *
208 * tsbep = TSBE va (ro)
209 * tte = TSBE TTE (ro)
210 * tagtarget = TSBE tag (ro)
211 * %asi = ASI to use for TSB access
212 */
213
214#if defined(UTSB_PHYS)
215
216#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		\
217	add	tsbep, TSBE_TTE, tmp1					;\
218	stxa	tte, [tmp1]ASI_MEM		/* write tte data */	;\
219	membar #StoreStore						;\
220	add	tsbep, TSBE_TAG, tmp1					;\
221	stxa	tagtarget, [tmp1]ASI_MEM	/* write tte tag & unlock */
222
223#else /* UTSB_PHYS */
224
225#define	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget,tmp1)		\
226	stxa	tte, [tsbep + TSBE_TTE]%asi	/* write tte data */	;\
227	membar #StoreStore						;\
228	stxa	tagtarget, [tsbep + TSBE_TAG]%asi /* write tte tag & unlock */
229
230#endif /* UTSB_PHYS */
231
232/*
233 * Load an entry into the TSB at TL > 0.
234 *
235 * tsbep = pointer to the TSBE to load as va (ro)
236 * tte = value of the TTE retrieved and loaded (wo)
237 * tagtarget = tag target register.  To get TSBE tag to load,
238 *   we need to mask off the context and leave only the va (clobbered)
239 * ttepa = pointer to the TTE to retrieve/load as pa (ro)
240 * tmp1, tmp2 = scratch registers
241 * label = label to use for branches (text)
242 * %asi = ASI to use for TSB access
243 */
244
245#if defined(UTSB_PHYS)
246
247#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
248	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
249	/*								;\
250	 * I don't need to update the TSB then check for the valid tte.	;\
251	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
252	 * we always invalidate the hash table before we unload the TSB.;\
253	 */								;\
254	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
255	ldxa	[ttepa]ASI_MEM, tte					;\
256	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
257	sethi	%hi(TSBTAG_INVALID), tmp2				;\
258	add	tsbep, TSBE_TAG, tmp1					;\
259	brgez,a,pn tte, label/**/f					;\
260	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
261	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
262label:
263
264#else /* UTSB_PHYS */
265
266#define	TSB_UPDATE_TL(tsbep, tte, tagtarget, ttepa, tmp1, tmp2, label) \
267	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
268	/*								;\
269	 * I don't need to update the TSB then check for the valid tte.	;\
270	 * TSB invalidate will spin till the entry is unlocked.	Note,	;\
271	 * we always invalidate the hash table before we unload the TSB.;\
272	 */								;\
273	sllx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
274	ldxa	[ttepa]ASI_MEM, tte					;\
275	srlx	tagtarget, TTARGET_VA_SHIFT, tagtarget			;\
276	sethi	%hi(TSBTAG_INVALID), tmp2				;\
277	brgez,a,pn tte, label/**/f					;\
278	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
279	TSB_INSERT_UNLOCK_ENTRY(tsbep, tte, tagtarget, tmp1)		;\
280label:
281
282#endif /* UTSB_PHYS */
283
284/*
285 * Load an entry into the TSB at TL=0.
286 *
287 * tsbep = pointer to the TSBE to load as va (ro)
288 * tteva = pointer to the TTE to load as va (ro)
289 * tagtarget = TSBE tag to load (which contains no context), synthesized
290 * to match va of MMU tag target register only (ro)
291 * tmp1, tmp2 = scratch registers (clobbered)
292 * label = label to use for branches (text)
293 * %asi = ASI to use for TSB access
294 */
295
296#if defined(UTSB_PHYS)
297
298#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
299	/* can't rd tteva after locking tsb because it can tlb miss */	;\
300	ldx	[tteva], tteva			/* load tte */		;\
301	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
302	sethi	%hi(TSBTAG_INVALID), tmp2				;\
303	add	tsbep, TSBE_TAG, tmp1					;\
304	brgez,a,pn tteva, label/**/f					;\
305	 sta	tmp2, [tmp1]ASI_MEM			/* unlock */	;\
306	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
307label:
308
309#else /* UTSB_PHYS */
310
311#define	TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label)		\
312	/* can't rd tteva after locking tsb because it can tlb miss */	;\
313	ldx	[tteva], tteva			/* load tte */		;\
314	TSB_LOCK_ENTRY(tsbep, tmp1, tmp2, label)			;\
315	sethi	%hi(TSBTAG_INVALID), tmp2				;\
316	brgez,a,pn tteva, label/**/f					;\
317	 sta	tmp2, [tsbep + TSBE_TAG]%asi		/* unlock */	;\
318	TSB_INSERT_UNLOCK_ENTRY(tsbep, tteva, tagtarget, tmp1)		;\
319label:
320
321#endif /* UTSB_PHYS */
322
323/*
324 * Invalidate a TSB entry in the TSB.
325 *
326 * NOTE: TSBE_TAG is assumed to be zero.  There is a compile time check
327 *	 about this earlier to ensure this is true.  Thus when we are
328 *	 directly referencing tsbep below, we are referencing the tte_tag
329 *	 field of the TSBE.  If this  offset ever changes, the code below
330 *	 will need to be modified.
331 *
332 * tsbep = pointer to TSBE as va (ro)
333 * tag = invalidation is done if this matches the TSBE tag (ro)
334 * tmp1 - tmp3 = scratch registers (clobbered)
335 * label = label name to use for branches (text)
336 * %asi = ASI to use for TSB access
337 */
338
339#if defined(UTSB_PHYS)
340
341#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
342	lda	[tsbep]ASI_MEM, tmp1	/* tmp1 = tsbe tag */		;\
343	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
344label/**/1:								;\
345	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
346	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
347	  lda	[tsbep]ASI_MEM, tmp1	/* reloading value each time */	;\
348	ldxa	[tsbep]ASI_MEM, tmp3	/* tmp3 = tsbe tag */		;\
349	cmp	tag, tmp3		/* compare tags */		;\
350	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
351	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
352	casa	[tsbep]ASI_MEM, tmp1, tmp3 /* try to set tag invalid */	;\
353	cmp	tmp1, tmp3		/* if not successful */		;\
354	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
355	  lda	[tsbep]ASI_MEM, tmp1	/* reloading tsbe tag */	;\
356label/**/2:
357
358#else /* UTSB_PHYS */
359
360#define	TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label)		\
361	lda	[tsbep]%asi, tmp1	/* tmp1 = tsbe tag */		;\
362	sethi	%hi(TSBTAG_LOCKED), tmp2				;\
363label/**/1:								;\
364	cmp	tmp1, tmp2		/* see if tsbe is locked, if */	;\
365	be,a,pn	%icc, label/**/1	/* so, loop until unlocked */	;\
366	  lda	[tsbep]%asi, tmp1	/* reloading value each time */	;\
367	ldxa	[tsbep]%asi, tmp3	/* tmp3 = tsbe tag */		;\
368	cmp	tag, tmp3		/* compare tags */		;\
369	bne,pt	%xcc, label/**/2	/* if different, do nothing */	;\
370	  sethi	%hi(TSBTAG_INVALID), tmp3				;\
371	casa	[tsbep]%asi, tmp1, tmp3	/* try to set tag invalid */	;\
372	cmp	tmp1, tmp3		/* if not successful */		;\
373	bne,a,pn %icc, label/**/1	/* start over from the top */	;\
374	  lda	[tsbep]%asi, tmp1	/* reloading tsbe tag */	;\
375label/**/2:
376
377#endif /* UTSB_PHYS */
378
379#if TSB_SOFTSZ_MASK < TSB_SZ_MASK
380#error	- TSB_SOFTSZ_MASK too small
381#endif
382
383
384/*
385 * An implementation of setx which will be hot patched at run time.
386 * since it is being hot patched, there is no value passed in.
387 * Thus, essentially we are implementing
388 *	setx value, tmp, dest
389 * where value is RUNTIME_PATCH (aka 0) in this case.
390 */
391#define	RUNTIME_PATCH_SETX(dest, tmp)					\
392	sethi	%hh(RUNTIME_PATCH), tmp					;\
393	sethi	%lm(RUNTIME_PATCH), dest				;\
394	or	tmp, %hm(RUNTIME_PATCH), tmp				;\
395	or	dest, %lo(RUNTIME_PATCH), dest				;\
396	sllx	tmp, 32, tmp						;\
397	nop				/* for perf reasons */		;\
398	or	tmp, dest, dest		/* contents of patched value */
399
400
401#endif (lint)
402
403
404#if defined (lint)
405
406/*
407 * sfmmu related subroutines
408 */
409
410/*
411 * Use cas, if tte has changed underneath us then reread and try again.
412 * In the case of a retry, it will update sttep with the new original.
413 */
414/* ARGSUSED */
415int
416sfmmu_modifytte(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
417{ return(0); }
418
419/*
420 * Use cas, if tte has changed underneath us then return 1, else return 0
421 */
422/* ARGSUSED */
423int
424sfmmu_modifytte_try(tte_t *sttep, tte_t *stmodttep, tte_t *dttep)
425{ return(0); }
426
427/* ARGSUSED */
428void
429sfmmu_copytte(tte_t *sttep, tte_t *dttep)
430{}
431
432/*ARGSUSED*/
433struct tsbe *
434sfmmu_get_tsbe(uint64_t tsbeptr, caddr_t vaddr, int vpshift, int tsb_szc)
435{ return(0); }
436
437/*ARGSUSED*/
438uint64_t
439sfmmu_make_tsbtag(caddr_t va)
440{ return(0); }
441
442#else	/* lint */
443
444	.seg	".data"
445	.global	sfmmu_panic1
446sfmmu_panic1:
447	.asciz	"sfmmu_asm: interrupts already disabled"
448
449	.global	sfmmu_panic3
450sfmmu_panic3:
451	.asciz	"sfmmu_asm: sfmmu_vatopfn called for user"
452
453	.global	sfmmu_panic4
454sfmmu_panic4:
455	.asciz	"sfmmu_asm: 4M tsb pointer mis-match"
456
457	.global	sfmmu_panic5
458sfmmu_panic5:
459	.asciz	"sfmmu_asm: no unlocked TTEs in TLB 0"
460
461
462	ENTRY_NP(sfmmu_modifytte)
463	ldx	[%o2], %g3			/* current */
464	ldx	[%o0], %g1			/* original */
4652:
466	ldx	[%o1], %g2			/* modified */
467	cmp	%g2, %g3			/* is modified = current? */
468	be,a,pt	%xcc,1f				/* yes, don't write */
469	stx	%g3, [%o0]			/* update new original */
470	casx	[%o2], %g1, %g2
471	cmp	%g1, %g2
472	be,pt	%xcc, 1f			/* cas succeeded - return */
473	  nop
474	ldx	[%o2], %g3			/* new current */
475	stx	%g3, [%o0]			/* save as new original */
476	ba,pt	%xcc, 2b
477	  mov	%g3, %g1
4781:	retl
479	membar	#StoreLoad
480	SET_SIZE(sfmmu_modifytte)
481
482	ENTRY_NP(sfmmu_modifytte_try)
483	ldx	[%o1], %g2			/* modified */
484	ldx	[%o2], %g3			/* current */
485	ldx	[%o0], %g1			/* original */
486	cmp	%g3, %g2			/* is modified = current? */
487	be,a,pn %xcc,1f				/* yes, don't write */
488	mov	0, %o1				/* as if cas failed. */
489
490	casx	[%o2], %g1, %g2
491	membar	#StoreLoad
492	cmp	%g1, %g2
493	movne	%xcc, -1, %o1			/* cas failed. */
494	move	%xcc, 1, %o1			/* cas succeeded. */
4951:
496	stx	%g2, [%o0]			/* report "current" value */
497	retl
498	mov	%o1, %o0
499	SET_SIZE(sfmmu_modifytte_try)
500
501	ENTRY_NP(sfmmu_copytte)
502	ldx	[%o0], %g1
503	retl
504	stx	%g1, [%o1]
505	SET_SIZE(sfmmu_copytte)
506
507
508	/*
509	 * Calculate a TSB entry pointer for the given TSB, va, pagesize.
510	 * %o0 = TSB base address (in), pointer to TSB entry (out)
511	 * %o1 = vaddr (in)
512	 * %o2 = vpshift (in)
513	 * %o3 = tsb size code (in)
514	 * %o4 = scratch register
515	 */
516	ENTRY_NP(sfmmu_get_tsbe)
517	GET_TSBE_POINTER(%o2, %o0, %o1, %o3, %o4)
518	retl
519	nop
520	SET_SIZE(sfmmu_get_tsbe)
521
522	/*
523	 * Return a TSB tag for the given va.
524	 * %o0 = va (in/clobbered)
525	 * %o0 = va shifted to be in tsb tag format (with no context) (out)
526	 */
527	ENTRY_NP(sfmmu_make_tsbtag)
528	retl
529	srln	%o0, TTARGET_VA_SHIFT, %o0
530	SET_SIZE(sfmmu_make_tsbtag)
531
532#endif /* lint */
533
534/*
535 * Other sfmmu primitives
536 */
537
538
539#if defined (lint)
540void
541sfmmu_patch_ktsb(void)
542{
543}
544
545void
546sfmmu_kpm_patch_tlbm(void)
547{
548}
549
550void
551sfmmu_kpm_patch_tsbm(void)
552{
553}
554
555/* ARGSUSED */
556void
557sfmmu_load_tsbe(struct tsbe *tsbep, uint64_t vaddr, tte_t *ttep, int phys)
558{
559}
560
561/* ARGSUSED */
562void
563sfmmu_unload_tsbe(struct tsbe *tsbep, uint64_t vaddr, int phys)
564{
565}
566
567/* ARGSUSED */
568void
569sfmmu_kpm_load_tsb(caddr_t addr, tte_t *ttep, int vpshift)
570{
571}
572
573/* ARGSUSED */
574void
575sfmmu_kpm_unload_tsb(caddr_t addr, int vpshift)
576{
577}
578
579#else /* lint */
580
581#define	I_SIZE		4
582
583	ENTRY_NP(sfmmu_fix_ktlb_traptable)
584	/*
585	 * %o0 = start of patch area
586	 * %o1 = size code of TSB to patch
587	 * %o3 = scratch
588	 */
589	/* fix sll */
590	ld	[%o0], %o3			/* get sll */
591	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
592	st	%o3, [%o0]			/* write sll */
593	flush	%o0
594	/* fix srl */
595	add	%o0, I_SIZE, %o0		/* goto next instr. */
596	ld	[%o0], %o3			/* get srl */
597	sub	%o3, %o1, %o3			/* decrease shift by tsb szc */
598	st	%o3, [%o0]			/* write srl */
599	retl
600	flush	%o0
601	SET_SIZE(sfmmu_fix_ktlb_traptable)
602
603	ENTRY_NP(sfmmu_fixup_ktsbbase)
604	/*
605	 * %o0 = start of patch area
606	 * %o5 = kernel virtual or physical tsb base address
607	 * %o2, %o3 are used as scratch registers.
608	 */
609	/* fixup sethi instruction */
610	ld	[%o0], %o3
611	srl	%o5, 10, %o2			! offset is bits 32:10
612	or	%o3, %o2, %o3			! set imm22
613	st	%o3, [%o0]
614	/* fixup offset of lduw/ldx */
615	add	%o0, I_SIZE, %o0		! next instr
616	ld	[%o0], %o3
617	and	%o5, 0x3ff, %o2			! set imm13 to bits 9:0
618	or	%o3, %o2, %o3
619	st	%o3, [%o0]
620	retl
621	flush	%o0
622	SET_SIZE(sfmmu_fixup_ktsbbase)
623
624	ENTRY_NP(sfmmu_fixup_setx)
625	/*
626	 * %o0 = start of patch area
627	 * %o4 = 64 bit value to patch
628	 * %o2, %o3 are used as scratch registers.
629	 *
630	 * Note: Assuming that all parts of the instructions which need to be
631	 *	 patched correspond to RUNTIME_PATCH (aka 0)
632	 *
633	 * Note the implementation of setx which is being patched is as follows:
634	 *
635	 * sethi   %hh(RUNTIME_PATCH), tmp
636	 * sethi   %lm(RUNTIME_PATCH), dest
637	 * or      tmp, %hm(RUNTIME_PATCH), tmp
638	 * or      dest, %lo(RUNTIME_PATCH), dest
639	 * sllx    tmp, 32, tmp
640	 * nop
641	 * or      tmp, dest, dest
642	 *
643	 * which differs from the implementation in the
644	 * "SPARC Architecture Manual"
645	 */
646	/* fixup sethi instruction */
647	ld	[%o0], %o3
648	srlx	%o4, 42, %o2			! bits [63:42]
649	or	%o3, %o2, %o3			! set imm22
650	st	%o3, [%o0]
651	/* fixup sethi instruction */
652	add	%o0, I_SIZE, %o0		! next instr
653	ld	[%o0], %o3
654	sllx	%o4, 32, %o2			! clear upper bits
655	srlx	%o2, 42, %o2			! bits [31:10]
656	or	%o3, %o2, %o3			! set imm22
657	st	%o3, [%o0]
658	/* fixup or instruction */
659	add	%o0, I_SIZE, %o0		! next instr
660	ld	[%o0], %o3
661	srlx	%o4, 32, %o2			! bits [63:32]
662	and	%o2, 0x3ff, %o2			! bits [41:32]
663	or	%o3, %o2, %o3			! set imm
664	st	%o3, [%o0]
665	/* fixup or instruction */
666	add	%o0, I_SIZE, %o0		! next instr
667	ld	[%o0], %o3
668	and	%o4, 0x3ff, %o2			! bits [9:0]
669	or	%o3, %o2, %o3			! set imm
670	st	%o3, [%o0]
671	retl
672	flush	%o0
673	SET_SIZE(sfmmu_fixup_setx)
674
675	ENTRY_NP(sfmmu_fixup_or)
676	/*
677	 * %o0 = start of patch area
678	 * %o4 = 32 bit value to patch
679	 * %o2, %o3 are used as scratch registers.
680	 * Note: Assuming that all parts of the instructions which need to be
681	 *	 patched correspond to RUNTIME_PATCH (aka 0)
682	 */
683	ld	[%o0], %o3
684	and	%o4, 0x3ff, %o2			! bits [9:0]
685	or	%o3, %o2, %o3			! set imm
686	st	%o3, [%o0]
687	retl
688	flush	%o0
689	SET_SIZE(sfmmu_fixup_or)
690
691	ENTRY_NP(sfmmu_fixup_shiftx)
692	/*
693	 * %o0 = start of patch area
694	 * %o4 = signed int immediate value to add to sllx/srlx imm field
695	 * %o2, %o3 are used as scratch registers.
696	 *
697	 * sllx/srlx store the 6 bit immediate value in the lowest order bits
698	 * so we do a simple add.  The caller must be careful to prevent
699	 * overflow, which could easily occur if the initial value is nonzero!
700	 */
701	ld	[%o0], %o3			! %o3 = instruction to patch
702	and	%o3, 0x3f, %o2			! %o2 = existing imm value
703	add	%o2, %o4, %o2			! %o2 = new imm value
704	andn	%o3, 0x3f, %o3			! clear old imm value
705	and	%o2, 0x3f, %o2			! truncate new imm value
706	or	%o3, %o2, %o3			! set new imm value
707	st	%o3, [%o0]			! store updated instruction
708	retl
709	flush	%o0
710	SET_SIZE(sfmmu_fixup_shiftx)
711
712	ENTRY_NP(sfmmu_fixup_mmu_asi)
713	/*
714	 * Patch imm_asi of all ldda instructions in the MMU
715	 * trap handlers.  We search MMU_PATCH_INSTR instructions
716	 * starting from the itlb miss handler (trap 0x64).
717	 * %o0 = address of tt[0,1]_itlbmiss
718	 * %o1 = imm_asi to setup, shifted by appropriate offset.
719	 * %o3 = number of instructions to search
720	 * %o4 = reserved by caller: called from leaf routine
721	 */
7221:	ldsw	[%o0], %o2			! load instruction to %o2
723	brgez,pt %o2, 2f
724	  srl	%o2, 30, %o5
725	btst	1, %o5				! test bit 30; skip if not set
726	bz,pt	%icc, 2f
727	  sllx	%o2, 39, %o5			! bit 24 -> bit 63
728	srlx	%o5, 58, %o5			! isolate op3 part of opcode
729	xor	%o5, 0x13, %o5			! 01 0011 binary == ldda
730	brnz,pt	%o5, 2f				! skip if not a match
731	  or	%o2, %o1, %o2			! or in imm_asi
732	st	%o2, [%o0]			! write patched instruction
7332:	dec	%o3
734	brnz,a,pt %o3, 1b			! loop until we're done
735	  add	%o0, I_SIZE, %o0
736	retl
737	flush	%o0
738	SET_SIZE(sfmmu_fixup_mmu_asi)
739
740	/*
741	 * Patch immediate ASI used to access the TSB in the
742	 * trap table.
743	 * inputs: %o0 = value of ktsb_phys
744	 */
745	ENTRY_NP(sfmmu_patch_mmu_asi)
746	mov	%o7, %o4			! save return pc in %o4
747	movrnz	%o0, ASI_QUAD_LDD_PHYS, %o3
748	movrz	%o0, ASI_NQUAD_LD, %o3
749	sll	%o3, 5, %o1			! imm_asi offset
750	mov	6, %o3				! number of instructions
751	sethi	%hi(dktsb), %o0			! to search
752	call	sfmmu_fixup_mmu_asi		! patch kdtlb miss
753	  or	%o0, %lo(dktsb), %o0
754	mov	6, %o3				! number of instructions
755	sethi	%hi(dktsb4m), %o0		! to search
756	call	sfmmu_fixup_mmu_asi		! patch kdtlb4m miss
757	  or	%o0, %lo(dktsb4m), %o0
758	mov	6, %o3				! number of instructions
759	sethi	%hi(iktsb), %o0			! to search
760	call	sfmmu_fixup_mmu_asi		! patch kitlb miss
761	  or	%o0, %lo(iktsb), %o0
762	mov	%o4, %o7			! retore return pc -- leaf
763	retl
764	nop
765	SET_SIZE(sfmmu_patch_mmu_asi)
766
767	ENTRY_NP(sfmmu_patch_ktsb)
768	/*
769	 * We need to fix iktsb, dktsb, et. al.
770	 */
771	save	%sp, -SA(MINFRAME), %sp
772	set	ktsb_phys, %o1
773	ld	[%o1], %o4
774	set	ktsb_base, %o5
775	set	ktsb4m_base, %l1
776	brz,pt	%o4, 1f
777	  nop
778	set	ktsb_pbase, %o5
779	set	ktsb4m_pbase, %l1
7801:
781	sethi	%hi(ktsb_szcode), %o1
782	ld	[%o1 + %lo(ktsb_szcode)], %o1	/* %o1 = ktsb size code */
783
784	sethi	%hi(iktsb), %o0
785	call	sfmmu_fix_ktlb_traptable
786	  or	%o0, %lo(iktsb), %o0
787
788	sethi	%hi(dktsb), %o0
789	call	sfmmu_fix_ktlb_traptable
790	  or	%o0, %lo(dktsb), %o0
791
792	sethi	%hi(ktsb4m_szcode), %o1
793	ld	[%o1 + %lo(ktsb4m_szcode)], %o1	/* %o1 = ktsb4m size code */
794
795	sethi	%hi(dktsb4m), %o0
796	call	sfmmu_fix_ktlb_traptable
797	  or	%o0, %lo(dktsb4m), %o0
798
799#ifndef sun4v
800	mov	ASI_N, %o2
801	movrnz	%o4, ASI_MEM, %o2	! setup kernel 32bit ASI to patch
802	mov	%o2, %o4		! sfmmu_fixup_or needs this in %o4
803	sethi	%hi(tsb_kernel_patch_asi), %o0
804	call	sfmmu_fixup_or
805	  or	%o0, %lo(tsb_kernel_patch_asi), %o0
806#endif
807
808	ldx 	[%o5], %o4		! load ktsb base addr (VA or PA)
809
810	sethi	%hi(dktsbbase), %o0
811	call	sfmmu_fixup_setx	! patch value of ktsb base addr
812	  or	%o0, %lo(dktsbbase), %o0
813
814	sethi	%hi(iktsbbase), %o0
815	call	sfmmu_fixup_setx	! patch value of ktsb base addr
816	  or	%o0, %lo(iktsbbase), %o0
817
818	sethi	%hi(sfmmu_kprot_patch_ktsb_base), %o0
819	call	sfmmu_fixup_setx	! patch value of ktsb base addr
820	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_base), %o0
821
822#ifdef sun4v
823	sethi	%hi(sfmmu_dslow_patch_ktsb_base), %o0
824	call	sfmmu_fixup_setx	! patch value of ktsb base addr
825	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_base), %o0
826#endif /* sun4v */
827
828	ldx 	[%l1], %o4		! load ktsb4m base addr (VA or PA)
829
830	sethi	%hi(dktsb4mbase), %o0
831	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
832	  or	%o0, %lo(dktsb4mbase), %o0
833
834	sethi	%hi(sfmmu_kprot_patch_ktsb4m_base), %o0
835	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
836	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_base), %o0
837
838#ifdef sun4v
839	sethi	%hi(sfmmu_dslow_patch_ktsb4m_base), %o0
840	call	sfmmu_fixup_setx	! patch value of ktsb4m base addr
841	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_base), %o0
842#endif /* sun4v */
843
844	set	ktsb_szcode, %o4
845	ld	[%o4], %o4
846	sethi	%hi(sfmmu_kprot_patch_ktsb_szcode), %o0
847	call	sfmmu_fixup_or		! patch value of ktsb_szcode
848	  or	%o0, %lo(sfmmu_kprot_patch_ktsb_szcode), %o0
849
850#ifdef sun4v
851	sethi	%hi(sfmmu_dslow_patch_ktsb_szcode), %o0
852	call	sfmmu_fixup_or		! patch value of ktsb_szcode
853	  or	%o0, %lo(sfmmu_dslow_patch_ktsb_szcode), %o0
854#endif /* sun4v */
855
856	set	ktsb4m_szcode, %o4
857	ld	[%o4], %o4
858	sethi	%hi(sfmmu_kprot_patch_ktsb4m_szcode), %o0
859	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
860	  or	%o0, %lo(sfmmu_kprot_patch_ktsb4m_szcode), %o0
861
862#ifdef sun4v
863	sethi	%hi(sfmmu_dslow_patch_ktsb4m_szcode), %o0
864	call	sfmmu_fixup_or		! patch value of ktsb4m_szcode
865	  or	%o0, %lo(sfmmu_dslow_patch_ktsb4m_szcode), %o0
866#endif /* sun4v */
867
868	ret
869	restore
870	SET_SIZE(sfmmu_patch_ktsb)
871
872	ENTRY_NP(sfmmu_kpm_patch_tlbm)
873	/*
874	 * Fixup trap handlers in common segkpm case.  This is reserved
875	 * for future use should kpm TSB be changed to be other than the
876	 * kernel TSB.
877	 */
878	retl
879	nop
880	SET_SIZE(sfmmu_kpm_patch_tlbm)
881
882	ENTRY_NP(sfmmu_kpm_patch_tsbm)
883	/*
884	 * nop the branch to sfmmu_kpm_dtsb_miss_small
885	 * in the case where we are using large pages for
886	 * seg_kpm (and hence must probe the second TSB for
887	 * seg_kpm VAs)
888	 */
889	set	dktsb4m_kpmcheck_small, %o0
890	MAKE_NOP_INSTR(%o1)
891	st	%o1, [%o0]
892	flush	%o0
893	retl
894	nop
895	SET_SIZE(sfmmu_kpm_patch_tsbm)
896
897	ENTRY_NP(sfmmu_patch_utsb)
898#ifdef sun4v
899	retl
900	nop
901#else /* sun4v */
902	/*
903	 * We need to hot patch utsb_vabase and utsb4m_vabase
904	 */
905	save	%sp, -SA(MINFRAME), %sp
906
907	/* patch value of utsb_vabase */
908	set	utsb_vabase, %o1
909	ldx	[%o1], %o4
910	sethi	%hi(sfmmu_uprot_get_1st_tsbe_ptr), %o0
911	call	sfmmu_fixup_setx
912	  or	%o0, %lo(sfmmu_uprot_get_1st_tsbe_ptr), %o0
913	sethi	%hi(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
914	call	sfmmu_fixup_setx
915	  or	%o0, %lo(sfmmu_uitlb_get_1st_tsbe_ptr), %o0
916	sethi	%hi(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
917	call	sfmmu_fixup_setx
918	  or	%o0, %lo(sfmmu_udtlb_get_1st_tsbe_ptr), %o0
919
920	/* patch value of utsb4m_vabase */
921	set	utsb4m_vabase, %o1
922	ldx	[%o1], %o4
923	sethi	%hi(sfmmu_uprot_get_2nd_tsb_base), %o0
924	call	sfmmu_fixup_setx
925	  or	%o0, %lo(sfmmu_uprot_get_2nd_tsb_base), %o0
926	sethi	%hi(sfmmu_uitlb_get_2nd_tsb_base), %o0
927	call	sfmmu_fixup_setx
928	  or	%o0, %lo(sfmmu_uitlb_get_2nd_tsb_base), %o0
929	sethi	%hi(sfmmu_udtlb_get_2nd_tsb_base), %o0
930	call	sfmmu_fixup_setx
931	  or	%o0, %lo(sfmmu_udtlb_get_2nd_tsb_base), %o0
932
933	/*
934	 * Patch TSB base register masks and shifts if needed.
935	 * By default the TSB base register contents are set up for 4M slab.
936	 * If we're using a smaller slab size and reserved VA range we need
937	 * to patch up those values here.
938	 */
939	set	tsb_slab_shift, %o1
940	set	MMU_PAGESHIFT4M, %o4
941	ldsw	[%o1], %o3
942	subcc	%o4, %o3, %o4
943	bz,pt	%icc, 1f
944	  /* delay slot safe */
945
946	/* patch reserved VA range size if needed. */
947	sethi	%hi(sfmmu_tsb_1st_resv_offset), %o0
948	call	sfmmu_fixup_shiftx
949	  or	%o0, %lo(sfmmu_tsb_1st_resv_offset), %o0
950	call	sfmmu_fixup_shiftx
951	  add	%o0, I_SIZE, %o0
952	sethi	%hi(sfmmu_tsb_2nd_resv_offset), %o0
953	call	sfmmu_fixup_shiftx
954	  or	%o0, %lo(sfmmu_tsb_2nd_resv_offset), %o0
955	call	sfmmu_fixup_shiftx
956	  add	%o0, I_SIZE, %o0
9571:
958	/* patch TSBREG_VAMASK used to set up TSB base register */
959	set	tsb_slab_mask, %o1
960	lduw	[%o1], %o4
961	sethi	%hi(sfmmu_tsb_1st_tsbreg_vamask), %o0
962	call	sfmmu_fixup_or
963	  or	%o0, %lo(sfmmu_tsb_1st_tsbreg_vamask), %o0
964	sethi	%hi(sfmmu_tsb_2nd_tsbreg_vamask), %o0
965	call	sfmmu_fixup_or
966	  or	%o0, %lo(sfmmu_tsb_2nd_tsbreg_vamask), %o0
967
968	ret
969	restore
970#endif /* sun4v */
971	SET_SIZE(sfmmu_patch_utsb)
972
973
974	/*
975	 * Routine that loads an entry into a tsb using virtual addresses.
976	 * Locking is required since all cpus can use the same TSB.
977	 * Note that it is no longer required to have a valid context
978	 * when calling this function.
979	 */
980	ENTRY_NP(sfmmu_load_tsbe)
981	/*
982	 * %o0 = pointer to tsbe to load
983	 * %o1 = tsb tag
984	 * %o2 = virtual pointer to TTE
985	 * %o3 = 1 if physical address in %o0 else 0
986	 */
987	rdpr	%pstate, %o5
988#ifdef DEBUG
989	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
990	bnz,pt 	%icc, 1f			/* disabled, panic	 */
991	  nop
992
993	sethi	%hi(panicstr), %g1
994	ldx	[%g1 + %lo(panicstr)], %g1
995	tst	%g1
996	bnz,pt	%icc, 1f
997	  nop
998
999	save	%sp, -SA(MINFRAME), %sp
1000	sethi	%hi(sfmmu_panic1), %o0
1001	call	panic
1002	 or	%o0, %lo(sfmmu_panic1), %o0
10031:
1004#endif /* DEBUG */
1005
1006	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1007
1008	SETUP_TSB_ASI(%o3, %g3)
1009	TSB_UPDATE(%o0, %o2, %o1, %g1, %g2, 1)
1010
1011	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1012
1013	retl
1014	membar	#StoreStore|#StoreLoad
1015	SET_SIZE(sfmmu_load_tsbe)
1016
1017	/*
1018	 * Flush TSB of a given entry if the tag matches.
1019	 */
1020	ENTRY(sfmmu_unload_tsbe)
1021	/*
1022	 * %o0 = pointer to tsbe to be flushed
1023	 * %o1 = tag to match
1024	 * %o2 = 1 if physical address in %o0 else 0
1025	 */
1026	SETUP_TSB_ASI(%o2, %g1)
1027	TSB_INVALIDATE(%o0, %o1, %g1, %o2, %o3, unload_tsbe)
1028	retl
1029	membar	#StoreStore|#StoreLoad
1030	SET_SIZE(sfmmu_unload_tsbe)
1031
1032	/*
1033	 * Routine that loads a TTE into the kpm TSB from C code.
1034	 * Locking is required since kpm TSB is shared among all CPUs.
1035	 */
1036	ENTRY_NP(sfmmu_kpm_load_tsb)
1037	/*
1038	 * %o0 = vaddr
1039	 * %o1 = ttep
1040	 * %o2 = virtpg to TSB index shift (e.g. TTE pagesize shift)
1041	 */
1042	rdpr	%pstate, %o5			! %o5 = saved pstate
1043#ifdef DEBUG
1044	andcc	%o5, PSTATE_IE, %g0		! if interrupts already
1045	bnz,pt	%icc, 1f			! disabled, panic
1046	  nop
1047
1048	sethi	%hi(panicstr), %g1
1049	ldx	[%g1 + %lo(panicstr)], %g1
1050	tst	%g1
1051	bnz,pt	%icc, 1f
1052	  nop
1053
1054	save	%sp, -SA(MINFRAME), %sp
1055	sethi	%hi(sfmmu_panic1), %o0
1056	call	panic
1057	  or	%o0, %lo(sfmmu_panic1), %o0
10581:
1059#endif /* DEBUG */
1060	wrpr	%o5, PSTATE_IE, %pstate		! disable interrupts
1061
1062#ifndef sun4v
1063	sethi	%hi(ktsb_phys), %o4
1064	mov	ASI_N, %o3
1065	ld	[%o4 + %lo(ktsb_phys)], %o4
1066	movrnz	%o4, ASI_MEM, %o3
1067	mov	%o3, %asi
1068#endif
1069	mov	%o0, %g1			! %g1 = vaddr
1070
1071	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1072	GET_KPM_TSBE_POINTER(%o2, %g2, %g1, %o3, %o4)
1073	/* %g2 = tsbep, %g1 clobbered */
1074
1075	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1076	/* TSB_UPDATE(tsbep, tteva, tagtarget, tmp1, tmp2, label) */
1077	TSB_UPDATE(%g2, %o1, %g1, %o3, %o4, 1)
1078
1079	wrpr	%g0, %o5, %pstate		! enable interrupts
1080	retl
1081	  membar #StoreStore|#StoreLoad
1082	SET_SIZE(sfmmu_kpm_load_tsb)
1083
1084	/*
1085	 * Routine that shoots down a TTE in the kpm TSB or in the
1086	 * kernel TSB depending on virtpg. Locking is required since
1087	 * kpm/kernel TSB is shared among all CPUs.
1088	 */
1089	ENTRY_NP(sfmmu_kpm_unload_tsb)
1090	/*
1091	 * %o0 = vaddr
1092	 * %o1 = virtpg to TSB index shift (e.g. TTE page shift)
1093	 */
1094#ifndef sun4v
1095	sethi	%hi(ktsb_phys), %o4
1096	mov	ASI_N, %o3
1097	ld	[%o4 + %lo(ktsb_phys)], %o4
1098	movrnz	%o4, ASI_MEM, %o3
1099	mov	%o3, %asi
1100#endif
1101	mov	%o0, %g1			! %g1 = vaddr
1102
1103	/* GET_KPM_TSBE_POINTER(vpshift, tsbp, vaddr (clobbers), tmp1, tmp2) */
1104	GET_KPM_TSBE_POINTER(%o1, %g2, %g1, %o3, %o4)
1105	/* %g2 = tsbep, %g1 clobbered */
1106
1107	srlx	%o0, TTARGET_VA_SHIFT, %g1;	! %g1 = tag target
1108	/* TSB_INVALIDATE(tsbep, tag, tmp1, tmp2, tmp3, label) */
1109	TSB_INVALIDATE(%g2, %g1, %o3, %o4, %o1, kpm_tsbinval)
1110
1111	retl
1112	  membar	#StoreStore|#StoreLoad
1113	SET_SIZE(sfmmu_kpm_unload_tsb)
1114
1115#endif /* lint */
1116
1117
1118#if defined (lint)
1119
1120/*ARGSUSED*/
1121pfn_t
1122sfmmu_ttetopfn(tte_t *tte, caddr_t vaddr)
1123{ return(0); }
1124
1125#else /* lint */
1126
1127	ENTRY_NP(sfmmu_ttetopfn)
1128	ldx	[%o0], %g1			/* read tte */
1129	TTETOPFN(%g1, %o1, sfmmu_ttetopfn_l1, %g2, %g3, %g4)
1130	/*
1131	 * g1 = pfn
1132	 */
1133	retl
1134	mov	%g1, %o0
1135	SET_SIZE(sfmmu_ttetopfn)
1136
1137#endif /* !lint */
1138
1139
1140#if defined (lint)
1141/*
1142 * The sfmmu_hblk_hash_add is the assembly primitive for adding hmeblks to the
1143 * the hash list.
1144 */
1145/* ARGSUSED */
1146void
1147sfmmu_hblk_hash_add(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1148	uint64_t hblkpa)
1149{
1150}
1151
1152/*
1153 * The sfmmu_hblk_hash_rm is the assembly primitive to remove hmeblks from the
1154 * hash list.
1155 */
1156/* ARGSUSED */
1157void
1158sfmmu_hblk_hash_rm(struct hmehash_bucket *hmebp, struct hme_blk *hmeblkp,
1159	uint64_t hblkpa, struct hme_blk *prev_hblkp)
1160{
1161}
1162#else /* lint */
1163
1164/*
1165 * Functions to grab/release hme bucket list lock.  I only use a byte
1166 * instead of the whole int because eventually we might want to
1167 * put some counters on the other bytes (of course, these routines would
1168 * have to change).  The code that grab this lock should execute
1169 * with interrupts disabled and hold the lock for the least amount of time
1170 * possible.
1171 */
1172
1173/*
1174 * Even though hmeh_listlock is updated using pa there's no need to flush
1175 * dcache since hmeh_listlock will be restored to the original value (0)
1176 * before interrupts are reenabled.
1177 */
1178
1179/*
1180 * For sparcv9 hme hash buckets may not be in the nucleus.  hme hash update
1181 * routines still use virtual addresses to update the bucket fields. But they
1182 * must not cause a TLB miss after grabbing the low level bucket lock. To
1183 * achieve this we must make sure the bucket structure is completely within an
1184 * 8K page.
1185 */
1186
1187#if (HMEBUCK_SIZE & (HMEBUCK_SIZE - 1))
1188#error - the size of hmehash_bucket structure is not power of 2
1189#endif
1190
1191#define HMELOCK_ENTER(hmebp, tmp1, tmp2, label1, asi)           \
1192	mov     0xff, tmp2                                      ;\
1193	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1194label1:                                                         ;\
1195	casa    [tmp1]asi, %g0, tmp2                            ;\
1196	brnz,pn tmp2, label1                                    ;\
1197	mov     0xff, tmp2                                      ;\
1198	membar  #LoadLoad
1199
1200#define HMELOCK_EXIT(hmebp, tmp1, asi)                          \
1201	membar  #LoadStore|#StoreStore                          ;\
1202	add     hmebp, HMEBUCK_LOCK, tmp1                       ;\
1203	sta     %g0, [tmp1]asi
1204
1205	.seg	".data"
1206hblk_add_panic1:
1207	.ascii	"sfmmu_hblk_hash_add: interrupts disabled"
1208	.byte	0
1209hblk_add_panic2:
1210	.ascii	"sfmmu_hblk_hash_add: va hmeblkp is NULL but pa is not"
1211	.byte	0
1212	.align	4
1213	.seg	".text"
1214
1215	ENTRY_NP(sfmmu_hblk_hash_add)
1216	/*
1217	 * %o0 = hmebp
1218	 * %o1 = hmeblkp
1219	 * %o2 = hblkpa
1220	 */
1221	rdpr	%pstate, %o5
1222#ifdef DEBUG
1223	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1224	bnz,pt %icc, 3f				/* disabled, panic	 */
1225	  nop
1226	save	%sp, -SA(MINFRAME), %sp
1227	sethi	%hi(hblk_add_panic1), %o0
1228	call	panic
1229	 or	%o0, %lo(hblk_add_panic1), %o0
1230	ret
1231	restore
1232
12333:
1234#endif /* DEBUG */
1235	wrpr	%o5, PSTATE_IE, %pstate		/* disable interrupts */
1236	mov	%o2, %g1
1237
1238	/*
1239	 * g1 = hblkpa
1240	 */
1241	ldn	[%o0 + HMEBUCK_HBLK], %o4	/* next hmeblk */
1242	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = next hblkpa */
1243#ifdef	DEBUG
1244	cmp	%o4, %g0
1245	bne,pt %xcc, 1f
1246	 nop
1247	brz,pt %g2, 1f
1248	 nop
1249	wrpr	%g0, %o5, %pstate		/* enable interrupts */
1250	save	%sp, -SA(MINFRAME), %sp
1251	sethi	%hi(hblk_add_panic2), %o0
1252	call	panic
1253	  or	%o0, %lo(hblk_add_panic2), %o0
1254	ret
1255	restore
12561:
1257#endif /* DEBUG */
1258	/*
1259	 * We update hmeblks entries before grabbing lock because the stores
1260	 * could take a tlb miss and require the hash lock.  The buckets
1261	 * are part of the nucleus so we are cool with those stores.
1262	 *
1263	 * if buckets are not part of the nucleus our game is to
1264	 * not touch any other page via va until we drop the lock.
1265	 * This guarantees we won't get a tlb miss before the lock release
1266	 * since interrupts are disabled.
1267	 */
1268	stn	%o4, [%o1 + HMEBLK_NEXT]	/* update hmeblk's next */
1269	stx	%g2, [%o1 + HMEBLK_NEXTPA]	/* update hmeblk's next pa */
1270	HMELOCK_ENTER(%o0, %o2, %o3, hashadd1, ASI_N)
1271	stn	%o1, [%o0 + HMEBUCK_HBLK]	/* update bucket hblk next */
1272	stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* add hmeblk to list */
1273	HMELOCK_EXIT(%o0, %g2, ASI_N)
1274	retl
1275	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1276	SET_SIZE(sfmmu_hblk_hash_add)
1277
1278	ENTRY_NP(sfmmu_hblk_hash_rm)
1279	/*
1280	 * This function removes an hmeblk from the hash chain.
1281	 * It is written to guarantee we don't take a tlb miss
1282	 * by using physical addresses to update the list.
1283	 *
1284	 * %o0 = hmebp
1285	 * %o1 = hmeblkp
1286	 * %o2 = hmeblkp previous pa
1287	 * %o3 = hmeblkp previous
1288	 */
1289
1290	mov	%o3, %o4			/* o4 = hmeblkp previous */
1291
1292	rdpr	%pstate, %o5
1293#ifdef DEBUG
1294	andcc	%o5, PSTATE_IE, %g0		/* if interrupts already */
1295	bnz,pt 	%icc, 3f			/* disabled, panic	 */
1296	  nop
1297
1298	sethi	%hi(panicstr), %g1
1299	ldx	[%g1 + %lo(panicstr)], %g1
1300	tst	%g1
1301	bnz,pt	%icc, 3f
1302	  nop
1303
1304	sethi	%hi(sfmmu_panic1), %o0
1305	call	panic
1306	 or	%o0, %lo(sfmmu_panic1), %o0
13073:
1308#endif /* DEBUG */
1309	/*
1310	 * disable interrupts, clear Address Mask to access 64 bit physaddr
1311	 */
1312	andn    %o5, PSTATE_IE, %g1
1313	wrpr    %g1, 0, %pstate
1314
1315#ifndef sun4v
1316	sethi   %hi(dcache_line_mask), %g4
1317	ld      [%g4 + %lo(dcache_line_mask)], %g4
1318#endif /* sun4v */
1319
1320	/*
1321	 * if buckets are not part of the nucleus our game is to
1322	 * not touch any other page via va until we drop the lock.
1323	 * This guarantees we won't get a tlb miss before the lock release
1324	 * since interrupts are disabled.
1325	 */
1326	HMELOCK_ENTER(%o0, %g1, %g3, hashrm1, ASI_N)
1327	ldn	[%o0 + HMEBUCK_HBLK], %g2	/* first hmeblk in list */
1328	cmp	%g2, %o1
1329	bne,pt	%ncc,1f
1330	 mov	ASI_MEM, %asi
1331	/*
1332	 * hmeblk is first on list
1333	 */
1334	ldx	[%o0 + HMEBUCK_NEXTPA], %g2	/* g2 = hmeblk pa */
1335	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1336	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1337	stn	%o3, [%o0 + HMEBUCK_HBLK]	/* write va */
1338	ba,pt	%xcc, 2f
1339	  stx	%g1, [%o0 + HMEBUCK_NEXTPA]	/* write pa */
13401:
1341	/* hmeblk is not first on list */
1342
1343	mov	%o2, %g3
1344#ifndef sun4v
1345	GET_CPU_IMPL(%g2)
1346	cmp %g2, CHEETAH_IMPL
1347	bge %icc, hblk_hash_rm_1
1348	and	%o4, %g4, %g2
1349	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev pa from dcache */
1350	add	%o4, HMEBLK_NEXT, %o4
1351	and	%o4, %g4, %g2
1352	ba	hblk_hash_rm_2
1353	stxa	%g0, [%g2]ASI_DC_TAG		/* flush prev va from dcache */
1354hblk_hash_rm_1:
1355
1356	stxa	%g0, [%g3]ASI_DC_INVAL		/* flush prev pa from dcache */
1357	membar	#Sync
1358	add     %g3, HMEBLK_NEXT, %g2
1359	stxa	%g0, [%g2]ASI_DC_INVAL		/* flush prev va from dcache */
1360hblk_hash_rm_2:
1361	membar	#Sync
1362#endif /* sun4v */
1363	ldxa	[%g3 + HMEBLK_NEXTPA] %asi, %g2	/* g2 = hmeblk pa */
1364	ldna	[%g2 + HMEBLK_NEXT] %asi, %o3	/* read next hmeblk va */
1365	ldxa	[%g2 + HMEBLK_NEXTPA] %asi, %g1	/* read next hmeblk pa */
1366	stna	%o3, [%g3 + HMEBLK_NEXT] %asi	/* write va */
1367	stxa	%g1, [%g3 + HMEBLK_NEXTPA] %asi	/* write pa */
13682:
1369	HMELOCK_EXIT(%o0, %g2, ASI_N)
1370	retl
1371	  wrpr	%g0, %o5, %pstate		/* enable interrupts */
1372	SET_SIZE(sfmmu_hblk_hash_rm)
1373
1374#endif /* lint */
1375
1376/*
1377 * These macros are used to update global sfmmu hme hash statistics
1378 * in perf critical paths. It is only enabled in debug kernels or
1379 * if SFMMU_STAT_GATHER is defined
1380 */
1381#if defined(DEBUG) || defined(SFMMU_STAT_GATHER)
1382#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1383	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1384	mov	HATSTAT_KHASH_SEARCH, tmp2				;\
1385	cmp	tmp1, hatid						;\
1386	movne	%ncc, HATSTAT_UHASH_SEARCH, tmp2			;\
1387	set	sfmmu_global_stat, tmp1					;\
1388	add	tmp1, tmp2, tmp1					;\
1389	ld	[tmp1], tmp2						;\
1390	inc	tmp2							;\
1391	st	tmp2, [tmp1]
1392
1393#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)			\
1394	ldn	[tsbarea + TSBMISS_KHATID], tmp1			;\
1395	mov	HATSTAT_KHASH_LINKS, tmp2				;\
1396	cmp	tmp1, hatid						;\
1397	movne	%ncc, HATSTAT_UHASH_LINKS, tmp2				;\
1398	set	sfmmu_global_stat, tmp1					;\
1399	add	tmp1, tmp2, tmp1					;\
1400	ld	[tmp1], tmp2						;\
1401	inc	tmp2							;\
1402	st	tmp2, [tmp1]
1403
1404
1405#else /* DEBUG || SFMMU_STAT_GATHER */
1406
1407#define	HAT_HSEARCH_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1408
1409#define	HAT_HLINK_DBSTAT(hatid, tsbarea, tmp1, tmp2)
1410
1411#endif  /* DEBUG || SFMMU_STAT_GATHER */
1412
1413/*
1414 * This macro is used to update global sfmmu kstas in non
1415 * perf critical areas so they are enabled all the time
1416 */
1417#define	HAT_GLOBAL_STAT(statname, tmp1, tmp2)				\
1418	sethi	%hi(sfmmu_global_stat), tmp1				;\
1419	add	tmp1, statname, tmp1					;\
1420	ld	[tmp1 + %lo(sfmmu_global_stat)], tmp2			;\
1421	inc	tmp2							;\
1422	st	tmp2, [tmp1 + %lo(sfmmu_global_stat)]
1423
1424/*
1425 * These macros are used to update per cpu stats in non perf
1426 * critical areas so they are enabled all the time
1427 */
1428#define	HAT_PERCPU_STAT32(tsbarea, stat, tmp1)				\
1429	ld	[tsbarea + stat], tmp1					;\
1430	inc	tmp1							;\
1431	st	tmp1, [tsbarea + stat]
1432
1433/*
1434 * These macros are used to update per cpu stats in non perf
1435 * critical areas so they are enabled all the time
1436 */
1437#define	HAT_PERCPU_STAT16(tsbarea, stat, tmp1)				\
1438	lduh	[tsbarea + stat], tmp1					;\
1439	inc	tmp1							;\
1440	stuh	tmp1, [tsbarea + stat]
1441
1442#if defined(KPM_TLBMISS_STATS_GATHER)
1443	/*
1444	 * Count kpm dtlb misses separately to allow a different
1445	 * evaluation of hme and kpm tlbmisses. kpm tsb hits can
1446	 * be calculated by (kpm_dtlb_misses - kpm_tsb_misses).
1447	 */
1448#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)		\
1449	brgez	tagacc, label	/* KPM VA? */				;\
1450	nop								;\
1451	CPU_INDEX(tmp1, tsbma)						;\
1452	sethi	%hi(kpmtsbm_area), tsbma				;\
1453	sllx	tmp1, KPMTSBM_SHIFT, tmp1				;\
1454	or	tsbma, %lo(kpmtsbm_area), tsbma				;\
1455	add	tsbma, tmp1, tsbma		/* kpmtsbm area */	;\
1456	/* VA range check */						;\
1457	ldx	[tsbma + KPMTSBM_VBASE], val				;\
1458	cmp	tagacc, val						;\
1459	blu,pn	%xcc, label						;\
1460	  ldx	[tsbma + KPMTSBM_VEND], tmp1				;\
1461	cmp	tagacc, tmp1						;\
1462	bgeu,pn	%xcc, label						;\
1463	  lduw	[tsbma + KPMTSBM_DTLBMISS], val				;\
1464	inc	val							;\
1465	st	val, [tsbma + KPMTSBM_DTLBMISS]				;\
1466label:
1467#else
1468#define	KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label)
1469#endif	/* KPM_TLBMISS_STATS_GATHER */
1470
1471#if defined (lint)
1472/*
1473 * The following routines are jumped to from the mmu trap handlers to do
1474 * the setting up to call systrap.  They are separate routines instead of
1475 * being part of the handlers because the handlers would exceed 32
1476 * instructions and since this is part of the slow path the jump
1477 * cost is irrelevant.
1478 */
1479void
1480sfmmu_pagefault(void)
1481{
1482}
1483
1484void
1485sfmmu_mmu_trap(void)
1486{
1487}
1488
1489void
1490sfmmu_window_trap(void)
1491{
1492}
1493
1494void
1495sfmmu_kpm_exception(void)
1496{
1497}
1498
1499#else /* lint */
1500
1501#ifdef	PTL1_PANIC_DEBUG
1502	.seg	".data"
1503	.global	test_ptl1_panic
1504test_ptl1_panic:
1505	.word	0
1506	.align	8
1507
1508	.seg	".text"
1509	.align	4
1510#endif	/* PTL1_PANIC_DEBUG */
1511
1512
1513	ENTRY_NP(sfmmu_pagefault)
1514	USE_ALTERNATE_GLOBALS(%g5)
1515	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g6, %g4)
1516	rdpr	%tt, %g6
1517	cmp	%g6, FAST_IMMU_MISS_TT
1518	be,a,pn	%icc, 1f
1519	  mov	T_INSTR_MMU_MISS, %g3
1520	cmp	%g6, T_INSTR_MMU_MISS
1521	be,a,pn	%icc, 1f
1522	  mov	T_INSTR_MMU_MISS, %g3
1523	mov	%g5, %g2
1524	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1525	cmp	%g6, FAST_DMMU_MISS_TT
1526	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1527	cmp	%g6, T_DATA_MMU_MISS
1528	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1529
1530#ifdef  PTL1_PANIC_DEBUG
1531	/* check if we want to test the tl1 panic */
1532	sethi	%hi(test_ptl1_panic), %g4
1533	ld	[%g4 + %lo(test_ptl1_panic)], %g1
1534	st	%g0, [%g4 + %lo(test_ptl1_panic)]
1535	cmp	%g1, %g0
1536	bne,a,pn %icc, ptl1_panic
1537	  or	%g0, PTL1_BAD_DEBUG, %g1
1538#endif	/* PTL1_PANIC_DEBUG */
15391:
1540	HAT_GLOBAL_STAT(HATSTAT_PAGEFAULT, %g6, %g4)
1541	/*
1542	 * g2 = tag access reg
1543	 * g3.l = type
1544	 * g3.h = 0
1545	 */
1546	sethi	%hi(trap), %g1
1547	or	%g1, %lo(trap), %g1
15482:
1549	ba,pt	%xcc, sys_trap
1550	  mov	-1, %g4
1551	SET_SIZE(sfmmu_pagefault)
1552
1553	ENTRY_NP(sfmmu_mmu_trap)
1554	USE_ALTERNATE_GLOBALS(%g5)
1555	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g6)
1556	rdpr	%tt, %g6
1557	cmp	%g6, FAST_IMMU_MISS_TT
1558	be,a,pn	%icc, 1f
1559	  mov	T_INSTR_MMU_MISS, %g3
1560	cmp	%g6, T_INSTR_MMU_MISS
1561	be,a,pn	%icc, 1f
1562	  mov	T_INSTR_MMU_MISS, %g3
1563	mov	%g5, %g2
1564	mov	T_DATA_PROT, %g3		/* arg2 = traptype */
1565	cmp	%g6, FAST_DMMU_MISS_TT
1566	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1567	cmp	%g6, T_DATA_MMU_MISS
1568	move	%icc, T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
15691:
1570	/*
1571	 * g2 = tag access reg
1572	 * g3 = type
1573	 */
1574	sethi	%hi(sfmmu_tsbmiss_exception), %g1
1575	or	%g1, %lo(sfmmu_tsbmiss_exception), %g1
1576	ba,pt	%xcc, sys_trap
1577	  mov	-1, %g4
1578	/*NOTREACHED*/
1579	SET_SIZE(sfmmu_mmu_trap)
1580
1581	ENTRY_NP(sfmmu_suspend_tl)
1582	USE_ALTERNATE_GLOBALS(%g5)
1583	GET_MMU_BOTH_TAGACC(%g5 /*dtag*/, %g2 /*itag*/, %g4, %g3)
1584	rdpr	%tt, %g6
1585	cmp	%g6, FAST_IMMU_MISS_TT
1586	be,a,pn	%icc, 1f
1587	  mov	T_INSTR_MMU_MISS, %g3
1588	mov	%g5, %g2
1589	cmp	%g6, FAST_DMMU_MISS_TT
1590	move	%icc, T_DATA_MMU_MISS, %g3
1591	movne	%icc, T_DATA_PROT, %g3
15921:
1593	sethi	%hi(sfmmu_tsbmiss_suspended), %g1
1594	or	%g1, %lo(sfmmu_tsbmiss_suspended), %g1
1595	/* g1 = TL0 handler, g2 = tagacc, g3 = trap type */
1596	ba,pt	%xcc, sys_trap
1597	  mov	PIL_15, %g4
1598	/*NOTREACHED*/
1599	SET_SIZE(sfmmu_suspend_tl)
1600
1601	/*
1602	 * No %g registers in use at this point.
1603	 */
1604	ENTRY_NP(sfmmu_window_trap)
1605	rdpr	%tpc, %g1
1606#ifdef sun4v
1607#ifdef DEBUG
1608	/* We assume previous %gl was 1 */
1609	rdpr	%tstate, %g4
1610	srlx	%g4, TSTATE_GL_SHIFT, %g4
1611	and	%g4, TSTATE_GL_MASK, %g4
1612	cmp	%g4, 1
1613	bne,a,pn %icc, ptl1_panic
1614	  mov	PTL1_BAD_WTRAP, %g1
1615#endif /* DEBUG */
1616	/* user miss at tl>1. better be the window handler or user_rtt */
1617	/* in user_rtt? */
1618	set	rtt_fill_start, %g4
1619	cmp	%g1, %g4
1620	blu,pn %xcc, 6f
1621	 .empty
1622	set	rtt_fill_end, %g4
1623	cmp	%g1, %g4
1624	bgeu,pn %xcc, 6f
1625	 nop
1626	set	fault_rtt_fn1, %g1
1627	wrpr	%g0, %g1, %tnpc
1628	ba,a	7f
16296:
1630	! must save this trap level before descending trap stack
1631	! no need to save %tnpc, either overwritten or discarded
1632	! already got it: rdpr	%tpc, %g1
1633	rdpr	%tstate, %g6
1634	rdpr	%tt, %g7
1635	! trap level saved, go get underlying trap type
1636	rdpr	%tl, %g5
1637	sub	%g5, 1, %g3
1638	wrpr	%g3, %tl
1639	rdpr	%tt, %g2
1640	wrpr	%g5, %tl
1641	! restore saved trap level
1642	wrpr	%g1, %tpc
1643	wrpr	%g6, %tstate
1644	wrpr	%g7, %tt
1645#else /* sun4v */
1646	/* user miss at tl>1. better be the window handler */
1647	rdpr	%tl, %g5
1648	sub	%g5, 1, %g3
1649	wrpr	%g3, %tl
1650	rdpr	%tt, %g2
1651	wrpr	%g5, %tl
1652#endif /* sun4v */
1653	and	%g2, WTRAP_TTMASK, %g4
1654	cmp	%g4, WTRAP_TYPE
1655	bne,pn	%xcc, 1f
1656	 nop
1657	/* tpc should be in the trap table */
1658	set	trap_table, %g4
1659	cmp	%g1, %g4
1660	blt,pn %xcc, 1f
1661	 .empty
1662	set	etrap_table, %g4
1663	cmp	%g1, %g4
1664	bge,pn %xcc, 1f
1665	 .empty
1666	andn	%g1, WTRAP_ALIGN, %g1	/* 128 byte aligned */
1667	add	%g1, WTRAP_FAULTOFF, %g1
1668	wrpr	%g0, %g1, %tnpc
16697:
1670	/*
1671	 * some wbuf handlers will call systrap to resolve the fault
1672	 * we pass the trap type so they figure out the correct parameters.
1673	 * g5 = trap type, g6 = tag access reg
1674	 */
1675
1676	/*
1677	 * only use g5, g6, g7 registers after we have switched to alternate
1678	 * globals.
1679	 */
1680	SET_GL_REG(1)
1681	USE_ALTERNATE_GLOBALS(%g5)
1682	GET_MMU_D_TAGACC(%g6 /*dtag*/, %g5 /*scratch*/)
1683	rdpr	%tt, %g7
1684	cmp	%g7, FAST_IMMU_MISS_TT
1685	be,a,pn	%icc, ptl1_panic
1686	  mov	PTL1_BAD_WTRAP, %g1
1687	cmp	%g7, T_INSTR_MMU_MISS
1688	be,a,pn	%icc, ptl1_panic
1689	  mov	PTL1_BAD_WTRAP, %g1
1690	mov	T_DATA_PROT, %g5
1691	cmp	%g7, FAST_DMMU_MISS_TT
1692	move	%icc, T_DATA_MMU_MISS, %g5
1693	cmp	%g7, T_DATA_MMU_MISS
1694	move	%icc, T_DATA_MMU_MISS, %g5
1695	! XXXQ AGS re-check out this one
1696	done
16971:
1698	CPU_ADDR(%g1, %g4)
1699	ld	[%g1 + CPU_TL1_HDLR], %g4
1700	brnz,a,pt %g4, sfmmu_mmu_trap
1701	  st	%g0, [%g1 + CPU_TL1_HDLR]
1702	ba,pt	%icc, ptl1_panic
1703	  mov	PTL1_BAD_TRAP, %g1
1704	SET_SIZE(sfmmu_window_trap)
1705
1706	ENTRY_NP(sfmmu_kpm_exception)
1707	/*
1708	 * We have accessed an unmapped segkpm address or a legal segkpm
1709	 * address which is involved in a VAC alias conflict prevention.
1710	 * Before we go to trap(), check to see if CPU_DTRACE_NOFAULT is
1711	 * set. If it is, we will instead note that a fault has occurred
1712	 * by setting CPU_DTRACE_BADADDR and issue a "done" (instead of
1713	 * a "retry"). This will step over the faulting instruction.
1714	 * Note that this means that a legal segkpm address involved in
1715	 * a VAC alias conflict prevention (a rare case to begin with)
1716	 * cannot be used in DTrace.
1717	 */
1718	CPU_INDEX(%g1, %g2)
1719	set	cpu_core, %g2
1720	sllx	%g1, CPU_CORE_SHIFT, %g1
1721	add	%g1, %g2, %g1
1722	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
1723	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
1724	bz	0f
1725	or	%g2, CPU_DTRACE_BADADDR, %g2
1726	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
1727	GET_MMU_D_ADDR(%g3, /*scratch*/ %g4)
1728	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
1729	done
17300:
1731	TSTAT_CHECK_TL1(1f, %g1, %g2)
17321:
1733	USE_ALTERNATE_GLOBALS(%g5)
1734	GET_MMU_D_TAGACC(%g2 /* tagacc */, %g4 /*scratch*/)
1735	mov	T_DATA_MMU_MISS, %g3	/* arg2 = traptype */
1736	/*
1737	 * g2=tagacc g3.l=type g3.h=0
1738	 */
1739	sethi	%hi(trap), %g1
1740	or	%g1, %lo(trap), %g1
1741	ba,pt	%xcc, sys_trap
1742	mov	-1, %g4
1743	SET_SIZE(sfmmu_kpm_exception)
1744
1745#endif /* lint */
1746
1747#if defined (lint)
1748
1749void
1750sfmmu_tsb_miss(void)
1751{
1752}
1753
1754void
1755sfmmu_kpm_dtsb_miss(void)
1756{
1757}
1758
1759void
1760sfmmu_kpm_dtsb_miss_small(void)
1761{
1762}
1763
1764#else /* lint */
1765
1766
1767#if (CTX_SIZE != (1 << CTX_SZ_SHIFT))
1768#error - size of context struct does not match with CTX_SZ_SHIFT
1769#endif
1770
1771#if (IMAP_SEG != 0)
1772#error - ism_map->ism_seg offset is not zero
1773#endif
1774
1775/*
1776 * Copies ism mapping for this ctx in param "ism" if this is a ISM
1777 * tlb miss and branches to label "ismhit". If this is not an ISM
1778 * process or an ISM tlb miss it falls thru.
1779 *
1780 * Checks to see if the vaddr passed in via tagacc is in an ISM segment for
1781 * this process.
1782 * If so, it will branch to label "ismhit".  If not, it will fall through.
1783 *
1784 * Also hat_unshare() will set the context for this process to INVALID_CONTEXT
1785 * so that any other threads of this process will not try and walk the ism
1786 * maps while they are being changed.
1787 *
1788 * NOTE: We will never have any holes in our ISM maps. sfmmu_share/unshare
1789 *       will make sure of that. This means we can terminate our search on
1790 *       the first zero mapping we find.
1791 *
1792 * Parameters:
1793 * tagacc	= tag access register (vaddr + ctx) (in)
1794 * tsbmiss	= address of tsb miss area (in)
1795 * ismseg	= contents of ism_seg for this ism map (out)
1796 * ismhat	= physical address of imap_ismhat for this ism map (out)
1797 * tmp1		= scratch reg (CLOBBERED)
1798 * tmp2		= scratch reg (CLOBBERED)
1799 * tmp3		= scratch reg (CLOBBERED)
1800 * label:    temporary labels
1801 * ismhit:   label where to jump to if an ism dtlb miss
1802 * exitlabel:label where to jump if hat is busy due to hat_unshare.
1803 */
1804#define ISM_CHECK(tagacc, tsbmiss, ismseg, ismhat, tmp1, tmp2, tmp3 \
1805	label, ismhit)							\
1806	ldx	[tsbmiss + TSBMISS_ISMBLKPA], tmp1 /* tmp1 = &ismblk */	;\
1807	brlz,pt  tmp1, label/**/3		/* exit if -1 */	;\
1808	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0] */	;\
1809label/**/1:								;\
1810	ldxa	[ismhat]ASI_MEM, ismseg	/* ismblk.map[0].ism_seg */	;\
1811	mov	tmp1, tmp3	/* update current ismblkpa head */	;\
1812label/**/2:								;\
1813	brz,pt  ismseg, label/**/3		/* no mapping */	;\
1814	  add	ismhat, IMAP_VB_SHIFT, tmp1 /* tmp1 = vb_shift addr */	;\
1815	lduha	[tmp1]ASI_MEM, tmp1 		/* tmp1 = vb shift*/	;\
1816	srlx	ismseg, tmp1, tmp2		/* tmp2 = vbase */	;\
1817	srlx	tagacc, tmp1, tmp1		/* tmp1 =  va seg*/	;\
1818	sub	tmp1, tmp2, tmp2		/* tmp2 = va - vbase */	;\
1819	add	ismhat, IMAP_SZ_MASK, tmp1 /* tmp1 = sz_mask addr */	;\
1820	lda	[tmp1]ASI_MEM, tmp1		/* tmp1 = sz_mask */	;\
1821	and	ismseg, tmp1, tmp1		/* tmp1 = size */	;\
1822	cmp	tmp2, tmp1		 	/* check va <= offset*/	;\
1823	blu,a,pt  %xcc, ismhit			/* ism hit */		;\
1824	  add	ismhat, IMAP_ISMHAT, ismhat 	/* ismhat = &ism_sfmmu*/ ;\
1825									;\
1826	add	ismhat, ISM_MAP_SZ, ismhat /* ismhat += sizeof(map) */ 	;\
1827	add	tmp3, (IBLK_MAPS + ISM_MAP_SLOTS * ISM_MAP_SZ), tmp1	;\
1828	cmp	ismhat, tmp1						;\
1829	bl,pt	%xcc, label/**/2		/* keep looking  */	;\
1830	  ldxa	[ismhat]ASI_MEM, ismseg	/* ismseg = map[ismhat] */	;\
1831									;\
1832	add	tmp3, IBLK_NEXTPA, tmp1					;\
1833	ldxa	[tmp1]ASI_MEM, tmp1		/* check blk->nextpa */	;\
1834	brgez,pt tmp1, label/**/1		/* continue if not -1*/	;\
1835	  add	tmp1, IBLK_MAPS, ismhat	/* ismhat = &ismblk.map[0]*/	;\
1836label/**/3:
1837
1838/*
1839 * Returns the hme hash bucket (hmebp) given the vaddr, and the hatid
1840 * It also returns the virtual pg for vaddr (ie. vaddr << hmeshift)
1841 * Parameters:
1842 * vaddr = reg containing virtual address
1843 * hatid = reg containing sfmmu pointer
1844 * hmeshift = constant/register to shift vaddr to obtain vapg
1845 * hmebp = register where bucket pointer will be stored
1846 * vapg = register where virtual page will be stored
1847 * tmp1, tmp2 = tmp registers
1848 */
1849
1850
1851#define	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, hmebp,	\
1852	vapg, label, tmp1, tmp2)					\
1853	sllx	tagacc, TAGACC_CTX_LSHIFT, tmp1				;\
1854	brnz,a,pt tmp1, label/**/1					;\
1855	  ld    [tsbarea + TSBMISS_UHASHSZ], hmebp			;\
1856	ld	[tsbarea + TSBMISS_KHASHSZ], hmebp			;\
1857	ba,pt	%xcc, label/**/2					;\
1858	  ldx	[tsbarea + TSBMISS_KHASHSTART], tmp1			;\
1859label/**/1:								;\
1860	ldx	[tsbarea + TSBMISS_UHASHSTART], tmp1			;\
1861label/**/2:								;\
1862	srlx	tagacc, hmeshift, vapg					;\
1863	xor	vapg, hatid, tmp2	/* hatid ^ (vaddr >> shift) */	;\
1864	and	tmp2, hmebp, hmebp	/* index into hme_hash */	;\
1865	mulx	hmebp, HMEBUCK_SIZE, hmebp				;\
1866	add	hmebp, tmp1, hmebp
1867
1868/*
1869 * hashtag includes bspage + hashno (64 bits).
1870 */
1871
1872#define	MAKE_HASHTAG(vapg, hatid, hmeshift, hashno, hblktag)		\
1873	sllx	vapg, hmeshift, vapg					;\
1874	or	vapg, hashno, hblktag
1875
1876/*
1877 * Function to traverse hmeblk hash link list and find corresponding match.
1878 * The search is done using physical pointers. It returns the physical address
1879 * and virtual address pointers to the hmeblk that matches with the tag
1880 * provided.
1881 * Parameters:
1882 * hmebp	= register that points to hme hash bucket, also used as
1883 *		  tmp reg (clobbered)
1884 * hmeblktag	= register with hmeblk tag match
1885 * hatid	= register with hatid
1886 * hmeblkpa	= register where physical ptr will be stored
1887 * hmeblkva	= register where virtual ptr will be stored
1888 * tmp1		= tmp reg
1889 * label: temporary label
1890 */
1891
1892#define	HMEHASH_SEARCH(hmebp, hmeblktag, hatid, hmeblkpa, hmeblkva,	\
1893	tsbarea, tmp1, label)					 	\
1894	add     hmebp, HMEBUCK_NEXTPA, hmeblkpa				;\
1895	ldxa    [hmeblkpa]ASI_MEM, hmeblkpa				;\
1896	add     hmebp, HMEBUCK_HBLK, hmeblkva				;\
1897	ldxa    [hmeblkva]ASI_MEM, hmeblkva				;\
1898	HAT_HSEARCH_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1899label/**/1:								;\
1900	brz,pn	hmeblkva, label/**/2					;\
1901	HAT_HLINK_DBSTAT(hatid, tsbarea, hmebp, tmp1)			;\
1902	add	hmeblkpa, HMEBLK_TAG, hmebp				;\
1903	ldxa	[hmebp]ASI_MEM, tmp1	 /* read 1st part of tag */	;\
1904	add	hmebp, CLONGSIZE, hmebp					;\
1905	ldxa	[hmebp]ASI_MEM, hmebp 	/* read 2nd part of tag */	;\
1906	xor	tmp1, hmeblktag, tmp1					;\
1907	xor	hmebp, hatid, hmebp					;\
1908	or	hmebp, tmp1, hmebp					;\
1909	brz,pn	hmebp, label/**/2	/* branch on hit */		;\
1910	  add	hmeblkpa, HMEBLK_NEXT, hmebp				;\
1911	ldna	[hmebp]ASI_MEM, hmeblkva	/* hmeblk ptr va */	;\
1912	add	hmeblkpa, HMEBLK_NEXTPA, hmebp				;\
1913	ba,pt	%xcc, label/**/1					;\
1914	  ldxa	[hmebp]ASI_MEM, hmeblkpa	/* hmeblk ptr pa */	;\
1915label/**/2:
1916
1917
1918#if ((1 << SFHME_SHIFT) != SFHME_SIZE)
1919#error HMEBLK_TO_HMENT assumes sf_hment is power of 2 in size
1920#endif
1921
1922/*
1923 * HMEBLK_TO_HMENT is a macro that given an hmeblk and a vaddr returns
1924 * he offset for the corresponding hment.
1925 * Parameters:
1926 * vaddr = register with virtual address
1927 * hmeblkpa = physical pointer to hme_blk
1928 * hment = register where address of hment will be stored
1929 * hmentoff = register where hment offset will be stored
1930 * label1 = temporary label
1931 */
1932#define	HMEBLK_TO_HMENT(vaddr, hmeblkpa, hmentoff, tmp1, label1)	\
1933	add	hmeblkpa, HMEBLK_MISC, hmentoff				;\
1934	lda	[hmentoff]ASI_MEM, tmp1 				;\
1935	andcc	tmp1, HBLK_SZMASK, %g0	 /* tmp1 = get_hblk_sz(%g5) */	;\
1936	bnz,a,pn  %icc, label1		/* if sz != TTE8K branch */	;\
1937	  or	%g0, HMEBLK_HME1, hmentoff				;\
1938	srl	vaddr, MMU_PAGESHIFT, tmp1				;\
1939	and	tmp1, NHMENTS - 1, tmp1		/* tmp1 = index */	;\
1940	sllx	tmp1, SFHME_SHIFT, tmp1					;\
1941	add	tmp1, HMEBLK_HME1, hmentoff				;\
1942label1:
1943
1944/*
1945 * GET_TTE is a macro that returns a TTE given a tag and hatid.
1946 *
1947 * tagacc	= tag access register (vaddr + ctx) (in)
1948 * hatid	= sfmmu pointer for TSB miss (in)
1949 * tte		= tte for TLB miss if found, otherwise clobbered (out)
1950 * hmeblkpa	= PA of hment if found, otherwise clobbered (out)
1951 * hmeblkva	= VA of hment if found, otherwise clobbered (out)
1952 * tsbarea	= pointer to the tsbmiss area for this cpu. (in)
1953 * hmentoff	= temporarily stores hment offset (clobbered)
1954 * hmeshift	= constant/register to shift VA to obtain the virtual pfn
1955 *		  for this page size.
1956 * hashno	= constant/register hash number
1957 * label	= temporary label for branching within macro.
1958 * foundlabel	= label to jump to when tte is found.
1959 * suspendlabel= label to jump to when tte is suspended.
1960 * exitlabel	= label to jump to when tte is not found.  The hmebp lock
1961 *		  is still held at this time.
1962 *
1963 * The caller should set up the tsbmiss->scratch[2] field correctly before
1964 * calling this funciton  (aka TSBMISS_SCRATCH + TSBMISS_HATID)
1965 */
1966#define GET_TTE(tagacc, hatid, tte, hmeblkpa, hmeblkva, tsbarea, hmentoff, \
1967		hmeshift, hashno, label, foundlabel, suspendlabel, exitlabel) \
1968									;\
1969	stn	tagacc, [tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)]	;\
1970	stn	hatid, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)]	;\
1971	HMEHASH_FUNC_ASM(tagacc, hatid, tsbarea, hmeshift, tte,		\
1972		hmeblkpa, label/**/5, hmentoff, hmeblkva)		;\
1973									;\
1974	/*								;\
1975	 * tagacc = tagacc						;\
1976	 * hatid = hatid						;\
1977	 * tsbarea = tsbarea						;\
1978	 * tte   = hmebp (hme bucket pointer)				;\
1979	 * hmeblkpa  = vapg  (virtual page)				;\
1980	 * hmentoff, hmeblkva = scratch					;\
1981	 */								;\
1982	MAKE_HASHTAG(hmeblkpa, hatid, hmeshift, hashno, hmentoff)	;\
1983									;\
1984	/*								;\
1985	 * tagacc = tagacc						;\
1986	 * hatid = hatid						;\
1987	 * tte   = hmebp						;\
1988	 * hmeblkpa  = CLOBBERED					;\
1989	 * hmentoff  = htag_bspage & hashno				;\
1990	 * hmeblkva  = scratch						;\
1991	 */								;\
1992	stn	tte, [tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)]	;\
1993	HMELOCK_ENTER(tte, hmeblkpa, hmeblkva, label/**/3, ASI_MEM)	;\
1994	HMEHASH_SEARCH(tte, hmentoff, hatid, hmeblkpa, hmeblkva, 	\
1995		tsbarea, tagacc, label/**/1)				;\
1996	/*								;\
1997	 * tagacc = CLOBBERED						;\
1998	 * tte = CLOBBERED						;\
1999	 * hmeblkpa = hmeblkpa						;\
2000	 * hmeblkva = hmeblkva						;\
2001	 */								;\
2002	brnz,pt	hmeblkva, label/**/4	/* branch if hmeblk found */	;\
2003	  ldn	[tsbarea + (TSBMISS_SCRATCH + TSB_TAGACC)], tagacc	;\
2004	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmeblkva	;\
2005	HMELOCK_EXIT(hmeblkva, hmeblkva, ASI_MEM)  /* drop lock */	;\
2006	ba,pt	%xcc, exitlabel		/* exit if hblk not found */	;\
2007	  nop								;\
2008label/**/4:								;\
2009	/*								;\
2010	 * We have found the hmeblk containing the hment.		;\
2011	 * Now we calculate the corresponding tte.			;\
2012	 *								;\
2013	 * tagacc = tagacc						;\
2014	 * hatid = clobbered						;\
2015	 * tte   = hmebp						;\
2016	 * hmeblkpa  = hmeblkpa						;\
2017	 * hmentoff  = hblktag						;\
2018	 * hmeblkva  = hmeblkva 					;\
2019	 */								;\
2020	HMEBLK_TO_HMENT(tagacc, hmeblkpa, hmentoff, hatid, label/**/2)	;\
2021									;\
2022	add	hmentoff, SFHME_TTE, hmentoff				;\
2023	add	hmeblkpa, hmentoff, hmeblkpa				;\
2024	ldxa	[hmeblkpa]ASI_MEM, tte	/* MMU_READTTE through pa */	;\
2025	add	hmeblkva, hmentoff, hmeblkva				;\
2026	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HATID)], hatid	;\
2027	ldn	[tsbarea + (TSBMISS_SCRATCH + TSBMISS_HMEBP)], hmentoff ;\
2028	HMELOCK_EXIT(hmentoff, hmentoff, ASI_MEM)	/* drop lock */	;\
2029	set	TTE_SUSPEND, hmentoff					;\
2030	TTE_SUSPEND_INT_SHIFT(hmentoff)					;\
2031	btst	tte, hmentoff						;\
2032	bz,pt	%xcc, foundlabel					;\
2033	 nop								;\
2034									;\
2035	/*								;\
2036	 * Mapping is suspended, so goto suspend label.			;\
2037	 */								;\
2038	ba,pt	%xcc, suspendlabel					;\
2039	  nop
2040
2041	/*
2042	 * KERNEL PROTECTION HANDLER
2043	 *
2044	 * g1 = tsb8k pointer register (clobbered)
2045	 * g2 = tag access register (ro)
2046	 * g3 - g7 = scratch registers
2047	 *
2048	 * Note: This function is patched at runtime for performance reasons.
2049	 * 	 Any changes here require sfmmu_patch_ktsb fixed.
2050	 */
2051	ENTRY_NP(sfmmu_kprot_trap)
2052	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2053sfmmu_kprot_patch_ktsb_base:
2054	RUNTIME_PATCH_SETX(%g1, %g6)
2055	/* %g1 = contents of ktsb_base or ktsb_pbase */
2056sfmmu_kprot_patch_ktsb_szcode:
2057	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
2058
2059	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
2060	! %g1 = First TSB entry pointer, as TSB miss handler expects
2061
2062	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
2063sfmmu_kprot_patch_ktsb4m_base:
2064	RUNTIME_PATCH_SETX(%g3, %g6)
2065	/* %g3 = contents of ktsb4m_base or ktsb4m_pbase */
2066sfmmu_kprot_patch_ktsb4m_szcode:
2067	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
2068
2069	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
2070	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
2071
2072	CPU_TSBMISS_AREA(%g6, %g7)
2073	HAT_PERCPU_STAT16(%g6, TSBMISS_KPROTS, %g7)
2074	ba,pt	%xcc, sfmmu_tsb_miss_tt
2075	  nop
2076
2077	/*
2078	 * USER PROTECTION HANDLER
2079	 *
2080	 * g1 = tsb8k pointer register (ro)
2081	 * g2 = tag access register (ro)
2082	 * g3 = faulting context (clobbered, currently not used)
2083	 * g4 - g7 = scratch registers
2084	 */
2085	ALTENTRY(sfmmu_uprot_trap)
2086#ifdef sun4v
2087	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2088	/* %g1 = first TSB entry ptr now, %g2 preserved */
2089
2090	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
2091	brlz,pt %g3, 9f			/* check for 2nd TSB */
2092	  mov	%g0, %g3		/* clear second tsbe ptr */
2093
2094	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2095	/* %g3 = second TSB entry ptr now, %g2 preserved */
2096
2097#else /* sun4v */
2098
2099	brgez,pt %g1, 9f		/* check for 2nd TSB */
2100	  mov	%g0, %g3		/* clear second tsbe ptr */
2101
2102	mov	%g2, %g7
2103	GET_2ND_TSBE_PTR(%g7, %g1, %g3, %g4, %g5, sfmmu_uprot)
2104	/* %g3 = second TSB entry ptr now, %g7 clobbered */
2105	mov	%g1, %g7
2106	GET_1ST_TSBE_PTR(%g7, %g1, %g5, sfmmu_uprot)
2107
2108#endif /* sun4v */
21099:
2110	CPU_TSBMISS_AREA(%g6, %g7)
2111	HAT_PERCPU_STAT16(%g6, TSBMISS_UPROTS, %g7)
2112	ba,pt	%xcc, sfmmu_tsb_miss_tt		/* branch TSB miss handler */
2113	  nop
2114
2115	/*
2116	 * Kernel 8K page iTLB miss.  We also get here if we took a
2117	 * fast instruction access mmu miss trap while running in
2118	 * invalid context.
2119	 *
2120	 * %g1 = 8K TSB pointer register (not used, clobbered)
2121	 * %g2 = tag access register (used)
2122	 * %g3 = faulting context id (used)
2123	 * %g7 = 4M virtual page number for tag matching  (used)
2124	 */
2125	.align	64
2126	ALTENTRY(sfmmu_kitlb_miss)
2127	brnz,pn %g3, tsb_tl0_noctxt
2128	  nop
2129
2130	/* kernel miss */
2131	/* get kernel tsb pointer */
2132	/* we patch the next set of instructions at run time */
2133	/* NOTE: any changes here require sfmmu_patch_ktsb fixed */
2134iktsbbase:
2135	RUNTIME_PATCH_SETX(%g4, %g5)
2136	/* %g4 = contents of ktsb_base or ktsb_pbase */
2137
2138iktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2139	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2140	or	%g4, %g1, %g1			! form tsb ptr
2141	ldda	[%g1]RUNTIME_PATCH, %g4		! %g4 = tag, %g5 = data
2142	cmp	%g4, %g7
2143	bne,pn	%xcc, sfmmu_tsb_miss_tt		! branch on miss
2144	  andcc %g5, TTE_EXECPRM_INT, %g0	! check exec bit
2145	bz,pn	%icc, exec_fault
2146	  nop
2147	TT_TRACE(trace_tsbhit)			! 2 instr traptrace
2148	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2149	retry
2150
2151	/*
2152	 * Kernel dTLB miss.  We also get here if we took a fast data
2153	 * access mmu miss trap while running in invalid context.
2154	 *
2155	 * Note: for now we store kpm TTEs in the kernel TSB as usual.
2156	 *	We select the TSB miss handler to branch to depending on
2157	 *	the virtual address of the access.  In the future it may
2158	 *	be desirable to separate kpm TTEs into their own TSB,
2159	 *	in which case all that needs to be done is to set
2160	 *	kpm_tsbbase/kpm_tsbsz to point to the new TSB and branch
2161	 *	early in the miss if we detect a kpm VA to a new handler.
2162	 *
2163	 * %g1 = 8K TSB pointer register (not used, clobbered)
2164	 * %g2 = tag access register (used)
2165	 * %g3 = faulting context id (used)
2166	 */
2167	.align	64
2168	ALTENTRY(sfmmu_kdtlb_miss)
2169	brnz,pn	%g3, tsb_tl0_noctxt		/* invalid context? */
2170	  nop
2171
2172	/* Gather some stats for kpm misses in the TLB. */
2173	/* KPM_TLBMISS_STAT_INCR(tagacc, val, tsbma, tmp1, label) */
2174	KPM_TLBMISS_STAT_INCR(%g2, %g4, %g5, %g6, kpmtlbm_stat_out)
2175
2176	/*
2177	 * Get first TSB offset and look for 8K/64K/512K mapping
2178	 * using the 8K virtual page as the index.
2179	 *
2180	 * We patch the next set of instructions at run time;
2181	 * any changes here require sfmmu_patch_ktsb changes too.
2182	 */
2183dktsbbase:
2184	RUNTIME_PATCH_SETX(%g7, %g6)
2185	/* %g7 = contents of ktsb_base or ktsb_pbase */
2186
2187dktsb:	sllx	%g2, 64-(TAGACC_SHIFT + TSB_START_SIZE + RUNTIME_PATCH), %g1
2188	srlx	%g1, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g1
2189
2190	/*
2191	 * At this point %g1 is our index into the TSB.
2192	 * We just masked off enough bits of the VA depending
2193	 * on our TSB size code.
2194	 */
2195	ldda	[%g7 + %g1]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2196	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2197	cmp	%g6, %g4			! compare tag
2198	bne,pn	%xcc, dktsb4m_kpmcheck_small
2199	  add	%g7, %g1, %g1			/* form tsb ptr */
2200	TT_TRACE(trace_tsbhit)
2201	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2202	/* trapstat expects tte in %g5 */
2203	retry
2204
2205	/*
2206	 * If kpm is using large pages, the following instruction needs
2207	 * to be patched to a nop at boot time (by sfmmu_kpm_patch_tsbm)
2208	 * so that we will probe the 4M TSB regardless of the VA.  In
2209	 * the case kpm is using small pages, we know no large kernel
2210	 * mappings are located above 0x80000000.00000000 so we skip the
2211	 * probe as an optimization.
2212	 */
2213dktsb4m_kpmcheck_small:
2214	brlz,pn %g2, sfmmu_kpm_dtsb_miss_small
2215	  /* delay slot safe, below */
2216
2217	/*
2218	 * Get second TSB offset and look for 4M mapping
2219	 * using 4M virtual page as the TSB index.
2220	 *
2221	 * Here:
2222	 * %g1 = 8K TSB pointer.  Don't squash it.
2223	 * %g2 = tag access register (we still need it)
2224	 */
2225	srlx	%g2, MMU_PAGESHIFT4M, %g3
2226
2227	/*
2228	 * We patch the next set of instructions at run time;
2229	 * any changes here require sfmmu_patch_ktsb changes too.
2230	 */
2231dktsb4mbase:
2232	RUNTIME_PATCH_SETX(%g7, %g6)
2233	/* %g7 = contents of ktsb4m_base or ktsb4m_pbase */
2234dktsb4m:
2235	sllx	%g3, 64-(TSB_START_SIZE + RUNTIME_PATCH), %g3
2236	srlx	%g3, 64-(TSB_START_SIZE + TSB_ENTRY_SHIFT + RUNTIME_PATCH), %g3
2237
2238	/*
2239	 * At this point %g3 is our index into the TSB.
2240	 * We just masked off enough bits of the VA depending
2241	 * on our TSB size code.
2242	 */
2243	ldda	[%g7 + %g3]RUNTIME_PATCH, %g4	! %g4 = tag, %g5 = data
2244	srlx	%g2, TAG_VALO_SHIFT, %g6	! make tag to compare
2245	cmp	%g6, %g4			! compare tag
2246
2247dktsb4m_tsbmiss:
2248	bne,pn	%xcc, dktsb4m_kpmcheck
2249	  add	%g7, %g3, %g3			! %g3 = kernel second TSB ptr
2250	TT_TRACE(trace_tsbhit)
2251	/* we don't check TTE size here since we assume 4M TSB is separate */
2252	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2253	/* trapstat expects tte in %g5 */
2254	retry
2255
2256	/*
2257	 * So, we failed to find a valid TTE to match the faulting
2258	 * address in either TSB.  There are a few cases that could land
2259	 * us here:
2260	 *
2261	 * 1) This is a kernel VA below 0x80000000.00000000.  We branch
2262	 *    to sfmmu_tsb_miss_tt to handle the miss.
2263	 * 2) We missed on a kpm VA, and we didn't find the mapping in the
2264	 *    4M TSB.  Let segkpm handle it.
2265	 *
2266	 * Note that we shouldn't land here in the case of a kpm VA when
2267	 * kpm_smallpages is active -- we handled that case earlier at
2268	 * dktsb4m_kpmcheck_small.
2269	 *
2270	 * At this point:
2271	 *  g1 = 8K-indexed primary TSB pointer
2272	 *  g2 = tag access register
2273	 *  g3 = 4M-indexed secondary TSB pointer
2274	 */
2275dktsb4m_kpmcheck:
2276	cmp	%g2, %g0
2277	bl,pn	%xcc, sfmmu_kpm_dtsb_miss
2278	  nop
2279	ba,a,pt	%icc, sfmmu_tsb_miss_tt
2280	  nop
2281
2282#ifdef sun4v
2283	/*
2284	 * User instruction miss w/ single TSB.
2285	 * The first probe covers 8K, 64K, and 512K page sizes,
2286	 * because 64K and 512K mappings are replicated off 8K
2287	 * pointer.
2288	 *
2289	 * g1 = tsb8k pointer register
2290	 * g2 = tag access register
2291	 * g3 - g6 = scratch registers
2292	 * g7 = TSB tag to match
2293	 */
2294	.align	64
2295	ALTENTRY(sfmmu_uitlb_fastpath)
2296
2297	SETUP_UTSB_ATOMIC_ASI(%g4, %g5)
2298	PROBE_1ST_ITSB(%g1, %g7, uitlb_fast_8k_probefail)
2299	/* g4 - g5 = clobbered by PROBE_1ST_ITSB */
2300	ba,pn	%xcc, sfmmu_tsb_miss_tt
2301	  mov	%g0, %g3
2302
2303	/*
2304	 * User data miss w/ single TSB.
2305	 * The first probe covers 8K, 64K, and 512K page sizes,
2306	 * because 64K and 512K mappings are replicated off 8K
2307	 * pointer.
2308	 *
2309	 * g1 = tsb8k pointer register
2310	 * g2 = tag access register
2311	 * g3 - g6 = scratch registers
2312	 * g7 = TSB tag to match
2313	 */
2314	.align 64
2315	ALTENTRY(sfmmu_udtlb_fastpath)
2316
2317	SETUP_UTSB_ATOMIC_ASI(%g4, %g6)
2318	PROBE_1ST_DTSB(%g1, %g7, udtlb_fast_8k_probefail)
2319	/* g4 - g5 = clobbered by PROBE_1ST_DTSB */
2320	ba,pn	%xcc, sfmmu_tsb_miss_tt
2321	  mov	%g0, %g3
2322
2323#endif /* sun4v */
2324
2325	/*
2326	 * User instruction miss w/ multiple TSBs.
2327	 * The first probe covers 8K, 64K, and 512K page sizes,
2328	 * because 64K and 512K mappings are replicated off 8K
2329	 * pointer.  Second probe covers 4M page size only.
2330	 *
2331	 * Just like sfmmu_udtlb_slowpath, except:
2332	 *   o Uses ASI_ITLB_IN
2333	 *   o checks for execute permission
2334	 *   o No ISM prediction.
2335	 *
2336	 * g1 = tsb8k pointer register
2337	 * g2 = tag access register
2338	 * g3 - g6 = scratch registers
2339	 * g7 = TSB tag to match
2340	 */
2341	.align	64
2342	ALTENTRY(sfmmu_uitlb_slowpath)
2343
2344#ifdef sun4v
2345	SETUP_UTSB_ATOMIC_ASI(%g4, %g5)
2346	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2347
2348	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2349	/* g4 - g5 = clobbered here */
2350
2351	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2352	/* g1 = first TSB pointer, g3 = second TSB pointer */
2353	srlx	%g2, TAG_VALO_SHIFT, %g7
2354	PROBE_2ND_ITSB(%g3, %g7)
2355	/* NOT REACHED */
2356#else /* sun4v */
2357	mov	%g1, %g3	/* save tsb8k reg in %g3 */
2358	SETUP_UTSB_ATOMIC_ASI(%g4, %g5)
2359	GET_1ST_TSBE_PTR(%g3, %g1, %g5, sfmmu_uitlb)
2360
2361	PROBE_1ST_ITSB(%g1, %g7, uitlb_8k_probefail)
2362	/* g4 - g5 = clobbered here */
2363
2364	mov	%g2, %g6	/* GET_2ND_TSBE_PTR clobbers tagacc */
2365	mov	%g3, %g7	/* copy tsb8k reg in %g7 */
2366	GET_2ND_TSBE_PTR(%g6, %g7, %g3, %g4, %g5, sfmmu_uitlb)
2367	/* g1 = first TSB pointer, g3 = second TSB pointer */
2368	srlx	%g2, TAG_VALO_SHIFT, %g7
2369	PROBE_2ND_ITSB(%g3, %g7)
2370	/* NOT REACHED */
2371#endif /* sun4v */
2372
2373	/*
2374	 * User data miss w/ multiple TSBs.
2375	 * The first probe covers 8K, 64K, and 512K page sizes,
2376	 * because 64K and 512K mappings are replicated off 8K
2377	 * pointer.  Second probe covers 4M page size only.
2378	 *
2379	 * We consider probing for 4M pages first if the VA falls
2380	 * in a range that's likely to be ISM.
2381	 *
2382	 * g1 = tsb8k pointer register
2383	 * g2 = tag access register
2384	 * g3 - g6 = scratch registers
2385	 * g7 = TSB tag to match
2386	 */
2387	.align 64
2388	ALTENTRY(sfmmu_udtlb_slowpath)
2389
2390	SETUP_UTSB_ATOMIC_ASI(%g4, %g6)
2391
2392	/*
2393	 * Check for ISM.  If it exists, look for 4M mappings in the second TSB
2394	 * first, then probe for other mappings in the first TSB if that fails.
2395	 */
2396	srax	%g2, PREDISM_BASESHIFT, %g6	/* g6 > 0 : ISM predicted */
2397	brgz,pn %g6, udtlb_miss_probesecond	/* check for ISM */
2398	  mov	%g1, %g3
2399
2400udtlb_miss_probefirst:
2401	/*
2402	 * g1 = 8K TSB pointer register
2403	 * g2 = tag access register
2404	 * g3 = (potentially) second TSB entry ptr
2405	 * g6 = ism pred.
2406	 * g7 = vpg_4m
2407	 */
2408#ifdef sun4v
2409	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
2410	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2411
2412	/*
2413	 * Here:
2414	 *   g1 = first TSB pointer
2415	 *   g2 = tag access reg
2416	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2417	 */
2418	brgz,pn	%g6, sfmmu_tsb_miss_tt
2419	  nop
2420#else /* sun4v */
2421	mov	%g1, %g4
2422	GET_1ST_TSBE_PTR(%g4, %g1, %g5, sfmmu_udtlb)
2423	PROBE_1ST_DTSB(%g1, %g7, udtlb_first_probefail)
2424
2425	/*
2426	 * Here:
2427	 *   g1 = first TSB pointer
2428	 *   g2 = tag access reg
2429	 *   g3 = second TSB ptr IFF ISM pred. (else don't care)
2430	 */
2431	brgz,pn	%g6, sfmmu_tsb_miss_tt
2432	  nop
2433	ldxa	[%g0]ASI_DMMU_TSB_8K, %g3
2434	/* fall through in 8K->4M probe order */
2435#endif /* sun4v */
2436
2437udtlb_miss_probesecond:
2438	/*
2439	 * Look in the second TSB for the TTE
2440	 * g1 = First TSB entry ptr if !ISM pred, TSB8K ptr reg if ISM pred.
2441	 * g2 = tag access reg
2442	 * g3 = 8K TSB pointer register
2443	 * g6 = ism pred.
2444	 * g7 = vpg_4m
2445	 */
2446#ifdef sun4v
2447	/* GET_2ND_TSBE_PTR(tagacc, tsbe_ptr, tmp1, tmp2) */
2448	/* tagacc (%g2) not destroyed */
2449	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
2450	/* %g2 is okay, no need to reload, %g3 = second tsbe ptr */
2451#else
2452	mov	%g3, %g7
2453	GET_2ND_TSBE_PTR(%g2, %g7, %g3, %g4, %g5, sfmmu_udtlb)
2454	/* %g2 clobbered, %g3 =second tsbe ptr */
2455	mov	MMU_TAG_ACCESS, %g2
2456	ldxa	[%g2]ASI_DMMU, %g2
2457#endif
2458
2459	srlx	%g2, TAG_VALO_SHIFT, %g7
2460	PROBE_2ND_DTSB(%g3, %g7, udtlb_4m_probefail)
2461	/* g4 - g5 = clobbered here; %g7 still vpg_4m at this point */
2462	brgz,pn	%g6, udtlb_miss_probefirst
2463	  nop
2464
2465	/* fall through to sfmmu_tsb_miss_tt */
2466
2467	ALTENTRY(sfmmu_tsb_miss_tt)
2468	TT_TRACE(trace_tsbmiss)
2469	/*
2470	 * We get here if there is a TSB miss OR a write protect trap.
2471	 *
2472	 * g1 = First TSB entry pointer
2473	 * g2 = tag access register
2474	 * g3 = 4M TSB entry pointer; NULL if no 2nd TSB
2475	 * g4 - g7 = scratch registers
2476	 */
2477
2478	ALTENTRY(sfmmu_tsb_miss)
2479
2480	/*
2481	 * If trapstat is running, we need to shift the %tpc and %tnpc to
2482	 * point to trapstat's TSB miss return code (note that trapstat
2483	 * itself will patch the correct offset to add).
2484	 */
2485	rdpr	%tl, %g7
2486	cmp	%g7, 1
2487	ble,pt	%xcc, 0f
2488	  sethi	%hi(KERNELBASE), %g6
2489	rdpr	%tpc, %g7
2490	or	%g6, %lo(KERNELBASE), %g6
2491	cmp	%g7, %g6
2492	bgeu,pt	%xcc, 0f
2493	/* delay slot safe */
2494
2495	ALTENTRY(tsbmiss_trapstat_patch_point)
2496	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
2497	wrpr	%g7, %tpc
2498	add	%g7, 4, %g7
2499	wrpr	%g7, %tnpc
25000:
2501	CPU_TSBMISS_AREA(%g6, %g7)
2502
2503	stn	%g1, [%g6 + TSBMISS_TSBPTR]	/* save first tsb pointer */
2504	stn	%g3, [%g6 + TSBMISS_TSBPTR4M]	/* save second tsb pointer */
2505
2506	sllx	%g2, TAGACC_CTX_LSHIFT, %g3
2507	brz,a,pn %g3, 1f			/* skip ahead if kernel */
2508	  ldn	[%g6 + TSBMISS_KHATID], %g7
2509	srlx	%g3, TAGACC_CTX_LSHIFT, %g3	/* g3 = ctxnum */
2510	ldn	[%g6 + TSBMISS_UHATID], %g7     /* g7 = hatid */
2511
2512	HAT_PERCPU_STAT32(%g6, TSBMISS_UTSBMISS, %g5)
2513
2514	cmp	%g3, INVALID_CONTEXT
2515	be,pn	%icc, tsb_tl0_noctxt		/* no ctx miss exception */
2516	  stn	%g7, [%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)]
2517
2518	ISM_CHECK(%g2, %g6, %g3, %g4, %g5, %g7, %g1, tsb_l1, tsb_ism)
2519	/*
2520	 * The miss wasn't in an ISM segment.
2521	 *
2522	 * %g1 %g3, %g4, %g5, %g7 all clobbered
2523	 * %g2 = tag access (vaddr + ctx)
2524	 */
2525
2526	ba,pt	%icc, 2f
2527	  ldn	[%g6 + (TSBMISS_SCRATCH + TSBMISS_HATID)], %g7
2528
25291:
2530	HAT_PERCPU_STAT32(%g6, TSBMISS_KTSBMISS, %g5)
2531	/*
2532	 * 8K and 64K hash.
2533	 */
25342:
2535
2536	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2537		MMU_PAGESHIFT64K, TTE64K, tsb_l8K, tsb_checktte,
2538		sfmmu_suspend_tl, tsb_512K)
2539	/* NOT REACHED */
2540
2541tsb_512K:
2542	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2543	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2544	brz,pn	%g5, 3f
2545	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2546	and	%g4, HAT_512K_FLAG, %g5
2547
2548	/*
2549	 * Note that there is a small window here where we may have
2550	 * a 512k page in the hash list but have not set the HAT_512K_FLAG
2551	 * flag yet, so we will skip searching the 512k hash list.
2552	 * In this case we will end up in pagefault which will find
2553	 * the mapping and return.  So, in this instance we will end up
2554	 * spending a bit more time resolving this TSB miss, but it can
2555	 * only happen once per process and even then, the chances of that
2556	 * are very small, so it's not worth the extra overhead it would
2557	 * take to close this window.
2558	 */
2559	brz,pn	%g5, tsb_4M
2560	  nop
25613:
2562	/*
2563	 * 512K hash
2564	 */
2565
2566	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2567		MMU_PAGESHIFT512K, TTE512K, tsb_l512K, tsb_checktte,
2568		sfmmu_suspend_tl, tsb_4M)
2569	/* NOT REACHED */
2570
2571tsb_4M:
2572	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2573	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2574	brz,pn	%g5, 4f
2575	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2576	and	%g4, HAT_4M_FLAG, %g5
2577	brz,pn	%g5, tsb_32M
2578	  nop
25794:
2580	/*
2581	 * 4M hash
2582	 */
2583
2584	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2585		MMU_PAGESHIFT4M, TTE4M, tsb_l4M, tsb_checktte,
2586		sfmmu_suspend_tl, tsb_32M)
2587	/* NOT REACHED */
2588
2589tsb_32M:
2590#ifndef sun4v
2591	GET_CPU_IMPL(%g5)
2592	cmp	%g5, PANTHER_IMPL
2593	bne,pt	%xcc, tsb_pagefault
2594	  nop
2595#endif
2596
2597	ldn	[%g6 + (TSBMISS_SCRATCH + TSB_TAGACC)], %g3
2598	sllx	%g3, TAGACC_CTX_LSHIFT, %g5
2599#ifdef sun4v
2600	brz,pn	%g5, 6f
2601#else
2602	brz,pn	%g5, tsb_pagefault
2603#endif
2604	  lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2605	and	%g4, HAT_32M_FLAG, %g5
2606	brz,pn	%g5, tsb_256M
2607	  nop
26085:
2609	/*
2610	 * 32M hash
2611	 */
2612
2613	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2614		MMU_PAGESHIFT32M, TTE32M, tsb_l32M, tsb_checktte,
2615		sfmmu_suspend_tl, tsb_256M)
2616	/* NOT REACHED */
2617
2618tsb_256M:
2619	lduh	[%g6 + TSBMISS_HATFLAGS], %g4
2620	and	%g4, HAT_256M_FLAG, %g5
2621	brz,pn	%g5, tsb_pagefault
2622	  nop
26236:
2624	/*
2625	 * 256M hash
2626	 */
2627
2628	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1,
2629	    MMU_PAGESHIFT256M, TTE256M, tsb_l256M, tsb_checktte,
2630	    sfmmu_suspend_tl, tsb_pagefault)
2631	/* NOT REACHED */
2632
2633tsb_checktte:
2634	/*
2635	 * g3 = tte
2636	 * g4 = tte pa
2637	 * g5 = tte va
2638	 * g6 = tsbmiss area
2639	 */
2640	brgez,pn %g3, tsb_pagefault	/* if tte invalid branch */
2641	  nop
2642
2643tsb_validtte:
2644	/*
2645	 * Set ref/mod bits if this is a prot trap.  Usually, it isn't.
2646	 */
2647	rdpr	%tt, %g7
2648	cmp	%g7, FAST_PROT_TT
2649	bne,pt	%icc, 4f
2650	  nop
2651
2652	TTE_SET_REFMOD_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_refmod,
2653	    tsb_protfault)
2654
2655	rdpr	%tt, %g5
2656	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
2657	ba,pt	%xcc, tsb_update_tl1
2658	  nop
2659
26604:
2661	/*
2662	 * If ITLB miss check exec bit.
2663	 * If not set treat as invalid TTE.
2664	 */
2665	cmp     %g7, T_INSTR_MMU_MISS
2666	be,pn	%icc, 5f
2667	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
2668	cmp     %g7, FAST_IMMU_MISS_TT
2669	bne,pt %icc, 3f
2670	  andcc   %g3, TTE_EXECPRM_INT, %g0	/* check execute bit is set */
26715:
2672	bz,pn %icc, tsb_protfault
2673	  nop
2674
26753:
2676	/*
2677	 * Set reference bit if not already set
2678	 */
2679	TTE_SET_REF_ML(%g3, %g4, %g5, %g6, %g7, tsb_lset_ref)
2680
2681	/*
2682	 * Now, load into TSB/TLB.  At this point:
2683	 * g3 = tte
2684	 * g4 = patte
2685	 * g6 = tsbmiss area
2686	 */
2687	rdpr	%tt, %g5
2688#ifdef sun4v
2689	MMU_FAULT_STATUS_AREA(%g2)
2690	cmp	%g5, T_INSTR_MMU_MISS
2691	be,a,pt	%icc, 9f
2692	  nop
2693	cmp	%g5, FAST_IMMU_MISS_TT
2694	be,a,pt	%icc, 9f
2695	  nop
2696	add	%g2, MMFSA_D_, %g2
26979:
2698	ldx	[%g2 + MMFSA_CTX_], %g7
2699	sllx	%g7, TTARGET_CTX_SHIFT, %g7
2700	ldx	[%g2 + MMFSA_ADDR_], %g2
2701	srlx	%g2, TTARGET_VA_SHIFT, %g2
2702	or	%g2, %g7, %g2
2703#else
2704	cmp	%g5, FAST_IMMU_MISS_TT
2705	be,a,pt	%icc, tsb_update_tl1
2706	  ldxa	[%g0]ASI_IMMU, %g2
2707	ldxa	[%g0]ASI_DMMU, %g2
2708#endif
2709tsb_update_tl1:
2710	srlx	%g2, TTARGET_CTX_SHIFT, %g7
2711	brz,pn	%g7, tsb_kernel
2712#ifdef sun4v
2713	  and	%g3, TTE_SZ_BITS, %g7	! assumes TTE_SZ_SHFT is 0
2714#else
2715	  srlx	%g3, TTE_SZ_SHFT, %g7
2716#endif
2717
2718tsb_user:
2719#ifdef sun4v
2720	cmp	%g7, TTE4M
2721	bge,pn	%icc, tsb_user4m
2722	  nop
2723#else
2724	cmp	%g7, TTESZ_VALID | TTE4M
2725	be,pn	%icc, tsb_user4m
2726	  srlx	%g3, TTE_SZ2_SHFT, %g7
2727	andcc	%g7, TTE_SZ2_BITS, %g7		! check 32/256MB
2728	bnz,a,pn %icc, tsb_user4m
2729	  nop
2730#endif
2731
2732tsb_user8k:
2733	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = first TSB ptr
2734
2735#ifndef sun4v
2736	mov	ASI_N, %g7	! user TSBs always accessed by VA
2737	mov	%g7, %asi
2738#endif /* sun4v */
2739
2740	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 5)
2741
2742#ifdef sun4v
2743	cmp	%g5, T_INSTR_MMU_MISS
2744	be,a,pn	%xcc, 9f
2745	  mov	%g3, %g5
2746#endif /* sun4v */
2747	cmp	%g5, FAST_IMMU_MISS_TT
2748	be,pn	%xcc, 9f
2749	  mov	%g3, %g5
2750
2751	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2752	! trapstat wants TTE in %g5
2753	retry
27549:
2755	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2756	! trapstat wants TTE in %g5
2757	retry
2758
2759tsb_user4m:
2760	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1		/* g1 = tsbp */
2761	brz,pn	%g1, 5f	/* Check to see if we have 2nd TSB programmed */
2762	  nop
2763
2764#ifndef sun4v
2765	mov	ASI_N, %g7	! user TSBs always accessed by VA
2766	mov	%g7, %asi
2767#endif
2768
2769	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 6)
2770
27715:
2772#ifdef sun4v
2773	cmp	%g5, T_INSTR_MMU_MISS
2774	be,a,pn	%xcc, 9f
2775	  mov	%g3, %g5
2776#endif /* sun4v */
2777	cmp	%g5, FAST_IMMU_MISS_TT
2778	be,pn	%xcc, 9f
2779	mov	%g3, %g5
2780
2781	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2782	! trapstat wants TTE in %g5
2783	retry
27849:
2785	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2786	! trapstat wants TTE in %g5
2787	retry
2788
2789tsb_kernel:					! no 32M or 256M support
2790#ifdef sun4v
2791	cmp	%g7, TTE4M
2792#else
2793	cmp	%g7, TTESZ_VALID | TTE4M
2794#endif
2795	be,pn	%icc, 5f
2796	  nop
2797	ldn	[%g6 + TSBMISS_TSBPTR], %g1	! g1 = 8k tsbptr
2798	ba,pt	%xcc, 6f
2799	  nop
28005:
2801	ldn	[%g6 + TSBMISS_TSBPTR4M], %g1	! g1 = 4m tsbptr
2802	brz,pn	%g1, 3f		/* skip programming if 4m TSB ptr is NULL */
2803	  nop
28046:
2805#ifndef sun4v
2806tsb_kernel_patch_asi:
2807	or	%g0, RUNTIME_PATCH, %g6
2808	mov	%g6, %asi	! XXX avoid writing to %asi !!
2809#endif
2810	TSB_UPDATE_TL(%g1, %g3, %g2, %g4, %g7, %g6, 7)
28113:
2812#ifdef sun4v
2813	cmp	%g5, T_INSTR_MMU_MISS
2814	be,a,pn	%icc, 1f
2815	  mov	%g3, %g5			! trapstat wants TTE in %g5
2816#endif /* sun4v */
2817	cmp	%g5, FAST_IMMU_MISS_TT
2818	be,pn	%icc, 1f
2819	  mov	%g3, %g5			! trapstat wants TTE in %g5
2820	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2821	! trapstat wants TTE in %g5
2822	retry
28231:
2824	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)
2825	! trapstat wants TTE in %g5
2826	retry
2827
2828tsb_ism:
2829	/*
2830	 * This is an ISM [i|d]tlb miss.  We optimize for largest
2831	 * page size down to smallest.
2832	 *
2833	 * g2 = vaddr + ctx	aka tag access register
2834	 * g3 = ismmap->ism_seg
2835	 * g4 = physical address of ismmap->ism_sfmmu
2836	 * g6 = tsbmiss area
2837	 */
2838	ldna	[%g4]ASI_MEM, %g7		/* g7 = ism hatid */
2839	brz,a,pn %g7, ptl1_panic		/* if zero jmp ahead */
2840	  mov	PTL1_BAD_ISM, %g1
2841						/* g5 = pa of imap_vb_shift */
2842	sub	%g4, (IMAP_ISMHAT - IMAP_VB_SHIFT), %g5
2843	lduha	[%g5]ASI_MEM, %g4		/* g4 = imap_vb_shift */
2844	srlx	%g3, %g4, %g3			/* clr size field */
2845	set	TAGACC_CTX_MASK, %g1		/* mask off ctx number */
2846	sllx	%g3, %g4, %g3			/* g3 = ism vbase */
2847	and	%g2, %g1, %g4			/* g4 = ctx number */
2848	andn	%g2, %g1, %g1			/* g1 = tlb miss vaddr */
2849	sub	%g1, %g3, %g2			/* g2 = offset in ISM seg */
2850	or	%g2, %g4, %g2			/* g2 = tagacc (vaddr + ctx) */
2851
2852	/*
2853	 * ISM pages are always locked down.
2854	 * If we can't find the tte then pagefault
2855	 * and let the spt segment driver resovle it.
2856	 *
2857	 * g2 = ISM vaddr (offset in ISM seg)
2858	 * g6 = tsb miss area
2859	 * g7 = ISM hatid
2860	 */
2861	sub	%g5, (IMAP_VB_SHIFT - IMAP_HATFLAGS), %g5
2862	lduha	[%g5]ASI_MEM, %g4		/* g5 = pa of imap_hatflags */
2863	and	%g4, HAT_4M_FLAG, %g5		/* g4 = imap_hatflags */
2864	brnz,pt	%g5, tsb_ism_4M			/* branch if 4M pages */
2865	  nop
2866
2867tsb_ism_32M:
2868	and	%g4, HAT_32M_FLAG, %g5		/* check default 32M next */
2869	brz,pn	%g5, tsb_ism_256M
2870	  nop
2871
2872	/*
2873	 * 32M hash.
2874	 */
2875
2876	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT32M,
2877	    TTE32M, tsb_ism_l32M, tsb_ism_32M_found, sfmmu_suspend_tl,
2878	    tsb_ism_4M)
2879	/* NOT REACHED */
2880
2881tsb_ism_32M_found:
2882	brlz,pt %g3, tsb_validtte
2883	  nop
2884	ba,pt	%xcc, tsb_ism_4M
2885	  nop
2886
2887tsb_ism_256M:
2888	and	%g4, HAT_256M_FLAG, %g5		/* 256M is last resort */
2889	brz,a,pn %g5, ptl1_panic
2890	  mov	PTL1_BAD_ISM, %g1
2891
2892	/*
2893	 * 256M hash.
2894	 */
2895	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT256M,
2896	    TTE256M, tsb_ism_l256M, tsb_ism_256M_found, sfmmu_suspend_tl,
2897	    tsb_ism_4M)
2898
2899tsb_ism_256M_found:
2900	brlz,pt %g3, tsb_validtte
2901	  nop
2902
2903tsb_ism_4M:
2904	/*
2905	 * 4M hash.
2906	 */
2907	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT4M,
2908	    TTE4M, tsb_ism_l4M, tsb_ism_4M_found, sfmmu_suspend_tl,
2909	    tsb_ism_8K)
2910	/* NOT REACHED */
2911
2912tsb_ism_4M_found:
2913	brlz,pt %g3, tsb_validtte
2914	  nop
2915
2916tsb_ism_8K:
2917	/*
2918	 * 8K and 64K hash.
2919	 */
2920
2921	GET_TTE(%g2, %g7, %g3, %g4, %g5, %g6, %g1, MMU_PAGESHIFT64K,
2922	    TTE64K, tsb_ism_l8K, tsb_ism_8K_found, sfmmu_suspend_tl,
2923	    tsb_pagefault)
2924	/* NOT REACHED */
2925
2926tsb_ism_8K_found:
2927	brlz,pt	%g3, tsb_validtte
2928	  nop
2929
2930tsb_pagefault:
2931	rdpr	%tt, %g7
2932	cmp	%g7, FAST_PROT_TT
2933	be,a,pn	%icc, tsb_protfault
2934	  wrpr	%g0, FAST_DMMU_MISS_TT, %tt
2935
2936tsb_protfault:
2937	/*
2938	 * we get here if we couldn't find a valid tte in the hash.
2939	 *
2940	 * If user and we are at tl>1 we go to window handling code.
2941	 *
2942	 * If kernel and the fault is on the same page as our stack
2943	 * pointer, then we know the stack is bad and the trap handler
2944	 * will fail, so we call ptl1_panic with PTL1_BAD_STACK.
2945	 *
2946	 * If this is a kernel trap and tl>1, panic.
2947	 *
2948	 * Otherwise we call pagefault.
2949	 */
2950	cmp	%g7, FAST_IMMU_MISS_TT
2951#ifdef sun4v
2952	MMU_FAULT_STATUS_AREA(%g4)
2953	ldx	[%g4 + MMFSA_I_CTX], %g5
2954	ldx	[%g4 + MMFSA_D_CTX], %g4
2955	move	%icc, %g5, %g4
2956	cmp	%g7, T_INSTR_MMU_MISS
2957	move	%icc, %g5, %g4
2958#else
2959	mov	MMU_TAG_ACCESS, %g4
2960	ldxa	[%g4]ASI_DMMU, %g2
2961	ldxa	[%g4]ASI_IMMU, %g5
2962	move	%icc, %g5, %g2
2963	cmp	%g7, T_INSTR_MMU_MISS
2964	move	%icc, %g5, %g2
2965	sllx	%g2, TAGACC_CTX_LSHIFT, %g4
2966#endif
2967	brnz,pn	%g4, 3f				/* skip if not kernel */
2968	  rdpr	%tl, %g5
2969
2970	add	%sp, STACK_BIAS, %g3
2971	srlx	%g3, MMU_PAGESHIFT, %g3
2972	srlx	%g2, MMU_PAGESHIFT, %g4
2973	cmp	%g3, %g4
2974	be,a,pn	%icc, ptl1_panic		/* panic if bad %sp */
2975	  mov	PTL1_BAD_STACK, %g1
2976
2977	cmp	%g5, 1
2978	ble,pt	%icc, 2f
2979	  nop
2980	TSTAT_CHECK_TL1(2f, %g1, %g2)
2981	rdpr	%tt, %g2
2982	cmp	%g2, FAST_PROT_TT
2983	mov	PTL1_BAD_KPROT_FAULT, %g1
2984	movne	%icc, PTL1_BAD_KMISS, %g1
2985	ba,pt	%icc, ptl1_panic
2986	  nop
2987
29882:
2989	/*
2990	 * We are taking a pagefault in the kernel on a kernel address.  If
2991	 * CPU_DTRACE_NOFAULT is set in the cpuc_dtrace_flags, we don't actually
2992	 * want to call sfmmu_pagefault -- we will instead note that a fault
2993	 * has occurred by setting CPU_DTRACE_BADADDR and issue a "done"
2994	 * (instead of a "retry").  This will step over the faulting
2995	 * instruction.
2996	 */
2997	CPU_INDEX(%g1, %g2)
2998	set	cpu_core, %g2
2999	sllx	%g1, CPU_CORE_SHIFT, %g1
3000	add	%g1, %g2, %g1
3001	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3002	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3003	bz	sfmmu_pagefault
3004	or	%g2, CPU_DTRACE_BADADDR, %g2
3005	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3006	GET_MMU_D_ADDR(%g3, %g4)
3007	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3008	done
3009
30103:
3011	cmp	%g5, 1
3012	ble,pt	%icc, 4f
3013	  nop
3014	TSTAT_CHECK_TL1(4f, %g1, %g2)
3015	ba,pt	%icc, sfmmu_window_trap
3016	  nop
3017
30184:
3019	/*
3020	 * We are taking a pagefault on a non-kernel address.  If we are in
3021	 * the kernel (e.g., due to a copyin()), we will check cpuc_dtrace_flags
3022	 * and (if CPU_DTRACE_NOFAULT is set) will proceed as outlined above.
3023	 */
3024	CPU_INDEX(%g1, %g2)
3025	set	cpu_core, %g2
3026	sllx	%g1, CPU_CORE_SHIFT, %g1
3027	add	%g1, %g2, %g1
3028	lduh	[%g1 + CPUC_DTRACE_FLAGS], %g2
3029	andcc	%g2, CPU_DTRACE_NOFAULT, %g0
3030	bz	sfmmu_pagefault
3031	or	%g2, CPU_DTRACE_BADADDR, %g2
3032	stuh	%g2, [%g1 + CPUC_DTRACE_FLAGS]
3033	GET_MMU_D_ADDR(%g3, %g4)
3034	stx	%g3, [%g1 + CPUC_DTRACE_ILLVAL]
3035
3036	/*
3037	 * Be sure that we're actually taking this miss from the kernel --
3038	 * otherwise we have managed to return to user-level with
3039	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3040	 */
3041	rdpr	%tstate, %g2
3042	btst	TSTATE_PRIV, %g2
3043	bz,a	ptl1_panic
3044	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3045	done
3046
3047	ALTENTRY(tsb_tl0_noctxt)
3048	/*
3049	 * If we have no context, check to see if CPU_DTRACE_NOFAULT is set;
3050	 * if it is, indicated that we have faulted and issue a done.
3051	 */
3052	CPU_INDEX(%g5, %g6)
3053	set	cpu_core, %g6
3054	sllx	%g5, CPU_CORE_SHIFT, %g5
3055	add	%g5, %g6, %g5
3056	lduh	[%g5 + CPUC_DTRACE_FLAGS], %g6
3057	andcc	%g6, CPU_DTRACE_NOFAULT, %g0
3058	bz	1f
3059	or	%g6, CPU_DTRACE_BADADDR, %g6
3060	stuh	%g6, [%g5 + CPUC_DTRACE_FLAGS]
3061	GET_MMU_D_ADDR(%g3, %g4)
3062	stx	%g3, [%g5 + CPUC_DTRACE_ILLVAL]
3063
3064	/*
3065	 * Be sure that we're actually taking this miss from the kernel --
3066	 * otherwise we have managed to return to user-level with
3067	 * CPU_DTRACE_NOFAULT set in cpuc_dtrace_flags.
3068	 */
3069	rdpr	%tstate, %g5
3070	btst	TSTATE_PRIV, %g5
3071	bz,a	ptl1_panic
3072	  mov	PTL1_BAD_DTRACE_FLAGS, %g1
3073	done
3074
30751:
3076	rdpr	%tt, %g5
3077	cmp	%g5, FAST_IMMU_MISS_TT
3078#ifdef sun4v
3079	MMU_FAULT_STATUS_AREA(%g2)
3080	be,a,pt	%icc, 2f
3081	  ldx	[%g2 + MMFSA_I_CTX], %g3
3082	cmp	%g5, T_INSTR_MMU_MISS
3083	be,a,pt	%icc, 2f
3084	  ldx	[%g2 + MMFSA_I_CTX], %g3
3085	ldx	[%g2 + MMFSA_D_CTX], %g3
30862:
3087#else
3088	mov	MMU_TAG_ACCESS, %g2
3089	be,a,pt	%icc, 2f
3090	  ldxa	[%g2]ASI_IMMU, %g3
3091	ldxa	[%g2]ASI_DMMU, %g3
30922:	sllx	%g3, TAGACC_CTX_LSHIFT, %g3
3093#endif
3094	brz,a,pn %g3, ptl1_panic		! panic if called for kernel
3095	  mov	PTL1_BAD_CTX_STEAL, %g1		! since kernel ctx was stolen
3096	rdpr	%tl, %g5
3097	cmp	%g5, 1
3098	ble,pt	%icc, sfmmu_mmu_trap
3099	  nop
3100	TSTAT_CHECK_TL1(sfmmu_mmu_trap, %g1, %g2)
3101	ba,pt	%icc, sfmmu_window_trap
3102	  nop
3103	SET_SIZE(sfmmu_tsb_miss)
3104
3105#if (1<< TSBMISS_SHIFT) != TSBMISS_SIZE
3106#error - TSBMISS_SHIFT does not correspond to size of tsbmiss struct
3107#endif
3108
3109#endif /* lint */
3110
3111#if defined (lint)
3112/*
3113 * This routine will look for a user or kernel vaddr in the hash
3114 * structure.  It returns a valid pfn or PFN_INVALID.  It doesn't
3115 * grab any locks.  It should only be used by other sfmmu routines.
3116 */
3117/* ARGSUSED */
3118pfn_t
3119sfmmu_vatopfn(caddr_t vaddr, sfmmu_t *sfmmup, tte_t *ttep)
3120{
3121	return(0);
3122}
3123
3124#else /* lint */
3125
3126	ENTRY_NP(sfmmu_vatopfn)
3127 	/*
3128 	 * disable interrupts
3129 	 */
3130 	rdpr	%pstate, %o3
3131#ifdef DEBUG
3132	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
3133	bnz,pt	%icc, 1f			/* disabled, panic	 */
3134	  nop
3135
3136	sethi	%hi(panicstr), %g1
3137	ldx	[%g1 + %lo(panicstr)], %g1
3138	tst	%g1
3139	bnz,pt	%icc, 1f
3140	  nop
3141
3142	save	%sp, -SA(MINFRAME), %sp
3143	sethi	%hi(sfmmu_panic1), %o0
3144	call	panic
3145	 or	%o0, %lo(sfmmu_panic1), %o0
31461:
3147#endif
3148	/*
3149	 * disable interrupts to protect the TSBMISS area
3150	 */
3151	andn    %o3, PSTATE_IE, %o5
3152	wrpr    %o5, 0, %pstate
3153
3154	/*
3155	 * o0 = vaddr
3156	 * o1 = sfmmup
3157	 * o2 = ttep
3158	 */
3159	CPU_TSBMISS_AREA(%g1, %o5)
3160	ldn	[%g1 + TSBMISS_KHATID], %o4
3161	cmp	%o4, %o1
3162	bne,pn	%ncc, vatopfn_nokernel
3163	  mov	TTE64K, %g5			/* g5 = rehash # */
3164	mov %g1,%o5				/* o5 = tsbmiss_area */
3165	/*
3166	 * o0 = vaddr
3167	 * o1 & o4 = hatid
3168	 * o2 = ttep
3169	 * o5 = tsbmiss area
3170	 */
3171	mov	HBLK_RANGE_SHIFT, %g6
31721:
3173
3174	/*
3175	 * o0 = vaddr
3176	 * o1 = sfmmup
3177	 * o2 = ttep
3178	 * o3 = old %pstate
3179	 * o4 = hatid
3180	 * o5 = tsbmiss
3181	 * g5 = rehash #
3182	 * g6 = hmeshift
3183	 *
3184	 * The first arg to GET_TTE is actually tagaccess register
3185	 * not just vaddr. Since this call is for kernel we need to clear
3186	 * any lower vaddr bits that would be interpreted as ctx bits.
3187	 */
3188	set     TAGACC_CTX_MASK, %g1
3189	andn    %o0, %g1, %o0
3190	GET_TTE(%o0, %o4, %g1, %g2, %g3, %o5, %g4, %g6, %g5,
3191		vatopfn_l1, kvtop_hblk_found, tsb_suspend, kvtop_nohblk)
3192
3193kvtop_hblk_found:
3194	/*
3195	 * o0 = vaddr
3196	 * o1 = sfmmup
3197	 * o2 = ttep
3198	 * g1 = tte
3199	 * g2 = tte pa
3200	 * g3 = tte va
3201	 * o2 = tsbmiss area
3202	 * o1 = hat id
3203	 */
3204	brgez,a,pn %g1, 6f			/* if tte invalid goto tl0 */
3205	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3206	stx %g1,[%o2]				/* put tte into *ttep */
3207	TTETOPFN(%g1, %o0, vatopfn_l2, %g2, %g3, %g4)
3208	/*
3209	 * o0 = vaddr
3210	 * o1 = sfmmup
3211	 * o2 = ttep
3212	 * g1 = pfn
3213	 */
3214	ba,pt	%xcc, 6f
3215	  mov	%g1, %o0
3216
3217kvtop_nohblk:
3218	/*
3219	 * we get here if we couldn't find valid hblk in hash.  We rehash
3220	 * if neccesary.
3221	 */
3222	ldn	[%o5 + (TSBMISS_SCRATCH + TSB_TAGACC)], %o0
3223#ifdef sun4v
3224	cmp	%g5, MAX_HASHCNT
3225#else
3226	cmp	%g5, DEFAULT_MAX_HASHCNT	/* no 32/256M kernel pages */
3227#endif
3228	be,a,pn	%icc, 6f
3229	  mov	-1, %o0				/* output = -1 (PFN_INVALID) */
3230	mov	%o1, %o4			/* restore hatid */
3231#ifdef sun4v
3232        add	%g5, 2, %g5
3233	cmp	%g5, 3
3234	move	%icc, MMU_PAGESHIFT4M, %g6
3235	ba,pt	%icc, 1b
3236	movne	%icc, MMU_PAGESHIFT256M, %g6
3237#else
3238        inc	%g5
3239	cmp	%g5, 2
3240	move	%icc, MMU_PAGESHIFT512K, %g6
3241	ba,pt	%icc, 1b
3242	movne	%icc, MMU_PAGESHIFT4M, %g6
3243#endif
32446:
3245	retl
3246 	  wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3247
3248tsb_suspend:
3249	/*
3250	 * o0 = vaddr
3251	 * o1 = sfmmup
3252	 * o2 = ttep
3253	 * g1 = tte
3254	 * g2 = tte pa
3255	 * g3 = tte va
3256	 * o2 = tsbmiss area  use o5 instead of o2 for tsbmiss
3257	 */
3258	stx %g1,[%o2]				/* put tte into *ttep */
3259	brgez,a,pn %g1, 8f			/* if tte invalid goto 8: */
3260	  sub	%g0, 1, %o0			/* output = -1 (PFN_INVALID) */
3261	TTETOPFN(%g1, %o0, vatopfn_l3, %g2, %g3, %g4)
3262	/*
3263	 * o0 = PFN return value PFN_INVALID, PFN_SUSPENDED, or pfn#
3264	 * o1 = sfmmup
3265	 * o2 = ttep
3266	 * g1 = pfn
3267	 */
3268	sub	%g0, 2, %o0			/* output = PFN_SUSPENDED */
32698:
3270	retl
3271	 wrpr	%g0, %o3, %pstate		/* enable interrupts */
3272
3273vatopfn_nokernel:
3274	/*
3275	 * This routine does NOT support user addresses
3276	 * There is a routine in C that supports this.
3277	 * The only reason why we don't have the C routine
3278	 * support kernel addresses as well is because
3279	 * we do va_to_pa while holding the hashlock.
3280	 */
3281 	wrpr	%g0, %o3, %pstate		/* re-enable interrupts */
3282	save	%sp, -SA(MINFRAME), %sp
3283	sethi	%hi(sfmmu_panic3), %o0
3284	call	panic
3285	 or	%o0, %lo(sfmmu_panic3), %o0
3286
3287	SET_SIZE(sfmmu_vatopfn)
3288#endif /* lint */
3289
3290
3291
3292#if !defined(lint)
3293
3294/*
3295 * kpm lock used between trap level tsbmiss handler and kpm C level.
3296 */
3297#define KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi)			\
3298	mov     0xff, tmp1						;\
3299label1:									;\
3300	casa    [kpmlckp]asi, %g0, tmp1					;\
3301	brnz,pn tmp1, label1						;\
3302	mov     0xff, tmp1						;\
3303	membar  #LoadLoad
3304
3305#define KPMLOCK_EXIT(kpmlckp, asi)					\
3306	membar  #LoadStore|#StoreStore					;\
3307	sta     %g0, [kpmlckp]asi
3308
3309/*
3310 * Lookup a memseg for a given pfn and if found, return the physical
3311 * address of the corresponding struct memseg in mseg, otherwise
3312 * return MSEG_NULLPTR_PA. The kpmtsbm pointer must be provided in
3313 * tsbmp, %asi is assumed to be ASI_MEM.
3314 * This lookup is done by strictly traversing only the physical memseg
3315 * linkage. The more generic approach, to check the virtual linkage
3316 * before using the physical (used e.g. with hmehash buckets), cannot
3317 * be used here. Memory DR operations can run in parallel to this
3318 * lookup w/o any locks and updates of the physical and virtual linkage
3319 * cannot be done atomically wrt. to each other. Because physical
3320 * address zero can be valid physical address, MSEG_NULLPTR_PA acts
3321 * as "physical NULL" pointer.
3322 */
3323#define	PAGE_NUM2MEMSEG_NOLOCK_PA(pfn, mseg, tsbmp, tmp1, tmp2, tmp3, label) \
3324	sethi	%hi(mhash_per_slot), tmp3 /* no tsbmp use due to DR */	;\
3325	ldx	[tmp3 + %lo(mhash_per_slot)], mseg			;\
3326	udivx	pfn, mseg, mseg						;\
3327	ldx	[tsbmp + KPMTSBM_MSEGPHASHPA], tmp1			;\
3328	and	mseg, SFMMU_N_MEM_SLOTS - 1, mseg			;\
3329	sllx	mseg, SFMMU_MEM_HASH_ENTRY_SHIFT, mseg			;\
3330	add	tmp1, mseg, tmp1					;\
3331	ldxa	[tmp1]%asi, mseg					;\
3332	cmp	mseg, MSEG_NULLPTR_PA					;\
3333	be,pn	%xcc, label/**/1		/* if not found */	;\
3334	  nop								;\
3335	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3336	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3337	blu,pn	%xcc, label/**/1					;\
3338	  ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3339	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3340	bgeu,pn	%xcc, label/**/1					;\
3341	  sub	pfn, tmp1, tmp1			/* pfn - pages_base */	;\
3342	mulx	tmp1, PAGE_SIZE, tmp1					;\
3343	ldxa	[mseg + MEMSEG_PAGESPA]%asi, tmp2	/* pages */	;\
3344	add	tmp2, tmp1, tmp1			/* pp */	;\
3345	lduwa	[tmp1 + PAGE_PAGENUM]%asi, tmp2				;\
3346	cmp	tmp2, pfn						;\
3347	be,pt	%xcc, label/**/_ok			/* found */	;\
3348label/**/1:								;\
3349	/* brute force lookup */					;\
3350	sethi	%hi(memsegspa), tmp3 /* no tsbmp use due to DR */	;\
3351	ldx	[tmp3 + %lo(memsegspa)], mseg				;\
3352label/**/2:								;\
3353	cmp	mseg, MSEG_NULLPTR_PA					;\
3354	be,pn	%xcc, label/**/_ok		/* if not found */	;\
3355	  nop								;\
3356	ldxa	[mseg + MEMSEG_PAGES_BASE]%asi, tmp1			;\
3357	cmp	pfn, tmp1			/* pfn - pages_base */	;\
3358	blu,a,pt %xcc, label/**/2					;\
3359	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3360	ldxa	[mseg + MEMSEG_PAGES_END]%asi, tmp2			;\
3361	cmp	pfn, tmp2			/* pfn - pages_end */	;\
3362	bgeu,a,pt %xcc, label/**/2					;\
3363	  ldxa	[mseg + MEMSEG_NEXTPA]%asi, mseg			;\
3364label/**/_ok:
3365
3366	/*
3367	 * kpm tsb miss handler large pages
3368	 * g1 = 8K kpm TSB entry pointer
3369	 * g2 = tag access register
3370	 * g3 = 4M kpm TSB entry pointer
3371	 */
3372	ALTENTRY(sfmmu_kpm_dtsb_miss)
3373	TT_TRACE(trace_tsbmiss)
3374
3375	CPU_INDEX(%g7, %g6)
3376	sethi	%hi(kpmtsbm_area), %g6
3377	sllx	%g7, KPMTSBM_SHIFT, %g7
3378	or	%g6, %lo(kpmtsbm_area), %g6
3379	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
3380
3381	/* check enable flag */
3382	ldub	[%g6 + KPMTSBM_FLAGS], %g4
3383	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
3384	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
3385	  nop
3386
3387	/* VA range check */
3388	ldx	[%g6 + KPMTSBM_VBASE], %g7
3389	cmp	%g2, %g7
3390	blu,pn	%xcc, sfmmu_tsb_miss
3391	  ldx	[%g6 + KPMTSBM_VEND], %g5
3392	cmp	%g2, %g5
3393	bgeu,pn	%xcc, sfmmu_tsb_miss
3394	  stx	%g3, [%g6 + KPMTSBM_TSBPTR]
3395
3396	/*
3397	 * check TL tsbmiss handling flag
3398	 * bump tsbmiss counter
3399	 */
3400	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
3401#ifdef	DEBUG
3402	and	%g4, KPMTSBM_TLTSBM_FLAG, %g3
3403	inc	%g5
3404	brz,pn	%g3, sfmmu_kpm_exception
3405	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
3406#else
3407	inc	%g5
3408	st	%g5, [%g6 + KPMTSBM_TSBMISS]
3409#endif
3410	/*
3411	 * At this point:
3412	 *  g1 = 8K kpm TSB pointer (not used)
3413	 *  g2 = tag access register
3414	 *  g3 = clobbered
3415	 *  g6 = per-CPU kpm tsbmiss area
3416	 *  g7 = kpm_vbase
3417	 */
3418
3419	/* vaddr2pfn */
3420	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
3421	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
3422	srax    %g4, %g3, %g2			/* which alias range (r) */
3423	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
3424	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
3425
3426	/*
3427	 * Setup %asi
3428	 * mseg_pa = page_numtomemseg_nolock(pfn)
3429	 * if (mseg_pa == NULL) sfmmu_kpm_exception
3430	 * g2=pfn
3431	 */
3432	mov	ASI_MEM, %asi
3433	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmp2m)
3434	cmp	%g3, MSEG_NULLPTR_PA
3435	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
3436	  nop
3437
3438	/*
3439	 * inx = ptokpmp((kpmptop((ptopkpmp(pfn))) - mseg_pa->kpm_pbase));
3440	 * g2=pfn g3=mseg_pa
3441	 */
3442	ldub	[%g6 + KPMTSBM_KPMP2PSHFT], %g5
3443	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
3444	srlx	%g2, %g5, %g4
3445	sllx	%g4, %g5, %g4
3446	sub	%g4, %g7, %g4
3447	srlx	%g4, %g5, %g4
3448
3449	/*
3450	 * Validate inx value
3451	 * g2=pfn g3=mseg_pa g4=inx
3452	 */
3453#ifdef	DEBUG
3454	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
3455	cmp	%g4, %g5			/* inx - nkpmpgs */
3456	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
3457	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3458#else
3459	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3460#endif
3461	/*
3462	 * kp = &mseg_pa->kpm_pages[inx]
3463	 */
3464	sllx	%g4, KPMPAGE_SHIFT, %g4		/* kpm_pages offset */
3465	ldxa	[%g3 + MEMSEG_KPM_PAGES]%asi, %g5 /* kpm_pages */
3466	add	%g5, %g4, %g5			/* kp */
3467
3468	/*
3469	 * KPMP_HASH(kp)
3470	 * g2=pfn g3=mseg_pa g4=offset g5=kp g7=kpmp_table_sz
3471	 */
3472	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
3473	sub	%g7, 1, %g7			/* mask */
3474	srlx	%g5, %g1, %g1			/* x = ksp >> kpmp_shift */
3475	add	%g5, %g1, %g5			/* y = ksp + x */
3476	and 	%g5, %g7, %g5			/* hashinx = y & mask */
3477
3478	/*
3479	 * Calculate physical kpm_page pointer
3480	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
3481	 */
3482	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_pagespa */
3483	add	%g1, %g4, %g1			/* kp_pa */
3484
3485	/*
3486	 * Calculate physical hash lock address
3487	 * g1=kp_refcntc_pa g2=pfn g5=hashinx
3488	 */
3489	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_tablepa */
3490	sllx	%g5, KPMHLK_SHIFT, %g5
3491	add	%g4, %g5, %g3
3492	add	%g3, KPMHLK_LOCK, %g3		/* hlck_pa */
3493
3494	/*
3495	 * Assemble tte
3496	 * g1=kp_pa g2=pfn g3=hlck_pa
3497	 */
3498#ifdef sun4v
3499	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
3500	sllx	%g5, 32, %g5
3501	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3502	or	%g4, TTE4M, %g4
3503	or	%g5, %g4, %g5
3504#else
3505	sethi	%hi(TTE_VALID_INT), %g4
3506	mov	TTE4M, %g5
3507	sllx	%g5, TTE_SZ_SHFT_INT, %g5
3508	or	%g5, %g4, %g5			/* upper part */
3509	sllx	%g5, 32, %g5
3510	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3511	or	%g5, %g4, %g5
3512#endif
3513	sllx	%g2, MMU_PAGESHIFT, %g4
3514	or	%g5, %g4, %g5			/* tte */
3515	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
3516	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3517
3518	/*
3519	 * tsb dropin
3520	 * g1=kp_pa g2=ttarget g3=hlck_pa g4=kpmtsbp4m g5=tte g6=kpmtsbm_area
3521	 */
3522
3523	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
3524	KPMLOCK_ENTER(%g3, %g7, kpmtsbmhdlr1, ASI_MEM)
3525
3526	/* use C-handler if there's no go for dropin */
3527	ldsha	[%g1 + KPMPAGE_REFCNTC]%asi, %g7 /* kp_refcntc */
3528	cmp	%g7, -1
3529	bne,pn	%xcc, 5f	/* use C-handler if there's no go for dropin */
3530	  nop
3531
3532#ifdef	DEBUG
3533	/* double check refcnt */
3534	ldsha	[%g1 + KPMPAGE_REFCNT]%asi, %g7
3535	brz,pn	%g7, 5f			/* let C-handler deal with this */
3536	  nop
3537#endif
3538
3539#ifndef sun4v
3540	ldub	[%g6 + KPMTSBM_FLAGS], %g7
3541	mov	ASI_N, %g1
3542	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
3543	movnz	%icc, ASI_MEM, %g1
3544	mov	%g1, %asi
3545#endif
3546
3547	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
3548	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
3549
3550	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
3551	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
3552
3553	DTLB_STUFF(%g5, %g1, %g2, %g4, %g6)
3554
3555	/* KPMLOCK_EXIT(kpmlckp, asi) */
3556	KPMLOCK_EXIT(%g3, ASI_MEM)
3557
3558	/*
3559	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3560	 * point to trapstat's TSB miss return code (note that trapstat
3561	 * itself will patch the correct offset to add).
3562	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
3563	 */
3564	rdpr	%tl, %g7
3565	cmp	%g7, 1
3566	ble	%icc, 0f
3567	sethi	%hi(KERNELBASE), %g6
3568	rdpr	%tpc, %g7
3569	or	%g6, %lo(KERNELBASE), %g6
3570	cmp	%g7, %g6
3571	bgeu	%xcc, 0f
3572	ALTENTRY(tsbmiss_trapstat_patch_point_kpm)
3573	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3574	wrpr	%g7, %tpc
3575	add	%g7, 4, %g7
3576	wrpr	%g7, %tnpc
35770:
3578	retry
35795:
3580	/* g3=hlck_pa */
3581	KPMLOCK_EXIT(%g3, ASI_MEM)
3582	ba,pt	%icc, sfmmu_kpm_exception
3583	  nop
3584	SET_SIZE(sfmmu_kpm_dtsb_miss)
3585
3586	/*
3587	 * kpm tsbmiss handler for smallpages
3588	 * g1 = 8K kpm TSB pointer
3589	 * g2 = tag access register
3590	 * g3 = 4M kpm TSB pointer
3591	 */
3592	ALTENTRY(sfmmu_kpm_dtsb_miss_small)
3593	TT_TRACE(trace_tsbmiss)
3594	CPU_INDEX(%g7, %g6)
3595	sethi	%hi(kpmtsbm_area), %g6
3596	sllx	%g7, KPMTSBM_SHIFT, %g7
3597	or	%g6, %lo(kpmtsbm_area), %g6
3598	add	%g6, %g7, %g6			/* g6 = kpmtsbm ptr */
3599
3600	/* check enable flag */
3601	ldub	[%g6 + KPMTSBM_FLAGS], %g4
3602	and	%g4, KPMTSBM_ENABLE_FLAG, %g5
3603	brz,pn	%g5, sfmmu_tsb_miss		/* if kpm not enabled */
3604	  nop
3605
3606	/*
3607	 * VA range check
3608	 * On fail: goto sfmmu_tsb_miss
3609	 */
3610	ldx	[%g6 + KPMTSBM_VBASE], %g7
3611	cmp	%g2, %g7
3612	blu,pn	%xcc, sfmmu_tsb_miss
3613	  ldx	[%g6 + KPMTSBM_VEND], %g5
3614	cmp	%g2, %g5
3615	bgeu,pn	%xcc, sfmmu_tsb_miss
3616	  stx	%g1, [%g6 + KPMTSBM_TSBPTR]	/* save 8K kpm TSB pointer */
3617
3618	/*
3619	 * check TL tsbmiss handling flag
3620	 * bump tsbmiss counter
3621	 */
3622	lduw	[%g6 + KPMTSBM_TSBMISS], %g5
3623#ifdef	DEBUG
3624	and	%g4, KPMTSBM_TLTSBM_FLAG, %g1
3625	inc	%g5
3626	brz,pn	%g1, sfmmu_kpm_exception
3627	  st	%g5, [%g6 + KPMTSBM_TSBMISS]
3628#else
3629	inc	%g5
3630	st	%g5, [%g6 + KPMTSBM_TSBMISS]
3631#endif
3632	/*
3633	 * At this point:
3634	 *  g1 = clobbered
3635	 *  g2 = tag access register
3636	 *  g3 = 4M kpm TSB pointer (not used)
3637	 *  g6 = per-CPU kpm tsbmiss area
3638	 *  g7 = kpm_vbase
3639	 */
3640
3641	/* vaddr2pfn */
3642	ldub	[%g6 + KPMTSBM_SZSHIFT], %g3
3643	sub	%g2, %g7, %g4			/* paddr = vaddr-kpm_vbase */
3644	srax    %g4, %g3, %g2			/* which alias range (r) */
3645	brnz,pn	%g2, sfmmu_kpm_exception	/* if (r != 0) goto C handler */
3646	  srlx	%g4, MMU_PAGESHIFT, %g2		/* %g2 = pfn */
3647
3648	/*
3649	 * Setup %asi
3650	 * mseg_pa = page_numtomemseg_nolock_pa(pfn)
3651	 * if (mseg not found) sfmmu_kpm_exception
3652	 * g2=pfn
3653	 */
3654	mov	ASI_MEM, %asi
3655	PAGE_NUM2MEMSEG_NOLOCK_PA(%g2, %g3, %g6, %g4, %g5, %g7, kpmtsbmsp2m)
3656	cmp	%g3, MSEG_NULLPTR_PA
3657	be,pn	%xcc, sfmmu_kpm_exception	/* if mseg not found */
3658	  nop
3659
3660	/*
3661	 * inx = pfn - mseg_pa->kpm_pbase
3662	 * g2=pfn g3=mseg_pa
3663	 */
3664	ldxa	[%g3 + MEMSEG_KPM_PBASE]%asi, %g7
3665	sub	%g2, %g7, %g4
3666
3667#ifdef	DEBUG
3668	/*
3669	 * Validate inx value
3670	 * g2=pfn g3=mseg_pa g4=inx
3671	 */
3672	ldxa	[%g3 + MEMSEG_KPM_NKPMPGS]%asi, %g5
3673	cmp	%g4, %g5			/* inx - nkpmpgs */
3674	bgeu,pn	%xcc, sfmmu_kpm_exception	/* if out of range */
3675	  ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3676#else
3677	ld	[%g6 + KPMTSBM_KPMPTABLESZ], %g7
3678#endif
3679	/* ksp = &mseg_pa->kpm_spages[inx] */
3680	ldxa	[%g3 + MEMSEG_KPM_SPAGES]%asi, %g5
3681	add	%g5, %g4, %g5			/* ksp */
3682
3683	/*
3684	 * KPMP_SHASH(kp)
3685	 * g2=pfn g3=mseg_pa g4=inx g5=ksp g7=kpmp_stable_sz
3686	 */
3687	ldub	[%g6 + KPMTSBM_KPMPSHIFT], %g1	/* kpmp_shift */
3688	sub	%g7, 1, %g7			/* mask */
3689	sllx	%g5, %g1, %g1			/* x = ksp << kpmp_shift */
3690	add	%g5, %g1, %g5			/* y = ksp + x */
3691	and 	%g5, %g7, %g5			/* hashinx = y & mask */
3692
3693	/*
3694	 * Calculate physical kpm_spage pointer
3695	 * g2=pfn g3=mseg_pa g4=offset g5=hashinx
3696	 */
3697	ldxa	[%g3 + MEMSEG_KPM_PAGESPA]%asi, %g1 /* kpm_spagespa */
3698	add	%g1, %g4, %g1			/* ksp_pa */
3699
3700	/*
3701	 * Calculate physical hash lock address.
3702	 * Note: Changes in kpm_shlk_t must be reflected here.
3703	 * g1=ksp_pa g2=pfn g5=hashinx
3704	 */
3705	ldx	[%g6 + KPMTSBM_KPMPTABLEPA], %g4 /* kpmp_stablepa */
3706	sllx	%g5, KPMSHLK_SHIFT, %g5
3707	add	%g4, %g5, %g3			/* hlck_pa */
3708
3709	/*
3710	 * Assemble tte
3711	 * g1=ksp_pa g2=pfn g3=hlck_pa
3712	 */
3713	sethi	%hi(TTE_VALID_INT), %g5		/* upper part */
3714	sllx	%g5, 32, %g5
3715	mov	(TTE_CP_INT|TTE_CV_INT|TTE_PRIV_INT|TTE_HWWR_INT), %g4
3716	or	%g5, %g4, %g5
3717	sllx	%g2, MMU_PAGESHIFT, %g4
3718	or	%g5, %g4, %g5			/* tte */
3719	ldx	[%g6 + KPMTSBM_TSBPTR], %g4
3720	GET_MMU_D_TTARGET(%g2, %g7)		/* %g2 = ttarget */
3721
3722	/*
3723	 * tsb dropin
3724	 * g1=ksp_pa g2=ttarget g3=hlck_pa g4=ktsbp g5=tte
3725	 */
3726
3727	/* KPMLOCK_ENTER(kpmlckp, tmp1, label1, asi) */
3728	KPMLOCK_ENTER(%g3, %g7, kpmtsbsmlock, ASI_MEM)
3729
3730	/* use C-handler if there's no go for dropin */
3731	ldsba	[%g1 + KPMSPAGE_MAPPED]%asi, %g7 /* kp_mapped */
3732	cmp	%g7, -1
3733	bne,pn	%xcc, 5f
3734	  nop
3735
3736#ifndef sun4v
3737	ldub	[%g6 + KPMTSBM_FLAGS], %g7
3738	mov	ASI_N, %g1
3739	andcc	%g7, KPMTSBM_TSBPHYS_FLAG, %g0
3740	movnz	%icc, ASI_MEM, %g1
3741	mov	%g1, %asi
3742#endif
3743
3744	/* TSB_LOCK_ENTRY(tsbp, tmp1, tmp2, label) (needs %asi set) */
3745	TSB_LOCK_ENTRY(%g4, %g1, %g7, 6)
3746
3747	/* TSB_INSERT_UNLOCK_ENTRY(tsbp, tte, tagtarget, tmp) */
3748	TSB_INSERT_UNLOCK_ENTRY(%g4, %g5, %g2, %g7)
3749
3750	DTLB_STUFF(%g5, %g2, %g4, %g5, %g6)
3751
3752	/* KPMLOCK_EXIT(kpmlckp, asi) */
3753	KPMLOCK_EXIT(%g3, ASI_MEM)
3754
3755	/*
3756	 * If trapstat is running, we need to shift the %tpc and %tnpc to
3757	 * point to trapstat's TSB miss return code (note that trapstat
3758	 * itself will patch the correct offset to add).
3759	 * Note: TTE is expected in %g5 (allows per pagesize reporting).
3760	 */
3761	rdpr	%tl, %g7
3762	cmp	%g7, 1
3763	ble	%icc, 0f
3764	sethi	%hi(KERNELBASE), %g6
3765	rdpr	%tpc, %g7
3766	or	%g6, %lo(KERNELBASE), %g6
3767	cmp	%g7, %g6
3768	bgeu	%xcc, 0f
3769	ALTENTRY(tsbmiss_trapstat_patch_point_kpm_small)
3770	add	%g7, RUNTIME_PATCH, %g7	/* must match TSTAT_TSBMISS_INSTR */
3771	wrpr	%g7, %tpc
3772	add	%g7, 4, %g7
3773	wrpr	%g7, %tnpc
37740:
3775	retry
37765:
3777	/* g3=hlck_pa */
3778	KPMLOCK_EXIT(%g3, ASI_MEM)
3779	ba,pt	%icc, sfmmu_kpm_exception
3780	  nop
3781	SET_SIZE(sfmmu_kpm_dtsb_miss_small)
3782
3783#if (1<< KPMTSBM_SHIFT) != KPMTSBM_SIZE
3784#error - KPMTSBM_SHIFT does not correspond to size of kpmtsbm struct
3785#endif
3786
3787#endif /* lint */
3788
3789#ifdef	lint
3790/*
3791 * Enable/disable tsbmiss handling at trap level for a kpm (large) page.
3792 * Called from C-level, sets/clears "go" indication for trap level handler.
3793 * khl_lock is a low level spin lock to protect the kp_tsbmtl field.
3794 * Assumed that &kp->kp_refcntc is checked for zero or -1 at C-level.
3795 * Assumes khl_mutex is held when called from C-level.
3796 */
3797/* ARGSUSED */
3798void
3799sfmmu_kpm_tsbmtl(short *kp_refcntc, uint_t *khl_lock, int cmd)
3800{
3801}
3802
3803/*
3804 * kpm_smallpages: stores val to byte at address mapped within
3805 * low level lock brackets. The old value is returned.
3806 * Called from C-level.
3807 */
3808/* ARGSUSED */
3809int
3810sfmmu_kpm_stsbmtl(char *mapped, uint_t *kshl_lock, int val)
3811{
3812	return (0);
3813}
3814
3815#else /* lint */
3816
3817	.seg	".data"
3818sfmmu_kpm_tsbmtl_panic:
3819	.ascii	"sfmmu_kpm_tsbmtl: interrupts disabled"
3820	.byte	0
3821sfmmu_kpm_stsbmtl_panic:
3822	.ascii	"sfmmu_kpm_stsbmtl: interrupts disabled"
3823	.byte	0
3824	.align	4
3825	.seg	".text"
3826
3827	ENTRY_NP(sfmmu_kpm_tsbmtl)
3828	rdpr	%pstate, %o3
3829	/*
3830	 * %o0 = &kp_refcntc
3831	 * %o1 = &khl_lock
3832	 * %o2 = 0/1 (off/on)
3833	 * %o3 = pstate save
3834	 */
3835#ifdef DEBUG
3836	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
3837	bnz,pt %icc, 1f				/* disabled, panic	 */
3838	  nop
3839	save	%sp, -SA(MINFRAME), %sp
3840	sethi	%hi(sfmmu_kpm_tsbmtl_panic), %o0
3841	call	panic
3842	 or	%o0, %lo(sfmmu_kpm_tsbmtl_panic), %o0
3843	ret
3844	restore
38451:
3846#endif /* DEBUG */
3847	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
3848
3849	KPMLOCK_ENTER(%o1, %o4, kpmtsbmtl1, ASI_N)
3850	mov	-1, %o5
3851	brz,a	%o2, 2f
3852	  mov	0, %o5
38532:
3854	sth	%o5, [%o0]
3855	KPMLOCK_EXIT(%o1, ASI_N)
3856
3857	retl
3858	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
3859	SET_SIZE(sfmmu_kpm_tsbmtl)
3860
3861	ENTRY_NP(sfmmu_kpm_stsbmtl)
3862	rdpr	%pstate, %o3
3863	/*
3864	 * %o0 = &mapped
3865	 * %o1 = &kshl_lock
3866	 * %o2 = val
3867	 * %o3 = pstate save
3868	 */
3869#ifdef DEBUG
3870	andcc	%o3, PSTATE_IE, %g0		/* if interrupts already */
3871	bnz,pt %icc, 1f				/* disabled, panic	 */
3872	  nop
3873	save	%sp, -SA(MINFRAME), %sp
3874	sethi	%hi(sfmmu_kpm_stsbmtl_panic), %o0
3875	call	panic
3876	  or	%o0, %lo(sfmmu_kpm_stsbmtl_panic), %o0
3877	ret
3878	restore
38791:
3880#endif /* DEBUG */
3881	wrpr	%o3, PSTATE_IE, %pstate		/* disable interrupts */
3882
3883	KPMLOCK_ENTER(%o1, %o4, kpmstsbmtl1, ASI_N)
3884	ldsb	[%o0], %o5
3885	stb	%o2, [%o0]
3886	KPMLOCK_EXIT(%o1, ASI_N)
3887
3888	mov	%o5, %o0			/* return old val */
3889	retl
3890	  wrpr	%g0, %o3, %pstate		/* enable interrupts */
3891	SET_SIZE(sfmmu_kpm_stsbmtl)
3892
3893#endif /* lint */
3894
3895#ifndef lint
3896#ifdef sun4v
3897	/*
3898	 * User/kernel data miss w// multiple TSBs
3899	 * The first probe covers 8K, 64K, and 512K page sizes,
3900	 * because 64K and 512K mappings are replicated off 8K
3901	 * pointer.  Second probe covers 4M page size only.
3902	 *
3903	 * MMU fault area contains miss address and context.
3904	 */
3905	ALTENTRY(sfmmu_slow_dmmu_miss)
3906	GET_MMU_D_TAGACC_CTX(%g2, %g3)	! %g2 = tagacc, %g3 = ctx
3907
3908slow_miss_common:
3909	/*
3910	 *  %g2 = tagacc register (needed for sfmmu_tsb_miss_tt)
3911	 *  %g3 = ctx (cannot be INVALID_CONTEXT)
3912	 */
3913	brnz,pt	%g3, 8f			! check for user context
3914	  nop
3915
3916	/*
3917	 * Kernel miss
3918	 * Get 8K and 4M TSB pointers in %g1 and %g3 and
3919	 * branch to sfmmu_tsb_miss_tt to handle it.
3920	 */
3921	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
3922sfmmu_dslow_patch_ktsb_base:
3923	RUNTIME_PATCH_SETX(%g1, %g6)	! %g1 = contents of ktsb_pbase
3924sfmmu_dslow_patch_ktsb_szcode:
3925	or	%g0, RUNTIME_PATCH, %g3	! ktsb_szcode (hot patched)
3926
3927	GET_TSBE_POINTER(MMU_PAGESHIFT, %g1, %g7, %g3, %g5)
3928	! %g1 = First TSB entry pointer, as TSB miss handler expects
3929
3930	mov	%g2, %g7		! TSB pointer macro clobbers tagacc
3931sfmmu_dslow_patch_ktsb4m_base:
3932	RUNTIME_PATCH_SETX(%g3, %g6)	! %g3 = contents of ktsb4m_pbase
3933sfmmu_dslow_patch_ktsb4m_szcode:
3934	or	%g0, RUNTIME_PATCH, %g6	! ktsb4m_szcode (hot patched)
3935
3936	GET_TSBE_POINTER(MMU_PAGESHIFT4M, %g3, %g7, %g6, %g5)
3937	! %g3 = 4M tsb entry pointer, as TSB miss handler expects
3938	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
3939	.empty
3940
39418:
3942	/*
3943	 * User miss
3944	 * Get first TSB pointer in %g1
3945	 * Get second TSB pointer (or NULL if no second TSB) in %g3
3946	 * Branch to sfmmu_tsb_miss_tt to handle it
3947	 */
3948	GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)
3949	/* %g1 = first TSB entry ptr now, %g2 preserved */
3950
3951	GET_UTSBREG(SCRATCHPAD_UTSBREG2, %g3)	/* get 2nd utsbreg */
3952	brlz,a,pt %g3, sfmmu_tsb_miss_tt	/* done if no 2nd TSB */
3953	  mov	%g0, %g3
3954
3955	GET_2ND_TSBE_PTR(%g2, %g3, %g4, %g5)
3956	/* %g3 = second TSB entry ptr now, %g2 preserved */
39579:
3958	ba,a,pt	%xcc, sfmmu_tsb_miss_tt
3959	.empty
3960	SET_SIZE(sfmmu_slow_dmmu_miss)
3961
3962
3963	/*
3964	 * User/kernel instruction miss w/ multiple TSBs
3965	 * The first probe covers 8K, 64K, and 512K page sizes,
3966	 * because 64K and 512K mappings are replicated off 8K
3967	 * pointer.  Second probe covers 4M page size only.
3968	 *
3969	 * MMU fault area contains miss address and context.
3970	 */
3971	ALTENTRY(sfmmu_slow_immu_miss)
3972	MMU_FAULT_STATUS_AREA(%g2)
3973	ldx	[%g2 + MMFSA_I_CTX], %g3
3974	ldx	[%g2 + MMFSA_I_ADDR], %g2
3975	srlx	%g2, MMU_PAGESHIFT, %g2	! align address to page boundry
3976	sllx	%g2, MMU_PAGESHIFT, %g2
3977	ba,pt	%xcc, slow_miss_common
3978	or	%g2, %g3, %g2
3979	SET_SIZE(sfmmu_slow_immu_miss)
3980
3981#endif /* sun4v */
3982#endif	/* lint */
3983
3984#ifndef lint
3985
3986/*
3987 * Per-CPU tsbmiss areas to avoid cache misses in TSB miss handlers.
3988 */
3989	.seg	".data"
3990	.align	64
3991	.global tsbmiss_area
3992tsbmiss_area:
3993	.skip	(TSBMISS_SIZE * NCPU)
3994
3995	.align	64
3996	.global kpmtsbm_area
3997kpmtsbm_area:
3998	.skip	(KPMTSBM_SIZE * NCPU)
3999#endif	/* lint */
4000