xref: /illumos-gate/usr/src/uts/sun4v/vm/mach_sfmmu.h (revision 5d9d9091)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * VM - Hardware Address Translation management.
 *
 * This file describes the contents of the sun reference mmu (sfmmu)
 * specific hat data structures and the sfmmu specific hat procedures.
 * The machine independent interface is described in <vm/hat.h>.
 */

#ifndef _VM_MACH_SFMMU_H
#define	_VM_MACH_SFMMU_H

#include <sys/x_call.h>
#include <sys/hypervisor_api.h>

#ifdef	__cplusplus
extern "C" {
#endif

/*
 * Define UTSB_PHYS if user TSB is always accessed via physical address.
 * On sun4v platform, user TSB is accessed via physical address.
 */
#define	UTSB_PHYS	1

/*
 * Hypervisor TSB info
 */
#define	NHV_TSB_INFO	4

#ifndef _ASM

struct hv_tsb_block {
	uint64_t	hv_tsb_info_pa;	/* hypervisor TSB info PA */
	uint64_t	hv_tsb_info_cnt; /* hypervisor TSB info count */
	hv_tsb_info_t	hv_tsb_info[NHV_TSB_INFO]; /* hypervisor TSB info */
};

#endif /* _ASM */

#ifdef _ASM

/*
 * This macro is used to set private/shared secondary context register in
 * sfmmu_alloc_ctx().
 * Input:
 * cnum     = cnum
 * is_shctx = sfmmu private/shared flag (0: private, 1: shared)
 * tmp2 is only used in the sun4u version of this macro
 */
#define	SET_SECCTX(cnum, is_shctx, tmp1, tmp2, label)			\
	mov	MMU_SCONTEXT, tmp1;					\
	movrnz	is_shctx, MMU_SCONTEXT1, tmp1;				\
	stxa    cnum, [tmp1]ASI_MMU_CTX;  /* set 2nd ctx reg. */	\
	membar  #Sync;							\

/*
 * This macro is used in the MMU code to check if TL should be lowered from
 * 2 to 1 to pop trapstat's state.  See the block comment in trapstat.c
 * for details.
 */

#define	TSTAT_CHECK_TL1(label, scr1, scr2)			\
	rdpr	%tpc, scr1;					\
	sethi	%hi(KERNELBASE), scr2;				\
	or	scr2, %lo(KERNELBASE), scr2; 			\
	cmp	scr1, scr2; 					\
	bgeu	%xcc, 9f;					\
	nop;							\
	wrpr	%g0, 1, %gl;					\
	ba	label;						\
	wrpr	%g0, 1, %tl;					\
9:

/*
 * The following macros allow us to share majority of the
 * SFMMU code between sun4u and sun4v platforms.
 */

#define	SETUP_TSB_ASI(qlp, tmp)

#define	SETUP_UTSB_ATOMIC_ASI(tmp1, tmp2)

/*
 * Macro to swtich to alternate global register on sun4u platforms
 * (not applicable to sun4v platforms)
 */
#define	USE_ALTERNATE_GLOBALS(scr)

/*
 * Macro to set %gl register value on sun4v platforms
 * (not applicable to sun4u platforms)
 */
#define	SET_GL_REG(val)						\
	wrpr	%g0, val, %gl

/*
 * Get pseudo-tagacc value and context from the MMU fault area.  Pseudo-tagacc
 * is the faulting virtual address OR'd with 0 for KCONTEXT, INVALID_CONTEXT
 * (1) for invalid context, and USER_CONTEXT (2) for user context.
 *
 * In:
 *   tagacc, ctxtype = scratch registers
 * Out:
 *   tagacc = MMU data tag access register value
 *   ctx = context type (KCONTEXT, INVALID_CONTEXT or USER_CONTEXT)
 */
#define	GET_MMU_D_PTAGACC_CTXTYPE(ptagacc, ctxtype)			\
	MMU_FAULT_STATUS_AREA(ctxtype);					\
	ldx	[ctxtype + MMFSA_D_ADDR], ptagacc;			\
	ldx	[ctxtype + MMFSA_D_CTX], ctxtype;			\
	srlx	ptagacc, MMU_PAGESHIFT, ptagacc; /* align to page boundary */ \
	cmp	ctxtype, USER_CONTEXT_TYPE;				\
	sllx	ptagacc, MMU_PAGESHIFT, ptagacc;			\
	movgu	%icc, USER_CONTEXT_TYPE, ctxtype;			\
	or	ptagacc, ctxtype, ptagacc

/*
 * Synthesize/get data tag access register value from the MMU fault area
 *
 * In:
 *   tagacc, scr1 = scratch registers
 * Out:
 *   tagacc = MMU data tag access register value
 */
#define	GET_MMU_D_TAGACC(tagacc, scr1)				\
	GET_MMU_D_PTAGACC_CTXTYPE(tagacc, scr1)

/*
 * Synthesize/get data tag target register value from the MMU fault area
 *
 * In:
 *   ttarget, scr1 = scratch registers
 * Out:
 *   ttarget = MMU data tag target register value
 */
#define	GET_MMU_D_TTARGET(ttarget, scr1)			\
	MMU_FAULT_STATUS_AREA(ttarget);				\
	ldx	[ttarget + MMFSA_D_CTX], scr1;			\
	sllx	scr1, TTARGET_CTX_SHIFT, scr1;			\
	ldx	[ttarget + MMFSA_D_ADDR], ttarget;		\
	srlx	ttarget, TTARGET_VA_SHIFT, ttarget;		\
	or	ttarget, scr1, ttarget

/*
 * Synthesize/get data/instruction psuedo tag access register values
 * from the MMU fault area (context is 0 for kernel, 1 for invalid, 2 for user)
 *
 * In:
 *   dtagacc, itagacc, scr1, scr2 = scratch registers
 * Out:
 *   dtagacc = MMU data tag access register value w/psuedo-context
 *   itagacc = MMU instruction tag access register value w/pseudo-context
 */
#define	GET_MMU_BOTH_TAGACC(dtagacc, itagacc, scr1, scr2)	\
	MMU_FAULT_STATUS_AREA(scr1);				\
	ldx	[scr1 + MMFSA_D_ADDR], scr2;			\
	ldx	[scr1 + MMFSA_D_CTX], dtagacc;			\
	srlx	scr2, MMU_PAGESHIFT, scr2;	/* align to page boundary */ \
	cmp	dtagacc, USER_CONTEXT_TYPE;			\
	sllx	scr2, MMU_PAGESHIFT, scr2;			\
	movgu	%icc, USER_CONTEXT_TYPE, dtagacc;		\
	or	scr2, dtagacc, dtagacc;				\
	ldx	[scr1 + MMFSA_I_ADDR], scr2;			\
	ldx	[scr1 + MMFSA_I_CTX], itagacc;			\
	srlx	scr2, MMU_PAGESHIFT, scr2;	/* align to page boundry */ \
	cmp	itagacc, USER_CONTEXT_TYPE;			\
	sllx	scr2, MMU_PAGESHIFT, scr2;			\
	movgu	%icc, USER_CONTEXT_TYPE, itagacc;		\
	or	scr2, itagacc, itagacc

/*
 * Synthesize/get MMU data fault address from the MMU fault area
 *
 * In:
 *   daddr, scr1 = scratch registers
 * Out:
 *   daddr = MMU data fault address
 */
#define	GET_MMU_D_ADDR(daddr, scr1)				\
	MMU_FAULT_STATUS_AREA(scr1);				\
	ldx	[scr1 + MMFSA_D_ADDR], daddr

/*
 * Get pseudo-tagacc value and context from the MMU fault area.  Pseudo-tagacc
 * is the faulting virtual address OR'd with 0 for KCONTEXT, INVALID_CONTEXT
 * (1) for invalid context, and USER_CONTEXT (2) for user context.
 *
 * In:
 *   tagacc, ctxtype = scratch registers
 * Out:
 *   tagacc = MMU instruction tag access register value
 *   ctxtype = context type (KCONTEXT, INVALID_CONTEXT or USER_CONTEXT)
 */
#define	GET_MMU_I_PTAGACC_CTXTYPE(ptagacc, ctxtype)			\
	MMU_FAULT_STATUS_AREA(ctxtype);					\
	ldx	[ctxtype + MMFSA_I_ADDR], ptagacc;			\
	ldx	[ctxtype + MMFSA_I_CTX], ctxtype;			\
	srlx	ptagacc, MMU_PAGESHIFT, ptagacc; /* align to page boundary */ \
	cmp	ctxtype, USER_CONTEXT_TYPE;				\
	sllx	ptagacc, MMU_PAGESHIFT, ptagacc;			\
	movgu	%icc, USER_CONTEXT_TYPE, ctxtype;			\
	or	ptagacc, ctxtype, ptagacc

/*
 * Load ITLB entry
 *
 * In:
 *   tte = reg containing tte
 *   scr1, scr2, scr3, scr4 = scratch registers
 */
#define	ITLB_STUFF(tte, scr1, scr2, scr3, scr4)		\
	mov	%o0, scr1;				\
	mov	%o1, scr2;				\
	mov	%o2, scr3;				\
	mov	%o3, scr4;				\
	MMU_FAULT_STATUS_AREA(%o2);			\
	ldx	[%o2 + MMFSA_I_ADDR], %o0;		\
	ldx	[%o2 + MMFSA_I_CTX], %o1;		\
	mov	tte, %o2;				\
	mov	MAP_ITLB, %o3;				\
	ta	MMU_MAP_ADDR;				\
	/* BEGIN CSTYLED */				\
	brnz,a,pn %o0, ptl1_panic;			\
	  mov	PTL1_BAD_HCALL, %g1;			\
	/* END CSTYLED */				\
	mov	scr1, %o0;				\
	mov	scr2, %o1;				\
	mov	scr3, %o2;				\
	mov	scr4, %o3

/*
 * Load DTLB entry
 *
 * In:
 *   tte = reg containing tte
 *   scr1, scr2, scr3, scr4 = scratch registers
 */
#define	DTLB_STUFF(tte, scr1, scr2, scr3, scr4)		\
	mov	%o0, scr1;				\
	mov	%o1, scr2;				\
	mov	%o2, scr3;				\
	mov	%o3, scr4;				\
	MMU_FAULT_STATUS_AREA(%o2);			\
	ldx	[%o2 + MMFSA_D_ADDR], %o0;		\
	ldx	[%o2 + MMFSA_D_CTX], %o1;		\
	mov	tte, %o2;				\
	mov	MAP_DTLB, %o3;				\
	ta	MMU_MAP_ADDR;				\
	/* BEGIN CSTYLED */				\
	brnz,a,pn %o0, ptl1_panic;			\
	  mov	PTL1_BAD_HCALL, %g1;			\
	/* END CSTYLED */				\
	mov	scr1, %o0;				\
	mov	scr2, %o1;				\
	mov	scr3, %o2;				\
	mov	scr4, %o3

/*
 * Returns PFN given the TTE and vaddr
 *
 * In:
 *   tte = reg containing tte
 *   vaddr = reg containing vaddr
 *   scr1, scr2, scr3 = scratch registers
 * Out:
 *   tte = PFN value
 */
#define	TTETOPFN(tte, vaddr, label, scr1, scr2, scr3)			\
	and	tte, TTE_SZ_BITS, scr1;		/* scr1 = ttesz */	\
	sllx	tte, TTE_PA_LSHIFT, tte;				\
	sllx	scr1, 1, scr2;						\
	add	scr2, scr1, scr2;		/* mulx 3 */		\
	add	scr2, MMU_PAGESHIFT + TTE_PA_LSHIFT, scr3;		\
	/* CSTYLED */							\
	brz,pt	scr2, label##1;					\
	srlx	tte, scr3, tte;						\
	sllx	tte, scr2, tte;						\
	set	1, scr1;						\
	add	scr2, MMU_PAGESHIFT, scr3;				\
	sllx	scr1, scr3, scr1;					\
	sub	scr1, 1, scr1;	/* scr1=TTE_PAGE_OFFSET(ttesz) */	\
	and	vaddr, scr1, scr2;					\
	srln	scr2, MMU_PAGESHIFT, scr2;				\
	or	tte, scr2, tte;						\
	/* CSTYLED */							\
label##1:

/*
 * TTE_SET_REF_ML is a macro that updates the reference bit if it is
 * not already set.
 *
 * Parameters:
 * tte      = reg containing tte
 * ttepa    = physical pointer to tte
 * tsbarea  = tsb miss area
 * tmp1     = tmp reg
 * tmp2     = tmp reg
 * label    = temporary label
 */

#define	TTE_SET_REF_ML(tte, ttepa, tsbarea, tmp1, tmp2, label)		\
	/* BEGIN CSTYLED */						\
	/* check reference bit */					\
	btst	TTE_REF_INT, tte;					\
	bnz,pt	%xcc, label##2;	/* if ref bit set-skip ahead */	\
	nop;								\
	/* update reference bit */					\
label##1:								\
	or	tte, TTE_REF_INT, tmp1;					\
	casxa	[ttepa]ASI_MEM, tte, tmp1; 	/* update ref bit */	\
	cmp	tte, tmp1;						\
	bne,a,pn %xcc, label##1;					\
	ldxa	[ttepa]ASI_MEM, tte;	/* MMU_READTTE through pa */	\
	or	tte, TTE_REF_INT, tte;					\
label##2:								\
	/* END CSTYLED */


/*
 * TTE_SET_REFMOD_ML is a macro that updates the reference and modify bits
 * if not already set.
 *
 * Parameters:
 * tte      = reg containing tte
 * ttepa    = physical pointer to tte
 * tsbarea  = tsb miss area
 * tmp1     = tmp reg
 * tmp2     = tmp reg
 * label    = temporary label
 * exitlabel = label where to jump to if write perm bit not set.
 */

#define	TTE_SET_REFMOD_ML(tte, ttepa, tsbarea, tmp1, tmp2, label,	\
	exitlabel)							\
	/* BEGIN CSTYLED */						\
	/* check reference bit */					\
	btst	TTE_WRPRM_INT, tte;					\
	bz,pn	%xcc, exitlabel;	/* exit if wr_perm no set */	\
	  btst	TTE_HWWR_INT, tte;					\
	bnz,pn	%xcc, label##2;	/* nothing to do */		\
	  nop;								\
	/* update reference bit */					\
label##1:								\
	or	tte, TTE_HWWR_INT | TTE_REF_INT, tmp1;			\
	casxa	[ttepa]ASI_MEM, tte, tmp1; /* update ref/mod bit */	\
	cmp	tte, tmp1;						\
	bne,a,pn %xcc, label##1;					\
	  ldxa	[ttepa]ASI_MEM, tte;	/* MMU_READTTE through pa */	\
	or	tte, TTE_HWWR_INT | TTE_REF_INT, tte;			\
label##2:								\
	/* END CSTYLED */
/*
 * Get TSB base register from the scratchpad for
 * shared contexts
 *
 * In:
 *   tsbmiss = pointer to tsbmiss area
 *   tsbmissoffset = offset to right tsb pointer
 *   tsbreg = scratch
 * Out:
 *   tsbreg = tsbreg from the specified scratchpad register
 */
#define	GET_UTSBREG_SHCTX(tsbmiss, tsbmissoffset, tsbreg)		\
	ldx	[tsbmiss + tsbmissoffset], tsbreg


/*
 * Get the location of the TSB entry in the first TSB to probe
 *
 * In:
 *   tagacc = tag access register (not clobbered)
 *   tsbe, tmp1, tmp2 = scratch registers
 * Out:
 *   tsbe = pointer to the tsbe in the 1st TSB
 */

#define	GET_1ST_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)			\
	/* BEGIN CSTYLED */						\
	mov	SCRATCHPAD_UTSBREG1, tmp1				;\
	ldxa	[tmp1]ASI_SCRATCHPAD, tsbe	/* get tsbreg */	;\
	and	tsbe, TSB_SOFTSZ_MASK, tmp2	/* tmp2=szc */		;\
	andn	tsbe, TSB_SOFTSZ_MASK, tsbe	/* tsbbase */		;\
	mov	TSB_ENTRIES(0), tmp1	/* nentries in TSB size 0 */	;\
	sllx	tmp1, tmp2, tmp1	/* tmp1 = nentries in TSB */	;\
	sub	tmp1, 1, tmp1		/* mask = nentries - 1 */	;\
	srlx	tagacc, MMU_PAGESHIFT, tmp2 				;\
	and	tmp2, tmp1, tmp1	/* tsbent = virtpage & mask */	;\
	sllx	tmp1, TSB_ENTRY_SHIFT, tmp1	/* entry num --> ptr */	;\
	add	tsbe, tmp1, tsbe	/* add entry offset to TSB base */ ;\
	/* END CSTYLED */


/*
 * Will probe the first TSB, and if it finds a match, will insert it
 * into the TLB and retry.
 *
 * tsbe_ptr = precomputed first TSB entry pointer (in, ro)
 * vpg_4m = 4M virtual page number for tag matching  (in, ro)
 * label = where to branch to if this is a miss (text)
 * %asi = atomic ASI to use for the TSB access
 *
 * For trapstat, we have to explicily use these registers.
 * g4 = location tag will be retrieved into from TSB (out)
 * g5 = location data(tte) will be retrieved into from TSB (out)
 */
#define	PROBE_1ST_DTSB(tsbe_ptr, vpg_4m, label)	/* g4/g5 clobbered */	\
	/* BEGIN CSTYLED */						\
	ldda	[tsbe_ptr]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */ ;\
	cmp	%g4, vpg_4m		/* compare tag w/ TSB */	;\
	bne,pn	%xcc, label##1	/* branch if !match */		;\
	  nop								;\
	brgez,pn %g5, label##1					;\
	  nop								;\
	TT_TRACE(trace_tsbhit)						;\
	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)				;\
	/* trapstat expects tte in %g5 */				;\
	retry				/* retry faulted instruction */	;\
label##1:								\
	/* END CSTYLED */


/*
 * Same as above, only if the TTE doesn't have the execute
 * bit set, will branch to exec_fault directly.
 */
#define	PROBE_1ST_ITSB(tsbe_ptr, vpg_4m, label)				\
	/* BEGIN CSTYLED */						\
	ldda	[tsbe_ptr]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */ ;\
	cmp	%g4, vpg_4m		/* compare tag w/ TSB */	;\
	bne,pn	%xcc, label##1	/* branch if !match */		;\
	  nop								;\
	brgez,pn %g5, label##1					;\
	  nop								;\
	andcc	%g5, TTE_EXECPRM_INT, %g0  /* check execute bit */	;\
	bz,pn	%icc, exec_fault					;\
	  nop								;\
	TT_TRACE(trace_tsbhit)						;\
	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)				;\
	retry				/* retry faulted instruction */	;\
label##1:								\
	/* END CSTYLED */

/*
 * vpg_4m = 4M virtual page number for tag matching (in)
 * tsbe_ptr = precomputed second TSB entry pointer (in)
 * label = label to use to make branch targets unique (text)
 *
 * For trapstat, we have to explicity use these registers.
 * g4 = tag portion of TSBE (out)
 * g5 = data portion of TSBE (out)
 */
#define	PROBE_2ND_DTSB(tsbe_ptr, vpg_4m, label)				\
	/* BEGIN CSTYLED */						\
	ldda	[tsbe_ptr]ASI_QUAD_LDD_PHYS, %g4  /* g4 = tag, g5 = data */ ;\
	/* since we are looking at 2nd tsb, if it's valid, it must be 4M */ ;\
	cmp	%g4, vpg_4m						;\
	bne,pn	%xcc, label##1					;\
	  nop								;\
	brgez,pn %g5, label##1					;\
	  nop								;\
	mov	tsbe_ptr, %g1		/* trace_tsbhit wants ptr in %g1 */ ;\
	TT_TRACE(trace_tsbhit)						;\
	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)				;\
	/* trapstat expects tte in %g5 */				;\
	retry				/* retry faulted instruction */	;\
label##1:								\
	/* END CSTYLED */


/*
 * Same as above, with the following additions:
 * If the TTE found is not executable, branch directly
 * to exec_fault.  If a TSB miss, branch to TSB miss handler.
 */
#define	PROBE_2ND_ITSB(tsbe_ptr, vpg_4m)				\
	/* BEGIN CSTYLED */						\
	ldda	[tsbe_ptr]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */ ;\
	cmp	%g4, vpg_4m		/* compare tag w/ TSB */	;\
	bne,pn	%xcc, sfmmu_tsb_miss_tt	/* branch if !match */		;\
	  nop								;\
	brgez,pn %g5, sfmmu_tsb_miss_tt					;\
	  nop								;\
	andcc	%g5, TTE_EXECPRM_INT, %g0  /* check execute bit */	;\
	bz,pn	%icc, exec_fault					;\
	  mov	tsbe_ptr, %g1		/* trap trace wants ptr in %g1 */ ;\
	TT_TRACE(trace_tsbhit)						;\
	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)				;\
	retry				/* retry faulted instruction */	\
	/* END CSTYLED */

/*
 * 1. Get ctx1. The traptype is supplied by caller.
 * 2. If iTSB miss, store in MMFSA_I_CTX
 * 3. if dTSB miss, store in MMFSA_D_CTX
 * 4. Thus the [D|I]TLB_STUFF will work as expected.
 */
#define	SAVE_CTX1(traptype, ctx1, tmp, label)				\
	/* BEGIN CSTYLED */						\
	mov	MMU_SCONTEXT1, tmp					;\
	ldxa	[tmp]ASI_MMU_CTX, ctx1					;\
	MMU_FAULT_STATUS_AREA(tmp)					;\
	cmp     traptype, FAST_IMMU_MISS_TT				;\
	be,a,pn %icc, label						;\
	  stx	ctx1, [tmp + MMFSA_I_CTX] 				;\
	cmp     traptype, T_INSTR_MMU_MISS				;\
	be,a,pn %icc, label						;\
	  stx	ctx1, [tmp + MMFSA_I_CTX]				;\
	stx	ctx1, [tmp + MMFSA_D_CTX]				;\
label:
	/* END CSTYLED */

#endif /* _ASM */

#ifdef	__cplusplus
}
#endif

#endif	/* _VM_MACH_SFMMU_H */