xref: /illumos-gate/usr/src/uts/sun4u/sys/fpras_impl.h (revision 5d9d9091)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef	_SYS_FPRAS_IMPL_H
#define	_SYS_FPRAS_IMPL_H

#include <sys/fpras.h>

#if !defined(_ASM)
#include <sys/types.h>
#else
#include <sys/intreg.h>
#include <sys/errno.h>
#endif	/* _ASM */

#ifdef	__cplusplus
extern "C" {
#endif

/*
 * sun4u/cheetah fpRAS implementation.  Arrays etc will be allocated in sun4u
 * post_startup() if fpras_implemented is set.  This file may belong at
 * the cpu level (eg, cheetahregs.h) but most of it should be common
 * when fpRAS support is added for additional cpu types so we introduce
 * it at the sun4u level (and set fpras_implemented in cpu_setup).
 *
 * If fpRAS is implemented on a sun4u/cpu combination that does not use
 * an ASR for %stick then the FPRAS_INTERVAL macro will need some
 * modification.
 */

/*
 * Upper bound for check frequency per cpu and per operation.  For example, if
 * this is 100 then for cpuid N performing a bcopy if that cpu has not
 * performed a checked bcopy in the the last 1/100th of a second then
 * we'll check the current operation.  A value of 0 will check every operation.
 * Modifying fpras_frequency from its default is not recommended.
 * fpras_interval is computed from fpras_frequency.
 */
#if !defined(_ASM)
extern int fpras_frequency;
extern int64_t fpras_interval;
#endif	/* _ASM */
#define	FPRAS_DEFAULT_FREQUENCY	100

#if !defined(_ASM)

/*
 * Structure of a check function.  The preamble prepares registers for the
 * upcoming calculation that is performed in blk0 and blk1.  One of those
 * blocks will be rewritten as part of an FPRAS_REWRITE operation.  Finally
 * the result checked in chkresult should be as predetermined, and we should
 * return zero on success and nonzero on failure.  If an illegal instruction
 * is encountered in the execution of the check function then we trampoline
 * to the final three instructions to return a different value.
 *
 * Note that the size of this structure is a power of 2 as is the
 * size of a struct fpras_chkfngrp.  The asm macros below rely on this
 * in performing bit shifts instead of mulx.
 */
struct fpras_chkfn {
	uint32_t	fpras_preamble[16];
	uint32_t	fpras_blk0[16];
	uint32_t	fpras_blk1[16];
	uint32_t	fpras_chkresult[13];
	uint32_t	fpras_trampoline[3];
};

/*
 * Check function constructed to match a struct fpras_chkfn
 */
extern int fpras_chkfn_type1(void);

/*
 * A group of check functions, one for each operation type.  These will
 * be the check functions for copy operations on a particular processor.
 */
struct fpras_chkfngrp {
	struct fpras_chkfn fpras_fn[FPRAS_NCOPYOPS];
};

/*
 * Where we store check functions for execution.  Indexed by cpuid and
 * function within that for cacheline friendliness.  Startup code
 * copies the check function into this array.  The fpRAS mechanism will
 * rewrite one of fpras_blk0 or fpras_blk1 before calling the check function
 * for a cpuid & copy function combination.
 */
extern struct fpras_chkfngrp *fpras_chkfngrps;

#endif	/* !_ASM */

#if defined(_ASM)

/* BEGIN CSTYLED */

/*
 * The INTERVAL macro decides whether we will check this copy operation,
 * based on performing no more than 1 check per cpu & operation in a specified
 * time interval.  If it decides to abort this check (ie, we have checked
 * recently) then it returns doex NULL, otherwise doex is the address of the
 * check function to execute later.  Migration must have been prevented before
 * calling this macro.  Args:
 *
 *	operation (immediate): one of FPRAS_BCOPY etc
 *	blk (immediate): which block to copy
 *	doex (register): register in which to return check function address
 *	tmp1 (register): used for scratch, not preserved
 *	tmp2 (register): used for scratch, not preserved
 *	tmp3 (register): used for scratch, not preserved
 *	tmp4 (register): used for scratch, not preserved
 *	label: free local numeric label
 */

#define	FPRAS_INTERVAL(operation, blk, doex, tmp1, tmp2, tmp3, tmp4, label) \
	sethi	%hi(fpras_interval), tmp1				;\
	ldx	[tmp1 + %lo(fpras_interval)], tmp1			;\
	brlz,pn	tmp1, label##f	/* not initialized? */		;\
	  clr	doex							;\
	sethi	%hi(fpras_disableids), tmp2				;\
	ld	[tmp2 + %lo(fpras_disableids)], tmp2			;\
	mov	0x1, tmp3						;\
	sll	tmp3, operation, tmp3					;\
	btst	tmp3, tmp2						;\
	bnz,a,pn %icc, label##f	/* disabled for this op? */	;\
	  nop								;\
	set	fpras_chkfn_type1, tmp2					;\
	prefetch [tmp2 + (FPRAS_BLK0 + blk * 64)], #one_read		;\
	ldn	[THREAD_REG + T_CPU], tmp2				;\
	ldn	[tmp2 + CPU_PRIVATE], tmp2				;\
	brz,pn	tmp2, label##f	/* early in startup? */		;\
	  mov	operation, tmp3						;\
	sll	tmp3, 3, tmp3						;\
	set	CHPR_FPRAS_TIMESTAMP, tmp4				;\
	add	tmp2, tmp4, tmp2					;\
	add	tmp2, tmp3, tmp2	/* keep ptr for update */	;\
	ldx	[tmp2], tmp3		/* last timestamp */		;\
	rd	STICK, doex		/* doex is a scratch here */	;\
	sub	doex, tmp3, tmp4	/* delta since last check */	;\
	cmp	tmp4, tmp1		/* compare delta to interval */	;\
	blu,a,pn %xcc, label##f					;\
	  clr	doex							;\
	stx	doex, [tmp2]		/* updated timestamp */		;\
	ldn	[THREAD_REG + T_CPU], tmp1				;\
	ld	[tmp1 + CPU_ID], tmp1					;\
	sethi	%hi(fpras_chkfngrps), doex				;\
	ldn	[doex + %lo(fpras_chkfngrps)], doex			;\
	sll	tmp1, FPRAS_CHKFNGRP_SIZE_SHIFT, tmp1			;\
	add	doex, tmp1, doex					;\
	mov	operation, tmp1						;\
	sll	tmp1, FPRAS_CHKFN_SIZE_SHIFT, tmp1			;\
	add	doex, tmp1, doex	/* address of check function */	;\
label:

/*
 * The REWRITE macro copies an instruction block from fpras_chkfn_type1
 * into a per-cpu fpras check function.
 * If doex is NULL it must not attempt any copy, and must leave doex NULL.
 * CPU migration of this thread must be prevented before we call this macro.
 * We must have checked for fp in use (and saved state, including the
 * quadrant of registers indicated by the fpq argument and fp enabled before
 * using this macro.  Args:
 *
 *	blk (immediate): as above
 *	doex (register): register in which to return check function addr
 *	[fpq (fp register): frf quadrant to be used (%f0/%f16/%f32/%f48)]
 *		This is used on type 1 rewrite only - on others the
 *		quadrant is implicit/hardcoded in the macro name.
 *	tmp1 (register): used for scratch, not preserved
 *	label1: free local numeric label
 *	[label2: free local numeric label]
 *		This is used in type 2 only.
 *
 * Note that the REWRITE macros do not perform a flush instruction -
 * flush is not necessary on Cheetah derivative processors in which
 * i$ snoops for invalidations.
 */

/*
 * Rewrite type 1 will work with any instruction pattern - it just block
 * loads and block stores the given block.  A membar after block store
 * forces the block store to complete before upcoming reuse of the
 * fpregs in the block;  the block load is blocking on sun4u/cheetah
 * so no need for a membar after it.
 */

#define	FPRAS_REWRITE_TYPE1(blk, doex, fpq, tmp1, label)	\
	brz,pn  doex, label##f				;\
	  sethi	%hi(fpras_chkfn_type1), tmp1			;\
	add	tmp1, %lo(fpras_chkfn_type1), tmp1		;\
	add	tmp1, FPRAS_BLK0 + blk * 64, tmp1		;\
	ldda	[tmp1]ASI_BLK_P, fpq				;\
	add	doex, FPRAS_BLK0 + blk * 64, tmp1		;\
	stda	fpq, [tmp1]ASI_BLK_P				;\
	membar	#Sync						;\
label:

/*
 * Rewrite type 2 will only work with instruction blocks that satisfy
 * this particular repeat pattern.  Note that the frf quadrant to
 * use is implicit in the macro name and had better match what the
 * copy function is preserving.
*
 * The odd looking repetition in the initial loop is designed to open
 * up boths paths from prefetch cache to the frf - unrolling the loop
 * would defeat this.  In addition we perform idempotent faligndata
 * manipulations using %tick as a randomly aligned address (this only
 * works for address that aren't doubleword aligned).
 */
#define	FPRAS_REWRITE_TYPE2Q1(blk, doex, tmp1, tmp2, label1, label2)	\
	brz,pn	doex, label1##f					;\
	  mov	0x2, tmp1						;\
	set	fpras_chkfn_type1, tmp2					;\
label2:									;\
	deccc		tmp1						;\
	ldd		[tmp2 + (FPRAS_BLK0 + blk * 64)], %f4		;\
	ldd		[tmp2 + (FPRAS_BLK0 + blk * 64) + 8], %f2	;\
	bnz,a,pt	%icc, label2##b				;\
	  fsrc1		%f4, %f0					;\
	rdpr		%tick, tmp1					;\
	fsrc1		%f4, %f8					;\
	fsrc1		%f2, %f10					;\
	btst		0x7, tmp1					;\
	alignaddr	tmp1, %g0, %g0	/* changes %gsr */		;\
	bz,pn		%icc, label2##f				;\
	  faligndata	%f2, %f4, %f6					;\
	faligndata	%f0, %f2, %f12					;\
	alignaddrl	tmp1, %g0, %g0					;\
	faligndata	%f12, %f6, %f6					;\
label2:									;\
	add		doex, FPRAS_BLK0 + blk * 64, tmp1		;\
	fsrc2		%f8, %f12					;\
	fsrc1		%f6, %f14					;\
	stda		%f0, [tmp1]ASI_BLK_P				;\
	membar		#Sync						;\
label1:

#define	FPRAS_REWRITE_TYPE2Q2(blk, doex, tmp1, tmp2, label1, label2)	\
	brz,pn	doex, label1##f					;\
	  mov	0x2, tmp1						;\
	set	fpras_chkfn_type1, tmp2					;\
label2:									;\
	deccc		tmp1						;\
	ldd		[tmp2 + (FPRAS_BLK0 + blk * 64)], %f20	;\
	ldd		[tmp2 + (FPRAS_BLK0 + blk * 64) + 8], %f18	;\
	bnz,a,pt	%icc, label2##b				;\
	  fsrc1		%f20, %f16					;\
	rdpr		%tick, tmp1					;\
	fsrc1		%f20, %f24					;\
	fsrc1		%f18, %f26					;\
	btst		0x7, tmp1					;\
	alignaddr	tmp1, %g0, %g0	/* changes %gsr */		;\
	bz,pn		%icc, label2##f				;\
	  faligndata	%f18, %f20, %f22				;\
	faligndata	%f16, %f18, %f28				;\
	alignaddrl	tmp1, %g0, %g0					;\
	faligndata	%f28, %f22, %f22				;\
label2:									;\
	add		doex, FPRAS_BLK0 + blk * 64, tmp1		;\
	fsrc2		%f24, %f28					;\
	fsrc1		%f22, %f30					;\
	stda		%f16, [tmp1]ASI_BLK_P				;\
	membar		#Sync						;\
label1:

/*
 * The CHECK macro takes the 'doex' address of the check function to
 * execute and jumps to it (if not NULL). If the check function returns
 * nonzero then the check has failed and the CHECK macro must initiate
 * an appropriate failure action.  Illegal instruction trap handlers
 * will also recognise traps in this PC range as fp failures.  Thread
 * migration must only be reallowed after completion of this check.  The
 * CHECK macro should be treated as a CALL/JMPL - output registers are
 * forfeit after using it.  If the call to fpras_failure returns
 * (it may decide to panic) then invoke lofault handler (which must exist)
 * to return an error (be sure to use this macro before restoring original
 * lofault setup in copy functions).  Note that the lofault handler is the
 * copyops aware proxy handler which will perform other tidy up operations
 * (unbind, fp state restore) that would normally have been done in the tail
 * of the copy function.
 *
 *	operation (immedidate): as above
 *	doex (register): doex value returned from the REWRITE
 *	label: free local numeric label
 */

#define	FPRAS_CHECK(operation, doex, label)				\
	brz,pn	doex, label##f					;\
	  nop								;\
	jmpl	doex, %o7						;\
	  nop								;\
	cmp	%o0, FPRAS_OK						;\
	be	%icc, label##f					;\
	  nop								;\
	mov	%o0, %o1	/* how detected */			;\
	call	fpras_failure	/* take failure action */		;\
	  mov	operation, %o0						;\
	ldn	[THREAD_REG + T_LOFAULT], doex				;\
	jmp	doex							;\
	  mov	EFAULT, %g1						;\
label:

/* END CSTYLED */

#endif	/* _ASM */

#ifdef	__cplusplus
}
#endif

#endif	/* _SYS_FPRAS_IMPL_H */