xref: /illumos-gate/usr/src/uts/sun4u/cpu/us3_common.c (revision 56870e8c)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/systm.h>
#include <sys/ddi.h>
#include <sys/sysmacros.h>
#include <sys/archsystm.h>
#include <sys/vmsystm.h>
#include <sys/machparam.h>
#include <sys/machsystm.h>
#include <sys/machthread.h>
#include <sys/cpu.h>
#include <sys/cmp.h>
#include <sys/elf_SPARC.h>
#include <vm/vm_dep.h>
#include <vm/hat_sfmmu.h>
#include <vm/seg_kpm.h>
#include <sys/cpuvar.h>
#include <sys/cheetahregs.h>
#include <sys/us3_module.h>
#include <sys/async.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <sys/dditypes.h>
#include <sys/prom_debug.h>
#include <sys/prom_plat.h>
#include <sys/cpu_module.h>
#include <sys/sysmacros.h>
#include <sys/intreg.h>
#include <sys/clock.h>
#include <sys/platform_module.h>
#include <sys/machtrap.h>
#include <sys/ontrap.h>
#include <sys/panic.h>
#include <sys/memlist.h>
#include <sys/bootconf.h>
#include <sys/ivintr.h>
#include <sys/atomic.h>
#include <sys/taskq.h>
#include <sys/note.h>
#include <sys/ndifm.h>
#include <sys/ddifm.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/fm/cpu/UltraSPARC-III.h>
#include <sys/fpras_impl.h>
#include <sys/dtrace.h>
#include <sys/watchpoint.h>
#include <sys/plat_ecc_unum.h>
#include <sys/cyclic.h>
#include <sys/errorq.h>
#include <sys/errclassify.h>
#include <sys/pghw.h>
#include <sys/clock_impl.h>

#ifdef	CHEETAHPLUS_ERRATUM_25
#include <sys/xc_impl.h>
#endif	/* CHEETAHPLUS_ERRATUM_25 */

ch_cpu_logout_t	clop_before_flush;
ch_cpu_logout_t	clop_after_flush;
uint_t	flush_retries_done = 0;
/*
 * Note that 'Cheetah PRM' refers to:
 *   SPARC V9 JPS1 Implementation Supplement: Sun UltraSPARC-III
 */

/*
 * Per CPU pointers to physical address of TL>0 logout data areas.
 * These pointers have to be in the kernel nucleus to avoid MMU
 * misses.
 */
uint64_t ch_err_tl1_paddrs[NCPU];

/*
 * One statically allocated structure to use during startup/DR
 * to prevent unnecessary panics.
 */
ch_err_tl1_data_t ch_err_tl1_data;

/*
 * Per CPU pending error at TL>0, used by level15 softint handler
 */
uchar_t ch_err_tl1_pending[NCPU];

/*
 * For deferred CE re-enable after trap.
 */
taskq_t		*ch_check_ce_tq;

/*
 * Internal functions.
 */
static int cpu_async_log_err(void *flt, errorq_elem_t *eqep);
static void cpu_log_diag_info(ch_async_flt_t *ch_flt);
static void cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason,
    ecc_type_to_info_t *eccp, ch_diag_data_t *cdp);
static int cpu_flt_in_memory_one_event(ch_async_flt_t *ch_flt,
    uint64_t t_afsr_bit);
static int clear_ecc(struct async_flt *ecc);
#if defined(CPU_IMP_ECACHE_ASSOC)
static int cpu_ecache_line_valid(ch_async_flt_t *ch_flt);
#endif
int cpu_ecache_set_size(struct cpu *cp);
static int cpu_ectag_line_invalid(int cachesize, uint64_t tag);
int cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr);
uint64_t cpu_ectag_to_pa(int setsize, uint64_t tag);
int cpu_ectag_pa_to_subblk_state(int cachesize,
				uint64_t subaddr, uint64_t tag);
static void cpu_flush_ecache_line(ch_async_flt_t *ch_flt);
static int afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit);
static int afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit);
static int afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit);
static int afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit);
static int synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit);
static int cpu_get_mem_unum_synd(int synd_code, struct async_flt *, char *buf);
static void cpu_uninit_ecache_scrub_dr(struct cpu *cp);
static void cpu_scrubphys(struct async_flt *aflt);
static void cpu_payload_add_aflt(struct async_flt *, nvlist_t *, nvlist_t *,
    int *, int *);
static void cpu_payload_add_ecache(struct async_flt *, nvlist_t *);
static void cpu_ereport_init(struct async_flt *aflt);
static int cpu_check_secondary_errors(ch_async_flt_t *, uint64_t, uint64_t);
static uint8_t cpu_flt_bit_to_plat_error(struct async_flt *aflt);
static void cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen,
    uint64_t nceen, ch_cpu_logout_t *clop);
static int cpu_ce_delayed_ec_logout(uint64_t);
static int cpu_matching_ecache_line(uint64_t, void *, int, int *);
static int cpu_error_is_ecache_data(int, uint64_t);
static void cpu_fmri_cpu_set(nvlist_t *, int);
static int cpu_error_to_resource_type(struct async_flt *aflt);

#ifdef	CHEETAHPLUS_ERRATUM_25
static int mondo_recover_proc(uint16_t, int);
static void cheetah_nudge_init(void);
static void cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr,
    cyc_time_t *when);
static void cheetah_nudge_buddy(void);
#endif	/* CHEETAHPLUS_ERRATUM_25 */

#if defined(CPU_IMP_L1_CACHE_PARITY)
static void cpu_dcache_parity_info(ch_async_flt_t *ch_flt);
static void cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index);
static void cpu_record_dc_data_parity(ch_async_flt_t *ch_flt,
    ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word);
static void cpu_icache_parity_info(ch_async_flt_t *ch_flt);
static void cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index);
static void cpu_pcache_parity_info(ch_async_flt_t *ch_flt);
static void cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index);
static void cpu_payload_add_dcache(struct async_flt *, nvlist_t *);
static void cpu_payload_add_icache(struct async_flt *, nvlist_t *);
#endif	/* CPU_IMP_L1_CACHE_PARITY */

int (*p2get_mem_info)(int synd_code, uint64_t paddr,
    uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep,
    int *segsp, int *banksp, int *mcidp);

/*
 * This table is used to determine which bit(s) is(are) bad when an ECC
 * error occurs.  The array is indexed by an 9-bit syndrome.  The entries
 * of this array have the following semantics:
 *
 *      00-127  The number of the bad bit, when only one bit is bad.
 *      128     ECC bit C0 is bad.
 *      129     ECC bit C1 is bad.
 *      130     ECC bit C2 is bad.
 *      131     ECC bit C3 is bad.
 *      132     ECC bit C4 is bad.
 *      133     ECC bit C5 is bad.
 *      134     ECC bit C6 is bad.
 *      135     ECC bit C7 is bad.
 *      136     ECC bit C8 is bad.
 *	137-143 reserved for Mtag Data and ECC.
 *      144(M2) Two bits are bad within a nibble.
 *      145(M3) Three bits are bad within a nibble.
 *      146(M3) Four bits are bad within a nibble.
 *      147(M)  Multiple bits (5 or more) are bad.
 *      148     NO bits are bad.
 * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-4,11-5.
 */

#define	C0	128
#define	C1	129
#define	C2	130
#define	C3	131
#define	C4	132
#define	C5	133
#define	C6	134
#define	C7	135
#define	C8	136
#define	MT0	137	/* Mtag Data bit 0 */
#define	MT1	138
#define	MT2	139
#define	MTC0	140	/* Mtag Check bit 0 */
#define	MTC1	141
#define	MTC2	142
#define	MTC3	143
#define	M2	144
#define	M3	145
#define	M4	146
#define	M	147
#define	NA	148
#if defined(JALAPENO) || defined(SERRANO)
#define	S003	149	/* Syndrome 0x003 => likely from CPU/EDU:ST/FRU/BP */
#define	S003MEM	150	/* Syndrome 0x003 => likely from WDU/WBP */
#define	SLAST	S003MEM	/* last special syndrome */
#else /* JALAPENO || SERRANO */
#define	S003	149	/* Syndrome 0x003 => likely from EDU:ST */
#define	S071	150	/* Syndrome 0x071 => likely from WDU/CPU */
#define	S11C	151	/* Syndrome 0x11c => likely from BERR/DBERR */
#define	SLAST	S11C	/* last special syndrome */
#endif /* JALAPENO || SERRANO */
#if defined(JALAPENO) || defined(SERRANO)
#define	BPAR0	152	/* syndrom 152 through 167 for bus parity */
#define	BPAR15	167
#endif	/* JALAPENO || SERRANO */

static uint8_t ecc_syndrome_tab[] =
{
NA,  C0,  C1, S003, C2,  M2,  M3,  47,  C3,  M2,  M2,  53,  M2,  41,  29,   M,
C4,   M,   M,  50,  M2,  38,  25,  M2,  M2,  33,  24,  M2,  11,   M,  M2,  16,
C5,   M,   M,  46,  M2,  37,  19,  M2,   M,  31,  32,   M,   7,  M2,  M2,  10,
M2,  40,  13,  M2,  59,   M,  M2,  66,   M,  M2,  M2,   0,  M2,  67,  71,   M,
C6,   M,   M,  43,   M,  36,  18,   M,  M2,  49,  15,   M,  63,  M2,  M2,   6,
M2,  44,  28,  M2,   M,  M2,  M2,  52,  68,  M2,  M2,  62,  M2,  M3,  M3,  M4,
M2,  26, 106,  M2,  64,   M,  M2,   2, 120,   M,  M2,  M3,   M,  M3,  M3,  M4,
#if defined(JALAPENO) || defined(SERRANO)
116, M2,  M2,  M3,  M2,  M3,   M,  M4,  M2,  58,  54,  M2,   M,  M4,  M4,  M3,
#else	/* JALAPENO || SERRANO */
116, S071, M2,  M3,  M2,  M3,   M,  M4,  M2,  58,  54,  M2,   M,  M4,  M4,  M3,
#endif	/* JALAPENO || SERRANO */
C7,  M2,   M,  42,   M,  35,  17,  M2,   M,  45,  14,  M2,  21,  M2,  M2,   5,
M,   27,   M,   M,  99,   M,   M,   3, 114,  M2,  M2,  20,  M2,  M3,  M3,   M,
M2,  23, 113,  M2, 112,  M2,   M,  51,  95,   M,  M2,  M3,  M2,  M3,  M3,  M2,
103,  M,  M2,  M3,  M2,  M3,  M3,  M4,  M2,  48,   M,   M,  73,  M2,   M,  M3,
M2,  22, 110,  M2, 109,  M2,   M,   9, 108,  M2,   M,  M3,  M2,  M3,  M3,   M,
102, M2,   M,   M,  M2,  M3,  M3,   M,  M2,  M3,  M3,  M2,   M,  M4,   M,  M3,
98,   M,  M2,  M3,  M2,   M,  M3,  M4,  M2,  M3,  M3,  M4,  M3,   M,   M,   M,
M2,  M3,  M3,   M,  M3,   M,   M,   M,  56,  M4,   M,  M3,  M4,   M,   M,   M,
C8,   M,  M2,  39,   M,  34, 105,  M2,   M,  30, 104,   M, 101,   M,   M,   4,
#if defined(JALAPENO) || defined(SERRANO)
M,    M, 100,   M,  83,   M,  M2,  12,  87,   M,   M,  57,  M2,   M,  M3,   M,
#else	/* JALAPENO || SERRANO */
M,    M, 100,   M,  83,   M,  M2,  12,  87,   M,   M,  57, S11C,  M,  M3,   M,
#endif	/* JALAPENO || SERRANO */
M2,  97,  82,  M2,  78,  M2,  M2,   1,  96,   M,   M,   M,   M,   M,  M3,  M2,
94,   M,  M2,  M3,  M2,   M,  M3,   M,  M2,   M,  79,   M,  69,   M,  M4,   M,
M2,  93,  92,   M,  91,   M,  M2,   8,  90,  M2,  M2,   M,   M,   M,   M,  M4,
89,   M,   M,  M3,  M2,  M3,  M3,   M,   M,   M,  M3,  M2,  M3,  M2,   M,  M3,
86,   M,  M2,  M3,  M2,   M,  M3,   M,  M2,   M,  M3,   M,  M3,   M,   M,  M3,
M,    M,  M3,  M2,  M3,  M2,  M4,   M,  60,   M,  M2,  M3,  M4,   M,   M,  M2,
M2,  88,  85,  M2,  84,   M,  M2,  55,  81,  M2,  M2,  M3,  M2,  M3,  M3,  M4,
77,   M,   M,   M,  M2,  M3,   M,   M,  M2,  M3,  M3,  M4,  M3,  M2,   M,   M,
74,   M,  M2,  M3,   M,   M,  M3,   M,   M,   M,  M3,   M,  M3,   M,  M4,  M3,
M2,  70, 107,  M4,  65,  M2,  M2,   M, 127,   M,   M,   M,  M2,  M3,  M3,   M,
80,  M2,  M2,  72,   M, 119, 118,   M,  M2, 126,  76,   M, 125,   M,  M4,  M3,
M2, 115, 124,   M,  75,   M,   M,  M3,  61,   M,  M4,   M,  M4,   M,   M,   M,
M,  123, 122,  M4, 121,  M4,   M,  M3, 117,  M2,  M2,  M3,  M4,  M3,   M,   M,
111,  M,   M,   M,  M4,  M3,  M3,   M,   M,   M,  M3,   M,  M3,  M2,   M,   M
};

#define	ESYND_TBL_SIZE	(sizeof (ecc_syndrome_tab) / sizeof (uint8_t))

#if !(defined(JALAPENO) || defined(SERRANO))
/*
 * This table is used to determine which bit(s) is(are) bad when a Mtag
 * error occurs.  The array is indexed by an 4-bit ECC syndrome. The entries
 * of this array have the following semantics:
 *
 *      -1	Invalid mtag syndrome.
 *      137     Mtag Data 0 is bad.
 *      138     Mtag Data 1 is bad.
 *      139     Mtag Data 2 is bad.
 *      140     Mtag ECC 0 is bad.
 *      141     Mtag ECC 1 is bad.
 *      142     Mtag ECC 2 is bad.
 *      143     Mtag ECC 3 is bad.
 * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-6.
 */
short mtag_syndrome_tab[] =
{
NA, MTC0, MTC1, M2, MTC2, M2, M2, MT0, MTC3, M2, M2,  MT1, M2, MT2, M2, M2
};

#define	MSYND_TBL_SIZE	(sizeof (mtag_syndrome_tab) / sizeof (short))

#else /* !(JALAPENO || SERRANO) */

#define	BSYND_TBL_SIZE	16

#endif /* !(JALAPENO || SERRANO) */

/*
 * Virtual Address bit flag in the data cache. This is actually bit 2 in the
 * dcache data tag.
 */
#define	VA13	INT64_C(0x0000000000000002)

/*
 * Types returned from cpu_error_to_resource_type()
 */
#define	ERRTYPE_UNKNOWN		0
#define	ERRTYPE_CPU		1
#define	ERRTYPE_MEMORY		2
#define	ERRTYPE_ECACHE_DATA	3

/*
 * CE initial classification and subsequent action lookup table
 */
static ce_dispact_t ce_disp_table[CE_INITDISPTBL_SIZE];
static int ce_disp_inited;

/*
 * Set to disable leaky and partner check for memory correctables
 */
int ce_xdiag_off;

/*
 * The following are not incremented atomically so are indicative only
 */
static int ce_xdiag_drops;
static int ce_xdiag_lkydrops;
static int ce_xdiag_ptnrdrops;
static int ce_xdiag_bad;

/*
 * CE leaky check callback structure
 */
typedef struct {
	struct async_flt *lkycb_aflt;
	errorq_t *lkycb_eqp;
	errorq_elem_t *lkycb_eqep;
} ce_lkychk_cb_t;

/*
 * defines for various ecache_flush_flag's
 */
#define	ECACHE_FLUSH_LINE	1
#define	ECACHE_FLUSH_ALL	2

/*
 * STICK sync
 */
#define	STICK_ITERATION 10
#define	MAX_TSKEW	1
#define	EV_A_START	0
#define	EV_A_END	1
#define	EV_B_START	2
#define	EV_B_END	3
#define	EVENTS		4

static int64_t stick_iter = STICK_ITERATION;
static int64_t stick_tsk = MAX_TSKEW;

typedef enum {
	EVENT_NULL = 0,
	SLAVE_START,
	SLAVE_CONT,
	MASTER_START
} event_cmd_t;

static volatile event_cmd_t stick_sync_cmd = EVENT_NULL;
static int64_t timestamp[EVENTS];
static volatile int slave_done;

#ifdef DEBUG
#define	DSYNC_ATTEMPTS 64
typedef struct {
	int64_t	skew_val[DSYNC_ATTEMPTS];
} ss_t;

ss_t stick_sync_stats[NCPU];
#endif /* DEBUG */

uint_t cpu_impl_dual_pgsz = 0;
#if defined(CPU_IMP_DUAL_PAGESIZE)
uint_t disable_dual_pgsz = 0;
#endif	/* CPU_IMP_DUAL_PAGESIZE */

/*
 * Save the cache bootup state for use when internal
 * caches are to be re-enabled after an error occurs.
 */
uint64_t cache_boot_state;

/*
 * PA[22:0] represent Displacement in Safari configuration space.
 */
uint_t	root_phys_addr_lo_mask = 0x7fffffu;

bus_config_eclk_t bus_config_eclk[] = {
#if defined(JALAPENO) || defined(SERRANO)
	{JBUS_CONFIG_ECLK_1_DIV, JBUS_CONFIG_ECLK_1},
	{JBUS_CONFIG_ECLK_2_DIV, JBUS_CONFIG_ECLK_2},
	{JBUS_CONFIG_ECLK_32_DIV, JBUS_CONFIG_ECLK_32},
#else /* JALAPENO || SERRANO */
	{SAFARI_CONFIG_ECLK_1_DIV, SAFARI_CONFIG_ECLK_1},
	{SAFARI_CONFIG_ECLK_2_DIV, SAFARI_CONFIG_ECLK_2},
	{SAFARI_CONFIG_ECLK_32_DIV, SAFARI_CONFIG_ECLK_32},
#endif /* JALAPENO || SERRANO */
	{0, 0}
};

/*
 * Interval for deferred CEEN reenable
 */
int cpu_ceen_delay_secs = CPU_CEEN_DELAY_SECS;

/*
 * set in /etc/system to control logging of user BERR/TO's
 */
int cpu_berr_to_verbose = 0;

/*
 * set to 0 in /etc/system to defer CEEN reenable for all CEs
 */
uint64_t cpu_ce_not_deferred = CPU_CE_NOT_DEFERRED;
uint64_t cpu_ce_not_deferred_ext = CPU_CE_NOT_DEFERRED_EXT;

/*
 * Set of all offline cpus
 */
cpuset_t cpu_offline_set;

static void cpu_delayed_check_ce_errors(void *);
static void cpu_check_ce_errors(void *);
void cpu_error_ecache_flush(ch_async_flt_t *);
static int cpu_error_ecache_flush_required(ch_async_flt_t *);
static void cpu_log_and_clear_ce(ch_async_flt_t *);
void cpu_ce_detected(ch_cpu_errors_t *, int);

/*
 * CE Leaky check timeout in microseconds.  This is chosen to be twice the
 * memory refresh interval of current DIMMs (64ms).  After initial fix that
 * gives at least one full refresh cycle in which the cell can leak
 * (whereafter further refreshes simply reinforce any incorrect bit value).
 */
clock_t cpu_ce_lkychk_timeout_usec = 128000;

/*
 * CE partner check partner caching period in seconds
 */
int cpu_ce_ptnr_cachetime_sec = 60;

/*
 * Sets trap table entry ttentry by overwriting eight instructions from ttlabel
 */
#define	CH_SET_TRAP(ttentry, ttlabel)			\
		bcopy((const void *)&ttlabel, &ttentry, 32);		\
		flush_instr_mem((caddr_t)&ttentry, 32);

static int min_ecache_size;
static uint_t priv_hcl_1;
static uint_t priv_hcl_2;
static uint_t priv_hcl_4;
static uint_t priv_hcl_8;

void
cpu_setup(void)
{
	extern int at_flags;
	extern int cpc_has_overflow_intr;

	/*
	 * Setup chip-specific trap handlers.
	 */
	cpu_init_trap();

	cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT);

	at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1 | EF_SPARC_SUN_US3;

	/*
	 * save the cache bootup state.
	 */
	cache_boot_state = get_dcu() & DCU_CACHE;

	/*
	 * Due to the number of entries in the fully-associative tlb
	 * this may have to be tuned lower than in spitfire.
	 */
	pp_slots = MIN(8, MAXPP_SLOTS);

	/*
	 * Block stores do not invalidate all pages of the d$, pagecopy
	 * et. al. need virtual translations with virtual coloring taken
	 * into consideration.  prefetch/ldd will pollute the d$ on the
	 * load side.
	 */
	pp_consistent_coloring = PPAGE_STORE_VCOLORING | PPAGE_LOADS_POLLUTE;

	if (use_page_coloring) {
		do_pg_coloring = 1;
	}

	isa_list =
	    "sparcv9+vis2 sparcv9+vis sparcv9 "
	    "sparcv8plus+vis2 sparcv8plus+vis sparcv8plus "
	    "sparcv8 sparcv8-fsmuld sparcv7 sparc";

	/*
	 * On Panther-based machines, this should
	 * also include AV_SPARC_POPC too
	 */
	cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2;

	/*
	 * On cheetah, there's no hole in the virtual address space
	 */
	hole_start = hole_end = 0;

	/*
	 * The kpm mapping window.
	 * kpm_size:
	 *	The size of a single kpm range.
	 *	The overall size will be: kpm_size * vac_colors.
	 * kpm_vbase:
	 *	The virtual start address of the kpm range within the kernel
	 *	virtual address space. kpm_vbase has to be kpm_size aligned.
	 */
	kpm_size = (size_t)(8ull * 1024 * 1024 * 1024 * 1024); /* 8TB */
	kpm_size_shift = 43;
	kpm_vbase = (caddr_t)0x8000000000000000ull; /* 8EB */
	kpm_smallpages = 1;

	/*
	 * The traptrace code uses either %tick or %stick for
	 * timestamping.  We have %stick so we can use it.
	 */
	traptrace_use_stick = 1;

	/*
	 * Cheetah has a performance counter overflow interrupt
	 */
	cpc_has_overflow_intr = 1;

#if defined(CPU_IMP_DUAL_PAGESIZE)
	/*
	 * Use Cheetah+ and later dual page size support.
	 */
	if (!disable_dual_pgsz) {
		cpu_impl_dual_pgsz = 1;
	}
#endif	/* CPU_IMP_DUAL_PAGESIZE */

	/*
	 * Declare that this architecture/cpu combination does fpRAS.
	 */
	fpras_implemented = 1;

	/*
	 * Setup CE lookup table
	 */
	CE_INITDISPTBL_POPULATE(ce_disp_table);
	ce_disp_inited = 1;
}

/*
 * Called by setcpudelay
 */
void
cpu_init_tick_freq(void)
{
	/*
	 * For UltraSPARC III and beyond we want to use the
	 * system clock rate as the basis for low level timing,
	 * due to support of mixed speed CPUs and power managment.
	 */
	if (system_clock_freq == 0)
		cmn_err(CE_PANIC, "setcpudelay: invalid system_clock_freq");

	sys_tick_freq = system_clock_freq;
}

#ifdef CHEETAHPLUS_ERRATUM_25
/*
 * Tunables
 */
int cheetah_bpe_off = 0;
int cheetah_sendmondo_recover = 1;
int cheetah_sendmondo_fullscan = 0;
int cheetah_sendmondo_recover_delay = 5;

#define	CHEETAH_LIVELOCK_MIN_DELAY	1

/*
 * Recovery Statistics
 */
typedef struct cheetah_livelock_entry	{
	int cpuid;		/* fallen cpu */
	int buddy;		/* cpu that ran recovery */
	clock_t lbolt;		/* when recovery started */
	hrtime_t recovery_time;	/* time spent in recovery */
} cheetah_livelock_entry_t;

#define	CHEETAH_LIVELOCK_NENTRY	32

cheetah_livelock_entry_t cheetah_livelock_hist[CHEETAH_LIVELOCK_NENTRY];
int cheetah_livelock_entry_nxt;

#define	CHEETAH_LIVELOCK_ENTRY_NEXT(statp)	{			\
	statp = cheetah_livelock_hist + cheetah_livelock_entry_nxt;	\
	if (++cheetah_livelock_entry_nxt >= CHEETAH_LIVELOCK_NENTRY) {	\
		cheetah_livelock_entry_nxt = 0;				\
	}								\
}

#define	CHEETAH_LIVELOCK_ENTRY_SET(statp, item, val)	statp->item = val

struct {
	hrtime_t hrt;		/* maximum recovery time */
	int recovery;		/* recovered */
	int full_claimed;	/* maximum pages claimed in full recovery */
	int proc_entry;		/* attempted to claim TSB */
	int proc_tsb_scan;	/* tsb scanned */
	int proc_tsb_partscan;	/* tsb partially scanned */
	int proc_tsb_fullscan;	/* whole tsb scanned */
	int proc_claimed;	/* maximum pages claimed in tsb scan */
	int proc_user;		/* user thread */
	int proc_kernel;	/* kernel thread */
	int proc_onflt;		/* bad stack */
	int proc_cpu;		/* null cpu */
	int proc_thread;	/* null thread */
	int proc_proc;		/* null proc */
	int proc_as;		/* null as */
	int proc_hat;		/* null hat */
	int proc_hat_inval;	/* hat contents don't make sense */
	int proc_hat_busy;	/* hat is changing TSBs */
	int proc_tsb_reloc;	/* TSB skipped because being relocated */
	int proc_cnum_bad;	/* cnum out of range */
	int proc_cnum;		/* last cnum processed */
	tte_t proc_tte;		/* last tte processed */
} cheetah_livelock_stat;

#define	CHEETAH_LIVELOCK_STAT(item)	cheetah_livelock_stat.item++

#define	CHEETAH_LIVELOCK_STATSET(item, value)		\
	cheetah_livelock_stat.item = value

#define	CHEETAH_LIVELOCK_MAXSTAT(item, value)	{	\
	if (value > cheetah_livelock_stat.item)		\
		cheetah_livelock_stat.item = value;	\
}

/*
 * Attempt to recover a cpu by claiming every cache line as saved
 * in the TSB that the non-responsive cpu is using. Since we can't
 * grab any adaptive lock, this is at best an attempt to do so. Because
 * we don't grab any locks, we must operate under the protection of
 * on_fault().
 *
 * Return 1 if cpuid could be recovered, 0 if failed.
 */
int
mondo_recover_proc(uint16_t cpuid, int bn)
{
	label_t ljb;
	cpu_t *cp;
	kthread_t *t;
	proc_t *p;
	struct as *as;
	struct hat *hat;
	uint_t  cnum;
	struct tsb_info *tsbinfop;
	struct tsbe *tsbep;
	caddr_t tsbp;
	caddr_t end_tsbp;
	uint64_t paddr;
	uint64_t idsr;
	u_longlong_t pahi, palo;
	int pages_claimed = 0;
	tte_t tsbe_tte;
	int tried_kernel_tsb = 0;
	mmu_ctx_t *mmu_ctxp;

	CHEETAH_LIVELOCK_STAT(proc_entry);

	if (on_fault(&ljb)) {
		CHEETAH_LIVELOCK_STAT(proc_onflt);
		goto badstruct;
	}

	if ((cp = cpu[cpuid]) == NULL) {
		CHEETAH_LIVELOCK_STAT(proc_cpu);
		goto badstruct;
	}

	if ((t = cp->cpu_thread) == NULL) {
		CHEETAH_LIVELOCK_STAT(proc_thread);
		goto badstruct;
	}

	if ((p = ttoproc(t)) == NULL) {
		CHEETAH_LIVELOCK_STAT(proc_proc);
		goto badstruct;
	}

	if ((as = p->p_as) == NULL) {
		CHEETAH_LIVELOCK_STAT(proc_as);
		goto badstruct;
	}

	if ((hat = as->a_hat) == NULL) {
		CHEETAH_LIVELOCK_STAT(proc_hat);
		goto badstruct;
	}

	if (hat != ksfmmup) {
		CHEETAH_LIVELOCK_STAT(proc_user);
		if (hat->sfmmu_flags & (HAT_BUSY | HAT_SWAPPED | HAT_SWAPIN)) {
			CHEETAH_LIVELOCK_STAT(proc_hat_busy);
			goto badstruct;
		}
		tsbinfop = hat->sfmmu_tsb;
		if (tsbinfop == NULL) {
			CHEETAH_LIVELOCK_STAT(proc_hat_inval);
			goto badstruct;
		}
		tsbp = tsbinfop->tsb_va;
		end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc);
	} else {
		CHEETAH_LIVELOCK_STAT(proc_kernel);
		tsbinfop = NULL;
		tsbp = ktsb_base;
		end_tsbp = tsbp + TSB_BYTES(ktsb_sz);
	}

	/* Verify as */
	if (hat->sfmmu_as != as) {
		CHEETAH_LIVELOCK_STAT(proc_hat_inval);
		goto badstruct;
	}

	mmu_ctxp = CPU_MMU_CTXP(cp);
	ASSERT(mmu_ctxp);
	cnum = hat->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum;
	CHEETAH_LIVELOCK_STATSET(proc_cnum, cnum);

	if ((cnum < 0) || (cnum == INVALID_CONTEXT) ||
	    (cnum >= mmu_ctxp->mmu_nctxs)) {
		CHEETAH_LIVELOCK_STAT(proc_cnum_bad);
		goto badstruct;
	}

	do {
		CHEETAH_LIVELOCK_STAT(proc_tsb_scan);

		/*
		 * Skip TSBs being relocated.  This is important because
		 * we want to avoid the following deadlock scenario:
		 *
		 * 1) when we came in we set ourselves to "in recover" state.
		 * 2) when we try to touch TSB being relocated the mapping
		 *    will be in the suspended state so we'll spin waiting
		 *    for it to be unlocked.
		 * 3) when the CPU that holds the TSB mapping locked tries to
		 *    unlock it it will send a xtrap which will fail to xcall
		 *    us or the CPU we're trying to recover, and will in turn
		 *    enter the mondo code.
		 * 4) since we are still spinning on the locked mapping
		 *    no further progress will be made and the system will
		 *    inevitably hard hang.
		 *
		 * A TSB not being relocated can't begin being relocated
		 * while we're accessing it because we check
		 * sendmondo_in_recover before relocating TSBs.
		 */
		if (hat != ksfmmup &&
		    (tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) {
			CHEETAH_LIVELOCK_STAT(proc_tsb_reloc);
			goto next_tsbinfo;
		}

		for (tsbep = (struct tsbe *)tsbp;
		    tsbep < (struct tsbe *)end_tsbp; tsbep++) {
			tsbe_tte = tsbep->tte_data;

			if (tsbe_tte.tte_val == 0) {
				/*
				 * Invalid tte
				 */
				continue;
			}
			if (tsbe_tte.tte_se) {
				/*
				 * Don't want device registers
				 */
				continue;
			}
			if (tsbe_tte.tte_cp == 0) {
				/*
				 * Must be cached in E$
				 */
				continue;
			}
			if (tsbep->tte_tag.tag_invalid != 0) {
				/*
				 * Invalid tag, ingnore this entry.
				 */
				continue;
			}
			CHEETAH_LIVELOCK_STATSET(proc_tte, tsbe_tte);
			idsr = getidsr();
			if ((idsr & (IDSR_NACK_BIT(bn) |
			    IDSR_BUSY_BIT(bn))) == 0) {
				CHEETAH_LIVELOCK_STAT(proc_tsb_partscan);
				goto done;
			}
			pahi = tsbe_tte.tte_pahi;
			palo = tsbe_tte.tte_palo;
			paddr = (uint64_t)((pahi << 32) |
			    (palo << MMU_PAGESHIFT));
			claimlines(paddr, TTEBYTES(TTE_CSZ(&tsbe_tte)),
			    CH_ECACHE_SUBBLK_SIZE);
			if ((idsr & IDSR_BUSY_BIT(bn)) == 0) {
				shipit(cpuid, bn);
			}
			pages_claimed++;
		}
next_tsbinfo:
		if (tsbinfop != NULL)
			tsbinfop = tsbinfop->tsb_next;
		if (tsbinfop != NULL) {
			tsbp = tsbinfop->tsb_va;
			end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc);
		} else if (tsbp == ktsb_base) {
			tried_kernel_tsb = 1;
		} else if (!tried_kernel_tsb) {
			tsbp = ktsb_base;
			end_tsbp = tsbp + TSB_BYTES(ktsb_sz);
			hat = ksfmmup;
			tsbinfop = NULL;
		}
	} while (tsbinfop != NULL ||
	    ((tsbp == ktsb_base) && !tried_kernel_tsb));

	CHEETAH_LIVELOCK_STAT(proc_tsb_fullscan);
	CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed);
	no_fault();
	idsr = getidsr();
	if ((idsr & (IDSR_NACK_BIT(bn) |
	    IDSR_BUSY_BIT(bn))) == 0) {
		return (1);
	} else {
		return (0);
	}

done:
	no_fault();
	CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed);
	return (1);

badstruct:
	no_fault();
	return (0);
}

/*
 * Attempt to claim ownership, temporarily, of every cache line that a
 * non-responsive cpu might be using.  This might kick that cpu out of
 * this state.
 *
 * The return value indicates to the caller if we have exhausted all recovery
 * techniques. If 1 is returned, it is useless to call this function again
 * even for a different target CPU.
 */
int
mondo_recover(uint16_t cpuid, int bn)
{
	struct memseg *seg;
	uint64_t begin_pa, end_pa, cur_pa;
	hrtime_t begin_hrt, end_hrt;
	int retval = 0;
	int pages_claimed = 0;
	cheetah_livelock_entry_t *histp;
	uint64_t idsr;

	if (atomic_cas_32(&sendmondo_in_recover, 0, 1) != 0) {
		/*
		 * Wait while recovery takes place
		 */
		while (sendmondo_in_recover) {
			drv_usecwait(1);
		}
		/*
		 * Assume we didn't claim the whole memory. If
		 * the target of this caller is not recovered,
		 * it will come back.
		 */
		return (retval);
	}

	CHEETAH_LIVELOCK_ENTRY_NEXT(histp);
	CHEETAH_LIVELOCK_ENTRY_SET(histp, lbolt, LBOLT_WAITFREE);
	CHEETAH_LIVELOCK_ENTRY_SET(histp, cpuid, cpuid);
	CHEETAH_LIVELOCK_ENTRY_SET(histp, buddy, CPU->cpu_id);

	begin_hrt = gethrtime_waitfree();
	/*
	 * First try to claim the lines in the TSB the target
	 * may have been using.
	 */
	if (mondo_recover_proc(cpuid, bn) == 1) {
		/*
		 * Didn't claim the whole memory
		 */
		goto done;
	}

	/*
	 * We tried using the TSB. The target is still
	 * not recovered. Check if complete memory scan is
	 * enabled.
	 */
	if (cheetah_sendmondo_fullscan == 0) {
		/*
		 * Full memory scan is disabled.
		 */
		retval = 1;
		goto done;
	}

	/*
	 * Try claiming the whole memory.
	 */
	for (seg = memsegs; seg; seg = seg->next) {
		begin_pa = (uint64_t)(seg->pages_base) << MMU_PAGESHIFT;
		end_pa = (uint64_t)(seg->pages_end) << MMU_PAGESHIFT;
		for (cur_pa = begin_pa; cur_pa < end_pa;
		    cur_pa += MMU_PAGESIZE) {
			idsr = getidsr();
			if ((idsr & (IDSR_NACK_BIT(bn) |
			    IDSR_BUSY_BIT(bn))) == 0) {
				/*
				 * Didn't claim all memory
				 */
				goto done;
			}
			claimlines(cur_pa, MMU_PAGESIZE,
			    CH_ECACHE_SUBBLK_SIZE);
			if ((idsr & IDSR_BUSY_BIT(bn)) == 0) {
				shipit(cpuid, bn);
			}
			pages_claimed++;
		}
	}

	/*
	 * We did all we could.
	 */
	retval = 1;

done:
	/*
	 * Update statistics
	 */
	end_hrt = gethrtime_waitfree();
	CHEETAH_LIVELOCK_STAT(recovery);
	CHEETAH_LIVELOCK_MAXSTAT(hrt, (end_hrt - begin_hrt));
	CHEETAH_LIVELOCK_MAXSTAT(full_claimed, pages_claimed);
	CHEETAH_LIVELOCK_ENTRY_SET(histp, recovery_time, \
	    (end_hrt -  begin_hrt));

	while (atomic_cas_32(&sendmondo_in_recover, 1, 0) != 1)
		;

	return (retval);
}

/*
 * This is called by the cyclic framework when this CPU becomes online
 */
/*ARGSUSED*/
static void
cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when)
{

	hdlr->cyh_func = (cyc_func_t)cheetah_nudge_buddy;
	hdlr->cyh_level = CY_LOW_LEVEL;
	hdlr->cyh_arg = NULL;

	/*
	 * Stagger the start time
	 */
	when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU);
	if (cheetah_sendmondo_recover_delay < CHEETAH_LIVELOCK_MIN_DELAY) {
		cheetah_sendmondo_recover_delay = CHEETAH_LIVELOCK_MIN_DELAY;
	}
	when->cyt_interval = cheetah_sendmondo_recover_delay * NANOSEC;
}

/*
 * Create a low level cyclic to send a xtrap to the next cpu online.
 * However, there's no need to have this running on a uniprocessor system.
 */
static void
cheetah_nudge_init(void)
{
	cyc_omni_handler_t hdlr;

	if (max_ncpus == 1) {
		return;
	}

	hdlr.cyo_online = cheetah_nudge_onln;
	hdlr.cyo_offline = NULL;
	hdlr.cyo_arg = NULL;

	mutex_enter(&cpu_lock);
	(void) cyclic_add_omni(&hdlr);
	mutex_exit(&cpu_lock);
}

/*
 * Cyclic handler to wake up buddy
 */
void
cheetah_nudge_buddy(void)
{
	/*
	 * Disable kernel preemption to protect the cpu list
	 */
	kpreempt_disable();
	if ((CPU->cpu_next_onln != CPU) && (sendmondo_in_recover == 0)) {
		xt_one(CPU->cpu_next_onln->cpu_id, (xcfunc_t *)xt_sync_tl1,
		    0, 0);
	}
	kpreempt_enable();
}

#endif	/* CHEETAHPLUS_ERRATUM_25 */

#ifdef SEND_MONDO_STATS
uint32_t x_one_stimes[64];
uint32_t x_one_ltimes[16];
uint32_t x_set_stimes[64];
uint32_t x_set_ltimes[16];
uint32_t x_set_cpus[NCPU];
uint32_t x_nack_stimes[64];
#endif

/*
 * Note: A version of this function is used by the debugger via the KDI,
 * and must be kept in sync with this version.  Any changes made to this
 * function to support new chips or to accomodate errata must also be included
 * in the KDI-specific version.  See us3_kdi.c.
 */
void
send_one_mondo(int cpuid)
{
	int busy, nack;
	uint64_t idsr, starttick, endtick, tick, lasttick;
	uint64_t busymask;
#ifdef	CHEETAHPLUS_ERRATUM_25
	int recovered = 0;
#endif

	CPU_STATS_ADDQ(CPU, sys, xcalls, 1);
	starttick = lasttick = gettick();
	shipit(cpuid, 0);
	endtick = starttick + xc_tick_limit;
	busy = nack = 0;
#if defined(JALAPENO) || defined(SERRANO)
	/*
	 * Lower 2 bits of the agent ID determine which BUSY/NACK pair
	 * will be used for dispatching interrupt. For now, assume
	 * there are no more than IDSR_BN_SETS CPUs, hence no aliasing
	 * issues with respect to BUSY/NACK pair usage.
	 */
	busymask  = IDSR_BUSY_BIT(cpuid);
#else /* JALAPENO || SERRANO */
	busymask = IDSR_BUSY;
#endif /* JALAPENO || SERRANO */
	for (;;) {
		idsr = getidsr();
		if (idsr == 0)
			break;

		tick = gettick();
		/*
		 * If there is a big jump between the current tick
		 * count and lasttick, we have probably hit a break
		 * point.  Adjust endtick accordingly to avoid panic.
		 */
		if (tick > (lasttick + xc_tick_jump_limit))
			endtick += (tick - lasttick);
		lasttick = tick;
		if (tick > endtick) {
			if (panic_quiesce)
				return;
#ifdef	CHEETAHPLUS_ERRATUM_25
			if (cheetah_sendmondo_recover && recovered == 0) {
				if (mondo_recover(cpuid, 0)) {
					/*
					 * We claimed the whole memory or
					 * full scan is disabled.
					 */
					recovered++;
				}
				tick = gettick();
				endtick = tick + xc_tick_limit;
				lasttick = tick;
				/*
				 * Recheck idsr
				 */
				continue;
			} else
#endif	/* CHEETAHPLUS_ERRATUM_25 */
			{
				cmn_err(CE_PANIC, "send mondo timeout "
				    "(target 0x%x) [%d NACK %d BUSY]",
				    cpuid, nack, busy);
			}
		}

		if (idsr & busymask) {
			busy++;
			continue;
		}
		drv_usecwait(1);
		shipit(cpuid, 0);
		nack++;
		busy = 0;
	}
#ifdef SEND_MONDO_STATS
	{
		int n = gettick() - starttick;
		if (n < 8192)
			x_one_stimes[n >> 7]++;
		else
			x_one_ltimes[(n >> 13) & 0xf]++;
	}
#endif
}

void
syncfpu(void)
{
}

/*
 * Return processor specific async error structure
 * size used.
 */
int
cpu_aflt_size(void)
{
	return (sizeof (ch_async_flt_t));
}

/*
 * Tunable to disable the checking of other cpu logout areas during panic for
 * potential syndrome 71 generating errors.
 */
int enable_check_other_cpus_logout = 1;

/*
 * Check other cpus logout area for potential synd 71 generating
 * errors.
 */
static void
cpu_check_cpu_logout(int cpuid, caddr_t tpc, int tl, int ecc_type,
    ch_cpu_logout_t *clop)
{
	struct async_flt *aflt;
	ch_async_flt_t ch_flt;
	uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs;

	if (clop == NULL || clop->clo_data.chd_afar == LOGOUT_INVALID) {
		return;
	}

	bzero(&ch_flt, sizeof (ch_async_flt_t));

	t_afar = clop->clo_data.chd_afar;
	t_afsr = clop->clo_data.chd_afsr;
	t_afsr_ext = clop->clo_data.chd_afsr_ext;
#if defined(SERRANO)
	ch_flt.afar2 = clop->clo_data.chd_afar2;
#endif	/* SERRANO */

	/*
	 * In order to simplify code, we maintain this afsr_errs
	 * variable which holds the aggregate of AFSR and AFSR_EXT
	 * sticky bits.
	 */
	t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) |
	    (t_afsr & C_AFSR_ALL_ERRS);

	/* Setup the async fault structure */
	aflt = (struct async_flt *)&ch_flt;
	aflt->flt_id = gethrtime_waitfree();
	ch_flt.afsr_ext = t_afsr_ext;
	ch_flt.afsr_errs = t_afsr_errs;
	aflt->flt_stat = t_afsr;
	aflt->flt_addr = t_afar;
	aflt->flt_bus_id = cpuid;
	aflt->flt_inst = cpuid;
	aflt->flt_pc = tpc;
	aflt->flt_prot = AFLT_PROT_NONE;
	aflt->flt_class = CPU_FAULT;
	aflt->flt_priv = ((t_afsr & C_AFSR_PRIV) != 0);
	aflt->flt_tl = tl;
	aflt->flt_status = ecc_type;
	aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs);

	/*
	 * Queue events on the async event queue, one event per error bit.
	 * If no events are queued, queue an event to complain.
	 */
	if (cpu_queue_events(&ch_flt, NULL, t_afsr_errs, clop) == 0) {
		ch_flt.flt_type = CPU_INV_AFSR;
		cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR,
		    (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue,
		    aflt->flt_panic);
	}

	/*
	 * Zero out + invalidate CPU logout.
	 */
	bzero(clop, sizeof (ch_cpu_logout_t));
	clop->clo_data.chd_afar = LOGOUT_INVALID;
}

/*
 * Check the logout areas of all other cpus for unlogged errors.
 */
static void
cpu_check_other_cpus_logout(void)
{
	int i, j;
	processorid_t myid;
	struct cpu *cp;
	ch_err_tl1_data_t *cl1p;

	myid = CPU->cpu_id;
	for (i = 0; i < NCPU; i++) {
		cp = cpu[i];

		if ((cp == NULL) || !(cp->cpu_flags & CPU_EXISTS) ||
		    (cp->cpu_id == myid) || (CPU_PRIVATE(cp) == NULL)) {
			continue;
		}

		/*
		 * Check each of the tl>0 logout areas
		 */
		cl1p = CPU_PRIVATE_PTR(cp, chpr_tl1_err_data[0]);
		for (j = 0; j < CH_ERR_TL1_TLMAX; j++, cl1p++) {
			if (cl1p->ch_err_tl1_flags == 0)
				continue;

			cpu_check_cpu_logout(i, (caddr_t)cl1p->ch_err_tl1_tpc,
			    1, ECC_F_TRAP, &cl1p->ch_err_tl1_logout);
		}

		/*
		 * Check each of the remaining logout areas
		 */
		cpu_check_cpu_logout(i, NULL, 0, ECC_F_TRAP,
		    CPU_PRIVATE_PTR(cp, chpr_fecctl0_logout));
		cpu_check_cpu_logout(i, NULL, 0, ECC_C_TRAP,
		    CPU_PRIVATE_PTR(cp, chpr_cecc_logout));
		cpu_check_cpu_logout(i, NULL, 0, ECC_D_TRAP,
		    CPU_PRIVATE_PTR(cp, chpr_async_logout));
	}
}

/*
 * The fast_ecc_err handler transfers control here for UCU, UCC events.
 * Note that we flush Ecache twice, once in the fast_ecc_err handler to
 * flush the error that caused the UCU/UCC, then again here at the end to
 * flush the TL=1 trap handler code out of the Ecache, so we can minimize
 * the probability of getting a TL>1 Fast ECC trap when we're fielding
 * another Fast ECC trap.
 *
 * Cheetah+ also handles: TSCE: No additional processing required.
 * Panther adds L3_UCU and L3_UCC which are reported in AFSR_EXT.
 *
 * Note that the p_clo_flags input is only valid in cases where the
 * cpu_private struct is not yet initialized (since that is the only
 * time that information cannot be obtained from the logout struct.)
 */
/*ARGSUSED*/
void
cpu_fast_ecc_error(struct regs *rp, ulong_t p_clo_flags)
{
	ch_cpu_logout_t *clop;
	uint64_t ceen, nceen;

	/*
	 * Get the CPU log out info. If we can't find our CPU private
	 * pointer, then we will have to make due without any detailed
	 * logout information.
	 */
	if (CPU_PRIVATE(CPU) == NULL) {
		clop = NULL;
		ceen = p_clo_flags & EN_REG_CEEN;
		nceen = p_clo_flags & EN_REG_NCEEN;
	} else {
		clop = CPU_PRIVATE_PTR(CPU, chpr_fecctl0_logout);
		ceen = clop->clo_flags & EN_REG_CEEN;
		nceen = clop->clo_flags & EN_REG_NCEEN;
	}

	cpu_log_fast_ecc_error((caddr_t)rp->r_pc,
	    (rp->r_tstate & TSTATE_PRIV) ? 1 : 0, 0, ceen, nceen, clop);
}

/*
 * Log fast ecc error, called from either Fast ECC at TL=0 or Fast
 * ECC at TL>0.  Need to supply either a error register pointer or a
 * cpu logout structure pointer.
 */
static void
cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen,
    uint64_t nceen, ch_cpu_logout_t *clop)
{
	struct async_flt *aflt;
	ch_async_flt_t ch_flt;
	uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs;
	char pr_reason[MAX_REASON_STRING];
	ch_cpu_errors_t cpu_error_regs;

	bzero(&ch_flt, sizeof (ch_async_flt_t));
	/*
	 * If no cpu logout data, then we will have to make due without
	 * any detailed logout information.
	 */
	if (clop == NULL) {
		ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID;
		get_cpu_error_state(&cpu_error_regs);
		set_cpu_error_state(&cpu_error_regs);
		t_afar = cpu_error_regs.afar;
		t_afsr = cpu_error_regs.afsr;
		t_afsr_ext = cpu_error_regs.afsr_ext;
#if defined(SERRANO)
		ch_flt.afar2 = cpu_error_regs.afar2;
#endif	/* SERRANO */
	} else {
		t_afar = clop->clo_data.chd_afar;
		t_afsr = clop->clo_data.chd_afsr;
		t_afsr_ext = clop->clo_data.chd_afsr_ext;
#if defined(SERRANO)
		ch_flt.afar2 = clop->clo_data.chd_afar2;
#endif	/* SERRANO */
	}

	/*
	 * In order to simplify code, we maintain this afsr_errs
	 * variable which holds the aggregate of AFSR and AFSR_EXT
	 * sticky bits.
	 */
	t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) |
	    (t_afsr & C_AFSR_ALL_ERRS);
	pr_reason[0] = '\0';

	/* Setup the async fault structure */
	aflt = (struct async_flt *)&ch_flt;
	aflt->flt_id = gethrtime_waitfree();
	ch_flt.afsr_ext = t_afsr_ext;
	ch_flt.afsr_errs = t_afsr_errs;
	aflt->flt_stat = t_afsr;
	aflt->flt_addr = t_afar;
	aflt->flt_bus_id = getprocessorid();
	aflt->flt_inst = CPU->cpu_id;
	aflt->flt_pc = tpc;
	aflt->flt_prot = AFLT_PROT_NONE;
	aflt->flt_class = CPU_FAULT;
	aflt->flt_priv = priv;
	aflt->flt_tl = tl;
	aflt->flt_status = ECC_F_TRAP;
	aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs);

	/*
	 * XXXX - Phenomenal hack to get around Solaris not getting all the
	 * cmn_err messages out to the console.  The situation is a UCU (in
	 * priv mode) which causes a WDU which causes a UE (on the retry).
	 * The messages for the UCU and WDU are enqueued and then pulled off
	 * the async queue via softint and syslogd starts to process them
	 * but doesn't get them to the console.  The UE causes a panic, but
	 * since the UCU/WDU messages are already in transit, those aren't
	 * on the async queue.  The hack is to check if we have a matching
	 * WDU event for the UCU, and if it matches, we're more than likely
	 * going to panic with a UE, unless we're under protection.  So, we
	 * check to see if we got a matching WDU event and if we're under
	 * protection.
	 *
	 * For Cheetah/Cheetah+/Jaguar/Jalapeno, the sequence we care about
	 * looks like this:
	 *    UCU->WDU->UE
	 * For Panther, it could look like either of these:
	 *    UCU---->WDU->L3_WDU->UE
	 *    L3_UCU->WDU->L3_WDU->UE
	 */
	if ((t_afsr_errs & (C_AFSR_UCU | C_AFSR_L3_UCU)) &&
	    aflt->flt_panic == 0 && aflt->flt_priv != 0 &&
	    curthread->t_ontrap == NULL &&
	    curthread->t_lofault == (uintptr_t)NULL) {
		get_cpu_error_state(&cpu_error_regs);
		if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
			aflt->flt_panic |=
			    ((cpu_error_regs.afsr & C_AFSR_WDU) &&
			    (cpu_error_regs.afsr_ext & C_AFSR_L3_WDU) &&
			    (cpu_error_regs.afar == t_afar));
			aflt->flt_panic |= ((clop == NULL) &&
			    (t_afsr_errs & C_AFSR_WDU) &&
			    (t_afsr_errs & C_AFSR_L3_WDU));
		} else {
			aflt->flt_panic |=
			    ((cpu_error_regs.afsr & C_AFSR_WDU) &&
			    (cpu_error_regs.afar == t_afar));
			aflt->flt_panic |= ((clop == NULL) &&
			    (t_afsr_errs & C_AFSR_WDU));
		}
	}

	/*
	 * Queue events on the async event queue, one event per error bit.
	 * If no events are queued or no Fast ECC events are on in the AFSR,
	 * queue an event to complain.
	 */
	if (cpu_queue_events(&ch_flt, pr_reason, t_afsr_errs, clop) == 0 ||
	    ((t_afsr_errs & (C_AFSR_FECC_ERRS | C_AFSR_EXT_FECC_ERRS)) == 0)) {
		ch_flt.flt_type = CPU_INV_AFSR;
		cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR,
		    (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue,
		    aflt->flt_panic);
	}

	/*
	 * Zero out + invalidate CPU logout.
	 */
	if (clop) {
		bzero(clop, sizeof (ch_cpu_logout_t));
		clop->clo_data.chd_afar = LOGOUT_INVALID;
	}

	/*
	 * We carefully re-enable NCEEN and CEEN and then check if any deferred
	 * or disrupting errors have happened.  We do this because if a
	 * deferred or disrupting error had occurred with NCEEN/CEEN off, the
	 * trap will not be taken when NCEEN/CEEN is re-enabled.  Note that
	 * CEEN works differently on Cheetah than on Spitfire.  Also, we enable
	 * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a
	 * deferred or disrupting error happening between checking the AFSR and
	 * enabling NCEEN/CEEN.
	 *
	 * Note: CEEN and NCEEN are only reenabled if they were on when trap
	 * taken.
	 */
	set_error_enable(get_error_enable() | (nceen | ceen));
	if (clear_errors(&ch_flt)) {
		aflt->flt_panic |= ((ch_flt.afsr_errs &
		    (C_AFSR_EXT_ASYNC_ERRS | C_AFSR_ASYNC_ERRS)) != 0);
		(void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs,
		    NULL);
	}

	/*
	 * Panic here if aflt->flt_panic has been set.  Enqueued errors will
	 * be logged as part of the panic flow.
	 */
	if (aflt->flt_panic)
		fm_panic("%sError(s)", pr_reason);

	/*
	 * Flushing the Ecache here gets the part of the trap handler that
	 * is run at TL=1 out of the Ecache.
	 */
	cpu_flush_ecache();
}

/*
 * This is called via sys_trap from pil15_interrupt code if the
 * corresponding entry in ch_err_tl1_pending is set.  Checks the
 * various ch_err_tl1_data structures for valid entries based on the bit
 * settings in the ch_err_tl1_flags entry of the structure.
 */
/*ARGSUSED*/
void
cpu_tl1_error(struct regs *rp, int panic)
{
	ch_err_tl1_data_t *cl1p, cl1;
	int i, ncl1ps;
	uint64_t me_flags;
	uint64_t ceen, nceen;

	if (ch_err_tl1_paddrs[CPU->cpu_id] == 0) {
		cl1p = &ch_err_tl1_data;
		ncl1ps = 1;
	} else if (CPU_PRIVATE(CPU) != NULL) {
		cl1p = CPU_PRIVATE_PTR(CPU, chpr_tl1_err_data[0]);
		ncl1ps = CH_ERR_TL1_TLMAX;
	} else {
		ncl1ps = 0;
	}

	for (i = 0; i < ncl1ps; i++, cl1p++) {
		if (cl1p->ch_err_tl1_flags == 0)
			continue;

		/*
		 * Grab a copy of the logout data and invalidate
		 * the logout area.
		 */
		cl1 = *cl1p;
		bzero(cl1p, sizeof (ch_err_tl1_data_t));
		cl1p->ch_err_tl1_logout.clo_data.chd_afar = LOGOUT_INVALID;
		me_flags = CH_ERR_ME_FLAGS(cl1.ch_err_tl1_flags);

		/*
		 * Log "first error" in ch_err_tl1_data.
		 */
		if (cl1.ch_err_tl1_flags & CH_ERR_FECC) {
			ceen = get_error_enable() & EN_REG_CEEN;
			nceen = get_error_enable() & EN_REG_NCEEN;
			cpu_log_fast_ecc_error((caddr_t)cl1.ch_err_tl1_tpc, 1,
			    1, ceen, nceen, &cl1.ch_err_tl1_logout);
		}
#if defined(CPU_IMP_L1_CACHE_PARITY)
		if (cl1.ch_err_tl1_flags & (CH_ERR_IPE | CH_ERR_DPE)) {
			cpu_parity_error(rp, cl1.ch_err_tl1_flags,
			    (caddr_t)cl1.ch_err_tl1_tpc);
		}
#endif	/* CPU_IMP_L1_CACHE_PARITY */

		/*
		 * Log "multiple events" in ch_err_tl1_data.  Note that
		 * we don't read and clear the AFSR/AFAR in the TL>0 code
		 * if the structure is busy, we just do the cache flushing
		 * we have to do and then do the retry.  So the AFSR/AFAR
		 * at this point *should* have some relevant info.  If there
		 * are no valid errors in the AFSR, we'll assume they've
		 * already been picked up and logged.  For I$/D$ parity,
		 * we just log an event with an "Unknown" (NULL) TPC.
		 */
		if (me_flags & CH_ERR_FECC) {
			ch_cpu_errors_t cpu_error_regs;
			uint64_t t_afsr_errs;

			/*
			 * Get the error registers and see if there's
			 * a pending error.  If not, don't bother
			 * generating an "Invalid AFSR" error event.
			 */
			get_cpu_error_state(&cpu_error_regs);
			t_afsr_errs = (cpu_error_regs.afsr_ext &
			    C_AFSR_EXT_ALL_ERRS) |
			    (cpu_error_regs.afsr & C_AFSR_ALL_ERRS);
			if (t_afsr_errs != 0) {
				ceen = get_error_enable() & EN_REG_CEEN;
				nceen = get_error_enable() & EN_REG_NCEEN;
				cpu_log_fast_ecc_error((caddr_t)NULL, 1,
				    1, ceen, nceen, NULL);
			}
		}
#if defined(CPU_IMP_L1_CACHE_PARITY)
		if (me_flags & (CH_ERR_IPE | CH_ERR_DPE)) {
			cpu_parity_error(rp, me_flags, (caddr_t)NULL);
		}
#endif	/* CPU_IMP_L1_CACHE_PARITY */
	}
}

/*
 * Called from Fast ECC TL>0 handler in case of fatal error.
 * cpu_tl1_error should always find an associated ch_err_tl1_data structure,
 * but if we don't, we'll panic with something reasonable.
 */
/*ARGSUSED*/
void
cpu_tl1_err_panic(struct regs *rp, ulong_t flags)
{
	cpu_tl1_error(rp, 1);
	/*
	 * Should never return, but just in case.
	 */
	fm_panic("Unsurvivable ECC Error at TL>0");
}

/*
 * The ce_err/ce_err_tl1 handlers transfer control here for CE, EMC, EDU:ST,
 * EDC, WDU, WDC, CPU, CPC, IVU, IVC events.
 * Disrupting errors controlled by NCEEN: EDU:ST, WDU, CPU, IVU
 * Disrupting errors controlled by CEEN: CE, EMC, EDC, WDC, CPC, IVC
 *
 * Cheetah+ also handles (No additional processing required):
 *    DUE, DTO, DBERR	(NCEEN controlled)
 *    THCE		(CEEN and ET_ECC_en controlled)
 *    TUE		(ET_ECC_en controlled)
 *
 * Panther further adds:
 *    IMU, L3_EDU, L3_WDU, L3_CPU		(NCEEN controlled)
 *    IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE	(CEEN controlled)
 *    TUE_SH, TUE		(NCEEN and L2_tag_ECC_en controlled)
 *    L3_TUE, L3_TUE_SH		(NCEEN and ET_ECC_en controlled)
 *    THCE			(CEEN and L2_tag_ECC_en controlled)
 *    L3_THCE			(CEEN and ET_ECC_en controlled)
 *
 * Note that the p_clo_flags input is only valid in cases where the
 * cpu_private struct is not yet initialized (since that is the only
 * time that information cannot be obtained from the logout struct.)
 */
/*ARGSUSED*/
void
cpu_disrupting_error(struct regs *rp, ulong_t p_clo_flags)
{
	struct async_flt *aflt;
	ch_async_flt_t ch_flt;
	char pr_reason[MAX_REASON_STRING];
	ch_cpu_logout_t *clop;
	uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs;
	ch_cpu_errors_t cpu_error_regs;

	bzero(&ch_flt, sizeof (ch_async_flt_t));
	/*
	 * Get the CPU log out info. If we can't find our CPU private
	 * pointer, then we will have to make due without any detailed
	 * logout information.
	 */
	if (CPU_PRIVATE(CPU) == NULL) {
		clop = NULL;
		ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID;
		get_cpu_error_state(&cpu_error_regs);
		set_cpu_error_state(&cpu_error_regs);
		t_afar = cpu_error_regs.afar;
		t_afsr = cpu_error_regs.afsr;
		t_afsr_ext = cpu_error_regs.afsr_ext;
#if defined(SERRANO)
		ch_flt.afar2 = cpu_error_regs.afar2;
#endif	/* SERRANO */
	} else {
		clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout);
		t_afar = clop->clo_data.chd_afar;
		t_afsr = clop->clo_data.chd_afsr;
		t_afsr_ext = clop->clo_data.chd_afsr_ext;
#if defined(SERRANO)
		ch_flt.afar2 = clop->clo_data.chd_afar2;
#endif	/* SERRANO */
	}

	/*
	 * In order to simplify code, we maintain this afsr_errs
	 * variable which holds the aggregate of AFSR and AFSR_EXT
	 * sticky bits.
	 */
	t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) |
	    (t_afsr & C_AFSR_ALL_ERRS);

	pr_reason[0] = '\0';
	/* Setup the async fault structure */
	aflt = (struct async_flt *)&ch_flt;
	ch_flt.afsr_ext = t_afsr_ext;
	ch_flt.afsr_errs = t_afsr_errs;
	aflt->flt_stat = t_afsr;
	aflt->flt_addr = t_afar;
	aflt->flt_pc = (caddr_t)rp->r_pc;
	aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ?  1 : 0;
	aflt->flt_tl = 0;
	aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs);

	/*
	 * If this trap is a result of one of the errors not masked
	 * by cpu_ce_not_deferred, we don't reenable CEEN. Instead
	 * indicate that a timeout is to be set later.
	 */
	if (!(t_afsr_errs & (cpu_ce_not_deferred | cpu_ce_not_deferred_ext)) &&
	    !aflt->flt_panic)
		ch_flt.flt_trapped_ce = CE_CEEN_DEFER | CE_CEEN_TRAPPED;
	else
		ch_flt.flt_trapped_ce = CE_CEEN_NODEFER | CE_CEEN_TRAPPED;

	/*
	 * log the CE and clean up
	 */
	cpu_log_and_clear_ce(&ch_flt);

	/*
	 * We re-enable CEEN (if required) and check if any disrupting errors
	 * have happened.  We do this because if a disrupting error had occurred
	 * with CEEN off, the trap will not be taken when CEEN is re-enabled.
	 * Note that CEEN works differently on Cheetah than on Spitfire.  Also,
	 * we enable CEEN *before* checking the AFSR to avoid the small window
	 * of a error happening between checking the AFSR and enabling CEEN.
	 */
	if (ch_flt.flt_trapped_ce & CE_CEEN_NODEFER)
		set_error_enable(get_error_enable() | EN_REG_CEEN);
	if (clear_errors(&ch_flt)) {
		(void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs,
		    NULL);
	}

	/*
	 * Panic here if aflt->flt_panic has been set.  Enqueued errors will
	 * be logged as part of the panic flow.
	 */
	if (aflt->flt_panic)
		fm_panic("%sError(s)", pr_reason);
}

/*
 * The async_err handler transfers control here for UE, EMU, EDU:BLD,
 * L3_EDU:BLD, TO, and BERR events.
 * Deferred errors controlled by NCEEN: UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR
 *
 * Cheetah+: No additional errors handled.
 *
 * Note that the p_clo_flags input is only valid in cases where the
 * cpu_private struct is not yet initialized (since that is the only
 * time that information cannot be obtained from the logout struct.)
 */
/*ARGSUSED*/
void
cpu_deferred_error(struct regs *rp, ulong_t p_clo_flags)
{
	ushort_t ttype, tl;
	ch_async_flt_t ch_flt;
	struct async_flt *aflt;
	int trampolined = 0;
	char pr_reason[MAX_REASON_STRING];
	ch_cpu_logout_t *clop;
	uint64_t ceen, clo_flags;
	uint64_t log_afsr;
	uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs;
	ch_cpu_errors_t cpu_error_regs;
	int expected = DDI_FM_ERR_UNEXPECTED;
	ddi_acc_hdl_t *hp;

	/*
	 * We need to look at p_flag to determine if the thread detected an
	 * error while dumping core.  We can't grab p_lock here, but it's ok
	 * because we just need a consistent snapshot and we know that everyone
	 * else will store a consistent set of bits while holding p_lock.  We
	 * don't have to worry about a race because SDOCORE is set once prior
	 * to doing i/o from the process's address space and is never cleared.
	 */
	uint_t pflag = ttoproc(curthread)->p_flag;

	bzero(&ch_flt, sizeof (ch_async_flt_t));
	/*
	 * Get the CPU log out info. If we can't find our CPU private
	 * pointer then we will have to make due without any detailed
	 * logout information.
	 */
	if (CPU_PRIVATE(CPU) == NULL) {
		clop = NULL;
		ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID;
		get_cpu_error_state(&cpu_error_regs);
		set_cpu_error_state(&cpu_error_regs);
		t_afar = cpu_error_regs.afar;
		t_afsr = cpu_error_regs.afsr;
		t_afsr_ext = cpu_error_regs.afsr_ext;
#if defined(SERRANO)
		ch_flt.afar2 = cpu_error_regs.afar2;
#endif	/* SERRANO */
		clo_flags = p_clo_flags;
	} else {
		clop = CPU_PRIVATE_PTR(CPU, chpr_async_logout);
		t_afar = clop->clo_data.chd_afar;
		t_afsr = clop->clo_data.chd_afsr;
		t_afsr_ext = clop->clo_data.chd_afsr_ext;
#if defined(SERRANO)
		ch_flt.afar2 = clop->clo_data.chd_afar2;
#endif	/* SERRANO */
		clo_flags = clop->clo_flags;
	}

	/*
	 * In order to simplify code, we maintain this afsr_errs
	 * variable which holds the aggregate of AFSR and AFSR_EXT
	 * sticky bits.
	 */
	t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) |
	    (t_afsr & C_AFSR_ALL_ERRS);
	pr_reason[0] = '\0';

	/*
	 * Grab information encoded into our clo_flags field.
	 */
	ceen = clo_flags & EN_REG_CEEN;
	tl = (clo_flags & CLO_FLAGS_TL_MASK) >> CLO_FLAGS_TL_SHIFT;
	ttype = (clo_flags & CLO_FLAGS_TT_MASK) >> CLO_FLAGS_TT_SHIFT;

	/*
	 * handle the specific error
	 */
	aflt = (struct async_flt *)&ch_flt;
	aflt->flt_id = gethrtime_waitfree();
	aflt->flt_bus_id = getprocessorid();
	aflt->flt_inst = CPU->cpu_id;
	ch_flt.afsr_ext = t_afsr_ext;
	ch_flt.afsr_errs = t_afsr_errs;
	aflt->flt_stat = t_afsr;
	aflt->flt_addr = t_afar;
	aflt->flt_pc = (caddr_t)rp->r_pc;
	aflt->flt_prot = AFLT_PROT_NONE;
	aflt->flt_class = CPU_FAULT;
	aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ?  1 : 0;
	aflt->flt_tl = (uchar_t)tl;
	aflt->flt_panic = ((tl != 0) || (aft_testfatal != 0) ||
	    C_AFSR_PANIC(t_afsr_errs));
	aflt->flt_core = (pflag & SDOCORE) ? 1 : 0;
	aflt->flt_status = ((ttype == T_DATA_ERROR) ? ECC_D_TRAP : ECC_I_TRAP);

	/*
	 * If the trap occurred in privileged mode at TL=0, we need to check to
	 * see if we were executing in the kernel under on_trap() or t_lofault
	 * protection.  If so, modify the saved registers so that we return
	 * from the trap to the appropriate trampoline routine.
	 */
	if (aflt->flt_priv && tl == 0) {
		if (curthread->t_ontrap != NULL) {
			on_trap_data_t *otp = curthread->t_ontrap;

			if (otp->ot_prot & OT_DATA_EC) {
				aflt->flt_prot = AFLT_PROT_EC;
				otp->ot_trap |= OT_DATA_EC;
				rp->r_pc = otp->ot_trampoline;
				rp->r_npc = rp->r_pc + 4;
				trampolined = 1;
			}

			if ((t_afsr & (C_AFSR_TO | C_AFSR_BERR)) &&
			    (otp->ot_prot & OT_DATA_ACCESS)) {
				aflt->flt_prot = AFLT_PROT_ACCESS;
				otp->ot_trap |= OT_DATA_ACCESS;
				rp->r_pc = otp->ot_trampoline;
				rp->r_npc = rp->r_pc + 4;
				trampolined = 1;
				/*
				 * for peeks and caut_gets errors are expected
				 */
				hp = (ddi_acc_hdl_t *)otp->ot_handle;
				if (!hp)
					expected = DDI_FM_ERR_PEEK;
				else if (hp->ah_acc.devacc_attr_access ==
				    DDI_CAUTIOUS_ACC)
					expected = DDI_FM_ERR_EXPECTED;
			}

		} else if (curthread->t_lofault) {
			aflt->flt_prot = AFLT_PROT_COPY;
			rp->r_g1 = EFAULT;
			rp->r_pc = curthread->t_lofault;
			rp->r_npc = rp->r_pc + 4;
			trampolined = 1;
		}
	}

	/*
	 * If we're in user mode or we're doing a protected copy, we either
	 * want the ASTON code below to send a signal to the user process
	 * or we want to panic if aft_panic is set.
	 *
	 * If we're in privileged mode and we're not doing a copy, then we
	 * need to check if we've trampolined.  If we haven't trampolined,
	 * we should panic.
	 */
	if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) {
		if (t_afsr_errs &
		    ((C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS) &
		    ~(C_AFSR_BERR | C_AFSR_TO)))
			aflt->flt_panic |= aft_panic;
	} else if (!trampolined) {
			aflt->flt_panic = 1;
	}

	/*
	 * If we've trampolined due to a privileged TO or BERR, or if an
	 * unprivileged TO or BERR occurred, we don't want to enqueue an
	 * event for that TO or BERR.  Queue all other events (if any) besides
	 * the TO/BERR.  Since we may not be enqueing any events, we need to
	 * ignore the number of events queued.  If we haven't trampolined due
	 * to a TO or BERR, just enqueue events normally.
	 */
	log_afsr = t_afsr_errs;
	if (trampolined) {
		log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR);
	} else if (!aflt->flt_priv) {
		/*
		 * User mode, suppress messages if
		 * cpu_berr_to_verbose is not set.
		 */
		if (!cpu_berr_to_verbose)
			log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR);
	}

	/*
	 * Log any errors that occurred
	 */
	if (((log_afsr &
	    ((C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS) & ~C_AFSR_ME)) &&
	    cpu_queue_events(&ch_flt, pr_reason, log_afsr, clop) == 0) ||
	    (t_afsr_errs & (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) == 0) {
		ch_flt.flt_type = CPU_INV_AFSR;
		cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR,
		    (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue,
		    aflt->flt_panic);
	}

	/*
	 * Zero out + invalidate CPU logout.
	 */
	if (clop) {
		bzero(clop, sizeof (ch_cpu_logout_t));
		clop->clo_data.chd_afar = LOGOUT_INVALID;
	}

#if defined(JALAPENO) || defined(SERRANO)
	/*
	 * UE/RUE/BERR/TO: Call our bus nexus friends to check for
	 * IO errors that may have resulted in this trap.
	 */
	if (t_afsr & (C_AFSR_UE|C_AFSR_RUE|C_AFSR_TO|C_AFSR_BERR)) {
		cpu_run_bus_error_handlers(aflt, expected);
	}

	/*
	 * UE/RUE: If UE or RUE is in memory, we need to flush the bad
	 * line from the Ecache.  We also need to query the bus nexus for
	 * fatal errors.  Attempts to do diagnostic read on caches may
	 * introduce more errors (especially when the module is bad).
	 */
	if (t_afsr & (C_AFSR_UE|C_AFSR_RUE)) {
		/*
		 * Ask our bus nexus friends if they have any fatal errors.  If
		 * so, they will log appropriate error messages.
		 */
		if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL)
			aflt->flt_panic = 1;

		/*
		 * We got a UE or RUE and are panicking, save the fault PA in
		 * a known location so that the platform specific panic code
		 * can check for copyback errors.
		 */
		if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) {
			panic_aflt = *aflt;
		}
	}

	/*
	 * Flush Ecache line or entire Ecache
	 */
	if (t_afsr & (C_AFSR_UE | C_AFSR_RUE | C_AFSR_EDU | C_AFSR_BERR))
		cpu_error_ecache_flush(&ch_flt);
#else /* JALAPENO || SERRANO */
	/*
	 * UE/BERR/TO: Call our bus nexus friends to check for
	 * IO errors that may have resulted in this trap.
	 */
	if (t_afsr & (C_AFSR_UE|C_AFSR_TO|C_AFSR_BERR)) {
		cpu_run_bus_error_handlers(aflt, expected);
	}

	/*
	 * UE: If the UE is in memory, we need to flush the bad
	 * line from the Ecache.  We also need to query the bus nexus for
	 * fatal errors.  Attempts to do diagnostic read on caches may
	 * introduce more errors (especially when the module is bad).
	 */
	if (t_afsr & C_AFSR_UE) {
		/*
		 * Ask our legacy bus nexus friends if they have any fatal
		 * errors.  If so, they will log appropriate error messages.
		 */
		if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL)
			aflt->flt_panic = 1;

		/*
		 * We got a UE and are panicking, save the fault PA in a known
		 * location so that the platform specific panic code can check
		 * for copyback errors.
		 */
		if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) {
			panic_aflt = *aflt;
		}
	}

	/*
	 * Flush Ecache line or entire Ecache
	 */
	if (t_afsr_errs &
	    (C_AFSR_UE | C_AFSR_EDU | C_AFSR_BERR | C_AFSR_L3_EDU))
		cpu_error_ecache_flush(&ch_flt);
#endif /* JALAPENO || SERRANO */

	/*
	 * We carefully re-enable NCEEN and CEEN and then check if any deferred
	 * or disrupting errors have happened.  We do this because if a
	 * deferred or disrupting error had occurred with NCEEN/CEEN off, the
	 * trap will not be taken when NCEEN/CEEN is re-enabled.  Note that
	 * CEEN works differently on Cheetah than on Spitfire.  Also, we enable
	 * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a
	 * deferred or disrupting error happening between checking the AFSR and
	 * enabling NCEEN/CEEN.
	 *
	 * Note: CEEN reenabled only if it was on when trap taken.
	 */
	set_error_enable(get_error_enable() | (EN_REG_NCEEN | ceen));
	if (clear_errors(&ch_flt)) {
		/*
		 * Check for secondary errors, and avoid panicking if we
		 * have them
		 */
		if (cpu_check_secondary_errors(&ch_flt, t_afsr_errs,
		    t_afar) == 0) {
			aflt->flt_panic |= ((ch_flt.afsr_errs &
			    (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) != 0);
		}
		(void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs,
		    NULL);
	}

	/*
	 * Panic here if aflt->flt_panic has been set.  Enqueued errors will
	 * be logged as part of the panic flow.
	 */
	if (aflt->flt_panic)
		fm_panic("%sError(s)", pr_reason);

	/*
	 * If we queued an error and we are going to return from the trap and
	 * the error was in user mode or inside of a copy routine, set AST flag
	 * so the queue will be drained before returning to user mode.  The
	 * AST processing will also act on our failure policy.
	 */
	if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) {
		int pcb_flag = 0;

		if (t_afsr_errs &
		    (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS &
		    ~(C_AFSR_BERR | C_AFSR_TO)))
			pcb_flag |= ASYNC_HWERR;

		if (t_afsr & C_AFSR_BERR)
			pcb_flag |= ASYNC_BERR;

		if (t_afsr & C_AFSR_TO)
			pcb_flag |= ASYNC_BTO;

		ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag;
		aston(curthread);
	}
}

#if defined(CPU_IMP_L1_CACHE_PARITY)
/*
 * Handling of data and instruction parity errors (traps 0x71, 0x72).
 *
 * For Panther, P$ data parity errors during floating point load hits
 * are also detected (reported as TT 0x71) and handled by this trap
 * handler.
 *
 * AFSR/AFAR are not set for parity errors, only TPC (a virtual address)
 * is available.
 */
/*ARGSUSED*/
void
cpu_parity_error(struct regs *rp, uint_t flags, caddr_t tpc)
{
	ch_async_flt_t ch_flt;
	struct async_flt *aflt;
	uchar_t tl = ((flags & CH_ERR_TL) != 0);
	uchar_t iparity = ((flags & CH_ERR_IPE) != 0);
	uchar_t panic = ((flags & CH_ERR_PANIC) != 0);
	char *error_class;
	int index, way, word;
	ch_dc_data_t tmp_dcp;
	int dc_set_size = dcache_size / CH_DCACHE_NWAY;
	uint64_t parity_bits, pbits;
	/* The parity bit array corresponds to the result of summing two bits */
	static int parity_bits_popc[] = { 0, 1, 1, 0 };

	/*
	 * Log the error.
	 * For icache parity errors the fault address is the trap PC.
	 * For dcache/pcache parity errors the instruction would have to
	 * be decoded to determine the address and that isn't possible
	 * at high PIL.
	 */
	bzero(&ch_flt, sizeof (ch_async_flt_t));
	aflt = (struct async_flt *)&ch_flt;
	aflt->flt_id = gethrtime_waitfree();
	aflt->flt_bus_id = getprocessorid();
	aflt->flt_inst = CPU->cpu_id;
	aflt->flt_pc = tpc;
	aflt->flt_addr = iparity ? (uint64_t)tpc : AFLT_INV_ADDR;
	aflt->flt_prot = AFLT_PROT_NONE;
	aflt->flt_class = CPU_FAULT;
	aflt->flt_priv = (tl || (rp->r_tstate & TSTATE_PRIV)) ?  1 : 0;
	aflt->flt_tl = tl;
	aflt->flt_panic = panic;
	aflt->flt_status = iparity ? ECC_IP_TRAP : ECC_DP_TRAP;
	ch_flt.flt_type = iparity ? CPU_IC_PARITY : CPU_DC_PARITY;

	if (iparity) {
		cpu_icache_parity_info(&ch_flt);
		if (ch_flt.parity_data.ipe.cpl_off != -1)
			error_class = FM_EREPORT_CPU_USIII_IDSPE;
		else if (ch_flt.parity_data.ipe.cpl_way != -1)
			error_class = FM_EREPORT_CPU_USIII_ITSPE;
		else
			error_class = FM_EREPORT_CPU_USIII_IPE;
		aflt->flt_payload = FM_EREPORT_PAYLOAD_ICACHE_PE;
	} else {
		cpu_dcache_parity_info(&ch_flt);
		if (ch_flt.parity_data.dpe.cpl_off != -1) {
			/*
			 * If not at TL 0 and running on a Jalapeno processor,
			 * then process as a true ddspe.  A true
			 * ddspe error can only occur if the way == 0
			 */
			way = ch_flt.parity_data.dpe.cpl_way;
			if ((tl == 0) && (way != 0) &&
			    IS_JALAPENO(cpunodes[CPU->cpu_id].implementation)) {
				for (index = 0; index < dc_set_size;
				    index += dcache_linesize) {
					get_dcache_dtag(index + way *
					    dc_set_size,
					    (uint64_t *)&tmp_dcp);
					/*
					 * Check data array for even parity.
					 * The 8 parity bits are grouped into
					 * 4 pairs each of which covers a 64-bit
					 * word.  The endianness is reversed
					 * -- the low-order parity bits cover
					 *  the high-order data words.
					 */
					parity_bits = tmp_dcp.dc_utag >> 8;
					for (word = 0; word < 4; word++) {
						pbits = (parity_bits >>
						    (6 - word * 2)) & 3;
						if (((popc64(
						    tmp_dcp.dc_data[word]) +
						    parity_bits_popc[pbits]) &
						    1) && (tmp_dcp.dc_tag &
						    VA13)) {
							/* cleanup */
							correct_dcache_parity(
							    dcache_size,
							    dcache_linesize);
							if (cache_boot_state &
							    DCU_DC) {
								flush_dcache();
							}

							set_dcu(get_dcu() |
							    cache_boot_state);
							return;
						}
					}
				}
			} /* (tl == 0) && (way != 0) && IS JALAPENO */
			error_class = FM_EREPORT_CPU_USIII_DDSPE;
		} else if (ch_flt.parity_data.dpe.cpl_way != -1)
			error_class = FM_EREPORT_CPU_USIII_DTSPE;
		else
			error_class = FM_EREPORT_CPU_USIII_DPE;
		aflt->flt_payload = FM_EREPORT_PAYLOAD_DCACHE_PE;
		/*
		 * For panther we also need to check the P$ for parity errors.
		 */
		if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
			cpu_pcache_parity_info(&ch_flt);
			if (ch_flt.parity_data.dpe.cpl_cache == CPU_PC_PARITY) {
				error_class = FM_EREPORT_CPU_USIII_PDSPE;
				aflt->flt_payload =
				    FM_EREPORT_PAYLOAD_PCACHE_PE;
			}
		}
	}

	cpu_errorq_dispatch(error_class, (void *)&ch_flt,
	    sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic);

	if (iparity) {
		/*
		 * Invalidate entire I$.
		 * This is required due to the use of diagnostic ASI
		 * accesses that may result in a loss of I$ coherency.
		 */
		if (cache_boot_state & DCU_IC) {
			flush_icache();
		}
		/*
		 * According to section P.3.1 of the Panther PRM, we
		 * need to do a little more for recovery on those
		 * CPUs after encountering an I$ parity error.
		 */
		if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
			flush_ipb();
			correct_dcache_parity(dcache_size,
			    dcache_linesize);
			flush_pcache();
		}
	} else {
		/*
		 * Since the valid bit is ignored when checking parity the
		 * D$ data and tag must also be corrected.  Set D$ data bits
		 * to zero and set utag to 0, 1, 2, 3.
		 */
		correct_dcache_parity(dcache_size, dcache_linesize);

		/*
		 * According to section P.3.3 of the Panther PRM, we
		 * need to do a little more for recovery on those
		 * CPUs after encountering a D$ or P$ parity error.
		 *
		 * As far as clearing P$ parity errors, it is enough to
		 * simply invalidate all entries in the P$ since P$ parity
		 * error traps are only generated for floating point load
		 * hits.
		 */
		if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
			flush_icache();
			flush_ipb();
			flush_pcache();
		}
	}

	/*
	 * Invalidate entire D$ if it was enabled.
	 * This is done to avoid stale data in the D$ which might
	 * occur with the D$ disabled and the trap handler doing
	 * stores affecting lines already in the D$.
	 */
	if (cache_boot_state & DCU_DC) {
		flush_dcache();
	}

	/*
	 * Restore caches to their bootup state.
	 */
	set_dcu(get_dcu() | cache_boot_state);

	/*
	 * Panic here if aflt->flt_panic has been set.  Enqueued errors will
	 * be logged as part of the panic flow.
	 */
	if (aflt->flt_panic)
		fm_panic("%sError(s)", iparity ? "IPE " : "DPE ");

	/*
	 * If this error occurred at TL>0 then flush the E$ here to reduce
	 * the chance of getting an unrecoverable Fast ECC error.  This
	 * flush will evict the part of the parity trap handler that is run
	 * at TL>1.
	 */
	if (tl) {
		cpu_flush_ecache();
	}
}

/*
 * On an I$ parity error, mark the appropriate entries in the ch_async_flt_t
 * to indicate which portions of the captured data should be in the ereport.
 */
void
cpu_async_log_ic_parity_err(ch_async_flt_t *ch_flt)
{
	int way = ch_flt->parity_data.ipe.cpl_way;
	int offset = ch_flt->parity_data.ipe.cpl_off;
	int tag_index;
	struct async_flt *aflt = (struct async_flt *)ch_flt;


	if ((offset != -1) || (way != -1)) {
		/*
		 * Parity error in I$ tag or data
		 */
		tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx;
		if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation))
			ch_flt->parity_data.ipe.cpl_ic[way].ic_way =
			    PN_ICIDX_TO_WAY(tag_index);
		else
			ch_flt->parity_data.ipe.cpl_ic[way].ic_way =
			    CH_ICIDX_TO_WAY(tag_index);
		ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag =
		    IC_LOGFLAG_MAGIC;
	} else {
		/*
		 * Parity error was not identified.
		 * Log tags and data for all ways.
		 */
		for (way = 0; way < CH_ICACHE_NWAY; way++) {
			tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx;
			if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation))
				ch_flt->parity_data.ipe.cpl_ic[way].ic_way =
				    PN_ICIDX_TO_WAY(tag_index);
			else
				ch_flt->parity_data.ipe.cpl_ic[way].ic_way =
				    CH_ICIDX_TO_WAY(tag_index);
			ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag =
			    IC_LOGFLAG_MAGIC;
		}
	}
}

/*
 * On an D$ parity error, mark the appropriate entries in the ch_async_flt_t
 * to indicate which portions of the captured data should be in the ereport.
 */
void
cpu_async_log_dc_parity_err(ch_async_flt_t *ch_flt)
{
	int way = ch_flt->parity_data.dpe.cpl_way;
	int offset = ch_flt->parity_data.dpe.cpl_off;
	int tag_index;

	if (offset != -1) {
		/*
		 * Parity error in D$ or P$ data array.
		 *
		 * First check to see whether the parity error is in D$ or P$
		 * since P$ data parity errors are reported in Panther using
		 * the same trap.
		 */
		if (ch_flt->parity_data.dpe.cpl_cache == CPU_PC_PARITY) {
			tag_index = ch_flt->parity_data.dpe.cpl_pc[way].pc_idx;
			ch_flt->parity_data.dpe.cpl_pc[way].pc_way =
			    CH_PCIDX_TO_WAY(tag_index);
			ch_flt->parity_data.dpe.cpl_pc[way].pc_logflag =
			    PC_LOGFLAG_MAGIC;
		} else {
			tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx;
			ch_flt->parity_data.dpe.cpl_dc[way].dc_way =
			    CH_DCIDX_TO_WAY(tag_index);
			ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag =
			    DC_LOGFLAG_MAGIC;
		}
	} else if (way != -1) {
		/*
		 * Parity error in D$ tag.
		 */
		tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx;
		ch_flt->parity_data.dpe.cpl_dc[way].dc_way =
		    CH_DCIDX_TO_WAY(tag_index);
		ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag =
		    DC_LOGFLAG_MAGIC;
	}
}
#endif	/* CPU_IMP_L1_CACHE_PARITY */

/*
 * The cpu_async_log_err() function is called via the [uc]e_drain() function to
 * post-process CPU events that are dequeued.  As such, it can be invoked
 * from softint context, from AST processing in the trap() flow, or from the
 * panic flow.  We decode the CPU-specific data, and take appropriate actions.
 * Historically this entry point was used to log the actual cmn_err(9F) text;
 * now with FMA it is used to prepare 'flt' to be converted into an ereport.
 * With FMA this function now also returns a flag which indicates to the
 * caller whether the ereport should be posted (1) or suppressed (0).
 */
static int
cpu_async_log_err(void *flt, errorq_elem_t *eqep)
{
	ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt;
	struct async_flt *aflt = (struct async_flt *)flt;
	uint64_t errors;
	extern void memscrub_induced_error(void);

	switch (ch_flt->flt_type) {
	case CPU_INV_AFSR:
		/*
		 * If it is a disrupting trap and the AFSR is zero, then
		 * the event has probably already been noted. Do not post
		 * an ereport.
		 */
		if ((aflt->flt_status & ECC_C_TRAP) &&
		    (!(aflt->flt_stat & C_AFSR_MASK)))
			return (0);
		else
			return (1);
	case CPU_TO:
	case CPU_BERR:
	case CPU_FATAL:
	case CPU_FPUERR:
		return (1);

	case CPU_UE_ECACHE_RETIRE:
		cpu_log_err(aflt);
		cpu_page_retire(ch_flt);
		return (1);

	/*
	 * Cases where we may want to suppress logging or perform
	 * extended diagnostics.
	 */
	case CPU_CE:
	case CPU_EMC:
		/*
		 * We want to skip logging and further classification
		 * only if ALL the following conditions are true:
		 *
		 *	1. There is only one error
		 *	2. That error is a correctable memory error
		 *	3. The error is caused by the memory scrubber (in
		 *	   which case the error will have occurred under
		 *	   on_trap protection)
		 *	4. The error is on a retired page
		 *
		 * Note: AFLT_PROT_EC is used places other than the memory
		 * scrubber.  However, none of those errors should occur
		 * on a retired page.
		 */
		if ((ch_flt->afsr_errs &
		    (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_CE &&
		    aflt->flt_prot == AFLT_PROT_EC) {

			if (page_retire_check(aflt->flt_addr, NULL) == 0) {
				if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) {

				/*
				 * Since we're skipping logging, we'll need
				 * to schedule the re-enabling of CEEN
				 */
				(void) timeout(cpu_delayed_check_ce_errors,
				    (void *)(uintptr_t)aflt->flt_inst,
				    drv_usectohz((clock_t)cpu_ceen_delay_secs
				    * MICROSEC));
				}

				/*
				 * Inform memscrubber - scrubbing induced
				 * CE on a retired page.
				 */
				memscrub_induced_error();
				return (0);
			}
		}

		/*
		 * Perform/schedule further classification actions, but
		 * only if the page is healthy (we don't want bad
		 * pages inducing too much diagnostic activity).  If we could
		 * not find a page pointer then we also skip this.  If
		 * ce_scrub_xdiag_recirc returns nonzero then it has chosen
		 * to copy and recirculate the event (for further diagnostics)
		 * and we should not proceed to log it here.
		 *
		 * This must be the last step here before the cpu_log_err()
		 * below - if an event recirculates cpu_ce_log_err() will
		 * not call the current function but just proceed directly
		 * to cpu_ereport_post after the cpu_log_err() avoided below.
		 *
		 * Note: Check cpu_impl_async_log_err if changing this
		 */
		if (page_retire_check(aflt->flt_addr, &errors) == EINVAL) {
			CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
			    CE_XDIAG_SKIP_NOPP);
		} else {
			if (errors != PR_OK) {
				CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
				    CE_XDIAG_SKIP_PAGEDET);
			} else if (ce_scrub_xdiag_recirc(aflt, ce_queue, eqep,
			    offsetof(ch_async_flt_t, cmn_asyncflt))) {
				return (0);
			}
		}
		/*FALLTHRU*/

	/*
	 * Cases where we just want to report the error and continue.
	 */
	case CPU_CE_ECACHE:
	case CPU_UE_ECACHE:
	case CPU_IV:
	case CPU_ORPH:
		cpu_log_err(aflt);
		return (1);

	/*
	 * Cases where we want to fall through to handle panicking.
	 */
	case CPU_UE:
		/*
		 * We want to skip logging in the same conditions as the
		 * CE case.  In addition, we want to make sure we're not
		 * panicking.
		 */
		if (!panicstr && (ch_flt->afsr_errs &
		    (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_UE &&
		    aflt->flt_prot == AFLT_PROT_EC) {
			if (page_retire_check(aflt->flt_addr, NULL) == 0) {
				/* Zero the address to clear the error */
				softcall(ecc_page_zero, (void *)aflt->flt_addr);
				/*
				 * Inform memscrubber - scrubbing induced
				 * UE on a retired page.
				 */
				memscrub_induced_error();
				return (0);
			}
		}
		cpu_log_err(aflt);
		break;

	default:
		/*
		 * If the us3_common.c code doesn't know the flt_type, it may
		 * be an implementation-specific code.  Call into the impldep
		 * backend to find out what to do: if it tells us to continue,
		 * break and handle as if falling through from a UE; if not,
		 * the impldep backend has handled the error and we're done.
		 */
		switch (cpu_impl_async_log_err(flt, eqep)) {
		case CH_ASYNC_LOG_DONE:
			return (1);
		case CH_ASYNC_LOG_RECIRC:
			return (0);
		case CH_ASYNC_LOG_CONTINUE:
			break; /* continue on to handle UE-like error */
		default:
			cmn_err(CE_WARN, "discarding error 0x%p with "
			    "invalid fault type (0x%x)",
			    (void *)aflt, ch_flt->flt_type);
			return (0);
		}
	}

	/* ... fall through from the UE case */

	if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) {
		if (!panicstr) {
			cpu_page_retire(ch_flt);
		} else {
			/*
			 * Clear UEs on panic so that we don't
			 * get haunted by them during panic or
			 * after reboot
			 */
			cpu_clearphys(aflt);
			(void) clear_errors(NULL);
		}
	}

	return (1);
}

/*
 * Retire the bad page that may contain the flushed error.
 */
void
cpu_page_retire(ch_async_flt_t *ch_flt)
{
	struct async_flt *aflt = (struct async_flt *)ch_flt;
	(void) page_retire(aflt->flt_addr, PR_UE);
}

/*
 * Return true if the error specified in the AFSR indicates
 * an E$ data error (L2$ for Cheetah/Cheetah+/Jaguar, L3$
 * for Panther, none for Jalapeno/Serrano).
 */
/* ARGSUSED */
static int
cpu_error_is_ecache_data(int cpuid, uint64_t t_afsr)
{
#if defined(JALAPENO) || defined(SERRANO)
	return (0);
#elif defined(CHEETAH_PLUS)
	if (IS_PANTHER(cpunodes[cpuid].implementation))
		return ((t_afsr & C_AFSR_EXT_L3_DATA_ERRS) != 0);
	return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0);
#else	/* CHEETAH_PLUS */
	return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0);
#endif
}

/*
 * The cpu_log_err() function is called by cpu_async_log_err() to perform the
 * generic event post-processing for correctable and uncorrectable memory,
 * E$, and MTag errors.  Historically this entry point was used to log bits of
 * common cmn_err(9F) text; now with FMA it is used to prepare 'flt' to be
 * converted into an ereport.  In addition, it transmits the error to any
 * platform-specific service-processor FRU logging routines, if available.
 */
void
cpu_log_err(struct async_flt *aflt)
{
	char unum[UNUM_NAMLEN];
	int synd_status, synd_code, afar_status;
	ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt;

	if (cpu_error_is_ecache_data(aflt->flt_inst, ch_flt->flt_bit))
		aflt->flt_status |= ECC_ECACHE;
	else
		aflt->flt_status &= ~ECC_ECACHE;
	/*
	 * Determine syndrome status.
	 */
	synd_status = afsr_to_synd_status(aflt->flt_inst,
	    ch_flt->afsr_errs, ch_flt->flt_bit);

	/*
	 * Determine afar status.
	 */
	if (pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT))
		afar_status = afsr_to_afar_status(ch_flt->afsr_errs,
		    ch_flt->flt_bit);
	else
		afar_status = AFLT_STAT_INVALID;

	synd_code = synd_to_synd_code(synd_status,
	    aflt->flt_synd, ch_flt->flt_bit);

	/*
	 * If afar status is not invalid do a unum lookup.
	 */
	if (afar_status != AFLT_STAT_INVALID) {
		(void) cpu_get_mem_unum_synd(synd_code, aflt, unum);
	} else {
		unum[0] = '\0';
	}

	/*
	 * Do not send the fruid message (plat_ecc_error_data_t)
	 * to the SC if it can handle the enhanced error information
	 * (plat_ecc_error2_data_t) or when the tunable
	 * ecc_log_fruid_enable is set to 0.
	 */

	if (&plat_ecc_capability_sc_get &&
	    plat_ecc_capability_sc_get(PLAT_ECC_ERROR_MESSAGE)) {
		if (&plat_log_fruid_error)
			plat_log_fruid_error(synd_code, aflt, unum,
			    ch_flt->flt_bit);
	}

	if (aflt->flt_func != NULL)
		aflt->flt_func(aflt, unum);

	if (afar_status != AFLT_STAT_INVALID)
		cpu_log_diag_info(ch_flt);

	/*
	 * If we have a CEEN error , we do not reenable CEEN until after
	 * we exit the trap handler. Otherwise, another error may
	 * occur causing the handler to be entered recursively.
	 * We set a timeout to trigger in cpu_ceen_delay_secs seconds,
	 * to try and ensure that the CPU makes progress in the face
	 * of a CE storm.
	 */
	if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) {
		(void) timeout(cpu_delayed_check_ce_errors,
		    (void *)(uintptr_t)aflt->flt_inst,
		    drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC));
	}
}

/*
 * Invoked by error_init() early in startup and therefore before
 * startup_errorq() is called to drain any error Q -
 *
 * startup()
 *   startup_end()
 *     error_init()
 *       cpu_error_init()
 * errorq_init()
 *   errorq_drain()
 * start_other_cpus()
 *
 * The purpose of this routine is to create error-related taskqs.  Taskqs
 * are used for this purpose because cpu_lock can't be grabbed from interrupt
 * context.
 */
void
cpu_error_init(int items)
{
	/*
	 * Create taskq(s) to reenable CE
	 */
	ch_check_ce_tq = taskq_create("cheetah_check_ce", 1, minclsyspri,
	    items, items, TASKQ_PREPOPULATE);
}

void
cpu_ce_log_err(struct async_flt *aflt, errorq_elem_t *eqep)
{
	char unum[UNUM_NAMLEN];
	int len;

	switch (aflt->flt_class) {
	case CPU_FAULT:
		cpu_ereport_init(aflt);
		if (cpu_async_log_err(aflt, eqep))
			cpu_ereport_post(aflt);
		break;

	case BUS_FAULT:
		if (aflt->flt_func != NULL) {
			(void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt,
			    unum, UNUM_NAMLEN, &len);
			aflt->flt_func(aflt, unum);
		}
		break;

	case RECIRC_CPU_FAULT:
		aflt->flt_class = CPU_FAULT;
		cpu_log_err(aflt);
		cpu_ereport_post(aflt);
		break;

	case RECIRC_BUS_FAULT:
		ASSERT(aflt->flt_class != RECIRC_BUS_FAULT);
		/*FALLTHRU*/
	default:
		cmn_err(CE_WARN, "discarding CE error 0x%p with invalid "
		    "fault class (0x%x)", (void *)aflt, aflt->flt_class);
		return;
	}
}

/*
 * Scrub and classify a CE.  This function must not modify the
 * fault structure passed to it but instead should return the classification
 * information.
 */

static uchar_t
cpu_ce_scrub_mem_err_common(struct async_flt *ecc, boolean_t logout_tried)
{
	uchar_t disp = CE_XDIAG_EXTALG;
	on_trap_data_t otd;
	uint64_t orig_err;
	ch_cpu_logout_t *clop;

	/*
	 * Clear CEEN.  CPU CE TL > 0 trap handling will already have done
	 * this, but our other callers have not.  Disable preemption to
	 * avoid CPU migration so that we restore CEEN on the correct
	 * cpu later.
	 *
	 * CEEN is cleared so that further CEs that our instruction and
	 * data footprint induce do not cause use to either creep down
	 * kernel stack to the point of overflow, or do so much CE
	 * notification as to make little real forward progress.
	 *
	 * NCEEN must not be cleared.  However it is possible that
	 * our accesses to the flt_addr may provoke a bus error or timeout
	 * if the offending address has just been unconfigured as part of
	 * a DR action.  So we must operate under on_trap protection.
	 */
	kpreempt_disable();
	orig_err = get_error_enable();
	if (orig_err & EN_REG_CEEN)
		set_error_enable(orig_err & ~EN_REG_CEEN);

	/*
	 * Our classification algorithm includes the line state before
	 * the scrub; we'd like this captured after the detection and
	 * before the algorithm below - the earlier the better.
	 *
	 * If we've come from a cpu CE trap then this info already exists
	 * in the cpu logout area.
	 *
	 * For a CE detected by memscrub for which there was no trap
	 * (running with CEEN off) cpu_log_and_clear_ce has called
	 * cpu_ce_delayed_ec_logout to capture some cache data, and
	 * marked the fault structure as incomplete as a flag to later
	 * logging code.
	 *
	 * If called directly from an IO detected CE there has been
	 * no line data capture.  In this case we logout to the cpu logout
	 * area - that's appropriate since it's the cpu cache data we need
	 * for classification.  We thus borrow the cpu logout area for a
	 * short time, and cpu_ce_delayed_ec_logout will mark it as busy in
	 * this time (we will invalidate it again below).
	 *
	 * If called from the partner check xcall handler then this cpu
	 * (the partner) has not necessarily experienced a CE at this
	 * address.  But we want to capture line state before its scrub
	 * attempt since we use that in our classification.
	 */
	if (logout_tried == B_FALSE) {
		if (!cpu_ce_delayed_ec_logout(ecc->flt_addr))
			disp |= CE_XDIAG_NOLOGOUT;
	}

	/*
	 * Scrub memory, then check AFSR for errors.  The AFAR we scrub may
	 * no longer be valid (if DR'd since the initial event) so we
	 * perform this scrub under on_trap protection.  If this access is
	 * ok then further accesses below will also be ok - DR cannot
	 * proceed while this thread is active (preemption is disabled);
	 * to be safe we'll nonetheless use on_trap again below.
	 */
	if (!on_trap(&otd, OT_DATA_ACCESS)) {
		cpu_scrubphys(ecc);
	} else {
		no_trap();
		if (orig_err & EN_REG_CEEN)
			set_error_enable(orig_err);
		kpreempt_enable();
		return (disp);
	}
	no_trap();

	/*
	 * Did the casx read of the scrub log a CE that matches the AFAR?
	 * Note that it's quite possible that the read sourced the data from
	 * another cpu.
	 */
	if (clear_ecc(ecc))
		disp |= CE_XDIAG_CE1;

	/*
	 * Read the data again.  This time the read is very likely to
	 * come from memory since the scrub induced a writeback to memory.
	 */
	if (!on_trap(&otd, OT_DATA_ACCESS)) {
		(void) lddphys(P2ALIGN(ecc->flt_addr, 8));
	} else {
		no_trap();
		if (orig_err & EN_REG_CEEN)
			set_error_enable(orig_err);
		kpreempt_enable();
		return (disp);
	}
	no_trap();

	/* Did that read induce a CE that matches the AFAR? */
	if (clear_ecc(ecc))
		disp |= CE_XDIAG_CE2;

	/*
	 * Look at the logout information and record whether we found the
	 * line in l2/l3 cache.  For Panther we are interested in whether
	 * we found it in either cache (it won't reside in both but
	 * it is possible to read it that way given the moving target).
	 */
	clop = CPU_PRIVATE(CPU) ? CPU_PRIVATE_PTR(CPU, chpr_cecc_logout) : NULL;
	if (!(disp & CE_XDIAG_NOLOGOUT) && clop &&
	    clop->clo_data.chd_afar != LOGOUT_INVALID) {
		int hit, level;
		int state;
		int totalsize;
		ch_ec_data_t *ecp;

		/*
		 * If hit is nonzero then a match was found and hit will
		 * be one greater than the index which hit.  For Panther we
		 * also need to pay attention to level to see which of l2$ or
		 * l3$ it hit in.
		 */
		hit = cpu_matching_ecache_line(ecc->flt_addr, &clop->clo_data,
		    0, &level);

		if (hit) {
			--hit;
			disp |= CE_XDIAG_AFARMATCH;

			if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) {
				if (level == 2)
					ecp = &clop->clo_data.chd_l2_data[hit];
				else
					ecp = &clop->clo_data.chd_ec_data[hit];
			} else {
				ASSERT(level == 2);
				ecp = &clop->clo_data.chd_ec_data[hit];
			}
			totalsize = cpunodes[CPU->cpu_id].ecache_size;
			state = cpu_ectag_pa_to_subblk_state(totalsize,
			    ecc->flt_addr, ecp->ec_tag);

			/*
			 * Cheetah variants use different state encodings -
			 * the CH_ECSTATE_* defines vary depending on the
			 * module we're compiled for.  Translate into our
			 * one true version.  Conflate Owner-Shared state
			 * of SSM mode with Owner as victimisation of such
			 * lines may cause a writeback.
			 */
			switch (state) {
			case CH_ECSTATE_MOD:
				disp |= EC_STATE_M;
				break;

			case CH_ECSTATE_OWN:
			case CH_ECSTATE_OWS:
				disp |= EC_STATE_O;
				break;

			case CH_ECSTATE_EXL:
				disp |= EC_STATE_E;
				break;

			case CH_ECSTATE_SHR:
				disp |= EC_STATE_S;
				break;

			default:
				disp |= EC_STATE_I;
				break;
			}
		}

		/*
		 * If we initiated the delayed logout then we are responsible
		 * for invalidating the logout area.
		 */
		if (logout_tried == B_FALSE) {
			bzero(clop, sizeof (ch_cpu_logout_t));
			clop->clo_data.chd_afar = LOGOUT_INVALID;
		}
	}

	/*
	 * Re-enable CEEN if we turned it off.
	 */
	if (orig_err & EN_REG_CEEN)
		set_error_enable(orig_err);
	kpreempt_enable();

	return (disp);
}

/*
 * Scrub a correctable memory error and collect data for classification
 * of CE type.  This function is called in the detection path, ie tl0 handling
 * of a correctable error trap (cpus) or interrupt (IO) at high PIL.
 */
void
cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t logout_tried)
{
	/*
	 * Cheetah CE classification does not set any bits in flt_status.
	 * Instead we will record classification datapoints in flt_disp.
	 */
	ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY);

	/*
	 * To check if the error detected by IO is persistent, sticky or
	 * intermittent.  This is noticed by clear_ecc().
	 */
	if (ecc->flt_status & ECC_IOBUS)
		ecc->flt_stat = C_AFSR_MEMORY;

	/*
	 * Record information from this first part of the algorithm in
	 * flt_disp.
	 */
	ecc->flt_disp = cpu_ce_scrub_mem_err_common(ecc, logout_tried);
}

/*
 * Select a partner to perform a further CE classification check from.
 * Must be called with kernel preemption disabled (to stop the cpu list
 * from changing).  The detecting cpu we are partnering has cpuid
 * aflt->flt_inst; we might not be running on the detecting cpu.
 *
 * Restrict choice to active cpus in the same cpu partition as ourselves in
 * an effort to stop bad cpus in one partition causing other partitions to
 * perform excessive diagnostic activity.  Actually since the errorq drain
 * is run from a softint most of the time and that is a global mechanism
 * this isolation is only partial.  Return NULL if we fail to find a
 * suitable partner.
 *
 * We prefer a partner that is in a different latency group to ourselves as
 * we will share fewer datapaths.  If such a partner is unavailable then
 * choose one in the same lgroup but prefer a different chip and only allow
 * a sibling core if flags includes PTNR_SIBLINGOK.  If all else fails and
 * flags includes PTNR_SELFOK then permit selection of the original detector.
 *
 * We keep a cache of the last partner selected for a cpu, and we'll try to
 * use that previous partner if no more than cpu_ce_ptnr_cachetime_sec seconds
 * have passed since that selection was made.  This provides the benefit
 * of the point-of-view of different partners over time but without
 * requiring frequent cpu list traversals.
 */

#define	PTNR_SIBLINGOK	0x1	/* Allow selection of sibling core */
#define	PTNR_SELFOK	0x2	/* Allow selection of cpu to "partner" itself */

static cpu_t *
ce_ptnr_select(struct async_flt *aflt, int flags, int *typep)
{
	cpu_t *sp, *dtcr, *ptnr, *locptnr, *sibptnr;
	hrtime_t lasttime, thistime;

	ASSERT(curthread->t_preempt > 0 || getpil() >= DISP_LEVEL);

	dtcr = cpu[aflt->flt_inst];

	/*
	 * Short-circuit for the following cases:
	 *	. the dtcr is not flagged active
	 *	. there is just one cpu present
	 *	. the detector has disappeared
	 *	. we were given a bad flt_inst cpuid; this should not happen
	 *	  (eg PCI code now fills flt_inst) but if it does it is no
	 *	  reason to panic.
	 *	. there is just one cpu left online in the cpu partition
	 *
	 * If we return NULL after this point then we do not update the
	 * chpr_ceptnr_seltime which will cause us to perform a full lookup
	 * again next time; this is the case where the only other cpu online
	 * in the detector's partition is on the same chip as the detector
	 * and since CEEN re-enable is throttled even that case should not
	 * hurt performance.
	 */
	if (dtcr == NULL || !cpu_flagged_active(dtcr->cpu_flags)) {
		return (NULL);
	}
	if (ncpus == 1 || dtcr->cpu_part->cp_ncpus == 1) {
		if (flags & PTNR_SELFOK) {
			*typep = CE_XDIAG_PTNR_SELF;
			return (dtcr);
		} else {
			return (NULL);
		}
	}

	thistime = gethrtime();
	lasttime = CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime);

	/*
	 * Select a starting point.
	 */
	if (!lasttime) {
		/*
		 * We've never selected a partner for this detector before.
		 * Start the scan at the next online cpu in the same cpu
		 * partition.
		 */
		sp = dtcr->cpu_next_part;
	} else if (thistime - lasttime < cpu_ce_ptnr_cachetime_sec * NANOSEC) {
		/*
		 * Our last selection has not aged yet.  If this partner:
		 *	. is still a valid cpu,
		 *	. is still in the same partition as the detector
		 *	. is still marked active
		 *	. satisfies the 'flags' argument criteria
		 * then select it again without updating the timestamp.
		 */
		sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)];
		if (sp == NULL || sp->cpu_part != dtcr->cpu_part ||
		    !cpu_flagged_active(sp->cpu_flags) ||
		    (sp == dtcr && !(flags & PTNR_SELFOK)) ||
		    (pg_plat_cpus_share(sp, dtcr, PGHW_CHIP) &&
		    !(flags & PTNR_SIBLINGOK))) {
			sp = dtcr->cpu_next_part;
		} else {
			if (sp->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) {
				*typep = CE_XDIAG_PTNR_REMOTE;
			} else if (sp == dtcr) {
				*typep = CE_XDIAG_PTNR_SELF;
			} else if (pg_plat_cpus_share(sp, dtcr, PGHW_CHIP)) {
				*typep = CE_XDIAG_PTNR_SIBLING;
			} else {
				*typep = CE_XDIAG_PTNR_LOCAL;
			}
			return (sp);
		}
	} else {
		/*
		 * Our last selection has aged.  If it is nonetheless still a
		 * valid cpu then start the scan at the next cpu in the
		 * partition after our last partner.  If the last selection
		 * is no longer a valid cpu then go with our default.  In
		 * this way we slowly cycle through possible partners to
		 * obtain multiple viewpoints over time.
		 */
		sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)];
		if (sp == NULL) {
			sp = dtcr->cpu_next_part;
		} else {
			sp = sp->cpu_next_part;		/* may be dtcr */
			if (sp->cpu_part != dtcr->cpu_part)
				sp = dtcr;
		}
	}

	/*
	 * We have a proposed starting point for our search, but if this
	 * cpu is offline then its cpu_next_part will point to itself
	 * so we can't use that to iterate over cpus in this partition in
	 * the loop below.  We still want to avoid iterating over cpus not
	 * in our partition, so in the case that our starting point is offline
	 * we will repoint it to be the detector itself;  and if the detector
	 * happens to be offline we'll return NULL from the following loop.
	 */
	if (!cpu_flagged_active(sp->cpu_flags)) {
		sp = dtcr;
	}

	ptnr = sp;
	locptnr = NULL;
	sibptnr = NULL;
	do {
		if (ptnr == dtcr || !cpu_flagged_active(ptnr->cpu_flags))
			continue;
		if (ptnr->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) {
			CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = ptnr->cpu_id;
			CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime;
			*typep = CE_XDIAG_PTNR_REMOTE;
			return (ptnr);
		}
		if (pg_plat_cpus_share(ptnr, dtcr, PGHW_CHIP)) {
			if (sibptnr == NULL)
				sibptnr = ptnr;
			continue;
		}
		if (locptnr == NULL)
			locptnr = ptnr;
	} while ((ptnr = ptnr->cpu_next_part) != sp);

	/*
	 * A foreign partner has already been returned if one was available.
	 *
	 * If locptnr is not NULL it is a cpu in the same lgroup as the
	 * detector, is active, and is not a sibling of the detector.
	 *
	 * If sibptnr is not NULL it is a sibling of the detector, and is
	 * active.
	 *
	 * If we have to resort to using the detector itself we have already
	 * checked that it is active.
	 */
	if (locptnr) {
		CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = locptnr->cpu_id;
		CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime;
		*typep = CE_XDIAG_PTNR_LOCAL;
		return (locptnr);
	} else if (sibptnr && flags & PTNR_SIBLINGOK) {
		CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = sibptnr->cpu_id;
		CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime;
		*typep = CE_XDIAG_PTNR_SIBLING;
		return (sibptnr);
	} else if (flags & PTNR_SELFOK) {
		CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = dtcr->cpu_id;
		CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime;
		*typep = CE_XDIAG_PTNR_SELF;
		return (dtcr);
	}

	return (NULL);
}

/*
 * Cross call handler that is requested to run on the designated partner of
 * a cpu that experienced a possibly sticky or possibly persistnet CE.
 */
static void
ce_ptnrchk_xc(struct async_flt *aflt, uchar_t *dispp)
{
	*dispp = cpu_ce_scrub_mem_err_common(aflt, B_FALSE);
}

/*
 * The associated errorqs are never destroyed so we do not need to deal with
 * them disappearing before this timeout fires.  If the affected memory
 * has been DR'd out since the original event the scrub algrithm will catch
 * any errors and return null disposition info.  If the original detecting
 * cpu has been DR'd out then ereport detector info will not be able to
 * lookup CPU type;  with a small timeout this is unlikely.
 */
static void
ce_lkychk_cb(ce_lkychk_cb_t *cbarg)
{
	struct async_flt *aflt = cbarg->lkycb_aflt;
	uchar_t disp;
	cpu_t *cp;
	int ptnrtype;

	kpreempt_disable();
	if (cp = ce_ptnr_select(aflt, PTNR_SIBLINGOK | PTNR_SELFOK,
	    &ptnrtype)) {
		xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, (uint64_t)aflt,
		    (uint64_t)&disp);
		CE_XDIAG_SETLKYINFO(aflt->flt_disp, disp);
		CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id);
		CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype);
	} else {
		ce_xdiag_lkydrops++;
		if (ncpus > 1)
			CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
			    CE_XDIAG_SKIP_NOPTNR);
	}
	kpreempt_enable();

	errorq_commit(cbarg->lkycb_eqp, cbarg->lkycb_eqep, ERRORQ_ASYNC);
	kmem_free(cbarg, sizeof (ce_lkychk_cb_t));
}

/*
 * Called from errorq drain code when processing a CE error, both from
 * CPU and PCI drain functions.  Decide what further classification actions,
 * if any, we will perform.  Perform immediate actions now, and schedule
 * delayed actions as required.  Note that we are no longer necessarily running
 * on the detecting cpu, and that the async_flt structure will not persist on
 * return from this function.
 *
 * Calls to this function should aim to be self-throtlling in some way.  With
 * the delayed re-enable of CEEN the absolute rate of calls should not
 * be excessive.  Callers should also avoid performing in-depth classification
 * for events in pages that are already known to be suspect.
 *
 * We return nonzero to indicate that the event has been copied and
 * recirculated for further testing.  The caller should not log the event
 * in this case - it will be logged when further test results are available.
 *
 * Our possible contexts are that of errorq_drain: below lock level or from
 * panic context.  We can assume that the cpu we are running on is online.
 */


#ifdef DEBUG
static int ce_xdiag_forceaction;
#endif

int
ce_scrub_xdiag_recirc(struct async_flt *aflt, errorq_t *eqp,
    errorq_elem_t *eqep, size_t afltoffset)
{
	ce_dispact_t dispact, action;
	cpu_t *cp;
	uchar_t dtcrinfo, disp;
	int ptnrtype;

	if (!ce_disp_inited || panicstr || ce_xdiag_off) {
		ce_xdiag_drops++;
		return (0);
	} else if (!aflt->flt_in_memory) {
		ce_xdiag_drops++;
		CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOTMEM);
		return (0);
	}

	dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp);

	/*
	 * Some correctable events are not scrubbed/classified, such as those
	 * noticed at the tail of cpu_deferred_error.  So if there is no
	 * initial detector classification go no further.
	 */
	if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) {
		ce_xdiag_drops++;
		CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOSCRUB);
		return (0);
	}

	dispact = CE_DISPACT(ce_disp_table,
	    CE_XDIAG_AFARMATCHED(dtcrinfo),
	    CE_XDIAG_STATE(dtcrinfo),
	    CE_XDIAG_CE1SEEN(dtcrinfo),
	    CE_XDIAG_CE2SEEN(dtcrinfo));


	action = CE_ACT(dispact);	/* bad lookup caught below */
#ifdef DEBUG
	if (ce_xdiag_forceaction != 0)
		action = ce_xdiag_forceaction;
#endif

	switch (action) {
	case CE_ACT_LKYCHK: {
		caddr_t ndata;
		errorq_elem_t *neqep;
		struct async_flt *ecc;
		ce_lkychk_cb_t *cbargp;

		if ((ndata = errorq_elem_dup(eqp, eqep, &neqep)) == NULL) {
			ce_xdiag_lkydrops++;
			CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
			    CE_XDIAG_SKIP_DUPFAIL);
			break;
		}
		ecc = (struct async_flt *)(ndata + afltoffset);

		ASSERT(ecc->flt_class == CPU_FAULT ||
		    ecc->flt_class == BUS_FAULT);
		ecc->flt_class = (ecc->flt_class == CPU_FAULT) ?
		    RECIRC_CPU_FAULT : RECIRC_BUS_FAULT;

		cbargp = kmem_alloc(sizeof (ce_lkychk_cb_t), KM_SLEEP);
		cbargp->lkycb_aflt = ecc;
		cbargp->lkycb_eqp = eqp;
		cbargp->lkycb_eqep = neqep;

		(void) timeout((void (*)(void *))ce_lkychk_cb,
		    (void *)cbargp, drv_usectohz(cpu_ce_lkychk_timeout_usec));
		return (1);
	}

	case CE_ACT_PTNRCHK:
		kpreempt_disable();	/* stop cpu list changing */
		if ((cp = ce_ptnr_select(aflt, 0, &ptnrtype)) != NULL) {
			xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc,
			    (uint64_t)aflt, (uint64_t)&disp);
			CE_XDIAG_SETPTNRINFO(aflt->flt_disp, disp);
			CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id);
			CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype);
		} else if (ncpus > 1) {
			ce_xdiag_ptnrdrops++;
			CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
			    CE_XDIAG_SKIP_NOPTNR);
		} else {
			ce_xdiag_ptnrdrops++;
			CE_XDIAG_SETSKIPCODE(aflt->flt_disp,
			    CE_XDIAG_SKIP_UNIPROC);
		}
		kpreempt_enable();
		break;

	case CE_ACT_DONE:
		break;

	case CE_DISP_BAD:
	default:
#ifdef DEBUG
		cmn_err(CE_PANIC, "ce_scrub_post: Bad action '%d'", action);
#endif
		ce_xdiag_bad++;
		CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_ACTBAD);
		break;
	}

	return (0);
}

/*
 * We route all errors through a single switch statement.
 */
void
cpu_ue_log_err(struct async_flt *aflt)
{
	switch (aflt->flt_class) {
	case CPU_FAULT:
		cpu_ereport_init(aflt);
		if (cpu_async_log_err(aflt, NULL))
			cpu_ereport_post(aflt);
		break;

	case BUS_FAULT:
		bus_async_log_err(aflt);
		break;

	default:
		cmn_err(CE_WARN, "discarding async error %p with invalid "
		    "fault class (0x%x)", (void *)aflt, aflt->flt_class);
		return;
	}
}

/*
 * Routine for panic hook callback from panic_idle().
 */
void
cpu_async_panic_callb(void)
{
	ch_async_flt_t ch_flt;
	struct async_flt *aflt;
	ch_cpu_errors_t cpu_error_regs;
	uint64_t afsr_errs;

	get_cpu_error_state(&cpu_error_regs);

	afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) |
	    (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS);

	if (afsr_errs) {

		bzero(&ch_flt, sizeof (ch_async_flt_t));
		aflt = (struct async_flt *)&ch_flt;
		aflt->flt_id = gethrtime_waitfree();
		aflt->flt_bus_id = getprocessorid();
		aflt->flt_inst = CPU->cpu_id;
		aflt->flt_stat = cpu_error_regs.afsr;
		aflt->flt_addr = cpu_error_regs.afar;
		aflt->flt_prot = AFLT_PROT_NONE;
		aflt->flt_class = CPU_FAULT;
		aflt->flt_priv = ((cpu_error_regs.afsr & C_AFSR_PRIV) != 0);
		aflt->flt_panic = 1;
		ch_flt.afsr_ext = cpu_error_regs.afsr_ext;
		ch_flt.afsr_errs = afsr_errs;
#if defined(SERRANO)
		ch_flt.afar2 = cpu_error_regs.afar2;
#endif	/* SERRANO */
		(void) cpu_queue_events(&ch_flt, NULL, afsr_errs, NULL);
	}
}

/*
 * Routine to convert a syndrome into a syndrome code.
 */
static int
synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit)
{
	if (synd_status == AFLT_STAT_INVALID)
		return (-1);

	/*
	 * Use the syndrome to index the appropriate syndrome table,
	 * to get the code indicating which bit(s) is(are) bad.
	 */
	if (afsr_bit &
	    (C_AFSR_MSYND_ERRS | C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) {
		if (afsr_bit & C_AFSR_MSYND_ERRS) {
#if defined(JALAPENO) || defined(SERRANO)
			if ((synd == 0) || (synd >= BSYND_TBL_SIZE))
				return (-1);
			else
				return (BPAR0 + synd);
#else /* JALAPENO || SERRANO */
			if ((synd == 0) || (synd >= MSYND_TBL_SIZE))
				return (-1);
			else
				return (mtag_syndrome_tab[synd]);
#endif /* JALAPENO || SERRANO */
		} else {
			if ((synd == 0) || (synd >= ESYND_TBL_SIZE))
				return (-1);
			else
				return (ecc_syndrome_tab[synd]);
		}
	} else {
		return (-1);
	}
}

int
cpu_get_mem_sid(char *unum, char *buf, int buflen, int *lenp)
{
	if (&plat_get_mem_sid)
		return (plat_get_mem_sid(unum, buf, buflen, lenp));
	else
		return (ENOTSUP);
}

int
cpu_get_mem_offset(uint64_t flt_addr, uint64_t *offp)
{
	if (&plat_get_mem_offset)
		return (plat_get_mem_offset(flt_addr, offp));
	else
		return (ENOTSUP);
}

int
cpu_get_mem_addr(char *unum, char *sid, uint64_t offset, uint64_t *addrp)
{
	if (&plat_get_mem_addr)
		return (plat_get_mem_addr(unum, sid, offset, addrp));
	else
		return (ENOTSUP);
}

/*
 * Routine to return a string identifying the physical name
 * associated with a memory/cache error.
 */
int
cpu_get_mem_unum(int synd_status, ushort_t flt_synd, uint64_t flt_stat,
    uint64_t flt_addr, int flt_bus_id, int flt_in_memory,
    ushort_t flt_status, char *buf, int buflen, int *lenp)
{
	int synd_code;
	int ret;

	/*
	 * An AFSR of -1 defaults to a memory syndrome.
	 */
	if (flt_stat == (uint64_t)-1)
		flt_stat = C_AFSR_CE;

	synd_code = synd_to_synd_code(synd_status, flt_synd, flt_stat);

	/*
	 * Syndrome code must be either a single-bit error code
	 * (0...143) or -1 for unum lookup.
	 */
	if (synd_code < 0 || synd_code >= M2)
		synd_code = -1;
	if (&plat_get_mem_unum) {
		if ((ret = plat_get_mem_unum(synd_code, flt_addr, flt_bus_id,
		    flt_in_memory, flt_status, buf, buflen, lenp)) != 0) {
			buf[0] = '\0';
			*lenp = 0;
		}

		return (ret);
	}

	return (ENOTSUP);
}

/*
 * Wrapper for cpu_get_mem_unum() routine that takes an
 * async_flt struct rather than explicit arguments.
 */
int
cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt,
    char *buf, int buflen, int *lenp)
{
	/*
	 * If we come thru here for an IO bus error aflt->flt_stat will
	 * not be the CPU AFSR, and we pass in a -1 to cpu_get_mem_unum()
	 * so it will interpret this as a memory error.
	 */
	return (cpu_get_mem_unum(synd_status, aflt->flt_synd,
	    (aflt->flt_class == BUS_FAULT) ?
	    (uint64_t)-1 : ((ch_async_flt_t *)aflt)->flt_bit,
	    aflt->flt_addr, aflt->flt_bus_id, aflt->flt_in_memory,
	    aflt->flt_status, buf, buflen, lenp));
}

/*
 * Return unum string given synd_code and async_flt into
 * the buf with size UNUM_NAMLEN
 */
static int
cpu_get_mem_unum_synd(int synd_code, struct async_flt *aflt, char *buf)
{
	int ret, len;

	/*
	 * Syndrome code must be either a single-bit error code
	 * (0...143) or -1 for unum lookup.
	 */
	if (synd_code < 0 || synd_code >= M2)
		synd_code = -1;
	if (&plat_get_mem_unum) {
		if ((ret = plat_get_mem_unum(synd_code, aflt->flt_addr,
		    aflt->flt_bus_id, aflt->flt_in_memory,
		    aflt->flt_status, buf, UNUM_NAMLEN, &len)) != 0) {
			buf[0] = '\0';
		}
		return (ret);
	}

	buf[0] = '\0';
	return (ENOTSUP);
}

/*
 * This routine is a more generic interface to cpu_get_mem_unum()
 * that may be used by other modules (e.g. the 'mm' driver, through
 * the 'MEM_NAME' ioctl, which is used by fmd to resolve unum's
 * for Jalapeno/Serrano FRC/RCE or FRU/RUE paired events).
 */
int
cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar,
    char *buf, int buflen, int *lenp)
{
	int synd_status, flt_in_memory, ret;
	ushort_t flt_status = 0;
	char unum[UNUM_NAMLEN];
	uint64_t t_afsr_errs;

	/*
	 * Check for an invalid address.
	 */
	if (afar == (uint64_t)-1)
		return (ENXIO);

	if (synd == (uint64_t)-1)
		synd_status = AFLT_STAT_INVALID;
	else
		synd_status = AFLT_STAT_VALID;

	flt_in_memory = (*afsr & C_AFSR_MEMORY) &&
	    pf_is_memory(afar >> MMU_PAGESHIFT);

	/*
	 * Get aggregate AFSR for call to cpu_error_is_ecache_data.
	 */
	if (*afsr == (uint64_t)-1)
		t_afsr_errs = C_AFSR_CE;
	else {
		t_afsr_errs = (*afsr & C_AFSR_ALL_ERRS);
#if defined(CHEETAH_PLUS)
		if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation))
			t_afsr_errs |= (*(afsr + 1) & C_AFSR_EXT_ALL_ERRS);
#endif	/* CHEETAH_PLUS */
	}

	/*
	 * Turn on ECC_ECACHE if error type is E$ Data.
	 */
	if (cpu_error_is_ecache_data(CPU->cpu_id, t_afsr_errs))
		flt_status |= ECC_ECACHE;

	ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, t_afsr_errs, afar,
	    CPU->cpu_id, flt_in_memory, flt_status, unum, UNUM_NAMLEN, lenp);
	if (ret != 0)
		return (ret);

	if (*lenp >= buflen)
		return (ENAMETOOLONG);

	(void) strncpy(buf, unum, buflen);

	return (0);
}

/*
 * Routine to return memory information associated
 * with a physical address and syndrome.
 */
int
cpu_get_mem_info(uint64_t synd, uint64_t afar,
    uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep,
    int *segsp, int *banksp, int *mcidp)
{
	int synd_status, synd_code;

	if (afar == (uint64_t)-1)
		return (ENXIO);

	if (synd == (uint64_t)-1)
		synd_status = AFLT_STAT_INVALID;
	else
		synd_status = AFLT_STAT_VALID;

	synd_code = synd_to_synd_code(synd_status, synd, C_AFSR_CE);

	if (p2get_mem_info != NULL)
		return ((p2get_mem_info)(synd_code, afar,
		    mem_sizep, seg_sizep, bank_sizep,
		    segsp, banksp, mcidp));
	else
		return (ENOTSUP);
}

/*
 * Routine to return a string identifying the physical
 * name associated with a cpuid.
 */
int
cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp)
{
	int ret;
	char unum[UNUM_NAMLEN];

	if (&plat_get_cpu_unum) {
		if ((ret = plat_get_cpu_unum(cpuid, unum, UNUM_NAMLEN, lenp))
		    != 0)
			return (ret);
	} else {
		return (ENOTSUP);
	}

	if (*lenp >= buflen)
		return (ENAMETOOLONG);

	(void) strncpy(buf, unum, buflen);

	return (0);
}

/*
 * This routine exports the name buffer size.
 */
size_t
cpu_get_name_bufsize()
{
	return (UNUM_NAMLEN);
}

/*
 * Historical function, apparantly not used.
 */
/* ARGSUSED */
void
cpu_read_paddr(struct async_flt *ecc, short verbose, short