17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5*1e2e7a75Shuah * Common Development and Distribution License (the "License"). 6*1e2e7a75Shuah * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 227aec1d6eScindi * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 277c478bd9Sstevel@tonic-gate 287c478bd9Sstevel@tonic-gate #include <sys/types.h> 297c478bd9Sstevel@tonic-gate #include <sys/systm.h> 307c478bd9Sstevel@tonic-gate #include <sys/ddi.h> 317c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 327c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 337c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 347c478bd9Sstevel@tonic-gate #include <sys/machparam.h> 357c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 367c478bd9Sstevel@tonic-gate #include <sys/machthread.h> 377c478bd9Sstevel@tonic-gate #include <sys/cpu.h> 387c478bd9Sstevel@tonic-gate #include <sys/cmp.h> 397c478bd9Sstevel@tonic-gate #include <sys/elf_SPARC.h> 407c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 417c478bd9Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 427c478bd9Sstevel@tonic-gate #include <vm/seg_kpm.h> 437c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 447c478bd9Sstevel@tonic-gate #include <sys/cheetahregs.h> 457c478bd9Sstevel@tonic-gate #include <sys/us3_module.h> 467c478bd9Sstevel@tonic-gate #include <sys/async.h> 477c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 487c478bd9Sstevel@tonic-gate #include <sys/debug.h> 497c478bd9Sstevel@tonic-gate #include <sys/dditypes.h> 507c478bd9Sstevel@tonic-gate #include <sys/prom_debug.h> 517c478bd9Sstevel@tonic-gate #include <sys/prom_plat.h> 527c478bd9Sstevel@tonic-gate #include <sys/cpu_module.h> 537c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 547c478bd9Sstevel@tonic-gate #include <sys/intreg.h> 557c478bd9Sstevel@tonic-gate #include <sys/clock.h> 567c478bd9Sstevel@tonic-gate #include <sys/platform_module.h> 577c478bd9Sstevel@tonic-gate #include <sys/machtrap.h> 587c478bd9Sstevel@tonic-gate #include <sys/ontrap.h> 597c478bd9Sstevel@tonic-gate #include <sys/panic.h> 607c478bd9Sstevel@tonic-gate #include <sys/memlist.h> 617c478bd9Sstevel@tonic-gate #include <sys/bootconf.h> 627c478bd9Sstevel@tonic-gate #include <sys/ivintr.h> 637c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 647c478bd9Sstevel@tonic-gate #include <sys/taskq.h> 657c478bd9Sstevel@tonic-gate #include <sys/note.h> 667c478bd9Sstevel@tonic-gate #include <sys/ndifm.h> 677c478bd9Sstevel@tonic-gate #include <sys/ddifm.h> 687c478bd9Sstevel@tonic-gate #include <sys/fm/protocol.h> 697c478bd9Sstevel@tonic-gate #include <sys/fm/util.h> 707c478bd9Sstevel@tonic-gate #include <sys/fm/cpu/UltraSPARC-III.h> 717c478bd9Sstevel@tonic-gate #include <sys/fpras_impl.h> 727c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 737c478bd9Sstevel@tonic-gate #include <sys/watchpoint.h> 747c478bd9Sstevel@tonic-gate #include <sys/plat_ecc_unum.h> 757c478bd9Sstevel@tonic-gate #include <sys/cyclic.h> 767c478bd9Sstevel@tonic-gate #include <sys/errorq.h> 777c478bd9Sstevel@tonic-gate #include <sys/errclassify.h> 787c478bd9Sstevel@tonic-gate 797c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 807c478bd9Sstevel@tonic-gate #include <sys/xc_impl.h> 817c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 827c478bd9Sstevel@tonic-gate 837c478bd9Sstevel@tonic-gate /* 847c478bd9Sstevel@tonic-gate * Note that 'Cheetah PRM' refers to: 857c478bd9Sstevel@tonic-gate * SPARC V9 JPS1 Implementation Supplement: Sun UltraSPARC-III 867c478bd9Sstevel@tonic-gate */ 877c478bd9Sstevel@tonic-gate 887c478bd9Sstevel@tonic-gate /* 897c478bd9Sstevel@tonic-gate * Per CPU pointers to physical address of TL>0 logout data areas. 907c478bd9Sstevel@tonic-gate * These pointers have to be in the kernel nucleus to avoid MMU 917c478bd9Sstevel@tonic-gate * misses. 927c478bd9Sstevel@tonic-gate */ 937c478bd9Sstevel@tonic-gate uint64_t ch_err_tl1_paddrs[NCPU]; 947c478bd9Sstevel@tonic-gate 957c478bd9Sstevel@tonic-gate /* 967c478bd9Sstevel@tonic-gate * One statically allocated structure to use during startup/DR 977c478bd9Sstevel@tonic-gate * to prevent unnecessary panics. 987c478bd9Sstevel@tonic-gate */ 997c478bd9Sstevel@tonic-gate ch_err_tl1_data_t ch_err_tl1_data; 1007c478bd9Sstevel@tonic-gate 1017c478bd9Sstevel@tonic-gate /* 1027c478bd9Sstevel@tonic-gate * Per CPU pending error at TL>0, used by level15 softint handler 1037c478bd9Sstevel@tonic-gate */ 1047c478bd9Sstevel@tonic-gate uchar_t ch_err_tl1_pending[NCPU]; 1057c478bd9Sstevel@tonic-gate 1067c478bd9Sstevel@tonic-gate /* 1077c478bd9Sstevel@tonic-gate * For deferred CE re-enable after trap. 1087c478bd9Sstevel@tonic-gate */ 1097c478bd9Sstevel@tonic-gate taskq_t *ch_check_ce_tq; 1107c478bd9Sstevel@tonic-gate 1117c478bd9Sstevel@tonic-gate /* 1127c478bd9Sstevel@tonic-gate * Internal functions. 1137c478bd9Sstevel@tonic-gate */ 1147c478bd9Sstevel@tonic-gate static int cpu_async_log_err(void *flt, errorq_elem_t *eqep); 1157c478bd9Sstevel@tonic-gate static void cpu_log_diag_info(ch_async_flt_t *ch_flt); 1167c478bd9Sstevel@tonic-gate static void cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 1177c478bd9Sstevel@tonic-gate ecc_type_to_info_t *eccp, ch_diag_data_t *cdp); 1187c478bd9Sstevel@tonic-gate static int clear_ecc(struct async_flt *ecc); 1197c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 1207c478bd9Sstevel@tonic-gate static int cpu_ecache_line_valid(ch_async_flt_t *ch_flt); 1217c478bd9Sstevel@tonic-gate #endif 1227c478bd9Sstevel@tonic-gate static int cpu_ecache_set_size(struct cpu *cp); 1237c478bd9Sstevel@tonic-gate static int cpu_ectag_line_invalid(int cachesize, uint64_t tag); 1247c478bd9Sstevel@tonic-gate static int cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr); 1257c478bd9Sstevel@tonic-gate static uint64_t cpu_ectag_to_pa(int setsize, uint64_t tag); 1267c478bd9Sstevel@tonic-gate static int cpu_ectag_pa_to_subblk_state(int cachesize, 1277c478bd9Sstevel@tonic-gate uint64_t subaddr, uint64_t tag); 1287c478bd9Sstevel@tonic-gate static void cpu_flush_ecache_line(ch_async_flt_t *ch_flt); 1297c478bd9Sstevel@tonic-gate static int afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit); 1307c478bd9Sstevel@tonic-gate static int afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit); 1317c478bd9Sstevel@tonic-gate static int afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit); 1327c478bd9Sstevel@tonic-gate static int afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit); 1337c478bd9Sstevel@tonic-gate static int synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit); 1347c478bd9Sstevel@tonic-gate static void cpu_uninit_ecache_scrub_dr(struct cpu *cp); 1357c478bd9Sstevel@tonic-gate static void cpu_scrubphys(struct async_flt *aflt); 1367c478bd9Sstevel@tonic-gate static void cpu_payload_add_aflt(struct async_flt *, nvlist_t *, nvlist_t *, 1377c478bd9Sstevel@tonic-gate int *, int *); 1387c478bd9Sstevel@tonic-gate static void cpu_payload_add_ecache(struct async_flt *, nvlist_t *); 1397c478bd9Sstevel@tonic-gate static void cpu_ereport_init(struct async_flt *aflt); 1407c478bd9Sstevel@tonic-gate static int cpu_check_secondary_errors(ch_async_flt_t *, uint64_t, uint64_t); 1417c478bd9Sstevel@tonic-gate static uint8_t cpu_flt_bit_to_plat_error(struct async_flt *aflt); 1427c478bd9Sstevel@tonic-gate static void cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 1434fd7ecabSdilpreet uint64_t nceen, ch_cpu_logout_t *clop); 1447c478bd9Sstevel@tonic-gate static int cpu_ce_delayed_ec_logout(uint64_t); 1457c478bd9Sstevel@tonic-gate static int cpu_matching_ecache_line(uint64_t, void *, int, int *); 1467c478bd9Sstevel@tonic-gate 1477c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 1487c478bd9Sstevel@tonic-gate static int mondo_recover_proc(uint16_t, int); 1497c478bd9Sstevel@tonic-gate static void cheetah_nudge_init(void); 1507c478bd9Sstevel@tonic-gate static void cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 1517c478bd9Sstevel@tonic-gate cyc_time_t *when); 1527c478bd9Sstevel@tonic-gate static void cheetah_nudge_buddy(void); 1537c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 1547c478bd9Sstevel@tonic-gate 1557c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 1567c478bd9Sstevel@tonic-gate static void cpu_dcache_parity_info(ch_async_flt_t *ch_flt); 1577c478bd9Sstevel@tonic-gate static void cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index); 1587c478bd9Sstevel@tonic-gate static void cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 1597c478bd9Sstevel@tonic-gate ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word); 1607c478bd9Sstevel@tonic-gate static void cpu_icache_parity_info(ch_async_flt_t *ch_flt); 1617c478bd9Sstevel@tonic-gate static void cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index); 1627c478bd9Sstevel@tonic-gate static void cpu_pcache_parity_info(ch_async_flt_t *ch_flt); 1637c478bd9Sstevel@tonic-gate static void cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index); 1647c478bd9Sstevel@tonic-gate static void cpu_payload_add_dcache(struct async_flt *, nvlist_t *); 1657c478bd9Sstevel@tonic-gate static void cpu_payload_add_icache(struct async_flt *, nvlist_t *); 1667c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 1677c478bd9Sstevel@tonic-gate 1687c478bd9Sstevel@tonic-gate int (*p2get_mem_info)(int synd_code, uint64_t paddr, 1697c478bd9Sstevel@tonic-gate uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 1707c478bd9Sstevel@tonic-gate int *segsp, int *banksp, int *mcidp); 1717c478bd9Sstevel@tonic-gate 1727c478bd9Sstevel@tonic-gate /* 1737c478bd9Sstevel@tonic-gate * This table is used to determine which bit(s) is(are) bad when an ECC 1747c478bd9Sstevel@tonic-gate * error occurs. The array is indexed by an 9-bit syndrome. The entries 1757c478bd9Sstevel@tonic-gate * of this array have the following semantics: 1767c478bd9Sstevel@tonic-gate * 1777c478bd9Sstevel@tonic-gate * 00-127 The number of the bad bit, when only one bit is bad. 1787c478bd9Sstevel@tonic-gate * 128 ECC bit C0 is bad. 1797c478bd9Sstevel@tonic-gate * 129 ECC bit C1 is bad. 1807c478bd9Sstevel@tonic-gate * 130 ECC bit C2 is bad. 1817c478bd9Sstevel@tonic-gate * 131 ECC bit C3 is bad. 1827c478bd9Sstevel@tonic-gate * 132 ECC bit C4 is bad. 1837c478bd9Sstevel@tonic-gate * 133 ECC bit C5 is bad. 1847c478bd9Sstevel@tonic-gate * 134 ECC bit C6 is bad. 1857c478bd9Sstevel@tonic-gate * 135 ECC bit C7 is bad. 1867c478bd9Sstevel@tonic-gate * 136 ECC bit C8 is bad. 1877c478bd9Sstevel@tonic-gate * 137-143 reserved for Mtag Data and ECC. 1887c478bd9Sstevel@tonic-gate * 144(M2) Two bits are bad within a nibble. 1897c478bd9Sstevel@tonic-gate * 145(M3) Three bits are bad within a nibble. 1907c478bd9Sstevel@tonic-gate * 146(M3) Four bits are bad within a nibble. 1917c478bd9Sstevel@tonic-gate * 147(M) Multiple bits (5 or more) are bad. 1927c478bd9Sstevel@tonic-gate * 148 NO bits are bad. 1937c478bd9Sstevel@tonic-gate * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-4,11-5. 1947c478bd9Sstevel@tonic-gate */ 1957c478bd9Sstevel@tonic-gate 1967c478bd9Sstevel@tonic-gate #define C0 128 1977c478bd9Sstevel@tonic-gate #define C1 129 1987c478bd9Sstevel@tonic-gate #define C2 130 1997c478bd9Sstevel@tonic-gate #define C3 131 2007c478bd9Sstevel@tonic-gate #define C4 132 2017c478bd9Sstevel@tonic-gate #define C5 133 2027c478bd9Sstevel@tonic-gate #define C6 134 2037c478bd9Sstevel@tonic-gate #define C7 135 2047c478bd9Sstevel@tonic-gate #define C8 136 2057c478bd9Sstevel@tonic-gate #define MT0 137 /* Mtag Data bit 0 */ 2067c478bd9Sstevel@tonic-gate #define MT1 138 2077c478bd9Sstevel@tonic-gate #define MT2 139 2087c478bd9Sstevel@tonic-gate #define MTC0 140 /* Mtag Check bit 0 */ 2097c478bd9Sstevel@tonic-gate #define MTC1 141 2107c478bd9Sstevel@tonic-gate #define MTC2 142 2117c478bd9Sstevel@tonic-gate #define MTC3 143 2127c478bd9Sstevel@tonic-gate #define M2 144 2137c478bd9Sstevel@tonic-gate #define M3 145 2147c478bd9Sstevel@tonic-gate #define M4 146 2157c478bd9Sstevel@tonic-gate #define M 147 2167c478bd9Sstevel@tonic-gate #define NA 148 2177c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2187c478bd9Sstevel@tonic-gate #define S003 149 /* Syndrome 0x003 => likely from CPU/EDU:ST/FRU/BP */ 2197c478bd9Sstevel@tonic-gate #define S003MEM 150 /* Syndrome 0x003 => likely from WDU/WBP */ 2207c478bd9Sstevel@tonic-gate #define SLAST S003MEM /* last special syndrome */ 2217c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 2227c478bd9Sstevel@tonic-gate #define S003 149 /* Syndrome 0x003 => likely from EDU:ST */ 2237c478bd9Sstevel@tonic-gate #define S071 150 /* Syndrome 0x071 => likely from WDU/CPU */ 2247c478bd9Sstevel@tonic-gate #define S11C 151 /* Syndrome 0x11c => likely from BERR/DBERR */ 2257c478bd9Sstevel@tonic-gate #define SLAST S11C /* last special syndrome */ 2267c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2277c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2287c478bd9Sstevel@tonic-gate #define BPAR0 152 /* syndrom 152 through 167 for bus parity */ 2297c478bd9Sstevel@tonic-gate #define BPAR15 167 2307c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2317c478bd9Sstevel@tonic-gate 2327c478bd9Sstevel@tonic-gate static uint8_t ecc_syndrome_tab[] = 2337c478bd9Sstevel@tonic-gate { 2347c478bd9Sstevel@tonic-gate NA, C0, C1, S003, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M, 2357c478bd9Sstevel@tonic-gate C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16, 2367c478bd9Sstevel@tonic-gate C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10, 2377c478bd9Sstevel@tonic-gate M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M, 2387c478bd9Sstevel@tonic-gate C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6, 2397c478bd9Sstevel@tonic-gate M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4, 2407c478bd9Sstevel@tonic-gate M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4, 2417c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2427c478bd9Sstevel@tonic-gate 116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 2437c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 2447c478bd9Sstevel@tonic-gate 116, S071, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 2457c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2467c478bd9Sstevel@tonic-gate C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5, 2477c478bd9Sstevel@tonic-gate M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M, 2487c478bd9Sstevel@tonic-gate M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2, 2497c478bd9Sstevel@tonic-gate 103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3, 2507c478bd9Sstevel@tonic-gate M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M, 2517c478bd9Sstevel@tonic-gate 102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3, 2527c478bd9Sstevel@tonic-gate 98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M, 2537c478bd9Sstevel@tonic-gate M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M, 2547c478bd9Sstevel@tonic-gate C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4, 2557c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2567c478bd9Sstevel@tonic-gate M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M, 2577c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 2587c478bd9Sstevel@tonic-gate M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, S11C, M, M3, M, 2597c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2607c478bd9Sstevel@tonic-gate M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2, 2617c478bd9Sstevel@tonic-gate 94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M, 2627c478bd9Sstevel@tonic-gate M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4, 2637c478bd9Sstevel@tonic-gate 89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3, 2647c478bd9Sstevel@tonic-gate 86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3, 2657c478bd9Sstevel@tonic-gate M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2, 2667c478bd9Sstevel@tonic-gate M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4, 2677c478bd9Sstevel@tonic-gate 77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M, 2687c478bd9Sstevel@tonic-gate 74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3, 2697c478bd9Sstevel@tonic-gate M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M, 2707c478bd9Sstevel@tonic-gate 80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3, 2717c478bd9Sstevel@tonic-gate M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M, 2727c478bd9Sstevel@tonic-gate M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M, 2737c478bd9Sstevel@tonic-gate 111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M 2747c478bd9Sstevel@tonic-gate }; 2757c478bd9Sstevel@tonic-gate 2767c478bd9Sstevel@tonic-gate #define ESYND_TBL_SIZE (sizeof (ecc_syndrome_tab) / sizeof (uint8_t)) 2777c478bd9Sstevel@tonic-gate 2787c478bd9Sstevel@tonic-gate #if !(defined(JALAPENO) || defined(SERRANO)) 2797c478bd9Sstevel@tonic-gate /* 2807c478bd9Sstevel@tonic-gate * This table is used to determine which bit(s) is(are) bad when a Mtag 2817c478bd9Sstevel@tonic-gate * error occurs. The array is indexed by an 4-bit ECC syndrome. The entries 2827c478bd9Sstevel@tonic-gate * of this array have the following semantics: 2837c478bd9Sstevel@tonic-gate * 2847c478bd9Sstevel@tonic-gate * -1 Invalid mtag syndrome. 2857c478bd9Sstevel@tonic-gate * 137 Mtag Data 0 is bad. 2867c478bd9Sstevel@tonic-gate * 138 Mtag Data 1 is bad. 2877c478bd9Sstevel@tonic-gate * 139 Mtag Data 2 is bad. 2887c478bd9Sstevel@tonic-gate * 140 Mtag ECC 0 is bad. 2897c478bd9Sstevel@tonic-gate * 141 Mtag ECC 1 is bad. 2907c478bd9Sstevel@tonic-gate * 142 Mtag ECC 2 is bad. 2917c478bd9Sstevel@tonic-gate * 143 Mtag ECC 3 is bad. 2927c478bd9Sstevel@tonic-gate * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-6. 2937c478bd9Sstevel@tonic-gate */ 2947c478bd9Sstevel@tonic-gate short mtag_syndrome_tab[] = 2957c478bd9Sstevel@tonic-gate { 2967c478bd9Sstevel@tonic-gate NA, MTC0, MTC1, M2, MTC2, M2, M2, MT0, MTC3, M2, M2, MT1, M2, MT2, M2, M2 2977c478bd9Sstevel@tonic-gate }; 2987c478bd9Sstevel@tonic-gate 2997c478bd9Sstevel@tonic-gate #define MSYND_TBL_SIZE (sizeof (mtag_syndrome_tab) / sizeof (short)) 3007c478bd9Sstevel@tonic-gate 3017c478bd9Sstevel@tonic-gate #else /* !(JALAPENO || SERRANO) */ 3027c478bd9Sstevel@tonic-gate 3037c478bd9Sstevel@tonic-gate #define BSYND_TBL_SIZE 16 3047c478bd9Sstevel@tonic-gate 3057c478bd9Sstevel@tonic-gate #endif /* !(JALAPENO || SERRANO) */ 3067c478bd9Sstevel@tonic-gate 3077c478bd9Sstevel@tonic-gate /* 3087c478bd9Sstevel@tonic-gate * CE initial classification and subsequent action lookup table 3097c478bd9Sstevel@tonic-gate */ 3107c478bd9Sstevel@tonic-gate static ce_dispact_t ce_disp_table[CE_INITDISPTBL_SIZE]; 3117c478bd9Sstevel@tonic-gate static int ce_disp_inited; 3127c478bd9Sstevel@tonic-gate 3137c478bd9Sstevel@tonic-gate /* 3147c478bd9Sstevel@tonic-gate * Set to disable leaky and partner check for memory correctables 3157c478bd9Sstevel@tonic-gate */ 3167c478bd9Sstevel@tonic-gate int ce_xdiag_off; 3177c478bd9Sstevel@tonic-gate 3187c478bd9Sstevel@tonic-gate /* 3197c478bd9Sstevel@tonic-gate * The following are not incremented atomically so are indicative only 3207c478bd9Sstevel@tonic-gate */ 3217c478bd9Sstevel@tonic-gate static int ce_xdiag_drops; 3227c478bd9Sstevel@tonic-gate static int ce_xdiag_lkydrops; 3237c478bd9Sstevel@tonic-gate static int ce_xdiag_ptnrdrops; 3247c478bd9Sstevel@tonic-gate static int ce_xdiag_bad; 3257c478bd9Sstevel@tonic-gate 3267c478bd9Sstevel@tonic-gate /* 3277c478bd9Sstevel@tonic-gate * CE leaky check callback structure 3287c478bd9Sstevel@tonic-gate */ 3297c478bd9Sstevel@tonic-gate typedef struct { 3307c478bd9Sstevel@tonic-gate struct async_flt *lkycb_aflt; 3317c478bd9Sstevel@tonic-gate errorq_t *lkycb_eqp; 3327c478bd9Sstevel@tonic-gate errorq_elem_t *lkycb_eqep; 3337c478bd9Sstevel@tonic-gate } ce_lkychk_cb_t; 3347c478bd9Sstevel@tonic-gate 3357c478bd9Sstevel@tonic-gate /* 3367c478bd9Sstevel@tonic-gate * defines for various ecache_flush_flag's 3377c478bd9Sstevel@tonic-gate */ 3387c478bd9Sstevel@tonic-gate #define ECACHE_FLUSH_LINE 1 3397c478bd9Sstevel@tonic-gate #define ECACHE_FLUSH_ALL 2 3407c478bd9Sstevel@tonic-gate 3417c478bd9Sstevel@tonic-gate /* 3427c478bd9Sstevel@tonic-gate * STICK sync 3437c478bd9Sstevel@tonic-gate */ 3447c478bd9Sstevel@tonic-gate #define STICK_ITERATION 10 3457c478bd9Sstevel@tonic-gate #define MAX_TSKEW 1 3467c478bd9Sstevel@tonic-gate #define EV_A_START 0 3477c478bd9Sstevel@tonic-gate #define EV_A_END 1 3487c478bd9Sstevel@tonic-gate #define EV_B_START 2 3497c478bd9Sstevel@tonic-gate #define EV_B_END 3 3507c478bd9Sstevel@tonic-gate #define EVENTS 4 3517c478bd9Sstevel@tonic-gate 3527c478bd9Sstevel@tonic-gate static int64_t stick_iter = STICK_ITERATION; 3537c478bd9Sstevel@tonic-gate static int64_t stick_tsk = MAX_TSKEW; 3547c478bd9Sstevel@tonic-gate 3557c478bd9Sstevel@tonic-gate typedef enum { 3567c478bd9Sstevel@tonic-gate EVENT_NULL = 0, 3577c478bd9Sstevel@tonic-gate SLAVE_START, 3587c478bd9Sstevel@tonic-gate SLAVE_CONT, 3597c478bd9Sstevel@tonic-gate MASTER_START 3607c478bd9Sstevel@tonic-gate } event_cmd_t; 3617c478bd9Sstevel@tonic-gate 3627c478bd9Sstevel@tonic-gate static volatile event_cmd_t stick_sync_cmd = EVENT_NULL; 3637c478bd9Sstevel@tonic-gate static int64_t timestamp[EVENTS]; 3647c478bd9Sstevel@tonic-gate static volatile int slave_done; 3657c478bd9Sstevel@tonic-gate 3667c478bd9Sstevel@tonic-gate #ifdef DEBUG 3677c478bd9Sstevel@tonic-gate #define DSYNC_ATTEMPTS 64 3687c478bd9Sstevel@tonic-gate typedef struct { 3697c478bd9Sstevel@tonic-gate int64_t skew_val[DSYNC_ATTEMPTS]; 3707c478bd9Sstevel@tonic-gate } ss_t; 3717c478bd9Sstevel@tonic-gate 3727c478bd9Sstevel@tonic-gate ss_t stick_sync_stats[NCPU]; 3737c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 3747c478bd9Sstevel@tonic-gate 375*1e2e7a75Shuah uint_t cpu_impl_dual_pgsz = 0; 3767c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_DUAL_PAGESIZE) 3777c478bd9Sstevel@tonic-gate uint_t disable_dual_pgsz = 0; 3787c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_DUAL_PAGESIZE */ 3797c478bd9Sstevel@tonic-gate 3807c478bd9Sstevel@tonic-gate /* 3817c478bd9Sstevel@tonic-gate * Save the cache bootup state for use when internal 3827c478bd9Sstevel@tonic-gate * caches are to be re-enabled after an error occurs. 3837c478bd9Sstevel@tonic-gate */ 3847c478bd9Sstevel@tonic-gate uint64_t cache_boot_state; 3857c478bd9Sstevel@tonic-gate 3867c478bd9Sstevel@tonic-gate /* 3877c478bd9Sstevel@tonic-gate * PA[22:0] represent Displacement in Safari configuration space. 3887c478bd9Sstevel@tonic-gate */ 3897c478bd9Sstevel@tonic-gate uint_t root_phys_addr_lo_mask = 0x7fffffu; 3907c478bd9Sstevel@tonic-gate 3917c478bd9Sstevel@tonic-gate bus_config_eclk_t bus_config_eclk[] = { 3927c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 3937c478bd9Sstevel@tonic-gate {JBUS_CONFIG_ECLK_1_DIV, JBUS_CONFIG_ECLK_1}, 3947c478bd9Sstevel@tonic-gate {JBUS_CONFIG_ECLK_2_DIV, JBUS_CONFIG_ECLK_2}, 3957c478bd9Sstevel@tonic-gate {JBUS_CONFIG_ECLK_32_DIV, JBUS_CONFIG_ECLK_32}, 3967c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 3977c478bd9Sstevel@tonic-gate {SAFARI_CONFIG_ECLK_1_DIV, SAFARI_CONFIG_ECLK_1}, 3987c478bd9Sstevel@tonic-gate {SAFARI_CONFIG_ECLK_2_DIV, SAFARI_CONFIG_ECLK_2}, 3997c478bd9Sstevel@tonic-gate {SAFARI_CONFIG_ECLK_32_DIV, SAFARI_CONFIG_ECLK_32}, 4007c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 4017c478bd9Sstevel@tonic-gate {0, 0} 4027c478bd9Sstevel@tonic-gate }; 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate /* 4057c478bd9Sstevel@tonic-gate * Interval for deferred CEEN reenable 4067c478bd9Sstevel@tonic-gate */ 4077c478bd9Sstevel@tonic-gate int cpu_ceen_delay_secs = CPU_CEEN_DELAY_SECS; 4087c478bd9Sstevel@tonic-gate 4097c478bd9Sstevel@tonic-gate /* 4107c478bd9Sstevel@tonic-gate * set in /etc/system to control logging of user BERR/TO's 4117c478bd9Sstevel@tonic-gate */ 4127c478bd9Sstevel@tonic-gate int cpu_berr_to_verbose = 0; 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate /* 4157c478bd9Sstevel@tonic-gate * set to 0 in /etc/system to defer CEEN reenable for all CEs 4167c478bd9Sstevel@tonic-gate */ 4177c478bd9Sstevel@tonic-gate uint64_t cpu_ce_not_deferred = CPU_CE_NOT_DEFERRED; 4187c478bd9Sstevel@tonic-gate uint64_t cpu_ce_not_deferred_ext = CPU_CE_NOT_DEFERRED_EXT; 4197c478bd9Sstevel@tonic-gate 4207c478bd9Sstevel@tonic-gate /* 4217c478bd9Sstevel@tonic-gate * Set of all offline cpus 4227c478bd9Sstevel@tonic-gate */ 4237c478bd9Sstevel@tonic-gate cpuset_t cpu_offline_set; 4247c478bd9Sstevel@tonic-gate 4257c478bd9Sstevel@tonic-gate static void cpu_delayed_check_ce_errors(void *); 4267c478bd9Sstevel@tonic-gate static void cpu_check_ce_errors(void *); 4277c478bd9Sstevel@tonic-gate void cpu_error_ecache_flush(ch_async_flt_t *); 4287c478bd9Sstevel@tonic-gate static int cpu_error_ecache_flush_required(ch_async_flt_t *); 4297c478bd9Sstevel@tonic-gate static void cpu_log_and_clear_ce(ch_async_flt_t *); 4307c478bd9Sstevel@tonic-gate void cpu_ce_detected(ch_cpu_errors_t *, int); 4317c478bd9Sstevel@tonic-gate 4327c478bd9Sstevel@tonic-gate /* 4337c478bd9Sstevel@tonic-gate * CE Leaky check timeout in microseconds. This is chosen to be twice the 4347c478bd9Sstevel@tonic-gate * memory refresh interval of current DIMMs (64ms). After initial fix that 4357c478bd9Sstevel@tonic-gate * gives at least one full refresh cycle in which the cell can leak 4367c478bd9Sstevel@tonic-gate * (whereafter further refreshes simply reinforce any incorrect bit value). 4377c478bd9Sstevel@tonic-gate */ 4387c478bd9Sstevel@tonic-gate clock_t cpu_ce_lkychk_timeout_usec = 128000; 4397c478bd9Sstevel@tonic-gate 4407c478bd9Sstevel@tonic-gate /* 4417c478bd9Sstevel@tonic-gate * CE partner check partner caching period in seconds 4427c478bd9Sstevel@tonic-gate */ 4437c478bd9Sstevel@tonic-gate int cpu_ce_ptnr_cachetime_sec = 60; 4447c478bd9Sstevel@tonic-gate 4457c478bd9Sstevel@tonic-gate /* 4467c478bd9Sstevel@tonic-gate * Sets trap table entry ttentry by overwriting eight instructions from ttlabel 4477c478bd9Sstevel@tonic-gate */ 4487c478bd9Sstevel@tonic-gate #define CH_SET_TRAP(ttentry, ttlabel) \ 4497c478bd9Sstevel@tonic-gate bcopy((const void *)&ttlabel, &ttentry, 32); \ 4507c478bd9Sstevel@tonic-gate flush_instr_mem((caddr_t)&ttentry, 32); 4517c478bd9Sstevel@tonic-gate 4527c478bd9Sstevel@tonic-gate static int min_ecache_size; 4537c478bd9Sstevel@tonic-gate static uint_t priv_hcl_1; 4547c478bd9Sstevel@tonic-gate static uint_t priv_hcl_2; 4557c478bd9Sstevel@tonic-gate static uint_t priv_hcl_4; 4567c478bd9Sstevel@tonic-gate static uint_t priv_hcl_8; 4577c478bd9Sstevel@tonic-gate 4587c478bd9Sstevel@tonic-gate void 4597c478bd9Sstevel@tonic-gate cpu_setup(void) 4607c478bd9Sstevel@tonic-gate { 4617c478bd9Sstevel@tonic-gate extern int at_flags; 4627c478bd9Sstevel@tonic-gate extern int disable_delay_tlb_flush, delay_tlb_flush; 4637c478bd9Sstevel@tonic-gate extern int cpc_has_overflow_intr; 4647c478bd9Sstevel@tonic-gate extern int disable_text_largepages; 4657c478bd9Sstevel@tonic-gate extern int use_text_pgsz4m; 4667c478bd9Sstevel@tonic-gate 4677c478bd9Sstevel@tonic-gate /* 4687c478bd9Sstevel@tonic-gate * Setup chip-specific trap handlers. 4697c478bd9Sstevel@tonic-gate */ 4707c478bd9Sstevel@tonic-gate cpu_init_trap(); 4717c478bd9Sstevel@tonic-gate 4727c478bd9Sstevel@tonic-gate cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT); 4737c478bd9Sstevel@tonic-gate 4747c478bd9Sstevel@tonic-gate at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1 | EF_SPARC_SUN_US3; 4757c478bd9Sstevel@tonic-gate 4767c478bd9Sstevel@tonic-gate /* 4777c478bd9Sstevel@tonic-gate * save the cache bootup state. 4787c478bd9Sstevel@tonic-gate */ 4797c478bd9Sstevel@tonic-gate cache_boot_state = get_dcu() & DCU_CACHE; 4807c478bd9Sstevel@tonic-gate 4817c478bd9Sstevel@tonic-gate /* 4827c478bd9Sstevel@tonic-gate * Due to the number of entries in the fully-associative tlb 4837c478bd9Sstevel@tonic-gate * this may have to be tuned lower than in spitfire. 4847c478bd9Sstevel@tonic-gate */ 4857c478bd9Sstevel@tonic-gate pp_slots = MIN(8, MAXPP_SLOTS); 4867c478bd9Sstevel@tonic-gate 4877c478bd9Sstevel@tonic-gate /* 4887c478bd9Sstevel@tonic-gate * Block stores do not invalidate all pages of the d$, pagecopy 4897c478bd9Sstevel@tonic-gate * et. al. need virtual translations with virtual coloring taken 4907c478bd9Sstevel@tonic-gate * into consideration. prefetch/ldd will pollute the d$ on the 4917c478bd9Sstevel@tonic-gate * load side. 4927c478bd9Sstevel@tonic-gate */ 4937c478bd9Sstevel@tonic-gate pp_consistent_coloring = PPAGE_STORE_VCOLORING | PPAGE_LOADS_POLLUTE; 4947c478bd9Sstevel@tonic-gate 4957c478bd9Sstevel@tonic-gate if (use_page_coloring) { 4967c478bd9Sstevel@tonic-gate do_pg_coloring = 1; 4977c478bd9Sstevel@tonic-gate if (use_virtual_coloring) 4987c478bd9Sstevel@tonic-gate do_virtual_coloring = 1; 4997c478bd9Sstevel@tonic-gate } 5007c478bd9Sstevel@tonic-gate 5017c478bd9Sstevel@tonic-gate isa_list = 5027c478bd9Sstevel@tonic-gate "sparcv9+vis2 sparcv9+vis sparcv9 " 5037c478bd9Sstevel@tonic-gate "sparcv8plus+vis2 sparcv8plus+vis sparcv8plus " 5047c478bd9Sstevel@tonic-gate "sparcv8 sparcv8-fsmuld sparcv7 sparc"; 5057c478bd9Sstevel@tonic-gate 5067c478bd9Sstevel@tonic-gate /* 5077c478bd9Sstevel@tonic-gate * On Panther-based machines, this should 5087c478bd9Sstevel@tonic-gate * also include AV_SPARC_POPC too 5097c478bd9Sstevel@tonic-gate */ 5107c478bd9Sstevel@tonic-gate cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2; 5117c478bd9Sstevel@tonic-gate 5127c478bd9Sstevel@tonic-gate /* 5137c478bd9Sstevel@tonic-gate * On cheetah, there's no hole in the virtual address space 5147c478bd9Sstevel@tonic-gate */ 5157c478bd9Sstevel@tonic-gate hole_start = hole_end = 0; 5167c478bd9Sstevel@tonic-gate 5177c478bd9Sstevel@tonic-gate /* 5187c478bd9Sstevel@tonic-gate * The kpm mapping window. 5197c478bd9Sstevel@tonic-gate * kpm_size: 5207c478bd9Sstevel@tonic-gate * The size of a single kpm range. 5217c478bd9Sstevel@tonic-gate * The overall size will be: kpm_size * vac_colors. 5227c478bd9Sstevel@tonic-gate * kpm_vbase: 5237c478bd9Sstevel@tonic-gate * The virtual start address of the kpm range within the kernel 5247c478bd9Sstevel@tonic-gate * virtual address space. kpm_vbase has to be kpm_size aligned. 5257c478bd9Sstevel@tonic-gate */ 5267c478bd9Sstevel@tonic-gate kpm_size = (size_t)(8ull * 1024 * 1024 * 1024 * 1024); /* 8TB */ 5277c478bd9Sstevel@tonic-gate kpm_size_shift = 43; 5287c478bd9Sstevel@tonic-gate kpm_vbase = (caddr_t)0x8000000000000000ull; /* 8EB */ 5297c478bd9Sstevel@tonic-gate kpm_smallpages = 1; 5307c478bd9Sstevel@tonic-gate 5317c478bd9Sstevel@tonic-gate /* 5327c478bd9Sstevel@tonic-gate * The traptrace code uses either %tick or %stick for 5337c478bd9Sstevel@tonic-gate * timestamping. We have %stick so we can use it. 5347c478bd9Sstevel@tonic-gate */ 5357c478bd9Sstevel@tonic-gate traptrace_use_stick = 1; 5367c478bd9Sstevel@tonic-gate 5377c478bd9Sstevel@tonic-gate /* 5387c478bd9Sstevel@tonic-gate * Cheetah has a performance counter overflow interrupt 5397c478bd9Sstevel@tonic-gate */ 5407c478bd9Sstevel@tonic-gate cpc_has_overflow_intr = 1; 5417c478bd9Sstevel@tonic-gate 5427c478bd9Sstevel@tonic-gate /* 5437c478bd9Sstevel@tonic-gate * Use cheetah flush-all support 5447c478bd9Sstevel@tonic-gate */ 5457c478bd9Sstevel@tonic-gate if (!disable_delay_tlb_flush) 5467c478bd9Sstevel@tonic-gate delay_tlb_flush = 1; 5477c478bd9Sstevel@tonic-gate 5487c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_DUAL_PAGESIZE) 5497c478bd9Sstevel@tonic-gate /* 5507c478bd9Sstevel@tonic-gate * Use Cheetah+ and later dual page size support. 5517c478bd9Sstevel@tonic-gate */ 5527c478bd9Sstevel@tonic-gate if (!disable_dual_pgsz) { 553*1e2e7a75Shuah cpu_impl_dual_pgsz = 1; 5547c478bd9Sstevel@tonic-gate } 5557c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_DUAL_PAGESIZE */ 5567c478bd9Sstevel@tonic-gate 5577c478bd9Sstevel@tonic-gate /* 5587c478bd9Sstevel@tonic-gate * Declare that this architecture/cpu combination does fpRAS. 5597c478bd9Sstevel@tonic-gate */ 5607c478bd9Sstevel@tonic-gate fpras_implemented = 1; 5617c478bd9Sstevel@tonic-gate 5627c478bd9Sstevel@tonic-gate /* 5637c478bd9Sstevel@tonic-gate * Enable 4M pages to be used for mapping user text by default. Don't 5647c478bd9Sstevel@tonic-gate * use large pages for initialized data segments since we may not know 5657c478bd9Sstevel@tonic-gate * at exec() time what should be the preferred large page size for DTLB 5667c478bd9Sstevel@tonic-gate * programming. 5677c478bd9Sstevel@tonic-gate */ 5687c478bd9Sstevel@tonic-gate use_text_pgsz4m = 1; 5697c478bd9Sstevel@tonic-gate disable_text_largepages = (1 << TTE64K) | (1 << TTE512K) | 5707c478bd9Sstevel@tonic-gate (1 << TTE32M) | (1 << TTE256M); 5717c478bd9Sstevel@tonic-gate 5727c478bd9Sstevel@tonic-gate /* 5737c478bd9Sstevel@tonic-gate * Setup CE lookup table 5747c478bd9Sstevel@tonic-gate */ 5757c478bd9Sstevel@tonic-gate CE_INITDISPTBL_POPULATE(ce_disp_table); 5767c478bd9Sstevel@tonic-gate ce_disp_inited = 1; 5777c478bd9Sstevel@tonic-gate } 5787c478bd9Sstevel@tonic-gate 5797c478bd9Sstevel@tonic-gate /* 5807c478bd9Sstevel@tonic-gate * Called by setcpudelay 5817c478bd9Sstevel@tonic-gate */ 5827c478bd9Sstevel@tonic-gate void 5837c478bd9Sstevel@tonic-gate cpu_init_tick_freq(void) 5847c478bd9Sstevel@tonic-gate { 5857c478bd9Sstevel@tonic-gate /* 5867c478bd9Sstevel@tonic-gate * For UltraSPARC III and beyond we want to use the 5877c478bd9Sstevel@tonic-gate * system clock rate as the basis for low level timing, 5887c478bd9Sstevel@tonic-gate * due to support of mixed speed CPUs and power managment. 5897c478bd9Sstevel@tonic-gate */ 5907c478bd9Sstevel@tonic-gate if (system_clock_freq == 0) 5917c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "setcpudelay: invalid system_clock_freq"); 5927c478bd9Sstevel@tonic-gate 5937c478bd9Sstevel@tonic-gate sys_tick_freq = system_clock_freq; 5947c478bd9Sstevel@tonic-gate } 5957c478bd9Sstevel@tonic-gate 5967c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 5977c478bd9Sstevel@tonic-gate /* 5987c478bd9Sstevel@tonic-gate * Tunables 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate int cheetah_bpe_off = 0; 6017c478bd9Sstevel@tonic-gate int cheetah_sendmondo_recover = 1; 6027c478bd9Sstevel@tonic-gate int cheetah_sendmondo_fullscan = 0; 6037c478bd9Sstevel@tonic-gate int cheetah_sendmondo_recover_delay = 5; 6047c478bd9Sstevel@tonic-gate 6057c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_MIN_DELAY 1 6067c478bd9Sstevel@tonic-gate 6077c478bd9Sstevel@tonic-gate /* 6087c478bd9Sstevel@tonic-gate * Recovery Statistics 6097c478bd9Sstevel@tonic-gate */ 6107c478bd9Sstevel@tonic-gate typedef struct cheetah_livelock_entry { 6117c478bd9Sstevel@tonic-gate int cpuid; /* fallen cpu */ 6127c478bd9Sstevel@tonic-gate int buddy; /* cpu that ran recovery */ 6137c478bd9Sstevel@tonic-gate clock_t lbolt; /* when recovery started */ 6147c478bd9Sstevel@tonic-gate hrtime_t recovery_time; /* time spent in recovery */ 6157c478bd9Sstevel@tonic-gate } cheetah_livelock_entry_t; 6167c478bd9Sstevel@tonic-gate 6177c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_NENTRY 32 6187c478bd9Sstevel@tonic-gate 6197c478bd9Sstevel@tonic-gate cheetah_livelock_entry_t cheetah_livelock_hist[CHEETAH_LIVELOCK_NENTRY]; 6207c478bd9Sstevel@tonic-gate int cheetah_livelock_entry_nxt; 6217c478bd9Sstevel@tonic-gate 6227c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_ENTRY_NEXT(statp) { \ 6237c478bd9Sstevel@tonic-gate statp = cheetah_livelock_hist + cheetah_livelock_entry_nxt; \ 6247c478bd9Sstevel@tonic-gate if (++cheetah_livelock_entry_nxt >= CHEETAH_LIVELOCK_NENTRY) { \ 6257c478bd9Sstevel@tonic-gate cheetah_livelock_entry_nxt = 0; \ 6267c478bd9Sstevel@tonic-gate } \ 6277c478bd9Sstevel@tonic-gate } 6287c478bd9Sstevel@tonic-gate 6297c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_ENTRY_SET(statp, item, val) statp->item = val 6307c478bd9Sstevel@tonic-gate 6317c478bd9Sstevel@tonic-gate struct { 6327c478bd9Sstevel@tonic-gate hrtime_t hrt; /* maximum recovery time */ 6337c478bd9Sstevel@tonic-gate int recovery; /* recovered */ 6347c478bd9Sstevel@tonic-gate int full_claimed; /* maximum pages claimed in full recovery */ 6357c478bd9Sstevel@tonic-gate int proc_entry; /* attempted to claim TSB */ 6367c478bd9Sstevel@tonic-gate int proc_tsb_scan; /* tsb scanned */ 6377c478bd9Sstevel@tonic-gate int proc_tsb_partscan; /* tsb partially scanned */ 6387c478bd9Sstevel@tonic-gate int proc_tsb_fullscan; /* whole tsb scanned */ 6397c478bd9Sstevel@tonic-gate int proc_claimed; /* maximum pages claimed in tsb scan */ 6407c478bd9Sstevel@tonic-gate int proc_user; /* user thread */ 6417c478bd9Sstevel@tonic-gate int proc_kernel; /* kernel thread */ 6427c478bd9Sstevel@tonic-gate int proc_onflt; /* bad stack */ 6437c478bd9Sstevel@tonic-gate int proc_cpu; /* null cpu */ 6447c478bd9Sstevel@tonic-gate int proc_thread; /* null thread */ 6457c478bd9Sstevel@tonic-gate int proc_proc; /* null proc */ 6467c478bd9Sstevel@tonic-gate int proc_as; /* null as */ 6477c478bd9Sstevel@tonic-gate int proc_hat; /* null hat */ 6487c478bd9Sstevel@tonic-gate int proc_hat_inval; /* hat contents don't make sense */ 6497c478bd9Sstevel@tonic-gate int proc_hat_busy; /* hat is changing TSBs */ 6507c478bd9Sstevel@tonic-gate int proc_tsb_reloc; /* TSB skipped because being relocated */ 6517c478bd9Sstevel@tonic-gate int proc_cnum_bad; /* cnum out of range */ 6527c478bd9Sstevel@tonic-gate int proc_cnum; /* last cnum processed */ 6537c478bd9Sstevel@tonic-gate tte_t proc_tte; /* last tte processed */ 6547c478bd9Sstevel@tonic-gate } cheetah_livelock_stat; 6557c478bd9Sstevel@tonic-gate 6567c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_STAT(item) cheetah_livelock_stat.item++ 6577c478bd9Sstevel@tonic-gate 6587c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_STATSET(item, value) \ 6597c478bd9Sstevel@tonic-gate cheetah_livelock_stat.item = value 6607c478bd9Sstevel@tonic-gate 6617c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_MAXSTAT(item, value) { \ 6627c478bd9Sstevel@tonic-gate if (value > cheetah_livelock_stat.item) \ 6637c478bd9Sstevel@tonic-gate cheetah_livelock_stat.item = value; \ 6647c478bd9Sstevel@tonic-gate } 6657c478bd9Sstevel@tonic-gate 6667c478bd9Sstevel@tonic-gate /* 6677c478bd9Sstevel@tonic-gate * Attempt to recover a cpu by claiming every cache line as saved 6687c478bd9Sstevel@tonic-gate * in the TSB that the non-responsive cpu is using. Since we can't 6697c478bd9Sstevel@tonic-gate * grab any adaptive lock, this is at best an attempt to do so. Because 6707c478bd9Sstevel@tonic-gate * we don't grab any locks, we must operate under the protection of 6717c478bd9Sstevel@tonic-gate * on_fault(). 6727c478bd9Sstevel@tonic-gate * 6737c478bd9Sstevel@tonic-gate * Return 1 if cpuid could be recovered, 0 if failed. 6747c478bd9Sstevel@tonic-gate */ 6757c478bd9Sstevel@tonic-gate int 6767c478bd9Sstevel@tonic-gate mondo_recover_proc(uint16_t cpuid, int bn) 6777c478bd9Sstevel@tonic-gate { 6787c478bd9Sstevel@tonic-gate label_t ljb; 6797c478bd9Sstevel@tonic-gate cpu_t *cp; 6807c478bd9Sstevel@tonic-gate kthread_t *t; 6817c478bd9Sstevel@tonic-gate proc_t *p; 6827c478bd9Sstevel@tonic-gate struct as *as; 6837c478bd9Sstevel@tonic-gate struct hat *hat; 684*1e2e7a75Shuah uint_t cnum; 6857c478bd9Sstevel@tonic-gate struct tsb_info *tsbinfop; 6867c478bd9Sstevel@tonic-gate struct tsbe *tsbep; 6877c478bd9Sstevel@tonic-gate caddr_t tsbp; 6887c478bd9Sstevel@tonic-gate caddr_t end_tsbp; 6897c478bd9Sstevel@tonic-gate uint64_t paddr; 6907c478bd9Sstevel@tonic-gate uint64_t idsr; 6917c478bd9Sstevel@tonic-gate u_longlong_t pahi, palo; 6927c478bd9Sstevel@tonic-gate int pages_claimed = 0; 6937c478bd9Sstevel@tonic-gate tte_t tsbe_tte; 6947c478bd9Sstevel@tonic-gate int tried_kernel_tsb = 0; 695*1e2e7a75Shuah mmu_ctx_t *mmu_ctxp; 6967c478bd9Sstevel@tonic-gate 6977c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_entry); 6987c478bd9Sstevel@tonic-gate 6997c478bd9Sstevel@tonic-gate if (on_fault(&ljb)) { 7007c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_onflt); 7017c478bd9Sstevel@tonic-gate goto badstruct; 7027c478bd9Sstevel@tonic-gate } 7037c478bd9Sstevel@tonic-gate 7047c478bd9Sstevel@tonic-gate if ((cp = cpu[cpuid]) == NULL) { 7057c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_cpu); 7067c478bd9Sstevel@tonic-gate goto badstruct; 7077c478bd9Sstevel@tonic-gate } 7087c478bd9Sstevel@tonic-gate 7097c478bd9Sstevel@tonic-gate if ((t = cp->cpu_thread) == NULL) { 7107c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_thread); 7117c478bd9Sstevel@tonic-gate goto badstruct; 7127c478bd9Sstevel@tonic-gate } 7137c478bd9Sstevel@tonic-gate 7147c478bd9Sstevel@tonic-gate if ((p = ttoproc(t)) == NULL) { 7157c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_proc); 7167c478bd9Sstevel@tonic-gate goto badstruct; 7177c478bd9Sstevel@tonic-gate } 7187c478bd9Sstevel@tonic-gate 7197c478bd9Sstevel@tonic-gate if ((as = p->p_as) == NULL) { 7207c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_as); 7217c478bd9Sstevel@tonic-gate goto badstruct; 7227c478bd9Sstevel@tonic-gate } 7237c478bd9Sstevel@tonic-gate 7247c478bd9Sstevel@tonic-gate if ((hat = as->a_hat) == NULL) { 7257c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat); 7267c478bd9Sstevel@tonic-gate goto badstruct; 7277c478bd9Sstevel@tonic-gate } 7287c478bd9Sstevel@tonic-gate 7297c478bd9Sstevel@tonic-gate if (hat != ksfmmup) { 7307c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_user); 7317c478bd9Sstevel@tonic-gate if (hat->sfmmu_flags & (HAT_BUSY | HAT_SWAPPED | HAT_SWAPIN)) { 7327c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat_busy); 7337c478bd9Sstevel@tonic-gate goto badstruct; 7347c478bd9Sstevel@tonic-gate } 7357c478bd9Sstevel@tonic-gate tsbinfop = hat->sfmmu_tsb; 7367c478bd9Sstevel@tonic-gate if (tsbinfop == NULL) { 7377c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat_inval); 7387c478bd9Sstevel@tonic-gate goto badstruct; 7397c478bd9Sstevel@tonic-gate } 7407c478bd9Sstevel@tonic-gate tsbp = tsbinfop->tsb_va; 7417c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 7427c478bd9Sstevel@tonic-gate } else { 7437c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_kernel); 7447c478bd9Sstevel@tonic-gate tsbinfop = NULL; 7457c478bd9Sstevel@tonic-gate tsbp = ktsb_base; 7467c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 7477c478bd9Sstevel@tonic-gate } 7487c478bd9Sstevel@tonic-gate 7497c478bd9Sstevel@tonic-gate /* Verify as */ 7507c478bd9Sstevel@tonic-gate if (hat->sfmmu_as != as) { 7517c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat_inval); 7527c478bd9Sstevel@tonic-gate goto badstruct; 7537c478bd9Sstevel@tonic-gate } 7547c478bd9Sstevel@tonic-gate 755*1e2e7a75Shuah mmu_ctxp = CPU_MMU_CTXP(cp); 756*1e2e7a75Shuah ASSERT(mmu_ctxp); 757*1e2e7a75Shuah cnum = hat->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; 7587c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STATSET(proc_cnum, cnum); 7597c478bd9Sstevel@tonic-gate 760*1e2e7a75Shuah if ((cnum < 0) || (cnum == INVALID_CONTEXT) || 761*1e2e7a75Shuah (cnum >= mmu_ctxp->mmu_nctxs)) { 7627c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_cnum_bad); 7637c478bd9Sstevel@tonic-gate goto badstruct; 7647c478bd9Sstevel@tonic-gate } 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate do { 7677c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_scan); 7687c478bd9Sstevel@tonic-gate 7697c478bd9Sstevel@tonic-gate /* 7707c478bd9Sstevel@tonic-gate * Skip TSBs being relocated. This is important because 7717c478bd9Sstevel@tonic-gate * we want to avoid the following deadlock scenario: 7727c478bd9Sstevel@tonic-gate * 7737c478bd9Sstevel@tonic-gate * 1) when we came in we set ourselves to "in recover" state. 7747c478bd9Sstevel@tonic-gate * 2) when we try to touch TSB being relocated the mapping 7757c478bd9Sstevel@tonic-gate * will be in the suspended state so we'll spin waiting 7767c478bd9Sstevel@tonic-gate * for it to be unlocked. 7777c478bd9Sstevel@tonic-gate * 3) when the CPU that holds the TSB mapping locked tries to 7787c478bd9Sstevel@tonic-gate * unlock it it will send a xtrap which will fail to xcall 7797c478bd9Sstevel@tonic-gate * us or the CPU we're trying to recover, and will in turn 7807c478bd9Sstevel@tonic-gate * enter the mondo code. 7817c478bd9Sstevel@tonic-gate * 4) since we are still spinning on the locked mapping 7827c478bd9Sstevel@tonic-gate * no further progress will be made and the system will 7837c478bd9Sstevel@tonic-gate * inevitably hard hang. 7847c478bd9Sstevel@tonic-gate * 7857c478bd9Sstevel@tonic-gate * A TSB not being relocated can't begin being relocated 7867c478bd9Sstevel@tonic-gate * while we're accessing it because we check 7877c478bd9Sstevel@tonic-gate * sendmondo_in_recover before relocating TSBs. 7887c478bd9Sstevel@tonic-gate */ 7897c478bd9Sstevel@tonic-gate if (hat != ksfmmup && 7907c478bd9Sstevel@tonic-gate (tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 7917c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_reloc); 7927c478bd9Sstevel@tonic-gate goto next_tsbinfo; 7937c478bd9Sstevel@tonic-gate } 7947c478bd9Sstevel@tonic-gate 7957c478bd9Sstevel@tonic-gate for (tsbep = (struct tsbe *)tsbp; 7967c478bd9Sstevel@tonic-gate tsbep < (struct tsbe *)end_tsbp; tsbep++) { 7977c478bd9Sstevel@tonic-gate tsbe_tte = tsbep->tte_data; 7987c478bd9Sstevel@tonic-gate 7997c478bd9Sstevel@tonic-gate if (tsbe_tte.tte_val == 0) { 8007c478bd9Sstevel@tonic-gate /* 8017c478bd9Sstevel@tonic-gate * Invalid tte 8027c478bd9Sstevel@tonic-gate */ 8037c478bd9Sstevel@tonic-gate continue; 8047c478bd9Sstevel@tonic-gate } 8057c478bd9Sstevel@tonic-gate if (tsbe_tte.tte_se) { 8067c478bd9Sstevel@tonic-gate /* 8077c478bd9Sstevel@tonic-gate * Don't want device registers 8087c478bd9Sstevel@tonic-gate */ 8097c478bd9Sstevel@tonic-gate continue; 8107c478bd9Sstevel@tonic-gate } 8117c478bd9Sstevel@tonic-gate if (tsbe_tte.tte_cp == 0) { 8127c478bd9Sstevel@tonic-gate /* 8137c478bd9Sstevel@tonic-gate * Must be cached in E$ 8147c478bd9Sstevel@tonic-gate */ 8157c478bd9Sstevel@tonic-gate continue; 8167c478bd9Sstevel@tonic-gate } 8177c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STATSET(proc_tte, tsbe_tte); 8187c478bd9Sstevel@tonic-gate idsr = getidsr(); 8197c478bd9Sstevel@tonic-gate if ((idsr & (IDSR_NACK_BIT(bn) | 8207c478bd9Sstevel@tonic-gate IDSR_BUSY_BIT(bn))) == 0) { 8217c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_partscan); 8227c478bd9Sstevel@tonic-gate goto done; 8237c478bd9Sstevel@tonic-gate } 8247c478bd9Sstevel@tonic-gate pahi = tsbe_tte.tte_pahi; 8257c478bd9Sstevel@tonic-gate palo = tsbe_tte.tte_palo; 8267c478bd9Sstevel@tonic-gate paddr = (uint64_t)((pahi << 32) | 8277c478bd9Sstevel@tonic-gate (palo << MMU_PAGESHIFT)); 8287c478bd9Sstevel@tonic-gate claimlines(paddr, TTEBYTES(TTE_CSZ(&tsbe_tte)), 8297c478bd9Sstevel@tonic-gate CH_ECACHE_SUBBLK_SIZE); 8307c478bd9Sstevel@tonic-gate if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 8317c478bd9Sstevel@tonic-gate shipit(cpuid, bn); 8327c478bd9Sstevel@tonic-gate } 8337c478bd9Sstevel@tonic-gate pages_claimed++; 8347c478bd9Sstevel@tonic-gate } 8357c478bd9Sstevel@tonic-gate next_tsbinfo: 8367c478bd9Sstevel@tonic-gate if (tsbinfop != NULL) 8377c478bd9Sstevel@tonic-gate tsbinfop = tsbinfop->tsb_next; 8387c478bd9Sstevel@tonic-gate if (tsbinfop != NULL) { 8397c478bd9Sstevel@tonic-gate tsbp = tsbinfop->tsb_va; 8407c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 8417c478bd9Sstevel@tonic-gate } else if (tsbp == ktsb_base) { 8427c478bd9Sstevel@tonic-gate tried_kernel_tsb = 1; 8437c478bd9Sstevel@tonic-gate } else if (!tried_kernel_tsb) { 8447c478bd9Sstevel@tonic-gate tsbp = ktsb_base; 8457c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 8467c478bd9Sstevel@tonic-gate hat = ksfmmup; 8477c478bd9Sstevel@tonic-gate tsbinfop = NULL; 8487c478bd9Sstevel@tonic-gate } 8497c478bd9Sstevel@tonic-gate } while (tsbinfop != NULL || 8507c478bd9Sstevel@tonic-gate ((tsbp == ktsb_base) && !tried_kernel_tsb)); 8517c478bd9Sstevel@tonic-gate 8527c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_fullscan); 8537c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 8547c478bd9Sstevel@tonic-gate no_fault(); 8557c478bd9Sstevel@tonic-gate idsr = getidsr(); 8567c478bd9Sstevel@tonic-gate if ((idsr & (IDSR_NACK_BIT(bn) | 8577c478bd9Sstevel@tonic-gate IDSR_BUSY_BIT(bn))) == 0) { 8587c478bd9Sstevel@tonic-gate return (1); 8597c478bd9Sstevel@tonic-gate } else { 8607c478bd9Sstevel@tonic-gate return (0); 8617c478bd9Sstevel@tonic-gate } 8627c478bd9Sstevel@tonic-gate 8637c478bd9Sstevel@tonic-gate done: 8647c478bd9Sstevel@tonic-gate no_fault(); 8657c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 8667c478bd9Sstevel@tonic-gate return (1); 8677c478bd9Sstevel@tonic-gate 8687c478bd9Sstevel@tonic-gate badstruct: 8697c478bd9Sstevel@tonic-gate no_fault(); 8707c478bd9Sstevel@tonic-gate return (0); 8717c478bd9Sstevel@tonic-gate } 8727c478bd9Sstevel@tonic-gate 8737c478bd9Sstevel@tonic-gate /* 8747c478bd9Sstevel@tonic-gate * Attempt to claim ownership, temporarily, of every cache line that a 8757c478bd9Sstevel@tonic-gate * non-responsive cpu might be using. This might kick that cpu out of 8767c478bd9Sstevel@tonic-gate * this state. 8777c478bd9Sstevel@tonic-gate * 8787c478bd9Sstevel@tonic-gate * The return value indicates to the caller if we have exhausted all recovery 8797c478bd9Sstevel@tonic-gate * techniques. If 1 is returned, it is useless to call this function again 8807c478bd9Sstevel@tonic-gate * even for a different target CPU. 8817c478bd9Sstevel@tonic-gate */ 8827c478bd9Sstevel@tonic-gate int 8837c478bd9Sstevel@tonic-gate mondo_recover(uint16_t cpuid, int bn) 8847c478bd9Sstevel@tonic-gate { 8857c478bd9Sstevel@tonic-gate struct memseg *seg; 8867c478bd9Sstevel@tonic-gate uint64_t begin_pa, end_pa, cur_pa; 8877c478bd9Sstevel@tonic-gate hrtime_t begin_hrt, end_hrt; 8887c478bd9Sstevel@tonic-gate int retval = 0; 8897c478bd9Sstevel@tonic-gate int pages_claimed = 0; 8907c478bd9Sstevel@tonic-gate cheetah_livelock_entry_t *histp; 8917c478bd9Sstevel@tonic-gate uint64_t idsr; 8927c478bd9Sstevel@tonic-gate 8937c478bd9Sstevel@tonic-gate if (cas32(&sendmondo_in_recover, 0, 1) != 0) { 8947c478bd9Sstevel@tonic-gate /* 8957c478bd9Sstevel@tonic-gate * Wait while recovery takes place 8967c478bd9Sstevel@tonic-gate */ 8977c478bd9Sstevel@tonic-gate while (sendmondo_in_recover) { 8987c478bd9Sstevel@tonic-gate drv_usecwait(1); 8997c478bd9Sstevel@tonic-gate } 9007c478bd9Sstevel@tonic-gate /* 9017c478bd9Sstevel@tonic-gate * Assume we didn't claim the whole memory. If 9027c478bd9Sstevel@tonic-gate * the target of this caller is not recovered, 9037c478bd9Sstevel@tonic-gate * it will come back. 9047c478bd9Sstevel@tonic-gate */ 9057c478bd9Sstevel@tonic-gate return (retval); 9067c478bd9Sstevel@tonic-gate } 9077c478bd9Sstevel@tonic-gate 9087c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_NEXT(histp) 9097c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, lbolt, lbolt); 9107c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, cpuid, cpuid); 9117c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, buddy, CPU->cpu_id); 9127c478bd9Sstevel@tonic-gate 9137c478bd9Sstevel@tonic-gate begin_hrt = gethrtime_waitfree(); 9147c478bd9Sstevel@tonic-gate /* 9157c478bd9Sstevel@tonic-gate * First try to claim the lines in the TSB the target 9167c478bd9Sstevel@tonic-gate * may have been using. 9177c478bd9Sstevel@tonic-gate */ 9187c478bd9Sstevel@tonic-gate if (mondo_recover_proc(cpuid, bn) == 1) { 9197c478bd9Sstevel@tonic-gate /* 9207c478bd9Sstevel@tonic-gate * Didn't claim the whole memory 9217c478bd9Sstevel@tonic-gate */ 9227c478bd9Sstevel@tonic-gate goto done; 9237c478bd9Sstevel@tonic-gate } 9247c478bd9Sstevel@tonic-gate 9257c478bd9Sstevel@tonic-gate /* 9267c478bd9Sstevel@tonic-gate * We tried using the TSB. The target is still 9277c478bd9Sstevel@tonic-gate * not recovered. Check if complete memory scan is 9287c478bd9Sstevel@tonic-gate * enabled. 9297c478bd9Sstevel@tonic-gate */ 9307c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_fullscan == 0) { 9317c478bd9Sstevel@tonic-gate /* 9327c478bd9Sstevel@tonic-gate * Full memory scan is disabled. 9337c478bd9Sstevel@tonic-gate */ 9347c478bd9Sstevel@tonic-gate retval = 1; 9357c478bd9Sstevel@tonic-gate goto done; 9367c478bd9Sstevel@tonic-gate } 9377c478bd9Sstevel@tonic-gate 9387c478bd9Sstevel@tonic-gate /* 9397c478bd9Sstevel@tonic-gate * Try claiming the whole memory. 9407c478bd9Sstevel@tonic-gate */ 9417c478bd9Sstevel@tonic-gate for (seg = memsegs; seg; seg = seg->next) { 9427c478bd9Sstevel@tonic-gate begin_pa = (uint64_t)(seg->pages_base) << MMU_PAGESHIFT; 9437c478bd9Sstevel@tonic-gate end_pa = (uint64_t)(seg->pages_end) << MMU_PAGESHIFT; 9447c478bd9Sstevel@tonic-gate for (cur_pa = begin_pa; cur_pa < end_pa; 9457c478bd9Sstevel@tonic-gate cur_pa += MMU_PAGESIZE) { 9467c478bd9Sstevel@tonic-gate idsr = getidsr(); 9477c478bd9Sstevel@tonic-gate if ((idsr & (IDSR_NACK_BIT(bn) | 9487c478bd9Sstevel@tonic-gate IDSR_BUSY_BIT(bn))) == 0) { 9497c478bd9Sstevel@tonic-gate /* 9507c478bd9Sstevel@tonic-gate * Didn't claim all memory 9517c478bd9Sstevel@tonic-gate */ 9527c478bd9Sstevel@tonic-gate goto done; 9537c478bd9Sstevel@tonic-gate } 9547c478bd9Sstevel@tonic-gate claimlines(cur_pa, MMU_PAGESIZE, 9557c478bd9Sstevel@tonic-gate CH_ECACHE_SUBBLK_SIZE); 9567c478bd9Sstevel@tonic-gate if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 9577c478bd9Sstevel@tonic-gate shipit(cpuid, bn); 9587c478bd9Sstevel@tonic-gate } 9597c478bd9Sstevel@tonic-gate pages_claimed++; 9607c478bd9Sstevel@tonic-gate } 9617c478bd9Sstevel@tonic-gate } 9627c478bd9Sstevel@tonic-gate 9637c478bd9Sstevel@tonic-gate /* 9647c478bd9Sstevel@tonic-gate * We did all we could. 9657c478bd9Sstevel@tonic-gate */ 9667c478bd9Sstevel@tonic-gate retval = 1; 9677c478bd9Sstevel@tonic-gate 9687c478bd9Sstevel@tonic-gate done: 9697c478bd9Sstevel@tonic-gate /* 9707c478bd9Sstevel@tonic-gate * Update statistics 9717c478bd9Sstevel@tonic-gate */ 9727c478bd9Sstevel@tonic-gate end_hrt = gethrtime_waitfree(); 9737c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(recovery); 9747c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(hrt, (end_hrt - begin_hrt)); 9757c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(full_claimed, pages_claimed); 9767c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, recovery_time, \ 9777c478bd9Sstevel@tonic-gate (end_hrt - begin_hrt)); 9787c478bd9Sstevel@tonic-gate 9797c478bd9Sstevel@tonic-gate while (cas32(&sendmondo_in_recover, 1, 0) != 1); 9807c478bd9Sstevel@tonic-gate 9817c478bd9Sstevel@tonic-gate return (retval); 9827c478bd9Sstevel@tonic-gate } 9837c478bd9Sstevel@tonic-gate 9847c478bd9Sstevel@tonic-gate /* 9857c478bd9Sstevel@tonic-gate * This is called by the cyclic framework when this CPU becomes online 9867c478bd9Sstevel@tonic-gate */ 9877c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 9887c478bd9Sstevel@tonic-gate static void 9897c478bd9Sstevel@tonic-gate cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when) 9907c478bd9Sstevel@tonic-gate { 9917c478bd9Sstevel@tonic-gate 9927c478bd9Sstevel@tonic-gate hdlr->cyh_func = (cyc_func_t)cheetah_nudge_buddy; 9937c478bd9Sstevel@tonic-gate hdlr->cyh_level = CY_LOW_LEVEL; 9947c478bd9Sstevel@tonic-gate hdlr->cyh_arg = NULL; 9957c478bd9Sstevel@tonic-gate 9967c478bd9Sstevel@tonic-gate /* 9977c478bd9Sstevel@tonic-gate * Stagger the start time 9987c478bd9Sstevel@tonic-gate */ 9997c478bd9Sstevel@tonic-gate when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU); 10007c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_recover_delay < CHEETAH_LIVELOCK_MIN_DELAY) { 10017c478bd9Sstevel@tonic-gate cheetah_sendmondo_recover_delay = CHEETAH_LIVELOCK_MIN_DELAY; 10027c478bd9Sstevel@tonic-gate } 10037c478bd9Sstevel@tonic-gate when->cyt_interval = cheetah_sendmondo_recover_delay * NANOSEC; 10047c478bd9Sstevel@tonic-gate } 10057c478bd9Sstevel@tonic-gate 10067c478bd9Sstevel@tonic-gate /* 10077c478bd9Sstevel@tonic-gate * Create a low level cyclic to send a xtrap to the next cpu online. 10087c478bd9Sstevel@tonic-gate * However, there's no need to have this running on a uniprocessor system. 10097c478bd9Sstevel@tonic-gate */ 10107c478bd9Sstevel@tonic-gate static void 10117c478bd9Sstevel@tonic-gate cheetah_nudge_init(void) 10127c478bd9Sstevel@tonic-gate { 10137c478bd9Sstevel@tonic-gate cyc_omni_handler_t hdlr; 10147c478bd9Sstevel@tonic-gate 10157c478bd9Sstevel@tonic-gate if (max_ncpus == 1) { 10167c478bd9Sstevel@tonic-gate return; 10177c478bd9Sstevel@tonic-gate } 10187c478bd9Sstevel@tonic-gate 10197c478bd9Sstevel@tonic-gate hdlr.cyo_online = cheetah_nudge_onln; 10207c478bd9Sstevel@tonic-gate hdlr.cyo_offline = NULL; 10217c478bd9Sstevel@tonic-gate hdlr.cyo_arg = NULL; 10227c478bd9Sstevel@tonic-gate 10237c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 10247c478bd9Sstevel@tonic-gate (void) cyclic_add_omni(&hdlr); 10257c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 10267c478bd9Sstevel@tonic-gate } 10277c478bd9Sstevel@tonic-gate 10287c478bd9Sstevel@tonic-gate /* 10297c478bd9Sstevel@tonic-gate * Cyclic handler to wake up buddy 10307c478bd9Sstevel@tonic-gate */ 10317c478bd9Sstevel@tonic-gate void 10327c478bd9Sstevel@tonic-gate cheetah_nudge_buddy(void) 10337c478bd9Sstevel@tonic-gate { 10347c478bd9Sstevel@tonic-gate /* 10357c478bd9Sstevel@tonic-gate * Disable kernel preemption to protect the cpu list 10367c478bd9Sstevel@tonic-gate */ 10377c478bd9Sstevel@tonic-gate kpreempt_disable(); 10387c478bd9Sstevel@tonic-gate if ((CPU->cpu_next_onln != CPU) && (sendmondo_in_recover == 0)) { 10397c478bd9Sstevel@tonic-gate xt_one(CPU->cpu_next_onln->cpu_id, (xcfunc_t *)xt_sync_tl1, 10407c478bd9Sstevel@tonic-gate 0, 0); 10417c478bd9Sstevel@tonic-gate } 10427c478bd9Sstevel@tonic-gate kpreempt_enable(); 10437c478bd9Sstevel@tonic-gate } 10447c478bd9Sstevel@tonic-gate 10457c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 10467c478bd9Sstevel@tonic-gate 10477c478bd9Sstevel@tonic-gate #ifdef SEND_MONDO_STATS 10487c478bd9Sstevel@tonic-gate uint32_t x_one_stimes[64]; 10497c478bd9Sstevel@tonic-gate uint32_t x_one_ltimes[16]; 10507c478bd9Sstevel@tonic-gate uint32_t x_set_stimes[64]; 10517c478bd9Sstevel@tonic-gate uint32_t x_set_ltimes[16]; 10527c478bd9Sstevel@tonic-gate uint32_t x_set_cpus[NCPU]; 10537c478bd9Sstevel@tonic-gate uint32_t x_nack_stimes[64]; 10547c478bd9Sstevel@tonic-gate #endif 10557c478bd9Sstevel@tonic-gate 10567c478bd9Sstevel@tonic-gate /* 10577c478bd9Sstevel@tonic-gate * Note: A version of this function is used by the debugger via the KDI, 10587c478bd9Sstevel@tonic-gate * and must be kept in sync with this version. Any changes made to this 10597c478bd9Sstevel@tonic-gate * function to support new chips or to accomodate errata must also be included 10607c478bd9Sstevel@tonic-gate * in the KDI-specific version. See us3_kdi.c. 10617c478bd9Sstevel@tonic-gate */ 10627c478bd9Sstevel@tonic-gate void 10637c478bd9Sstevel@tonic-gate send_one_mondo(int cpuid) 10647c478bd9Sstevel@tonic-gate { 10657c478bd9Sstevel@tonic-gate int busy, nack; 10667c478bd9Sstevel@tonic-gate uint64_t idsr, starttick, endtick, tick, lasttick; 10677c478bd9Sstevel@tonic-gate uint64_t busymask; 10687c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 10697c478bd9Sstevel@tonic-gate int recovered = 0; 10707c478bd9Sstevel@tonic-gate #endif 10717c478bd9Sstevel@tonic-gate 10727c478bd9Sstevel@tonic-gate CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 10737c478bd9Sstevel@tonic-gate starttick = lasttick = gettick(); 10747c478bd9Sstevel@tonic-gate shipit(cpuid, 0); 10757c478bd9Sstevel@tonic-gate endtick = starttick + xc_tick_limit; 10767c478bd9Sstevel@tonic-gate busy = nack = 0; 10777c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 10787c478bd9Sstevel@tonic-gate /* 10797c478bd9Sstevel@tonic-gate * Lower 2 bits of the agent ID determine which BUSY/NACK pair 10807c478bd9Sstevel@tonic-gate * will be used for dispatching interrupt. For now, assume 10817c478bd9Sstevel@tonic-gate * there are no more than IDSR_BN_SETS CPUs, hence no aliasing 10827c478bd9Sstevel@tonic-gate * issues with respect to BUSY/NACK pair usage. 10837c478bd9Sstevel@tonic-gate */ 10847c478bd9Sstevel@tonic-gate busymask = IDSR_BUSY_BIT(cpuid); 10857c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 10867c478bd9Sstevel@tonic-gate busymask = IDSR_BUSY; 10877c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 10887c478bd9Sstevel@tonic-gate for (;;) { 10897c478bd9Sstevel@tonic-gate idsr = getidsr(); 10907c478bd9Sstevel@tonic-gate if (idsr == 0) 10917c478bd9Sstevel@tonic-gate break; 10927c478bd9Sstevel@tonic-gate 10937c478bd9Sstevel@tonic-gate tick = gettick(); 10947c478bd9Sstevel@tonic-gate /* 10957c478bd9Sstevel@tonic-gate * If there is a big jump between the current tick 10967c478bd9Sstevel@tonic-gate * count and lasttick, we have probably hit a break 10977c478bd9Sstevel@tonic-gate * point. Adjust endtick accordingly to avoid panic. 10987c478bd9Sstevel@tonic-gate */ 10997c478bd9Sstevel@tonic-gate if (tick > (lasttick + xc_tick_jump_limit)) 11007c478bd9Sstevel@tonic-gate endtick += (tick - lasttick); 11017c478bd9Sstevel@tonic-gate lasttick = tick; 11027c478bd9Sstevel@tonic-gate if (tick > endtick) { 11037c478bd9Sstevel@tonic-gate if (panic_quiesce) 11047c478bd9Sstevel@tonic-gate return; 11057c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 11067c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_recover && recovered == 0) { 11077c478bd9Sstevel@tonic-gate if (mondo_recover(cpuid, 0)) { 11087c478bd9Sstevel@tonic-gate /* 11097c478bd9Sstevel@tonic-gate * We claimed the whole memory or 11107c478bd9Sstevel@tonic-gate * full scan is disabled. 11117c478bd9Sstevel@tonic-gate */ 11127c478bd9Sstevel@tonic-gate recovered++; 11137c478bd9Sstevel@tonic-gate } 11147c478bd9Sstevel@tonic-gate tick = gettick(); 11157c478bd9Sstevel@tonic-gate endtick = tick + xc_tick_limit; 11167c478bd9Sstevel@tonic-gate lasttick = tick; 11177c478bd9Sstevel@tonic-gate /* 11187c478bd9Sstevel@tonic-gate * Recheck idsr 11197c478bd9Sstevel@tonic-gate */ 11207c478bd9Sstevel@tonic-gate continue; 11217c478bd9Sstevel@tonic-gate } else 11227c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 11237c478bd9Sstevel@tonic-gate { 11247c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "send mondo timeout " 11257c478bd9Sstevel@tonic-gate "(target 0x%x) [%d NACK %d BUSY]", 11267c478bd9Sstevel@tonic-gate cpuid, nack, busy); 11277c478bd9Sstevel@tonic-gate } 11287c478bd9Sstevel@tonic-gate } 11297c478bd9Sstevel@tonic-gate 11307c478bd9Sstevel@tonic-gate if (idsr & busymask) { 11317c478bd9Sstevel@tonic-gate busy++; 11327c478bd9Sstevel@tonic-gate continue; 11337c478bd9Sstevel@tonic-gate } 11347c478bd9Sstevel@tonic-gate drv_usecwait(1); 11357c478bd9Sstevel@tonic-gate shipit(cpuid, 0); 11367c478bd9Sstevel@tonic-gate nack++; 11377c478bd9Sstevel@tonic-gate busy = 0; 11387c478bd9Sstevel@tonic-gate } 11397c478bd9Sstevel@tonic-gate #ifdef SEND_MONDO_STATS 11407c478bd9Sstevel@tonic-gate { 11417c478bd9Sstevel@tonic-gate int n = gettick() - starttick; 11427c478bd9Sstevel@tonic-gate if (n < 8192) 11437c478bd9Sstevel@tonic-gate x_one_stimes[n >> 7]++; 11447c478bd9Sstevel@tonic-gate else 11457c478bd9Sstevel@tonic-gate x_one_ltimes[(n >> 13) & 0xf]++; 11467c478bd9Sstevel@tonic-gate } 11477c478bd9Sstevel@tonic-gate #endif 11487c478bd9Sstevel@tonic-gate } 11497c478bd9Sstevel@tonic-gate 11507c478bd9Sstevel@tonic-gate void 11517c478bd9Sstevel@tonic-gate syncfpu(void) 11527c478bd9Sstevel@tonic-gate { 11537c478bd9Sstevel@tonic-gate } 11547c478bd9Sstevel@tonic-gate 11557c478bd9Sstevel@tonic-gate /* 11567c478bd9Sstevel@tonic-gate * Return processor specific async error structure 11577c478bd9Sstevel@tonic-gate * size used. 11587c478bd9Sstevel@tonic-gate */ 11597c478bd9Sstevel@tonic-gate int 11607c478bd9Sstevel@tonic-gate cpu_aflt_size(void) 11617c478bd9Sstevel@tonic-gate { 11627c478bd9Sstevel@tonic-gate return (sizeof (ch_async_flt_t)); 11637c478bd9Sstevel@tonic-gate } 11647c478bd9Sstevel@tonic-gate 1165db7840cdSrscott /* 1166db7840cdSrscott * Tunable to disable the checking of other cpu logout areas during panic for 1167db7840cdSrscott * potential syndrome 71 generating errors. 1168db7840cdSrscott */ 1169db7840cdSrscott int enable_check_other_cpus_logout = 1; 1170db7840cdSrscott 1171db7840cdSrscott /* 1172db7840cdSrscott * Check other cpus logout area for potential synd 71 generating 1173db7840cdSrscott * errors. 1174db7840cdSrscott */ 1175db7840cdSrscott static void 1176db7840cdSrscott cpu_check_cpu_logout(int cpuid, caddr_t tpc, int tl, int ecc_type, 1177db7840cdSrscott ch_cpu_logout_t *clop) 1178db7840cdSrscott { 1179db7840cdSrscott struct async_flt *aflt; 1180db7840cdSrscott ch_async_flt_t ch_flt; 1181db7840cdSrscott uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1182db7840cdSrscott 1183db7840cdSrscott if (clop == NULL || clop->clo_data.chd_afar == LOGOUT_INVALID) { 1184db7840cdSrscott return; 1185db7840cdSrscott } 1186db7840cdSrscott 1187db7840cdSrscott bzero(&ch_flt, sizeof (ch_async_flt_t)); 1188db7840cdSrscott 1189db7840cdSrscott t_afar = clop->clo_data.chd_afar; 1190db7840cdSrscott t_afsr = clop->clo_data.chd_afsr; 1191db7840cdSrscott t_afsr_ext = clop->clo_data.chd_afsr_ext; 1192db7840cdSrscott #if defined(SERRANO) 1193db7840cdSrscott ch_flt.afar2 = clop->clo_data.chd_afar2; 1194db7840cdSrscott #endif /* SERRANO */ 1195db7840cdSrscott 1196db7840cdSrscott /* 1197db7840cdSrscott * In order to simplify code, we maintain this afsr_errs 1198db7840cdSrscott * variable which holds the aggregate of AFSR and AFSR_EXT 1199db7840cdSrscott * sticky bits. 1200db7840cdSrscott */ 1201db7840cdSrscott t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1202db7840cdSrscott (t_afsr & C_AFSR_ALL_ERRS); 1203db7840cdSrscott 1204db7840cdSrscott /* Setup the async fault structure */ 1205db7840cdSrscott aflt = (struct async_flt *)&ch_flt; 1206db7840cdSrscott aflt->flt_id = gethrtime_waitfree(); 1207db7840cdSrscott ch_flt.afsr_ext = t_afsr_ext; 1208db7840cdSrscott ch_flt.afsr_errs = t_afsr_errs; 1209db7840cdSrscott aflt->flt_stat = t_afsr; 1210db7840cdSrscott aflt->flt_addr = t_afar; 1211db7840cdSrscott aflt->flt_bus_id = cpuid; 1212db7840cdSrscott aflt->flt_inst = cpuid; 1213db7840cdSrscott aflt->flt_pc = tpc; 1214db7840cdSrscott aflt->flt_prot = AFLT_PROT_NONE; 1215db7840cdSrscott aflt->flt_class = CPU_FAULT; 1216db7840cdSrscott aflt->flt_priv = ((t_afsr & C_AFSR_PRIV) != 0); 1217db7840cdSrscott aflt->flt_tl = tl; 1218db7840cdSrscott aflt->flt_status = ecc_type; 1219db7840cdSrscott aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 1220db7840cdSrscott 1221db7840cdSrscott /* 1222db7840cdSrscott * Queue events on the async event queue, one event per error bit. 1223db7840cdSrscott * If no events are queued, queue an event to complain. 1224db7840cdSrscott */ 1225db7840cdSrscott if (cpu_queue_events(&ch_flt, NULL, t_afsr_errs, clop) == 0) { 1226db7840cdSrscott ch_flt.flt_type = CPU_INV_AFSR; 1227db7840cdSrscott cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 1228db7840cdSrscott (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 1229db7840cdSrscott aflt->flt_panic); 1230db7840cdSrscott } 1231db7840cdSrscott 1232db7840cdSrscott /* 1233db7840cdSrscott * Zero out + invalidate CPU logout. 1234db7840cdSrscott */ 1235db7840cdSrscott bzero(clop, sizeof (ch_cpu_logout_t)); 1236db7840cdSrscott clop->clo_data.chd_afar = LOGOUT_INVALID; 1237db7840cdSrscott } 1238db7840cdSrscott 1239db7840cdSrscott /* 1240db7840cdSrscott * Check the logout areas of all other cpus for unlogged errors. 1241db7840cdSrscott */ 1242db7840cdSrscott static void 1243db7840cdSrscott cpu_check_other_cpus_logout(void) 1244db7840cdSrscott { 1245db7840cdSrscott int i, j; 1246db7840cdSrscott processorid_t myid; 1247db7840cdSrscott struct cpu *cp; 1248db7840cdSrscott ch_err_tl1_data_t *cl1p; 1249db7840cdSrscott 1250db7840cdSrscott myid = CPU->cpu_id; 1251db7840cdSrscott for (i = 0; i < NCPU; i++) { 1252db7840cdSrscott cp = cpu[i]; 1253db7840cdSrscott 1254db7840cdSrscott if ((cp == NULL) || !(cp->cpu_flags & CPU_EXISTS) || 1255db7840cdSrscott (cp->cpu_id == myid) || (CPU_PRIVATE(cp) == NULL)) { 1256db7840cdSrscott continue; 1257db7840cdSrscott } 1258db7840cdSrscott 1259db7840cdSrscott /* 1260db7840cdSrscott * Check each of the tl>0 logout areas 1261db7840cdSrscott */ 1262db7840cdSrscott cl1p = CPU_PRIVATE_PTR(cp, chpr_tl1_err_data[0]); 1263db7840cdSrscott for (j = 0; j < CH_ERR_TL1_TLMAX; j++, cl1p++) { 1264db7840cdSrscott if (cl1p->ch_err_tl1_flags == 0) 1265db7840cdSrscott continue; 1266db7840cdSrscott 1267db7840cdSrscott cpu_check_cpu_logout(i, (caddr_t)cl1p->ch_err_tl1_tpc, 1268db7840cdSrscott 1, ECC_F_TRAP, &cl1p->ch_err_tl1_logout); 1269db7840cdSrscott } 1270db7840cdSrscott 1271db7840cdSrscott /* 1272db7840cdSrscott * Check each of the remaining logout areas 1273db7840cdSrscott */ 1274db7840cdSrscott cpu_check_cpu_logout(i, NULL, 0, ECC_F_TRAP, 1275db7840cdSrscott CPU_PRIVATE_PTR(cp, chpr_fecctl0_logout)); 1276db7840cdSrscott cpu_check_cpu_logout(i, NULL, 0, ECC_C_TRAP, 1277db7840cdSrscott CPU_PRIVATE_PTR(cp, chpr_cecc_logout)); 1278db7840cdSrscott cpu_check_cpu_logout(i, NULL, 0, ECC_D_TRAP, 1279db7840cdSrscott CPU_PRIVATE_PTR(cp, chpr_async_logout)); 1280db7840cdSrscott } 1281db7840cdSrscott } 1282db7840cdSrscott 12837c478bd9Sstevel@tonic-gate /* 12847c478bd9Sstevel@tonic-gate * The fast_ecc_err handler transfers control here for UCU, UCC events. 12857c478bd9Sstevel@tonic-gate * Note that we flush Ecache twice, once in the fast_ecc_err handler to 12867c478bd9Sstevel@tonic-gate * flush the error that caused the UCU/UCC, then again here at the end to 12877c478bd9Sstevel@tonic-gate * flush the TL=1 trap handler code out of the Ecache, so we can minimize 12887c478bd9Sstevel@tonic-gate * the probability of getting a TL>1 Fast ECC trap when we're fielding 12897c478bd9Sstevel@tonic-gate * another Fast ECC trap. 12907c478bd9Sstevel@tonic-gate * 12917c478bd9Sstevel@tonic-gate * Cheetah+ also handles: TSCE: No additional processing required. 12927c478bd9Sstevel@tonic-gate * Panther adds L3_UCU and L3_UCC which are reported in AFSR_EXT. 12937c478bd9Sstevel@tonic-gate * 12947c478bd9Sstevel@tonic-gate * Note that the p_clo_flags input is only valid in cases where the 12957c478bd9Sstevel@tonic-gate * cpu_private struct is not yet initialized (since that is the only 12967c478bd9Sstevel@tonic-gate * time that information cannot be obtained from the logout struct.) 12977c478bd9Sstevel@tonic-gate */ 12987c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 12997c478bd9Sstevel@tonic-gate void 13007c478bd9Sstevel@tonic-gate cpu_fast_ecc_error(struct regs *rp, ulong_t p_clo_flags) 13017c478bd9Sstevel@tonic-gate { 13027c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 13034fd7ecabSdilpreet uint64_t ceen, nceen; 13047c478bd9Sstevel@tonic-gate 13057c478bd9Sstevel@tonic-gate /* 13067c478bd9Sstevel@tonic-gate * Get the CPU log out info. If we can't find our CPU private 13077c478bd9Sstevel@tonic-gate * pointer, then we will have to make due without any detailed 13087c478bd9Sstevel@tonic-gate * logout information. 13097c478bd9Sstevel@tonic-gate */ 13107c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 13117c478bd9Sstevel@tonic-gate clop = NULL; 13127c478bd9Sstevel@tonic-gate ceen = p_clo_flags & EN_REG_CEEN; 13134fd7ecabSdilpreet nceen = p_clo_flags & EN_REG_NCEEN; 13147c478bd9Sstevel@tonic-gate } else { 13157c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_fecctl0_logout); 13167c478bd9Sstevel@tonic-gate ceen = clop->clo_flags & EN_REG_CEEN; 13174fd7ecabSdilpreet nceen = clop->clo_flags & EN_REG_NCEEN; 13187c478bd9Sstevel@tonic-gate } 13197c478bd9Sstevel@tonic-gate 13207c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error((caddr_t)rp->r_pc, 13214fd7ecabSdilpreet (rp->r_tstate & TSTATE_PRIV) ? 1 : 0, 0, ceen, nceen, clop); 13227c478bd9Sstevel@tonic-gate } 13237c478bd9Sstevel@tonic-gate 13247c478bd9Sstevel@tonic-gate /* 13257c478bd9Sstevel@tonic-gate * Log fast ecc error, called from either Fast ECC at TL=0 or Fast 13267c478bd9Sstevel@tonic-gate * ECC at TL>0. Need to supply either a error register pointer or a 13277c478bd9Sstevel@tonic-gate * cpu logout structure pointer. 13287c478bd9Sstevel@tonic-gate */ 13297c478bd9Sstevel@tonic-gate static void 13307c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 13314fd7ecabSdilpreet uint64_t nceen, ch_cpu_logout_t *clop) 13327c478bd9Sstevel@tonic-gate { 13337c478bd9Sstevel@tonic-gate struct async_flt *aflt; 13347c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 13357c478bd9Sstevel@tonic-gate uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 13367c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 13377c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 13387c478bd9Sstevel@tonic-gate 13397c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 13407c478bd9Sstevel@tonic-gate /* 13417c478bd9Sstevel@tonic-gate * If no cpu logout data, then we will have to make due without 13427c478bd9Sstevel@tonic-gate * any detailed logout information. 13437c478bd9Sstevel@tonic-gate */ 13447c478bd9Sstevel@tonic-gate if (clop == NULL) { 13457c478bd9Sstevel@tonic-gate ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 13467c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 13477c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 13487c478bd9Sstevel@tonic-gate t_afar = cpu_error_regs.afar; 13497c478bd9Sstevel@tonic-gate t_afsr = cpu_error_regs.afsr; 13507c478bd9Sstevel@tonic-gate t_afsr_ext = cpu_error_regs.afsr_ext; 13517c478bd9Sstevel@tonic-gate #if defined(SERRANO) 13527c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 13537c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 13547c478bd9Sstevel@tonic-gate } else { 13557c478bd9Sstevel@tonic-gate t_afar = clop->clo_data.chd_afar; 13567c478bd9Sstevel@tonic-gate t_afsr = clop->clo_data.chd_afsr; 13577c478bd9Sstevel@tonic-gate t_afsr_ext = clop->clo_data.chd_afsr_ext; 13587c478bd9Sstevel@tonic-gate #if defined(SERRANO) 13597c478bd9Sstevel@tonic-gate ch_flt.afar2 = clop->clo_data.chd_afar2; 13607c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 13617c478bd9Sstevel@tonic-gate } 13627c478bd9Sstevel@tonic-gate 13637c478bd9Sstevel@tonic-gate /* 13647c478bd9Sstevel@tonic-gate * In order to simplify code, we maintain this afsr_errs 13657c478bd9Sstevel@tonic-gate * variable which holds the aggregate of AFSR and AFSR_EXT 13667c478bd9Sstevel@tonic-gate * sticky bits. 13677c478bd9Sstevel@tonic-gate */ 13687c478bd9Sstevel@tonic-gate t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 13697c478bd9Sstevel@tonic-gate (t_afsr & C_AFSR_ALL_ERRS); 13707c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 13717c478bd9Sstevel@tonic-gate 13727c478bd9Sstevel@tonic-gate /* Setup the async fault structure */ 13737c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 13747c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 13757c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = t_afsr_ext; 13767c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = t_afsr_errs; 13777c478bd9Sstevel@tonic-gate aflt->flt_stat = t_afsr; 13787c478bd9Sstevel@tonic-gate aflt->flt_addr = t_afar; 13797c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 13807c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 13817c478bd9Sstevel@tonic-gate aflt->flt_pc = tpc; 13827c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 13837c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 13847c478bd9Sstevel@tonic-gate aflt->flt_priv = priv; 13857c478bd9Sstevel@tonic-gate aflt->flt_tl = tl; 13867c478bd9Sstevel@tonic-gate aflt->flt_status = ECC_F_TRAP; 13877c478bd9Sstevel@tonic-gate aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 13887c478bd9Sstevel@tonic-gate 13897c478bd9Sstevel@tonic-gate /* 13907c478bd9Sstevel@tonic-gate * XXXX - Phenomenal hack to get around Solaris not getting all the 13917c478bd9Sstevel@tonic-gate * cmn_err messages out to the console. The situation is a UCU (in 13927c478bd9Sstevel@tonic-gate * priv mode) which causes a WDU which causes a UE (on the retry). 13937c478bd9Sstevel@tonic-gate * The messages for the UCU and WDU are enqueued and then pulled off 13947c478bd9Sstevel@tonic-gate * the async queue via softint and syslogd starts to process them 13957c478bd9Sstevel@tonic-gate * but doesn't get them to the console. The UE causes a panic, but 13967c478bd9Sstevel@tonic-gate * since the UCU/WDU messages are already in transit, those aren't 13977c478bd9Sstevel@tonic-gate * on the async queue. The hack is to check if we have a matching 13987c478bd9Sstevel@tonic-gate * WDU event for the UCU, and if it matches, we're more than likely 13997c478bd9Sstevel@tonic-gate * going to panic with a UE, unless we're under protection. So, we 14007c478bd9Sstevel@tonic-gate * check to see if we got a matching WDU event and if we're under 14017c478bd9Sstevel@tonic-gate * protection. 14027c478bd9Sstevel@tonic-gate * 14037c478bd9Sstevel@tonic-gate * For Cheetah/Cheetah+/Jaguar/Jalapeno, the sequence we care about 14047c478bd9Sstevel@tonic-gate * looks like this: 14057c478bd9Sstevel@tonic-gate * UCU->WDU->UE 14067c478bd9Sstevel@tonic-gate * For Panther, it could look like either of these: 14077c478bd9Sstevel@tonic-gate * UCU---->WDU->L3_WDU->UE 14087c478bd9Sstevel@tonic-gate * L3_UCU->WDU->L3_WDU->UE 14097c478bd9Sstevel@tonic-gate */ 14107c478bd9Sstevel@tonic-gate if ((t_afsr_errs & (C_AFSR_UCU | C_AFSR_L3_UCU)) && 14117c478bd9Sstevel@tonic-gate aflt->flt_panic == 0 && aflt->flt_priv != 0 && 14127c478bd9Sstevel@tonic-gate curthread->t_ontrap == NULL && curthread->t_lofault == NULL) { 14137c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 14147c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((cpu_error_regs.afsr & C_AFSR_WDU) && 14157c478bd9Sstevel@tonic-gate (cpu_error_regs.afar == t_afar)); 14167c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((clop == NULL) && 14177c478bd9Sstevel@tonic-gate (t_afsr_errs & C_AFSR_WDU)); 14187c478bd9Sstevel@tonic-gate } 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate /* 14217c478bd9Sstevel@tonic-gate * Queue events on the async event queue, one event per error bit. 14227c478bd9Sstevel@tonic-gate * If no events are queued or no Fast ECC events are on in the AFSR, 14237c478bd9Sstevel@tonic-gate * queue an event to complain. 14247c478bd9Sstevel@tonic-gate */ 14257c478bd9Sstevel@tonic-gate if (cpu_queue_events(&ch_flt, pr_reason, t_afsr_errs, clop) == 0 || 14267c478bd9Sstevel@tonic-gate ((t_afsr_errs & (C_AFSR_FECC_ERRS | C_AFSR_EXT_FECC_ERRS)) == 0)) { 14277c478bd9Sstevel@tonic-gate ch_flt.flt_type = CPU_INV_AFSR; 14287c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 14297c478bd9Sstevel@tonic-gate (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 14307c478bd9Sstevel@tonic-gate aflt->flt_panic); 14317c478bd9Sstevel@tonic-gate } 14327c478bd9Sstevel@tonic-gate 14337c478bd9Sstevel@tonic-gate /* 14347c478bd9Sstevel@tonic-gate * Zero out + invalidate CPU logout. 14357c478bd9Sstevel@tonic-gate */ 14367c478bd9Sstevel@tonic-gate if (clop) { 14377c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 14387c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 14397c478bd9Sstevel@tonic-gate } 14407c478bd9Sstevel@tonic-gate 14417c478bd9Sstevel@tonic-gate /* 14427c478bd9Sstevel@tonic-gate * We carefully re-enable NCEEN and CEEN and then check if any deferred 14437c478bd9Sstevel@tonic-gate * or disrupting errors have happened. We do this because if a 14447c478bd9Sstevel@tonic-gate * deferred or disrupting error had occurred with NCEEN/CEEN off, the 14457c478bd9Sstevel@tonic-gate * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 14467c478bd9Sstevel@tonic-gate * CEEN works differently on Cheetah than on Spitfire. Also, we enable 14477c478bd9Sstevel@tonic-gate * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 14487c478bd9Sstevel@tonic-gate * deferred or disrupting error happening between checking the AFSR and 14497c478bd9Sstevel@tonic-gate * enabling NCEEN/CEEN. 14507c478bd9Sstevel@tonic-gate * 14514fd7ecabSdilpreet * Note: CEEN and NCEEN are only reenabled if they were on when trap 14524fd7ecabSdilpreet * taken. 14537c478bd9Sstevel@tonic-gate */ 14544fd7ecabSdilpreet set_error_enable(get_error_enable() | (nceen | ceen)); 14557c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 14567c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((ch_flt.afsr_errs & 14577c478bd9Sstevel@tonic-gate (C_AFSR_EXT_ASYNC_ERRS | C_AFSR_ASYNC_ERRS)) != 0); 14587c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 14597c478bd9Sstevel@tonic-gate NULL); 14607c478bd9Sstevel@tonic-gate } 14617c478bd9Sstevel@tonic-gate 14627c478bd9Sstevel@tonic-gate /* 14637c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 14647c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 14657c478bd9Sstevel@tonic-gate */ 14667c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 14677c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", pr_reason); 14687c478bd9Sstevel@tonic-gate 14697c478bd9Sstevel@tonic-gate /* 14707c478bd9Sstevel@tonic-gate * Flushing the Ecache here gets the part of the trap handler that 14717c478bd9Sstevel@tonic-gate * is run at TL=1 out of the Ecache. 14727c478bd9Sstevel@tonic-gate */ 14737c478bd9Sstevel@tonic-gate cpu_flush_ecache(); 14747c478bd9Sstevel@tonic-gate } 14757c478bd9Sstevel@tonic-gate 14767c478bd9Sstevel@tonic-gate /* 14777c478bd9Sstevel@tonic-gate * This is called via sys_trap from pil15_interrupt code if the 14787c478bd9Sstevel@tonic-gate * corresponding entry in ch_err_tl1_pending is set. Checks the 14797c478bd9Sstevel@tonic-gate * various ch_err_tl1_data structures for valid entries based on the bit 14807c478bd9Sstevel@tonic-gate * settings in the ch_err_tl1_flags entry of the structure. 14817c478bd9Sstevel@tonic-gate */ 14827c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 14837c478bd9Sstevel@tonic-gate void 14847c478bd9Sstevel@tonic-gate cpu_tl1_error(struct regs *rp, int panic) 14857c478bd9Sstevel@tonic-gate { 14867c478bd9Sstevel@tonic-gate ch_err_tl1_data_t *cl1p, cl1; 14877c478bd9Sstevel@tonic-gate int i, ncl1ps; 14887c478bd9Sstevel@tonic-gate uint64_t me_flags; 14894fd7ecabSdilpreet uint64_t ceen, nceen; 14907c478bd9Sstevel@tonic-gate 14917c478bd9Sstevel@tonic-gate if (ch_err_tl1_paddrs[CPU->cpu_id] == 0) { 14927c478bd9Sstevel@tonic-gate cl1p = &ch_err_tl1_data; 14937c478bd9Sstevel@tonic-gate ncl1ps = 1; 14947c478bd9Sstevel@tonic-gate } else if (CPU_PRIVATE(CPU) != NULL) { 14957c478bd9Sstevel@tonic-gate cl1p = CPU_PRIVATE_PTR(CPU, chpr_tl1_err_data[0]); 14967c478bd9Sstevel@tonic-gate ncl1ps = CH_ERR_TL1_TLMAX; 14977c478bd9Sstevel@tonic-gate } else { 14987c478bd9Sstevel@tonic-gate ncl1ps = 0; 14997c478bd9Sstevel@tonic-gate } 15007c478bd9Sstevel@tonic-gate 15017c478bd9Sstevel@tonic-gate for (i = 0; i < ncl1ps; i++, cl1p++) { 15027c478bd9Sstevel@tonic-gate if (cl1p->ch_err_tl1_flags == 0) 15037c478bd9Sstevel@tonic-gate continue; 15047c478bd9Sstevel@tonic-gate 15057c478bd9Sstevel@tonic-gate /* 15067c478bd9Sstevel@tonic-gate * Grab a copy of the logout data and invalidate 15077c478bd9Sstevel@tonic-gate * the logout area. 15087c478bd9Sstevel@tonic-gate */ 15097c478bd9Sstevel@tonic-gate cl1 = *cl1p; 15107c478bd9Sstevel@tonic-gate bzero(cl1p, sizeof (ch_err_tl1_data_t)); 15117c478bd9Sstevel@tonic-gate cl1p->ch_err_tl1_logout.clo_data.chd_afar = LOGOUT_INVALID; 15127c478bd9Sstevel@tonic-gate me_flags = CH_ERR_ME_FLAGS(cl1.ch_err_tl1_flags); 15137c478bd9Sstevel@tonic-gate 15147c478bd9Sstevel@tonic-gate /* 15157c478bd9Sstevel@tonic-gate * Log "first error" in ch_err_tl1_data. 15167c478bd9Sstevel@tonic-gate */ 15177c478bd9Sstevel@tonic-gate if (cl1.ch_err_tl1_flags & CH_ERR_FECC) { 15187c478bd9Sstevel@tonic-gate ceen = get_error_enable() & EN_REG_CEEN; 15194fd7ecabSdilpreet nceen = get_error_enable() & EN_REG_NCEEN; 15207c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error((caddr_t)cl1.ch_err_tl1_tpc, 1, 15214fd7ecabSdilpreet 1, ceen, nceen, &cl1.ch_err_tl1_logout); 15227c478bd9Sstevel@tonic-gate } 15237c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 15247c478bd9Sstevel@tonic-gate if (cl1.ch_err_tl1_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 15257c478bd9Sstevel@tonic-gate cpu_parity_error(rp, cl1.ch_err_tl1_flags, 15267c478bd9Sstevel@tonic-gate (caddr_t)cl1.ch_err_tl1_tpc); 15277c478bd9Sstevel@tonic-gate } 15287c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 15297c478bd9Sstevel@tonic-gate 15307c478bd9Sstevel@tonic-gate /* 15317c478bd9Sstevel@tonic-gate * Log "multiple events" in ch_err_tl1_data. Note that 15327c478bd9Sstevel@tonic-gate * we don't read and clear the AFSR/AFAR in the TL>0 code 15337c478bd9Sstevel@tonic-gate * if the structure is busy, we just do the cache flushing 15347c478bd9Sstevel@tonic-gate * we have to do and then do the retry. So the AFSR/AFAR 15357c478bd9Sstevel@tonic-gate * at this point *should* have some relevant info. If there 15367c478bd9Sstevel@tonic-gate * are no valid errors in the AFSR, we'll assume they've 15377c478bd9Sstevel@tonic-gate * already been picked up and logged. For I$/D$ parity, 15387c478bd9Sstevel@tonic-gate * we just log an event with an "Unknown" (NULL) TPC. 15397c478bd9Sstevel@tonic-gate */ 15407c478bd9Sstevel@tonic-gate if (me_flags & CH_ERR_FECC) { 15417c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 15427c478bd9Sstevel@tonic-gate uint64_t t_afsr_errs; 15437c478bd9Sstevel@tonic-gate 15447c478bd9Sstevel@tonic-gate /* 15457c478bd9Sstevel@tonic-gate * Get the error registers and see if there's 15467c478bd9Sstevel@tonic-gate * a pending error. If not, don't bother 15477c478bd9Sstevel@tonic-gate * generating an "Invalid AFSR" error event. 15487c478bd9Sstevel@tonic-gate */ 15497c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 15507c478bd9Sstevel@tonic-gate t_afsr_errs = (cpu_error_regs.afsr_ext & 15517c478bd9Sstevel@tonic-gate C_AFSR_EXT_ALL_ERRS) | 15527c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr & C_AFSR_ALL_ERRS); 15537c478bd9Sstevel@tonic-gate if (t_afsr_errs != 0) { 15547c478bd9Sstevel@tonic-gate ceen = get_error_enable() & EN_REG_CEEN; 15554fd7ecabSdilpreet nceen = get_error_enable() & EN_REG_NCEEN; 15567c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error((caddr_t)NULL, 1, 15574fd7ecabSdilpreet 1, ceen, nceen, NULL); 15587c478bd9Sstevel@tonic-gate } 15597c478bd9Sstevel@tonic-gate } 15607c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 15617c478bd9Sstevel@tonic-gate if (me_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 15627c478bd9Sstevel@tonic-gate cpu_parity_error(rp, me_flags, (caddr_t)NULL); 15637c478bd9Sstevel@tonic-gate } 15647c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 15657c478bd9Sstevel@tonic-gate } 15667c478bd9Sstevel@tonic-gate } 15677c478bd9Sstevel@tonic-gate 15687c478bd9Sstevel@tonic-gate /* 15697c478bd9Sstevel@tonic-gate * Called from Fast ECC TL>0 handler in case of fatal error. 15707c478bd9Sstevel@tonic-gate * cpu_tl1_error should always find an associated ch_err_tl1_data structure, 15717c478bd9Sstevel@tonic-gate * but if we don't, we'll panic with something reasonable. 15727c478bd9Sstevel@tonic-gate */ 15737c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 15747c478bd9Sstevel@tonic-gate void 15757c478bd9Sstevel@tonic-gate cpu_tl1_err_panic(struct regs *rp, ulong_t flags) 15767c478bd9Sstevel@tonic-gate { 15777c478bd9Sstevel@tonic-gate cpu_tl1_error(rp, 1); 15787c478bd9Sstevel@tonic-gate /* 15797c478bd9Sstevel@tonic-gate * Should never return, but just in case. 15807c478bd9Sstevel@tonic-gate */ 15817c478bd9Sstevel@tonic-gate fm_panic("Unsurvivable ECC Error at TL>0"); 15827c478bd9Sstevel@tonic-gate } 15837c478bd9Sstevel@tonic-gate 15847c478bd9Sstevel@tonic-gate /* 15857c478bd9Sstevel@tonic-gate * The ce_err/ce_err_tl1 handlers transfer control here for CE, EMC, EDU:ST, 15867c478bd9Sstevel@tonic-gate * EDC, WDU, WDC, CPU, CPC, IVU, IVC events. 15877c478bd9Sstevel@tonic-gate * Disrupting errors controlled by NCEEN: EDU:ST, WDU, CPU, IVU 15887c478bd9Sstevel@tonic-gate * Disrupting errors controlled by CEEN: CE, EMC, EDC, WDC, CPC, IVC 15897c478bd9Sstevel@tonic-gate * 15907c478bd9Sstevel@tonic-gate * Cheetah+ also handles (No additional processing required): 15917c478bd9Sstevel@tonic-gate * DUE, DTO, DBERR (NCEEN controlled) 15927c478bd9Sstevel@tonic-gate * THCE (CEEN and ET_ECC_en controlled) 15937c478bd9Sstevel@tonic-gate * TUE (ET_ECC_en controlled) 15947c478bd9Sstevel@tonic-gate * 15957c478bd9Sstevel@tonic-gate * Panther further adds: 15967c478bd9Sstevel@tonic-gate * IMU, L3_EDU, L3_WDU, L3_CPU (NCEEN controlled) 15977c478bd9Sstevel@tonic-gate * IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE (CEEN controlled) 15987c478bd9Sstevel@tonic-gate * TUE_SH, TUE (NCEEN and L2_tag_ECC_en controlled) 15997c478bd9Sstevel@tonic-gate * L3_TUE, L3_TUE_SH (NCEEN and ET_ECC_en controlled) 16007c478bd9Sstevel@tonic-gate * THCE (CEEN and L2_tag_ECC_en controlled) 16017c478bd9Sstevel@tonic-gate * L3_THCE (CEEN and ET_ECC_en controlled) 16027c478bd9Sstevel@tonic-gate * 16037c478bd9Sstevel@tonic-gate * Note that the p_clo_flags input is only valid in cases where the 16047c478bd9Sstevel@tonic-gate * cpu_private struct is not yet initialized (since that is the only 16057c478bd9Sstevel@tonic-gate * time that information cannot be obtained from the logout struct.) 16067c478bd9Sstevel@tonic-gate */ 16077c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 16087c478bd9Sstevel@tonic-gate void 16097c478bd9Sstevel@tonic-gate cpu_disrupting_error(struct regs *rp, ulong_t p_clo_flags) 16107c478bd9Sstevel@tonic-gate { 16117c478bd9Sstevel@tonic-gate struct async_flt *aflt; 16127c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 16137c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 16147c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 16157c478bd9Sstevel@tonic-gate uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 16167c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 16177c478bd9Sstevel@tonic-gate 16187c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 16197c478bd9Sstevel@tonic-gate /* 16207c478bd9Sstevel@tonic-gate * Get the CPU log out info. If we can't find our CPU private 16217c478bd9Sstevel@tonic-gate * pointer, then we will have to make due without any detailed 16227c478bd9Sstevel@tonic-gate * logout information. 16237c478bd9Sstevel@tonic-gate */ 16247c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 16257c478bd9Sstevel@tonic-gate clop = NULL; 16267c478bd9Sstevel@tonic-gate ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 16277c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 16287c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 16297c478bd9Sstevel@tonic-gate t_afar = cpu_error_regs.afar; 16307c478bd9Sstevel@tonic-gate t_afsr = cpu_error_regs.afsr; 16317c478bd9Sstevel@tonic-gate t_afsr_ext = cpu_error_regs.afsr_ext; 16327c478bd9Sstevel@tonic-gate #if defined(SERRANO) 16337c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 16347c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 16357c478bd9Sstevel@tonic-gate } else { 16367c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 16377c478bd9Sstevel@tonic-gate t_afar = clop->clo_data.chd_afar; 16387c478bd9Sstevel@tonic-gate t_afsr = clop->clo_data.chd_afsr; 16397c478bd9Sstevel@tonic-gate t_afsr_ext = clop->clo_data.chd_afsr_ext; 16407c478bd9Sstevel@tonic-gate #if defined(SERRANO) 16417c478bd9Sstevel@tonic-gate ch_flt.afar2 = clop->clo_data.chd_afar2; 16427c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 16437c478bd9Sstevel@tonic-gate } 16447c478bd9Sstevel@tonic-gate 16457c478bd9Sstevel@tonic-gate /* 16467c478bd9Sstevel@tonic-gate * In order to simplify code, we maintain this afsr_errs 16477c478bd9Sstevel@tonic-gate * variable which holds the aggregate of AFSR and AFSR_EXT 16487c478bd9Sstevel@tonic-gate * sticky bits. 16497c478bd9Sstevel@tonic-gate */ 16507c478bd9Sstevel@tonic-gate t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 16517c478bd9Sstevel@tonic-gate (t_afsr & C_AFSR_ALL_ERRS); 16527c478bd9Sstevel@tonic-gate 16537c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 16547c478bd9Sstevel@tonic-gate /* Setup the async fault structure */ 16557c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 16567c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = t_afsr_ext; 16577c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = t_afsr_errs; 16587c478bd9Sstevel@tonic-gate aflt->flt_stat = t_afsr; 16597c478bd9Sstevel@tonic-gate aflt->flt_addr = t_afar; 16607c478bd9Sstevel@tonic-gate aflt->flt_pc = (caddr_t)rp->r_pc; 16617c478bd9Sstevel@tonic-gate aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 16627c478bd9Sstevel@tonic-gate aflt->flt_tl = 0; 16637c478bd9Sstevel@tonic-gate aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 16647c478bd9Sstevel@tonic-gate 16657c478bd9Sstevel@tonic-gate /* 16667c478bd9Sstevel@tonic-gate * If this trap is a result of one of the errors not masked 16677c478bd9Sstevel@tonic-gate * by cpu_ce_not_deferred, we don't reenable CEEN. Instead 16687c478bd9Sstevel@tonic-gate * indicate that a timeout is to be set later. 16697c478bd9Sstevel@tonic-gate */ 16707c478bd9Sstevel@tonic-gate if (!(t_afsr_errs & (cpu_ce_not_deferred | cpu_ce_not_deferred_ext)) && 16717c478bd9Sstevel@tonic-gate !aflt->flt_panic) 16727c478bd9Sstevel@tonic-gate ch_flt.flt_trapped_ce = CE_CEEN_DEFER | CE_CEEN_TRAPPED; 16737c478bd9Sstevel@tonic-gate else 16747c478bd9Sstevel@tonic-gate ch_flt.flt_trapped_ce = CE_CEEN_NODEFER | CE_CEEN_TRAPPED; 16757c478bd9Sstevel@tonic-gate 16767c478bd9Sstevel@tonic-gate /* 16777c478bd9Sstevel@tonic-gate * log the CE and clean up 16787c478bd9Sstevel@tonic-gate */ 16797c478bd9Sstevel@tonic-gate cpu_log_and_clear_ce(&ch_flt); 16807c478bd9Sstevel@tonic-gate 16817c478bd9Sstevel@tonic-gate /* 16827c478bd9Sstevel@tonic-gate * We re-enable CEEN (if required) and check if any disrupting errors 16837c478bd9Sstevel@tonic-gate * have happened. We do this because if a disrupting error had occurred 16847c478bd9Sstevel@tonic-gate * with CEEN off, the trap will not be taken when CEEN is re-enabled. 16857c478bd9Sstevel@tonic-gate * Note that CEEN works differently on Cheetah than on Spitfire. Also, 16867c478bd9Sstevel@tonic-gate * we enable CEEN *before* checking the AFSR to avoid the small window 16877c478bd9Sstevel@tonic-gate * of a error happening between checking the AFSR and enabling CEEN. 16887c478bd9Sstevel@tonic-gate */ 16897c478bd9Sstevel@tonic-gate if (ch_flt.flt_trapped_ce & CE_CEEN_NODEFER) 16907c478bd9Sstevel@tonic-gate set_error_enable(get_error_enable() | EN_REG_CEEN); 16917c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 16927c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 16937c478bd9Sstevel@tonic-gate NULL); 16947c478bd9Sstevel@tonic-gate } 16957c478bd9Sstevel@tonic-gate 16967c478bd9Sstevel@tonic-gate /* 16977c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 16987c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 16997c478bd9Sstevel@tonic-gate */ 17007c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 17017c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", pr_reason); 17027c478bd9Sstevel@tonic-gate } 17037c478bd9Sstevel@tonic-gate 17047c478bd9Sstevel@tonic-gate /* 17057c478bd9Sstevel@tonic-gate * The async_err handler transfers control here for UE, EMU, EDU:BLD, 17067c478bd9Sstevel@tonic-gate * L3_EDU:BLD, TO, and BERR events. 17077c478bd9Sstevel@tonic-gate * Deferred errors controlled by NCEEN: UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR 17087c478bd9Sstevel@tonic-gate * 17097c478bd9Sstevel@tonic-gate * Cheetah+: No additional errors handled. 17107c478bd9Sstevel@tonic-gate * 17117c478bd9Sstevel@tonic-gate * Note that the p_clo_flags input is only valid in cases where the 17127c478bd9Sstevel@tonic-gate * cpu_private struct is not yet initialized (since that is the only 17137c478bd9Sstevel@tonic-gate * time that information cannot be obtained from the logout struct.) 17147c478bd9Sstevel@tonic-gate */ 17157c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 17167c478bd9Sstevel@tonic-gate void 17177c478bd9Sstevel@tonic-gate cpu_deferred_error(struct regs *rp, ulong_t p_clo_flags) 17187c478bd9Sstevel@tonic-gate { 17197c478bd9Sstevel@tonic-gate ushort_t ttype, tl; 17207c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 17217c478bd9Sstevel@tonic-gate struct async_flt *aflt; 17227c478bd9Sstevel@tonic-gate int trampolined = 0; 17237c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 17247c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 17257c478bd9Sstevel@tonic-gate uint64_t ceen, clo_flags; 17267c478bd9Sstevel@tonic-gate uint64_t log_afsr; 17277c478bd9Sstevel@tonic-gate uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 17287c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 17297c478bd9Sstevel@tonic-gate int expected = DDI_FM_ERR_UNEXPECTED; 17307c478bd9Sstevel@tonic-gate ddi_acc_hdl_t *hp; 17317c478bd9Sstevel@tonic-gate 17327c478bd9Sstevel@tonic-gate /* 17337c478bd9Sstevel@tonic-gate * We need to look at p_flag to determine if the thread detected an 17347c478bd9Sstevel@tonic-gate * error while dumping core. We can't grab p_lock here, but it's ok 17357c478bd9Sstevel@tonic-gate * because we just need a consistent snapshot and we know that everyone 17367c478bd9Sstevel@tonic-gate * else will store a consistent set of bits while holding p_lock. We 17377c478bd9Sstevel@tonic-gate * don't have to worry about a race because SDOCORE is set once prior 17387c478bd9Sstevel@tonic-gate * to doing i/o from the process's address space and is never cleared. 17397c478bd9Sstevel@tonic-gate */ 17407c478bd9Sstevel@tonic-gate uint_t pflag = ttoproc(curthread)->p_flag; 17417c478bd9Sstevel@tonic-gate 17427c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 17437c478bd9Sstevel@tonic-gate /* 17447c478bd9Sstevel@tonic-gate * Get the CPU log out info. If we can't find our CPU private 17457c478bd9Sstevel@tonic-gate * pointer then we will have to make due without any detailed 17467c478bd9Sstevel@tonic-gate * logout information. 17477c478bd9Sstevel@tonic-gate */ 17487c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 17497c478bd9Sstevel@tonic-gate clop = NULL; 17507c478bd9Sstevel@tonic-gate ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 17517c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 17527c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 17537c478bd9Sstevel@tonic-gate t_afar = cpu_error_regs.afar; 17547c478bd9Sstevel@tonic-gate t_afsr = cpu_error_regs.afsr; 17557c478bd9Sstevel@tonic-gate t_afsr_ext = cpu_error_regs.afsr_ext; 17567c478bd9Sstevel@tonic-gate #if defined(SERRANO) 17577c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 17587c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 17597c478bd9Sstevel@tonic-gate clo_flags = p_clo_flags; 17607c478bd9Sstevel@tonic-gate } else { 17617c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_async_logout); 17627c478bd9Sstevel@tonic-gate t_afar = clop->clo_data.chd_afar; 17637c478bd9Sstevel@tonic-gate t_afsr = clop->clo_data.chd_afsr; 17647c478bd9Sstevel@tonic-gate t_afsr_ext = clop->clo_data.chd_afsr_ext; 17657c478bd9Sstevel@tonic-gate #if defined(SERRANO) 17667c478bd9Sstevel@tonic-gate ch_flt.afar2 = clop->clo_data.chd_afar2; 17677c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 17687c478bd9Sstevel@tonic-gate clo_flags = clop->clo_flags; 17697c478bd9Sstevel@tonic-gate } 17707c478bd9Sstevel@tonic-gate 17717c478bd9Sstevel@tonic-gate /* 17727c478bd9Sstevel@tonic-gate * In order to simplify code, we maintain this afsr_errs 17737c478bd9Sstevel@tonic-gate * variable which holds the aggregate of AFSR and AFSR_EXT 17747c478bd9Sstevel@tonic-gate * sticky bits. 17757c478bd9Sstevel@tonic-gate */ 17767c478bd9Sstevel@tonic-gate t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 17777c478bd9Sstevel@tonic-gate (t_afsr & C_AFSR_ALL_ERRS); 17787c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 17797c478bd9Sstevel@tonic-gate 17807c478bd9Sstevel@tonic-gate /* 17817c478bd9Sstevel@tonic-gate * Grab information encoded into our clo_flags field. 17827c478bd9Sstevel@tonic-gate */ 17837c478bd9Sstevel@tonic-gate ceen = clo_flags & EN_REG_CEEN; 17847c478bd9Sstevel@tonic-gate tl = (clo_flags & CLO_FLAGS_TL_MASK) >> CLO_FLAGS_TL_SHIFT; 17857c478bd9Sstevel@tonic-gate ttype = (clo_flags & CLO_FLAGS_TT_MASK) >> CLO_FLAGS_TT_SHIFT; 17867c478bd9Sstevel@tonic-gate 17877c478bd9Sstevel@tonic-gate /* 17887c478bd9Sstevel@tonic-gate * handle the specific error 17897c478bd9Sstevel@tonic-gate */ 17907c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 17917c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 17927c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 17937c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 17947c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = t_afsr_ext; 17957c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = t_afsr_errs; 17967c478bd9Sstevel@tonic-gate aflt->flt_stat = t_afsr; 17977c478bd9Sstevel@tonic-gate aflt->flt_addr = t_afar; 17987c478bd9Sstevel@tonic-gate aflt->flt_pc = (caddr_t)rp->r_pc; 17997c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 18007c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 18017c478bd9Sstevel@tonic-gate aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 18027c478bd9Sstevel@tonic-gate aflt->flt_tl = (uchar_t)tl; 18037c478bd9Sstevel@tonic-gate aflt->flt_panic = ((tl != 0) || (aft_testfatal != 0) || 18047c478bd9Sstevel@tonic-gate C_AFSR_PANIC(t_afsr_errs)); 18057c478bd9Sstevel@tonic-gate aflt->flt_core = (pflag & SDOCORE) ? 1 : 0; 18067c478bd9Sstevel@tonic-gate aflt->flt_status = ((ttype == T_DATA_ERROR) ? ECC_D_TRAP : ECC_I_TRAP); 18077c478bd9Sstevel@tonic-gate 18087c478bd9Sstevel@tonic-gate /* 18097c478bd9Sstevel@tonic-gate * If the trap occurred in privileged mode at TL=0, we need to check to 18107c478bd9Sstevel@tonic-gate * see if we were executing in the kernel under on_trap() or t_lofault 18117c478bd9Sstevel@tonic-gate * protection. If so, modify the saved registers so that we return 18127c478bd9Sstevel@tonic-gate * from the trap to the appropriate trampoline routine. 18137c478bd9Sstevel@tonic-gate */ 18147c478bd9Sstevel@tonic-gate if (aflt->flt_priv && tl == 0) { 18157c478bd9Sstevel@tonic-gate if (curthread->t_ontrap != NULL) { 18167c478bd9Sstevel@tonic-gate on_trap_data_t *otp = curthread->t_ontrap; 18177c478bd9Sstevel@tonic-gate 18187c478bd9Sstevel@tonic-gate if (otp->ot_prot & OT_DATA_EC) { 18197c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_EC; 18207c478bd9Sstevel@tonic-gate otp->ot_trap |= OT_DATA_EC; 18217c478bd9Sstevel@tonic-gate rp->r_pc = otp->ot_trampoline; 18227c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 18237c478bd9Sstevel@tonic-gate trampolined = 1; 18247c478bd9Sstevel@tonic-gate } 18257c478bd9Sstevel@tonic-gate 18267c478bd9Sstevel@tonic-gate if ((t_afsr & (C_AFSR_TO | C_AFSR_BERR)) && 18277c478bd9Sstevel@tonic-gate (otp->ot_prot & OT_DATA_ACCESS)) { 18287c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_ACCESS; 18297c478bd9Sstevel@tonic-gate otp->ot_trap |= OT_DATA_ACCESS; 18307c478bd9Sstevel@tonic-gate rp->r_pc = otp->ot_trampoline; 18317c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 18327c478bd9Sstevel@tonic-gate trampolined = 1; 18337c478bd9Sstevel@tonic-gate /* 18347c478bd9Sstevel@tonic-gate * for peeks and caut_gets errors are expected 18357c478bd9Sstevel@tonic-gate */ 18367c478bd9Sstevel@tonic-gate hp = (ddi_acc_hdl_t *)otp->ot_handle; 18377c478bd9Sstevel@tonic-gate if (!hp) 18387c478bd9Sstevel@tonic-gate expected = DDI_FM_ERR_PEEK; 18397c478bd9Sstevel@tonic-gate else if (hp->ah_acc.devacc_attr_access == 18407c478bd9Sstevel@tonic-gate DDI_CAUTIOUS_ACC) 18417c478bd9Sstevel@tonic-gate expected = DDI_FM_ERR_EXPECTED; 18427c478bd9Sstevel@tonic-gate } 18437c478bd9Sstevel@tonic-gate 18447c478bd9Sstevel@tonic-gate } else if (curthread->t_lofault) { 18457c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_COPY; 18467c478bd9Sstevel@tonic-gate rp->r_g1 = EFAULT; 18477c478bd9Sstevel@tonic-gate rp->r_pc = curthread->t_lofault; 18487c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 18497c478bd9Sstevel@tonic-gate trampolined = 1; 18507c478bd9Sstevel@tonic-gate } 18517c478bd9Sstevel@tonic-gate } 18527c478bd9Sstevel@tonic-gate 18537c478bd9Sstevel@tonic-gate /* 18547c478bd9Sstevel@tonic-gate * If we're in user mode or we're doing a protected copy, we either 18557c478bd9Sstevel@tonic-gate * want the ASTON code below to send a signal to the user process 18567c478bd9Sstevel@tonic-gate * or we want to panic if aft_panic is set. 18577c478bd9Sstevel@tonic-gate * 18587c478bd9Sstevel@tonic-gate * If we're in privileged mode and we're not doing a copy, then we 18597c478bd9Sstevel@tonic-gate * need to check if we've trampolined. If we haven't trampolined, 18607c478bd9Sstevel@tonic-gate * we should panic. 18617c478bd9Sstevel@tonic-gate */ 18627c478bd9Sstevel@tonic-gate if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 18637c478bd9Sstevel@tonic-gate if (t_afsr_errs & 18647c478bd9Sstevel@tonic-gate ((C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS) & 18657c478bd9Sstevel@tonic-gate ~(C_AFSR_BERR | C_AFSR_TO))) 18667c478bd9Sstevel@tonic-gate aflt->flt_panic |= aft_panic; 18677c478bd9Sstevel@tonic-gate } else if (!trampolined) { 18687c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 18697c478bd9Sstevel@tonic-gate } 18707c478bd9Sstevel@tonic-gate 18717c478bd9Sstevel@tonic-gate /* 18727c478bd9Sstevel@tonic-gate * If we've trampolined due to a privileged TO or BERR, or if an 18737c478bd9Sstevel@tonic-gate * unprivileged TO or BERR occurred, we don't want to enqueue an 18747c478bd9Sstevel@tonic-gate * event for that TO or BERR. Queue all other events (if any) besides 18757c478bd9Sstevel@tonic-gate * the TO/BERR. Since we may not be enqueing any events, we need to 18767c478bd9Sstevel@tonic-gate * ignore the number of events queued. If we haven't trampolined due 18777c478bd9Sstevel@tonic-gate * to a TO or BERR, just enqueue events normally. 18787c478bd9Sstevel@tonic-gate */ 18797c478bd9Sstevel@tonic-gate log_afsr = t_afsr_errs; 18807c478bd9Sstevel@tonic-gate if (trampolined) { 18817c478bd9Sstevel@tonic-gate log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 18827c478bd9Sstevel@tonic-gate } else if (!aflt->flt_priv) { 18837c478bd9Sstevel@tonic-gate /* 18847c478bd9Sstevel@tonic-gate * User mode, suppress messages if 18857c478bd9Sstevel@tonic-gate * cpu_berr_to_verbose is not set. 18867c478bd9Sstevel@tonic-gate */ 18877c478bd9Sstevel@tonic-gate if (!cpu_berr_to_verbose) 18887c478bd9Sstevel@tonic-gate log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 18897c478bd9Sstevel@tonic-gate } 18907c478bd9Sstevel@tonic-gate 18917c478bd9Sstevel@tonic-gate /* 18927c478bd9Sstevel@tonic-gate * Log any errors that occurred 18937c478bd9Sstevel@tonic-gate */ 18947c478bd9Sstevel@tonic-gate if (((log_afsr & 18957c478bd9Sstevel@tonic-gate ((C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS) & ~C_AFSR_ME)) && 18967c478bd9Sstevel@tonic-gate cpu_queue_events(&ch_flt, pr_reason, log_afsr, clop) == 0) || 18977c478bd9Sstevel@tonic-gate (t_afsr_errs & 18987c478bd9Sstevel@tonic-gate (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) == 0) { 18997c478bd9Sstevel@tonic-gate ch_flt.flt_type = CPU_INV_AFSR; 19007c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 19017c478bd9Sstevel@tonic-gate (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 19027c478bd9Sstevel@tonic-gate aflt->flt_panic); 19037c478bd9Sstevel@tonic-gate } 19047c478bd9Sstevel@tonic-gate 19057c478bd9Sstevel@tonic-gate /* 19067c478bd9Sstevel@tonic-gate * Zero out + invalidate CPU logout. 19077c478bd9Sstevel@tonic-gate */ 19087c478bd9Sstevel@tonic-gate if (clop) { 19097c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 19107c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 19117c478bd9Sstevel@tonic-gate } 19127c478bd9Sstevel@tonic-gate 19137c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 19147c478bd9Sstevel@tonic-gate /* 19157c478bd9Sstevel@tonic-gate * UE/RUE/BERR/TO: Call our bus nexus friends to check for 19167c478bd9Sstevel@tonic-gate * IO errors that may have resulted in this trap. 19177c478bd9Sstevel@tonic-gate */ 19187c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE|C_AFSR_RUE|C_AFSR_TO|C_AFSR_BERR)) { 19197c478bd9Sstevel@tonic-gate cpu_run_bus_error_handlers(aflt, expected); 19207c478bd9Sstevel@tonic-gate } 19217c478bd9Sstevel@tonic-gate 19227c478bd9Sstevel@tonic-gate /* 19237c478bd9Sstevel@tonic-gate * UE/RUE: If UE or RUE is in memory, we need to flush the bad 19247c478bd9Sstevel@tonic-gate * line from the Ecache. We also need to query the bus nexus for 19257c478bd9Sstevel@tonic-gate * fatal errors. Attempts to do diagnostic read on caches may 19267c478bd9Sstevel@tonic-gate * introduce more errors (especially when the module is bad). 19277c478bd9Sstevel@tonic-gate */ 19287c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE|C_AFSR_RUE)) { 19297c478bd9Sstevel@tonic-gate /* 19307c478bd9Sstevel@tonic-gate * Ask our bus nexus friends if they have any fatal errors. If 19317c478bd9Sstevel@tonic-gate * so, they will log appropriate error messages. 19327c478bd9Sstevel@tonic-gate */ 19337c478bd9Sstevel@tonic-gate if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 19347c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 19357c478bd9Sstevel@tonic-gate 19367c478bd9Sstevel@tonic-gate /* 19377c478bd9Sstevel@tonic-gate * We got a UE or RUE and are panicking, save the fault PA in 19387c478bd9Sstevel@tonic-gate * a known location so that the platform specific panic code 19397c478bd9Sstevel@tonic-gate * can check for copyback errors. 19407c478bd9Sstevel@tonic-gate */ 19417c478bd9Sstevel@tonic-gate if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 19427c478bd9Sstevel@tonic-gate panic_aflt = *aflt; 19437c478bd9Sstevel@tonic-gate } 19447c478bd9Sstevel@tonic-gate } 19457c478bd9Sstevel@tonic-gate 19467c478bd9Sstevel@tonic-gate /* 19477c478bd9Sstevel@tonic-gate * Flush Ecache line or entire Ecache 19487c478bd9Sstevel@tonic-gate */ 19497c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE | C_AFSR_RUE | C_AFSR_EDU | C_AFSR_BERR)) 19507c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(&ch_flt); 19517c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 19527c478bd9Sstevel@tonic-gate /* 19537c478bd9Sstevel@tonic-gate * UE/BERR/TO: Call our bus nexus friends to check for 19547c478bd9Sstevel@tonic-gate * IO errors that may have resulted in this trap. 19557c478bd9Sstevel@tonic-gate */ 19567c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE|C_AFSR_TO|C_AFSR_BERR)) { 19577c478bd9Sstevel@tonic-gate cpu_run_bus_error_handlers(aflt, expected); 19587c478bd9Sstevel@tonic-gate } 19597c478bd9Sstevel@tonic-gate 19607c478bd9Sstevel@tonic-gate /* 19617c478bd9Sstevel@tonic-gate * UE: If the UE is in memory, we need to flush the bad 19627c478bd9Sstevel@tonic-gate * line from the Ecache. We also need to query the bus nexus for 19637c478bd9Sstevel@tonic-gate * fatal errors. Attempts to do diagnostic read on caches may 19647c478bd9Sstevel@tonic-gate * introduce more errors (especially when the module is bad). 19657c478bd9Sstevel@tonic-gate */ 19667c478bd9Sstevel@tonic-gate if (t_afsr & C_AFSR_UE) { 19677c478bd9Sstevel@tonic-gate /* 19687c478bd9Sstevel@tonic-gate * Ask our legacy bus nexus friends if they have any fatal 19697c478bd9Sstevel@tonic-gate * errors. If so, they will log appropriate error messages. 19707c478bd9Sstevel@tonic-gate */ 19717c478bd9Sstevel@tonic-gate if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 19727c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 19737c478bd9Sstevel@tonic-gate 19747c478bd9Sstevel@tonic-gate /* 19757c478bd9Sstevel@tonic-gate * We got a UE and are panicking, save the fault PA in a known 19767c478bd9Sstevel@tonic-gate * location so that the platform specific panic code can check 19777c478bd9Sstevel@tonic-gate * for copyback errors. 19787c478bd9Sstevel@tonic-gate */ 19797c478bd9Sstevel@tonic-gate if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 19807c478bd9Sstevel@tonic-gate panic_aflt = *aflt; 19817c478bd9Sstevel@tonic-gate } 19827c478bd9Sstevel@tonic-gate } 19837c478bd9Sstevel@tonic-gate 19847c478bd9Sstevel@tonic-gate /* 19857c478bd9Sstevel@tonic-gate * Flush Ecache line or entire Ecache 19867c478bd9Sstevel@tonic-gate */ 19877c478bd9Sstevel@tonic-gate if (t_afsr_errs & 19887c478bd9Sstevel@tonic-gate (C_AFSR_UE | C_AFSR_EDU | C_AFSR_BERR | C_AFSR_L3_EDU)) 19897c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(&ch_flt); 19907c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 19917c478bd9Sstevel@tonic-gate 19927c478bd9Sstevel@tonic-gate /* 19937c478bd9Sstevel@tonic-gate * We carefully re-enable NCEEN and CEEN and then check if any deferred 19947c478bd9Sstevel@tonic-gate * or disrupting errors have happened. We do this because if a 19957c478bd9Sstevel@tonic-gate * deferred or disrupting error had occurred with NCEEN/CEEN off, the 19967c478bd9Sstevel@tonic-gate * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 19977c478bd9Sstevel@tonic-gate * CEEN works differently on Cheetah than on Spitfire. Also, we enable 19987c478bd9Sstevel@tonic-gate * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 19997c478bd9Sstevel@tonic-gate * deferred or disrupting error happening between checking the AFSR and 20007c478bd9Sstevel@tonic-gate * enabling NCEEN/CEEN. 20017c478bd9Sstevel@tonic-gate * 20027c478bd9Sstevel@tonic-gate * Note: CEEN reenabled only if it was on when trap taken. 20037c478bd9Sstevel@tonic-gate */ 20047c478bd9Sstevel@tonic-gate set_error_enable(get_error_enable() | (EN_REG_NCEEN | ceen)); 20057c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 20067c478bd9Sstevel@tonic-gate /* 20077c478bd9Sstevel@tonic-gate * Check for secondary errors, and avoid panicking if we 20087c478bd9Sstevel@tonic-gate * have them 20097c478bd9Sstevel@tonic-gate */ 20107c478bd9Sstevel@tonic-gate if (cpu_check_secondary_errors(&ch_flt, t_afsr_errs, 20117c478bd9Sstevel@tonic-gate t_afar) == 0) { 20127c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((ch_flt.afsr_errs & 20137c478bd9Sstevel@tonic-gate (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) != 0); 20147c478bd9Sstevel@tonic-gate } 20157c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 20167c478bd9Sstevel@tonic-gate NULL); 20177c478bd9Sstevel@tonic-gate } 20187c478bd9Sstevel@tonic-gate 20197c478bd9Sstevel@tonic-gate /* 20207c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 20217c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 20227c478bd9Sstevel@tonic-gate */ 20237c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 20247c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", pr_reason); 20257c478bd9Sstevel@tonic-gate 20267c478bd9Sstevel@tonic-gate /* 20277c478bd9Sstevel@tonic-gate * If we queued an error and we are going to return from the trap and 20287c478bd9Sstevel@tonic-gate * the error was in user mode or inside of a copy routine, set AST flag 20297c478bd9Sstevel@tonic-gate * so the queue will be drained before returning to user mode. The 20307c478bd9Sstevel@tonic-gate * AST processing will also act on our failure policy. 20317c478bd9Sstevel@tonic-gate */ 20327c478bd9Sstevel@tonic-gate if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 20337c478bd9Sstevel@tonic-gate int pcb_flag = 0; 20347c478bd9Sstevel@tonic-gate 20357c478bd9Sstevel@tonic-gate if (t_afsr_errs & 20367c478bd9Sstevel@tonic-gate (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS & 20377c478bd9Sstevel@tonic-gate ~(C_AFSR_BERR | C_AFSR_TO))) 20387c478bd9Sstevel@tonic-gate pcb_flag |= ASYNC_HWERR; 20397c478bd9Sstevel@tonic-gate 20407c478bd9Sstevel@tonic-gate if (t_afsr & C_AFSR_BERR) 20417c478bd9Sstevel@tonic-gate pcb_flag |= ASYNC_BERR; 20427c478bd9Sstevel@tonic-gate 20437c478bd9Sstevel@tonic-gate if (t_afsr & C_AFSR_TO) 20447c478bd9Sstevel@tonic-gate pcb_flag |= ASYNC_BTO; 20457c478bd9Sstevel@tonic-gate 20467c478bd9Sstevel@tonic-gate ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag; 20477c478bd9Sstevel@tonic-gate aston(curthread); 20487c478bd9Sstevel@tonic-gate } 20497c478bd9Sstevel@tonic-gate } 20507c478bd9Sstevel@tonic-gate 20517c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 20527c478bd9Sstevel@tonic-gate /* 20537c478bd9Sstevel@tonic-gate * Handling of data and instruction parity errors (traps 0x71, 0x72). 20547c478bd9Sstevel@tonic-gate * 20557c478bd9Sstevel@tonic-gate * For Panther, P$ data parity errors during floating point load hits 20567c478bd9Sstevel@tonic-gate * are also detected (reported as TT 0x71) and handled by this trap 20577c478bd9Sstevel@tonic-gate * handler. 20587c478bd9Sstevel@tonic-gate * 20597c478bd9Sstevel@tonic-gate * AFSR/AFAR are not set for parity errors, only TPC (a virtual address) 20607c478bd9Sstevel@tonic-gate * is available. 20617c478bd9Sstevel@tonic-gate */ 20627c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 20637c478bd9Sstevel@tonic-gate void 20647c478bd9Sstevel@tonic-gate cpu_parity_error(struct regs *rp, uint_t flags, caddr_t tpc) 20657c478bd9Sstevel@tonic-gate { 20667c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 20677c478bd9Sstevel@tonic-gate struct async_flt *aflt; 20687c478bd9Sstevel@tonic-gate uchar_t tl = ((flags & CH_ERR_TL) != 0); 20697c478bd9Sstevel@tonic-gate uchar_t iparity = ((flags & CH_ERR_IPE) != 0); 20707c478bd9Sstevel@tonic-gate uchar_t panic = ((flags & CH_ERR_PANIC) != 0); 20717c478bd9Sstevel@tonic-gate char *error_class; 20727c478bd9Sstevel@tonic-gate 20737c478bd9Sstevel@tonic-gate /* 20747c478bd9Sstevel@tonic-gate * Log the error. 20757c478bd9Sstevel@tonic-gate * For icache parity errors the fault address is the trap PC. 20767c478bd9Sstevel@tonic-gate * For dcache/pcache parity errors the instruction would have to 20777c478bd9Sstevel@tonic-gate * be decoded to determine the address and that isn't possible 20787c478bd9Sstevel@tonic-gate * at high PIL. 20797c478bd9Sstevel@tonic-gate */ 20807c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 20817c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 20827c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 20837c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 20847c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 20857c478bd9Sstevel@tonic-gate aflt->flt_pc = tpc; 20867c478bd9Sstevel@tonic-gate aflt->flt_addr = iparity ? (uint64_t)tpc : AFLT_INV_ADDR; 20877c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 20887c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 20897c478bd9Sstevel@tonic-gate aflt->flt_priv = (tl || (rp->r_tstate & TSTATE_PRIV)) ? 1 : 0; 20907c478bd9Sstevel@tonic-gate aflt->flt_tl = tl; 20917c478bd9Sstevel@tonic-gate aflt->flt_panic = panic; 20927c478bd9Sstevel@tonic-gate aflt->flt_status = iparity ? ECC_IP_TRAP : ECC_DP_TRAP; 20937c478bd9Sstevel@tonic-gate ch_flt.flt_type = iparity ? CPU_IC_PARITY : CPU_DC_PARITY; 20947c478bd9Sstevel@tonic-gate 20957c478bd9Sstevel@tonic-gate if (iparity) { 20967c478bd9Sstevel@tonic-gate cpu_icache_parity_info(&ch_flt); 20977c478bd9Sstevel@tonic-gate if (ch_flt.parity_data.ipe.cpl_off != -1) 20987c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_IDSPE; 20997c478bd9Sstevel@tonic-gate else if (ch_flt.parity_data.ipe.cpl_way != -1) 21007c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_ITSPE; 21017c478bd9Sstevel@tonic-gate else 21027c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_IPE; 21037c478bd9Sstevel@tonic-gate aflt->flt_payload = FM_EREPORT_PAYLOAD_ICACHE_PE; 21047c478bd9Sstevel@tonic-gate } else { 21057c478bd9Sstevel@tonic-gate cpu_dcache_parity_info(&ch_flt); 21067c478bd9Sstevel@tonic-gate if (ch_flt.parity_data.dpe.cpl_off != -1) 21077c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_DDSPE; 21087c478bd9Sstevel@tonic-gate else if (ch_flt.parity_data.dpe.cpl_way != -1) 21097c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_DTSPE; 21107c478bd9Sstevel@tonic-gate else 21117c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_DPE; 21127c478bd9Sstevel@tonic-gate aflt->flt_payload = FM_EREPORT_PAYLOAD_DCACHE_PE; 21137c478bd9Sstevel@tonic-gate /* 21147c478bd9Sstevel@tonic-gate * For panther we also need to check the P$ for parity errors. 21157c478bd9Sstevel@tonic-gate */ 21167c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 21177c478bd9Sstevel@tonic-gate cpu_pcache_parity_info(&ch_flt); 21187c478bd9Sstevel@tonic-gate if (ch_flt.parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 21197c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_PDSPE; 21207c478bd9Sstevel@tonic-gate aflt->flt_payload = 21217c478bd9Sstevel@tonic-gate FM_EREPORT_PAYLOAD_PCACHE_PE; 21227c478bd9Sstevel@tonic-gate } 21237c478bd9Sstevel@tonic-gate } 21247c478bd9Sstevel@tonic-gate } 21257c478bd9Sstevel@tonic-gate 21267c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(error_class, (void *)&ch_flt, 21277c478bd9Sstevel@tonic-gate sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 21287c478bd9Sstevel@tonic-gate 21297c478bd9Sstevel@tonic-gate if (iparity) { 21307c478bd9Sstevel@tonic-gate /* 21317c478bd9Sstevel@tonic-gate * Invalidate entire I$. 21327c478bd9Sstevel@tonic-gate * This is required due to the use of diagnostic ASI 21337c478bd9Sstevel@tonic-gate * accesses that may result in a loss of I$ coherency. 21347c478bd9Sstevel@tonic-gate */ 21357c478bd9Sstevel@tonic-gate if (cache_boot_state & DCU_IC) { 21367c478bd9Sstevel@tonic-gate flush_icache(); 21377c478bd9Sstevel@tonic-gate } 21387c478bd9Sstevel@tonic-gate /* 21397c478bd9Sstevel@tonic-gate * According to section P.3.1 of the Panther PRM, we 21407c478bd9Sstevel@tonic-gate * need to do a little more for recovery on those 21417c478bd9Sstevel@tonic-gate * CPUs after encountering an I$ parity error. 21427c478bd9Sstevel@tonic-gate */ 21437c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 21447c478bd9Sstevel@tonic-gate flush_ipb(); 21457c478bd9Sstevel@tonic-gate correct_dcache_parity(dcache_size, 21467c478bd9Sstevel@tonic-gate dcache_linesize); 21477c478bd9Sstevel@tonic-gate flush_pcache(); 21487c478bd9Sstevel@tonic-gate } 21497c478bd9Sstevel@tonic-gate } else { 21507c478bd9Sstevel@tonic-gate /* 21517c478bd9Sstevel@tonic-gate * Since the valid bit is ignored when checking parity the 21527c478bd9Sstevel@tonic-gate * D$ data and tag must also be corrected. Set D$ data bits 21537c478bd9Sstevel@tonic-gate * to zero and set utag to 0, 1, 2, 3. 21547c478bd9Sstevel@tonic-gate */ 21557c478bd9Sstevel@tonic-gate correct_dcache_parity(dcache_size, dcache_linesize); 21567c478bd9Sstevel@tonic-gate 21577c478bd9Sstevel@tonic-gate /* 21587c478bd9Sstevel@tonic-gate * According to section P.3.3 of the Panther PRM, we 21597c478bd9Sstevel@tonic-gate * need to do a little more for recovery on those 21607c478bd9Sstevel@tonic-gate * CPUs after encountering a D$ or P$ parity error. 21617c478bd9Sstevel@tonic-gate * 21627c478bd9Sstevel@tonic-gate * As far as clearing P$ parity errors, it is enough to 21637c478bd9Sstevel@tonic-gate * simply invalidate all entries in the P$ since P$ parity 21647c478bd9Sstevel@tonic-gate * error traps are only generated for floating point load 21657c478bd9Sstevel@tonic-gate * hits. 21667c478bd9Sstevel@tonic-gate */ 21677c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 21687c478bd9Sstevel@tonic-gate flush_icache(); 21697c478bd9Sstevel@tonic-gate flush_ipb(); 21707c478bd9Sstevel@tonic-gate flush_pcache(); 21717c478bd9Sstevel@tonic-gate } 21727c478bd9Sstevel@tonic-gate } 21737c478bd9Sstevel@tonic-gate 21747c478bd9Sstevel@tonic-gate /* 21757c478bd9Sstevel@tonic-gate * Invalidate entire D$ if it was enabled. 21767c478bd9Sstevel@tonic-gate * This is done to avoid stale data in the D$ which might 21777c478bd9Sstevel@tonic-gate * occur with the D$ disabled and the trap handler doing 21787c478bd9Sstevel@tonic-gate * stores affecting lines already in the D$. 21797c478bd9Sstevel@tonic-gate */ 21807c478bd9Sstevel@tonic-gate if (cache_boot_state & DCU_DC) { 21817c478bd9Sstevel@tonic-gate flush_dcache(); 21827c478bd9Sstevel@tonic-gate } 21837c478bd9Sstevel@tonic-gate 21847c478bd9Sstevel@tonic-gate /* 21857c478bd9Sstevel@tonic-gate * Restore caches to their bootup state. 21867c478bd9Sstevel@tonic-gate */ 21877c478bd9Sstevel@tonic-gate set_dcu(get_dcu() | cache_boot_state); 21887c478bd9Sstevel@tonic-gate 21897c478bd9Sstevel@tonic-gate /* 21907c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 21917c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 21927c478bd9Sstevel@tonic-gate */ 21937c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 21947c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", iparity ? "IPE " : "DPE "); 21957c478bd9Sstevel@tonic-gate 21967c478bd9Sstevel@tonic-gate /* 21977c478bd9Sstevel@tonic-gate * If this error occurred at TL>0 then flush the E$ here to reduce 21987c478bd9Sstevel@tonic-gate * the chance of getting an unrecoverable Fast ECC error. This 21997c478bd9Sstevel@tonic-gate * flush will evict the part of the parity trap handler that is run 22007c478bd9Sstevel@tonic-gate * at TL>1. 22017c478bd9Sstevel@tonic-gate */ 22027c478bd9Sstevel@tonic-gate if (tl) { 22037c478bd9Sstevel@tonic-gate cpu_flush_ecache(); 22047c478bd9Sstevel@tonic-gate } 22057c478bd9Sstevel@tonic-gate } 22067c478bd9Sstevel@tonic-gate 22077c478bd9Sstevel@tonic-gate /* 22087c478bd9Sstevel@tonic-gate * On an I$ parity error, mark the appropriate entries in the ch_async_flt_t 22097c478bd9Sstevel@tonic-gate * to indicate which portions of the captured data should be in the ereport. 22107c478bd9Sstevel@tonic-gate */ 22117c478bd9Sstevel@tonic-gate void 22127c478bd9Sstevel@tonic-gate cpu_async_log_ic_parity_err(ch_async_flt_t *ch_flt) 22137c478bd9Sstevel@tonic-gate { 22147c478bd9Sstevel@tonic-gate int way = ch_flt->parity_data.ipe.cpl_way; 22157c478bd9Sstevel@tonic-gate int offset = ch_flt->parity_data.ipe.cpl_off; 22167c478bd9Sstevel@tonic-gate int tag_index; 22177c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 22187c478bd9Sstevel@tonic-gate 22197c478bd9Sstevel@tonic-gate 22207c478bd9Sstevel@tonic-gate if ((offset != -1) || (way != -1)) { 22217c478bd9Sstevel@tonic-gate /* 22227c478bd9Sstevel@tonic-gate * Parity error in I$ tag or data 22237c478bd9Sstevel@tonic-gate */ 22247c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 22257c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 22267c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 22277c478bd9Sstevel@tonic-gate PN_ICIDX_TO_WAY(tag_index); 22287c478bd9Sstevel@tonic-gate else 22297c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 22307c478bd9Sstevel@tonic-gate CH_ICIDX_TO_WAY(tag_index); 22317c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 22327c478bd9Sstevel@tonic-gate IC_LOGFLAG_MAGIC; 22337c478bd9Sstevel@tonic-gate } else { 22347c478bd9Sstevel@tonic-gate /* 22357c478bd9Sstevel@tonic-gate * Parity error was not identified. 22367c478bd9Sstevel@tonic-gate * Log tags and data for all ways. 22377c478bd9Sstevel@tonic-gate */ 22387c478bd9Sstevel@tonic-gate for (way = 0; way < CH_ICACHE_NWAY; way++) { 22397c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 22407c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 22417c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 22427c478bd9Sstevel@tonic-gate PN_ICIDX_TO_WAY(tag_index); 22437c478bd9Sstevel@tonic-gate else 22447c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 22457c478bd9Sstevel@tonic-gate CH_ICIDX_TO_WAY(tag_index); 22467c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 22477c478bd9Sstevel@tonic-gate IC_LOGFLAG_MAGIC; 22487c478bd9Sstevel@tonic-gate } 22497c478bd9Sstevel@tonic-gate } 22507c478bd9Sstevel@tonic-gate } 22517c478bd9Sstevel@tonic-gate 22527c478bd9Sstevel@tonic-gate /* 22537c478bd9Sstevel@tonic-gate * On an D$ parity error, mark the appropriate entries in the ch_async_flt_t 22547c478bd9Sstevel@tonic-gate * to indicate which portions of the captured data should be in the ereport. 22557c478bd9Sstevel@tonic-gate */ 22567c478bd9Sstevel@tonic-gate void 22577c478bd9Sstevel@tonic-gate cpu_async_log_dc_parity_err(ch_async_flt_t *ch_flt) 22587c478bd9Sstevel@tonic-gate { 22597c478bd9Sstevel@tonic-gate int way = ch_flt->parity_data.dpe.cpl_way; 22607c478bd9Sstevel@tonic-gate int offset = ch_flt->parity_data.dpe.cpl_off; 22617c478bd9Sstevel@tonic-gate int tag_index; 22627c478bd9Sstevel@tonic-gate 22637c478bd9Sstevel@tonic-gate if (offset != -1) { 22647c478bd9Sstevel@tonic-gate /* 22657c478bd9Sstevel@tonic-gate * Parity error in D$ or P$ data array. 22667c478bd9Sstevel@tonic-gate * 22677c478bd9Sstevel@tonic-gate * First check to see whether the parity error is in D$ or P$ 22687c478bd9Sstevel@tonic-gate * since P$ data parity errors are reported in Panther using 22697c478bd9Sstevel@tonic-gate * the same trap. 22707c478bd9Sstevel@tonic-gate */ 22717c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 22727c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.dpe.cpl_pc[way].pc_idx; 22737c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_pc[way].pc_way = 22747c478bd9Sstevel@tonic-gate CH_PCIDX_TO_WAY(tag_index); 22757c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_pc[way].pc_logflag = 22767c478bd9Sstevel@tonic-gate PC_LOGFLAG_MAGIC; 22777c478bd9Sstevel@tonic-gate } else { 22787c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 22797c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 22807c478bd9Sstevel@tonic-gate CH_DCIDX_TO_WAY(tag_index); 22817c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 22827c478bd9Sstevel@tonic-gate DC_LOGFLAG_MAGIC; 22837c478bd9Sstevel@tonic-gate } 22847c478bd9Sstevel@tonic-gate } else if (way != -1) { 22857c478bd9Sstevel@tonic-gate /* 22867c478bd9Sstevel@tonic-gate * Parity error in D$ tag. 22877c478bd9Sstevel@tonic-gate */ 22887c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 22897c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 22907c478bd9Sstevel@tonic-gate CH_DCIDX_TO_WAY(tag_index); 22917c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 22927c478bd9Sstevel@tonic-gate DC_LOGFLAG_MAGIC; 22937c478bd9Sstevel@tonic-gate } 22947c478bd9Sstevel@tonic-gate } 22957c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 22967c478bd9Sstevel@tonic-gate 22977c478bd9Sstevel@tonic-gate /* 22987c478bd9Sstevel@tonic-gate * The cpu_async_log_err() function is called via the [uc]e_drain() function to 22997c478bd9Sstevel@tonic-gate * post-process CPU events that are dequeued. As such, it can be invoked 23007c478bd9Sstevel@tonic-gate * from softint context, from AST processing in the trap() flow, or from the 23017c478bd9Sstevel@tonic-gate * panic flow. We decode the CPU-specific data, and take appropriate actions. 23027c478bd9Sstevel@tonic-gate * Historically this entry point was used to log the actual cmn_err(9F) text; 23037c478bd9Sstevel@tonic-gate * now with FMA it is used to prepare 'flt' to be converted into an ereport. 23047c478bd9Sstevel@tonic-gate * With FMA this function now also returns a flag which indicates to the 23057c478bd9Sstevel@tonic-gate * caller whether the ereport should be posted (1) or suppressed (0). 23067c478bd9Sstevel@tonic-gate */ 23077c478bd9Sstevel@tonic-gate static int 23087c478bd9Sstevel@tonic-gate cpu_async_log_err(void *flt, errorq_elem_t *eqep) 23097c478bd9Sstevel@tonic-gate { 23107c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt; 23117c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)flt; 2312db874c57Selowe uint64_t errors; 23137c478bd9Sstevel@tonic-gate 23147c478bd9Sstevel@tonic-gate switch (ch_flt->flt_type) { 23157c478bd9Sstevel@tonic-gate case CPU_INV_AFSR: 23167c478bd9Sstevel@tonic-gate /* 23177c478bd9Sstevel@tonic-gate * If it is a disrupting trap and the AFSR is zero, then 23187c478bd9Sstevel@tonic-gate * the event has probably already been noted. Do not post 23197c478bd9Sstevel@tonic-gate * an ereport. 23207c478bd9Sstevel@tonic-gate */ 23217c478bd9Sstevel@tonic-gate if ((aflt->flt_status & ECC_C_TRAP) && 23227c478bd9Sstevel@tonic-gate (!(aflt->flt_stat & C_AFSR_MASK))) 23237c478bd9Sstevel@tonic-gate return (0); 23247c478bd9Sstevel@tonic-gate else 23257c478bd9Sstevel@tonic-gate return (1); 23267c478bd9Sstevel@tonic-gate case CPU_TO: 23277c478bd9Sstevel@tonic-gate case CPU_BERR: 23287c478bd9Sstevel@tonic-gate case CPU_FATAL: 23297c478bd9Sstevel@tonic-gate case CPU_FPUERR: 23307c478bd9Sstevel@tonic-gate return (1); 23317c478bd9Sstevel@tonic-gate 23327c478bd9Sstevel@tonic-gate case CPU_UE_ECACHE_RETIRE: 23337c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 23347c478bd9Sstevel@tonic-gate cpu_page_retire(ch_flt); 23357c478bd9Sstevel@tonic-gate return (1); 23367c478bd9Sstevel@tonic-gate 23377c478bd9Sstevel@tonic-gate /* 23387c478bd9Sstevel@tonic-gate * Cases where we may want to suppress logging or perform 23397c478bd9Sstevel@tonic-gate * extended diagnostics. 23407c478bd9Sstevel@tonic-gate */ 23417c478bd9Sstevel@tonic-gate case CPU_CE: 23427c478bd9Sstevel@tonic-gate case CPU_EMC: 23437c478bd9Sstevel@tonic-gate /* 23447c478bd9Sstevel@tonic-gate * We want to skip logging and further classification 23457c478bd9Sstevel@tonic-gate * only if ALL the following conditions are true: 23467c478bd9Sstevel@tonic-gate * 23477c478bd9Sstevel@tonic-gate * 1. There is only one error 23487c478bd9Sstevel@tonic-gate * 2. That error is a correctable memory error 23497c478bd9Sstevel@tonic-gate * 3. The error is caused by the memory scrubber (in 23507c478bd9Sstevel@tonic-gate * which case the error will have occurred under 23517c478bd9Sstevel@tonic-gate * on_trap protection) 23527c478bd9Sstevel@tonic-gate * 4. The error is on a retired page 23537c478bd9Sstevel@tonic-gate * 23547c478bd9Sstevel@tonic-gate * Note: AFLT_PROT_EC is used places other than the memory 23557c478bd9Sstevel@tonic-gate * scrubber. However, none of those errors should occur 23567c478bd9Sstevel@tonic-gate * on a retired page. 23577c478bd9Sstevel@tonic-gate */ 23587c478bd9Sstevel@tonic-gate if ((ch_flt->afsr_errs & 23597c478bd9Sstevel@tonic-gate (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_CE && 23607c478bd9Sstevel@tonic-gate aflt->flt_prot == AFLT_PROT_EC) { 23617c478bd9Sstevel@tonic-gate 2362db874c57Selowe if (page_retire_check(aflt->flt_addr, NULL) == 0) { 23637c478bd9Sstevel@tonic-gate if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 23647c478bd9Sstevel@tonic-gate 23657c478bd9Sstevel@tonic-gate /* 23667c478bd9Sstevel@tonic-gate * Since we're skipping logging, we'll need 23677c478bd9Sstevel@tonic-gate * to schedule the re-enabling of CEEN 23687c478bd9Sstevel@tonic-gate */ 23697c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, 2370f47a9c50Smathue (void *)(uintptr_t)aflt->flt_inst, 2371f47a9c50Smathue drv_usectohz((clock_t)cpu_ceen_delay_secs 2372f47a9c50Smathue * MICROSEC)); 23737c478bd9Sstevel@tonic-gate } 23747c478bd9Sstevel@tonic-gate return (0); 23757c478bd9Sstevel@tonic-gate } 23767c478bd9Sstevel@tonic-gate } 23777c478bd9Sstevel@tonic-gate 23787c478bd9Sstevel@tonic-gate /* 23797c478bd9Sstevel@tonic-gate * Perform/schedule further classification actions, but 23807c478bd9Sstevel@tonic-gate * only if the page is healthy (we don't want bad 23817c478bd9Sstevel@tonic-gate * pages inducing too much diagnostic activity). If we could 23827c478bd9Sstevel@tonic-gate * not find a page pointer then we also skip this. If 23837c478bd9Sstevel@tonic-gate * ce_scrub_xdiag_recirc returns nonzero then it has chosen 23847c478bd9Sstevel@tonic-gate * to copy and recirculate the event (for further diagnostics) 23857c478bd9Sstevel@tonic-gate * and we should not proceed to log it here. 23867c478bd9Sstevel@tonic-gate * 23877c478bd9Sstevel@tonic-gate * This must be the last step here before the cpu_log_err() 23887c478bd9Sstevel@tonic-gate * below - if an event recirculates cpu_ce_log_err() will 23897c478bd9Sstevel@tonic-gate * not call the current function but just proceed directly 23907c478bd9Sstevel@tonic-gate * to cpu_ereport_post after the cpu_log_err() avoided below. 23917c478bd9Sstevel@tonic-gate * 23927c478bd9Sstevel@tonic-gate * Note: Check cpu_impl_async_log_err if changing this 23937c478bd9Sstevel@tonic-gate */ 2394db874c57Selowe if (page_retire_check(aflt->flt_addr, &errors) == EINVAL) { 2395db874c57Selowe CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 2396db874c57Selowe CE_XDIAG_SKIP_NOPP); 2397db874c57Selowe } else { 2398db874c57Selowe if (errors != PR_OK) { 23997c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 24007c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_PAGEDET); 24017c478bd9Sstevel@tonic-gate } else if (ce_scrub_xdiag_recirc(aflt, ce_queue, eqep, 24027c478bd9Sstevel@tonic-gate offsetof(ch_async_flt_t, cmn_asyncflt))) { 24037c478bd9Sstevel@tonic-gate return (0); 24047c478bd9Sstevel@tonic-gate } 24057c478bd9Sstevel@tonic-gate } 24067c478bd9Sstevel@tonic-gate /*FALLTHRU*/ 24077c478bd9Sstevel@tonic-gate 24087c478bd9Sstevel@tonic-gate /* 24097c478bd9Sstevel@tonic-gate * Cases where we just want to report the error and continue. 24107c478bd9Sstevel@tonic-gate */ 24117c478bd9Sstevel@tonic-gate case CPU_CE_ECACHE: 24127c478bd9Sstevel@tonic-gate case CPU_UE_ECACHE: 24137c478bd9Sstevel@tonic-gate case CPU_IV: 24147c478bd9Sstevel@tonic-gate case CPU_ORPH: 24157c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 24167c478bd9Sstevel@tonic-gate return (1); 24177c478bd9Sstevel@tonic-gate 24187c478bd9Sstevel@tonic-gate /* 24197c478bd9Sstevel@tonic-gate * Cases where we want to fall through to handle panicking. 24207c478bd9Sstevel@tonic-gate */ 24217c478bd9Sstevel@tonic-gate case CPU_UE: 24227c478bd9Sstevel@tonic-gate /* 24237c478bd9Sstevel@tonic-gate * We want to skip logging in the same conditions as the 24247c478bd9Sstevel@tonic-gate * CE case. In addition, we want to make sure we're not 24257c478bd9Sstevel@tonic-gate * panicking. 24267c478bd9Sstevel@tonic-gate */ 24277c478bd9Sstevel@tonic-gate if (!panicstr && (ch_flt->afsr_errs & 24287c478bd9Sstevel@tonic-gate (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_UE && 24297c478bd9Sstevel@tonic-gate aflt->flt_prot == AFLT_PROT_EC) { 2430db874c57Selowe if (page_retire_check(aflt->flt_addr, NULL) == 0) { 24317c478bd9Sstevel@tonic-gate /* Zero the address to clear the error */ 24327c478bd9Sstevel@tonic-gate softcall(ecc_page_zero, (void *)aflt->flt_addr); 24337c478bd9Sstevel@tonic-gate return (0); 24347c478bd9Sstevel@tonic-gate } 24357c478bd9Sstevel@tonic-gate } 24367c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 24377c478bd9Sstevel@tonic-gate break; 24387c478bd9Sstevel@tonic-gate 24397c478bd9Sstevel@tonic-gate default: 24407c478bd9Sstevel@tonic-gate /* 24417c478bd9Sstevel@tonic-gate * If the us3_common.c code doesn't know the flt_type, it may 24427c478bd9Sstevel@tonic-gate * be an implementation-specific code. Call into the impldep 24437c478bd9Sstevel@tonic-gate * backend to find out what to do: if it tells us to continue, 24447c478bd9Sstevel@tonic-gate * break and handle as if falling through from a UE; if not, 24457c478bd9Sstevel@tonic-gate * the impldep backend has handled the error and we're done. 24467c478bd9Sstevel@tonic-gate */ 24477c478bd9Sstevel@tonic-gate switch (cpu_impl_async_log_err(flt, eqep)) { 24487c478bd9Sstevel@tonic-gate case CH_ASYNC_LOG_DONE: 24497c478bd9Sstevel@tonic-gate return (1); 24507c478bd9Sstevel@tonic-gate case CH_ASYNC_LOG_RECIRC: 24517c478bd9Sstevel@tonic-gate return (0); 24527c478bd9Sstevel@tonic-gate case CH_ASYNC_LOG_CONTINUE: 24537c478bd9Sstevel@tonic-gate break; /* continue on to handle UE-like error */ 24547c478bd9Sstevel@tonic-gate default: 24557c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "discarding error 0x%p with " 24567c478bd9Sstevel@tonic-gate "invalid fault type (0x%x)", 24577c478bd9Sstevel@tonic-gate (void *)aflt, ch_flt->flt_type); 24587c478bd9Sstevel@tonic-gate return (0); 24597c478bd9Sstevel@tonic-gate } 24607c478bd9Sstevel@tonic-gate } 24617c478bd9Sstevel@tonic-gate 24627c478bd9Sstevel@tonic-gate /* ... fall through from the UE case */ 24637c478bd9Sstevel@tonic-gate 24647c478bd9Sstevel@tonic-gate if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) { 24657c478bd9Sstevel@tonic-gate if (!panicstr) { 24667c478bd9Sstevel@tonic-gate cpu_page_retire(ch_flt); 24677c478bd9Sstevel@tonic-gate } else { 24687c478bd9Sstevel@tonic-gate /* 24697c478bd9Sstevel@tonic-gate * Clear UEs on panic so that we don't 24707c478bd9Sstevel@tonic-gate * get haunted by them during panic or 24717c478bd9Sstevel@tonic-gate * after reboot 24727c478bd9Sstevel@tonic-gate */ 24737c478bd9Sstevel@tonic-gate cpu_clearphys(aflt); 24747c478bd9Sstevel@tonic-gate (void) clear_errors(NULL); 24757c478bd9Sstevel@tonic-gate } 24767c478bd9Sstevel@tonic-gate } 24777c478bd9Sstevel@tonic-gate 24787c478bd9Sstevel@tonic-gate return (1); 24797c478bd9Sstevel@tonic-gate } 24807c478bd9Sstevel@tonic-gate 24817c478bd9Sstevel@tonic-gate /* 24827c478bd9Sstevel@tonic-gate * Retire the bad page that may contain the flushed error. 24837c478bd9Sstevel@tonic-gate */ 24847c478bd9Sstevel@tonic-gate void 24857c478bd9Sstevel@tonic-gate cpu_page_retire(ch_async_flt_t *ch_flt) 24867c478bd9Sstevel@tonic-gate { 24877c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 2488db874c57Selowe (void) page_retire(aflt->flt_addr, PR_UE); 24897c478bd9Sstevel@tonic-gate } 24907c478bd9Sstevel@tonic-gate 24917c478bd9Sstevel@tonic-gate /* 24927c478bd9Sstevel@tonic-gate * The cpu_log_err() function is called by cpu_async_log_err() to perform the 24937c478bd9Sstevel@tonic-gate * generic event post-processing for correctable and uncorrectable memory, 24947c478bd9Sstevel@tonic-gate * E$, and MTag errors. Historically this entry point was used to log bits of 24957c478bd9Sstevel@tonic-gate * common cmn_err(9F) text; now with FMA it is used to prepare 'flt' to be 24967c478bd9Sstevel@tonic-gate * converted into an ereport. In addition, it transmits the error to any 24977c478bd9Sstevel@tonic-gate * platform-specific service-processor FRU logging routines, if available. 24987c478bd9Sstevel@tonic-gate */ 24997c478bd9Sstevel@tonic-gate void 25007c478bd9Sstevel@tonic-gate cpu_log_err(struct async_flt *aflt) 25017c478bd9Sstevel@tonic-gate { 25027c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 25037c478bd9Sstevel@tonic-gate int len = 0; 25047c478bd9Sstevel@tonic-gate int synd_status, synd_code, afar_status; 25057c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 25067c478bd9Sstevel@tonic-gate 25077c478bd9Sstevel@tonic-gate /* 25087c478bd9Sstevel@tonic-gate * Need to turn on ECC_ECACHE for plat_get_mem_unum(). 25097c478bd9Sstevel@tonic-gate * For Panther, L2$ is not external, so we don't want to 25107c478bd9Sstevel@tonic-gate * generate an E$ unum for those errors. 25117c478bd9Sstevel@tonic-gate */ 25127c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 25137c478bd9Sstevel@tonic-gate if (ch_flt->flt_bit & C_AFSR_EXT_L3_ERRS) 25147c478bd9Sstevel@tonic-gate aflt->flt_status |= ECC_ECACHE; 25157c478bd9Sstevel@tonic-gate } else { 25167c478bd9Sstevel@tonic-gate if (ch_flt->flt_bit & C_AFSR_ECACHE) 25177c478bd9Sstevel@tonic-gate aflt->flt_status |= ECC_ECACHE; 25187c478bd9Sstevel@tonic-gate } 25197c478bd9Sstevel@tonic-gate 25207c478bd9Sstevel@tonic-gate /* 25217c478bd9Sstevel@tonic-gate * Determine syndrome status. 25227c478bd9Sstevel@tonic-gate */ 25237c478bd9Sstevel@tonic-gate synd_status = afsr_to_synd_status(aflt->flt_inst, 25247c478bd9Sstevel@tonic-gate ch_flt->afsr_errs, ch_flt->flt_bit); 25257c478bd9Sstevel@tonic-gate 25267c478bd9Sstevel@tonic-gate /* 25277c478bd9Sstevel@tonic-gate * Determine afar status. 25287c478bd9Sstevel@tonic-gate */ 25297c478bd9Sstevel@tonic-gate if (pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)) 25307c478bd9Sstevel@tonic-gate afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 25317c478bd9Sstevel@tonic-gate ch_flt->flt_bit); 25327c478bd9Sstevel@tonic-gate else 25337c478bd9Sstevel@tonic-gate afar_status = AFLT_STAT_INVALID; 25347c478bd9Sstevel@tonic-gate 25357c478bd9Sstevel@tonic-gate /* 25367c478bd9Sstevel@tonic-gate * If afar status is not invalid do a unum lookup. 25377c478bd9Sstevel@tonic-gate */ 25387c478bd9Sstevel@tonic-gate if (afar_status != AFLT_STAT_INVALID) { 25397c478bd9Sstevel@tonic-gate (void) cpu_get_mem_unum_aflt(synd_status, aflt, unum, 25407c478bd9Sstevel@tonic-gate UNUM_NAMLEN, &len); 25417c478bd9Sstevel@tonic-gate } else { 25427c478bd9Sstevel@tonic-gate unum[0] = '\0'; 25437c478bd9Sstevel@tonic-gate } 25447c478bd9Sstevel@tonic-gate 25457c478bd9Sstevel@tonic-gate synd_code = synd_to_synd_code(synd_status, 25467c478bd9Sstevel@tonic-gate aflt->flt_synd, ch_flt->flt_bit); 25477c478bd9Sstevel@tonic-gate 25487c478bd9Sstevel@tonic-gate /* 25497c478bd9Sstevel@tonic-gate * Do not send the fruid message (plat_ecc_error_data_t) 25507c478bd9Sstevel@tonic-gate * to the SC if it can handle the enhanced error information 25517c478bd9Sstevel@tonic-gate * (plat_ecc_error2_data_t) or when the tunable 25527c478bd9Sstevel@tonic-gate * ecc_log_fruid_enable is set to 0. 25537c478bd9Sstevel@tonic-gate */ 25547c478bd9Sstevel@tonic-gate 25557c478bd9Sstevel@tonic-gate if (&plat_ecc_capability_sc_get && 25567c478bd9Sstevel@tonic-gate plat_ecc_capability_sc_get(PLAT_ECC_ERROR_MESSAGE)) { 25577c478bd9Sstevel@tonic-gate if (&plat_log_fruid_error) 25587c478bd9Sstevel@tonic-gate plat_log_fruid_error(synd_code, aflt, unum, 25597c478bd9Sstevel@tonic-gate ch_flt->flt_bit); 25607c478bd9Sstevel@tonic-gate } 25617c478bd9Sstevel@tonic-gate 25627c478bd9Sstevel@tonic-gate if (aflt->flt_func != NULL) 25637c478bd9Sstevel@tonic-gate aflt->flt_func(aflt, unum); 25647c478bd9Sstevel@tonic-gate 25657c478bd9Sstevel@tonic-gate if (afar_status != AFLT_STAT_INVALID) 25667c478bd9Sstevel@tonic-gate cpu_log_diag_info(ch_flt); 25677c478bd9Sstevel@tonic-gate 25687c478bd9Sstevel@tonic-gate /* 25697c478bd9Sstevel@tonic-gate * If we have a CEEN error , we do not reenable CEEN until after 25707c478bd9Sstevel@tonic-gate * we exit the trap handler. Otherwise, another error may 25717c478bd9Sstevel@tonic-gate * occur causing the handler to be entered recursively. 25727c478bd9Sstevel@tonic-gate * We set a timeout to trigger in cpu_ceen_delay_secs seconds, 25737c478bd9Sstevel@tonic-gate * to try and ensure that the CPU makes progress in the face 25747c478bd9Sstevel@tonic-gate * of a CE storm. 25757c478bd9Sstevel@tonic-gate */ 25767c478bd9Sstevel@tonic-gate if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 25777c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, 2578f47a9c50Smathue (void *)(uintptr_t)aflt->flt_inst, 25797c478bd9Sstevel@tonic-gate drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 25807c478bd9Sstevel@tonic-gate } 25817c478bd9Sstevel@tonic-gate } 25827c478bd9Sstevel@tonic-gate 25837c478bd9Sstevel@tonic-gate /* 25847c478bd9Sstevel@tonic-gate * Invoked by error_init() early in startup and therefore before 25857c478bd9Sstevel@tonic-gate * startup_errorq() is called to drain any error Q - 25867c478bd9Sstevel@tonic-gate * 25877c478bd9Sstevel@tonic-gate * startup() 25887c478bd9Sstevel@tonic-gate * startup_end() 25897c478bd9Sstevel@tonic-gate * error_init() 25907c478bd9Sstevel@tonic-gate * cpu_error_init() 25917c478bd9Sstevel@tonic-gate * errorq_init() 25927c478bd9Sstevel@tonic-gate * errorq_drain() 25937c478bd9Sstevel@tonic-gate * start_other_cpus() 25947c478bd9Sstevel@tonic-gate * 25957c478bd9Sstevel@tonic-gate * The purpose of this routine is to create error-related taskqs. Taskqs 25967c478bd9Sstevel@tonic-gate * are used for this purpose because cpu_lock can't be grabbed from interrupt 25977c478bd9Sstevel@tonic-gate * context. 25987c478bd9Sstevel@tonic-gate */ 25997c478bd9Sstevel@tonic-gate void 26007c478bd9Sstevel@tonic-gate cpu_error_init(int items) 26017c478bd9Sstevel@tonic-gate { 26027c478bd9Sstevel@tonic-gate /* 26037c478bd9Sstevel@tonic-gate * Create taskq(s) to reenable CE 26047c478bd9Sstevel@tonic-gate */ 26057c478bd9Sstevel@tonic-gate ch_check_ce_tq = taskq_create("cheetah_check_ce", 1, minclsyspri, 26067c478bd9Sstevel@tonic-gate items, items, TASKQ_PREPOPULATE); 26077c478bd9Sstevel@tonic-gate } 26087c478bd9Sstevel@tonic-gate 26097c478bd9Sstevel@tonic-gate void 26107c478bd9Sstevel@tonic-gate cpu_ce_log_err(struct async_flt *aflt, errorq_elem_t *eqep) 26117c478bd9Sstevel@tonic-gate { 26127c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 26137c478bd9Sstevel@tonic-gate int len; 26147c478bd9Sstevel@tonic-gate 26157c478bd9Sstevel@tonic-gate switch (aflt->flt_class) { 26167c478bd9Sstevel@tonic-gate case CPU_FAULT: 26177c478bd9Sstevel@tonic-gate cpu_ereport_init(aflt); 26187c478bd9Sstevel@tonic-gate if (cpu_async_log_err(aflt, eqep)) 26197c478bd9Sstevel@tonic-gate cpu_ereport_post(aflt); 26207c478bd9Sstevel@tonic-gate break; 26217c478bd9Sstevel@tonic-gate 26227c478bd9Sstevel@tonic-gate case BUS_FAULT: 26237c478bd9Sstevel@tonic-gate if (aflt->flt_func != NULL) { 26247c478bd9Sstevel@tonic-gate (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, 26257c478bd9Sstevel@tonic-gate unum, UNUM_NAMLEN, &len); 26267c478bd9Sstevel@tonic-gate aflt->flt_func(aflt, unum); 26277c478bd9Sstevel@tonic-gate } 26287c478bd9Sstevel@tonic-gate break; 26297c478bd9Sstevel@tonic-gate 26307c478bd9Sstevel@tonic-gate case RECIRC_CPU_FAULT: 26317c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 26327c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 26337c478bd9Sstevel@tonic-gate cpu_ereport_post(aflt); 26347c478bd9Sstevel@tonic-gate break; 26357c478bd9Sstevel@tonic-gate 26367c478bd9Sstevel@tonic-gate case RECIRC_BUS_FAULT: 26377c478bd9Sstevel@tonic-gate ASSERT(aflt->flt_class != RECIRC_BUS_FAULT); 26387c478bd9Sstevel@tonic-gate /*FALLTHRU*/ 26397c478bd9Sstevel@tonic-gate default: 26407c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "discarding CE error 0x%p with invalid " 26417c478bd9Sstevel@tonic-gate "fault class (0x%x)", (void *)aflt, aflt->flt_class); 26427c478bd9Sstevel@tonic-gate return; 26437c478bd9Sstevel@tonic-gate } 26447c478bd9Sstevel@tonic-gate } 26457c478bd9Sstevel@tonic-gate 26467c478bd9Sstevel@tonic-gate /* 26477c478bd9Sstevel@tonic-gate * Scrub and classify a CE. This function must not modify the 26487c478bd9Sstevel@tonic-gate * fault structure passed to it but instead should return the classification 26497c478bd9Sstevel@tonic-gate * information. 26507c478bd9Sstevel@tonic-gate */ 26517c478bd9Sstevel@tonic-gate 26527c478bd9Sstevel@tonic-gate static uchar_t 26537c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err_common(struct async_flt *ecc, boolean_t logout_tried) 26547c478bd9Sstevel@tonic-gate { 26557c478bd9Sstevel@tonic-gate uchar_t disp = CE_XDIAG_EXTALG; 26567c478bd9Sstevel@tonic-gate on_trap_data_t otd; 26577c478bd9Sstevel@tonic-gate uint64_t orig_err; 26587c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 26597c478bd9Sstevel@tonic-gate 26607c478bd9Sstevel@tonic-gate /* 26617c478bd9Sstevel@tonic-gate * Clear CEEN. CPU CE TL > 0 trap handling will already have done 26627c478bd9Sstevel@tonic-gate * this, but our other callers have not. Disable preemption to 26637c478bd9Sstevel@tonic-gate * avoid CPU migration so that we restore CEEN on the correct 26647c478bd9Sstevel@tonic-gate * cpu later. 26657c478bd9Sstevel@tonic-gate * 26667c478bd9Sstevel@tonic-gate * CEEN is cleared so that further CEs that our instruction and 26677c478bd9Sstevel@tonic-gate * data footprint induce do not cause use to either creep down 26687c478bd9Sstevel@tonic-gate * kernel stack to the point of overflow, or do so much CE 26697c478bd9Sstevel@tonic-gate * notification as to make little real forward progress. 26707c478bd9Sstevel@tonic-gate * 26717c478bd9Sstevel@tonic-gate * NCEEN must not be cleared. However it is possible that 26727c478bd9Sstevel@tonic-gate * our accesses to the flt_addr may provoke a bus error or timeout 26737c478bd9Sstevel@tonic-gate * if the offending address has just been unconfigured as part of 26747c478bd9Sstevel@tonic-gate * a DR action. So we must operate under on_trap protection. 26757c478bd9Sstevel@tonic-gate */ 26767c478bd9Sstevel@tonic-gate kpreempt_disable(); 26777c478bd9Sstevel@tonic-gate orig_err = get_error_enable(); 26787c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 26797c478bd9Sstevel@tonic-gate set_error_enable(orig_err & ~EN_REG_CEEN); 26807c478bd9Sstevel@tonic-gate 26817c478bd9Sstevel@tonic-gate /* 26827c478bd9Sstevel@tonic-gate * Our classification algorithm includes the line state before 26837c478bd9Sstevel@tonic-gate * the scrub; we'd like this captured after the detection and 26847c478bd9Sstevel@tonic-gate * before the algorithm below - the earlier the better. 26857c478bd9Sstevel@tonic-gate * 26867c478bd9Sstevel@tonic-gate * If we've come from a cpu CE trap then this info already exists 26877c478bd9Sstevel@tonic-gate * in the cpu logout area. 26887c478bd9Sstevel@tonic-gate * 26897c478bd9Sstevel@tonic-gate * For a CE detected by memscrub for which there was no trap 26907c478bd9Sstevel@tonic-gate * (running with CEEN off) cpu_log_and_clear_ce has called 26917c478bd9Sstevel@tonic-gate * cpu_ce_delayed_ec_logout to capture some cache data, and 26927c478bd9Sstevel@tonic-gate * marked the fault structure as incomplete as a flag to later 26937c478bd9Sstevel@tonic-gate * logging code. 26947c478bd9Sstevel@tonic-gate * 26957c478bd9Sstevel@tonic-gate * If called directly from an IO detected CE there has been 26967c478bd9Sstevel@tonic-gate * no line data capture. In this case we logout to the cpu logout 26977c478bd9Sstevel@tonic-gate * area - that's appropriate since it's the cpu cache data we need 26987c478bd9Sstevel@tonic-gate * for classification. We thus borrow the cpu logout area for a 26997c478bd9Sstevel@tonic-gate * short time, and cpu_ce_delayed_ec_logout will mark it as busy in 27007c478bd9Sstevel@tonic-gate * this time (we will invalidate it again below). 27017c478bd9Sstevel@tonic-gate * 27027c478bd9Sstevel@tonic-gate * If called from the partner check xcall handler then this cpu 27037c478bd9Sstevel@tonic-gate * (the partner) has not necessarily experienced a CE at this 27047c478bd9Sstevel@tonic-gate * address. But we want to capture line state before its scrub 27057c478bd9Sstevel@tonic-gate * attempt since we use that in our classification. 27067c478bd9Sstevel@tonic-gate */ 27077c478bd9Sstevel@tonic-gate if (logout_tried == B_FALSE) { 27087c478bd9Sstevel@tonic-gate if (!cpu_ce_delayed_ec_logout(ecc->flt_addr)) 27097c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_NOLOGOUT; 27107c478bd9Sstevel@tonic-gate } 27117c478bd9Sstevel@tonic-gate 27127c478bd9Sstevel@tonic-gate /* 27137c478bd9Sstevel@tonic-gate * Scrub memory, then check AFSR for errors. The AFAR we scrub may 27147c478bd9Sstevel@tonic-gate * no longer be valid (if DR'd since the initial event) so we 27157c478bd9Sstevel@tonic-gate * perform this scrub under on_trap protection. If this access is 27167c478bd9Sstevel@tonic-gate * ok then further accesses below will also be ok - DR cannot 27177c478bd9Sstevel@tonic-gate * proceed while this thread is active (preemption is disabled); 27187c478bd9Sstevel@tonic-gate * to be safe we'll nonetheless use on_trap again below. 27197c478bd9Sstevel@tonic-gate */ 27207c478bd9Sstevel@tonic-gate if (!on_trap(&otd, OT_DATA_ACCESS)) { 27217c478bd9Sstevel@tonic-gate cpu_scrubphys(ecc); 27227c478bd9Sstevel@tonic-gate } else { 27237c478bd9Sstevel@tonic-gate no_trap(); 27247c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 27257c478bd9Sstevel@tonic-gate set_error_enable(orig_err); 27267c478bd9Sstevel@tonic-gate kpreempt_enable(); 27277c478bd9Sstevel@tonic-gate return (disp); 27287c478bd9Sstevel@tonic-gate } 27297c478bd9Sstevel@tonic-gate no_trap(); 27307c478bd9Sstevel@tonic-gate 27317c478bd9Sstevel@tonic-gate /* 27327c478bd9Sstevel@tonic-gate * Did the casx read of the scrub log a CE that matches the AFAR? 27337c478bd9Sstevel@tonic-gate * Note that it's quite possible that the read sourced the data from 27347c478bd9Sstevel@tonic-gate * another cpu. 27357c478bd9Sstevel@tonic-gate */ 27367c478bd9Sstevel@tonic-gate if (clear_ecc(ecc)) 27377c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_CE1; 27387c478bd9Sstevel@tonic-gate 27397c478bd9Sstevel@tonic-gate /* 27407c478bd9Sstevel@tonic-gate * Read the data again. This time the read is very likely to 27417c478bd9Sstevel@tonic-gate * come from memory since the scrub induced a writeback to memory. 27427c478bd9Sstevel@tonic-gate */ 27437c478bd9Sstevel@tonic-gate if (!on_trap(&otd, OT_DATA_ACCESS)) { 27447c478bd9Sstevel@tonic-gate (void) lddphys(P2ALIGN(ecc->flt_addr, 8)); 27457c478bd9Sstevel@tonic-gate } else { 27467c478bd9Sstevel@tonic-gate no_trap(); 27477c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 27487c478bd9Sstevel@tonic-gate set_error_enable(orig_err); 27497c478bd9Sstevel@tonic-gate kpreempt_enable(); 27507c478bd9Sstevel@tonic-gate return (disp); 27517c478bd9Sstevel@tonic-gate } 27527c478bd9Sstevel@tonic-gate no_trap(); 27537c478bd9Sstevel@tonic-gate 27547c478bd9Sstevel@tonic-gate /* Did that read induce a CE that matches the AFAR? */ 27557c478bd9Sstevel@tonic-gate if (clear_ecc(ecc)) 27567c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_CE2; 27577c478bd9Sstevel@tonic-gate 27587c478bd9Sstevel@tonic-gate /* 27597c478bd9Sstevel@tonic-gate * Look at the logout information and record whether we found the 27607c478bd9Sstevel@tonic-gate * line in l2/l3 cache. For Panther we are interested in whether 27617c478bd9Sstevel@tonic-gate * we found it in either cache (it won't reside in both but 27627c478bd9Sstevel@tonic-gate * it is possible to read it that way given the moving target). 27637c478bd9Sstevel@tonic-gate */ 27647c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE(CPU) ? CPU_PRIVATE_PTR(CPU, chpr_cecc_logout) : NULL; 27657c478bd9Sstevel@tonic-gate if (!(disp & CE_XDIAG_NOLOGOUT) && clop && 27667c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar != LOGOUT_INVALID) { 27677c478bd9Sstevel@tonic-gate int hit, level; 27687c478bd9Sstevel@tonic-gate int state; 27697c478bd9Sstevel@tonic-gate int totalsize; 27707c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp; 27717c478bd9Sstevel@tonic-gate 27727c478bd9Sstevel@tonic-gate /* 27737c478bd9Sstevel@tonic-gate * If hit is nonzero then a match was found and hit will 27747c478bd9Sstevel@tonic-gate * be one greater than the index which hit. For Panther we 27757c478bd9Sstevel@tonic-gate * also need to pay attention to level to see which of l2$ or 27767c478bd9Sstevel@tonic-gate * l3$ it hit in. 27777c478bd9Sstevel@tonic-gate */ 27787c478bd9Sstevel@tonic-gate hit = cpu_matching_ecache_line(ecc->flt_addr, &clop->clo_data, 27797c478bd9Sstevel@tonic-gate 0, &level); 27807c478bd9Sstevel@tonic-gate 27817c478bd9Sstevel@tonic-gate if (hit) { 27827c478bd9Sstevel@tonic-gate --hit; 27837c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_AFARMATCH; 27847c478bd9Sstevel@tonic-gate 27857c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 27867c478bd9Sstevel@tonic-gate if (level == 2) 27877c478bd9Sstevel@tonic-gate ecp = &clop->clo_data.chd_l2_data[hit]; 27887c478bd9Sstevel@tonic-gate else 27897c478bd9Sstevel@tonic-gate ecp = &clop->clo_data.chd_ec_data[hit]; 27907c478bd9Sstevel@tonic-gate } else { 27917c478bd9Sstevel@tonic-gate ASSERT(level == 2); 27927c478bd9Sstevel@tonic-gate ecp = &clop->clo_data.chd_ec_data[hit]; 27937c478bd9Sstevel@tonic-gate } 27947c478bd9Sstevel@tonic-gate totalsize = cpunodes[CPU->cpu_id].ecache_size; 27957c478bd9Sstevel@tonic-gate state = cpu_ectag_pa_to_subblk_state(totalsize, 27967c478bd9Sstevel@tonic-gate ecc->flt_addr, ecp->ec_tag); 27977c478bd9Sstevel@tonic-gate 27987c478bd9Sstevel@tonic-gate /* 27997c478bd9Sstevel@tonic-gate * Cheetah variants use different state encodings - 28007c478bd9Sstevel@tonic-gate * the CH_ECSTATE_* defines vary depending on the 28017c478bd9Sstevel@tonic-gate * module we're compiled for. Translate into our 28027c478bd9Sstevel@tonic-gate * one true version. Conflate Owner-Shared state 28037c478bd9Sstevel@tonic-gate * of SSM mode with Owner as victimisation of such 28047c478bd9Sstevel@tonic-gate * lines may cause a writeback. 28057c478bd9Sstevel@tonic-gate */ 28067c478bd9Sstevel@tonic-gate switch (state) { 28077c478bd9Sstevel@tonic-gate case CH_ECSTATE_MOD: 28087c478bd9Sstevel@tonic-gate disp |= EC_STATE_M; 28097c478bd9Sstevel@tonic-gate break; 28107c478bd9Sstevel@tonic-gate 28117c478bd9Sstevel@tonic-gate case CH_ECSTATE_OWN: 28127c478bd9Sstevel@tonic-gate case CH_ECSTATE_OWS: 28137c478bd9Sstevel@tonic-gate disp |= EC_STATE_O; 28147c478bd9Sstevel@tonic-gate break; 28157c478bd9Sstevel@tonic-gate 28167c478bd9Sstevel@tonic-gate case CH_ECSTATE_EXL: 28177c478bd9Sstevel@tonic-gate disp |= EC_STATE_E; 28187c478bd9Sstevel@tonic-gate break; 28197c478bd9Sstevel@tonic-gate 28207c478bd9Sstevel@tonic-gate case CH_ECSTATE_SHR: 28217c478bd9Sstevel@tonic-gate disp |= EC_STATE_S; 28227c478bd9Sstevel@tonic-gate break; 28237c478bd9Sstevel@tonic-gate 28247c478bd9Sstevel@tonic-gate default: 28257c478bd9Sstevel@tonic-gate disp |= EC_STATE_I; 28267c478bd9Sstevel@tonic-gate break; 28277c478bd9Sstevel@tonic-gate } 28287c478bd9Sstevel@tonic-gate } 28297c478bd9Sstevel@tonic-gate 28307c478bd9Sstevel@tonic-gate /* 28317c478bd9Sstevel@tonic-gate * If we initiated the delayed logout then we are responsible 28327c478bd9Sstevel@tonic-gate * for invalidating the logout area. 28337c478bd9Sstevel@tonic-gate */ 28347c478bd9Sstevel@tonic-gate if (logout_tried == B_FALSE) { 28357c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 28367c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 28377c478bd9Sstevel@tonic-gate } 28387c478bd9Sstevel@tonic-gate } 28397c478bd9Sstevel@tonic-gate 28407c478bd9Sstevel@tonic-gate /* 28417c478bd9Sstevel@tonic-gate * Re-enable CEEN if we turned it off. 28427c478bd9Sstevel@tonic-gate */ 28437c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 28447c478bd9Sstevel@tonic-gate set_error_enable(orig_err); 28457c478bd9Sstevel@tonic-gate kpreempt_enable(); 28467c478bd9Sstevel@tonic-gate 28477c478bd9Sstevel@tonic-gate return (disp); 28487c478bd9Sstevel@tonic-gate } 28497c478bd9Sstevel@tonic-gate 28507c478bd9Sstevel@tonic-gate /* 28517c478bd9Sstevel@tonic-gate * Scrub a correctable memory error and collect data for classification 28527c478bd9Sstevel@tonic-gate * of CE type. This function is called in the detection path, ie tl0 handling 28537c478bd9Sstevel@tonic-gate * of a correctable error trap (cpus) or interrupt (IO) at high PIL. 28547c478bd9Sstevel@tonic-gate */ 28557c478bd9Sstevel@tonic-gate void 28567c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t logout_tried) 28577c478bd9Sstevel@tonic-gate { 28587c478bd9Sstevel@tonic-gate /* 28597c478bd9Sstevel@tonic-gate * Cheetah CE classification does not set any bits in flt_status. 28607c478bd9Sstevel@tonic-gate * Instead we will record classification datapoints in flt_disp. 28617c478bd9Sstevel@tonic-gate */ 28627c478bd9Sstevel@tonic-gate ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY); 28637c478bd9Sstevel@tonic-gate 28647c478bd9Sstevel@tonic-gate /* 28657c478bd9Sstevel@tonic-gate * To check if the error detected by IO is persistent, sticky or 28667c478bd9Sstevel@tonic-gate * intermittent. This is noticed by clear_ecc(). 28677c478bd9Sstevel@tonic-gate */ 28687c478bd9Sstevel@tonic-gate if (ecc->flt_status & ECC_IOBUS) 28697c478bd9Sstevel@tonic-gate ecc->flt_stat = C_AFSR_MEMORY; 28707c478bd9Sstevel@tonic-gate 28717c478bd9Sstevel@tonic-gate /* 28727c478bd9Sstevel@tonic-gate * Record information from this first part of the algorithm in 28737c478bd9Sstevel@tonic-gate * flt_disp. 28747c478bd9Sstevel@tonic-gate */ 28757c478bd9Sstevel@tonic-gate ecc->flt_disp = cpu_ce_scrub_mem_err_common(ecc, logout_tried); 28767c478bd9Sstevel@tonic-gate } 28777c478bd9Sstevel@tonic-gate 28787c478bd9Sstevel@tonic-gate /* 28797c478bd9Sstevel@tonic-gate * Select a partner to perform a further CE classification check from. 28807c478bd9Sstevel@tonic-gate * Must be called with kernel preemption disabled (to stop the cpu list 28817c478bd9Sstevel@tonic-gate * from changing). The detecting cpu we are partnering has cpuid 28827c478bd9Sstevel@tonic-gate * aflt->flt_inst; we might not be running on the detecting cpu. 28837c478bd9Sstevel@tonic-gate * 28847c478bd9Sstevel@tonic-gate * Restrict choice to active cpus in the same cpu partition as ourselves in 28857c478bd9Sstevel@tonic-gate * an effort to stop bad cpus in one partition causing other partitions to 28867c478bd9Sstevel@tonic-gate * perform excessive diagnostic activity. Actually since the errorq drain 28877c478bd9Sstevel@tonic-gate * is run from a softint most of the time and that is a global mechanism 28887c478bd9Sstevel@tonic-gate * this isolation is only partial. Return NULL if we fail to find a 28897c478bd9Sstevel@tonic-gate * suitable partner. 28907c478bd9Sstevel@tonic-gate * 28917c478bd9Sstevel@tonic-gate * We prefer a partner that is in a different latency group to ourselves as 28927c478bd9Sstevel@tonic-gate * we will share fewer datapaths. If such a partner is unavailable then 28937c478bd9Sstevel@tonic-gate * choose one in the same lgroup but prefer a different chip and only allow 28947c478bd9Sstevel@tonic-gate * a sibling core if flags includes PTNR_SIBLINGOK. If all else fails and 28957c478bd9Sstevel@tonic-gate * flags includes PTNR_SELFOK then permit selection of the original detector. 28967c478bd9Sstevel@tonic-gate * 28977c478bd9Sstevel@tonic-gate * We keep a cache of the last partner selected for a cpu, and we'll try to 28987c478bd9Sstevel@tonic-gate * use that previous partner if no more than cpu_ce_ptnr_cachetime_sec seconds 28997c478bd9Sstevel@tonic-gate * have passed since that selection was made. This provides the benefit 29007c478bd9Sstevel@tonic-gate * of the point-of-view of different partners over time but without 29017c478bd9Sstevel@tonic-gate * requiring frequent cpu list traversals. 29027c478bd9Sstevel@tonic-gate */ 29037c478bd9Sstevel@tonic-gate 29047c478bd9Sstevel@tonic-gate #define PTNR_SIBLINGOK 0x1 /* Allow selection of sibling core */ 29057c478bd9Sstevel@tonic-gate #define PTNR_SELFOK 0x2 /* Allow selection of cpu to "partner" itself */ 29067c478bd9Sstevel@tonic-gate 29077c478bd9Sstevel@tonic-gate static cpu_t * 29087c478bd9Sstevel@tonic-gate ce_ptnr_select(struct async_flt *aflt, int flags, int *typep) 29097c478bd9Sstevel@tonic-gate { 29107c478bd9Sstevel@tonic-gate cpu_t *sp, *dtcr, *ptnr, *locptnr, *sibptnr; 29117c478bd9Sstevel@tonic-gate hrtime_t lasttime, thistime; 29127c478bd9Sstevel@tonic-gate 29137c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt > 0 || getpil() >= DISP_LEVEL); 29147c478bd9Sstevel@tonic-gate 29157c478bd9Sstevel@tonic-gate dtcr = cpu[aflt->flt_inst]; 29167c478bd9Sstevel@tonic-gate 29177c478bd9Sstevel@tonic-gate /* 29187c478bd9Sstevel@tonic-gate * Short-circuit for the following cases: 29197c478bd9Sstevel@tonic-gate * . the dtcr is not flagged active 29207c478bd9Sstevel@tonic-gate * . there is just one cpu present 29217c478bd9Sstevel@tonic-gate * . the detector has disappeared 29227c478bd9Sstevel@tonic-gate * . we were given a bad flt_inst cpuid; this should not happen 29237c478bd9Sstevel@tonic-gate * (eg PCI code now fills flt_inst) but if it does it is no 29247c478bd9Sstevel@tonic-gate * reason to panic. 29257c478bd9Sstevel@tonic-gate * . there is just one cpu left online in the cpu partition 29267c478bd9Sstevel@tonic-gate * 29277c478bd9Sstevel@tonic-gate * If we return NULL after this point then we do not update the 29287c478bd9Sstevel@tonic-gate * chpr_ceptnr_seltime which will cause us to perform a full lookup 29297c478bd9Sstevel@tonic-gate * again next time; this is the case where the only other cpu online 29307c478bd9Sstevel@tonic-gate * in the detector's partition is on the same chip as the detector 29317c478bd9Sstevel@tonic-gate * and since CEEN re-enable is throttled even that case should not 29327c478bd9Sstevel@tonic-gate * hurt performance. 29337c478bd9Sstevel@tonic-gate */ 29347c478bd9Sstevel@tonic-gate if (dtcr == NULL || !cpu_flagged_active(dtcr->cpu_flags)) { 29357c478bd9Sstevel@tonic-gate return (NULL); 29367c478bd9Sstevel@tonic-gate } 29377c478bd9Sstevel@tonic-gate if (ncpus == 1 || dtcr->cpu_part->cp_ncpus == 1) { 29387c478bd9Sstevel@tonic-gate if (flags & PTNR_SELFOK) { 29397c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SELF; 29407c478bd9Sstevel@tonic-gate return (dtcr); 29417c478bd9Sstevel@tonic-gate } else { 29427c478bd9Sstevel@tonic-gate return (NULL); 29437c478bd9Sstevel@tonic-gate } 29447c478bd9Sstevel@tonic-gate } 29457c478bd9Sstevel@tonic-gate 29467c478bd9Sstevel@tonic-gate thistime = gethrtime(); 29477c478bd9Sstevel@tonic-gate lasttime = CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime); 29487c478bd9Sstevel@tonic-gate 29497c478bd9Sstevel@tonic-gate /* 29507c478bd9Sstevel@tonic-gate * Select a starting point. 29517c478bd9Sstevel@tonic-gate */ 29527c478bd9Sstevel@tonic-gate if (!lasttime) { 29537c478bd9Sstevel@tonic-gate /* 29547c478bd9Sstevel@tonic-gate * We've never selected a partner for this detector before. 29557c478bd9Sstevel@tonic-gate * Start the scan at the next online cpu in the same cpu 29567c478bd9Sstevel@tonic-gate * partition. 29577c478bd9Sstevel@tonic-gate */ 29587c478bd9Sstevel@tonic-gate sp = dtcr->cpu_next_part; 29597c478bd9Sstevel@tonic-gate } else if (thistime - lasttime < cpu_ce_ptnr_cachetime_sec * NANOSEC) { 29607c478bd9Sstevel@tonic-gate /* 29617c478bd9Sstevel@tonic-gate * Our last selection has not aged yet. If this partner: 29627c478bd9Sstevel@tonic-gate * . is still a valid cpu, 29637c478bd9Sstevel@tonic-gate * . is still in the same partition as the detector 29647c478bd9Sstevel@tonic-gate * . is still marked active 29657c478bd9Sstevel@tonic-gate * . satisfies the 'flags' argument criteria 29667c478bd9Sstevel@tonic-gate * then select it again without updating the timestamp. 29677c478bd9Sstevel@tonic-gate */ 29687c478bd9Sstevel@tonic-gate sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 29697c478bd9Sstevel@tonic-gate if (sp == NULL || sp->cpu_part != dtcr->cpu_part || 29707c478bd9Sstevel@tonic-gate !cpu_flagged_active(sp->cpu_flags) || 29717c478bd9Sstevel@tonic-gate (sp == dtcr && !(flags & PTNR_SELFOK)) || 29727c478bd9Sstevel@tonic-gate (sp->cpu_chip->chip_id == dtcr->cpu_chip->chip_id && 29737c478bd9Sstevel@tonic-gate !(flags & PTNR_SIBLINGOK))) { 29747c478bd9Sstevel@tonic-gate sp = dtcr->cpu_next_part; 29757c478bd9Sstevel@tonic-gate } else { 29767c478bd9Sstevel@tonic-gate if (sp->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 29777c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_REMOTE; 29787c478bd9Sstevel@tonic-gate } else if (sp == dtcr) { 29797c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SELF; 29807c478bd9Sstevel@tonic-gate } else if (sp->cpu_chip->chip_id == 29817c478bd9Sstevel@tonic-gate dtcr->cpu_chip->chip_id) { 29827c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SIBLING; 29837c478bd9Sstevel@tonic-gate } else { 29847c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_LOCAL; 29857c478bd9Sstevel@tonic-gate } 29867c478bd9Sstevel@tonic-gate return (sp); 29877c478bd9Sstevel@tonic-gate } 29887c478bd9Sstevel@tonic-gate } else { 29897c478bd9Sstevel@tonic-gate /* 29907c478bd9Sstevel@tonic-gate * Our last selection has aged. If it is nonetheless still a 29917c478bd9Sstevel@tonic-gate * valid cpu then start the scan at the next cpu in the 29927c478bd9Sstevel@tonic-gate * partition after our last partner. If the last selection 29937c478bd9Sstevel@tonic-gate * is no longer a valid cpu then go with our default. In 29947c478bd9Sstevel@tonic-gate * this way we slowly cycle through possible partners to 29957c478bd9Sstevel@tonic-gate * obtain multiple viewpoints over time. 29967c478bd9Sstevel@tonic-gate */ 29977c478bd9Sstevel@tonic-gate sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 29987c478bd9Sstevel@tonic-gate if (sp == NULL) { 29997c478bd9Sstevel@tonic-gate sp = dtcr->cpu_next_part; 30007c478bd9Sstevel@tonic-gate } else { 30017c478bd9Sstevel@tonic-gate sp = sp->cpu_next_part; /* may be dtcr */ 30027c478bd9Sstevel@tonic-gate if (sp->cpu_part != dtcr->cpu_part) 30037c478bd9Sstevel@tonic-gate sp = dtcr; 30047c478bd9Sstevel@tonic-gate } 30057c478bd9Sstevel@tonic-gate } 30067c478bd9Sstevel@tonic-gate 30077c478bd9Sstevel@tonic-gate /* 30087c478bd9Sstevel@tonic-gate * We have a proposed starting point for our search, but if this 30097c478bd9Sstevel@tonic-gate * cpu is offline then its cpu_next_part will point to itself 30107c478bd9Sstevel@tonic-gate * so we can't use that to iterate over cpus in this partition in 30117c478bd9Sstevel@tonic-gate * the loop below. We still want to avoid iterating over cpus not 30127c478bd9Sstevel@tonic-gate * in our partition, so in the case that our starting point is offline 30137c478bd9Sstevel@tonic-gate * we will repoint it to be the detector itself; and if the detector 30147c478bd9Sstevel@tonic-gate * happens to be offline we'll return NULL from the following loop. 30157c478bd9Sstevel@tonic-gate */ 30167c478bd9Sstevel@tonic-gate if (!cpu_flagged_active(sp->cpu_flags)) { 30177c478bd9Sstevel@tonic-gate sp = dtcr; 30187c478bd9Sstevel@tonic-gate } 30197c478bd9Sstevel@tonic-gate 30207c478bd9Sstevel@tonic-gate ptnr = sp; 30217c478bd9Sstevel@tonic-gate locptnr = NULL; 30227c478bd9Sstevel@tonic-gate sibptnr = NULL; 30237c478bd9Sstevel@tonic-gate do { 30247c478bd9Sstevel@tonic-gate if (ptnr == dtcr || !cpu_flagged_active(ptnr->cpu_flags)) 30257c478bd9Sstevel@tonic-gate continue; 30267c478bd9Sstevel@tonic-gate if (ptnr->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 30277c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = ptnr->cpu_id; 30287c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 30297c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_REMOTE; 30307c478bd9Sstevel@tonic-gate return (ptnr); 30317c478bd9Sstevel@tonic-gate } 30327c478bd9Sstevel@tonic-gate if (ptnr->cpu_chip->chip_id == dtcr->cpu_chip->chip_id) { 30337c478bd9Sstevel@tonic-gate if (sibptnr == NULL) 30347c478bd9Sstevel@tonic-gate sibptnr = ptnr; 30357c478bd9Sstevel@tonic-gate continue; 30367c478bd9Sstevel@tonic-gate } 30377c478bd9Sstevel@tonic-gate if (locptnr == NULL) 30387c478bd9Sstevel@tonic-gate locptnr = ptnr; 30397c478bd9Sstevel@tonic-gate } while ((ptnr = ptnr->cpu_next_part) != sp); 30407c478bd9Sstevel@tonic-gate 30417c478bd9Sstevel@tonic-gate /* 30427c478bd9Sstevel@tonic-gate * A foreign partner has already been returned if one was available. 30437c478bd9Sstevel@tonic-gate * 30447c478bd9Sstevel@tonic-gate * If locptnr is not NULL it is a cpu in the same lgroup as the 30457c478bd9Sstevel@tonic-gate * detector, is active, and is not a sibling of the detector. 30467c478bd9Sstevel@tonic-gate * 30477c478bd9Sstevel@tonic-gate * If sibptnr is not NULL it is a sibling of the detector, and is 30487c478bd9Sstevel@tonic-gate * active. 30497c478bd9Sstevel@tonic-gate * 30507c478bd9Sstevel@tonic-gate * If we have to resort to using the detector itself we have already 30517c478bd9Sstevel@tonic-gate * checked that it is active. 30527c478bd9Sstevel@tonic-gate */ 30537c478bd9Sstevel@tonic-gate if (locptnr) { 30547c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = locptnr->cpu_id; 30557c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 30567c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_LOCAL; 30577c478bd9Sstevel@tonic-gate return (locptnr); 30587c478bd9Sstevel@tonic-gate } else if (sibptnr && flags & PTNR_SIBLINGOK) { 30597c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = sibptnr->cpu_id; 30607c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 30617c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SIBLING; 30627c478bd9Sstevel@tonic-gate return (sibptnr); 30637c478bd9Sstevel@tonic-gate } else if (flags & PTNR_SELFOK) { 30647c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = dtcr->cpu_id; 30657c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 30667c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SELF; 30677c478bd9Sstevel@tonic-gate return (dtcr); 30687c478bd9Sstevel@tonic-gate } 30697c478bd9Sstevel@tonic-gate 30707c478bd9Sstevel@tonic-gate return (NULL); 30717c478bd9Sstevel@tonic-gate } 30727c478bd9Sstevel@tonic-gate 30737c478bd9Sstevel@tonic-gate /* 30747c478bd9Sstevel@tonic-gate * Cross call handler that is requested to run on the designated partner of 30757c478bd9Sstevel@tonic-gate * a cpu that experienced a possibly sticky or possibly persistnet CE. 30767c478bd9Sstevel@tonic-gate */ 30777c478bd9Sstevel@tonic-gate static void 30787c478bd9Sstevel@tonic-gate ce_ptnrchk_xc(struct async_flt *aflt, uchar_t *dispp) 30797c478bd9Sstevel@tonic-gate { 30807c478bd9Sstevel@tonic-gate *dispp = cpu_ce_scrub_mem_err_common(aflt, B_FALSE); 30817c478bd9Sstevel@tonic-gate } 30827c478bd9Sstevel@tonic-gate 30837c478bd9Sstevel@tonic-gate /* 30847c478bd9Sstevel@tonic-gate * The associated errorqs are never destroyed so we do not need to deal with 30857c478bd9Sstevel@tonic-gate * them disappearing before this timeout fires. If the affected memory 30867c478bd9Sstevel@tonic-gate * has been DR'd out since the original event the scrub algrithm will catch 30877c478bd9Sstevel@tonic-gate * any errors and return null disposition info. If the original detecting 30887c478bd9Sstevel@tonic-gate * cpu has been DR'd out then ereport detector info will not be able to 30897c478bd9Sstevel@tonic-gate * lookup CPU type; with a small timeout this is unlikely. 30907c478bd9Sstevel@tonic-gate */ 30917c478bd9Sstevel@tonic-gate static void 30927c478bd9Sstevel@tonic-gate ce_lkychk_cb(ce_lkychk_cb_t *cbarg) 30937c478bd9Sstevel@tonic-gate { 30947c478bd9Sstevel@tonic-gate struct async_flt *aflt = cbarg->lkycb_aflt; 30957c478bd9Sstevel@tonic-gate uchar_t disp; 30967c478bd9Sstevel@tonic-gate cpu_t *cp; 30977c478bd9Sstevel@tonic-gate int ptnrtype; 30987c478bd9Sstevel@tonic-gate 30997c478bd9Sstevel@tonic-gate kpreempt_disable(); 31007c478bd9Sstevel@tonic-gate if (cp = ce_ptnr_select(aflt, PTNR_SIBLINGOK | PTNR_SELFOK, 31017c478bd9Sstevel@tonic-gate &ptnrtype)) { 31027c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, (uint64_t)aflt, 31037c478bd9Sstevel@tonic-gate (uint64_t)&disp); 31047c478bd9Sstevel@tonic-gate CE_XDIAG_SETLKYINFO(aflt->flt_disp, disp); 31057c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 31067c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 31077c478bd9Sstevel@tonic-gate } else { 31087c478bd9Sstevel@tonic-gate ce_xdiag_lkydrops++; 31097c478bd9Sstevel@tonic-gate if (ncpus > 1) 31107c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 31117c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_NOPTNR); 31127c478bd9Sstevel@tonic-gate } 31137c478bd9Sstevel@tonic-gate kpreempt_enable(); 31147c478bd9Sstevel@tonic-gate 31157c478bd9Sstevel@tonic-gate errorq_commit(cbarg->lkycb_eqp, cbarg->lkycb_eqep, ERRORQ_ASYNC); 31167c478bd9Sstevel@tonic-gate kmem_free(cbarg, sizeof (ce_lkychk_cb_t)); 31177c478bd9Sstevel@tonic-gate } 31187c478bd9Sstevel@tonic-gate 31197c478bd9Sstevel@tonic-gate /* 31207c478bd9Sstevel@tonic-gate * Called from errorq drain code when processing a CE error, both from 31217c478bd9Sstevel@tonic-gate * CPU and PCI drain functions. Decide what further classification actions, 31227c478bd9Sstevel@tonic-gate * if any, we will perform. Perform immediate actions now, and schedule 31237c478bd9Sstevel@tonic-gate * delayed actions as required. Note that we are no longer necessarily running 31247c478bd9Sstevel@tonic-gate * on the detecting cpu, and that the async_flt structure will not persist on 31257c478bd9Sstevel@tonic-gate * return from this function. 31267c478bd9Sstevel@tonic-gate * 31277c478bd9Sstevel@tonic-gate * Calls to this function should aim to be self-throtlling in some way. With 31287c478bd9Sstevel@tonic-gate * the delayed re-enable of CEEN the absolute rate of calls should not 31297c478bd9Sstevel@tonic-gate * be excessive. Callers should also avoid performing in-depth classification 31307c478bd9Sstevel@tonic-gate * for events in pages that are already known to be suspect. 31317c478bd9Sstevel@tonic-gate * 31327c478bd9Sstevel@tonic-gate * We return nonzero to indicate that the event has been copied and 31337c478bd9Sstevel@tonic-gate * recirculated for further testing. The caller should not log the event 31347c478bd9Sstevel@tonic-gate * in this case - it will be logged when further test results are available. 31357c478bd9Sstevel@tonic-gate * 31367c478bd9Sstevel@tonic-gate * Our possible contexts are that of errorq_drain: below lock level or from 31377c478bd9Sstevel@tonic-gate * panic context. We can assume that the cpu we are running on is online. 31387c478bd9Sstevel@tonic-gate */ 31397c478bd9Sstevel@tonic-gate 31407c478bd9Sstevel@tonic-gate 31417c478bd9Sstevel@tonic-gate #ifdef DEBUG 31427c478bd9Sstevel@tonic-gate static int ce_xdiag_forceaction; 31437c478bd9Sstevel@tonic-gate #endif 31447c478bd9Sstevel@tonic-gate 31457c478bd9Sstevel@tonic-gate int 31467c478bd9Sstevel@tonic-gate ce_scrub_xdiag_recirc(struct async_flt *aflt, errorq_t *eqp, 31477c478bd9Sstevel@tonic-gate errorq_elem_t *eqep, size_t afltoffset) 31487c478bd9Sstevel@tonic-gate { 31497c478bd9Sstevel@tonic-gate ce_dispact_t dispact, action; 31507c478bd9Sstevel@tonic-gate cpu_t *cp; 31517c478bd9Sstevel@tonic-gate uchar_t dtcrinfo, disp; 31527c478bd9Sstevel@tonic-gate int ptnrtype; 31537c478bd9Sstevel@tonic-gate 31547c478bd9Sstevel@tonic-gate if (!ce_disp_inited || panicstr || ce_xdiag_off) { 31557c478bd9Sstevel@tonic-gate ce_xdiag_drops++; 31567c478bd9Sstevel@tonic-gate return (0); 31577c478bd9Sstevel@tonic-gate } else if (!aflt->flt_in_memory) { 31587c478bd9Sstevel@tonic-gate ce_xdiag_drops++; 31597c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOTMEM); 31607c478bd9Sstevel@tonic-gate return (0); 31617c478bd9Sstevel@tonic-gate } 31627c478bd9Sstevel@tonic-gate 31637c478bd9Sstevel@tonic-gate dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 31647c478bd9Sstevel@tonic-gate 31657c478bd9Sstevel@tonic-gate /* 31667c478bd9Sstevel@tonic-gate * Some correctable events are not scrubbed/classified, such as those 31677c478bd9Sstevel@tonic-gate * noticed at the tail of cpu_deferred_error. So if there is no 31687c478bd9Sstevel@tonic-gate * initial detector classification go no further. 31697c478bd9Sstevel@tonic-gate */ 31707c478bd9Sstevel@tonic-gate if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) { 31717c478bd9Sstevel@tonic-gate ce_xdiag_drops++; 31727c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOSCRUB); 31737c478bd9Sstevel@tonic-gate return (0); 31747c478bd9Sstevel@tonic-gate } 31757c478bd9Sstevel@tonic-gate 31767c478bd9Sstevel@tonic-gate dispact = CE_DISPACT(ce_disp_table, 31777c478bd9Sstevel@tonic-gate CE_XDIAG_AFARMATCHED(dtcrinfo), 31787c478bd9Sstevel@tonic-gate CE_XDIAG_STATE(dtcrinfo), 31797c478bd9Sstevel@tonic-gate CE_XDIAG_CE1SEEN(dtcrinfo), 31807c478bd9Sstevel@tonic-gate CE_XDIAG_CE2SEEN(dtcrinfo)); 31817c478bd9Sstevel@tonic-gate 31827c478bd9Sstevel@tonic-gate 31837c478bd9Sstevel@tonic-gate action = CE_ACT(dispact); /* bad lookup caught below */ 31847c478bd9Sstevel@tonic-gate #ifdef DEBUG 31857c478bd9Sstevel@tonic-gate if (ce_xdiag_forceaction != 0) 31867c478bd9Sstevel@tonic-gate action = ce_xdiag_forceaction; 31877c478bd9Sstevel@tonic-gate #endif 31887c478bd9Sstevel@tonic-gate 31897c478bd9Sstevel@tonic-gate switch (action) { 31907c478bd9Sstevel@tonic-gate case CE_ACT_LKYCHK: { 31917c478bd9Sstevel@tonic-gate caddr_t ndata; 31927c478bd9Sstevel@tonic-gate errorq_elem_t *neqep; 31937c478bd9Sstevel@tonic-gate struct async_flt *ecc; 31947c478bd9Sstevel@tonic-gate ce_lkychk_cb_t *cbargp; 31957c478bd9Sstevel@tonic-gate 31967c478bd9Sstevel@tonic-gate if ((ndata = errorq_elem_dup(eqp, eqep, &neqep)) == NULL) { 31977c478bd9Sstevel@tonic-gate ce_xdiag_lkydrops++; 31987c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 31997c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_DUPFAIL); 32007c478bd9Sstevel@tonic-gate break; 32017c478bd9Sstevel@tonic-gate } 32027c478bd9Sstevel@tonic-gate ecc = (struct async_flt *)(ndata + afltoffset); 32037c478bd9Sstevel@tonic-gate 32047c478bd9Sstevel@tonic-gate ASSERT(ecc->flt_class == CPU_FAULT || 32057c478bd9Sstevel@tonic-gate ecc->flt_class == BUS_FAULT); 32067c478bd9Sstevel@tonic-gate ecc->flt_class = (ecc->flt_class == CPU_FAULT) ? 32077c478bd9Sstevel@tonic-gate RECIRC_CPU_FAULT : RECIRC_BUS_FAULT; 32087c478bd9Sstevel@tonic-gate 32097c478bd9Sstevel@tonic-gate cbargp = kmem_alloc(sizeof (ce_lkychk_cb_t), KM_SLEEP); 32107c478bd9Sstevel@tonic-gate cbargp->lkycb_aflt = ecc; 32117c478bd9Sstevel@tonic-gate cbargp->lkycb_eqp = eqp; 32127c478bd9Sstevel@tonic-gate cbargp->lkycb_eqep = neqep; 32137c478bd9Sstevel@tonic-gate 32147c478bd9Sstevel@tonic-gate (void) timeout((void (*)(void *))ce_lkychk_cb, 32157c478bd9Sstevel@tonic-gate (void *)cbargp, drv_usectohz(cpu_ce_lkychk_timeout_usec)); 32167c478bd9Sstevel@tonic-gate return (1); 32177c478bd9Sstevel@tonic-gate } 32187c478bd9Sstevel@tonic-gate 32197c478bd9Sstevel@tonic-gate case CE_ACT_PTNRCHK: 32207c478bd9Sstevel@tonic-gate kpreempt_disable(); /* stop cpu list changing */ 32217c478bd9Sstevel@tonic-gate if ((cp = ce_ptnr_select(aflt, 0, &ptnrtype)) != NULL) { 32227c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, 32237c478bd9Sstevel@tonic-gate (uint64_t)aflt, (uint64_t)&disp); 32247c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRINFO(aflt->flt_disp, disp); 32257c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 32267c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 32277c478bd9Sstevel@tonic-gate } else if (ncpus > 1) { 32287c478bd9Sstevel@tonic-gate ce_xdiag_ptnrdrops++; 32297c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 32307c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_NOPTNR); 32317c478bd9Sstevel@tonic-gate } else { 32327c478bd9Sstevel@tonic-gate ce_xdiag_ptnrdrops++; 32337c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 32347c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_UNIPROC); 32357c478bd9Sstevel@tonic-gate } 32367c478bd9Sstevel@tonic-gate kpreempt_enable(); 32377c478bd9Sstevel@tonic-gate break; 32387c478bd9Sstevel@tonic-gate 32397c478bd9Sstevel@tonic-gate case CE_ACT_DONE: 32407c478bd9Sstevel@tonic-gate break; 32417c478bd9Sstevel@tonic-gate 32427c478bd9Sstevel@tonic-gate case CE_ACT(CE_DISP_BAD): 32437c478bd9Sstevel@tonic-gate default: 32447c478bd9Sstevel@tonic-gate #ifdef DEBUG 32457c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "ce_scrub_post: Bad action '%d'", action); 32467c478bd9Sstevel@tonic-gate #endif 32477c478bd9Sstevel@tonic-gate ce_xdiag_bad++; 32487c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_ACTBAD); 32497c478bd9Sstevel@tonic-gate break; 32507c478bd9Sstevel@tonic-gate } 32517c478bd9Sstevel@tonic-gate 32527c478bd9Sstevel@tonic-gate return (0); 32537c478bd9Sstevel@tonic-gate } 32547c478bd9Sstevel@tonic-gate 32557c478bd9Sstevel@tonic-gate /* 32567c478bd9Sstevel@tonic-gate * We route all errors through a single switch statement. 32577c478bd9Sstevel@tonic-gate */ 32587c478bd9Sstevel@tonic-gate void 32597c478bd9Sstevel@tonic-gate cpu_ue_log_err(struct async_flt *aflt) 32607c478bd9Sstevel@tonic-gate { 32617c478bd9Sstevel@tonic-gate switch (aflt->flt_class) { 32627c478bd9Sstevel@tonic-gate case CPU_FAULT: 32637c478bd9Sstevel@tonic-gate cpu_ereport_init(aflt); 32647c478bd9Sstevel@tonic-gate if (cpu_async_log_err(aflt, NULL)) 32657c478bd9Sstevel@tonic-gate cpu_ereport_post(aflt); 32667c478bd9Sstevel@tonic-gate break; 32677c478bd9Sstevel@tonic-gate 32687c478bd9Sstevel@tonic-gate case BUS_FAULT: 32697c478bd9Sstevel@tonic-gate bus_async_log_err(aflt); 32707c478bd9Sstevel@tonic-gate break; 32717c478bd9Sstevel@tonic-gate 32727c478bd9Sstevel@tonic-gate default: 32737c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "discarding async error %p with invalid " 32747c478bd9Sstevel@tonic-gate "fault class (0x%x)", (void *)aflt, aflt->flt_class); 32757c478bd9Sstevel@tonic-gate return; 32767c478bd9Sstevel@tonic-gate } 32777c478bd9Sstevel@tonic-gate } 32787c478bd9Sstevel@tonic-gate 32797c478bd9Sstevel@tonic-gate /* 32807c478bd9Sstevel@tonic-gate * Routine for panic hook callback from panic_idle(). 32817c478bd9Sstevel@tonic-gate */ 32827c478bd9Sstevel@tonic-gate void 32837c478bd9Sstevel@tonic-gate cpu_async_panic_callb(void) 32847c478bd9Sstevel@tonic-gate { 32857c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 32867c478bd9Sstevel@tonic-gate struct async_flt *aflt; 32877c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 32887c478bd9Sstevel@tonic-gate uint64_t afsr_errs; 32897c478bd9Sstevel@tonic-gate 32907c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 32917c478bd9Sstevel@tonic-gate 32927c478bd9Sstevel@tonic-gate afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 32937c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr_ext & C_AFSR_EXT_L3_ERRS); 32947c478bd9Sstevel@tonic-gate 32957c478bd9Sstevel@tonic-gate if (afsr_errs) { 32967c478bd9Sstevel@tonic-gate 32977c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 32987c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 32997c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 33007c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 33017c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 33027c478bd9Sstevel@tonic-gate aflt->flt_stat = cpu_error_regs.afsr; 33037c478bd9Sstevel@tonic-gate aflt->flt_addr = cpu_error_regs.afar; 33047c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 33057c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 33067c478bd9Sstevel@tonic-gate aflt->flt_priv = ((cpu_error_regs.afsr & C_AFSR_PRIV) != 0); 33077c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 33087c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = cpu_error_regs.afsr_ext; 33097c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = afsr_errs; 33107c478bd9Sstevel@tonic-gate #if defined(SERRANO) 33117c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 33127c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 33137c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, NULL, afsr_errs, NULL); 33147c478bd9Sstevel@tonic-gate } 33157c478bd9Sstevel@tonic-gate } 33167c478bd9Sstevel@tonic-gate 33177c478bd9Sstevel@tonic-gate /* 33187c478bd9Sstevel@tonic-gate * Routine to convert a syndrome into a syndrome code. 33197c478bd9Sstevel@tonic-gate */ 33207c478bd9Sstevel@tonic-gate static int 33217c478bd9Sstevel@tonic-gate synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit) 33227c478bd9Sstevel@tonic-gate { 33237c478bd9Sstevel@tonic-gate if (synd_status == AFLT_STAT_INVALID) 33247c478bd9Sstevel@tonic-gate return (-1); 33257c478bd9Sstevel@tonic-gate 33267c478bd9Sstevel@tonic-gate /* 33277c478bd9Sstevel@tonic-gate * Use the syndrome to index the appropriate syndrome table, 33287c478bd9Sstevel@tonic-gate * to get the code indicating which bit(s) is(are) bad. 33297c478bd9Sstevel@tonic-gate */ 33307c478bd9Sstevel@tonic-gate if (afsr_bit & 33317c478bd9Sstevel@tonic-gate (C_AFSR_MSYND_ERRS | C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 33327c478bd9Sstevel@tonic-gate if (afsr_bit & C_AFSR_MSYND_ERRS) { 33337c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 33347c478bd9Sstevel@tonic-gate if ((synd == 0) || (synd >= BSYND_TBL_SIZE)) 33357c478bd9Sstevel@tonic-gate return (-1); 33367c478bd9Sstevel@tonic-gate else 33377c478bd9Sstevel@tonic-gate return (BPAR0 + synd); 33387c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 33397c478bd9Sstevel@tonic-gate if ((synd == 0) || (synd >= MSYND_TBL_SIZE)) 33407c478bd9Sstevel@tonic-gate return (-1); 33417c478bd9Sstevel@tonic-gate else 33427c478bd9Sstevel@tonic-gate return (mtag_syndrome_tab[synd]); 33437c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 33447c478bd9Sstevel@tonic-gate } else { 33457c478bd9Sstevel@tonic-gate if ((synd == 0) || (synd >= ESYND_TBL_SIZE)) 33467c478bd9Sstevel@tonic-gate return (-1); 33477c478bd9Sstevel@tonic-gate else 33487c478bd9Sstevel@tonic-gate return (ecc_syndrome_tab[synd]); 33497c478bd9Sstevel@tonic-gate } 33507c478bd9Sstevel@tonic-gate } else { 33517c478bd9Sstevel@tonic-gate return (-1); 33527c478bd9Sstevel@tonic-gate } 33537c478bd9Sstevel@tonic-gate } 33547c478bd9Sstevel@tonic-gate 3355d00f0155Sayznaga int 3356d00f0155Sayznaga cpu_get_mem_sid(char *unum, char *buf, int buflen, int *lenp) 3357d00f0155Sayznaga { 3358d00f0155Sayznaga if (&plat_get_mem_sid) 3359d00f0155Sayznaga return (plat_get_mem_sid(unum, buf, buflen, lenp)); 3360d00f0155Sayznaga else 3361d00f0155Sayznaga return (ENOTSUP); 3362d00f0155Sayznaga } 3363d00f0155Sayznaga 3364d00f0155Sayznaga int 3365d00f0155Sayznaga cpu_get_mem_offset(uint64_t flt_addr, uint64_t *offp) 3366d00f0155Sayznaga { 3367d00f0155Sayznaga if (&plat_get_mem_offset) 3368d00f0155Sayznaga return (plat_get_mem_offset(flt_addr, offp)); 3369d00f0155Sayznaga else 3370d00f0155Sayznaga return (ENOTSUP); 3371d00f0155Sayznaga } 3372d00f0155Sayznaga 3373d00f0155Sayznaga int 3374d00f0155Sayznaga cpu_get_mem_addr(char *unum, char *sid, uint64_t offset, uint64_t *addrp) 3375d00f0155Sayznaga { 3376d00f0155Sayznaga if (&plat_get_mem_addr) 3377d00f0155Sayznaga return (plat_get_mem_addr(unum, sid, offset, addrp)); 3378d00f0155Sayznaga else 3379d00f0155Sayznaga return (ENOTSUP); 3380d00f0155Sayznaga } 3381d00f0155Sayznaga 33827c478bd9Sstevel@tonic-gate /* 33837c478bd9Sstevel@tonic-gate * Routine to return a string identifying the physical name 33847c478bd9Sstevel@tonic-gate * associated with a memory/cache error. 33857c478bd9Sstevel@tonic-gate */ 33867c478bd9Sstevel@tonic-gate int 33877c478bd9Sstevel@tonic-gate cpu_get_mem_unum(int synd_status, ushort_t flt_synd, uint64_t flt_stat, 33887c478bd9Sstevel@tonic-gate uint64_t flt_addr, int flt_bus_id, int flt_in_memory, 33897c478bd9Sstevel@tonic-gate ushort_t flt_status, char *buf, int buflen, int *lenp) 33907c478bd9Sstevel@tonic-gate { 33917c478bd9Sstevel@tonic-gate int synd_code; 33927c478bd9Sstevel@tonic-gate int ret; 33937c478bd9Sstevel@tonic-gate 33947c478bd9Sstevel@tonic-gate /* 33957c478bd9Sstevel@tonic-gate * An AFSR of -1 defaults to a memory syndrome. 33967c478bd9Sstevel@tonic-gate */ 33977c478bd9Sstevel@tonic-gate if (flt_stat == (uint64_t)-1) 33987c478bd9Sstevel@tonic-gate flt_stat = C_AFSR_CE; 33997c478bd9Sstevel@tonic-gate 34007c478bd9Sstevel@tonic-gate synd_code = synd_to_synd_code(synd_status, flt_synd, flt_stat); 34017c478bd9Sstevel@tonic-gate 34027c478bd9Sstevel@tonic-gate /* 34037c478bd9Sstevel@tonic-gate * Syndrome code must be either a single-bit error code 34047c478bd9Sstevel@tonic-gate * (0...143) or -1 for unum lookup. 34057c478bd9Sstevel@tonic-gate */ 34067c478bd9Sstevel@tonic-gate if (synd_code < 0 || synd_code >= M2) 34077c478bd9Sstevel@tonic-gate synd_code = -1; 34087c478bd9Sstevel@tonic-gate if (&plat_get_mem_unum) { 34097c478bd9Sstevel@tonic-gate if ((ret = plat_get_mem_unum(synd_code, flt_addr, flt_bus_id, 34107c478bd9Sstevel@tonic-gate flt_in_memory, flt_status, buf, buflen, lenp)) != 0) { 34117c478bd9Sstevel@tonic-gate buf[0] = '\0'; 34127c478bd9Sstevel@tonic-gate *lenp = 0; 34137c478bd9Sstevel@tonic-gate } 34147c478bd9Sstevel@tonic-gate 34157c478bd9Sstevel@tonic-gate return (ret); 34167c478bd9Sstevel@tonic-gate } 34177c478bd9Sstevel@tonic-gate 34187c478bd9Sstevel@tonic-gate return (ENOTSUP); 34197c478bd9Sstevel@tonic-gate } 34207c478bd9Sstevel@tonic-gate 34217c478bd9Sstevel@tonic-gate /* 34227c478bd9Sstevel@tonic-gate * Wrapper for cpu_get_mem_unum() routine that takes an 34237c478bd9Sstevel@tonic-gate * async_flt struct rather than explicit arguments. 34247c478bd9Sstevel@tonic-gate */ 34257c478bd9Sstevel@tonic-gate int 34267c478bd9Sstevel@tonic-gate cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt, 34277c478bd9Sstevel@tonic-gate char *buf, int buflen, int *lenp) 34287c478bd9Sstevel@tonic-gate { 34297c478bd9Sstevel@tonic-gate /* 34307c478bd9Sstevel@tonic-gate * If we come thru here for an IO bus error aflt->flt_stat will 34317c478bd9Sstevel@tonic-gate * not be the CPU AFSR, and we pass in a -1 to cpu_get_mem_unum() 34327c478bd9Sstevel@tonic-gate * so it will interpret this as a memory error. 34337c478bd9Sstevel@tonic-gate */ 34347c478bd9Sstevel@tonic-gate return (cpu_get_mem_unum(synd_status, aflt->flt_synd, 34357c478bd9Sstevel@tonic-gate (aflt->flt_class == BUS_FAULT) ? 34367c478bd9Sstevel@tonic-gate (uint64_t)-1 : ((ch_async_flt_t *)(aflt))->afsr_errs, 34377c478bd9Sstevel@tonic-gate aflt->flt_addr, aflt->flt_bus_id, aflt->flt_in_memory, 34387c478bd9Sstevel@tonic-gate aflt->flt_status, buf, buflen, lenp)); 34397c478bd9Sstevel@tonic-gate } 34407c478bd9Sstevel@tonic-gate 34417c478bd9Sstevel@tonic-gate /* 34427c478bd9Sstevel@tonic-gate * This routine is a more generic interface to cpu_get_mem_unum() 34437c478bd9Sstevel@tonic-gate * that may be used by other modules (e.g. mm). 34447c478bd9Sstevel@tonic-gate */ 34457c478bd9Sstevel@tonic-gate int 34467c478bd9Sstevel@tonic-gate cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar, 34477c478bd9Sstevel@tonic-gate char *buf, int buflen, int *lenp) 34487c478bd9Sstevel@tonic-gate { 34497c478bd9Sstevel@tonic-gate int synd_status, flt_in_memory, ret; 34507c478bd9Sstevel@tonic-gate ushort_t flt_status = 0; 34517c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 34527c478bd9Sstevel@tonic-gate 34537c478bd9Sstevel@tonic-gate /* 34547c478bd9Sstevel@tonic-gate * Check for an invalid address. 34557c478bd9Sstevel@tonic-gate */ 34567c478bd9Sstevel@tonic-gate if (afar == (uint64_t)-1) 34577c478bd9Sstevel@tonic-gate return (ENXIO); 34587c478bd9Sstevel@tonic-gate 34597c478bd9Sstevel@tonic-gate if (synd == (uint64_t)-1) 34607c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_INVALID; 34617c478bd9Sstevel@tonic-gate else 34627c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_VALID; 34637c478bd9Sstevel@tonic-gate 34647c478bd9Sstevel@tonic-gate flt_in_memory = (*afsr & C_AFSR_MEMORY) && 34657c478bd9Sstevel@tonic-gate pf_is_memory(afar >> MMU_PAGESHIFT); 34667c478bd9Sstevel@tonic-gate 34677c478bd9Sstevel@tonic-gate /* 34687c478bd9Sstevel@tonic-gate * Need to turn on ECC_ECACHE for plat_get_mem_unum(). 34697c478bd9Sstevel@tonic-gate * For Panther, L2$ is not external, so we don't want to 34707c478bd9Sstevel@tonic-gate * generate an E$ unum for those errors. 34717c478bd9Sstevel@tonic-gate */ 34727c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 34737c478bd9Sstevel@tonic-gate if (*(afsr + 1) & C_AFSR_EXT_L3_ERRS) 34747c478bd9Sstevel@tonic-gate flt_status |= ECC_ECACHE; 34757c478bd9Sstevel@tonic-gate } else { 34767c478bd9Sstevel@tonic-gate if (*afsr & C_AFSR_ECACHE) 34777c478bd9Sstevel@tonic-gate flt_status |= ECC_ECACHE; 34787c478bd9Sstevel@tonic-gate } 34797c478bd9Sstevel@tonic-gate 34807c478bd9Sstevel@tonic-gate ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, *afsr, afar, 34817c478bd9Sstevel@tonic-gate CPU->cpu_id, flt_in_memory, flt_status, unum, UNUM_NAMLEN, lenp); 34827c478bd9Sstevel@tonic-gate if (ret != 0) 34837c478bd9Sstevel@tonic-gate return (ret); 34847c478bd9Sstevel@tonic-gate 34857c478bd9Sstevel@tonic-gate if (*lenp >= buflen) 34867c478bd9Sstevel@tonic-gate return (ENAMETOOLONG); 34877c478bd9Sstevel@tonic-gate 34887c478bd9Sstevel@tonic-gate (void) strncpy(buf, unum, buflen); 34897c478bd9Sstevel@tonic-gate 34907c478bd9Sstevel@tonic-gate return (0); 34917c478bd9Sstevel@tonic-gate } 34927c478bd9Sstevel@tonic-gate 34937c478bd9Sstevel@tonic-gate /* 34947c478bd9Sstevel@tonic-gate * Routine to return memory information associated 34957c478bd9Sstevel@tonic-gate * with a physical address and syndrome. 34967c478bd9Sstevel@tonic-gate */ 34977c478bd9Sstevel@tonic-gate int 34987c478bd9Sstevel@tonic-gate cpu_get_mem_info(uint64_t synd, uint64_t afar, 34997c478bd9Sstevel@tonic-gate uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 35007c478bd9Sstevel@tonic-gate int *segsp, int *banksp, int *mcidp) 35017c478bd9Sstevel@tonic-gate { 35027c478bd9Sstevel@tonic-gate int synd_status, synd_code; 35037c478bd9Sstevel@tonic-gate 35047c478bd9Sstevel@tonic-gate if (afar == (uint64_t)-1) 35057c478bd9Sstevel@tonic-gate return (ENXIO); 35067c478bd9Sstevel@tonic-gate 35077c478bd9Sstevel@tonic-gate if (synd == (uint64_t)-1) 35087c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_INVALID; 35097c478bd9Sstevel@tonic-gate else 35107c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_VALID; 35117c478bd9Sstevel@tonic-gate 35127c478bd9Sstevel@tonic-gate synd_code = synd_to_synd_code(synd_status, synd, C_AFSR_CE); 35137c478bd9Sstevel@tonic-gate 35147c478bd9Sstevel@tonic-gate if (p2get_mem_info != NULL) 35157c478bd9Sstevel@tonic-gate return ((p2get_mem_info)(synd_code, afar, 35167c478bd9Sstevel@tonic-gate mem_sizep, seg_sizep, bank_sizep, 35177c478bd9Sstevel@tonic-gate segsp, banksp, mcidp)); 35187c478bd9Sstevel@tonic-gate else 35197c478bd9Sstevel@tonic-gate return (ENOTSUP); 35207c478bd9Sstevel@tonic-gate } 35217c478bd9Sstevel@tonic-gate 35227c478bd9Sstevel@tonic-gate /* 35237c478bd9Sstevel@tonic-gate * Routine to return a string identifying the physical 35247c478bd9Sstevel@tonic-gate * name associated with a cpuid. 35257c478bd9Sstevel@tonic-gate */ 35267c478bd9Sstevel@tonic-gate int 35277c478bd9Sstevel@tonic-gate cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp) 35287c478bd9Sstevel@tonic-gate { 35297c478bd9Sstevel@tonic-gate int ret; 35307c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 35317c478bd9Sstevel@tonic-gate 35327c478bd9Sstevel@tonic-gate if (&plat_get_cpu_unum) { 35337c478bd9Sstevel@tonic-gate if ((ret = plat_get_cpu_unum(cpuid, unum, UNUM_NAMLEN, lenp)) 35347c478bd9Sstevel@tonic-gate != 0) 35357c478bd9Sstevel@tonic-gate return (ret); 35367c478bd9Sstevel@tonic-gate } else { 35377c478bd9Sstevel@tonic-gate return (ENOTSUP); 35387c478bd9Sstevel@tonic-gate } 35397c478bd9Sstevel@tonic-gate 35407c478bd9Sstevel@tonic-gate if (*lenp >= buflen) 35417c478bd9Sstevel@tonic-gate return (ENAMETOOLONG); 35427c478bd9Sstevel@tonic-gate 35437c478bd9Sstevel@tonic-gate (void) strncpy(buf, unum, buflen); 35447c478bd9Sstevel@tonic-gate 35457c478bd9Sstevel@tonic-gate return (0); 35467c478bd9Sstevel@tonic-gate } 35477c478bd9Sstevel@tonic-gate 35487c478bd9Sstevel@tonic-gate /* 35497c478bd9Sstevel@tonic-gate * This routine exports the name buffer size. 35507c478bd9Sstevel@tonic-gate */ 35517c478bd9Sstevel@tonic-gate size_t 35527c478bd9Sstevel@tonic-gate cpu_get_name_bufsize() 35537c478bd9Sstevel@tonic-gate { 35547c478bd9Sstevel@tonic-gate return (UNUM_NAMLEN); 35557c478bd9Sstevel@tonic-gate } 35567c478bd9Sstevel@tonic-gate 35577c478bd9Sstevel@tonic-gate /* 35587c478bd9Sstevel@tonic-gate * Historical function, apparantly not used. 35597c478bd9Sstevel@tonic-gate */ 35607c478bd9Sstevel@tonic-gate /* ARGSUSED */ 35617c478bd9Sstevel@tonic-gate void 35627c478bd9Sstevel@tonic-gate cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err) 35637c478bd9Sstevel@tonic-gate {} 35647c478bd9Sstevel@tonic-gate 35657c478bd9Sstevel@tonic-gate /* 35667c478bd9Sstevel@tonic-gate * Historical function only called for SBus errors in debugging. 35677c478bd9Sstevel@tonic-gate */ 35687c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 35697c478bd9Sstevel@tonic-gate void 35707c478bd9Sstevel@tonic-gate read_ecc_data(struct async_flt *aflt, short verbose, short ce_err) 35717c478bd9Sstevel@tonic-gate {} 35727c478bd9Sstevel@tonic-gate 35737c478bd9Sstevel@tonic-gate /* 35747c478bd9Sstevel@tonic-gate * Clear the AFSR sticky bits. The routine returns a non-zero value if 35757c478bd9Sstevel@tonic-gate * any of the AFSR's sticky errors are detected. If a non-null pointer to 35767c478bd9Sstevel@tonic-gate * an async fault structure argument is passed in, the captured error state 35777c478bd9Sstevel@tonic-gate * (AFSR, AFAR) info will be returned in the structure. 35787c478bd9Sstevel@tonic-gate */ 35797c478bd9Sstevel@tonic-gate int 35807c478bd9Sstevel@tonic-gate clear_errors(ch_async_flt_t *ch_flt) 35817c478bd9Sstevel@tonic-gate { 35827c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 35837c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 35847c478bd9Sstevel@tonic-gate 35857c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 35867c478bd9Sstevel@tonic-gate 35877c478bd9Sstevel@tonic-gate if (ch_flt != NULL) { 35887c478bd9Sstevel@tonic-gate aflt->flt_stat = cpu_error_regs.afsr & C_AFSR_MASK; 35897c478bd9Sstevel@tonic-gate aflt->flt_addr = cpu_error_regs.afar; 35907c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = cpu_error_regs.afsr_ext; 35917c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 35927c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS); 35937c478bd9Sstevel@tonic-gate #if defined(SERRANO) 35947c478bd9Sstevel@tonic-gate ch_flt->afar2 = cpu_error_regs.afar2; 35957c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 35967c478bd9Sstevel@tonic-gate } 35977c478bd9Sstevel@tonic-gate 35987c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 35997c478bd9Sstevel@tonic-gate 36007c478bd9Sstevel@tonic-gate return (((cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 36017c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS)) != 0); 36027c478bd9Sstevel@tonic-gate } 36037c478bd9Sstevel@tonic-gate 36047c478bd9Sstevel@tonic-gate /* 36057c478bd9Sstevel@tonic-gate * Clear any AFSR error bits, and check for persistence. 36067c478bd9Sstevel@tonic-gate * 36077c478bd9Sstevel@tonic-gate * It would be desirable to also insist that syndrome match. PCI handling 36087c478bd9Sstevel@tonic-gate * has already filled flt_synd. For errors trapped by CPU we only fill 36097c478bd9Sstevel@tonic-gate * flt_synd when we queue the event, so we do not have a valid flt_synd 36107c478bd9Sstevel@tonic-gate * during initial classification (it is valid if we're called as part of 36117c478bd9Sstevel@tonic-gate * subsequent low-pil additional classification attempts). We could try 36127c478bd9Sstevel@tonic-gate * to determine which syndrome to use: we know we're only called for 36137c478bd9Sstevel@tonic-gate * CE/RCE (Jalapeno & Serrano) and CE/EMC (others) so the syndrome to use 36147c478bd9Sstevel@tonic-gate * would be esynd/none and esynd/msynd, respectively. If that is 36157c478bd9Sstevel@tonic-gate * implemented then what do we do in the case that we do experience an 36167c478bd9Sstevel@tonic-gate * error on the same afar but with different syndrome? At the very least 36177c478bd9Sstevel@tonic-gate * we should count such occurences. Anyway, for now, we'll leave it as 36187c478bd9Sstevel@tonic-gate * it has been for ages. 36197c478bd9Sstevel@tonic-gate */ 36207c478bd9Sstevel@tonic-gate static int 36217c478bd9Sstevel@tonic-gate clear_ecc(struct async_flt *aflt) 36227c478bd9Sstevel@tonic-gate { 36237c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 36247c478bd9Sstevel@tonic-gate 36257c478bd9Sstevel@tonic-gate /* 36267c478bd9Sstevel@tonic-gate * Snapshot the AFSR and AFAR and clear any errors 36277c478bd9Sstevel@tonic-gate */ 36287c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 36297c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 36307c478bd9Sstevel@tonic-gate 36317c478bd9Sstevel@tonic-gate /* 36327c478bd9Sstevel@tonic-gate * If any of the same memory access error bits are still on and 36337c478bd9Sstevel@tonic-gate * the AFAR matches, return that the error is persistent. 36347c478bd9Sstevel@tonic-gate */ 36357c478bd9Sstevel@tonic-gate return ((cpu_error_regs.afsr & (C_AFSR_MEMORY & aflt->flt_stat)) != 0 && 36367c478bd9Sstevel@tonic-gate cpu_error_regs.afar == aflt->flt_addr); 36377c478bd9Sstevel@tonic-gate } 36387c478bd9Sstevel@tonic-gate 36397c478bd9Sstevel@tonic-gate /* 36407c478bd9Sstevel@tonic-gate * Turn off all cpu error detection, normally only used for panics. 36417c478bd9Sstevel@tonic-gate */ 36427c478bd9Sstevel@tonic-gate void 36437c478bd9Sstevel@tonic-gate cpu_disable_errors(void) 36447c478bd9Sstevel@tonic-gate { 36457c478bd9Sstevel@tonic-gate xt_all(set_error_enable_tl1, EN_REG_DISABLE, EER_SET_ABSOLUTE); 3646db7840cdSrscott 3647db7840cdSrscott /* 3648db7840cdSrscott * With error detection now turned off, check the other cpus 3649db7840cdSrscott * logout areas for any unlogged errors. 3650db7840cdSrscott */ 3651db7840cdSrscott if (enable_check_other_cpus_logout) { 3652db7840cdSrscott cpu_check_other_cpus_logout(); 3653db7840cdSrscott /* 3654db7840cdSrscott * Make a second pass over the logout areas, in case 3655db7840cdSrscott * there is a failing CPU in an error-trap loop which 3656db7840cdSrscott * will write to the logout area once it is emptied. 3657db7840cdSrscott */ 3658db7840cdSrscott cpu_check_other_cpus_logout(); 3659db7840cdSrscott } 36607c478bd9Sstevel@tonic-gate } 36617c478bd9Sstevel@tonic-gate 36627c478bd9Sstevel@tonic-gate /* 36637c478bd9Sstevel@tonic-gate * Enable errors. 36647c478bd9Sstevel@tonic-gate */ 36657c478bd9Sstevel@tonic-gate void 36667c478bd9Sstevel@tonic-gate cpu_enable_errors(void) 36677c478bd9Sstevel@tonic-gate { 36687c478bd9Sstevel@tonic-gate xt_all(set_error_enable_tl1, EN_REG_ENABLE, EER_SET_ABSOLUTE); 36697c478bd9Sstevel@tonic-gate } 36707c478bd9Sstevel@tonic-gate 36717c478bd9Sstevel@tonic-gate /* 36727c478bd9Sstevel@tonic-gate * Flush the entire ecache using displacement flush by reading through a 36737c478bd9Sstevel@tonic-gate * physical address range twice as large as the Ecache. 36747c478bd9Sstevel@tonic-gate */ 36757c478bd9Sstevel@tonic-gate void 36767c478bd9Sstevel@tonic-gate cpu_flush_ecache(void) 36777c478bd9Sstevel@tonic-gate { 36787c478bd9Sstevel@tonic-gate flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size, 36797c478bd9Sstevel@tonic-gate cpunodes[CPU->cpu_id].ecache_linesize); 36807c478bd9Sstevel@tonic-gate } 36817c478bd9Sstevel@tonic-gate 36827c478bd9Sstevel@tonic-gate /* 36837c478bd9Sstevel@tonic-gate * Return CPU E$ set size - E$ size divided by the associativity. 36847c478bd9Sstevel@tonic-gate * We use this function in places where the CPU_PRIVATE ptr may not be 36857c478bd9Sstevel@tonic-gate * initialized yet. Note that for send_mondo and in the Ecache scrubber, 36867c478bd9Sstevel@tonic-gate * we're guaranteed that CPU_PRIVATE is initialized. Also, cpunodes is set 36877c478bd9Sstevel@tonic-gate * up before the kernel switches from OBP's to the kernel's trap table, so 36887c478bd9Sstevel@tonic-gate * we don't have to worry about cpunodes being unitialized. 36897c478bd9Sstevel@tonic-gate */ 36907c478bd9Sstevel@tonic-gate int 36917c478bd9Sstevel@tonic-gate cpu_ecache_set_size(struct cpu *cp) 36927c478bd9Sstevel@tonic-gate { 36937c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(cp)) 36947c478bd9Sstevel@tonic-gate return (CPU_PRIVATE_VAL(cp, chpr_ec_set_size)); 36957c478bd9Sstevel@tonic-gate 36967c478bd9Sstevel@tonic-gate return (cpunodes[cp->cpu_id].ecache_size / cpu_ecache_nway()); 36977c478bd9Sstevel@tonic-gate } 36987c478bd9Sstevel@tonic-gate 36997c478bd9Sstevel@tonic-gate /* 37007c478bd9Sstevel@tonic-gate * Flush Ecache line. 37017c478bd9Sstevel@tonic-gate * Uses ASI_EC_DIAG for Cheetah+ and Jalapeno. 37027c478bd9Sstevel@tonic-gate * Uses normal displacement flush for Cheetah. 37037c478bd9Sstevel@tonic-gate */ 37047c478bd9Sstevel@tonic-gate static void 37057c478bd9Sstevel@tonic-gate cpu_flush_ecache_line(ch_async_flt_t *ch_flt) 37067c478bd9Sstevel@tonic-gate { 37077c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 37087c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 37097c478bd9Sstevel@tonic-gate 37107c478bd9Sstevel@tonic-gate ecache_flush_line(aflt->flt_addr, ec_set_size); 37117c478bd9Sstevel@tonic-gate } 37127c478bd9Sstevel@tonic-gate 37137c478bd9Sstevel@tonic-gate /* 37147c478bd9Sstevel@tonic-gate * Scrub physical address. 37157c478bd9Sstevel@tonic-gate * Scrub code is different depending upon whether this a Cheetah+ with 2-way 37167c478bd9Sstevel@tonic-gate * Ecache or direct-mapped Ecache. 37177c478bd9Sstevel@tonic-gate */ 37187c478bd9Sstevel@tonic-gate static void 37197c478bd9Sstevel@tonic-gate cpu_scrubphys(struct async_flt *aflt) 37207c478bd9Sstevel@tonic-gate { 37217c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 37227c478bd9Sstevel@tonic-gate 37237c478bd9Sstevel@tonic-gate scrubphys(aflt->flt_addr, ec_set_size); 37247c478bd9Sstevel@tonic-gate } 37257c478bd9Sstevel@tonic-gate 37267c478bd9Sstevel@tonic-gate /* 37277c478bd9Sstevel@tonic-gate * Clear physical address. 37287c478bd9Sstevel@tonic-gate * Scrub code is different depending upon whether this a Cheetah+ with 2-way 37297c478bd9Sstevel@tonic-gate * Ecache or direct-mapped Ecache. 37307c478bd9Sstevel@tonic-gate */ 37317c478bd9Sstevel@tonic-gate void 37327c478bd9Sstevel@tonic-gate cpu_clearphys(struct async_flt *aflt) 37337c478bd9Sstevel@tonic-gate { 37347c478bd9Sstevel@tonic-gate int lsize = cpunodes[CPU->cpu_id].ecache_linesize; 37357c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 37367c478bd9Sstevel@tonic-gate 37377c478bd9Sstevel@tonic-gate 37387c478bd9Sstevel@tonic-gate clearphys(P2ALIGN(aflt->flt_addr, lsize), ec_set_size, lsize); 37397c478bd9Sstevel@tonic-gate } 37407c478bd9Sstevel@tonic-gate 37417c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 37427c478bd9Sstevel@tonic-gate /* 37437c478bd9Sstevel@tonic-gate * Check for a matching valid line in all the sets. 37447c478bd9Sstevel@tonic-gate * If found, return set# + 1. Otherwise return 0. 37457c478bd9Sstevel@tonic-gate */ 37467c478bd9Sstevel@tonic-gate static int 37477c478bd9Sstevel@tonic-gate cpu_ecache_line_valid(ch_async_flt_t *ch_flt) 37487c478bd9Sstevel@tonic-gate { 37497c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 37507c478bd9Sstevel@tonic-gate int totalsize = cpunodes[CPU->cpu_id].ecache_size; 37517c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 37527c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 37537c478bd9Sstevel@tonic-gate int nway = cpu_ecache_nway(); 37547c478bd9Sstevel@tonic-gate int i; 37557c478bd9Sstevel@tonic-gate 37567c478bd9Sstevel@tonic-gate for (i = 0; i < nway; i++, ecp++) { 37577c478bd9Sstevel@tonic-gate if (!cpu_ectag_line_invalid(totalsize, ecp->ec_tag) && 37587c478bd9Sstevel@tonic-gate (aflt->flt_addr & P2ALIGN(C_AFAR_PA, ec_set_size)) == 37597c478bd9Sstevel@tonic-gate cpu_ectag_to_pa(ec_set_size, ecp->ec_tag)) 37607c478bd9Sstevel@tonic-gate return (i+1); 37617c478bd9Sstevel@tonic-gate } 37627c478bd9Sstevel@tonic-gate return (0); 37637c478bd9Sstevel@tonic-gate } 37647c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_ECACHE_ASSOC */ 37657c478bd9Sstevel@tonic-gate 37667c478bd9Sstevel@tonic-gate /* 37677c478bd9Sstevel@tonic-gate * Check whether a line in the given logout info matches the specified 37687c478bd9Sstevel@tonic-gate * fault address. If reqval is set then the line must not be Invalid. 37697c478bd9Sstevel@tonic-gate * Returns 0 on failure; on success (way + 1) is returned an *level is 37707c478bd9Sstevel@tonic-gate * set to 2 for l2$ or 3 for l3$. 37717c478bd9Sstevel@tonic-gate */ 37727c478bd9Sstevel@tonic-gate static int 37737c478bd9Sstevel@tonic-gate cpu_matching_ecache_line(uint64_t faddr, void *data, int reqval, int *level) 37747c478bd9Sstevel@tonic-gate { 37757c478bd9Sstevel@tonic-gate ch_diag_data_t *cdp = data; 37767c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp; 37777c478bd9Sstevel@tonic-gate int totalsize, ec_set_size; 37787c478bd9Sstevel@tonic-gate int i, ways; 37797c478bd9Sstevel@tonic-gate int match = 0; 37807c478bd9Sstevel@tonic-gate int tagvalid; 37817c478bd9Sstevel@tonic-gate uint64_t addr, tagpa; 37827c478bd9Sstevel@tonic-gate int ispanther = IS_PANTHER(cpunodes[CPU->cpu_id].implementation); 37837c478bd9Sstevel@tonic-gate 37847c478bd9Sstevel@tonic-gate /* 37857c478bd9Sstevel@tonic-gate * Check the l2$ logout data 37867c478bd9Sstevel@tonic-gate */ 37877c478bd9Sstevel@tonic-gate if (ispanther) { 37887c478bd9Sstevel@tonic-gate ecp = &cdp->chd_l2_data[0]; 37897c478bd9Sstevel@tonic-gate ec_set_size = PN_L2_SET_SIZE; 37907c478bd9Sstevel@tonic-gate ways = PN_L2_NWAYS; 37917c478bd9Sstevel@tonic-gate } else { 37927c478bd9Sstevel@tonic-gate ecp = &cdp->chd_ec_data[0]; 37937c478bd9Sstevel@tonic-gate ec_set_size = cpu_ecache_set_size(CPU); 37947c478bd9Sstevel@tonic-gate ways = cpu_ecache_nway(); 37957c478bd9Sstevel@tonic-gate totalsize = cpunodes[CPU->cpu_id].ecache_size; 37967c478bd9Sstevel@tonic-gate } 37977c478bd9Sstevel@tonic-gate /* remove low order PA bits from fault address not used in PA tag */ 37987c478bd9Sstevel@tonic-gate addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 37997c478bd9Sstevel@tonic-gate for (i = 0; i < ways; i++, ecp++) { 38007c478bd9Sstevel@tonic-gate if (ispanther) { 38017c478bd9Sstevel@tonic-gate tagpa = PN_L2TAG_TO_PA(ecp->ec_tag); 38027c478bd9Sstevel@tonic-gate tagvalid = !PN_L2_LINE_INVALID(ecp->ec_tag); 38037c478bd9Sstevel@tonic-gate } else { 38047c478bd9Sstevel@tonic-gate tagpa = cpu_ectag_to_pa(ec_set_size, ecp->ec_tag); 38057c478bd9Sstevel@tonic-gate tagvalid = !cpu_ectag_line_invalid(totalsize, 38067c478bd9Sstevel@tonic-gate ecp->ec_tag); 38077c478bd9Sstevel@tonic-gate } 38087c478bd9Sstevel@tonic-gate if (tagpa == addr && (!reqval || tagvalid)) { 38097c478bd9Sstevel@tonic-gate match = i + 1; 38107c478bd9Sstevel@tonic-gate *level = 2; 38117c478bd9Sstevel@tonic-gate break; 38127c478bd9Sstevel@tonic-gate } 38137c478bd9Sstevel@tonic-gate } 38147c478bd9Sstevel@tonic-gate 38157c478bd9Sstevel@tonic-gate if (match || !ispanther) 38167c478bd9Sstevel@tonic-gate return (match); 38177c478bd9Sstevel@tonic-gate 38187c478bd9Sstevel@tonic-gate /* For Panther we also check the l3$ */ 38197c478bd9Sstevel@tonic-gate ecp = &cdp->chd_ec_data[0]; 38207c478bd9Sstevel@tonic-gate ec_set_size = PN_L3_SET_SIZE; 38217c478bd9Sstevel@tonic-gate ways = PN_L3_NWAYS; 38227c478bd9Sstevel@tonic-gate addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 38237c478bd9Sstevel@tonic-gate 38247c478bd9Sstevel@tonic-gate for (i = 0; i < ways; i++, ecp++) { 38257c478bd9Sstevel@tonic-gate if (PN_L3TAG_TO_PA(ecp->ec_tag) == addr && (!reqval || 38267c478bd9Sstevel@tonic-gate !PN_L3_LINE_INVALID(ecp->ec_tag))) { 38277c478bd9Sstevel@tonic-gate match = i + 1; 38287c478bd9Sstevel@tonic-gate *level = 3; 38297c478bd9Sstevel@tonic-gate break; 38307c478bd9Sstevel@tonic-gate } 38317c478bd9Sstevel@tonic-gate } 38327c478bd9Sstevel@tonic-gate 38337c478bd9Sstevel@tonic-gate return (match); 38347c478bd9Sstevel@tonic-gate } 38357c478bd9Sstevel@tonic-gate 38367c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 38377c478bd9Sstevel@tonic-gate /* 38387c478bd9Sstevel@tonic-gate * Record information related to the source of an Dcache Parity Error. 38397c478bd9Sstevel@tonic-gate */ 38407c478bd9Sstevel@tonic-gate static void 38417c478bd9Sstevel@tonic-gate cpu_dcache_parity_info(ch_async_flt_t *ch_flt) 38427c478bd9Sstevel@tonic-gate { 38437c478bd9Sstevel@tonic-gate int dc_set_size = dcache_size / CH_DCACHE_NWAY; 38447c478bd9Sstevel@tonic-gate int index; 38457c478bd9Sstevel@tonic-gate 38467c478bd9Sstevel@tonic-gate /* 38477c478bd9Sstevel@tonic-gate * Since instruction decode cannot be done at high PIL 38487c478bd9Sstevel@tonic-gate * just examine the entire Dcache to locate the error. 38497c478bd9Sstevel@tonic-gate */ 38507c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 38517c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = -1; 38527c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = -1; 38537c478bd9Sstevel@tonic-gate } 38547c478bd9Sstevel@tonic-gate for (index = 0; index < dc_set_size; index += dcache_linesize) 38557c478bd9Sstevel@tonic-gate cpu_dcache_parity_check(ch_flt, index); 38567c478bd9Sstevel@tonic-gate } 38577c478bd9Sstevel@tonic-gate 38587c478bd9Sstevel@tonic-gate /* 38597c478bd9Sstevel@tonic-gate * Check all ways of the Dcache at a specified index for good parity. 38607c478bd9Sstevel@tonic-gate */ 38617c478bd9Sstevel@tonic-gate static void 38627c478bd9Sstevel@tonic-gate cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index) 38637c478bd9Sstevel@tonic-gate { 38647c478bd9Sstevel@tonic-gate int dc_set_size = dcache_size / CH_DCACHE_NWAY; 38657c478bd9Sstevel@tonic-gate uint64_t parity_bits, pbits, data_word; 38667c478bd9Sstevel@tonic-gate static int parity_bits_popc[] = { 0, 1, 1, 0 }; 38677c478bd9Sstevel@tonic-gate int way, word, data_byte; 38687c478bd9Sstevel@tonic-gate ch_dc_data_t *dcp = &ch_flt->parity_data.dpe.cpl_dc[0]; 38697c478bd9Sstevel@tonic-gate ch_dc_data_t tmp_dcp; 38707c478bd9Sstevel@tonic-gate 38717c478bd9Sstevel@tonic-gate for (way = 0; way < CH_DCACHE_NWAY; way++, dcp++) { 38727c478bd9Sstevel@tonic-gate /* 38737c478bd9Sstevel@tonic-gate * Perform diagnostic read. 38747c478bd9Sstevel@tonic-gate */ 38757c478bd9Sstevel@tonic-gate get_dcache_dtag(index + way * dc_set_size, 38767c478bd9Sstevel@tonic-gate (uint64_t *)&tmp_dcp); 38777c478bd9Sstevel@tonic-gate 38787c478bd9Sstevel@tonic-gate /* 38797c478bd9Sstevel@tonic-gate * Check tag for even parity. 38807c478bd9Sstevel@tonic-gate * Sum of 1 bits (including parity bit) should be even. 38817c478bd9Sstevel@tonic-gate */ 38827c478bd9Sstevel@tonic-gate if (popc64(tmp_dcp.dc_tag & CHP_DCTAG_PARMASK) & 1) { 38837c478bd9Sstevel@tonic-gate /* 38847c478bd9Sstevel@tonic-gate * If this is the first error log detailed information 38857c478bd9Sstevel@tonic-gate * about it and check the snoop tag. Otherwise just 38867c478bd9Sstevel@tonic-gate * record the fact that we found another error. 38877c478bd9Sstevel@tonic-gate */ 38887c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 38897c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = way; 38907c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_cache = 38917c478bd9Sstevel@tonic-gate CPU_DC_PARITY; 38927c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_tag |= CHP_DC_TAG; 38937c478bd9Sstevel@tonic-gate 38947c478bd9Sstevel@tonic-gate if (popc64(tmp_dcp.dc_sntag & 38957c478bd9Sstevel@tonic-gate CHP_DCSNTAG_PARMASK) & 1) { 38967c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_tag |= 38977c478bd9Sstevel@tonic-gate CHP_DC_SNTAG; 38987c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 38997c478bd9Sstevel@tonic-gate } 39007c478bd9Sstevel@tonic-gate 39017c478bd9Sstevel@tonic-gate bcopy(&tmp_dcp, dcp, sizeof (ch_dc_data_t)); 39027c478bd9Sstevel@tonic-gate } 39037c478bd9Sstevel@tonic-gate 39047c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 39057c478bd9Sstevel@tonic-gate } 39067c478bd9Sstevel@tonic-gate 39077c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 39087c478bd9Sstevel@tonic-gate /* 39097c478bd9Sstevel@tonic-gate * Panther has more parity bits than the other 39107c478bd9Sstevel@tonic-gate * processors for covering dcache data and so each 39117c478bd9Sstevel@tonic-gate * byte of data in each word has its own parity bit. 39127c478bd9Sstevel@tonic-gate */ 39137c478bd9Sstevel@tonic-gate parity_bits = tmp_dcp.dc_pn_data_parity; 39147c478bd9Sstevel@tonic-gate for (word = 0; word < 4; word++) { 39157c478bd9Sstevel@tonic-gate data_word = tmp_dcp.dc_data[word]; 39167c478bd9Sstevel@tonic-gate pbits = parity_bits & PN_DC_DATA_PARITY_MASK; 39177c478bd9Sstevel@tonic-gate for (data_byte = 0; data_byte < 8; 39187c478bd9Sstevel@tonic-gate data_byte++) { 39197c478bd9Sstevel@tonic-gate if (((popc64(data_word & 39207c478bd9Sstevel@tonic-gate PN_DC_DATA_PARITY_MASK)) & 1) ^ 39217c478bd9Sstevel@tonic-gate (pbits & 1)) { 39227c478bd9Sstevel@tonic-gate cpu_record_dc_data_parity( 39237c478bd9Sstevel@tonic-gate ch_flt, dcp, &tmp_dcp, way, 39247c478bd9Sstevel@tonic-gate word); 39257c478bd9Sstevel@tonic-gate } 39267c478bd9Sstevel@tonic-gate pbits >>= 1; 39277c478bd9Sstevel@tonic-gate data_word >>= 8; 39287c478bd9Sstevel@tonic-gate } 39297c478bd9Sstevel@tonic-gate parity_bits >>= 8; 39307c478bd9Sstevel@tonic-gate } 39317c478bd9Sstevel@tonic-gate } else { 39327c478bd9Sstevel@tonic-gate /* 39337c478bd9Sstevel@tonic-gate * Check data array for even parity. 39347c478bd9Sstevel@tonic-gate * The 8 parity bits are grouped into 4 pairs each 39357c478bd9Sstevel@tonic-gate * of which covers a 64-bit word. The endianness is 39367c478bd9Sstevel@tonic-gate * reversed -- the low-order parity bits cover the 39377c478bd9Sstevel@tonic-gate * high-order data words. 39387c478bd9Sstevel@tonic-gate */ 39397c478bd9Sstevel@tonic-gate parity_bits = tmp_dcp.dc_utag >> 8; 39407c478bd9Sstevel@tonic-gate for (word = 0; word < 4; word++) { 39417c478bd9Sstevel@tonic-gate pbits = (parity_bits >> (6 - word * 2)) & 3; 39427c478bd9Sstevel@tonic-gate if ((popc64(tmp_dcp.dc_data[word]) + 39437c478bd9Sstevel@tonic-gate parity_bits_popc[pbits]) & 1) { 39447c478bd9Sstevel@tonic-gate cpu_record_dc_data_parity(ch_flt, dcp, 39457c478bd9Sstevel@tonic-gate &tmp_dcp, way, word); 39467c478bd9Sstevel@tonic-gate } 39477c478bd9Sstevel@tonic-gate } 39487c478bd9Sstevel@tonic-gate } 39497c478bd9Sstevel@tonic-gate } 39507c478bd9Sstevel@tonic-gate } 39517c478bd9Sstevel@tonic-gate 39527c478bd9Sstevel@tonic-gate static void 39537c478bd9Sstevel@tonic-gate cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 39547c478bd9Sstevel@tonic-gate ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word) 39557c478bd9Sstevel@tonic-gate { 39567c478bd9Sstevel@tonic-gate /* 39577c478bd9Sstevel@tonic-gate * If this is the first error log detailed information about it. 39587c478bd9Sstevel@tonic-gate * Otherwise just record the fact that we found another error. 39597c478bd9Sstevel@tonic-gate */ 39607c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 39617c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = way; 39627c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_cache = CPU_DC_PARITY; 39637c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = word * 8; 39647c478bd9Sstevel@tonic-gate bcopy(src_dcp, dest_dcp, sizeof (ch_dc_data_t)); 39657c478bd9Sstevel@tonic-gate } 39667c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 39677c478bd9Sstevel@tonic-gate } 39687c478bd9Sstevel@tonic-gate 39697c478bd9Sstevel@tonic-gate /* 39707c478bd9Sstevel@tonic-gate * Record information related to the source of an Icache Parity Error. 39717c478bd9Sstevel@tonic-gate * 39727c478bd9Sstevel@tonic-gate * Called with the Icache disabled so any diagnostic accesses are safe. 39737c478bd9Sstevel@tonic-gate */ 39747c478bd9Sstevel@tonic-gate static void 39757c478bd9Sstevel@tonic-gate cpu_icache_parity_info(ch_async_flt_t *ch_flt) 39767c478bd9Sstevel@tonic-gate { 39777c478bd9Sstevel@tonic-gate int ic_set_size; 39787c478bd9Sstevel@tonic-gate int ic_linesize; 39797c478bd9Sstevel@tonic-gate int index; 39807c478bd9Sstevel@tonic-gate 39817c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU)) { 39827c478bd9Sstevel@tonic-gate ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 39837c478bd9Sstevel@tonic-gate CH_ICACHE_NWAY; 39847c478bd9Sstevel@tonic-gate ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 39857c478bd9Sstevel@tonic-gate } else { 39867c478bd9Sstevel@tonic-gate ic_set_size = icache_size / CH_ICACHE_NWAY; 39877c478bd9Sstevel@tonic-gate ic_linesize = icache_linesize; 39887c478bd9Sstevel@tonic-gate } 39897c478bd9Sstevel@tonic-gate 39907c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_way = -1; 39917c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_off = -1; 39927c478bd9Sstevel@tonic-gate 39937c478bd9Sstevel@tonic-gate for (index = 0; index < ic_set_size; index += ic_linesize) 39947c478bd9Sstevel@tonic-gate cpu_icache_parity_check(ch_flt, index); 39957c478bd9Sstevel@tonic-gate } 39967c478bd9Sstevel@tonic-gate 39977c478bd9Sstevel@tonic-gate /* 39987c478bd9Sstevel@tonic-gate * Check all ways of the Icache at a specified index for good parity. 39997c478bd9Sstevel@tonic-gate */ 40007c478bd9Sstevel@tonic-gate static void 40017c478bd9Sstevel@tonic-gate cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index) 40027c478bd9Sstevel@tonic-gate { 40037c478bd9Sstevel@tonic-gate uint64_t parmask, pn_inst_parity; 40047c478bd9Sstevel@tonic-gate int ic_set_size; 40057c478bd9Sstevel@tonic-gate int ic_linesize; 40067c478bd9Sstevel@tonic-gate int flt_index, way, instr, num_instr; 40077c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 40087c478bd9Sstevel@tonic-gate ch_ic_data_t *icp = &ch_flt->parity_data.ipe.cpl_ic[0]; 40097c478bd9Sstevel@tonic-gate ch_ic_data_t tmp_icp; 40107c478bd9Sstevel@tonic-gate 40117c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU)) { 40127c478bd9Sstevel@tonic-gate ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 40137c478bd9Sstevel@tonic-gate CH_ICACHE_NWAY; 40147c478bd9Sstevel@tonic-gate ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 40157c478bd9Sstevel@tonic-gate } else { 40167c478bd9Sstevel@tonic-gate ic_set_size = icache_size / CH_ICACHE_NWAY; 40177c478bd9Sstevel@tonic-gate ic_linesize = icache_linesize; 40187c478bd9Sstevel@tonic-gate } 40197c478bd9Sstevel@tonic-gate 40207c478bd9Sstevel@tonic-gate /* 40217c478bd9Sstevel@tonic-gate * Panther has twice as many instructions per icache line and the 40227c478bd9Sstevel@tonic-gate * instruction parity bit is in a different location. 40237c478bd9Sstevel@tonic-gate */ 40247c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 40257c478bd9Sstevel@tonic-gate num_instr = PN_IC_DATA_REG_SIZE / sizeof (uint64_t); 40267c478bd9Sstevel@tonic-gate pn_inst_parity = PN_ICDATA_PARITY_BIT_MASK; 40277c478bd9Sstevel@tonic-gate } else { 40287c478bd9Sstevel@tonic-gate num_instr = CH_IC_DATA_REG_SIZE / sizeof (uint64_t); 40297c478bd9Sstevel@tonic-gate pn_inst_parity = 0; 40307c478bd9Sstevel@tonic-gate } 40317c478bd9Sstevel@tonic-gate 40327c478bd9Sstevel@tonic-gate /* 40337c478bd9Sstevel@tonic-gate * Index at which we expect to find the parity error. 40347c478bd9Sstevel@tonic-gate */ 40357c478bd9Sstevel@tonic-gate flt_index = P2ALIGN(aflt->flt_addr % ic_set_size, ic_linesize); 40367c478bd9Sstevel@tonic-gate 40377c478bd9Sstevel@tonic-gate for (way = 0; way < CH_ICACHE_NWAY; way++, icp++) { 40387c478bd9Sstevel@tonic-gate /* 40397c478bd9Sstevel@tonic-gate * Diagnostic reads expect address argument in ASI format. 40407c478bd9Sstevel@tonic-gate */ 40417c478bd9Sstevel@tonic-gate get_icache_dtag(2 * (index + way * ic_set_size), 40427c478bd9Sstevel@tonic-gate (uint64_t *)&tmp_icp); 40437c478bd9Sstevel@tonic-gate 40447c478bd9Sstevel@tonic-gate /* 40457c478bd9Sstevel@tonic-gate * If this is the index in which we expect to find the 40467c478bd9Sstevel@tonic-gate * error log detailed information about each of the ways. 40477c478bd9Sstevel@tonic-gate * This information will be displayed later if we can't 40487c478bd9Sstevel@tonic-gate * determine the exact way in which the error is located. 40497c478bd9Sstevel@tonic-gate */ 40507c478bd9Sstevel@tonic-gate if (flt_index == index) 40517c478bd9Sstevel@tonic-gate bcopy(&tmp_icp, icp, sizeof (ch_ic_data_t)); 40527c478bd9Sstevel@tonic-gate 40537c478bd9Sstevel@tonic-gate /* 40547c478bd9Sstevel@tonic-gate * Check tag for even parity. 40557c478bd9Sstevel@tonic-gate * Sum of 1 bits (including parity bit) should be even. 40567c478bd9Sstevel@tonic-gate */ 40577c478bd9Sstevel@tonic-gate if (popc64(tmp_icp.ic_patag & CHP_ICPATAG_PARMASK) & 1) { 40587c478bd9Sstevel@tonic-gate /* 40597c478bd9Sstevel@tonic-gate * If this way is the one in which we expected 40607c478bd9Sstevel@tonic-gate * to find the error record the way and check the 40617c478bd9Sstevel@tonic-gate * snoop tag. Otherwise just record the fact we 40627c478bd9Sstevel@tonic-gate * found another error. 40637c478bd9Sstevel@tonic-gate */ 40647c478bd9Sstevel@tonic-gate if (flt_index == index) { 40657c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_way = way; 40667c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_tag |= CHP_IC_TAG; 40677c478bd9Sstevel@tonic-gate 40687c478bd9Sstevel@tonic-gate if (popc64(tmp_icp.ic_sntag & 40697c478bd9Sstevel@tonic-gate CHP_ICSNTAG_PARMASK) & 1) { 40707c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_tag |= 40717c478bd9Sstevel@tonic-gate CHP_IC_SNTAG; 40727c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_lcnt++; 40737c478bd9Sstevel@tonic-gate } 40747c478bd9Sstevel@tonic-gate 40757c478bd9Sstevel@tonic-gate } 40767c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_lcnt++; 40777c478bd9Sstevel@tonic-gate continue; 40787c478bd9Sstevel@tonic-gate } 40797c478bd9Sstevel@tonic-gate 40807c478bd9Sstevel@tonic-gate /* 40817c478bd9Sstevel@tonic-gate * Check instruction data for even parity. 40827c478bd9Sstevel@tonic-gate * Bits participating in parity differ for PC-relative 40837c478bd9Sstevel@tonic-gate * versus non-PC-relative instructions. 40847c478bd9Sstevel@tonic-gate */ 40857c478bd9Sstevel@tonic-gate for (instr = 0; instr < num_instr; instr++) { 40867c478bd9Sstevel@tonic-gate parmask = (tmp_icp.ic_data[instr] & 40877c478bd9Sstevel@tonic-gate CH_ICDATA_PRED_ISPCREL) ? 40887c478bd9Sstevel@tonic-gate (CHP_ICDATA_PCREL_PARMASK | pn_inst_parity) : 40897c478bd9Sstevel@tonic-gate (CHP_ICDATA_NPCREL_PARMASK | pn_inst_parity); 40907c478bd9Sstevel@tonic-gate if (popc64(tmp_icp.ic_data[instr] & parmask) & 1) { 40917c478bd9Sstevel@tonic-gate /* 40927c478bd9Sstevel@tonic-gate * If this way is the one in which we expected 40937c478bd9Sstevel@tonic-gate * to find the error record the way and offset. 40947c478bd9Sstevel@tonic-gate * Otherwise just log the fact we found another 40957c478bd9Sstevel@tonic-gate * error. 40967c478bd9Sstevel@tonic-gate */ 40977c478bd9Sstevel@tonic-gate if (flt_index == index) { 40987c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_way = way; 40997c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_off = 41007c478bd9Sstevel@tonic-gate instr * 4; 41017c478bd9Sstevel@tonic-gate } 41027c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_lcnt++; 41037c478bd9Sstevel@tonic-gate continue; 41047c478bd9Sstevel@tonic-gate } 41057c478bd9Sstevel@tonic-gate } 41067c478bd9Sstevel@tonic-gate } 41077c478bd9Sstevel@tonic-gate } 41087c478bd9Sstevel@tonic-gate 41097c478bd9Sstevel@tonic-gate /* 41107c478bd9Sstevel@tonic-gate * Record information related to the source of an Pcache Parity Error. 41117c478bd9Sstevel@tonic-gate */ 41127c478bd9Sstevel@tonic-gate static void 41137c478bd9Sstevel@tonic-gate cpu_pcache_parity_info(ch_async_flt_t *ch_flt) 41147c478bd9Sstevel@tonic-gate { 41157c478bd9Sstevel@tonic-gate int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 41167c478bd9Sstevel@tonic-gate int index; 41177c478bd9Sstevel@tonic-gate 41187c478bd9Sstevel@tonic-gate /* 41197c478bd9Sstevel@tonic-gate * Since instruction decode cannot be done at high PIL just 41207c478bd9Sstevel@tonic-gate * examine the entire Pcache to check for any parity errors. 41217c478bd9Sstevel@tonic-gate */ 41227c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 41237c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = -1; 41247c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = -1; 41257c478bd9Sstevel@tonic-gate } 41267c478bd9Sstevel@tonic-gate for (index = 0; index < pc_set_size; index += CH_PCACHE_LSIZE) 41277c478bd9Sstevel@tonic-gate cpu_pcache_parity_check(ch_flt, index); 41287c478bd9Sstevel@tonic-gate } 41297c478bd9Sstevel@tonic-gate 41307c478bd9Sstevel@tonic-gate /* 41317c478bd9Sstevel@tonic-gate * Check all ways of the Pcache at a specified index for good parity. 41327c478bd9Sstevel@tonic-gate */ 41337c478bd9Sstevel@tonic-gate static void 41347c478bd9Sstevel@tonic-gate cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index) 41357c478bd9Sstevel@tonic-gate { 41367c478bd9Sstevel@tonic-gate int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 41377c478bd9Sstevel@tonic-gate int pc_data_words = CH_PC_DATA_REG_SIZE / sizeof (uint64_t); 41387c478bd9Sstevel@tonic-gate int way, word, pbit, parity_bits; 41397c478bd9Sstevel@tonic-gate ch_pc_data_t *pcp = &ch_flt->parity_data.dpe.cpl_pc[0]; 41407c478bd9Sstevel@tonic-gate ch_pc_data_t tmp_pcp; 41417c478bd9Sstevel@tonic-gate 41427c478bd9Sstevel@tonic-gate for (way = 0; way < CH_PCACHE_NWAY; way++, pcp++) { 41437c478bd9Sstevel@tonic-gate /* 41447c478bd9Sstevel@tonic-gate * Perform diagnostic read. 41457c478bd9Sstevel@tonic-gate */ 41467c478bd9Sstevel@tonic-gate get_pcache_dtag(index + way * pc_set_size, 41477c478bd9Sstevel@tonic-gate (uint64_t *)&tmp_pcp); 41487c478bd9Sstevel@tonic-gate /* 41497c478bd9Sstevel@tonic-gate * Check data array for odd parity. There are 8 parity 41507c478bd9Sstevel@tonic-gate * bits (bits 57:50 of ASI_PCACHE_STATUS_DATA) and each 41517c478bd9Sstevel@tonic-gate * of those bits covers exactly 8 bytes of the data 41527c478bd9Sstevel@tonic-gate * array: 41537c478bd9Sstevel@tonic-gate * 41547c478bd9Sstevel@tonic-gate * parity bit P$ data bytes covered 41557c478bd9Sstevel@tonic-gate * ---------- --------------------- 41567c478bd9Sstevel@tonic-gate * 50 63:56 41577c478bd9Sstevel@tonic-gate * 51 55:48 41587c478bd9Sstevel@tonic-gate * 52 47:40 41597c478bd9Sstevel@tonic-gate * 53 39:32 41607c478bd9Sstevel@tonic-gate * 54 31:24 41617c478bd9Sstevel@tonic-gate * 55 23:16 41627c478bd9Sstevel@tonic-gate * 56 15:8 41637c478bd9Sstevel@tonic-gate * 57 7:0 41647c478bd9Sstevel@tonic-gate */ 41657c478bd9Sstevel@tonic-gate parity_bits = PN_PC_PARITY_BITS(tmp_pcp.pc_status); 41667c478bd9Sstevel@tonic-gate for (word = 0; word < pc_data_words; word++) { 41677c478bd9Sstevel@tonic-gate pbit = (parity_bits >> (pc_data_words - word - 1)) & 1; 41687c478bd9Sstevel@tonic-gate if ((popc64(tmp_pcp.pc_data[word]) & 1) ^ pbit) { 41697c478bd9Sstevel@tonic-gate /* 41707c478bd9Sstevel@tonic-gate * If this is the first error log detailed 41717c478bd9Sstevel@tonic-gate * information about it. Otherwise just record 41727c478bd9Sstevel@tonic-gate * the fact that we found another error. 41737c478bd9Sstevel@tonic-gate */ 41747c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 41757c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = way; 41767c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_cache = 41777c478bd9Sstevel@tonic-gate CPU_PC_PARITY; 41787c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = 41797c478bd9Sstevel@tonic-gate word * sizeof (uint64_t); 41807c478bd9Sstevel@tonic-gate bcopy(&tmp_pcp, pcp, 41817c478bd9Sstevel@tonic-gate sizeof (ch_pc_data_t)); 41827c478bd9Sstevel@tonic-gate } 41837c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 41847c478bd9Sstevel@tonic-gate } 41857c478bd9Sstevel@tonic-gate } 41867c478bd9Sstevel@tonic-gate } 41877c478bd9Sstevel@tonic-gate } 41887c478bd9Sstevel@tonic-gate 41897c478bd9Sstevel@tonic-gate 41907c478bd9Sstevel@tonic-gate /* 41917c478bd9Sstevel@tonic-gate * Add L1 Data cache data to the ereport payload. 41927c478bd9Sstevel@tonic-gate */ 41937c478bd9Sstevel@tonic-gate static void 41947c478bd9Sstevel@tonic-gate cpu_payload_add_dcache(struct async_flt *aflt, nvlist_t *nvl) 41957c478bd9Sstevel@tonic-gate { 41967c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 41977c478bd9Sstevel@tonic-gate ch_dc_data_t *dcp; 41987c478bd9Sstevel@tonic-gate ch_dc_data_t dcdata[CH_DCACHE_NWAY]; 41997c478bd9Sstevel@tonic-gate uint_t nelem; 42007c478bd9Sstevel@tonic-gate int i, ways_to_check, ways_logged = 0; 42017c478bd9Sstevel@tonic-gate 42027c478bd9Sstevel@tonic-gate /* 42037c478bd9Sstevel@tonic-gate * If this is an D$ fault then there may be multiple 42047c478bd9Sstevel@tonic-gate * ways captured in the ch_parity_log_t structure. 42057c478bd9Sstevel@tonic-gate * Otherwise, there will be at most one way captured 42067c478bd9Sstevel@tonic-gate * in the ch_diag_data_t struct. 42077c478bd9Sstevel@tonic-gate * Check each way to see if it should be encoded. 42087c478bd9Sstevel@tonic-gate */ 42097c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_DC_PARITY) 42107c478bd9Sstevel@tonic-gate ways_to_check = CH_DCACHE_NWAY; 42117c478bd9Sstevel@tonic-gate else 42127c478bd9Sstevel@tonic-gate ways_to_check = 1; 42137c478bd9Sstevel@tonic-gate for (i = 0; i < ways_to_check; i++) { 42147c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_DC_PARITY) 42157c478bd9Sstevel@tonic-gate dcp = &ch_flt->parity_data.dpe.cpl_dc[i]; 42167c478bd9Sstevel@tonic-gate else 42177c478bd9Sstevel@tonic-gate dcp = &ch_flt->flt_diag_data.chd_dc_data; 42187c478bd9Sstevel@tonic-gate if (dcp->dc_logflag == DC_LOGFLAG_MAGIC) { 42197c478bd9Sstevel@tonic-gate bcopy(dcp, &dcdata[ways_logged], 42207c478bd9Sstevel@tonic-gate sizeof (ch_dc_data_t)); 42217c478bd9Sstevel@tonic-gate ways_logged++; 42227c478bd9Sstevel@tonic-gate } 42237c478bd9Sstevel@tonic-gate } 42247c478bd9Sstevel@tonic-gate 42257c478bd9Sstevel@tonic-gate /* 42267c478bd9Sstevel@tonic-gate * Add the dcache data to the payload. 42277c478bd9Sstevel@tonic-gate */ 42287c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_WAYS, 42297c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 42307c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 42317c478bd9Sstevel@tonic-gate nelem = sizeof (ch_dc_data_t) / sizeof (uint64_t) * ways_logged; 42327c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_DATA, 42337c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)dcdata, NULL); 42347c478bd9Sstevel@tonic-gate } 42357c478bd9Sstevel@tonic-gate } 42367c478bd9Sstevel@tonic-gate 42377c478bd9Sstevel@tonic-gate /* 42387c478bd9Sstevel@tonic-gate * Add L1 Instruction cache data to the ereport payload. 42397c478bd9Sstevel@tonic-gate */ 42407c478bd9Sstevel@tonic-gate static void 42417c478bd9Sstevel@tonic-gate cpu_payload_add_icache(struct async_flt *aflt, nvlist_t *nvl) 42427c478bd9Sstevel@tonic-gate { 42437c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 42447c478bd9Sstevel@tonic-gate ch_ic_data_t *icp; 42457c478bd9Sstevel@tonic-gate ch_ic_data_t icdata[CH_ICACHE_NWAY]; 42467c478bd9Sstevel@tonic-gate uint_t nelem; 42477c478bd9Sstevel@tonic-gate int i, ways_to_check, ways_logged = 0; 42487c478bd9Sstevel@tonic-gate 42497c478bd9Sstevel@tonic-gate /* 42507c478bd9Sstevel@tonic-gate * If this is an I$ fault then there may be multiple 42517c478bd9Sstevel@tonic-gate * ways captured in the ch_parity_log_t structure. 42527c478bd9Sstevel@tonic-gate * Otherwise, there will be at most one way captured 42537c478bd9Sstevel@tonic-gate * in the ch_diag_data_t struct. 42547c478bd9Sstevel@tonic-gate * Check each way to see if it should be encoded. 42557c478bd9Sstevel@tonic-gate */ 42567c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_IC_PARITY) 42577c478bd9Sstevel@tonic-gate ways_to_check = CH_ICACHE_NWAY; 42587c478bd9Sstevel@tonic-gate else 42597c478bd9Sstevel@tonic-gate ways_to_check = 1; 42607c478bd9Sstevel@tonic-gate for (i = 0; i < ways_to_check; i++) { 42617c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_IC_PARITY) 42627c478bd9Sstevel@tonic-gate icp = &ch_flt->parity_data.ipe.cpl_ic[i]; 42637c478bd9Sstevel@tonic-gate else 42647c478bd9Sstevel@tonic-gate icp = &ch_flt->flt_diag_data.chd_ic_data; 42657c478bd9Sstevel@tonic-gate if (icp->ic_logflag == IC_LOGFLAG_MAGIC) { 42667c478bd9Sstevel@tonic-gate bcopy(icp, &icdata[ways_logged], 42677c478bd9Sstevel@tonic-gate sizeof (ch_ic_data_t)); 42687c478bd9Sstevel@tonic-gate ways_logged++; 42697c478bd9Sstevel@tonic-gate } 42707c478bd9Sstevel@tonic-gate } 42717c478bd9Sstevel@tonic-gate 42727c478bd9Sstevel@tonic-gate /* 42737c478bd9Sstevel@tonic-gate * Add the icache data to the payload. 42747c478bd9Sstevel@tonic-gate */ 42757c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_WAYS, 42767c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 42777c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 42787c478bd9Sstevel@tonic-gate nelem = sizeof (ch_ic_data_t) / sizeof (uint64_t) * ways_logged; 42797c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_DATA, 42807c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)icdata, NULL); 42817c478bd9Sstevel@tonic-gate } 42827c478bd9Sstevel@tonic-gate } 42837c478bd9Sstevel@tonic-gate 42847c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 42857c478bd9Sstevel@tonic-gate 42867c478bd9Sstevel@tonic-gate /* 42877c478bd9Sstevel@tonic-gate * Add ecache data to payload. 42887c478bd9Sstevel@tonic-gate */ 42897c478bd9Sstevel@tonic-gate static void 42907c478bd9Sstevel@tonic-gate cpu_payload_add_ecache(struct async_flt *aflt, nvlist_t *nvl) 42917c478bd9Sstevel@tonic-gate { 42927c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 42937c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp; 42947c478bd9Sstevel@tonic-gate ch_ec_data_t ecdata[CHD_EC_DATA_SETS]; 42957c478bd9Sstevel@tonic-gate uint_t nelem; 42967c478bd9Sstevel@tonic-gate int i, ways_logged = 0; 42977c478bd9Sstevel@tonic-gate 42987c478bd9Sstevel@tonic-gate /* 42997c478bd9Sstevel@tonic-gate * Check each way to see if it should be encoded 43007c478bd9Sstevel@tonic-gate * and concatinate it into a temporary buffer. 43017c478bd9Sstevel@tonic-gate */ 43027c478bd9Sstevel@tonic-gate for (i = 0; i < CHD_EC_DATA_SETS; i++) { 43037c478bd9Sstevel@tonic-gate ecp = &ch_flt->flt_diag_data.chd_ec_data[i]; 43047c478bd9Sstevel@tonic-gate if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 43057c478bd9Sstevel@tonic-gate bcopy(ecp, &ecdata[ways_logged], 43067c478bd9Sstevel@tonic-gate sizeof (ch_ec_data_t)); 43077c478bd9Sstevel@tonic-gate ways_logged++; 43087c478bd9Sstevel@tonic-gate } 43097c478bd9Sstevel@tonic-gate } 43107c478bd9Sstevel@tonic-gate 43117c478bd9Sstevel@tonic-gate /* 43127c478bd9Sstevel@tonic-gate * Panther CPUs have an additional level of cache and so 43137c478bd9Sstevel@tonic-gate * what we just collected was the L3 (ecache) and not the 43147c478bd9Sstevel@tonic-gate * L2 cache. 43157c478bd9Sstevel@tonic-gate */ 43167c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 43177c478bd9Sstevel@tonic-gate /* 43187c478bd9Sstevel@tonic-gate * Add the L3 (ecache) data to the payload. 43197c478bd9Sstevel@tonic-gate */ 43207c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_WAYS, 43217c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 43227c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 43237c478bd9Sstevel@tonic-gate nelem = sizeof (ch_ec_data_t) / 43247c478bd9Sstevel@tonic-gate sizeof (uint64_t) * ways_logged; 43257c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_DATA, 43267c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, 43277c478bd9Sstevel@tonic-gate (uint64_t *)ecdata, NULL); 43287c478bd9Sstevel@tonic-gate } 43297c478bd9Sstevel@tonic-gate 43307c478bd9Sstevel@tonic-gate /* 43317c478bd9Sstevel@tonic-gate * Now collect the L2 cache. 43327c478bd9Sstevel@tonic-gate */ 43337c478bd9Sstevel@tonic-gate ways_logged = 0; 43347c478bd9Sstevel@tonic-gate for (i = 0; i < PN_L2_NWAYS; i++) { 43357c478bd9Sstevel@tonic-gate ecp = &ch_flt->flt_diag_data.chd_l2_data[i]; 43367c478bd9Sstevel@tonic-gate if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 43377c478bd9Sstevel@tonic-gate bcopy(ecp, &ecdata[ways_logged], 43387c478bd9Sstevel@tonic-gate sizeof (ch_ec_data_t)); 43397c478bd9Sstevel@tonic-gate ways_logged++; 43407c478bd9Sstevel@tonic-gate } 43417c478bd9Sstevel@tonic-gate } 43427c478bd9Sstevel@tonic-gate } 43437c478bd9Sstevel@tonic-gate 43447c478bd9Sstevel@tonic-gate /* 43457c478bd9Sstevel@tonic-gate * Add the L2 cache data to the payload. 43467c478bd9Sstevel@tonic-gate */ 43477c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_WAYS, 43487c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 43497c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 43507c478bd9Sstevel@tonic-gate nelem = sizeof (ch_ec_data_t) / 43517c478bd9Sstevel@tonic-gate sizeof (uint64_t) * ways_logged; 43527c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_DATA, 43537c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)ecdata, NULL); 43547c478bd9Sstevel@tonic-gate } 43557c478bd9Sstevel@tonic-gate } 43567c478bd9Sstevel@tonic-gate 43577c478bd9Sstevel@tonic-gate /* 43587c478bd9Sstevel@tonic-gate * Encode the data saved in the ch_async_flt_t struct into 43597c478bd9Sstevel@tonic-gate * the FM ereport payload. 43607c478bd9Sstevel@tonic-gate */ 43617c478bd9Sstevel@tonic-gate static void 43627c478bd9Sstevel@tonic-gate cpu_payload_add_aflt(struct async_flt *aflt, nvlist_t *payload, 43637c478bd9Sstevel@tonic-gate nvlist_t *resource, int *afar_status, int *synd_status) 43647c478bd9Sstevel@tonic-gate { 43657c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 43667c478bd9Sstevel@tonic-gate *synd_status = AFLT_STAT_INVALID; 43677c478bd9Sstevel@tonic-gate *afar_status = AFLT_STAT_INVALID; 43687c478bd9Sstevel@tonic-gate 43697c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR) { 43707c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR, 43717c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, aflt->flt_stat, NULL); 43727c478bd9Sstevel@tonic-gate } 43737c478bd9Sstevel@tonic-gate 43747c478bd9Sstevel@tonic-gate if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR_EXT) && 43757c478bd9Sstevel@tonic-gate IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 43767c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR_EXT, 43777c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, ch_flt->afsr_ext, NULL); 43787c478bd9Sstevel@tonic-gate } 43797c478bd9Sstevel@tonic-gate 43807c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR_STATUS) { 43817c478bd9Sstevel@tonic-gate *afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 43827c478bd9Sstevel@tonic-gate ch_flt->flt_bit); 43837c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR_STATUS, 43847c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)*afar_status, NULL); 43857c478bd9Sstevel@tonic-gate } 43867c478bd9Sstevel@tonic-gate 43877c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR) { 43887c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR, 43897c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, aflt->flt_addr, NULL); 43907c478bd9Sstevel@tonic-gate } 43917c478bd9Sstevel@tonic-gate 43927c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PC) { 43937c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PC, 43947c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, (uint64_t)aflt->flt_pc, NULL); 43957c478bd9Sstevel@tonic-gate } 43967c478bd9Sstevel@tonic-gate 43977c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TL) { 43987c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TL, 43997c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)aflt->flt_tl, NULL); 44007c478bd9Sstevel@tonic-gate } 44017c478bd9Sstevel@tonic-gate 44027c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TT) { 44037c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TT, 44047c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, flt_to_trap_type(aflt), NULL); 44057c478bd9Sstevel@tonic-gate } 44067c478bd9Sstevel@tonic-gate 44077c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PRIV) { 44087c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PRIV, 44097c478bd9Sstevel@tonic-gate DATA_TYPE_BOOLEAN_VALUE, 44107c478bd9Sstevel@tonic-gate (aflt->flt_priv ? B_TRUE : B_FALSE), NULL); 44117c478bd9Sstevel@tonic-gate } 44127c478bd9Sstevel@tonic-gate 44137c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ME) { 44147c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ME, 44157c478bd9Sstevel@tonic-gate DATA_TYPE_BOOLEAN_VALUE, 44167c478bd9Sstevel@tonic-gate (aflt->flt_stat & C_AFSR_ME) ? B_TRUE : B_FALSE, NULL); 44177c478bd9Sstevel@tonic-gate } 44187c478bd9Sstevel@tonic-gate 44197c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND_STATUS) { 44207c478bd9Sstevel@tonic-gate *synd_status = afsr_to_synd_status(aflt->flt_inst, 44217c478bd9Sstevel@tonic-gate ch_flt->afsr_errs, ch_flt->flt_bit); 44227c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND_STATUS, 44237c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)*synd_status, NULL); 44247c478bd9Sstevel@tonic-gate } 44257c478bd9Sstevel@tonic-gate 44267c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND) { 44277c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND, 44287c478bd9Sstevel@tonic-gate DATA_TYPE_UINT16, (uint16_t)aflt->flt_synd, NULL); 44297c478bd9Sstevel@tonic-gate } 44307c478bd9Sstevel@tonic-gate 44317c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_TYPE) { 44327c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_TYPE, 44337c478bd9Sstevel@tonic-gate DATA_TYPE_STRING, flt_to_error_type(aflt), NULL); 44347c478bd9Sstevel@tonic-gate } 44357c478bd9Sstevel@tonic-gate 44367c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_DISP) { 44377c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_DISP, 44387c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, aflt->flt_disp, NULL); 44397c478bd9Sstevel@tonic-gate } 44407c478bd9Sstevel@tonic-gate 44417c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L2) 44427c478bd9Sstevel@tonic-gate cpu_payload_add_ecache(aflt, payload); 44437c478bd9Sstevel@tonic-gate 44447c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_COPYFUNCTION) { 44457c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_COPYFUNCTION, 44467c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)aflt->flt_status & 0xff, NULL); 44477c478bd9Sstevel@tonic-gate } 44487c478bd9Sstevel@tonic-gate 44497c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_HOWDETECTED) { 44507c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_HOWDETECTED, 44517c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)(aflt->flt_status >> 8), NULL); 44527c478bd9Sstevel@tonic-gate } 44537c478bd9Sstevel@tonic-gate 44547c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_INSTRBLOCK) { 44557c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_INSTRBLOCK, 44567c478bd9Sstevel@tonic-gate DATA_TYPE_UINT32_ARRAY, 16, 44577c478bd9Sstevel@tonic-gate (uint32_t *)&ch_flt->flt_fpdata, NULL); 44587c478bd9Sstevel@tonic-gate } 44597c478bd9Sstevel@tonic-gate 44607c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 44617c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1D) 44627c478bd9Sstevel@tonic-gate cpu_payload_add_dcache(aflt, payload); 44637c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1I) 44647c478bd9Sstevel@tonic-gate cpu_payload_add_icache(aflt, payload); 44657c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 44667c478bd9Sstevel@tonic-gate 44677c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 44687c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1P) 44697c478bd9Sstevel@tonic-gate cpu_payload_add_pcache(aflt, payload); 44707c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_TLB) 44717c478bd9Sstevel@tonic-gate cpu_payload_add_tlb(aflt, payload); 44727c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 44737c478bd9Sstevel@tonic-gate /* 44747c478bd9Sstevel@tonic-gate * Create the FMRI that goes into the payload 44757c478bd9Sstevel@tonic-gate * and contains the unum info if necessary. 44767c478bd9Sstevel@tonic-gate */ 44777c478bd9Sstevel@tonic-gate if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_RESOURCE) && 44787c478bd9Sstevel@tonic-gate (*afar_status == AFLT_STAT_VALID)) { 4479d00f0155Sayznaga char unum[UNUM_NAMLEN] = ""; 4480d00f0155Sayznaga char sid[DIMM_SERIAL_ID_LEN] = ""; 44817c478bd9Sstevel@tonic-gate int len; 44827c478bd9Sstevel@tonic-gate 44837c478bd9Sstevel@tonic-gate if (cpu_get_mem_unum_aflt(*synd_status, aflt, unum, 44847c478bd9Sstevel@tonic-gate UNUM_NAMLEN, &len) == 0) { 4485d00f0155Sayznaga uint64_t offset = (uint64_t)-1; 4486d00f0155Sayznaga int ret; 4487d00f0155Sayznaga 4488d00f0155Sayznaga ret = cpu_get_mem_sid(unum, sid, DIMM_SERIAL_ID_LEN, 4489d00f0155Sayznaga &len); 4490d00f0155Sayznaga 4491d00f0155Sayznaga if (ret == 0) { 4492d00f0155Sayznaga (void) cpu_get_mem_offset(aflt->flt_addr, 4493d00f0155Sayznaga &offset); 4494d00f0155Sayznaga } 4495d00f0155Sayznaga 44967c478bd9Sstevel@tonic-gate fm_fmri_mem_set(resource, FM_MEM_SCHEME_VERSION, 4497d00f0155Sayznaga NULL, unum, (ret == 0) ? sid : NULL, offset); 44987c478bd9Sstevel@tonic-gate fm_payload_set(payload, 44997c478bd9Sstevel@tonic-gate FM_EREPORT_PAYLOAD_NAME_RESOURCE, 45007c478bd9Sstevel@tonic-gate DATA_TYPE_NVLIST, resource, NULL); 45017c478bd9Sstevel@tonic-gate } 45027c478bd9Sstevel@tonic-gate } 45037c478bd9Sstevel@tonic-gate } 45047c478bd9Sstevel@tonic-gate 45057c478bd9Sstevel@tonic-gate /* 45067c478bd9Sstevel@tonic-gate * Initialize the way info if necessary. 45077c478bd9Sstevel@tonic-gate */ 45087c478bd9Sstevel@tonic-gate void 45097c478bd9Sstevel@tonic-gate cpu_ereport_init(struct async_flt *aflt) 45107c478bd9Sstevel@tonic-gate { 45117c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 45127c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 45137c478bd9Sstevel@tonic-gate ch_ec_data_t *l2p = &ch_flt->flt_diag_data.chd_l2_data[0]; 45147c478bd9Sstevel@tonic-gate int i; 45157c478bd9Sstevel@tonic-gate 45167c478bd9Sstevel@tonic-gate /* 45177c478bd9Sstevel@tonic-gate * Initialize the info in the CPU logout structure. 45187c478bd9Sstevel@tonic-gate * The I$/D$ way information is not initialized here 45197c478bd9Sstevel@tonic-gate * since it is captured in the logout assembly code. 45207c478bd9Sstevel@tonic-gate */ 45217c478bd9Sstevel@tonic-gate for (i = 0; i < CHD_EC_DATA_SETS; i++) 45227c478bd9Sstevel@tonic-gate (ecp + i)->ec_way = i; 45237c478bd9Sstevel@tonic-gate 45247c478bd9Sstevel@tonic-gate for (i = 0; i < PN_L2_NWAYS; i++) 45257c478bd9Sstevel@tonic-gate (l2p + i)->ec_way = i; 45267c478bd9Sstevel@tonic-gate } 45277c478bd9Sstevel@tonic-gate 45287c478bd9Sstevel@tonic-gate /* 45297c478bd9Sstevel@tonic-gate * Returns whether fault address is valid for this error bit and 45307c478bd9Sstevel@tonic-gate * whether the address is "in memory" (i.e. pf_is_memory returns 1). 45317c478bd9Sstevel@tonic-gate */ 45327c478bd9Sstevel@tonic-gate int 45337c478bd9Sstevel@tonic-gate cpu_flt_in_memory(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit) 45347c478bd9Sstevel@tonic-gate { 45357c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 45367c478bd9Sstevel@tonic-gate 45377c478bd9Sstevel@tonic-gate return ((aflt->flt_stat & C_AFSR_MEMORY) && 45387c478bd9Sstevel@tonic-gate afsr_to_afar_status(ch_flt->afsr_errs, t_afsr_bit) == 45397c478bd9Sstevel@tonic-gate AFLT_STAT_VALID && 45407c478bd9Sstevel@tonic-gate pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)); 45417c478bd9Sstevel@tonic-gate } 45427c478bd9Sstevel@tonic-gate 45437c478bd9Sstevel@tonic-gate static void 45447c478bd9Sstevel@tonic-gate cpu_log_diag_info(ch_async_flt_t *ch_flt) 45457c478bd9Sstevel@tonic-gate { 45467c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 45477c478bd9Sstevel@tonic-gate ch_dc_data_t *dcp = &ch_flt->flt_diag_data.chd_dc_data; 45487c478bd9Sstevel@tonic-gate ch_ic_data_t *icp = &ch_flt->flt_diag_data.chd_ic_data; 45497c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 45507c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 45517c478bd9Sstevel@tonic-gate int i, nway; 45527c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_ECACHE_ASSOC */ 45537c478bd9Sstevel@tonic-gate 45547c478bd9Sstevel@tonic-gate /* 45557c478bd9Sstevel@tonic-gate * Check if the CPU log out captured was valid. 45567c478bd9Sstevel@tonic-gate */ 45577c478bd9Sstevel@tonic-gate if (ch_flt->flt_diag_data.chd_afar == LOGOUT_INVALID || 45587c478bd9Sstevel@tonic-gate ch_flt->flt_data_incomplete) 45597c478bd9Sstevel@tonic-gate return; 45607c478bd9Sstevel@tonic-gate 45617c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 45627c478bd9Sstevel@tonic-gate nway = cpu_ecache_nway(); 45637c478bd9Sstevel@tonic-gate i = cpu_ecache_line_valid(ch_flt); 45647c478bd9Sstevel@tonic-gate if (i == 0 || i > nway) { 45657c478bd9Sstevel@tonic-gate for (i = 0; i < nway; i++) 45667c478bd9Sstevel@tonic-gate ecp[i].ec_logflag = EC_LOGFLAG_MAGIC; 45677c478bd9Sstevel@tonic-gate } else 45687c478bd9Sstevel@tonic-gate ecp[i - 1].ec_logflag = EC_LOGFLAG_MAGIC; 45697c478bd9Sstevel@tonic-gate #else /* CPU_IMP_ECACHE_ASSOC */ 45707c478bd9Sstevel@tonic-gate ecp->ec_logflag = EC_LOGFLAG_MAGIC; 45717c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_ECACHE_ASSOC */ 45727c478bd9Sstevel@tonic-gate 45737c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 45747c478bd9Sstevel@tonic-gate pn_cpu_log_diag_l2_info(ch_flt); 45757c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 45767c478bd9Sstevel@tonic-gate 45777c478bd9Sstevel@tonic-gate if (CH_DCTAG_MATCH(dcp->dc_tag, aflt->flt_addr)) { 45787c478bd9Sstevel@tonic-gate dcp->dc_way = CH_DCIDX_TO_WAY(dcp->dc_idx); 45797c478bd9Sstevel@tonic-gate dcp->dc_logflag = DC_LOGFLAG_MAGIC; 45807c478bd9Sstevel@tonic-gate } 45817c478bd9Sstevel@tonic-gate 45827c478bd9Sstevel@tonic-gate if (CH_ICTAG_MATCH(icp, aflt->flt_addr)) { 45837c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 45847c478bd9Sstevel@tonic-gate icp->ic_way = PN_ICIDX_TO_WAY(icp->ic_idx); 45857c478bd9Sstevel@tonic-gate else 45867c478bd9Sstevel@tonic-gate icp->ic_way = CH_ICIDX_TO_WAY(icp->ic_idx); 45877c478bd9Sstevel@tonic-gate icp->ic_logflag = IC_LOGFLAG_MAGIC; 45887c478bd9Sstevel@tonic-gate } 45897c478bd9Sstevel@tonic-gate } 45907c478bd9Sstevel@tonic-gate 45917c478bd9Sstevel@tonic-gate /* 45927c478bd9Sstevel@tonic-gate * Cheetah ECC calculation. 45937c478bd9Sstevel@tonic-gate * 45947c478bd9Sstevel@tonic-gate * We only need to do the calculation on the data bits and can ignore check 45957c478bd9Sstevel@tonic-gate * bit and Mtag bit terms in the calculation. 45967c478bd9Sstevel@tonic-gate */ 45977c478bd9Sstevel@tonic-gate static uint64_t ch_ecc_table[9][2] = { 45987c478bd9Sstevel@tonic-gate /* 45997c478bd9Sstevel@tonic-gate * low order 64-bits high-order 64-bits 46007c478bd9Sstevel@tonic-gate */ 46017c478bd9Sstevel@tonic-gate { 0x46bffffeccd1177f, 0x488800022100014c }, 46027c478bd9Sstevel@tonic-gate { 0x42fccc81331ff77f, 0x14424f1010249184 }, 46037c478bd9Sstevel@tonic-gate { 0x8898827c222f1ffe, 0x22c1222808184aaf }, 46047c478bd9Sstevel@tonic-gate { 0xf7632203e131ccf1, 0xe1241121848292b8 }, 46057c478bd9Sstevel@tonic-gate { 0x7f5511421b113809, 0x901c88d84288aafe }, 46067c478bd9Sstevel@tonic-gate { 0x1d49412184882487, 0x8f338c87c044c6ef }, 46077c478bd9Sstevel@tonic-gate { 0xf552181014448344, 0x7ff8f4443e411911 }, 46087c478bd9Sstevel@tonic-gate { 0x2189240808f24228, 0xfeeff8cc81333f42 }, 46097c478bd9Sstevel@tonic-gate { 0x3280008440001112, 0xfee88b337ffffd62 }, 46107c478bd9Sstevel@tonic-gate }; 46117c478bd9Sstevel@tonic-gate 46127c478bd9Sstevel@tonic-gate /* 46137c478bd9Sstevel@tonic-gate * 64-bit population count, use well-known popcnt trick. 46147c478bd9Sstevel@tonic-gate * We could use the UltraSPARC V9 POPC instruction, but some 46157c478bd9Sstevel@tonic-gate * CPUs including Cheetahplus and Jaguar do not support that 46167c478bd9Sstevel@tonic-gate * instruction. 46177c478bd9Sstevel@tonic-gate */ 46187c478bd9Sstevel@tonic-gate int 46197c478bd9Sstevel@tonic-gate popc64(uint64_t val) 46207c478bd9Sstevel@tonic-gate { 46217c478bd9Sstevel@tonic-gate int cnt; 46227c478bd9Sstevel@tonic-gate 46237c478bd9Sstevel@tonic-gate for (cnt = 0; val != 0; val &= val - 1) 46247c478bd9Sstevel@tonic-gate cnt++; 46257c478bd9Sstevel@tonic-gate return (cnt); 46267c478bd9Sstevel@tonic-gate } 46277c478bd9Sstevel@tonic-gate 46287c478bd9Sstevel@tonic-gate /* 46297c478bd9Sstevel@tonic-gate * Generate the 9 ECC bits for the 128-bit chunk based on the table above. 46307c478bd9Sstevel@tonic-gate * Note that xor'ing an odd number of 1 bits == 1 and xor'ing an even number 46317c478bd9Sstevel@tonic-gate * of 1 bits == 0, so we can just use the least significant bit of the popcnt 46327c478bd9Sstevel@tonic-gate * instead of doing all the xor's. 46337c478bd9Sstevel@tonic-gate */ 46347c478bd9Sstevel@tonic-gate uint32_t 46357c478bd9Sstevel@tonic-gate us3_gen_ecc(uint64_t data_low, uint64_t data_high) 46367c478bd9Sstevel@tonic-gate { 46377c478bd9Sstevel@tonic-gate int bitno, s; 46387c478bd9Sstevel@tonic-gate int synd = 0; 46397c478bd9Sstevel@tonic-gate 46407c478bd9Sstevel@tonic-gate for (bitno = 0; bitno < 9; bitno++) { 46417c478bd9Sstevel@tonic-gate s = (popc64(data_low & ch_ecc_table[bitno][0]) + 46427c478bd9Sstevel@tonic-gate popc64(data_high & ch_ecc_table[bitno][1])) & 1; 46437c478bd9Sstevel@tonic-gate synd |= (s << bitno); 46447c478bd9Sstevel@tonic-gate } 46457c478bd9Sstevel@tonic-gate return (synd); 46467c478bd9Sstevel@tonic-gate 46477c478bd9Sstevel@tonic-gate } 46487c478bd9Sstevel@tonic-gate 46497c478bd9Sstevel@tonic-gate /* 46507c478bd9Sstevel@tonic-gate * Queue one event based on ecc_type_to_info entry. If the event has an AFT1 46517c478bd9Sstevel@tonic-gate * tag associated with it or is a fatal event (aflt_panic set), it is sent to 46527c478bd9Sstevel@tonic-gate * the UE event queue. Otherwise it is dispatched to the CE event queue. 46537c478bd9Sstevel@tonic-gate */ 46547c478bd9Sstevel@tonic-gate static void 46557c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 46567c478bd9Sstevel@tonic-gate ecc_type_to_info_t *eccp, ch_diag_data_t *cdp) 46577c478bd9Sstevel@tonic-gate { 46587c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 46597c478bd9Sstevel@tonic-gate 46607c478bd9Sstevel@tonic-gate if (reason && 46617c478bd9Sstevel@tonic-gate strlen(reason) + strlen(eccp->ec_reason) < MAX_REASON_STRING) { 46627c478bd9Sstevel@tonic-gate (void) strcat(reason, eccp->ec_reason); 46637c478bd9Sstevel@tonic-gate } 46647c478bd9Sstevel@tonic-gate 46657c478bd9Sstevel@tonic-gate ch_flt->flt_bit = eccp->ec_afsr_bit; 46667c478bd9Sstevel@tonic-gate ch_flt->flt_type = eccp->ec_flt_type; 46677c478bd9Sstevel@tonic-gate if (cdp != NULL && cdp->chd_afar != LOGOUT_INVALID) 46687c478bd9Sstevel@tonic-gate ch_flt->flt_diag_data = *cdp; 46697c478bd9Sstevel@tonic-gate else 46707c478bd9Sstevel@tonic-gate ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 46717c478bd9Sstevel@tonic-gate aflt->flt_in_memory = cpu_flt_in_memory(ch_flt, ch_flt->flt_bit); 46727c478bd9Sstevel@tonic-gate 46737c478bd9Sstevel@tonic-gate if (ch_flt->flt_bit & C_AFSR_MSYND_ERRS) 46747c478bd9Sstevel@tonic-gate aflt->flt_synd = GET_M_SYND(aflt->flt_stat); 46757c478bd9Sstevel@tonic-gate else if (ch_flt->flt_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) 46767c478bd9Sstevel@tonic-gate aflt->flt_synd = GET_E_SYND(aflt->flt_stat); 46777c478bd9Sstevel@tonic-gate else 46787c478bd9Sstevel@tonic-gate aflt->flt_synd = 0; 46797c478bd9Sstevel@tonic-gate 46807c478bd9Sstevel@tonic-gate aflt->flt_payload = eccp->ec_err_payload; 46817c478bd9Sstevel@tonic-gate 46827c478bd9Sstevel@tonic-gate if (aflt->flt_panic || (eccp->ec_afsr_bit & 46837c478bd9Sstevel@tonic-gate (C_AFSR_LEVEL1 | C_AFSR_EXT_LEVEL1))) 46847c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(eccp->ec_err_class, 46857c478bd9Sstevel@tonic-gate (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 46867c478bd9Sstevel@tonic-gate aflt->flt_panic); 46877c478bd9Sstevel@tonic-gate else 46887c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(eccp->ec_err_class, 46897c478bd9Sstevel@tonic-gate (void *)ch_flt, sizeof (ch_async_flt_t), ce_queue, 46907c478bd9Sstevel@tonic-gate aflt->flt_panic); 46917c478bd9Sstevel@tonic-gate } 46927c478bd9Sstevel@tonic-gate 46937c478bd9Sstevel@tonic-gate /* 46947c478bd9Sstevel@tonic-gate * Queue events on async event queue one event per error bit. First we 46957c478bd9Sstevel@tonic-gate * queue the events that we "expect" for the given trap, then we queue events 46967c478bd9Sstevel@tonic-gate * that we may not expect. Return number of events queued. 46977c478bd9Sstevel@tonic-gate */ 46987c478bd9Sstevel@tonic-gate int 46997c478bd9Sstevel@tonic-gate cpu_queue_events(ch_async_flt_t *ch_flt, char *reason, uint64_t t_afsr_errs, 47007c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop) 47017c478bd9Sstevel@tonic-gate { 47027c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 47037c478bd9Sstevel@tonic-gate ecc_type_to_info_t *eccp; 47047c478bd9Sstevel@tonic-gate int nevents = 0; 47057c478bd9Sstevel@tonic-gate uint64_t primary_afar = aflt->flt_addr, primary_afsr = aflt->flt_stat; 47067c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 47077c478bd9Sstevel@tonic-gate uint64_t orig_t_afsr_errs; 47087c478bd9Sstevel@tonic-gate #endif 47097c478bd9Sstevel@tonic-gate uint64_t primary_afsr_ext = ch_flt->afsr_ext; 47107c478bd9Sstevel@tonic-gate uint64_t primary_afsr_errs = ch_flt->afsr_errs; 47117c478bd9Sstevel@tonic-gate ch_diag_data_t *cdp = NULL; 47127c478bd9Sstevel@tonic-gate 47137c478bd9Sstevel@tonic-gate t_afsr_errs &= ((C_AFSR_ALL_ERRS & ~C_AFSR_ME) | C_AFSR_EXT_ALL_ERRS); 47147c478bd9Sstevel@tonic-gate 47157c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 47167c478bd9Sstevel@tonic-gate orig_t_afsr_errs = t_afsr_errs; 47177c478bd9Sstevel@tonic-gate 47187c478bd9Sstevel@tonic-gate /* 47197c478bd9Sstevel@tonic-gate * For Cheetah+, log the shadow AFSR/AFAR bits first. 47207c478bd9Sstevel@tonic-gate */ 47217c478bd9Sstevel@tonic-gate if (clop != NULL) { 47227c478bd9Sstevel@tonic-gate /* 47237c478bd9Sstevel@tonic-gate * Set the AFSR and AFAR fields to the shadow registers. The 47247c478bd9Sstevel@tonic-gate * flt_addr and flt_stat fields will be reset to the primaries 47257c478bd9Sstevel@tonic-gate * below, but the sdw_addr and sdw_stat will stay as the 47267c478bd9Sstevel@tonic-gate * secondaries. 47277c478bd9Sstevel@tonic-gate */ 47287c478bd9Sstevel@tonic-gate cdp = &clop->clo_sdw_data; 47297c478bd9Sstevel@tonic-gate aflt->flt_addr = ch_flt->flt_sdw_afar = cdp->chd_afar; 47307c478bd9Sstevel@tonic-gate aflt->flt_stat = ch_flt->flt_sdw_afsr = cdp->chd_afsr; 47317c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = ch_flt->flt_sdw_afsr_ext = cdp->chd_afsr_ext; 47327c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = (cdp->chd_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 47337c478bd9Sstevel@tonic-gate (cdp->chd_afsr & C_AFSR_ALL_ERRS); 47347c478bd9Sstevel@tonic-gate 47357c478bd9Sstevel@tonic-gate /* 47367c478bd9Sstevel@tonic-gate * If the primary and shadow AFSR differ, tag the shadow as 47377c478bd9Sstevel@tonic-gate * the first fault. 47387c478bd9Sstevel@tonic-gate */ 47397c478bd9Sstevel@tonic-gate if ((primary_afar != cdp->chd_afar) || 47407c478bd9Sstevel@tonic-gate (primary_afsr_errs != ch_flt->afsr_errs)) { 47417c478bd9Sstevel@tonic-gate aflt->flt_stat |= (1ull << C_AFSR_FIRSTFLT_SHIFT); 47427c478bd9Sstevel@tonic-gate } 47437c478bd9Sstevel@tonic-gate 47447c478bd9Sstevel@tonic-gate /* 47457c478bd9Sstevel@tonic-gate * Check AFSR bits as well as AFSR_EXT bits in order of 47467c478bd9Sstevel@tonic-gate * the AFAR overwrite priority. Our stored AFSR_EXT value 47477c478bd9Sstevel@tonic-gate * is expected to be zero for those CPUs which do not have 47487c478bd9Sstevel@tonic-gate * an AFSR_EXT register. 47497c478bd9Sstevel@tonic-gate */ 47507c478bd9Sstevel@tonic-gate for (eccp = ecc_type_to_info; eccp->ec_desc != NULL; eccp++) { 47517c478bd9Sstevel@tonic-gate if ((eccp->ec_afsr_bit & 47527c478bd9Sstevel@tonic-gate (ch_flt->afsr_errs & t_afsr_errs)) && 47537c478bd9Sstevel@tonic-gate ((eccp->ec_flags & aflt->flt_status) != 0)) { 47547c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_flt, reason, eccp, cdp); 47557c478bd9Sstevel@tonic-gate cdp = NULL; 47567c478bd9Sstevel@tonic-gate t_afsr_errs &= ~eccp->ec_afsr_bit; 47577c478bd9Sstevel@tonic-gate nevents++; 47587c478bd9Sstevel@tonic-gate } 47597c478bd9Sstevel@tonic-gate } 47607c478bd9Sstevel@tonic-gate 47617c478bd9Sstevel@tonic-gate /* 47627c478bd9Sstevel@tonic-gate * If the ME bit is on in the primary AFSR turn all the 47637c478bd9Sstevel@tonic-gate * error bits on again that may set the ME bit to make 47647c478bd9Sstevel@tonic-gate * sure we see the ME AFSR error logs. 47657c478bd9Sstevel@tonic-gate */ 47667c478bd9Sstevel@tonic-gate if ((primary_afsr & C_AFSR_ME) != 0) 47677c478bd9Sstevel@tonic-gate t_afsr_errs = (orig_t_afsr_errs & C_AFSR_ALL_ME_ERRS); 47687c478bd9Sstevel@tonic-gate } 47697c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 47707c478bd9Sstevel@tonic-gate 47717c478bd9Sstevel@tonic-gate if (clop != NULL) 47727c478bd9Sstevel@tonic-gate cdp = &clop->clo_data; 47737c478bd9Sstevel@tonic-gate 47747c478bd9Sstevel@tonic-gate /* 47757c478bd9Sstevel@tonic-gate * Queue expected errors, error bit and fault type must match 47767c478bd9Sstevel@tonic-gate * in the ecc_type_to_info table. 47777c478bd9Sstevel@tonic-gate */ 47787c478bd9Sstevel@tonic-gate for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 47797c478bd9Sstevel@tonic-gate eccp++) { 47807c478bd9Sstevel@tonic-gate if ((eccp->ec_afsr_bit & t_afsr_errs) != 0 && 47817c478bd9Sstevel@tonic-gate (eccp->ec_flags & aflt->flt_status) != 0) { 47827c478bd9Sstevel@tonic-gate #if defined(SERRANO) 47837c478bd9Sstevel@tonic-gate /* 47847c478bd9Sstevel@tonic-gate * For FRC/FRU errors on Serrano the afar2 captures 47857c478bd9Sstevel@tonic-gate * the address and the associated data is 47867c478bd9Sstevel@tonic-gate * in the shadow logout area. 47877c478bd9Sstevel@tonic-gate */ 47887c478bd9Sstevel@tonic-gate if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 47897c478bd9Sstevel@tonic-gate if (clop != NULL) 47907c478bd9Sstevel@tonic-gate cdp = &clop->clo_sdw_data; 47917c478bd9Sstevel@tonic-gate aflt->flt_addr = ch_flt->afar2; 47927c478bd9Sstevel@tonic-gate } else { 47937c478bd9Sstevel@tonic-gate if (clop != NULL) 47947c478bd9Sstevel@tonic-gate cdp = &clop->clo_data; 47957c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 47967c478bd9Sstevel@tonic-gate } 47977c478bd9Sstevel@tonic-gate #else /* SERRANO */ 47987c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 47997c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 48007c478bd9Sstevel@tonic-gate aflt->flt_stat = primary_afsr; 48017c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = primary_afsr_ext; 48027c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = primary_afsr_errs; 48037c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_flt, reason, eccp, cdp); 48047c478bd9Sstevel@tonic-gate cdp = NULL; 48057c478bd9Sstevel@tonic-gate t_afsr_errs &= ~eccp->ec_afsr_bit; 48067c478bd9Sstevel@tonic-gate nevents++; 48077c478bd9Sstevel@tonic-gate } 48087c478bd9Sstevel@tonic-gate } 48097c478bd9Sstevel@tonic-gate 48107c478bd9Sstevel@tonic-gate /* 48117c478bd9Sstevel@tonic-gate * Queue unexpected errors, error bit only match. 48127c478bd9Sstevel@tonic-gate */ 48137c478bd9Sstevel@tonic-gate for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 48147c478bd9Sstevel@tonic-gate eccp++) { 48157c478bd9Sstevel@tonic-gate if (eccp->ec_afsr_bit & t_afsr_errs) { 48167c478bd9Sstevel@tonic-gate #if defined(SERRANO) 48177c478bd9Sstevel@tonic-gate /* 48187c478bd9Sstevel@tonic-gate * For FRC/FRU errors on Serrano the afar2 captures 48197c478bd9Sstevel@tonic-gate * the address and the associated data is 48207c478bd9Sstevel@tonic-gate * in the shadow logout area. 48217c478bd9Sstevel@tonic-gate */ 48227c478bd9Sstevel@tonic-gate if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 48237c478bd9Sstevel@tonic-gate if (clop != NULL) 48247c478bd9Sstevel@tonic-gate cdp = &clop->clo_sdw_data; 48257c478bd9Sstevel@tonic-gate aflt->flt_addr = ch_flt->afar2; 48267c478bd9Sstevel@tonic-gate } else { 48277c478bd9Sstevel@tonic-gate if (clop != NULL) 48287c478bd9Sstevel@tonic-gate cdp = &clop->clo_data; 48297c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 48307c478bd9Sstevel@tonic-gate } 48317c478bd9Sstevel@tonic-gate #else /* SERRANO */ 48327c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 48337c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 48347c478bd9Sstevel@tonic-gate aflt->flt_stat = primary_afsr; 48357c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = primary_afsr_ext; 48367c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = primary_afsr_errs; 48377c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_flt, reason, eccp, cdp); 48387c478bd9Sstevel@tonic-gate cdp = NULL; 48397c478bd9Sstevel@tonic-gate t_afsr_errs &= ~eccp->ec_afsr_bit; 48407c478bd9Sstevel@tonic-gate nevents++; 48417c478bd9Sstevel@tonic-gate } 48427c478bd9Sstevel@tonic-gate } 48437c478bd9Sstevel@tonic-gate return (nevents); 48447c478bd9Sstevel@tonic-gate } 48457c478bd9Sstevel@tonic-gate 48467c478bd9Sstevel@tonic-gate /* 48477c478bd9Sstevel@tonic-gate * Return trap type number. 48487c478bd9Sstevel@tonic-gate */ 48497c478bd9Sstevel@tonic-gate uint8_t 48507c478bd9Sstevel@tonic-gate flt_to_trap_type(struct async_flt *aflt) 48517c478bd9Sstevel@tonic-gate { 48527c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_I_TRAP) 48537c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_I); 48547c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_D_TRAP) 48557c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_D); 48567c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_F_TRAP) 48577c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_F); 48587c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_C_TRAP) 48597c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_C); 48607c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_DP_TRAP) 48617c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_DP); 48627c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_IP_TRAP) 48637c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_IP); 48647c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_ITLB_TRAP) 48657c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_ITLB); 48667c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_DTLB_TRAP) 48677c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_DTLB); 48687c478bd9Sstevel@tonic-gate return (TRAP_TYPE_UNKNOWN); 48697c478bd9Sstevel@tonic-gate } 48707c478bd9Sstevel@tonic-gate 48717c478bd9Sstevel@tonic-gate /* 48727c478bd9Sstevel@tonic-gate * Decide an error type based on detector and leaky/partner tests. 48737c478bd9Sstevel@tonic-gate * The following array is used for quick translation - it must 48747c478bd9Sstevel@tonic-gate * stay in sync with ce_dispact_t. 48757c478bd9Sstevel@tonic-gate */ 48767c478bd9Sstevel@tonic-gate 48777c478bd9Sstevel@tonic-gate static char *cetypes[] = { 48787c478bd9Sstevel@tonic-gate CE_DISP_DESC_U, 48797c478bd9Sstevel@tonic-gate CE_DISP_DESC_I, 48807c478bd9Sstevel@tonic-gate CE_DISP_DESC_PP, 48817c478bd9Sstevel@tonic-gate CE_DISP_DESC_P, 48827c478bd9Sstevel@tonic-gate CE_DISP_DESC_L, 48837c478bd9Sstevel@tonic-gate CE_DISP_DESC_PS, 48847c478bd9Sstevel@tonic-gate CE_DISP_DESC_S 48857c478bd9Sstevel@tonic-gate }; 48867c478bd9Sstevel@tonic-gate 48877c478bd9Sstevel@tonic-gate char * 48887c478bd9Sstevel@tonic-gate flt_to_error_type(struct async_flt *aflt) 48897c478bd9Sstevel@tonic-gate { 48907c478bd9Sstevel@tonic-gate ce_dispact_t dispact, disp; 48917c478bd9Sstevel@tonic-gate uchar_t dtcrinfo, ptnrinfo, lkyinfo; 48927c478bd9Sstevel@tonic-gate 48937c478bd9Sstevel@tonic-gate /* 48947c478bd9Sstevel@tonic-gate * The memory payload bundle is shared by some events that do 48957c478bd9Sstevel@tonic-gate * not perform any classification. For those flt_disp will be 48967c478bd9Sstevel@tonic-gate * 0 and we will return "unknown". 48977c478bd9Sstevel@tonic-gate */ 48987c478bd9Sstevel@tonic-gate if (!ce_disp_inited || !aflt->flt_in_memory || aflt->flt_disp == 0) 48997c478bd9Sstevel@tonic-gate return (cetypes[CE_DISP_UNKNOWN]); 49007c478bd9Sstevel@tonic-gate 49017c478bd9Sstevel@tonic-gate dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 49027c478bd9Sstevel@tonic-gate 49037c478bd9Sstevel@tonic-gate /* 49047c478bd9Sstevel@tonic-gate * It is also possible that no scrub/classification was performed 49057c478bd9Sstevel@tonic-gate * by the detector, for instance where a disrupting error logged 49067c478bd9Sstevel@tonic-gate * in the AFSR while CEEN was off in cpu_deferred_error. 49077c478bd9Sstevel@tonic-gate */ 49087c478bd9Sstevel@tonic-gate if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) 49097c478bd9Sstevel@tonic-gate return (cetypes[CE_DISP_UNKNOWN]); 49107c478bd9Sstevel@tonic-gate 49117c478bd9Sstevel@tonic-gate /* 49127c478bd9Sstevel@tonic-gate * Lookup type in initial classification/action table 49137c478bd9Sstevel@tonic-gate */ 49147c478bd9Sstevel@tonic-gate dispact = CE_DISPACT(ce_disp_table, 49157c478bd9Sstevel@tonic-gate CE_XDIAG_AFARMATCHED(dtcrinfo), 49167c478bd9Sstevel@tonic-gate CE_XDIAG_STATE(dtcrinfo), 49177c478bd9Sstevel@tonic-gate CE_XDIAG_CE1SEEN(dtcrinfo), 49187c478bd9Sstevel@tonic-gate CE_XDIAG_CE2SEEN(dtcrinfo)); 49197c478bd9Sstevel@tonic-gate 49207c478bd9Sstevel@tonic-gate /* 49217c478bd9Sstevel@tonic-gate * A bad lookup is not something to panic production systems for. 49227c478bd9Sstevel@tonic-gate */ 49237c478bd9Sstevel@tonic-gate ASSERT(dispact != CE_DISP_BAD); 49247c478bd9Sstevel@tonic-gate if (dispact == CE_DISP_BAD) 49257c478bd9Sstevel@tonic-gate return (cetypes[CE_DISP_UNKNOWN]); 49267c478bd9Sstevel@tonic-gate 49277c478bd9Sstevel@tonic-gate disp = CE_DISP(dispact); 49287c478bd9Sstevel@tonic-gate 49297c478bd9Sstevel@tonic-gate switch (disp) { 49307c478bd9Sstevel@tonic-gate case CE_DISP_UNKNOWN: 49317c478bd9Sstevel@tonic-gate case CE_DISP_INTERMITTENT: 49327c478bd9Sstevel@tonic-gate break; 49337c478bd9Sstevel@tonic-gate 49347c478bd9Sstevel@tonic-gate case CE_DISP_POSS_PERS: 49357c478bd9Sstevel@tonic-gate /* 49367c478bd9Sstevel@tonic-gate * "Possible persistent" errors to which we have applied a valid 49377c478bd9Sstevel@tonic-gate * leaky test can be separated into "persistent" or "leaky". 49387c478bd9Sstevel@tonic-gate */ 49397c478bd9Sstevel@tonic-gate lkyinfo = CE_XDIAG_LKYINFO(aflt->flt_disp); 49407c478bd9Sstevel@tonic-gate if (CE_XDIAG_TESTVALID(lkyinfo)) { 49417c478bd9Sstevel@tonic-gate if (CE_XDIAG_CE1SEEN(lkyinfo) || 49427c478bd9Sstevel@tonic-gate CE_XDIAG_CE2SEEN(lkyinfo)) 49437c478bd9Sstevel@tonic-gate disp = CE_DISP_LEAKY; 49447c478bd9Sstevel@tonic-gate else 49457c478bd9Sstevel@tonic-gate disp = CE_DISP_PERS; 49467c478bd9Sstevel@tonic-gate } 49477c478bd9Sstevel@tonic-gate break; 49487c478bd9Sstevel@tonic-gate 49497c478bd9Sstevel@tonic-gate case CE_DISP_POSS_STICKY: 49507c478bd9Sstevel@tonic-gate /* 49517c478bd9Sstevel@tonic-gate * Promote "possible sticky" results that have been 49527c478bd9Sstevel@tonic-gate * confirmed by a partner test to "sticky". Unconfirmed 49537c478bd9Sstevel@tonic-gate * "possible sticky" events are left at that status - we do not 49547c478bd9Sstevel@tonic-gate * guess at any bad reader/writer etc status here. 49557c478bd9Sstevel@tonic-gate */ 49567c478bd9Sstevel@tonic-gate ptnrinfo = CE_XDIAG_PTNRINFO(aflt->flt_disp); 49577c478bd9Sstevel@tonic-gate if (CE_XDIAG_TESTVALID(ptnrinfo) && 49587c478bd9Sstevel@tonic-gate CE_XDIAG_CE1SEEN(ptnrinfo) && CE_XDIAG_CE2SEEN(ptnrinfo)) 49597c478bd9Sstevel@tonic-gate disp = CE_DISP_STICKY; 49607c478bd9Sstevel@tonic-gate 49617c478bd9Sstevel@tonic-gate /* 49627c478bd9Sstevel@tonic-gate * Promote "possible sticky" results on a uniprocessor 49637c478bd9Sstevel@tonic-gate * to "sticky" 49647c478bd9Sstevel@tonic-gate */ 49657c478bd9Sstevel@tonic-gate if (disp == CE_DISP_POSS_STICKY && 49667c478bd9Sstevel@tonic-gate CE_XDIAG_SKIPCODE(disp) == CE_XDIAG_SKIP_UNIPROC) 49677c478bd9Sstevel@tonic-gate disp = CE_DISP_STICKY; 49687c478bd9Sstevel@tonic-gate break; 49697c478bd9Sstevel@tonic-gate 49707c478bd9Sstevel@tonic-gate default: 49717c478bd9Sstevel@tonic-gate disp = CE_DISP_UNKNOWN; 49727c478bd9Sstevel@tonic-gate break; 49737c478bd9Sstevel@tonic-gate } 49747c478bd9Sstevel@tonic-gate 49757c478bd9Sstevel@tonic-gate return (cetypes[disp]); 49767c478bd9Sstevel@tonic-gate } 49777c478bd9Sstevel@tonic-gate 49787c478bd9Sstevel@tonic-gate /* 49797c478bd9Sstevel@tonic-gate * Given the entire afsr, the specific bit to check and a prioritized list of 49807c478bd9Sstevel@tonic-gate * error bits, determine the validity of the various overwrite priority 49817c478bd9Sstevel@tonic-gate * features of the AFSR/AFAR: AFAR, ESYND and MSYND, each of which have 49827c478bd9Sstevel@tonic-gate * different overwrite priorities. 49837c478bd9Sstevel@tonic-gate * 49847c478bd9Sstevel@tonic-gate * Given a specific afsr error bit and the entire afsr, there are three cases: 49857c478bd9Sstevel@tonic-gate * INVALID: The specified bit is lower overwrite priority than some other 49867c478bd9Sstevel@tonic-gate * error bit which is on in the afsr (or IVU/IVC). 49877c478bd9Sstevel@tonic-gate * VALID: The specified bit is higher priority than all other error bits 49887c478bd9Sstevel@tonic-gate * which are on in the afsr. 49897c478bd9Sstevel@tonic-gate * AMBIGUOUS: Another error bit (or bits) of equal priority to the specified 49907c478bd9Sstevel@tonic-gate * bit is on in the afsr. 49917c478bd9Sstevel@tonic-gate */ 49927c478bd9Sstevel@tonic-gate int 49937c478bd9Sstevel@tonic-gate afsr_to_overw_status(uint64_t afsr, uint64_t afsr_bit, uint64_t *ow_bits) 49947c478bd9Sstevel@tonic-gate { 49957c478bd9Sstevel@tonic-gate uint64_t afsr_ow; 49967c478bd9Sstevel@tonic-gate 49977c478bd9Sstevel@tonic-gate while ((afsr_ow = *ow_bits++) != 0) { 49987c478bd9Sstevel@tonic-gate /* 49997c478bd9Sstevel@tonic-gate * If bit is in the priority class, check to see if another 50007c478bd9Sstevel@tonic-gate * bit in the same class is on => ambiguous. Otherwise, 50017c478bd9Sstevel@tonic-gate * the value is valid. If the bit is not on at this priority 50027c478bd9Sstevel@tonic-gate * class, but a higher priority bit is on, then the value is 50037c478bd9Sstevel@tonic-gate * invalid. 50047c478bd9Sstevel@tonic-gate */ 50057c478bd9Sstevel@tonic-gate if (afsr_ow & afsr_bit) { 50067c478bd9Sstevel@tonic-gate /* 50077c478bd9Sstevel@tonic-gate * If equal pri bit is on, ambiguous. 50087c478bd9Sstevel@tonic-gate */ 50097c478bd9Sstevel@tonic-gate if (afsr & (afsr_ow & ~afsr_bit)) 50107c478bd9Sstevel@tonic-gate return (AFLT_STAT_AMBIGUOUS); 50117c478bd9Sstevel@tonic-gate return (AFLT_STAT_VALID); 50127c478bd9Sstevel@tonic-gate } else if (afsr & afsr_ow) 50137c478bd9Sstevel@tonic-gate break; 50147c478bd9Sstevel@tonic-gate } 50157c478bd9Sstevel@tonic-gate 50167c478bd9Sstevel@tonic-gate /* 50177c478bd9Sstevel@tonic-gate * We didn't find a match or a higher priority bit was on. Not 50187c478bd9Sstevel@tonic-gate * finding a match handles the case of invalid AFAR for IVC, IVU. 50197c478bd9Sstevel@tonic-gate */ 50207c478bd9Sstevel@tonic-gate return (AFLT_STAT_INVALID); 50217c478bd9Sstevel@tonic-gate } 50227c478bd9Sstevel@tonic-gate 50237c478bd9Sstevel@tonic-gate static int 50247c478bd9Sstevel@tonic-gate afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit) 50257c478bd9Sstevel@tonic-gate { 50267c478bd9Sstevel@tonic-gate #if defined(SERRANO) 50277c478bd9Sstevel@tonic-gate if (afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) 50287c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, afar2_overwrite)); 50297c478bd9Sstevel@tonic-gate else 50307c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 50317c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, afar_overwrite)); 50327c478bd9Sstevel@tonic-gate } 50337c478bd9Sstevel@tonic-gate 50347c478bd9Sstevel@tonic-gate static int 50357c478bd9Sstevel@tonic-gate afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit) 50367c478bd9Sstevel@tonic-gate { 50377c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, esynd_overwrite)); 50387c478bd9Sstevel@tonic-gate } 50397c478bd9Sstevel@tonic-gate 50407c478bd9Sstevel@tonic-gate static int 50417c478bd9Sstevel@tonic-gate afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit) 50427c478bd9Sstevel@tonic-gate { 50437c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, msynd_overwrite)); 50447c478bd9Sstevel@tonic-gate } 50457c478bd9Sstevel@tonic-gate 50467c478bd9Sstevel@tonic-gate static int 50477c478bd9Sstevel@tonic-gate afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit) 50487c478bd9Sstevel@tonic-gate { 50497c478bd9Sstevel@tonic-gate #ifdef lint 50507c478bd9Sstevel@tonic-gate cpuid = cpuid; 50517c478bd9Sstevel@tonic-gate #endif 50527c478bd9Sstevel@tonic-gate if (afsr_bit & C_AFSR_MSYND_ERRS) { 50537c478bd9Sstevel@tonic-gate return (afsr_to_msynd_status(afsr, afsr_bit)); 50547c478bd9Sstevel@tonic-gate } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 50557c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 50567c478bd9Sstevel@tonic-gate /* 50577c478bd9Sstevel@tonic-gate * The E_SYND overwrite policy is slightly different 50587c478bd9Sstevel@tonic-gate * for Panther CPUs. 50597c478bd9Sstevel@tonic-gate */ 50607c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[cpuid].implementation)) 50617c478bd9Sstevel@tonic-gate return (afsr_to_pn_esynd_status(afsr, afsr_bit)); 50627c478bd9Sstevel@tonic-gate else 50637c478bd9Sstevel@tonic-gate return (afsr_to_esynd_status(afsr, afsr_bit)); 50647c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */ 50657c478bd9Sstevel@tonic-gate return (afsr_to_esynd_status(afsr, afsr_bit)); 50667c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 50677c478bd9Sstevel@tonic-gate } else { 50687c478bd9Sstevel@tonic-gate return (AFLT_STAT_INVALID); 50697c478bd9Sstevel@tonic-gate } 50707c478bd9Sstevel@tonic-gate } 50717c478bd9Sstevel@tonic-gate 50727c478bd9Sstevel@tonic-gate /* 50737c478bd9Sstevel@tonic-gate * Slave CPU stick synchronization. 50747c478bd9Sstevel@tonic-gate */ 50757c478bd9Sstevel@tonic-gate void 50767c478bd9Sstevel@tonic-gate sticksync_slave(void) 50777c478bd9Sstevel@tonic-gate { 50787c478bd9Sstevel@tonic-gate int i; 50797c478bd9Sstevel@tonic-gate int tries = 0; 50807c478bd9Sstevel@tonic-gate int64_t tskew; 50817c478bd9Sstevel@tonic-gate int64_t av_tskew; 50827c478bd9Sstevel@tonic-gate 50837c478bd9Sstevel@tonic-gate kpreempt_disable(); 50847c478bd9Sstevel@tonic-gate /* wait for the master side */ 50857c478bd9Sstevel@tonic-gate while (stick_sync_cmd != SLAVE_START) 50867c478bd9Sstevel@tonic-gate ; 50877c478bd9Sstevel@tonic-gate /* 50887c478bd9Sstevel@tonic-gate * Synchronization should only take a few tries at most. But in the 50897c478bd9Sstevel@tonic-gate * odd case where the cpu isn't cooperating we'll keep trying. A cpu 50907c478bd9Sstevel@tonic-gate * without it's stick synchronized wouldn't be a good citizen. 50917c478bd9Sstevel@tonic-gate */ 50927c478bd9Sstevel@tonic-gate while (slave_done == 0) { 50937c478bd9Sstevel@tonic-gate /* 50947c478bd9Sstevel@tonic-gate * Time skew calculation. 50957c478bd9Sstevel@tonic-gate */ 50967c478bd9Sstevel@tonic-gate av_tskew = tskew = 0; 50977c478bd9Sstevel@tonic-gate 50987c478bd9Sstevel@tonic-gate for (i = 0; i < stick_iter; i++) { 50997c478bd9Sstevel@tonic-gate /* make location hot */ 51007c478bd9Sstevel@tonic-gate timestamp[EV_A_START] = 0; 51017c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_A_START]); 51027c478bd9Sstevel@tonic-gate 51037c478bd9Sstevel@tonic-gate /* tell the master we're ready */ 51047c478bd9Sstevel@tonic-gate stick_sync_cmd = MASTER_START; 51057c478bd9Sstevel@tonic-gate 51067c478bd9Sstevel@tonic-gate /* and wait */ 51077c478bd9Sstevel@tonic-gate while (stick_sync_cmd != SLAVE_CONT) 51087c478bd9Sstevel@tonic-gate ; 51097c478bd9Sstevel@tonic-gate /* Event B end */ 51107c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_B_END]); 51117c478bd9Sstevel@tonic-gate 51127c478bd9Sstevel@tonic-gate /* calculate time skew */ 51137c478bd9Sstevel@tonic-gate tskew = ((timestamp[EV_B_END] - timestamp[EV_B_START]) 51147c478bd9Sstevel@tonic-gate - (timestamp[EV_A_END] - 51157c478bd9Sstevel@tonic-gate timestamp[EV_A_START])) / 2; 51167c478bd9Sstevel@tonic-gate 51177c478bd9Sstevel@tonic-gate /* keep running count */ 51187c478bd9Sstevel@tonic-gate av_tskew += tskew; 51197c478bd9Sstevel@tonic-gate } /* for */ 51207c478bd9Sstevel@tonic-gate 51217c478bd9Sstevel@tonic-gate /* 51227c478bd9Sstevel@tonic-gate * Adjust stick for time skew if not within the max allowed; 51237c478bd9Sstevel@tonic-gate * otherwise we're all done. 51247c478bd9Sstevel@tonic-gate */ 51257c478bd9Sstevel@tonic-gate if (stick_iter != 0) 51267c478bd9Sstevel@tonic-gate av_tskew = av_tskew/stick_iter; 51277c478bd9Sstevel@tonic-gate if (ABS(av_tskew) > stick_tsk) { 51287c478bd9Sstevel@tonic-gate /* 51297c478bd9Sstevel@tonic-gate * If the skew is 1 (the slave's STICK register 51307c478bd9Sstevel@tonic-gate * is 1 STICK ahead of the master's), stick_adj 51317c478bd9Sstevel@tonic-gate * could fail to adjust the slave's STICK register 51327c478bd9Sstevel@tonic-gate * if the STICK read on the slave happens to 51337c478bd9Sstevel@tonic-gate * align with the increment of the STICK. 51347c478bd9Sstevel@tonic-gate * Therefore, we increment the skew to 2. 51357c478bd9Sstevel@tonic-gate */ 51367c478bd9Sstevel@tonic-gate if (av_tskew == 1) 51377c478bd9Sstevel@tonic-gate av_tskew++; 51387c478bd9Sstevel@tonic-gate stick_adj(-av_tskew); 51397c478bd9Sstevel@tonic-gate } else 51407c478bd9Sstevel@tonic-gate slave_done = 1; 51417c478bd9Sstevel@tonic-gate #ifdef DEBUG 51427c478bd9Sstevel@tonic-gate if (tries < DSYNC_ATTEMPTS) 51437c478bd9Sstevel@tonic-gate stick_sync_stats[CPU->cpu_id].skew_val[tries] = 51447c478bd9Sstevel@tonic-gate av_tskew; 51457c478bd9Sstevel@tonic-gate ++tries; 51467c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 51477c478bd9Sstevel@tonic-gate #ifdef lint 51487c478bd9Sstevel@tonic-gate tries = tries; 51497c478bd9Sstevel@tonic-gate #endif 51507c478bd9Sstevel@tonic-gate 51517c478bd9Sstevel@tonic-gate } /* while */ 51527c478bd9Sstevel@tonic-gate 51537c478bd9Sstevel@tonic-gate /* allow the master to finish */ 51547c478bd9Sstevel@tonic-gate stick_sync_cmd = EVENT_NULL; 51557c478bd9Sstevel@tonic-gate kpreempt_enable(); 51567c478bd9Sstevel@tonic-gate } 51577c478bd9Sstevel@tonic-gate 51587c478bd9Sstevel@tonic-gate /* 51597c478bd9Sstevel@tonic-gate * Master CPU side of stick synchronization. 51607c478bd9Sstevel@tonic-gate * - timestamp end of Event A 51617c478bd9Sstevel@tonic-gate * - timestamp beginning of Event B 51627c478bd9Sstevel@tonic-gate */ 51637c478bd9Sstevel@tonic-gate void 51647c478bd9Sstevel@tonic-gate sticksync_master(void) 51657c478bd9Sstevel@tonic-gate { 51667c478bd9Sstevel@tonic-gate int i; 51677c478bd9Sstevel@tonic-gate 51687c478bd9Sstevel@tonic-gate kpreempt_disable(); 51697c478bd9Sstevel@tonic-gate /* tell the slave we've started */ 51707c478bd9Sstevel@tonic-gate slave_done = 0; 51717c478bd9Sstevel@tonic-gate stick_sync_cmd = SLAVE_START; 51727c478bd9Sstevel@tonic-gate 51737c478bd9Sstevel@tonic-gate while (slave_done == 0) { 51747c478bd9Sstevel@tonic-gate for (i = 0; i < stick_iter; i++) { 51757c478bd9Sstevel@tonic-gate /* wait for the slave */ 51767c478bd9Sstevel@tonic-gate while (stick_sync_cmd != MASTER_START) 51777c478bd9Sstevel@tonic-gate ; 51787c478bd9Sstevel@tonic-gate /* Event A end */ 51797c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_A_END]); 51807c478bd9Sstevel@tonic-gate 51817c478bd9Sstevel@tonic-gate /* make location hot */ 51827c478bd9Sstevel@tonic-gate timestamp[EV_B_START] = 0; 51837c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_B_START]); 51847c478bd9Sstevel@tonic-gate 51857c478bd9Sstevel@tonic-gate /* tell the slave to continue */ 51867c478bd9Sstevel@tonic-gate stick_sync_cmd = SLAVE_CONT; 51877c478bd9Sstevel@tonic-gate } /* for */ 51887c478bd9Sstevel@tonic-gate 51897c478bd9Sstevel@tonic-gate /* wait while slave calculates time skew */ 51907c478bd9Sstevel@tonic-gate while (stick_sync_cmd == SLAVE_CONT) 51917c478bd9Sstevel@tonic-gate ; 51927c478bd9Sstevel@tonic-gate } /* while */ 51937c478bd9Sstevel@tonic-gate kpreempt_enable(); 51947c478bd9Sstevel@tonic-gate } 51957c478bd9Sstevel@tonic-gate 51967c478bd9Sstevel@tonic-gate /* 51977c478bd9Sstevel@tonic-gate * Cheetah/Cheetah+ have disrupting error for copyback's, so we don't need to 51987c478bd9Sstevel@tonic-gate * do Spitfire hack of xcall'ing all the cpus to ask to check for them. Also, 51997c478bd9Sstevel@tonic-gate * in cpu_async_panic_callb, each cpu checks for CPU events on its way to 52007c478bd9Sstevel@tonic-gate * panic idle. 52017c478bd9Sstevel@tonic-gate */ 52027c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 52037c478bd9Sstevel@tonic-gate void 52047c478bd9Sstevel@tonic-gate cpu_check_allcpus(struct async_flt *aflt) 52057c478bd9Sstevel@tonic-gate {} 52067c478bd9Sstevel@tonic-gate 52077c478bd9Sstevel@tonic-gate struct kmem_cache *ch_private_cache; 52087c478bd9Sstevel@tonic-gate 52097c478bd9Sstevel@tonic-gate /* 52107c478bd9Sstevel@tonic-gate * Cpu private unitialization. Uninitialize the Ecache scrubber and 52117c478bd9Sstevel@tonic-gate * deallocate the scrubber data structures and cpu_private data structure. 52127c478bd9Sstevel@tonic-gate */ 52137c478bd9Sstevel@tonic-gate void 52147c478bd9Sstevel@tonic-gate cpu_uninit_private(struct cpu *cp) 52157c478bd9Sstevel@tonic-gate { 52167c478bd9Sstevel@tonic-gate cheetah_private_t *chprp = CPU_PRIVATE(cp); 52177c478bd9Sstevel@tonic-gate 52187c478bd9Sstevel@tonic-gate ASSERT(chprp); 52197c478bd9Sstevel@tonic-gate cpu_uninit_ecache_scrub_dr(cp); 52207c478bd9Sstevel@tonic-gate CPU_PRIVATE(cp) = NULL; 52217c478bd9Sstevel@tonic-gate ch_err_tl1_paddrs[cp->cpu_id] = NULL; 52227c478bd9Sstevel@tonic-gate kmem_cache_free(ch_private_cache, chprp); 52237c478bd9Sstevel@tonic-gate cmp_delete_cpu(cp->cpu_id); 52247c478bd9Sstevel@tonic-gate 52257c478bd9Sstevel@tonic-gate } 52267c478bd9Sstevel@tonic-gate 52277c478bd9Sstevel@tonic-gate /* 52287c478bd9Sstevel@tonic-gate * Cheetah Cache Scrubbing 52297c478bd9Sstevel@tonic-gate * 52307c478bd9Sstevel@tonic-gate * The primary purpose of Cheetah cache scrubbing is to reduce the exposure 52317c478bd9Sstevel@tonic-gate * of E$ tags, D$ data, and I$ data to cosmic ray events since they are not 52327c478bd9Sstevel@tonic-gate * protected by either parity or ECC. 52337c478bd9Sstevel@tonic-gate * 52347c478bd9Sstevel@tonic-gate * We currently default the E$ and D$ scan rate to 100 (scan 10% of the 52357c478bd9Sstevel@tonic-gate * cache per second). Due to the the specifics of how the I$ control 52367c478bd9Sstevel@tonic-gate * logic works with respect to the ASI used to scrub I$ lines, the entire 52377c478bd9Sstevel@tonic-gate * I$ is scanned at once. 52387c478bd9Sstevel@tonic-gate */ 52397c478bd9Sstevel@tonic-gate 52407c478bd9Sstevel@tonic-gate /* 52417c478bd9Sstevel@tonic-gate * Tuneables to enable and disable the scrubbing of the caches, and to tune 52427c478bd9Sstevel@tonic-gate * scrubbing behavior. These may be changed via /etc/system or using mdb 52437c478bd9Sstevel@tonic-gate * on a running system. 52447c478bd9Sstevel@tonic-gate */ 52457c478bd9Sstevel@tonic-gate int dcache_scrub_enable = 1; /* D$ scrubbing is on by default */ 52467c478bd9Sstevel@tonic-gate 52477c478bd9Sstevel@tonic-gate /* 52487c478bd9Sstevel@tonic-gate * The following are the PIL levels that the softints/cross traps will fire at. 52497c478bd9Sstevel@tonic-gate */ 52507c478bd9Sstevel@tonic-gate uint_t ecache_scrub_pil = PIL_9; /* E$ scrub PIL for cross traps */ 52517c478bd9Sstevel@tonic-gate uint_t dcache_scrub_pil = PIL_9; /* D$ scrub PIL for cross traps */ 52527c478bd9Sstevel@tonic-gate uint_t icache_scrub_pil = PIL_9; /* I$ scrub PIL for cross traps */ 52537c478bd9Sstevel@tonic-gate 52547c478bd9Sstevel@tonic-gate #if defined(JALAPENO) 52557c478bd9Sstevel@tonic-gate 52567c478bd9Sstevel@tonic-gate /* 52577c478bd9Sstevel@tonic-gate * Due to several errata (82, 85, 86), we don't enable the L2$ scrubber 52587c478bd9Sstevel@tonic-gate * on Jalapeno. 52597c478bd9Sstevel@tonic-gate */ 52607c478bd9Sstevel@tonic-gate int ecache_scrub_enable = 0; 52617c478bd9Sstevel@tonic-gate 52627c478bd9Sstevel@tonic-gate #else /* JALAPENO */ 52637c478bd9Sstevel@tonic-gate 52647c478bd9Sstevel@tonic-gate /* 52657c478bd9Sstevel@tonic-gate * With all other cpu types, E$ scrubbing is on by default 52667c478bd9Sstevel@tonic-gate */ 52677c478bd9Sstevel@tonic-gate int ecache_scrub_enable = 1; 52687c478bd9Sstevel@tonic-gate 52697c478bd9Sstevel@tonic-gate #endif /* JALAPENO */ 52707c478bd9Sstevel@tonic-gate 52717c478bd9Sstevel@tonic-gate 52727c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) || defined(JALAPENO) || defined(SERRANO) 52737c478bd9Sstevel@tonic-gate 52747c478bd9Sstevel@tonic-gate /* 52757c478bd9Sstevel@tonic-gate * The I$ scrubber tends to cause latency problems for real-time SW, so it 52767c478bd9Sstevel@tonic-gate * is disabled by default on non-Cheetah systems 52777c478bd9Sstevel@tonic-gate */ 52787c478bd9Sstevel@tonic-gate int icache_scrub_enable = 0; 52797c478bd9Sstevel@tonic-gate 52807c478bd9Sstevel@tonic-gate /* 52817c478bd9Sstevel@tonic-gate * Tuneables specifying the scrub calls per second and the scan rate 52827c478bd9Sstevel@tonic-gate * for each cache 52837c478bd9Sstevel@tonic-gate * 52847c478bd9Sstevel@tonic-gate * The cyclic times are set during boot based on the following values. 52857c478bd9Sstevel@tonic-gate * Changing these values in mdb after this time will have no effect. If 52867c478bd9Sstevel@tonic-gate * a different value is desired, it must be set in /etc/system before a 52877c478bd9Sstevel@tonic-gate * reboot. 52887c478bd9Sstevel@tonic-gate */ 52897c478bd9Sstevel@tonic-gate int ecache_calls_a_sec = 1; 52907c478bd9Sstevel@tonic-gate int dcache_calls_a_sec = 2; 52917c478bd9Sstevel@tonic-gate int icache_calls_a_sec = 2; 52927c478bd9Sstevel@tonic-gate 52937c478bd9Sstevel@tonic-gate int ecache_scan_rate_idle = 1; 52947c478bd9Sstevel@tonic-gate int ecache_scan_rate_busy = 1; 52957c478bd9Sstevel@tonic-gate int dcache_scan_rate_idle = 1; 52967c478bd9Sstevel@tonic-gate int dcache_scan_rate_busy = 1; 52977c478bd9Sstevel@tonic-gate int icache_scan_rate_idle = 1; 52987c478bd9Sstevel@tonic-gate int icache_scan_rate_busy = 1; 52997c478bd9Sstevel@tonic-gate 53007c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS || JALAPENO || SERRANO */ 53017c478bd9Sstevel@tonic-gate 53027c478bd9Sstevel@tonic-gate int icache_scrub_enable = 1; /* I$ scrubbing is on by default */ 53037c478bd9Sstevel@tonic-gate 53047c478bd9Sstevel@tonic-gate int ecache_calls_a_sec = 100; /* E$ scrub calls per seconds */ 53057c478bd9Sstevel@tonic-gate int dcache_calls_a_sec = 100; /* D$ scrub calls per seconds */ 53067c478bd9Sstevel@tonic-gate int icache_calls_a_sec = 100; /* I$ scrub calls per seconds */ 53077c478bd9Sstevel@tonic-gate 53087c478bd9Sstevel@tonic-gate int ecache_scan_rate_idle = 100; /* E$ scan rate (in tenths of a %) */ 53097c478bd9Sstevel@tonic-gate int ecache_scan_rate_busy = 100; /* E$ scan rate (in tenths of a %) */ 53107c478bd9Sstevel@tonic-gate int dcache_scan_rate_idle = 100; /* D$ scan rate (in tenths of a %) */ 53117c478bd9Sstevel@tonic-gate int dcache_scan_rate_busy = 100; /* D$ scan rate (in tenths of a %) */ 53127c478bd9Sstevel@tonic-gate int icache_scan_rate_idle = 100; /* I$ scan rate (in tenths of a %) */ 53137c478bd9Sstevel@tonic-gate int icache_scan_rate_busy = 100; /* I$ scan rate (in tenths of a %) */ 53147c478bd9Sstevel@tonic-gate 53157c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS || JALAPENO || SERRANO */ 53167c478bd9Sstevel@tonic-gate 53177c478bd9Sstevel@tonic-gate /* 53187c478bd9Sstevel@tonic-gate * In order to scrub on offline cpus, a cross trap is sent. The handler will 53197c478bd9Sstevel@tonic-gate * increment the outstanding request counter and schedule a softint to run 53207c478bd9Sstevel@tonic-gate * the scrubber. 53217c478bd9Sstevel@tonic-gate */ 53227c478bd9Sstevel@tonic-gate extern xcfunc_t cache_scrubreq_tl1; 53237c478bd9Sstevel@tonic-gate 53247c478bd9Sstevel@tonic-gate /* 53257c478bd9Sstevel@tonic-gate * These are the softint functions for each cache scrubber 53267c478bd9Sstevel@tonic-gate */ 53277c478bd9Sstevel@tonic-gate static uint_t scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2); 53287c478bd9Sstevel@tonic-gate static uint_t scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2); 53297c478bd9Sstevel@tonic-gate static uint_t scrub_icache_line_intr(caddr_t arg1, caddr_t arg2); 53307c478bd9Sstevel@tonic-gate 53317c478bd9Sstevel@tonic-gate /* 53327c478bd9Sstevel@tonic-gate * The cache scrub info table contains cache specific information 53337c478bd9Sstevel@tonic-gate * and allows for some of the scrub code to be table driven, reducing 53347c478bd9Sstevel@tonic-gate * duplication of cache similar code. 53357c478bd9Sstevel@tonic-gate * 53367c478bd9Sstevel@tonic-gate * This table keeps a copy of the value in the calls per second variable 53377c478bd9Sstevel@tonic-gate * (?cache_calls_a_sec). This makes it much more difficult for someone 53387c478bd9Sstevel@tonic-gate * to cause us problems (for example, by setting ecache_calls_a_sec to 0 in 53397c478bd9Sstevel@tonic-gate * mdb in a misguided attempt to disable the scrubber). 53407c478bd9Sstevel@tonic-gate */ 53417c478bd9Sstevel@tonic-gate struct scrub_info { 53427c478bd9Sstevel@tonic-gate int *csi_enable; /* scrubber enable flag */ 53437c478bd9Sstevel@tonic-gate int csi_freq; /* scrubber calls per second */ 53447c478bd9Sstevel@tonic-gate int csi_index; /* index to chsm_outstanding[] */ 53457c478bd9Sstevel@tonic-gate uint_t csi_inum; /* scrubber interrupt number */ 53467c478bd9Sstevel@tonic-gate cyclic_id_t csi_omni_cyc_id; /* omni cyclic ID */ 53477c478bd9Sstevel@tonic-gate cyclic_id_t csi_offline_cyc_id; /* offline cyclic ID */ 53487c478bd9Sstevel@tonic-gate char csi_name[3]; /* cache name for this scrub entry */ 53497c478bd9Sstevel@tonic-gate } cache_scrub_info[] = { 53507c478bd9Sstevel@tonic-gate { &ecache_scrub_enable, 0, CACHE_SCRUBBER_INFO_E, 0, 0, 0, "E$"}, 53517c478bd9Sstevel@tonic-gate { &dcache_scrub_enable, 0, CACHE_SCRUBBER_INFO_D, 0, 0, 0, "D$"}, 53527c478bd9Sstevel@tonic-gate { &icache_scrub_enable, 0, CACHE_SCRUBBER_INFO_I, 0, 0, 0, "I$"} 53537c478bd9Sstevel@tonic-gate }; 53547c478bd9Sstevel@tonic-gate 53557c478bd9Sstevel@tonic-gate /* 53567c478bd9Sstevel@tonic-gate * If scrubbing is enabled, increment the outstanding request counter. If it 53577c478bd9Sstevel@tonic-gate * is 1 (meaning there were no previous requests outstanding), call 53587c478bd9Sstevel@tonic-gate * setsoftint_tl1 through xt_one_unchecked, which eventually ends up doing 53597c478bd9Sstevel@tonic-gate * a self trap. 53607c478bd9Sstevel@tonic-gate */ 53617c478bd9Sstevel@tonic-gate static void 53627c478bd9Sstevel@tonic-gate do_scrub(struct scrub_info *csi) 53637c478bd9Sstevel@tonic-gate { 53647c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 53657c478bd9Sstevel@tonic-gate int index = csi->csi_index; 53667c478bd9Sstevel@tonic-gate uint32_t *outstanding = &csmp->chsm_outstanding[index]; 53677c478bd9Sstevel@tonic-gate 53687c478bd9Sstevel@tonic-gate if (*(csi->csi_enable) && (csmp->chsm_enable[index])) { 53697c478bd9Sstevel@tonic-gate if (atomic_add_32_nv(outstanding, 1) == 1) { 53707c478bd9Sstevel@tonic-gate xt_one_unchecked(CPU->cpu_id, setsoftint_tl1, 53717c478bd9Sstevel@tonic-gate csi->csi_inum, 0); 53727c478bd9Sstevel@tonic-gate } 53737c478bd9Sstevel@tonic-gate } 53747c478bd9Sstevel@tonic-gate } 53757c478bd9Sstevel@tonic-gate 53767c478bd9Sstevel@tonic-gate /* 53777c478bd9Sstevel@tonic-gate * Omni cyclics don't fire on offline cpus, so we use another cyclic to 53787c478bd9Sstevel@tonic-gate * cross-trap the offline cpus. 53797c478bd9Sstevel@tonic-gate */ 53807c478bd9Sstevel@tonic-gate static void 53817c478bd9Sstevel@tonic-gate do_scrub_offline(struct scrub_info *csi) 53827c478bd9Sstevel@tonic-gate { 53837c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 53847c478bd9Sstevel@tonic-gate 53857c478bd9Sstevel@tonic-gate if (CPUSET_ISNULL(cpu_offline_set)) { 53867c478bd9Sstevel@tonic-gate /* 53877c478bd9Sstevel@tonic-gate * No offline cpus - nothing to do 53887c478bd9Sstevel@tonic-gate */ 53897c478bd9Sstevel@tonic-gate return; 53907c478bd9Sstevel@tonic-gate } 53917c478bd9Sstevel@tonic-gate 53927c478bd9Sstevel@tonic-gate if (*(csi->csi_enable) && (csmp->chsm_enable[csi->csi_index])) { 53937c478bd9Sstevel@tonic-gate xt_some(cpu_offline_set, cache_scrubreq_tl1, csi->csi_inum, 53947c478bd9Sstevel@tonic-gate csi->csi_index); 53957c478bd9Sstevel@tonic-gate } 53967c478bd9Sstevel@tonic-gate } 53977c478bd9Sstevel@tonic-gate 53987c478bd9Sstevel@tonic-gate /* 53997c478bd9Sstevel@tonic-gate * This is the initial setup for the scrubber cyclics - it sets the 54007c478bd9Sstevel@tonic-gate * interrupt level, frequency, and function to call. 54017c478bd9Sstevel@tonic-gate */ 54027c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 54037c478bd9Sstevel@tonic-gate static void 54047c478bd9Sstevel@tonic-gate cpu_scrub_cyclic_setup(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 54057c478bd9Sstevel@tonic-gate cyc_time_t *when) 54067c478bd9Sstevel@tonic-gate { 54077c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg; 54087c478bd9Sstevel@tonic-gate 54097c478bd9Sstevel@tonic-gate ASSERT(csi != NULL); 54107c478bd9Sstevel@tonic-gate hdlr->cyh_func = (cyc_func_t)do_scrub; 54117c478bd9Sstevel@tonic-gate hdlr->cyh_level = CY_LOW_LEVEL; 54127c478bd9Sstevel@tonic-gate hdlr->cyh_arg = arg; 54137c478bd9Sstevel@tonic-gate 54147c478bd9Sstevel@tonic-gate when->cyt_when = 0; /* Start immediately */ 54157c478bd9Sstevel@tonic-gate when->cyt_interval = NANOSEC / csi->csi_freq; 54167c478bd9Sstevel@tonic-gate } 54177c478bd9Sstevel@tonic-gate 54187c478bd9Sstevel@tonic-gate /* 54197c478bd9Sstevel@tonic-gate * Initialization for cache scrubbing. 54207c478bd9Sstevel@tonic-gate * This routine is called AFTER all cpus have had cpu_init_private called 54217c478bd9Sstevel@tonic-gate * to initialize their private data areas. 54227c478bd9Sstevel@tonic-gate */ 54237c478bd9Sstevel@tonic-gate void 54247c478bd9Sstevel@tonic-gate cpu_init_cache_scrub(void) 54257c478bd9Sstevel@tonic-gate { 54267c478bd9Sstevel@tonic-gate int i; 54277c478bd9Sstevel@tonic-gate struct scrub_info *csi; 54287c478bd9Sstevel@tonic-gate cyc_omni_handler_t omni_hdlr; 54297c478bd9Sstevel@tonic-gate cyc_handler_t offline_hdlr; 54307c478bd9Sstevel@tonic-gate cyc_time_t when; 54317c478bd9Sstevel@tonic-gate 54327c478bd9Sstevel@tonic-gate /* 54337c478bd9Sstevel@tonic-gate * save away the maximum number of lines for the D$ 54347c478bd9Sstevel@tonic-gate */ 54357c478bd9Sstevel@tonic-gate dcache_nlines = dcache_size / dcache_linesize; 54367c478bd9Sstevel@tonic-gate 54377c478bd9Sstevel@tonic-gate /* 54387c478bd9Sstevel@tonic-gate * register the softints for the cache scrubbing 54397c478bd9Sstevel@tonic-gate */ 54407c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_inum = 54417c478bd9Sstevel@tonic-gate add_softintr(ecache_scrub_pil, scrub_ecache_line_intr, 54427c478bd9Sstevel@tonic-gate (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_E]); 54437c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_freq = ecache_calls_a_sec; 54447c478bd9Sstevel@tonic-gate 54457c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_inum = 54467c478bd9Sstevel@tonic-gate add_softintr(dcache_scrub_pil, scrub_dcache_line_intr, 54477c478bd9Sstevel@tonic-gate (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_D]); 54487c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_freq = dcache_calls_a_sec; 54497c478bd9Sstevel@tonic-gate 54507c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_inum = 54517c478bd9Sstevel@tonic-gate add_softintr(icache_scrub_pil, scrub_icache_line_intr, 54527c478bd9Sstevel@tonic-gate (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_I]); 54537c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_freq = icache_calls_a_sec; 54547c478bd9Sstevel@tonic-gate 54557c478bd9Sstevel@tonic-gate /* 54567c478bd9Sstevel@tonic-gate * start the scrubbing for all the caches 54577c478bd9Sstevel@tonic-gate */ 54587c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 54597c478bd9Sstevel@tonic-gate for (i = 0; i < CACHE_SCRUBBER_COUNT; i++) { 54607c478bd9Sstevel@tonic-gate 54617c478bd9Sstevel@tonic-gate csi = &cache_scrub_info[i]; 54627c478bd9Sstevel@tonic-gate 54637c478bd9Sstevel@tonic-gate if (!(*csi->csi_enable)) 54647c478bd9Sstevel@tonic-gate continue; 54657c478bd9Sstevel@tonic-gate 54667c478bd9Sstevel@tonic-gate /* 54677c478bd9Sstevel@tonic-gate * force the following to be true: 54687c478bd9Sstevel@tonic-gate * 1 <= calls_a_sec <= hz 54697c478bd9Sstevel@tonic-gate */ 54707c478bd9Sstevel@tonic-gate if (csi->csi_freq > hz) { 54717c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "%s scrub calls_a_sec set too high " 54727c478bd9Sstevel@tonic-gate "(%d); resetting to hz (%d)", csi->csi_name, 54737c478bd9Sstevel@tonic-gate csi->csi_freq, hz); 54747c478bd9Sstevel@tonic-gate csi->csi_freq = hz; 54757c478bd9Sstevel@tonic-gate } else if (csi->csi_freq < 1) { 54767c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "%s scrub calls_a_sec set too low " 54777c478bd9Sstevel@tonic-gate "(%d); resetting to 1", csi->csi_name, 54787c478bd9Sstevel@tonic-gate csi->csi_freq); 54797c478bd9Sstevel@tonic-gate csi->csi_freq = 1; 54807c478bd9Sstevel@tonic-gate } 54817c478bd9Sstevel@tonic-gate 54827c478bd9Sstevel@tonic-gate omni_hdlr.cyo_online = cpu_scrub_cyclic_setup; 54837c478bd9Sstevel@tonic-gate omni_hdlr.cyo_offline = NULL; 54847c478bd9Sstevel@tonic-gate omni_hdlr.cyo_arg = (void *)csi; 54857c478bd9Sstevel@tonic-gate 54867c478bd9Sstevel@tonic-gate offline_hdlr.cyh_func = (cyc_func_t)do_scrub_offline; 54877c478bd9Sstevel@tonic-gate offline_hdlr.cyh_arg = (void *)csi; 54887c478bd9Sstevel@tonic-gate offline_hdlr.cyh_level = CY_LOW_LEVEL; 54897c478bd9Sstevel@tonic-gate 54907c478bd9Sstevel@tonic-gate when.cyt_when = 0; /* Start immediately */ 54917c478bd9Sstevel@tonic-gate when.cyt_interval = NANOSEC / csi->csi_freq; 54927c478bd9Sstevel@tonic-gate 54937c478bd9Sstevel@tonic-gate csi->csi_omni_cyc_id = cyclic_add_omni(&omni_hdlr); 54947c478bd9Sstevel@tonic-gate csi->csi_offline_cyc_id = cyclic_add(&offline_hdlr, &when); 54957c478bd9Sstevel@tonic-gate } 54967c478bd9Sstevel@tonic-gate register_cpu_setup_func(cpu_scrub_cpu_setup, NULL); 54977c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 54987c478bd9Sstevel@tonic-gate } 54997c478bd9Sstevel@tonic-gate 55007c478bd9Sstevel@tonic-gate /* 55017c478bd9Sstevel@tonic-gate * Indicate that the specified cpu is idle. 55027c478bd9Sstevel@tonic-gate */ 55037c478bd9Sstevel@tonic-gate void 55047c478bd9Sstevel@tonic-gate cpu_idle_ecache_scrub(struct cpu *cp) 55057c478bd9Sstevel@tonic-gate { 55067c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(cp) != NULL) { 55077c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 55087c478bd9Sstevel@tonic-gate csmp->chsm_ecache_busy = ECACHE_CPU_IDLE; 55097c478bd9Sstevel@tonic-gate } 55107c478bd9Sstevel@tonic-gate } 55117c478bd9Sstevel@tonic-gate 55127c478bd9Sstevel@tonic-gate /* 55137c478bd9Sstevel@tonic-gate * Indicate that the specified cpu is busy. 55147c478bd9Sstevel@tonic-gate */ 55157c478bd9Sstevel@tonic-gate void 55167c478bd9Sstevel@tonic-gate cpu_busy_ecache_scrub(struct cpu *cp) 55177c478bd9Sstevel@tonic-gate { 55187c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(cp) != NULL) { 55197c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 55207c478bd9Sstevel@tonic-gate csmp->chsm_ecache_busy = ECACHE_CPU_BUSY; 55217c478bd9Sstevel@tonic-gate } 55227c478bd9Sstevel@tonic-gate } 55237c478bd9Sstevel@tonic-gate 55247c478bd9Sstevel@tonic-gate /* 55257c478bd9Sstevel@tonic-gate * Initialization for cache scrubbing for the specified cpu. 55267c478bd9Sstevel@tonic-gate */ 55277c478bd9Sstevel@tonic-gate void 55287c478bd9Sstevel@tonic-gate cpu_init_ecache_scrub_dr(struct cpu *cp) 55297c478bd9Sstevel@tonic-gate { 55307c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 55317c478bd9Sstevel@tonic-gate int cpuid = cp->cpu_id; 55327c478bd9Sstevel@tonic-gate 55337c478bd9Sstevel@tonic-gate /* initialize the number of lines in the caches */ 55347c478bd9Sstevel@tonic-gate csmp->chsm_ecache_nlines = cpunodes[cpuid].ecache_size / 55357c478bd9Sstevel@tonic-gate cpunodes[cpuid].ecache_linesize; 55367c478bd9Sstevel@tonic-gate csmp->chsm_icache_nlines = CPU_PRIVATE_VAL(cp, chpr_icache_size) / 55377c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(cp, chpr_icache_linesize); 55387c478bd9Sstevel@tonic-gate 55397c478bd9Sstevel@tonic-gate /* 55407c478bd9Sstevel@tonic-gate * do_scrub() and do_scrub_offline() check both the global 55417c478bd9Sstevel@tonic-gate * ?cache_scrub_enable and this per-cpu enable variable. All scrubbers 55427c478bd9Sstevel@tonic-gate * check this value before scrubbing. Currently, we use it to 55437c478bd9Sstevel@tonic-gate * disable the E$ scrubber on multi-core cpus or while running at 55447c478bd9Sstevel@tonic-gate * slowed speed. For now, just turn everything on and allow 55457c478bd9Sstevel@tonic-gate * cpu_init_private() to change it if necessary. 55467c478bd9Sstevel@tonic-gate */ 55477c478bd9Sstevel@tonic-gate csmp->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1; 55487c478bd9Sstevel@tonic-gate csmp->chsm_enable[CACHE_SCRUBBER_INFO_D] = 1; 55497c478bd9Sstevel@tonic-gate csmp->chsm_enable[CACHE_SCRUBBER_INFO_I] = 1; 55507c478bd9Sstevel@tonic-gate 55517c478bd9Sstevel@tonic-gate cpu_busy_ecache_scrub(cp); 55527c478bd9Sstevel@tonic-gate } 55537c478bd9Sstevel@tonic-gate 55547c478bd9Sstevel@tonic-gate /* 55557c478bd9Sstevel@tonic-gate * Un-initialization for cache scrubbing for the specified cpu. 55567c478bd9Sstevel@tonic-gate */ 55577c478bd9Sstevel@tonic-gate static void 55587c478bd9Sstevel@tonic-gate cpu_uninit_ecache_scrub_dr(struct cpu *cp) 55597c478bd9Sstevel@tonic-gate { 55607c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 55617c478bd9Sstevel@tonic-gate 55627c478bd9Sstevel@tonic-gate /* 55637c478bd9Sstevel@tonic-gate * un-initialize bookkeeping for cache scrubbing 55647c478bd9Sstevel@tonic-gate */ 55657c478bd9Sstevel@tonic-gate bzero(csmp, sizeof (ch_scrub_misc_t)); 55667c478bd9Sstevel@tonic-gate 55677c478bd9Sstevel@tonic-gate cpu_idle_ecache_scrub(cp); 55687c478bd9Sstevel@tonic-gate } 55697c478bd9Sstevel@tonic-gate 55707c478bd9Sstevel@tonic-gate /* 55717c478bd9Sstevel@tonic-gate * Called periodically on each CPU to scrub the D$. 55727c478bd9Sstevel@tonic-gate */ 55737c478bd9Sstevel@tonic-gate static void 55747c478bd9Sstevel@tonic-gate scrub_dcache(int how_many) 55757c478bd9Sstevel@tonic-gate { 55767c478bd9Sstevel@tonic-gate int i; 55777c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 55787c478bd9Sstevel@tonic-gate int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D]; 55797c478bd9Sstevel@tonic-gate 55807c478bd9Sstevel@tonic-gate /* 55817c478bd9Sstevel@tonic-gate * scrub the desired number of lines 55827c478bd9Sstevel@tonic-gate */ 55837c478bd9Sstevel@tonic-gate for (i = 0; i < how_many; i++) { 55847c478bd9Sstevel@tonic-gate /* 55857c478bd9Sstevel@tonic-gate * scrub a D$ line 55867c478bd9Sstevel@tonic-gate */ 55877c478bd9Sstevel@tonic-gate dcache_inval_line(index); 55887c478bd9Sstevel@tonic-gate 55897c478bd9Sstevel@tonic-gate /* 55907c478bd9Sstevel@tonic-gate * calculate the next D$ line to scrub, assumes 55917c478bd9Sstevel@tonic-gate * that dcache_nlines is a power of 2 55927c478bd9Sstevel@tonic-gate */ 55937c478bd9Sstevel@tonic-gate index = (index + 1) & (dcache_nlines - 1); 55947c478bd9Sstevel@tonic-gate } 55957c478bd9Sstevel@tonic-gate 55967c478bd9Sstevel@tonic-gate /* 55977c478bd9Sstevel@tonic-gate * set the scrub index for the next visit 55987c478bd9Sstevel@tonic-gate */ 55997c478bd9Sstevel@tonic-gate csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D] = index; 56007c478bd9Sstevel@tonic-gate } 56017c478bd9Sstevel@tonic-gate 56027c478bd9Sstevel@tonic-gate /* 56037c478bd9Sstevel@tonic-gate * Handler for D$ scrub inum softint. Call scrub_dcache until 56047c478bd9Sstevel@tonic-gate * we decrement the outstanding request count to zero. 56057c478bd9Sstevel@tonic-gate */ 56067c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 56077c478bd9Sstevel@tonic-gate static uint_t 56087c478bd9Sstevel@tonic-gate scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2) 56097c478bd9Sstevel@tonic-gate { 56107c478bd9Sstevel@tonic-gate int i; 56117c478bd9Sstevel@tonic-gate int how_many; 56127c478bd9Sstevel@tonic-gate int outstanding; 56137c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 56147c478bd9Sstevel@tonic-gate uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_D]; 56157c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg1; 56167c478bd9Sstevel@tonic-gate int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 56177c478bd9Sstevel@tonic-gate dcache_scan_rate_idle : dcache_scan_rate_busy; 56187c478bd9Sstevel@tonic-gate 56197c478bd9Sstevel@tonic-gate /* 56207c478bd9Sstevel@tonic-gate * The scan rates are expressed in units of tenths of a 56217c478bd9Sstevel@tonic-gate * percent. A scan rate of 1000 (100%) means the whole 56227c478bd9Sstevel@tonic-gate * cache is scanned every second. 56237c478bd9Sstevel@tonic-gate */ 56247c478bd9Sstevel@tonic-gate how_many = (dcache_nlines * scan_rate) / (1000 * csi->csi_freq); 56257c478bd9Sstevel@tonic-gate 56267c478bd9Sstevel@tonic-gate do { 56277c478bd9Sstevel@tonic-gate outstanding = *countp; 56287c478bd9Sstevel@tonic-gate for (i = 0; i < outstanding; i++) { 56297c478bd9Sstevel@tonic-gate scrub_dcache(how_many); 56307c478bd9Sstevel@tonic-gate } 56317c478bd9Sstevel@tonic-gate } while (atomic_add_32_nv(countp, -outstanding)); 56327c478bd9Sstevel@tonic-gate 56337c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 56347c478bd9Sstevel@tonic-gate } 56357c478bd9Sstevel@tonic-gate 56367c478bd9Sstevel@tonic-gate /* 56377c478bd9Sstevel@tonic-gate * Called periodically on each CPU to scrub the I$. The I$ is scrubbed 56387c478bd9Sstevel@tonic-gate * by invalidating lines. Due to the characteristics of the ASI which 56397c478bd9Sstevel@tonic-gate * is used to invalidate an I$ line, the entire I$ must be invalidated 56407c478bd9Sstevel@tonic-gate * vs. an individual I$ line. 56417c478bd9Sstevel@tonic-gate */ 56427c478bd9Sstevel@tonic-gate static void 56437c478bd9Sstevel@tonic-gate scrub_icache(int how_many) 56447c478bd9Sstevel@tonic-gate { 56457c478bd9Sstevel@tonic-gate int i; 56467c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 56477c478bd9Sstevel@tonic-gate int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I]; 56487c478bd9Sstevel@tonic-gate int icache_nlines = csmp->chsm_icache_nlines; 56497c478bd9Sstevel@tonic-gate 56507c478bd9Sstevel@tonic-gate /* 56517c478bd9Sstevel@tonic-gate * scrub the desired number of lines 56527c478bd9Sstevel@tonic-gate */ 56537c478bd9Sstevel@tonic-gate for (i = 0; i < how_many; i++) { 56547c478bd9Sstevel@tonic-gate /* 56557c478bd9Sstevel@tonic-gate * since the entire I$ must be scrubbed at once, 56567c478bd9Sstevel@tonic-gate * wait until the index wraps to zero to invalidate 56577c478bd9Sstevel@tonic-gate * the entire I$ 56587c478bd9Sstevel@tonic-gate */ 56597c478bd9Sstevel@tonic-gate if (index == 0) { 56607c478bd9Sstevel@tonic-gate icache_inval_all(); 56617c478bd9Sstevel@tonic-gate } 56627c478bd9Sstevel@tonic-gate 56637c478bd9Sstevel@tonic-gate /* 56647c478bd9Sstevel@tonic-gate * calculate the next I$ line to scrub, assumes 56657c478bd9Sstevel@tonic-gate * that chsm_icache_nlines is a power of 2 56667c478bd9Sstevel@tonic-gate */ 56677c478bd9Sstevel@tonic-gate index = (index + 1) & (icache_nlines - 1); 56687c478bd9Sstevel@tonic-gate } 56697c478bd9Sstevel@tonic-gate 56707c478bd9Sstevel@tonic-gate /* 56717c478bd9Sstevel@tonic-gate * set the scrub index for the next visit 56727c478bd9Sstevel@tonic-gate */ 56737c478bd9Sstevel@tonic-gate csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I] = index; 56747c478bd9Sstevel@tonic-gate } 56757c478bd9Sstevel@tonic-gate 56767c478bd9Sstevel@tonic-gate /* 56777c478bd9Sstevel@tonic-gate * Handler for I$ scrub inum softint. Call scrub_icache until 56787c478bd9Sstevel@tonic-gate * we decrement the outstanding request count to zero. 56797c478bd9Sstevel@tonic-gate */ 56807c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 56817c478bd9Sstevel@tonic-gate static uint_t 56827c478bd9Sstevel@tonic-gate scrub_icache_line_intr(caddr_t arg1, caddr_t arg2) 56837c478bd9Sstevel@tonic-gate { 56847c478bd9Sstevel@tonic-gate int i; 56857c478bd9Sstevel@tonic-gate int how_many; 56867c478bd9Sstevel@tonic-gate int outstanding; 56877c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 56887c478bd9Sstevel@tonic-gate uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_I]; 56897c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg1; 56907c478bd9Sstevel@tonic-gate int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 56917c478bd9Sstevel@tonic-gate icache_scan_rate_idle : icache_scan_rate_busy; 56927c478bd9Sstevel@tonic-gate int icache_nlines = csmp->chsm_icache_nlines; 56937c478bd9Sstevel@tonic-gate 56947c478bd9Sstevel@tonic-gate /* 56957c478bd9Sstevel@tonic-gate * The scan rates are expressed in units of tenths of a 56967c478bd9Sstevel@tonic-gate * percent. A scan rate of 1000 (100%) means the whole 56977c478bd9Sstevel@tonic-gate * cache is scanned every second. 56987c478bd9Sstevel@tonic-gate */ 56997c478bd9Sstevel@tonic-gate how_many = (icache_nlines * scan_rate) / (1000 * csi->csi_freq); 57007c478bd9Sstevel@tonic-gate 57017c478bd9Sstevel@tonic-gate do { 57027c478bd9Sstevel@tonic-gate outstanding = *countp; 57037c478bd9Sstevel@tonic-gate for (i = 0; i < outstanding; i++) { 57047c478bd9Sstevel@tonic-gate scrub_icache(how_many); 57057c478bd9Sstevel@tonic-gate } 57067c478bd9Sstevel@tonic-gate } while (atomic_add_32_nv(countp, -outstanding)); 57077c478bd9Sstevel@tonic-gate 57087c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 57097c478bd9Sstevel@tonic-gate } 57107c478bd9Sstevel@tonic-gate 57117c478bd9Sstevel@tonic-gate /* 57127c478bd9Sstevel@tonic-gate * Called periodically on each CPU to scrub the E$. 57137c478bd9Sstevel@tonic-gate */ 57147c478bd9Sstevel@tonic-gate static void 57157c478bd9Sstevel@tonic-gate scrub_ecache(int how_many) 57167c478bd9Sstevel@tonic-gate { 57177c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 57187c478bd9Sstevel@tonic-gate int i; 57197c478bd9Sstevel@tonic-gate int cpuid = CPU->cpu_id; 57207c478bd9Sstevel@tonic-gate int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E]; 57217c478bd9Sstevel@tonic-gate int nlines = csmp->chsm_ecache_nlines; 57227c478bd9Sstevel@tonic-gate int linesize = cpunodes[cpuid].ecache_linesize; 57237c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 57247c478bd9Sstevel@tonic-gate 57257c478bd9Sstevel@tonic-gate /* 57267c478bd9Sstevel@tonic-gate * scrub the desired number of lines 57277c478bd9Sstevel@tonic-gate */ 57287c478bd9Sstevel@tonic-gate for (i = 0; i < how_many; i++) { 57297c478bd9Sstevel@tonic-gate /* 57307c478bd9Sstevel@tonic-gate * scrub the E$ line 57317c478bd9Sstevel@tonic-gate */ 57327c478bd9Sstevel@tonic-gate ecache_flush_line(ecache_flushaddr + (index * linesize), 57337c478bd9Sstevel@tonic-gate ec_set_size); 57347c478bd9Sstevel@tonic-gate 57357c478bd9Sstevel@tonic-gate /* 57367c478bd9Sstevel@tonic-gate * calculate the next E$ line to scrub based on twice 57377c478bd9Sstevel@tonic-gate * the number of E$ lines (to displace lines containing 57387c478bd9Sstevel@tonic-gate * flush area data), assumes that the number of lines 57397c478bd9Sstevel@tonic-gate * is a power of 2 57407c478bd9Sstevel@tonic-gate */ 57417c478bd9Sstevel@tonic-gate index = (index + 1) & ((nlines << 1) - 1); 57427c478bd9Sstevel@tonic-gate } 57437c478bd9Sstevel@tonic-gate 57447c478bd9Sstevel@tonic-gate /* 57457c478bd9Sstevel@tonic-gate * set the ecache scrub index for the next visit 57467c478bd9Sstevel@tonic-gate */ 57477c478bd9Sstevel@tonic-gate csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E] = index; 57487c478bd9Sstevel@tonic-gate } 57497c478bd9Sstevel@tonic-gate 57507c478bd9Sstevel@tonic-gate /* 57517c478bd9Sstevel@tonic-gate * Handler for E$ scrub inum softint. Call the E$ scrubber until 57527c478bd9Sstevel@tonic-gate * we decrement the outstanding request count to zero. 57536319f043Srscott * 57546319f043Srscott * Due to interactions with cpu_scrub_cpu_setup(), the outstanding count may 57556319f043Srscott * become negative after the atomic_add_32_nv(). This is not a problem, as 57566319f043Srscott * the next trip around the loop won't scrub anything, and the next add will 57576319f043Srscott * reset the count back to zero. 57587c478bd9Sstevel@tonic-gate */ 57597c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 57607c478bd9Sstevel@tonic-gate static uint_t 57617c478bd9Sstevel@tonic-gate scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2) 57627c478bd9Sstevel@tonic-gate { 57637c478bd9Sstevel@tonic-gate int i; 57647c478bd9Sstevel@tonic-gate int how_many; 57657c478bd9Sstevel@tonic-gate int outstanding; 57667c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 57677c478bd9Sstevel@tonic-gate uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_E]; 57687c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg1; 57697c478bd9Sstevel@tonic-gate int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 57707c478bd9Sstevel@tonic-gate ecache_scan_rate_idle : ecache_scan_rate_busy; 57717c478bd9Sstevel@tonic-gate int ecache_nlines = csmp->chsm_ecache_nlines; 57727c478bd9Sstevel@tonic-gate 57737c478bd9Sstevel@tonic-gate /* 57747c478bd9Sstevel@tonic-gate * The scan rates are expressed in units of tenths of a 57757c478bd9Sstevel@tonic-gate * percent. A scan rate of 1000 (100%) means the whole 57767c478bd9Sstevel@tonic-gate * cache is scanned every second. 57777c478bd9Sstevel@tonic-gate */ 57787c478bd9Sstevel@tonic-gate how_many = (ecache_nlines * scan_rate) / (1000 * csi->csi_freq); 57797c478bd9Sstevel@tonic-gate 57807c478bd9Sstevel@tonic-gate do { 57817c478bd9Sstevel@tonic-gate outstanding = *countp; 57827c478bd9Sstevel@tonic-gate for (i = 0; i < outstanding; i++) { 57837c478bd9Sstevel@tonic-gate scrub_ecache(how_many); 57847c478bd9Sstevel@tonic-gate } 57857c478bd9Sstevel@tonic-gate } while (atomic_add_32_nv(countp, -outstanding)); 57867c478bd9Sstevel@tonic-gate 57877c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 57887c478bd9Sstevel@tonic-gate } 57897c478bd9Sstevel@tonic-gate 57907c478bd9Sstevel@tonic-gate /* 57917c478bd9Sstevel@tonic-gate * Timeout function to reenable CE 57927c478bd9Sstevel@tonic-gate */ 57937c478bd9Sstevel@tonic-gate static void 57947c478bd9Sstevel@tonic-gate cpu_delayed_check_ce_errors(void *arg) 57957c478bd9Sstevel@tonic-gate { 57967c478bd9Sstevel@tonic-gate if (!taskq_dispatch(ch_check_ce_tq, cpu_check_ce_errors, arg, 57977c478bd9Sstevel@tonic-gate TQ_NOSLEEP)) { 57987c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, arg, 57997c478bd9Sstevel@tonic-gate drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 58007c478bd9Sstevel@tonic-gate } 58017c478bd9Sstevel@tonic-gate } 58027c478bd9Sstevel@tonic-gate 58037c478bd9Sstevel@tonic-gate /* 58047c478bd9Sstevel@tonic-gate * CE Deferred Re-enable after trap. 58057c478bd9Sstevel@tonic-gate * 58067c478bd9Sstevel@tonic-gate * When the CPU gets a disrupting trap for any of the errors 58077c478bd9Sstevel@tonic-gate * controlled by the CEEN bit, CEEN is disabled in the trap handler 58087c478bd9Sstevel@tonic-gate * immediately. To eliminate the possibility of multiple CEs causing 58097c478bd9Sstevel@tonic-gate * recursive stack overflow in the trap handler, we cannot 58107c478bd9Sstevel@tonic-gate * reenable CEEN while still running in the trap handler. Instead, 58117c478bd9Sstevel@tonic-gate * after a CE is logged on a CPU, we schedule a timeout function, 58127c478bd9Sstevel@tonic-gate * cpu_check_ce_errors(), to trigger after cpu_ceen_delay_secs 58137c478bd9Sstevel@tonic-gate * seconds. This function will check whether any further CEs 58147c478bd9Sstevel@tonic-gate * have occurred on that CPU, and if none have, will reenable CEEN. 58157c478bd9Sstevel@tonic-gate * 58167c478bd9Sstevel@tonic-gate * If further CEs have occurred while CEEN is disabled, another 58177c478bd9Sstevel@tonic-gate * timeout will be scheduled. This is to ensure that the CPU can 58187c478bd9Sstevel@tonic-gate * make progress in the face of CE 'storms', and that it does not 58197c478bd9Sstevel@tonic-gate * spend all its time logging CE errors. 58207c478bd9Sstevel@tonic-gate */ 58217c478bd9Sstevel@tonic-gate static void 58227c478bd9Sstevel@tonic-gate cpu_check_ce_errors(void *arg) 58237c478bd9Sstevel@tonic-gate { 5824f47a9c50Smathue int cpuid = (int)(uintptr_t)arg; 58257c478bd9Sstevel@tonic-gate cpu_t *cp; 58267c478bd9Sstevel@tonic-gate 58277c478bd9Sstevel@tonic-gate /* 58287c478bd9Sstevel@tonic-gate * We acquire cpu_lock. 58297c478bd9Sstevel@tonic-gate */ 58307c478bd9Sstevel@tonic-gate ASSERT(curthread->t_pil == 0); 58317c478bd9Sstevel@tonic-gate 58327c478bd9Sstevel@tonic-gate /* 58337c478bd9Sstevel@tonic-gate * verify that the cpu is still around, DR 58347c478bd9Sstevel@tonic-gate * could have got there first ... 58357c478bd9Sstevel@tonic-gate */ 58367c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 58377c478bd9Sstevel@tonic-gate cp = cpu_get(cpuid); 58387c478bd9Sstevel@tonic-gate if (cp == NULL) { 58397c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 58407c478bd9Sstevel@tonic-gate return; 58417c478bd9Sstevel@tonic-gate } 58427c478bd9Sstevel@tonic-gate /* 58437c478bd9Sstevel@tonic-gate * make sure we don't migrate across CPUs 58447c478bd9Sstevel@tonic-gate * while checking our CE status. 58457c478bd9Sstevel@tonic-gate */ 58467c478bd9Sstevel@tonic-gate kpreempt_disable(); 58477c478bd9Sstevel@tonic-gate 58487c478bd9Sstevel@tonic-gate /* 58497c478bd9Sstevel@tonic-gate * If we are running on the CPU that got the 58507c478bd9Sstevel@tonic-gate * CE, we can do the checks directly. 58517c478bd9Sstevel@tonic-gate */ 58527c478bd9Sstevel@tonic-gate if (cp->cpu_id == CPU->cpu_id) { 58537c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 58547c478bd9Sstevel@tonic-gate cpu_check_ce(TIMEOUT_CEEN_CHECK, 0, 0, 0); 58557c478bd9Sstevel@tonic-gate kpreempt_enable(); 58567c478bd9Sstevel@tonic-gate return; 58577c478bd9Sstevel@tonic-gate } 58587c478bd9Sstevel@tonic-gate kpreempt_enable(); 58597c478bd9Sstevel@tonic-gate 58607c478bd9Sstevel@tonic-gate /* 58617c478bd9Sstevel@tonic-gate * send an x-call to get the CPU that originally 58627c478bd9Sstevel@tonic-gate * got the CE to do the necessary checks. If we can't 58637c478bd9Sstevel@tonic-gate * send the x-call, reschedule the timeout, otherwise we 58647c478bd9Sstevel@tonic-gate * lose CEEN forever on that CPU. 58657c478bd9Sstevel@tonic-gate */ 58667c478bd9Sstevel@tonic-gate if (CPU_XCALL_READY(cp->cpu_id) && (!(cp->cpu_flags & CPU_QUIESCED))) { 58677c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)cpu_check_ce, 58687c478bd9Sstevel@tonic-gate TIMEOUT_CEEN_CHECK, 0); 58697c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 58707c478bd9Sstevel@tonic-gate } else { 58717c478bd9Sstevel@tonic-gate /* 58727c478bd9Sstevel@tonic-gate * When the CPU is not accepting xcalls, or 58737c478bd9Sstevel@tonic-gate * the processor is offlined, we don't want to 58747c478bd9Sstevel@tonic-gate * incur the extra overhead of trying to schedule the 58757c478bd9Sstevel@tonic-gate * CE timeout indefinitely. However, we don't want to lose 58767c478bd9Sstevel@tonic-gate * CE checking forever. 58777c478bd9Sstevel@tonic-gate * 58787c478bd9Sstevel@tonic-gate * Keep rescheduling the timeout, accepting the additional 58797c478bd9Sstevel@tonic-gate * overhead as the cost of correctness in the case where we get 58807c478bd9Sstevel@tonic-gate * a CE, disable CEEN, offline the CPU during the 58817c478bd9Sstevel@tonic-gate * the timeout interval, and then online it at some 58827c478bd9Sstevel@tonic-gate * point in the future. This is unlikely given the short 58837c478bd9Sstevel@tonic-gate * cpu_ceen_delay_secs. 58847c478bd9Sstevel@tonic-gate */ 58857c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 5886f47a9c50Smathue (void) timeout(cpu_delayed_check_ce_errors, 5887f47a9c50Smathue (void *)(uintptr_t)cp->cpu_id, 58887c478bd9Sstevel@tonic-gate drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 58897c478bd9Sstevel@tonic-gate } 58907c478bd9Sstevel@tonic-gate } 58917c478bd9Sstevel@tonic-gate 58927c478bd9Sstevel@tonic-gate /* 58937c478bd9Sstevel@tonic-gate * This routine will check whether CEs have occurred while 58947c478bd9Sstevel@tonic-gate * CEEN is disabled. Any CEs detected will be logged and, if 58957c478bd9Sstevel@tonic-gate * possible, scrubbed. 58967c478bd9Sstevel@tonic-gate * 58977c478bd9Sstevel@tonic-gate * The memscrubber will also use this routine to clear any errors 58987c478bd9Sstevel@tonic-gate * caused by its scrubbing with CEEN disabled. 58997c478bd9Sstevel@tonic-gate * 59007c478bd9Sstevel@tonic-gate * flag == SCRUBBER_CEEN_CHECK 59017c478bd9Sstevel@tonic-gate * called from memscrubber, just check/scrub, no reset 59027c478bd9Sstevel@tonic-gate * paddr physical addr. for start of scrub pages 59037c478bd9Sstevel@tonic-gate * vaddr virtual addr. for scrub area 59047c478bd9Sstevel@tonic-gate * psz page size of area to be scrubbed 59057c478bd9Sstevel@tonic-gate * 59067c478bd9Sstevel@tonic-gate * flag == TIMEOUT_CEEN_CHECK 59077c478bd9Sstevel@tonic-gate * timeout function has triggered, reset timeout or CEEN 59087c478bd9Sstevel@tonic-gate * 59097c478bd9Sstevel@tonic-gate * Note: We must not migrate cpus during this function. This can be 59107c478bd9Sstevel@tonic-gate * achieved by one of: 59117c478bd9Sstevel@tonic-gate * - invoking as target of an x-call in which case we're at XCALL_PIL 59127c478bd9Sstevel@tonic-gate * The flag value must be first xcall argument. 59137c478bd9Sstevel@tonic-gate * - disabling kernel preemption. This should be done for very short 59147c478bd9Sstevel@tonic-gate * periods so is not suitable for SCRUBBER_CEEN_CHECK where we might 59157c478bd9Sstevel@tonic-gate * scrub an extended area with cpu_check_block. The call for 59167c478bd9Sstevel@tonic-gate * TIMEOUT_CEEN_CHECK uses this so cpu_check_ce must be kept 59177c478bd9Sstevel@tonic-gate * brief for this case. 59187c478bd9Sstevel@tonic-gate * - binding to a cpu, eg with thread_affinity_set(). This is used 59197c478bd9Sstevel@tonic-gate * in the SCRUBBER_CEEN_CHECK case, but is not practical for 59207c478bd9Sstevel@tonic-gate * the TIMEOUT_CEEN_CHECK because both need cpu_lock. 59217c478bd9Sstevel@tonic-gate */ 59227c478bd9Sstevel@tonic-gate void 59237c478bd9Sstevel@tonic-gate cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t psz) 59247c478bd9Sstevel@tonic-gate { 59257c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 59267c478bd9Sstevel@tonic-gate uint64_t ec_err_enable; 59277c478bd9Sstevel@tonic-gate uint64_t page_offset; 59287c478bd9Sstevel@tonic-gate 59297c478bd9Sstevel@tonic-gate /* Read AFSR */ 59307c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 59317c478bd9Sstevel@tonic-gate 59327c478bd9Sstevel@tonic-gate /* 59337c478bd9Sstevel@tonic-gate * If no CEEN errors have occurred during the timeout 59347c478bd9Sstevel@tonic-gate * interval, it is safe to re-enable CEEN and exit. 59357c478bd9Sstevel@tonic-gate */ 59367c478bd9Sstevel@tonic-gate if ((cpu_error_regs.afsr & C_AFSR_CECC_ERRS) == 0) { 59377c478bd9Sstevel@tonic-gate if (flag == TIMEOUT_CEEN_CHECK && 59387c478bd9Sstevel@tonic-gate !((ec_err_enable = get_error_enable()) & EN_REG_CEEN)) 59397c478bd9Sstevel@tonic-gate set_error_enable(ec_err_enable | EN_REG_CEEN); 59407c478bd9Sstevel@tonic-gate return; 59417c478bd9Sstevel@tonic-gate } 59427c478bd9Sstevel@tonic-gate 59437c478bd9Sstevel@tonic-gate /* 59447c478bd9Sstevel@tonic-gate * Ensure that CEEN was not reenabled (maybe by DR) before 59457c478bd9Sstevel@tonic-gate * we log/clear the error. 59467c478bd9Sstevel@tonic-gate */ 59477c478bd9Sstevel@tonic-gate if ((ec_err_enable = get_error_enable()) & EN_REG_CEEN) 59487c478bd9Sstevel@tonic-gate set_error_enable(ec_err_enable & ~EN_REG_CEEN); 59497c478bd9Sstevel@tonic-gate 59507c478bd9Sstevel@tonic-gate /* 59517c478bd9Sstevel@tonic-gate * log/clear the CE. If CE_CEEN_DEFER is passed, the 59527c478bd9Sstevel@tonic-gate * timeout will be rescheduled when the error is logged. 59537c478bd9Sstevel@tonic-gate */ 59547c478bd9Sstevel@tonic-gate if (!(cpu_error_regs.afsr & cpu_ce_not_deferred)) 59557c478bd9Sstevel@tonic-gate cpu_ce_detected(&cpu_error_regs, 59567c478bd9Sstevel@tonic-gate CE_CEEN_DEFER | CE_CEEN_TIMEOUT); 59577c478bd9Sstevel@tonic-gate else 59587c478bd9Sstevel@tonic-gate cpu_ce_detected(&cpu_error_regs, CE_CEEN_TIMEOUT); 59597c478bd9Sstevel@tonic-gate 59607c478bd9Sstevel@tonic-gate /* 59617c478bd9Sstevel@tonic-gate * If the memory scrubber runs while CEEN is 59627c478bd9Sstevel@tonic-gate * disabled, (or if CEEN is disabled during the 59637c478bd9Sstevel@tonic-gate * scrub as a result of a CE being triggered by 59647c478bd9Sstevel@tonic-gate * it), the range being scrubbed will not be 59657c478bd9Sstevel@tonic-gate * completely cleaned. If there are multiple CEs 59667c478bd9Sstevel@tonic-gate * in the range at most two of these will be dealt 59677c478bd9Sstevel@tonic-gate * with, (one by the trap handler and one by the 59687c478bd9Sstevel@tonic-gate * timeout). It is also possible that none are dealt 59697c478bd9Sstevel@tonic-gate * with, (CEEN disabled and another CE occurs before 59707c478bd9Sstevel@tonic-gate * the timeout triggers). So to ensure that the 59717c478bd9Sstevel@tonic-gate * memory is actually scrubbed, we have to access each 59727c478bd9Sstevel@tonic-gate * memory location in the range and then check whether 59737c478bd9Sstevel@tonic-gate * that access causes a CE. 59747c478bd9Sstevel@tonic-gate */ 59757c478bd9Sstevel@tonic-gate if (flag == SCRUBBER_CEEN_CHECK && va) { 59767c478bd9Sstevel@tonic-gate if ((cpu_error_regs.afar >= pa) && 59777c478bd9Sstevel@tonic-gate (cpu_error_regs.afar < (pa + psz))) { 59787c478bd9Sstevel@tonic-gate /* 59797c478bd9Sstevel@tonic-gate * Force a load from physical memory for each 59807c478bd9Sstevel@tonic-gate * 64-byte block, then check AFSR to determine 59817c478bd9Sstevel@tonic-gate * whether this access caused an error. 59827c478bd9Sstevel@tonic-gate * 59837c478bd9Sstevel@tonic-gate * This is a slow way to do a scrub, but as it will 59847c478bd9Sstevel@tonic-gate * only be invoked when the memory scrubber actually 59857c478bd9Sstevel@tonic-gate * triggered a CE, it should not happen too 59867c478bd9Sstevel@tonic-gate * frequently. 59877c478bd9Sstevel@tonic-gate * 59887c478bd9Sstevel@tonic-gate * cut down what we need to check as the scrubber 59897c478bd9Sstevel@tonic-gate * has verified up to AFAR, so get it's offset 59907c478bd9Sstevel@tonic-gate * into the page and start there. 59917c478bd9Sstevel@tonic-gate */ 59927c478bd9Sstevel@tonic-gate page_offset = (uint64_t)(cpu_error_regs.afar & 59937c478bd9Sstevel@tonic-gate (psz - 1)); 59947c478bd9Sstevel@tonic-gate va = (caddr_t)(va + (P2ALIGN(page_offset, 64))); 59957c478bd9Sstevel@tonic-gate psz -= (uint_t)(P2ALIGN(page_offset, 64)); 59967c478bd9Sstevel@tonic-gate cpu_check_block((caddr_t)(P2ALIGN((uint64_t)va, 64)), 59977c478bd9Sstevel@tonic-gate psz); 59987c478bd9Sstevel@tonic-gate } 59997c478bd9Sstevel@tonic-gate } 60007c478bd9Sstevel@tonic-gate 60017c478bd9Sstevel@tonic-gate /* 60027c478bd9Sstevel@tonic-gate * Reset error enable if this CE is not masked. 60037c478bd9Sstevel@tonic-gate */ 60047c478bd9Sstevel@tonic-gate if ((flag == TIMEOUT_CEEN_CHECK) && 60057c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr & cpu_ce_not_deferred)) 60067c478bd9Sstevel@tonic-gate set_error_enable(ec_err_enable | EN_REG_CEEN); 60077c478bd9Sstevel@tonic-gate 60087c478bd9Sstevel@tonic-gate } 60097c478bd9Sstevel@tonic-gate 60107c478bd9Sstevel@tonic-gate /* 60117c478bd9Sstevel@tonic-gate * Attempt a cpu logout for an error that we did not trap for, such 60127c478bd9Sstevel@tonic-gate * as a CE noticed with CEEN off. It is assumed that we are still running 60137c478bd9Sstevel@tonic-gate * on the cpu that took the error and that we cannot migrate. Returns 60147c478bd9Sstevel@tonic-gate * 0 on success, otherwise nonzero. 60157c478bd9Sstevel@tonic-gate */ 60167c478bd9Sstevel@tonic-gate static int 60177c478bd9Sstevel@tonic-gate cpu_ce_delayed_ec_logout(uint64_t afar) 60187c478bd9Sstevel@tonic-gate { 60197c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 60207c478bd9Sstevel@tonic-gate 60217c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) 60227c478bd9Sstevel@tonic-gate return (0); 60237c478bd9Sstevel@tonic-gate 60247c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 60257c478bd9Sstevel@tonic-gate if (cas64(&clop->clo_data.chd_afar, LOGOUT_INVALID, afar) != 60267c478bd9Sstevel@tonic-gate LOGOUT_INVALID) 60277c478bd9Sstevel@tonic-gate return (0); 60287c478bd9Sstevel@tonic-gate 60297c478bd9Sstevel@tonic-gate cpu_delayed_logout(afar, clop); 60307c478bd9Sstevel@tonic-gate return (1); 60317c478bd9Sstevel@tonic-gate } 60327c478bd9Sstevel@tonic-gate 60337c478bd9Sstevel@tonic-gate /* 60347c478bd9Sstevel@tonic-gate * We got an error while CEEN was disabled. We 60357c478bd9Sstevel@tonic-gate * need to clean up after it and log whatever 60367c478bd9Sstevel@tonic-gate * information we have on the CE. 60377c478bd9Sstevel@tonic-gate */ 60387c478bd9Sstevel@tonic-gate void 60397c478bd9Sstevel@tonic-gate cpu_ce_detected(ch_cpu_errors_t *cpu_error_regs, int flag) 60407c478bd9Sstevel@tonic-gate { 60417c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 60427c478bd9Sstevel@tonic-gate struct async_flt *aflt; 60437c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 60447c478bd9Sstevel@tonic-gate 60457c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 60467c478bd9Sstevel@tonic-gate ch_flt.flt_trapped_ce = flag; 60477c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 60487c478bd9Sstevel@tonic-gate aflt->flt_stat = cpu_error_regs->afsr & C_AFSR_MASK; 60497c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = cpu_error_regs->afsr_ext; 60507c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = (cpu_error_regs->afsr_ext & C_AFSR_EXT_ALL_ERRS) | 60517c478bd9Sstevel@tonic-gate (cpu_error_regs->afsr & C_AFSR_ALL_ERRS); 60527c478bd9Sstevel@tonic-gate aflt->flt_addr = cpu_error_regs->afar; 60537c478bd9Sstevel@tonic-gate #if defined(SERRANO) 60547c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs->afar2; 60557c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 60567c478bd9Sstevel@tonic-gate aflt->flt_pc = NULL; 60577c478bd9Sstevel@tonic-gate aflt->flt_priv = ((cpu_error_regs->afsr & C_AFSR_PRIV) != 0); 60587c478bd9Sstevel@tonic-gate aflt->flt_tl = 0; 60597c478bd9Sstevel@tonic-gate aflt->flt_panic = 0; 60607c478bd9Sstevel@tonic-gate cpu_log_and_clear_ce(&ch_flt); 60617c478bd9Sstevel@tonic-gate 60627c478bd9Sstevel@tonic-gate /* 60637c478bd9Sstevel@tonic-gate * check if we caused any errors during cleanup 60647c478bd9Sstevel@tonic-gate */ 60657c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 60667c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 60677c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 60687c478bd9Sstevel@tonic-gate NULL); 60697c478bd9Sstevel@tonic-gate } 60707c478bd9Sstevel@tonic-gate } 60717c478bd9Sstevel@tonic-gate 60727c478bd9Sstevel@tonic-gate /* 60737c478bd9Sstevel@tonic-gate * Log/clear CEEN-controlled disrupting errors 60747c478bd9Sstevel@tonic-gate */ 60757c478bd9Sstevel@tonic-gate static void 60767c478bd9Sstevel@tonic-gate cpu_log_and_clear_ce(ch_async_flt_t *ch_flt) 60777c478bd9Sstevel@tonic-gate { 60787c478bd9Sstevel@tonic-gate struct async_flt *aflt; 60797c478bd9Sstevel@tonic-gate uint64_t afsr, afsr_errs; 60807c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 60817c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 60827c478bd9Sstevel@tonic-gate on_trap_data_t *otp = curthread->t_ontrap; 60837c478bd9Sstevel@tonic-gate 60847c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)ch_flt; 60857c478bd9Sstevel@tonic-gate afsr = aflt->flt_stat; 60867c478bd9Sstevel@tonic-gate afsr_errs = ch_flt->afsr_errs; 60877c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 60887c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 60897c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 60907c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 60917c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 60927c478bd9Sstevel@tonic-gate aflt->flt_status = ECC_C_TRAP; 60937c478bd9Sstevel@tonic-gate 60947c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 60957c478bd9Sstevel@tonic-gate /* 60967c478bd9Sstevel@tonic-gate * Get the CPU log out info for Disrupting Trap. 60977c478bd9Sstevel@tonic-gate */ 60987c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 60997c478bd9Sstevel@tonic-gate clop = NULL; 61007c478bd9Sstevel@tonic-gate ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 61017c478bd9Sstevel@tonic-gate } else { 61027c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 61037c478bd9Sstevel@tonic-gate } 61047c478bd9Sstevel@tonic-gate 61057c478bd9Sstevel@tonic-gate if (clop && ch_flt->flt_trapped_ce & CE_CEEN_TIMEOUT) { 61067c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 61077c478bd9Sstevel@tonic-gate 61087c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 61097c478bd9Sstevel@tonic-gate (void) cpu_ce_delayed_ec_logout(cpu_error_regs.afar); 61107c478bd9Sstevel@tonic-gate clop->clo_data.chd_afsr = cpu_error_regs.afsr; 61117c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = cpu_error_regs.afar; 61127c478bd9Sstevel@tonic-gate clop->clo_data.chd_afsr_ext = cpu_error_regs.afsr_ext; 61137c478bd9Sstevel@tonic-gate clop->clo_sdw_data.chd_afsr = cpu_error_regs.shadow_afsr; 61147c478bd9Sstevel@tonic-gate clop->clo_sdw_data.chd_afar = cpu_error_regs.shadow_afar; 61157c478bd9Sstevel@tonic-gate clop->clo_sdw_data.chd_afsr_ext = 61167c478bd9Sstevel@tonic-gate cpu_error_regs.shadow_afsr_ext; 61177c478bd9Sstevel@tonic-gate #if defined(SERRANO) 61187c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar2 = cpu_error_regs.afar2; 61197c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 61207c478bd9Sstevel@tonic-gate ch_flt->flt_data_incomplete = 1; 61217c478bd9Sstevel@tonic-gate 61227c478bd9Sstevel@tonic-gate /* 61237c478bd9Sstevel@tonic-gate * The logging/clear code expects AFSR/AFAR to be cleared. 61247c478bd9Sstevel@tonic-gate * The trap handler does it for CEEN enabled errors 61257c478bd9Sstevel@tonic-gate * so we need to do it here. 61267c478bd9Sstevel@tonic-gate */ 61277c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 61287c478bd9Sstevel@tonic-gate } 61297c478bd9Sstevel@tonic-gate 61307c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 61317c478bd9Sstevel@tonic-gate /* 61327c478bd9Sstevel@tonic-gate * FRC: Can't scrub memory as we don't have AFAR for Jalapeno. 61337c478bd9Sstevel@tonic-gate * For Serrano, even thou we do have the AFAR, we still do the 61347c478bd9Sstevel@tonic-gate * scrub on the RCE side since that's where the error type can 61357c478bd9Sstevel@tonic-gate * be properly classified as intermittent, persistent, etc. 61367c478bd9Sstevel@tonic-gate * 61377c478bd9Sstevel@tonic-gate * CE/RCE: If error is in memory and AFAR is valid, scrub the memory. 61387c478bd9Sstevel@tonic-gate * Must scrub memory before cpu_queue_events, as scrubbing memory sets 61397c478bd9Sstevel@tonic-gate * the flt_status bits. 61407c478bd9Sstevel@tonic-gate */ 61417c478bd9Sstevel@tonic-gate if ((afsr & (C_AFSR_CE|C_AFSR_RCE)) && 61427c478bd9Sstevel@tonic-gate (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 61437c478bd9Sstevel@tonic-gate cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_RCE)))) { 61447c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err(aflt, B_TRUE); 61457c478bd9Sstevel@tonic-gate } 61467c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 61477c478bd9Sstevel@tonic-gate /* 61487c478bd9Sstevel@tonic-gate * CE/EMC: If error is in memory and AFAR is valid, scrub the memory. 61497c478bd9Sstevel@tonic-gate * Must scrub memory before cpu_queue_events, as scrubbing memory sets 61507c478bd9Sstevel@tonic-gate * the flt_status bits. 61517c478bd9Sstevel@tonic-gate */ 61527c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_CE|C_AFSR_EMC)) { 61537c478bd9Sstevel@tonic-gate if (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 61547c478bd9Sstevel@tonic-gate cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_EMC))) { 61557c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err(aflt, B_TRUE); 61567c478bd9Sstevel@tonic-gate } 61577c478bd9Sstevel@tonic-gate } 61587c478bd9Sstevel@tonic-gate 61597c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 61607c478bd9Sstevel@tonic-gate 61617c478bd9Sstevel@tonic-gate /* 61627c478bd9Sstevel@tonic-gate * Update flt_prot if this error occurred under on_trap protection. 61637c478bd9Sstevel@tonic-gate */ 61647c478bd9Sstevel@tonic-gate if (otp != NULL && (otp->ot_prot & OT_DATA_EC)) 61657c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_EC; 61667c478bd9Sstevel@tonic-gate 61677c478bd9Sstevel@tonic-gate /* 61687c478bd9Sstevel@tonic-gate * Queue events on the async event queue, one event per error bit. 61697c478bd9Sstevel@tonic-gate */ 61707c478bd9Sstevel@tonic-gate if (cpu_queue_events(ch_flt, pr_reason, afsr_errs, clop) == 0 || 61717c478bd9Sstevel@tonic-gate (afsr_errs & (C_AFSR_CECC_ERRS | C_AFSR_EXT_CECC_ERRS)) == 0) { 61727c478bd9Sstevel@tonic-gate ch_flt->flt_type = CPU_INV_AFSR; 61737c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 61747c478bd9Sstevel@tonic-gate (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 61757c478bd9Sstevel@tonic-gate aflt->flt_panic); 61767c478bd9Sstevel@tonic-gate } 61777c478bd9Sstevel@tonic-gate 61787c478bd9Sstevel@tonic-gate /* 61797c478bd9Sstevel@tonic-gate * Zero out + invalidate CPU logout. 61807c478bd9Sstevel@tonic-gate */ 61817c478bd9Sstevel@tonic-gate if (clop) { 61827c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 61837c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 61847c478bd9Sstevel@tonic-gate } 61857c478bd9Sstevel@tonic-gate 61867c478bd9Sstevel@tonic-gate /* 61877c478bd9Sstevel@tonic-gate * If either a CPC, WDC or EDC error has occurred while CEEN 61887c478bd9Sstevel@tonic-gate * was disabled, we need to flush either the entire 61897c478bd9Sstevel@tonic-gate * E$ or an E$ line. 61907c478bd9Sstevel@tonic-gate */ 61917c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 61927c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_CPU | C_AFSR_WDC)) 61937c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 61947c478bd9Sstevel@tonic-gate if (afsr_errs & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_WDC | C_AFSR_L3_EDC | 61957c478bd9Sstevel@tonic-gate C_AFSR_L3_CPC | C_AFSR_L3_WDC)) 61967c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 61977c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(ch_flt); 61987c478bd9Sstevel@tonic-gate 61997c478bd9Sstevel@tonic-gate } 62007c478bd9Sstevel@tonic-gate 62017c478bd9Sstevel@tonic-gate /* 62027c478bd9Sstevel@tonic-gate * depending on the error type, we determine whether we 62037c478bd9Sstevel@tonic-gate * need to flush the entire ecache or just a line. 62047c478bd9Sstevel@tonic-gate */ 62057c478bd9Sstevel@tonic-gate static int 62067c478bd9Sstevel@tonic-gate cpu_error_ecache_flush_required(ch_async_flt_t *ch_flt) 62077c478bd9Sstevel@tonic-gate { 62087c478bd9Sstevel@tonic-gate struct async_flt *aflt; 62097c478bd9Sstevel@tonic-gate uint64_t afsr; 62107c478bd9Sstevel@tonic-gate uint64_t afsr_errs = ch_flt->afsr_errs; 62117c478bd9Sstevel@tonic-gate 62127c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)ch_flt; 62137c478bd9Sstevel@tonic-gate afsr = aflt->flt_stat; 62147c478bd9Sstevel@tonic-gate 62157c478bd9Sstevel@tonic-gate /* 62167c478bd9Sstevel@tonic-gate * If we got multiple errors, no point in trying 62177c478bd9Sstevel@tonic-gate * the individual cases, just flush the whole cache 62187c478bd9Sstevel@tonic-gate */ 62197c478bd9Sstevel@tonic-gate if (afsr & C_AFSR_ME) { 62207c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 62217c478bd9Sstevel@tonic-gate } 62227c478bd9Sstevel@tonic-gate 62237c478bd9Sstevel@tonic-gate /* 62247c478bd9Sstevel@tonic-gate * If either a CPC, WDC or EDC error has occurred while CEEN 62257c478bd9Sstevel@tonic-gate * was disabled, we need to flush entire E$. We can't just 62267c478bd9Sstevel@tonic-gate * flush the cache line affected as the ME bit 62277c478bd9Sstevel@tonic-gate * is not set when multiple correctable errors of the same 62287c478bd9Sstevel@tonic-gate * type occur, so we might have multiple CPC or EDC errors, 62297c478bd9Sstevel@tonic-gate * with only the first recorded. 62307c478bd9Sstevel@tonic-gate */ 62317c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 62327c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_CPC | C_AFSR_CPU | C_AFSR_EDC | C_AFSR_WDC)) { 62337c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 62347c478bd9Sstevel@tonic-gate if (afsr_errs & (C_AFSR_CPC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_L3_CPC | 62357c478bd9Sstevel@tonic-gate C_AFSR_L3_EDC | C_AFSR_L3_WDC)) { 62367c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 62377c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 62387c478bd9Sstevel@tonic-gate } 62397c478bd9Sstevel@tonic-gate 62407c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 62417c478bd9Sstevel@tonic-gate /* 62427c478bd9Sstevel@tonic-gate * If only UE or RUE is set, flush the Ecache line, otherwise 62437c478bd9Sstevel@tonic-gate * flush the entire Ecache. 62447c478bd9Sstevel@tonic-gate */ 62457c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_UE|C_AFSR_RUE)) { 62467c478bd9Sstevel@tonic-gate if ((afsr & C_AFSR_ALL_ERRS) == C_AFSR_UE || 62477c478bd9Sstevel@tonic-gate (afsr & C_AFSR_ALL_ERRS) == C_AFSR_RUE) { 62487c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 62497c478bd9Sstevel@tonic-gate } else { 62507c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 62517c478bd9Sstevel@tonic-gate } 62527c478bd9Sstevel@tonic-gate } 62537c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 62547c478bd9Sstevel@tonic-gate /* 62557c478bd9Sstevel@tonic-gate * If UE only is set, flush the Ecache line, otherwise 62567c478bd9Sstevel@tonic-gate * flush the entire Ecache. 62577c478bd9Sstevel@tonic-gate */ 62587c478bd9Sstevel@tonic-gate if (afsr_errs & C_AFSR_UE) { 62597c478bd9Sstevel@tonic-gate if ((afsr_errs & (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == 62607c478bd9Sstevel@tonic-gate C_AFSR_UE) { 62617c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 62627c478bd9Sstevel@tonic-gate } else { 62637c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 62647c478bd9Sstevel@tonic-gate } 62657c478bd9Sstevel@tonic-gate } 62667c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 62677c478bd9Sstevel@tonic-gate 62687c478bd9Sstevel@tonic-gate /* 62697c478bd9Sstevel@tonic-gate * EDU: If EDU only is set, flush the ecache line, otherwise 62707c478bd9Sstevel@tonic-gate * flush the entire Ecache. 62717c478bd9Sstevel@tonic-gate */ 62727c478bd9Sstevel@tonic-gate if (afsr_errs & (C_AFSR_EDU | C_AFSR_L3_EDU)) { 62737c478bd9Sstevel@tonic-gate if (((afsr_errs & ~C_AFSR_EDU) == 0) || 62747c478bd9Sstevel@tonic-gate ((afsr_errs & ~C_AFSR_L3_EDU) == 0)) { 62757c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 62767c478bd9Sstevel@tonic-gate } else { 62777c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 62787c478bd9Sstevel@tonic-gate } 62797c478bd9Sstevel@tonic-gate } 62807c478bd9Sstevel@tonic-gate 62817c478bd9Sstevel@tonic-gate /* 62827c478bd9Sstevel@tonic-gate * BERR: If BERR only is set, flush the Ecache line, otherwise 62837c478bd9Sstevel@tonic-gate * flush the entire Ecache. 62847c478bd9Sstevel@tonic-gate */ 62857c478bd9Sstevel@tonic-gate if (afsr_errs & C_AFSR_BERR) { 62867c478bd9Sstevel@tonic-gate if ((afsr_errs & ~C_AFSR_BERR) == 0) { 62877c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 62887c478bd9Sstevel@tonic-gate } else { 62897c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 62907c478bd9Sstevel@tonic-gate } 62917c478bd9Sstevel@tonic-gate } 62927c478bd9Sstevel@tonic-gate 62937c478bd9Sstevel@tonic-gate return (0); 62947c478bd9Sstevel@tonic-gate } 62957c478bd9Sstevel@tonic-gate 62967c478bd9Sstevel@tonic-gate void 62977c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(ch_async_flt_t *ch_flt) 62987c478bd9Sstevel@tonic-gate { 62997c478bd9Sstevel@tonic-gate int ecache_flush_flag = 63007c478bd9Sstevel@tonic-gate cpu_error_ecache_flush_required(ch_flt); 63017c478bd9Sstevel@tonic-gate 63027c478bd9Sstevel@tonic-gate /* 63037c478bd9Sstevel@tonic-gate * Flush Ecache line or entire Ecache based on above checks. 63047c478bd9Sstevel@tonic-gate */ 63057c478bd9Sstevel@tonic-gate if (ecache_flush_flag == ECACHE_FLUSH_ALL) 63067c478bd9Sstevel@tonic-gate cpu_flush_ecache(); 63077c478bd9Sstevel@tonic-gate else if (ecache_flush_flag == ECACHE_FLUSH_LINE) { 63087c478bd9Sstevel@tonic-gate cpu_flush_ecache_line(ch_flt); 63097c478bd9Sstevel@tonic-gate } 63107c478bd9Sstevel@tonic-gate 63117c478bd9Sstevel@tonic-gate } 63127c478bd9Sstevel@tonic-gate 63137c478bd9Sstevel@tonic-gate /* 63147c478bd9Sstevel@tonic-gate * Extract the PA portion from the E$ tag. 63157c478bd9Sstevel@tonic-gate */ 63167c478bd9Sstevel@tonic-gate uint64_t 63177c478bd9Sstevel@tonic-gate cpu_ectag_to_pa(int setsize, uint64_t tag) 63187c478bd9Sstevel@tonic-gate { 63197c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 63207c478bd9Sstevel@tonic-gate return (JG_ECTAG_TO_PA(setsize, tag)); 63217c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 63227c478bd9Sstevel@tonic-gate return (PN_L3TAG_TO_PA(tag)); 63237c478bd9Sstevel@tonic-gate else 63247c478bd9Sstevel@tonic-gate return (CH_ECTAG_TO_PA(setsize, tag)); 63257c478bd9Sstevel@tonic-gate } 63267c478bd9Sstevel@tonic-gate 63277c478bd9Sstevel@tonic-gate /* 63287c478bd9Sstevel@tonic-gate * Convert the E$ tag PA into an E$ subblock index. 63297c478bd9Sstevel@tonic-gate */ 63307c478bd9Sstevel@tonic-gate static int 63317c478bd9Sstevel@tonic-gate cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr) 63327c478bd9Sstevel@tonic-gate { 63337c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 63347c478bd9Sstevel@tonic-gate return (JG_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 63357c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 63367c478bd9Sstevel@tonic-gate /* Panther has only one subblock per line */ 63377c478bd9Sstevel@tonic-gate return (0); 63387c478bd9Sstevel@tonic-gate else 63397c478bd9Sstevel@tonic-gate return (CH_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 63407c478bd9Sstevel@tonic-gate } 63417c478bd9Sstevel@tonic-gate 63427c478bd9Sstevel@tonic-gate /* 63437c478bd9Sstevel@tonic-gate * All subblocks in an E$ line must be invalid for 63447c478bd9Sstevel@tonic-gate * the line to be invalid. 63457c478bd9Sstevel@tonic-gate */ 63467c478bd9Sstevel@tonic-gate int 63477c478bd9Sstevel@tonic-gate cpu_ectag_line_invalid(int cachesize, uint64_t tag) 63487c478bd9Sstevel@tonic-gate { 63497c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 63507c478bd9Sstevel@tonic-gate return (JG_ECTAG_LINE_INVALID(cachesize, tag)); 63517c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 63527c478bd9Sstevel@tonic-gate return (PN_L3_LINE_INVALID(tag)); 63537c478bd9Sstevel@tonic-gate else 63547c478bd9Sstevel@tonic-gate return (CH_ECTAG_LINE_INVALID(cachesize, tag)); 63557c478bd9Sstevel@tonic-gate } 63567c478bd9Sstevel@tonic-gate 63577c478bd9Sstevel@tonic-gate /* 63587c478bd9Sstevel@tonic-gate * Extract state bits for a subblock given the tag. Note that for Panther 63597c478bd9Sstevel@tonic-gate * this works on both l2 and l3 tags. 63607c478bd9Sstevel@tonic-gate */ 63617c478bd9Sstevel@tonic-gate static int 63627c478bd9Sstevel@tonic-gate cpu_ectag_pa_to_subblk_state(int cachesize, uint64_t subaddr, uint64_t tag) 63637c478bd9Sstevel@tonic-gate { 63647c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 63657c478bd9Sstevel@tonic-gate return (JG_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 63667c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 63677c478bd9Sstevel@tonic-gate return (tag & CH_ECSTATE_MASK); 63687c478bd9Sstevel@tonic-gate else 63697c478bd9Sstevel@tonic-gate return (CH_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 63707c478bd9Sstevel@tonic-gate } 63717c478bd9Sstevel@tonic-gate 63727c478bd9Sstevel@tonic-gate /* 63737c478bd9Sstevel@tonic-gate * Cpu specific initialization. 63747c478bd9Sstevel@tonic-gate */ 63757c478bd9Sstevel@tonic-gate void 63767c478bd9Sstevel@tonic-gate cpu_mp_init(void) 63777c478bd9Sstevel@tonic-gate { 63787c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 63797c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_recover) { 63807c478bd9Sstevel@tonic-gate cheetah_nudge_init(); 63817c478bd9Sstevel@tonic-gate } 63827c478bd9Sstevel@tonic-gate #endif 63837c478bd9Sstevel@tonic-gate } 63847c478bd9Sstevel@tonic-gate 63857c478bd9Sstevel@tonic-gate void 63867c478bd9Sstevel@tonic-gate cpu_ereport_post(struct async_flt *aflt) 63877c478bd9Sstevel@tonic-gate { 63887c478bd9Sstevel@tonic-gate char *cpu_type, buf[FM_MAX_CLASS]; 63897aec1d6eScindi char sbuf[21]; /* sizeof (UINT64_MAX) + '\0' */ 63907c478bd9Sstevel@tonic-gate nv_alloc_t *nva = NULL; 63917c478bd9Sstevel@tonic-gate nvlist_t *ereport, *detector, *resource; 63927c478bd9Sstevel@tonic-gate errorq_elem_t *eqep; 63937c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 63947c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 63957c478bd9Sstevel@tonic-gate int len = 0; 63967aec1d6eScindi uint8_t msg_type, mask; 63977c478bd9Sstevel@tonic-gate plat_ecc_ch_async_flt_t plat_ecc_ch_flt; 63987c478bd9Sstevel@tonic-gate 63997c478bd9Sstevel@tonic-gate if (aflt->flt_panic || panicstr) { 64007c478bd9Sstevel@tonic-gate eqep = errorq_reserve(ereport_errorq); 64017c478bd9Sstevel@tonic-gate if (eqep == NULL) 64027c478bd9Sstevel@tonic-gate return; 64037c478bd9Sstevel@tonic-gate ereport = errorq_elem_nvl(ereport_errorq, eqep); 64047c478bd9Sstevel@tonic-gate nva = errorq_elem_nva(ereport_errorq, eqep); 64057c478bd9Sstevel@tonic-gate } else { 64067c478bd9Sstevel@tonic-gate ereport = fm_nvlist_create(nva); 64077c478bd9Sstevel@tonic-gate } 64087c478bd9Sstevel@tonic-gate 64097c478bd9Sstevel@tonic-gate /* 64107c478bd9Sstevel@tonic-gate * Create the scheme "cpu" FMRI. 64117c478bd9Sstevel@tonic-gate */ 64127c478bd9Sstevel@tonic-gate detector = fm_nvlist_create(nva); 64137c478bd9Sstevel@tonic-gate resource = fm_nvlist_create(nva); 64147c478bd9Sstevel@tonic-gate switch (cpunodes[aflt->flt_inst].implementation) { 64157c478bd9Sstevel@tonic-gate case CHEETAH_IMPL: 64167c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIII; 64177c478bd9Sstevel@tonic-gate break; 64187c478bd9Sstevel@tonic-gate case CHEETAH_PLUS_IMPL: 64197c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIIIplus; 64207c478bd9Sstevel@tonic-gate break; 64217c478bd9Sstevel@tonic-gate case JALAPENO_IMPL: 64227c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIIIi; 64237c478bd9Sstevel@tonic-gate break; 64247c478bd9Sstevel@tonic-gate case SERRANO_IMPL: 64257c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIIIiplus; 64267c478bd9Sstevel@tonic-gate break; 64277c478bd9Sstevel@tonic-gate case JAGUAR_IMPL: 64287c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIV; 64297c478bd9Sstevel@tonic-gate break; 64307c478bd9Sstevel@tonic-gate case PANTHER_IMPL: 64317c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIVplus; 64327c478bd9Sstevel@tonic-gate break; 64337c478bd9Sstevel@tonic-gate default: 64347c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_UNSUPPORTED; 64357c478bd9Sstevel@tonic-gate break; 64367c478bd9Sstevel@tonic-gate } 64377aec1d6eScindi mask = cpunodes[aflt->flt_inst].version; 64387aec1d6eScindi (void) snprintf(sbuf, sizeof (sbuf), "%llX", 64397aec1d6eScindi (u_longlong_t)cpunodes[aflt->flt_inst].device_id); 64407c478bd9Sstevel@tonic-gate (void) fm_fmri_cpu_set(detector, FM_CPU_SCHEME_VERSION, NULL, 64417aec1d6eScindi aflt->flt_inst, &mask, (const char *)sbuf); 64427c478bd9Sstevel@tonic-gate 64437c478bd9Sstevel@tonic-gate /* 64447c478bd9Sstevel@tonic-gate * Encode all the common data into the ereport. 64457c478bd9Sstevel@tonic-gate */ 64467c478bd9Sstevel@tonic-gate (void) snprintf(buf, FM_MAX_CLASS, "%s.%s.%s", 64477c478bd9Sstevel@tonic-gate FM_ERROR_CPU, cpu_type, aflt->flt_erpt_class); 64487c478bd9Sstevel@tonic-gate 64497c478bd9Sstevel@tonic-gate fm_ereport_set(ereport, FM_EREPORT_VERSION, buf, 64507c478bd9Sstevel@tonic-gate fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, FM_ENA_FMT1), 64517c478bd9Sstevel@tonic-gate detector, NULL); 64527c478bd9Sstevel@tonic-gate 64537c478bd9Sstevel@tonic-gate /* 64547c478bd9Sstevel@tonic-gate * Encode the error specific data that was saved in 64557c478bd9Sstevel@tonic-gate * the async_flt structure into the ereport. 64567c478bd9Sstevel@tonic-gate */ 64577c478bd9Sstevel@tonic-gate cpu_payload_add_aflt(aflt, ereport, resource, 64587c478bd9Sstevel@tonic-gate &plat_ecc_ch_flt.ecaf_afar_status, 64597c478bd9Sstevel@tonic-gate &plat_ecc_ch_flt.ecaf_synd_status); 64607c478bd9Sstevel@tonic-gate 64617c478bd9Sstevel@tonic-gate if (aflt->flt_panic || panicstr) { 64627c478bd9Sstevel@tonic-gate errorq_commit(ereport_errorq, eqep, ERRORQ_SYNC); 64637c478bd9Sstevel@tonic-gate } else { 64647c478bd9Sstevel@tonic-gate (void) fm_ereport_post(ereport, EVCH_TRYHARD); 64657c478bd9Sstevel@tonic-gate fm_nvlist_destroy(ereport, FM_NVA_FREE); 64667c478bd9Sstevel@tonic-gate fm_nvlist_destroy(detector, FM_NVA_FREE); 64677c478bd9Sstevel@tonic-gate fm_nvlist_destroy(resource, FM_NVA_FREE); 64687c478bd9Sstevel@tonic-gate } 64697c478bd9Sstevel@tonic-gate /* 64707c478bd9Sstevel@tonic-gate * Send the enhanced error information (plat_ecc_error2_data_t) 64717c478bd9Sstevel@tonic-gate * to the SC olny if it can process it. 64727c478bd9Sstevel@tonic-gate */ 64737c478bd9Sstevel@tonic-gate 64747c478bd9Sstevel@tonic-gate if (&plat_ecc_capability_sc_get && 64757c478bd9Sstevel@tonic-gate plat_ecc_capability_sc_get(PLAT_ECC_ERROR2_MESSAGE)) { 64767c478bd9Sstevel@tonic-gate msg_type = cpu_flt_bit_to_plat_error(aflt); 64777c478bd9Sstevel@tonic-gate if (msg_type != PLAT_ECC_ERROR2_NONE) { 64787c478bd9Sstevel@tonic-gate /* 64797c478bd9Sstevel@tonic-gate * If afar status is not invalid do a unum lookup. 64807c478bd9Sstevel@tonic-gate */ 64817c478bd9Sstevel@tonic-gate if (plat_ecc_ch_flt.ecaf_afar_status != 64827c478bd9Sstevel@tonic-gate AFLT_STAT_INVALID) { 64837c478bd9Sstevel@tonic-gate (void) cpu_get_mem_unum_aflt( 64847c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_synd_status, aflt, 64857c478bd9Sstevel@tonic-gate unum, UNUM_NAMLEN, &len); 64867c478bd9Sstevel@tonic-gate } else { 64877c478bd9Sstevel@tonic-gate unum[0] = '\0'; 64887c478bd9Sstevel@tonic-gate } 64897c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_sdw_afar = ch_flt->flt_sdw_afar; 64907c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_sdw_afsr = ch_flt->flt_sdw_afsr; 64917c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_afsr_ext = ch_flt->afsr_ext; 64927c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_sdw_afsr_ext = 64937c478bd9Sstevel@tonic-gate ch_flt->flt_sdw_afsr_ext; 64947c478bd9Sstevel@tonic-gate 64957c478bd9Sstevel@tonic-gate if (&plat_log_fruid_error2) 64967c478bd9Sstevel@tonic-gate plat_log_fruid_error2(msg_type, unum, aflt, 64977c478bd9Sstevel@tonic-gate &plat_ecc_ch_flt); 64987c478bd9Sstevel@tonic-gate } 64997c478bd9Sstevel@tonic-gate } 65007c478bd9Sstevel@tonic-gate } 65017c478bd9Sstevel@tonic-gate 65027c478bd9Sstevel@tonic-gate void 65037c478bd9Sstevel@tonic-gate cpu_run_bus_error_handlers(struct async_flt *aflt, int expected) 65047c478bd9Sstevel@tonic-gate { 65057c478bd9Sstevel@tonic-gate int status; 65067c478bd9Sstevel@tonic-gate ddi_fm_error_t de; 65077c478bd9Sstevel@tonic-gate 65087c478bd9Sstevel@tonic-gate bzero(&de, sizeof (ddi_fm_error_t)); 65097c478bd9Sstevel@tonic-gate 65107c478bd9Sstevel@tonic-gate de.fme_version = DDI_FME_VERSION; 65117c478bd9Sstevel@tonic-gate de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, 65127c478bd9Sstevel@tonic-gate FM_ENA_FMT1); 65137c478bd9Sstevel@tonic-gate de.fme_flag = expected; 65147c478bd9Sstevel@tonic-gate de.fme_bus_specific = (void *)aflt->flt_addr; 65157c478bd9Sstevel@tonic-gate status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de); 65167c478bd9Sstevel@tonic-gate if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL)) 65177c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 65187c478bd9Sstevel@tonic-gate } 65197c478bd9Sstevel@tonic-gate 65207c478bd9Sstevel@tonic-gate void 65217c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz, 65227c478bd9Sstevel@tonic-gate errorq_t *eqp, uint_t flag) 65237c478bd9Sstevel@tonic-gate { 65247c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)payload; 65257c478bd9Sstevel@tonic-gate 65267c478bd9Sstevel@tonic-gate aflt->flt_erpt_class = error_class; 65277c478bd9Sstevel@tonic-gate errorq_dispatch(eqp, payload, payload_sz, flag); 65287c478bd9Sstevel@tonic-gate } 65297c478bd9Sstevel@tonic-gate 65307c478bd9Sstevel@tonic-gate /* 65317c478bd9Sstevel@tonic-gate * This routine may be called by the IO module, but does not do 65327c478bd9Sstevel@tonic-gate * anything in this cpu module. The SERD algorithm is handled by 65337c478bd9Sstevel@tonic-gate * cpumem-diagnosis engine instead. 65347c478bd9Sstevel@tonic-gate */ 65357c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 65367c478bd9Sstevel@tonic-gate void 65377c478bd9Sstevel@tonic-gate cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum) 65387c478bd9Sstevel@tonic-gate {} 65397c478bd9Sstevel@tonic-gate 65407c478bd9Sstevel@tonic-gate void 65417c478bd9Sstevel@tonic-gate adjust_hw_copy_limits(int ecache_size) 65427c478bd9Sstevel@tonic-gate { 65437c478bd9Sstevel@tonic-gate /* 65447c478bd9Sstevel@tonic-gate * Set hw copy limits. 65457c478bd9Sstevel@tonic-gate * 65467c478bd9Sstevel@tonic-gate * /etc/system will be parsed later and can override one or more 65477c478bd9Sstevel@tonic-gate * of these settings. 65487c478bd9Sstevel@tonic-gate * 65497c478bd9Sstevel@tonic-gate * At this time, ecache size seems only mildly relevant. 65507c478bd9Sstevel@tonic-gate * We seem to run into issues with the d-cache and stalls 65517c478bd9Sstevel@tonic-gate * we see on misses. 65527c478bd9Sstevel@tonic-gate * 65537c478bd9Sstevel@tonic-gate * Cycle measurement indicates that 2 byte aligned copies fare 65547c478bd9Sstevel@tonic-gate * little better than doing things with VIS at around 512 bytes. 65557c478bd9Sstevel@tonic-gate * 4 byte aligned shows promise until around 1024 bytes. 8 Byte 65567c478bd9Sstevel@tonic-gate * aligned is faster whenever the source and destination data 65577c478bd9Sstevel@tonic-gate * in cache and the total size is less than 2 Kbytes. The 2K 65587c478bd9Sstevel@tonic-gate * limit seems to be driven by the 2K write cache. 65597c478bd9Sstevel@tonic-gate * When more than 2K of copies are done in non-VIS mode, stores 65607c478bd9Sstevel@tonic-gate * backup in the write cache. In VIS mode, the write cache is 65617c478bd9Sstevel@tonic-gate * bypassed, allowing faster cache-line writes aligned on cache 65627c478bd9Sstevel@tonic-gate * boundaries. 65637c478bd9Sstevel@tonic-gate * 65647c478bd9Sstevel@tonic-gate * In addition, in non-VIS mode, there is no prefetching, so 65657c478bd9Sstevel@tonic-gate * for larger copies, the advantage of prefetching to avoid even 65667c478bd9Sstevel@tonic-gate * occasional cache misses is enough to justify using the VIS code. 65677c478bd9Sstevel@tonic-gate * 65687c478bd9Sstevel@tonic-gate * During testing, it was discovered that netbench ran 3% slower 65697c478bd9Sstevel@tonic-gate * when hw_copy_limit_8 was 2K or larger. Apparently for server 65707c478bd9Sstevel@tonic-gate * applications, data is only used once (copied to the output 65717c478bd9Sstevel@tonic-gate * buffer, then copied by the network device off the system). Using 65727c478bd9Sstevel@tonic-gate * the VIS copy saves more L2 cache state. Network copies are 65737c478bd9Sstevel@tonic-gate * around 1.3K to 1.5K in size for historical reasons. 65747c478bd9Sstevel@tonic-gate * 65757c478bd9Sstevel@tonic-gate * Therefore, a limit of 1K bytes will be used for the 8 byte 65767c478bd9Sstevel@tonic-gate * aligned copy even for large caches and 8 MB ecache. The 65777c478bd9Sstevel@tonic-gate * infrastructure to allow different limits for different sized 65787c478bd9Sstevel@tonic-gate * caches is kept to allow further tuning in later releases. 65797c478bd9Sstevel@tonic-gate */ 65807c478bd9Sstevel@tonic-gate 65817c478bd9Sstevel@tonic-gate if (min_ecache_size == 0 && use_hw_bcopy) { 65827c478bd9Sstevel@tonic-gate /* 65837c478bd9Sstevel@tonic-gate * First time through - should be before /etc/system 65847c478bd9Sstevel@tonic-gate * is read. 65857c478bd9Sstevel@tonic-gate * Could skip the checks for zero but this lets us 65867c478bd9Sstevel@tonic-gate * preserve any debugger rewrites. 65877c478bd9Sstevel@tonic-gate */ 65887c478bd9Sstevel@tonic-gate if (hw_copy_limit_1 == 0) { 65897c478bd9Sstevel@tonic-gate hw_copy_limit_1 = VIS_COPY_THRESHOLD; 65907c478bd9Sstevel@tonic-gate priv_hcl_1 = hw_copy_limit_1; 65917c478bd9Sstevel@tonic-gate } 65927c478bd9Sstevel@tonic-gate if (hw_copy_limit_2 == 0) { 65937c478bd9Sstevel@tonic-gate hw_copy_limit_2 = 2 * VIS_COPY_THRESHOLD; 65947c478bd9Sstevel@tonic-gate priv_hcl_2 = hw_copy_limit_2; 65957c478bd9Sstevel@tonic-gate } 65967c478bd9Sstevel@tonic-gate if (hw_copy_limit_4 == 0) { 65977c478bd9Sstevel@tonic-gate hw_copy_limit_4 = 4 * VIS_COPY_THRESHOLD; 65987c478bd9Sstevel@tonic-gate priv_hcl_4 = hw_copy_limit_4; 65997c478bd9Sstevel@tonic-gate } 66007c478bd9Sstevel@tonic-gate if (hw_copy_limit_8 == 0) { 66017c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD; 66027c478bd9Sstevel@tonic-gate priv_hcl_8 = hw_copy_limit_8; 66037c478bd9Sstevel@tonic-gate } 66047c478bd9Sstevel@tonic-gate min_ecache_size = ecache_size; 66057c478bd9Sstevel@tonic-gate } else { 66067c478bd9Sstevel@tonic-gate /* 66077c478bd9Sstevel@tonic-gate * MP initialization. Called *after* /etc/system has 66087c478bd9Sstevel@tonic-gate * been parsed. One CPU has already been initialized. 66097c478bd9Sstevel@tonic-gate * Need to cater for /etc/system having scragged one 66107c478bd9Sstevel@tonic-gate * of our values. 66117c478bd9Sstevel@tonic-gate */ 66127c478bd9Sstevel@tonic-gate if (ecache_size == min_ecache_size) { 66137c478bd9Sstevel@tonic-gate /* 66147c478bd9Sstevel@tonic-gate * Same size ecache. We do nothing unless we 66157c478bd9Sstevel@tonic-gate * have a pessimistic ecache setting. In that 66167c478bd9Sstevel@tonic-gate * case we become more optimistic (if the cache is 66177c478bd9Sstevel@tonic-gate * large enough). 66187c478bd9Sstevel@tonic-gate */ 66197c478bd9Sstevel@tonic-gate if (hw_copy_limit_8 == 4 * VIS_COPY_THRESHOLD) { 66207c478bd9Sstevel@tonic-gate /* 66217c478bd9Sstevel@tonic-gate * Need to adjust hw_copy_limit* from our 66227c478bd9Sstevel@tonic-gate * pessimistic uniprocessor value to a more 66237c478bd9Sstevel@tonic-gate * optimistic UP value *iff* it hasn't been 66247c478bd9Sstevel@tonic-gate * reset. 66257c478bd9Sstevel@tonic-gate */ 66267c478bd9Sstevel@tonic-gate if ((ecache_size > 1048576) && 66277c478bd9Sstevel@tonic-gate (priv_hcl_8 == hw_copy_limit_8)) { 66287c478bd9Sstevel@tonic-gate if (ecache_size <= 2097152) 66297c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * 66307c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 66317c478bd9Sstevel@tonic-gate else if (ecache_size <= 4194304) 66327c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * 66337c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 66347c478bd9Sstevel@tonic-gate else 66357c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * 66367c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 66377c478bd9Sstevel@tonic-gate priv_hcl_8 = hw_copy_limit_8; 66387c478bd9Sstevel@tonic-gate } 66397c478bd9Sstevel@tonic-gate } 66407c478bd9Sstevel@tonic-gate } else if (ecache_size < min_ecache_size) { 66417c478bd9Sstevel@tonic-gate /* 66427c478bd9Sstevel@tonic-gate * A different ecache size. Can this even happen? 66437c478bd9Sstevel@tonic-gate */ 66447c478bd9Sstevel@tonic-gate if (priv_hcl_8 == hw_copy_limit_8) { 66457c478bd9Sstevel@tonic-gate /* 66467c478bd9Sstevel@tonic-gate * The previous value that we set 66477c478bd9Sstevel@tonic-gate * is unchanged (i.e., it hasn't been 66487c478bd9Sstevel@tonic-gate * scragged by /etc/system). Rewrite it. 66497c478bd9Sstevel@tonic-gate */ 66507c478bd9Sstevel@tonic-gate if (ecache_size <= 1048576) 66517c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 8 * 66527c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 66537c478bd9Sstevel@tonic-gate else if (ecache_size <= 2097152) 66547c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 8 * 66557c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 66567c478bd9Sstevel@tonic-gate else if (ecache_size <= 4194304) 66577c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 8 * 66587c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 66597c478bd9Sstevel@tonic-gate else 66607c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 10 * 66617c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 66627c478bd9Sstevel@tonic-gate priv_hcl_8 = hw_copy_limit_8; 66637c478bd9Sstevel@tonic-gate min_ecache_size = ecache_size; 66647c478bd9Sstevel@tonic-gate } 66657c478bd9Sstevel@tonic-gate } 66667c478bd9Sstevel@tonic-gate } 66677c478bd9Sstevel@tonic-gate } 66687c478bd9Sstevel@tonic-gate 66697c478bd9Sstevel@tonic-gate /* 66707c478bd9Sstevel@tonic-gate * Called from illegal instruction trap handler to see if we can attribute 66717c478bd9Sstevel@tonic-gate * the trap to a fpras check. 66727c478bd9Sstevel@tonic-gate */ 66737c478bd9Sstevel@tonic-gate int 66747c478bd9Sstevel@tonic-gate fpras_chktrap(struct regs *rp) 66757c478bd9Sstevel@tonic-gate { 66767c478bd9Sstevel@tonic-gate int op; 66777c478bd9Sstevel@tonic-gate struct fpras_chkfngrp *cgp; 66787c478bd9Sstevel@tonic-gate uintptr_t tpc = (uintptr_t)rp->r_pc; 66797c478bd9Sstevel@tonic-gate 66807c478bd9Sstevel@tonic-gate if (fpras_chkfngrps == NULL) 66817c478bd9Sstevel@tonic-gate return (0); 66827c478bd9Sstevel@tonic-gate 66837c478bd9Sstevel@tonic-gate cgp = &fpras_chkfngrps[CPU->cpu_id]; 66847c478bd9Sstevel@tonic-gate for (op = 0; op < FPRAS_NCOPYOPS; ++op) { 66857c478bd9Sstevel@tonic-gate if (tpc >= (uintptr_t)&cgp->fpras_fn[op].fpras_blk0 && 66867c478bd9Sstevel@tonic-gate tpc < (uintptr_t)&cgp->fpras_fn[op].fpras_chkresult) 66877c478bd9Sstevel@tonic-gate break; 66887c478bd9Sstevel@tonic-gate } 66897c478bd9Sstevel@tonic-gate if (op == FPRAS_NCOPYOPS) 66907c478bd9Sstevel@tonic-gate return (0); 66917c478bd9Sstevel@tonic-gate 66927c478bd9Sstevel@tonic-gate /* 66937c478bd9Sstevel@tonic-gate * This is an fpRAS failure caught through an illegal 66947c478bd9Sstevel@tonic-gate * instruction - trampoline. 66957c478bd9Sstevel@tonic-gate */ 66967c478bd9Sstevel@tonic-gate rp->r_pc = (uintptr_t)&cgp->fpras_fn[op].fpras_trampoline; 66977c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 66987c478bd9Sstevel@tonic-gate return (1); 66997c478bd9Sstevel@tonic-gate } 67007c478bd9Sstevel@tonic-gate 67017c478bd9Sstevel@tonic-gate /* 67027c478bd9Sstevel@tonic-gate * fpras_failure is called when a fpras check detects a bad calculation 67037c478bd9Sstevel@tonic-gate * result or an illegal instruction trap is attributed to an fpras 67047c478bd9Sstevel@tonic-gate * check. In all cases we are still bound to CPU. 67057c478bd9Sstevel@tonic-gate */ 67067c478bd9Sstevel@tonic-gate int 67077c478bd9Sstevel@tonic-gate fpras_failure(int op, int how) 67087c478bd9Sstevel@tonic-gate { 67097c478bd9Sstevel@tonic-gate int use_hw_bcopy_orig, use_hw_bzero_orig; 67107c478bd9Sstevel@tonic-gate uint_t hcl1_orig, hcl2_orig, hcl4_orig, hcl8_orig; 67117c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 67127c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)&ch_flt; 67137c478bd9Sstevel@tonic-gate struct fpras_chkfn *sfp, *cfp; 67147c478bd9Sstevel@tonic-gate uint32_t *sip, *cip; 67157c478bd9Sstevel@tonic-gate int i; 67167c478bd9Sstevel@tonic-gate 67177c478bd9Sstevel@tonic-gate /* 67187c478bd9Sstevel@tonic-gate * We're running on a sick CPU. Avoid further FPU use at least for 67197c478bd9Sstevel@tonic-gate * the time in which we dispatch an ereport and (if applicable) panic. 67207c478bd9Sstevel@tonic-gate */ 67217c478bd9Sstevel@tonic-gate use_hw_bcopy_orig = use_hw_bcopy; 67227c478bd9Sstevel@tonic-gate use_hw_bzero_orig = use_hw_bzero; 67237c478bd9Sstevel@tonic-gate hcl1_orig = hw_copy_limit_1; 67247c478bd9Sstevel@tonic-gate hcl2_orig = hw_copy_limit_2; 67257c478bd9Sstevel@tonic-gate hcl4_orig = hw_copy_limit_4; 67267c478bd9Sstevel@tonic-gate hcl8_orig = hw_copy_limit_8; 67277c478bd9Sstevel@tonic-gate use_hw_bcopy = use_hw_bzero = 0; 67287c478bd9Sstevel@tonic-gate hw_copy_limit_1 = hw_copy_limit_2 = hw_copy_limit_4 = 67297c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 0; 67307c478bd9Sstevel@tonic-gate 67317c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 67327c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 67337c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 67347c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 67357c478bd9Sstevel@tonic-gate aflt->flt_status = (how << 8) | op; 67367c478bd9Sstevel@tonic-gate aflt->flt_payload = FM_EREPORT_PAYLOAD_FPU_HWCOPY; 67377c478bd9Sstevel@tonic-gate ch_flt.flt_type = CPU_FPUERR; 67387c478bd9Sstevel@tonic-gate 67397c478bd9Sstevel@tonic-gate /* 67407c478bd9Sstevel@tonic-gate * We must panic if the copy operation had no lofault protection - 67417c478bd9Sstevel@tonic-gate * ie, don't panic for copyin, copyout, kcopy and bcopy called 67427c478bd9Sstevel@tonic-gate * under on_fault and do panic for unprotected bcopy and hwblkpagecopy. 67437c478bd9Sstevel@tonic-gate */ 67447c478bd9Sstevel@tonic-gate aflt->flt_panic = (curthread->t_lofault == NULL); 67457c478bd9Sstevel@tonic-gate 67467c478bd9Sstevel@tonic-gate /* 67477c478bd9Sstevel@tonic-gate * XOR the source instruction block with the copied instruction 67487c478bd9Sstevel@tonic-gate * block - this will show us which bit(s) are corrupted. 67497c478bd9Sstevel@tonic-gate */ 67507c478bd9Sstevel@tonic-gate sfp = (struct fpras_chkfn *)fpras_chkfn_type1; 67517c478bd9Sstevel@tonic-gate cfp = &fpras_chkfngrps[CPU->cpu_id].fpras_fn[op]; 67527c478bd9Sstevel@tonic-gate if (op == FPRAS_BCOPY || op == FPRAS_COPYOUT) { 67537c478bd9Sstevel@tonic-gate sip = &sfp->fpras_blk0[0]; 67547c478bd9Sstevel@tonic-gate cip = &cfp->fpras_blk0[0]; 67557c478bd9Sstevel@tonic-gate } else { 67567c478bd9Sstevel@tonic-gate sip = &sfp->fpras_blk1[0]; 67577c478bd9Sstevel@tonic-gate cip = &cfp->fpras_blk1[0]; 67587c478bd9Sstevel@tonic-gate } 67597c478bd9Sstevel@tonic-gate for (i = 0; i < 16; ++i, ++sip, ++cip) 67607c478bd9Sstevel@tonic-gate ch_flt.flt_fpdata[i] = *sip ^ *cip; 67617c478bd9Sstevel@tonic-gate 67627c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_FPU_HWCOPY, (void *)&ch_flt, 67637c478bd9Sstevel@tonic-gate sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 67647c478bd9Sstevel@tonic-gate 67657c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 67667c478bd9Sstevel@tonic-gate fm_panic("FPU failure on CPU %d", CPU->cpu_id); 67677c478bd9Sstevel@tonic-gate 67687c478bd9Sstevel@tonic-gate /* 67697c478bd9Sstevel@tonic-gate * We get here for copyin/copyout and kcopy or bcopy where the 67707c478bd9Sstevel@tonic-gate * caller has used on_fault. We will flag the error so that 67717c478bd9Sstevel@tonic-gate * the process may be killed The trap_async_hwerr mechanism will 67727c478bd9Sstevel@tonic-gate * take appropriate further action (such as a reboot, contract 67737c478bd9Sstevel@tonic-gate * notification etc). Since we may be continuing we will 67747c478bd9Sstevel@tonic-gate * restore the global hardware copy acceleration switches. 67757c478bd9Sstevel@tonic-gate * 67767c478bd9Sstevel@tonic-gate * When we return from this function to the copy function we want to 67777c478bd9Sstevel@tonic-gate * avoid potentially bad data being used, ie we want the affected 67787c478bd9Sstevel@tonic-gate * copy function to return an error. The caller should therefore 67797c478bd9Sstevel@tonic-gate * invoke its lofault handler (which always exists for these functions) 67807c478bd9Sstevel@tonic-gate * which will return the appropriate error. 67817c478bd9Sstevel@tonic-gate */ 67827c478bd9Sstevel@tonic-gate ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR; 67837c478bd9Sstevel@tonic-gate aston(curthread); 67847c478bd9Sstevel@tonic-gate 67857c478bd9Sstevel@tonic-gate use_hw_bcopy = use_hw_bcopy_orig; 67867c478bd9Sstevel@tonic-gate use_hw_bzero = use_hw_bzero_orig; 67877c478bd9Sstevel@tonic-gate hw_copy_limit_1 = hcl1_orig; 67887c478bd9Sstevel@tonic-gate hw_copy_limit_2 = hcl2_orig; 67897c478bd9Sstevel@tonic-gate hw_copy_limit_4 = hcl4_orig; 67907c478bd9Sstevel@tonic-gate hw_copy_limit_8 = hcl8_orig; 67917c478bd9Sstevel@tonic-gate 67927c478bd9Sstevel@tonic-gate return (1); 67937c478bd9Sstevel@tonic-gate } 67947c478bd9Sstevel@tonic-gate 67957c478bd9Sstevel@tonic-gate #define VIS_BLOCKSIZE 64 67967c478bd9Sstevel@tonic-gate 67977c478bd9Sstevel@tonic-gate int 67987c478bd9Sstevel@tonic-gate dtrace_blksuword32_err(uintptr_t addr, uint32_t *data) 67997c478bd9Sstevel@tonic-gate { 68007c478bd9Sstevel@tonic-gate int ret, watched; 68017c478bd9Sstevel@tonic-gate 68027c478bd9Sstevel@tonic-gate watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 68037c478bd9Sstevel@tonic-gate ret = dtrace_blksuword32(addr, data, 0); 68047c478bd9Sstevel@tonic-gate if (watched) 68057c478bd9Sstevel@tonic-gate watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 68067c478bd9Sstevel@tonic-gate 68077c478bd9Sstevel@tonic-gate return (ret); 68087c478bd9Sstevel@tonic-gate } 68097c478bd9Sstevel@tonic-gate 68107c478bd9Sstevel@tonic-gate /* 68117c478bd9Sstevel@tonic-gate * Called when a cpu enters the CPU_FAULTED state (by the cpu placing the 68127c478bd9Sstevel@tonic-gate * faulted cpu into that state). Cross-trap to the faulted cpu to clear 68137c478bd9Sstevel@tonic-gate * CEEN from the EER to disable traps for further disrupting error types 68147c478bd9Sstevel@tonic-gate * on that cpu. We could cross-call instead, but that has a larger 68157c478bd9Sstevel@tonic-gate * instruction and data footprint than cross-trapping, and the cpu is known 68167c478bd9Sstevel@tonic-gate * to be faulted. 68177c478bd9Sstevel@tonic-gate */ 68187c478bd9Sstevel@tonic-gate 68197c478bd9Sstevel@tonic-gate void 68207c478bd9Sstevel@tonic-gate cpu_faulted_enter(struct cpu *cp) 68217c478bd9Sstevel@tonic-gate { 68227c478bd9Sstevel@tonic-gate xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_CLRBITS); 68237c478bd9Sstevel@tonic-gate } 68247c478bd9Sstevel@tonic-gate 68257c478bd9Sstevel@tonic-gate /* 68267c478bd9Sstevel@tonic-gate * Called when a cpu leaves the CPU_FAULTED state to return to one of 68277c478bd9Sstevel@tonic-gate * offline, spare, or online (by the cpu requesting this state change). 68287c478bd9Sstevel@tonic-gate * First we cross-call to clear the AFSR (and AFSR_EXT on Panther) of 68297c478bd9Sstevel@tonic-gate * disrupting error bits that have accumulated without trapping, then 68307c478bd9Sstevel@tonic-gate * we cross-trap to re-enable CEEN controlled traps. 68317c478bd9Sstevel@tonic-gate */ 68327c478bd9Sstevel@tonic-gate void 68337c478bd9Sstevel@tonic-gate cpu_faulted_exit(struct cpu *cp) 68347c478bd9Sstevel@tonic-gate { 68357c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 68367c478bd9Sstevel@tonic-gate 68377c478bd9Sstevel@tonic-gate cpu_error_regs.afsr = C_AFSR_CECC_ERRS; 68387c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) 68397c478bd9Sstevel@tonic-gate cpu_error_regs.afsr_ext &= C_AFSR_EXT_CECC_ERRS; 68407c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)set_cpu_error_state, 68417c478bd9Sstevel@tonic-gate (uint64_t)&cpu_error_regs, 0); 68427c478bd9Sstevel@tonic-gate 68437c478bd9Sstevel@tonic-gate xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_SETBITS); 68447c478bd9Sstevel@tonic-gate } 68457c478bd9Sstevel@tonic-gate 68467c478bd9Sstevel@tonic-gate /* 68477c478bd9Sstevel@tonic-gate * Return 1 if the errors in ch_flt's AFSR are secondary errors caused by 68487c478bd9Sstevel@tonic-gate * the errors in the original AFSR, 0 otherwise. 68497c478bd9Sstevel@tonic-gate * 68507c478bd9Sstevel@tonic-gate * For all procs if the initial error was a BERR or TO, then it is possible 68517c478bd9Sstevel@tonic-gate * that we may have caused a secondary BERR or TO in the process of logging the 68527c478bd9Sstevel@tonic-gate * inital error via cpu_run_bus_error_handlers(). If this is the case then 68537c478bd9Sstevel@tonic-gate * if the request was protected then a panic is still not necessary, if not 68547c478bd9Sstevel@tonic-gate * protected then aft_panic is already set - so either way there's no need 68557c478bd9Sstevel@tonic-gate * to set aft_panic for the secondary error. 68567c478bd9Sstevel@tonic-gate * 68577c478bd9Sstevel@tonic-gate * For Cheetah and Jalapeno, if the original error was a UE which occurred on 68587c478bd9Sstevel@tonic-gate * a store merge, then the error handling code will call cpu_deferred_error(). 68597c478bd9Sstevel@tonic-gate * When clear_errors() is called, it will determine that secondary errors have 68607c478bd9Sstevel@tonic-gate * occurred - in particular, the store merge also caused a EDU and WDU that 68617c478bd9Sstevel@tonic-gate * weren't discovered until this point. 68627c478bd9Sstevel@tonic-gate * 68637c478bd9Sstevel@tonic-gate * We do three checks to verify that we are in this case. If we pass all three 68647c478bd9Sstevel@tonic-gate * checks, we return 1 to indicate that we should not panic. If any unexpected 68657c478bd9Sstevel@tonic-gate * errors occur, we return 0. 68667c478bd9Sstevel@tonic-gate * 68677c478bd9Sstevel@tonic-gate * For Cheetah+ and derivative procs, the store merge causes a DUE, which is 68687c478bd9Sstevel@tonic-gate * handled in cpu_disrupting_errors(). Since this function is not even called 68697c478bd9Sstevel@tonic-gate * in the case we are interested in, we just return 0 for these processors. 68707c478bd9Sstevel@tonic-gate */ 68717c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 68727c478bd9Sstevel@tonic-gate static int 68737c478bd9Sstevel@tonic-gate cpu_check_secondary_errors(ch_async_flt_t *ch_flt, uint64_t t_afsr_errs, 68747c478bd9Sstevel@tonic-gate uint64_t t_afar) 68757c478bd9Sstevel@tonic-gate { 68767c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 68777c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */ 68787c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 68797c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 68807c478bd9Sstevel@tonic-gate 68817c478bd9Sstevel@tonic-gate /* 68827c478bd9Sstevel@tonic-gate * Was the original error a BERR or TO and only a BERR or TO 68837c478bd9Sstevel@tonic-gate * (multiple errors are also OK) 68847c478bd9Sstevel@tonic-gate */ 68857c478bd9Sstevel@tonic-gate if ((t_afsr_errs & ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) { 68867c478bd9Sstevel@tonic-gate /* 68877c478bd9Sstevel@tonic-gate * Is the new error a BERR or TO and only a BERR or TO 68887c478bd9Sstevel@tonic-gate * (multiple errors are also OK) 68897c478bd9Sstevel@tonic-gate */ 68907c478bd9Sstevel@tonic-gate if ((ch_flt->afsr_errs & 68917c478bd9Sstevel@tonic-gate ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) 68927c478bd9Sstevel@tonic-gate return (1); 68937c478bd9Sstevel@tonic-gate } 68947c478bd9Sstevel@tonic-gate 68957c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 68967c478bd9Sstevel@tonic-gate return (0); 68977c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */ 68987c478bd9Sstevel@tonic-gate /* 68997c478bd9Sstevel@tonic-gate * Now look for secondary effects of a UE on cheetah/jalapeno 69007c478bd9Sstevel@tonic-gate * 69017c478bd9Sstevel@tonic-gate * Check the original error was a UE, and only a UE. Note that 69027c478bd9Sstevel@tonic-gate * the ME bit will cause us to fail this check. 69037c478bd9Sstevel@tonic-gate */ 69047c478bd9Sstevel@tonic-gate if (t_afsr_errs != C_AFSR_UE) 69057c478bd9Sstevel@tonic-gate return (0); 69067c478bd9Sstevel@tonic-gate 69077c478bd9Sstevel@tonic-gate /* 69087c478bd9Sstevel@tonic-gate * Check the secondary errors were exclusively an EDU and/or WDU. 69097c478bd9Sstevel@tonic-gate */ 69107c478bd9Sstevel@tonic-gate if ((ch_flt->afsr_errs & ~(C_AFSR_EDU|C_AFSR_WDU)) != 0) 69117c478bd9Sstevel@tonic-gate return (0); 69127c478bd9Sstevel@tonic-gate 69137c478bd9Sstevel@tonic-gate /* 69147c478bd9Sstevel@tonic-gate * Check the AFAR of the original error and secondary errors 69157c478bd9Sstevel@tonic-gate * match to the 64-byte boundary 69167c478bd9Sstevel@tonic-gate */ 69177c478bd9Sstevel@tonic-gate if (P2ALIGN(aflt->flt_addr, 64) != P2ALIGN(t_afar, 64)) 69187c478bd9Sstevel@tonic-gate return (0); 69197c478bd9Sstevel@tonic-gate 69207c478bd9Sstevel@tonic-gate /* 69217c478bd9Sstevel@tonic-gate * We've passed all the checks, so it's a secondary error! 69227c478bd9Sstevel@tonic-gate */ 69237c478bd9Sstevel@tonic-gate return (1); 69247c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 69257c478bd9Sstevel@tonic-gate } 69267c478bd9Sstevel@tonic-gate 69277c478bd9Sstevel@tonic-gate /* 69287c478bd9Sstevel@tonic-gate * Translate the flt_bit or flt_type into an error type. First, flt_bit 69297c478bd9Sstevel@tonic-gate * is checked for any valid errors. If found, the error type is 69307c478bd9Sstevel@tonic-gate * returned. If not found, the flt_type is checked for L1$ parity errors. 69317c478bd9Sstevel@tonic-gate */ 69327c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 69337c478bd9Sstevel@tonic-gate static uint8_t 69347c478bd9Sstevel@tonic-gate cpu_flt_bit_to_plat_error(struct async_flt *aflt) 69357c478bd9Sstevel@tonic-gate { 69367c478bd9Sstevel@tonic-gate #if defined(JALAPENO) 69377c478bd9Sstevel@tonic-gate /* 69387c478bd9Sstevel@tonic-gate * Currently, logging errors to the SC is not supported on Jalapeno 69397c478bd9Sstevel@tonic-gate */ 69407c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_NONE); 69417c478bd9Sstevel@tonic-gate #else 69427c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 69437c478bd9Sstevel@tonic-gate 69447c478bd9Sstevel@tonic-gate switch (ch_flt->flt_bit) { 69457c478bd9Sstevel@tonic-gate case C_AFSR_CE: 69467c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_CE); 69477c478bd9Sstevel@tonic-gate case C_AFSR_UCC: 69487c478bd9Sstevel@tonic-gate case C_AFSR_EDC: 69497c478bd9Sstevel@tonic-gate case C_AFSR_WDC: 69507c478bd9Sstevel@tonic-gate case C_AFSR_CPC: 69517c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_CE); 69527c478bd9Sstevel@tonic-gate case C_AFSR_EMC: 69537c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_EMC); 69547c478bd9Sstevel@tonic-gate case C_AFSR_IVC: 69557c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IVC); 69567c478bd9Sstevel@tonic-gate case C_AFSR_UE: 69577c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_UE); 69587c478bd9Sstevel@tonic-gate case C_AFSR_UCU: 69597c478bd9Sstevel@tonic-gate case C_AFSR_EDU: 69607c478bd9Sstevel@tonic-gate case C_AFSR_WDU: 69617c478bd9Sstevel@tonic-gate case C_AFSR_CPU: 69627c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_UE); 69637c478bd9Sstevel@tonic-gate case C_AFSR_IVU: 69647c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IVU); 69657c478bd9Sstevel@tonic-gate case C_AFSR_TO: 69667c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_TO); 69677c478bd9Sstevel@tonic-gate case C_AFSR_BERR: 69687c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_BERR); 69697c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 69707c478bd9Sstevel@tonic-gate case C_AFSR_L3_EDC: 69717c478bd9Sstevel@tonic-gate case C_AFSR_L3_UCC: 69727c478bd9Sstevel@tonic-gate case C_AFSR_L3_CPC: 69737c478bd9Sstevel@tonic-gate case C_AFSR_L3_WDC: 69747c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_CE); 69757c478bd9Sstevel@tonic-gate case C_AFSR_IMC: 69767c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IMC); 69777c478bd9Sstevel@tonic-gate case C_AFSR_TSCE: 69787c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_TSCE); 69797c478bd9Sstevel@tonic-gate case C_AFSR_THCE: 69807c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_THCE); 69817c478bd9Sstevel@tonic-gate case C_AFSR_L3_MECC: 69827c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_MECC); 69837c478bd9Sstevel@tonic-gate case C_AFSR_L3_THCE: 69847c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_THCE); 69857c478bd9Sstevel@tonic-gate case C_AFSR_L3_CPU: 69867c478bd9Sstevel@tonic-gate case C_AFSR_L3_EDU: 69877c478bd9Sstevel@tonic-gate case C_AFSR_L3_UCU: 69887c478bd9Sstevel@tonic-gate case C_AFSR_L3_WDU: 69897c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_UE); 69907c478bd9Sstevel@tonic-gate case C_AFSR_DUE: 69917c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DUE); 69927c478bd9Sstevel@tonic-gate case C_AFSR_DTO: 69937c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DTO); 69947c478bd9Sstevel@tonic-gate case C_AFSR_DBERR: 69957c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DBERR); 69967c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 69977c478bd9Sstevel@tonic-gate default: 69987c478bd9Sstevel@tonic-gate switch (ch_flt->flt_type) { 69997c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 70007c478bd9Sstevel@tonic-gate case CPU_IC_PARITY: 70017c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IPE); 70027c478bd9Sstevel@tonic-gate case CPU_DC_PARITY: 70037c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 70047c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_cache == 70057c478bd9Sstevel@tonic-gate CPU_PC_PARITY) { 70067c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_PCACHE); 70077c478bd9Sstevel@tonic-gate } 70087c478bd9Sstevel@tonic-gate } 70097c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DPE); 70107c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 70117c478bd9Sstevel@tonic-gate case CPU_ITLB_PARITY: 70127c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_ITLB); 70137c478bd9Sstevel@tonic-gate case CPU_DTLB_PARITY: 70147c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DTLB); 70157c478bd9Sstevel@tonic-gate default: 70167c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_NONE); 70177c478bd9Sstevel@tonic-gate } 70187c478bd9Sstevel@tonic-gate } 70197c478bd9Sstevel@tonic-gate #endif /* JALAPENO */ 70207c478bd9Sstevel@tonic-gate } 7021