17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57c478bd9Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only 67c478bd9Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance 77c478bd9Sstevel@tonic-gate * with the License. 87c478bd9Sstevel@tonic-gate * 97c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 107c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 117c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 127c478bd9Sstevel@tonic-gate * and limitations under the License. 137c478bd9Sstevel@tonic-gate * 147c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 157c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 167c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 177c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 187c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 197c478bd9Sstevel@tonic-gate * 207c478bd9Sstevel@tonic-gate * CDDL HEADER END 217c478bd9Sstevel@tonic-gate */ 227c478bd9Sstevel@tonic-gate /* 237c478bd9Sstevel@tonic-gate * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/types.h> 307c478bd9Sstevel@tonic-gate #include <sys/systm.h> 317c478bd9Sstevel@tonic-gate #include <sys/ddi.h> 327c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 337c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 347c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 357c478bd9Sstevel@tonic-gate #include <sys/machparam.h> 367c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 377c478bd9Sstevel@tonic-gate #include <sys/machthread.h> 387c478bd9Sstevel@tonic-gate #include <sys/cpu.h> 397c478bd9Sstevel@tonic-gate #include <sys/cmp.h> 407c478bd9Sstevel@tonic-gate #include <sys/elf_SPARC.h> 417c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 427c478bd9Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 437c478bd9Sstevel@tonic-gate #include <vm/seg_kpm.h> 447c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 457c478bd9Sstevel@tonic-gate #include <sys/cheetahregs.h> 467c478bd9Sstevel@tonic-gate #include <sys/us3_module.h> 477c478bd9Sstevel@tonic-gate #include <sys/async.h> 487c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 497c478bd9Sstevel@tonic-gate #include <sys/debug.h> 507c478bd9Sstevel@tonic-gate #include <sys/dditypes.h> 517c478bd9Sstevel@tonic-gate #include <sys/prom_debug.h> 527c478bd9Sstevel@tonic-gate #include <sys/prom_plat.h> 537c478bd9Sstevel@tonic-gate #include <sys/cpu_module.h> 547c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 557c478bd9Sstevel@tonic-gate #include <sys/intreg.h> 567c478bd9Sstevel@tonic-gate #include <sys/clock.h> 577c478bd9Sstevel@tonic-gate #include <sys/platform_module.h> 587c478bd9Sstevel@tonic-gate #include <sys/machtrap.h> 597c478bd9Sstevel@tonic-gate #include <sys/ontrap.h> 607c478bd9Sstevel@tonic-gate #include <sys/panic.h> 617c478bd9Sstevel@tonic-gate #include <sys/memlist.h> 627c478bd9Sstevel@tonic-gate #include <sys/bootconf.h> 637c478bd9Sstevel@tonic-gate #include <sys/ivintr.h> 647c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 657c478bd9Sstevel@tonic-gate #include <sys/taskq.h> 667c478bd9Sstevel@tonic-gate #include <sys/note.h> 677c478bd9Sstevel@tonic-gate #include <sys/ndifm.h> 687c478bd9Sstevel@tonic-gate #include <sys/ddifm.h> 697c478bd9Sstevel@tonic-gate #include <sys/fm/protocol.h> 707c478bd9Sstevel@tonic-gate #include <sys/fm/util.h> 717c478bd9Sstevel@tonic-gate #include <sys/fm/cpu/UltraSPARC-III.h> 727c478bd9Sstevel@tonic-gate #include <sys/fpras_impl.h> 737c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 747c478bd9Sstevel@tonic-gate #include <sys/watchpoint.h> 757c478bd9Sstevel@tonic-gate #include <sys/plat_ecc_unum.h> 767c478bd9Sstevel@tonic-gate #include <sys/cyclic.h> 777c478bd9Sstevel@tonic-gate #include <sys/errorq.h> 787c478bd9Sstevel@tonic-gate #include <sys/errclassify.h> 797c478bd9Sstevel@tonic-gate 807c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 817c478bd9Sstevel@tonic-gate #include <sys/xc_impl.h> 827c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 837c478bd9Sstevel@tonic-gate 847c478bd9Sstevel@tonic-gate /* 857c478bd9Sstevel@tonic-gate * Note that 'Cheetah PRM' refers to: 867c478bd9Sstevel@tonic-gate * SPARC V9 JPS1 Implementation Supplement: Sun UltraSPARC-III 877c478bd9Sstevel@tonic-gate */ 887c478bd9Sstevel@tonic-gate 897c478bd9Sstevel@tonic-gate /* 907c478bd9Sstevel@tonic-gate * Per CPU pointers to physical address of TL>0 logout data areas. 917c478bd9Sstevel@tonic-gate * These pointers have to be in the kernel nucleus to avoid MMU 927c478bd9Sstevel@tonic-gate * misses. 937c478bd9Sstevel@tonic-gate */ 947c478bd9Sstevel@tonic-gate uint64_t ch_err_tl1_paddrs[NCPU]; 957c478bd9Sstevel@tonic-gate 967c478bd9Sstevel@tonic-gate /* 977c478bd9Sstevel@tonic-gate * One statically allocated structure to use during startup/DR 987c478bd9Sstevel@tonic-gate * to prevent unnecessary panics. 997c478bd9Sstevel@tonic-gate */ 1007c478bd9Sstevel@tonic-gate ch_err_tl1_data_t ch_err_tl1_data; 1017c478bd9Sstevel@tonic-gate 1027c478bd9Sstevel@tonic-gate /* 1037c478bd9Sstevel@tonic-gate * Per CPU pending error at TL>0, used by level15 softint handler 1047c478bd9Sstevel@tonic-gate */ 1057c478bd9Sstevel@tonic-gate uchar_t ch_err_tl1_pending[NCPU]; 1067c478bd9Sstevel@tonic-gate 1077c478bd9Sstevel@tonic-gate /* 1087c478bd9Sstevel@tonic-gate * For deferred CE re-enable after trap. 1097c478bd9Sstevel@tonic-gate */ 1107c478bd9Sstevel@tonic-gate taskq_t *ch_check_ce_tq; 1117c478bd9Sstevel@tonic-gate 1127c478bd9Sstevel@tonic-gate /* 1137c478bd9Sstevel@tonic-gate * Internal functions. 1147c478bd9Sstevel@tonic-gate */ 1157c478bd9Sstevel@tonic-gate static int cpu_async_log_err(void *flt, errorq_elem_t *eqep); 1167c478bd9Sstevel@tonic-gate static void cpu_log_diag_info(ch_async_flt_t *ch_flt); 1177c478bd9Sstevel@tonic-gate static void cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 1187c478bd9Sstevel@tonic-gate ecc_type_to_info_t *eccp, ch_diag_data_t *cdp); 1197c478bd9Sstevel@tonic-gate static int clear_ecc(struct async_flt *ecc); 1207c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 1217c478bd9Sstevel@tonic-gate static int cpu_ecache_line_valid(ch_async_flt_t *ch_flt); 1227c478bd9Sstevel@tonic-gate #endif 1237c478bd9Sstevel@tonic-gate static int cpu_ecache_set_size(struct cpu *cp); 1247c478bd9Sstevel@tonic-gate static int cpu_ectag_line_invalid(int cachesize, uint64_t tag); 1257c478bd9Sstevel@tonic-gate static int cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr); 1267c478bd9Sstevel@tonic-gate static uint64_t cpu_ectag_to_pa(int setsize, uint64_t tag); 1277c478bd9Sstevel@tonic-gate static int cpu_ectag_pa_to_subblk_state(int cachesize, 1287c478bd9Sstevel@tonic-gate uint64_t subaddr, uint64_t tag); 1297c478bd9Sstevel@tonic-gate static void cpu_flush_ecache_line(ch_async_flt_t *ch_flt); 1307c478bd9Sstevel@tonic-gate static int afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit); 1317c478bd9Sstevel@tonic-gate static int afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit); 1327c478bd9Sstevel@tonic-gate static int afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit); 1337c478bd9Sstevel@tonic-gate static int afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit); 1347c478bd9Sstevel@tonic-gate static int synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit); 1357c478bd9Sstevel@tonic-gate static void cpu_uninit_ecache_scrub_dr(struct cpu *cp); 1367c478bd9Sstevel@tonic-gate static void cpu_scrubphys(struct async_flt *aflt); 1377c478bd9Sstevel@tonic-gate static void cpu_payload_add_aflt(struct async_flt *, nvlist_t *, nvlist_t *, 1387c478bd9Sstevel@tonic-gate int *, int *); 1397c478bd9Sstevel@tonic-gate static void cpu_payload_add_ecache(struct async_flt *, nvlist_t *); 1407c478bd9Sstevel@tonic-gate static void cpu_ereport_init(struct async_flt *aflt); 1417c478bd9Sstevel@tonic-gate static int cpu_check_secondary_errors(ch_async_flt_t *, uint64_t, uint64_t); 1427c478bd9Sstevel@tonic-gate static uint8_t cpu_flt_bit_to_plat_error(struct async_flt *aflt); 1437c478bd9Sstevel@tonic-gate static void cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 144*4fd7ecabSdilpreet uint64_t nceen, ch_cpu_logout_t *clop); 1457c478bd9Sstevel@tonic-gate static int cpu_ce_delayed_ec_logout(uint64_t); 1467c478bd9Sstevel@tonic-gate static int cpu_matching_ecache_line(uint64_t, void *, int, int *); 1477c478bd9Sstevel@tonic-gate 1487c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 1497c478bd9Sstevel@tonic-gate static int mondo_recover_proc(uint16_t, int); 1507c478bd9Sstevel@tonic-gate static void cheetah_nudge_init(void); 1517c478bd9Sstevel@tonic-gate static void cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 1527c478bd9Sstevel@tonic-gate cyc_time_t *when); 1537c478bd9Sstevel@tonic-gate static void cheetah_nudge_buddy(void); 1547c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 1557c478bd9Sstevel@tonic-gate 1567c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 1577c478bd9Sstevel@tonic-gate static void cpu_dcache_parity_info(ch_async_flt_t *ch_flt); 1587c478bd9Sstevel@tonic-gate static void cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index); 1597c478bd9Sstevel@tonic-gate static void cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 1607c478bd9Sstevel@tonic-gate ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word); 1617c478bd9Sstevel@tonic-gate static void cpu_icache_parity_info(ch_async_flt_t *ch_flt); 1627c478bd9Sstevel@tonic-gate static void cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index); 1637c478bd9Sstevel@tonic-gate static void cpu_pcache_parity_info(ch_async_flt_t *ch_flt); 1647c478bd9Sstevel@tonic-gate static void cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index); 1657c478bd9Sstevel@tonic-gate static void cpu_payload_add_dcache(struct async_flt *, nvlist_t *); 1667c478bd9Sstevel@tonic-gate static void cpu_payload_add_icache(struct async_flt *, nvlist_t *); 1677c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 1687c478bd9Sstevel@tonic-gate 1697c478bd9Sstevel@tonic-gate int (*p2get_mem_info)(int synd_code, uint64_t paddr, 1707c478bd9Sstevel@tonic-gate uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 1717c478bd9Sstevel@tonic-gate int *segsp, int *banksp, int *mcidp); 1727c478bd9Sstevel@tonic-gate 1737c478bd9Sstevel@tonic-gate /* 1747c478bd9Sstevel@tonic-gate * This table is used to determine which bit(s) is(are) bad when an ECC 1757c478bd9Sstevel@tonic-gate * error occurs. The array is indexed by an 9-bit syndrome. The entries 1767c478bd9Sstevel@tonic-gate * of this array have the following semantics: 1777c478bd9Sstevel@tonic-gate * 1787c478bd9Sstevel@tonic-gate * 00-127 The number of the bad bit, when only one bit is bad. 1797c478bd9Sstevel@tonic-gate * 128 ECC bit C0 is bad. 1807c478bd9Sstevel@tonic-gate * 129 ECC bit C1 is bad. 1817c478bd9Sstevel@tonic-gate * 130 ECC bit C2 is bad. 1827c478bd9Sstevel@tonic-gate * 131 ECC bit C3 is bad. 1837c478bd9Sstevel@tonic-gate * 132 ECC bit C4 is bad. 1847c478bd9Sstevel@tonic-gate * 133 ECC bit C5 is bad. 1857c478bd9Sstevel@tonic-gate * 134 ECC bit C6 is bad. 1867c478bd9Sstevel@tonic-gate * 135 ECC bit C7 is bad. 1877c478bd9Sstevel@tonic-gate * 136 ECC bit C8 is bad. 1887c478bd9Sstevel@tonic-gate * 137-143 reserved for Mtag Data and ECC. 1897c478bd9Sstevel@tonic-gate * 144(M2) Two bits are bad within a nibble. 1907c478bd9Sstevel@tonic-gate * 145(M3) Three bits are bad within a nibble. 1917c478bd9Sstevel@tonic-gate * 146(M3) Four bits are bad within a nibble. 1927c478bd9Sstevel@tonic-gate * 147(M) Multiple bits (5 or more) are bad. 1937c478bd9Sstevel@tonic-gate * 148 NO bits are bad. 1947c478bd9Sstevel@tonic-gate * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-4,11-5. 1957c478bd9Sstevel@tonic-gate */ 1967c478bd9Sstevel@tonic-gate 1977c478bd9Sstevel@tonic-gate #define C0 128 1987c478bd9Sstevel@tonic-gate #define C1 129 1997c478bd9Sstevel@tonic-gate #define C2 130 2007c478bd9Sstevel@tonic-gate #define C3 131 2017c478bd9Sstevel@tonic-gate #define C4 132 2027c478bd9Sstevel@tonic-gate #define C5 133 2037c478bd9Sstevel@tonic-gate #define C6 134 2047c478bd9Sstevel@tonic-gate #define C7 135 2057c478bd9Sstevel@tonic-gate #define C8 136 2067c478bd9Sstevel@tonic-gate #define MT0 137 /* Mtag Data bit 0 */ 2077c478bd9Sstevel@tonic-gate #define MT1 138 2087c478bd9Sstevel@tonic-gate #define MT2 139 2097c478bd9Sstevel@tonic-gate #define MTC0 140 /* Mtag Check bit 0 */ 2107c478bd9Sstevel@tonic-gate #define MTC1 141 2117c478bd9Sstevel@tonic-gate #define MTC2 142 2127c478bd9Sstevel@tonic-gate #define MTC3 143 2137c478bd9Sstevel@tonic-gate #define M2 144 2147c478bd9Sstevel@tonic-gate #define M3 145 2157c478bd9Sstevel@tonic-gate #define M4 146 2167c478bd9Sstevel@tonic-gate #define M 147 2177c478bd9Sstevel@tonic-gate #define NA 148 2187c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2197c478bd9Sstevel@tonic-gate #define S003 149 /* Syndrome 0x003 => likely from CPU/EDU:ST/FRU/BP */ 2207c478bd9Sstevel@tonic-gate #define S003MEM 150 /* Syndrome 0x003 => likely from WDU/WBP */ 2217c478bd9Sstevel@tonic-gate #define SLAST S003MEM /* last special syndrome */ 2227c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 2237c478bd9Sstevel@tonic-gate #define S003 149 /* Syndrome 0x003 => likely from EDU:ST */ 2247c478bd9Sstevel@tonic-gate #define S071 150 /* Syndrome 0x071 => likely from WDU/CPU */ 2257c478bd9Sstevel@tonic-gate #define S11C 151 /* Syndrome 0x11c => likely from BERR/DBERR */ 2267c478bd9Sstevel@tonic-gate #define SLAST S11C /* last special syndrome */ 2277c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2287c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2297c478bd9Sstevel@tonic-gate #define BPAR0 152 /* syndrom 152 through 167 for bus parity */ 2307c478bd9Sstevel@tonic-gate #define BPAR15 167 2317c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2327c478bd9Sstevel@tonic-gate 2337c478bd9Sstevel@tonic-gate static uint8_t ecc_syndrome_tab[] = 2347c478bd9Sstevel@tonic-gate { 2357c478bd9Sstevel@tonic-gate NA, C0, C1, S003, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M, 2367c478bd9Sstevel@tonic-gate C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16, 2377c478bd9Sstevel@tonic-gate C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10, 2387c478bd9Sstevel@tonic-gate M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M, 2397c478bd9Sstevel@tonic-gate C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6, 2407c478bd9Sstevel@tonic-gate M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4, 2417c478bd9Sstevel@tonic-gate M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4, 2427c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2437c478bd9Sstevel@tonic-gate 116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 2447c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 2457c478bd9Sstevel@tonic-gate 116, S071, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 2467c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2477c478bd9Sstevel@tonic-gate C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5, 2487c478bd9Sstevel@tonic-gate M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M, 2497c478bd9Sstevel@tonic-gate M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2, 2507c478bd9Sstevel@tonic-gate 103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3, 2517c478bd9Sstevel@tonic-gate M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M, 2527c478bd9Sstevel@tonic-gate 102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3, 2537c478bd9Sstevel@tonic-gate 98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M, 2547c478bd9Sstevel@tonic-gate M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M, 2557c478bd9Sstevel@tonic-gate C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4, 2567c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2577c478bd9Sstevel@tonic-gate M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M, 2587c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 2597c478bd9Sstevel@tonic-gate M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, S11C, M, M3, M, 2607c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2617c478bd9Sstevel@tonic-gate M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2, 2627c478bd9Sstevel@tonic-gate 94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M, 2637c478bd9Sstevel@tonic-gate M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4, 2647c478bd9Sstevel@tonic-gate 89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3, 2657c478bd9Sstevel@tonic-gate 86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3, 2667c478bd9Sstevel@tonic-gate M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2, 2677c478bd9Sstevel@tonic-gate M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4, 2687c478bd9Sstevel@tonic-gate 77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M, 2697c478bd9Sstevel@tonic-gate 74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3, 2707c478bd9Sstevel@tonic-gate M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M, 2717c478bd9Sstevel@tonic-gate 80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3, 2727c478bd9Sstevel@tonic-gate M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M, 2737c478bd9Sstevel@tonic-gate M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M, 2747c478bd9Sstevel@tonic-gate 111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M 2757c478bd9Sstevel@tonic-gate }; 2767c478bd9Sstevel@tonic-gate 2777c478bd9Sstevel@tonic-gate #define ESYND_TBL_SIZE (sizeof (ecc_syndrome_tab) / sizeof (uint8_t)) 2787c478bd9Sstevel@tonic-gate 2797c478bd9Sstevel@tonic-gate #if !(defined(JALAPENO) || defined(SERRANO)) 2807c478bd9Sstevel@tonic-gate /* 2817c478bd9Sstevel@tonic-gate * This table is used to determine which bit(s) is(are) bad when a Mtag 2827c478bd9Sstevel@tonic-gate * error occurs. The array is indexed by an 4-bit ECC syndrome. The entries 2837c478bd9Sstevel@tonic-gate * of this array have the following semantics: 2847c478bd9Sstevel@tonic-gate * 2857c478bd9Sstevel@tonic-gate * -1 Invalid mtag syndrome. 2867c478bd9Sstevel@tonic-gate * 137 Mtag Data 0 is bad. 2877c478bd9Sstevel@tonic-gate * 138 Mtag Data 1 is bad. 2887c478bd9Sstevel@tonic-gate * 139 Mtag Data 2 is bad. 2897c478bd9Sstevel@tonic-gate * 140 Mtag ECC 0 is bad. 2907c478bd9Sstevel@tonic-gate * 141 Mtag ECC 1 is bad. 2917c478bd9Sstevel@tonic-gate * 142 Mtag ECC 2 is bad. 2927c478bd9Sstevel@tonic-gate * 143 Mtag ECC 3 is bad. 2937c478bd9Sstevel@tonic-gate * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-6. 2947c478bd9Sstevel@tonic-gate */ 2957c478bd9Sstevel@tonic-gate short mtag_syndrome_tab[] = 2967c478bd9Sstevel@tonic-gate { 2977c478bd9Sstevel@tonic-gate NA, MTC0, MTC1, M2, MTC2, M2, M2, MT0, MTC3, M2, M2, MT1, M2, MT2, M2, M2 2987c478bd9Sstevel@tonic-gate }; 2997c478bd9Sstevel@tonic-gate 3007c478bd9Sstevel@tonic-gate #define MSYND_TBL_SIZE (sizeof (mtag_syndrome_tab) / sizeof (short)) 3017c478bd9Sstevel@tonic-gate 3027c478bd9Sstevel@tonic-gate #else /* !(JALAPENO || SERRANO) */ 3037c478bd9Sstevel@tonic-gate 3047c478bd9Sstevel@tonic-gate #define BSYND_TBL_SIZE 16 3057c478bd9Sstevel@tonic-gate 3067c478bd9Sstevel@tonic-gate #endif /* !(JALAPENO || SERRANO) */ 3077c478bd9Sstevel@tonic-gate 3087c478bd9Sstevel@tonic-gate /* 3097c478bd9Sstevel@tonic-gate * CE initial classification and subsequent action lookup table 3107c478bd9Sstevel@tonic-gate */ 3117c478bd9Sstevel@tonic-gate static ce_dispact_t ce_disp_table[CE_INITDISPTBL_SIZE]; 3127c478bd9Sstevel@tonic-gate static int ce_disp_inited; 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate /* 3157c478bd9Sstevel@tonic-gate * Set to disable leaky and partner check for memory correctables 3167c478bd9Sstevel@tonic-gate */ 3177c478bd9Sstevel@tonic-gate int ce_xdiag_off; 3187c478bd9Sstevel@tonic-gate 3197c478bd9Sstevel@tonic-gate /* 3207c478bd9Sstevel@tonic-gate * The following are not incremented atomically so are indicative only 3217c478bd9Sstevel@tonic-gate */ 3227c478bd9Sstevel@tonic-gate static int ce_xdiag_drops; 3237c478bd9Sstevel@tonic-gate static int ce_xdiag_lkydrops; 3247c478bd9Sstevel@tonic-gate static int ce_xdiag_ptnrdrops; 3257c478bd9Sstevel@tonic-gate static int ce_xdiag_bad; 3267c478bd9Sstevel@tonic-gate 3277c478bd9Sstevel@tonic-gate /* 3287c478bd9Sstevel@tonic-gate * CE leaky check callback structure 3297c478bd9Sstevel@tonic-gate */ 3307c478bd9Sstevel@tonic-gate typedef struct { 3317c478bd9Sstevel@tonic-gate struct async_flt *lkycb_aflt; 3327c478bd9Sstevel@tonic-gate errorq_t *lkycb_eqp; 3337c478bd9Sstevel@tonic-gate errorq_elem_t *lkycb_eqep; 3347c478bd9Sstevel@tonic-gate } ce_lkychk_cb_t; 3357c478bd9Sstevel@tonic-gate 3367c478bd9Sstevel@tonic-gate /* 3377c478bd9Sstevel@tonic-gate * defines for various ecache_flush_flag's 3387c478bd9Sstevel@tonic-gate */ 3397c478bd9Sstevel@tonic-gate #define ECACHE_FLUSH_LINE 1 3407c478bd9Sstevel@tonic-gate #define ECACHE_FLUSH_ALL 2 3417c478bd9Sstevel@tonic-gate 3427c478bd9Sstevel@tonic-gate /* 3437c478bd9Sstevel@tonic-gate * STICK sync 3447c478bd9Sstevel@tonic-gate */ 3457c478bd9Sstevel@tonic-gate #define STICK_ITERATION 10 3467c478bd9Sstevel@tonic-gate #define MAX_TSKEW 1 3477c478bd9Sstevel@tonic-gate #define EV_A_START 0 3487c478bd9Sstevel@tonic-gate #define EV_A_END 1 3497c478bd9Sstevel@tonic-gate #define EV_B_START 2 3507c478bd9Sstevel@tonic-gate #define EV_B_END 3 3517c478bd9Sstevel@tonic-gate #define EVENTS 4 3527c478bd9Sstevel@tonic-gate 3537c478bd9Sstevel@tonic-gate static int64_t stick_iter = STICK_ITERATION; 3547c478bd9Sstevel@tonic-gate static int64_t stick_tsk = MAX_TSKEW; 3557c478bd9Sstevel@tonic-gate 3567c478bd9Sstevel@tonic-gate typedef enum { 3577c478bd9Sstevel@tonic-gate EVENT_NULL = 0, 3587c478bd9Sstevel@tonic-gate SLAVE_START, 3597c478bd9Sstevel@tonic-gate SLAVE_CONT, 3607c478bd9Sstevel@tonic-gate MASTER_START 3617c478bd9Sstevel@tonic-gate } event_cmd_t; 3627c478bd9Sstevel@tonic-gate 3637c478bd9Sstevel@tonic-gate static volatile event_cmd_t stick_sync_cmd = EVENT_NULL; 3647c478bd9Sstevel@tonic-gate static int64_t timestamp[EVENTS]; 3657c478bd9Sstevel@tonic-gate static volatile int slave_done; 3667c478bd9Sstevel@tonic-gate 3677c478bd9Sstevel@tonic-gate #ifdef DEBUG 3687c478bd9Sstevel@tonic-gate #define DSYNC_ATTEMPTS 64 3697c478bd9Sstevel@tonic-gate typedef struct { 3707c478bd9Sstevel@tonic-gate int64_t skew_val[DSYNC_ATTEMPTS]; 3717c478bd9Sstevel@tonic-gate } ss_t; 3727c478bd9Sstevel@tonic-gate 3737c478bd9Sstevel@tonic-gate ss_t stick_sync_stats[NCPU]; 3747c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 3757c478bd9Sstevel@tonic-gate 3767c478bd9Sstevel@tonic-gate /* 3777c478bd9Sstevel@tonic-gate * Maximum number of contexts for Cheetah. 3787c478bd9Sstevel@tonic-gate */ 3797c478bd9Sstevel@tonic-gate #define MAX_NCTXS (1 << 13) 3807c478bd9Sstevel@tonic-gate 3817c478bd9Sstevel@tonic-gate /* Will be set !NULL for Cheetah+ and derivatives. */ 3827c478bd9Sstevel@tonic-gate uchar_t *ctx_pgsz_array = NULL; 3837c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_DUAL_PAGESIZE) 3847c478bd9Sstevel@tonic-gate static uchar_t ctx_pgsz_arr[MAX_NCTXS]; 3857c478bd9Sstevel@tonic-gate uint_t disable_dual_pgsz = 0; 3867c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_DUAL_PAGESIZE */ 3877c478bd9Sstevel@tonic-gate 3887c478bd9Sstevel@tonic-gate /* 3897c478bd9Sstevel@tonic-gate * Save the cache bootup state for use when internal 3907c478bd9Sstevel@tonic-gate * caches are to be re-enabled after an error occurs. 3917c478bd9Sstevel@tonic-gate */ 3927c478bd9Sstevel@tonic-gate uint64_t cache_boot_state; 3937c478bd9Sstevel@tonic-gate 3947c478bd9Sstevel@tonic-gate /* 3957c478bd9Sstevel@tonic-gate * PA[22:0] represent Displacement in Safari configuration space. 3967c478bd9Sstevel@tonic-gate */ 3977c478bd9Sstevel@tonic-gate uint_t root_phys_addr_lo_mask = 0x7fffffu; 3987c478bd9Sstevel@tonic-gate 3997c478bd9Sstevel@tonic-gate bus_config_eclk_t bus_config_eclk[] = { 4007c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 4017c478bd9Sstevel@tonic-gate {JBUS_CONFIG_ECLK_1_DIV, JBUS_CONFIG_ECLK_1}, 4027c478bd9Sstevel@tonic-gate {JBUS_CONFIG_ECLK_2_DIV, JBUS_CONFIG_ECLK_2}, 4037c478bd9Sstevel@tonic-gate {JBUS_CONFIG_ECLK_32_DIV, JBUS_CONFIG_ECLK_32}, 4047c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 4057c478bd9Sstevel@tonic-gate {SAFARI_CONFIG_ECLK_1_DIV, SAFARI_CONFIG_ECLK_1}, 4067c478bd9Sstevel@tonic-gate {SAFARI_CONFIG_ECLK_2_DIV, SAFARI_CONFIG_ECLK_2}, 4077c478bd9Sstevel@tonic-gate {SAFARI_CONFIG_ECLK_32_DIV, SAFARI_CONFIG_ECLK_32}, 4087c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 4097c478bd9Sstevel@tonic-gate {0, 0} 4107c478bd9Sstevel@tonic-gate }; 4117c478bd9Sstevel@tonic-gate 4127c478bd9Sstevel@tonic-gate /* 4137c478bd9Sstevel@tonic-gate * Interval for deferred CEEN reenable 4147c478bd9Sstevel@tonic-gate */ 4157c478bd9Sstevel@tonic-gate int cpu_ceen_delay_secs = CPU_CEEN_DELAY_SECS; 4167c478bd9Sstevel@tonic-gate 4177c478bd9Sstevel@tonic-gate /* 4187c478bd9Sstevel@tonic-gate * set in /etc/system to control logging of user BERR/TO's 4197c478bd9Sstevel@tonic-gate */ 4207c478bd9Sstevel@tonic-gate int cpu_berr_to_verbose = 0; 4217c478bd9Sstevel@tonic-gate 4227c478bd9Sstevel@tonic-gate /* 4237c478bd9Sstevel@tonic-gate * set to 0 in /etc/system to defer CEEN reenable for all CEs 4247c478bd9Sstevel@tonic-gate */ 4257c478bd9Sstevel@tonic-gate uint64_t cpu_ce_not_deferred = CPU_CE_NOT_DEFERRED; 4267c478bd9Sstevel@tonic-gate uint64_t cpu_ce_not_deferred_ext = CPU_CE_NOT_DEFERRED_EXT; 4277c478bd9Sstevel@tonic-gate 4287c478bd9Sstevel@tonic-gate /* 4297c478bd9Sstevel@tonic-gate * Set of all offline cpus 4307c478bd9Sstevel@tonic-gate */ 4317c478bd9Sstevel@tonic-gate cpuset_t cpu_offline_set; 4327c478bd9Sstevel@tonic-gate 4337c478bd9Sstevel@tonic-gate static void cpu_delayed_check_ce_errors(void *); 4347c478bd9Sstevel@tonic-gate static void cpu_check_ce_errors(void *); 4357c478bd9Sstevel@tonic-gate void cpu_error_ecache_flush(ch_async_flt_t *); 4367c478bd9Sstevel@tonic-gate static int cpu_error_ecache_flush_required(ch_async_flt_t *); 4377c478bd9Sstevel@tonic-gate static void cpu_log_and_clear_ce(ch_async_flt_t *); 4387c478bd9Sstevel@tonic-gate void cpu_ce_detected(ch_cpu_errors_t *, int); 4397c478bd9Sstevel@tonic-gate 4407c478bd9Sstevel@tonic-gate /* 4417c478bd9Sstevel@tonic-gate * CE Leaky check timeout in microseconds. This is chosen to be twice the 4427c478bd9Sstevel@tonic-gate * memory refresh interval of current DIMMs (64ms). After initial fix that 4437c478bd9Sstevel@tonic-gate * gives at least one full refresh cycle in which the cell can leak 4447c478bd9Sstevel@tonic-gate * (whereafter further refreshes simply reinforce any incorrect bit value). 4457c478bd9Sstevel@tonic-gate */ 4467c478bd9Sstevel@tonic-gate clock_t cpu_ce_lkychk_timeout_usec = 128000; 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate /* 4497c478bd9Sstevel@tonic-gate * CE partner check partner caching period in seconds 4507c478bd9Sstevel@tonic-gate */ 4517c478bd9Sstevel@tonic-gate int cpu_ce_ptnr_cachetime_sec = 60; 4527c478bd9Sstevel@tonic-gate 4537c478bd9Sstevel@tonic-gate /* 4547c478bd9Sstevel@tonic-gate * Sets trap table entry ttentry by overwriting eight instructions from ttlabel 4557c478bd9Sstevel@tonic-gate */ 4567c478bd9Sstevel@tonic-gate #define CH_SET_TRAP(ttentry, ttlabel) \ 4577c478bd9Sstevel@tonic-gate bcopy((const void *)&ttlabel, &ttentry, 32); \ 4587c478bd9Sstevel@tonic-gate flush_instr_mem((caddr_t)&ttentry, 32); 4597c478bd9Sstevel@tonic-gate 4607c478bd9Sstevel@tonic-gate static int min_ecache_size; 4617c478bd9Sstevel@tonic-gate static uint_t priv_hcl_1; 4627c478bd9Sstevel@tonic-gate static uint_t priv_hcl_2; 4637c478bd9Sstevel@tonic-gate static uint_t priv_hcl_4; 4647c478bd9Sstevel@tonic-gate static uint_t priv_hcl_8; 4657c478bd9Sstevel@tonic-gate 4667c478bd9Sstevel@tonic-gate void 4677c478bd9Sstevel@tonic-gate cpu_setup(void) 4687c478bd9Sstevel@tonic-gate { 4697c478bd9Sstevel@tonic-gate extern int at_flags; 4707c478bd9Sstevel@tonic-gate extern int disable_delay_tlb_flush, delay_tlb_flush; 4717c478bd9Sstevel@tonic-gate extern int cpc_has_overflow_intr; 4727c478bd9Sstevel@tonic-gate extern int disable_text_largepages; 4737c478bd9Sstevel@tonic-gate extern int use_text_pgsz4m; 4747c478bd9Sstevel@tonic-gate 4757c478bd9Sstevel@tonic-gate /* 4767c478bd9Sstevel@tonic-gate * Setup chip-specific trap handlers. 4777c478bd9Sstevel@tonic-gate */ 4787c478bd9Sstevel@tonic-gate cpu_init_trap(); 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT); 4817c478bd9Sstevel@tonic-gate 4827c478bd9Sstevel@tonic-gate at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1 | EF_SPARC_SUN_US3; 4837c478bd9Sstevel@tonic-gate 4847c478bd9Sstevel@tonic-gate /* 4857c478bd9Sstevel@tonic-gate * save the cache bootup state. 4867c478bd9Sstevel@tonic-gate */ 4877c478bd9Sstevel@tonic-gate cache_boot_state = get_dcu() & DCU_CACHE; 4887c478bd9Sstevel@tonic-gate 4897c478bd9Sstevel@tonic-gate /* 4907c478bd9Sstevel@tonic-gate * Use the maximum number of contexts available for Cheetah 4917c478bd9Sstevel@tonic-gate * unless it has been tuned for debugging. 4927c478bd9Sstevel@tonic-gate * We are checking against 0 here since this value can be patched 4937c478bd9Sstevel@tonic-gate * while booting. It can not be patched via /etc/system since it 4947c478bd9Sstevel@tonic-gate * will be patched too late and thus cause the system to panic. 4957c478bd9Sstevel@tonic-gate */ 4967c478bd9Sstevel@tonic-gate if (nctxs == 0) 4977c478bd9Sstevel@tonic-gate nctxs = MAX_NCTXS; 4987c478bd9Sstevel@tonic-gate 4997c478bd9Sstevel@tonic-gate /* 5007c478bd9Sstevel@tonic-gate * Due to the number of entries in the fully-associative tlb 5017c478bd9Sstevel@tonic-gate * this may have to be tuned lower than in spitfire. 5027c478bd9Sstevel@tonic-gate */ 5037c478bd9Sstevel@tonic-gate pp_slots = MIN(8, MAXPP_SLOTS); 5047c478bd9Sstevel@tonic-gate 5057c478bd9Sstevel@tonic-gate /* 5067c478bd9Sstevel@tonic-gate * Block stores do not invalidate all pages of the d$, pagecopy 5077c478bd9Sstevel@tonic-gate * et. al. need virtual translations with virtual coloring taken 5087c478bd9Sstevel@tonic-gate * into consideration. prefetch/ldd will pollute the d$ on the 5097c478bd9Sstevel@tonic-gate * load side. 5107c478bd9Sstevel@tonic-gate */ 5117c478bd9Sstevel@tonic-gate pp_consistent_coloring = PPAGE_STORE_VCOLORING | PPAGE_LOADS_POLLUTE; 5127c478bd9Sstevel@tonic-gate 5137c478bd9Sstevel@tonic-gate if (use_page_coloring) { 5147c478bd9Sstevel@tonic-gate do_pg_coloring = 1; 5157c478bd9Sstevel@tonic-gate if (use_virtual_coloring) 5167c478bd9Sstevel@tonic-gate do_virtual_coloring = 1; 5177c478bd9Sstevel@tonic-gate } 5187c478bd9Sstevel@tonic-gate 5197c478bd9Sstevel@tonic-gate isa_list = 5207c478bd9Sstevel@tonic-gate "sparcv9+vis2 sparcv9+vis sparcv9 " 5217c478bd9Sstevel@tonic-gate "sparcv8plus+vis2 sparcv8plus+vis sparcv8plus " 5227c478bd9Sstevel@tonic-gate "sparcv8 sparcv8-fsmuld sparcv7 sparc"; 5237c478bd9Sstevel@tonic-gate 5247c478bd9Sstevel@tonic-gate /* 5257c478bd9Sstevel@tonic-gate * On Panther-based machines, this should 5267c478bd9Sstevel@tonic-gate * also include AV_SPARC_POPC too 5277c478bd9Sstevel@tonic-gate */ 5287c478bd9Sstevel@tonic-gate cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2; 5297c478bd9Sstevel@tonic-gate 5307c478bd9Sstevel@tonic-gate /* 5317c478bd9Sstevel@tonic-gate * On cheetah, there's no hole in the virtual address space 5327c478bd9Sstevel@tonic-gate */ 5337c478bd9Sstevel@tonic-gate hole_start = hole_end = 0; 5347c478bd9Sstevel@tonic-gate 5357c478bd9Sstevel@tonic-gate /* 5367c478bd9Sstevel@tonic-gate * The kpm mapping window. 5377c478bd9Sstevel@tonic-gate * kpm_size: 5387c478bd9Sstevel@tonic-gate * The size of a single kpm range. 5397c478bd9Sstevel@tonic-gate * The overall size will be: kpm_size * vac_colors. 5407c478bd9Sstevel@tonic-gate * kpm_vbase: 5417c478bd9Sstevel@tonic-gate * The virtual start address of the kpm range within the kernel 5427c478bd9Sstevel@tonic-gate * virtual address space. kpm_vbase has to be kpm_size aligned. 5437c478bd9Sstevel@tonic-gate */ 5447c478bd9Sstevel@tonic-gate kpm_size = (size_t)(8ull * 1024 * 1024 * 1024 * 1024); /* 8TB */ 5457c478bd9Sstevel@tonic-gate kpm_size_shift = 43; 5467c478bd9Sstevel@tonic-gate kpm_vbase = (caddr_t)0x8000000000000000ull; /* 8EB */ 5477c478bd9Sstevel@tonic-gate kpm_smallpages = 1; 5487c478bd9Sstevel@tonic-gate 5497c478bd9Sstevel@tonic-gate /* 5507c478bd9Sstevel@tonic-gate * The traptrace code uses either %tick or %stick for 5517c478bd9Sstevel@tonic-gate * timestamping. We have %stick so we can use it. 5527c478bd9Sstevel@tonic-gate */ 5537c478bd9Sstevel@tonic-gate traptrace_use_stick = 1; 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate /* 5567c478bd9Sstevel@tonic-gate * Cheetah has a performance counter overflow interrupt 5577c478bd9Sstevel@tonic-gate */ 5587c478bd9Sstevel@tonic-gate cpc_has_overflow_intr = 1; 5597c478bd9Sstevel@tonic-gate 5607c478bd9Sstevel@tonic-gate /* 5617c478bd9Sstevel@tonic-gate * Use cheetah flush-all support 5627c478bd9Sstevel@tonic-gate */ 5637c478bd9Sstevel@tonic-gate if (!disable_delay_tlb_flush) 5647c478bd9Sstevel@tonic-gate delay_tlb_flush = 1; 5657c478bd9Sstevel@tonic-gate 5667c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_DUAL_PAGESIZE) 5677c478bd9Sstevel@tonic-gate /* 5687c478bd9Sstevel@tonic-gate * Use Cheetah+ and later dual page size support. 5697c478bd9Sstevel@tonic-gate */ 5707c478bd9Sstevel@tonic-gate if (!disable_dual_pgsz) { 5717c478bd9Sstevel@tonic-gate ctx_pgsz_array = ctx_pgsz_arr; 5727c478bd9Sstevel@tonic-gate } 5737c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_DUAL_PAGESIZE */ 5747c478bd9Sstevel@tonic-gate 5757c478bd9Sstevel@tonic-gate /* 5767c478bd9Sstevel@tonic-gate * Declare that this architecture/cpu combination does fpRAS. 5777c478bd9Sstevel@tonic-gate */ 5787c478bd9Sstevel@tonic-gate fpras_implemented = 1; 5797c478bd9Sstevel@tonic-gate 5807c478bd9Sstevel@tonic-gate /* 5817c478bd9Sstevel@tonic-gate * Enable 4M pages to be used for mapping user text by default. Don't 5827c478bd9Sstevel@tonic-gate * use large pages for initialized data segments since we may not know 5837c478bd9Sstevel@tonic-gate * at exec() time what should be the preferred large page size for DTLB 5847c478bd9Sstevel@tonic-gate * programming. 5857c478bd9Sstevel@tonic-gate */ 5867c478bd9Sstevel@tonic-gate use_text_pgsz4m = 1; 5877c478bd9Sstevel@tonic-gate disable_text_largepages = (1 << TTE64K) | (1 << TTE512K) | 5887c478bd9Sstevel@tonic-gate (1 << TTE32M) | (1 << TTE256M); 5897c478bd9Sstevel@tonic-gate 5907c478bd9Sstevel@tonic-gate /* 5917c478bd9Sstevel@tonic-gate * Setup CE lookup table 5927c478bd9Sstevel@tonic-gate */ 5937c478bd9Sstevel@tonic-gate CE_INITDISPTBL_POPULATE(ce_disp_table); 5947c478bd9Sstevel@tonic-gate ce_disp_inited = 1; 5957c478bd9Sstevel@tonic-gate } 5967c478bd9Sstevel@tonic-gate 5977c478bd9Sstevel@tonic-gate /* 5987c478bd9Sstevel@tonic-gate * Called by setcpudelay 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate void 6017c478bd9Sstevel@tonic-gate cpu_init_tick_freq(void) 6027c478bd9Sstevel@tonic-gate { 6037c478bd9Sstevel@tonic-gate /* 6047c478bd9Sstevel@tonic-gate * For UltraSPARC III and beyond we want to use the 6057c478bd9Sstevel@tonic-gate * system clock rate as the basis for low level timing, 6067c478bd9Sstevel@tonic-gate * due to support of mixed speed CPUs and power managment. 6077c478bd9Sstevel@tonic-gate */ 6087c478bd9Sstevel@tonic-gate if (system_clock_freq == 0) 6097c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "setcpudelay: invalid system_clock_freq"); 6107c478bd9Sstevel@tonic-gate 6117c478bd9Sstevel@tonic-gate sys_tick_freq = system_clock_freq; 6127c478bd9Sstevel@tonic-gate } 6137c478bd9Sstevel@tonic-gate 6147c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 6157c478bd9Sstevel@tonic-gate /* 6167c478bd9Sstevel@tonic-gate * Tunables 6177c478bd9Sstevel@tonic-gate */ 6187c478bd9Sstevel@tonic-gate int cheetah_bpe_off = 0; 6197c478bd9Sstevel@tonic-gate int cheetah_sendmondo_recover = 1; 6207c478bd9Sstevel@tonic-gate int cheetah_sendmondo_fullscan = 0; 6217c478bd9Sstevel@tonic-gate int cheetah_sendmondo_recover_delay = 5; 6227c478bd9Sstevel@tonic-gate 6237c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_MIN_DELAY 1 6247c478bd9Sstevel@tonic-gate 6257c478bd9Sstevel@tonic-gate /* 6267c478bd9Sstevel@tonic-gate * Recovery Statistics 6277c478bd9Sstevel@tonic-gate */ 6287c478bd9Sstevel@tonic-gate typedef struct cheetah_livelock_entry { 6297c478bd9Sstevel@tonic-gate int cpuid; /* fallen cpu */ 6307c478bd9Sstevel@tonic-gate int buddy; /* cpu that ran recovery */ 6317c478bd9Sstevel@tonic-gate clock_t lbolt; /* when recovery started */ 6327c478bd9Sstevel@tonic-gate hrtime_t recovery_time; /* time spent in recovery */ 6337c478bd9Sstevel@tonic-gate } cheetah_livelock_entry_t; 6347c478bd9Sstevel@tonic-gate 6357c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_NENTRY 32 6367c478bd9Sstevel@tonic-gate 6377c478bd9Sstevel@tonic-gate cheetah_livelock_entry_t cheetah_livelock_hist[CHEETAH_LIVELOCK_NENTRY]; 6387c478bd9Sstevel@tonic-gate int cheetah_livelock_entry_nxt; 6397c478bd9Sstevel@tonic-gate 6407c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_ENTRY_NEXT(statp) { \ 6417c478bd9Sstevel@tonic-gate statp = cheetah_livelock_hist + cheetah_livelock_entry_nxt; \ 6427c478bd9Sstevel@tonic-gate if (++cheetah_livelock_entry_nxt >= CHEETAH_LIVELOCK_NENTRY) { \ 6437c478bd9Sstevel@tonic-gate cheetah_livelock_entry_nxt = 0; \ 6447c478bd9Sstevel@tonic-gate } \ 6457c478bd9Sstevel@tonic-gate } 6467c478bd9Sstevel@tonic-gate 6477c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_ENTRY_SET(statp, item, val) statp->item = val 6487c478bd9Sstevel@tonic-gate 6497c478bd9Sstevel@tonic-gate struct { 6507c478bd9Sstevel@tonic-gate hrtime_t hrt; /* maximum recovery time */ 6517c478bd9Sstevel@tonic-gate int recovery; /* recovered */ 6527c478bd9Sstevel@tonic-gate int full_claimed; /* maximum pages claimed in full recovery */ 6537c478bd9Sstevel@tonic-gate int proc_entry; /* attempted to claim TSB */ 6547c478bd9Sstevel@tonic-gate int proc_tsb_scan; /* tsb scanned */ 6557c478bd9Sstevel@tonic-gate int proc_tsb_partscan; /* tsb partially scanned */ 6567c478bd9Sstevel@tonic-gate int proc_tsb_fullscan; /* whole tsb scanned */ 6577c478bd9Sstevel@tonic-gate int proc_claimed; /* maximum pages claimed in tsb scan */ 6587c478bd9Sstevel@tonic-gate int proc_user; /* user thread */ 6597c478bd9Sstevel@tonic-gate int proc_kernel; /* kernel thread */ 6607c478bd9Sstevel@tonic-gate int proc_onflt; /* bad stack */ 6617c478bd9Sstevel@tonic-gate int proc_cpu; /* null cpu */ 6627c478bd9Sstevel@tonic-gate int proc_thread; /* null thread */ 6637c478bd9Sstevel@tonic-gate int proc_proc; /* null proc */ 6647c478bd9Sstevel@tonic-gate int proc_as; /* null as */ 6657c478bd9Sstevel@tonic-gate int proc_hat; /* null hat */ 6667c478bd9Sstevel@tonic-gate int proc_hat_inval; /* hat contents don't make sense */ 6677c478bd9Sstevel@tonic-gate int proc_hat_busy; /* hat is changing TSBs */ 6687c478bd9Sstevel@tonic-gate int proc_tsb_reloc; /* TSB skipped because being relocated */ 6697c478bd9Sstevel@tonic-gate int proc_cnum_bad; /* cnum out of range */ 6707c478bd9Sstevel@tonic-gate int proc_cnum; /* last cnum processed */ 6717c478bd9Sstevel@tonic-gate tte_t proc_tte; /* last tte processed */ 6727c478bd9Sstevel@tonic-gate } cheetah_livelock_stat; 6737c478bd9Sstevel@tonic-gate 6747c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_STAT(item) cheetah_livelock_stat.item++ 6757c478bd9Sstevel@tonic-gate 6767c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_STATSET(item, value) \ 6777c478bd9Sstevel@tonic-gate cheetah_livelock_stat.item = value 6787c478bd9Sstevel@tonic-gate 6797c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_MAXSTAT(item, value) { \ 6807c478bd9Sstevel@tonic-gate if (value > cheetah_livelock_stat.item) \ 6817c478bd9Sstevel@tonic-gate cheetah_livelock_stat.item = value; \ 6827c478bd9Sstevel@tonic-gate } 6837c478bd9Sstevel@tonic-gate 6847c478bd9Sstevel@tonic-gate /* 6857c478bd9Sstevel@tonic-gate * Attempt to recover a cpu by claiming every cache line as saved 6867c478bd9Sstevel@tonic-gate * in the TSB that the non-responsive cpu is using. Since we can't 6877c478bd9Sstevel@tonic-gate * grab any adaptive lock, this is at best an attempt to do so. Because 6887c478bd9Sstevel@tonic-gate * we don't grab any locks, we must operate under the protection of 6897c478bd9Sstevel@tonic-gate * on_fault(). 6907c478bd9Sstevel@tonic-gate * 6917c478bd9Sstevel@tonic-gate * Return 1 if cpuid could be recovered, 0 if failed. 6927c478bd9Sstevel@tonic-gate */ 6937c478bd9Sstevel@tonic-gate int 6947c478bd9Sstevel@tonic-gate mondo_recover_proc(uint16_t cpuid, int bn) 6957c478bd9Sstevel@tonic-gate { 6967c478bd9Sstevel@tonic-gate label_t ljb; 6977c478bd9Sstevel@tonic-gate cpu_t *cp; 6987c478bd9Sstevel@tonic-gate kthread_t *t; 6997c478bd9Sstevel@tonic-gate proc_t *p; 7007c478bd9Sstevel@tonic-gate struct as *as; 7017c478bd9Sstevel@tonic-gate struct hat *hat; 7027c478bd9Sstevel@tonic-gate short cnum; 7037c478bd9Sstevel@tonic-gate struct tsb_info *tsbinfop; 7047c478bd9Sstevel@tonic-gate struct tsbe *tsbep; 7057c478bd9Sstevel@tonic-gate caddr_t tsbp; 7067c478bd9Sstevel@tonic-gate caddr_t end_tsbp; 7077c478bd9Sstevel@tonic-gate uint64_t paddr; 7087c478bd9Sstevel@tonic-gate uint64_t idsr; 7097c478bd9Sstevel@tonic-gate u_longlong_t pahi, palo; 7107c478bd9Sstevel@tonic-gate int pages_claimed = 0; 7117c478bd9Sstevel@tonic-gate tte_t tsbe_tte; 7127c478bd9Sstevel@tonic-gate int tried_kernel_tsb = 0; 7137c478bd9Sstevel@tonic-gate 7147c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_entry); 7157c478bd9Sstevel@tonic-gate 7167c478bd9Sstevel@tonic-gate if (on_fault(&ljb)) { 7177c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_onflt); 7187c478bd9Sstevel@tonic-gate goto badstruct; 7197c478bd9Sstevel@tonic-gate } 7207c478bd9Sstevel@tonic-gate 7217c478bd9Sstevel@tonic-gate if ((cp = cpu[cpuid]) == NULL) { 7227c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_cpu); 7237c478bd9Sstevel@tonic-gate goto badstruct; 7247c478bd9Sstevel@tonic-gate } 7257c478bd9Sstevel@tonic-gate 7267c478bd9Sstevel@tonic-gate if ((t = cp->cpu_thread) == NULL) { 7277c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_thread); 7287c478bd9Sstevel@tonic-gate goto badstruct; 7297c478bd9Sstevel@tonic-gate } 7307c478bd9Sstevel@tonic-gate 7317c478bd9Sstevel@tonic-gate if ((p = ttoproc(t)) == NULL) { 7327c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_proc); 7337c478bd9Sstevel@tonic-gate goto badstruct; 7347c478bd9Sstevel@tonic-gate } 7357c478bd9Sstevel@tonic-gate 7367c478bd9Sstevel@tonic-gate if ((as = p->p_as) == NULL) { 7377c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_as); 7387c478bd9Sstevel@tonic-gate goto badstruct; 7397c478bd9Sstevel@tonic-gate } 7407c478bd9Sstevel@tonic-gate 7417c478bd9Sstevel@tonic-gate if ((hat = as->a_hat) == NULL) { 7427c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat); 7437c478bd9Sstevel@tonic-gate goto badstruct; 7447c478bd9Sstevel@tonic-gate } 7457c478bd9Sstevel@tonic-gate 7467c478bd9Sstevel@tonic-gate if (hat != ksfmmup) { 7477c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_user); 7487c478bd9Sstevel@tonic-gate if (hat->sfmmu_flags & (HAT_BUSY | HAT_SWAPPED | HAT_SWAPIN)) { 7497c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat_busy); 7507c478bd9Sstevel@tonic-gate goto badstruct; 7517c478bd9Sstevel@tonic-gate } 7527c478bd9Sstevel@tonic-gate tsbinfop = hat->sfmmu_tsb; 7537c478bd9Sstevel@tonic-gate if (tsbinfop == NULL) { 7547c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat_inval); 7557c478bd9Sstevel@tonic-gate goto badstruct; 7567c478bd9Sstevel@tonic-gate } 7577c478bd9Sstevel@tonic-gate tsbp = tsbinfop->tsb_va; 7587c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 7597c478bd9Sstevel@tonic-gate } else { 7607c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_kernel); 7617c478bd9Sstevel@tonic-gate tsbinfop = NULL; 7627c478bd9Sstevel@tonic-gate tsbp = ktsb_base; 7637c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 7647c478bd9Sstevel@tonic-gate } 7657c478bd9Sstevel@tonic-gate 7667c478bd9Sstevel@tonic-gate /* Verify as */ 7677c478bd9Sstevel@tonic-gate if (hat->sfmmu_as != as) { 7687c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat_inval); 7697c478bd9Sstevel@tonic-gate goto badstruct; 7707c478bd9Sstevel@tonic-gate } 7717c478bd9Sstevel@tonic-gate 7727c478bd9Sstevel@tonic-gate cnum = hat->sfmmu_cnum; 7737c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STATSET(proc_cnum, cnum); 7747c478bd9Sstevel@tonic-gate 7757c478bd9Sstevel@tonic-gate if ((cnum < 0) || (cnum == INVALID_CONTEXT) || (cnum >= nctxs)) { 7767c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_cnum_bad); 7777c478bd9Sstevel@tonic-gate goto badstruct; 7787c478bd9Sstevel@tonic-gate } 7797c478bd9Sstevel@tonic-gate 7807c478bd9Sstevel@tonic-gate do { 7817c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_scan); 7827c478bd9Sstevel@tonic-gate 7837c478bd9Sstevel@tonic-gate /* 7847c478bd9Sstevel@tonic-gate * Skip TSBs being relocated. This is important because 7857c478bd9Sstevel@tonic-gate * we want to avoid the following deadlock scenario: 7867c478bd9Sstevel@tonic-gate * 7877c478bd9Sstevel@tonic-gate * 1) when we came in we set ourselves to "in recover" state. 7887c478bd9Sstevel@tonic-gate * 2) when we try to touch TSB being relocated the mapping 7897c478bd9Sstevel@tonic-gate * will be in the suspended state so we'll spin waiting 7907c478bd9Sstevel@tonic-gate * for it to be unlocked. 7917c478bd9Sstevel@tonic-gate * 3) when the CPU that holds the TSB mapping locked tries to 7927c478bd9Sstevel@tonic-gate * unlock it it will send a xtrap which will fail to xcall 7937c478bd9Sstevel@tonic-gate * us or the CPU we're trying to recover, and will in turn 7947c478bd9Sstevel@tonic-gate * enter the mondo code. 7957c478bd9Sstevel@tonic-gate * 4) since we are still spinning on the locked mapping 7967c478bd9Sstevel@tonic-gate * no further progress will be made and the system will 7977c478bd9Sstevel@tonic-gate * inevitably hard hang. 7987c478bd9Sstevel@tonic-gate * 7997c478bd9Sstevel@tonic-gate * A TSB not being relocated can't begin being relocated 8007c478bd9Sstevel@tonic-gate * while we're accessing it because we check 8017c478bd9Sstevel@tonic-gate * sendmondo_in_recover before relocating TSBs. 8027c478bd9Sstevel@tonic-gate */ 8037c478bd9Sstevel@tonic-gate if (hat != ksfmmup && 8047c478bd9Sstevel@tonic-gate (tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 8057c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_reloc); 8067c478bd9Sstevel@tonic-gate goto next_tsbinfo; 8077c478bd9Sstevel@tonic-gate } 8087c478bd9Sstevel@tonic-gate 8097c478bd9Sstevel@tonic-gate for (tsbep = (struct tsbe *)tsbp; 8107c478bd9Sstevel@tonic-gate tsbep < (struct tsbe *)end_tsbp; tsbep++) { 8117c478bd9Sstevel@tonic-gate tsbe_tte = tsbep->tte_data; 8127c478bd9Sstevel@tonic-gate 8137c478bd9Sstevel@tonic-gate if (tsbe_tte.tte_val == 0) { 8147c478bd9Sstevel@tonic-gate /* 8157c478bd9Sstevel@tonic-gate * Invalid tte 8167c478bd9Sstevel@tonic-gate */ 8177c478bd9Sstevel@tonic-gate continue; 8187c478bd9Sstevel@tonic-gate } 8197c478bd9Sstevel@tonic-gate if (tsbe_tte.tte_se) { 8207c478bd9Sstevel@tonic-gate /* 8217c478bd9Sstevel@tonic-gate * Don't want device registers 8227c478bd9Sstevel@tonic-gate */ 8237c478bd9Sstevel@tonic-gate continue; 8247c478bd9Sstevel@tonic-gate } 8257c478bd9Sstevel@tonic-gate if (tsbe_tte.tte_cp == 0) { 8267c478bd9Sstevel@tonic-gate /* 8277c478bd9Sstevel@tonic-gate * Must be cached in E$ 8287c478bd9Sstevel@tonic-gate */ 8297c478bd9Sstevel@tonic-gate continue; 8307c478bd9Sstevel@tonic-gate } 8317c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STATSET(proc_tte, tsbe_tte); 8327c478bd9Sstevel@tonic-gate idsr = getidsr(); 8337c478bd9Sstevel@tonic-gate if ((idsr & (IDSR_NACK_BIT(bn) | 8347c478bd9Sstevel@tonic-gate IDSR_BUSY_BIT(bn))) == 0) { 8357c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_partscan); 8367c478bd9Sstevel@tonic-gate goto done; 8377c478bd9Sstevel@tonic-gate } 8387c478bd9Sstevel@tonic-gate pahi = tsbe_tte.tte_pahi; 8397c478bd9Sstevel@tonic-gate palo = tsbe_tte.tte_palo; 8407c478bd9Sstevel@tonic-gate paddr = (uint64_t)((pahi << 32) | 8417c478bd9Sstevel@tonic-gate (palo << MMU_PAGESHIFT)); 8427c478bd9Sstevel@tonic-gate claimlines(paddr, TTEBYTES(TTE_CSZ(&tsbe_tte)), 8437c478bd9Sstevel@tonic-gate CH_ECACHE_SUBBLK_SIZE); 8447c478bd9Sstevel@tonic-gate if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 8457c478bd9Sstevel@tonic-gate shipit(cpuid, bn); 8467c478bd9Sstevel@tonic-gate } 8477c478bd9Sstevel@tonic-gate pages_claimed++; 8487c478bd9Sstevel@tonic-gate } 8497c478bd9Sstevel@tonic-gate next_tsbinfo: 8507c478bd9Sstevel@tonic-gate if (tsbinfop != NULL) 8517c478bd9Sstevel@tonic-gate tsbinfop = tsbinfop->tsb_next; 8527c478bd9Sstevel@tonic-gate if (tsbinfop != NULL) { 8537c478bd9Sstevel@tonic-gate tsbp = tsbinfop->tsb_va; 8547c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 8557c478bd9Sstevel@tonic-gate } else if (tsbp == ktsb_base) { 8567c478bd9Sstevel@tonic-gate tried_kernel_tsb = 1; 8577c478bd9Sstevel@tonic-gate } else if (!tried_kernel_tsb) { 8587c478bd9Sstevel@tonic-gate tsbp = ktsb_base; 8597c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 8607c478bd9Sstevel@tonic-gate hat = ksfmmup; 8617c478bd9Sstevel@tonic-gate tsbinfop = NULL; 8627c478bd9Sstevel@tonic-gate } 8637c478bd9Sstevel@tonic-gate } while (tsbinfop != NULL || 8647c478bd9Sstevel@tonic-gate ((tsbp == ktsb_base) && !tried_kernel_tsb)); 8657c478bd9Sstevel@tonic-gate 8667c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_fullscan); 8677c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 8687c478bd9Sstevel@tonic-gate no_fault(); 8697c478bd9Sstevel@tonic-gate idsr = getidsr(); 8707c478bd9Sstevel@tonic-gate if ((idsr & (IDSR_NACK_BIT(bn) | 8717c478bd9Sstevel@tonic-gate IDSR_BUSY_BIT(bn))) == 0) { 8727c478bd9Sstevel@tonic-gate return (1); 8737c478bd9Sstevel@tonic-gate } else { 8747c478bd9Sstevel@tonic-gate return (0); 8757c478bd9Sstevel@tonic-gate } 8767c478bd9Sstevel@tonic-gate 8777c478bd9Sstevel@tonic-gate done: 8787c478bd9Sstevel@tonic-gate no_fault(); 8797c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 8807c478bd9Sstevel@tonic-gate return (1); 8817c478bd9Sstevel@tonic-gate 8827c478bd9Sstevel@tonic-gate badstruct: 8837c478bd9Sstevel@tonic-gate no_fault(); 8847c478bd9Sstevel@tonic-gate return (0); 8857c478bd9Sstevel@tonic-gate } 8867c478bd9Sstevel@tonic-gate 8877c478bd9Sstevel@tonic-gate /* 8887c478bd9Sstevel@tonic-gate * Attempt to claim ownership, temporarily, of every cache line that a 8897c478bd9Sstevel@tonic-gate * non-responsive cpu might be using. This might kick that cpu out of 8907c478bd9Sstevel@tonic-gate * this state. 8917c478bd9Sstevel@tonic-gate * 8927c478bd9Sstevel@tonic-gate * The return value indicates to the caller if we have exhausted all recovery 8937c478bd9Sstevel@tonic-gate * techniques. If 1 is returned, it is useless to call this function again 8947c478bd9Sstevel@tonic-gate * even for a different target CPU. 8957c478bd9Sstevel@tonic-gate */ 8967c478bd9Sstevel@tonic-gate int 8977c478bd9Sstevel@tonic-gate mondo_recover(uint16_t cpuid, int bn) 8987c478bd9Sstevel@tonic-gate { 8997c478bd9Sstevel@tonic-gate struct memseg *seg; 9007c478bd9Sstevel@tonic-gate uint64_t begin_pa, end_pa, cur_pa; 9017c478bd9Sstevel@tonic-gate hrtime_t begin_hrt, end_hrt; 9027c478bd9Sstevel@tonic-gate int retval = 0; 9037c478bd9Sstevel@tonic-gate int pages_claimed = 0; 9047c478bd9Sstevel@tonic-gate cheetah_livelock_entry_t *histp; 9057c478bd9Sstevel@tonic-gate uint64_t idsr; 9067c478bd9Sstevel@tonic-gate 9077c478bd9Sstevel@tonic-gate if (cas32(&sendmondo_in_recover, 0, 1) != 0) { 9087c478bd9Sstevel@tonic-gate /* 9097c478bd9Sstevel@tonic-gate * Wait while recovery takes place 9107c478bd9Sstevel@tonic-gate */ 9117c478bd9Sstevel@tonic-gate while (sendmondo_in_recover) { 9127c478bd9Sstevel@tonic-gate drv_usecwait(1); 9137c478bd9Sstevel@tonic-gate } 9147c478bd9Sstevel@tonic-gate /* 9157c478bd9Sstevel@tonic-gate * Assume we didn't claim the whole memory. If 9167c478bd9Sstevel@tonic-gate * the target of this caller is not recovered, 9177c478bd9Sstevel@tonic-gate * it will come back. 9187c478bd9Sstevel@tonic-gate */ 9197c478bd9Sstevel@tonic-gate return (retval); 9207c478bd9Sstevel@tonic-gate } 9217c478bd9Sstevel@tonic-gate 9227c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_NEXT(histp) 9237c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, lbolt, lbolt); 9247c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, cpuid, cpuid); 9257c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, buddy, CPU->cpu_id); 9267c478bd9Sstevel@tonic-gate 9277c478bd9Sstevel@tonic-gate begin_hrt = gethrtime_waitfree(); 9287c478bd9Sstevel@tonic-gate /* 9297c478bd9Sstevel@tonic-gate * First try to claim the lines in the TSB the target 9307c478bd9Sstevel@tonic-gate * may have been using. 9317c478bd9Sstevel@tonic-gate */ 9327c478bd9Sstevel@tonic-gate if (mondo_recover_proc(cpuid, bn) == 1) { 9337c478bd9Sstevel@tonic-gate /* 9347c478bd9Sstevel@tonic-gate * Didn't claim the whole memory 9357c478bd9Sstevel@tonic-gate */ 9367c478bd9Sstevel@tonic-gate goto done; 9377c478bd9Sstevel@tonic-gate } 9387c478bd9Sstevel@tonic-gate 9397c478bd9Sstevel@tonic-gate /* 9407c478bd9Sstevel@tonic-gate * We tried using the TSB. The target is still 9417c478bd9Sstevel@tonic-gate * not recovered. Check if complete memory scan is 9427c478bd9Sstevel@tonic-gate * enabled. 9437c478bd9Sstevel@tonic-gate */ 9447c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_fullscan == 0) { 9457c478bd9Sstevel@tonic-gate /* 9467c478bd9Sstevel@tonic-gate * Full memory scan is disabled. 9477c478bd9Sstevel@tonic-gate */ 9487c478bd9Sstevel@tonic-gate retval = 1; 9497c478bd9Sstevel@tonic-gate goto done; 9507c478bd9Sstevel@tonic-gate } 9517c478bd9Sstevel@tonic-gate 9527c478bd9Sstevel@tonic-gate /* 9537c478bd9Sstevel@tonic-gate * Try claiming the whole memory. 9547c478bd9Sstevel@tonic-gate */ 9557c478bd9Sstevel@tonic-gate for (seg = memsegs; seg; seg = seg->next) { 9567c478bd9Sstevel@tonic-gate begin_pa = (uint64_t)(seg->pages_base) << MMU_PAGESHIFT; 9577c478bd9Sstevel@tonic-gate end_pa = (uint64_t)(seg->pages_end) << MMU_PAGESHIFT; 9587c478bd9Sstevel@tonic-gate for (cur_pa = begin_pa; cur_pa < end_pa; 9597c478bd9Sstevel@tonic-gate cur_pa += MMU_PAGESIZE) { 9607c478bd9Sstevel@tonic-gate idsr = getidsr(); 9617c478bd9Sstevel@tonic-gate if ((idsr & (IDSR_NACK_BIT(bn) | 9627c478bd9Sstevel@tonic-gate IDSR_BUSY_BIT(bn))) == 0) { 9637c478bd9Sstevel@tonic-gate /* 9647c478bd9Sstevel@tonic-gate * Didn't claim all memory 9657c478bd9Sstevel@tonic-gate */ 9667c478bd9Sstevel@tonic-gate goto done; 9677c478bd9Sstevel@tonic-gate } 9687c478bd9Sstevel@tonic-gate claimlines(cur_pa, MMU_PAGESIZE, 9697c478bd9Sstevel@tonic-gate CH_ECACHE_SUBBLK_SIZE); 9707c478bd9Sstevel@tonic-gate if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 9717c478bd9Sstevel@tonic-gate shipit(cpuid, bn); 9727c478bd9Sstevel@tonic-gate } 9737c478bd9Sstevel@tonic-gate pages_claimed++; 9747c478bd9Sstevel@tonic-gate } 9757c478bd9Sstevel@tonic-gate } 9767c478bd9Sstevel@tonic-gate 9777c478bd9Sstevel@tonic-gate /* 9787c478bd9Sstevel@tonic-gate * We did all we could. 9797c478bd9Sstevel@tonic-gate */ 9807c478bd9Sstevel@tonic-gate retval = 1; 9817c478bd9Sstevel@tonic-gate 9827c478bd9Sstevel@tonic-gate done: 9837c478bd9Sstevel@tonic-gate /* 9847c478bd9Sstevel@tonic-gate * Update statistics 9857c478bd9Sstevel@tonic-gate */ 9867c478bd9Sstevel@tonic-gate end_hrt = gethrtime_waitfree(); 9877c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(recovery); 9887c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(hrt, (end_hrt - begin_hrt)); 9897c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(full_claimed, pages_claimed); 9907c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, recovery_time, \ 9917c478bd9Sstevel@tonic-gate (end_hrt - begin_hrt)); 9927c478bd9Sstevel@tonic-gate 9937c478bd9Sstevel@tonic-gate while (cas32(&sendmondo_in_recover, 1, 0) != 1); 9947c478bd9Sstevel@tonic-gate 9957c478bd9Sstevel@tonic-gate return (retval); 9967c478bd9Sstevel@tonic-gate } 9977c478bd9Sstevel@tonic-gate 9987c478bd9Sstevel@tonic-gate /* 9997c478bd9Sstevel@tonic-gate * This is called by the cyclic framework when this CPU becomes online 10007c478bd9Sstevel@tonic-gate */ 10017c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 10027c478bd9Sstevel@tonic-gate static void 10037c478bd9Sstevel@tonic-gate cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when) 10047c478bd9Sstevel@tonic-gate { 10057c478bd9Sstevel@tonic-gate 10067c478bd9Sstevel@tonic-gate hdlr->cyh_func = (cyc_func_t)cheetah_nudge_buddy; 10077c478bd9Sstevel@tonic-gate hdlr->cyh_level = CY_LOW_LEVEL; 10087c478bd9Sstevel@tonic-gate hdlr->cyh_arg = NULL; 10097c478bd9Sstevel@tonic-gate 10107c478bd9Sstevel@tonic-gate /* 10117c478bd9Sstevel@tonic-gate * Stagger the start time 10127c478bd9Sstevel@tonic-gate */ 10137c478bd9Sstevel@tonic-gate when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU); 10147c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_recover_delay < CHEETAH_LIVELOCK_MIN_DELAY) { 10157c478bd9Sstevel@tonic-gate cheetah_sendmondo_recover_delay = CHEETAH_LIVELOCK_MIN_DELAY; 10167c478bd9Sstevel@tonic-gate } 10177c478bd9Sstevel@tonic-gate when->cyt_interval = cheetah_sendmondo_recover_delay * NANOSEC; 10187c478bd9Sstevel@tonic-gate } 10197c478bd9Sstevel@tonic-gate 10207c478bd9Sstevel@tonic-gate /* 10217c478bd9Sstevel@tonic-gate * Create a low level cyclic to send a xtrap to the next cpu online. 10227c478bd9Sstevel@tonic-gate * However, there's no need to have this running on a uniprocessor system. 10237c478bd9Sstevel@tonic-gate */ 10247c478bd9Sstevel@tonic-gate static void 10257c478bd9Sstevel@tonic-gate cheetah_nudge_init(void) 10267c478bd9Sstevel@tonic-gate { 10277c478bd9Sstevel@tonic-gate cyc_omni_handler_t hdlr; 10287c478bd9Sstevel@tonic-gate 10297c478bd9Sstevel@tonic-gate if (max_ncpus == 1) { 10307c478bd9Sstevel@tonic-gate return; 10317c478bd9Sstevel@tonic-gate } 10327c478bd9Sstevel@tonic-gate 10337c478bd9Sstevel@tonic-gate hdlr.cyo_online = cheetah_nudge_onln; 10347c478bd9Sstevel@tonic-gate hdlr.cyo_offline = NULL; 10357c478bd9Sstevel@tonic-gate hdlr.cyo_arg = NULL; 10367c478bd9Sstevel@tonic-gate 10377c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 10387c478bd9Sstevel@tonic-gate (void) cyclic_add_omni(&hdlr); 10397c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 10407c478bd9Sstevel@tonic-gate } 10417c478bd9Sstevel@tonic-gate 10427c478bd9Sstevel@tonic-gate /* 10437c478bd9Sstevel@tonic-gate * Cyclic handler to wake up buddy 10447c478bd9Sstevel@tonic-gate */ 10457c478bd9Sstevel@tonic-gate void 10467c478bd9Sstevel@tonic-gate cheetah_nudge_buddy(void) 10477c478bd9Sstevel@tonic-gate { 10487c478bd9Sstevel@tonic-gate /* 10497c478bd9Sstevel@tonic-gate * Disable kernel preemption to protect the cpu list 10507c478bd9Sstevel@tonic-gate */ 10517c478bd9Sstevel@tonic-gate kpreempt_disable(); 10527c478bd9Sstevel@tonic-gate if ((CPU->cpu_next_onln != CPU) && (sendmondo_in_recover == 0)) { 10537c478bd9Sstevel@tonic-gate xt_one(CPU->cpu_next_onln->cpu_id, (xcfunc_t *)xt_sync_tl1, 10547c478bd9Sstevel@tonic-gate 0, 0); 10557c478bd9Sstevel@tonic-gate } 10567c478bd9Sstevel@tonic-gate kpreempt_enable(); 10577c478bd9Sstevel@tonic-gate } 10587c478bd9Sstevel@tonic-gate 10597c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 10607c478bd9Sstevel@tonic-gate 10617c478bd9Sstevel@tonic-gate #ifdef SEND_MONDO_STATS 10627c478bd9Sstevel@tonic-gate uint32_t x_one_stimes[64]; 10637c478bd9Sstevel@tonic-gate uint32_t x_one_ltimes[16]; 10647c478bd9Sstevel@tonic-gate uint32_t x_set_stimes[64]; 10657c478bd9Sstevel@tonic-gate uint32_t x_set_ltimes[16]; 10667c478bd9Sstevel@tonic-gate uint32_t x_set_cpus[NCPU]; 10677c478bd9Sstevel@tonic-gate uint32_t x_nack_stimes[64]; 10687c478bd9Sstevel@tonic-gate #endif 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate /* 10717c478bd9Sstevel@tonic-gate * Note: A version of this function is used by the debugger via the KDI, 10727c478bd9Sstevel@tonic-gate * and must be kept in sync with this version. Any changes made to this 10737c478bd9Sstevel@tonic-gate * function to support new chips or to accomodate errata must also be included 10747c478bd9Sstevel@tonic-gate * in the KDI-specific version. See us3_kdi.c. 10757c478bd9Sstevel@tonic-gate */ 10767c478bd9Sstevel@tonic-gate void 10777c478bd9Sstevel@tonic-gate send_one_mondo(int cpuid) 10787c478bd9Sstevel@tonic-gate { 10797c478bd9Sstevel@tonic-gate int busy, nack; 10807c478bd9Sstevel@tonic-gate uint64_t idsr, starttick, endtick, tick, lasttick; 10817c478bd9Sstevel@tonic-gate uint64_t busymask; 10827c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 10837c478bd9Sstevel@tonic-gate int recovered = 0; 10847c478bd9Sstevel@tonic-gate #endif 10857c478bd9Sstevel@tonic-gate 10867c478bd9Sstevel@tonic-gate CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 10877c478bd9Sstevel@tonic-gate starttick = lasttick = gettick(); 10887c478bd9Sstevel@tonic-gate shipit(cpuid, 0); 10897c478bd9Sstevel@tonic-gate endtick = starttick + xc_tick_limit; 10907c478bd9Sstevel@tonic-gate busy = nack = 0; 10917c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 10927c478bd9Sstevel@tonic-gate /* 10937c478bd9Sstevel@tonic-gate * Lower 2 bits of the agent ID determine which BUSY/NACK pair 10947c478bd9Sstevel@tonic-gate * will be used for dispatching interrupt. For now, assume 10957c478bd9Sstevel@tonic-gate * there are no more than IDSR_BN_SETS CPUs, hence no aliasing 10967c478bd9Sstevel@tonic-gate * issues with respect to BUSY/NACK pair usage. 10977c478bd9Sstevel@tonic-gate */ 10987c478bd9Sstevel@tonic-gate busymask = IDSR_BUSY_BIT(cpuid); 10997c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 11007c478bd9Sstevel@tonic-gate busymask = IDSR_BUSY; 11017c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 11027c478bd9Sstevel@tonic-gate for (;;) { 11037c478bd9Sstevel@tonic-gate idsr = getidsr(); 11047c478bd9Sstevel@tonic-gate if (idsr == 0) 11057c478bd9Sstevel@tonic-gate break; 11067c478bd9Sstevel@tonic-gate 11077c478bd9Sstevel@tonic-gate tick = gettick(); 11087c478bd9Sstevel@tonic-gate /* 11097c478bd9Sstevel@tonic-gate * If there is a big jump between the current tick 11107c478bd9Sstevel@tonic-gate * count and lasttick, we have probably hit a break 11117c478bd9Sstevel@tonic-gate * point. Adjust endtick accordingly to avoid panic. 11127c478bd9Sstevel@tonic-gate */ 11137c478bd9Sstevel@tonic-gate if (tick > (lasttick + xc_tick_jump_limit)) 11147c478bd9Sstevel@tonic-gate endtick += (tick - lasttick); 11157c478bd9Sstevel@tonic-gate lasttick = tick; 11167c478bd9Sstevel@tonic-gate if (tick > endtick) { 11177c478bd9Sstevel@tonic-gate if (panic_quiesce) 11187c478bd9Sstevel@tonic-gate return; 11197c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 11207c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_recover && recovered == 0) { 11217c478bd9Sstevel@tonic-gate if (mondo_recover(cpuid, 0)) { 11227c478bd9Sstevel@tonic-gate /* 11237c478bd9Sstevel@tonic-gate * We claimed the whole memory or 11247c478bd9Sstevel@tonic-gate * full scan is disabled. 11257c478bd9Sstevel@tonic-gate */ 11267c478bd9Sstevel@tonic-gate recovered++; 11277c478bd9Sstevel@tonic-gate } 11287c478bd9Sstevel@tonic-gate tick = gettick(); 11297c478bd9Sstevel@tonic-gate endtick = tick + xc_tick_limit; 11307c478bd9Sstevel@tonic-gate lasttick = tick; 11317c478bd9Sstevel@tonic-gate /* 11327c478bd9Sstevel@tonic-gate * Recheck idsr 11337c478bd9Sstevel@tonic-gate */ 11347c478bd9Sstevel@tonic-gate continue; 11357c478bd9Sstevel@tonic-gate } else 11367c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 11377c478bd9Sstevel@tonic-gate { 11387c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "send mondo timeout " 11397c478bd9Sstevel@tonic-gate "(target 0x%x) [%d NACK %d BUSY]", 11407c478bd9Sstevel@tonic-gate cpuid, nack, busy); 11417c478bd9Sstevel@tonic-gate } 11427c478bd9Sstevel@tonic-gate } 11437c478bd9Sstevel@tonic-gate 11447c478bd9Sstevel@tonic-gate if (idsr & busymask) { 11457c478bd9Sstevel@tonic-gate busy++; 11467c478bd9Sstevel@tonic-gate continue; 11477c478bd9Sstevel@tonic-gate } 11487c478bd9Sstevel@tonic-gate drv_usecwait(1); 11497c478bd9Sstevel@tonic-gate shipit(cpuid, 0); 11507c478bd9Sstevel@tonic-gate nack++; 11517c478bd9Sstevel@tonic-gate busy = 0; 11527c478bd9Sstevel@tonic-gate } 11537c478bd9Sstevel@tonic-gate #ifdef SEND_MONDO_STATS 11547c478bd9Sstevel@tonic-gate { 11557c478bd9Sstevel@tonic-gate int n = gettick() - starttick; 11567c478bd9Sstevel@tonic-gate if (n < 8192) 11577c478bd9Sstevel@tonic-gate x_one_stimes[n >> 7]++; 11587c478bd9Sstevel@tonic-gate else 11597c478bd9Sstevel@tonic-gate x_one_ltimes[(n >> 13) & 0xf]++; 11607c478bd9Sstevel@tonic-gate } 11617c478bd9Sstevel@tonic-gate #endif 11627c478bd9Sstevel@tonic-gate } 11637c478bd9Sstevel@tonic-gate 11647c478bd9Sstevel@tonic-gate void 11657c478bd9Sstevel@tonic-gate syncfpu(void) 11667c478bd9Sstevel@tonic-gate { 11677c478bd9Sstevel@tonic-gate } 11687c478bd9Sstevel@tonic-gate 11697c478bd9Sstevel@tonic-gate /* 11707c478bd9Sstevel@tonic-gate * Return processor specific async error structure 11717c478bd9Sstevel@tonic-gate * size used. 11727c478bd9Sstevel@tonic-gate */ 11737c478bd9Sstevel@tonic-gate int 11747c478bd9Sstevel@tonic-gate cpu_aflt_size(void) 11757c478bd9Sstevel@tonic-gate { 11767c478bd9Sstevel@tonic-gate return (sizeof (ch_async_flt_t)); 11777c478bd9Sstevel@tonic-gate } 11787c478bd9Sstevel@tonic-gate 11797c478bd9Sstevel@tonic-gate /* 11807c478bd9Sstevel@tonic-gate * The fast_ecc_err handler transfers control here for UCU, UCC events. 11817c478bd9Sstevel@tonic-gate * Note that we flush Ecache twice, once in the fast_ecc_err handler to 11827c478bd9Sstevel@tonic-gate * flush the error that caused the UCU/UCC, then again here at the end to 11837c478bd9Sstevel@tonic-gate * flush the TL=1 trap handler code out of the Ecache, so we can minimize 11847c478bd9Sstevel@tonic-gate * the probability of getting a TL>1 Fast ECC trap when we're fielding 11857c478bd9Sstevel@tonic-gate * another Fast ECC trap. 11867c478bd9Sstevel@tonic-gate * 11877c478bd9Sstevel@tonic-gate * Cheetah+ also handles: TSCE: No additional processing required. 11887c478bd9Sstevel@tonic-gate * Panther adds L3_UCU and L3_UCC which are reported in AFSR_EXT. 11897c478bd9Sstevel@tonic-gate * 11907c478bd9Sstevel@tonic-gate * Note that the p_clo_flags input is only valid in cases where the 11917c478bd9Sstevel@tonic-gate * cpu_private struct is not yet initialized (since that is the only 11927c478bd9Sstevel@tonic-gate * time that information cannot be obtained from the logout struct.) 11937c478bd9Sstevel@tonic-gate */ 11947c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 11957c478bd9Sstevel@tonic-gate void 11967c478bd9Sstevel@tonic-gate cpu_fast_ecc_error(struct regs *rp, ulong_t p_clo_flags) 11977c478bd9Sstevel@tonic-gate { 11987c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 1199*4fd7ecabSdilpreet uint64_t ceen, nceen; 12007c478bd9Sstevel@tonic-gate 12017c478bd9Sstevel@tonic-gate /* 12027c478bd9Sstevel@tonic-gate * Get the CPU log out info. If we can't find our CPU private 12037c478bd9Sstevel@tonic-gate * pointer, then we will have to make due without any detailed 12047c478bd9Sstevel@tonic-gate * logout information. 12057c478bd9Sstevel@tonic-gate */ 12067c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 12077c478bd9Sstevel@tonic-gate clop = NULL; 12087c478bd9Sstevel@tonic-gate ceen = p_clo_flags & EN_REG_CEEN; 1209*4fd7ecabSdilpreet nceen = p_clo_flags & EN_REG_NCEEN; 12107c478bd9Sstevel@tonic-gate } else { 12117c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_fecctl0_logout); 12127c478bd9Sstevel@tonic-gate ceen = clop->clo_flags & EN_REG_CEEN; 1213*4fd7ecabSdilpreet nceen = clop->clo_flags & EN_REG_NCEEN; 12147c478bd9Sstevel@tonic-gate } 12157c478bd9Sstevel@tonic-gate 12167c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error((caddr_t)rp->r_pc, 1217*4fd7ecabSdilpreet (rp->r_tstate & TSTATE_PRIV) ? 1 : 0, 0, ceen, nceen, clop); 12187c478bd9Sstevel@tonic-gate } 12197c478bd9Sstevel@tonic-gate 12207c478bd9Sstevel@tonic-gate /* 12217c478bd9Sstevel@tonic-gate * Log fast ecc error, called from either Fast ECC at TL=0 or Fast 12227c478bd9Sstevel@tonic-gate * ECC at TL>0. Need to supply either a error register pointer or a 12237c478bd9Sstevel@tonic-gate * cpu logout structure pointer. 12247c478bd9Sstevel@tonic-gate */ 12257c478bd9Sstevel@tonic-gate static void 12267c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 1227*4fd7ecabSdilpreet uint64_t nceen, ch_cpu_logout_t *clop) 12287c478bd9Sstevel@tonic-gate { 12297c478bd9Sstevel@tonic-gate struct async_flt *aflt; 12307c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 12317c478bd9Sstevel@tonic-gate uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 12327c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 12337c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 12347c478bd9Sstevel@tonic-gate 12357c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 12367c478bd9Sstevel@tonic-gate /* 12377c478bd9Sstevel@tonic-gate * If no cpu logout data, then we will have to make due without 12387c478bd9Sstevel@tonic-gate * any detailed logout information. 12397c478bd9Sstevel@tonic-gate */ 12407c478bd9Sstevel@tonic-gate if (clop == NULL) { 12417c478bd9Sstevel@tonic-gate ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 12427c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 12437c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 12447c478bd9Sstevel@tonic-gate t_afar = cpu_error_regs.afar; 12457c478bd9Sstevel@tonic-gate t_afsr = cpu_error_regs.afsr; 12467c478bd9Sstevel@tonic-gate t_afsr_ext = cpu_error_regs.afsr_ext; 12477c478bd9Sstevel@tonic-gate #if defined(SERRANO) 12487c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 12497c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 12507c478bd9Sstevel@tonic-gate } else { 12517c478bd9Sstevel@tonic-gate t_afar = clop->clo_data.chd_afar; 12527c478bd9Sstevel@tonic-gate t_afsr = clop->clo_data.chd_afsr; 12537c478bd9Sstevel@tonic-gate t_afsr_ext = clop->clo_data.chd_afsr_ext; 12547c478bd9Sstevel@tonic-gate #if defined(SERRANO) 12557c478bd9Sstevel@tonic-gate ch_flt.afar2 = clop->clo_data.chd_afar2; 12567c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 12577c478bd9Sstevel@tonic-gate } 12587c478bd9Sstevel@tonic-gate 12597c478bd9Sstevel@tonic-gate /* 12607c478bd9Sstevel@tonic-gate * In order to simplify code, we maintain this afsr_errs 12617c478bd9Sstevel@tonic-gate * variable which holds the aggregate of AFSR and AFSR_EXT 12627c478bd9Sstevel@tonic-gate * sticky bits. 12637c478bd9Sstevel@tonic-gate */ 12647c478bd9Sstevel@tonic-gate t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 12657c478bd9Sstevel@tonic-gate (t_afsr & C_AFSR_ALL_ERRS); 12667c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 12677c478bd9Sstevel@tonic-gate 12687c478bd9Sstevel@tonic-gate /* Setup the async fault structure */ 12697c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 12707c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 12717c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = t_afsr_ext; 12727c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = t_afsr_errs; 12737c478bd9Sstevel@tonic-gate aflt->flt_stat = t_afsr; 12747c478bd9Sstevel@tonic-gate aflt->flt_addr = t_afar; 12757c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 12767c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 12777c478bd9Sstevel@tonic-gate aflt->flt_pc = tpc; 12787c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 12797c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 12807c478bd9Sstevel@tonic-gate aflt->flt_priv = priv; 12817c478bd9Sstevel@tonic-gate aflt->flt_tl = tl; 12827c478bd9Sstevel@tonic-gate aflt->flt_status = ECC_F_TRAP; 12837c478bd9Sstevel@tonic-gate aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 12847c478bd9Sstevel@tonic-gate 12857c478bd9Sstevel@tonic-gate /* 12867c478bd9Sstevel@tonic-gate * XXXX - Phenomenal hack to get around Solaris not getting all the 12877c478bd9Sstevel@tonic-gate * cmn_err messages out to the console. The situation is a UCU (in 12887c478bd9Sstevel@tonic-gate * priv mode) which causes a WDU which causes a UE (on the retry). 12897c478bd9Sstevel@tonic-gate * The messages for the UCU and WDU are enqueued and then pulled off 12907c478bd9Sstevel@tonic-gate * the async queue via softint and syslogd starts to process them 12917c478bd9Sstevel@tonic-gate * but doesn't get them to the console. The UE causes a panic, but 12927c478bd9Sstevel@tonic-gate * since the UCU/WDU messages are already in transit, those aren't 12937c478bd9Sstevel@tonic-gate * on the async queue. The hack is to check if we have a matching 12947c478bd9Sstevel@tonic-gate * WDU event for the UCU, and if it matches, we're more than likely 12957c478bd9Sstevel@tonic-gate * going to panic with a UE, unless we're under protection. So, we 12967c478bd9Sstevel@tonic-gate * check to see if we got a matching WDU event and if we're under 12977c478bd9Sstevel@tonic-gate * protection. 12987c478bd9Sstevel@tonic-gate * 12997c478bd9Sstevel@tonic-gate * For Cheetah/Cheetah+/Jaguar/Jalapeno, the sequence we care about 13007c478bd9Sstevel@tonic-gate * looks like this: 13017c478bd9Sstevel@tonic-gate * UCU->WDU->UE 13027c478bd9Sstevel@tonic-gate * For Panther, it could look like either of these: 13037c478bd9Sstevel@tonic-gate * UCU---->WDU->L3_WDU->UE 13047c478bd9Sstevel@tonic-gate * L3_UCU->WDU->L3_WDU->UE 13057c478bd9Sstevel@tonic-gate */ 13067c478bd9Sstevel@tonic-gate if ((t_afsr_errs & (C_AFSR_UCU | C_AFSR_L3_UCU)) && 13077c478bd9Sstevel@tonic-gate aflt->flt_panic == 0 && aflt->flt_priv != 0 && 13087c478bd9Sstevel@tonic-gate curthread->t_ontrap == NULL && curthread->t_lofault == NULL) { 13097c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 13107c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((cpu_error_regs.afsr & C_AFSR_WDU) && 13117c478bd9Sstevel@tonic-gate (cpu_error_regs.afar == t_afar)); 13127c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((clop == NULL) && 13137c478bd9Sstevel@tonic-gate (t_afsr_errs & C_AFSR_WDU)); 13147c478bd9Sstevel@tonic-gate } 13157c478bd9Sstevel@tonic-gate 13167c478bd9Sstevel@tonic-gate /* 13177c478bd9Sstevel@tonic-gate * Queue events on the async event queue, one event per error bit. 13187c478bd9Sstevel@tonic-gate * If no events are queued or no Fast ECC events are on in the AFSR, 13197c478bd9Sstevel@tonic-gate * queue an event to complain. 13207c478bd9Sstevel@tonic-gate */ 13217c478bd9Sstevel@tonic-gate if (cpu_queue_events(&ch_flt, pr_reason, t_afsr_errs, clop) == 0 || 13227c478bd9Sstevel@tonic-gate ((t_afsr_errs & (C_AFSR_FECC_ERRS | C_AFSR_EXT_FECC_ERRS)) == 0)) { 13237c478bd9Sstevel@tonic-gate ch_flt.flt_type = CPU_INV_AFSR; 13247c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 13257c478bd9Sstevel@tonic-gate (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 13267c478bd9Sstevel@tonic-gate aflt->flt_panic); 13277c478bd9Sstevel@tonic-gate } 13287c478bd9Sstevel@tonic-gate 13297c478bd9Sstevel@tonic-gate /* 13307c478bd9Sstevel@tonic-gate * Zero out + invalidate CPU logout. 13317c478bd9Sstevel@tonic-gate */ 13327c478bd9Sstevel@tonic-gate if (clop) { 13337c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 13347c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 13357c478bd9Sstevel@tonic-gate } 13367c478bd9Sstevel@tonic-gate 13377c478bd9Sstevel@tonic-gate /* 13387c478bd9Sstevel@tonic-gate * We carefully re-enable NCEEN and CEEN and then check if any deferred 13397c478bd9Sstevel@tonic-gate * or disrupting errors have happened. We do this because if a 13407c478bd9Sstevel@tonic-gate * deferred or disrupting error had occurred with NCEEN/CEEN off, the 13417c478bd9Sstevel@tonic-gate * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 13427c478bd9Sstevel@tonic-gate * CEEN works differently on Cheetah than on Spitfire. Also, we enable 13437c478bd9Sstevel@tonic-gate * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 13447c478bd9Sstevel@tonic-gate * deferred or disrupting error happening between checking the AFSR and 13457c478bd9Sstevel@tonic-gate * enabling NCEEN/CEEN. 13467c478bd9Sstevel@tonic-gate * 1347*4fd7ecabSdilpreet * Note: CEEN and NCEEN are only reenabled if they were on when trap 1348*4fd7ecabSdilpreet * taken. 13497c478bd9Sstevel@tonic-gate */ 1350*4fd7ecabSdilpreet set_error_enable(get_error_enable() | (nceen | ceen)); 13517c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 13527c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((ch_flt.afsr_errs & 13537c478bd9Sstevel@tonic-gate (C_AFSR_EXT_ASYNC_ERRS | C_AFSR_ASYNC_ERRS)) != 0); 13547c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 13557c478bd9Sstevel@tonic-gate NULL); 13567c478bd9Sstevel@tonic-gate } 13577c478bd9Sstevel@tonic-gate 13587c478bd9Sstevel@tonic-gate /* 13597c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 13607c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 13617c478bd9Sstevel@tonic-gate */ 13627c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 13637c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", pr_reason); 13647c478bd9Sstevel@tonic-gate 13657c478bd9Sstevel@tonic-gate /* 13667c478bd9Sstevel@tonic-gate * Flushing the Ecache here gets the part of the trap handler that 13677c478bd9Sstevel@tonic-gate * is run at TL=1 out of the Ecache. 13687c478bd9Sstevel@tonic-gate */ 13697c478bd9Sstevel@tonic-gate cpu_flush_ecache(); 13707c478bd9Sstevel@tonic-gate } 13717c478bd9Sstevel@tonic-gate 13727c478bd9Sstevel@tonic-gate /* 13737c478bd9Sstevel@tonic-gate * This is called via sys_trap from pil15_interrupt code if the 13747c478bd9Sstevel@tonic-gate * corresponding entry in ch_err_tl1_pending is set. Checks the 13757c478bd9Sstevel@tonic-gate * various ch_err_tl1_data structures for valid entries based on the bit 13767c478bd9Sstevel@tonic-gate * settings in the ch_err_tl1_flags entry of the structure. 13777c478bd9Sstevel@tonic-gate */ 13787c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 13797c478bd9Sstevel@tonic-gate void 13807c478bd9Sstevel@tonic-gate cpu_tl1_error(struct regs *rp, int panic) 13817c478bd9Sstevel@tonic-gate { 13827c478bd9Sstevel@tonic-gate ch_err_tl1_data_t *cl1p, cl1; 13837c478bd9Sstevel@tonic-gate int i, ncl1ps; 13847c478bd9Sstevel@tonic-gate uint64_t me_flags; 1385*4fd7ecabSdilpreet uint64_t ceen, nceen; 13867c478bd9Sstevel@tonic-gate 13877c478bd9Sstevel@tonic-gate if (ch_err_tl1_paddrs[CPU->cpu_id] == 0) { 13887c478bd9Sstevel@tonic-gate cl1p = &ch_err_tl1_data; 13897c478bd9Sstevel@tonic-gate ncl1ps = 1; 13907c478bd9Sstevel@tonic-gate } else if (CPU_PRIVATE(CPU) != NULL) { 13917c478bd9Sstevel@tonic-gate cl1p = CPU_PRIVATE_PTR(CPU, chpr_tl1_err_data[0]); 13927c478bd9Sstevel@tonic-gate ncl1ps = CH_ERR_TL1_TLMAX; 13937c478bd9Sstevel@tonic-gate } else { 13947c478bd9Sstevel@tonic-gate ncl1ps = 0; 13957c478bd9Sstevel@tonic-gate } 13967c478bd9Sstevel@tonic-gate 13977c478bd9Sstevel@tonic-gate for (i = 0; i < ncl1ps; i++, cl1p++) { 13987c478bd9Sstevel@tonic-gate if (cl1p->ch_err_tl1_flags == 0) 13997c478bd9Sstevel@tonic-gate continue; 14007c478bd9Sstevel@tonic-gate 14017c478bd9Sstevel@tonic-gate /* 14027c478bd9Sstevel@tonic-gate * Grab a copy of the logout data and invalidate 14037c478bd9Sstevel@tonic-gate * the logout area. 14047c478bd9Sstevel@tonic-gate */ 14057c478bd9Sstevel@tonic-gate cl1 = *cl1p; 14067c478bd9Sstevel@tonic-gate bzero(cl1p, sizeof (ch_err_tl1_data_t)); 14077c478bd9Sstevel@tonic-gate cl1p->ch_err_tl1_logout.clo_data.chd_afar = LOGOUT_INVALID; 14087c478bd9Sstevel@tonic-gate me_flags = CH_ERR_ME_FLAGS(cl1.ch_err_tl1_flags); 14097c478bd9Sstevel@tonic-gate 14107c478bd9Sstevel@tonic-gate /* 14117c478bd9Sstevel@tonic-gate * Log "first error" in ch_err_tl1_data. 14127c478bd9Sstevel@tonic-gate */ 14137c478bd9Sstevel@tonic-gate if (cl1.ch_err_tl1_flags & CH_ERR_FECC) { 14147c478bd9Sstevel@tonic-gate ceen = get_error_enable() & EN_REG_CEEN; 1415*4fd7ecabSdilpreet nceen = get_error_enable() & EN_REG_NCEEN; 14167c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error((caddr_t)cl1.ch_err_tl1_tpc, 1, 1417*4fd7ecabSdilpreet 1, ceen, nceen, &cl1.ch_err_tl1_logout); 14187c478bd9Sstevel@tonic-gate } 14197c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 14207c478bd9Sstevel@tonic-gate if (cl1.ch_err_tl1_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 14217c478bd9Sstevel@tonic-gate cpu_parity_error(rp, cl1.ch_err_tl1_flags, 14227c478bd9Sstevel@tonic-gate (caddr_t)cl1.ch_err_tl1_tpc); 14237c478bd9Sstevel@tonic-gate } 14247c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 14257c478bd9Sstevel@tonic-gate 14267c478bd9Sstevel@tonic-gate /* 14277c478bd9Sstevel@tonic-gate * Log "multiple events" in ch_err_tl1_data. Note that 14287c478bd9Sstevel@tonic-gate * we don't read and clear the AFSR/AFAR in the TL>0 code 14297c478bd9Sstevel@tonic-gate * if the structure is busy, we just do the cache flushing 14307c478bd9Sstevel@tonic-gate * we have to do and then do the retry. So the AFSR/AFAR 14317c478bd9Sstevel@tonic-gate * at this point *should* have some relevant info. If there 14327c478bd9Sstevel@tonic-gate * are no valid errors in the AFSR, we'll assume they've 14337c478bd9Sstevel@tonic-gate * already been picked up and logged. For I$/D$ parity, 14347c478bd9Sstevel@tonic-gate * we just log an event with an "Unknown" (NULL) TPC. 14357c478bd9Sstevel@tonic-gate */ 14367c478bd9Sstevel@tonic-gate if (me_flags & CH_ERR_FECC) { 14377c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 14387c478bd9Sstevel@tonic-gate uint64_t t_afsr_errs; 14397c478bd9Sstevel@tonic-gate 14407c478bd9Sstevel@tonic-gate /* 14417c478bd9Sstevel@tonic-gate * Get the error registers and see if there's 14427c478bd9Sstevel@tonic-gate * a pending error. If not, don't bother 14437c478bd9Sstevel@tonic-gate * generating an "Invalid AFSR" error event. 14447c478bd9Sstevel@tonic-gate */ 14457c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 14467c478bd9Sstevel@tonic-gate t_afsr_errs = (cpu_error_regs.afsr_ext & 14477c478bd9Sstevel@tonic-gate C_AFSR_EXT_ALL_ERRS) | 14487c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr & C_AFSR_ALL_ERRS); 14497c478bd9Sstevel@tonic-gate if (t_afsr_errs != 0) { 14507c478bd9Sstevel@tonic-gate ceen = get_error_enable() & EN_REG_CEEN; 1451*4fd7ecabSdilpreet nceen = get_error_enable() & EN_REG_NCEEN; 14527c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error((caddr_t)NULL, 1, 1453*4fd7ecabSdilpreet 1, ceen, nceen, NULL); 14547c478bd9Sstevel@tonic-gate } 14557c478bd9Sstevel@tonic-gate } 14567c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 14577c478bd9Sstevel@tonic-gate if (me_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 14587c478bd9Sstevel@tonic-gate cpu_parity_error(rp, me_flags, (caddr_t)NULL); 14597c478bd9Sstevel@tonic-gate } 14607c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 14617c478bd9Sstevel@tonic-gate } 14627c478bd9Sstevel@tonic-gate } 14637c478bd9Sstevel@tonic-gate 14647c478bd9Sstevel@tonic-gate /* 14657c478bd9Sstevel@tonic-gate * Called from Fast ECC TL>0 handler in case of fatal error. 14667c478bd9Sstevel@tonic-gate * cpu_tl1_error should always find an associated ch_err_tl1_data structure, 14677c478bd9Sstevel@tonic-gate * but if we don't, we'll panic with something reasonable. 14687c478bd9Sstevel@tonic-gate */ 14697c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 14707c478bd9Sstevel@tonic-gate void 14717c478bd9Sstevel@tonic-gate cpu_tl1_err_panic(struct regs *rp, ulong_t flags) 14727c478bd9Sstevel@tonic-gate { 14737c478bd9Sstevel@tonic-gate cpu_tl1_error(rp, 1); 14747c478bd9Sstevel@tonic-gate /* 14757c478bd9Sstevel@tonic-gate * Should never return, but just in case. 14767c478bd9Sstevel@tonic-gate */ 14777c478bd9Sstevel@tonic-gate fm_panic("Unsurvivable ECC Error at TL>0"); 14787c478bd9Sstevel@tonic-gate } 14797c478bd9Sstevel@tonic-gate 14807c478bd9Sstevel@tonic-gate /* 14817c478bd9Sstevel@tonic-gate * The ce_err/ce_err_tl1 handlers transfer control here for CE, EMC, EDU:ST, 14827c478bd9Sstevel@tonic-gate * EDC, WDU, WDC, CPU, CPC, IVU, IVC events. 14837c478bd9Sstevel@tonic-gate * Disrupting errors controlled by NCEEN: EDU:ST, WDU, CPU, IVU 14847c478bd9Sstevel@tonic-gate * Disrupting errors controlled by CEEN: CE, EMC, EDC, WDC, CPC, IVC 14857c478bd9Sstevel@tonic-gate * 14867c478bd9Sstevel@tonic-gate * Cheetah+ also handles (No additional processing required): 14877c478bd9Sstevel@tonic-gate * DUE, DTO, DBERR (NCEEN controlled) 14887c478bd9Sstevel@tonic-gate * THCE (CEEN and ET_ECC_en controlled) 14897c478bd9Sstevel@tonic-gate * TUE (ET_ECC_en controlled) 14907c478bd9Sstevel@tonic-gate * 14917c478bd9Sstevel@tonic-gate * Panther further adds: 14927c478bd9Sstevel@tonic-gate * IMU, L3_EDU, L3_WDU, L3_CPU (NCEEN controlled) 14937c478bd9Sstevel@tonic-gate * IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE (CEEN controlled) 14947c478bd9Sstevel@tonic-gate * TUE_SH, TUE (NCEEN and L2_tag_ECC_en controlled) 14957c478bd9Sstevel@tonic-gate * L3_TUE, L3_TUE_SH (NCEEN and ET_ECC_en controlled) 14967c478bd9Sstevel@tonic-gate * THCE (CEEN and L2_tag_ECC_en controlled) 14977c478bd9Sstevel@tonic-gate * L3_THCE (CEEN and ET_ECC_en controlled) 14987c478bd9Sstevel@tonic-gate * 14997c478bd9Sstevel@tonic-gate * Note that the p_clo_flags input is only valid in cases where the 15007c478bd9Sstevel@tonic-gate * cpu_private struct is not yet initialized (since that is the only 15017c478bd9Sstevel@tonic-gate * time that information cannot be obtained from the logout struct.) 15027c478bd9Sstevel@tonic-gate */ 15037c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 15047c478bd9Sstevel@tonic-gate void 15057c478bd9Sstevel@tonic-gate cpu_disrupting_error(struct regs *rp, ulong_t p_clo_flags) 15067c478bd9Sstevel@tonic-gate { 15077c478bd9Sstevel@tonic-gate struct async_flt *aflt; 15087c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 15097c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 15107c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 15117c478bd9Sstevel@tonic-gate uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 15127c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 15137c478bd9Sstevel@tonic-gate 15147c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 15157c478bd9Sstevel@tonic-gate /* 15167c478bd9Sstevel@tonic-gate * Get the CPU log out info. If we can't find our CPU private 15177c478bd9Sstevel@tonic-gate * pointer, then we will have to make due without any detailed 15187c478bd9Sstevel@tonic-gate * logout information. 15197c478bd9Sstevel@tonic-gate */ 15207c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 15217c478bd9Sstevel@tonic-gate clop = NULL; 15227c478bd9Sstevel@tonic-gate ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 15237c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 15247c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 15257c478bd9Sstevel@tonic-gate t_afar = cpu_error_regs.afar; 15267c478bd9Sstevel@tonic-gate t_afsr = cpu_error_regs.afsr; 15277c478bd9Sstevel@tonic-gate t_afsr_ext = cpu_error_regs.afsr_ext; 15287c478bd9Sstevel@tonic-gate #if defined(SERRANO) 15297c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 15307c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 15317c478bd9Sstevel@tonic-gate } else { 15327c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 15337c478bd9Sstevel@tonic-gate t_afar = clop->clo_data.chd_afar; 15347c478bd9Sstevel@tonic-gate t_afsr = clop->clo_data.chd_afsr; 15357c478bd9Sstevel@tonic-gate t_afsr_ext = clop->clo_data.chd_afsr_ext; 15367c478bd9Sstevel@tonic-gate #if defined(SERRANO) 15377c478bd9Sstevel@tonic-gate ch_flt.afar2 = clop->clo_data.chd_afar2; 15387c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 15397c478bd9Sstevel@tonic-gate } 15407c478bd9Sstevel@tonic-gate 15417c478bd9Sstevel@tonic-gate /* 15427c478bd9Sstevel@tonic-gate * In order to simplify code, we maintain this afsr_errs 15437c478bd9Sstevel@tonic-gate * variable which holds the aggregate of AFSR and AFSR_EXT 15447c478bd9Sstevel@tonic-gate * sticky bits. 15457c478bd9Sstevel@tonic-gate */ 15467c478bd9Sstevel@tonic-gate t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 15477c478bd9Sstevel@tonic-gate (t_afsr & C_AFSR_ALL_ERRS); 15487c478bd9Sstevel@tonic-gate 15497c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 15507c478bd9Sstevel@tonic-gate /* Setup the async fault structure */ 15517c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 15527c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = t_afsr_ext; 15537c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = t_afsr_errs; 15547c478bd9Sstevel@tonic-gate aflt->flt_stat = t_afsr; 15557c478bd9Sstevel@tonic-gate aflt->flt_addr = t_afar; 15567c478bd9Sstevel@tonic-gate aflt->flt_pc = (caddr_t)rp->r_pc; 15577c478bd9Sstevel@tonic-gate aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 15587c478bd9Sstevel@tonic-gate aflt->flt_tl = 0; 15597c478bd9Sstevel@tonic-gate aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 15607c478bd9Sstevel@tonic-gate 15617c478bd9Sstevel@tonic-gate /* 15627c478bd9Sstevel@tonic-gate * If this trap is a result of one of the errors not masked 15637c478bd9Sstevel@tonic-gate * by cpu_ce_not_deferred, we don't reenable CEEN. Instead 15647c478bd9Sstevel@tonic-gate * indicate that a timeout is to be set later. 15657c478bd9Sstevel@tonic-gate */ 15667c478bd9Sstevel@tonic-gate if (!(t_afsr_errs & (cpu_ce_not_deferred | cpu_ce_not_deferred_ext)) && 15677c478bd9Sstevel@tonic-gate !aflt->flt_panic) 15687c478bd9Sstevel@tonic-gate ch_flt.flt_trapped_ce = CE_CEEN_DEFER | CE_CEEN_TRAPPED; 15697c478bd9Sstevel@tonic-gate else 15707c478bd9Sstevel@tonic-gate ch_flt.flt_trapped_ce = CE_CEEN_NODEFER | CE_CEEN_TRAPPED; 15717c478bd9Sstevel@tonic-gate 15727c478bd9Sstevel@tonic-gate /* 15737c478bd9Sstevel@tonic-gate * log the CE and clean up 15747c478bd9Sstevel@tonic-gate */ 15757c478bd9Sstevel@tonic-gate cpu_log_and_clear_ce(&ch_flt); 15767c478bd9Sstevel@tonic-gate 15777c478bd9Sstevel@tonic-gate /* 15787c478bd9Sstevel@tonic-gate * We re-enable CEEN (if required) and check if any disrupting errors 15797c478bd9Sstevel@tonic-gate * have happened. We do this because if a disrupting error had occurred 15807c478bd9Sstevel@tonic-gate * with CEEN off, the trap will not be taken when CEEN is re-enabled. 15817c478bd9Sstevel@tonic-gate * Note that CEEN works differently on Cheetah than on Spitfire. Also, 15827c478bd9Sstevel@tonic-gate * we enable CEEN *before* checking the AFSR to avoid the small window 15837c478bd9Sstevel@tonic-gate * of a error happening between checking the AFSR and enabling CEEN. 15847c478bd9Sstevel@tonic-gate */ 15857c478bd9Sstevel@tonic-gate if (ch_flt.flt_trapped_ce & CE_CEEN_NODEFER) 15867c478bd9Sstevel@tonic-gate set_error_enable(get_error_enable() | EN_REG_CEEN); 15877c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 15887c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 15897c478bd9Sstevel@tonic-gate NULL); 15907c478bd9Sstevel@tonic-gate } 15917c478bd9Sstevel@tonic-gate 15927c478bd9Sstevel@tonic-gate /* 15937c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 15947c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 15957c478bd9Sstevel@tonic-gate */ 15967c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 15977c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", pr_reason); 15987c478bd9Sstevel@tonic-gate } 15997c478bd9Sstevel@tonic-gate 16007c478bd9Sstevel@tonic-gate /* 16017c478bd9Sstevel@tonic-gate * The async_err handler transfers control here for UE, EMU, EDU:BLD, 16027c478bd9Sstevel@tonic-gate * L3_EDU:BLD, TO, and BERR events. 16037c478bd9Sstevel@tonic-gate * Deferred errors controlled by NCEEN: UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR 16047c478bd9Sstevel@tonic-gate * 16057c478bd9Sstevel@tonic-gate * Cheetah+: No additional errors handled. 16067c478bd9Sstevel@tonic-gate * 16077c478bd9Sstevel@tonic-gate * Note that the p_clo_flags input is only valid in cases where the 16087c478bd9Sstevel@tonic-gate * cpu_private struct is not yet initialized (since that is the only 16097c478bd9Sstevel@tonic-gate * time that information cannot be obtained from the logout struct.) 16107c478bd9Sstevel@tonic-gate */ 16117c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 16127c478bd9Sstevel@tonic-gate void 16137c478bd9Sstevel@tonic-gate cpu_deferred_error(struct regs *rp, ulong_t p_clo_flags) 16147c478bd9Sstevel@tonic-gate { 16157c478bd9Sstevel@tonic-gate ushort_t ttype, tl; 16167c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 16177c478bd9Sstevel@tonic-gate struct async_flt *aflt; 16187c478bd9Sstevel@tonic-gate int trampolined = 0; 16197c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 16207c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 16217c478bd9Sstevel@tonic-gate uint64_t ceen, clo_flags; 16227c478bd9Sstevel@tonic-gate uint64_t log_afsr; 16237c478bd9Sstevel@tonic-gate uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 16247c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 16257c478bd9Sstevel@tonic-gate int expected = DDI_FM_ERR_UNEXPECTED; 16267c478bd9Sstevel@tonic-gate ddi_acc_hdl_t *hp; 16277c478bd9Sstevel@tonic-gate 16287c478bd9Sstevel@tonic-gate /* 16297c478bd9Sstevel@tonic-gate * We need to look at p_flag to determine if the thread detected an 16307c478bd9Sstevel@tonic-gate * error while dumping core. We can't grab p_lock here, but it's ok 16317c478bd9Sstevel@tonic-gate * because we just need a consistent snapshot and we know that everyone 16327c478bd9Sstevel@tonic-gate * else will store a consistent set of bits while holding p_lock. We 16337c478bd9Sstevel@tonic-gate * don't have to worry about a race because SDOCORE is set once prior 16347c478bd9Sstevel@tonic-gate * to doing i/o from the process's address space and is never cleared. 16357c478bd9Sstevel@tonic-gate */ 16367c478bd9Sstevel@tonic-gate uint_t pflag = ttoproc(curthread)->p_flag; 16377c478bd9Sstevel@tonic-gate 16387c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 16397c478bd9Sstevel@tonic-gate /* 16407c478bd9Sstevel@tonic-gate * Get the CPU log out info. If we can't find our CPU private 16417c478bd9Sstevel@tonic-gate * pointer then we will have to make due without any detailed 16427c478bd9Sstevel@tonic-gate * logout information. 16437c478bd9Sstevel@tonic-gate */ 16447c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 16457c478bd9Sstevel@tonic-gate clop = NULL; 16467c478bd9Sstevel@tonic-gate ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 16477c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 16487c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 16497c478bd9Sstevel@tonic-gate t_afar = cpu_error_regs.afar; 16507c478bd9Sstevel@tonic-gate t_afsr = cpu_error_regs.afsr; 16517c478bd9Sstevel@tonic-gate t_afsr_ext = cpu_error_regs.afsr_ext; 16527c478bd9Sstevel@tonic-gate #if defined(SERRANO) 16537c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 16547c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 16557c478bd9Sstevel@tonic-gate clo_flags = p_clo_flags; 16567c478bd9Sstevel@tonic-gate } else { 16577c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_async_logout); 16587c478bd9Sstevel@tonic-gate t_afar = clop->clo_data.chd_afar; 16597c478bd9Sstevel@tonic-gate t_afsr = clop->clo_data.chd_afsr; 16607c478bd9Sstevel@tonic-gate t_afsr_ext = clop->clo_data.chd_afsr_ext; 16617c478bd9Sstevel@tonic-gate #if defined(SERRANO) 16627c478bd9Sstevel@tonic-gate ch_flt.afar2 = clop->clo_data.chd_afar2; 16637c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 16647c478bd9Sstevel@tonic-gate clo_flags = clop->clo_flags; 16657c478bd9Sstevel@tonic-gate } 16667c478bd9Sstevel@tonic-gate 16677c478bd9Sstevel@tonic-gate /* 16687c478bd9Sstevel@tonic-gate * In order to simplify code, we maintain this afsr_errs 16697c478bd9Sstevel@tonic-gate * variable which holds the aggregate of AFSR and AFSR_EXT 16707c478bd9Sstevel@tonic-gate * sticky bits. 16717c478bd9Sstevel@tonic-gate */ 16727c478bd9Sstevel@tonic-gate t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 16737c478bd9Sstevel@tonic-gate (t_afsr & C_AFSR_ALL_ERRS); 16747c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 16757c478bd9Sstevel@tonic-gate 16767c478bd9Sstevel@tonic-gate /* 16777c478bd9Sstevel@tonic-gate * Grab information encoded into our clo_flags field. 16787c478bd9Sstevel@tonic-gate */ 16797c478bd9Sstevel@tonic-gate ceen = clo_flags & EN_REG_CEEN; 16807c478bd9Sstevel@tonic-gate tl = (clo_flags & CLO_FLAGS_TL_MASK) >> CLO_FLAGS_TL_SHIFT; 16817c478bd9Sstevel@tonic-gate ttype = (clo_flags & CLO_FLAGS_TT_MASK) >> CLO_FLAGS_TT_SHIFT; 16827c478bd9Sstevel@tonic-gate 16837c478bd9Sstevel@tonic-gate /* 16847c478bd9Sstevel@tonic-gate * handle the specific error 16857c478bd9Sstevel@tonic-gate */ 16867c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 16877c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 16887c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 16897c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 16907c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = t_afsr_ext; 16917c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = t_afsr_errs; 16927c478bd9Sstevel@tonic-gate aflt->flt_stat = t_afsr; 16937c478bd9Sstevel@tonic-gate aflt->flt_addr = t_afar; 16947c478bd9Sstevel@tonic-gate aflt->flt_pc = (caddr_t)rp->r_pc; 16957c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 16967c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 16977c478bd9Sstevel@tonic-gate aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 16987c478bd9Sstevel@tonic-gate aflt->flt_tl = (uchar_t)tl; 16997c478bd9Sstevel@tonic-gate aflt->flt_panic = ((tl != 0) || (aft_testfatal != 0) || 17007c478bd9Sstevel@tonic-gate C_AFSR_PANIC(t_afsr_errs)); 17017c478bd9Sstevel@tonic-gate aflt->flt_core = (pflag & SDOCORE) ? 1 : 0; 17027c478bd9Sstevel@tonic-gate aflt->flt_status = ((ttype == T_DATA_ERROR) ? ECC_D_TRAP : ECC_I_TRAP); 17037c478bd9Sstevel@tonic-gate 17047c478bd9Sstevel@tonic-gate /* 17057c478bd9Sstevel@tonic-gate * If the trap occurred in privileged mode at TL=0, we need to check to 17067c478bd9Sstevel@tonic-gate * see if we were executing in the kernel under on_trap() or t_lofault 17077c478bd9Sstevel@tonic-gate * protection. If so, modify the saved registers so that we return 17087c478bd9Sstevel@tonic-gate * from the trap to the appropriate trampoline routine. 17097c478bd9Sstevel@tonic-gate */ 17107c478bd9Sstevel@tonic-gate if (aflt->flt_priv && tl == 0) { 17117c478bd9Sstevel@tonic-gate if (curthread->t_ontrap != NULL) { 17127c478bd9Sstevel@tonic-gate on_trap_data_t *otp = curthread->t_ontrap; 17137c478bd9Sstevel@tonic-gate 17147c478bd9Sstevel@tonic-gate if (otp->ot_prot & OT_DATA_EC) { 17157c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_EC; 17167c478bd9Sstevel@tonic-gate otp->ot_trap |= OT_DATA_EC; 17177c478bd9Sstevel@tonic-gate rp->r_pc = otp->ot_trampoline; 17187c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 17197c478bd9Sstevel@tonic-gate trampolined = 1; 17207c478bd9Sstevel@tonic-gate } 17217c478bd9Sstevel@tonic-gate 17227c478bd9Sstevel@tonic-gate if ((t_afsr & (C_AFSR_TO | C_AFSR_BERR)) && 17237c478bd9Sstevel@tonic-gate (otp->ot_prot & OT_DATA_ACCESS)) { 17247c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_ACCESS; 17257c478bd9Sstevel@tonic-gate otp->ot_trap |= OT_DATA_ACCESS; 17267c478bd9Sstevel@tonic-gate rp->r_pc = otp->ot_trampoline; 17277c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 17287c478bd9Sstevel@tonic-gate trampolined = 1; 17297c478bd9Sstevel@tonic-gate /* 17307c478bd9Sstevel@tonic-gate * for peeks and caut_gets errors are expected 17317c478bd9Sstevel@tonic-gate */ 17327c478bd9Sstevel@tonic-gate hp = (ddi_acc_hdl_t *)otp->ot_handle; 17337c478bd9Sstevel@tonic-gate if (!hp) 17347c478bd9Sstevel@tonic-gate expected = DDI_FM_ERR_PEEK; 17357c478bd9Sstevel@tonic-gate else if (hp->ah_acc.devacc_attr_access == 17367c478bd9Sstevel@tonic-gate DDI_CAUTIOUS_ACC) 17377c478bd9Sstevel@tonic-gate expected = DDI_FM_ERR_EXPECTED; 17387c478bd9Sstevel@tonic-gate } 17397c478bd9Sstevel@tonic-gate 17407c478bd9Sstevel@tonic-gate } else if (curthread->t_lofault) { 17417c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_COPY; 17427c478bd9Sstevel@tonic-gate rp->r_g1 = EFAULT; 17437c478bd9Sstevel@tonic-gate rp->r_pc = curthread->t_lofault; 17447c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 17457c478bd9Sstevel@tonic-gate trampolined = 1; 17467c478bd9Sstevel@tonic-gate } 17477c478bd9Sstevel@tonic-gate } 17487c478bd9Sstevel@tonic-gate 17497c478bd9Sstevel@tonic-gate /* 17507c478bd9Sstevel@tonic-gate * If we're in user mode or we're doing a protected copy, we either 17517c478bd9Sstevel@tonic-gate * want the ASTON code below to send a signal to the user process 17527c478bd9Sstevel@tonic-gate * or we want to panic if aft_panic is set. 17537c478bd9Sstevel@tonic-gate * 17547c478bd9Sstevel@tonic-gate * If we're in privileged mode and we're not doing a copy, then we 17557c478bd9Sstevel@tonic-gate * need to check if we've trampolined. If we haven't trampolined, 17567c478bd9Sstevel@tonic-gate * we should panic. 17577c478bd9Sstevel@tonic-gate */ 17587c478bd9Sstevel@tonic-gate if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 17597c478bd9Sstevel@tonic-gate if (t_afsr_errs & 17607c478bd9Sstevel@tonic-gate ((C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS) & 17617c478bd9Sstevel@tonic-gate ~(C_AFSR_BERR | C_AFSR_TO))) 17627c478bd9Sstevel@tonic-gate aflt->flt_panic |= aft_panic; 17637c478bd9Sstevel@tonic-gate } else if (!trampolined) { 17647c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 17657c478bd9Sstevel@tonic-gate } 17667c478bd9Sstevel@tonic-gate 17677c478bd9Sstevel@tonic-gate /* 17687c478bd9Sstevel@tonic-gate * If we've trampolined due to a privileged TO or BERR, or if an 17697c478bd9Sstevel@tonic-gate * unprivileged TO or BERR occurred, we don't want to enqueue an 17707c478bd9Sstevel@tonic-gate * event for that TO or BERR. Queue all other events (if any) besides 17717c478bd9Sstevel@tonic-gate * the TO/BERR. Since we may not be enqueing any events, we need to 17727c478bd9Sstevel@tonic-gate * ignore the number of events queued. If we haven't trampolined due 17737c478bd9Sstevel@tonic-gate * to a TO or BERR, just enqueue events normally. 17747c478bd9Sstevel@tonic-gate */ 17757c478bd9Sstevel@tonic-gate log_afsr = t_afsr_errs; 17767c478bd9Sstevel@tonic-gate if (trampolined) { 17777c478bd9Sstevel@tonic-gate log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 17787c478bd9Sstevel@tonic-gate } else if (!aflt->flt_priv) { 17797c478bd9Sstevel@tonic-gate /* 17807c478bd9Sstevel@tonic-gate * User mode, suppress messages if 17817c478bd9Sstevel@tonic-gate * cpu_berr_to_verbose is not set. 17827c478bd9Sstevel@tonic-gate */ 17837c478bd9Sstevel@tonic-gate if (!cpu_berr_to_verbose) 17847c478bd9Sstevel@tonic-gate log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 17857c478bd9Sstevel@tonic-gate } 17867c478bd9Sstevel@tonic-gate 17877c478bd9Sstevel@tonic-gate /* 17887c478bd9Sstevel@tonic-gate * Log any errors that occurred 17897c478bd9Sstevel@tonic-gate */ 17907c478bd9Sstevel@tonic-gate if (((log_afsr & 17917c478bd9Sstevel@tonic-gate ((C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS) & ~C_AFSR_ME)) && 17927c478bd9Sstevel@tonic-gate cpu_queue_events(&ch_flt, pr_reason, log_afsr, clop) == 0) || 17937c478bd9Sstevel@tonic-gate (t_afsr_errs & 17947c478bd9Sstevel@tonic-gate (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) == 0) { 17957c478bd9Sstevel@tonic-gate ch_flt.flt_type = CPU_INV_AFSR; 17967c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 17977c478bd9Sstevel@tonic-gate (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 17987c478bd9Sstevel@tonic-gate aflt->flt_panic); 17997c478bd9Sstevel@tonic-gate } 18007c478bd9Sstevel@tonic-gate 18017c478bd9Sstevel@tonic-gate /* 18027c478bd9Sstevel@tonic-gate * Zero out + invalidate CPU logout. 18037c478bd9Sstevel@tonic-gate */ 18047c478bd9Sstevel@tonic-gate if (clop) { 18057c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 18067c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 18077c478bd9Sstevel@tonic-gate } 18087c478bd9Sstevel@tonic-gate 18097c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 18107c478bd9Sstevel@tonic-gate /* 18117c478bd9Sstevel@tonic-gate * UE/RUE/BERR/TO: Call our bus nexus friends to check for 18127c478bd9Sstevel@tonic-gate * IO errors that may have resulted in this trap. 18137c478bd9Sstevel@tonic-gate */ 18147c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE|C_AFSR_RUE|C_AFSR_TO|C_AFSR_BERR)) { 18157c478bd9Sstevel@tonic-gate cpu_run_bus_error_handlers(aflt, expected); 18167c478bd9Sstevel@tonic-gate } 18177c478bd9Sstevel@tonic-gate 18187c478bd9Sstevel@tonic-gate /* 18197c478bd9Sstevel@tonic-gate * UE/RUE: If UE or RUE is in memory, we need to flush the bad 18207c478bd9Sstevel@tonic-gate * line from the Ecache. We also need to query the bus nexus for 18217c478bd9Sstevel@tonic-gate * fatal errors. Attempts to do diagnostic read on caches may 18227c478bd9Sstevel@tonic-gate * introduce more errors (especially when the module is bad). 18237c478bd9Sstevel@tonic-gate */ 18247c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE|C_AFSR_RUE)) { 18257c478bd9Sstevel@tonic-gate /* 18267c478bd9Sstevel@tonic-gate * Ask our bus nexus friends if they have any fatal errors. If 18277c478bd9Sstevel@tonic-gate * so, they will log appropriate error messages. 18287c478bd9Sstevel@tonic-gate */ 18297c478bd9Sstevel@tonic-gate if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 18307c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 18317c478bd9Sstevel@tonic-gate 18327c478bd9Sstevel@tonic-gate /* 18337c478bd9Sstevel@tonic-gate * We got a UE or RUE and are panicking, save the fault PA in 18347c478bd9Sstevel@tonic-gate * a known location so that the platform specific panic code 18357c478bd9Sstevel@tonic-gate * can check for copyback errors. 18367c478bd9Sstevel@tonic-gate */ 18377c478bd9Sstevel@tonic-gate if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 18387c478bd9Sstevel@tonic-gate panic_aflt = *aflt; 18397c478bd9Sstevel@tonic-gate } 18407c478bd9Sstevel@tonic-gate } 18417c478bd9Sstevel@tonic-gate 18427c478bd9Sstevel@tonic-gate /* 18437c478bd9Sstevel@tonic-gate * Flush Ecache line or entire Ecache 18447c478bd9Sstevel@tonic-gate */ 18457c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE | C_AFSR_RUE | C_AFSR_EDU | C_AFSR_BERR)) 18467c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(&ch_flt); 18477c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 18487c478bd9Sstevel@tonic-gate /* 18497c478bd9Sstevel@tonic-gate * UE/BERR/TO: Call our bus nexus friends to check for 18507c478bd9Sstevel@tonic-gate * IO errors that may have resulted in this trap. 18517c478bd9Sstevel@tonic-gate */ 18527c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE|C_AFSR_TO|C_AFSR_BERR)) { 18537c478bd9Sstevel@tonic-gate cpu_run_bus_error_handlers(aflt, expected); 18547c478bd9Sstevel@tonic-gate } 18557c478bd9Sstevel@tonic-gate 18567c478bd9Sstevel@tonic-gate /* 18577c478bd9Sstevel@tonic-gate * UE: If the UE is in memory, we need to flush the bad 18587c478bd9Sstevel@tonic-gate * line from the Ecache. We also need to query the bus nexus for 18597c478bd9Sstevel@tonic-gate * fatal errors. Attempts to do diagnostic read on caches may 18607c478bd9Sstevel@tonic-gate * introduce more errors (especially when the module is bad). 18617c478bd9Sstevel@tonic-gate */ 18627c478bd9Sstevel@tonic-gate if (t_afsr & C_AFSR_UE) { 18637c478bd9Sstevel@tonic-gate /* 18647c478bd9Sstevel@tonic-gate * Ask our legacy bus nexus friends if they have any fatal 18657c478bd9Sstevel@tonic-gate * errors. If so, they will log appropriate error messages. 18667c478bd9Sstevel@tonic-gate */ 18677c478bd9Sstevel@tonic-gate if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 18687c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 18697c478bd9Sstevel@tonic-gate 18707c478bd9Sstevel@tonic-gate /* 18717c478bd9Sstevel@tonic-gate * We got a UE and are panicking, save the fault PA in a known 18727c478bd9Sstevel@tonic-gate * location so that the platform specific panic code can check 18737c478bd9Sstevel@tonic-gate * for copyback errors. 18747c478bd9Sstevel@tonic-gate */ 18757c478bd9Sstevel@tonic-gate if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 18767c478bd9Sstevel@tonic-gate panic_aflt = *aflt; 18777c478bd9Sstevel@tonic-gate } 18787c478bd9Sstevel@tonic-gate } 18797c478bd9Sstevel@tonic-gate 18807c478bd9Sstevel@tonic-gate /* 18817c478bd9Sstevel@tonic-gate * Flush Ecache line or entire Ecache 18827c478bd9Sstevel@tonic-gate */ 18837c478bd9Sstevel@tonic-gate if (t_afsr_errs & 18847c478bd9Sstevel@tonic-gate (C_AFSR_UE | C_AFSR_EDU | C_AFSR_BERR | C_AFSR_L3_EDU)) 18857c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(&ch_flt); 18867c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 18877c478bd9Sstevel@tonic-gate 18887c478bd9Sstevel@tonic-gate /* 18897c478bd9Sstevel@tonic-gate * We carefully re-enable NCEEN and CEEN and then check if any deferred 18907c478bd9Sstevel@tonic-gate * or disrupting errors have happened. We do this because if a 18917c478bd9Sstevel@tonic-gate * deferred or disrupting error had occurred with NCEEN/CEEN off, the 18927c478bd9Sstevel@tonic-gate * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 18937c478bd9Sstevel@tonic-gate * CEEN works differently on Cheetah than on Spitfire. Also, we enable 18947c478bd9Sstevel@tonic-gate * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 18957c478bd9Sstevel@tonic-gate * deferred or disrupting error happening between checking the AFSR and 18967c478bd9Sstevel@tonic-gate * enabling NCEEN/CEEN. 18977c478bd9Sstevel@tonic-gate * 18987c478bd9Sstevel@tonic-gate * Note: CEEN reenabled only if it was on when trap taken. 18997c478bd9Sstevel@tonic-gate */ 19007c478bd9Sstevel@tonic-gate set_error_enable(get_error_enable() | (EN_REG_NCEEN | ceen)); 19017c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 19027c478bd9Sstevel@tonic-gate /* 19037c478bd9Sstevel@tonic-gate * Check for secondary errors, and avoid panicking if we 19047c478bd9Sstevel@tonic-gate * have them 19057c478bd9Sstevel@tonic-gate */ 19067c478bd9Sstevel@tonic-gate if (cpu_check_secondary_errors(&ch_flt, t_afsr_errs, 19077c478bd9Sstevel@tonic-gate t_afar) == 0) { 19087c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((ch_flt.afsr_errs & 19097c478bd9Sstevel@tonic-gate (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) != 0); 19107c478bd9Sstevel@tonic-gate } 19117c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 19127c478bd9Sstevel@tonic-gate NULL); 19137c478bd9Sstevel@tonic-gate } 19147c478bd9Sstevel@tonic-gate 19157c478bd9Sstevel@tonic-gate /* 19167c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 19177c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 19187c478bd9Sstevel@tonic-gate */ 19197c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 19207c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", pr_reason); 19217c478bd9Sstevel@tonic-gate 19227c478bd9Sstevel@tonic-gate /* 19237c478bd9Sstevel@tonic-gate * If we queued an error and we are going to return from the trap and 19247c478bd9Sstevel@tonic-gate * the error was in user mode or inside of a copy routine, set AST flag 19257c478bd9Sstevel@tonic-gate * so the queue will be drained before returning to user mode. The 19267c478bd9Sstevel@tonic-gate * AST processing will also act on our failure policy. 19277c478bd9Sstevel@tonic-gate */ 19287c478bd9Sstevel@tonic-gate if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 19297c478bd9Sstevel@tonic-gate int pcb_flag = 0; 19307c478bd9Sstevel@tonic-gate 19317c478bd9Sstevel@tonic-gate if (t_afsr_errs & 19327c478bd9Sstevel@tonic-gate (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS & 19337c478bd9Sstevel@tonic-gate ~(C_AFSR_BERR | C_AFSR_TO))) 19347c478bd9Sstevel@tonic-gate pcb_flag |= ASYNC_HWERR; 19357c478bd9Sstevel@tonic-gate 19367c478bd9Sstevel@tonic-gate if (t_afsr & C_AFSR_BERR) 19377c478bd9Sstevel@tonic-gate pcb_flag |= ASYNC_BERR; 19387c478bd9Sstevel@tonic-gate 19397c478bd9Sstevel@tonic-gate if (t_afsr & C_AFSR_TO) 19407c478bd9Sstevel@tonic-gate pcb_flag |= ASYNC_BTO; 19417c478bd9Sstevel@tonic-gate 19427c478bd9Sstevel@tonic-gate ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag; 19437c478bd9Sstevel@tonic-gate aston(curthread); 19447c478bd9Sstevel@tonic-gate } 19457c478bd9Sstevel@tonic-gate } 19467c478bd9Sstevel@tonic-gate 19477c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 19487c478bd9Sstevel@tonic-gate /* 19497c478bd9Sstevel@tonic-gate * Handling of data and instruction parity errors (traps 0x71, 0x72). 19507c478bd9Sstevel@tonic-gate * 19517c478bd9Sstevel@tonic-gate * For Panther, P$ data parity errors during floating point load hits 19527c478bd9Sstevel@tonic-gate * are also detected (reported as TT 0x71) and handled by this trap 19537c478bd9Sstevel@tonic-gate * handler. 19547c478bd9Sstevel@tonic-gate * 19557c478bd9Sstevel@tonic-gate * AFSR/AFAR are not set for parity errors, only TPC (a virtual address) 19567c478bd9Sstevel@tonic-gate * is available. 19577c478bd9Sstevel@tonic-gate */ 19587c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 19597c478bd9Sstevel@tonic-gate void 19607c478bd9Sstevel@tonic-gate cpu_parity_error(struct regs *rp, uint_t flags, caddr_t tpc) 19617c478bd9Sstevel@tonic-gate { 19627c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 19637c478bd9Sstevel@tonic-gate struct async_flt *aflt; 19647c478bd9Sstevel@tonic-gate uchar_t tl = ((flags & CH_ERR_TL) != 0); 19657c478bd9Sstevel@tonic-gate uchar_t iparity = ((flags & CH_ERR_IPE) != 0); 19667c478bd9Sstevel@tonic-gate uchar_t panic = ((flags & CH_ERR_PANIC) != 0); 19677c478bd9Sstevel@tonic-gate char *error_class; 19687c478bd9Sstevel@tonic-gate 19697c478bd9Sstevel@tonic-gate /* 19707c478bd9Sstevel@tonic-gate * Log the error. 19717c478bd9Sstevel@tonic-gate * For icache parity errors the fault address is the trap PC. 19727c478bd9Sstevel@tonic-gate * For dcache/pcache parity errors the instruction would have to 19737c478bd9Sstevel@tonic-gate * be decoded to determine the address and that isn't possible 19747c478bd9Sstevel@tonic-gate * at high PIL. 19757c478bd9Sstevel@tonic-gate */ 19767c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 19777c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 19787c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 19797c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 19807c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 19817c478bd9Sstevel@tonic-gate aflt->flt_pc = tpc; 19827c478bd9Sstevel@tonic-gate aflt->flt_addr = iparity ? (uint64_t)tpc : AFLT_INV_ADDR; 19837c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 19847c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 19857c478bd9Sstevel@tonic-gate aflt->flt_priv = (tl || (rp->r_tstate & TSTATE_PRIV)) ? 1 : 0; 19867c478bd9Sstevel@tonic-gate aflt->flt_tl = tl; 19877c478bd9Sstevel@tonic-gate aflt->flt_panic = panic; 19887c478bd9Sstevel@tonic-gate aflt->flt_status = iparity ? ECC_IP_TRAP : ECC_DP_TRAP; 19897c478bd9Sstevel@tonic-gate ch_flt.flt_type = iparity ? CPU_IC_PARITY : CPU_DC_PARITY; 19907c478bd9Sstevel@tonic-gate 19917c478bd9Sstevel@tonic-gate if (iparity) { 19927c478bd9Sstevel@tonic-gate cpu_icache_parity_info(&ch_flt); 19937c478bd9Sstevel@tonic-gate if (ch_flt.parity_data.ipe.cpl_off != -1) 19947c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_IDSPE; 19957c478bd9Sstevel@tonic-gate else if (ch_flt.parity_data.ipe.cpl_way != -1) 19967c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_ITSPE; 19977c478bd9Sstevel@tonic-gate else 19987c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_IPE; 19997c478bd9Sstevel@tonic-gate aflt->flt_payload = FM_EREPORT_PAYLOAD_ICACHE_PE; 20007c478bd9Sstevel@tonic-gate } else { 20017c478bd9Sstevel@tonic-gate cpu_dcache_parity_info(&ch_flt); 20027c478bd9Sstevel@tonic-gate if (ch_flt.parity_data.dpe.cpl_off != -1) 20037c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_DDSPE; 20047c478bd9Sstevel@tonic-gate else if (ch_flt.parity_data.dpe.cpl_way != -1) 20057c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_DTSPE; 20067c478bd9Sstevel@tonic-gate else 20077c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_DPE; 20087c478bd9Sstevel@tonic-gate aflt->flt_payload = FM_EREPORT_PAYLOAD_DCACHE_PE; 20097c478bd9Sstevel@tonic-gate /* 20107c478bd9Sstevel@tonic-gate * For panther we also need to check the P$ for parity errors. 20117c478bd9Sstevel@tonic-gate */ 20127c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 20137c478bd9Sstevel@tonic-gate cpu_pcache_parity_info(&ch_flt); 20147c478bd9Sstevel@tonic-gate if (ch_flt.parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 20157c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_PDSPE; 20167c478bd9Sstevel@tonic-gate aflt->flt_payload = 20177c478bd9Sstevel@tonic-gate FM_EREPORT_PAYLOAD_PCACHE_PE; 20187c478bd9Sstevel@tonic-gate } 20197c478bd9Sstevel@tonic-gate } 20207c478bd9Sstevel@tonic-gate } 20217c478bd9Sstevel@tonic-gate 20227c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(error_class, (void *)&ch_flt, 20237c478bd9Sstevel@tonic-gate sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 20247c478bd9Sstevel@tonic-gate 20257c478bd9Sstevel@tonic-gate if (iparity) { 20267c478bd9Sstevel@tonic-gate /* 20277c478bd9Sstevel@tonic-gate * Invalidate entire I$. 20287c478bd9Sstevel@tonic-gate * This is required due to the use of diagnostic ASI 20297c478bd9Sstevel@tonic-gate * accesses that may result in a loss of I$ coherency. 20307c478bd9Sstevel@tonic-gate */ 20317c478bd9Sstevel@tonic-gate if (cache_boot_state & DCU_IC) { 20327c478bd9Sstevel@tonic-gate flush_icache(); 20337c478bd9Sstevel@tonic-gate } 20347c478bd9Sstevel@tonic-gate /* 20357c478bd9Sstevel@tonic-gate * According to section P.3.1 of the Panther PRM, we 20367c478bd9Sstevel@tonic-gate * need to do a little more for recovery on those 20377c478bd9Sstevel@tonic-gate * CPUs after encountering an I$ parity error. 20387c478bd9Sstevel@tonic-gate */ 20397c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 20407c478bd9Sstevel@tonic-gate flush_ipb(); 20417c478bd9Sstevel@tonic-gate correct_dcache_parity(dcache_size, 20427c478bd9Sstevel@tonic-gate dcache_linesize); 20437c478bd9Sstevel@tonic-gate flush_pcache(); 20447c478bd9Sstevel@tonic-gate } 20457c478bd9Sstevel@tonic-gate } else { 20467c478bd9Sstevel@tonic-gate /* 20477c478bd9Sstevel@tonic-gate * Since the valid bit is ignored when checking parity the 20487c478bd9Sstevel@tonic-gate * D$ data and tag must also be corrected. Set D$ data bits 20497c478bd9Sstevel@tonic-gate * to zero and set utag to 0, 1, 2, 3. 20507c478bd9Sstevel@tonic-gate */ 20517c478bd9Sstevel@tonic-gate correct_dcache_parity(dcache_size, dcache_linesize); 20527c478bd9Sstevel@tonic-gate 20537c478bd9Sstevel@tonic-gate /* 20547c478bd9Sstevel@tonic-gate * According to section P.3.3 of the Panther PRM, we 20557c478bd9Sstevel@tonic-gate * need to do a little more for recovery on those 20567c478bd9Sstevel@tonic-gate * CPUs after encountering a D$ or P$ parity error. 20577c478bd9Sstevel@tonic-gate * 20587c478bd9Sstevel@tonic-gate * As far as clearing P$ parity errors, it is enough to 20597c478bd9Sstevel@tonic-gate * simply invalidate all entries in the P$ since P$ parity 20607c478bd9Sstevel@tonic-gate * error traps are only generated for floating point load 20617c478bd9Sstevel@tonic-gate * hits. 20627c478bd9Sstevel@tonic-gate */ 20637c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 20647c478bd9Sstevel@tonic-gate flush_icache(); 20657c478bd9Sstevel@tonic-gate flush_ipb(); 20667c478bd9Sstevel@tonic-gate flush_pcache(); 20677c478bd9Sstevel@tonic-gate } 20687c478bd9Sstevel@tonic-gate } 20697c478bd9Sstevel@tonic-gate 20707c478bd9Sstevel@tonic-gate /* 20717c478bd9Sstevel@tonic-gate * Invalidate entire D$ if it was enabled. 20727c478bd9Sstevel@tonic-gate * This is done to avoid stale data in the D$ which might 20737c478bd9Sstevel@tonic-gate * occur with the D$ disabled and the trap handler doing 20747c478bd9Sstevel@tonic-gate * stores affecting lines already in the D$. 20757c478bd9Sstevel@tonic-gate */ 20767c478bd9Sstevel@tonic-gate if (cache_boot_state & DCU_DC) { 20777c478bd9Sstevel@tonic-gate flush_dcache(); 20787c478bd9Sstevel@tonic-gate } 20797c478bd9Sstevel@tonic-gate 20807c478bd9Sstevel@tonic-gate /* 20817c478bd9Sstevel@tonic-gate * Restore caches to their bootup state. 20827c478bd9Sstevel@tonic-gate */ 20837c478bd9Sstevel@tonic-gate set_dcu(get_dcu() | cache_boot_state); 20847c478bd9Sstevel@tonic-gate 20857c478bd9Sstevel@tonic-gate /* 20867c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 20877c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 20887c478bd9Sstevel@tonic-gate */ 20897c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 20907c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", iparity ? "IPE " : "DPE "); 20917c478bd9Sstevel@tonic-gate 20927c478bd9Sstevel@tonic-gate /* 20937c478bd9Sstevel@tonic-gate * If this error occurred at TL>0 then flush the E$ here to reduce 20947c478bd9Sstevel@tonic-gate * the chance of getting an unrecoverable Fast ECC error. This 20957c478bd9Sstevel@tonic-gate * flush will evict the part of the parity trap handler that is run 20967c478bd9Sstevel@tonic-gate * at TL>1. 20977c478bd9Sstevel@tonic-gate */ 20987c478bd9Sstevel@tonic-gate if (tl) { 20997c478bd9Sstevel@tonic-gate cpu_flush_ecache(); 21007c478bd9Sstevel@tonic-gate } 21017c478bd9Sstevel@tonic-gate } 21027c478bd9Sstevel@tonic-gate 21037c478bd9Sstevel@tonic-gate /* 21047c478bd9Sstevel@tonic-gate * On an I$ parity error, mark the appropriate entries in the ch_async_flt_t 21057c478bd9Sstevel@tonic-gate * to indicate which portions of the captured data should be in the ereport. 21067c478bd9Sstevel@tonic-gate */ 21077c478bd9Sstevel@tonic-gate void 21087c478bd9Sstevel@tonic-gate cpu_async_log_ic_parity_err(ch_async_flt_t *ch_flt) 21097c478bd9Sstevel@tonic-gate { 21107c478bd9Sstevel@tonic-gate int way = ch_flt->parity_data.ipe.cpl_way; 21117c478bd9Sstevel@tonic-gate int offset = ch_flt->parity_data.ipe.cpl_off; 21127c478bd9Sstevel@tonic-gate int tag_index; 21137c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 21147c478bd9Sstevel@tonic-gate 21157c478bd9Sstevel@tonic-gate 21167c478bd9Sstevel@tonic-gate if ((offset != -1) || (way != -1)) { 21177c478bd9Sstevel@tonic-gate /* 21187c478bd9Sstevel@tonic-gate * Parity error in I$ tag or data 21197c478bd9Sstevel@tonic-gate */ 21207c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 21217c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 21227c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 21237c478bd9Sstevel@tonic-gate PN_ICIDX_TO_WAY(tag_index); 21247c478bd9Sstevel@tonic-gate else 21257c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 21267c478bd9Sstevel@tonic-gate CH_ICIDX_TO_WAY(tag_index); 21277c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 21287c478bd9Sstevel@tonic-gate IC_LOGFLAG_MAGIC; 21297c478bd9Sstevel@tonic-gate } else { 21307c478bd9Sstevel@tonic-gate /* 21317c478bd9Sstevel@tonic-gate * Parity error was not identified. 21327c478bd9Sstevel@tonic-gate * Log tags and data for all ways. 21337c478bd9Sstevel@tonic-gate */ 21347c478bd9Sstevel@tonic-gate for (way = 0; way < CH_ICACHE_NWAY; way++) { 21357c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 21367c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 21377c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 21387c478bd9Sstevel@tonic-gate PN_ICIDX_TO_WAY(tag_index); 21397c478bd9Sstevel@tonic-gate else 21407c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 21417c478bd9Sstevel@tonic-gate CH_ICIDX_TO_WAY(tag_index); 21427c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 21437c478bd9Sstevel@tonic-gate IC_LOGFLAG_MAGIC; 21447c478bd9Sstevel@tonic-gate } 21457c478bd9Sstevel@tonic-gate } 21467c478bd9Sstevel@tonic-gate } 21477c478bd9Sstevel@tonic-gate 21487c478bd9Sstevel@tonic-gate /* 21497c478bd9Sstevel@tonic-gate * On an D$ parity error, mark the appropriate entries in the ch_async_flt_t 21507c478bd9Sstevel@tonic-gate * to indicate which portions of the captured data should be in the ereport. 21517c478bd9Sstevel@tonic-gate */ 21527c478bd9Sstevel@tonic-gate void 21537c478bd9Sstevel@tonic-gate cpu_async_log_dc_parity_err(ch_async_flt_t *ch_flt) 21547c478bd9Sstevel@tonic-gate { 21557c478bd9Sstevel@tonic-gate int way = ch_flt->parity_data.dpe.cpl_way; 21567c478bd9Sstevel@tonic-gate int offset = ch_flt->parity_data.dpe.cpl_off; 21577c478bd9Sstevel@tonic-gate int tag_index; 21587c478bd9Sstevel@tonic-gate 21597c478bd9Sstevel@tonic-gate if (offset != -1) { 21607c478bd9Sstevel@tonic-gate /* 21617c478bd9Sstevel@tonic-gate * Parity error in D$ or P$ data array. 21627c478bd9Sstevel@tonic-gate * 21637c478bd9Sstevel@tonic-gate * First check to see whether the parity error is in D$ or P$ 21647c478bd9Sstevel@tonic-gate * since P$ data parity errors are reported in Panther using 21657c478bd9Sstevel@tonic-gate * the same trap. 21667c478bd9Sstevel@tonic-gate */ 21677c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 21687c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.dpe.cpl_pc[way].pc_idx; 21697c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_pc[way].pc_way = 21707c478bd9Sstevel@tonic-gate CH_PCIDX_TO_WAY(tag_index); 21717c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_pc[way].pc_logflag = 21727c478bd9Sstevel@tonic-gate PC_LOGFLAG_MAGIC; 21737c478bd9Sstevel@tonic-gate } else { 21747c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 21757c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 21767c478bd9Sstevel@tonic-gate CH_DCIDX_TO_WAY(tag_index); 21777c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 21787c478bd9Sstevel@tonic-gate DC_LOGFLAG_MAGIC; 21797c478bd9Sstevel@tonic-gate } 21807c478bd9Sstevel@tonic-gate } else if (way != -1) { 21817c478bd9Sstevel@tonic-gate /* 21827c478bd9Sstevel@tonic-gate * Parity error in D$ tag. 21837c478bd9Sstevel@tonic-gate */ 21847c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 21857c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 21867c478bd9Sstevel@tonic-gate CH_DCIDX_TO_WAY(tag_index); 21877c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 21887c478bd9Sstevel@tonic-gate DC_LOGFLAG_MAGIC; 21897c478bd9Sstevel@tonic-gate } 21907c478bd9Sstevel@tonic-gate } 21917c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 21927c478bd9Sstevel@tonic-gate 21937c478bd9Sstevel@tonic-gate /* 21947c478bd9Sstevel@tonic-gate * The cpu_async_log_err() function is called via the [uc]e_drain() function to 21957c478bd9Sstevel@tonic-gate * post-process CPU events that are dequeued. As such, it can be invoked 21967c478bd9Sstevel@tonic-gate * from softint context, from AST processing in the trap() flow, or from the 21977c478bd9Sstevel@tonic-gate * panic flow. We decode the CPU-specific data, and take appropriate actions. 21987c478bd9Sstevel@tonic-gate * Historically this entry point was used to log the actual cmn_err(9F) text; 21997c478bd9Sstevel@tonic-gate * now with FMA it is used to prepare 'flt' to be converted into an ereport. 22007c478bd9Sstevel@tonic-gate * With FMA this function now also returns a flag which indicates to the 22017c478bd9Sstevel@tonic-gate * caller whether the ereport should be posted (1) or suppressed (0). 22027c478bd9Sstevel@tonic-gate */ 22037c478bd9Sstevel@tonic-gate static int 22047c478bd9Sstevel@tonic-gate cpu_async_log_err(void *flt, errorq_elem_t *eqep) 22057c478bd9Sstevel@tonic-gate { 22067c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt; 22077c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)flt; 22087c478bd9Sstevel@tonic-gate page_t *pp; 22097c478bd9Sstevel@tonic-gate 22107c478bd9Sstevel@tonic-gate switch (ch_flt->flt_type) { 22117c478bd9Sstevel@tonic-gate case CPU_INV_AFSR: 22127c478bd9Sstevel@tonic-gate /* 22137c478bd9Sstevel@tonic-gate * If it is a disrupting trap and the AFSR is zero, then 22147c478bd9Sstevel@tonic-gate * the event has probably already been noted. Do not post 22157c478bd9Sstevel@tonic-gate * an ereport. 22167c478bd9Sstevel@tonic-gate */ 22177c478bd9Sstevel@tonic-gate if ((aflt->flt_status & ECC_C_TRAP) && 22187c478bd9Sstevel@tonic-gate (!(aflt->flt_stat & C_AFSR_MASK))) 22197c478bd9Sstevel@tonic-gate return (0); 22207c478bd9Sstevel@tonic-gate else 22217c478bd9Sstevel@tonic-gate return (1); 22227c478bd9Sstevel@tonic-gate case CPU_TO: 22237c478bd9Sstevel@tonic-gate case CPU_BERR: 22247c478bd9Sstevel@tonic-gate case CPU_FATAL: 22257c478bd9Sstevel@tonic-gate case CPU_FPUERR: 22267c478bd9Sstevel@tonic-gate return (1); 22277c478bd9Sstevel@tonic-gate 22287c478bd9Sstevel@tonic-gate case CPU_UE_ECACHE_RETIRE: 22297c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 22307c478bd9Sstevel@tonic-gate cpu_page_retire(ch_flt); 22317c478bd9Sstevel@tonic-gate return (1); 22327c478bd9Sstevel@tonic-gate 22337c478bd9Sstevel@tonic-gate /* 22347c478bd9Sstevel@tonic-gate * Cases where we may want to suppress logging or perform 22357c478bd9Sstevel@tonic-gate * extended diagnostics. 22367c478bd9Sstevel@tonic-gate */ 22377c478bd9Sstevel@tonic-gate case CPU_CE: 22387c478bd9Sstevel@tonic-gate case CPU_EMC: 22397c478bd9Sstevel@tonic-gate pp = page_numtopp_nolock((pfn_t) 22407c478bd9Sstevel@tonic-gate (aflt->flt_addr >> MMU_PAGESHIFT)); 22417c478bd9Sstevel@tonic-gate 22427c478bd9Sstevel@tonic-gate /* 22437c478bd9Sstevel@tonic-gate * We want to skip logging and further classification 22447c478bd9Sstevel@tonic-gate * only if ALL the following conditions are true: 22457c478bd9Sstevel@tonic-gate * 22467c478bd9Sstevel@tonic-gate * 1. There is only one error 22477c478bd9Sstevel@tonic-gate * 2. That error is a correctable memory error 22487c478bd9Sstevel@tonic-gate * 3. The error is caused by the memory scrubber (in 22497c478bd9Sstevel@tonic-gate * which case the error will have occurred under 22507c478bd9Sstevel@tonic-gate * on_trap protection) 22517c478bd9Sstevel@tonic-gate * 4. The error is on a retired page 22527c478bd9Sstevel@tonic-gate * 22537c478bd9Sstevel@tonic-gate * Note: AFLT_PROT_EC is used places other than the memory 22547c478bd9Sstevel@tonic-gate * scrubber. However, none of those errors should occur 22557c478bd9Sstevel@tonic-gate * on a retired page. 22567c478bd9Sstevel@tonic-gate */ 22577c478bd9Sstevel@tonic-gate if ((ch_flt->afsr_errs & 22587c478bd9Sstevel@tonic-gate (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_CE && 22597c478bd9Sstevel@tonic-gate aflt->flt_prot == AFLT_PROT_EC) { 22607c478bd9Sstevel@tonic-gate 22617c478bd9Sstevel@tonic-gate if (pp != NULL && page_isretired(pp)) { 22627c478bd9Sstevel@tonic-gate if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 22637c478bd9Sstevel@tonic-gate 22647c478bd9Sstevel@tonic-gate /* 22657c478bd9Sstevel@tonic-gate * Since we're skipping logging, we'll need 22667c478bd9Sstevel@tonic-gate * to schedule the re-enabling of CEEN 22677c478bd9Sstevel@tonic-gate */ 22687c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, 22697c478bd9Sstevel@tonic-gate (void *)aflt->flt_inst, drv_usectohz( 22707c478bd9Sstevel@tonic-gate (clock_t)cpu_ceen_delay_secs * MICROSEC)); 22717c478bd9Sstevel@tonic-gate } 22727c478bd9Sstevel@tonic-gate return (0); 22737c478bd9Sstevel@tonic-gate } 22747c478bd9Sstevel@tonic-gate } 22757c478bd9Sstevel@tonic-gate 22767c478bd9Sstevel@tonic-gate /* 22777c478bd9Sstevel@tonic-gate * Perform/schedule further classification actions, but 22787c478bd9Sstevel@tonic-gate * only if the page is healthy (we don't want bad 22797c478bd9Sstevel@tonic-gate * pages inducing too much diagnostic activity). If we could 22807c478bd9Sstevel@tonic-gate * not find a page pointer then we also skip this. If 22817c478bd9Sstevel@tonic-gate * ce_scrub_xdiag_recirc returns nonzero then it has chosen 22827c478bd9Sstevel@tonic-gate * to copy and recirculate the event (for further diagnostics) 22837c478bd9Sstevel@tonic-gate * and we should not proceed to log it here. 22847c478bd9Sstevel@tonic-gate * 22857c478bd9Sstevel@tonic-gate * This must be the last step here before the cpu_log_err() 22867c478bd9Sstevel@tonic-gate * below - if an event recirculates cpu_ce_log_err() will 22877c478bd9Sstevel@tonic-gate * not call the current function but just proceed directly 22887c478bd9Sstevel@tonic-gate * to cpu_ereport_post after the cpu_log_err() avoided below. 22897c478bd9Sstevel@tonic-gate * 22907c478bd9Sstevel@tonic-gate * Note: Check cpu_impl_async_log_err if changing this 22917c478bd9Sstevel@tonic-gate */ 22927c478bd9Sstevel@tonic-gate if (pp) { 22937c478bd9Sstevel@tonic-gate if (page_isretired(pp) || page_deteriorating(pp)) { 22947c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 22957c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_PAGEDET); 22967c478bd9Sstevel@tonic-gate } else if (ce_scrub_xdiag_recirc(aflt, ce_queue, eqep, 22977c478bd9Sstevel@tonic-gate offsetof(ch_async_flt_t, cmn_asyncflt))) { 22987c478bd9Sstevel@tonic-gate return (0); 22997c478bd9Sstevel@tonic-gate } 23007c478bd9Sstevel@tonic-gate } else { 23017c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 23027c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_NOPP); 23037c478bd9Sstevel@tonic-gate } 23047c478bd9Sstevel@tonic-gate /*FALLTHRU*/ 23057c478bd9Sstevel@tonic-gate 23067c478bd9Sstevel@tonic-gate /* 23077c478bd9Sstevel@tonic-gate * Cases where we just want to report the error and continue. 23087c478bd9Sstevel@tonic-gate */ 23097c478bd9Sstevel@tonic-gate case CPU_CE_ECACHE: 23107c478bd9Sstevel@tonic-gate case CPU_UE_ECACHE: 23117c478bd9Sstevel@tonic-gate case CPU_IV: 23127c478bd9Sstevel@tonic-gate case CPU_ORPH: 23137c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 23147c478bd9Sstevel@tonic-gate return (1); 23157c478bd9Sstevel@tonic-gate 23167c478bd9Sstevel@tonic-gate /* 23177c478bd9Sstevel@tonic-gate * Cases where we want to fall through to handle panicking. 23187c478bd9Sstevel@tonic-gate */ 23197c478bd9Sstevel@tonic-gate case CPU_UE: 23207c478bd9Sstevel@tonic-gate /* 23217c478bd9Sstevel@tonic-gate * We want to skip logging in the same conditions as the 23227c478bd9Sstevel@tonic-gate * CE case. In addition, we want to make sure we're not 23237c478bd9Sstevel@tonic-gate * panicking. 23247c478bd9Sstevel@tonic-gate */ 23257c478bd9Sstevel@tonic-gate if (!panicstr && (ch_flt->afsr_errs & 23267c478bd9Sstevel@tonic-gate (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_UE && 23277c478bd9Sstevel@tonic-gate aflt->flt_prot == AFLT_PROT_EC) { 23287c478bd9Sstevel@tonic-gate page_t *pp = page_numtopp_nolock((pfn_t) 23297c478bd9Sstevel@tonic-gate (aflt->flt_addr >> MMU_PAGESHIFT)); 23307c478bd9Sstevel@tonic-gate 23317c478bd9Sstevel@tonic-gate if (pp != NULL && page_isretired(pp)) { 23327c478bd9Sstevel@tonic-gate 23337c478bd9Sstevel@tonic-gate /* Zero the address to clear the error */ 23347c478bd9Sstevel@tonic-gate softcall(ecc_page_zero, (void *)aflt->flt_addr); 23357c478bd9Sstevel@tonic-gate return (0); 23367c478bd9Sstevel@tonic-gate } 23377c478bd9Sstevel@tonic-gate } 23387c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 23397c478bd9Sstevel@tonic-gate break; 23407c478bd9Sstevel@tonic-gate 23417c478bd9Sstevel@tonic-gate default: 23427c478bd9Sstevel@tonic-gate /* 23437c478bd9Sstevel@tonic-gate * If the us3_common.c code doesn't know the flt_type, it may 23447c478bd9Sstevel@tonic-gate * be an implementation-specific code. Call into the impldep 23457c478bd9Sstevel@tonic-gate * backend to find out what to do: if it tells us to continue, 23467c478bd9Sstevel@tonic-gate * break and handle as if falling through from a UE; if not, 23477c478bd9Sstevel@tonic-gate * the impldep backend has handled the error and we're done. 23487c478bd9Sstevel@tonic-gate */ 23497c478bd9Sstevel@tonic-gate switch (cpu_impl_async_log_err(flt, eqep)) { 23507c478bd9Sstevel@tonic-gate case CH_ASYNC_LOG_DONE: 23517c478bd9Sstevel@tonic-gate return (1); 23527c478bd9Sstevel@tonic-gate case CH_ASYNC_LOG_RECIRC: 23537c478bd9Sstevel@tonic-gate return (0); 23547c478bd9Sstevel@tonic-gate case CH_ASYNC_LOG_CONTINUE: 23557c478bd9Sstevel@tonic-gate break; /* continue on to handle UE-like error */ 23567c478bd9Sstevel@tonic-gate default: 23577c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "discarding error 0x%p with " 23587c478bd9Sstevel@tonic-gate "invalid fault type (0x%x)", 23597c478bd9Sstevel@tonic-gate (void *)aflt, ch_flt->flt_type); 23607c478bd9Sstevel@tonic-gate return (0); 23617c478bd9Sstevel@tonic-gate } 23627c478bd9Sstevel@tonic-gate } 23637c478bd9Sstevel@tonic-gate 23647c478bd9Sstevel@tonic-gate /* ... fall through from the UE case */ 23657c478bd9Sstevel@tonic-gate 23667c478bd9Sstevel@tonic-gate if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) { 23677c478bd9Sstevel@tonic-gate if (!panicstr) { 23687c478bd9Sstevel@tonic-gate cpu_page_retire(ch_flt); 23697c478bd9Sstevel@tonic-gate } else { 23707c478bd9Sstevel@tonic-gate /* 23717c478bd9Sstevel@tonic-gate * Clear UEs on panic so that we don't 23727c478bd9Sstevel@tonic-gate * get haunted by them during panic or 23737c478bd9Sstevel@tonic-gate * after reboot 23747c478bd9Sstevel@tonic-gate */ 23757c478bd9Sstevel@tonic-gate cpu_clearphys(aflt); 23767c478bd9Sstevel@tonic-gate (void) clear_errors(NULL); 23777c478bd9Sstevel@tonic-gate } 23787c478bd9Sstevel@tonic-gate } 23797c478bd9Sstevel@tonic-gate 23807c478bd9Sstevel@tonic-gate return (1); 23817c478bd9Sstevel@tonic-gate } 23827c478bd9Sstevel@tonic-gate 23837c478bd9Sstevel@tonic-gate /* 23847c478bd9Sstevel@tonic-gate * Retire the bad page that may contain the flushed error. 23857c478bd9Sstevel@tonic-gate */ 23867c478bd9Sstevel@tonic-gate void 23877c478bd9Sstevel@tonic-gate cpu_page_retire(ch_async_flt_t *ch_flt) 23887c478bd9Sstevel@tonic-gate { 23897c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 23907c478bd9Sstevel@tonic-gate page_t *pp = page_numtopp_nolock(aflt->flt_addr >> MMU_PAGESHIFT); 23917c478bd9Sstevel@tonic-gate 23927c478bd9Sstevel@tonic-gate if (pp != NULL) { 23937c478bd9Sstevel@tonic-gate page_settoxic(pp, PAGE_IS_FAULTY); 23947c478bd9Sstevel@tonic-gate (void) page_retire(pp, PAGE_IS_TOXIC); 23957c478bd9Sstevel@tonic-gate } 23967c478bd9Sstevel@tonic-gate } 23977c478bd9Sstevel@tonic-gate 23987c478bd9Sstevel@tonic-gate /* 23997c478bd9Sstevel@tonic-gate * The cpu_log_err() function is called by cpu_async_log_err() to perform the 24007c478bd9Sstevel@tonic-gate * generic event post-processing for correctable and uncorrectable memory, 24017c478bd9Sstevel@tonic-gate * E$, and MTag errors. Historically this entry point was used to log bits of 24027c478bd9Sstevel@tonic-gate * common cmn_err(9F) text; now with FMA it is used to prepare 'flt' to be 24037c478bd9Sstevel@tonic-gate * converted into an ereport. In addition, it transmits the error to any 24047c478bd9Sstevel@tonic-gate * platform-specific service-processor FRU logging routines, if available. 24057c478bd9Sstevel@tonic-gate */ 24067c478bd9Sstevel@tonic-gate void 24077c478bd9Sstevel@tonic-gate cpu_log_err(struct async_flt *aflt) 24087c478bd9Sstevel@tonic-gate { 24097c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 24107c478bd9Sstevel@tonic-gate int len = 0; 24117c478bd9Sstevel@tonic-gate int synd_status, synd_code, afar_status; 24127c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 24137c478bd9Sstevel@tonic-gate 24147c478bd9Sstevel@tonic-gate /* 24157c478bd9Sstevel@tonic-gate * Need to turn on ECC_ECACHE for plat_get_mem_unum(). 24167c478bd9Sstevel@tonic-gate * For Panther, L2$ is not external, so we don't want to 24177c478bd9Sstevel@tonic-gate * generate an E$ unum for those errors. 24187c478bd9Sstevel@tonic-gate */ 24197c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 24207c478bd9Sstevel@tonic-gate if (ch_flt->flt_bit & C_AFSR_EXT_L3_ERRS) 24217c478bd9Sstevel@tonic-gate aflt->flt_status |= ECC_ECACHE; 24227c478bd9Sstevel@tonic-gate } else { 24237c478bd9Sstevel@tonic-gate if (ch_flt->flt_bit & C_AFSR_ECACHE) 24247c478bd9Sstevel@tonic-gate aflt->flt_status |= ECC_ECACHE; 24257c478bd9Sstevel@tonic-gate } 24267c478bd9Sstevel@tonic-gate 24277c478bd9Sstevel@tonic-gate /* 24287c478bd9Sstevel@tonic-gate * Determine syndrome status. 24297c478bd9Sstevel@tonic-gate */ 24307c478bd9Sstevel@tonic-gate synd_status = afsr_to_synd_status(aflt->flt_inst, 24317c478bd9Sstevel@tonic-gate ch_flt->afsr_errs, ch_flt->flt_bit); 24327c478bd9Sstevel@tonic-gate 24337c478bd9Sstevel@tonic-gate /* 24347c478bd9Sstevel@tonic-gate * Determine afar status. 24357c478bd9Sstevel@tonic-gate */ 24367c478bd9Sstevel@tonic-gate if (pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)) 24377c478bd9Sstevel@tonic-gate afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 24387c478bd9Sstevel@tonic-gate ch_flt->flt_bit); 24397c478bd9Sstevel@tonic-gate else 24407c478bd9Sstevel@tonic-gate afar_status = AFLT_STAT_INVALID; 24417c478bd9Sstevel@tonic-gate 24427c478bd9Sstevel@tonic-gate /* 24437c478bd9Sstevel@tonic-gate * If afar status is not invalid do a unum lookup. 24447c478bd9Sstevel@tonic-gate */ 24457c478bd9Sstevel@tonic-gate if (afar_status != AFLT_STAT_INVALID) { 24467c478bd9Sstevel@tonic-gate (void) cpu_get_mem_unum_aflt(synd_status, aflt, unum, 24477c478bd9Sstevel@tonic-gate UNUM_NAMLEN, &len); 24487c478bd9Sstevel@tonic-gate } else { 24497c478bd9Sstevel@tonic-gate unum[0] = '\0'; 24507c478bd9Sstevel@tonic-gate } 24517c478bd9Sstevel@tonic-gate 24527c478bd9Sstevel@tonic-gate synd_code = synd_to_synd_code(synd_status, 24537c478bd9Sstevel@tonic-gate aflt->flt_synd, ch_flt->flt_bit); 24547c478bd9Sstevel@tonic-gate 24557c478bd9Sstevel@tonic-gate /* 24567c478bd9Sstevel@tonic-gate * Do not send the fruid message (plat_ecc_error_data_t) 24577c478bd9Sstevel@tonic-gate * to the SC if it can handle the enhanced error information 24587c478bd9Sstevel@tonic-gate * (plat_ecc_error2_data_t) or when the tunable 24597c478bd9Sstevel@tonic-gate * ecc_log_fruid_enable is set to 0. 24607c478bd9Sstevel@tonic-gate */ 24617c478bd9Sstevel@tonic-gate 24627c478bd9Sstevel@tonic-gate if (&plat_ecc_capability_sc_get && 24637c478bd9Sstevel@tonic-gate plat_ecc_capability_sc_get(PLAT_ECC_ERROR_MESSAGE)) { 24647c478bd9Sstevel@tonic-gate if (&plat_log_fruid_error) 24657c478bd9Sstevel@tonic-gate plat_log_fruid_error(synd_code, aflt, unum, 24667c478bd9Sstevel@tonic-gate ch_flt->flt_bit); 24677c478bd9Sstevel@tonic-gate } 24687c478bd9Sstevel@tonic-gate 24697c478bd9Sstevel@tonic-gate if (aflt->flt_func != NULL) 24707c478bd9Sstevel@tonic-gate aflt->flt_func(aflt, unum); 24717c478bd9Sstevel@tonic-gate 24727c478bd9Sstevel@tonic-gate if (afar_status != AFLT_STAT_INVALID) 24737c478bd9Sstevel@tonic-gate cpu_log_diag_info(ch_flt); 24747c478bd9Sstevel@tonic-gate 24757c478bd9Sstevel@tonic-gate /* 24767c478bd9Sstevel@tonic-gate * If we have a CEEN error , we do not reenable CEEN until after 24777c478bd9Sstevel@tonic-gate * we exit the trap handler. Otherwise, another error may 24787c478bd9Sstevel@tonic-gate * occur causing the handler to be entered recursively. 24797c478bd9Sstevel@tonic-gate * We set a timeout to trigger in cpu_ceen_delay_secs seconds, 24807c478bd9Sstevel@tonic-gate * to try and ensure that the CPU makes progress in the face 24817c478bd9Sstevel@tonic-gate * of a CE storm. 24827c478bd9Sstevel@tonic-gate */ 24837c478bd9Sstevel@tonic-gate if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 24847c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, 24857c478bd9Sstevel@tonic-gate (void *)aflt->flt_inst, 24867c478bd9Sstevel@tonic-gate drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 24877c478bd9Sstevel@tonic-gate } 24887c478bd9Sstevel@tonic-gate } 24897c478bd9Sstevel@tonic-gate 24907c478bd9Sstevel@tonic-gate /* 24917c478bd9Sstevel@tonic-gate * Invoked by error_init() early in startup and therefore before 24927c478bd9Sstevel@tonic-gate * startup_errorq() is called to drain any error Q - 24937c478bd9Sstevel@tonic-gate * 24947c478bd9Sstevel@tonic-gate * startup() 24957c478bd9Sstevel@tonic-gate * startup_end() 24967c478bd9Sstevel@tonic-gate * error_init() 24977c478bd9Sstevel@tonic-gate * cpu_error_init() 24987c478bd9Sstevel@tonic-gate * errorq_init() 24997c478bd9Sstevel@tonic-gate * errorq_drain() 25007c478bd9Sstevel@tonic-gate * start_other_cpus() 25017c478bd9Sstevel@tonic-gate * 25027c478bd9Sstevel@tonic-gate * The purpose of this routine is to create error-related taskqs. Taskqs 25037c478bd9Sstevel@tonic-gate * are used for this purpose because cpu_lock can't be grabbed from interrupt 25047c478bd9Sstevel@tonic-gate * context. 25057c478bd9Sstevel@tonic-gate */ 25067c478bd9Sstevel@tonic-gate void 25077c478bd9Sstevel@tonic-gate cpu_error_init(int items) 25087c478bd9Sstevel@tonic-gate { 25097c478bd9Sstevel@tonic-gate /* 25107c478bd9Sstevel@tonic-gate * Create taskq(s) to reenable CE 25117c478bd9Sstevel@tonic-gate */ 25127c478bd9Sstevel@tonic-gate ch_check_ce_tq = taskq_create("cheetah_check_ce", 1, minclsyspri, 25137c478bd9Sstevel@tonic-gate items, items, TASKQ_PREPOPULATE); 25147c478bd9Sstevel@tonic-gate } 25157c478bd9Sstevel@tonic-gate 25167c478bd9Sstevel@tonic-gate void 25177c478bd9Sstevel@tonic-gate cpu_ce_log_err(struct async_flt *aflt, errorq_elem_t *eqep) 25187c478bd9Sstevel@tonic-gate { 25197c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 25207c478bd9Sstevel@tonic-gate int len; 25217c478bd9Sstevel@tonic-gate 25227c478bd9Sstevel@tonic-gate switch (aflt->flt_class) { 25237c478bd9Sstevel@tonic-gate case CPU_FAULT: 25247c478bd9Sstevel@tonic-gate cpu_ereport_init(aflt); 25257c478bd9Sstevel@tonic-gate if (cpu_async_log_err(aflt, eqep)) 25267c478bd9Sstevel@tonic-gate cpu_ereport_post(aflt); 25277c478bd9Sstevel@tonic-gate break; 25287c478bd9Sstevel@tonic-gate 25297c478bd9Sstevel@tonic-gate case BUS_FAULT: 25307c478bd9Sstevel@tonic-gate if (aflt->flt_func != NULL) { 25317c478bd9Sstevel@tonic-gate (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, 25327c478bd9Sstevel@tonic-gate unum, UNUM_NAMLEN, &len); 25337c478bd9Sstevel@tonic-gate aflt->flt_func(aflt, unum); 25347c478bd9Sstevel@tonic-gate } 25357c478bd9Sstevel@tonic-gate break; 25367c478bd9Sstevel@tonic-gate 25377c478bd9Sstevel@tonic-gate case RECIRC_CPU_FAULT: 25387c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 25397c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 25407c478bd9Sstevel@tonic-gate cpu_ereport_post(aflt); 25417c478bd9Sstevel@tonic-gate break; 25427c478bd9Sstevel@tonic-gate 25437c478bd9Sstevel@tonic-gate case RECIRC_BUS_FAULT: 25447c478bd9Sstevel@tonic-gate ASSERT(aflt->flt_class != RECIRC_BUS_FAULT); 25457c478bd9Sstevel@tonic-gate /*FALLTHRU*/ 25467c478bd9Sstevel@tonic-gate default: 25477c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "discarding CE error 0x%p with invalid " 25487c478bd9Sstevel@tonic-gate "fault class (0x%x)", (void *)aflt, aflt->flt_class); 25497c478bd9Sstevel@tonic-gate return; 25507c478bd9Sstevel@tonic-gate } 25517c478bd9Sstevel@tonic-gate } 25527c478bd9Sstevel@tonic-gate 25537c478bd9Sstevel@tonic-gate /* 25547c478bd9Sstevel@tonic-gate * Scrub and classify a CE. This function must not modify the 25557c478bd9Sstevel@tonic-gate * fault structure passed to it but instead should return the classification 25567c478bd9Sstevel@tonic-gate * information. 25577c478bd9Sstevel@tonic-gate */ 25587c478bd9Sstevel@tonic-gate 25597c478bd9Sstevel@tonic-gate static uchar_t 25607c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err_common(struct async_flt *ecc, boolean_t logout_tried) 25617c478bd9Sstevel@tonic-gate { 25627c478bd9Sstevel@tonic-gate uchar_t disp = CE_XDIAG_EXTALG; 25637c478bd9Sstevel@tonic-gate on_trap_data_t otd; 25647c478bd9Sstevel@tonic-gate uint64_t orig_err; 25657c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 25667c478bd9Sstevel@tonic-gate 25677c478bd9Sstevel@tonic-gate /* 25687c478bd9Sstevel@tonic-gate * Clear CEEN. CPU CE TL > 0 trap handling will already have done 25697c478bd9Sstevel@tonic-gate * this, but our other callers have not. Disable preemption to 25707c478bd9Sstevel@tonic-gate * avoid CPU migration so that we restore CEEN on the correct 25717c478bd9Sstevel@tonic-gate * cpu later. 25727c478bd9Sstevel@tonic-gate * 25737c478bd9Sstevel@tonic-gate * CEEN is cleared so that further CEs that our instruction and 25747c478bd9Sstevel@tonic-gate * data footprint induce do not cause use to either creep down 25757c478bd9Sstevel@tonic-gate * kernel stack to the point of overflow, or do so much CE 25767c478bd9Sstevel@tonic-gate * notification as to make little real forward progress. 25777c478bd9Sstevel@tonic-gate * 25787c478bd9Sstevel@tonic-gate * NCEEN must not be cleared. However it is possible that 25797c478bd9Sstevel@tonic-gate * our accesses to the flt_addr may provoke a bus error or timeout 25807c478bd9Sstevel@tonic-gate * if the offending address has just been unconfigured as part of 25817c478bd9Sstevel@tonic-gate * a DR action. So we must operate under on_trap protection. 25827c478bd9Sstevel@tonic-gate */ 25837c478bd9Sstevel@tonic-gate kpreempt_disable(); 25847c478bd9Sstevel@tonic-gate orig_err = get_error_enable(); 25857c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 25867c478bd9Sstevel@tonic-gate set_error_enable(orig_err & ~EN_REG_CEEN); 25877c478bd9Sstevel@tonic-gate 25887c478bd9Sstevel@tonic-gate /* 25897c478bd9Sstevel@tonic-gate * Our classification algorithm includes the line state before 25907c478bd9Sstevel@tonic-gate * the scrub; we'd like this captured after the detection and 25917c478bd9Sstevel@tonic-gate * before the algorithm below - the earlier the better. 25927c478bd9Sstevel@tonic-gate * 25937c478bd9Sstevel@tonic-gate * If we've come from a cpu CE trap then this info already exists 25947c478bd9Sstevel@tonic-gate * in the cpu logout area. 25957c478bd9Sstevel@tonic-gate * 25967c478bd9Sstevel@tonic-gate * For a CE detected by memscrub for which there was no trap 25977c478bd9Sstevel@tonic-gate * (running with CEEN off) cpu_log_and_clear_ce has called 25987c478bd9Sstevel@tonic-gate * cpu_ce_delayed_ec_logout to capture some cache data, and 25997c478bd9Sstevel@tonic-gate * marked the fault structure as incomplete as a flag to later 26007c478bd9Sstevel@tonic-gate * logging code. 26017c478bd9Sstevel@tonic-gate * 26027c478bd9Sstevel@tonic-gate * If called directly from an IO detected CE there has been 26037c478bd9Sstevel@tonic-gate * no line data capture. In this case we logout to the cpu logout 26047c478bd9Sstevel@tonic-gate * area - that's appropriate since it's the cpu cache data we need 26057c478bd9Sstevel@tonic-gate * for classification. We thus borrow the cpu logout area for a 26067c478bd9Sstevel@tonic-gate * short time, and cpu_ce_delayed_ec_logout will mark it as busy in 26077c478bd9Sstevel@tonic-gate * this time (we will invalidate it again below). 26087c478bd9Sstevel@tonic-gate * 26097c478bd9Sstevel@tonic-gate * If called from the partner check xcall handler then this cpu 26107c478bd9Sstevel@tonic-gate * (the partner) has not necessarily experienced a CE at this 26117c478bd9Sstevel@tonic-gate * address. But we want to capture line state before its scrub 26127c478bd9Sstevel@tonic-gate * attempt since we use that in our classification. 26137c478bd9Sstevel@tonic-gate */ 26147c478bd9Sstevel@tonic-gate if (logout_tried == B_FALSE) { 26157c478bd9Sstevel@tonic-gate if (!cpu_ce_delayed_ec_logout(ecc->flt_addr)) 26167c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_NOLOGOUT; 26177c478bd9Sstevel@tonic-gate } 26187c478bd9Sstevel@tonic-gate 26197c478bd9Sstevel@tonic-gate /* 26207c478bd9Sstevel@tonic-gate * Scrub memory, then check AFSR for errors. The AFAR we scrub may 26217c478bd9Sstevel@tonic-gate * no longer be valid (if DR'd since the initial event) so we 26227c478bd9Sstevel@tonic-gate * perform this scrub under on_trap protection. If this access is 26237c478bd9Sstevel@tonic-gate * ok then further accesses below will also be ok - DR cannot 26247c478bd9Sstevel@tonic-gate * proceed while this thread is active (preemption is disabled); 26257c478bd9Sstevel@tonic-gate * to be safe we'll nonetheless use on_trap again below. 26267c478bd9Sstevel@tonic-gate */ 26277c478bd9Sstevel@tonic-gate if (!on_trap(&otd, OT_DATA_ACCESS)) { 26287c478bd9Sstevel@tonic-gate cpu_scrubphys(ecc); 26297c478bd9Sstevel@tonic-gate } else { 26307c478bd9Sstevel@tonic-gate no_trap(); 26317c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 26327c478bd9Sstevel@tonic-gate set_error_enable(orig_err); 26337c478bd9Sstevel@tonic-gate kpreempt_enable(); 26347c478bd9Sstevel@tonic-gate return (disp); 26357c478bd9Sstevel@tonic-gate } 26367c478bd9Sstevel@tonic-gate no_trap(); 26377c478bd9Sstevel@tonic-gate 26387c478bd9Sstevel@tonic-gate /* 26397c478bd9Sstevel@tonic-gate * Did the casx read of the scrub log a CE that matches the AFAR? 26407c478bd9Sstevel@tonic-gate * Note that it's quite possible that the read sourced the data from 26417c478bd9Sstevel@tonic-gate * another cpu. 26427c478bd9Sstevel@tonic-gate */ 26437c478bd9Sstevel@tonic-gate if (clear_ecc(ecc)) 26447c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_CE1; 26457c478bd9Sstevel@tonic-gate 26467c478bd9Sstevel@tonic-gate /* 26477c478bd9Sstevel@tonic-gate * Read the data again. This time the read is very likely to 26487c478bd9Sstevel@tonic-gate * come from memory since the scrub induced a writeback to memory. 26497c478bd9Sstevel@tonic-gate */ 26507c478bd9Sstevel@tonic-gate if (!on_trap(&otd, OT_DATA_ACCESS)) { 26517c478bd9Sstevel@tonic-gate (void) lddphys(P2ALIGN(ecc->flt_addr, 8)); 26527c478bd9Sstevel@tonic-gate } else { 26537c478bd9Sstevel@tonic-gate no_trap(); 26547c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 26557c478bd9Sstevel@tonic-gate set_error_enable(orig_err); 26567c478bd9Sstevel@tonic-gate kpreempt_enable(); 26577c478bd9Sstevel@tonic-gate return (disp); 26587c478bd9Sstevel@tonic-gate } 26597c478bd9Sstevel@tonic-gate no_trap(); 26607c478bd9Sstevel@tonic-gate 26617c478bd9Sstevel@tonic-gate /* Did that read induce a CE that matches the AFAR? */ 26627c478bd9Sstevel@tonic-gate if (clear_ecc(ecc)) 26637c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_CE2; 26647c478bd9Sstevel@tonic-gate 26657c478bd9Sstevel@tonic-gate /* 26667c478bd9Sstevel@tonic-gate * Look at the logout information and record whether we found the 26677c478bd9Sstevel@tonic-gate * line in l2/l3 cache. For Panther we are interested in whether 26687c478bd9Sstevel@tonic-gate * we found it in either cache (it won't reside in both but 26697c478bd9Sstevel@tonic-gate * it is possible to read it that way given the moving target). 26707c478bd9Sstevel@tonic-gate */ 26717c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE(CPU) ? CPU_PRIVATE_PTR(CPU, chpr_cecc_logout) : NULL; 26727c478bd9Sstevel@tonic-gate if (!(disp & CE_XDIAG_NOLOGOUT) && clop && 26737c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar != LOGOUT_INVALID) { 26747c478bd9Sstevel@tonic-gate int hit, level; 26757c478bd9Sstevel@tonic-gate int state; 26767c478bd9Sstevel@tonic-gate int totalsize; 26777c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp; 26787c478bd9Sstevel@tonic-gate 26797c478bd9Sstevel@tonic-gate /* 26807c478bd9Sstevel@tonic-gate * If hit is nonzero then a match was found and hit will 26817c478bd9Sstevel@tonic-gate * be one greater than the index which hit. For Panther we 26827c478bd9Sstevel@tonic-gate * also need to pay attention to level to see which of l2$ or 26837c478bd9Sstevel@tonic-gate * l3$ it hit in. 26847c478bd9Sstevel@tonic-gate */ 26857c478bd9Sstevel@tonic-gate hit = cpu_matching_ecache_line(ecc->flt_addr, &clop->clo_data, 26867c478bd9Sstevel@tonic-gate 0, &level); 26877c478bd9Sstevel@tonic-gate 26887c478bd9Sstevel@tonic-gate if (hit) { 26897c478bd9Sstevel@tonic-gate --hit; 26907c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_AFARMATCH; 26917c478bd9Sstevel@tonic-gate 26927c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 26937c478bd9Sstevel@tonic-gate if (level == 2) 26947c478bd9Sstevel@tonic-gate ecp = &clop->clo_data.chd_l2_data[hit]; 26957c478bd9Sstevel@tonic-gate else 26967c478bd9Sstevel@tonic-gate ecp = &clop->clo_data.chd_ec_data[hit]; 26977c478bd9Sstevel@tonic-gate } else { 26987c478bd9Sstevel@tonic-gate ASSERT(level == 2); 26997c478bd9Sstevel@tonic-gate ecp = &clop->clo_data.chd_ec_data[hit]; 27007c478bd9Sstevel@tonic-gate } 27017c478bd9Sstevel@tonic-gate totalsize = cpunodes[CPU->cpu_id].ecache_size; 27027c478bd9Sstevel@tonic-gate state = cpu_ectag_pa_to_subblk_state(totalsize, 27037c478bd9Sstevel@tonic-gate ecc->flt_addr, ecp->ec_tag); 27047c478bd9Sstevel@tonic-gate 27057c478bd9Sstevel@tonic-gate /* 27067c478bd9Sstevel@tonic-gate * Cheetah variants use different state encodings - 27077c478bd9Sstevel@tonic-gate * the CH_ECSTATE_* defines vary depending on the 27087c478bd9Sstevel@tonic-gate * module we're compiled for. Translate into our 27097c478bd9Sstevel@tonic-gate * one true version. Conflate Owner-Shared state 27107c478bd9Sstevel@tonic-gate * of SSM mode with Owner as victimisation of such 27117c478bd9Sstevel@tonic-gate * lines may cause a writeback. 27127c478bd9Sstevel@tonic-gate */ 27137c478bd9Sstevel@tonic-gate switch (state) { 27147c478bd9Sstevel@tonic-gate case CH_ECSTATE_MOD: 27157c478bd9Sstevel@tonic-gate disp |= EC_STATE_M; 27167c478bd9Sstevel@tonic-gate break; 27177c478bd9Sstevel@tonic-gate 27187c478bd9Sstevel@tonic-gate case CH_ECSTATE_OWN: 27197c478bd9Sstevel@tonic-gate case CH_ECSTATE_OWS: 27207c478bd9Sstevel@tonic-gate disp |= EC_STATE_O; 27217c478bd9Sstevel@tonic-gate break; 27227c478bd9Sstevel@tonic-gate 27237c478bd9Sstevel@tonic-gate case CH_ECSTATE_EXL: 27247c478bd9Sstevel@tonic-gate disp |= EC_STATE_E; 27257c478bd9Sstevel@tonic-gate break; 27267c478bd9Sstevel@tonic-gate 27277c478bd9Sstevel@tonic-gate case CH_ECSTATE_SHR: 27287c478bd9Sstevel@tonic-gate disp |= EC_STATE_S; 27297c478bd9Sstevel@tonic-gate break; 27307c478bd9Sstevel@tonic-gate 27317c478bd9Sstevel@tonic-gate default: 27327c478bd9Sstevel@tonic-gate disp |= EC_STATE_I; 27337c478bd9Sstevel@tonic-gate break; 27347c478bd9Sstevel@tonic-gate } 27357c478bd9Sstevel@tonic-gate } 27367c478bd9Sstevel@tonic-gate 27377c478bd9Sstevel@tonic-gate /* 27387c478bd9Sstevel@tonic-gate * If we initiated the delayed logout then we are responsible 27397c478bd9Sstevel@tonic-gate * for invalidating the logout area. 27407c478bd9Sstevel@tonic-gate */ 27417c478bd9Sstevel@tonic-gate if (logout_tried == B_FALSE) { 27427c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 27437c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 27447c478bd9Sstevel@tonic-gate } 27457c478bd9Sstevel@tonic-gate } 27467c478bd9Sstevel@tonic-gate 27477c478bd9Sstevel@tonic-gate /* 27487c478bd9Sstevel@tonic-gate * Re-enable CEEN if we turned it off. 27497c478bd9Sstevel@tonic-gate */ 27507c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 27517c478bd9Sstevel@tonic-gate set_error_enable(orig_err); 27527c478bd9Sstevel@tonic-gate kpreempt_enable(); 27537c478bd9Sstevel@tonic-gate 27547c478bd9Sstevel@tonic-gate return (disp); 27557c478bd9Sstevel@tonic-gate } 27567c478bd9Sstevel@tonic-gate 27577c478bd9Sstevel@tonic-gate /* 27587c478bd9Sstevel@tonic-gate * Scrub a correctable memory error and collect data for classification 27597c478bd9Sstevel@tonic-gate * of CE type. This function is called in the detection path, ie tl0 handling 27607c478bd9Sstevel@tonic-gate * of a correctable error trap (cpus) or interrupt (IO) at high PIL. 27617c478bd9Sstevel@tonic-gate */ 27627c478bd9Sstevel@tonic-gate void 27637c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t logout_tried) 27647c478bd9Sstevel@tonic-gate { 27657c478bd9Sstevel@tonic-gate /* 27667c478bd9Sstevel@tonic-gate * Cheetah CE classification does not set any bits in flt_status. 27677c478bd9Sstevel@tonic-gate * Instead we will record classification datapoints in flt_disp. 27687c478bd9Sstevel@tonic-gate */ 27697c478bd9Sstevel@tonic-gate ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY); 27707c478bd9Sstevel@tonic-gate 27717c478bd9Sstevel@tonic-gate /* 27727c478bd9Sstevel@tonic-gate * To check if the error detected by IO is persistent, sticky or 27737c478bd9Sstevel@tonic-gate * intermittent. This is noticed by clear_ecc(). 27747c478bd9Sstevel@tonic-gate */ 27757c478bd9Sstevel@tonic-gate if (ecc->flt_status & ECC_IOBUS) 27767c478bd9Sstevel@tonic-gate ecc->flt_stat = C_AFSR_MEMORY; 27777c478bd9Sstevel@tonic-gate 27787c478bd9Sstevel@tonic-gate /* 27797c478bd9Sstevel@tonic-gate * Record information from this first part of the algorithm in 27807c478bd9Sstevel@tonic-gate * flt_disp. 27817c478bd9Sstevel@tonic-gate */ 27827c478bd9Sstevel@tonic-gate ecc->flt_disp = cpu_ce_scrub_mem_err_common(ecc, logout_tried); 27837c478bd9Sstevel@tonic-gate } 27847c478bd9Sstevel@tonic-gate 27857c478bd9Sstevel@tonic-gate /* 27867c478bd9Sstevel@tonic-gate * Select a partner to perform a further CE classification check from. 27877c478bd9Sstevel@tonic-gate * Must be called with kernel preemption disabled (to stop the cpu list 27887c478bd9Sstevel@tonic-gate * from changing). The detecting cpu we are partnering has cpuid 27897c478bd9Sstevel@tonic-gate * aflt->flt_inst; we might not be running on the detecting cpu. 27907c478bd9Sstevel@tonic-gate * 27917c478bd9Sstevel@tonic-gate * Restrict choice to active cpus in the same cpu partition as ourselves in 27927c478bd9Sstevel@tonic-gate * an effort to stop bad cpus in one partition causing other partitions to 27937c478bd9Sstevel@tonic-gate * perform excessive diagnostic activity. Actually since the errorq drain 27947c478bd9Sstevel@tonic-gate * is run from a softint most of the time and that is a global mechanism 27957c478bd9Sstevel@tonic-gate * this isolation is only partial. Return NULL if we fail to find a 27967c478bd9Sstevel@tonic-gate * suitable partner. 27977c478bd9Sstevel@tonic-gate * 27987c478bd9Sstevel@tonic-gate * We prefer a partner that is in a different latency group to ourselves as 27997c478bd9Sstevel@tonic-gate * we will share fewer datapaths. If such a partner is unavailable then 28007c478bd9Sstevel@tonic-gate * choose one in the same lgroup but prefer a different chip and only allow 28017c478bd9Sstevel@tonic-gate * a sibling core if flags includes PTNR_SIBLINGOK. If all else fails and 28027c478bd9Sstevel@tonic-gate * flags includes PTNR_SELFOK then permit selection of the original detector. 28037c478bd9Sstevel@tonic-gate * 28047c478bd9Sstevel@tonic-gate * We keep a cache of the last partner selected for a cpu, and we'll try to 28057c478bd9Sstevel@tonic-gate * use that previous partner if no more than cpu_ce_ptnr_cachetime_sec seconds 28067c478bd9Sstevel@tonic-gate * have passed since that selection was made. This provides the benefit 28077c478bd9Sstevel@tonic-gate * of the point-of-view of different partners over time but without 28087c478bd9Sstevel@tonic-gate * requiring frequent cpu list traversals. 28097c478bd9Sstevel@tonic-gate */ 28107c478bd9Sstevel@tonic-gate 28117c478bd9Sstevel@tonic-gate #define PTNR_SIBLINGOK 0x1 /* Allow selection of sibling core */ 28127c478bd9Sstevel@tonic-gate #define PTNR_SELFOK 0x2 /* Allow selection of cpu to "partner" itself */ 28137c478bd9Sstevel@tonic-gate 28147c478bd9Sstevel@tonic-gate static cpu_t * 28157c478bd9Sstevel@tonic-gate ce_ptnr_select(struct async_flt *aflt, int flags, int *typep) 28167c478bd9Sstevel@tonic-gate { 28177c478bd9Sstevel@tonic-gate cpu_t *sp, *dtcr, *ptnr, *locptnr, *sibptnr; 28187c478bd9Sstevel@tonic-gate hrtime_t lasttime, thistime; 28197c478bd9Sstevel@tonic-gate 28207c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt > 0 || getpil() >= DISP_LEVEL); 28217c478bd9Sstevel@tonic-gate 28227c478bd9Sstevel@tonic-gate dtcr = cpu[aflt->flt_inst]; 28237c478bd9Sstevel@tonic-gate 28247c478bd9Sstevel@tonic-gate /* 28257c478bd9Sstevel@tonic-gate * Short-circuit for the following cases: 28267c478bd9Sstevel@tonic-gate * . the dtcr is not flagged active 28277c478bd9Sstevel@tonic-gate * . there is just one cpu present 28287c478bd9Sstevel@tonic-gate * . the detector has disappeared 28297c478bd9Sstevel@tonic-gate * . we were given a bad flt_inst cpuid; this should not happen 28307c478bd9Sstevel@tonic-gate * (eg PCI code now fills flt_inst) but if it does it is no 28317c478bd9Sstevel@tonic-gate * reason to panic. 28327c478bd9Sstevel@tonic-gate * . there is just one cpu left online in the cpu partition 28337c478bd9Sstevel@tonic-gate * 28347c478bd9Sstevel@tonic-gate * If we return NULL after this point then we do not update the 28357c478bd9Sstevel@tonic-gate * chpr_ceptnr_seltime which will cause us to perform a full lookup 28367c478bd9Sstevel@tonic-gate * again next time; this is the case where the only other cpu online 28377c478bd9Sstevel@tonic-gate * in the detector's partition is on the same chip as the detector 28387c478bd9Sstevel@tonic-gate * and since CEEN re-enable is throttled even that case should not 28397c478bd9Sstevel@tonic-gate * hurt performance. 28407c478bd9Sstevel@tonic-gate */ 28417c478bd9Sstevel@tonic-gate if (dtcr == NULL || !cpu_flagged_active(dtcr->cpu_flags)) { 28427c478bd9Sstevel@tonic-gate return (NULL); 28437c478bd9Sstevel@tonic-gate } 28447c478bd9Sstevel@tonic-gate if (ncpus == 1 || dtcr->cpu_part->cp_ncpus == 1) { 28457c478bd9Sstevel@tonic-gate if (flags & PTNR_SELFOK) { 28467c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SELF; 28477c478bd9Sstevel@tonic-gate return (dtcr); 28487c478bd9Sstevel@tonic-gate } else { 28497c478bd9Sstevel@tonic-gate return (NULL); 28507c478bd9Sstevel@tonic-gate } 28517c478bd9Sstevel@tonic-gate } 28527c478bd9Sstevel@tonic-gate 28537c478bd9Sstevel@tonic-gate thistime = gethrtime(); 28547c478bd9Sstevel@tonic-gate lasttime = CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime); 28557c478bd9Sstevel@tonic-gate 28567c478bd9Sstevel@tonic-gate /* 28577c478bd9Sstevel@tonic-gate * Select a starting point. 28587c478bd9Sstevel@tonic-gate */ 28597c478bd9Sstevel@tonic-gate if (!lasttime) { 28607c478bd9Sstevel@tonic-gate /* 28617c478bd9Sstevel@tonic-gate * We've never selected a partner for this detector before. 28627c478bd9Sstevel@tonic-gate * Start the scan at the next online cpu in the same cpu 28637c478bd9Sstevel@tonic-gate * partition. 28647c478bd9Sstevel@tonic-gate */ 28657c478bd9Sstevel@tonic-gate sp = dtcr->cpu_next_part; 28667c478bd9Sstevel@tonic-gate } else if (thistime - lasttime < cpu_ce_ptnr_cachetime_sec * NANOSEC) { 28677c478bd9Sstevel@tonic-gate /* 28687c478bd9Sstevel@tonic-gate * Our last selection has not aged yet. If this partner: 28697c478bd9Sstevel@tonic-gate * . is still a valid cpu, 28707c478bd9Sstevel@tonic-gate * . is still in the same partition as the detector 28717c478bd9Sstevel@tonic-gate * . is still marked active 28727c478bd9Sstevel@tonic-gate * . satisfies the 'flags' argument criteria 28737c478bd9Sstevel@tonic-gate * then select it again without updating the timestamp. 28747c478bd9Sstevel@tonic-gate */ 28757c478bd9Sstevel@tonic-gate sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 28767c478bd9Sstevel@tonic-gate if (sp == NULL || sp->cpu_part != dtcr->cpu_part || 28777c478bd9Sstevel@tonic-gate !cpu_flagged_active(sp->cpu_flags) || 28787c478bd9Sstevel@tonic-gate (sp == dtcr && !(flags & PTNR_SELFOK)) || 28797c478bd9Sstevel@tonic-gate (sp->cpu_chip->chip_id == dtcr->cpu_chip->chip_id && 28807c478bd9Sstevel@tonic-gate !(flags & PTNR_SIBLINGOK))) { 28817c478bd9Sstevel@tonic-gate sp = dtcr->cpu_next_part; 28827c478bd9Sstevel@tonic-gate } else { 28837c478bd9Sstevel@tonic-gate if (sp->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 28847c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_REMOTE; 28857c478bd9Sstevel@tonic-gate } else if (sp == dtcr) { 28867c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SELF; 28877c478bd9Sstevel@tonic-gate } else if (sp->cpu_chip->chip_id == 28887c478bd9Sstevel@tonic-gate dtcr->cpu_chip->chip_id) { 28897c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SIBLING; 28907c478bd9Sstevel@tonic-gate } else { 28917c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_LOCAL; 28927c478bd9Sstevel@tonic-gate } 28937c478bd9Sstevel@tonic-gate return (sp); 28947c478bd9Sstevel@tonic-gate } 28957c478bd9Sstevel@tonic-gate } else { 28967c478bd9Sstevel@tonic-gate /* 28977c478bd9Sstevel@tonic-gate * Our last selection has aged. If it is nonetheless still a 28987c478bd9Sstevel@tonic-gate * valid cpu then start the scan at the next cpu in the 28997c478bd9Sstevel@tonic-gate * partition after our last partner. If the last selection 29007c478bd9Sstevel@tonic-gate * is no longer a valid cpu then go with our default. In 29017c478bd9Sstevel@tonic-gate * this way we slowly cycle through possible partners to 29027c478bd9Sstevel@tonic-gate * obtain multiple viewpoints over time. 29037c478bd9Sstevel@tonic-gate */ 29047c478bd9Sstevel@tonic-gate sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 29057c478bd9Sstevel@tonic-gate if (sp == NULL) { 29067c478bd9Sstevel@tonic-gate sp = dtcr->cpu_next_part; 29077c478bd9Sstevel@tonic-gate } else { 29087c478bd9Sstevel@tonic-gate sp = sp->cpu_next_part; /* may be dtcr */ 29097c478bd9Sstevel@tonic-gate if (sp->cpu_part != dtcr->cpu_part) 29107c478bd9Sstevel@tonic-gate sp = dtcr; 29117c478bd9Sstevel@tonic-gate } 29127c478bd9Sstevel@tonic-gate } 29137c478bd9Sstevel@tonic-gate 29147c478bd9Sstevel@tonic-gate /* 29157c478bd9Sstevel@tonic-gate * We have a proposed starting point for our search, but if this 29167c478bd9Sstevel@tonic-gate * cpu is offline then its cpu_next_part will point to itself 29177c478bd9Sstevel@tonic-gate * so we can't use that to iterate over cpus in this partition in 29187c478bd9Sstevel@tonic-gate * the loop below. We still want to avoid iterating over cpus not 29197c478bd9Sstevel@tonic-gate * in our partition, so in the case that our starting point is offline 29207c478bd9Sstevel@tonic-gate * we will repoint it to be the detector itself; and if the detector 29217c478bd9Sstevel@tonic-gate * happens to be offline we'll return NULL from the following loop. 29227c478bd9Sstevel@tonic-gate */ 29237c478bd9Sstevel@tonic-gate if (!cpu_flagged_active(sp->cpu_flags)) { 29247c478bd9Sstevel@tonic-gate sp = dtcr; 29257c478bd9Sstevel@tonic-gate } 29267c478bd9Sstevel@tonic-gate 29277c478bd9Sstevel@tonic-gate ptnr = sp; 29287c478bd9Sstevel@tonic-gate locptnr = NULL; 29297c478bd9Sstevel@tonic-gate sibptnr = NULL; 29307c478bd9Sstevel@tonic-gate do { 29317c478bd9Sstevel@tonic-gate if (ptnr == dtcr || !cpu_flagged_active(ptnr->cpu_flags)) 29327c478bd9Sstevel@tonic-gate continue; 29337c478bd9Sstevel@tonic-gate if (ptnr->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 29347c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = ptnr->cpu_id; 29357c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 29367c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_REMOTE; 29377c478bd9Sstevel@tonic-gate return (ptnr); 29387c478bd9Sstevel@tonic-gate } 29397c478bd9Sstevel@tonic-gate if (ptnr->cpu_chip->chip_id == dtcr->cpu_chip->chip_id) { 29407c478bd9Sstevel@tonic-gate if (sibptnr == NULL) 29417c478bd9Sstevel@tonic-gate sibptnr = ptnr; 29427c478bd9Sstevel@tonic-gate continue; 29437c478bd9Sstevel@tonic-gate } 29447c478bd9Sstevel@tonic-gate if (locptnr == NULL) 29457c478bd9Sstevel@tonic-gate locptnr = ptnr; 29467c478bd9Sstevel@tonic-gate } while ((ptnr = ptnr->cpu_next_part) != sp); 29477c478bd9Sstevel@tonic-gate 29487c478bd9Sstevel@tonic-gate /* 29497c478bd9Sstevel@tonic-gate * A foreign partner has already been returned if one was available. 29507c478bd9Sstevel@tonic-gate * 29517c478bd9Sstevel@tonic-gate * If locptnr is not NULL it is a cpu in the same lgroup as the 29527c478bd9Sstevel@tonic-gate * detector, is active, and is not a sibling of the detector. 29537c478bd9Sstevel@tonic-gate * 29547c478bd9Sstevel@tonic-gate * If sibptnr is not NULL it is a sibling of the detector, and is 29557c478bd9Sstevel@tonic-gate * active. 29567c478bd9Sstevel@tonic-gate * 29577c478bd9Sstevel@tonic-gate * If we have to resort to using the detector itself we have already 29587c478bd9Sstevel@tonic-gate * checked that it is active. 29597c478bd9Sstevel@tonic-gate */ 29607c478bd9Sstevel@tonic-gate if (locptnr) { 29617c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = locptnr->cpu_id; 29627c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 29637c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_LOCAL; 29647c478bd9Sstevel@tonic-gate return (locptnr); 29657c478bd9Sstevel@tonic-gate } else if (sibptnr && flags & PTNR_SIBLINGOK) { 29667c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = sibptnr->cpu_id; 29677c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 29687c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SIBLING; 29697c478bd9Sstevel@tonic-gate return (sibptnr); 29707c478bd9Sstevel@tonic-gate } else if (flags & PTNR_SELFOK) { 29717c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = dtcr->cpu_id; 29727c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 29737c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SELF; 29747c478bd9Sstevel@tonic-gate return (dtcr); 29757c478bd9Sstevel@tonic-gate } 29767c478bd9Sstevel@tonic-gate 29777c478bd9Sstevel@tonic-gate return (NULL); 29787c478bd9Sstevel@tonic-gate } 29797c478bd9Sstevel@tonic-gate 29807c478bd9Sstevel@tonic-gate /* 29817c478bd9Sstevel@tonic-gate * Cross call handler that is requested to run on the designated partner of 29827c478bd9Sstevel@tonic-gate * a cpu that experienced a possibly sticky or possibly persistnet CE. 29837c478bd9Sstevel@tonic-gate */ 29847c478bd9Sstevel@tonic-gate static void 29857c478bd9Sstevel@tonic-gate ce_ptnrchk_xc(struct async_flt *aflt, uchar_t *dispp) 29867c478bd9Sstevel@tonic-gate { 29877c478bd9Sstevel@tonic-gate *dispp = cpu_ce_scrub_mem_err_common(aflt, B_FALSE); 29887c478bd9Sstevel@tonic-gate } 29897c478bd9Sstevel@tonic-gate 29907c478bd9Sstevel@tonic-gate /* 29917c478bd9Sstevel@tonic-gate * The associated errorqs are never destroyed so we do not need to deal with 29927c478bd9Sstevel@tonic-gate * them disappearing before this timeout fires. If the affected memory 29937c478bd9Sstevel@tonic-gate * has been DR'd out since the original event the scrub algrithm will catch 29947c478bd9Sstevel@tonic-gate * any errors and return null disposition info. If the original detecting 29957c478bd9Sstevel@tonic-gate * cpu has been DR'd out then ereport detector info will not be able to 29967c478bd9Sstevel@tonic-gate * lookup CPU type; with a small timeout this is unlikely. 29977c478bd9Sstevel@tonic-gate */ 29987c478bd9Sstevel@tonic-gate static void 29997c478bd9Sstevel@tonic-gate ce_lkychk_cb(ce_lkychk_cb_t *cbarg) 30007c478bd9Sstevel@tonic-gate { 30017c478bd9Sstevel@tonic-gate struct async_flt *aflt = cbarg->lkycb_aflt; 30027c478bd9Sstevel@tonic-gate uchar_t disp; 30037c478bd9Sstevel@tonic-gate cpu_t *cp; 30047c478bd9Sstevel@tonic-gate int ptnrtype; 30057c478bd9Sstevel@tonic-gate 30067c478bd9Sstevel@tonic-gate kpreempt_disable(); 30077c478bd9Sstevel@tonic-gate if (cp = ce_ptnr_select(aflt, PTNR_SIBLINGOK | PTNR_SELFOK, 30087c478bd9Sstevel@tonic-gate &ptnrtype)) { 30097c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, (uint64_t)aflt, 30107c478bd9Sstevel@tonic-gate (uint64_t)&disp); 30117c478bd9Sstevel@tonic-gate CE_XDIAG_SETLKYINFO(aflt->flt_disp, disp); 30127c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 30137c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 30147c478bd9Sstevel@tonic-gate } else { 30157c478bd9Sstevel@tonic-gate ce_xdiag_lkydrops++; 30167c478bd9Sstevel@tonic-gate if (ncpus > 1) 30177c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 30187c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_NOPTNR); 30197c478bd9Sstevel@tonic-gate } 30207c478bd9Sstevel@tonic-gate kpreempt_enable(); 30217c478bd9Sstevel@tonic-gate 30227c478bd9Sstevel@tonic-gate errorq_commit(cbarg->lkycb_eqp, cbarg->lkycb_eqep, ERRORQ_ASYNC); 30237c478bd9Sstevel@tonic-gate kmem_free(cbarg, sizeof (ce_lkychk_cb_t)); 30247c478bd9Sstevel@tonic-gate } 30257c478bd9Sstevel@tonic-gate 30267c478bd9Sstevel@tonic-gate /* 30277c478bd9Sstevel@tonic-gate * Called from errorq drain code when processing a CE error, both from 30287c478bd9Sstevel@tonic-gate * CPU and PCI drain functions. Decide what further classification actions, 30297c478bd9Sstevel@tonic-gate * if any, we will perform. Perform immediate actions now, and schedule 30307c478bd9Sstevel@tonic-gate * delayed actions as required. Note that we are no longer necessarily running 30317c478bd9Sstevel@tonic-gate * on the detecting cpu, and that the async_flt structure will not persist on 30327c478bd9Sstevel@tonic-gate * return from this function. 30337c478bd9Sstevel@tonic-gate * 30347c478bd9Sstevel@tonic-gate * Calls to this function should aim to be self-throtlling in some way. With 30357c478bd9Sstevel@tonic-gate * the delayed re-enable of CEEN the absolute rate of calls should not 30367c478bd9Sstevel@tonic-gate * be excessive. Callers should also avoid performing in-depth classification 30377c478bd9Sstevel@tonic-gate * for events in pages that are already known to be suspect. 30387c478bd9Sstevel@tonic-gate * 30397c478bd9Sstevel@tonic-gate * We return nonzero to indicate that the event has been copied and 30407c478bd9Sstevel@tonic-gate * recirculated for further testing. The caller should not log the event 30417c478bd9Sstevel@tonic-gate * in this case - it will be logged when further test results are available. 30427c478bd9Sstevel@tonic-gate * 30437c478bd9Sstevel@tonic-gate * Our possible contexts are that of errorq_drain: below lock level or from 30447c478bd9Sstevel@tonic-gate * panic context. We can assume that the cpu we are running on is online. 30457c478bd9Sstevel@tonic-gate */ 30467c478bd9Sstevel@tonic-gate 30477c478bd9Sstevel@tonic-gate 30487c478bd9Sstevel@tonic-gate #ifdef DEBUG 30497c478bd9Sstevel@tonic-gate static int ce_xdiag_forceaction; 30507c478bd9Sstevel@tonic-gate #endif 30517c478bd9Sstevel@tonic-gate 30527c478bd9Sstevel@tonic-gate int 30537c478bd9Sstevel@tonic-gate ce_scrub_xdiag_recirc(struct async_flt *aflt, errorq_t *eqp, 30547c478bd9Sstevel@tonic-gate errorq_elem_t *eqep, size_t afltoffset) 30557c478bd9Sstevel@tonic-gate { 30567c478bd9Sstevel@tonic-gate ce_dispact_t dispact, action; 30577c478bd9Sstevel@tonic-gate cpu_t *cp; 30587c478bd9Sstevel@tonic-gate uchar_t dtcrinfo, disp; 30597c478bd9Sstevel@tonic-gate int ptnrtype; 30607c478bd9Sstevel@tonic-gate 30617c478bd9Sstevel@tonic-gate if (!ce_disp_inited || panicstr || ce_xdiag_off) { 30627c478bd9Sstevel@tonic-gate ce_xdiag_drops++; 30637c478bd9Sstevel@tonic-gate return (0); 30647c478bd9Sstevel@tonic-gate } else if (!aflt->flt_in_memory) { 30657c478bd9Sstevel@tonic-gate ce_xdiag_drops++; 30667c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOTMEM); 30677c478bd9Sstevel@tonic-gate return (0); 30687c478bd9Sstevel@tonic-gate } 30697c478bd9Sstevel@tonic-gate 30707c478bd9Sstevel@tonic-gate dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 30717c478bd9Sstevel@tonic-gate 30727c478bd9Sstevel@tonic-gate /* 30737c478bd9Sstevel@tonic-gate * Some correctable events are not scrubbed/classified, such as those 30747c478bd9Sstevel@tonic-gate * noticed at the tail of cpu_deferred_error. So if there is no 30757c478bd9Sstevel@tonic-gate * initial detector classification go no further. 30767c478bd9Sstevel@tonic-gate */ 30777c478bd9Sstevel@tonic-gate if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) { 30787c478bd9Sstevel@tonic-gate ce_xdiag_drops++; 30797c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOSCRUB); 30807c478bd9Sstevel@tonic-gate return (0); 30817c478bd9Sstevel@tonic-gate } 30827c478bd9Sstevel@tonic-gate 30837c478bd9Sstevel@tonic-gate dispact = CE_DISPACT(ce_disp_table, 30847c478bd9Sstevel@tonic-gate CE_XDIAG_AFARMATCHED(dtcrinfo), 30857c478bd9Sstevel@tonic-gate CE_XDIAG_STATE(dtcrinfo), 30867c478bd9Sstevel@tonic-gate CE_XDIAG_CE1SEEN(dtcrinfo), 30877c478bd9Sstevel@tonic-gate CE_XDIAG_CE2SEEN(dtcrinfo)); 30887c478bd9Sstevel@tonic-gate 30897c478bd9Sstevel@tonic-gate 30907c478bd9Sstevel@tonic-gate action = CE_ACT(dispact); /* bad lookup caught below */ 30917c478bd9Sstevel@tonic-gate #ifdef DEBUG 30927c478bd9Sstevel@tonic-gate if (ce_xdiag_forceaction != 0) 30937c478bd9Sstevel@tonic-gate action = ce_xdiag_forceaction; 30947c478bd9Sstevel@tonic-gate #endif 30957c478bd9Sstevel@tonic-gate 30967c478bd9Sstevel@tonic-gate switch (action) { 30977c478bd9Sstevel@tonic-gate case CE_ACT_LKYCHK: { 30987c478bd9Sstevel@tonic-gate caddr_t ndata; 30997c478bd9Sstevel@tonic-gate errorq_elem_t *neqep; 31007c478bd9Sstevel@tonic-gate struct async_flt *ecc; 31017c478bd9Sstevel@tonic-gate ce_lkychk_cb_t *cbargp; 31027c478bd9Sstevel@tonic-gate 31037c478bd9Sstevel@tonic-gate if ((ndata = errorq_elem_dup(eqp, eqep, &neqep)) == NULL) { 31047c478bd9Sstevel@tonic-gate ce_xdiag_lkydrops++; 31057c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 31067c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_DUPFAIL); 31077c478bd9Sstevel@tonic-gate break; 31087c478bd9Sstevel@tonic-gate } 31097c478bd9Sstevel@tonic-gate ecc = (struct async_flt *)(ndata + afltoffset); 31107c478bd9Sstevel@tonic-gate 31117c478bd9Sstevel@tonic-gate ASSERT(ecc->flt_class == CPU_FAULT || 31127c478bd9Sstevel@tonic-gate ecc->flt_class == BUS_FAULT); 31137c478bd9Sstevel@tonic-gate ecc->flt_class = (ecc->flt_class == CPU_FAULT) ? 31147c478bd9Sstevel@tonic-gate RECIRC_CPU_FAULT : RECIRC_BUS_FAULT; 31157c478bd9Sstevel@tonic-gate 31167c478bd9Sstevel@tonic-gate cbargp = kmem_alloc(sizeof (ce_lkychk_cb_t), KM_SLEEP); 31177c478bd9Sstevel@tonic-gate cbargp->lkycb_aflt = ecc; 31187c478bd9Sstevel@tonic-gate cbargp->lkycb_eqp = eqp; 31197c478bd9Sstevel@tonic-gate cbargp->lkycb_eqep = neqep; 31207c478bd9Sstevel@tonic-gate 31217c478bd9Sstevel@tonic-gate (void) timeout((void (*)(void *))ce_lkychk_cb, 31227c478bd9Sstevel@tonic-gate (void *)cbargp, drv_usectohz(cpu_ce_lkychk_timeout_usec)); 31237c478bd9Sstevel@tonic-gate return (1); 31247c478bd9Sstevel@tonic-gate } 31257c478bd9Sstevel@tonic-gate 31267c478bd9Sstevel@tonic-gate case CE_ACT_PTNRCHK: 31277c478bd9Sstevel@tonic-gate kpreempt_disable(); /* stop cpu list changing */ 31287c478bd9Sstevel@tonic-gate if ((cp = ce_ptnr_select(aflt, 0, &ptnrtype)) != NULL) { 31297c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, 31307c478bd9Sstevel@tonic-gate (uint64_t)aflt, (uint64_t)&disp); 31317c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRINFO(aflt->flt_disp, disp); 31327c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 31337c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 31347c478bd9Sstevel@tonic-gate } else if (ncpus > 1) { 31357c478bd9Sstevel@tonic-gate ce_xdiag_ptnrdrops++; 31367c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 31377c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_NOPTNR); 31387c478bd9Sstevel@tonic-gate } else { 31397c478bd9Sstevel@tonic-gate ce_xdiag_ptnrdrops++; 31407c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 31417c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_UNIPROC); 31427c478bd9Sstevel@tonic-gate } 31437c478bd9Sstevel@tonic-gate kpreempt_enable(); 31447c478bd9Sstevel@tonic-gate break; 31457c478bd9Sstevel@tonic-gate 31467c478bd9Sstevel@tonic-gate case CE_ACT_DONE: 31477c478bd9Sstevel@tonic-gate break; 31487c478bd9Sstevel@tonic-gate 31497c478bd9Sstevel@tonic-gate case CE_ACT(CE_DISP_BAD): 31507c478bd9Sstevel@tonic-gate default: 31517c478bd9Sstevel@tonic-gate #ifdef DEBUG 31527c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "ce_scrub_post: Bad action '%d'", action); 31537c478bd9Sstevel@tonic-gate #endif 31547c478bd9Sstevel@tonic-gate ce_xdiag_bad++; 31557c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_ACTBAD); 31567c478bd9Sstevel@tonic-gate break; 31577c478bd9Sstevel@tonic-gate } 31587c478bd9Sstevel@tonic-gate 31597c478bd9Sstevel@tonic-gate return (0); 31607c478bd9Sstevel@tonic-gate } 31617c478bd9Sstevel@tonic-gate 31627c478bd9Sstevel@tonic-gate /* 31637c478bd9Sstevel@tonic-gate * We route all errors through a single switch statement. 31647c478bd9Sstevel@tonic-gate */ 31657c478bd9Sstevel@tonic-gate void 31667c478bd9Sstevel@tonic-gate cpu_ue_log_err(struct async_flt *aflt) 31677c478bd9Sstevel@tonic-gate { 31687c478bd9Sstevel@tonic-gate switch (aflt->flt_class) { 31697c478bd9Sstevel@tonic-gate case CPU_FAULT: 31707c478bd9Sstevel@tonic-gate cpu_ereport_init(aflt); 31717c478bd9Sstevel@tonic-gate if (cpu_async_log_err(aflt, NULL)) 31727c478bd9Sstevel@tonic-gate cpu_ereport_post(aflt); 31737c478bd9Sstevel@tonic-gate break; 31747c478bd9Sstevel@tonic-gate 31757c478bd9Sstevel@tonic-gate case BUS_FAULT: 31767c478bd9Sstevel@tonic-gate bus_async_log_err(aflt); 31777c478bd9Sstevel@tonic-gate break; 31787c478bd9Sstevel@tonic-gate 31797c478bd9Sstevel@tonic-gate default: 31807c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "discarding async error %p with invalid " 31817c478bd9Sstevel@tonic-gate "fault class (0x%x)", (void *)aflt, aflt->flt_class); 31827c478bd9Sstevel@tonic-gate return; 31837c478bd9Sstevel@tonic-gate } 31847c478bd9Sstevel@tonic-gate } 31857c478bd9Sstevel@tonic-gate 31867c478bd9Sstevel@tonic-gate /* 31877c478bd9Sstevel@tonic-gate * Routine for panic hook callback from panic_idle(). 31887c478bd9Sstevel@tonic-gate */ 31897c478bd9Sstevel@tonic-gate void 31907c478bd9Sstevel@tonic-gate cpu_async_panic_callb(void) 31917c478bd9Sstevel@tonic-gate { 31927c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 31937c478bd9Sstevel@tonic-gate struct async_flt *aflt; 31947c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 31957c478bd9Sstevel@tonic-gate uint64_t afsr_errs; 31967c478bd9Sstevel@tonic-gate 31977c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 31987c478bd9Sstevel@tonic-gate 31997c478bd9Sstevel@tonic-gate afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 32007c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr_ext & C_AFSR_EXT_L3_ERRS); 32017c478bd9Sstevel@tonic-gate 32027c478bd9Sstevel@tonic-gate if (afsr_errs) { 32037c478bd9Sstevel@tonic-gate 32047c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 32057c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 32067c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 32077c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 32087c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 32097c478bd9Sstevel@tonic-gate aflt->flt_stat = cpu_error_regs.afsr; 32107c478bd9Sstevel@tonic-gate aflt->flt_addr = cpu_error_regs.afar; 32117c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 32127c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 32137c478bd9Sstevel@tonic-gate aflt->flt_priv = ((cpu_error_regs.afsr & C_AFSR_PRIV) != 0); 32147c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 32157c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = cpu_error_regs.afsr_ext; 32167c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = afsr_errs; 32177c478bd9Sstevel@tonic-gate #if defined(SERRANO) 32187c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 32197c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 32207c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, NULL, afsr_errs, NULL); 32217c478bd9Sstevel@tonic-gate } 32227c478bd9Sstevel@tonic-gate } 32237c478bd9Sstevel@tonic-gate 32247c478bd9Sstevel@tonic-gate /* 32257c478bd9Sstevel@tonic-gate * Routine to convert a syndrome into a syndrome code. 32267c478bd9Sstevel@tonic-gate */ 32277c478bd9Sstevel@tonic-gate static int 32287c478bd9Sstevel@tonic-gate synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit) 32297c478bd9Sstevel@tonic-gate { 32307c478bd9Sstevel@tonic-gate if (synd_status == AFLT_STAT_INVALID) 32317c478bd9Sstevel@tonic-gate return (-1); 32327c478bd9Sstevel@tonic-gate 32337c478bd9Sstevel@tonic-gate /* 32347c478bd9Sstevel@tonic-gate * Use the syndrome to index the appropriate syndrome table, 32357c478bd9Sstevel@tonic-gate * to get the code indicating which bit(s) is(are) bad. 32367c478bd9Sstevel@tonic-gate */ 32377c478bd9Sstevel@tonic-gate if (afsr_bit & 32387c478bd9Sstevel@tonic-gate (C_AFSR_MSYND_ERRS | C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 32397c478bd9Sstevel@tonic-gate if (afsr_bit & C_AFSR_MSYND_ERRS) { 32407c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 32417c478bd9Sstevel@tonic-gate if ((synd == 0) || (synd >= BSYND_TBL_SIZE)) 32427c478bd9Sstevel@tonic-gate return (-1); 32437c478bd9Sstevel@tonic-gate else 32447c478bd9Sstevel@tonic-gate return (BPAR0 + synd); 32457c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 32467c478bd9Sstevel@tonic-gate if ((synd == 0) || (synd >= MSYND_TBL_SIZE)) 32477c478bd9Sstevel@tonic-gate return (-1); 32487c478bd9Sstevel@tonic-gate else 32497c478bd9Sstevel@tonic-gate return (mtag_syndrome_tab[synd]); 32507c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 32517c478bd9Sstevel@tonic-gate } else { 32527c478bd9Sstevel@tonic-gate if ((synd == 0) || (synd >= ESYND_TBL_SIZE)) 32537c478bd9Sstevel@tonic-gate return (-1); 32547c478bd9Sstevel@tonic-gate else 32557c478bd9Sstevel@tonic-gate return (ecc_syndrome_tab[synd]); 32567c478bd9Sstevel@tonic-gate } 32577c478bd9Sstevel@tonic-gate } else { 32587c478bd9Sstevel@tonic-gate return (-1); 32597c478bd9Sstevel@tonic-gate } 32607c478bd9Sstevel@tonic-gate } 32617c478bd9Sstevel@tonic-gate 32627c478bd9Sstevel@tonic-gate /* 32637c478bd9Sstevel@tonic-gate * Routine to return a string identifying the physical name 32647c478bd9Sstevel@tonic-gate * associated with a memory/cache error. 32657c478bd9Sstevel@tonic-gate */ 32667c478bd9Sstevel@tonic-gate int 32677c478bd9Sstevel@tonic-gate cpu_get_mem_unum(int synd_status, ushort_t flt_synd, uint64_t flt_stat, 32687c478bd9Sstevel@tonic-gate uint64_t flt_addr, int flt_bus_id, int flt_in_memory, 32697c478bd9Sstevel@tonic-gate ushort_t flt_status, char *buf, int buflen, int *lenp) 32707c478bd9Sstevel@tonic-gate { 32717c478bd9Sstevel@tonic-gate int synd_code; 32727c478bd9Sstevel@tonic-gate int ret; 32737c478bd9Sstevel@tonic-gate 32747c478bd9Sstevel@tonic-gate /* 32757c478bd9Sstevel@tonic-gate * An AFSR of -1 defaults to a memory syndrome. 32767c478bd9Sstevel@tonic-gate */ 32777c478bd9Sstevel@tonic-gate if (flt_stat == (uint64_t)-1) 32787c478bd9Sstevel@tonic-gate flt_stat = C_AFSR_CE; 32797c478bd9Sstevel@tonic-gate 32807c478bd9Sstevel@tonic-gate synd_code = synd_to_synd_code(synd_status, flt_synd, flt_stat); 32817c478bd9Sstevel@tonic-gate 32827c478bd9Sstevel@tonic-gate /* 32837c478bd9Sstevel@tonic-gate * Syndrome code must be either a single-bit error code 32847c478bd9Sstevel@tonic-gate * (0...143) or -1 for unum lookup. 32857c478bd9Sstevel@tonic-gate */ 32867c478bd9Sstevel@tonic-gate if (synd_code < 0 || synd_code >= M2) 32877c478bd9Sstevel@tonic-gate synd_code = -1; 32887c478bd9Sstevel@tonic-gate if (&plat_get_mem_unum) { 32897c478bd9Sstevel@tonic-gate if ((ret = plat_get_mem_unum(synd_code, flt_addr, flt_bus_id, 32907c478bd9Sstevel@tonic-gate flt_in_memory, flt_status, buf, buflen, lenp)) != 0) { 32917c478bd9Sstevel@tonic-gate buf[0] = '\0'; 32927c478bd9Sstevel@tonic-gate *lenp = 0; 32937c478bd9Sstevel@tonic-gate } 32947c478bd9Sstevel@tonic-gate 32957c478bd9Sstevel@tonic-gate return (ret); 32967c478bd9Sstevel@tonic-gate } 32977c478bd9Sstevel@tonic-gate 32987c478bd9Sstevel@tonic-gate return (ENOTSUP); 32997c478bd9Sstevel@tonic-gate } 33007c478bd9Sstevel@tonic-gate 33017c478bd9Sstevel@tonic-gate /* 33027c478bd9Sstevel@tonic-gate * Wrapper for cpu_get_mem_unum() routine that takes an 33037c478bd9Sstevel@tonic-gate * async_flt struct rather than explicit arguments. 33047c478bd9Sstevel@tonic-gate */ 33057c478bd9Sstevel@tonic-gate int 33067c478bd9Sstevel@tonic-gate cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt, 33077c478bd9Sstevel@tonic-gate char *buf, int buflen, int *lenp) 33087c478bd9Sstevel@tonic-gate { 33097c478bd9Sstevel@tonic-gate /* 33107c478bd9Sstevel@tonic-gate * If we come thru here for an IO bus error aflt->flt_stat will 33117c478bd9Sstevel@tonic-gate * not be the CPU AFSR, and we pass in a -1 to cpu_get_mem_unum() 33127c478bd9Sstevel@tonic-gate * so it will interpret this as a memory error. 33137c478bd9Sstevel@tonic-gate */ 33147c478bd9Sstevel@tonic-gate return (cpu_get_mem_unum(synd_status, aflt->flt_synd, 33157c478bd9Sstevel@tonic-gate (aflt->flt_class == BUS_FAULT) ? 33167c478bd9Sstevel@tonic-gate (uint64_t)-1 : ((ch_async_flt_t *)(aflt))->afsr_errs, 33177c478bd9Sstevel@tonic-gate aflt->flt_addr, aflt->flt_bus_id, aflt->flt_in_memory, 33187c478bd9Sstevel@tonic-gate aflt->flt_status, buf, buflen, lenp)); 33197c478bd9Sstevel@tonic-gate } 33207c478bd9Sstevel@tonic-gate 33217c478bd9Sstevel@tonic-gate /* 33227c478bd9Sstevel@tonic-gate * This routine is a more generic interface to cpu_get_mem_unum() 33237c478bd9Sstevel@tonic-gate * that may be used by other modules (e.g. mm). 33247c478bd9Sstevel@tonic-gate */ 33257c478bd9Sstevel@tonic-gate int 33267c478bd9Sstevel@tonic-gate cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar, 33277c478bd9Sstevel@tonic-gate char *buf, int buflen, int *lenp) 33287c478bd9Sstevel@tonic-gate { 33297c478bd9Sstevel@tonic-gate int synd_status, flt_in_memory, ret; 33307c478bd9Sstevel@tonic-gate ushort_t flt_status = 0; 33317c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 33327c478bd9Sstevel@tonic-gate 33337c478bd9Sstevel@tonic-gate /* 33347c478bd9Sstevel@tonic-gate * Check for an invalid address. 33357c478bd9Sstevel@tonic-gate */ 33367c478bd9Sstevel@tonic-gate if (afar == (uint64_t)-1) 33377c478bd9Sstevel@tonic-gate return (ENXIO); 33387c478bd9Sstevel@tonic-gate 33397c478bd9Sstevel@tonic-gate if (synd == (uint64_t)-1) 33407c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_INVALID; 33417c478bd9Sstevel@tonic-gate else 33427c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_VALID; 33437c478bd9Sstevel@tonic-gate 33447c478bd9Sstevel@tonic-gate flt_in_memory = (*afsr & C_AFSR_MEMORY) && 33457c478bd9Sstevel@tonic-gate pf_is_memory(afar >> MMU_PAGESHIFT); 33467c478bd9Sstevel@tonic-gate 33477c478bd9Sstevel@tonic-gate /* 33487c478bd9Sstevel@tonic-gate * Need to turn on ECC_ECACHE for plat_get_mem_unum(). 33497c478bd9Sstevel@tonic-gate * For Panther, L2$ is not external, so we don't want to 33507c478bd9Sstevel@tonic-gate * generate an E$ unum for those errors. 33517c478bd9Sstevel@tonic-gate */ 33527c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 33537c478bd9Sstevel@tonic-gate if (*(afsr + 1) & C_AFSR_EXT_L3_ERRS) 33547c478bd9Sstevel@tonic-gate flt_status |= ECC_ECACHE; 33557c478bd9Sstevel@tonic-gate } else { 33567c478bd9Sstevel@tonic-gate if (*afsr & C_AFSR_ECACHE) 33577c478bd9Sstevel@tonic-gate flt_status |= ECC_ECACHE; 33587c478bd9Sstevel@tonic-gate } 33597c478bd9Sstevel@tonic-gate 33607c478bd9Sstevel@tonic-gate ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, *afsr, afar, 33617c478bd9Sstevel@tonic-gate CPU->cpu_id, flt_in_memory, flt_status, unum, UNUM_NAMLEN, lenp); 33627c478bd9Sstevel@tonic-gate if (ret != 0) 33637c478bd9Sstevel@tonic-gate return (ret); 33647c478bd9Sstevel@tonic-gate 33657c478bd9Sstevel@tonic-gate if (*lenp >= buflen) 33667c478bd9Sstevel@tonic-gate return (ENAMETOOLONG); 33677c478bd9Sstevel@tonic-gate 33687c478bd9Sstevel@tonic-gate (void) strncpy(buf, unum, buflen); 33697c478bd9Sstevel@tonic-gate 33707c478bd9Sstevel@tonic-gate return (0); 33717c478bd9Sstevel@tonic-gate } 33727c478bd9Sstevel@tonic-gate 33737c478bd9Sstevel@tonic-gate /* 33747c478bd9Sstevel@tonic-gate * Routine to return memory information associated 33757c478bd9Sstevel@tonic-gate * with a physical address and syndrome. 33767c478bd9Sstevel@tonic-gate */ 33777c478bd9Sstevel@tonic-gate int 33787c478bd9Sstevel@tonic-gate cpu_get_mem_info(uint64_t synd, uint64_t afar, 33797c478bd9Sstevel@tonic-gate uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 33807c478bd9Sstevel@tonic-gate int *segsp, int *banksp, int *mcidp) 33817c478bd9Sstevel@tonic-gate { 33827c478bd9Sstevel@tonic-gate int synd_status, synd_code; 33837c478bd9Sstevel@tonic-gate 33847c478bd9Sstevel@tonic-gate if (afar == (uint64_t)-1) 33857c478bd9Sstevel@tonic-gate return (ENXIO); 33867c478bd9Sstevel@tonic-gate 33877c478bd9Sstevel@tonic-gate if (synd == (uint64_t)-1) 33887c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_INVALID; 33897c478bd9Sstevel@tonic-gate else 33907c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_VALID; 33917c478bd9Sstevel@tonic-gate 33927c478bd9Sstevel@tonic-gate synd_code = synd_to_synd_code(synd_status, synd, C_AFSR_CE); 33937c478bd9Sstevel@tonic-gate 33947c478bd9Sstevel@tonic-gate if (p2get_mem_info != NULL) 33957c478bd9Sstevel@tonic-gate return ((p2get_mem_info)(synd_code, afar, 33967c478bd9Sstevel@tonic-gate mem_sizep, seg_sizep, bank_sizep, 33977c478bd9Sstevel@tonic-gate segsp, banksp, mcidp)); 33987c478bd9Sstevel@tonic-gate else 33997c478bd9Sstevel@tonic-gate return (ENOTSUP); 34007c478bd9Sstevel@tonic-gate } 34017c478bd9Sstevel@tonic-gate 34027c478bd9Sstevel@tonic-gate /* 34037c478bd9Sstevel@tonic-gate * Routine to return a string identifying the physical 34047c478bd9Sstevel@tonic-gate * name associated with a cpuid. 34057c478bd9Sstevel@tonic-gate */ 34067c478bd9Sstevel@tonic-gate int 34077c478bd9Sstevel@tonic-gate cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp) 34087c478bd9Sstevel@tonic-gate { 34097c478bd9Sstevel@tonic-gate int ret; 34107c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 34117c478bd9Sstevel@tonic-gate 34127c478bd9Sstevel@tonic-gate if (&plat_get_cpu_unum) { 34137c478bd9Sstevel@tonic-gate if ((ret = plat_get_cpu_unum(cpuid, unum, UNUM_NAMLEN, lenp)) 34147c478bd9Sstevel@tonic-gate != 0) 34157c478bd9Sstevel@tonic-gate return (ret); 34167c478bd9Sstevel@tonic-gate } else { 34177c478bd9Sstevel@tonic-gate return (ENOTSUP); 34187c478bd9Sstevel@tonic-gate } 34197c478bd9Sstevel@tonic-gate 34207c478bd9Sstevel@tonic-gate if (*lenp >= buflen) 34217c478bd9Sstevel@tonic-gate return (ENAMETOOLONG); 34227c478bd9Sstevel@tonic-gate 34237c478bd9Sstevel@tonic-gate (void) strncpy(buf, unum, buflen); 34247c478bd9Sstevel@tonic-gate 34257c478bd9Sstevel@tonic-gate return (0); 34267c478bd9Sstevel@tonic-gate } 34277c478bd9Sstevel@tonic-gate 34287c478bd9Sstevel@tonic-gate /* 34297c478bd9Sstevel@tonic-gate * This routine exports the name buffer size. 34307c478bd9Sstevel@tonic-gate */ 34317c478bd9Sstevel@tonic-gate size_t 34327c478bd9Sstevel@tonic-gate cpu_get_name_bufsize() 34337c478bd9Sstevel@tonic-gate { 34347c478bd9Sstevel@tonic-gate return (UNUM_NAMLEN); 34357c478bd9Sstevel@tonic-gate } 34367c478bd9Sstevel@tonic-gate 34377c478bd9Sstevel@tonic-gate /* 34387c478bd9Sstevel@tonic-gate * Historical function, apparantly not used. 34397c478bd9Sstevel@tonic-gate */ 34407c478bd9Sstevel@tonic-gate /* ARGSUSED */ 34417c478bd9Sstevel@tonic-gate void 34427c478bd9Sstevel@tonic-gate cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err) 34437c478bd9Sstevel@tonic-gate {} 34447c478bd9Sstevel@tonic-gate 34457c478bd9Sstevel@tonic-gate /* 34467c478bd9Sstevel@tonic-gate * Historical function only called for SBus errors in debugging. 34477c478bd9Sstevel@tonic-gate */ 34487c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 34497c478bd9Sstevel@tonic-gate void 34507c478bd9Sstevel@tonic-gate read_ecc_data(struct async_flt *aflt, short verbose, short ce_err) 34517c478bd9Sstevel@tonic-gate {} 34527c478bd9Sstevel@tonic-gate 34537c478bd9Sstevel@tonic-gate /* 34547c478bd9Sstevel@tonic-gate * Clear the AFSR sticky bits. The routine returns a non-zero value if 34557c478bd9Sstevel@tonic-gate * any of the AFSR's sticky errors are detected. If a non-null pointer to 34567c478bd9Sstevel@tonic-gate * an async fault structure argument is passed in, the captured error state 34577c478bd9Sstevel@tonic-gate * (AFSR, AFAR) info will be returned in the structure. 34587c478bd9Sstevel@tonic-gate */ 34597c478bd9Sstevel@tonic-gate int 34607c478bd9Sstevel@tonic-gate clear_errors(ch_async_flt_t *ch_flt) 34617c478bd9Sstevel@tonic-gate { 34627c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 34637c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 34647c478bd9Sstevel@tonic-gate 34657c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 34667c478bd9Sstevel@tonic-gate 34677c478bd9Sstevel@tonic-gate if (ch_flt != NULL) { 34687c478bd9Sstevel@tonic-gate aflt->flt_stat = cpu_error_regs.afsr & C_AFSR_MASK; 34697c478bd9Sstevel@tonic-gate aflt->flt_addr = cpu_error_regs.afar; 34707c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = cpu_error_regs.afsr_ext; 34717c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 34727c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS); 34737c478bd9Sstevel@tonic-gate #if defined(SERRANO) 34747c478bd9Sstevel@tonic-gate ch_flt->afar2 = cpu_error_regs.afar2; 34757c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 34767c478bd9Sstevel@tonic-gate } 34777c478bd9Sstevel@tonic-gate 34787c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 34797c478bd9Sstevel@tonic-gate 34807c478bd9Sstevel@tonic-gate return (((cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 34817c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS)) != 0); 34827c478bd9Sstevel@tonic-gate } 34837c478bd9Sstevel@tonic-gate 34847c478bd9Sstevel@tonic-gate /* 34857c478bd9Sstevel@tonic-gate * Clear any AFSR error bits, and check for persistence. 34867c478bd9Sstevel@tonic-gate * 34877c478bd9Sstevel@tonic-gate * It would be desirable to also insist that syndrome match. PCI handling 34887c478bd9Sstevel@tonic-gate * has already filled flt_synd. For errors trapped by CPU we only fill 34897c478bd9Sstevel@tonic-gate * flt_synd when we queue the event, so we do not have a valid flt_synd 34907c478bd9Sstevel@tonic-gate * during initial classification (it is valid if we're called as part of 34917c478bd9Sstevel@tonic-gate * subsequent low-pil additional classification attempts). We could try 34927c478bd9Sstevel@tonic-gate * to determine which syndrome to use: we know we're only called for 34937c478bd9Sstevel@tonic-gate * CE/RCE (Jalapeno & Serrano) and CE/EMC (others) so the syndrome to use 34947c478bd9Sstevel@tonic-gate * would be esynd/none and esynd/msynd, respectively. If that is 34957c478bd9Sstevel@tonic-gate * implemented then what do we do in the case that we do experience an 34967c478bd9Sstevel@tonic-gate * error on the same afar but with different syndrome? At the very least 34977c478bd9Sstevel@tonic-gate * we should count such occurences. Anyway, for now, we'll leave it as 34987c478bd9Sstevel@tonic-gate * it has been for ages. 34997c478bd9Sstevel@tonic-gate */ 35007c478bd9Sstevel@tonic-gate static int 35017c478bd9Sstevel@tonic-gate clear_ecc(struct async_flt *aflt) 35027c478bd9Sstevel@tonic-gate { 35037c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 35047c478bd9Sstevel@tonic-gate 35057c478bd9Sstevel@tonic-gate /* 35067c478bd9Sstevel@tonic-gate * Snapshot the AFSR and AFAR and clear any errors 35077c478bd9Sstevel@tonic-gate */ 35087c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 35097c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 35107c478bd9Sstevel@tonic-gate 35117c478bd9Sstevel@tonic-gate /* 35127c478bd9Sstevel@tonic-gate * If any of the same memory access error bits are still on and 35137c478bd9Sstevel@tonic-gate * the AFAR matches, return that the error is persistent. 35147c478bd9Sstevel@tonic-gate */ 35157c478bd9Sstevel@tonic-gate return ((cpu_error_regs.afsr & (C_AFSR_MEMORY & aflt->flt_stat)) != 0 && 35167c478bd9Sstevel@tonic-gate cpu_error_regs.afar == aflt->flt_addr); 35177c478bd9Sstevel@tonic-gate } 35187c478bd9Sstevel@tonic-gate 35197c478bd9Sstevel@tonic-gate /* 35207c478bd9Sstevel@tonic-gate * Turn off all cpu error detection, normally only used for panics. 35217c478bd9Sstevel@tonic-gate */ 35227c478bd9Sstevel@tonic-gate void 35237c478bd9Sstevel@tonic-gate cpu_disable_errors(void) 35247c478bd9Sstevel@tonic-gate { 35257c478bd9Sstevel@tonic-gate xt_all(set_error_enable_tl1, EN_REG_DISABLE, EER_SET_ABSOLUTE); 35267c478bd9Sstevel@tonic-gate } 35277c478bd9Sstevel@tonic-gate 35287c478bd9Sstevel@tonic-gate /* 35297c478bd9Sstevel@tonic-gate * Enable errors. 35307c478bd9Sstevel@tonic-gate */ 35317c478bd9Sstevel@tonic-gate void 35327c478bd9Sstevel@tonic-gate cpu_enable_errors(void) 35337c478bd9Sstevel@tonic-gate { 35347c478bd9Sstevel@tonic-gate xt_all(set_error_enable_tl1, EN_REG_ENABLE, EER_SET_ABSOLUTE); 35357c478bd9Sstevel@tonic-gate } 35367c478bd9Sstevel@tonic-gate 35377c478bd9Sstevel@tonic-gate /* 35387c478bd9Sstevel@tonic-gate * Flush the entire ecache using displacement flush by reading through a 35397c478bd9Sstevel@tonic-gate * physical address range twice as large as the Ecache. 35407c478bd9Sstevel@tonic-gate */ 35417c478bd9Sstevel@tonic-gate void 35427c478bd9Sstevel@tonic-gate cpu_flush_ecache(void) 35437c478bd9Sstevel@tonic-gate { 35447c478bd9Sstevel@tonic-gate flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size, 35457c478bd9Sstevel@tonic-gate cpunodes[CPU->cpu_id].ecache_linesize); 35467c478bd9Sstevel@tonic-gate } 35477c478bd9Sstevel@tonic-gate 35487c478bd9Sstevel@tonic-gate /* 35497c478bd9Sstevel@tonic-gate * Return CPU E$ set size - E$ size divided by the associativity. 35507c478bd9Sstevel@tonic-gate * We use this function in places where the CPU_PRIVATE ptr may not be 35517c478bd9Sstevel@tonic-gate * initialized yet. Note that for send_mondo and in the Ecache scrubber, 35527c478bd9Sstevel@tonic-gate * we're guaranteed that CPU_PRIVATE is initialized. Also, cpunodes is set 35537c478bd9Sstevel@tonic-gate * up before the kernel switches from OBP's to the kernel's trap table, so 35547c478bd9Sstevel@tonic-gate * we don't have to worry about cpunodes being unitialized. 35557c478bd9Sstevel@tonic-gate */ 35567c478bd9Sstevel@tonic-gate int 35577c478bd9Sstevel@tonic-gate cpu_ecache_set_size(struct cpu *cp) 35587c478bd9Sstevel@tonic-gate { 35597c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(cp)) 35607c478bd9Sstevel@tonic-gate return (CPU_PRIVATE_VAL(cp, chpr_ec_set_size)); 35617c478bd9Sstevel@tonic-gate 35627c478bd9Sstevel@tonic-gate return (cpunodes[cp->cpu_id].ecache_size / cpu_ecache_nway()); 35637c478bd9Sstevel@tonic-gate } 35647c478bd9Sstevel@tonic-gate 35657c478bd9Sstevel@tonic-gate /* 35667c478bd9Sstevel@tonic-gate * Flush Ecache line. 35677c478bd9Sstevel@tonic-gate * Uses ASI_EC_DIAG for Cheetah+ and Jalapeno. 35687c478bd9Sstevel@tonic-gate * Uses normal displacement flush for Cheetah. 35697c478bd9Sstevel@tonic-gate */ 35707c478bd9Sstevel@tonic-gate static void 35717c478bd9Sstevel@tonic-gate cpu_flush_ecache_line(ch_async_flt_t *ch_flt) 35727c478bd9Sstevel@tonic-gate { 35737c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 35747c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 35757c478bd9Sstevel@tonic-gate 35767c478bd9Sstevel@tonic-gate ecache_flush_line(aflt->flt_addr, ec_set_size); 35777c478bd9Sstevel@tonic-gate } 35787c478bd9Sstevel@tonic-gate 35797c478bd9Sstevel@tonic-gate /* 35807c478bd9Sstevel@tonic-gate * Scrub physical address. 35817c478bd9Sstevel@tonic-gate * Scrub code is different depending upon whether this a Cheetah+ with 2-way 35827c478bd9Sstevel@tonic-gate * Ecache or direct-mapped Ecache. 35837c478bd9Sstevel@tonic-gate */ 35847c478bd9Sstevel@tonic-gate static void 35857c478bd9Sstevel@tonic-gate cpu_scrubphys(struct async_flt *aflt) 35867c478bd9Sstevel@tonic-gate { 35877c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 35887c478bd9Sstevel@tonic-gate 35897c478bd9Sstevel@tonic-gate scrubphys(aflt->flt_addr, ec_set_size); 35907c478bd9Sstevel@tonic-gate } 35917c478bd9Sstevel@tonic-gate 35927c478bd9Sstevel@tonic-gate /* 35937c478bd9Sstevel@tonic-gate * Clear physical address. 35947c478bd9Sstevel@tonic-gate * Scrub code is different depending upon whether this a Cheetah+ with 2-way 35957c478bd9Sstevel@tonic-gate * Ecache or direct-mapped Ecache. 35967c478bd9Sstevel@tonic-gate */ 35977c478bd9Sstevel@tonic-gate void 35987c478bd9Sstevel@tonic-gate cpu_clearphys(struct async_flt *aflt) 35997c478bd9Sstevel@tonic-gate { 36007c478bd9Sstevel@tonic-gate int lsize = cpunodes[CPU->cpu_id].ecache_linesize; 36017c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 36027c478bd9Sstevel@tonic-gate 36037c478bd9Sstevel@tonic-gate 36047c478bd9Sstevel@tonic-gate clearphys(P2ALIGN(aflt->flt_addr, lsize), ec_set_size, lsize); 36057c478bd9Sstevel@tonic-gate } 36067c478bd9Sstevel@tonic-gate 36077c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 36087c478bd9Sstevel@tonic-gate /* 36097c478bd9Sstevel@tonic-gate * Check for a matching valid line in all the sets. 36107c478bd9Sstevel@tonic-gate * If found, return set# + 1. Otherwise return 0. 36117c478bd9Sstevel@tonic-gate */ 36127c478bd9Sstevel@tonic-gate static int 36137c478bd9Sstevel@tonic-gate cpu_ecache_line_valid(ch_async_flt_t *ch_flt) 36147c478bd9Sstevel@tonic-gate { 36157c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 36167c478bd9Sstevel@tonic-gate int totalsize = cpunodes[CPU->cpu_id].ecache_size; 36177c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 36187c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 36197c478bd9Sstevel@tonic-gate int nway = cpu_ecache_nway(); 36207c478bd9Sstevel@tonic-gate int i; 36217c478bd9Sstevel@tonic-gate 36227c478bd9Sstevel@tonic-gate for (i = 0; i < nway; i++, ecp++) { 36237c478bd9Sstevel@tonic-gate if (!cpu_ectag_line_invalid(totalsize, ecp->ec_tag) && 36247c478bd9Sstevel@tonic-gate (aflt->flt_addr & P2ALIGN(C_AFAR_PA, ec_set_size)) == 36257c478bd9Sstevel@tonic-gate cpu_ectag_to_pa(ec_set_size, ecp->ec_tag)) 36267c478bd9Sstevel@tonic-gate return (i+1); 36277c478bd9Sstevel@tonic-gate } 36287c478bd9Sstevel@tonic-gate return (0); 36297c478bd9Sstevel@tonic-gate } 36307c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_ECACHE_ASSOC */ 36317c478bd9Sstevel@tonic-gate 36327c478bd9Sstevel@tonic-gate /* 36337c478bd9Sstevel@tonic-gate * Check whether a line in the given logout info matches the specified 36347c478bd9Sstevel@tonic-gate * fault address. If reqval is set then the line must not be Invalid. 36357c478bd9Sstevel@tonic-gate * Returns 0 on failure; on success (way + 1) is returned an *level is 36367c478bd9Sstevel@tonic-gate * set to 2 for l2$ or 3 for l3$. 36377c478bd9Sstevel@tonic-gate */ 36387c478bd9Sstevel@tonic-gate static int 36397c478bd9Sstevel@tonic-gate cpu_matching_ecache_line(uint64_t faddr, void *data, int reqval, int *level) 36407c478bd9Sstevel@tonic-gate { 36417c478bd9Sstevel@tonic-gate ch_diag_data_t *cdp = data; 36427c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp; 36437c478bd9Sstevel@tonic-gate int totalsize, ec_set_size; 36447c478bd9Sstevel@tonic-gate int i, ways; 36457c478bd9Sstevel@tonic-gate int match = 0; 36467c478bd9Sstevel@tonic-gate int tagvalid; 36477c478bd9Sstevel@tonic-gate uint64_t addr, tagpa; 36487c478bd9Sstevel@tonic-gate int ispanther = IS_PANTHER(cpunodes[CPU->cpu_id].implementation); 36497c478bd9Sstevel@tonic-gate 36507c478bd9Sstevel@tonic-gate /* 36517c478bd9Sstevel@tonic-gate * Check the l2$ logout data 36527c478bd9Sstevel@tonic-gate */ 36537c478bd9Sstevel@tonic-gate if (ispanther) { 36547c478bd9Sstevel@tonic-gate ecp = &cdp->chd_l2_data[0]; 36557c478bd9Sstevel@tonic-gate ec_set_size = PN_L2_SET_SIZE; 36567c478bd9Sstevel@tonic-gate ways = PN_L2_NWAYS; 36577c478bd9Sstevel@tonic-gate } else { 36587c478bd9Sstevel@tonic-gate ecp = &cdp->chd_ec_data[0]; 36597c478bd9Sstevel@tonic-gate ec_set_size = cpu_ecache_set_size(CPU); 36607c478bd9Sstevel@tonic-gate ways = cpu_ecache_nway(); 36617c478bd9Sstevel@tonic-gate totalsize = cpunodes[CPU->cpu_id].ecache_size; 36627c478bd9Sstevel@tonic-gate } 36637c478bd9Sstevel@tonic-gate /* remove low order PA bits from fault address not used in PA tag */ 36647c478bd9Sstevel@tonic-gate addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 36657c478bd9Sstevel@tonic-gate for (i = 0; i < ways; i++, ecp++) { 36667c478bd9Sstevel@tonic-gate if (ispanther) { 36677c478bd9Sstevel@tonic-gate tagpa = PN_L2TAG_TO_PA(ecp->ec_tag); 36687c478bd9Sstevel@tonic-gate tagvalid = !PN_L2_LINE_INVALID(ecp->ec_tag); 36697c478bd9Sstevel@tonic-gate } else { 36707c478bd9Sstevel@tonic-gate tagpa = cpu_ectag_to_pa(ec_set_size, ecp->ec_tag); 36717c478bd9Sstevel@tonic-gate tagvalid = !cpu_ectag_line_invalid(totalsize, 36727c478bd9Sstevel@tonic-gate ecp->ec_tag); 36737c478bd9Sstevel@tonic-gate } 36747c478bd9Sstevel@tonic-gate if (tagpa == addr && (!reqval || tagvalid)) { 36757c478bd9Sstevel@tonic-gate match = i + 1; 36767c478bd9Sstevel@tonic-gate *level = 2; 36777c478bd9Sstevel@tonic-gate break; 36787c478bd9Sstevel@tonic-gate } 36797c478bd9Sstevel@tonic-gate } 36807c478bd9Sstevel@tonic-gate 36817c478bd9Sstevel@tonic-gate if (match || !ispanther) 36827c478bd9Sstevel@tonic-gate return (match); 36837c478bd9Sstevel@tonic-gate 36847c478bd9Sstevel@tonic-gate /* For Panther we also check the l3$ */ 36857c478bd9Sstevel@tonic-gate ecp = &cdp->chd_ec_data[0]; 36867c478bd9Sstevel@tonic-gate ec_set_size = PN_L3_SET_SIZE; 36877c478bd9Sstevel@tonic-gate ways = PN_L3_NWAYS; 36887c478bd9Sstevel@tonic-gate addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 36897c478bd9Sstevel@tonic-gate 36907c478bd9Sstevel@tonic-gate for (i = 0; i < ways; i++, ecp++) { 36917c478bd9Sstevel@tonic-gate if (PN_L3TAG_TO_PA(ecp->ec_tag) == addr && (!reqval || 36927c478bd9Sstevel@tonic-gate !PN_L3_LINE_INVALID(ecp->ec_tag))) { 36937c478bd9Sstevel@tonic-gate match = i + 1; 36947c478bd9Sstevel@tonic-gate *level = 3; 36957c478bd9Sstevel@tonic-gate break; 36967c478bd9Sstevel@tonic-gate } 36977c478bd9Sstevel@tonic-gate } 36987c478bd9Sstevel@tonic-gate 36997c478bd9Sstevel@tonic-gate return (match); 37007c478bd9Sstevel@tonic-gate } 37017c478bd9Sstevel@tonic-gate 37027c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 37037c478bd9Sstevel@tonic-gate /* 37047c478bd9Sstevel@tonic-gate * Record information related to the source of an Dcache Parity Error. 37057c478bd9Sstevel@tonic-gate */ 37067c478bd9Sstevel@tonic-gate static void 37077c478bd9Sstevel@tonic-gate cpu_dcache_parity_info(ch_async_flt_t *ch_flt) 37087c478bd9Sstevel@tonic-gate { 37097c478bd9Sstevel@tonic-gate int dc_set_size = dcache_size / CH_DCACHE_NWAY; 37107c478bd9Sstevel@tonic-gate int index; 37117c478bd9Sstevel@tonic-gate 37127c478bd9Sstevel@tonic-gate /* 37137c478bd9Sstevel@tonic-gate * Since instruction decode cannot be done at high PIL 37147c478bd9Sstevel@tonic-gate * just examine the entire Dcache to locate the error. 37157c478bd9Sstevel@tonic-gate */ 37167c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 37177c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = -1; 37187c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = -1; 37197c478bd9Sstevel@tonic-gate } 37207c478bd9Sstevel@tonic-gate for (index = 0; index < dc_set_size; index += dcache_linesize) 37217c478bd9Sstevel@tonic-gate cpu_dcache_parity_check(ch_flt, index); 37227c478bd9Sstevel@tonic-gate } 37237c478bd9Sstevel@tonic-gate 37247c478bd9Sstevel@tonic-gate /* 37257c478bd9Sstevel@tonic-gate * Check all ways of the Dcache at a specified index for good parity. 37267c478bd9Sstevel@tonic-gate */ 37277c478bd9Sstevel@tonic-gate static void 37287c478bd9Sstevel@tonic-gate cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index) 37297c478bd9Sstevel@tonic-gate { 37307c478bd9Sstevel@tonic-gate int dc_set_size = dcache_size / CH_DCACHE_NWAY; 37317c478bd9Sstevel@tonic-gate uint64_t parity_bits, pbits, data_word; 37327c478bd9Sstevel@tonic-gate static int parity_bits_popc[] = { 0, 1, 1, 0 }; 37337c478bd9Sstevel@tonic-gate int way, word, data_byte; 37347c478bd9Sstevel@tonic-gate ch_dc_data_t *dcp = &ch_flt->parity_data.dpe.cpl_dc[0]; 37357c478bd9Sstevel@tonic-gate ch_dc_data_t tmp_dcp; 37367c478bd9Sstevel@tonic-gate 37377c478bd9Sstevel@tonic-gate for (way = 0; way < CH_DCACHE_NWAY; way++, dcp++) { 37387c478bd9Sstevel@tonic-gate /* 37397c478bd9Sstevel@tonic-gate * Perform diagnostic read. 37407c478bd9Sstevel@tonic-gate */ 37417c478bd9Sstevel@tonic-gate get_dcache_dtag(index + way * dc_set_size, 37427c478bd9Sstevel@tonic-gate (uint64_t *)&tmp_dcp); 37437c478bd9Sstevel@tonic-gate 37447c478bd9Sstevel@tonic-gate /* 37457c478bd9Sstevel@tonic-gate * Check tag for even parity. 37467c478bd9Sstevel@tonic-gate * Sum of 1 bits (including parity bit) should be even. 37477c478bd9Sstevel@tonic-gate */ 37487c478bd9Sstevel@tonic-gate if (popc64(tmp_dcp.dc_tag & CHP_DCTAG_PARMASK) & 1) { 37497c478bd9Sstevel@tonic-gate /* 37507c478bd9Sstevel@tonic-gate * If this is the first error log detailed information 37517c478bd9Sstevel@tonic-gate * about it and check the snoop tag. Otherwise just 37527c478bd9Sstevel@tonic-gate * record the fact that we found another error. 37537c478bd9Sstevel@tonic-gate */ 37547c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 37557c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = way; 37567c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_cache = 37577c478bd9Sstevel@tonic-gate CPU_DC_PARITY; 37587c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_tag |= CHP_DC_TAG; 37597c478bd9Sstevel@tonic-gate 37607c478bd9Sstevel@tonic-gate if (popc64(tmp_dcp.dc_sntag & 37617c478bd9Sstevel@tonic-gate CHP_DCSNTAG_PARMASK) & 1) { 37627c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_tag |= 37637c478bd9Sstevel@tonic-gate CHP_DC_SNTAG; 37647c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 37657c478bd9Sstevel@tonic-gate } 37667c478bd9Sstevel@tonic-gate 37677c478bd9Sstevel@tonic-gate bcopy(&tmp_dcp, dcp, sizeof (ch_dc_data_t)); 37687c478bd9Sstevel@tonic-gate } 37697c478bd9Sstevel@tonic-gate 37707c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 37717c478bd9Sstevel@tonic-gate } 37727c478bd9Sstevel@tonic-gate 37737c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 37747c478bd9Sstevel@tonic-gate /* 37757c478bd9Sstevel@tonic-gate * Panther has more parity bits than the other 37767c478bd9Sstevel@tonic-gate * processors for covering dcache data and so each 37777c478bd9Sstevel@tonic-gate * byte of data in each word has its own parity bit. 37787c478bd9Sstevel@tonic-gate */ 37797c478bd9Sstevel@tonic-gate parity_bits = tmp_dcp.dc_pn_data_parity; 37807c478bd9Sstevel@tonic-gate for (word = 0; word < 4; word++) { 37817c478bd9Sstevel@tonic-gate data_word = tmp_dcp.dc_data[word]; 37827c478bd9Sstevel@tonic-gate pbits = parity_bits & PN_DC_DATA_PARITY_MASK; 37837c478bd9Sstevel@tonic-gate for (data_byte = 0; data_byte < 8; 37847c478bd9Sstevel@tonic-gate data_byte++) { 37857c478bd9Sstevel@tonic-gate if (((popc64(data_word & 37867c478bd9Sstevel@tonic-gate PN_DC_DATA_PARITY_MASK)) & 1) ^ 37877c478bd9Sstevel@tonic-gate (pbits & 1)) { 37887c478bd9Sstevel@tonic-gate cpu_record_dc_data_parity( 37897c478bd9Sstevel@tonic-gate ch_flt, dcp, &tmp_dcp, way, 37907c478bd9Sstevel@tonic-gate word); 37917c478bd9Sstevel@tonic-gate } 37927c478bd9Sstevel@tonic-gate pbits >>= 1; 37937c478bd9Sstevel@tonic-gate data_word >>= 8; 37947c478bd9Sstevel@tonic-gate } 37957c478bd9Sstevel@tonic-gate parity_bits >>= 8; 37967c478bd9Sstevel@tonic-gate } 37977c478bd9Sstevel@tonic-gate } else { 37987c478bd9Sstevel@tonic-gate /* 37997c478bd9Sstevel@tonic-gate * Check data array for even parity. 38007c478bd9Sstevel@tonic-gate * The 8 parity bits are grouped into 4 pairs each 38017c478bd9Sstevel@tonic-gate * of which covers a 64-bit word. The endianness is 38027c478bd9Sstevel@tonic-gate * reversed -- the low-order parity bits cover the 38037c478bd9Sstevel@tonic-gate * high-order data words. 38047c478bd9Sstevel@tonic-gate */ 38057c478bd9Sstevel@tonic-gate parity_bits = tmp_dcp.dc_utag >> 8; 38067c478bd9Sstevel@tonic-gate for (word = 0; word < 4; word++) { 38077c478bd9Sstevel@tonic-gate pbits = (parity_bits >> (6 - word * 2)) & 3; 38087c478bd9Sstevel@tonic-gate if ((popc64(tmp_dcp.dc_data[word]) + 38097c478bd9Sstevel@tonic-gate parity_bits_popc[pbits]) & 1) { 38107c478bd9Sstevel@tonic-gate cpu_record_dc_data_parity(ch_flt, dcp, 38117c478bd9Sstevel@tonic-gate &tmp_dcp, way, word); 38127c478bd9Sstevel@tonic-gate } 38137c478bd9Sstevel@tonic-gate } 38147c478bd9Sstevel@tonic-gate } 38157c478bd9Sstevel@tonic-gate } 38167c478bd9Sstevel@tonic-gate } 38177c478bd9Sstevel@tonic-gate 38187c478bd9Sstevel@tonic-gate static void 38197c478bd9Sstevel@tonic-gate cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 38207c478bd9Sstevel@tonic-gate ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word) 38217c478bd9Sstevel@tonic-gate { 38227c478bd9Sstevel@tonic-gate /* 38237c478bd9Sstevel@tonic-gate * If this is the first error log detailed information about it. 38247c478bd9Sstevel@tonic-gate * Otherwise just record the fact that we found another error. 38257c478bd9Sstevel@tonic-gate */ 38267c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 38277c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = way; 38287c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_cache = CPU_DC_PARITY; 38297c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = word * 8; 38307c478bd9Sstevel@tonic-gate bcopy(src_dcp, dest_dcp, sizeof (ch_dc_data_t)); 38317c478bd9Sstevel@tonic-gate } 38327c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 38337c478bd9Sstevel@tonic-gate } 38347c478bd9Sstevel@tonic-gate 38357c478bd9Sstevel@tonic-gate /* 38367c478bd9Sstevel@tonic-gate * Record information related to the source of an Icache Parity Error. 38377c478bd9Sstevel@tonic-gate * 38387c478bd9Sstevel@tonic-gate * Called with the Icache disabled so any diagnostic accesses are safe. 38397c478bd9Sstevel@tonic-gate */ 38407c478bd9Sstevel@tonic-gate static void 38417c478bd9Sstevel@tonic-gate cpu_icache_parity_info(ch_async_flt_t *ch_flt) 38427c478bd9Sstevel@tonic-gate { 38437c478bd9Sstevel@tonic-gate int ic_set_size; 38447c478bd9Sstevel@tonic-gate int ic_linesize; 38457c478bd9Sstevel@tonic-gate int index; 38467c478bd9Sstevel@tonic-gate 38477c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU)) { 38487c478bd9Sstevel@tonic-gate ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 38497c478bd9Sstevel@tonic-gate CH_ICACHE_NWAY; 38507c478bd9Sstevel@tonic-gate ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 38517c478bd9Sstevel@tonic-gate } else { 38527c478bd9Sstevel@tonic-gate ic_set_size = icache_size / CH_ICACHE_NWAY; 38537c478bd9Sstevel@tonic-gate ic_linesize = icache_linesize; 38547c478bd9Sstevel@tonic-gate } 38557c478bd9Sstevel@tonic-gate 38567c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_way = -1; 38577c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_off = -1; 38587c478bd9Sstevel@tonic-gate 38597c478bd9Sstevel@tonic-gate for (index = 0; index < ic_set_size; index += ic_linesize) 38607c478bd9Sstevel@tonic-gate cpu_icache_parity_check(ch_flt, index); 38617c478bd9Sstevel@tonic-gate } 38627c478bd9Sstevel@tonic-gate 38637c478bd9Sstevel@tonic-gate /* 38647c478bd9Sstevel@tonic-gate * Check all ways of the Icache at a specified index for good parity. 38657c478bd9Sstevel@tonic-gate */ 38667c478bd9Sstevel@tonic-gate static void 38677c478bd9Sstevel@tonic-gate cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index) 38687c478bd9Sstevel@tonic-gate { 38697c478bd9Sstevel@tonic-gate uint64_t parmask, pn_inst_parity; 38707c478bd9Sstevel@tonic-gate int ic_set_size; 38717c478bd9Sstevel@tonic-gate int ic_linesize; 38727c478bd9Sstevel@tonic-gate int flt_index, way, instr, num_instr; 38737c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 38747c478bd9Sstevel@tonic-gate ch_ic_data_t *icp = &ch_flt->parity_data.ipe.cpl_ic[0]; 38757c478bd9Sstevel@tonic-gate ch_ic_data_t tmp_icp; 38767c478bd9Sstevel@tonic-gate 38777c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU)) { 38787c478bd9Sstevel@tonic-gate ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 38797c478bd9Sstevel@tonic-gate CH_ICACHE_NWAY; 38807c478bd9Sstevel@tonic-gate ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 38817c478bd9Sstevel@tonic-gate } else { 38827c478bd9Sstevel@tonic-gate ic_set_size = icache_size / CH_ICACHE_NWAY; 38837c478bd9Sstevel@tonic-gate ic_linesize = icache_linesize; 38847c478bd9Sstevel@tonic-gate } 38857c478bd9Sstevel@tonic-gate 38867c478bd9Sstevel@tonic-gate /* 38877c478bd9Sstevel@tonic-gate * Panther has twice as many instructions per icache line and the 38887c478bd9Sstevel@tonic-gate * instruction parity bit is in a different location. 38897c478bd9Sstevel@tonic-gate */ 38907c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 38917c478bd9Sstevel@tonic-gate num_instr = PN_IC_DATA_REG_SIZE / sizeof (uint64_t); 38927c478bd9Sstevel@tonic-gate pn_inst_parity = PN_ICDATA_PARITY_BIT_MASK; 38937c478bd9Sstevel@tonic-gate } else { 38947c478bd9Sstevel@tonic-gate num_instr = CH_IC_DATA_REG_SIZE / sizeof (uint64_t); 38957c478bd9Sstevel@tonic-gate pn_inst_parity = 0; 38967c478bd9Sstevel@tonic-gate } 38977c478bd9Sstevel@tonic-gate 38987c478bd9Sstevel@tonic-gate /* 38997c478bd9Sstevel@tonic-gate * Index at which we expect to find the parity error. 39007c478bd9Sstevel@tonic-gate */ 39017c478bd9Sstevel@tonic-gate flt_index = P2ALIGN(aflt->flt_addr % ic_set_size, ic_linesize); 39027c478bd9Sstevel@tonic-gate 39037c478bd9Sstevel@tonic-gate for (way = 0; way < CH_ICACHE_NWAY; way++, icp++) { 39047c478bd9Sstevel@tonic-gate /* 39057c478bd9Sstevel@tonic-gate * Diagnostic reads expect address argument in ASI format. 39067c478bd9Sstevel@tonic-gate */ 39077c478bd9Sstevel@tonic-gate get_icache_dtag(2 * (index + way * ic_set_size), 39087c478bd9Sstevel@tonic-gate (uint64_t *)&tmp_icp); 39097c478bd9Sstevel@tonic-gate 39107c478bd9Sstevel@tonic-gate /* 39117c478bd9Sstevel@tonic-gate * If this is the index in which we expect to find the 39127c478bd9Sstevel@tonic-gate * error log detailed information about each of the ways. 39137c478bd9Sstevel@tonic-gate * This information will be displayed later if we can't 39147c478bd9Sstevel@tonic-gate * determine the exact way in which the error is located. 39157c478bd9Sstevel@tonic-gate */ 39167c478bd9Sstevel@tonic-gate if (flt_index == index) 39177c478bd9Sstevel@tonic-gate bcopy(&tmp_icp, icp, sizeof (ch_ic_data_t)); 39187c478bd9Sstevel@tonic-gate 39197c478bd9Sstevel@tonic-gate /* 39207c478bd9Sstevel@tonic-gate * Check tag for even parity. 39217c478bd9Sstevel@tonic-gate * Sum of 1 bits (including parity bit) should be even. 39227c478bd9Sstevel@tonic-gate */ 39237c478bd9Sstevel@tonic-gate if (popc64(tmp_icp.ic_patag & CHP_ICPATAG_PARMASK) & 1) { 39247c478bd9Sstevel@tonic-gate /* 39257c478bd9Sstevel@tonic-gate * If this way is the one in which we expected 39267c478bd9Sstevel@tonic-gate * to find the error record the way and check the 39277c478bd9Sstevel@tonic-gate * snoop tag. Otherwise just record the fact we 39287c478bd9Sstevel@tonic-gate * found another error. 39297c478bd9Sstevel@tonic-gate */ 39307c478bd9Sstevel@tonic-gate if (flt_index == index) { 39317c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_way = way; 39327c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_tag |= CHP_IC_TAG; 39337c478bd9Sstevel@tonic-gate 39347c478bd9Sstevel@tonic-gate if (popc64(tmp_icp.ic_sntag & 39357c478bd9Sstevel@tonic-gate CHP_ICSNTAG_PARMASK) & 1) { 39367c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_tag |= 39377c478bd9Sstevel@tonic-gate CHP_IC_SNTAG; 39387c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_lcnt++; 39397c478bd9Sstevel@tonic-gate } 39407c478bd9Sstevel@tonic-gate 39417c478bd9Sstevel@tonic-gate } 39427c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_lcnt++; 39437c478bd9Sstevel@tonic-gate continue; 39447c478bd9Sstevel@tonic-gate } 39457c478bd9Sstevel@tonic-gate 39467c478bd9Sstevel@tonic-gate /* 39477c478bd9Sstevel@tonic-gate * Check instruction data for even parity. 39487c478bd9Sstevel@tonic-gate * Bits participating in parity differ for PC-relative 39497c478bd9Sstevel@tonic-gate * versus non-PC-relative instructions. 39507c478bd9Sstevel@tonic-gate */ 39517c478bd9Sstevel@tonic-gate for (instr = 0; instr < num_instr; instr++) { 39527c478bd9Sstevel@tonic-gate parmask = (tmp_icp.ic_data[instr] & 39537c478bd9Sstevel@tonic-gate CH_ICDATA_PRED_ISPCREL) ? 39547c478bd9Sstevel@tonic-gate (CHP_ICDATA_PCREL_PARMASK | pn_inst_parity) : 39557c478bd9Sstevel@tonic-gate (CHP_ICDATA_NPCREL_PARMASK | pn_inst_parity); 39567c478bd9Sstevel@tonic-gate if (popc64(tmp_icp.ic_data[instr] & parmask) & 1) { 39577c478bd9Sstevel@tonic-gate /* 39587c478bd9Sstevel@tonic-gate * If this way is the one in which we expected 39597c478bd9Sstevel@tonic-gate * to find the error record the way and offset. 39607c478bd9Sstevel@tonic-gate * Otherwise just log the fact we found another 39617c478bd9Sstevel@tonic-gate * error. 39627c478bd9Sstevel@tonic-gate */ 39637c478bd9Sstevel@tonic-gate if (flt_index == index) { 39647c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_way = way; 39657c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_off = 39667c478bd9Sstevel@tonic-gate instr * 4; 39677c478bd9Sstevel@tonic-gate } 39687c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_lcnt++; 39697c478bd9Sstevel@tonic-gate continue; 39707c478bd9Sstevel@tonic-gate } 39717c478bd9Sstevel@tonic-gate } 39727c478bd9Sstevel@tonic-gate } 39737c478bd9Sstevel@tonic-gate } 39747c478bd9Sstevel@tonic-gate 39757c478bd9Sstevel@tonic-gate /* 39767c478bd9Sstevel@tonic-gate * Record information related to the source of an Pcache Parity Error. 39777c478bd9Sstevel@tonic-gate */ 39787c478bd9Sstevel@tonic-gate static void 39797c478bd9Sstevel@tonic-gate cpu_pcache_parity_info(ch_async_flt_t *ch_flt) 39807c478bd9Sstevel@tonic-gate { 39817c478bd9Sstevel@tonic-gate int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 39827c478bd9Sstevel@tonic-gate int index; 39837c478bd9Sstevel@tonic-gate 39847c478bd9Sstevel@tonic-gate /* 39857c478bd9Sstevel@tonic-gate * Since instruction decode cannot be done at high PIL just 39867c478bd9Sstevel@tonic-gate * examine the entire Pcache to check for any parity errors. 39877c478bd9Sstevel@tonic-gate */ 39887c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 39897c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = -1; 39907c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = -1; 39917c478bd9Sstevel@tonic-gate } 39927c478bd9Sstevel@tonic-gate for (index = 0; index < pc_set_size; index += CH_PCACHE_LSIZE) 39937c478bd9Sstevel@tonic-gate cpu_pcache_parity_check(ch_flt, index); 39947c478bd9Sstevel@tonic-gate } 39957c478bd9Sstevel@tonic-gate 39967c478bd9Sstevel@tonic-gate /* 39977c478bd9Sstevel@tonic-gate * Check all ways of the Pcache at a specified index for good parity. 39987c478bd9Sstevel@tonic-gate */ 39997c478bd9Sstevel@tonic-gate static void 40007c478bd9Sstevel@tonic-gate cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index) 40017c478bd9Sstevel@tonic-gate { 40027c478bd9Sstevel@tonic-gate int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 40037c478bd9Sstevel@tonic-gate int pc_data_words = CH_PC_DATA_REG_SIZE / sizeof (uint64_t); 40047c478bd9Sstevel@tonic-gate int way, word, pbit, parity_bits; 40057c478bd9Sstevel@tonic-gate ch_pc_data_t *pcp = &ch_flt->parity_data.dpe.cpl_pc[0]; 40067c478bd9Sstevel@tonic-gate ch_pc_data_t tmp_pcp; 40077c478bd9Sstevel@tonic-gate 40087c478bd9Sstevel@tonic-gate for (way = 0; way < CH_PCACHE_NWAY; way++, pcp++) { 40097c478bd9Sstevel@tonic-gate /* 40107c478bd9Sstevel@tonic-gate * Perform diagnostic read. 40117c478bd9Sstevel@tonic-gate */ 40127c478bd9Sstevel@tonic-gate get_pcache_dtag(index + way * pc_set_size, 40137c478bd9Sstevel@tonic-gate (uint64_t *)&tmp_pcp); 40147c478bd9Sstevel@tonic-gate /* 40157c478bd9Sstevel@tonic-gate * Check data array for odd parity. There are 8 parity 40167c478bd9Sstevel@tonic-gate * bits (bits 57:50 of ASI_PCACHE_STATUS_DATA) and each 40177c478bd9Sstevel@tonic-gate * of those bits covers exactly 8 bytes of the data 40187c478bd9Sstevel@tonic-gate * array: 40197c478bd9Sstevel@tonic-gate * 40207c478bd9Sstevel@tonic-gate * parity bit P$ data bytes covered 40217c478bd9Sstevel@tonic-gate * ---------- --------------------- 40227c478bd9Sstevel@tonic-gate * 50 63:56 40237c478bd9Sstevel@tonic-gate * 51 55:48 40247c478bd9Sstevel@tonic-gate * 52 47:40 40257c478bd9Sstevel@tonic-gate * 53 39:32 40267c478bd9Sstevel@tonic-gate * 54 31:24 40277c478bd9Sstevel@tonic-gate * 55 23:16 40287c478bd9Sstevel@tonic-gate * 56 15:8 40297c478bd9Sstevel@tonic-gate * 57 7:0 40307c478bd9Sstevel@tonic-gate */ 40317c478bd9Sstevel@tonic-gate parity_bits = PN_PC_PARITY_BITS(tmp_pcp.pc_status); 40327c478bd9Sstevel@tonic-gate for (word = 0; word < pc_data_words; word++) { 40337c478bd9Sstevel@tonic-gate pbit = (parity_bits >> (pc_data_words - word - 1)) & 1; 40347c478bd9Sstevel@tonic-gate if ((popc64(tmp_pcp.pc_data[word]) & 1) ^ pbit) { 40357c478bd9Sstevel@tonic-gate /* 40367c478bd9Sstevel@tonic-gate * If this is the first error log detailed 40377c478bd9Sstevel@tonic-gate * information about it. Otherwise just record 40387c478bd9Sstevel@tonic-gate * the fact that we found another error. 40397c478bd9Sstevel@tonic-gate */ 40407c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 40417c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = way; 40427c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_cache = 40437c478bd9Sstevel@tonic-gate CPU_PC_PARITY; 40447c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = 40457c478bd9Sstevel@tonic-gate word * sizeof (uint64_t); 40467c478bd9Sstevel@tonic-gate bcopy(&tmp_pcp, pcp, 40477c478bd9Sstevel@tonic-gate sizeof (ch_pc_data_t)); 40487c478bd9Sstevel@tonic-gate } 40497c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 40507c478bd9Sstevel@tonic-gate } 40517c478bd9Sstevel@tonic-gate } 40527c478bd9Sstevel@tonic-gate } 40537c478bd9Sstevel@tonic-gate } 40547c478bd9Sstevel@tonic-gate 40557c478bd9Sstevel@tonic-gate 40567c478bd9Sstevel@tonic-gate /* 40577c478bd9Sstevel@tonic-gate * Add L1 Data cache data to the ereport payload. 40587c478bd9Sstevel@tonic-gate */ 40597c478bd9Sstevel@tonic-gate static void 40607c478bd9Sstevel@tonic-gate cpu_payload_add_dcache(struct async_flt *aflt, nvlist_t *nvl) 40617c478bd9Sstevel@tonic-gate { 40627c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 40637c478bd9Sstevel@tonic-gate ch_dc_data_t *dcp; 40647c478bd9Sstevel@tonic-gate ch_dc_data_t dcdata[CH_DCACHE_NWAY]; 40657c478bd9Sstevel@tonic-gate uint_t nelem; 40667c478bd9Sstevel@tonic-gate int i, ways_to_check, ways_logged = 0; 40677c478bd9Sstevel@tonic-gate 40687c478bd9Sstevel@tonic-gate /* 40697c478bd9Sstevel@tonic-gate * If this is an D$ fault then there may be multiple 40707c478bd9Sstevel@tonic-gate * ways captured in the ch_parity_log_t structure. 40717c478bd9Sstevel@tonic-gate * Otherwise, there will be at most one way captured 40727c478bd9Sstevel@tonic-gate * in the ch_diag_data_t struct. 40737c478bd9Sstevel@tonic-gate * Check each way to see if it should be encoded. 40747c478bd9Sstevel@tonic-gate */ 40757c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_DC_PARITY) 40767c478bd9Sstevel@tonic-gate ways_to_check = CH_DCACHE_NWAY; 40777c478bd9Sstevel@tonic-gate else 40787c478bd9Sstevel@tonic-gate ways_to_check = 1; 40797c478bd9Sstevel@tonic-gate for (i = 0; i < ways_to_check; i++) { 40807c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_DC_PARITY) 40817c478bd9Sstevel@tonic-gate dcp = &ch_flt->parity_data.dpe.cpl_dc[i]; 40827c478bd9Sstevel@tonic-gate else 40837c478bd9Sstevel@tonic-gate dcp = &ch_flt->flt_diag_data.chd_dc_data; 40847c478bd9Sstevel@tonic-gate if (dcp->dc_logflag == DC_LOGFLAG_MAGIC) { 40857c478bd9Sstevel@tonic-gate bcopy(dcp, &dcdata[ways_logged], 40867c478bd9Sstevel@tonic-gate sizeof (ch_dc_data_t)); 40877c478bd9Sstevel@tonic-gate ways_logged++; 40887c478bd9Sstevel@tonic-gate } 40897c478bd9Sstevel@tonic-gate } 40907c478bd9Sstevel@tonic-gate 40917c478bd9Sstevel@tonic-gate /* 40927c478bd9Sstevel@tonic-gate * Add the dcache data to the payload. 40937c478bd9Sstevel@tonic-gate */ 40947c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_WAYS, 40957c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 40967c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 40977c478bd9Sstevel@tonic-gate nelem = sizeof (ch_dc_data_t) / sizeof (uint64_t) * ways_logged; 40987c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_DATA, 40997c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)dcdata, NULL); 41007c478bd9Sstevel@tonic-gate } 41017c478bd9Sstevel@tonic-gate } 41027c478bd9Sstevel@tonic-gate 41037c478bd9Sstevel@tonic-gate /* 41047c478bd9Sstevel@tonic-gate * Add L1 Instruction cache data to the ereport payload. 41057c478bd9Sstevel@tonic-gate */ 41067c478bd9Sstevel@tonic-gate static void 41077c478bd9Sstevel@tonic-gate cpu_payload_add_icache(struct async_flt *aflt, nvlist_t *nvl) 41087c478bd9Sstevel@tonic-gate { 41097c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 41107c478bd9Sstevel@tonic-gate ch_ic_data_t *icp; 41117c478bd9Sstevel@tonic-gate ch_ic_data_t icdata[CH_ICACHE_NWAY]; 41127c478bd9Sstevel@tonic-gate uint_t nelem; 41137c478bd9Sstevel@tonic-gate int i, ways_to_check, ways_logged = 0; 41147c478bd9Sstevel@tonic-gate 41157c478bd9Sstevel@tonic-gate /* 41167c478bd9Sstevel@tonic-gate * If this is an I$ fault then there may be multiple 41177c478bd9Sstevel@tonic-gate * ways captured in the ch_parity_log_t structure. 41187c478bd9Sstevel@tonic-gate * Otherwise, there will be at most one way captured 41197c478bd9Sstevel@tonic-gate * in the ch_diag_data_t struct. 41207c478bd9Sstevel@tonic-gate * Check each way to see if it should be encoded. 41217c478bd9Sstevel@tonic-gate */ 41227c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_IC_PARITY) 41237c478bd9Sstevel@tonic-gate ways_to_check = CH_ICACHE_NWAY; 41247c478bd9Sstevel@tonic-gate else 41257c478bd9Sstevel@tonic-gate ways_to_check = 1; 41267c478bd9Sstevel@tonic-gate for (i = 0; i < ways_to_check; i++) { 41277c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_IC_PARITY) 41287c478bd9Sstevel@tonic-gate icp = &ch_flt->parity_data.ipe.cpl_ic[i]; 41297c478bd9Sstevel@tonic-gate else 41307c478bd9Sstevel@tonic-gate icp = &ch_flt->flt_diag_data.chd_ic_data; 41317c478bd9Sstevel@tonic-gate if (icp->ic_logflag == IC_LOGFLAG_MAGIC) { 41327c478bd9Sstevel@tonic-gate bcopy(icp, &icdata[ways_logged], 41337c478bd9Sstevel@tonic-gate sizeof (ch_ic_data_t)); 41347c478bd9Sstevel@tonic-gate ways_logged++; 41357c478bd9Sstevel@tonic-gate } 41367c478bd9Sstevel@tonic-gate } 41377c478bd9Sstevel@tonic-gate 41387c478bd9Sstevel@tonic-gate /* 41397c478bd9Sstevel@tonic-gate * Add the icache data to the payload. 41407c478bd9Sstevel@tonic-gate */ 41417c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_WAYS, 41427c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 41437c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 41447c478bd9Sstevel@tonic-gate nelem = sizeof (ch_ic_data_t) / sizeof (uint64_t) * ways_logged; 41457c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_DATA, 41467c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)icdata, NULL); 41477c478bd9Sstevel@tonic-gate } 41487c478bd9Sstevel@tonic-gate } 41497c478bd9Sstevel@tonic-gate 41507c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 41517c478bd9Sstevel@tonic-gate 41527c478bd9Sstevel@tonic-gate /* 41537c478bd9Sstevel@tonic-gate * Add ecache data to payload. 41547c478bd9Sstevel@tonic-gate */ 41557c478bd9Sstevel@tonic-gate static void 41567c478bd9Sstevel@tonic-gate cpu_payload_add_ecache(struct async_flt *aflt, nvlist_t *nvl) 41577c478bd9Sstevel@tonic-gate { 41587c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 41597c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp; 41607c478bd9Sstevel@tonic-gate ch_ec_data_t ecdata[CHD_EC_DATA_SETS]; 41617c478bd9Sstevel@tonic-gate uint_t nelem; 41627c478bd9Sstevel@tonic-gate int i, ways_logged = 0; 41637c478bd9Sstevel@tonic-gate 41647c478bd9Sstevel@tonic-gate /* 41657c478bd9Sstevel@tonic-gate * Check each way to see if it should be encoded 41667c478bd9Sstevel@tonic-gate * and concatinate it into a temporary buffer. 41677c478bd9Sstevel@tonic-gate */ 41687c478bd9Sstevel@tonic-gate for (i = 0; i < CHD_EC_DATA_SETS; i++) { 41697c478bd9Sstevel@tonic-gate ecp = &ch_flt->flt_diag_data.chd_ec_data[i]; 41707c478bd9Sstevel@tonic-gate if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 41717c478bd9Sstevel@tonic-gate bcopy(ecp, &ecdata[ways_logged], 41727c478bd9Sstevel@tonic-gate sizeof (ch_ec_data_t)); 41737c478bd9Sstevel@tonic-gate ways_logged++; 41747c478bd9Sstevel@tonic-gate } 41757c478bd9Sstevel@tonic-gate } 41767c478bd9Sstevel@tonic-gate 41777c478bd9Sstevel@tonic-gate /* 41787c478bd9Sstevel@tonic-gate * Panther CPUs have an additional level of cache and so 41797c478bd9Sstevel@tonic-gate * what we just collected was the L3 (ecache) and not the 41807c478bd9Sstevel@tonic-gate * L2 cache. 41817c478bd9Sstevel@tonic-gate */ 41827c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 41837c478bd9Sstevel@tonic-gate /* 41847c478bd9Sstevel@tonic-gate * Add the L3 (ecache) data to the payload. 41857c478bd9Sstevel@tonic-gate */ 41867c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_WAYS, 41877c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 41887c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 41897c478bd9Sstevel@tonic-gate nelem = sizeof (ch_ec_data_t) / 41907c478bd9Sstevel@tonic-gate sizeof (uint64_t) * ways_logged; 41917c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_DATA, 41927c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, 41937c478bd9Sstevel@tonic-gate (uint64_t *)ecdata, NULL); 41947c478bd9Sstevel@tonic-gate } 41957c478bd9Sstevel@tonic-gate 41967c478bd9Sstevel@tonic-gate /* 41977c478bd9Sstevel@tonic-gate * Now collect the L2 cache. 41987c478bd9Sstevel@tonic-gate */ 41997c478bd9Sstevel@tonic-gate ways_logged = 0; 42007c478bd9Sstevel@tonic-gate for (i = 0; i < PN_L2_NWAYS; i++) { 42017c478bd9Sstevel@tonic-gate ecp = &ch_flt->flt_diag_data.chd_l2_data[i]; 42027c478bd9Sstevel@tonic-gate if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 42037c478bd9Sstevel@tonic-gate bcopy(ecp, &ecdata[ways_logged], 42047c478bd9Sstevel@tonic-gate sizeof (ch_ec_data_t)); 42057c478bd9Sstevel@tonic-gate ways_logged++; 42067c478bd9Sstevel@tonic-gate } 42077c478bd9Sstevel@tonic-gate } 42087c478bd9Sstevel@tonic-gate } 42097c478bd9Sstevel@tonic-gate 42107c478bd9Sstevel@tonic-gate /* 42117c478bd9Sstevel@tonic-gate * Add the L2 cache data to the payload. 42127c478bd9Sstevel@tonic-gate */ 42137c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_WAYS, 42147c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 42157c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 42167c478bd9Sstevel@tonic-gate nelem = sizeof (ch_ec_data_t) / 42177c478bd9Sstevel@tonic-gate sizeof (uint64_t) * ways_logged; 42187c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_DATA, 42197c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)ecdata, NULL); 42207c478bd9Sstevel@tonic-gate } 42217c478bd9Sstevel@tonic-gate } 42227c478bd9Sstevel@tonic-gate 42237c478bd9Sstevel@tonic-gate /* 42247c478bd9Sstevel@tonic-gate * Encode the data saved in the ch_async_flt_t struct into 42257c478bd9Sstevel@tonic-gate * the FM ereport payload. 42267c478bd9Sstevel@tonic-gate */ 42277c478bd9Sstevel@tonic-gate static void 42287c478bd9Sstevel@tonic-gate cpu_payload_add_aflt(struct async_flt *aflt, nvlist_t *payload, 42297c478bd9Sstevel@tonic-gate nvlist_t *resource, int *afar_status, int *synd_status) 42307c478bd9Sstevel@tonic-gate { 42317c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 42327c478bd9Sstevel@tonic-gate *synd_status = AFLT_STAT_INVALID; 42337c478bd9Sstevel@tonic-gate *afar_status = AFLT_STAT_INVALID; 42347c478bd9Sstevel@tonic-gate 42357c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR) { 42367c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR, 42377c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, aflt->flt_stat, NULL); 42387c478bd9Sstevel@tonic-gate } 42397c478bd9Sstevel@tonic-gate 42407c478bd9Sstevel@tonic-gate if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR_EXT) && 42417c478bd9Sstevel@tonic-gate IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 42427c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR_EXT, 42437c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, ch_flt->afsr_ext, NULL); 42447c478bd9Sstevel@tonic-gate } 42457c478bd9Sstevel@tonic-gate 42467c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR_STATUS) { 42477c478bd9Sstevel@tonic-gate *afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 42487c478bd9Sstevel@tonic-gate ch_flt->flt_bit); 42497c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR_STATUS, 42507c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)*afar_status, NULL); 42517c478bd9Sstevel@tonic-gate } 42527c478bd9Sstevel@tonic-gate 42537c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR) { 42547c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR, 42557c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, aflt->flt_addr, NULL); 42567c478bd9Sstevel@tonic-gate } 42577c478bd9Sstevel@tonic-gate 42587c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PC) { 42597c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PC, 42607c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, (uint64_t)aflt->flt_pc, NULL); 42617c478bd9Sstevel@tonic-gate } 42627c478bd9Sstevel@tonic-gate 42637c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TL) { 42647c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TL, 42657c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)aflt->flt_tl, NULL); 42667c478bd9Sstevel@tonic-gate } 42677c478bd9Sstevel@tonic-gate 42687c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TT) { 42697c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TT, 42707c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, flt_to_trap_type(aflt), NULL); 42717c478bd9Sstevel@tonic-gate } 42727c478bd9Sstevel@tonic-gate 42737c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PRIV) { 42747c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PRIV, 42757c478bd9Sstevel@tonic-gate DATA_TYPE_BOOLEAN_VALUE, 42767c478bd9Sstevel@tonic-gate (aflt->flt_priv ? B_TRUE : B_FALSE), NULL); 42777c478bd9Sstevel@tonic-gate } 42787c478bd9Sstevel@tonic-gate 42797c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ME) { 42807c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ME, 42817c478bd9Sstevel@tonic-gate DATA_TYPE_BOOLEAN_VALUE, 42827c478bd9Sstevel@tonic-gate (aflt->flt_stat & C_AFSR_ME) ? B_TRUE : B_FALSE, NULL); 42837c478bd9Sstevel@tonic-gate } 42847c478bd9Sstevel@tonic-gate 42857c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND_STATUS) { 42867c478bd9Sstevel@tonic-gate *synd_status = afsr_to_synd_status(aflt->flt_inst, 42877c478bd9Sstevel@tonic-gate ch_flt->afsr_errs, ch_flt->flt_bit); 42887c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND_STATUS, 42897c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)*synd_status, NULL); 42907c478bd9Sstevel@tonic-gate } 42917c478bd9Sstevel@tonic-gate 42927c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND) { 42937c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND, 42947c478bd9Sstevel@tonic-gate DATA_TYPE_UINT16, (uint16_t)aflt->flt_synd, NULL); 42957c478bd9Sstevel@tonic-gate } 42967c478bd9Sstevel@tonic-gate 42977c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_TYPE) { 42987c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_TYPE, 42997c478bd9Sstevel@tonic-gate DATA_TYPE_STRING, flt_to_error_type(aflt), NULL); 43007c478bd9Sstevel@tonic-gate } 43017c478bd9Sstevel@tonic-gate 43027c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_DISP) { 43037c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_DISP, 43047c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, aflt->flt_disp, NULL); 43057c478bd9Sstevel@tonic-gate } 43067c478bd9Sstevel@tonic-gate 43077c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L2) 43087c478bd9Sstevel@tonic-gate cpu_payload_add_ecache(aflt, payload); 43097c478bd9Sstevel@tonic-gate 43107c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_COPYFUNCTION) { 43117c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_COPYFUNCTION, 43127c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)aflt->flt_status & 0xff, NULL); 43137c478bd9Sstevel@tonic-gate } 43147c478bd9Sstevel@tonic-gate 43157c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_HOWDETECTED) { 43167c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_HOWDETECTED, 43177c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)(aflt->flt_status >> 8), NULL); 43187c478bd9Sstevel@tonic-gate } 43197c478bd9Sstevel@tonic-gate 43207c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_INSTRBLOCK) { 43217c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_INSTRBLOCK, 43227c478bd9Sstevel@tonic-gate DATA_TYPE_UINT32_ARRAY, 16, 43237c478bd9Sstevel@tonic-gate (uint32_t *)&ch_flt->flt_fpdata, NULL); 43247c478bd9Sstevel@tonic-gate } 43257c478bd9Sstevel@tonic-gate 43267c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 43277c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1D) 43287c478bd9Sstevel@tonic-gate cpu_payload_add_dcache(aflt, payload); 43297c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1I) 43307c478bd9Sstevel@tonic-gate cpu_payload_add_icache(aflt, payload); 43317c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 43327c478bd9Sstevel@tonic-gate 43337c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 43347c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1P) 43357c478bd9Sstevel@tonic-gate cpu_payload_add_pcache(aflt, payload); 43367c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_TLB) 43377c478bd9Sstevel@tonic-gate cpu_payload_add_tlb(aflt, payload); 43387c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 43397c478bd9Sstevel@tonic-gate /* 43407c478bd9Sstevel@tonic-gate * Create the FMRI that goes into the payload 43417c478bd9Sstevel@tonic-gate * and contains the unum info if necessary. 43427c478bd9Sstevel@tonic-gate */ 43437c478bd9Sstevel@tonic-gate if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_RESOURCE) && 43447c478bd9Sstevel@tonic-gate (*afar_status == AFLT_STAT_VALID)) { 43457c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 43467c478bd9Sstevel@tonic-gate int len; 43477c478bd9Sstevel@tonic-gate 43487c478bd9Sstevel@tonic-gate if (cpu_get_mem_unum_aflt(*synd_status, aflt, unum, 43497c478bd9Sstevel@tonic-gate UNUM_NAMLEN, &len) == 0) { 43507c478bd9Sstevel@tonic-gate fm_fmri_mem_set(resource, FM_MEM_SCHEME_VERSION, 43517c478bd9Sstevel@tonic-gate NULL, unum, NULL); 43527c478bd9Sstevel@tonic-gate fm_payload_set(payload, 43537c478bd9Sstevel@tonic-gate FM_EREPORT_PAYLOAD_NAME_RESOURCE, 43547c478bd9Sstevel@tonic-gate DATA_TYPE_NVLIST, resource, NULL); 43557c478bd9Sstevel@tonic-gate } 43567c478bd9Sstevel@tonic-gate } 43577c478bd9Sstevel@tonic-gate } 43587c478bd9Sstevel@tonic-gate 43597c478bd9Sstevel@tonic-gate /* 43607c478bd9Sstevel@tonic-gate * Initialize the way info if necessary. 43617c478bd9Sstevel@tonic-gate */ 43627c478bd9Sstevel@tonic-gate void 43637c478bd9Sstevel@tonic-gate cpu_ereport_init(struct async_flt *aflt) 43647c478bd9Sstevel@tonic-gate { 43657c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 43667c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 43677c478bd9Sstevel@tonic-gate ch_ec_data_t *l2p = &ch_flt->flt_diag_data.chd_l2_data[0]; 43687c478bd9Sstevel@tonic-gate int i; 43697c478bd9Sstevel@tonic-gate 43707c478bd9Sstevel@tonic-gate /* 43717c478bd9Sstevel@tonic-gate * Initialize the info in the CPU logout structure. 43727c478bd9Sstevel@tonic-gate * The I$/D$ way information is not initialized here 43737c478bd9Sstevel@tonic-gate * since it is captured in the logout assembly code. 43747c478bd9Sstevel@tonic-gate */ 43757c478bd9Sstevel@tonic-gate for (i = 0; i < CHD_EC_DATA_SETS; i++) 43767c478bd9Sstevel@tonic-gate (ecp + i)->ec_way = i; 43777c478bd9Sstevel@tonic-gate 43787c478bd9Sstevel@tonic-gate for (i = 0; i < PN_L2_NWAYS; i++) 43797c478bd9Sstevel@tonic-gate (l2p + i)->ec_way = i; 43807c478bd9Sstevel@tonic-gate } 43817c478bd9Sstevel@tonic-gate 43827c478bd9Sstevel@tonic-gate /* 43837c478bd9Sstevel@tonic-gate * Returns whether fault address is valid for this error bit and 43847c478bd9Sstevel@tonic-gate * whether the address is "in memory" (i.e. pf_is_memory returns 1). 43857c478bd9Sstevel@tonic-gate */ 43867c478bd9Sstevel@tonic-gate int 43877c478bd9Sstevel@tonic-gate cpu_flt_in_memory(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit) 43887c478bd9Sstevel@tonic-gate { 43897c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 43907c478bd9Sstevel@tonic-gate 43917c478bd9Sstevel@tonic-gate return ((aflt->flt_stat & C_AFSR_MEMORY) && 43927c478bd9Sstevel@tonic-gate afsr_to_afar_status(ch_flt->afsr_errs, t_afsr_bit) == 43937c478bd9Sstevel@tonic-gate AFLT_STAT_VALID && 43947c478bd9Sstevel@tonic-gate pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)); 43957c478bd9Sstevel@tonic-gate } 43967c478bd9Sstevel@tonic-gate 43977c478bd9Sstevel@tonic-gate static void 43987c478bd9Sstevel@tonic-gate cpu_log_diag_info(ch_async_flt_t *ch_flt) 43997c478bd9Sstevel@tonic-gate { 44007c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 44017c478bd9Sstevel@tonic-gate ch_dc_data_t *dcp = &ch_flt->flt_diag_data.chd_dc_data; 44027c478bd9Sstevel@tonic-gate ch_ic_data_t *icp = &ch_flt->flt_diag_data.chd_ic_data; 44037c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 44047c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 44057c478bd9Sstevel@tonic-gate int i, nway; 44067c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_ECACHE_ASSOC */ 44077c478bd9Sstevel@tonic-gate 44087c478bd9Sstevel@tonic-gate /* 44097c478bd9Sstevel@tonic-gate * Check if the CPU log out captured was valid. 44107c478bd9Sstevel@tonic-gate */ 44117c478bd9Sstevel@tonic-gate if (ch_flt->flt_diag_data.chd_afar == LOGOUT_INVALID || 44127c478bd9Sstevel@tonic-gate ch_flt->flt_data_incomplete) 44137c478bd9Sstevel@tonic-gate return; 44147c478bd9Sstevel@tonic-gate 44157c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 44167c478bd9Sstevel@tonic-gate nway = cpu_ecache_nway(); 44177c478bd9Sstevel@tonic-gate i = cpu_ecache_line_valid(ch_flt); 44187c478bd9Sstevel@tonic-gate if (i == 0 || i > nway) { 44197c478bd9Sstevel@tonic-gate for (i = 0; i < nway; i++) 44207c478bd9Sstevel@tonic-gate ecp[i].ec_logflag = EC_LOGFLAG_MAGIC; 44217c478bd9Sstevel@tonic-gate } else 44227c478bd9Sstevel@tonic-gate ecp[i - 1].ec_logflag = EC_LOGFLAG_MAGIC; 44237c478bd9Sstevel@tonic-gate #else /* CPU_IMP_ECACHE_ASSOC */ 44247c478bd9Sstevel@tonic-gate ecp->ec_logflag = EC_LOGFLAG_MAGIC; 44257c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_ECACHE_ASSOC */ 44267c478bd9Sstevel@tonic-gate 44277c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 44287c478bd9Sstevel@tonic-gate pn_cpu_log_diag_l2_info(ch_flt); 44297c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 44307c478bd9Sstevel@tonic-gate 44317c478bd9Sstevel@tonic-gate if (CH_DCTAG_MATCH(dcp->dc_tag, aflt->flt_addr)) { 44327c478bd9Sstevel@tonic-gate dcp->dc_way = CH_DCIDX_TO_WAY(dcp->dc_idx); 44337c478bd9Sstevel@tonic-gate dcp->dc_logflag = DC_LOGFLAG_MAGIC; 44347c478bd9Sstevel@tonic-gate } 44357c478bd9Sstevel@tonic-gate 44367c478bd9Sstevel@tonic-gate if (CH_ICTAG_MATCH(icp, aflt->flt_addr)) { 44377c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 44387c478bd9Sstevel@tonic-gate icp->ic_way = PN_ICIDX_TO_WAY(icp->ic_idx); 44397c478bd9Sstevel@tonic-gate else 44407c478bd9Sstevel@tonic-gate icp->ic_way = CH_ICIDX_TO_WAY(icp->ic_idx); 44417c478bd9Sstevel@tonic-gate icp->ic_logflag = IC_LOGFLAG_MAGIC; 44427c478bd9Sstevel@tonic-gate } 44437c478bd9Sstevel@tonic-gate } 44447c478bd9Sstevel@tonic-gate 44457c478bd9Sstevel@tonic-gate /* 44467c478bd9Sstevel@tonic-gate * Cheetah ECC calculation. 44477c478bd9Sstevel@tonic-gate * 44487c478bd9Sstevel@tonic-gate * We only need to do the calculation on the data bits and can ignore check 44497c478bd9Sstevel@tonic-gate * bit and Mtag bit terms in the calculation. 44507c478bd9Sstevel@tonic-gate */ 44517c478bd9Sstevel@tonic-gate static uint64_t ch_ecc_table[9][2] = { 44527c478bd9Sstevel@tonic-gate /* 44537c478bd9Sstevel@tonic-gate * low order 64-bits high-order 64-bits 44547c478bd9Sstevel@tonic-gate */ 44557c478bd9Sstevel@tonic-gate { 0x46bffffeccd1177f, 0x488800022100014c }, 44567c478bd9Sstevel@tonic-gate { 0x42fccc81331ff77f, 0x14424f1010249184 }, 44577c478bd9Sstevel@tonic-gate { 0x8898827c222f1ffe, 0x22c1222808184aaf }, 44587c478bd9Sstevel@tonic-gate { 0xf7632203e131ccf1, 0xe1241121848292b8 }, 44597c478bd9Sstevel@tonic-gate { 0x7f5511421b113809, 0x901c88d84288aafe }, 44607c478bd9Sstevel@tonic-gate { 0x1d49412184882487, 0x8f338c87c044c6ef }, 44617c478bd9Sstevel@tonic-gate { 0xf552181014448344, 0x7ff8f4443e411911 }, 44627c478bd9Sstevel@tonic-gate { 0x2189240808f24228, 0xfeeff8cc81333f42 }, 44637c478bd9Sstevel@tonic-gate { 0x3280008440001112, 0xfee88b337ffffd62 }, 44647c478bd9Sstevel@tonic-gate }; 44657c478bd9Sstevel@tonic-gate 44667c478bd9Sstevel@tonic-gate /* 44677c478bd9Sstevel@tonic-gate * 64-bit population count, use well-known popcnt trick. 44687c478bd9Sstevel@tonic-gate * We could use the UltraSPARC V9 POPC instruction, but some 44697c478bd9Sstevel@tonic-gate * CPUs including Cheetahplus and Jaguar do not support that 44707c478bd9Sstevel@tonic-gate * instruction. 44717c478bd9Sstevel@tonic-gate */ 44727c478bd9Sstevel@tonic-gate int 44737c478bd9Sstevel@tonic-gate popc64(uint64_t val) 44747c478bd9Sstevel@tonic-gate { 44757c478bd9Sstevel@tonic-gate int cnt; 44767c478bd9Sstevel@tonic-gate 44777c478bd9Sstevel@tonic-gate for (cnt = 0; val != 0; val &= val - 1) 44787c478bd9Sstevel@tonic-gate cnt++; 44797c478bd9Sstevel@tonic-gate return (cnt); 44807c478bd9Sstevel@tonic-gate } 44817c478bd9Sstevel@tonic-gate 44827c478bd9Sstevel@tonic-gate /* 44837c478bd9Sstevel@tonic-gate * Generate the 9 ECC bits for the 128-bit chunk based on the table above. 44847c478bd9Sstevel@tonic-gate * Note that xor'ing an odd number of 1 bits == 1 and xor'ing an even number 44857c478bd9Sstevel@tonic-gate * of 1 bits == 0, so we can just use the least significant bit of the popcnt 44867c478bd9Sstevel@tonic-gate * instead of doing all the xor's. 44877c478bd9Sstevel@tonic-gate */ 44887c478bd9Sstevel@tonic-gate uint32_t 44897c478bd9Sstevel@tonic-gate us3_gen_ecc(uint64_t data_low, uint64_t data_high) 44907c478bd9Sstevel@tonic-gate { 44917c478bd9Sstevel@tonic-gate int bitno, s; 44927c478bd9Sstevel@tonic-gate int synd = 0; 44937c478bd9Sstevel@tonic-gate 44947c478bd9Sstevel@tonic-gate for (bitno = 0; bitno < 9; bitno++) { 44957c478bd9Sstevel@tonic-gate s = (popc64(data_low & ch_ecc_table[bitno][0]) + 44967c478bd9Sstevel@tonic-gate popc64(data_high & ch_ecc_table[bitno][1])) & 1; 44977c478bd9Sstevel@tonic-gate synd |= (s << bitno); 44987c478bd9Sstevel@tonic-gate } 44997c478bd9Sstevel@tonic-gate return (synd); 45007c478bd9Sstevel@tonic-gate 45017c478bd9Sstevel@tonic-gate } 45027c478bd9Sstevel@tonic-gate 45037c478bd9Sstevel@tonic-gate /* 45047c478bd9Sstevel@tonic-gate * Queue one event based on ecc_type_to_info entry. If the event has an AFT1 45057c478bd9Sstevel@tonic-gate * tag associated with it or is a fatal event (aflt_panic set), it is sent to 45067c478bd9Sstevel@tonic-gate * the UE event queue. Otherwise it is dispatched to the CE event queue. 45077c478bd9Sstevel@tonic-gate */ 45087c478bd9Sstevel@tonic-gate static void 45097c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 45107c478bd9Sstevel@tonic-gate ecc_type_to_info_t *eccp, ch_diag_data_t *cdp) 45117c478bd9Sstevel@tonic-gate { 45127c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 45137c478bd9Sstevel@tonic-gate 45147c478bd9Sstevel@tonic-gate if (reason && 45157c478bd9Sstevel@tonic-gate strlen(reason) + strlen(eccp->ec_reason) < MAX_REASON_STRING) { 45167c478bd9Sstevel@tonic-gate (void) strcat(reason, eccp->ec_reason); 45177c478bd9Sstevel@tonic-gate } 45187c478bd9Sstevel@tonic-gate 45197c478bd9Sstevel@tonic-gate ch_flt->flt_bit = eccp->ec_afsr_bit; 45207c478bd9Sstevel@tonic-gate ch_flt->flt_type = eccp->ec_flt_type; 45217c478bd9Sstevel@tonic-gate if (cdp != NULL && cdp->chd_afar != LOGOUT_INVALID) 45227c478bd9Sstevel@tonic-gate ch_flt->flt_diag_data = *cdp; 45237c478bd9Sstevel@tonic-gate else 45247c478bd9Sstevel@tonic-gate ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 45257c478bd9Sstevel@tonic-gate aflt->flt_in_memory = cpu_flt_in_memory(ch_flt, ch_flt->flt_bit); 45267c478bd9Sstevel@tonic-gate 45277c478bd9Sstevel@tonic-gate if (ch_flt->flt_bit & C_AFSR_MSYND_ERRS) 45287c478bd9Sstevel@tonic-gate aflt->flt_synd = GET_M_SYND(aflt->flt_stat); 45297c478bd9Sstevel@tonic-gate else if (ch_flt->flt_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) 45307c478bd9Sstevel@tonic-gate aflt->flt_synd = GET_E_SYND(aflt->flt_stat); 45317c478bd9Sstevel@tonic-gate else 45327c478bd9Sstevel@tonic-gate aflt->flt_synd = 0; 45337c478bd9Sstevel@tonic-gate 45347c478bd9Sstevel@tonic-gate aflt->flt_payload = eccp->ec_err_payload; 45357c478bd9Sstevel@tonic-gate 45367c478bd9Sstevel@tonic-gate if (aflt->flt_panic || (eccp->ec_afsr_bit & 45377c478bd9Sstevel@tonic-gate (C_AFSR_LEVEL1 | C_AFSR_EXT_LEVEL1))) 45387c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(eccp->ec_err_class, 45397c478bd9Sstevel@tonic-gate (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 45407c478bd9Sstevel@tonic-gate aflt->flt_panic); 45417c478bd9Sstevel@tonic-gate else 45427c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(eccp->ec_err_class, 45437c478bd9Sstevel@tonic-gate (void *)ch_flt, sizeof (ch_async_flt_t), ce_queue, 45447c478bd9Sstevel@tonic-gate aflt->flt_panic); 45457c478bd9Sstevel@tonic-gate } 45467c478bd9Sstevel@tonic-gate 45477c478bd9Sstevel@tonic-gate /* 45487c478bd9Sstevel@tonic-gate * Queue events on async event queue one event per error bit. First we 45497c478bd9Sstevel@tonic-gate * queue the events that we "expect" for the given trap, then we queue events 45507c478bd9Sstevel@tonic-gate * that we may not expect. Return number of events queued. 45517c478bd9Sstevel@tonic-gate */ 45527c478bd9Sstevel@tonic-gate int 45537c478bd9Sstevel@tonic-gate cpu_queue_events(ch_async_flt_t *ch_flt, char *reason, uint64_t t_afsr_errs, 45547c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop) 45557c478bd9Sstevel@tonic-gate { 45567c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 45577c478bd9Sstevel@tonic-gate ecc_type_to_info_t *eccp; 45587c478bd9Sstevel@tonic-gate int nevents = 0; 45597c478bd9Sstevel@tonic-gate uint64_t primary_afar = aflt->flt_addr, primary_afsr = aflt->flt_stat; 45607c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 45617c478bd9Sstevel@tonic-gate uint64_t orig_t_afsr_errs; 45627c478bd9Sstevel@tonic-gate #endif 45637c478bd9Sstevel@tonic-gate uint64_t primary_afsr_ext = ch_flt->afsr_ext; 45647c478bd9Sstevel@tonic-gate uint64_t primary_afsr_errs = ch_flt->afsr_errs; 45657c478bd9Sstevel@tonic-gate ch_diag_data_t *cdp = NULL; 45667c478bd9Sstevel@tonic-gate 45677c478bd9Sstevel@tonic-gate t_afsr_errs &= ((C_AFSR_ALL_ERRS & ~C_AFSR_ME) | C_AFSR_EXT_ALL_ERRS); 45687c478bd9Sstevel@tonic-gate 45697c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 45707c478bd9Sstevel@tonic-gate orig_t_afsr_errs = t_afsr_errs; 45717c478bd9Sstevel@tonic-gate 45727c478bd9Sstevel@tonic-gate /* 45737c478bd9Sstevel@tonic-gate * For Cheetah+, log the shadow AFSR/AFAR bits first. 45747c478bd9Sstevel@tonic-gate */ 45757c478bd9Sstevel@tonic-gate if (clop != NULL) { 45767c478bd9Sstevel@tonic-gate /* 45777c478bd9Sstevel@tonic-gate * Set the AFSR and AFAR fields to the shadow registers. The 45787c478bd9Sstevel@tonic-gate * flt_addr and flt_stat fields will be reset to the primaries 45797c478bd9Sstevel@tonic-gate * below, but the sdw_addr and sdw_stat will stay as the 45807c478bd9Sstevel@tonic-gate * secondaries. 45817c478bd9Sstevel@tonic-gate */ 45827c478bd9Sstevel@tonic-gate cdp = &clop->clo_sdw_data; 45837c478bd9Sstevel@tonic-gate aflt->flt_addr = ch_flt->flt_sdw_afar = cdp->chd_afar; 45847c478bd9Sstevel@tonic-gate aflt->flt_stat = ch_flt->flt_sdw_afsr = cdp->chd_afsr; 45857c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = ch_flt->flt_sdw_afsr_ext = cdp->chd_afsr_ext; 45867c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = (cdp->chd_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 45877c478bd9Sstevel@tonic-gate (cdp->chd_afsr & C_AFSR_ALL_ERRS); 45887c478bd9Sstevel@tonic-gate 45897c478bd9Sstevel@tonic-gate /* 45907c478bd9Sstevel@tonic-gate * If the primary and shadow AFSR differ, tag the shadow as 45917c478bd9Sstevel@tonic-gate * the first fault. 45927c478bd9Sstevel@tonic-gate */ 45937c478bd9Sstevel@tonic-gate if ((primary_afar != cdp->chd_afar) || 45947c478bd9Sstevel@tonic-gate (primary_afsr_errs != ch_flt->afsr_errs)) { 45957c478bd9Sstevel@tonic-gate aflt->flt_stat |= (1ull << C_AFSR_FIRSTFLT_SHIFT); 45967c478bd9Sstevel@tonic-gate } 45977c478bd9Sstevel@tonic-gate 45987c478bd9Sstevel@tonic-gate /* 45997c478bd9Sstevel@tonic-gate * Check AFSR bits as well as AFSR_EXT bits in order of 46007c478bd9Sstevel@tonic-gate * the AFAR overwrite priority. Our stored AFSR_EXT value 46017c478bd9Sstevel@tonic-gate * is expected to be zero for those CPUs which do not have 46027c478bd9Sstevel@tonic-gate * an AFSR_EXT register. 46037c478bd9Sstevel@tonic-gate */ 46047c478bd9Sstevel@tonic-gate for (eccp = ecc_type_to_info; eccp->ec_desc != NULL; eccp++) { 46057c478bd9Sstevel@tonic-gate if ((eccp->ec_afsr_bit & 46067c478bd9Sstevel@tonic-gate (ch_flt->afsr_errs & t_afsr_errs)) && 46077c478bd9Sstevel@tonic-gate ((eccp->ec_flags & aflt->flt_status) != 0)) { 46087c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_flt, reason, eccp, cdp); 46097c478bd9Sstevel@tonic-gate cdp = NULL; 46107c478bd9Sstevel@tonic-gate t_afsr_errs &= ~eccp->ec_afsr_bit; 46117c478bd9Sstevel@tonic-gate nevents++; 46127c478bd9Sstevel@tonic-gate } 46137c478bd9Sstevel@tonic-gate } 46147c478bd9Sstevel@tonic-gate 46157c478bd9Sstevel@tonic-gate /* 46167c478bd9Sstevel@tonic-gate * If the ME bit is on in the primary AFSR turn all the 46177c478bd9Sstevel@tonic-gate * error bits on again that may set the ME bit to make 46187c478bd9Sstevel@tonic-gate * sure we see the ME AFSR error logs. 46197c478bd9Sstevel@tonic-gate */ 46207c478bd9Sstevel@tonic-gate if ((primary_afsr & C_AFSR_ME) != 0) 46217c478bd9Sstevel@tonic-gate t_afsr_errs = (orig_t_afsr_errs & C_AFSR_ALL_ME_ERRS); 46227c478bd9Sstevel@tonic-gate } 46237c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 46247c478bd9Sstevel@tonic-gate 46257c478bd9Sstevel@tonic-gate if (clop != NULL) 46267c478bd9Sstevel@tonic-gate cdp = &clop->clo_data; 46277c478bd9Sstevel@tonic-gate 46287c478bd9Sstevel@tonic-gate /* 46297c478bd9Sstevel@tonic-gate * Queue expected errors, error bit and fault type must match 46307c478bd9Sstevel@tonic-gate * in the ecc_type_to_info table. 46317c478bd9Sstevel@tonic-gate */ 46327c478bd9Sstevel@tonic-gate for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 46337c478bd9Sstevel@tonic-gate eccp++) { 46347c478bd9Sstevel@tonic-gate if ((eccp->ec_afsr_bit & t_afsr_errs) != 0 && 46357c478bd9Sstevel@tonic-gate (eccp->ec_flags & aflt->flt_status) != 0) { 46367c478bd9Sstevel@tonic-gate #if defined(SERRANO) 46377c478bd9Sstevel@tonic-gate /* 46387c478bd9Sstevel@tonic-gate * For FRC/FRU errors on Serrano the afar2 captures 46397c478bd9Sstevel@tonic-gate * the address and the associated data is 46407c478bd9Sstevel@tonic-gate * in the shadow logout area. 46417c478bd9Sstevel@tonic-gate */ 46427c478bd9Sstevel@tonic-gate if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 46437c478bd9Sstevel@tonic-gate if (clop != NULL) 46447c478bd9Sstevel@tonic-gate cdp = &clop->clo_sdw_data; 46457c478bd9Sstevel@tonic-gate aflt->flt_addr = ch_flt->afar2; 46467c478bd9Sstevel@tonic-gate } else { 46477c478bd9Sstevel@tonic-gate if (clop != NULL) 46487c478bd9Sstevel@tonic-gate cdp = &clop->clo_data; 46497c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 46507c478bd9Sstevel@tonic-gate } 46517c478bd9Sstevel@tonic-gate #else /* SERRANO */ 46527c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 46537c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 46547c478bd9Sstevel@tonic-gate aflt->flt_stat = primary_afsr; 46557c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = primary_afsr_ext; 46567c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = primary_afsr_errs; 46577c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_flt, reason, eccp, cdp); 46587c478bd9Sstevel@tonic-gate cdp = NULL; 46597c478bd9Sstevel@tonic-gate t_afsr_errs &= ~eccp->ec_afsr_bit; 46607c478bd9Sstevel@tonic-gate nevents++; 46617c478bd9Sstevel@tonic-gate } 46627c478bd9Sstevel@tonic-gate } 46637c478bd9Sstevel@tonic-gate 46647c478bd9Sstevel@tonic-gate /* 46657c478bd9Sstevel@tonic-gate * Queue unexpected errors, error bit only match. 46667c478bd9Sstevel@tonic-gate */ 46677c478bd9Sstevel@tonic-gate for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 46687c478bd9Sstevel@tonic-gate eccp++) { 46697c478bd9Sstevel@tonic-gate if (eccp->ec_afsr_bit & t_afsr_errs) { 46707c478bd9Sstevel@tonic-gate #if defined(SERRANO) 46717c478bd9Sstevel@tonic-gate /* 46727c478bd9Sstevel@tonic-gate * For FRC/FRU errors on Serrano the afar2 captures 46737c478bd9Sstevel@tonic-gate * the address and the associated data is 46747c478bd9Sstevel@tonic-gate * in the shadow logout area. 46757c478bd9Sstevel@tonic-gate */ 46767c478bd9Sstevel@tonic-gate if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 46777c478bd9Sstevel@tonic-gate if (clop != NULL) 46787c478bd9Sstevel@tonic-gate cdp = &clop->clo_sdw_data; 46797c478bd9Sstevel@tonic-gate aflt->flt_addr = ch_flt->afar2; 46807c478bd9Sstevel@tonic-gate } else { 46817c478bd9Sstevel@tonic-gate if (clop != NULL) 46827c478bd9Sstevel@tonic-gate cdp = &clop->clo_data; 46837c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 46847c478bd9Sstevel@tonic-gate } 46857c478bd9Sstevel@tonic-gate #else /* SERRANO */ 46867c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 46877c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 46887c478bd9Sstevel@tonic-gate aflt->flt_stat = primary_afsr; 46897c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = primary_afsr_ext; 46907c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = primary_afsr_errs; 46917c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_flt, reason, eccp, cdp); 46927c478bd9Sstevel@tonic-gate cdp = NULL; 46937c478bd9Sstevel@tonic-gate t_afsr_errs &= ~eccp->ec_afsr_bit; 46947c478bd9Sstevel@tonic-gate nevents++; 46957c478bd9Sstevel@tonic-gate } 46967c478bd9Sstevel@tonic-gate } 46977c478bd9Sstevel@tonic-gate return (nevents); 46987c478bd9Sstevel@tonic-gate } 46997c478bd9Sstevel@tonic-gate 47007c478bd9Sstevel@tonic-gate /* 47017c478bd9Sstevel@tonic-gate * Return trap type number. 47027c478bd9Sstevel@tonic-gate */ 47037c478bd9Sstevel@tonic-gate uint8_t 47047c478bd9Sstevel@tonic-gate flt_to_trap_type(struct async_flt *aflt) 47057c478bd9Sstevel@tonic-gate { 47067c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_I_TRAP) 47077c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_I); 47087c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_D_TRAP) 47097c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_D); 47107c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_F_TRAP) 47117c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_F); 47127c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_C_TRAP) 47137c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_C); 47147c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_DP_TRAP) 47157c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_DP); 47167c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_IP_TRAP) 47177c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_IP); 47187c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_ITLB_TRAP) 47197c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_ITLB); 47207c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_DTLB_TRAP) 47217c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_DTLB); 47227c478bd9Sstevel@tonic-gate return (TRAP_TYPE_UNKNOWN); 47237c478bd9Sstevel@tonic-gate } 47247c478bd9Sstevel@tonic-gate 47257c478bd9Sstevel@tonic-gate /* 47267c478bd9Sstevel@tonic-gate * Decide an error type based on detector and leaky/partner tests. 47277c478bd9Sstevel@tonic-gate * The following array is used for quick translation - it must 47287c478bd9Sstevel@tonic-gate * stay in sync with ce_dispact_t. 47297c478bd9Sstevel@tonic-gate */ 47307c478bd9Sstevel@tonic-gate 47317c478bd9Sstevel@tonic-gate static char *cetypes[] = { 47327c478bd9Sstevel@tonic-gate CE_DISP_DESC_U, 47337c478bd9Sstevel@tonic-gate CE_DISP_DESC_I, 47347c478bd9Sstevel@tonic-gate CE_DISP_DESC_PP, 47357c478bd9Sstevel@tonic-gate CE_DISP_DESC_P, 47367c478bd9Sstevel@tonic-gate CE_DISP_DESC_L, 47377c478bd9Sstevel@tonic-gate CE_DISP_DESC_PS, 47387c478bd9Sstevel@tonic-gate CE_DISP_DESC_S 47397c478bd9Sstevel@tonic-gate }; 47407c478bd9Sstevel@tonic-gate 47417c478bd9Sstevel@tonic-gate char * 47427c478bd9Sstevel@tonic-gate flt_to_error_type(struct async_flt *aflt) 47437c478bd9Sstevel@tonic-gate { 47447c478bd9Sstevel@tonic-gate ce_dispact_t dispact, disp; 47457c478bd9Sstevel@tonic-gate uchar_t dtcrinfo, ptnrinfo, lkyinfo; 47467c478bd9Sstevel@tonic-gate 47477c478bd9Sstevel@tonic-gate /* 47487c478bd9Sstevel@tonic-gate * The memory payload bundle is shared by some events that do 47497c478bd9Sstevel@tonic-gate * not perform any classification. For those flt_disp will be 47507c478bd9Sstevel@tonic-gate * 0 and we will return "unknown". 47517c478bd9Sstevel@tonic-gate */ 47527c478bd9Sstevel@tonic-gate if (!ce_disp_inited || !aflt->flt_in_memory || aflt->flt_disp == 0) 47537c478bd9Sstevel@tonic-gate return (cetypes[CE_DISP_UNKNOWN]); 47547c478bd9Sstevel@tonic-gate 47557c478bd9Sstevel@tonic-gate dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 47567c478bd9Sstevel@tonic-gate 47577c478bd9Sstevel@tonic-gate /* 47587c478bd9Sstevel@tonic-gate * It is also possible that no scrub/classification was performed 47597c478bd9Sstevel@tonic-gate * by the detector, for instance where a disrupting error logged 47607c478bd9Sstevel@tonic-gate * in the AFSR while CEEN was off in cpu_deferred_error. 47617c478bd9Sstevel@tonic-gate */ 47627c478bd9Sstevel@tonic-gate if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) 47637c478bd9Sstevel@tonic-gate return (cetypes[CE_DISP_UNKNOWN]); 47647c478bd9Sstevel@tonic-gate 47657c478bd9Sstevel@tonic-gate /* 47667c478bd9Sstevel@tonic-gate * Lookup type in initial classification/action table 47677c478bd9Sstevel@tonic-gate */ 47687c478bd9Sstevel@tonic-gate dispact = CE_DISPACT(ce_disp_table, 47697c478bd9Sstevel@tonic-gate CE_XDIAG_AFARMATCHED(dtcrinfo), 47707c478bd9Sstevel@tonic-gate CE_XDIAG_STATE(dtcrinfo), 47717c478bd9Sstevel@tonic-gate CE_XDIAG_CE1SEEN(dtcrinfo), 47727c478bd9Sstevel@tonic-gate CE_XDIAG_CE2SEEN(dtcrinfo)); 47737c478bd9Sstevel@tonic-gate 47747c478bd9Sstevel@tonic-gate /* 47757c478bd9Sstevel@tonic-gate * A bad lookup is not something to panic production systems for. 47767c478bd9Sstevel@tonic-gate */ 47777c478bd9Sstevel@tonic-gate ASSERT(dispact != CE_DISP_BAD); 47787c478bd9Sstevel@tonic-gate if (dispact == CE_DISP_BAD) 47797c478bd9Sstevel@tonic-gate return (cetypes[CE_DISP_UNKNOWN]); 47807c478bd9Sstevel@tonic-gate 47817c478bd9Sstevel@tonic-gate disp = CE_DISP(dispact); 47827c478bd9Sstevel@tonic-gate 47837c478bd9Sstevel@tonic-gate switch (disp) { 47847c478bd9Sstevel@tonic-gate case CE_DISP_UNKNOWN: 47857c478bd9Sstevel@tonic-gate case CE_DISP_INTERMITTENT: 47867c478bd9Sstevel@tonic-gate break; 47877c478bd9Sstevel@tonic-gate 47887c478bd9Sstevel@tonic-gate case CE_DISP_POSS_PERS: 47897c478bd9Sstevel@tonic-gate /* 47907c478bd9Sstevel@tonic-gate * "Possible persistent" errors to which we have applied a valid 47917c478bd9Sstevel@tonic-gate * leaky test can be separated into "persistent" or "leaky". 47927c478bd9Sstevel@tonic-gate */ 47937c478bd9Sstevel@tonic-gate lkyinfo = CE_XDIAG_LKYINFO(aflt->flt_disp); 47947c478bd9Sstevel@tonic-gate if (CE_XDIAG_TESTVALID(lkyinfo)) { 47957c478bd9Sstevel@tonic-gate if (CE_XDIAG_CE1SEEN(lkyinfo) || 47967c478bd9Sstevel@tonic-gate CE_XDIAG_CE2SEEN(lkyinfo)) 47977c478bd9Sstevel@tonic-gate disp = CE_DISP_LEAKY; 47987c478bd9Sstevel@tonic-gate else 47997c478bd9Sstevel@tonic-gate disp = CE_DISP_PERS; 48007c478bd9Sstevel@tonic-gate } 48017c478bd9Sstevel@tonic-gate break; 48027c478bd9Sstevel@tonic-gate 48037c478bd9Sstevel@tonic-gate case CE_DISP_POSS_STICKY: 48047c478bd9Sstevel@tonic-gate /* 48057c478bd9Sstevel@tonic-gate * Promote "possible sticky" results that have been 48067c478bd9Sstevel@tonic-gate * confirmed by a partner test to "sticky". Unconfirmed 48077c478bd9Sstevel@tonic-gate * "possible sticky" events are left at that status - we do not 48087c478bd9Sstevel@tonic-gate * guess at any bad reader/writer etc status here. 48097c478bd9Sstevel@tonic-gate */ 48107c478bd9Sstevel@tonic-gate ptnrinfo = CE_XDIAG_PTNRINFO(aflt->flt_disp); 48117c478bd9Sstevel@tonic-gate if (CE_XDIAG_TESTVALID(ptnrinfo) && 48127c478bd9Sstevel@tonic-gate CE_XDIAG_CE1SEEN(ptnrinfo) && CE_XDIAG_CE2SEEN(ptnrinfo)) 48137c478bd9Sstevel@tonic-gate disp = CE_DISP_STICKY; 48147c478bd9Sstevel@tonic-gate 48157c478bd9Sstevel@tonic-gate /* 48167c478bd9Sstevel@tonic-gate * Promote "possible sticky" results on a uniprocessor 48177c478bd9Sstevel@tonic-gate * to "sticky" 48187c478bd9Sstevel@tonic-gate */ 48197c478bd9Sstevel@tonic-gate if (disp == CE_DISP_POSS_STICKY && 48207c478bd9Sstevel@tonic-gate CE_XDIAG_SKIPCODE(disp) == CE_XDIAG_SKIP_UNIPROC) 48217c478bd9Sstevel@tonic-gate disp = CE_DISP_STICKY; 48227c478bd9Sstevel@tonic-gate break; 48237c478bd9Sstevel@tonic-gate 48247c478bd9Sstevel@tonic-gate default: 48257c478bd9Sstevel@tonic-gate disp = CE_DISP_UNKNOWN; 48267c478bd9Sstevel@tonic-gate break; 48277c478bd9Sstevel@tonic-gate } 48287c478bd9Sstevel@tonic-gate 48297c478bd9Sstevel@tonic-gate return (cetypes[disp]); 48307c478bd9Sstevel@tonic-gate } 48317c478bd9Sstevel@tonic-gate 48327c478bd9Sstevel@tonic-gate /* 48337c478bd9Sstevel@tonic-gate * Given the entire afsr, the specific bit to check and a prioritized list of 48347c478bd9Sstevel@tonic-gate * error bits, determine the validity of the various overwrite priority 48357c478bd9Sstevel@tonic-gate * features of the AFSR/AFAR: AFAR, ESYND and MSYND, each of which have 48367c478bd9Sstevel@tonic-gate * different overwrite priorities. 48377c478bd9Sstevel@tonic-gate * 48387c478bd9Sstevel@tonic-gate * Given a specific afsr error bit and the entire afsr, there are three cases: 48397c478bd9Sstevel@tonic-gate * INVALID: The specified bit is lower overwrite priority than some other 48407c478bd9Sstevel@tonic-gate * error bit which is on in the afsr (or IVU/IVC). 48417c478bd9Sstevel@tonic-gate * VALID: The specified bit is higher priority than all other error bits 48427c478bd9Sstevel@tonic-gate * which are on in the afsr. 48437c478bd9Sstevel@tonic-gate * AMBIGUOUS: Another error bit (or bits) of equal priority to the specified 48447c478bd9Sstevel@tonic-gate * bit is on in the afsr. 48457c478bd9Sstevel@tonic-gate */ 48467c478bd9Sstevel@tonic-gate int 48477c478bd9Sstevel@tonic-gate afsr_to_overw_status(uint64_t afsr, uint64_t afsr_bit, uint64_t *ow_bits) 48487c478bd9Sstevel@tonic-gate { 48497c478bd9Sstevel@tonic-gate uint64_t afsr_ow; 48507c478bd9Sstevel@tonic-gate 48517c478bd9Sstevel@tonic-gate while ((afsr_ow = *ow_bits++) != 0) { 48527c478bd9Sstevel@tonic-gate /* 48537c478bd9Sstevel@tonic-gate * If bit is in the priority class, check to see if another 48547c478bd9Sstevel@tonic-gate * bit in the same class is on => ambiguous. Otherwise, 48557c478bd9Sstevel@tonic-gate * the value is valid. If the bit is not on at this priority 48567c478bd9Sstevel@tonic-gate * class, but a higher priority bit is on, then the value is 48577c478bd9Sstevel@tonic-gate * invalid. 48587c478bd9Sstevel@tonic-gate */ 48597c478bd9Sstevel@tonic-gate if (afsr_ow & afsr_bit) { 48607c478bd9Sstevel@tonic-gate /* 48617c478bd9Sstevel@tonic-gate * If equal pri bit is on, ambiguous. 48627c478bd9Sstevel@tonic-gate */ 48637c478bd9Sstevel@tonic-gate if (afsr & (afsr_ow & ~afsr_bit)) 48647c478bd9Sstevel@tonic-gate return (AFLT_STAT_AMBIGUOUS); 48657c478bd9Sstevel@tonic-gate return (AFLT_STAT_VALID); 48667c478bd9Sstevel@tonic-gate } else if (afsr & afsr_ow) 48677c478bd9Sstevel@tonic-gate break; 48687c478bd9Sstevel@tonic-gate } 48697c478bd9Sstevel@tonic-gate 48707c478bd9Sstevel@tonic-gate /* 48717c478bd9Sstevel@tonic-gate * We didn't find a match or a higher priority bit was on. Not 48727c478bd9Sstevel@tonic-gate * finding a match handles the case of invalid AFAR for IVC, IVU. 48737c478bd9Sstevel@tonic-gate */ 48747c478bd9Sstevel@tonic-gate return (AFLT_STAT_INVALID); 48757c478bd9Sstevel@tonic-gate } 48767c478bd9Sstevel@tonic-gate 48777c478bd9Sstevel@tonic-gate static int 48787c478bd9Sstevel@tonic-gate afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit) 48797c478bd9Sstevel@tonic-gate { 48807c478bd9Sstevel@tonic-gate #if defined(SERRANO) 48817c478bd9Sstevel@tonic-gate if (afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) 48827c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, afar2_overwrite)); 48837c478bd9Sstevel@tonic-gate else 48847c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 48857c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, afar_overwrite)); 48867c478bd9Sstevel@tonic-gate } 48877c478bd9Sstevel@tonic-gate 48887c478bd9Sstevel@tonic-gate static int 48897c478bd9Sstevel@tonic-gate afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit) 48907c478bd9Sstevel@tonic-gate { 48917c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, esynd_overwrite)); 48927c478bd9Sstevel@tonic-gate } 48937c478bd9Sstevel@tonic-gate 48947c478bd9Sstevel@tonic-gate static int 48957c478bd9Sstevel@tonic-gate afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit) 48967c478bd9Sstevel@tonic-gate { 48977c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, msynd_overwrite)); 48987c478bd9Sstevel@tonic-gate } 48997c478bd9Sstevel@tonic-gate 49007c478bd9Sstevel@tonic-gate static int 49017c478bd9Sstevel@tonic-gate afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit) 49027c478bd9Sstevel@tonic-gate { 49037c478bd9Sstevel@tonic-gate #ifdef lint 49047c478bd9Sstevel@tonic-gate cpuid = cpuid; 49057c478bd9Sstevel@tonic-gate #endif 49067c478bd9Sstevel@tonic-gate if (afsr_bit & C_AFSR_MSYND_ERRS) { 49077c478bd9Sstevel@tonic-gate return (afsr_to_msynd_status(afsr, afsr_bit)); 49087c478bd9Sstevel@tonic-gate } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 49097c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 49107c478bd9Sstevel@tonic-gate /* 49117c478bd9Sstevel@tonic-gate * The E_SYND overwrite policy is slightly different 49127c478bd9Sstevel@tonic-gate * for Panther CPUs. 49137c478bd9Sstevel@tonic-gate */ 49147c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[cpuid].implementation)) 49157c478bd9Sstevel@tonic-gate return (afsr_to_pn_esynd_status(afsr, afsr_bit)); 49167c478bd9Sstevel@tonic-gate else 49177c478bd9Sstevel@tonic-gate return (afsr_to_esynd_status(afsr, afsr_bit)); 49187c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */ 49197c478bd9Sstevel@tonic-gate return (afsr_to_esynd_status(afsr, afsr_bit)); 49207c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 49217c478bd9Sstevel@tonic-gate } else { 49227c478bd9Sstevel@tonic-gate return (AFLT_STAT_INVALID); 49237c478bd9Sstevel@tonic-gate } 49247c478bd9Sstevel@tonic-gate } 49257c478bd9Sstevel@tonic-gate 49267c478bd9Sstevel@tonic-gate /* 49277c478bd9Sstevel@tonic-gate * Slave CPU stick synchronization. 49287c478bd9Sstevel@tonic-gate */ 49297c478bd9Sstevel@tonic-gate void 49307c478bd9Sstevel@tonic-gate sticksync_slave(void) 49317c478bd9Sstevel@tonic-gate { 49327c478bd9Sstevel@tonic-gate int i; 49337c478bd9Sstevel@tonic-gate int tries = 0; 49347c478bd9Sstevel@tonic-gate int64_t tskew; 49357c478bd9Sstevel@tonic-gate int64_t av_tskew; 49367c478bd9Sstevel@tonic-gate 49377c478bd9Sstevel@tonic-gate kpreempt_disable(); 49387c478bd9Sstevel@tonic-gate /* wait for the master side */ 49397c478bd9Sstevel@tonic-gate while (stick_sync_cmd != SLAVE_START) 49407c478bd9Sstevel@tonic-gate ; 49417c478bd9Sstevel@tonic-gate /* 49427c478bd9Sstevel@tonic-gate * Synchronization should only take a few tries at most. But in the 49437c478bd9Sstevel@tonic-gate * odd case where the cpu isn't cooperating we'll keep trying. A cpu 49447c478bd9Sstevel@tonic-gate * without it's stick synchronized wouldn't be a good citizen. 49457c478bd9Sstevel@tonic-gate */ 49467c478bd9Sstevel@tonic-gate while (slave_done == 0) { 49477c478bd9Sstevel@tonic-gate /* 49487c478bd9Sstevel@tonic-gate * Time skew calculation. 49497c478bd9Sstevel@tonic-gate */ 49507c478bd9Sstevel@tonic-gate av_tskew = tskew = 0; 49517c478bd9Sstevel@tonic-gate 49527c478bd9Sstevel@tonic-gate for (i = 0; i < stick_iter; i++) { 49537c478bd9Sstevel@tonic-gate /* make location hot */ 49547c478bd9Sstevel@tonic-gate timestamp[EV_A_START] = 0; 49557c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_A_START]); 49567c478bd9Sstevel@tonic-gate 49577c478bd9Sstevel@tonic-gate /* tell the master we're ready */ 49587c478bd9Sstevel@tonic-gate stick_sync_cmd = MASTER_START; 49597c478bd9Sstevel@tonic-gate 49607c478bd9Sstevel@tonic-gate /* and wait */ 49617c478bd9Sstevel@tonic-gate while (stick_sync_cmd != SLAVE_CONT) 49627c478bd9Sstevel@tonic-gate ; 49637c478bd9Sstevel@tonic-gate /* Event B end */ 49647c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_B_END]); 49657c478bd9Sstevel@tonic-gate 49667c478bd9Sstevel@tonic-gate /* calculate time skew */ 49677c478bd9Sstevel@tonic-gate tskew = ((timestamp[EV_B_END] - timestamp[EV_B_START]) 49687c478bd9Sstevel@tonic-gate - (timestamp[EV_A_END] - 49697c478bd9Sstevel@tonic-gate timestamp[EV_A_START])) / 2; 49707c478bd9Sstevel@tonic-gate 49717c478bd9Sstevel@tonic-gate /* keep running count */ 49727c478bd9Sstevel@tonic-gate av_tskew += tskew; 49737c478bd9Sstevel@tonic-gate } /* for */ 49747c478bd9Sstevel@tonic-gate 49757c478bd9Sstevel@tonic-gate /* 49767c478bd9Sstevel@tonic-gate * Adjust stick for time skew if not within the max allowed; 49777c478bd9Sstevel@tonic-gate * otherwise we're all done. 49787c478bd9Sstevel@tonic-gate */ 49797c478bd9Sstevel@tonic-gate if (stick_iter != 0) 49807c478bd9Sstevel@tonic-gate av_tskew = av_tskew/stick_iter; 49817c478bd9Sstevel@tonic-gate if (ABS(av_tskew) > stick_tsk) { 49827c478bd9Sstevel@tonic-gate /* 49837c478bd9Sstevel@tonic-gate * If the skew is 1 (the slave's STICK register 49847c478bd9Sstevel@tonic-gate * is 1 STICK ahead of the master's), stick_adj 49857c478bd9Sstevel@tonic-gate * could fail to adjust the slave's STICK register 49867c478bd9Sstevel@tonic-gate * if the STICK read on the slave happens to 49877c478bd9Sstevel@tonic-gate * align with the increment of the STICK. 49887c478bd9Sstevel@tonic-gate * Therefore, we increment the skew to 2. 49897c478bd9Sstevel@tonic-gate */ 49907c478bd9Sstevel@tonic-gate if (av_tskew == 1) 49917c478bd9Sstevel@tonic-gate av_tskew++; 49927c478bd9Sstevel@tonic-gate stick_adj(-av_tskew); 49937c478bd9Sstevel@tonic-gate } else 49947c478bd9Sstevel@tonic-gate slave_done = 1; 49957c478bd9Sstevel@tonic-gate #ifdef DEBUG 49967c478bd9Sstevel@tonic-gate if (tries < DSYNC_ATTEMPTS) 49977c478bd9Sstevel@tonic-gate stick_sync_stats[CPU->cpu_id].skew_val[tries] = 49987c478bd9Sstevel@tonic-gate av_tskew; 49997c478bd9Sstevel@tonic-gate ++tries; 50007c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 50017c478bd9Sstevel@tonic-gate #ifdef lint 50027c478bd9Sstevel@tonic-gate tries = tries; 50037c478bd9Sstevel@tonic-gate #endif 50047c478bd9Sstevel@tonic-gate 50057c478bd9Sstevel@tonic-gate } /* while */ 50067c478bd9Sstevel@tonic-gate 50077c478bd9Sstevel@tonic-gate /* allow the master to finish */ 50087c478bd9Sstevel@tonic-gate stick_sync_cmd = EVENT_NULL; 50097c478bd9Sstevel@tonic-gate kpreempt_enable(); 50107c478bd9Sstevel@tonic-gate } 50117c478bd9Sstevel@tonic-gate 50127c478bd9Sstevel@tonic-gate /* 50137c478bd9Sstevel@tonic-gate * Master CPU side of stick synchronization. 50147c478bd9Sstevel@tonic-gate * - timestamp end of Event A 50157c478bd9Sstevel@tonic-gate * - timestamp beginning of Event B 50167c478bd9Sstevel@tonic-gate */ 50177c478bd9Sstevel@tonic-gate void 50187c478bd9Sstevel@tonic-gate sticksync_master(void) 50197c478bd9Sstevel@tonic-gate { 50207c478bd9Sstevel@tonic-gate int i; 50217c478bd9Sstevel@tonic-gate 50227c478bd9Sstevel@tonic-gate kpreempt_disable(); 50237c478bd9Sstevel@tonic-gate /* tell the slave we've started */ 50247c478bd9Sstevel@tonic-gate slave_done = 0; 50257c478bd9Sstevel@tonic-gate stick_sync_cmd = SLAVE_START; 50267c478bd9Sstevel@tonic-gate 50277c478bd9Sstevel@tonic-gate while (slave_done == 0) { 50287c478bd9Sstevel@tonic-gate for (i = 0; i < stick_iter; i++) { 50297c478bd9Sstevel@tonic-gate /* wait for the slave */ 50307c478bd9Sstevel@tonic-gate while (stick_sync_cmd != MASTER_START) 50317c478bd9Sstevel@tonic-gate ; 50327c478bd9Sstevel@tonic-gate /* Event A end */ 50337c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_A_END]); 50347c478bd9Sstevel@tonic-gate 50357c478bd9Sstevel@tonic-gate /* make location hot */ 50367c478bd9Sstevel@tonic-gate timestamp[EV_B_START] = 0; 50377c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_B_START]); 50387c478bd9Sstevel@tonic-gate 50397c478bd9Sstevel@tonic-gate /* tell the slave to continue */ 50407c478bd9Sstevel@tonic-gate stick_sync_cmd = SLAVE_CONT; 50417c478bd9Sstevel@tonic-gate } /* for */ 50427c478bd9Sstevel@tonic-gate 50437c478bd9Sstevel@tonic-gate /* wait while slave calculates time skew */ 50447c478bd9Sstevel@tonic-gate while (stick_sync_cmd == SLAVE_CONT) 50457c478bd9Sstevel@tonic-gate ; 50467c478bd9Sstevel@tonic-gate } /* while */ 50477c478bd9Sstevel@tonic-gate kpreempt_enable(); 50487c478bd9Sstevel@tonic-gate } 50497c478bd9Sstevel@tonic-gate 50507c478bd9Sstevel@tonic-gate /* 50517c478bd9Sstevel@tonic-gate * Cheetah/Cheetah+ have disrupting error for copyback's, so we don't need to 50527c478bd9Sstevel@tonic-gate * do Spitfire hack of xcall'ing all the cpus to ask to check for them. Also, 50537c478bd9Sstevel@tonic-gate * in cpu_async_panic_callb, each cpu checks for CPU events on its way to 50547c478bd9Sstevel@tonic-gate * panic idle. 50557c478bd9Sstevel@tonic-gate */ 50567c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 50577c478bd9Sstevel@tonic-gate void 50587c478bd9Sstevel@tonic-gate cpu_check_allcpus(struct async_flt *aflt) 50597c478bd9Sstevel@tonic-gate {} 50607c478bd9Sstevel@tonic-gate 50617c478bd9Sstevel@tonic-gate struct kmem_cache *ch_private_cache; 50627c478bd9Sstevel@tonic-gate 50637c478bd9Sstevel@tonic-gate /* 50647c478bd9Sstevel@tonic-gate * Cpu private unitialization. Uninitialize the Ecache scrubber and 50657c478bd9Sstevel@tonic-gate * deallocate the scrubber data structures and cpu_private data structure. 50667c478bd9Sstevel@tonic-gate */ 50677c478bd9Sstevel@tonic-gate void 50687c478bd9Sstevel@tonic-gate cpu_uninit_private(struct cpu *cp) 50697c478bd9Sstevel@tonic-gate { 50707c478bd9Sstevel@tonic-gate cheetah_private_t *chprp = CPU_PRIVATE(cp); 50717c478bd9Sstevel@tonic-gate 50727c478bd9Sstevel@tonic-gate ASSERT(chprp); 50737c478bd9Sstevel@tonic-gate cpu_uninit_ecache_scrub_dr(cp); 50747c478bd9Sstevel@tonic-gate CPU_PRIVATE(cp) = NULL; 50757c478bd9Sstevel@tonic-gate ch_err_tl1_paddrs[cp->cpu_id] = NULL; 50767c478bd9Sstevel@tonic-gate kmem_cache_free(ch_private_cache, chprp); 50777c478bd9Sstevel@tonic-gate cmp_delete_cpu(cp->cpu_id); 50787c478bd9Sstevel@tonic-gate 50797c478bd9Sstevel@tonic-gate } 50807c478bd9Sstevel@tonic-gate 50817c478bd9Sstevel@tonic-gate /* 50827c478bd9Sstevel@tonic-gate * Cheetah Cache Scrubbing 50837c478bd9Sstevel@tonic-gate * 50847c478bd9Sstevel@tonic-gate * The primary purpose of Cheetah cache scrubbing is to reduce the exposure 50857c478bd9Sstevel@tonic-gate * of E$ tags, D$ data, and I$ data to cosmic ray events since they are not 50867c478bd9Sstevel@tonic-gate * protected by either parity or ECC. 50877c478bd9Sstevel@tonic-gate * 50887c478bd9Sstevel@tonic-gate * We currently default the E$ and D$ scan rate to 100 (scan 10% of the 50897c478bd9Sstevel@tonic-gate * cache per second). Due to the the specifics of how the I$ control 50907c478bd9Sstevel@tonic-gate * logic works with respect to the ASI used to scrub I$ lines, the entire 50917c478bd9Sstevel@tonic-gate * I$ is scanned at once. 50927c478bd9Sstevel@tonic-gate */ 50937c478bd9Sstevel@tonic-gate 50947c478bd9Sstevel@tonic-gate /* 50957c478bd9Sstevel@tonic-gate * Tuneables to enable and disable the scrubbing of the caches, and to tune 50967c478bd9Sstevel@tonic-gate * scrubbing behavior. These may be changed via /etc/system or using mdb 50977c478bd9Sstevel@tonic-gate * on a running system. 50987c478bd9Sstevel@tonic-gate */ 50997c478bd9Sstevel@tonic-gate int dcache_scrub_enable = 1; /* D$ scrubbing is on by default */ 51007c478bd9Sstevel@tonic-gate 51017c478bd9Sstevel@tonic-gate /* 51027c478bd9Sstevel@tonic-gate * The following are the PIL levels that the softints/cross traps will fire at. 51037c478bd9Sstevel@tonic-gate */ 51047c478bd9Sstevel@tonic-gate uint_t ecache_scrub_pil = PIL_9; /* E$ scrub PIL for cross traps */ 51057c478bd9Sstevel@tonic-gate uint_t dcache_scrub_pil = PIL_9; /* D$ scrub PIL for cross traps */ 51067c478bd9Sstevel@tonic-gate uint_t icache_scrub_pil = PIL_9; /* I$ scrub PIL for cross traps */ 51077c478bd9Sstevel@tonic-gate 51087c478bd9Sstevel@tonic-gate #if defined(JALAPENO) 51097c478bd9Sstevel@tonic-gate 51107c478bd9Sstevel@tonic-gate /* 51117c478bd9Sstevel@tonic-gate * Due to several errata (82, 85, 86), we don't enable the L2$ scrubber 51127c478bd9Sstevel@tonic-gate * on Jalapeno. 51137c478bd9Sstevel@tonic-gate */ 51147c478bd9Sstevel@tonic-gate int ecache_scrub_enable = 0; 51157c478bd9Sstevel@tonic-gate 51167c478bd9Sstevel@tonic-gate #else /* JALAPENO */ 51177c478bd9Sstevel@tonic-gate 51187c478bd9Sstevel@tonic-gate /* 51197c478bd9Sstevel@tonic-gate * With all other cpu types, E$ scrubbing is on by default 51207c478bd9Sstevel@tonic-gate */ 51217c478bd9Sstevel@tonic-gate int ecache_scrub_enable = 1; 51227c478bd9Sstevel@tonic-gate 51237c478bd9Sstevel@tonic-gate #endif /* JALAPENO */ 51247c478bd9Sstevel@tonic-gate 51257c478bd9Sstevel@tonic-gate 51267c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) || defined(JALAPENO) || defined(SERRANO) 51277c478bd9Sstevel@tonic-gate 51287c478bd9Sstevel@tonic-gate /* 51297c478bd9Sstevel@tonic-gate * The I$ scrubber tends to cause latency problems for real-time SW, so it 51307c478bd9Sstevel@tonic-gate * is disabled by default on non-Cheetah systems 51317c478bd9Sstevel@tonic-gate */ 51327c478bd9Sstevel@tonic-gate int icache_scrub_enable = 0; 51337c478bd9Sstevel@tonic-gate 51347c478bd9Sstevel@tonic-gate /* 51357c478bd9Sstevel@tonic-gate * Tuneables specifying the scrub calls per second and the scan rate 51367c478bd9Sstevel@tonic-gate * for each cache 51377c478bd9Sstevel@tonic-gate * 51387c478bd9Sstevel@tonic-gate * The cyclic times are set during boot based on the following values. 51397c478bd9Sstevel@tonic-gate * Changing these values in mdb after this time will have no effect. If 51407c478bd9Sstevel@tonic-gate * a different value is desired, it must be set in /etc/system before a 51417c478bd9Sstevel@tonic-gate * reboot. 51427c478bd9Sstevel@tonic-gate */ 51437c478bd9Sstevel@tonic-gate int ecache_calls_a_sec = 1; 51447c478bd9Sstevel@tonic-gate int dcache_calls_a_sec = 2; 51457c478bd9Sstevel@tonic-gate int icache_calls_a_sec = 2; 51467c478bd9Sstevel@tonic-gate 51477c478bd9Sstevel@tonic-gate int ecache_scan_rate_idle = 1; 51487c478bd9Sstevel@tonic-gate int ecache_scan_rate_busy = 1; 51497c478bd9Sstevel@tonic-gate int dcache_scan_rate_idle = 1; 51507c478bd9Sstevel@tonic-gate int dcache_scan_rate_busy = 1; 51517c478bd9Sstevel@tonic-gate int icache_scan_rate_idle = 1; 51527c478bd9Sstevel@tonic-gate int icache_scan_rate_busy = 1; 51537c478bd9Sstevel@tonic-gate 51547c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS || JALAPENO || SERRANO */ 51557c478bd9Sstevel@tonic-gate 51567c478bd9Sstevel@tonic-gate int icache_scrub_enable = 1; /* I$ scrubbing is on by default */ 51577c478bd9Sstevel@tonic-gate 51587c478bd9Sstevel@tonic-gate int ecache_calls_a_sec = 100; /* E$ scrub calls per seconds */ 51597c478bd9Sstevel@tonic-gate int dcache_calls_a_sec = 100; /* D$ scrub calls per seconds */ 51607c478bd9Sstevel@tonic-gate int icache_calls_a_sec = 100; /* I$ scrub calls per seconds */ 51617c478bd9Sstevel@tonic-gate 51627c478bd9Sstevel@tonic-gate int ecache_scan_rate_idle = 100; /* E$ scan rate (in tenths of a %) */ 51637c478bd9Sstevel@tonic-gate int ecache_scan_rate_busy = 100; /* E$ scan rate (in tenths of a %) */ 51647c478bd9Sstevel@tonic-gate int dcache_scan_rate_idle = 100; /* D$ scan rate (in tenths of a %) */ 51657c478bd9Sstevel@tonic-gate int dcache_scan_rate_busy = 100; /* D$ scan rate (in tenths of a %) */ 51667c478bd9Sstevel@tonic-gate int icache_scan_rate_idle = 100; /* I$ scan rate (in tenths of a %) */ 51677c478bd9Sstevel@tonic-gate int icache_scan_rate_busy = 100; /* I$ scan rate (in tenths of a %) */ 51687c478bd9Sstevel@tonic-gate 51697c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS || JALAPENO || SERRANO */ 51707c478bd9Sstevel@tonic-gate 51717c478bd9Sstevel@tonic-gate /* 51727c478bd9Sstevel@tonic-gate * In order to scrub on offline cpus, a cross trap is sent. The handler will 51737c478bd9Sstevel@tonic-gate * increment the outstanding request counter and schedule a softint to run 51747c478bd9Sstevel@tonic-gate * the scrubber. 51757c478bd9Sstevel@tonic-gate */ 51767c478bd9Sstevel@tonic-gate extern xcfunc_t cache_scrubreq_tl1; 51777c478bd9Sstevel@tonic-gate 51787c478bd9Sstevel@tonic-gate /* 51797c478bd9Sstevel@tonic-gate * These are the softint functions for each cache scrubber 51807c478bd9Sstevel@tonic-gate */ 51817c478bd9Sstevel@tonic-gate static uint_t scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2); 51827c478bd9Sstevel@tonic-gate static uint_t scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2); 51837c478bd9Sstevel@tonic-gate static uint_t scrub_icache_line_intr(caddr_t arg1, caddr_t arg2); 51847c478bd9Sstevel@tonic-gate 51857c478bd9Sstevel@tonic-gate /* 51867c478bd9Sstevel@tonic-gate * The cache scrub info table contains cache specific information 51877c478bd9Sstevel@tonic-gate * and allows for some of the scrub code to be table driven, reducing 51887c478bd9Sstevel@tonic-gate * duplication of cache similar code. 51897c478bd9Sstevel@tonic-gate * 51907c478bd9Sstevel@tonic-gate * This table keeps a copy of the value in the calls per second variable 51917c478bd9Sstevel@tonic-gate * (?cache_calls_a_sec). This makes it much more difficult for someone 51927c478bd9Sstevel@tonic-gate * to cause us problems (for example, by setting ecache_calls_a_sec to 0 in 51937c478bd9Sstevel@tonic-gate * mdb in a misguided attempt to disable the scrubber). 51947c478bd9Sstevel@tonic-gate */ 51957c478bd9Sstevel@tonic-gate struct scrub_info { 51967c478bd9Sstevel@tonic-gate int *csi_enable; /* scrubber enable flag */ 51977c478bd9Sstevel@tonic-gate int csi_freq; /* scrubber calls per second */ 51987c478bd9Sstevel@tonic-gate int csi_index; /* index to chsm_outstanding[] */ 51997c478bd9Sstevel@tonic-gate uint_t csi_inum; /* scrubber interrupt number */ 52007c478bd9Sstevel@tonic-gate cyclic_id_t csi_omni_cyc_id; /* omni cyclic ID */ 52017c478bd9Sstevel@tonic-gate cyclic_id_t csi_offline_cyc_id; /* offline cyclic ID */ 52027c478bd9Sstevel@tonic-gate char csi_name[3]; /* cache name for this scrub entry */ 52037c478bd9Sstevel@tonic-gate } cache_scrub_info[] = { 52047c478bd9Sstevel@tonic-gate { &ecache_scrub_enable, 0, CACHE_SCRUBBER_INFO_E, 0, 0, 0, "E$"}, 52057c478bd9Sstevel@tonic-gate { &dcache_scrub_enable, 0, CACHE_SCRUBBER_INFO_D, 0, 0, 0, "D$"}, 52067c478bd9Sstevel@tonic-gate { &icache_scrub_enable, 0, CACHE_SCRUBBER_INFO_I, 0, 0, 0, "I$"} 52077c478bd9Sstevel@tonic-gate }; 52087c478bd9Sstevel@tonic-gate 52097c478bd9Sstevel@tonic-gate /* 52107c478bd9Sstevel@tonic-gate * If scrubbing is enabled, increment the outstanding request counter. If it 52117c478bd9Sstevel@tonic-gate * is 1 (meaning there were no previous requests outstanding), call 52127c478bd9Sstevel@tonic-gate * setsoftint_tl1 through xt_one_unchecked, which eventually ends up doing 52137c478bd9Sstevel@tonic-gate * a self trap. 52147c478bd9Sstevel@tonic-gate */ 52157c478bd9Sstevel@tonic-gate static void 52167c478bd9Sstevel@tonic-gate do_scrub(struct scrub_info *csi) 52177c478bd9Sstevel@tonic-gate { 52187c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 52197c478bd9Sstevel@tonic-gate int index = csi->csi_index; 52207c478bd9Sstevel@tonic-gate uint32_t *outstanding = &csmp->chsm_outstanding[index]; 52217c478bd9Sstevel@tonic-gate 52227c478bd9Sstevel@tonic-gate if (*(csi->csi_enable) && (csmp->chsm_enable[index])) { 52237c478bd9Sstevel@tonic-gate if (atomic_add_32_nv(outstanding, 1) == 1) { 52247c478bd9Sstevel@tonic-gate xt_one_unchecked(CPU->cpu_id, setsoftint_tl1, 52257c478bd9Sstevel@tonic-gate csi->csi_inum, 0); 52267c478bd9Sstevel@tonic-gate } 52277c478bd9Sstevel@tonic-gate } 52287c478bd9Sstevel@tonic-gate } 52297c478bd9Sstevel@tonic-gate 52307c478bd9Sstevel@tonic-gate /* 52317c478bd9Sstevel@tonic-gate * Omni cyclics don't fire on offline cpus, so we use another cyclic to 52327c478bd9Sstevel@tonic-gate * cross-trap the offline cpus. 52337c478bd9Sstevel@tonic-gate */ 52347c478bd9Sstevel@tonic-gate static void 52357c478bd9Sstevel@tonic-gate do_scrub_offline(struct scrub_info *csi) 52367c478bd9Sstevel@tonic-gate { 52377c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 52387c478bd9Sstevel@tonic-gate 52397c478bd9Sstevel@tonic-gate if (CPUSET_ISNULL(cpu_offline_set)) { 52407c478bd9Sstevel@tonic-gate /* 52417c478bd9Sstevel@tonic-gate * No offline cpus - nothing to do 52427c478bd9Sstevel@tonic-gate */ 52437c478bd9Sstevel@tonic-gate return; 52447c478bd9Sstevel@tonic-gate } 52457c478bd9Sstevel@tonic-gate 52467c478bd9Sstevel@tonic-gate if (*(csi->csi_enable) && (csmp->chsm_enable[csi->csi_index])) { 52477c478bd9Sstevel@tonic-gate xt_some(cpu_offline_set, cache_scrubreq_tl1, csi->csi_inum, 52487c478bd9Sstevel@tonic-gate csi->csi_index); 52497c478bd9Sstevel@tonic-gate } 52507c478bd9Sstevel@tonic-gate } 52517c478bd9Sstevel@tonic-gate 52527c478bd9Sstevel@tonic-gate /* 52537c478bd9Sstevel@tonic-gate * This is the initial setup for the scrubber cyclics - it sets the 52547c478bd9Sstevel@tonic-gate * interrupt level, frequency, and function to call. 52557c478bd9Sstevel@tonic-gate */ 52567c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 52577c478bd9Sstevel@tonic-gate static void 52587c478bd9Sstevel@tonic-gate cpu_scrub_cyclic_setup(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 52597c478bd9Sstevel@tonic-gate cyc_time_t *when) 52607c478bd9Sstevel@tonic-gate { 52617c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg; 52627c478bd9Sstevel@tonic-gate 52637c478bd9Sstevel@tonic-gate ASSERT(csi != NULL); 52647c478bd9Sstevel@tonic-gate hdlr->cyh_func = (cyc_func_t)do_scrub; 52657c478bd9Sstevel@tonic-gate hdlr->cyh_level = CY_LOW_LEVEL; 52667c478bd9Sstevel@tonic-gate hdlr->cyh_arg = arg; 52677c478bd9Sstevel@tonic-gate 52687c478bd9Sstevel@tonic-gate when->cyt_when = 0; /* Start immediately */ 52697c478bd9Sstevel@tonic-gate when->cyt_interval = NANOSEC / csi->csi_freq; 52707c478bd9Sstevel@tonic-gate } 52717c478bd9Sstevel@tonic-gate 52727c478bd9Sstevel@tonic-gate /* 52737c478bd9Sstevel@tonic-gate * Initialization for cache scrubbing. 52747c478bd9Sstevel@tonic-gate * This routine is called AFTER all cpus have had cpu_init_private called 52757c478bd9Sstevel@tonic-gate * to initialize their private data areas. 52767c478bd9Sstevel@tonic-gate */ 52777c478bd9Sstevel@tonic-gate void 52787c478bd9Sstevel@tonic-gate cpu_init_cache_scrub(void) 52797c478bd9Sstevel@tonic-gate { 52807c478bd9Sstevel@tonic-gate int i; 52817c478bd9Sstevel@tonic-gate struct scrub_info *csi; 52827c478bd9Sstevel@tonic-gate cyc_omni_handler_t omni_hdlr; 52837c478bd9Sstevel@tonic-gate cyc_handler_t offline_hdlr; 52847c478bd9Sstevel@tonic-gate cyc_time_t when; 52857c478bd9Sstevel@tonic-gate 52867c478bd9Sstevel@tonic-gate /* 52877c478bd9Sstevel@tonic-gate * save away the maximum number of lines for the D$ 52887c478bd9Sstevel@tonic-gate */ 52897c478bd9Sstevel@tonic-gate dcache_nlines = dcache_size / dcache_linesize; 52907c478bd9Sstevel@tonic-gate 52917c478bd9Sstevel@tonic-gate /* 52927c478bd9Sstevel@tonic-gate * register the softints for the cache scrubbing 52937c478bd9Sstevel@tonic-gate */ 52947c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_inum = 52957c478bd9Sstevel@tonic-gate add_softintr(ecache_scrub_pil, scrub_ecache_line_intr, 52967c478bd9Sstevel@tonic-gate (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_E]); 52977c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_freq = ecache_calls_a_sec; 52987c478bd9Sstevel@tonic-gate 52997c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_inum = 53007c478bd9Sstevel@tonic-gate add_softintr(dcache_scrub_pil, scrub_dcache_line_intr, 53017c478bd9Sstevel@tonic-gate (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_D]); 53027c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_freq = dcache_calls_a_sec; 53037c478bd9Sstevel@tonic-gate 53047c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_inum = 53057c478bd9Sstevel@tonic-gate add_softintr(icache_scrub_pil, scrub_icache_line_intr, 53067c478bd9Sstevel@tonic-gate (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_I]); 53077c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_freq = icache_calls_a_sec; 53087c478bd9Sstevel@tonic-gate 53097c478bd9Sstevel@tonic-gate /* 53107c478bd9Sstevel@tonic-gate * start the scrubbing for all the caches 53117c478bd9Sstevel@tonic-gate */ 53127c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 53137c478bd9Sstevel@tonic-gate for (i = 0; i < CACHE_SCRUBBER_COUNT; i++) { 53147c478bd9Sstevel@tonic-gate 53157c478bd9Sstevel@tonic-gate csi = &cache_scrub_info[i]; 53167c478bd9Sstevel@tonic-gate 53177c478bd9Sstevel@tonic-gate if (!(*csi->csi_enable)) 53187c478bd9Sstevel@tonic-gate continue; 53197c478bd9Sstevel@tonic-gate 53207c478bd9Sstevel@tonic-gate /* 53217c478bd9Sstevel@tonic-gate * force the following to be true: 53227c478bd9Sstevel@tonic-gate * 1 <= calls_a_sec <= hz 53237c478bd9Sstevel@tonic-gate */ 53247c478bd9Sstevel@tonic-gate if (csi->csi_freq > hz) { 53257c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "%s scrub calls_a_sec set too high " 53267c478bd9Sstevel@tonic-gate "(%d); resetting to hz (%d)", csi->csi_name, 53277c478bd9Sstevel@tonic-gate csi->csi_freq, hz); 53287c478bd9Sstevel@tonic-gate csi->csi_freq = hz; 53297c478bd9Sstevel@tonic-gate } else if (csi->csi_freq < 1) { 53307c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "%s scrub calls_a_sec set too low " 53317c478bd9Sstevel@tonic-gate "(%d); resetting to 1", csi->csi_name, 53327c478bd9Sstevel@tonic-gate csi->csi_freq); 53337c478bd9Sstevel@tonic-gate csi->csi_freq = 1; 53347c478bd9Sstevel@tonic-gate } 53357c478bd9Sstevel@tonic-gate 53367c478bd9Sstevel@tonic-gate omni_hdlr.cyo_online = cpu_scrub_cyclic_setup; 53377c478bd9Sstevel@tonic-gate omni_hdlr.cyo_offline = NULL; 53387c478bd9Sstevel@tonic-gate omni_hdlr.cyo_arg = (void *)csi; 53397c478bd9Sstevel@tonic-gate 53407c478bd9Sstevel@tonic-gate offline_hdlr.cyh_func = (cyc_func_t)do_scrub_offline; 53417c478bd9Sstevel@tonic-gate offline_hdlr.cyh_arg = (void *)csi; 53427c478bd9Sstevel@tonic-gate offline_hdlr.cyh_level = CY_LOW_LEVEL; 53437c478bd9Sstevel@tonic-gate 53447c478bd9Sstevel@tonic-gate when.cyt_when = 0; /* Start immediately */ 53457c478bd9Sstevel@tonic-gate when.cyt_interval = NANOSEC / csi->csi_freq; 53467c478bd9Sstevel@tonic-gate 53477c478bd9Sstevel@tonic-gate csi->csi_omni_cyc_id = cyclic_add_omni(&omni_hdlr); 53487c478bd9Sstevel@tonic-gate csi->csi_offline_cyc_id = cyclic_add(&offline_hdlr, &when); 53497c478bd9Sstevel@tonic-gate } 53507c478bd9Sstevel@tonic-gate register_cpu_setup_func(cpu_scrub_cpu_setup, NULL); 53517c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 53527c478bd9Sstevel@tonic-gate } 53537c478bd9Sstevel@tonic-gate 53547c478bd9Sstevel@tonic-gate /* 53557c478bd9Sstevel@tonic-gate * Indicate that the specified cpu is idle. 53567c478bd9Sstevel@tonic-gate */ 53577c478bd9Sstevel@tonic-gate void 53587c478bd9Sstevel@tonic-gate cpu_idle_ecache_scrub(struct cpu *cp) 53597c478bd9Sstevel@tonic-gate { 53607c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(cp) != NULL) { 53617c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 53627c478bd9Sstevel@tonic-gate csmp->chsm_ecache_busy = ECACHE_CPU_IDLE; 53637c478bd9Sstevel@tonic-gate } 53647c478bd9Sstevel@tonic-gate } 53657c478bd9Sstevel@tonic-gate 53667c478bd9Sstevel@tonic-gate /* 53677c478bd9Sstevel@tonic-gate * Indicate that the specified cpu is busy. 53687c478bd9Sstevel@tonic-gate */ 53697c478bd9Sstevel@tonic-gate void 53707c478bd9Sstevel@tonic-gate cpu_busy_ecache_scrub(struct cpu *cp) 53717c478bd9Sstevel@tonic-gate { 53727c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(cp) != NULL) { 53737c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 53747c478bd9Sstevel@tonic-gate csmp->chsm_ecache_busy = ECACHE_CPU_BUSY; 53757c478bd9Sstevel@tonic-gate } 53767c478bd9Sstevel@tonic-gate } 53777c478bd9Sstevel@tonic-gate 53787c478bd9Sstevel@tonic-gate /* 53797c478bd9Sstevel@tonic-gate * Initialization for cache scrubbing for the specified cpu. 53807c478bd9Sstevel@tonic-gate */ 53817c478bd9Sstevel@tonic-gate void 53827c478bd9Sstevel@tonic-gate cpu_init_ecache_scrub_dr(struct cpu *cp) 53837c478bd9Sstevel@tonic-gate { 53847c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 53857c478bd9Sstevel@tonic-gate int cpuid = cp->cpu_id; 53867c478bd9Sstevel@tonic-gate 53877c478bd9Sstevel@tonic-gate /* initialize the number of lines in the caches */ 53887c478bd9Sstevel@tonic-gate csmp->chsm_ecache_nlines = cpunodes[cpuid].ecache_size / 53897c478bd9Sstevel@tonic-gate cpunodes[cpuid].ecache_linesize; 53907c478bd9Sstevel@tonic-gate csmp->chsm_icache_nlines = CPU_PRIVATE_VAL(cp, chpr_icache_size) / 53917c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(cp, chpr_icache_linesize); 53927c478bd9Sstevel@tonic-gate 53937c478bd9Sstevel@tonic-gate /* 53947c478bd9Sstevel@tonic-gate * do_scrub() and do_scrub_offline() check both the global 53957c478bd9Sstevel@tonic-gate * ?cache_scrub_enable and this per-cpu enable variable. All scrubbers 53967c478bd9Sstevel@tonic-gate * check this value before scrubbing. Currently, we use it to 53977c478bd9Sstevel@tonic-gate * disable the E$ scrubber on multi-core cpus or while running at 53987c478bd9Sstevel@tonic-gate * slowed speed. For now, just turn everything on and allow 53997c478bd9Sstevel@tonic-gate * cpu_init_private() to change it if necessary. 54007c478bd9Sstevel@tonic-gate */ 54017c478bd9Sstevel@tonic-gate csmp->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1; 54027c478bd9Sstevel@tonic-gate csmp->chsm_enable[CACHE_SCRUBBER_INFO_D] = 1; 54037c478bd9Sstevel@tonic-gate csmp->chsm_enable[CACHE_SCRUBBER_INFO_I] = 1; 54047c478bd9Sstevel@tonic-gate 54057c478bd9Sstevel@tonic-gate cpu_busy_ecache_scrub(cp); 54067c478bd9Sstevel@tonic-gate } 54077c478bd9Sstevel@tonic-gate 54087c478bd9Sstevel@tonic-gate /* 54097c478bd9Sstevel@tonic-gate * Un-initialization for cache scrubbing for the specified cpu. 54107c478bd9Sstevel@tonic-gate */ 54117c478bd9Sstevel@tonic-gate static void 54127c478bd9Sstevel@tonic-gate cpu_uninit_ecache_scrub_dr(struct cpu *cp) 54137c478bd9Sstevel@tonic-gate { 54147c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 54157c478bd9Sstevel@tonic-gate 54167c478bd9Sstevel@tonic-gate /* 54177c478bd9Sstevel@tonic-gate * un-initialize bookkeeping for cache scrubbing 54187c478bd9Sstevel@tonic-gate */ 54197c478bd9Sstevel@tonic-gate bzero(csmp, sizeof (ch_scrub_misc_t)); 54207c478bd9Sstevel@tonic-gate 54217c478bd9Sstevel@tonic-gate cpu_idle_ecache_scrub(cp); 54227c478bd9Sstevel@tonic-gate } 54237c478bd9Sstevel@tonic-gate 54247c478bd9Sstevel@tonic-gate /* 54257c478bd9Sstevel@tonic-gate * Called periodically on each CPU to scrub the D$. 54267c478bd9Sstevel@tonic-gate */ 54277c478bd9Sstevel@tonic-gate static void 54287c478bd9Sstevel@tonic-gate scrub_dcache(int how_many) 54297c478bd9Sstevel@tonic-gate { 54307c478bd9Sstevel@tonic-gate int i; 54317c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 54327c478bd9Sstevel@tonic-gate int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D]; 54337c478bd9Sstevel@tonic-gate 54347c478bd9Sstevel@tonic-gate /* 54357c478bd9Sstevel@tonic-gate * scrub the desired number of lines 54367c478bd9Sstevel@tonic-gate */ 54377c478bd9Sstevel@tonic-gate for (i = 0; i < how_many; i++) { 54387c478bd9Sstevel@tonic-gate /* 54397c478bd9Sstevel@tonic-gate * scrub a D$ line 54407c478bd9Sstevel@tonic-gate */ 54417c478bd9Sstevel@tonic-gate dcache_inval_line(index); 54427c478bd9Sstevel@tonic-gate 54437c478bd9Sstevel@tonic-gate /* 54447c478bd9Sstevel@tonic-gate * calculate the next D$ line to scrub, assumes 54457c478bd9Sstevel@tonic-gate * that dcache_nlines is a power of 2 54467c478bd9Sstevel@tonic-gate */ 54477c478bd9Sstevel@tonic-gate index = (index + 1) & (dcache_nlines - 1); 54487c478bd9Sstevel@tonic-gate } 54497c478bd9Sstevel@tonic-gate 54507c478bd9Sstevel@tonic-gate /* 54517c478bd9Sstevel@tonic-gate * set the scrub index for the next visit 54527c478bd9Sstevel@tonic-gate */ 54537c478bd9Sstevel@tonic-gate csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D] = index; 54547c478bd9Sstevel@tonic-gate } 54557c478bd9Sstevel@tonic-gate 54567c478bd9Sstevel@tonic-gate /* 54577c478bd9Sstevel@tonic-gate * Handler for D$ scrub inum softint. Call scrub_dcache until 54587c478bd9Sstevel@tonic-gate * we decrement the outstanding request count to zero. 54597c478bd9Sstevel@tonic-gate */ 54607c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 54617c478bd9Sstevel@tonic-gate static uint_t 54627c478bd9Sstevel@tonic-gate scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2) 54637c478bd9Sstevel@tonic-gate { 54647c478bd9Sstevel@tonic-gate int i; 54657c478bd9Sstevel@tonic-gate int how_many; 54667c478bd9Sstevel@tonic-gate int outstanding; 54677c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 54687c478bd9Sstevel@tonic-gate uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_D]; 54697c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg1; 54707c478bd9Sstevel@tonic-gate int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 54717c478bd9Sstevel@tonic-gate dcache_scan_rate_idle : dcache_scan_rate_busy; 54727c478bd9Sstevel@tonic-gate 54737c478bd9Sstevel@tonic-gate /* 54747c478bd9Sstevel@tonic-gate * The scan rates are expressed in units of tenths of a 54757c478bd9Sstevel@tonic-gate * percent. A scan rate of 1000 (100%) means the whole 54767c478bd9Sstevel@tonic-gate * cache is scanned every second. 54777c478bd9Sstevel@tonic-gate */ 54787c478bd9Sstevel@tonic-gate how_many = (dcache_nlines * scan_rate) / (1000 * csi->csi_freq); 54797c478bd9Sstevel@tonic-gate 54807c478bd9Sstevel@tonic-gate do { 54817c478bd9Sstevel@tonic-gate outstanding = *countp; 54827c478bd9Sstevel@tonic-gate for (i = 0; i < outstanding; i++) { 54837c478bd9Sstevel@tonic-gate scrub_dcache(how_many); 54847c478bd9Sstevel@tonic-gate } 54857c478bd9Sstevel@tonic-gate } while (atomic_add_32_nv(countp, -outstanding)); 54867c478bd9Sstevel@tonic-gate 54877c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 54887c478bd9Sstevel@tonic-gate } 54897c478bd9Sstevel@tonic-gate 54907c478bd9Sstevel@tonic-gate /* 54917c478bd9Sstevel@tonic-gate * Called periodically on each CPU to scrub the I$. The I$ is scrubbed 54927c478bd9Sstevel@tonic-gate * by invalidating lines. Due to the characteristics of the ASI which 54937c478bd9Sstevel@tonic-gate * is used to invalidate an I$ line, the entire I$ must be invalidated 54947c478bd9Sstevel@tonic-gate * vs. an individual I$ line. 54957c478bd9Sstevel@tonic-gate */ 54967c478bd9Sstevel@tonic-gate static void 54977c478bd9Sstevel@tonic-gate scrub_icache(int how_many) 54987c478bd9Sstevel@tonic-gate { 54997c478bd9Sstevel@tonic-gate int i; 55007c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 55017c478bd9Sstevel@tonic-gate int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I]; 55027c478bd9Sstevel@tonic-gate int icache_nlines = csmp->chsm_icache_nlines; 55037c478bd9Sstevel@tonic-gate 55047c478bd9Sstevel@tonic-gate /* 55057c478bd9Sstevel@tonic-gate * scrub the desired number of lines 55067c478bd9Sstevel@tonic-gate */ 55077c478bd9Sstevel@tonic-gate for (i = 0; i < how_many; i++) { 55087c478bd9Sstevel@tonic-gate /* 55097c478bd9Sstevel@tonic-gate * since the entire I$ must be scrubbed at once, 55107c478bd9Sstevel@tonic-gate * wait until the index wraps to zero to invalidate 55117c478bd9Sstevel@tonic-gate * the entire I$ 55127c478bd9Sstevel@tonic-gate */ 55137c478bd9Sstevel@tonic-gate if (index == 0) { 55147c478bd9Sstevel@tonic-gate icache_inval_all(); 55157c478bd9Sstevel@tonic-gate } 55167c478bd9Sstevel@tonic-gate 55177c478bd9Sstevel@tonic-gate /* 55187c478bd9Sstevel@tonic-gate * calculate the next I$ line to scrub, assumes 55197c478bd9Sstevel@tonic-gate * that chsm_icache_nlines is a power of 2 55207c478bd9Sstevel@tonic-gate */ 55217c478bd9Sstevel@tonic-gate index = (index + 1) & (icache_nlines - 1); 55227c478bd9Sstevel@tonic-gate } 55237c478bd9Sstevel@tonic-gate 55247c478bd9Sstevel@tonic-gate /* 55257c478bd9Sstevel@tonic-gate * set the scrub index for the next visit 55267c478bd9Sstevel@tonic-gate */ 55277c478bd9Sstevel@tonic-gate csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I] = index; 55287c478bd9Sstevel@tonic-gate } 55297c478bd9Sstevel@tonic-gate 55307c478bd9Sstevel@tonic-gate /* 55317c478bd9Sstevel@tonic-gate * Handler for I$ scrub inum softint. Call scrub_icache until 55327c478bd9Sstevel@tonic-gate * we decrement the outstanding request count to zero. 55337c478bd9Sstevel@tonic-gate */ 55347c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 55357c478bd9Sstevel@tonic-gate static uint_t 55367c478bd9Sstevel@tonic-gate scrub_icache_line_intr(caddr_t arg1, caddr_t arg2) 55377c478bd9Sstevel@tonic-gate { 55387c478bd9Sstevel@tonic-gate int i; 55397c478bd9Sstevel@tonic-gate int how_many; 55407c478bd9Sstevel@tonic-gate int outstanding; 55417c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 55427c478bd9Sstevel@tonic-gate uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_I]; 55437c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg1; 55447c478bd9Sstevel@tonic-gate int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 55457c478bd9Sstevel@tonic-gate icache_scan_rate_idle : icache_scan_rate_busy; 55467c478bd9Sstevel@tonic-gate int icache_nlines = csmp->chsm_icache_nlines; 55477c478bd9Sstevel@tonic-gate 55487c478bd9Sstevel@tonic-gate /* 55497c478bd9Sstevel@tonic-gate * The scan rates are expressed in units of tenths of a 55507c478bd9Sstevel@tonic-gate * percent. A scan rate of 1000 (100%) means the whole 55517c478bd9Sstevel@tonic-gate * cache is scanned every second. 55527c478bd9Sstevel@tonic-gate */ 55537c478bd9Sstevel@tonic-gate how_many = (icache_nlines * scan_rate) / (1000 * csi->csi_freq); 55547c478bd9Sstevel@tonic-gate 55557c478bd9Sstevel@tonic-gate do { 55567c478bd9Sstevel@tonic-gate outstanding = *countp; 55577c478bd9Sstevel@tonic-gate for (i = 0; i < outstanding; i++) { 55587c478bd9Sstevel@tonic-gate scrub_icache(how_many); 55597c478bd9Sstevel@tonic-gate } 55607c478bd9Sstevel@tonic-gate } while (atomic_add_32_nv(countp, -outstanding)); 55617c478bd9Sstevel@tonic-gate 55627c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 55637c478bd9Sstevel@tonic-gate } 55647c478bd9Sstevel@tonic-gate 55657c478bd9Sstevel@tonic-gate /* 55667c478bd9Sstevel@tonic-gate * Called periodically on each CPU to scrub the E$. 55677c478bd9Sstevel@tonic-gate */ 55687c478bd9Sstevel@tonic-gate static void 55697c478bd9Sstevel@tonic-gate scrub_ecache(int how_many) 55707c478bd9Sstevel@tonic-gate { 55717c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 55727c478bd9Sstevel@tonic-gate int i; 55737c478bd9Sstevel@tonic-gate int cpuid = CPU->cpu_id; 55747c478bd9Sstevel@tonic-gate int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E]; 55757c478bd9Sstevel@tonic-gate int nlines = csmp->chsm_ecache_nlines; 55767c478bd9Sstevel@tonic-gate int linesize = cpunodes[cpuid].ecache_linesize; 55777c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 55787c478bd9Sstevel@tonic-gate 55797c478bd9Sstevel@tonic-gate /* 55807c478bd9Sstevel@tonic-gate * scrub the desired number of lines 55817c478bd9Sstevel@tonic-gate */ 55827c478bd9Sstevel@tonic-gate for (i = 0; i < how_many; i++) { 55837c478bd9Sstevel@tonic-gate /* 55847c478bd9Sstevel@tonic-gate * scrub the E$ line 55857c478bd9Sstevel@tonic-gate */ 55867c478bd9Sstevel@tonic-gate ecache_flush_line(ecache_flushaddr + (index * linesize), 55877c478bd9Sstevel@tonic-gate ec_set_size); 55887c478bd9Sstevel@tonic-gate 55897c478bd9Sstevel@tonic-gate /* 55907c478bd9Sstevel@tonic-gate * calculate the next E$ line to scrub based on twice 55917c478bd9Sstevel@tonic-gate * the number of E$ lines (to displace lines containing 55927c478bd9Sstevel@tonic-gate * flush area data), assumes that the number of lines 55937c478bd9Sstevel@tonic-gate * is a power of 2 55947c478bd9Sstevel@tonic-gate */ 55957c478bd9Sstevel@tonic-gate index = (index + 1) & ((nlines << 1) - 1); 55967c478bd9Sstevel@tonic-gate } 55977c478bd9Sstevel@tonic-gate 55987c478bd9Sstevel@tonic-gate /* 55997c478bd9Sstevel@tonic-gate * set the ecache scrub index for the next visit 56007c478bd9Sstevel@tonic-gate */ 56017c478bd9Sstevel@tonic-gate csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E] = index; 56027c478bd9Sstevel@tonic-gate } 56037c478bd9Sstevel@tonic-gate 56047c478bd9Sstevel@tonic-gate /* 56057c478bd9Sstevel@tonic-gate * Handler for E$ scrub inum softint. Call the E$ scrubber until 56067c478bd9Sstevel@tonic-gate * we decrement the outstanding request count to zero. 56076319f043Srscott * 56086319f043Srscott * Due to interactions with cpu_scrub_cpu_setup(), the outstanding count may 56096319f043Srscott * become negative after the atomic_add_32_nv(). This is not a problem, as 56106319f043Srscott * the next trip around the loop won't scrub anything, and the next add will 56116319f043Srscott * reset the count back to zero. 56127c478bd9Sstevel@tonic-gate */ 56137c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 56147c478bd9Sstevel@tonic-gate static uint_t 56157c478bd9Sstevel@tonic-gate scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2) 56167c478bd9Sstevel@tonic-gate { 56177c478bd9Sstevel@tonic-gate int i; 56187c478bd9Sstevel@tonic-gate int how_many; 56197c478bd9Sstevel@tonic-gate int outstanding; 56207c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 56217c478bd9Sstevel@tonic-gate uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_E]; 56227c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg1; 56237c478bd9Sstevel@tonic-gate int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 56247c478bd9Sstevel@tonic-gate ecache_scan_rate_idle : ecache_scan_rate_busy; 56257c478bd9Sstevel@tonic-gate int ecache_nlines = csmp->chsm_ecache_nlines; 56267c478bd9Sstevel@tonic-gate 56277c478bd9Sstevel@tonic-gate /* 56287c478bd9Sstevel@tonic-gate * The scan rates are expressed in units of tenths of a 56297c478bd9Sstevel@tonic-gate * percent. A scan rate of 1000 (100%) means the whole 56307c478bd9Sstevel@tonic-gate * cache is scanned every second. 56317c478bd9Sstevel@tonic-gate */ 56327c478bd9Sstevel@tonic-gate how_many = (ecache_nlines * scan_rate) / (1000 * csi->csi_freq); 56337c478bd9Sstevel@tonic-gate 56347c478bd9Sstevel@tonic-gate do { 56357c478bd9Sstevel@tonic-gate outstanding = *countp; 56367c478bd9Sstevel@tonic-gate for (i = 0; i < outstanding; i++) { 56377c478bd9Sstevel@tonic-gate scrub_ecache(how_many); 56387c478bd9Sstevel@tonic-gate } 56397c478bd9Sstevel@tonic-gate } while (atomic_add_32_nv(countp, -outstanding)); 56407c478bd9Sstevel@tonic-gate 56417c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 56427c478bd9Sstevel@tonic-gate } 56437c478bd9Sstevel@tonic-gate 56447c478bd9Sstevel@tonic-gate /* 56457c478bd9Sstevel@tonic-gate * Timeout function to reenable CE 56467c478bd9Sstevel@tonic-gate */ 56477c478bd9Sstevel@tonic-gate static void 56487c478bd9Sstevel@tonic-gate cpu_delayed_check_ce_errors(void *arg) 56497c478bd9Sstevel@tonic-gate { 56507c478bd9Sstevel@tonic-gate if (!taskq_dispatch(ch_check_ce_tq, cpu_check_ce_errors, arg, 56517c478bd9Sstevel@tonic-gate TQ_NOSLEEP)) { 56527c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, arg, 56537c478bd9Sstevel@tonic-gate drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 56547c478bd9Sstevel@tonic-gate } 56557c478bd9Sstevel@tonic-gate } 56567c478bd9Sstevel@tonic-gate 56577c478bd9Sstevel@tonic-gate /* 56587c478bd9Sstevel@tonic-gate * CE Deferred Re-enable after trap. 56597c478bd9Sstevel@tonic-gate * 56607c478bd9Sstevel@tonic-gate * When the CPU gets a disrupting trap for any of the errors 56617c478bd9Sstevel@tonic-gate * controlled by the CEEN bit, CEEN is disabled in the trap handler 56627c478bd9Sstevel@tonic-gate * immediately. To eliminate the possibility of multiple CEs causing 56637c478bd9Sstevel@tonic-gate * recursive stack overflow in the trap handler, we cannot 56647c478bd9Sstevel@tonic-gate * reenable CEEN while still running in the trap handler. Instead, 56657c478bd9Sstevel@tonic-gate * after a CE is logged on a CPU, we schedule a timeout function, 56667c478bd9Sstevel@tonic-gate * cpu_check_ce_errors(), to trigger after cpu_ceen_delay_secs 56677c478bd9Sstevel@tonic-gate * seconds. This function will check whether any further CEs 56687c478bd9Sstevel@tonic-gate * have occurred on that CPU, and if none have, will reenable CEEN. 56697c478bd9Sstevel@tonic-gate * 56707c478bd9Sstevel@tonic-gate * If further CEs have occurred while CEEN is disabled, another 56717c478bd9Sstevel@tonic-gate * timeout will be scheduled. This is to ensure that the CPU can 56727c478bd9Sstevel@tonic-gate * make progress in the face of CE 'storms', and that it does not 56737c478bd9Sstevel@tonic-gate * spend all its time logging CE errors. 56747c478bd9Sstevel@tonic-gate */ 56757c478bd9Sstevel@tonic-gate static void 56767c478bd9Sstevel@tonic-gate cpu_check_ce_errors(void *arg) 56777c478bd9Sstevel@tonic-gate { 56787c478bd9Sstevel@tonic-gate int cpuid = (int)arg; 56797c478bd9Sstevel@tonic-gate cpu_t *cp; 56807c478bd9Sstevel@tonic-gate 56817c478bd9Sstevel@tonic-gate /* 56827c478bd9Sstevel@tonic-gate * We acquire cpu_lock. 56837c478bd9Sstevel@tonic-gate */ 56847c478bd9Sstevel@tonic-gate ASSERT(curthread->t_pil == 0); 56857c478bd9Sstevel@tonic-gate 56867c478bd9Sstevel@tonic-gate /* 56877c478bd9Sstevel@tonic-gate * verify that the cpu is still around, DR 56887c478bd9Sstevel@tonic-gate * could have got there first ... 56897c478bd9Sstevel@tonic-gate */ 56907c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 56917c478bd9Sstevel@tonic-gate cp = cpu_get(cpuid); 56927c478bd9Sstevel@tonic-gate if (cp == NULL) { 56937c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 56947c478bd9Sstevel@tonic-gate return; 56957c478bd9Sstevel@tonic-gate } 56967c478bd9Sstevel@tonic-gate /* 56977c478bd9Sstevel@tonic-gate * make sure we don't migrate across CPUs 56987c478bd9Sstevel@tonic-gate * while checking our CE status. 56997c478bd9Sstevel@tonic-gate */ 57007c478bd9Sstevel@tonic-gate kpreempt_disable(); 57017c478bd9Sstevel@tonic-gate 57027c478bd9Sstevel@tonic-gate /* 57037c478bd9Sstevel@tonic-gate * If we are running on the CPU that got the 57047c478bd9Sstevel@tonic-gate * CE, we can do the checks directly. 57057c478bd9Sstevel@tonic-gate */ 57067c478bd9Sstevel@tonic-gate if (cp->cpu_id == CPU->cpu_id) { 57077c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 57087c478bd9Sstevel@tonic-gate cpu_check_ce(TIMEOUT_CEEN_CHECK, 0, 0, 0); 57097c478bd9Sstevel@tonic-gate kpreempt_enable(); 57107c478bd9Sstevel@tonic-gate return; 57117c478bd9Sstevel@tonic-gate } 57127c478bd9Sstevel@tonic-gate kpreempt_enable(); 57137c478bd9Sstevel@tonic-gate 57147c478bd9Sstevel@tonic-gate /* 57157c478bd9Sstevel@tonic-gate * send an x-call to get the CPU that originally 57167c478bd9Sstevel@tonic-gate * got the CE to do the necessary checks. If we can't 57177c478bd9Sstevel@tonic-gate * send the x-call, reschedule the timeout, otherwise we 57187c478bd9Sstevel@tonic-gate * lose CEEN forever on that CPU. 57197c478bd9Sstevel@tonic-gate */ 57207c478bd9Sstevel@tonic-gate if (CPU_XCALL_READY(cp->cpu_id) && (!(cp->cpu_flags & CPU_QUIESCED))) { 57217c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)cpu_check_ce, 57227c478bd9Sstevel@tonic-gate TIMEOUT_CEEN_CHECK, 0); 57237c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 57247c478bd9Sstevel@tonic-gate } else { 57257c478bd9Sstevel@tonic-gate /* 57267c478bd9Sstevel@tonic-gate * When the CPU is not accepting xcalls, or 57277c478bd9Sstevel@tonic-gate * the processor is offlined, we don't want to 57287c478bd9Sstevel@tonic-gate * incur the extra overhead of trying to schedule the 57297c478bd9Sstevel@tonic-gate * CE timeout indefinitely. However, we don't want to lose 57307c478bd9Sstevel@tonic-gate * CE checking forever. 57317c478bd9Sstevel@tonic-gate * 57327c478bd9Sstevel@tonic-gate * Keep rescheduling the timeout, accepting the additional 57337c478bd9Sstevel@tonic-gate * overhead as the cost of correctness in the case where we get 57347c478bd9Sstevel@tonic-gate * a CE, disable CEEN, offline the CPU during the 57357c478bd9Sstevel@tonic-gate * the timeout interval, and then online it at some 57367c478bd9Sstevel@tonic-gate * point in the future. This is unlikely given the short 57377c478bd9Sstevel@tonic-gate * cpu_ceen_delay_secs. 57387c478bd9Sstevel@tonic-gate */ 57397c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 57407c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, (void *)cp->cpu_id, 57417c478bd9Sstevel@tonic-gate drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 57427c478bd9Sstevel@tonic-gate } 57437c478bd9Sstevel@tonic-gate } 57447c478bd9Sstevel@tonic-gate 57457c478bd9Sstevel@tonic-gate /* 57467c478bd9Sstevel@tonic-gate * This routine will check whether CEs have occurred while 57477c478bd9Sstevel@tonic-gate * CEEN is disabled. Any CEs detected will be logged and, if 57487c478bd9Sstevel@tonic-gate * possible, scrubbed. 57497c478bd9Sstevel@tonic-gate * 57507c478bd9Sstevel@tonic-gate * The memscrubber will also use this routine to clear any errors 57517c478bd9Sstevel@tonic-gate * caused by its scrubbing with CEEN disabled. 57527c478bd9Sstevel@tonic-gate * 57537c478bd9Sstevel@tonic-gate * flag == SCRUBBER_CEEN_CHECK 57547c478bd9Sstevel@tonic-gate * called from memscrubber, just check/scrub, no reset 57557c478bd9Sstevel@tonic-gate * paddr physical addr. for start of scrub pages 57567c478bd9Sstevel@tonic-gate * vaddr virtual addr. for scrub area 57577c478bd9Sstevel@tonic-gate * psz page size of area to be scrubbed 57587c478bd9Sstevel@tonic-gate * 57597c478bd9Sstevel@tonic-gate * flag == TIMEOUT_CEEN_CHECK 57607c478bd9Sstevel@tonic-gate * timeout function has triggered, reset timeout or CEEN 57617c478bd9Sstevel@tonic-gate * 57627c478bd9Sstevel@tonic-gate * Note: We must not migrate cpus during this function. This can be 57637c478bd9Sstevel@tonic-gate * achieved by one of: 57647c478bd9Sstevel@tonic-gate * - invoking as target of an x-call in which case we're at XCALL_PIL 57657c478bd9Sstevel@tonic-gate * The flag value must be first xcall argument. 57667c478bd9Sstevel@tonic-gate * - disabling kernel preemption. This should be done for very short 57677c478bd9Sstevel@tonic-gate * periods so is not suitable for SCRUBBER_CEEN_CHECK where we might 57687c478bd9Sstevel@tonic-gate * scrub an extended area with cpu_check_block. The call for 57697c478bd9Sstevel@tonic-gate * TIMEOUT_CEEN_CHECK uses this so cpu_check_ce must be kept 57707c478bd9Sstevel@tonic-gate * brief for this case. 57717c478bd9Sstevel@tonic-gate * - binding to a cpu, eg with thread_affinity_set(). This is used 57727c478bd9Sstevel@tonic-gate * in the SCRUBBER_CEEN_CHECK case, but is not practical for 57737c478bd9Sstevel@tonic-gate * the TIMEOUT_CEEN_CHECK because both need cpu_lock. 57747c478bd9Sstevel@tonic-gate */ 57757c478bd9Sstevel@tonic-gate void 57767c478bd9Sstevel@tonic-gate cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t psz) 57777c478bd9Sstevel@tonic-gate { 57787c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 57797c478bd9Sstevel@tonic-gate uint64_t ec_err_enable; 57807c478bd9Sstevel@tonic-gate uint64_t page_offset; 57817c478bd9Sstevel@tonic-gate 57827c478bd9Sstevel@tonic-gate /* Read AFSR */ 57837c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 57847c478bd9Sstevel@tonic-gate 57857c478bd9Sstevel@tonic-gate /* 57867c478bd9Sstevel@tonic-gate * If no CEEN errors have occurred during the timeout 57877c478bd9Sstevel@tonic-gate * interval, it is safe to re-enable CEEN and exit. 57887c478bd9Sstevel@tonic-gate */ 57897c478bd9Sstevel@tonic-gate if ((cpu_error_regs.afsr & C_AFSR_CECC_ERRS) == 0) { 57907c478bd9Sstevel@tonic-gate if (flag == TIMEOUT_CEEN_CHECK && 57917c478bd9Sstevel@tonic-gate !((ec_err_enable = get_error_enable()) & EN_REG_CEEN)) 57927c478bd9Sstevel@tonic-gate set_error_enable(ec_err_enable | EN_REG_CEEN); 57937c478bd9Sstevel@tonic-gate return; 57947c478bd9Sstevel@tonic-gate } 57957c478bd9Sstevel@tonic-gate 57967c478bd9Sstevel@tonic-gate /* 57977c478bd9Sstevel@tonic-gate * Ensure that CEEN was not reenabled (maybe by DR) before 57987c478bd9Sstevel@tonic-gate * we log/clear the error. 57997c478bd9Sstevel@tonic-gate */ 58007c478bd9Sstevel@tonic-gate if ((ec_err_enable = get_error_enable()) & EN_REG_CEEN) 58017c478bd9Sstevel@tonic-gate set_error_enable(ec_err_enable & ~EN_REG_CEEN); 58027c478bd9Sstevel@tonic-gate 58037c478bd9Sstevel@tonic-gate /* 58047c478bd9Sstevel@tonic-gate * log/clear the CE. If CE_CEEN_DEFER is passed, the 58057c478bd9Sstevel@tonic-gate * timeout will be rescheduled when the error is logged. 58067c478bd9Sstevel@tonic-gate */ 58077c478bd9Sstevel@tonic-gate if (!(cpu_error_regs.afsr & cpu_ce_not_deferred)) 58087c478bd9Sstevel@tonic-gate cpu_ce_detected(&cpu_error_regs, 58097c478bd9Sstevel@tonic-gate CE_CEEN_DEFER | CE_CEEN_TIMEOUT); 58107c478bd9Sstevel@tonic-gate else 58117c478bd9Sstevel@tonic-gate cpu_ce_detected(&cpu_error_regs, CE_CEEN_TIMEOUT); 58127c478bd9Sstevel@tonic-gate 58137c478bd9Sstevel@tonic-gate /* 58147c478bd9Sstevel@tonic-gate * If the memory scrubber runs while CEEN is 58157c478bd9Sstevel@tonic-gate * disabled, (or if CEEN is disabled during the 58167c478bd9Sstevel@tonic-gate * scrub as a result of a CE being triggered by 58177c478bd9Sstevel@tonic-gate * it), the range being scrubbed will not be 58187c478bd9Sstevel@tonic-gate * completely cleaned. If there are multiple CEs 58197c478bd9Sstevel@tonic-gate * in the range at most two of these will be dealt 58207c478bd9Sstevel@tonic-gate * with, (one by the trap handler and one by the 58217c478bd9Sstevel@tonic-gate * timeout). It is also possible that none are dealt 58227c478bd9Sstevel@tonic-gate * with, (CEEN disabled and another CE occurs before 58237c478bd9Sstevel@tonic-gate * the timeout triggers). So to ensure that the 58247c478bd9Sstevel@tonic-gate * memory is actually scrubbed, we have to access each 58257c478bd9Sstevel@tonic-gate * memory location in the range and then check whether 58267c478bd9Sstevel@tonic-gate * that access causes a CE. 58277c478bd9Sstevel@tonic-gate */ 58287c478bd9Sstevel@tonic-gate if (flag == SCRUBBER_CEEN_CHECK && va) { 58297c478bd9Sstevel@tonic-gate if ((cpu_error_regs.afar >= pa) && 58307c478bd9Sstevel@tonic-gate (cpu_error_regs.afar < (pa + psz))) { 58317c478bd9Sstevel@tonic-gate /* 58327c478bd9Sstevel@tonic-gate * Force a load from physical memory for each 58337c478bd9Sstevel@tonic-gate * 64-byte block, then check AFSR to determine 58347c478bd9Sstevel@tonic-gate * whether this access caused an error. 58357c478bd9Sstevel@tonic-gate * 58367c478bd9Sstevel@tonic-gate * This is a slow way to do a scrub, but as it will 58377c478bd9Sstevel@tonic-gate * only be invoked when the memory scrubber actually 58387c478bd9Sstevel@tonic-gate * triggered a CE, it should not happen too 58397c478bd9Sstevel@tonic-gate * frequently. 58407c478bd9Sstevel@tonic-gate * 58417c478bd9Sstevel@tonic-gate * cut down what we need to check as the scrubber 58427c478bd9Sstevel@tonic-gate * has verified up to AFAR, so get it's offset 58437c478bd9Sstevel@tonic-gate * into the page and start there. 58447c478bd9Sstevel@tonic-gate */ 58457c478bd9Sstevel@tonic-gate page_offset = (uint64_t)(cpu_error_regs.afar & 58467c478bd9Sstevel@tonic-gate (psz - 1)); 58477c478bd9Sstevel@tonic-gate va = (caddr_t)(va + (P2ALIGN(page_offset, 64))); 58487c478bd9Sstevel@tonic-gate psz -= (uint_t)(P2ALIGN(page_offset, 64)); 58497c478bd9Sstevel@tonic-gate cpu_check_block((caddr_t)(P2ALIGN((uint64_t)va, 64)), 58507c478bd9Sstevel@tonic-gate psz); 58517c478bd9Sstevel@tonic-gate } 58527c478bd9Sstevel@tonic-gate } 58537c478bd9Sstevel@tonic-gate 58547c478bd9Sstevel@tonic-gate /* 58557c478bd9Sstevel@tonic-gate * Reset error enable if this CE is not masked. 58567c478bd9Sstevel@tonic-gate */ 58577c478bd9Sstevel@tonic-gate if ((flag == TIMEOUT_CEEN_CHECK) && 58587c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr & cpu_ce_not_deferred)) 58597c478bd9Sstevel@tonic-gate set_error_enable(ec_err_enable | EN_REG_CEEN); 58607c478bd9Sstevel@tonic-gate 58617c478bd9Sstevel@tonic-gate } 58627c478bd9Sstevel@tonic-gate 58637c478bd9Sstevel@tonic-gate /* 58647c478bd9Sstevel@tonic-gate * Attempt a cpu logout for an error that we did not trap for, such 58657c478bd9Sstevel@tonic-gate * as a CE noticed with CEEN off. It is assumed that we are still running 58667c478bd9Sstevel@tonic-gate * on the cpu that took the error and that we cannot migrate. Returns 58677c478bd9Sstevel@tonic-gate * 0 on success, otherwise nonzero. 58687c478bd9Sstevel@tonic-gate */ 58697c478bd9Sstevel@tonic-gate static int 58707c478bd9Sstevel@tonic-gate cpu_ce_delayed_ec_logout(uint64_t afar) 58717c478bd9Sstevel@tonic-gate { 58727c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 58737c478bd9Sstevel@tonic-gate 58747c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) 58757c478bd9Sstevel@tonic-gate return (0); 58767c478bd9Sstevel@tonic-gate 58777c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 58787c478bd9Sstevel@tonic-gate if (cas64(&clop->clo_data.chd_afar, LOGOUT_INVALID, afar) != 58797c478bd9Sstevel@tonic-gate LOGOUT_INVALID) 58807c478bd9Sstevel@tonic-gate return (0); 58817c478bd9Sstevel@tonic-gate 58827c478bd9Sstevel@tonic-gate cpu_delayed_logout(afar, clop); 58837c478bd9Sstevel@tonic-gate return (1); 58847c478bd9Sstevel@tonic-gate } 58857c478bd9Sstevel@tonic-gate 58867c478bd9Sstevel@tonic-gate /* 58877c478bd9Sstevel@tonic-gate * We got an error while CEEN was disabled. We 58887c478bd9Sstevel@tonic-gate * need to clean up after it and log whatever 58897c478bd9Sstevel@tonic-gate * information we have on the CE. 58907c478bd9Sstevel@tonic-gate */ 58917c478bd9Sstevel@tonic-gate void 58927c478bd9Sstevel@tonic-gate cpu_ce_detected(ch_cpu_errors_t *cpu_error_regs, int flag) 58937c478bd9Sstevel@tonic-gate { 58947c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 58957c478bd9Sstevel@tonic-gate struct async_flt *aflt; 58967c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 58977c478bd9Sstevel@tonic-gate 58987c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 58997c478bd9Sstevel@tonic-gate ch_flt.flt_trapped_ce = flag; 59007c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 59017c478bd9Sstevel@tonic-gate aflt->flt_stat = cpu_error_regs->afsr & C_AFSR_MASK; 59027c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = cpu_error_regs->afsr_ext; 59037c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = (cpu_error_regs->afsr_ext & C_AFSR_EXT_ALL_ERRS) | 59047c478bd9Sstevel@tonic-gate (cpu_error_regs->afsr & C_AFSR_ALL_ERRS); 59057c478bd9Sstevel@tonic-gate aflt->flt_addr = cpu_error_regs->afar; 59067c478bd9Sstevel@tonic-gate #if defined(SERRANO) 59077c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs->afar2; 59087c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 59097c478bd9Sstevel@tonic-gate aflt->flt_pc = NULL; 59107c478bd9Sstevel@tonic-gate aflt->flt_priv = ((cpu_error_regs->afsr & C_AFSR_PRIV) != 0); 59117c478bd9Sstevel@tonic-gate aflt->flt_tl = 0; 59127c478bd9Sstevel@tonic-gate aflt->flt_panic = 0; 59137c478bd9Sstevel@tonic-gate cpu_log_and_clear_ce(&ch_flt); 59147c478bd9Sstevel@tonic-gate 59157c478bd9Sstevel@tonic-gate /* 59167c478bd9Sstevel@tonic-gate * check if we caused any errors during cleanup 59177c478bd9Sstevel@tonic-gate */ 59187c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 59197c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 59207c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 59217c478bd9Sstevel@tonic-gate NULL); 59227c478bd9Sstevel@tonic-gate } 59237c478bd9Sstevel@tonic-gate } 59247c478bd9Sstevel@tonic-gate 59257c478bd9Sstevel@tonic-gate /* 59267c478bd9Sstevel@tonic-gate * Log/clear CEEN-controlled disrupting errors 59277c478bd9Sstevel@tonic-gate */ 59287c478bd9Sstevel@tonic-gate static void 59297c478bd9Sstevel@tonic-gate cpu_log_and_clear_ce(ch_async_flt_t *ch_flt) 59307c478bd9Sstevel@tonic-gate { 59317c478bd9Sstevel@tonic-gate struct async_flt *aflt; 59327c478bd9Sstevel@tonic-gate uint64_t afsr, afsr_errs; 59337c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 59347c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 59357c478bd9Sstevel@tonic-gate on_trap_data_t *otp = curthread->t_ontrap; 59367c478bd9Sstevel@tonic-gate 59377c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)ch_flt; 59387c478bd9Sstevel@tonic-gate afsr = aflt->flt_stat; 59397c478bd9Sstevel@tonic-gate afsr_errs = ch_flt->afsr_errs; 59407c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 59417c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 59427c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 59437c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 59447c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 59457c478bd9Sstevel@tonic-gate aflt->flt_status = ECC_C_TRAP; 59467c478bd9Sstevel@tonic-gate 59477c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 59487c478bd9Sstevel@tonic-gate /* 59497c478bd9Sstevel@tonic-gate * Get the CPU log out info for Disrupting Trap. 59507c478bd9Sstevel@tonic-gate */ 59517c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 59527c478bd9Sstevel@tonic-gate clop = NULL; 59537c478bd9Sstevel@tonic-gate ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 59547c478bd9Sstevel@tonic-gate } else { 59557c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 59567c478bd9Sstevel@tonic-gate } 59577c478bd9Sstevel@tonic-gate 59587c478bd9Sstevel@tonic-gate if (clop && ch_flt->flt_trapped_ce & CE_CEEN_TIMEOUT) { 59597c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 59607c478bd9Sstevel@tonic-gate 59617c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 59627c478bd9Sstevel@tonic-gate (void) cpu_ce_delayed_ec_logout(cpu_error_regs.afar); 59637c478bd9Sstevel@tonic-gate clop->clo_data.chd_afsr = cpu_error_regs.afsr; 59647c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = cpu_error_regs.afar; 59657c478bd9Sstevel@tonic-gate clop->clo_data.chd_afsr_ext = cpu_error_regs.afsr_ext; 59667c478bd9Sstevel@tonic-gate clop->clo_sdw_data.chd_afsr = cpu_error_regs.shadow_afsr; 59677c478bd9Sstevel@tonic-gate clop->clo_sdw_data.chd_afar = cpu_error_regs.shadow_afar; 59687c478bd9Sstevel@tonic-gate clop->clo_sdw_data.chd_afsr_ext = 59697c478bd9Sstevel@tonic-gate cpu_error_regs.shadow_afsr_ext; 59707c478bd9Sstevel@tonic-gate #if defined(SERRANO) 59717c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar2 = cpu_error_regs.afar2; 59727c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 59737c478bd9Sstevel@tonic-gate ch_flt->flt_data_incomplete = 1; 59747c478bd9Sstevel@tonic-gate 59757c478bd9Sstevel@tonic-gate /* 59767c478bd9Sstevel@tonic-gate * The logging/clear code expects AFSR/AFAR to be cleared. 59777c478bd9Sstevel@tonic-gate * The trap handler does it for CEEN enabled errors 59787c478bd9Sstevel@tonic-gate * so we need to do it here. 59797c478bd9Sstevel@tonic-gate */ 59807c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 59817c478bd9Sstevel@tonic-gate } 59827c478bd9Sstevel@tonic-gate 59837c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 59847c478bd9Sstevel@tonic-gate /* 59857c478bd9Sstevel@tonic-gate * FRC: Can't scrub memory as we don't have AFAR for Jalapeno. 59867c478bd9Sstevel@tonic-gate * For Serrano, even thou we do have the AFAR, we still do the 59877c478bd9Sstevel@tonic-gate * scrub on the RCE side since that's where the error type can 59887c478bd9Sstevel@tonic-gate * be properly classified as intermittent, persistent, etc. 59897c478bd9Sstevel@tonic-gate * 59907c478bd9Sstevel@tonic-gate * CE/RCE: If error is in memory and AFAR is valid, scrub the memory. 59917c478bd9Sstevel@tonic-gate * Must scrub memory before cpu_queue_events, as scrubbing memory sets 59927c478bd9Sstevel@tonic-gate * the flt_status bits. 59937c478bd9Sstevel@tonic-gate */ 59947c478bd9Sstevel@tonic-gate if ((afsr & (C_AFSR_CE|C_AFSR_RCE)) && 59957c478bd9Sstevel@tonic-gate (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 59967c478bd9Sstevel@tonic-gate cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_RCE)))) { 59977c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err(aflt, B_TRUE); 59987c478bd9Sstevel@tonic-gate } 59997c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 60007c478bd9Sstevel@tonic-gate /* 60017c478bd9Sstevel@tonic-gate * CE/EMC: If error is in memory and AFAR is valid, scrub the memory. 60027c478bd9Sstevel@tonic-gate * Must scrub memory before cpu_queue_events, as scrubbing memory sets 60037c478bd9Sstevel@tonic-gate * the flt_status bits. 60047c478bd9Sstevel@tonic-gate */ 60057c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_CE|C_AFSR_EMC)) { 60067c478bd9Sstevel@tonic-gate if (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 60077c478bd9Sstevel@tonic-gate cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_EMC))) { 60087c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err(aflt, B_TRUE); 60097c478bd9Sstevel@tonic-gate } 60107c478bd9Sstevel@tonic-gate } 60117c478bd9Sstevel@tonic-gate 60127c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 60137c478bd9Sstevel@tonic-gate 60147c478bd9Sstevel@tonic-gate /* 60157c478bd9Sstevel@tonic-gate * Update flt_prot if this error occurred under on_trap protection. 60167c478bd9Sstevel@tonic-gate */ 60177c478bd9Sstevel@tonic-gate if (otp != NULL && (otp->ot_prot & OT_DATA_EC)) 60187c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_EC; 60197c478bd9Sstevel@tonic-gate 60207c478bd9Sstevel@tonic-gate /* 60217c478bd9Sstevel@tonic-gate * Queue events on the async event queue, one event per error bit. 60227c478bd9Sstevel@tonic-gate */ 60237c478bd9Sstevel@tonic-gate if (cpu_queue_events(ch_flt, pr_reason, afsr_errs, clop) == 0 || 60247c478bd9Sstevel@tonic-gate (afsr_errs & (C_AFSR_CECC_ERRS | C_AFSR_EXT_CECC_ERRS)) == 0) { 60257c478bd9Sstevel@tonic-gate ch_flt->flt_type = CPU_INV_AFSR; 60267c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 60277c478bd9Sstevel@tonic-gate (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 60287c478bd9Sstevel@tonic-gate aflt->flt_panic); 60297c478bd9Sstevel@tonic-gate } 60307c478bd9Sstevel@tonic-gate 60317c478bd9Sstevel@tonic-gate /* 60327c478bd9Sstevel@tonic-gate * Zero out + invalidate CPU logout. 60337c478bd9Sstevel@tonic-gate */ 60347c478bd9Sstevel@tonic-gate if (clop) { 60357c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 60367c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 60377c478bd9Sstevel@tonic-gate } 60387c478bd9Sstevel@tonic-gate 60397c478bd9Sstevel@tonic-gate /* 60407c478bd9Sstevel@tonic-gate * If either a CPC, WDC or EDC error has occurred while CEEN 60417c478bd9Sstevel@tonic-gate * was disabled, we need to flush either the entire 60427c478bd9Sstevel@tonic-gate * E$ or an E$ line. 60437c478bd9Sstevel@tonic-gate */ 60447c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 60457c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_CPU | C_AFSR_WDC)) 60467c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 60477c478bd9Sstevel@tonic-gate if (afsr_errs & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_WDC | C_AFSR_L3_EDC | 60487c478bd9Sstevel@tonic-gate C_AFSR_L3_CPC | C_AFSR_L3_WDC)) 60497c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 60507c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(ch_flt); 60517c478bd9Sstevel@tonic-gate 60527c478bd9Sstevel@tonic-gate } 60537c478bd9Sstevel@tonic-gate 60547c478bd9Sstevel@tonic-gate /* 60557c478bd9Sstevel@tonic-gate * depending on the error type, we determine whether we 60567c478bd9Sstevel@tonic-gate * need to flush the entire ecache or just a line. 60577c478bd9Sstevel@tonic-gate */ 60587c478bd9Sstevel@tonic-gate static int 60597c478bd9Sstevel@tonic-gate cpu_error_ecache_flush_required(ch_async_flt_t *ch_flt) 60607c478bd9Sstevel@tonic-gate { 60617c478bd9Sstevel@tonic-gate struct async_flt *aflt; 60627c478bd9Sstevel@tonic-gate uint64_t afsr; 60637c478bd9Sstevel@tonic-gate uint64_t afsr_errs = ch_flt->afsr_errs; 60647c478bd9Sstevel@tonic-gate 60657c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)ch_flt; 60667c478bd9Sstevel@tonic-gate afsr = aflt->flt_stat; 60677c478bd9Sstevel@tonic-gate 60687c478bd9Sstevel@tonic-gate /* 60697c478bd9Sstevel@tonic-gate * If we got multiple errors, no point in trying 60707c478bd9Sstevel@tonic-gate * the individual cases, just flush the whole cache 60717c478bd9Sstevel@tonic-gate */ 60727c478bd9Sstevel@tonic-gate if (afsr & C_AFSR_ME) { 60737c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 60747c478bd9Sstevel@tonic-gate } 60757c478bd9Sstevel@tonic-gate 60767c478bd9Sstevel@tonic-gate /* 60777c478bd9Sstevel@tonic-gate * If either a CPC, WDC or EDC error has occurred while CEEN 60787c478bd9Sstevel@tonic-gate * was disabled, we need to flush entire E$. We can't just 60797c478bd9Sstevel@tonic-gate * flush the cache line affected as the ME bit 60807c478bd9Sstevel@tonic-gate * is not set when multiple correctable errors of the same 60817c478bd9Sstevel@tonic-gate * type occur, so we might have multiple CPC or EDC errors, 60827c478bd9Sstevel@tonic-gate * with only the first recorded. 60837c478bd9Sstevel@tonic-gate */ 60847c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 60857c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_CPC | C_AFSR_CPU | C_AFSR_EDC | C_AFSR_WDC)) { 60867c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 60877c478bd9Sstevel@tonic-gate if (afsr_errs & (C_AFSR_CPC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_L3_CPC | 60887c478bd9Sstevel@tonic-gate C_AFSR_L3_EDC | C_AFSR_L3_WDC)) { 60897c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 60907c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 60917c478bd9Sstevel@tonic-gate } 60927c478bd9Sstevel@tonic-gate 60937c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 60947c478bd9Sstevel@tonic-gate /* 60957c478bd9Sstevel@tonic-gate * If only UE or RUE is set, flush the Ecache line, otherwise 60967c478bd9Sstevel@tonic-gate * flush the entire Ecache. 60977c478bd9Sstevel@tonic-gate */ 60987c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_UE|C_AFSR_RUE)) { 60997c478bd9Sstevel@tonic-gate if ((afsr & C_AFSR_ALL_ERRS) == C_AFSR_UE || 61007c478bd9Sstevel@tonic-gate (afsr & C_AFSR_ALL_ERRS) == C_AFSR_RUE) { 61017c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 61027c478bd9Sstevel@tonic-gate } else { 61037c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 61047c478bd9Sstevel@tonic-gate } 61057c478bd9Sstevel@tonic-gate } 61067c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 61077c478bd9Sstevel@tonic-gate /* 61087c478bd9Sstevel@tonic-gate * If UE only is set, flush the Ecache line, otherwise 61097c478bd9Sstevel@tonic-gate * flush the entire Ecache. 61107c478bd9Sstevel@tonic-gate */ 61117c478bd9Sstevel@tonic-gate if (afsr_errs & C_AFSR_UE) { 61127c478bd9Sstevel@tonic-gate if ((afsr_errs & (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == 61137c478bd9Sstevel@tonic-gate C_AFSR_UE) { 61147c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 61157c478bd9Sstevel@tonic-gate } else { 61167c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 61177c478bd9Sstevel@tonic-gate } 61187c478bd9Sstevel@tonic-gate } 61197c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 61207c478bd9Sstevel@tonic-gate 61217c478bd9Sstevel@tonic-gate /* 61227c478bd9Sstevel@tonic-gate * EDU: If EDU only is set, flush the ecache line, otherwise 61237c478bd9Sstevel@tonic-gate * flush the entire Ecache. 61247c478bd9Sstevel@tonic-gate */ 61257c478bd9Sstevel@tonic-gate if (afsr_errs & (C_AFSR_EDU | C_AFSR_L3_EDU)) { 61267c478bd9Sstevel@tonic-gate if (((afsr_errs & ~C_AFSR_EDU) == 0) || 61277c478bd9Sstevel@tonic-gate ((afsr_errs & ~C_AFSR_L3_EDU) == 0)) { 61287c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 61297c478bd9Sstevel@tonic-gate } else { 61307c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 61317c478bd9Sstevel@tonic-gate } 61327c478bd9Sstevel@tonic-gate } 61337c478bd9Sstevel@tonic-gate 61347c478bd9Sstevel@tonic-gate /* 61357c478bd9Sstevel@tonic-gate * BERR: If BERR only is set, flush the Ecache line, otherwise 61367c478bd9Sstevel@tonic-gate * flush the entire Ecache. 61377c478bd9Sstevel@tonic-gate */ 61387c478bd9Sstevel@tonic-gate if (afsr_errs & C_AFSR_BERR) { 61397c478bd9Sstevel@tonic-gate if ((afsr_errs & ~C_AFSR_BERR) == 0) { 61407c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 61417c478bd9Sstevel@tonic-gate } else { 61427c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 61437c478bd9Sstevel@tonic-gate } 61447c478bd9Sstevel@tonic-gate } 61457c478bd9Sstevel@tonic-gate 61467c478bd9Sstevel@tonic-gate return (0); 61477c478bd9Sstevel@tonic-gate } 61487c478bd9Sstevel@tonic-gate 61497c478bd9Sstevel@tonic-gate void 61507c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(ch_async_flt_t *ch_flt) 61517c478bd9Sstevel@tonic-gate { 61527c478bd9Sstevel@tonic-gate int ecache_flush_flag = 61537c478bd9Sstevel@tonic-gate cpu_error_ecache_flush_required(ch_flt); 61547c478bd9Sstevel@tonic-gate 61557c478bd9Sstevel@tonic-gate /* 61567c478bd9Sstevel@tonic-gate * Flush Ecache line or entire Ecache based on above checks. 61577c478bd9Sstevel@tonic-gate */ 61587c478bd9Sstevel@tonic-gate if (ecache_flush_flag == ECACHE_FLUSH_ALL) 61597c478bd9Sstevel@tonic-gate cpu_flush_ecache(); 61607c478bd9Sstevel@tonic-gate else if (ecache_flush_flag == ECACHE_FLUSH_LINE) { 61617c478bd9Sstevel@tonic-gate cpu_flush_ecache_line(ch_flt); 61627c478bd9Sstevel@tonic-gate } 61637c478bd9Sstevel@tonic-gate 61647c478bd9Sstevel@tonic-gate } 61657c478bd9Sstevel@tonic-gate 61667c478bd9Sstevel@tonic-gate /* 61677c478bd9Sstevel@tonic-gate * Extract the PA portion from the E$ tag. 61687c478bd9Sstevel@tonic-gate */ 61697c478bd9Sstevel@tonic-gate uint64_t 61707c478bd9Sstevel@tonic-gate cpu_ectag_to_pa(int setsize, uint64_t tag) 61717c478bd9Sstevel@tonic-gate { 61727c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 61737c478bd9Sstevel@tonic-gate return (JG_ECTAG_TO_PA(setsize, tag)); 61747c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 61757c478bd9Sstevel@tonic-gate return (PN_L3TAG_TO_PA(tag)); 61767c478bd9Sstevel@tonic-gate else 61777c478bd9Sstevel@tonic-gate return (CH_ECTAG_TO_PA(setsize, tag)); 61787c478bd9Sstevel@tonic-gate } 61797c478bd9Sstevel@tonic-gate 61807c478bd9Sstevel@tonic-gate /* 61817c478bd9Sstevel@tonic-gate * Convert the E$ tag PA into an E$ subblock index. 61827c478bd9Sstevel@tonic-gate */ 61837c478bd9Sstevel@tonic-gate static int 61847c478bd9Sstevel@tonic-gate cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr) 61857c478bd9Sstevel@tonic-gate { 61867c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 61877c478bd9Sstevel@tonic-gate return (JG_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 61887c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 61897c478bd9Sstevel@tonic-gate /* Panther has only one subblock per line */ 61907c478bd9Sstevel@tonic-gate return (0); 61917c478bd9Sstevel@tonic-gate else 61927c478bd9Sstevel@tonic-gate return (CH_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 61937c478bd9Sstevel@tonic-gate } 61947c478bd9Sstevel@tonic-gate 61957c478bd9Sstevel@tonic-gate /* 61967c478bd9Sstevel@tonic-gate * All subblocks in an E$ line must be invalid for 61977c478bd9Sstevel@tonic-gate * the line to be invalid. 61987c478bd9Sstevel@tonic-gate */ 61997c478bd9Sstevel@tonic-gate int 62007c478bd9Sstevel@tonic-gate cpu_ectag_line_invalid(int cachesize, uint64_t tag) 62017c478bd9Sstevel@tonic-gate { 62027c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 62037c478bd9Sstevel@tonic-gate return (JG_ECTAG_LINE_INVALID(cachesize, tag)); 62047c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 62057c478bd9Sstevel@tonic-gate return (PN_L3_LINE_INVALID(tag)); 62067c478bd9Sstevel@tonic-gate else 62077c478bd9Sstevel@tonic-gate return (CH_ECTAG_LINE_INVALID(cachesize, tag)); 62087c478bd9Sstevel@tonic-gate } 62097c478bd9Sstevel@tonic-gate 62107c478bd9Sstevel@tonic-gate /* 62117c478bd9Sstevel@tonic-gate * Extract state bits for a subblock given the tag. Note that for Panther 62127c478bd9Sstevel@tonic-gate * this works on both l2 and l3 tags. 62137c478bd9Sstevel@tonic-gate */ 62147c478bd9Sstevel@tonic-gate static int 62157c478bd9Sstevel@tonic-gate cpu_ectag_pa_to_subblk_state(int cachesize, uint64_t subaddr, uint64_t tag) 62167c478bd9Sstevel@tonic-gate { 62177c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 62187c478bd9Sstevel@tonic-gate return (JG_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 62197c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 62207c478bd9Sstevel@tonic-gate return (tag & CH_ECSTATE_MASK); 62217c478bd9Sstevel@tonic-gate else 62227c478bd9Sstevel@tonic-gate return (CH_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 62237c478bd9Sstevel@tonic-gate } 62247c478bd9Sstevel@tonic-gate 62257c478bd9Sstevel@tonic-gate /* 62267c478bd9Sstevel@tonic-gate * Cpu specific initialization. 62277c478bd9Sstevel@tonic-gate */ 62287c478bd9Sstevel@tonic-gate void 62297c478bd9Sstevel@tonic-gate cpu_mp_init(void) 62307c478bd9Sstevel@tonic-gate { 62317c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 62327c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_recover) { 62337c478bd9Sstevel@tonic-gate cheetah_nudge_init(); 62347c478bd9Sstevel@tonic-gate } 62357c478bd9Sstevel@tonic-gate #endif 62367c478bd9Sstevel@tonic-gate } 62377c478bd9Sstevel@tonic-gate 62387c478bd9Sstevel@tonic-gate void 62397c478bd9Sstevel@tonic-gate cpu_ereport_post(struct async_flt *aflt) 62407c478bd9Sstevel@tonic-gate { 62417c478bd9Sstevel@tonic-gate char *cpu_type, buf[FM_MAX_CLASS]; 62427c478bd9Sstevel@tonic-gate nv_alloc_t *nva = NULL; 62437c478bd9Sstevel@tonic-gate nvlist_t *ereport, *detector, *resource; 62447c478bd9Sstevel@tonic-gate errorq_elem_t *eqep; 62457c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 62467c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 62477c478bd9Sstevel@tonic-gate int len = 0; 62487c478bd9Sstevel@tonic-gate uint8_t msg_type; 62497c478bd9Sstevel@tonic-gate plat_ecc_ch_async_flt_t plat_ecc_ch_flt; 62507c478bd9Sstevel@tonic-gate 62517c478bd9Sstevel@tonic-gate if (aflt->flt_panic || panicstr) { 62527c478bd9Sstevel@tonic-gate eqep = errorq_reserve(ereport_errorq); 62537c478bd9Sstevel@tonic-gate if (eqep == NULL) 62547c478bd9Sstevel@tonic-gate return; 62557c478bd9Sstevel@tonic-gate ereport = errorq_elem_nvl(ereport_errorq, eqep); 62567c478bd9Sstevel@tonic-gate nva = errorq_elem_nva(ereport_errorq, eqep); 62577c478bd9Sstevel@tonic-gate } else { 62587c478bd9Sstevel@tonic-gate ereport = fm_nvlist_create(nva); 62597c478bd9Sstevel@tonic-gate } 62607c478bd9Sstevel@tonic-gate 62617c478bd9Sstevel@tonic-gate /* 62627c478bd9Sstevel@tonic-gate * Create the scheme "cpu" FMRI. 62637c478bd9Sstevel@tonic-gate */ 62647c478bd9Sstevel@tonic-gate detector = fm_nvlist_create(nva); 62657c478bd9Sstevel@tonic-gate resource = fm_nvlist_create(nva); 62667c478bd9Sstevel@tonic-gate switch (cpunodes[aflt->flt_inst].implementation) { 62677c478bd9Sstevel@tonic-gate case CHEETAH_IMPL: 62687c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIII; 62697c478bd9Sstevel@tonic-gate break; 62707c478bd9Sstevel@tonic-gate case CHEETAH_PLUS_IMPL: 62717c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIIIplus; 62727c478bd9Sstevel@tonic-gate break; 62737c478bd9Sstevel@tonic-gate case JALAPENO_IMPL: 62747c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIIIi; 62757c478bd9Sstevel@tonic-gate break; 62767c478bd9Sstevel@tonic-gate case SERRANO_IMPL: 62777c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIIIiplus; 62787c478bd9Sstevel@tonic-gate break; 62797c478bd9Sstevel@tonic-gate case JAGUAR_IMPL: 62807c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIV; 62817c478bd9Sstevel@tonic-gate break; 62827c478bd9Sstevel@tonic-gate case PANTHER_IMPL: 62837c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIVplus; 62847c478bd9Sstevel@tonic-gate break; 62857c478bd9Sstevel@tonic-gate default: 62867c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_UNSUPPORTED; 62877c478bd9Sstevel@tonic-gate break; 62887c478bd9Sstevel@tonic-gate } 62897c478bd9Sstevel@tonic-gate (void) fm_fmri_cpu_set(detector, FM_CPU_SCHEME_VERSION, NULL, 62907c478bd9Sstevel@tonic-gate aflt->flt_inst, (uint8_t)cpunodes[aflt->flt_inst].version, 62917c478bd9Sstevel@tonic-gate cpunodes[aflt->flt_inst].device_id); 62927c478bd9Sstevel@tonic-gate 62937c478bd9Sstevel@tonic-gate /* 62947c478bd9Sstevel@tonic-gate * Encode all the common data into the ereport. 62957c478bd9Sstevel@tonic-gate */ 62967c478bd9Sstevel@tonic-gate (void) snprintf(buf, FM_MAX_CLASS, "%s.%s.%s", 62977c478bd9Sstevel@tonic-gate FM_ERROR_CPU, cpu_type, aflt->flt_erpt_class); 62987c478bd9Sstevel@tonic-gate 62997c478bd9Sstevel@tonic-gate fm_ereport_set(ereport, FM_EREPORT_VERSION, buf, 63007c478bd9Sstevel@tonic-gate fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, FM_ENA_FMT1), 63017c478bd9Sstevel@tonic-gate detector, NULL); 63027c478bd9Sstevel@tonic-gate 63037c478bd9Sstevel@tonic-gate /* 63047c478bd9Sstevel@tonic-gate * Encode the error specific data that was saved in 63057c478bd9Sstevel@tonic-gate * the async_flt structure into the ereport. 63067c478bd9Sstevel@tonic-gate */ 63077c478bd9Sstevel@tonic-gate cpu_payload_add_aflt(aflt, ereport, resource, 63087c478bd9Sstevel@tonic-gate &plat_ecc_ch_flt.ecaf_afar_status, 63097c478bd9Sstevel@tonic-gate &plat_ecc_ch_flt.ecaf_synd_status); 63107c478bd9Sstevel@tonic-gate 63117c478bd9Sstevel@tonic-gate if (aflt->flt_panic || panicstr) { 63127c478bd9Sstevel@tonic-gate errorq_commit(ereport_errorq, eqep, ERRORQ_SYNC); 63137c478bd9Sstevel@tonic-gate } else { 63147c478bd9Sstevel@tonic-gate (void) fm_ereport_post(ereport, EVCH_TRYHARD); 63157c478bd9Sstevel@tonic-gate fm_nvlist_destroy(ereport, FM_NVA_FREE); 63167c478bd9Sstevel@tonic-gate fm_nvlist_destroy(detector, FM_NVA_FREE); 63177c478bd9Sstevel@tonic-gate fm_nvlist_destroy(resource, FM_NVA_FREE); 63187c478bd9Sstevel@tonic-gate } 63197c478bd9Sstevel@tonic-gate /* 63207c478bd9Sstevel@tonic-gate * Send the enhanced error information (plat_ecc_error2_data_t) 63217c478bd9Sstevel@tonic-gate * to the SC olny if it can process it. 63227c478bd9Sstevel@tonic-gate */ 63237c478bd9Sstevel@tonic-gate 63247c478bd9Sstevel@tonic-gate if (&plat_ecc_capability_sc_get && 63257c478bd9Sstevel@tonic-gate plat_ecc_capability_sc_get(PLAT_ECC_ERROR2_MESSAGE)) { 63267c478bd9Sstevel@tonic-gate msg_type = cpu_flt_bit_to_plat_error(aflt); 63277c478bd9Sstevel@tonic-gate if (msg_type != PLAT_ECC_ERROR2_NONE) { 63287c478bd9Sstevel@tonic-gate /* 63297c478bd9Sstevel@tonic-gate * If afar status is not invalid do a unum lookup. 63307c478bd9Sstevel@tonic-gate */ 63317c478bd9Sstevel@tonic-gate if (plat_ecc_ch_flt.ecaf_afar_status != 63327c478bd9Sstevel@tonic-gate AFLT_STAT_INVALID) { 63337c478bd9Sstevel@tonic-gate (void) cpu_get_mem_unum_aflt( 63347c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_synd_status, aflt, 63357c478bd9Sstevel@tonic-gate unum, UNUM_NAMLEN, &len); 63367c478bd9Sstevel@tonic-gate } else { 63377c478bd9Sstevel@tonic-gate unum[0] = '\0'; 63387c478bd9Sstevel@tonic-gate } 63397c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_sdw_afar = ch_flt->flt_sdw_afar; 63407c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_sdw_afsr = ch_flt->flt_sdw_afsr; 63417c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_afsr_ext = ch_flt->afsr_ext; 63427c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_sdw_afsr_ext = 63437c478bd9Sstevel@tonic-gate ch_flt->flt_sdw_afsr_ext; 63447c478bd9Sstevel@tonic-gate 63457c478bd9Sstevel@tonic-gate if (&plat_log_fruid_error2) 63467c478bd9Sstevel@tonic-gate plat_log_fruid_error2(msg_type, unum, aflt, 63477c478bd9Sstevel@tonic-gate &plat_ecc_ch_flt); 63487c478bd9Sstevel@tonic-gate } 63497c478bd9Sstevel@tonic-gate } 63507c478bd9Sstevel@tonic-gate } 63517c478bd9Sstevel@tonic-gate 63527c478bd9Sstevel@tonic-gate void 63537c478bd9Sstevel@tonic-gate cpu_run_bus_error_handlers(struct async_flt *aflt, int expected) 63547c478bd9Sstevel@tonic-gate { 63557c478bd9Sstevel@tonic-gate int status; 63567c478bd9Sstevel@tonic-gate ddi_fm_error_t de; 63577c478bd9Sstevel@tonic-gate 63587c478bd9Sstevel@tonic-gate bzero(&de, sizeof (ddi_fm_error_t)); 63597c478bd9Sstevel@tonic-gate 63607c478bd9Sstevel@tonic-gate de.fme_version = DDI_FME_VERSION; 63617c478bd9Sstevel@tonic-gate de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, 63627c478bd9Sstevel@tonic-gate FM_ENA_FMT1); 63637c478bd9Sstevel@tonic-gate de.fme_flag = expected; 63647c478bd9Sstevel@tonic-gate de.fme_bus_specific = (void *)aflt->flt_addr; 63657c478bd9Sstevel@tonic-gate status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de); 63667c478bd9Sstevel@tonic-gate if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL)) 63677c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 63687c478bd9Sstevel@tonic-gate } 63697c478bd9Sstevel@tonic-gate 63707c478bd9Sstevel@tonic-gate void 63717c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz, 63727c478bd9Sstevel@tonic-gate errorq_t *eqp, uint_t flag) 63737c478bd9Sstevel@tonic-gate { 63747c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)payload; 63757c478bd9Sstevel@tonic-gate 63767c478bd9Sstevel@tonic-gate aflt->flt_erpt_class = error_class; 63777c478bd9Sstevel@tonic-gate errorq_dispatch(eqp, payload, payload_sz, flag); 63787c478bd9Sstevel@tonic-gate } 63797c478bd9Sstevel@tonic-gate 63807c478bd9Sstevel@tonic-gate /* 63817c478bd9Sstevel@tonic-gate * This routine may be called by the IO module, but does not do 63827c478bd9Sstevel@tonic-gate * anything in this cpu module. The SERD algorithm is handled by 63837c478bd9Sstevel@tonic-gate * cpumem-diagnosis engine instead. 63847c478bd9Sstevel@tonic-gate */ 63857c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 63867c478bd9Sstevel@tonic-gate void 63877c478bd9Sstevel@tonic-gate cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum) 63887c478bd9Sstevel@tonic-gate {} 63897c478bd9Sstevel@tonic-gate 63907c478bd9Sstevel@tonic-gate void 63917c478bd9Sstevel@tonic-gate adjust_hw_copy_limits(int ecache_size) 63927c478bd9Sstevel@tonic-gate { 63937c478bd9Sstevel@tonic-gate /* 63947c478bd9Sstevel@tonic-gate * Set hw copy limits. 63957c478bd9Sstevel@tonic-gate * 63967c478bd9Sstevel@tonic-gate * /etc/system will be parsed later and can override one or more 63977c478bd9Sstevel@tonic-gate * of these settings. 63987c478bd9Sstevel@tonic-gate * 63997c478bd9Sstevel@tonic-gate * At this time, ecache size seems only mildly relevant. 64007c478bd9Sstevel@tonic-gate * We seem to run into issues with the d-cache and stalls 64017c478bd9Sstevel@tonic-gate * we see on misses. 64027c478bd9Sstevel@tonic-gate * 64037c478bd9Sstevel@tonic-gate * Cycle measurement indicates that 2 byte aligned copies fare 64047c478bd9Sstevel@tonic-gate * little better than doing things with VIS at around 512 bytes. 64057c478bd9Sstevel@tonic-gate * 4 byte aligned shows promise until around 1024 bytes. 8 Byte 64067c478bd9Sstevel@tonic-gate * aligned is faster whenever the source and destination data 64077c478bd9Sstevel@tonic-gate * in cache and the total size is less than 2 Kbytes. The 2K 64087c478bd9Sstevel@tonic-gate * limit seems to be driven by the 2K write cache. 64097c478bd9Sstevel@tonic-gate * When more than 2K of copies are done in non-VIS mode, stores 64107c478bd9Sstevel@tonic-gate * backup in the write cache. In VIS mode, the write cache is 64117c478bd9Sstevel@tonic-gate * bypassed, allowing faster cache-line writes aligned on cache 64127c478bd9Sstevel@tonic-gate * boundaries. 64137c478bd9Sstevel@tonic-gate * 64147c478bd9Sstevel@tonic-gate * In addition, in non-VIS mode, there is no prefetching, so 64157c478bd9Sstevel@tonic-gate * for larger copies, the advantage of prefetching to avoid even 64167c478bd9Sstevel@tonic-gate * occasional cache misses is enough to justify using the VIS code. 64177c478bd9Sstevel@tonic-gate * 64187c478bd9Sstevel@tonic-gate * During testing, it was discovered that netbench ran 3% slower 64197c478bd9Sstevel@tonic-gate * when hw_copy_limit_8 was 2K or larger. Apparently for server 64207c478bd9Sstevel@tonic-gate * applications, data is only used once (copied to the output 64217c478bd9Sstevel@tonic-gate * buffer, then copied by the network device off the system). Using 64227c478bd9Sstevel@tonic-gate * the VIS copy saves more L2 cache state. Network copies are 64237c478bd9Sstevel@tonic-gate * around 1.3K to 1.5K in size for historical reasons. 64247c478bd9Sstevel@tonic-gate * 64257c478bd9Sstevel@tonic-gate * Therefore, a limit of 1K bytes will be used for the 8 byte 64267c478bd9Sstevel@tonic-gate * aligned copy even for large caches and 8 MB ecache. The 64277c478bd9Sstevel@tonic-gate * infrastructure to allow different limits for different sized 64287c478bd9Sstevel@tonic-gate * caches is kept to allow further tuning in later releases. 64297c478bd9Sstevel@tonic-gate */ 64307c478bd9Sstevel@tonic-gate 64317c478bd9Sstevel@tonic-gate if (min_ecache_size == 0 && use_hw_bcopy) { 64327c478bd9Sstevel@tonic-gate /* 64337c478bd9Sstevel@tonic-gate * First time through - should be before /etc/system 64347c478bd9Sstevel@tonic-gate * is read. 64357c478bd9Sstevel@tonic-gate * Could skip the checks for zero but this lets us 64367c478bd9Sstevel@tonic-gate * preserve any debugger rewrites. 64377c478bd9Sstevel@tonic-gate */ 64387c478bd9Sstevel@tonic-gate if (hw_copy_limit_1 == 0) { 64397c478bd9Sstevel@tonic-gate hw_copy_limit_1 = VIS_COPY_THRESHOLD; 64407c478bd9Sstevel@tonic-gate priv_hcl_1 = hw_copy_limit_1; 64417c478bd9Sstevel@tonic-gate } 64427c478bd9Sstevel@tonic-gate if (hw_copy_limit_2 == 0) { 64437c478bd9Sstevel@tonic-gate hw_copy_limit_2 = 2 * VIS_COPY_THRESHOLD; 64447c478bd9Sstevel@tonic-gate priv_hcl_2 = hw_copy_limit_2; 64457c478bd9Sstevel@tonic-gate } 64467c478bd9Sstevel@tonic-gate if (hw_copy_limit_4 == 0) { 64477c478bd9Sstevel@tonic-gate hw_copy_limit_4 = 4 * VIS_COPY_THRESHOLD; 64487c478bd9Sstevel@tonic-gate priv_hcl_4 = hw_copy_limit_4; 64497c478bd9Sstevel@tonic-gate } 64507c478bd9Sstevel@tonic-gate if (hw_copy_limit_8 == 0) { 64517c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD; 64527c478bd9Sstevel@tonic-gate priv_hcl_8 = hw_copy_limit_8; 64537c478bd9Sstevel@tonic-gate } 64547c478bd9Sstevel@tonic-gate min_ecache_size = ecache_size; 64557c478bd9Sstevel@tonic-gate } else { 64567c478bd9Sstevel@tonic-gate /* 64577c478bd9Sstevel@tonic-gate * MP initialization. Called *after* /etc/system has 64587c478bd9Sstevel@tonic-gate * been parsed. One CPU has already been initialized. 64597c478bd9Sstevel@tonic-gate * Need to cater for /etc/system having scragged one 64607c478bd9Sstevel@tonic-gate * of our values. 64617c478bd9Sstevel@tonic-gate */ 64627c478bd9Sstevel@tonic-gate if (ecache_size == min_ecache_size) { 64637c478bd9Sstevel@tonic-gate /* 64647c478bd9Sstevel@tonic-gate * Same size ecache. We do nothing unless we 64657c478bd9Sstevel@tonic-gate * have a pessimistic ecache setting. In that 64667c478bd9Sstevel@tonic-gate * case we become more optimistic (if the cache is 64677c478bd9Sstevel@tonic-gate * large enough). 64687c478bd9Sstevel@tonic-gate */ 64697c478bd9Sstevel@tonic-gate if (hw_copy_limit_8 == 4 * VIS_COPY_THRESHOLD) { 64707c478bd9Sstevel@tonic-gate /* 64717c478bd9Sstevel@tonic-gate * Need to adjust hw_copy_limit* from our 64727c478bd9Sstevel@tonic-gate * pessimistic uniprocessor value to a more 64737c478bd9Sstevel@tonic-gate * optimistic UP value *iff* it hasn't been 64747c478bd9Sstevel@tonic-gate * reset. 64757c478bd9Sstevel@tonic-gate */ 64767c478bd9Sstevel@tonic-gate if ((ecache_size > 1048576) && 64777c478bd9Sstevel@tonic-gate (priv_hcl_8 == hw_copy_limit_8)) { 64787c478bd9Sstevel@tonic-gate if (ecache_size <= 2097152) 64797c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * 64807c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 64817c478bd9Sstevel@tonic-gate else if (ecache_size <= 4194304) 64827c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * 64837c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 64847c478bd9Sstevel@tonic-gate else 64857c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * 64867c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 64877c478bd9Sstevel@tonic-gate priv_hcl_8 = hw_copy_limit_8; 64887c478bd9Sstevel@tonic-gate } 64897c478bd9Sstevel@tonic-gate } 64907c478bd9Sstevel@tonic-gate } else if (ecache_size < min_ecache_size) { 64917c478bd9Sstevel@tonic-gate /* 64927c478bd9Sstevel@tonic-gate * A different ecache size. Can this even happen? 64937c478bd9Sstevel@tonic-gate */ 64947c478bd9Sstevel@tonic-gate if (priv_hcl_8 == hw_copy_limit_8) { 64957c478bd9Sstevel@tonic-gate /* 64967c478bd9Sstevel@tonic-gate * The previous value that we set 64977c478bd9Sstevel@tonic-gate * is unchanged (i.e., it hasn't been 64987c478bd9Sstevel@tonic-gate * scragged by /etc/system). Rewrite it. 64997c478bd9Sstevel@tonic-gate */ 65007c478bd9Sstevel@tonic-gate if (ecache_size <= 1048576) 65017c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 8 * 65027c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 65037c478bd9Sstevel@tonic-gate else if (ecache_size <= 2097152) 65047c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 8 * 65057c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 65067c478bd9Sstevel@tonic-gate else if (ecache_size <= 4194304) 65077c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 8 * 65087c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 65097c478bd9Sstevel@tonic-gate else 65107c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 10 * 65117c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 65127c478bd9Sstevel@tonic-gate priv_hcl_8 = hw_copy_limit_8; 65137c478bd9Sstevel@tonic-gate min_ecache_size = ecache_size; 65147c478bd9Sstevel@tonic-gate } 65157c478bd9Sstevel@tonic-gate } 65167c478bd9Sstevel@tonic-gate } 65177c478bd9Sstevel@tonic-gate } 65187c478bd9Sstevel@tonic-gate 65197c478bd9Sstevel@tonic-gate /* 65207c478bd9Sstevel@tonic-gate * Called from illegal instruction trap handler to see if we can attribute 65217c478bd9Sstevel@tonic-gate * the trap to a fpras check. 65227c478bd9Sstevel@tonic-gate */ 65237c478bd9Sstevel@tonic-gate int 65247c478bd9Sstevel@tonic-gate fpras_chktrap(struct regs *rp) 65257c478bd9Sstevel@tonic-gate { 65267c478bd9Sstevel@tonic-gate int op; 65277c478bd9Sstevel@tonic-gate struct fpras_chkfngrp *cgp; 65287c478bd9Sstevel@tonic-gate uintptr_t tpc = (uintptr_t)rp->r_pc; 65297c478bd9Sstevel@tonic-gate 65307c478bd9Sstevel@tonic-gate if (fpras_chkfngrps == NULL) 65317c478bd9Sstevel@tonic-gate return (0); 65327c478bd9Sstevel@tonic-gate 65337c478bd9Sstevel@tonic-gate cgp = &fpras_chkfngrps[CPU->cpu_id]; 65347c478bd9Sstevel@tonic-gate for (op = 0; op < FPRAS_NCOPYOPS; ++op) { 65357c478bd9Sstevel@tonic-gate if (tpc >= (uintptr_t)&cgp->fpras_fn[op].fpras_blk0 && 65367c478bd9Sstevel@tonic-gate tpc < (uintptr_t)&cgp->fpras_fn[op].fpras_chkresult) 65377c478bd9Sstevel@tonic-gate break; 65387c478bd9Sstevel@tonic-gate } 65397c478bd9Sstevel@tonic-gate if (op == FPRAS_NCOPYOPS) 65407c478bd9Sstevel@tonic-gate return (0); 65417c478bd9Sstevel@tonic-gate 65427c478bd9Sstevel@tonic-gate /* 65437c478bd9Sstevel@tonic-gate * This is an fpRAS failure caught through an illegal 65447c478bd9Sstevel@tonic-gate * instruction - trampoline. 65457c478bd9Sstevel@tonic-gate */ 65467c478bd9Sstevel@tonic-gate rp->r_pc = (uintptr_t)&cgp->fpras_fn[op].fpras_trampoline; 65477c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 65487c478bd9Sstevel@tonic-gate return (1); 65497c478bd9Sstevel@tonic-gate } 65507c478bd9Sstevel@tonic-gate 65517c478bd9Sstevel@tonic-gate /* 65527c478bd9Sstevel@tonic-gate * fpras_failure is called when a fpras check detects a bad calculation 65537c478bd9Sstevel@tonic-gate * result or an illegal instruction trap is attributed to an fpras 65547c478bd9Sstevel@tonic-gate * check. In all cases we are still bound to CPU. 65557c478bd9Sstevel@tonic-gate */ 65567c478bd9Sstevel@tonic-gate int 65577c478bd9Sstevel@tonic-gate fpras_failure(int op, int how) 65587c478bd9Sstevel@tonic-gate { 65597c478bd9Sstevel@tonic-gate int use_hw_bcopy_orig, use_hw_bzero_orig; 65607c478bd9Sstevel@tonic-gate uint_t hcl1_orig, hcl2_orig, hcl4_orig, hcl8_orig; 65617c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 65627c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)&ch_flt; 65637c478bd9Sstevel@tonic-gate struct fpras_chkfn *sfp, *cfp; 65647c478bd9Sstevel@tonic-gate uint32_t *sip, *cip; 65657c478bd9Sstevel@tonic-gate int i; 65667c478bd9Sstevel@tonic-gate 65677c478bd9Sstevel@tonic-gate /* 65687c478bd9Sstevel@tonic-gate * We're running on a sick CPU. Avoid further FPU use at least for 65697c478bd9Sstevel@tonic-gate * the time in which we dispatch an ereport and (if applicable) panic. 65707c478bd9Sstevel@tonic-gate */ 65717c478bd9Sstevel@tonic-gate use_hw_bcopy_orig = use_hw_bcopy; 65727c478bd9Sstevel@tonic-gate use_hw_bzero_orig = use_hw_bzero; 65737c478bd9Sstevel@tonic-gate hcl1_orig = hw_copy_limit_1; 65747c478bd9Sstevel@tonic-gate hcl2_orig = hw_copy_limit_2; 65757c478bd9Sstevel@tonic-gate hcl4_orig = hw_copy_limit_4; 65767c478bd9Sstevel@tonic-gate hcl8_orig = hw_copy_limit_8; 65777c478bd9Sstevel@tonic-gate use_hw_bcopy = use_hw_bzero = 0; 65787c478bd9Sstevel@tonic-gate hw_copy_limit_1 = hw_copy_limit_2 = hw_copy_limit_4 = 65797c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 0; 65807c478bd9Sstevel@tonic-gate 65817c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 65827c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 65837c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 65847c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 65857c478bd9Sstevel@tonic-gate aflt->flt_status = (how << 8) | op; 65867c478bd9Sstevel@tonic-gate aflt->flt_payload = FM_EREPORT_PAYLOAD_FPU_HWCOPY; 65877c478bd9Sstevel@tonic-gate ch_flt.flt_type = CPU_FPUERR; 65887c478bd9Sstevel@tonic-gate 65897c478bd9Sstevel@tonic-gate /* 65907c478bd9Sstevel@tonic-gate * We must panic if the copy operation had no lofault protection - 65917c478bd9Sstevel@tonic-gate * ie, don't panic for copyin, copyout, kcopy and bcopy called 65927c478bd9Sstevel@tonic-gate * under on_fault and do panic for unprotected bcopy and hwblkpagecopy. 65937c478bd9Sstevel@tonic-gate */ 65947c478bd9Sstevel@tonic-gate aflt->flt_panic = (curthread->t_lofault == NULL); 65957c478bd9Sstevel@tonic-gate 65967c478bd9Sstevel@tonic-gate /* 65977c478bd9Sstevel@tonic-gate * XOR the source instruction block with the copied instruction 65987c478bd9Sstevel@tonic-gate * block - this will show us which bit(s) are corrupted. 65997c478bd9Sstevel@tonic-gate */ 66007c478bd9Sstevel@tonic-gate sfp = (struct fpras_chkfn *)fpras_chkfn_type1; 66017c478bd9Sstevel@tonic-gate cfp = &fpras_chkfngrps[CPU->cpu_id].fpras_fn[op]; 66027c478bd9Sstevel@tonic-gate if (op == FPRAS_BCOPY || op == FPRAS_COPYOUT) { 66037c478bd9Sstevel@tonic-gate sip = &sfp->fpras_blk0[0]; 66047c478bd9Sstevel@tonic-gate cip = &cfp->fpras_blk0[0]; 66057c478bd9Sstevel@tonic-gate } else { 66067c478bd9Sstevel@tonic-gate sip = &sfp->fpras_blk1[0]; 66077c478bd9Sstevel@tonic-gate cip = &cfp->fpras_blk1[0]; 66087c478bd9Sstevel@tonic-gate } 66097c478bd9Sstevel@tonic-gate for (i = 0; i < 16; ++i, ++sip, ++cip) 66107c478bd9Sstevel@tonic-gate ch_flt.flt_fpdata[i] = *sip ^ *cip; 66117c478bd9Sstevel@tonic-gate 66127c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_FPU_HWCOPY, (void *)&ch_flt, 66137c478bd9Sstevel@tonic-gate sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 66147c478bd9Sstevel@tonic-gate 66157c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 66167c478bd9Sstevel@tonic-gate fm_panic("FPU failure on CPU %d", CPU->cpu_id); 66177c478bd9Sstevel@tonic-gate 66187c478bd9Sstevel@tonic-gate /* 66197c478bd9Sstevel@tonic-gate * We get here for copyin/copyout and kcopy or bcopy where the 66207c478bd9Sstevel@tonic-gate * caller has used on_fault. We will flag the error so that 66217c478bd9Sstevel@tonic-gate * the process may be killed The trap_async_hwerr mechanism will 66227c478bd9Sstevel@tonic-gate * take appropriate further action (such as a reboot, contract 66237c478bd9Sstevel@tonic-gate * notification etc). Since we may be continuing we will 66247c478bd9Sstevel@tonic-gate * restore the global hardware copy acceleration switches. 66257c478bd9Sstevel@tonic-gate * 66267c478bd9Sstevel@tonic-gate * When we return from this function to the copy function we want to 66277c478bd9Sstevel@tonic-gate * avoid potentially bad data being used, ie we want the affected 66287c478bd9Sstevel@tonic-gate * copy function to return an error. The caller should therefore 66297c478bd9Sstevel@tonic-gate * invoke its lofault handler (which always exists for these functions) 66307c478bd9Sstevel@tonic-gate * which will return the appropriate error. 66317c478bd9Sstevel@tonic-gate */ 66327c478bd9Sstevel@tonic-gate ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR; 66337c478bd9Sstevel@tonic-gate aston(curthread); 66347c478bd9Sstevel@tonic-gate 66357c478bd9Sstevel@tonic-gate use_hw_bcopy = use_hw_bcopy_orig; 66367c478bd9Sstevel@tonic-gate use_hw_bzero = use_hw_bzero_orig; 66377c478bd9Sstevel@tonic-gate hw_copy_limit_1 = hcl1_orig; 66387c478bd9Sstevel@tonic-gate hw_copy_limit_2 = hcl2_orig; 66397c478bd9Sstevel@tonic-gate hw_copy_limit_4 = hcl4_orig; 66407c478bd9Sstevel@tonic-gate hw_copy_limit_8 = hcl8_orig; 66417c478bd9Sstevel@tonic-gate 66427c478bd9Sstevel@tonic-gate return (1); 66437c478bd9Sstevel@tonic-gate } 66447c478bd9Sstevel@tonic-gate 66457c478bd9Sstevel@tonic-gate #define VIS_BLOCKSIZE 64 66467c478bd9Sstevel@tonic-gate 66477c478bd9Sstevel@tonic-gate int 66487c478bd9Sstevel@tonic-gate dtrace_blksuword32_err(uintptr_t addr, uint32_t *data) 66497c478bd9Sstevel@tonic-gate { 66507c478bd9Sstevel@tonic-gate int ret, watched; 66517c478bd9Sstevel@tonic-gate 66527c478bd9Sstevel@tonic-gate watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 66537c478bd9Sstevel@tonic-gate ret = dtrace_blksuword32(addr, data, 0); 66547c478bd9Sstevel@tonic-gate if (watched) 66557c478bd9Sstevel@tonic-gate watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 66567c478bd9Sstevel@tonic-gate 66577c478bd9Sstevel@tonic-gate return (ret); 66587c478bd9Sstevel@tonic-gate } 66597c478bd9Sstevel@tonic-gate 66607c478bd9Sstevel@tonic-gate /* 66617c478bd9Sstevel@tonic-gate * Called when a cpu enters the CPU_FAULTED state (by the cpu placing the 66627c478bd9Sstevel@tonic-gate * faulted cpu into that state). Cross-trap to the faulted cpu to clear 66637c478bd9Sstevel@tonic-gate * CEEN from the EER to disable traps for further disrupting error types 66647c478bd9Sstevel@tonic-gate * on that cpu. We could cross-call instead, but that has a larger 66657c478bd9Sstevel@tonic-gate * instruction and data footprint than cross-trapping, and the cpu is known 66667c478bd9Sstevel@tonic-gate * to be faulted. 66677c478bd9Sstevel@tonic-gate */ 66687c478bd9Sstevel@tonic-gate 66697c478bd9Sstevel@tonic-gate void 66707c478bd9Sstevel@tonic-gate cpu_faulted_enter(struct cpu *cp) 66717c478bd9Sstevel@tonic-gate { 66727c478bd9Sstevel@tonic-gate xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_CLRBITS); 66737c478bd9Sstevel@tonic-gate } 66747c478bd9Sstevel@tonic-gate 66757c478bd9Sstevel@tonic-gate /* 66767c478bd9Sstevel@tonic-gate * Called when a cpu leaves the CPU_FAULTED state to return to one of 66777c478bd9Sstevel@tonic-gate * offline, spare, or online (by the cpu requesting this state change). 66787c478bd9Sstevel@tonic-gate * First we cross-call to clear the AFSR (and AFSR_EXT on Panther) of 66797c478bd9Sstevel@tonic-gate * disrupting error bits that have accumulated without trapping, then 66807c478bd9Sstevel@tonic-gate * we cross-trap to re-enable CEEN controlled traps. 66817c478bd9Sstevel@tonic-gate */ 66827c478bd9Sstevel@tonic-gate void 66837c478bd9Sstevel@tonic-gate cpu_faulted_exit(struct cpu *cp) 66847c478bd9Sstevel@tonic-gate { 66857c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 66867c478bd9Sstevel@tonic-gate 66877c478bd9Sstevel@tonic-gate cpu_error_regs.afsr = C_AFSR_CECC_ERRS; 66887c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) 66897c478bd9Sstevel@tonic-gate cpu_error_regs.afsr_ext &= C_AFSR_EXT_CECC_ERRS; 66907c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)set_cpu_error_state, 66917c478bd9Sstevel@tonic-gate (uint64_t)&cpu_error_regs, 0); 66927c478bd9Sstevel@tonic-gate 66937c478bd9Sstevel@tonic-gate xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_SETBITS); 66947c478bd9Sstevel@tonic-gate } 66957c478bd9Sstevel@tonic-gate 66967c478bd9Sstevel@tonic-gate /* 66977c478bd9Sstevel@tonic-gate * Return 1 if the errors in ch_flt's AFSR are secondary errors caused by 66987c478bd9Sstevel@tonic-gate * the errors in the original AFSR, 0 otherwise. 66997c478bd9Sstevel@tonic-gate * 67007c478bd9Sstevel@tonic-gate * For all procs if the initial error was a BERR or TO, then it is possible 67017c478bd9Sstevel@tonic-gate * that we may have caused a secondary BERR or TO in the process of logging the 67027c478bd9Sstevel@tonic-gate * inital error via cpu_run_bus_error_handlers(). If this is the case then 67037c478bd9Sstevel@tonic-gate * if the request was protected then a panic is still not necessary, if not 67047c478bd9Sstevel@tonic-gate * protected then aft_panic is already set - so either way there's no need 67057c478bd9Sstevel@tonic-gate * to set aft_panic for the secondary error. 67067c478bd9Sstevel@tonic-gate * 67077c478bd9Sstevel@tonic-gate * For Cheetah and Jalapeno, if the original error was a UE which occurred on 67087c478bd9Sstevel@tonic-gate * a store merge, then the error handling code will call cpu_deferred_error(). 67097c478bd9Sstevel@tonic-gate * When clear_errors() is called, it will determine that secondary errors have 67107c478bd9Sstevel@tonic-gate * occurred - in particular, the store merge also caused a EDU and WDU that 67117c478bd9Sstevel@tonic-gate * weren't discovered until this point. 67127c478bd9Sstevel@tonic-gate * 67137c478bd9Sstevel@tonic-gate * We do three checks to verify that we are in this case. If we pass all three 67147c478bd9Sstevel@tonic-gate * checks, we return 1 to indicate that we should not panic. If any unexpected 67157c478bd9Sstevel@tonic-gate * errors occur, we return 0. 67167c478bd9Sstevel@tonic-gate * 67177c478bd9Sstevel@tonic-gate * For Cheetah+ and derivative procs, the store merge causes a DUE, which is 67187c478bd9Sstevel@tonic-gate * handled in cpu_disrupting_errors(). Since this function is not even called 67197c478bd9Sstevel@tonic-gate * in the case we are interested in, we just return 0 for these processors. 67207c478bd9Sstevel@tonic-gate */ 67217c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 67227c478bd9Sstevel@tonic-gate static int 67237c478bd9Sstevel@tonic-gate cpu_check_secondary_errors(ch_async_flt_t *ch_flt, uint64_t t_afsr_errs, 67247c478bd9Sstevel@tonic-gate uint64_t t_afar) 67257c478bd9Sstevel@tonic-gate { 67267c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 67277c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */ 67287c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 67297c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 67307c478bd9Sstevel@tonic-gate 67317c478bd9Sstevel@tonic-gate /* 67327c478bd9Sstevel@tonic-gate * Was the original error a BERR or TO and only a BERR or TO 67337c478bd9Sstevel@tonic-gate * (multiple errors are also OK) 67347c478bd9Sstevel@tonic-gate */ 67357c478bd9Sstevel@tonic-gate if ((t_afsr_errs & ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) { 67367c478bd9Sstevel@tonic-gate /* 67377c478bd9Sstevel@tonic-gate * Is the new error a BERR or TO and only a BERR or TO 67387c478bd9Sstevel@tonic-gate * (multiple errors are also OK) 67397c478bd9Sstevel@tonic-gate */ 67407c478bd9Sstevel@tonic-gate if ((ch_flt->afsr_errs & 67417c478bd9Sstevel@tonic-gate ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) 67427c478bd9Sstevel@tonic-gate return (1); 67437c478bd9Sstevel@tonic-gate } 67447c478bd9Sstevel@tonic-gate 67457c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 67467c478bd9Sstevel@tonic-gate return (0); 67477c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */ 67487c478bd9Sstevel@tonic-gate /* 67497c478bd9Sstevel@tonic-gate * Now look for secondary effects of a UE on cheetah/jalapeno 67507c478bd9Sstevel@tonic-gate * 67517c478bd9Sstevel@tonic-gate * Check the original error was a UE, and only a UE. Note that 67527c478bd9Sstevel@tonic-gate * the ME bit will cause us to fail this check. 67537c478bd9Sstevel@tonic-gate */ 67547c478bd9Sstevel@tonic-gate if (t_afsr_errs != C_AFSR_UE) 67557c478bd9Sstevel@tonic-gate return (0); 67567c478bd9Sstevel@tonic-gate 67577c478bd9Sstevel@tonic-gate /* 67587c478bd9Sstevel@tonic-gate * Check the secondary errors were exclusively an EDU and/or WDU. 67597c478bd9Sstevel@tonic-gate */ 67607c478bd9Sstevel@tonic-gate if ((ch_flt->afsr_errs & ~(C_AFSR_EDU|C_AFSR_WDU)) != 0) 67617c478bd9Sstevel@tonic-gate return (0); 67627c478bd9Sstevel@tonic-gate 67637c478bd9Sstevel@tonic-gate /* 67647c478bd9Sstevel@tonic-gate * Check the AFAR of the original error and secondary errors 67657c478bd9Sstevel@tonic-gate * match to the 64-byte boundary 67667c478bd9Sstevel@tonic-gate */ 67677c478bd9Sstevel@tonic-gate if (P2ALIGN(aflt->flt_addr, 64) != P2ALIGN(t_afar, 64)) 67687c478bd9Sstevel@tonic-gate return (0); 67697c478bd9Sstevel@tonic-gate 67707c478bd9Sstevel@tonic-gate /* 67717c478bd9Sstevel@tonic-gate * We've passed all the checks, so it's a secondary error! 67727c478bd9Sstevel@tonic-gate */ 67737c478bd9Sstevel@tonic-gate return (1); 67747c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 67757c478bd9Sstevel@tonic-gate } 67767c478bd9Sstevel@tonic-gate 67777c478bd9Sstevel@tonic-gate /* 67787c478bd9Sstevel@tonic-gate * Translate the flt_bit or flt_type into an error type. First, flt_bit 67797c478bd9Sstevel@tonic-gate * is checked for any valid errors. If found, the error type is 67807c478bd9Sstevel@tonic-gate * returned. If not found, the flt_type is checked for L1$ parity errors. 67817c478bd9Sstevel@tonic-gate */ 67827c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 67837c478bd9Sstevel@tonic-gate static uint8_t 67847c478bd9Sstevel@tonic-gate cpu_flt_bit_to_plat_error(struct async_flt *aflt) 67857c478bd9Sstevel@tonic-gate { 67867c478bd9Sstevel@tonic-gate #if defined(JALAPENO) 67877c478bd9Sstevel@tonic-gate /* 67887c478bd9Sstevel@tonic-gate * Currently, logging errors to the SC is not supported on Jalapeno 67897c478bd9Sstevel@tonic-gate */ 67907c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_NONE); 67917c478bd9Sstevel@tonic-gate #else 67927c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 67937c478bd9Sstevel@tonic-gate 67947c478bd9Sstevel@tonic-gate switch (ch_flt->flt_bit) { 67957c478bd9Sstevel@tonic-gate case C_AFSR_CE: 67967c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_CE); 67977c478bd9Sstevel@tonic-gate case C_AFSR_UCC: 67987c478bd9Sstevel@tonic-gate case C_AFSR_EDC: 67997c478bd9Sstevel@tonic-gate case C_AFSR_WDC: 68007c478bd9Sstevel@tonic-gate case C_AFSR_CPC: 68017c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_CE); 68027c478bd9Sstevel@tonic-gate case C_AFSR_EMC: 68037c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_EMC); 68047c478bd9Sstevel@tonic-gate case C_AFSR_IVC: 68057c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IVC); 68067c478bd9Sstevel@tonic-gate case C_AFSR_UE: 68077c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_UE); 68087c478bd9Sstevel@tonic-gate case C_AFSR_UCU: 68097c478bd9Sstevel@tonic-gate case C_AFSR_EDU: 68107c478bd9Sstevel@tonic-gate case C_AFSR_WDU: 68117c478bd9Sstevel@tonic-gate case C_AFSR_CPU: 68127c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_UE); 68137c478bd9Sstevel@tonic-gate case C_AFSR_IVU: 68147c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IVU); 68157c478bd9Sstevel@tonic-gate case C_AFSR_TO: 68167c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_TO); 68177c478bd9Sstevel@tonic-gate case C_AFSR_BERR: 68187c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_BERR); 68197c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 68207c478bd9Sstevel@tonic-gate case C_AFSR_L3_EDC: 68217c478bd9Sstevel@tonic-gate case C_AFSR_L3_UCC: 68227c478bd9Sstevel@tonic-gate case C_AFSR_L3_CPC: 68237c478bd9Sstevel@tonic-gate case C_AFSR_L3_WDC: 68247c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_CE); 68257c478bd9Sstevel@tonic-gate case C_AFSR_IMC: 68267c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IMC); 68277c478bd9Sstevel@tonic-gate case C_AFSR_TSCE: 68287c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_TSCE); 68297c478bd9Sstevel@tonic-gate case C_AFSR_THCE: 68307c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_THCE); 68317c478bd9Sstevel@tonic-gate case C_AFSR_L3_MECC: 68327c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_MECC); 68337c478bd9Sstevel@tonic-gate case C_AFSR_L3_THCE: 68347c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_THCE); 68357c478bd9Sstevel@tonic-gate case C_AFSR_L3_CPU: 68367c478bd9Sstevel@tonic-gate case C_AFSR_L3_EDU: 68377c478bd9Sstevel@tonic-gate case C_AFSR_L3_UCU: 68387c478bd9Sstevel@tonic-gate case C_AFSR_L3_WDU: 68397c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_UE); 68407c478bd9Sstevel@tonic-gate case C_AFSR_DUE: 68417c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DUE); 68427c478bd9Sstevel@tonic-gate case C_AFSR_DTO: 68437c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DTO); 68447c478bd9Sstevel@tonic-gate case C_AFSR_DBERR: 68457c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DBERR); 68467c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 68477c478bd9Sstevel@tonic-gate default: 68487c478bd9Sstevel@tonic-gate switch (ch_flt->flt_type) { 68497c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 68507c478bd9Sstevel@tonic-gate case CPU_IC_PARITY: 68517c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IPE); 68527c478bd9Sstevel@tonic-gate case CPU_DC_PARITY: 68537c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 68547c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_cache == 68557c478bd9Sstevel@tonic-gate CPU_PC_PARITY) { 68567c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_PCACHE); 68577c478bd9Sstevel@tonic-gate } 68587c478bd9Sstevel@tonic-gate } 68597c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DPE); 68607c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 68617c478bd9Sstevel@tonic-gate case CPU_ITLB_PARITY: 68627c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_ITLB); 68637c478bd9Sstevel@tonic-gate case CPU_DTLB_PARITY: 68647c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DTLB); 68657c478bd9Sstevel@tonic-gate default: 68667c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_NONE); 68677c478bd9Sstevel@tonic-gate } 68687c478bd9Sstevel@tonic-gate } 68697c478bd9Sstevel@tonic-gate #endif /* JALAPENO */ 68707c478bd9Sstevel@tonic-gate } 6871