17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 51e2e7a75Shuah * Common Development and Distribution License (the "License"). 61e2e7a75Shuah * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 22*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #include <sys/types.h> 277c478bd9Sstevel@tonic-gate #include <sys/systm.h> 287c478bd9Sstevel@tonic-gate #include <sys/ddi.h> 297c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 307c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 317c478bd9Sstevel@tonic-gate #include <sys/vmsystm.h> 327c478bd9Sstevel@tonic-gate #include <sys/machparam.h> 337c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> 347c478bd9Sstevel@tonic-gate #include <sys/machthread.h> 357c478bd9Sstevel@tonic-gate #include <sys/cpu.h> 367c478bd9Sstevel@tonic-gate #include <sys/cmp.h> 377c478bd9Sstevel@tonic-gate #include <sys/elf_SPARC.h> 387c478bd9Sstevel@tonic-gate #include <vm/vm_dep.h> 397c478bd9Sstevel@tonic-gate #include <vm/hat_sfmmu.h> 407c478bd9Sstevel@tonic-gate #include <vm/seg_kpm.h> 417c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 427c478bd9Sstevel@tonic-gate #include <sys/cheetahregs.h> 437c478bd9Sstevel@tonic-gate #include <sys/us3_module.h> 447c478bd9Sstevel@tonic-gate #include <sys/async.h> 457c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 467c478bd9Sstevel@tonic-gate #include <sys/debug.h> 477c478bd9Sstevel@tonic-gate #include <sys/dditypes.h> 487c478bd9Sstevel@tonic-gate #include <sys/prom_debug.h> 497c478bd9Sstevel@tonic-gate #include <sys/prom_plat.h> 507c478bd9Sstevel@tonic-gate #include <sys/cpu_module.h> 517c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> 527c478bd9Sstevel@tonic-gate #include <sys/intreg.h> 537c478bd9Sstevel@tonic-gate #include <sys/clock.h> 547c478bd9Sstevel@tonic-gate #include <sys/platform_module.h> 557c478bd9Sstevel@tonic-gate #include <sys/machtrap.h> 567c478bd9Sstevel@tonic-gate #include <sys/ontrap.h> 577c478bd9Sstevel@tonic-gate #include <sys/panic.h> 587c478bd9Sstevel@tonic-gate #include <sys/memlist.h> 597c478bd9Sstevel@tonic-gate #include <sys/bootconf.h> 607c478bd9Sstevel@tonic-gate #include <sys/ivintr.h> 617c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 627c478bd9Sstevel@tonic-gate #include <sys/taskq.h> 637c478bd9Sstevel@tonic-gate #include <sys/note.h> 647c478bd9Sstevel@tonic-gate #include <sys/ndifm.h> 657c478bd9Sstevel@tonic-gate #include <sys/ddifm.h> 667c478bd9Sstevel@tonic-gate #include <sys/fm/protocol.h> 677c478bd9Sstevel@tonic-gate #include <sys/fm/util.h> 687c478bd9Sstevel@tonic-gate #include <sys/fm/cpu/UltraSPARC-III.h> 697c478bd9Sstevel@tonic-gate #include <sys/fpras_impl.h> 707c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 717c478bd9Sstevel@tonic-gate #include <sys/watchpoint.h> 727c478bd9Sstevel@tonic-gate #include <sys/plat_ecc_unum.h> 737c478bd9Sstevel@tonic-gate #include <sys/cyclic.h> 747c478bd9Sstevel@tonic-gate #include <sys/errorq.h> 757c478bd9Sstevel@tonic-gate #include <sys/errclassify.h> 76fb2f18f8Sesaxe #include <sys/pghw.h> 77d3d50737SRafael Vanoni #include <sys/clock_impl.h> 787c478bd9Sstevel@tonic-gate 797c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 807c478bd9Sstevel@tonic-gate #include <sys/xc_impl.h> 817c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 827c478bd9Sstevel@tonic-gate 837bebe46cSjc ch_cpu_logout_t clop_before_flush; 847bebe46cSjc ch_cpu_logout_t clop_after_flush; 857bebe46cSjc uint_t flush_retries_done = 0; 867c478bd9Sstevel@tonic-gate /* 877c478bd9Sstevel@tonic-gate * Note that 'Cheetah PRM' refers to: 887c478bd9Sstevel@tonic-gate * SPARC V9 JPS1 Implementation Supplement: Sun UltraSPARC-III 897c478bd9Sstevel@tonic-gate */ 907c478bd9Sstevel@tonic-gate 917c478bd9Sstevel@tonic-gate /* 927c478bd9Sstevel@tonic-gate * Per CPU pointers to physical address of TL>0 logout data areas. 937c478bd9Sstevel@tonic-gate * These pointers have to be in the kernel nucleus to avoid MMU 947c478bd9Sstevel@tonic-gate * misses. 957c478bd9Sstevel@tonic-gate */ 967c478bd9Sstevel@tonic-gate uint64_t ch_err_tl1_paddrs[NCPU]; 977c478bd9Sstevel@tonic-gate 987c478bd9Sstevel@tonic-gate /* 997c478bd9Sstevel@tonic-gate * One statically allocated structure to use during startup/DR 1007c478bd9Sstevel@tonic-gate * to prevent unnecessary panics. 1017c478bd9Sstevel@tonic-gate */ 1027c478bd9Sstevel@tonic-gate ch_err_tl1_data_t ch_err_tl1_data; 1037c478bd9Sstevel@tonic-gate 1047c478bd9Sstevel@tonic-gate /* 1057c478bd9Sstevel@tonic-gate * Per CPU pending error at TL>0, used by level15 softint handler 1067c478bd9Sstevel@tonic-gate */ 1077c478bd9Sstevel@tonic-gate uchar_t ch_err_tl1_pending[NCPU]; 1087c478bd9Sstevel@tonic-gate 1097c478bd9Sstevel@tonic-gate /* 1107c478bd9Sstevel@tonic-gate * For deferred CE re-enable after trap. 1117c478bd9Sstevel@tonic-gate */ 1127c478bd9Sstevel@tonic-gate taskq_t *ch_check_ce_tq; 1137c478bd9Sstevel@tonic-gate 1147c478bd9Sstevel@tonic-gate /* 1157c478bd9Sstevel@tonic-gate * Internal functions. 1167c478bd9Sstevel@tonic-gate */ 1177c478bd9Sstevel@tonic-gate static int cpu_async_log_err(void *flt, errorq_elem_t *eqep); 1187c478bd9Sstevel@tonic-gate static void cpu_log_diag_info(ch_async_flt_t *ch_flt); 1197c478bd9Sstevel@tonic-gate static void cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 1207c478bd9Sstevel@tonic-gate ecc_type_to_info_t *eccp, ch_diag_data_t *cdp); 12193743541Smb static int cpu_flt_in_memory_one_event(ch_async_flt_t *ch_flt, 12293743541Smb uint64_t t_afsr_bit); 1237c478bd9Sstevel@tonic-gate static int clear_ecc(struct async_flt *ecc); 1247c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 1257c478bd9Sstevel@tonic-gate static int cpu_ecache_line_valid(ch_async_flt_t *ch_flt); 1267c478bd9Sstevel@tonic-gate #endif 1277bebe46cSjc int cpu_ecache_set_size(struct cpu *cp); 1287c478bd9Sstevel@tonic-gate static int cpu_ectag_line_invalid(int cachesize, uint64_t tag); 1297bebe46cSjc int cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr); 1307bebe46cSjc uint64_t cpu_ectag_to_pa(int setsize, uint64_t tag); 1317bebe46cSjc int cpu_ectag_pa_to_subblk_state(int cachesize, 1327c478bd9Sstevel@tonic-gate uint64_t subaddr, uint64_t tag); 1337c478bd9Sstevel@tonic-gate static void cpu_flush_ecache_line(ch_async_flt_t *ch_flt); 1347c478bd9Sstevel@tonic-gate static int afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit); 1357c478bd9Sstevel@tonic-gate static int afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit); 1367c478bd9Sstevel@tonic-gate static int afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit); 1377c478bd9Sstevel@tonic-gate static int afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit); 1387c478bd9Sstevel@tonic-gate static int synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit); 13993743541Smb static int cpu_get_mem_unum_synd(int synd_code, struct async_flt *, char *buf); 1407c478bd9Sstevel@tonic-gate static void cpu_uninit_ecache_scrub_dr(struct cpu *cp); 1417c478bd9Sstevel@tonic-gate static void cpu_scrubphys(struct async_flt *aflt); 1427c478bd9Sstevel@tonic-gate static void cpu_payload_add_aflt(struct async_flt *, nvlist_t *, nvlist_t *, 1437c478bd9Sstevel@tonic-gate int *, int *); 1447c478bd9Sstevel@tonic-gate static void cpu_payload_add_ecache(struct async_flt *, nvlist_t *); 1457c478bd9Sstevel@tonic-gate static void cpu_ereport_init(struct async_flt *aflt); 1467c478bd9Sstevel@tonic-gate static int cpu_check_secondary_errors(ch_async_flt_t *, uint64_t, uint64_t); 1477c478bd9Sstevel@tonic-gate static uint8_t cpu_flt_bit_to_plat_error(struct async_flt *aflt); 1487c478bd9Sstevel@tonic-gate static void cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 1494fd7ecabSdilpreet uint64_t nceen, ch_cpu_logout_t *clop); 1507c478bd9Sstevel@tonic-gate static int cpu_ce_delayed_ec_logout(uint64_t); 1517c478bd9Sstevel@tonic-gate static int cpu_matching_ecache_line(uint64_t, void *, int, int *); 15238e9bdffSmikechr static int cpu_error_is_ecache_data(int, uint64_t); 15338e9bdffSmikechr static void cpu_fmri_cpu_set(nvlist_t *, int); 15438e9bdffSmikechr static int cpu_error_to_resource_type(struct async_flt *aflt); 1557c478bd9Sstevel@tonic-gate 1567c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 1577c478bd9Sstevel@tonic-gate static int mondo_recover_proc(uint16_t, int); 1587c478bd9Sstevel@tonic-gate static void cheetah_nudge_init(void); 1597c478bd9Sstevel@tonic-gate static void cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 1607c478bd9Sstevel@tonic-gate cyc_time_t *when); 1617c478bd9Sstevel@tonic-gate static void cheetah_nudge_buddy(void); 1627c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 1637c478bd9Sstevel@tonic-gate 1647c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 1657c478bd9Sstevel@tonic-gate static void cpu_dcache_parity_info(ch_async_flt_t *ch_flt); 1667c478bd9Sstevel@tonic-gate static void cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index); 1677c478bd9Sstevel@tonic-gate static void cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 1687c478bd9Sstevel@tonic-gate ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word); 1697c478bd9Sstevel@tonic-gate static void cpu_icache_parity_info(ch_async_flt_t *ch_flt); 1707c478bd9Sstevel@tonic-gate static void cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index); 1717c478bd9Sstevel@tonic-gate static void cpu_pcache_parity_info(ch_async_flt_t *ch_flt); 1727c478bd9Sstevel@tonic-gate static void cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index); 1737c478bd9Sstevel@tonic-gate static void cpu_payload_add_dcache(struct async_flt *, nvlist_t *); 1747c478bd9Sstevel@tonic-gate static void cpu_payload_add_icache(struct async_flt *, nvlist_t *); 1757c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 1767c478bd9Sstevel@tonic-gate 1777c478bd9Sstevel@tonic-gate int (*p2get_mem_info)(int synd_code, uint64_t paddr, 1787c478bd9Sstevel@tonic-gate uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 1797c478bd9Sstevel@tonic-gate int *segsp, int *banksp, int *mcidp); 1807c478bd9Sstevel@tonic-gate 1817c478bd9Sstevel@tonic-gate /* 1827c478bd9Sstevel@tonic-gate * This table is used to determine which bit(s) is(are) bad when an ECC 1837c478bd9Sstevel@tonic-gate * error occurs. The array is indexed by an 9-bit syndrome. The entries 1847c478bd9Sstevel@tonic-gate * of this array have the following semantics: 1857c478bd9Sstevel@tonic-gate * 1867c478bd9Sstevel@tonic-gate * 00-127 The number of the bad bit, when only one bit is bad. 1877c478bd9Sstevel@tonic-gate * 128 ECC bit C0 is bad. 1887c478bd9Sstevel@tonic-gate * 129 ECC bit C1 is bad. 1897c478bd9Sstevel@tonic-gate * 130 ECC bit C2 is bad. 1907c478bd9Sstevel@tonic-gate * 131 ECC bit C3 is bad. 1917c478bd9Sstevel@tonic-gate * 132 ECC bit C4 is bad. 1927c478bd9Sstevel@tonic-gate * 133 ECC bit C5 is bad. 1937c478bd9Sstevel@tonic-gate * 134 ECC bit C6 is bad. 1947c478bd9Sstevel@tonic-gate * 135 ECC bit C7 is bad. 1957c478bd9Sstevel@tonic-gate * 136 ECC bit C8 is bad. 1967c478bd9Sstevel@tonic-gate * 137-143 reserved for Mtag Data and ECC. 1977c478bd9Sstevel@tonic-gate * 144(M2) Two bits are bad within a nibble. 1987c478bd9Sstevel@tonic-gate * 145(M3) Three bits are bad within a nibble. 1997c478bd9Sstevel@tonic-gate * 146(M3) Four bits are bad within a nibble. 2007c478bd9Sstevel@tonic-gate * 147(M) Multiple bits (5 or more) are bad. 2017c478bd9Sstevel@tonic-gate * 148 NO bits are bad. 2027c478bd9Sstevel@tonic-gate * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-4,11-5. 2037c478bd9Sstevel@tonic-gate */ 2047c478bd9Sstevel@tonic-gate 2057c478bd9Sstevel@tonic-gate #define C0 128 2067c478bd9Sstevel@tonic-gate #define C1 129 2077c478bd9Sstevel@tonic-gate #define C2 130 2087c478bd9Sstevel@tonic-gate #define C3 131 2097c478bd9Sstevel@tonic-gate #define C4 132 2107c478bd9Sstevel@tonic-gate #define C5 133 2117c478bd9Sstevel@tonic-gate #define C6 134 2127c478bd9Sstevel@tonic-gate #define C7 135 2137c478bd9Sstevel@tonic-gate #define C8 136 2147c478bd9Sstevel@tonic-gate #define MT0 137 /* Mtag Data bit 0 */ 2157c478bd9Sstevel@tonic-gate #define MT1 138 2167c478bd9Sstevel@tonic-gate #define MT2 139 2177c478bd9Sstevel@tonic-gate #define MTC0 140 /* Mtag Check bit 0 */ 2187c478bd9Sstevel@tonic-gate #define MTC1 141 2197c478bd9Sstevel@tonic-gate #define MTC2 142 2207c478bd9Sstevel@tonic-gate #define MTC3 143 2217c478bd9Sstevel@tonic-gate #define M2 144 2227c478bd9Sstevel@tonic-gate #define M3 145 2237c478bd9Sstevel@tonic-gate #define M4 146 2247c478bd9Sstevel@tonic-gate #define M 147 2257c478bd9Sstevel@tonic-gate #define NA 148 2267c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2277c478bd9Sstevel@tonic-gate #define S003 149 /* Syndrome 0x003 => likely from CPU/EDU:ST/FRU/BP */ 2287c478bd9Sstevel@tonic-gate #define S003MEM 150 /* Syndrome 0x003 => likely from WDU/WBP */ 2297c478bd9Sstevel@tonic-gate #define SLAST S003MEM /* last special syndrome */ 2307c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 2317c478bd9Sstevel@tonic-gate #define S003 149 /* Syndrome 0x003 => likely from EDU:ST */ 2327c478bd9Sstevel@tonic-gate #define S071 150 /* Syndrome 0x071 => likely from WDU/CPU */ 2337c478bd9Sstevel@tonic-gate #define S11C 151 /* Syndrome 0x11c => likely from BERR/DBERR */ 2347c478bd9Sstevel@tonic-gate #define SLAST S11C /* last special syndrome */ 2357c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2367c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2377c478bd9Sstevel@tonic-gate #define BPAR0 152 /* syndrom 152 through 167 for bus parity */ 2387c478bd9Sstevel@tonic-gate #define BPAR15 167 2397c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2407c478bd9Sstevel@tonic-gate 2417c478bd9Sstevel@tonic-gate static uint8_t ecc_syndrome_tab[] = 2427c478bd9Sstevel@tonic-gate { 2437c478bd9Sstevel@tonic-gate NA, C0, C1, S003, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M, 2447c478bd9Sstevel@tonic-gate C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16, 2457c478bd9Sstevel@tonic-gate C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10, 2467c478bd9Sstevel@tonic-gate M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M, 2477c478bd9Sstevel@tonic-gate C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6, 2487c478bd9Sstevel@tonic-gate M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4, 2497c478bd9Sstevel@tonic-gate M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4, 2507c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2517c478bd9Sstevel@tonic-gate 116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 2527c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 2537c478bd9Sstevel@tonic-gate 116, S071, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, 2547c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2557c478bd9Sstevel@tonic-gate C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5, 2567c478bd9Sstevel@tonic-gate M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M, 2577c478bd9Sstevel@tonic-gate M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2, 2587c478bd9Sstevel@tonic-gate 103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3, 2597c478bd9Sstevel@tonic-gate M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M, 2607c478bd9Sstevel@tonic-gate 102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3, 2617c478bd9Sstevel@tonic-gate 98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M, 2627c478bd9Sstevel@tonic-gate M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M, 2637c478bd9Sstevel@tonic-gate C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4, 2647c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 2657c478bd9Sstevel@tonic-gate M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M, 2667c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 2677c478bd9Sstevel@tonic-gate M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, S11C, M, M3, M, 2687c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 2697c478bd9Sstevel@tonic-gate M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2, 2707c478bd9Sstevel@tonic-gate 94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M, 2717c478bd9Sstevel@tonic-gate M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4, 2727c478bd9Sstevel@tonic-gate 89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3, 2737c478bd9Sstevel@tonic-gate 86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3, 2747c478bd9Sstevel@tonic-gate M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2, 2757c478bd9Sstevel@tonic-gate M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4, 2767c478bd9Sstevel@tonic-gate 77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M, 2777c478bd9Sstevel@tonic-gate 74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3, 2787c478bd9Sstevel@tonic-gate M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M, 2797c478bd9Sstevel@tonic-gate 80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3, 2807c478bd9Sstevel@tonic-gate M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M, 2817c478bd9Sstevel@tonic-gate M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M, 2827c478bd9Sstevel@tonic-gate 111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M 2837c478bd9Sstevel@tonic-gate }; 2847c478bd9Sstevel@tonic-gate 2857c478bd9Sstevel@tonic-gate #define ESYND_TBL_SIZE (sizeof (ecc_syndrome_tab) / sizeof (uint8_t)) 2867c478bd9Sstevel@tonic-gate 2877c478bd9Sstevel@tonic-gate #if !(defined(JALAPENO) || defined(SERRANO)) 2887c478bd9Sstevel@tonic-gate /* 2897c478bd9Sstevel@tonic-gate * This table is used to determine which bit(s) is(are) bad when a Mtag 2907c478bd9Sstevel@tonic-gate * error occurs. The array is indexed by an 4-bit ECC syndrome. The entries 2917c478bd9Sstevel@tonic-gate * of this array have the following semantics: 2927c478bd9Sstevel@tonic-gate * 2937c478bd9Sstevel@tonic-gate * -1 Invalid mtag syndrome. 2947c478bd9Sstevel@tonic-gate * 137 Mtag Data 0 is bad. 2957c478bd9Sstevel@tonic-gate * 138 Mtag Data 1 is bad. 2967c478bd9Sstevel@tonic-gate * 139 Mtag Data 2 is bad. 2977c478bd9Sstevel@tonic-gate * 140 Mtag ECC 0 is bad. 2987c478bd9Sstevel@tonic-gate * 141 Mtag ECC 1 is bad. 2997c478bd9Sstevel@tonic-gate * 142 Mtag ECC 2 is bad. 3007c478bd9Sstevel@tonic-gate * 143 Mtag ECC 3 is bad. 3017c478bd9Sstevel@tonic-gate * Based on "Cheetah Programmer's Reference Manual" rev 1.1, Tables 11-6. 3027c478bd9Sstevel@tonic-gate */ 3037c478bd9Sstevel@tonic-gate short mtag_syndrome_tab[] = 3047c478bd9Sstevel@tonic-gate { 3057c478bd9Sstevel@tonic-gate NA, MTC0, MTC1, M2, MTC2, M2, M2, MT0, MTC3, M2, M2, MT1, M2, MT2, M2, M2 3067c478bd9Sstevel@tonic-gate }; 3077c478bd9Sstevel@tonic-gate 3087c478bd9Sstevel@tonic-gate #define MSYND_TBL_SIZE (sizeof (mtag_syndrome_tab) / sizeof (short)) 3097c478bd9Sstevel@tonic-gate 3107c478bd9Sstevel@tonic-gate #else /* !(JALAPENO || SERRANO) */ 3117c478bd9Sstevel@tonic-gate 3127c478bd9Sstevel@tonic-gate #define BSYND_TBL_SIZE 16 3137c478bd9Sstevel@tonic-gate 3147c478bd9Sstevel@tonic-gate #endif /* !(JALAPENO || SERRANO) */ 3157c478bd9Sstevel@tonic-gate 316*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States /* 317*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * Virtual Address bit flag in the data cache. This is actually bit 2 in the 318*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * dcache data tag. 319*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States */ 320*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States #define VA13 INT64_C(0x0000000000000002) 321*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States 32238e9bdffSmikechr /* 32338e9bdffSmikechr * Types returned from cpu_error_to_resource_type() 32438e9bdffSmikechr */ 32538e9bdffSmikechr #define ERRTYPE_UNKNOWN 0 32638e9bdffSmikechr #define ERRTYPE_CPU 1 32738e9bdffSmikechr #define ERRTYPE_MEMORY 2 32838e9bdffSmikechr #define ERRTYPE_ECACHE_DATA 3 32938e9bdffSmikechr 3307c478bd9Sstevel@tonic-gate /* 3317c478bd9Sstevel@tonic-gate * CE initial classification and subsequent action lookup table 3327c478bd9Sstevel@tonic-gate */ 3337c478bd9Sstevel@tonic-gate static ce_dispact_t ce_disp_table[CE_INITDISPTBL_SIZE]; 3347c478bd9Sstevel@tonic-gate static int ce_disp_inited; 3357c478bd9Sstevel@tonic-gate 3367c478bd9Sstevel@tonic-gate /* 3377c478bd9Sstevel@tonic-gate * Set to disable leaky and partner check for memory correctables 3387c478bd9Sstevel@tonic-gate */ 3397c478bd9Sstevel@tonic-gate int ce_xdiag_off; 3407c478bd9Sstevel@tonic-gate 3417c478bd9Sstevel@tonic-gate /* 3427c478bd9Sstevel@tonic-gate * The following are not incremented atomically so are indicative only 3437c478bd9Sstevel@tonic-gate */ 3447c478bd9Sstevel@tonic-gate static int ce_xdiag_drops; 3457c478bd9Sstevel@tonic-gate static int ce_xdiag_lkydrops; 3467c478bd9Sstevel@tonic-gate static int ce_xdiag_ptnrdrops; 3477c478bd9Sstevel@tonic-gate static int ce_xdiag_bad; 3487c478bd9Sstevel@tonic-gate 3497c478bd9Sstevel@tonic-gate /* 3507c478bd9Sstevel@tonic-gate * CE leaky check callback structure 3517c478bd9Sstevel@tonic-gate */ 3527c478bd9Sstevel@tonic-gate typedef struct { 3537c478bd9Sstevel@tonic-gate struct async_flt *lkycb_aflt; 3547c478bd9Sstevel@tonic-gate errorq_t *lkycb_eqp; 3557c478bd9Sstevel@tonic-gate errorq_elem_t *lkycb_eqep; 3567c478bd9Sstevel@tonic-gate } ce_lkychk_cb_t; 3577c478bd9Sstevel@tonic-gate 3587c478bd9Sstevel@tonic-gate /* 3597c478bd9Sstevel@tonic-gate * defines for various ecache_flush_flag's 3607c478bd9Sstevel@tonic-gate */ 3617c478bd9Sstevel@tonic-gate #define ECACHE_FLUSH_LINE 1 3627c478bd9Sstevel@tonic-gate #define ECACHE_FLUSH_ALL 2 3637c478bd9Sstevel@tonic-gate 3647c478bd9Sstevel@tonic-gate /* 3657c478bd9Sstevel@tonic-gate * STICK sync 3667c478bd9Sstevel@tonic-gate */ 3677c478bd9Sstevel@tonic-gate #define STICK_ITERATION 10 3687c478bd9Sstevel@tonic-gate #define MAX_TSKEW 1 3697c478bd9Sstevel@tonic-gate #define EV_A_START 0 3707c478bd9Sstevel@tonic-gate #define EV_A_END 1 3717c478bd9Sstevel@tonic-gate #define EV_B_START 2 3727c478bd9Sstevel@tonic-gate #define EV_B_END 3 3737c478bd9Sstevel@tonic-gate #define EVENTS 4 3747c478bd9Sstevel@tonic-gate 3757c478bd9Sstevel@tonic-gate static int64_t stick_iter = STICK_ITERATION; 3767c478bd9Sstevel@tonic-gate static int64_t stick_tsk = MAX_TSKEW; 3777c478bd9Sstevel@tonic-gate 3787c478bd9Sstevel@tonic-gate typedef enum { 3797c478bd9Sstevel@tonic-gate EVENT_NULL = 0, 3807c478bd9Sstevel@tonic-gate SLAVE_START, 3817c478bd9Sstevel@tonic-gate SLAVE_CONT, 3827c478bd9Sstevel@tonic-gate MASTER_START 3837c478bd9Sstevel@tonic-gate } event_cmd_t; 3847c478bd9Sstevel@tonic-gate 3857c478bd9Sstevel@tonic-gate static volatile event_cmd_t stick_sync_cmd = EVENT_NULL; 3867c478bd9Sstevel@tonic-gate static int64_t timestamp[EVENTS]; 3877c478bd9Sstevel@tonic-gate static volatile int slave_done; 3887c478bd9Sstevel@tonic-gate 3897c478bd9Sstevel@tonic-gate #ifdef DEBUG 3907c478bd9Sstevel@tonic-gate #define DSYNC_ATTEMPTS 64 3917c478bd9Sstevel@tonic-gate typedef struct { 3927c478bd9Sstevel@tonic-gate int64_t skew_val[DSYNC_ATTEMPTS]; 3937c478bd9Sstevel@tonic-gate } ss_t; 3947c478bd9Sstevel@tonic-gate 3957c478bd9Sstevel@tonic-gate ss_t stick_sync_stats[NCPU]; 3967c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 3977c478bd9Sstevel@tonic-gate 3981e2e7a75Shuah uint_t cpu_impl_dual_pgsz = 0; 3997c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_DUAL_PAGESIZE) 4007c478bd9Sstevel@tonic-gate uint_t disable_dual_pgsz = 0; 4017c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_DUAL_PAGESIZE */ 4027c478bd9Sstevel@tonic-gate 4037c478bd9Sstevel@tonic-gate /* 4047c478bd9Sstevel@tonic-gate * Save the cache bootup state for use when internal 4057c478bd9Sstevel@tonic-gate * caches are to be re-enabled after an error occurs. 4067c478bd9Sstevel@tonic-gate */ 4077c478bd9Sstevel@tonic-gate uint64_t cache_boot_state; 4087c478bd9Sstevel@tonic-gate 4097c478bd9Sstevel@tonic-gate /* 4107c478bd9Sstevel@tonic-gate * PA[22:0] represent Displacement in Safari configuration space. 4117c478bd9Sstevel@tonic-gate */ 4127c478bd9Sstevel@tonic-gate uint_t root_phys_addr_lo_mask = 0x7fffffu; 4137c478bd9Sstevel@tonic-gate 4147c478bd9Sstevel@tonic-gate bus_config_eclk_t bus_config_eclk[] = { 4157c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 4167c478bd9Sstevel@tonic-gate {JBUS_CONFIG_ECLK_1_DIV, JBUS_CONFIG_ECLK_1}, 4177c478bd9Sstevel@tonic-gate {JBUS_CONFIG_ECLK_2_DIV, JBUS_CONFIG_ECLK_2}, 4187c478bd9Sstevel@tonic-gate {JBUS_CONFIG_ECLK_32_DIV, JBUS_CONFIG_ECLK_32}, 4197c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 4207c478bd9Sstevel@tonic-gate {SAFARI_CONFIG_ECLK_1_DIV, SAFARI_CONFIG_ECLK_1}, 4217c478bd9Sstevel@tonic-gate {SAFARI_CONFIG_ECLK_2_DIV, SAFARI_CONFIG_ECLK_2}, 4227c478bd9Sstevel@tonic-gate {SAFARI_CONFIG_ECLK_32_DIV, SAFARI_CONFIG_ECLK_32}, 4237c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 4247c478bd9Sstevel@tonic-gate {0, 0} 4257c478bd9Sstevel@tonic-gate }; 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate /* 4287c478bd9Sstevel@tonic-gate * Interval for deferred CEEN reenable 4297c478bd9Sstevel@tonic-gate */ 4307c478bd9Sstevel@tonic-gate int cpu_ceen_delay_secs = CPU_CEEN_DELAY_SECS; 4317c478bd9Sstevel@tonic-gate 4327c478bd9Sstevel@tonic-gate /* 4337c478bd9Sstevel@tonic-gate * set in /etc/system to control logging of user BERR/TO's 4347c478bd9Sstevel@tonic-gate */ 4357c478bd9Sstevel@tonic-gate int cpu_berr_to_verbose = 0; 4367c478bd9Sstevel@tonic-gate 4377c478bd9Sstevel@tonic-gate /* 4387c478bd9Sstevel@tonic-gate * set to 0 in /etc/system to defer CEEN reenable for all CEs 4397c478bd9Sstevel@tonic-gate */ 4407c478bd9Sstevel@tonic-gate uint64_t cpu_ce_not_deferred = CPU_CE_NOT_DEFERRED; 4417c478bd9Sstevel@tonic-gate uint64_t cpu_ce_not_deferred_ext = CPU_CE_NOT_DEFERRED_EXT; 4427c478bd9Sstevel@tonic-gate 4437c478bd9Sstevel@tonic-gate /* 4447c478bd9Sstevel@tonic-gate * Set of all offline cpus 4457c478bd9Sstevel@tonic-gate */ 4467c478bd9Sstevel@tonic-gate cpuset_t cpu_offline_set; 4477c478bd9Sstevel@tonic-gate 4487c478bd9Sstevel@tonic-gate static void cpu_delayed_check_ce_errors(void *); 4497c478bd9Sstevel@tonic-gate static void cpu_check_ce_errors(void *); 4507c478bd9Sstevel@tonic-gate void cpu_error_ecache_flush(ch_async_flt_t *); 4517c478bd9Sstevel@tonic-gate static int cpu_error_ecache_flush_required(ch_async_flt_t *); 4527c478bd9Sstevel@tonic-gate static void cpu_log_and_clear_ce(ch_async_flt_t *); 4537c478bd9Sstevel@tonic-gate void cpu_ce_detected(ch_cpu_errors_t *, int); 4547c478bd9Sstevel@tonic-gate 4557c478bd9Sstevel@tonic-gate /* 4567c478bd9Sstevel@tonic-gate * CE Leaky check timeout in microseconds. This is chosen to be twice the 4577c478bd9Sstevel@tonic-gate * memory refresh interval of current DIMMs (64ms). After initial fix that 4587c478bd9Sstevel@tonic-gate * gives at least one full refresh cycle in which the cell can leak 4597c478bd9Sstevel@tonic-gate * (whereafter further refreshes simply reinforce any incorrect bit value). 4607c478bd9Sstevel@tonic-gate */ 4617c478bd9Sstevel@tonic-gate clock_t cpu_ce_lkychk_timeout_usec = 128000; 4627c478bd9Sstevel@tonic-gate 4637c478bd9Sstevel@tonic-gate /* 4647c478bd9Sstevel@tonic-gate * CE partner check partner caching period in seconds 4657c478bd9Sstevel@tonic-gate */ 4667c478bd9Sstevel@tonic-gate int cpu_ce_ptnr_cachetime_sec = 60; 4677c478bd9Sstevel@tonic-gate 4687c478bd9Sstevel@tonic-gate /* 4697c478bd9Sstevel@tonic-gate * Sets trap table entry ttentry by overwriting eight instructions from ttlabel 4707c478bd9Sstevel@tonic-gate */ 4717c478bd9Sstevel@tonic-gate #define CH_SET_TRAP(ttentry, ttlabel) \ 4727c478bd9Sstevel@tonic-gate bcopy((const void *)&ttlabel, &ttentry, 32); \ 4737c478bd9Sstevel@tonic-gate flush_instr_mem((caddr_t)&ttentry, 32); 4747c478bd9Sstevel@tonic-gate 4757c478bd9Sstevel@tonic-gate static int min_ecache_size; 4767c478bd9Sstevel@tonic-gate static uint_t priv_hcl_1; 4777c478bd9Sstevel@tonic-gate static uint_t priv_hcl_2; 4787c478bd9Sstevel@tonic-gate static uint_t priv_hcl_4; 4797c478bd9Sstevel@tonic-gate static uint_t priv_hcl_8; 4807c478bd9Sstevel@tonic-gate 4817c478bd9Sstevel@tonic-gate void 4827c478bd9Sstevel@tonic-gate cpu_setup(void) 4837c478bd9Sstevel@tonic-gate { 4847c478bd9Sstevel@tonic-gate extern int at_flags; 4857c478bd9Sstevel@tonic-gate extern int cpc_has_overflow_intr; 4867c478bd9Sstevel@tonic-gate 4877c478bd9Sstevel@tonic-gate /* 4887c478bd9Sstevel@tonic-gate * Setup chip-specific trap handlers. 4897c478bd9Sstevel@tonic-gate */ 4907c478bd9Sstevel@tonic-gate cpu_init_trap(); 4917c478bd9Sstevel@tonic-gate 4927c478bd9Sstevel@tonic-gate cache |= (CACHE_VAC | CACHE_PTAG | CACHE_IOCOHERENT); 4937c478bd9Sstevel@tonic-gate 4947c478bd9Sstevel@tonic-gate at_flags = EF_SPARC_32PLUS | EF_SPARC_SUN_US1 | EF_SPARC_SUN_US3; 4957c478bd9Sstevel@tonic-gate 4967c478bd9Sstevel@tonic-gate /* 4977c478bd9Sstevel@tonic-gate * save the cache bootup state. 4987c478bd9Sstevel@tonic-gate */ 4997c478bd9Sstevel@tonic-gate cache_boot_state = get_dcu() & DCU_CACHE; 5007c478bd9Sstevel@tonic-gate 5017c478bd9Sstevel@tonic-gate /* 5027c478bd9Sstevel@tonic-gate * Due to the number of entries in the fully-associative tlb 5037c478bd9Sstevel@tonic-gate * this may have to be tuned lower than in spitfire. 5047c478bd9Sstevel@tonic-gate */ 5057c478bd9Sstevel@tonic-gate pp_slots = MIN(8, MAXPP_SLOTS); 5067c478bd9Sstevel@tonic-gate 5077c478bd9Sstevel@tonic-gate /* 5087c478bd9Sstevel@tonic-gate * Block stores do not invalidate all pages of the d$, pagecopy 5097c478bd9Sstevel@tonic-gate * et. al. need virtual translations with virtual coloring taken 5107c478bd9Sstevel@tonic-gate * into consideration. prefetch/ldd will pollute the d$ on the 5117c478bd9Sstevel@tonic-gate * load side. 5127c478bd9Sstevel@tonic-gate */ 5137c478bd9Sstevel@tonic-gate pp_consistent_coloring = PPAGE_STORE_VCOLORING | PPAGE_LOADS_POLLUTE; 5147c478bd9Sstevel@tonic-gate 5157c478bd9Sstevel@tonic-gate if (use_page_coloring) { 5167c478bd9Sstevel@tonic-gate do_pg_coloring = 1; 5177c478bd9Sstevel@tonic-gate } 5187c478bd9Sstevel@tonic-gate 5197c478bd9Sstevel@tonic-gate isa_list = 5207c478bd9Sstevel@tonic-gate "sparcv9+vis2 sparcv9+vis sparcv9 " 5217c478bd9Sstevel@tonic-gate "sparcv8plus+vis2 sparcv8plus+vis sparcv8plus " 5227c478bd9Sstevel@tonic-gate "sparcv8 sparcv8-fsmuld sparcv7 sparc"; 5237c478bd9Sstevel@tonic-gate 5247c478bd9Sstevel@tonic-gate /* 5257c478bd9Sstevel@tonic-gate * On Panther-based machines, this should 5267c478bd9Sstevel@tonic-gate * also include AV_SPARC_POPC too 5277c478bd9Sstevel@tonic-gate */ 5287c478bd9Sstevel@tonic-gate cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2; 5297c478bd9Sstevel@tonic-gate 5307c478bd9Sstevel@tonic-gate /* 5317c478bd9Sstevel@tonic-gate * On cheetah, there's no hole in the virtual address space 5327c478bd9Sstevel@tonic-gate */ 5337c478bd9Sstevel@tonic-gate hole_start = hole_end = 0; 5347c478bd9Sstevel@tonic-gate 5357c478bd9Sstevel@tonic-gate /* 5367c478bd9Sstevel@tonic-gate * The kpm mapping window. 5377c478bd9Sstevel@tonic-gate * kpm_size: 5387c478bd9Sstevel@tonic-gate * The size of a single kpm range. 5397c478bd9Sstevel@tonic-gate * The overall size will be: kpm_size * vac_colors. 5407c478bd9Sstevel@tonic-gate * kpm_vbase: 5417c478bd9Sstevel@tonic-gate * The virtual start address of the kpm range within the kernel 5427c478bd9Sstevel@tonic-gate * virtual address space. kpm_vbase has to be kpm_size aligned. 5437c478bd9Sstevel@tonic-gate */ 5447c478bd9Sstevel@tonic-gate kpm_size = (size_t)(8ull * 1024 * 1024 * 1024 * 1024); /* 8TB */ 5457c478bd9Sstevel@tonic-gate kpm_size_shift = 43; 5467c478bd9Sstevel@tonic-gate kpm_vbase = (caddr_t)0x8000000000000000ull; /* 8EB */ 5477c478bd9Sstevel@tonic-gate kpm_smallpages = 1; 5487c478bd9Sstevel@tonic-gate 5497c478bd9Sstevel@tonic-gate /* 5507c478bd9Sstevel@tonic-gate * The traptrace code uses either %tick or %stick for 5517c478bd9Sstevel@tonic-gate * timestamping. We have %stick so we can use it. 5527c478bd9Sstevel@tonic-gate */ 5537c478bd9Sstevel@tonic-gate traptrace_use_stick = 1; 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate /* 5567c478bd9Sstevel@tonic-gate * Cheetah has a performance counter overflow interrupt 5577c478bd9Sstevel@tonic-gate */ 5587c478bd9Sstevel@tonic-gate cpc_has_overflow_intr = 1; 5597c478bd9Sstevel@tonic-gate 5607c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_DUAL_PAGESIZE) 5617c478bd9Sstevel@tonic-gate /* 5627c478bd9Sstevel@tonic-gate * Use Cheetah+ and later dual page size support. 5637c478bd9Sstevel@tonic-gate */ 5647c478bd9Sstevel@tonic-gate if (!disable_dual_pgsz) { 5651e2e7a75Shuah cpu_impl_dual_pgsz = 1; 5667c478bd9Sstevel@tonic-gate } 5677c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_DUAL_PAGESIZE */ 5687c478bd9Sstevel@tonic-gate 5697c478bd9Sstevel@tonic-gate /* 5707c478bd9Sstevel@tonic-gate * Declare that this architecture/cpu combination does fpRAS. 5717c478bd9Sstevel@tonic-gate */ 5727c478bd9Sstevel@tonic-gate fpras_implemented = 1; 5737c478bd9Sstevel@tonic-gate 5747c478bd9Sstevel@tonic-gate /* 5757c478bd9Sstevel@tonic-gate * Setup CE lookup table 5767c478bd9Sstevel@tonic-gate */ 5777c478bd9Sstevel@tonic-gate CE_INITDISPTBL_POPULATE(ce_disp_table); 5787c478bd9Sstevel@tonic-gate ce_disp_inited = 1; 5797c478bd9Sstevel@tonic-gate } 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate /* 5827c478bd9Sstevel@tonic-gate * Called by setcpudelay 5837c478bd9Sstevel@tonic-gate */ 5847c478bd9Sstevel@tonic-gate void 5857c478bd9Sstevel@tonic-gate cpu_init_tick_freq(void) 5867c478bd9Sstevel@tonic-gate { 5877c478bd9Sstevel@tonic-gate /* 5887c478bd9Sstevel@tonic-gate * For UltraSPARC III and beyond we want to use the 5897c478bd9Sstevel@tonic-gate * system clock rate as the basis for low level timing, 5907c478bd9Sstevel@tonic-gate * due to support of mixed speed CPUs and power managment. 5917c478bd9Sstevel@tonic-gate */ 5927c478bd9Sstevel@tonic-gate if (system_clock_freq == 0) 5937c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "setcpudelay: invalid system_clock_freq"); 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate sys_tick_freq = system_clock_freq; 5967c478bd9Sstevel@tonic-gate } 5977c478bd9Sstevel@tonic-gate 5987c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 5997c478bd9Sstevel@tonic-gate /* 6007c478bd9Sstevel@tonic-gate * Tunables 6017c478bd9Sstevel@tonic-gate */ 6027c478bd9Sstevel@tonic-gate int cheetah_bpe_off = 0; 6037c478bd9Sstevel@tonic-gate int cheetah_sendmondo_recover = 1; 6047c478bd9Sstevel@tonic-gate int cheetah_sendmondo_fullscan = 0; 6057c478bd9Sstevel@tonic-gate int cheetah_sendmondo_recover_delay = 5; 6067c478bd9Sstevel@tonic-gate 6077c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_MIN_DELAY 1 6087c478bd9Sstevel@tonic-gate 6097c478bd9Sstevel@tonic-gate /* 6107c478bd9Sstevel@tonic-gate * Recovery Statistics 6117c478bd9Sstevel@tonic-gate */ 6127c478bd9Sstevel@tonic-gate typedef struct cheetah_livelock_entry { 6137c478bd9Sstevel@tonic-gate int cpuid; /* fallen cpu */ 6147c478bd9Sstevel@tonic-gate int buddy; /* cpu that ran recovery */ 6157c478bd9Sstevel@tonic-gate clock_t lbolt; /* when recovery started */ 6167c478bd9Sstevel@tonic-gate hrtime_t recovery_time; /* time spent in recovery */ 6177c478bd9Sstevel@tonic-gate } cheetah_livelock_entry_t; 6187c478bd9Sstevel@tonic-gate 6197c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_NENTRY 32 6207c478bd9Sstevel@tonic-gate 6217c478bd9Sstevel@tonic-gate cheetah_livelock_entry_t cheetah_livelock_hist[CHEETAH_LIVELOCK_NENTRY]; 6227c478bd9Sstevel@tonic-gate int cheetah_livelock_entry_nxt; 6237c478bd9Sstevel@tonic-gate 6247c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_ENTRY_NEXT(statp) { \ 6257c478bd9Sstevel@tonic-gate statp = cheetah_livelock_hist + cheetah_livelock_entry_nxt; \ 6267c478bd9Sstevel@tonic-gate if (++cheetah_livelock_entry_nxt >= CHEETAH_LIVELOCK_NENTRY) { \ 6277c478bd9Sstevel@tonic-gate cheetah_livelock_entry_nxt = 0; \ 6287c478bd9Sstevel@tonic-gate } \ 6297c478bd9Sstevel@tonic-gate } 6307c478bd9Sstevel@tonic-gate 6317c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_ENTRY_SET(statp, item, val) statp->item = val 6327c478bd9Sstevel@tonic-gate 6337c478bd9Sstevel@tonic-gate struct { 6347c478bd9Sstevel@tonic-gate hrtime_t hrt; /* maximum recovery time */ 6357c478bd9Sstevel@tonic-gate int recovery; /* recovered */ 6367c478bd9Sstevel@tonic-gate int full_claimed; /* maximum pages claimed in full recovery */ 6377c478bd9Sstevel@tonic-gate int proc_entry; /* attempted to claim TSB */ 6387c478bd9Sstevel@tonic-gate int proc_tsb_scan; /* tsb scanned */ 6397c478bd9Sstevel@tonic-gate int proc_tsb_partscan; /* tsb partially scanned */ 6407c478bd9Sstevel@tonic-gate int proc_tsb_fullscan; /* whole tsb scanned */ 6417c478bd9Sstevel@tonic-gate int proc_claimed; /* maximum pages claimed in tsb scan */ 6427c478bd9Sstevel@tonic-gate int proc_user; /* user thread */ 6437c478bd9Sstevel@tonic-gate int proc_kernel; /* kernel thread */ 6447c478bd9Sstevel@tonic-gate int proc_onflt; /* bad stack */ 6457c478bd9Sstevel@tonic-gate int proc_cpu; /* null cpu */ 6467c478bd9Sstevel@tonic-gate int proc_thread; /* null thread */ 6477c478bd9Sstevel@tonic-gate int proc_proc; /* null proc */ 6487c478bd9Sstevel@tonic-gate int proc_as; /* null as */ 6497c478bd9Sstevel@tonic-gate int proc_hat; /* null hat */ 6507c478bd9Sstevel@tonic-gate int proc_hat_inval; /* hat contents don't make sense */ 6517c478bd9Sstevel@tonic-gate int proc_hat_busy; /* hat is changing TSBs */ 6527c478bd9Sstevel@tonic-gate int proc_tsb_reloc; /* TSB skipped because being relocated */ 6537c478bd9Sstevel@tonic-gate int proc_cnum_bad; /* cnum out of range */ 6547c478bd9Sstevel@tonic-gate int proc_cnum; /* last cnum processed */ 6557c478bd9Sstevel@tonic-gate tte_t proc_tte; /* last tte processed */ 6567c478bd9Sstevel@tonic-gate } cheetah_livelock_stat; 6577c478bd9Sstevel@tonic-gate 6587c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_STAT(item) cheetah_livelock_stat.item++ 6597c478bd9Sstevel@tonic-gate 6607c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_STATSET(item, value) \ 6617c478bd9Sstevel@tonic-gate cheetah_livelock_stat.item = value 6627c478bd9Sstevel@tonic-gate 6637c478bd9Sstevel@tonic-gate #define CHEETAH_LIVELOCK_MAXSTAT(item, value) { \ 6647c478bd9Sstevel@tonic-gate if (value > cheetah_livelock_stat.item) \ 6657c478bd9Sstevel@tonic-gate cheetah_livelock_stat.item = value; \ 6667c478bd9Sstevel@tonic-gate } 6677c478bd9Sstevel@tonic-gate 6687c478bd9Sstevel@tonic-gate /* 6697c478bd9Sstevel@tonic-gate * Attempt to recover a cpu by claiming every cache line as saved 6707c478bd9Sstevel@tonic-gate * in the TSB that the non-responsive cpu is using. Since we can't 6717c478bd9Sstevel@tonic-gate * grab any adaptive lock, this is at best an attempt to do so. Because 6727c478bd9Sstevel@tonic-gate * we don't grab any locks, we must operate under the protection of 6737c478bd9Sstevel@tonic-gate * on_fault(). 6747c478bd9Sstevel@tonic-gate * 6757c478bd9Sstevel@tonic-gate * Return 1 if cpuid could be recovered, 0 if failed. 6767c478bd9Sstevel@tonic-gate */ 6777c478bd9Sstevel@tonic-gate int 6787c478bd9Sstevel@tonic-gate mondo_recover_proc(uint16_t cpuid, int bn) 6797c478bd9Sstevel@tonic-gate { 6807c478bd9Sstevel@tonic-gate label_t ljb; 6817c478bd9Sstevel@tonic-gate cpu_t *cp; 6827c478bd9Sstevel@tonic-gate kthread_t *t; 6837c478bd9Sstevel@tonic-gate proc_t *p; 6847c478bd9Sstevel@tonic-gate struct as *as; 6857c478bd9Sstevel@tonic-gate struct hat *hat; 6861e2e7a75Shuah uint_t cnum; 6877c478bd9Sstevel@tonic-gate struct tsb_info *tsbinfop; 6887c478bd9Sstevel@tonic-gate struct tsbe *tsbep; 6897c478bd9Sstevel@tonic-gate caddr_t tsbp; 6907c478bd9Sstevel@tonic-gate caddr_t end_tsbp; 6917c478bd9Sstevel@tonic-gate uint64_t paddr; 6927c478bd9Sstevel@tonic-gate uint64_t idsr; 6937c478bd9Sstevel@tonic-gate u_longlong_t pahi, palo; 6947c478bd9Sstevel@tonic-gate int pages_claimed = 0; 6957c478bd9Sstevel@tonic-gate tte_t tsbe_tte; 6967c478bd9Sstevel@tonic-gate int tried_kernel_tsb = 0; 6971e2e7a75Shuah mmu_ctx_t *mmu_ctxp; 6987c478bd9Sstevel@tonic-gate 6997c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_entry); 7007c478bd9Sstevel@tonic-gate 7017c478bd9Sstevel@tonic-gate if (on_fault(&ljb)) { 7027c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_onflt); 7037c478bd9Sstevel@tonic-gate goto badstruct; 7047c478bd9Sstevel@tonic-gate } 7057c478bd9Sstevel@tonic-gate 7067c478bd9Sstevel@tonic-gate if ((cp = cpu[cpuid]) == NULL) { 7077c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_cpu); 7087c478bd9Sstevel@tonic-gate goto badstruct; 7097c478bd9Sstevel@tonic-gate } 7107c478bd9Sstevel@tonic-gate 7117c478bd9Sstevel@tonic-gate if ((t = cp->cpu_thread) == NULL) { 7127c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_thread); 7137c478bd9Sstevel@tonic-gate goto badstruct; 7147c478bd9Sstevel@tonic-gate } 7157c478bd9Sstevel@tonic-gate 7167c478bd9Sstevel@tonic-gate if ((p = ttoproc(t)) == NULL) { 7177c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_proc); 7187c478bd9Sstevel@tonic-gate goto badstruct; 7197c478bd9Sstevel@tonic-gate } 7207c478bd9Sstevel@tonic-gate 7217c478bd9Sstevel@tonic-gate if ((as = p->p_as) == NULL) { 7227c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_as); 7237c478bd9Sstevel@tonic-gate goto badstruct; 7247c478bd9Sstevel@tonic-gate } 7257c478bd9Sstevel@tonic-gate 7267c478bd9Sstevel@tonic-gate if ((hat = as->a_hat) == NULL) { 7277c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat); 7287c478bd9Sstevel@tonic-gate goto badstruct; 7297c478bd9Sstevel@tonic-gate } 7307c478bd9Sstevel@tonic-gate 7317c478bd9Sstevel@tonic-gate if (hat != ksfmmup) { 7327c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_user); 7337c478bd9Sstevel@tonic-gate if (hat->sfmmu_flags & (HAT_BUSY | HAT_SWAPPED | HAT_SWAPIN)) { 7347c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat_busy); 7357c478bd9Sstevel@tonic-gate goto badstruct; 7367c478bd9Sstevel@tonic-gate } 7377c478bd9Sstevel@tonic-gate tsbinfop = hat->sfmmu_tsb; 7387c478bd9Sstevel@tonic-gate if (tsbinfop == NULL) { 7397c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat_inval); 7407c478bd9Sstevel@tonic-gate goto badstruct; 7417c478bd9Sstevel@tonic-gate } 7427c478bd9Sstevel@tonic-gate tsbp = tsbinfop->tsb_va; 7437c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 7447c478bd9Sstevel@tonic-gate } else { 7457c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_kernel); 7467c478bd9Sstevel@tonic-gate tsbinfop = NULL; 7477c478bd9Sstevel@tonic-gate tsbp = ktsb_base; 7487c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 7497c478bd9Sstevel@tonic-gate } 7507c478bd9Sstevel@tonic-gate 7517c478bd9Sstevel@tonic-gate /* Verify as */ 7527c478bd9Sstevel@tonic-gate if (hat->sfmmu_as != as) { 7537c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_hat_inval); 7547c478bd9Sstevel@tonic-gate goto badstruct; 7557c478bd9Sstevel@tonic-gate } 7567c478bd9Sstevel@tonic-gate 7571e2e7a75Shuah mmu_ctxp = CPU_MMU_CTXP(cp); 7581e2e7a75Shuah ASSERT(mmu_ctxp); 7591e2e7a75Shuah cnum = hat->sfmmu_ctxs[mmu_ctxp->mmu_idx].cnum; 7607c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STATSET(proc_cnum, cnum); 7617c478bd9Sstevel@tonic-gate 7621e2e7a75Shuah if ((cnum < 0) || (cnum == INVALID_CONTEXT) || 7631e2e7a75Shuah (cnum >= mmu_ctxp->mmu_nctxs)) { 7647c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_cnum_bad); 7657c478bd9Sstevel@tonic-gate goto badstruct; 7667c478bd9Sstevel@tonic-gate } 7677c478bd9Sstevel@tonic-gate 7687c478bd9Sstevel@tonic-gate do { 7697c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_scan); 7707c478bd9Sstevel@tonic-gate 7717c478bd9Sstevel@tonic-gate /* 7727c478bd9Sstevel@tonic-gate * Skip TSBs being relocated. This is important because 7737c478bd9Sstevel@tonic-gate * we want to avoid the following deadlock scenario: 7747c478bd9Sstevel@tonic-gate * 7757c478bd9Sstevel@tonic-gate * 1) when we came in we set ourselves to "in recover" state. 7767c478bd9Sstevel@tonic-gate * 2) when we try to touch TSB being relocated the mapping 7777c478bd9Sstevel@tonic-gate * will be in the suspended state so we'll spin waiting 7787c478bd9Sstevel@tonic-gate * for it to be unlocked. 7797c478bd9Sstevel@tonic-gate * 3) when the CPU that holds the TSB mapping locked tries to 7807c478bd9Sstevel@tonic-gate * unlock it it will send a xtrap which will fail to xcall 7817c478bd9Sstevel@tonic-gate * us or the CPU we're trying to recover, and will in turn 7827c478bd9Sstevel@tonic-gate * enter the mondo code. 7837c478bd9Sstevel@tonic-gate * 4) since we are still spinning on the locked mapping 7847c478bd9Sstevel@tonic-gate * no further progress will be made and the system will 7857c478bd9Sstevel@tonic-gate * inevitably hard hang. 7867c478bd9Sstevel@tonic-gate * 7877c478bd9Sstevel@tonic-gate * A TSB not being relocated can't begin being relocated 7887c478bd9Sstevel@tonic-gate * while we're accessing it because we check 7897c478bd9Sstevel@tonic-gate * sendmondo_in_recover before relocating TSBs. 7907c478bd9Sstevel@tonic-gate */ 7917c478bd9Sstevel@tonic-gate if (hat != ksfmmup && 7927c478bd9Sstevel@tonic-gate (tsbinfop->tsb_flags & TSB_RELOC_FLAG) != 0) { 7937c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_reloc); 7947c478bd9Sstevel@tonic-gate goto next_tsbinfo; 7957c478bd9Sstevel@tonic-gate } 7967c478bd9Sstevel@tonic-gate 7977c478bd9Sstevel@tonic-gate for (tsbep = (struct tsbe *)tsbp; 7987c478bd9Sstevel@tonic-gate tsbep < (struct tsbe *)end_tsbp; tsbep++) { 7997c478bd9Sstevel@tonic-gate tsbe_tte = tsbep->tte_data; 8007c478bd9Sstevel@tonic-gate 8017c478bd9Sstevel@tonic-gate if (tsbe_tte.tte_val == 0) { 8027c478bd9Sstevel@tonic-gate /* 8037c478bd9Sstevel@tonic-gate * Invalid tte 8047c478bd9Sstevel@tonic-gate */ 8057c478bd9Sstevel@tonic-gate continue; 8067c478bd9Sstevel@tonic-gate } 8077c478bd9Sstevel@tonic-gate if (tsbe_tte.tte_se) { 8087c478bd9Sstevel@tonic-gate /* 8097c478bd9Sstevel@tonic-gate * Don't want device registers 8107c478bd9Sstevel@tonic-gate */ 8117c478bd9Sstevel@tonic-gate continue; 8127c478bd9Sstevel@tonic-gate } 8137c478bd9Sstevel@tonic-gate if (tsbe_tte.tte_cp == 0) { 8147c478bd9Sstevel@tonic-gate /* 8157c478bd9Sstevel@tonic-gate * Must be cached in E$ 8167c478bd9Sstevel@tonic-gate */ 8177c478bd9Sstevel@tonic-gate continue; 8187c478bd9Sstevel@tonic-gate } 819953394f3Sjesusm if (tsbep->tte_tag.tag_invalid != 0) { 820953394f3Sjesusm /* 821953394f3Sjesusm * Invalid tag, ingnore this entry. 822953394f3Sjesusm */ 823953394f3Sjesusm continue; 824953394f3Sjesusm } 8257c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STATSET(proc_tte, tsbe_tte); 8267c478bd9Sstevel@tonic-gate idsr = getidsr(); 8277c478bd9Sstevel@tonic-gate if ((idsr & (IDSR_NACK_BIT(bn) | 8287c478bd9Sstevel@tonic-gate IDSR_BUSY_BIT(bn))) == 0) { 8297c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_partscan); 8307c478bd9Sstevel@tonic-gate goto done; 8317c478bd9Sstevel@tonic-gate } 8327c478bd9Sstevel@tonic-gate pahi = tsbe_tte.tte_pahi; 8337c478bd9Sstevel@tonic-gate palo = tsbe_tte.tte_palo; 8347c478bd9Sstevel@tonic-gate paddr = (uint64_t)((pahi << 32) | 8357c478bd9Sstevel@tonic-gate (palo << MMU_PAGESHIFT)); 8367c478bd9Sstevel@tonic-gate claimlines(paddr, TTEBYTES(TTE_CSZ(&tsbe_tte)), 8377c478bd9Sstevel@tonic-gate CH_ECACHE_SUBBLK_SIZE); 8387c478bd9Sstevel@tonic-gate if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 8397c478bd9Sstevel@tonic-gate shipit(cpuid, bn); 8407c478bd9Sstevel@tonic-gate } 8417c478bd9Sstevel@tonic-gate pages_claimed++; 8427c478bd9Sstevel@tonic-gate } 8437c478bd9Sstevel@tonic-gate next_tsbinfo: 8447c478bd9Sstevel@tonic-gate if (tsbinfop != NULL) 8457c478bd9Sstevel@tonic-gate tsbinfop = tsbinfop->tsb_next; 8467c478bd9Sstevel@tonic-gate if (tsbinfop != NULL) { 8477c478bd9Sstevel@tonic-gate tsbp = tsbinfop->tsb_va; 8487c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(tsbinfop->tsb_szc); 8497c478bd9Sstevel@tonic-gate } else if (tsbp == ktsb_base) { 8507c478bd9Sstevel@tonic-gate tried_kernel_tsb = 1; 8517c478bd9Sstevel@tonic-gate } else if (!tried_kernel_tsb) { 8527c478bd9Sstevel@tonic-gate tsbp = ktsb_base; 8537c478bd9Sstevel@tonic-gate end_tsbp = tsbp + TSB_BYTES(ktsb_sz); 8547c478bd9Sstevel@tonic-gate hat = ksfmmup; 8557c478bd9Sstevel@tonic-gate tsbinfop = NULL; 8567c478bd9Sstevel@tonic-gate } 8577c478bd9Sstevel@tonic-gate } while (tsbinfop != NULL || 858953394f3Sjesusm ((tsbp == ktsb_base) && !tried_kernel_tsb)); 8597c478bd9Sstevel@tonic-gate 8607c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(proc_tsb_fullscan); 8617c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 8627c478bd9Sstevel@tonic-gate no_fault(); 8637c478bd9Sstevel@tonic-gate idsr = getidsr(); 8647c478bd9Sstevel@tonic-gate if ((idsr & (IDSR_NACK_BIT(bn) | 8657c478bd9Sstevel@tonic-gate IDSR_BUSY_BIT(bn))) == 0) { 8667c478bd9Sstevel@tonic-gate return (1); 8677c478bd9Sstevel@tonic-gate } else { 8687c478bd9Sstevel@tonic-gate return (0); 8697c478bd9Sstevel@tonic-gate } 8707c478bd9Sstevel@tonic-gate 8717c478bd9Sstevel@tonic-gate done: 8727c478bd9Sstevel@tonic-gate no_fault(); 8737c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(proc_claimed, pages_claimed); 8747c478bd9Sstevel@tonic-gate return (1); 8757c478bd9Sstevel@tonic-gate 8767c478bd9Sstevel@tonic-gate badstruct: 8777c478bd9Sstevel@tonic-gate no_fault(); 8787c478bd9Sstevel@tonic-gate return (0); 8797c478bd9Sstevel@tonic-gate } 8807c478bd9Sstevel@tonic-gate 8817c478bd9Sstevel@tonic-gate /* 8827c478bd9Sstevel@tonic-gate * Attempt to claim ownership, temporarily, of every cache line that a 8837c478bd9Sstevel@tonic-gate * non-responsive cpu might be using. This might kick that cpu out of 8847c478bd9Sstevel@tonic-gate * this state. 8857c478bd9Sstevel@tonic-gate * 8867c478bd9Sstevel@tonic-gate * The return value indicates to the caller if we have exhausted all recovery 8877c478bd9Sstevel@tonic-gate * techniques. If 1 is returned, it is useless to call this function again 8887c478bd9Sstevel@tonic-gate * even for a different target CPU. 8897c478bd9Sstevel@tonic-gate */ 8907c478bd9Sstevel@tonic-gate int 8917c478bd9Sstevel@tonic-gate mondo_recover(uint16_t cpuid, int bn) 8927c478bd9Sstevel@tonic-gate { 8937c478bd9Sstevel@tonic-gate struct memseg *seg; 8947c478bd9Sstevel@tonic-gate uint64_t begin_pa, end_pa, cur_pa; 8957c478bd9Sstevel@tonic-gate hrtime_t begin_hrt, end_hrt; 8967c478bd9Sstevel@tonic-gate int retval = 0; 8977c478bd9Sstevel@tonic-gate int pages_claimed = 0; 8987c478bd9Sstevel@tonic-gate cheetah_livelock_entry_t *histp; 8997c478bd9Sstevel@tonic-gate uint64_t idsr; 9007c478bd9Sstevel@tonic-gate 9017c478bd9Sstevel@tonic-gate if (cas32(&sendmondo_in_recover, 0, 1) != 0) { 9027c478bd9Sstevel@tonic-gate /* 9037c478bd9Sstevel@tonic-gate * Wait while recovery takes place 9047c478bd9Sstevel@tonic-gate */ 9057c478bd9Sstevel@tonic-gate while (sendmondo_in_recover) { 9067c478bd9Sstevel@tonic-gate drv_usecwait(1); 9077c478bd9Sstevel@tonic-gate } 9087c478bd9Sstevel@tonic-gate /* 9097c478bd9Sstevel@tonic-gate * Assume we didn't claim the whole memory. If 9107c478bd9Sstevel@tonic-gate * the target of this caller is not recovered, 9117c478bd9Sstevel@tonic-gate * it will come back. 9127c478bd9Sstevel@tonic-gate */ 9137c478bd9Sstevel@tonic-gate return (retval); 9147c478bd9Sstevel@tonic-gate } 9157c478bd9Sstevel@tonic-gate 916d3d50737SRafael Vanoni CHEETAH_LIVELOCK_ENTRY_NEXT(histp); 917d3d50737SRafael Vanoni CHEETAH_LIVELOCK_ENTRY_SET(histp, lbolt, LBOLT_WAITFREE); 9187c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, cpuid, cpuid); 9197c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, buddy, CPU->cpu_id); 9207c478bd9Sstevel@tonic-gate 9217c478bd9Sstevel@tonic-gate begin_hrt = gethrtime_waitfree(); 9227c478bd9Sstevel@tonic-gate /* 9237c478bd9Sstevel@tonic-gate * First try to claim the lines in the TSB the target 9247c478bd9Sstevel@tonic-gate * may have been using. 9257c478bd9Sstevel@tonic-gate */ 9267c478bd9Sstevel@tonic-gate if (mondo_recover_proc(cpuid, bn) == 1) { 9277c478bd9Sstevel@tonic-gate /* 9287c478bd9Sstevel@tonic-gate * Didn't claim the whole memory 9297c478bd9Sstevel@tonic-gate */ 9307c478bd9Sstevel@tonic-gate goto done; 9317c478bd9Sstevel@tonic-gate } 9327c478bd9Sstevel@tonic-gate 9337c478bd9Sstevel@tonic-gate /* 9347c478bd9Sstevel@tonic-gate * We tried using the TSB. The target is still 9357c478bd9Sstevel@tonic-gate * not recovered. Check if complete memory scan is 9367c478bd9Sstevel@tonic-gate * enabled. 9377c478bd9Sstevel@tonic-gate */ 9387c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_fullscan == 0) { 9397c478bd9Sstevel@tonic-gate /* 9407c478bd9Sstevel@tonic-gate * Full memory scan is disabled. 9417c478bd9Sstevel@tonic-gate */ 9427c478bd9Sstevel@tonic-gate retval = 1; 9437c478bd9Sstevel@tonic-gate goto done; 9447c478bd9Sstevel@tonic-gate } 9457c478bd9Sstevel@tonic-gate 9467c478bd9Sstevel@tonic-gate /* 9477c478bd9Sstevel@tonic-gate * Try claiming the whole memory. 9487c478bd9Sstevel@tonic-gate */ 9497c478bd9Sstevel@tonic-gate for (seg = memsegs; seg; seg = seg->next) { 9507c478bd9Sstevel@tonic-gate begin_pa = (uint64_t)(seg->pages_base) << MMU_PAGESHIFT; 9517c478bd9Sstevel@tonic-gate end_pa = (uint64_t)(seg->pages_end) << MMU_PAGESHIFT; 9527c478bd9Sstevel@tonic-gate for (cur_pa = begin_pa; cur_pa < end_pa; 9537c478bd9Sstevel@tonic-gate cur_pa += MMU_PAGESIZE) { 9547c478bd9Sstevel@tonic-gate idsr = getidsr(); 9557c478bd9Sstevel@tonic-gate if ((idsr & (IDSR_NACK_BIT(bn) | 9567c478bd9Sstevel@tonic-gate IDSR_BUSY_BIT(bn))) == 0) { 9577c478bd9Sstevel@tonic-gate /* 9587c478bd9Sstevel@tonic-gate * Didn't claim all memory 9597c478bd9Sstevel@tonic-gate */ 9607c478bd9Sstevel@tonic-gate goto done; 9617c478bd9Sstevel@tonic-gate } 9627c478bd9Sstevel@tonic-gate claimlines(cur_pa, MMU_PAGESIZE, 9637c478bd9Sstevel@tonic-gate CH_ECACHE_SUBBLK_SIZE); 9647c478bd9Sstevel@tonic-gate if ((idsr & IDSR_BUSY_BIT(bn)) == 0) { 9657c478bd9Sstevel@tonic-gate shipit(cpuid, bn); 9667c478bd9Sstevel@tonic-gate } 9677c478bd9Sstevel@tonic-gate pages_claimed++; 9687c478bd9Sstevel@tonic-gate } 9697c478bd9Sstevel@tonic-gate } 9707c478bd9Sstevel@tonic-gate 9717c478bd9Sstevel@tonic-gate /* 9727c478bd9Sstevel@tonic-gate * We did all we could. 9737c478bd9Sstevel@tonic-gate */ 9747c478bd9Sstevel@tonic-gate retval = 1; 9757c478bd9Sstevel@tonic-gate 9767c478bd9Sstevel@tonic-gate done: 9777c478bd9Sstevel@tonic-gate /* 9787c478bd9Sstevel@tonic-gate * Update statistics 9797c478bd9Sstevel@tonic-gate */ 9807c478bd9Sstevel@tonic-gate end_hrt = gethrtime_waitfree(); 9817c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_STAT(recovery); 9827c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(hrt, (end_hrt - begin_hrt)); 9837c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_MAXSTAT(full_claimed, pages_claimed); 9847c478bd9Sstevel@tonic-gate CHEETAH_LIVELOCK_ENTRY_SET(histp, recovery_time, \ 9857c478bd9Sstevel@tonic-gate (end_hrt - begin_hrt)); 9867c478bd9Sstevel@tonic-gate 987953394f3Sjesusm while (cas32(&sendmondo_in_recover, 1, 0) != 1) 988953394f3Sjesusm ; 9897c478bd9Sstevel@tonic-gate 9907c478bd9Sstevel@tonic-gate return (retval); 9917c478bd9Sstevel@tonic-gate } 9927c478bd9Sstevel@tonic-gate 9937c478bd9Sstevel@tonic-gate /* 9947c478bd9Sstevel@tonic-gate * This is called by the cyclic framework when this CPU becomes online 9957c478bd9Sstevel@tonic-gate */ 9967c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 9977c478bd9Sstevel@tonic-gate static void 9987c478bd9Sstevel@tonic-gate cheetah_nudge_onln(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, cyc_time_t *when) 9997c478bd9Sstevel@tonic-gate { 10007c478bd9Sstevel@tonic-gate 10017c478bd9Sstevel@tonic-gate hdlr->cyh_func = (cyc_func_t)cheetah_nudge_buddy; 10027c478bd9Sstevel@tonic-gate hdlr->cyh_level = CY_LOW_LEVEL; 10037c478bd9Sstevel@tonic-gate hdlr->cyh_arg = NULL; 10047c478bd9Sstevel@tonic-gate 10057c478bd9Sstevel@tonic-gate /* 10067c478bd9Sstevel@tonic-gate * Stagger the start time 10077c478bd9Sstevel@tonic-gate */ 10087c478bd9Sstevel@tonic-gate when->cyt_when = cpu->cpu_id * (NANOSEC / NCPU); 10097c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_recover_delay < CHEETAH_LIVELOCK_MIN_DELAY) { 10107c478bd9Sstevel@tonic-gate cheetah_sendmondo_recover_delay = CHEETAH_LIVELOCK_MIN_DELAY; 10117c478bd9Sstevel@tonic-gate } 10127c478bd9Sstevel@tonic-gate when->cyt_interval = cheetah_sendmondo_recover_delay * NANOSEC; 10137c478bd9Sstevel@tonic-gate } 10147c478bd9Sstevel@tonic-gate 10157c478bd9Sstevel@tonic-gate /* 10167c478bd9Sstevel@tonic-gate * Create a low level cyclic to send a xtrap to the next cpu online. 10177c478bd9Sstevel@tonic-gate * However, there's no need to have this running on a uniprocessor system. 10187c478bd9Sstevel@tonic-gate */ 10197c478bd9Sstevel@tonic-gate static void 10207c478bd9Sstevel@tonic-gate cheetah_nudge_init(void) 10217c478bd9Sstevel@tonic-gate { 10227c478bd9Sstevel@tonic-gate cyc_omni_handler_t hdlr; 10237c478bd9Sstevel@tonic-gate 10247c478bd9Sstevel@tonic-gate if (max_ncpus == 1) { 10257c478bd9Sstevel@tonic-gate return; 10267c478bd9Sstevel@tonic-gate } 10277c478bd9Sstevel@tonic-gate 10287c478bd9Sstevel@tonic-gate hdlr.cyo_online = cheetah_nudge_onln; 10297c478bd9Sstevel@tonic-gate hdlr.cyo_offline = NULL; 10307c478bd9Sstevel@tonic-gate hdlr.cyo_arg = NULL; 10317c478bd9Sstevel@tonic-gate 10327c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 10337c478bd9Sstevel@tonic-gate (void) cyclic_add_omni(&hdlr); 10347c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 10357c478bd9Sstevel@tonic-gate } 10367c478bd9Sstevel@tonic-gate 10377c478bd9Sstevel@tonic-gate /* 10387c478bd9Sstevel@tonic-gate * Cyclic handler to wake up buddy 10397c478bd9Sstevel@tonic-gate */ 10407c478bd9Sstevel@tonic-gate void 10417c478bd9Sstevel@tonic-gate cheetah_nudge_buddy(void) 10427c478bd9Sstevel@tonic-gate { 10437c478bd9Sstevel@tonic-gate /* 10447c478bd9Sstevel@tonic-gate * Disable kernel preemption to protect the cpu list 10457c478bd9Sstevel@tonic-gate */ 10467c478bd9Sstevel@tonic-gate kpreempt_disable(); 10477c478bd9Sstevel@tonic-gate if ((CPU->cpu_next_onln != CPU) && (sendmondo_in_recover == 0)) { 10487c478bd9Sstevel@tonic-gate xt_one(CPU->cpu_next_onln->cpu_id, (xcfunc_t *)xt_sync_tl1, 10497c478bd9Sstevel@tonic-gate 0, 0); 10507c478bd9Sstevel@tonic-gate } 10517c478bd9Sstevel@tonic-gate kpreempt_enable(); 10527c478bd9Sstevel@tonic-gate } 10537c478bd9Sstevel@tonic-gate 10547c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 10557c478bd9Sstevel@tonic-gate 10567c478bd9Sstevel@tonic-gate #ifdef SEND_MONDO_STATS 10577c478bd9Sstevel@tonic-gate uint32_t x_one_stimes[64]; 10587c478bd9Sstevel@tonic-gate uint32_t x_one_ltimes[16]; 10597c478bd9Sstevel@tonic-gate uint32_t x_set_stimes[64]; 10607c478bd9Sstevel@tonic-gate uint32_t x_set_ltimes[16]; 10617c478bd9Sstevel@tonic-gate uint32_t x_set_cpus[NCPU]; 10627c478bd9Sstevel@tonic-gate uint32_t x_nack_stimes[64]; 10637c478bd9Sstevel@tonic-gate #endif 10647c478bd9Sstevel@tonic-gate 10657c478bd9Sstevel@tonic-gate /* 10667c478bd9Sstevel@tonic-gate * Note: A version of this function is used by the debugger via the KDI, 10677c478bd9Sstevel@tonic-gate * and must be kept in sync with this version. Any changes made to this 10687c478bd9Sstevel@tonic-gate * function to support new chips or to accomodate errata must also be included 10697c478bd9Sstevel@tonic-gate * in the KDI-specific version. See us3_kdi.c. 10707c478bd9Sstevel@tonic-gate */ 10717c478bd9Sstevel@tonic-gate void 10727c478bd9Sstevel@tonic-gate send_one_mondo(int cpuid) 10737c478bd9Sstevel@tonic-gate { 10747c478bd9Sstevel@tonic-gate int busy, nack; 10757c478bd9Sstevel@tonic-gate uint64_t idsr, starttick, endtick, tick, lasttick; 10767c478bd9Sstevel@tonic-gate uint64_t busymask; 10777c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 10787c478bd9Sstevel@tonic-gate int recovered = 0; 10797c478bd9Sstevel@tonic-gate #endif 10807c478bd9Sstevel@tonic-gate 10817c478bd9Sstevel@tonic-gate CPU_STATS_ADDQ(CPU, sys, xcalls, 1); 10827c478bd9Sstevel@tonic-gate starttick = lasttick = gettick(); 10837c478bd9Sstevel@tonic-gate shipit(cpuid, 0); 10847c478bd9Sstevel@tonic-gate endtick = starttick + xc_tick_limit; 10857c478bd9Sstevel@tonic-gate busy = nack = 0; 10867c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 10877c478bd9Sstevel@tonic-gate /* 10887c478bd9Sstevel@tonic-gate * Lower 2 bits of the agent ID determine which BUSY/NACK pair 10897c478bd9Sstevel@tonic-gate * will be used for dispatching interrupt. For now, assume 10907c478bd9Sstevel@tonic-gate * there are no more than IDSR_BN_SETS CPUs, hence no aliasing 10917c478bd9Sstevel@tonic-gate * issues with respect to BUSY/NACK pair usage. 10927c478bd9Sstevel@tonic-gate */ 10937c478bd9Sstevel@tonic-gate busymask = IDSR_BUSY_BIT(cpuid); 10947c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 10957c478bd9Sstevel@tonic-gate busymask = IDSR_BUSY; 10967c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 10977c478bd9Sstevel@tonic-gate for (;;) { 10987c478bd9Sstevel@tonic-gate idsr = getidsr(); 10997c478bd9Sstevel@tonic-gate if (idsr == 0) 11007c478bd9Sstevel@tonic-gate break; 11017c478bd9Sstevel@tonic-gate 11027c478bd9Sstevel@tonic-gate tick = gettick(); 11037c478bd9Sstevel@tonic-gate /* 11047c478bd9Sstevel@tonic-gate * If there is a big jump between the current tick 11057c478bd9Sstevel@tonic-gate * count and lasttick, we have probably hit a break 11067c478bd9Sstevel@tonic-gate * point. Adjust endtick accordingly to avoid panic. 11077c478bd9Sstevel@tonic-gate */ 11087c478bd9Sstevel@tonic-gate if (tick > (lasttick + xc_tick_jump_limit)) 11097c478bd9Sstevel@tonic-gate endtick += (tick - lasttick); 11107c478bd9Sstevel@tonic-gate lasttick = tick; 11117c478bd9Sstevel@tonic-gate if (tick > endtick) { 11127c478bd9Sstevel@tonic-gate if (panic_quiesce) 11137c478bd9Sstevel@tonic-gate return; 11147c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 11157c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_recover && recovered == 0) { 11167c478bd9Sstevel@tonic-gate if (mondo_recover(cpuid, 0)) { 11177c478bd9Sstevel@tonic-gate /* 11187c478bd9Sstevel@tonic-gate * We claimed the whole memory or 11197c478bd9Sstevel@tonic-gate * full scan is disabled. 11207c478bd9Sstevel@tonic-gate */ 11217c478bd9Sstevel@tonic-gate recovered++; 11227c478bd9Sstevel@tonic-gate } 11237c478bd9Sstevel@tonic-gate tick = gettick(); 11247c478bd9Sstevel@tonic-gate endtick = tick + xc_tick_limit; 11257c478bd9Sstevel@tonic-gate lasttick = tick; 11267c478bd9Sstevel@tonic-gate /* 11277c478bd9Sstevel@tonic-gate * Recheck idsr 11287c478bd9Sstevel@tonic-gate */ 11297c478bd9Sstevel@tonic-gate continue; 11307c478bd9Sstevel@tonic-gate } else 11317c478bd9Sstevel@tonic-gate #endif /* CHEETAHPLUS_ERRATUM_25 */ 11327c478bd9Sstevel@tonic-gate { 11337c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "send mondo timeout " 11347c478bd9Sstevel@tonic-gate "(target 0x%x) [%d NACK %d BUSY]", 11357c478bd9Sstevel@tonic-gate cpuid, nack, busy); 11367c478bd9Sstevel@tonic-gate } 11377c478bd9Sstevel@tonic-gate } 11387c478bd9Sstevel@tonic-gate 11397c478bd9Sstevel@tonic-gate if (idsr & busymask) { 11407c478bd9Sstevel@tonic-gate busy++; 11417c478bd9Sstevel@tonic-gate continue; 11427c478bd9Sstevel@tonic-gate } 11437c478bd9Sstevel@tonic-gate drv_usecwait(1); 11447c478bd9Sstevel@tonic-gate shipit(cpuid, 0); 11457c478bd9Sstevel@tonic-gate nack++; 11467c478bd9Sstevel@tonic-gate busy = 0; 11477c478bd9Sstevel@tonic-gate } 11487c478bd9Sstevel@tonic-gate #ifdef SEND_MONDO_STATS 11497c478bd9Sstevel@tonic-gate { 11507c478bd9Sstevel@tonic-gate int n = gettick() - starttick; 11517c478bd9Sstevel@tonic-gate if (n < 8192) 11527c478bd9Sstevel@tonic-gate x_one_stimes[n >> 7]++; 11537c478bd9Sstevel@tonic-gate else 11547c478bd9Sstevel@tonic-gate x_one_ltimes[(n >> 13) & 0xf]++; 11557c478bd9Sstevel@tonic-gate } 11567c478bd9Sstevel@tonic-gate #endif 11577c478bd9Sstevel@tonic-gate } 11587c478bd9Sstevel@tonic-gate 11597c478bd9Sstevel@tonic-gate void 11607c478bd9Sstevel@tonic-gate syncfpu(void) 11617c478bd9Sstevel@tonic-gate { 11627c478bd9Sstevel@tonic-gate } 11637c478bd9Sstevel@tonic-gate 11647c478bd9Sstevel@tonic-gate /* 11657c478bd9Sstevel@tonic-gate * Return processor specific async error structure 11667c478bd9Sstevel@tonic-gate * size used. 11677c478bd9Sstevel@tonic-gate */ 11687c478bd9Sstevel@tonic-gate int 11697c478bd9Sstevel@tonic-gate cpu_aflt_size(void) 11707c478bd9Sstevel@tonic-gate { 11717c478bd9Sstevel@tonic-gate return (sizeof (ch_async_flt_t)); 11727c478bd9Sstevel@tonic-gate } 11737c478bd9Sstevel@tonic-gate 1174db7840cdSrscott /* 1175db7840cdSrscott * Tunable to disable the checking of other cpu logout areas during panic for 1176db7840cdSrscott * potential syndrome 71 generating errors. 1177db7840cdSrscott */ 1178db7840cdSrscott int enable_check_other_cpus_logout = 1; 1179db7840cdSrscott 1180db7840cdSrscott /* 1181db7840cdSrscott * Check other cpus logout area for potential synd 71 generating 1182db7840cdSrscott * errors. 1183db7840cdSrscott */ 1184db7840cdSrscott static void 1185db7840cdSrscott cpu_check_cpu_logout(int cpuid, caddr_t tpc, int tl, int ecc_type, 1186db7840cdSrscott ch_cpu_logout_t *clop) 1187db7840cdSrscott { 1188db7840cdSrscott struct async_flt *aflt; 1189db7840cdSrscott ch_async_flt_t ch_flt; 1190db7840cdSrscott uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 1191db7840cdSrscott 1192db7840cdSrscott if (clop == NULL || clop->clo_data.chd_afar == LOGOUT_INVALID) { 1193db7840cdSrscott return; 1194db7840cdSrscott } 1195db7840cdSrscott 1196db7840cdSrscott bzero(&ch_flt, sizeof (ch_async_flt_t)); 1197db7840cdSrscott 1198db7840cdSrscott t_afar = clop->clo_data.chd_afar; 1199db7840cdSrscott t_afsr = clop->clo_data.chd_afsr; 1200db7840cdSrscott t_afsr_ext = clop->clo_data.chd_afsr_ext; 1201db7840cdSrscott #if defined(SERRANO) 1202db7840cdSrscott ch_flt.afar2 = clop->clo_data.chd_afar2; 1203db7840cdSrscott #endif /* SERRANO */ 1204db7840cdSrscott 1205db7840cdSrscott /* 1206db7840cdSrscott * In order to simplify code, we maintain this afsr_errs 1207db7840cdSrscott * variable which holds the aggregate of AFSR and AFSR_EXT 1208db7840cdSrscott * sticky bits. 1209db7840cdSrscott */ 1210db7840cdSrscott t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 1211db7840cdSrscott (t_afsr & C_AFSR_ALL_ERRS); 1212db7840cdSrscott 1213db7840cdSrscott /* Setup the async fault structure */ 1214db7840cdSrscott aflt = (struct async_flt *)&ch_flt; 1215db7840cdSrscott aflt->flt_id = gethrtime_waitfree(); 1216db7840cdSrscott ch_flt.afsr_ext = t_afsr_ext; 1217db7840cdSrscott ch_flt.afsr_errs = t_afsr_errs; 1218db7840cdSrscott aflt->flt_stat = t_afsr; 1219db7840cdSrscott aflt->flt_addr = t_afar; 1220db7840cdSrscott aflt->flt_bus_id = cpuid; 1221db7840cdSrscott aflt->flt_inst = cpuid; 1222db7840cdSrscott aflt->flt_pc = tpc; 1223db7840cdSrscott aflt->flt_prot = AFLT_PROT_NONE; 1224db7840cdSrscott aflt->flt_class = CPU_FAULT; 1225db7840cdSrscott aflt->flt_priv = ((t_afsr & C_AFSR_PRIV) != 0); 1226db7840cdSrscott aflt->flt_tl = tl; 1227db7840cdSrscott aflt->flt_status = ecc_type; 1228db7840cdSrscott aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 1229db7840cdSrscott 1230db7840cdSrscott /* 1231db7840cdSrscott * Queue events on the async event queue, one event per error bit. 1232db7840cdSrscott * If no events are queued, queue an event to complain. 1233db7840cdSrscott */ 1234db7840cdSrscott if (cpu_queue_events(&ch_flt, NULL, t_afsr_errs, clop) == 0) { 1235db7840cdSrscott ch_flt.flt_type = CPU_INV_AFSR; 1236db7840cdSrscott cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 1237db7840cdSrscott (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 1238db7840cdSrscott aflt->flt_panic); 1239db7840cdSrscott } 1240db7840cdSrscott 1241db7840cdSrscott /* 1242db7840cdSrscott * Zero out + invalidate CPU logout. 1243db7840cdSrscott */ 1244db7840cdSrscott bzero(clop, sizeof (ch_cpu_logout_t)); 1245db7840cdSrscott clop->clo_data.chd_afar = LOGOUT_INVALID; 1246db7840cdSrscott } 1247db7840cdSrscott 1248db7840cdSrscott /* 1249db7840cdSrscott * Check the logout areas of all other cpus for unlogged errors. 1250db7840cdSrscott */ 1251db7840cdSrscott static void 1252db7840cdSrscott cpu_check_other_cpus_logout(void) 1253db7840cdSrscott { 1254db7840cdSrscott int i, j; 1255db7840cdSrscott processorid_t myid; 1256db7840cdSrscott struct cpu *cp; 1257db7840cdSrscott ch_err_tl1_data_t *cl1p; 1258db7840cdSrscott 1259db7840cdSrscott myid = CPU->cpu_id; 1260db7840cdSrscott for (i = 0; i < NCPU; i++) { 1261db7840cdSrscott cp = cpu[i]; 1262db7840cdSrscott 1263db7840cdSrscott if ((cp == NULL) || !(cp->cpu_flags & CPU_EXISTS) || 1264db7840cdSrscott (cp->cpu_id == myid) || (CPU_PRIVATE(cp) == NULL)) { 1265db7840cdSrscott continue; 1266db7840cdSrscott } 1267db7840cdSrscott 1268db7840cdSrscott /* 1269db7840cdSrscott * Check each of the tl>0 logout areas 1270db7840cdSrscott */ 1271db7840cdSrscott cl1p = CPU_PRIVATE_PTR(cp, chpr_tl1_err_data[0]); 1272db7840cdSrscott for (j = 0; j < CH_ERR_TL1_TLMAX; j++, cl1p++) { 1273db7840cdSrscott if (cl1p->ch_err_tl1_flags == 0) 1274db7840cdSrscott continue; 1275db7840cdSrscott 1276db7840cdSrscott cpu_check_cpu_logout(i, (caddr_t)cl1p->ch_err_tl1_tpc, 1277db7840cdSrscott 1, ECC_F_TRAP, &cl1p->ch_err_tl1_logout); 1278db7840cdSrscott } 1279db7840cdSrscott 1280db7840cdSrscott /* 1281db7840cdSrscott * Check each of the remaining logout areas 1282db7840cdSrscott */ 1283db7840cdSrscott cpu_check_cpu_logout(i, NULL, 0, ECC_F_TRAP, 1284db7840cdSrscott CPU_PRIVATE_PTR(cp, chpr_fecctl0_logout)); 1285db7840cdSrscott cpu_check_cpu_logout(i, NULL, 0, ECC_C_TRAP, 1286db7840cdSrscott CPU_PRIVATE_PTR(cp, chpr_cecc_logout)); 1287db7840cdSrscott cpu_check_cpu_logout(i, NULL, 0, ECC_D_TRAP, 1288db7840cdSrscott CPU_PRIVATE_PTR(cp, chpr_async_logout)); 1289db7840cdSrscott } 1290db7840cdSrscott } 1291db7840cdSrscott 12927c478bd9Sstevel@tonic-gate /* 12937c478bd9Sstevel@tonic-gate * The fast_ecc_err handler transfers control here for UCU, UCC events. 12947c478bd9Sstevel@tonic-gate * Note that we flush Ecache twice, once in the fast_ecc_err handler to 12957c478bd9Sstevel@tonic-gate * flush the error that caused the UCU/UCC, then again here at the end to 12967c478bd9Sstevel@tonic-gate * flush the TL=1 trap handler code out of the Ecache, so we can minimize 12977c478bd9Sstevel@tonic-gate * the probability of getting a TL>1 Fast ECC trap when we're fielding 12987c478bd9Sstevel@tonic-gate * another Fast ECC trap. 12997c478bd9Sstevel@tonic-gate * 13007c478bd9Sstevel@tonic-gate * Cheetah+ also handles: TSCE: No additional processing required. 13017c478bd9Sstevel@tonic-gate * Panther adds L3_UCU and L3_UCC which are reported in AFSR_EXT. 13027c478bd9Sstevel@tonic-gate * 13037c478bd9Sstevel@tonic-gate * Note that the p_clo_flags input is only valid in cases where the 13047c478bd9Sstevel@tonic-gate * cpu_private struct is not yet initialized (since that is the only 13057c478bd9Sstevel@tonic-gate * time that information cannot be obtained from the logout struct.) 13067c478bd9Sstevel@tonic-gate */ 13077c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 13087c478bd9Sstevel@tonic-gate void 13097c478bd9Sstevel@tonic-gate cpu_fast_ecc_error(struct regs *rp, ulong_t p_clo_flags) 13107c478bd9Sstevel@tonic-gate { 13117c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 13124fd7ecabSdilpreet uint64_t ceen, nceen; 13137c478bd9Sstevel@tonic-gate 13147c478bd9Sstevel@tonic-gate /* 13157c478bd9Sstevel@tonic-gate * Get the CPU log out info. If we can't find our CPU private 13167c478bd9Sstevel@tonic-gate * pointer, then we will have to make due without any detailed 13177c478bd9Sstevel@tonic-gate * logout information. 13187c478bd9Sstevel@tonic-gate */ 13197c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 13207c478bd9Sstevel@tonic-gate clop = NULL; 13217c478bd9Sstevel@tonic-gate ceen = p_clo_flags & EN_REG_CEEN; 13224fd7ecabSdilpreet nceen = p_clo_flags & EN_REG_NCEEN; 13237c478bd9Sstevel@tonic-gate } else { 13247c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_fecctl0_logout); 13257c478bd9Sstevel@tonic-gate ceen = clop->clo_flags & EN_REG_CEEN; 13264fd7ecabSdilpreet nceen = clop->clo_flags & EN_REG_NCEEN; 13277c478bd9Sstevel@tonic-gate } 13287c478bd9Sstevel@tonic-gate 13297c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error((caddr_t)rp->r_pc, 13304fd7ecabSdilpreet (rp->r_tstate & TSTATE_PRIV) ? 1 : 0, 0, ceen, nceen, clop); 13317c478bd9Sstevel@tonic-gate } 13327c478bd9Sstevel@tonic-gate 13337c478bd9Sstevel@tonic-gate /* 13347c478bd9Sstevel@tonic-gate * Log fast ecc error, called from either Fast ECC at TL=0 or Fast 13357c478bd9Sstevel@tonic-gate * ECC at TL>0. Need to supply either a error register pointer or a 13367c478bd9Sstevel@tonic-gate * cpu logout structure pointer. 13377c478bd9Sstevel@tonic-gate */ 13387c478bd9Sstevel@tonic-gate static void 13397c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error(caddr_t tpc, int priv, int tl, uint64_t ceen, 13404fd7ecabSdilpreet uint64_t nceen, ch_cpu_logout_t *clop) 13417c478bd9Sstevel@tonic-gate { 13427c478bd9Sstevel@tonic-gate struct async_flt *aflt; 13437c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 13447c478bd9Sstevel@tonic-gate uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 13457c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 13467c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 13477c478bd9Sstevel@tonic-gate 13487c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 13497c478bd9Sstevel@tonic-gate /* 13507c478bd9Sstevel@tonic-gate * If no cpu logout data, then we will have to make due without 13517c478bd9Sstevel@tonic-gate * any detailed logout information. 13527c478bd9Sstevel@tonic-gate */ 13537c478bd9Sstevel@tonic-gate if (clop == NULL) { 13547c478bd9Sstevel@tonic-gate ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 13557c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 13567c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 13577c478bd9Sstevel@tonic-gate t_afar = cpu_error_regs.afar; 13587c478bd9Sstevel@tonic-gate t_afsr = cpu_error_regs.afsr; 13597c478bd9Sstevel@tonic-gate t_afsr_ext = cpu_error_regs.afsr_ext; 13607c478bd9Sstevel@tonic-gate #if defined(SERRANO) 13617c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 13627c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 13637c478bd9Sstevel@tonic-gate } else { 13647c478bd9Sstevel@tonic-gate t_afar = clop->clo_data.chd_afar; 13657c478bd9Sstevel@tonic-gate t_afsr = clop->clo_data.chd_afsr; 13667c478bd9Sstevel@tonic-gate t_afsr_ext = clop->clo_data.chd_afsr_ext; 13677c478bd9Sstevel@tonic-gate #if defined(SERRANO) 13687c478bd9Sstevel@tonic-gate ch_flt.afar2 = clop->clo_data.chd_afar2; 13697c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 13707c478bd9Sstevel@tonic-gate } 13717c478bd9Sstevel@tonic-gate 13727c478bd9Sstevel@tonic-gate /* 13737c478bd9Sstevel@tonic-gate * In order to simplify code, we maintain this afsr_errs 13747c478bd9Sstevel@tonic-gate * variable which holds the aggregate of AFSR and AFSR_EXT 13757c478bd9Sstevel@tonic-gate * sticky bits. 13767c478bd9Sstevel@tonic-gate */ 13777c478bd9Sstevel@tonic-gate t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 13787c478bd9Sstevel@tonic-gate (t_afsr & C_AFSR_ALL_ERRS); 13797c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 13807c478bd9Sstevel@tonic-gate 13817c478bd9Sstevel@tonic-gate /* Setup the async fault structure */ 13827c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 13837c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 13847c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = t_afsr_ext; 13857c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = t_afsr_errs; 13867c478bd9Sstevel@tonic-gate aflt->flt_stat = t_afsr; 13877c478bd9Sstevel@tonic-gate aflt->flt_addr = t_afar; 13887c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 13897c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 13907c478bd9Sstevel@tonic-gate aflt->flt_pc = tpc; 13917c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 13927c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 13937c478bd9Sstevel@tonic-gate aflt->flt_priv = priv; 13947c478bd9Sstevel@tonic-gate aflt->flt_tl = tl; 13957c478bd9Sstevel@tonic-gate aflt->flt_status = ECC_F_TRAP; 13967c478bd9Sstevel@tonic-gate aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 13977c478bd9Sstevel@tonic-gate 13987c478bd9Sstevel@tonic-gate /* 13997c478bd9Sstevel@tonic-gate * XXXX - Phenomenal hack to get around Solaris not getting all the 14007c478bd9Sstevel@tonic-gate * cmn_err messages out to the console. The situation is a UCU (in 14017c478bd9Sstevel@tonic-gate * priv mode) which causes a WDU which causes a UE (on the retry). 14027c478bd9Sstevel@tonic-gate * The messages for the UCU and WDU are enqueued and then pulled off 14037c478bd9Sstevel@tonic-gate * the async queue via softint and syslogd starts to process them 14047c478bd9Sstevel@tonic-gate * but doesn't get them to the console. The UE causes a panic, but 14057c478bd9Sstevel@tonic-gate * since the UCU/WDU messages are already in transit, those aren't 14067c478bd9Sstevel@tonic-gate * on the async queue. The hack is to check if we have a matching 14077c478bd9Sstevel@tonic-gate * WDU event for the UCU, and if it matches, we're more than likely 14087c478bd9Sstevel@tonic-gate * going to panic with a UE, unless we're under protection. So, we 14097c478bd9Sstevel@tonic-gate * check to see if we got a matching WDU event and if we're under 14107c478bd9Sstevel@tonic-gate * protection. 14117c478bd9Sstevel@tonic-gate * 14127c478bd9Sstevel@tonic-gate * For Cheetah/Cheetah+/Jaguar/Jalapeno, the sequence we care about 14137c478bd9Sstevel@tonic-gate * looks like this: 14147c478bd9Sstevel@tonic-gate * UCU->WDU->UE 14157c478bd9Sstevel@tonic-gate * For Panther, it could look like either of these: 14167c478bd9Sstevel@tonic-gate * UCU---->WDU->L3_WDU->UE 14177c478bd9Sstevel@tonic-gate * L3_UCU->WDU->L3_WDU->UE 14187c478bd9Sstevel@tonic-gate */ 14197c478bd9Sstevel@tonic-gate if ((t_afsr_errs & (C_AFSR_UCU | C_AFSR_L3_UCU)) && 14207c478bd9Sstevel@tonic-gate aflt->flt_panic == 0 && aflt->flt_priv != 0 && 14217c478bd9Sstevel@tonic-gate curthread->t_ontrap == NULL && curthread->t_lofault == NULL) { 14227c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 14239ed9f144Skwmc if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 14249ed9f144Skwmc aflt->flt_panic |= 14259ed9f144Skwmc ((cpu_error_regs.afsr & C_AFSR_WDU) && 14269ed9f144Skwmc (cpu_error_regs.afsr_ext & C_AFSR_L3_WDU) && 14279ed9f144Skwmc (cpu_error_regs.afar == t_afar)); 14289ed9f144Skwmc aflt->flt_panic |= ((clop == NULL) && 14299ed9f144Skwmc (t_afsr_errs & C_AFSR_WDU) && 14309ed9f144Skwmc (t_afsr_errs & C_AFSR_L3_WDU)); 14319ed9f144Skwmc } else { 14329ed9f144Skwmc aflt->flt_panic |= 14339ed9f144Skwmc ((cpu_error_regs.afsr & C_AFSR_WDU) && 14349ed9f144Skwmc (cpu_error_regs.afar == t_afar)); 14359ed9f144Skwmc aflt->flt_panic |= ((clop == NULL) && 14369ed9f144Skwmc (t_afsr_errs & C_AFSR_WDU)); 14379ed9f144Skwmc } 14387c478bd9Sstevel@tonic-gate } 14397c478bd9Sstevel@tonic-gate 14407c478bd9Sstevel@tonic-gate /* 14417c478bd9Sstevel@tonic-gate * Queue events on the async event queue, one event per error bit. 14427c478bd9Sstevel@tonic-gate * If no events are queued or no Fast ECC events are on in the AFSR, 14437c478bd9Sstevel@tonic-gate * queue an event to complain. 14447c478bd9Sstevel@tonic-gate */ 14457c478bd9Sstevel@tonic-gate if (cpu_queue_events(&ch_flt, pr_reason, t_afsr_errs, clop) == 0 || 14467c478bd9Sstevel@tonic-gate ((t_afsr_errs & (C_AFSR_FECC_ERRS | C_AFSR_EXT_FECC_ERRS)) == 0)) { 14477c478bd9Sstevel@tonic-gate ch_flt.flt_type = CPU_INV_AFSR; 14487c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 14497c478bd9Sstevel@tonic-gate (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 14507c478bd9Sstevel@tonic-gate aflt->flt_panic); 14517c478bd9Sstevel@tonic-gate } 14527c478bd9Sstevel@tonic-gate 14537c478bd9Sstevel@tonic-gate /* 14547c478bd9Sstevel@tonic-gate * Zero out + invalidate CPU logout. 14557c478bd9Sstevel@tonic-gate */ 14567c478bd9Sstevel@tonic-gate if (clop) { 14577c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 14587c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 14597c478bd9Sstevel@tonic-gate } 14607c478bd9Sstevel@tonic-gate 14617c478bd9Sstevel@tonic-gate /* 14627c478bd9Sstevel@tonic-gate * We carefully re-enable NCEEN and CEEN and then check if any deferred 14637c478bd9Sstevel@tonic-gate * or disrupting errors have happened. We do this because if a 14647c478bd9Sstevel@tonic-gate * deferred or disrupting error had occurred with NCEEN/CEEN off, the 14657c478bd9Sstevel@tonic-gate * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 14667c478bd9Sstevel@tonic-gate * CEEN works differently on Cheetah than on Spitfire. Also, we enable 14677c478bd9Sstevel@tonic-gate * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 14687c478bd9Sstevel@tonic-gate * deferred or disrupting error happening between checking the AFSR and 14697c478bd9Sstevel@tonic-gate * enabling NCEEN/CEEN. 14707c478bd9Sstevel@tonic-gate * 14714fd7ecabSdilpreet * Note: CEEN and NCEEN are only reenabled if they were on when trap 14724fd7ecabSdilpreet * taken. 14737c478bd9Sstevel@tonic-gate */ 14744fd7ecabSdilpreet set_error_enable(get_error_enable() | (nceen | ceen)); 14757c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 14767c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((ch_flt.afsr_errs & 14777c478bd9Sstevel@tonic-gate (C_AFSR_EXT_ASYNC_ERRS | C_AFSR_ASYNC_ERRS)) != 0); 14787c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 14797c478bd9Sstevel@tonic-gate NULL); 14807c478bd9Sstevel@tonic-gate } 14817c478bd9Sstevel@tonic-gate 14827c478bd9Sstevel@tonic-gate /* 14837c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 14847c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 14857c478bd9Sstevel@tonic-gate */ 14867c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 14877c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", pr_reason); 14887c478bd9Sstevel@tonic-gate 14897c478bd9Sstevel@tonic-gate /* 14907c478bd9Sstevel@tonic-gate * Flushing the Ecache here gets the part of the trap handler that 14917c478bd9Sstevel@tonic-gate * is run at TL=1 out of the Ecache. 14927c478bd9Sstevel@tonic-gate */ 14937c478bd9Sstevel@tonic-gate cpu_flush_ecache(); 14947c478bd9Sstevel@tonic-gate } 14957c478bd9Sstevel@tonic-gate 14967c478bd9Sstevel@tonic-gate /* 14977c478bd9Sstevel@tonic-gate * This is called via sys_trap from pil15_interrupt code if the 14987c478bd9Sstevel@tonic-gate * corresponding entry in ch_err_tl1_pending is set. Checks the 14997c478bd9Sstevel@tonic-gate * various ch_err_tl1_data structures for valid entries based on the bit 15007c478bd9Sstevel@tonic-gate * settings in the ch_err_tl1_flags entry of the structure. 15017c478bd9Sstevel@tonic-gate */ 15027c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 15037c478bd9Sstevel@tonic-gate void 15047c478bd9Sstevel@tonic-gate cpu_tl1_error(struct regs *rp, int panic) 15057c478bd9Sstevel@tonic-gate { 15067c478bd9Sstevel@tonic-gate ch_err_tl1_data_t *cl1p, cl1; 15077c478bd9Sstevel@tonic-gate int i, ncl1ps; 15087c478bd9Sstevel@tonic-gate uint64_t me_flags; 15094fd7ecabSdilpreet uint64_t ceen, nceen; 15107c478bd9Sstevel@tonic-gate 15117c478bd9Sstevel@tonic-gate if (ch_err_tl1_paddrs[CPU->cpu_id] == 0) { 15127c478bd9Sstevel@tonic-gate cl1p = &ch_err_tl1_data; 15137c478bd9Sstevel@tonic-gate ncl1ps = 1; 15147c478bd9Sstevel@tonic-gate } else if (CPU_PRIVATE(CPU) != NULL) { 15157c478bd9Sstevel@tonic-gate cl1p = CPU_PRIVATE_PTR(CPU, chpr_tl1_err_data[0]); 15167c478bd9Sstevel@tonic-gate ncl1ps = CH_ERR_TL1_TLMAX; 15177c478bd9Sstevel@tonic-gate } else { 15187c478bd9Sstevel@tonic-gate ncl1ps = 0; 15197c478bd9Sstevel@tonic-gate } 15207c478bd9Sstevel@tonic-gate 15217c478bd9Sstevel@tonic-gate for (i = 0; i < ncl1ps; i++, cl1p++) { 15227c478bd9Sstevel@tonic-gate if (cl1p->ch_err_tl1_flags == 0) 15237c478bd9Sstevel@tonic-gate continue; 15247c478bd9Sstevel@tonic-gate 15257c478bd9Sstevel@tonic-gate /* 15267c478bd9Sstevel@tonic-gate * Grab a copy of the logout data and invalidate 15277c478bd9Sstevel@tonic-gate * the logout area. 15287c478bd9Sstevel@tonic-gate */ 15297c478bd9Sstevel@tonic-gate cl1 = *cl1p; 15307c478bd9Sstevel@tonic-gate bzero(cl1p, sizeof (ch_err_tl1_data_t)); 15317c478bd9Sstevel@tonic-gate cl1p->ch_err_tl1_logout.clo_data.chd_afar = LOGOUT_INVALID; 15327c478bd9Sstevel@tonic-gate me_flags = CH_ERR_ME_FLAGS(cl1.ch_err_tl1_flags); 15337c478bd9Sstevel@tonic-gate 15347c478bd9Sstevel@tonic-gate /* 15357c478bd9Sstevel@tonic-gate * Log "first error" in ch_err_tl1_data. 15367c478bd9Sstevel@tonic-gate */ 15377c478bd9Sstevel@tonic-gate if (cl1.ch_err_tl1_flags & CH_ERR_FECC) { 15387c478bd9Sstevel@tonic-gate ceen = get_error_enable() & EN_REG_CEEN; 15394fd7ecabSdilpreet nceen = get_error_enable() & EN_REG_NCEEN; 15407c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error((caddr_t)cl1.ch_err_tl1_tpc, 1, 15414fd7ecabSdilpreet 1, ceen, nceen, &cl1.ch_err_tl1_logout); 15427c478bd9Sstevel@tonic-gate } 15437c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 15447c478bd9Sstevel@tonic-gate if (cl1.ch_err_tl1_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 15457c478bd9Sstevel@tonic-gate cpu_parity_error(rp, cl1.ch_err_tl1_flags, 15467c478bd9Sstevel@tonic-gate (caddr_t)cl1.ch_err_tl1_tpc); 15477c478bd9Sstevel@tonic-gate } 15487c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 15497c478bd9Sstevel@tonic-gate 15507c478bd9Sstevel@tonic-gate /* 15517c478bd9Sstevel@tonic-gate * Log "multiple events" in ch_err_tl1_data. Note that 15527c478bd9Sstevel@tonic-gate * we don't read and clear the AFSR/AFAR in the TL>0 code 15537c478bd9Sstevel@tonic-gate * if the structure is busy, we just do the cache flushing 15547c478bd9Sstevel@tonic-gate * we have to do and then do the retry. So the AFSR/AFAR 15557c478bd9Sstevel@tonic-gate * at this point *should* have some relevant info. If there 15567c478bd9Sstevel@tonic-gate * are no valid errors in the AFSR, we'll assume they've 15577c478bd9Sstevel@tonic-gate * already been picked up and logged. For I$/D$ parity, 15587c478bd9Sstevel@tonic-gate * we just log an event with an "Unknown" (NULL) TPC. 15597c478bd9Sstevel@tonic-gate */ 15607c478bd9Sstevel@tonic-gate if (me_flags & CH_ERR_FECC) { 15617c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 15627c478bd9Sstevel@tonic-gate uint64_t t_afsr_errs; 15637c478bd9Sstevel@tonic-gate 15647c478bd9Sstevel@tonic-gate /* 15657c478bd9Sstevel@tonic-gate * Get the error registers and see if there's 15667c478bd9Sstevel@tonic-gate * a pending error. If not, don't bother 15677c478bd9Sstevel@tonic-gate * generating an "Invalid AFSR" error event. 15687c478bd9Sstevel@tonic-gate */ 15697c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 15707c478bd9Sstevel@tonic-gate t_afsr_errs = (cpu_error_regs.afsr_ext & 15717c478bd9Sstevel@tonic-gate C_AFSR_EXT_ALL_ERRS) | 15727c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr & C_AFSR_ALL_ERRS); 15737c478bd9Sstevel@tonic-gate if (t_afsr_errs != 0) { 15747c478bd9Sstevel@tonic-gate ceen = get_error_enable() & EN_REG_CEEN; 15754fd7ecabSdilpreet nceen = get_error_enable() & EN_REG_NCEEN; 15767c478bd9Sstevel@tonic-gate cpu_log_fast_ecc_error((caddr_t)NULL, 1, 15774fd7ecabSdilpreet 1, ceen, nceen, NULL); 15787c478bd9Sstevel@tonic-gate } 15797c478bd9Sstevel@tonic-gate } 15807c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 15817c478bd9Sstevel@tonic-gate if (me_flags & (CH_ERR_IPE | CH_ERR_DPE)) { 15827c478bd9Sstevel@tonic-gate cpu_parity_error(rp, me_flags, (caddr_t)NULL); 15837c478bd9Sstevel@tonic-gate } 15847c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 15857c478bd9Sstevel@tonic-gate } 15867c478bd9Sstevel@tonic-gate } 15877c478bd9Sstevel@tonic-gate 15887c478bd9Sstevel@tonic-gate /* 15897c478bd9Sstevel@tonic-gate * Called from Fast ECC TL>0 handler in case of fatal error. 15907c478bd9Sstevel@tonic-gate * cpu_tl1_error should always find an associated ch_err_tl1_data structure, 15917c478bd9Sstevel@tonic-gate * but if we don't, we'll panic with something reasonable. 15927c478bd9Sstevel@tonic-gate */ 15937c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 15947c478bd9Sstevel@tonic-gate void 15957c478bd9Sstevel@tonic-gate cpu_tl1_err_panic(struct regs *rp, ulong_t flags) 15967c478bd9Sstevel@tonic-gate { 15977c478bd9Sstevel@tonic-gate cpu_tl1_error(rp, 1); 15987c478bd9Sstevel@tonic-gate /* 15997c478bd9Sstevel@tonic-gate * Should never return, but just in case. 16007c478bd9Sstevel@tonic-gate */ 16017c478bd9Sstevel@tonic-gate fm_panic("Unsurvivable ECC Error at TL>0"); 16027c478bd9Sstevel@tonic-gate } 16037c478bd9Sstevel@tonic-gate 16047c478bd9Sstevel@tonic-gate /* 16057c478bd9Sstevel@tonic-gate * The ce_err/ce_err_tl1 handlers transfer control here for CE, EMC, EDU:ST, 16067c478bd9Sstevel@tonic-gate * EDC, WDU, WDC, CPU, CPC, IVU, IVC events. 16077c478bd9Sstevel@tonic-gate * Disrupting errors controlled by NCEEN: EDU:ST, WDU, CPU, IVU 16087c478bd9Sstevel@tonic-gate * Disrupting errors controlled by CEEN: CE, EMC, EDC, WDC, CPC, IVC 16097c478bd9Sstevel@tonic-gate * 16107c478bd9Sstevel@tonic-gate * Cheetah+ also handles (No additional processing required): 16117c478bd9Sstevel@tonic-gate * DUE, DTO, DBERR (NCEEN controlled) 16127c478bd9Sstevel@tonic-gate * THCE (CEEN and ET_ECC_en controlled) 16137c478bd9Sstevel@tonic-gate * TUE (ET_ECC_en controlled) 16147c478bd9Sstevel@tonic-gate * 16157c478bd9Sstevel@tonic-gate * Panther further adds: 16167c478bd9Sstevel@tonic-gate * IMU, L3_EDU, L3_WDU, L3_CPU (NCEEN controlled) 16177c478bd9Sstevel@tonic-gate * IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE (CEEN controlled) 16187c478bd9Sstevel@tonic-gate * TUE_SH, TUE (NCEEN and L2_tag_ECC_en controlled) 16197c478bd9Sstevel@tonic-gate * L3_TUE, L3_TUE_SH (NCEEN and ET_ECC_en controlled) 16207c478bd9Sstevel@tonic-gate * THCE (CEEN and L2_tag_ECC_en controlled) 16217c478bd9Sstevel@tonic-gate * L3_THCE (CEEN and ET_ECC_en controlled) 16227c478bd9Sstevel@tonic-gate * 16237c478bd9Sstevel@tonic-gate * Note that the p_clo_flags input is only valid in cases where the 16247c478bd9Sstevel@tonic-gate * cpu_private struct is not yet initialized (since that is the only 16257c478bd9Sstevel@tonic-gate * time that information cannot be obtained from the logout struct.) 16267c478bd9Sstevel@tonic-gate */ 16277c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 16287c478bd9Sstevel@tonic-gate void 16297c478bd9Sstevel@tonic-gate cpu_disrupting_error(struct regs *rp, ulong_t p_clo_flags) 16307c478bd9Sstevel@tonic-gate { 16317c478bd9Sstevel@tonic-gate struct async_flt *aflt; 16327c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 16337c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 16347c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 16357c478bd9Sstevel@tonic-gate uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 16367c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 16377c478bd9Sstevel@tonic-gate 16387c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 16397c478bd9Sstevel@tonic-gate /* 16407c478bd9Sstevel@tonic-gate * Get the CPU log out info. If we can't find our CPU private 16417c478bd9Sstevel@tonic-gate * pointer, then we will have to make due without any detailed 16427c478bd9Sstevel@tonic-gate * logout information. 16437c478bd9Sstevel@tonic-gate */ 16447c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 16457c478bd9Sstevel@tonic-gate clop = NULL; 16467c478bd9Sstevel@tonic-gate ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 16477c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 16487c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 16497c478bd9Sstevel@tonic-gate t_afar = cpu_error_regs.afar; 16507c478bd9Sstevel@tonic-gate t_afsr = cpu_error_regs.afsr; 16517c478bd9Sstevel@tonic-gate t_afsr_ext = cpu_error_regs.afsr_ext; 16527c478bd9Sstevel@tonic-gate #if defined(SERRANO) 16537c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 16547c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 16557c478bd9Sstevel@tonic-gate } else { 16567c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 16577c478bd9Sstevel@tonic-gate t_afar = clop->clo_data.chd_afar; 16587c478bd9Sstevel@tonic-gate t_afsr = clop->clo_data.chd_afsr; 16597c478bd9Sstevel@tonic-gate t_afsr_ext = clop->clo_data.chd_afsr_ext; 16607c478bd9Sstevel@tonic-gate #if defined(SERRANO) 16617c478bd9Sstevel@tonic-gate ch_flt.afar2 = clop->clo_data.chd_afar2; 16627c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 16637c478bd9Sstevel@tonic-gate } 16647c478bd9Sstevel@tonic-gate 16657c478bd9Sstevel@tonic-gate /* 16667c478bd9Sstevel@tonic-gate * In order to simplify code, we maintain this afsr_errs 16677c478bd9Sstevel@tonic-gate * variable which holds the aggregate of AFSR and AFSR_EXT 16687c478bd9Sstevel@tonic-gate * sticky bits. 16697c478bd9Sstevel@tonic-gate */ 16707c478bd9Sstevel@tonic-gate t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 16717c478bd9Sstevel@tonic-gate (t_afsr & C_AFSR_ALL_ERRS); 16727c478bd9Sstevel@tonic-gate 16737c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 16747c478bd9Sstevel@tonic-gate /* Setup the async fault structure */ 16757c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 16767c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = t_afsr_ext; 16777c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = t_afsr_errs; 16787c478bd9Sstevel@tonic-gate aflt->flt_stat = t_afsr; 16797c478bd9Sstevel@tonic-gate aflt->flt_addr = t_afar; 16807c478bd9Sstevel@tonic-gate aflt->flt_pc = (caddr_t)rp->r_pc; 16817c478bd9Sstevel@tonic-gate aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 16827c478bd9Sstevel@tonic-gate aflt->flt_tl = 0; 16837c478bd9Sstevel@tonic-gate aflt->flt_panic = C_AFSR_PANIC(t_afsr_errs); 16847c478bd9Sstevel@tonic-gate 16857c478bd9Sstevel@tonic-gate /* 16867c478bd9Sstevel@tonic-gate * If this trap is a result of one of the errors not masked 16877c478bd9Sstevel@tonic-gate * by cpu_ce_not_deferred, we don't reenable CEEN. Instead 16887c478bd9Sstevel@tonic-gate * indicate that a timeout is to be set later. 16897c478bd9Sstevel@tonic-gate */ 16907c478bd9Sstevel@tonic-gate if (!(t_afsr_errs & (cpu_ce_not_deferred | cpu_ce_not_deferred_ext)) && 16917c478bd9Sstevel@tonic-gate !aflt->flt_panic) 16927c478bd9Sstevel@tonic-gate ch_flt.flt_trapped_ce = CE_CEEN_DEFER | CE_CEEN_TRAPPED; 16937c478bd9Sstevel@tonic-gate else 16947c478bd9Sstevel@tonic-gate ch_flt.flt_trapped_ce = CE_CEEN_NODEFER | CE_CEEN_TRAPPED; 16957c478bd9Sstevel@tonic-gate 16967c478bd9Sstevel@tonic-gate /* 16977c478bd9Sstevel@tonic-gate * log the CE and clean up 16987c478bd9Sstevel@tonic-gate */ 16997c478bd9Sstevel@tonic-gate cpu_log_and_clear_ce(&ch_flt); 17007c478bd9Sstevel@tonic-gate 17017c478bd9Sstevel@tonic-gate /* 17027c478bd9Sstevel@tonic-gate * We re-enable CEEN (if required) and check if any disrupting errors 17037c478bd9Sstevel@tonic-gate * have happened. We do this because if a disrupting error had occurred 17047c478bd9Sstevel@tonic-gate * with CEEN off, the trap will not be taken when CEEN is re-enabled. 17057c478bd9Sstevel@tonic-gate * Note that CEEN works differently on Cheetah than on Spitfire. Also, 17067c478bd9Sstevel@tonic-gate * we enable CEEN *before* checking the AFSR to avoid the small window 17077c478bd9Sstevel@tonic-gate * of a error happening between checking the AFSR and enabling CEEN. 17087c478bd9Sstevel@tonic-gate */ 17097c478bd9Sstevel@tonic-gate if (ch_flt.flt_trapped_ce & CE_CEEN_NODEFER) 1710cbaac45eSkm set_error_enable(get_error_enable() | EN_REG_CEEN); 17117c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 17127c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 17137c478bd9Sstevel@tonic-gate NULL); 17147c478bd9Sstevel@tonic-gate } 17157c478bd9Sstevel@tonic-gate 17167c478bd9Sstevel@tonic-gate /* 17177c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 17187c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 17197c478bd9Sstevel@tonic-gate */ 17207c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 17217c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", pr_reason); 17227c478bd9Sstevel@tonic-gate } 17237c478bd9Sstevel@tonic-gate 17247c478bd9Sstevel@tonic-gate /* 17257c478bd9Sstevel@tonic-gate * The async_err handler transfers control here for UE, EMU, EDU:BLD, 17267c478bd9Sstevel@tonic-gate * L3_EDU:BLD, TO, and BERR events. 17277c478bd9Sstevel@tonic-gate * Deferred errors controlled by NCEEN: UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR 17287c478bd9Sstevel@tonic-gate * 17297c478bd9Sstevel@tonic-gate * Cheetah+: No additional errors handled. 17307c478bd9Sstevel@tonic-gate * 17317c478bd9Sstevel@tonic-gate * Note that the p_clo_flags input is only valid in cases where the 17327c478bd9Sstevel@tonic-gate * cpu_private struct is not yet initialized (since that is the only 17337c478bd9Sstevel@tonic-gate * time that information cannot be obtained from the logout struct.) 17347c478bd9Sstevel@tonic-gate */ 17357c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 17367c478bd9Sstevel@tonic-gate void 17377c478bd9Sstevel@tonic-gate cpu_deferred_error(struct regs *rp, ulong_t p_clo_flags) 17387c478bd9Sstevel@tonic-gate { 17397c478bd9Sstevel@tonic-gate ushort_t ttype, tl; 17407c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 17417c478bd9Sstevel@tonic-gate struct async_flt *aflt; 17427c478bd9Sstevel@tonic-gate int trampolined = 0; 17437c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 17447c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 17457c478bd9Sstevel@tonic-gate uint64_t ceen, clo_flags; 17467c478bd9Sstevel@tonic-gate uint64_t log_afsr; 17477c478bd9Sstevel@tonic-gate uint64_t t_afar, t_afsr, t_afsr_ext, t_afsr_errs; 17487c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 17497c478bd9Sstevel@tonic-gate int expected = DDI_FM_ERR_UNEXPECTED; 17507c478bd9Sstevel@tonic-gate ddi_acc_hdl_t *hp; 17517c478bd9Sstevel@tonic-gate 17527c478bd9Sstevel@tonic-gate /* 17537c478bd9Sstevel@tonic-gate * We need to look at p_flag to determine if the thread detected an 17547c478bd9Sstevel@tonic-gate * error while dumping core. We can't grab p_lock here, but it's ok 17557c478bd9Sstevel@tonic-gate * because we just need a consistent snapshot and we know that everyone 17567c478bd9Sstevel@tonic-gate * else will store a consistent set of bits while holding p_lock. We 17577c478bd9Sstevel@tonic-gate * don't have to worry about a race because SDOCORE is set once prior 17587c478bd9Sstevel@tonic-gate * to doing i/o from the process's address space and is never cleared. 17597c478bd9Sstevel@tonic-gate */ 17607c478bd9Sstevel@tonic-gate uint_t pflag = ttoproc(curthread)->p_flag; 17617c478bd9Sstevel@tonic-gate 17627c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 17637c478bd9Sstevel@tonic-gate /* 17647c478bd9Sstevel@tonic-gate * Get the CPU log out info. If we can't find our CPU private 17657c478bd9Sstevel@tonic-gate * pointer then we will have to make due without any detailed 17667c478bd9Sstevel@tonic-gate * logout information. 17677c478bd9Sstevel@tonic-gate */ 17687c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 17697c478bd9Sstevel@tonic-gate clop = NULL; 17707c478bd9Sstevel@tonic-gate ch_flt.flt_diag_data.chd_afar = LOGOUT_INVALID; 17717c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 17727c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 17737c478bd9Sstevel@tonic-gate t_afar = cpu_error_regs.afar; 17747c478bd9Sstevel@tonic-gate t_afsr = cpu_error_regs.afsr; 17757c478bd9Sstevel@tonic-gate t_afsr_ext = cpu_error_regs.afsr_ext; 17767c478bd9Sstevel@tonic-gate #if defined(SERRANO) 17777c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 17787c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 17797c478bd9Sstevel@tonic-gate clo_flags = p_clo_flags; 17807c478bd9Sstevel@tonic-gate } else { 17817c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_async_logout); 17827c478bd9Sstevel@tonic-gate t_afar = clop->clo_data.chd_afar; 17837c478bd9Sstevel@tonic-gate t_afsr = clop->clo_data.chd_afsr; 17847c478bd9Sstevel@tonic-gate t_afsr_ext = clop->clo_data.chd_afsr_ext; 17857c478bd9Sstevel@tonic-gate #if defined(SERRANO) 17867c478bd9Sstevel@tonic-gate ch_flt.afar2 = clop->clo_data.chd_afar2; 17877c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 17887c478bd9Sstevel@tonic-gate clo_flags = clop->clo_flags; 17897c478bd9Sstevel@tonic-gate } 17907c478bd9Sstevel@tonic-gate 17917c478bd9Sstevel@tonic-gate /* 17927c478bd9Sstevel@tonic-gate * In order to simplify code, we maintain this afsr_errs 17937c478bd9Sstevel@tonic-gate * variable which holds the aggregate of AFSR and AFSR_EXT 17947c478bd9Sstevel@tonic-gate * sticky bits. 17957c478bd9Sstevel@tonic-gate */ 17967c478bd9Sstevel@tonic-gate t_afsr_errs = (t_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 17977c478bd9Sstevel@tonic-gate (t_afsr & C_AFSR_ALL_ERRS); 17987c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 17997c478bd9Sstevel@tonic-gate 18007c478bd9Sstevel@tonic-gate /* 18017c478bd9Sstevel@tonic-gate * Grab information encoded into our clo_flags field. 18027c478bd9Sstevel@tonic-gate */ 18037c478bd9Sstevel@tonic-gate ceen = clo_flags & EN_REG_CEEN; 18047c478bd9Sstevel@tonic-gate tl = (clo_flags & CLO_FLAGS_TL_MASK) >> CLO_FLAGS_TL_SHIFT; 18057c478bd9Sstevel@tonic-gate ttype = (clo_flags & CLO_FLAGS_TT_MASK) >> CLO_FLAGS_TT_SHIFT; 18067c478bd9Sstevel@tonic-gate 18077c478bd9Sstevel@tonic-gate /* 18087c478bd9Sstevel@tonic-gate * handle the specific error 18097c478bd9Sstevel@tonic-gate */ 18107c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 18117c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 18127c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 18137c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 18147c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = t_afsr_ext; 18157c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = t_afsr_errs; 18167c478bd9Sstevel@tonic-gate aflt->flt_stat = t_afsr; 18177c478bd9Sstevel@tonic-gate aflt->flt_addr = t_afar; 18187c478bd9Sstevel@tonic-gate aflt->flt_pc = (caddr_t)rp->r_pc; 18197c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 18207c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 18217c478bd9Sstevel@tonic-gate aflt->flt_priv = (rp->r_tstate & TSTATE_PRIV) ? 1 : 0; 18227c478bd9Sstevel@tonic-gate aflt->flt_tl = (uchar_t)tl; 18237c478bd9Sstevel@tonic-gate aflt->flt_panic = ((tl != 0) || (aft_testfatal != 0) || 18247c478bd9Sstevel@tonic-gate C_AFSR_PANIC(t_afsr_errs)); 18257c478bd9Sstevel@tonic-gate aflt->flt_core = (pflag & SDOCORE) ? 1 : 0; 18267c478bd9Sstevel@tonic-gate aflt->flt_status = ((ttype == T_DATA_ERROR) ? ECC_D_TRAP : ECC_I_TRAP); 18277c478bd9Sstevel@tonic-gate 18287c478bd9Sstevel@tonic-gate /* 18297c478bd9Sstevel@tonic-gate * If the trap occurred in privileged mode at TL=0, we need to check to 18307c478bd9Sstevel@tonic-gate * see if we were executing in the kernel under on_trap() or t_lofault 18317c478bd9Sstevel@tonic-gate * protection. If so, modify the saved registers so that we return 18327c478bd9Sstevel@tonic-gate * from the trap to the appropriate trampoline routine. 18337c478bd9Sstevel@tonic-gate */ 18347c478bd9Sstevel@tonic-gate if (aflt->flt_priv && tl == 0) { 18357c478bd9Sstevel@tonic-gate if (curthread->t_ontrap != NULL) { 18367c478bd9Sstevel@tonic-gate on_trap_data_t *otp = curthread->t_ontrap; 18377c478bd9Sstevel@tonic-gate 18387c478bd9Sstevel@tonic-gate if (otp->ot_prot & OT_DATA_EC) { 18397c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_EC; 18407c478bd9Sstevel@tonic-gate otp->ot_trap |= OT_DATA_EC; 18417c478bd9Sstevel@tonic-gate rp->r_pc = otp->ot_trampoline; 18427c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 18437c478bd9Sstevel@tonic-gate trampolined = 1; 18447c478bd9Sstevel@tonic-gate } 18457c478bd9Sstevel@tonic-gate 18467c478bd9Sstevel@tonic-gate if ((t_afsr & (C_AFSR_TO | C_AFSR_BERR)) && 18477c478bd9Sstevel@tonic-gate (otp->ot_prot & OT_DATA_ACCESS)) { 18487c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_ACCESS; 18497c478bd9Sstevel@tonic-gate otp->ot_trap |= OT_DATA_ACCESS; 18507c478bd9Sstevel@tonic-gate rp->r_pc = otp->ot_trampoline; 18517c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 18527c478bd9Sstevel@tonic-gate trampolined = 1; 18537c478bd9Sstevel@tonic-gate /* 18547c478bd9Sstevel@tonic-gate * for peeks and caut_gets errors are expected 18557c478bd9Sstevel@tonic-gate */ 18567c478bd9Sstevel@tonic-gate hp = (ddi_acc_hdl_t *)otp->ot_handle; 18577c478bd9Sstevel@tonic-gate if (!hp) 18587c478bd9Sstevel@tonic-gate expected = DDI_FM_ERR_PEEK; 18597c478bd9Sstevel@tonic-gate else if (hp->ah_acc.devacc_attr_access == 18607c478bd9Sstevel@tonic-gate DDI_CAUTIOUS_ACC) 18617c478bd9Sstevel@tonic-gate expected = DDI_FM_ERR_EXPECTED; 18627c478bd9Sstevel@tonic-gate } 18637c478bd9Sstevel@tonic-gate 18647c478bd9Sstevel@tonic-gate } else if (curthread->t_lofault) { 18657c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_COPY; 18667c478bd9Sstevel@tonic-gate rp->r_g1 = EFAULT; 18677c478bd9Sstevel@tonic-gate rp->r_pc = curthread->t_lofault; 18687c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 18697c478bd9Sstevel@tonic-gate trampolined = 1; 18707c478bd9Sstevel@tonic-gate } 18717c478bd9Sstevel@tonic-gate } 18727c478bd9Sstevel@tonic-gate 18737c478bd9Sstevel@tonic-gate /* 18747c478bd9Sstevel@tonic-gate * If we're in user mode or we're doing a protected copy, we either 18757c478bd9Sstevel@tonic-gate * want the ASTON code below to send a signal to the user process 18767c478bd9Sstevel@tonic-gate * or we want to panic if aft_panic is set. 18777c478bd9Sstevel@tonic-gate * 18787c478bd9Sstevel@tonic-gate * If we're in privileged mode and we're not doing a copy, then we 18797c478bd9Sstevel@tonic-gate * need to check if we've trampolined. If we haven't trampolined, 18807c478bd9Sstevel@tonic-gate * we should panic. 18817c478bd9Sstevel@tonic-gate */ 18827c478bd9Sstevel@tonic-gate if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 18837c478bd9Sstevel@tonic-gate if (t_afsr_errs & 18847c478bd9Sstevel@tonic-gate ((C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS) & 18857c478bd9Sstevel@tonic-gate ~(C_AFSR_BERR | C_AFSR_TO))) 18867c478bd9Sstevel@tonic-gate aflt->flt_panic |= aft_panic; 18877c478bd9Sstevel@tonic-gate } else if (!trampolined) { 18887c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 18897c478bd9Sstevel@tonic-gate } 18907c478bd9Sstevel@tonic-gate 18917c478bd9Sstevel@tonic-gate /* 18927c478bd9Sstevel@tonic-gate * If we've trampolined due to a privileged TO or BERR, or if an 18937c478bd9Sstevel@tonic-gate * unprivileged TO or BERR occurred, we don't want to enqueue an 18947c478bd9Sstevel@tonic-gate * event for that TO or BERR. Queue all other events (if any) besides 18957c478bd9Sstevel@tonic-gate * the TO/BERR. Since we may not be enqueing any events, we need to 18967c478bd9Sstevel@tonic-gate * ignore the number of events queued. If we haven't trampolined due 18977c478bd9Sstevel@tonic-gate * to a TO or BERR, just enqueue events normally. 18987c478bd9Sstevel@tonic-gate */ 18997c478bd9Sstevel@tonic-gate log_afsr = t_afsr_errs; 19007c478bd9Sstevel@tonic-gate if (trampolined) { 19017c478bd9Sstevel@tonic-gate log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 19027c478bd9Sstevel@tonic-gate } else if (!aflt->flt_priv) { 19037c478bd9Sstevel@tonic-gate /* 19047c478bd9Sstevel@tonic-gate * User mode, suppress messages if 19057c478bd9Sstevel@tonic-gate * cpu_berr_to_verbose is not set. 19067c478bd9Sstevel@tonic-gate */ 19077c478bd9Sstevel@tonic-gate if (!cpu_berr_to_verbose) 19087c478bd9Sstevel@tonic-gate log_afsr &= ~(C_AFSR_TO | C_AFSR_BERR); 19097c478bd9Sstevel@tonic-gate } 19107c478bd9Sstevel@tonic-gate 19117c478bd9Sstevel@tonic-gate /* 19127c478bd9Sstevel@tonic-gate * Log any errors that occurred 19137c478bd9Sstevel@tonic-gate */ 19147c478bd9Sstevel@tonic-gate if (((log_afsr & 1915cbaac45eSkm ((C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS) & ~C_AFSR_ME)) && 1916cbaac45eSkm cpu_queue_events(&ch_flt, pr_reason, log_afsr, clop) == 0) || 1917cbaac45eSkm (t_afsr_errs & (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) == 0) { 19187c478bd9Sstevel@tonic-gate ch_flt.flt_type = CPU_INV_AFSR; 19197c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 19207c478bd9Sstevel@tonic-gate (void *)&ch_flt, sizeof (ch_async_flt_t), ue_queue, 19217c478bd9Sstevel@tonic-gate aflt->flt_panic); 19227c478bd9Sstevel@tonic-gate } 19237c478bd9Sstevel@tonic-gate 19247c478bd9Sstevel@tonic-gate /* 19257c478bd9Sstevel@tonic-gate * Zero out + invalidate CPU logout. 19267c478bd9Sstevel@tonic-gate */ 19277c478bd9Sstevel@tonic-gate if (clop) { 19287c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 19297c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 19307c478bd9Sstevel@tonic-gate } 19317c478bd9Sstevel@tonic-gate 19327c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 19337c478bd9Sstevel@tonic-gate /* 19347c478bd9Sstevel@tonic-gate * UE/RUE/BERR/TO: Call our bus nexus friends to check for 19357c478bd9Sstevel@tonic-gate * IO errors that may have resulted in this trap. 19367c478bd9Sstevel@tonic-gate */ 19377c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE|C_AFSR_RUE|C_AFSR_TO|C_AFSR_BERR)) { 19387c478bd9Sstevel@tonic-gate cpu_run_bus_error_handlers(aflt, expected); 19397c478bd9Sstevel@tonic-gate } 19407c478bd9Sstevel@tonic-gate 19417c478bd9Sstevel@tonic-gate /* 19427c478bd9Sstevel@tonic-gate * UE/RUE: If UE or RUE is in memory, we need to flush the bad 19437c478bd9Sstevel@tonic-gate * line from the Ecache. We also need to query the bus nexus for 19447c478bd9Sstevel@tonic-gate * fatal errors. Attempts to do diagnostic read on caches may 19457c478bd9Sstevel@tonic-gate * introduce more errors (especially when the module is bad). 19467c478bd9Sstevel@tonic-gate */ 19477c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE|C_AFSR_RUE)) { 19487c478bd9Sstevel@tonic-gate /* 19497c478bd9Sstevel@tonic-gate * Ask our bus nexus friends if they have any fatal errors. If 19507c478bd9Sstevel@tonic-gate * so, they will log appropriate error messages. 19517c478bd9Sstevel@tonic-gate */ 19527c478bd9Sstevel@tonic-gate if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 19537c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 19547c478bd9Sstevel@tonic-gate 19557c478bd9Sstevel@tonic-gate /* 19567c478bd9Sstevel@tonic-gate * We got a UE or RUE and are panicking, save the fault PA in 19577c478bd9Sstevel@tonic-gate * a known location so that the platform specific panic code 19587c478bd9Sstevel@tonic-gate * can check for copyback errors. 19597c478bd9Sstevel@tonic-gate */ 19607c478bd9Sstevel@tonic-gate if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 19617c478bd9Sstevel@tonic-gate panic_aflt = *aflt; 19627c478bd9Sstevel@tonic-gate } 19637c478bd9Sstevel@tonic-gate } 19647c478bd9Sstevel@tonic-gate 19657c478bd9Sstevel@tonic-gate /* 19667c478bd9Sstevel@tonic-gate * Flush Ecache line or entire Ecache 19677c478bd9Sstevel@tonic-gate */ 19687c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE | C_AFSR_RUE | C_AFSR_EDU | C_AFSR_BERR)) 19697c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(&ch_flt); 19707c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 19717c478bd9Sstevel@tonic-gate /* 19727c478bd9Sstevel@tonic-gate * UE/BERR/TO: Call our bus nexus friends to check for 19737c478bd9Sstevel@tonic-gate * IO errors that may have resulted in this trap. 19747c478bd9Sstevel@tonic-gate */ 19757c478bd9Sstevel@tonic-gate if (t_afsr & (C_AFSR_UE|C_AFSR_TO|C_AFSR_BERR)) { 19767c478bd9Sstevel@tonic-gate cpu_run_bus_error_handlers(aflt, expected); 19777c478bd9Sstevel@tonic-gate } 19787c478bd9Sstevel@tonic-gate 19797c478bd9Sstevel@tonic-gate /* 19807c478bd9Sstevel@tonic-gate * UE: If the UE is in memory, we need to flush the bad 19817c478bd9Sstevel@tonic-gate * line from the Ecache. We also need to query the bus nexus for 19827c478bd9Sstevel@tonic-gate * fatal errors. Attempts to do diagnostic read on caches may 19837c478bd9Sstevel@tonic-gate * introduce more errors (especially when the module is bad). 19847c478bd9Sstevel@tonic-gate */ 19857c478bd9Sstevel@tonic-gate if (t_afsr & C_AFSR_UE) { 19867c478bd9Sstevel@tonic-gate /* 19877c478bd9Sstevel@tonic-gate * Ask our legacy bus nexus friends if they have any fatal 19887c478bd9Sstevel@tonic-gate * errors. If so, they will log appropriate error messages. 19897c478bd9Sstevel@tonic-gate */ 19907c478bd9Sstevel@tonic-gate if (bus_func_invoke(BF_TYPE_UE) == BF_FATAL) 19917c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 19927c478bd9Sstevel@tonic-gate 19937c478bd9Sstevel@tonic-gate /* 19947c478bd9Sstevel@tonic-gate * We got a UE and are panicking, save the fault PA in a known 19957c478bd9Sstevel@tonic-gate * location so that the platform specific panic code can check 19967c478bd9Sstevel@tonic-gate * for copyback errors. 19977c478bd9Sstevel@tonic-gate */ 19987c478bd9Sstevel@tonic-gate if (aflt->flt_panic && cpu_flt_in_memory(&ch_flt, C_AFSR_UE)) { 19997c478bd9Sstevel@tonic-gate panic_aflt = *aflt; 20007c478bd9Sstevel@tonic-gate } 20017c478bd9Sstevel@tonic-gate } 20027c478bd9Sstevel@tonic-gate 20037c478bd9Sstevel@tonic-gate /* 20047c478bd9Sstevel@tonic-gate * Flush Ecache line or entire Ecache 20057c478bd9Sstevel@tonic-gate */ 20067c478bd9Sstevel@tonic-gate if (t_afsr_errs & 20077c478bd9Sstevel@tonic-gate (C_AFSR_UE | C_AFSR_EDU | C_AFSR_BERR | C_AFSR_L3_EDU)) 20087c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(&ch_flt); 20097c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 20107c478bd9Sstevel@tonic-gate 20117c478bd9Sstevel@tonic-gate /* 20127c478bd9Sstevel@tonic-gate * We carefully re-enable NCEEN and CEEN and then check if any deferred 20137c478bd9Sstevel@tonic-gate * or disrupting errors have happened. We do this because if a 20147c478bd9Sstevel@tonic-gate * deferred or disrupting error had occurred with NCEEN/CEEN off, the 20157c478bd9Sstevel@tonic-gate * trap will not be taken when NCEEN/CEEN is re-enabled. Note that 20167c478bd9Sstevel@tonic-gate * CEEN works differently on Cheetah than on Spitfire. Also, we enable 20177c478bd9Sstevel@tonic-gate * NCEEN/CEEN *before* checking the AFSR to avoid the small window of a 20187c478bd9Sstevel@tonic-gate * deferred or disrupting error happening between checking the AFSR and 20197c478bd9Sstevel@tonic-gate * enabling NCEEN/CEEN. 20207c478bd9Sstevel@tonic-gate * 20217c478bd9Sstevel@tonic-gate * Note: CEEN reenabled only if it was on when trap taken. 20227c478bd9Sstevel@tonic-gate */ 20237c478bd9Sstevel@tonic-gate set_error_enable(get_error_enable() | (EN_REG_NCEEN | ceen)); 20247c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 20257c478bd9Sstevel@tonic-gate /* 20267c478bd9Sstevel@tonic-gate * Check for secondary errors, and avoid panicking if we 20277c478bd9Sstevel@tonic-gate * have them 20287c478bd9Sstevel@tonic-gate */ 20297c478bd9Sstevel@tonic-gate if (cpu_check_secondary_errors(&ch_flt, t_afsr_errs, 20307c478bd9Sstevel@tonic-gate t_afar) == 0) { 20317c478bd9Sstevel@tonic-gate aflt->flt_panic |= ((ch_flt.afsr_errs & 20327c478bd9Sstevel@tonic-gate (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS)) != 0); 20337c478bd9Sstevel@tonic-gate } 20347c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 20357c478bd9Sstevel@tonic-gate NULL); 20367c478bd9Sstevel@tonic-gate } 20377c478bd9Sstevel@tonic-gate 20387c478bd9Sstevel@tonic-gate /* 20397c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 20407c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 20417c478bd9Sstevel@tonic-gate */ 20427c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 20437c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", pr_reason); 20447c478bd9Sstevel@tonic-gate 20457c478bd9Sstevel@tonic-gate /* 20467c478bd9Sstevel@tonic-gate * If we queued an error and we are going to return from the trap and 20477c478bd9Sstevel@tonic-gate * the error was in user mode or inside of a copy routine, set AST flag 20487c478bd9Sstevel@tonic-gate * so the queue will be drained before returning to user mode. The 20497c478bd9Sstevel@tonic-gate * AST processing will also act on our failure policy. 20507c478bd9Sstevel@tonic-gate */ 20517c478bd9Sstevel@tonic-gate if (!aflt->flt_priv || aflt->flt_prot == AFLT_PROT_COPY) { 20527c478bd9Sstevel@tonic-gate int pcb_flag = 0; 20537c478bd9Sstevel@tonic-gate 20547c478bd9Sstevel@tonic-gate if (t_afsr_errs & 20557c478bd9Sstevel@tonic-gate (C_AFSR_ASYNC_ERRS | C_AFSR_EXT_ASYNC_ERRS & 20567c478bd9Sstevel@tonic-gate ~(C_AFSR_BERR | C_AFSR_TO))) 20577c478bd9Sstevel@tonic-gate pcb_flag |= ASYNC_HWERR; 20587c478bd9Sstevel@tonic-gate 20597c478bd9Sstevel@tonic-gate if (t_afsr & C_AFSR_BERR) 20607c478bd9Sstevel@tonic-gate pcb_flag |= ASYNC_BERR; 20617c478bd9Sstevel@tonic-gate 20627c478bd9Sstevel@tonic-gate if (t_afsr & C_AFSR_TO) 20637c478bd9Sstevel@tonic-gate pcb_flag |= ASYNC_BTO; 20647c478bd9Sstevel@tonic-gate 20657c478bd9Sstevel@tonic-gate ttolwp(curthread)->lwp_pcb.pcb_flags |= pcb_flag; 20667c478bd9Sstevel@tonic-gate aston(curthread); 20677c478bd9Sstevel@tonic-gate } 20687c478bd9Sstevel@tonic-gate } 20697c478bd9Sstevel@tonic-gate 20707c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 20717c478bd9Sstevel@tonic-gate /* 20727c478bd9Sstevel@tonic-gate * Handling of data and instruction parity errors (traps 0x71, 0x72). 20737c478bd9Sstevel@tonic-gate * 20747c478bd9Sstevel@tonic-gate * For Panther, P$ data parity errors during floating point load hits 20757c478bd9Sstevel@tonic-gate * are also detected (reported as TT 0x71) and handled by this trap 20767c478bd9Sstevel@tonic-gate * handler. 20777c478bd9Sstevel@tonic-gate * 20787c478bd9Sstevel@tonic-gate * AFSR/AFAR are not set for parity errors, only TPC (a virtual address) 20797c478bd9Sstevel@tonic-gate * is available. 20807c478bd9Sstevel@tonic-gate */ 20817c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 20827c478bd9Sstevel@tonic-gate void 20837c478bd9Sstevel@tonic-gate cpu_parity_error(struct regs *rp, uint_t flags, caddr_t tpc) 20847c478bd9Sstevel@tonic-gate { 20857c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 20867c478bd9Sstevel@tonic-gate struct async_flt *aflt; 20877c478bd9Sstevel@tonic-gate uchar_t tl = ((flags & CH_ERR_TL) != 0); 20887c478bd9Sstevel@tonic-gate uchar_t iparity = ((flags & CH_ERR_IPE) != 0); 20897c478bd9Sstevel@tonic-gate uchar_t panic = ((flags & CH_ERR_PANIC) != 0); 20907c478bd9Sstevel@tonic-gate char *error_class; 2091*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States int index, way, word; 2092*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States ch_dc_data_t tmp_dcp; 2093*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States int dc_set_size = dcache_size / CH_DCACHE_NWAY; 2094*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States uint64_t parity_bits, pbits; 2095*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States /* The parity bit array corresponds to the result of summing two bits */ 2096*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States static int parity_bits_popc[] = { 0, 1, 1, 0 }; 20977c478bd9Sstevel@tonic-gate 20987c478bd9Sstevel@tonic-gate /* 20997c478bd9Sstevel@tonic-gate * Log the error. 21007c478bd9Sstevel@tonic-gate * For icache parity errors the fault address is the trap PC. 21017c478bd9Sstevel@tonic-gate * For dcache/pcache parity errors the instruction would have to 21027c478bd9Sstevel@tonic-gate * be decoded to determine the address and that isn't possible 21037c478bd9Sstevel@tonic-gate * at high PIL. 21047c478bd9Sstevel@tonic-gate */ 21057c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 21067c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 21077c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 21087c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 21097c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 21107c478bd9Sstevel@tonic-gate aflt->flt_pc = tpc; 21117c478bd9Sstevel@tonic-gate aflt->flt_addr = iparity ? (uint64_t)tpc : AFLT_INV_ADDR; 21127c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 21137c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 21147c478bd9Sstevel@tonic-gate aflt->flt_priv = (tl || (rp->r_tstate & TSTATE_PRIV)) ? 1 : 0; 21157c478bd9Sstevel@tonic-gate aflt->flt_tl = tl; 21167c478bd9Sstevel@tonic-gate aflt->flt_panic = panic; 21177c478bd9Sstevel@tonic-gate aflt->flt_status = iparity ? ECC_IP_TRAP : ECC_DP_TRAP; 21187c478bd9Sstevel@tonic-gate ch_flt.flt_type = iparity ? CPU_IC_PARITY : CPU_DC_PARITY; 21197c478bd9Sstevel@tonic-gate 21207c478bd9Sstevel@tonic-gate if (iparity) { 21217c478bd9Sstevel@tonic-gate cpu_icache_parity_info(&ch_flt); 21227c478bd9Sstevel@tonic-gate if (ch_flt.parity_data.ipe.cpl_off != -1) 21237c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_IDSPE; 21247c478bd9Sstevel@tonic-gate else if (ch_flt.parity_data.ipe.cpl_way != -1) 21257c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_ITSPE; 21267c478bd9Sstevel@tonic-gate else 21277c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_IPE; 21287c478bd9Sstevel@tonic-gate aflt->flt_payload = FM_EREPORT_PAYLOAD_ICACHE_PE; 21297c478bd9Sstevel@tonic-gate } else { 21307c478bd9Sstevel@tonic-gate cpu_dcache_parity_info(&ch_flt); 2131*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States if (ch_flt.parity_data.dpe.cpl_off != -1) { 2132*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States /* 2133*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * If not at TL 0 and running on a Jalapeno processor, 2134*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * then process as a true ddspe. A true 2135*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * ddspe error can only occur if the way == 0 2136*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States */ 2137*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States way = ch_flt.parity_data.dpe.cpl_way; 2138*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States if ((tl == 0) && (way != 0) && 2139*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States IS_JALAPENO(cpunodes[CPU->cpu_id].implementation)) { 2140*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States for (index = 0; index < dc_set_size; 2141*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States index += dcache_linesize) { 2142*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States get_dcache_dtag(index + way * 2143*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States dc_set_size, 2144*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States (uint64_t *)&tmp_dcp); 2145*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States /* 2146*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * Check data array for even parity. 2147*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * The 8 parity bits are grouped into 2148*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * 4 pairs each of which covers a 64-bit 2149*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * word. The endianness is reversed 2150*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * -- the low-order parity bits cover 2151*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States * the high-order data words. 2152*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States */ 2153*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States parity_bits = tmp_dcp.dc_utag >> 8; 2154*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States for (word = 0; word < 4; word++) { 2155*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States pbits = (parity_bits >> 2156*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States (6 - word * 2)) & 3; 2157*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States if (((popc64( 2158*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States tmp_dcp.dc_data[word]) + 2159*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States parity_bits_popc[pbits]) & 2160*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States 1) && (tmp_dcp.dc_tag & 2161*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States VA13)) { 2162*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States /* cleanup */ 2163*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States correct_dcache_parity( 2164*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States dcache_size, 2165*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States dcache_linesize); 2166*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States if (cache_boot_state & 2167*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States DCU_DC) { 2168*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States flush_dcache(); 2169*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States } 2170*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States 2171*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States set_dcu(get_dcu() | 2172*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States cache_boot_state); 2173*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States return; 2174*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States } 2175*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States } 2176*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States } 2177*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States } /* (tl == 0) && (way != 0) && IS JALAPENO */ 21787c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_DDSPE; 2179*f60f9424SChristopher Baumbauer - Sun Microsystems - San Diego United States } else if (ch_flt.parity_data.dpe.cpl_way != -1) 21807c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_DTSPE; 21817c478bd9Sstevel@tonic-gate else 21827c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_DPE; 21837c478bd9Sstevel@tonic-gate aflt->flt_payload = FM_EREPORT_PAYLOAD_DCACHE_PE; 21847c478bd9Sstevel@tonic-gate /* 21857c478bd9Sstevel@tonic-gate * For panther we also need to check the P$ for parity errors. 21867c478bd9Sstevel@tonic-gate */ 21877c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 21887c478bd9Sstevel@tonic-gate cpu_pcache_parity_info(&ch_flt); 21897c478bd9Sstevel@tonic-gate if (ch_flt.parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 21907c478bd9Sstevel@tonic-gate error_class = FM_EREPORT_CPU_USIII_PDSPE; 21917c478bd9Sstevel@tonic-gate aflt->flt_payload = 21927c478bd9Sstevel@tonic-gate FM_EREPORT_PAYLOAD_PCACHE_PE; 21937c478bd9Sstevel@tonic-gate } 21947c478bd9Sstevel@tonic-gate } 21957c478bd9Sstevel@tonic-gate } 21967c478bd9Sstevel@tonic-gate 21977c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(error_class, (void *)&ch_flt, 21987c478bd9Sstevel@tonic-gate sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 21997c478bd9Sstevel@tonic-gate 22007c478bd9Sstevel@tonic-gate if (iparity) { 22017c478bd9Sstevel@tonic-gate /* 22027c478bd9Sstevel@tonic-gate * Invalidate entire I$. 22037c478bd9Sstevel@tonic-gate * This is required due to the use of diagnostic ASI 22047c478bd9Sstevel@tonic-gate * accesses that may result in a loss of I$ coherency. 22057c478bd9Sstevel@tonic-gate */ 22067c478bd9Sstevel@tonic-gate if (cache_boot_state & DCU_IC) { 22077c478bd9Sstevel@tonic-gate flush_icache(); 22087c478bd9Sstevel@tonic-gate } 22097c478bd9Sstevel@tonic-gate /* 22107c478bd9Sstevel@tonic-gate * According to section P.3.1 of the Panther PRM, we 22117c478bd9Sstevel@tonic-gate * need to do a little more for recovery on those 22127c478bd9Sstevel@tonic-gate * CPUs after encountering an I$ parity error. 22137c478bd9Sstevel@tonic-gate */ 22147c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 22157c478bd9Sstevel@tonic-gate flush_ipb(); 22167c478bd9Sstevel@tonic-gate correct_dcache_parity(dcache_size, 22177c478bd9Sstevel@tonic-gate dcache_linesize); 22187c478bd9Sstevel@tonic-gate flush_pcache(); 22197c478bd9Sstevel@tonic-gate } 22207c478bd9Sstevel@tonic-gate } else { 22217c478bd9Sstevel@tonic-gate /* 22227c478bd9Sstevel@tonic-gate * Since the valid bit is ignored when checking parity the 22237c478bd9Sstevel@tonic-gate * D$ data and tag must also be corrected. Set D$ data bits 22247c478bd9Sstevel@tonic-gate * to zero and set utag to 0, 1, 2, 3. 22257c478bd9Sstevel@tonic-gate */ 22267c478bd9Sstevel@tonic-gate correct_dcache_parity(dcache_size, dcache_linesize); 22277c478bd9Sstevel@tonic-gate 22287c478bd9Sstevel@tonic-gate /* 22297c478bd9Sstevel@tonic-gate * According to section P.3.3 of the Panther PRM, we 22307c478bd9Sstevel@tonic-gate * need to do a little more for recovery on those 22317c478bd9Sstevel@tonic-gate * CPUs after encountering a D$ or P$ parity error. 22327c478bd9Sstevel@tonic-gate * 22337c478bd9Sstevel@tonic-gate * As far as clearing P$ parity errors, it is enough to 22347c478bd9Sstevel@tonic-gate * simply invalidate all entries in the P$ since P$ parity 22357c478bd9Sstevel@tonic-gate * error traps are only generated for floating point load 22367c478bd9Sstevel@tonic-gate * hits. 22377c478bd9Sstevel@tonic-gate */ 22387c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 22397c478bd9Sstevel@tonic-gate flush_icache(); 22407c478bd9Sstevel@tonic-gate flush_ipb(); 22417c478bd9Sstevel@tonic-gate flush_pcache(); 22427c478bd9Sstevel@tonic-gate } 22437c478bd9Sstevel@tonic-gate } 22447c478bd9Sstevel@tonic-gate 22457c478bd9Sstevel@tonic-gate /* 22467c478bd9Sstevel@tonic-gate * Invalidate entire D$ if it was enabled. 22477c478bd9Sstevel@tonic-gate * This is done to avoid stale data in the D$ which might 22487c478bd9Sstevel@tonic-gate * occur with the D$ disabled and the trap handler doing 22497c478bd9Sstevel@tonic-gate * stores affecting lines already in the D$. 22507c478bd9Sstevel@tonic-gate */ 22517c478bd9Sstevel@tonic-gate if (cache_boot_state & DCU_DC) { 22527c478bd9Sstevel@tonic-gate flush_dcache(); 22537c478bd9Sstevel@tonic-gate } 22547c478bd9Sstevel@tonic-gate 22557c478bd9Sstevel@tonic-gate /* 22567c478bd9Sstevel@tonic-gate * Restore caches to their bootup state. 22577c478bd9Sstevel@tonic-gate */ 22587c478bd9Sstevel@tonic-gate set_dcu(get_dcu() | cache_boot_state); 22597c478bd9Sstevel@tonic-gate 22607c478bd9Sstevel@tonic-gate /* 22617c478bd9Sstevel@tonic-gate * Panic here if aflt->flt_panic has been set. Enqueued errors will 22627c478bd9Sstevel@tonic-gate * be logged as part of the panic flow. 22637c478bd9Sstevel@tonic-gate */ 22647c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 22657c478bd9Sstevel@tonic-gate fm_panic("%sError(s)", iparity ? "IPE " : "DPE "); 22667c478bd9Sstevel@tonic-gate 22677c478bd9Sstevel@tonic-gate /* 22687c478bd9Sstevel@tonic-gate * If this error occurred at TL>0 then flush the E$ here to reduce 22697c478bd9Sstevel@tonic-gate * the chance of getting an unrecoverable Fast ECC error. This 22707c478bd9Sstevel@tonic-gate * flush will evict the part of the parity trap handler that is run 22717c478bd9Sstevel@tonic-gate * at TL>1. 22727c478bd9Sstevel@tonic-gate */ 22737c478bd9Sstevel@tonic-gate if (tl) { 22747c478bd9Sstevel@tonic-gate cpu_flush_ecache(); 22757c478bd9Sstevel@tonic-gate } 22767c478bd9Sstevel@tonic-gate } 22777c478bd9Sstevel@tonic-gate 22787c478bd9Sstevel@tonic-gate /* 22797c478bd9Sstevel@tonic-gate * On an I$ parity error, mark the appropriate entries in the ch_async_flt_t 22807c478bd9Sstevel@tonic-gate * to indicate which portions of the captured data should be in the ereport. 22817c478bd9Sstevel@tonic-gate */ 22827c478bd9Sstevel@tonic-gate void 22837c478bd9Sstevel@tonic-gate cpu_async_log_ic_parity_err(ch_async_flt_t *ch_flt) 22847c478bd9Sstevel@tonic-gate { 22857c478bd9Sstevel@tonic-gate int way = ch_flt->parity_data.ipe.cpl_way; 22867c478bd9Sstevel@tonic-gate int offset = ch_flt->parity_data.ipe.cpl_off; 22877c478bd9Sstevel@tonic-gate int tag_index; 22887c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 22897c478bd9Sstevel@tonic-gate 22907c478bd9Sstevel@tonic-gate 22917c478bd9Sstevel@tonic-gate if ((offset != -1) || (way != -1)) { 22927c478bd9Sstevel@tonic-gate /* 22937c478bd9Sstevel@tonic-gate * Parity error in I$ tag or data 22947c478bd9Sstevel@tonic-gate */ 22957c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 22967c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 22977c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 22987c478bd9Sstevel@tonic-gate PN_ICIDX_TO_WAY(tag_index); 22997c478bd9Sstevel@tonic-gate else 23007c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 23017c478bd9Sstevel@tonic-gate CH_ICIDX_TO_WAY(tag_index); 23027c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 23037c478bd9Sstevel@tonic-gate IC_LOGFLAG_MAGIC; 23047c478bd9Sstevel@tonic-gate } else { 23057c478bd9Sstevel@tonic-gate /* 23067c478bd9Sstevel@tonic-gate * Parity error was not identified. 23077c478bd9Sstevel@tonic-gate * Log tags and data for all ways. 23087c478bd9Sstevel@tonic-gate */ 23097c478bd9Sstevel@tonic-gate for (way = 0; way < CH_ICACHE_NWAY; way++) { 23107c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.ipe.cpl_ic[way].ic_idx; 23117c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 23127c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 23137c478bd9Sstevel@tonic-gate PN_ICIDX_TO_WAY(tag_index); 23147c478bd9Sstevel@tonic-gate else 23157c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_way = 23167c478bd9Sstevel@tonic-gate CH_ICIDX_TO_WAY(tag_index); 23177c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_ic[way].ic_logflag = 23187c478bd9Sstevel@tonic-gate IC_LOGFLAG_MAGIC; 23197c478bd9Sstevel@tonic-gate } 23207c478bd9Sstevel@tonic-gate } 23217c478bd9Sstevel@tonic-gate } 23227c478bd9Sstevel@tonic-gate 23237c478bd9Sstevel@tonic-gate /* 23247c478bd9Sstevel@tonic-gate * On an D$ parity error, mark the appropriate entries in the ch_async_flt_t 23257c478bd9Sstevel@tonic-gate * to indicate which portions of the captured data should be in the ereport. 23267c478bd9Sstevel@tonic-gate */ 23277c478bd9Sstevel@tonic-gate void 23287c478bd9Sstevel@tonic-gate cpu_async_log_dc_parity_err(ch_async_flt_t *ch_flt) 23297c478bd9Sstevel@tonic-gate { 23307c478bd9Sstevel@tonic-gate int way = ch_flt->parity_data.dpe.cpl_way; 23317c478bd9Sstevel@tonic-gate int offset = ch_flt->parity_data.dpe.cpl_off; 23327c478bd9Sstevel@tonic-gate int tag_index; 23337c478bd9Sstevel@tonic-gate 23347c478bd9Sstevel@tonic-gate if (offset != -1) { 23357c478bd9Sstevel@tonic-gate /* 23367c478bd9Sstevel@tonic-gate * Parity error in D$ or P$ data array. 23377c478bd9Sstevel@tonic-gate * 23387c478bd9Sstevel@tonic-gate * First check to see whether the parity error is in D$ or P$ 23397c478bd9Sstevel@tonic-gate * since P$ data parity errors are reported in Panther using 23407c478bd9Sstevel@tonic-gate * the same trap. 23417c478bd9Sstevel@tonic-gate */ 23427c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_cache == CPU_PC_PARITY) { 23437c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.dpe.cpl_pc[way].pc_idx; 23447c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_pc[way].pc_way = 23457c478bd9Sstevel@tonic-gate CH_PCIDX_TO_WAY(tag_index); 23467c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_pc[way].pc_logflag = 23477c478bd9Sstevel@tonic-gate PC_LOGFLAG_MAGIC; 23487c478bd9Sstevel@tonic-gate } else { 23497c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 23507c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 23517c478bd9Sstevel@tonic-gate CH_DCIDX_TO_WAY(tag_index); 23527c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 23537c478bd9Sstevel@tonic-gate DC_LOGFLAG_MAGIC; 23547c478bd9Sstevel@tonic-gate } 23557c478bd9Sstevel@tonic-gate } else if (way != -1) { 23567c478bd9Sstevel@tonic-gate /* 23577c478bd9Sstevel@tonic-gate * Parity error in D$ tag. 23587c478bd9Sstevel@tonic-gate */ 23597c478bd9Sstevel@tonic-gate tag_index = ch_flt->parity_data.dpe.cpl_dc[way].dc_idx; 23607c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_way = 23617c478bd9Sstevel@tonic-gate CH_DCIDX_TO_WAY(tag_index); 23627c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_dc[way].dc_logflag = 23637c478bd9Sstevel@tonic-gate DC_LOGFLAG_MAGIC; 23647c478bd9Sstevel@tonic-gate } 23657c478bd9Sstevel@tonic-gate } 23667c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 23677c478bd9Sstevel@tonic-gate 23687c478bd9Sstevel@tonic-gate /* 23697c478bd9Sstevel@tonic-gate * The cpu_async_log_err() function is called via the [uc]e_drain() function to 23707c478bd9Sstevel@tonic-gate * post-process CPU events that are dequeued. As such, it can be invoked 23717c478bd9Sstevel@tonic-gate * from softint context, from AST processing in the trap() flow, or from the 23727c478bd9Sstevel@tonic-gate * panic flow. We decode the CPU-specific data, and take appropriate actions. 23737c478bd9Sstevel@tonic-gate * Historically this entry point was used to log the actual cmn_err(9F) text; 23747c478bd9Sstevel@tonic-gate * now with FMA it is used to prepare 'flt' to be converted into an ereport. 23757c478bd9Sstevel@tonic-gate * With FMA this function now also returns a flag which indicates to the 23767c478bd9Sstevel@tonic-gate * caller whether the ereport should be posted (1) or suppressed (0). 23777c478bd9Sstevel@tonic-gate */ 23787c478bd9Sstevel@tonic-gate static int 23797c478bd9Sstevel@tonic-gate cpu_async_log_err(void *flt, errorq_elem_t *eqep) 23807c478bd9Sstevel@tonic-gate { 23817c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)flt; 23827c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)flt; 2383db874c57Selowe uint64_t errors; 238461ef38f7Svb extern void memscrub_induced_error(void); 23857c478bd9Sstevel@tonic-gate 23867c478bd9Sstevel@tonic-gate switch (ch_flt->flt_type) { 23877c478bd9Sstevel@tonic-gate case CPU_INV_AFSR: 23887c478bd9Sstevel@tonic-gate /* 23897c478bd9Sstevel@tonic-gate * If it is a disrupting trap and the AFSR is zero, then 23907c478bd9Sstevel@tonic-gate * the event has probably already been noted. Do not post 23917c478bd9Sstevel@tonic-gate * an ereport. 23927c478bd9Sstevel@tonic-gate */ 23937c478bd9Sstevel@tonic-gate if ((aflt->flt_status & ECC_C_TRAP) && 23947c478bd9Sstevel@tonic-gate (!(aflt->flt_stat & C_AFSR_MASK))) 23957c478bd9Sstevel@tonic-gate return (0); 23967c478bd9Sstevel@tonic-gate else 23977c478bd9Sstevel@tonic-gate return (1); 23987c478bd9Sstevel@tonic-gate case CPU_TO: 23997c478bd9Sstevel@tonic-gate case CPU_BERR: 24007c478bd9Sstevel@tonic-gate case CPU_FATAL: 24017c478bd9Sstevel@tonic-gate case CPU_FPUERR: 24027c478bd9Sstevel@tonic-gate return (1); 24037c478bd9Sstevel@tonic-gate 24047c478bd9Sstevel@tonic-gate case CPU_UE_ECACHE_RETIRE: 24057c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 24067c478bd9Sstevel@tonic-gate cpu_page_retire(ch_flt); 24077c478bd9Sstevel@tonic-gate return (1); 24087c478bd9Sstevel@tonic-gate 24097c478bd9Sstevel@tonic-gate /* 24107c478bd9Sstevel@tonic-gate * Cases where we may want to suppress logging or perform 24117c478bd9Sstevel@tonic-gate * extended diagnostics. 24127c478bd9Sstevel@tonic-gate */ 24137c478bd9Sstevel@tonic-gate case CPU_CE: 24147c478bd9Sstevel@tonic-gate case CPU_EMC: 24157c478bd9Sstevel@tonic-gate /* 24167c478bd9Sstevel@tonic-gate * We want to skip logging and further classification 24177c478bd9Sstevel@tonic-gate * only if ALL the following conditions are true: 24187c478bd9Sstevel@tonic-gate * 24197c478bd9Sstevel@tonic-gate * 1. There is only one error 24207c478bd9Sstevel@tonic-gate * 2. That error is a correctable memory error 24217c478bd9Sstevel@tonic-gate * 3. The error is caused by the memory scrubber (in 24227c478bd9Sstevel@tonic-gate * which case the error will have occurred under 24237c478bd9Sstevel@tonic-gate * on_trap protection) 24247c478bd9Sstevel@tonic-gate * 4. The error is on a retired page 24257c478bd9Sstevel@tonic-gate * 24267c478bd9Sstevel@tonic-gate * Note: AFLT_PROT_EC is used places other than the memory 24277c478bd9Sstevel@tonic-gate * scrubber. However, none of those errors should occur 24287c478bd9Sstevel@tonic-gate * on a retired page. 24297c478bd9Sstevel@tonic-gate */ 24307c478bd9Sstevel@tonic-gate if ((ch_flt->afsr_errs & 24317c478bd9Sstevel@tonic-gate (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_CE && 24327c478bd9Sstevel@tonic-gate aflt->flt_prot == AFLT_PROT_EC) { 24337c478bd9Sstevel@tonic-gate 2434db874c57Selowe if (page_retire_check(aflt->flt_addr, NULL) == 0) { 2435cbaac45eSkm if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 24367c478bd9Sstevel@tonic-gate 24377c478bd9Sstevel@tonic-gate /* 24387c478bd9Sstevel@tonic-gate * Since we're skipping logging, we'll need 24397c478bd9Sstevel@tonic-gate * to schedule the re-enabling of CEEN 24407c478bd9Sstevel@tonic-gate */ 24417c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, 2442f47a9c50Smathue (void *)(uintptr_t)aflt->flt_inst, 2443f47a9c50Smathue drv_usectohz((clock_t)cpu_ceen_delay_secs 2444cbaac45eSkm * MICROSEC)); 2445cbaac45eSkm } 2446cbaac45eSkm 244761ef38f7Svb /* 244861ef38f7Svb * Inform memscrubber - scrubbing induced 244961ef38f7Svb * CE on a retired page. 245061ef38f7Svb */ 245161ef38f7Svb memscrub_induced_error(); 245261ef38f7Svb return (0); 24537c478bd9Sstevel@tonic-gate } 24547c478bd9Sstevel@tonic-gate } 24557c478bd9Sstevel@tonic-gate 24567c478bd9Sstevel@tonic-gate /* 24577c478bd9Sstevel@tonic-gate * Perform/schedule further classification actions, but 24587c478bd9Sstevel@tonic-gate * only if the page is healthy (we don't want bad 24597c478bd9Sstevel@tonic-gate * pages inducing too much diagnostic activity). If we could 24607c478bd9Sstevel@tonic-gate * not find a page pointer then we also skip this. If 24617c478bd9Sstevel@tonic-gate * ce_scrub_xdiag_recirc returns nonzero then it has chosen 24627c478bd9Sstevel@tonic-gate * to copy and recirculate the event (for further diagnostics) 24637c478bd9Sstevel@tonic-gate * and we should not proceed to log it here. 24647c478bd9Sstevel@tonic-gate * 24657c478bd9Sstevel@tonic-gate * This must be the last step here before the cpu_log_err() 24667c478bd9Sstevel@tonic-gate * below - if an event recirculates cpu_ce_log_err() will 24677c478bd9Sstevel@tonic-gate * not call the current function but just proceed directly 24687c478bd9Sstevel@tonic-gate * to cpu_ereport_post after the cpu_log_err() avoided below. 24697c478bd9Sstevel@tonic-gate * 24707c478bd9Sstevel@tonic-gate * Note: Check cpu_impl_async_log_err if changing this 24717c478bd9Sstevel@tonic-gate */ 2472db874c57Selowe if (page_retire_check(aflt->flt_addr, &errors) == EINVAL) { 2473db874c57Selowe CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 2474db874c57Selowe CE_XDIAG_SKIP_NOPP); 2475db874c57Selowe } else { 2476db874c57Selowe if (errors != PR_OK) { 24777c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 24787c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_PAGEDET); 24797c478bd9Sstevel@tonic-gate } else if (ce_scrub_xdiag_recirc(aflt, ce_queue, eqep, 24807c478bd9Sstevel@tonic-gate offsetof(ch_async_flt_t, cmn_asyncflt))) { 24817c478bd9Sstevel@tonic-gate return (0); 24827c478bd9Sstevel@tonic-gate } 24837c478bd9Sstevel@tonic-gate } 24847c478bd9Sstevel@tonic-gate /*FALLTHRU*/ 24857c478bd9Sstevel@tonic-gate 24867c478bd9Sstevel@tonic-gate /* 24877c478bd9Sstevel@tonic-gate * Cases where we just want to report the error and continue. 24887c478bd9Sstevel@tonic-gate */ 24897c478bd9Sstevel@tonic-gate case CPU_CE_ECACHE: 24907c478bd9Sstevel@tonic-gate case CPU_UE_ECACHE: 24917c478bd9Sstevel@tonic-gate case CPU_IV: 24927c478bd9Sstevel@tonic-gate case CPU_ORPH: 24937c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 24947c478bd9Sstevel@tonic-gate return (1); 24957c478bd9Sstevel@tonic-gate 24967c478bd9Sstevel@tonic-gate /* 24977c478bd9Sstevel@tonic-gate * Cases where we want to fall through to handle panicking. 24987c478bd9Sstevel@tonic-gate */ 24997c478bd9Sstevel@tonic-gate case CPU_UE: 25007c478bd9Sstevel@tonic-gate /* 25017c478bd9Sstevel@tonic-gate * We want to skip logging in the same conditions as the 25027c478bd9Sstevel@tonic-gate * CE case. In addition, we want to make sure we're not 25037c478bd9Sstevel@tonic-gate * panicking. 25047c478bd9Sstevel@tonic-gate */ 25057c478bd9Sstevel@tonic-gate if (!panicstr && (ch_flt->afsr_errs & 25067c478bd9Sstevel@tonic-gate (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == C_AFSR_UE && 25077c478bd9Sstevel@tonic-gate aflt->flt_prot == AFLT_PROT_EC) { 2508db874c57Selowe if (page_retire_check(aflt->flt_addr, NULL) == 0) { 25097c478bd9Sstevel@tonic-gate /* Zero the address to clear the error */ 25107c478bd9Sstevel@tonic-gate softcall(ecc_page_zero, (void *)aflt->flt_addr); 251161ef38f7Svb /* 251261ef38f7Svb * Inform memscrubber - scrubbing induced 251361ef38f7Svb * UE on a retired page. 251461ef38f7Svb */ 251561ef38f7Svb memscrub_induced_error(); 25167c478bd9Sstevel@tonic-gate return (0); 25177c478bd9Sstevel@tonic-gate } 25187c478bd9Sstevel@tonic-gate } 25197c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 25207c478bd9Sstevel@tonic-gate break; 25217c478bd9Sstevel@tonic-gate 25227c478bd9Sstevel@tonic-gate default: 25237c478bd9Sstevel@tonic-gate /* 25247c478bd9Sstevel@tonic-gate * If the us3_common.c code doesn't know the flt_type, it may 25257c478bd9Sstevel@tonic-gate * be an implementation-specific code. Call into the impldep 25267c478bd9Sstevel@tonic-gate * backend to find out what to do: if it tells us to continue, 25277c478bd9Sstevel@tonic-gate * break and handle as if falling through from a UE; if not, 25287c478bd9Sstevel@tonic-gate * the impldep backend has handled the error and we're done. 25297c478bd9Sstevel@tonic-gate */ 25307c478bd9Sstevel@tonic-gate switch (cpu_impl_async_log_err(flt, eqep)) { 25317c478bd9Sstevel@tonic-gate case CH_ASYNC_LOG_DONE: 25327c478bd9Sstevel@tonic-gate return (1); 25337c478bd9Sstevel@tonic-gate case CH_ASYNC_LOG_RECIRC: 25347c478bd9Sstevel@tonic-gate return (0); 25357c478bd9Sstevel@tonic-gate case CH_ASYNC_LOG_CONTINUE: 25367c478bd9Sstevel@tonic-gate break; /* continue on to handle UE-like error */ 25377c478bd9Sstevel@tonic-gate default: 25387c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "discarding error 0x%p with " 25397c478bd9Sstevel@tonic-gate "invalid fault type (0x%x)", 25407c478bd9Sstevel@tonic-gate (void *)aflt, ch_flt->flt_type); 25417c478bd9Sstevel@tonic-gate return (0); 25427c478bd9Sstevel@tonic-gate } 25437c478bd9Sstevel@tonic-gate } 25447c478bd9Sstevel@tonic-gate 25457c478bd9Sstevel@tonic-gate /* ... fall through from the UE case */ 25467c478bd9Sstevel@tonic-gate 25477c478bd9Sstevel@tonic-gate if (aflt->flt_addr != AFLT_INV_ADDR && aflt->flt_in_memory) { 25487c478bd9Sstevel@tonic-gate if (!panicstr) { 25497c478bd9Sstevel@tonic-gate cpu_page_retire(ch_flt); 25507c478bd9Sstevel@tonic-gate } else { 25517c478bd9Sstevel@tonic-gate /* 25527c478bd9Sstevel@tonic-gate * Clear UEs on panic so that we don't 25537c478bd9Sstevel@tonic-gate * get haunted by them during panic or 25547c478bd9Sstevel@tonic-gate * after reboot 25557c478bd9Sstevel@tonic-gate */ 25567c478bd9Sstevel@tonic-gate cpu_clearphys(aflt); 25577c478bd9Sstevel@tonic-gate (void) clear_errors(NULL); 25587c478bd9Sstevel@tonic-gate } 25597c478bd9Sstevel@tonic-gate } 25607c478bd9Sstevel@tonic-gate 25617c478bd9Sstevel@tonic-gate return (1); 25627c478bd9Sstevel@tonic-gate } 25637c478bd9Sstevel@tonic-gate 25647c478bd9Sstevel@tonic-gate /* 25657c478bd9Sstevel@tonic-gate * Retire the bad page that may contain the flushed error. 25667c478bd9Sstevel@tonic-gate */ 25677c478bd9Sstevel@tonic-gate void 25687c478bd9Sstevel@tonic-gate cpu_page_retire(ch_async_flt_t *ch_flt) 25697c478bd9Sstevel@tonic-gate { 25707c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 2571db874c57Selowe (void) page_retire(aflt->flt_addr, PR_UE); 25727c478bd9Sstevel@tonic-gate } 25737c478bd9Sstevel@tonic-gate 257438e9bdffSmikechr /* 257538e9bdffSmikechr * Return true if the error specified in the AFSR indicates 257638e9bdffSmikechr * an E$ data error (L2$ for Cheetah/Cheetah+/Jaguar, L3$ 257738e9bdffSmikechr * for Panther, none for Jalapeno/Serrano). 257838e9bdffSmikechr */ 257938e9bdffSmikechr /* ARGSUSED */ 258038e9bdffSmikechr static int 258138e9bdffSmikechr cpu_error_is_ecache_data(int cpuid, uint64_t t_afsr) 258238e9bdffSmikechr { 258338e9bdffSmikechr #if defined(JALAPENO) || defined(SERRANO) 258438e9bdffSmikechr return (0); 258538e9bdffSmikechr #elif defined(CHEETAH_PLUS) 258638e9bdffSmikechr if (IS_PANTHER(cpunodes[cpuid].implementation)) 258738e9bdffSmikechr return ((t_afsr & C_AFSR_EXT_L3_DATA_ERRS) != 0); 258838e9bdffSmikechr return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0); 258938e9bdffSmikechr #else /* CHEETAH_PLUS */ 259038e9bdffSmikechr return ((t_afsr & C_AFSR_EC_DATA_ERRS) != 0); 259138e9bdffSmikechr #endif 259238e9bdffSmikechr } 259338e9bdffSmikechr 25947c478bd9Sstevel@tonic-gate /* 25957c478bd9Sstevel@tonic-gate * The cpu_log_err() function is called by cpu_async_log_err() to perform the 25967c478bd9Sstevel@tonic-gate * generic event post-processing for correctable and uncorrectable memory, 25977c478bd9Sstevel@tonic-gate * E$, and MTag errors. Historically this entry point was used to log bits of 25987c478bd9Sstevel@tonic-gate * common cmn_err(9F) text; now with FMA it is used to prepare 'flt' to be 25997c478bd9Sstevel@tonic-gate * converted into an ereport. In addition, it transmits the error to any 26007c478bd9Sstevel@tonic-gate * platform-specific service-processor FRU logging routines, if available. 26017c478bd9Sstevel@tonic-gate */ 26027c478bd9Sstevel@tonic-gate void 26037c478bd9Sstevel@tonic-gate cpu_log_err(struct async_flt *aflt) 26047c478bd9Sstevel@tonic-gate { 26057c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 26067c478bd9Sstevel@tonic-gate int synd_status, synd_code, afar_status; 26077c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 26087c478bd9Sstevel@tonic-gate 260938e9bdffSmikechr if (cpu_error_is_ecache_data(aflt->flt_inst, ch_flt->flt_bit)) 261038e9bdffSmikechr aflt->flt_status |= ECC_ECACHE; 261138e9bdffSmikechr else 261238e9bdffSmikechr aflt->flt_status &= ~ECC_ECACHE; 26137c478bd9Sstevel@tonic-gate /* 26147c478bd9Sstevel@tonic-gate * Determine syndrome status. 26157c478bd9Sstevel@tonic-gate */ 26167c478bd9Sstevel@tonic-gate synd_status = afsr_to_synd_status(aflt->flt_inst, 26177c478bd9Sstevel@tonic-gate ch_flt->afsr_errs, ch_flt->flt_bit); 26187c478bd9Sstevel@tonic-gate 26197c478bd9Sstevel@tonic-gate /* 26207c478bd9Sstevel@tonic-gate * Determine afar status. 26217c478bd9Sstevel@tonic-gate */ 26227c478bd9Sstevel@tonic-gate if (pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)) 26237c478bd9Sstevel@tonic-gate afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 2624cbaac45eSkm ch_flt->flt_bit); 26257c478bd9Sstevel@tonic-gate else 26267c478bd9Sstevel@tonic-gate afar_status = AFLT_STAT_INVALID; 26277c478bd9Sstevel@tonic-gate 262893743541Smb synd_code = synd_to_synd_code(synd_status, 262993743541Smb aflt->flt_synd, ch_flt->flt_bit); 263093743541Smb 26317c478bd9Sstevel@tonic-gate /* 26327c478bd9Sstevel@tonic-gate * If afar status is not invalid do a unum lookup. 26337c478bd9Sstevel@tonic-gate */ 26347c478bd9Sstevel@tonic-gate if (afar_status != AFLT_STAT_INVALID) { 263593743541Smb (void) cpu_get_mem_unum_synd(synd_code, aflt, unum); 26367c478bd9Sstevel@tonic-gate } else { 26377c478bd9Sstevel@tonic-gate unum[0] = '\0'; 26387c478bd9Sstevel@tonic-gate } 26397c478bd9Sstevel@tonic-gate 26407c478bd9Sstevel@tonic-gate /* 26417c478bd9Sstevel@tonic-gate * Do not send the fruid message (plat_ecc_error_data_t) 26427c478bd9Sstevel@tonic-gate * to the SC if it can handle the enhanced error information 26437c478bd9Sstevel@tonic-gate * (plat_ecc_error2_data_t) or when the tunable 26447c478bd9Sstevel@tonic-gate * ecc_log_fruid_enable is set to 0. 26457c478bd9Sstevel@tonic-gate */ 26467c478bd9Sstevel@tonic-gate 26477c478bd9Sstevel@tonic-gate if (&plat_ecc_capability_sc_get && 26487c478bd9Sstevel@tonic-gate plat_ecc_capability_sc_get(PLAT_ECC_ERROR_MESSAGE)) { 26497c478bd9Sstevel@tonic-gate if (&plat_log_fruid_error) 26507c478bd9Sstevel@tonic-gate plat_log_fruid_error(synd_code, aflt, unum, 26517c478bd9Sstevel@tonic-gate ch_flt->flt_bit); 26527c478bd9Sstevel@tonic-gate } 26537c478bd9Sstevel@tonic-gate 26547c478bd9Sstevel@tonic-gate if (aflt->flt_func != NULL) 26557c478bd9Sstevel@tonic-gate aflt->flt_func(aflt, unum); 26567c478bd9Sstevel@tonic-gate 26577c478bd9Sstevel@tonic-gate if (afar_status != AFLT_STAT_INVALID) 26587c478bd9Sstevel@tonic-gate cpu_log_diag_info(ch_flt); 26597c478bd9Sstevel@tonic-gate 26607c478bd9Sstevel@tonic-gate /* 26617c478bd9Sstevel@tonic-gate * If we have a CEEN error , we do not reenable CEEN until after 26627c478bd9Sstevel@tonic-gate * we exit the trap handler. Otherwise, another error may 26637c478bd9Sstevel@tonic-gate * occur causing the handler to be entered recursively. 26647c478bd9Sstevel@tonic-gate * We set a timeout to trigger in cpu_ceen_delay_secs seconds, 26657c478bd9Sstevel@tonic-gate * to try and ensure that the CPU makes progress in the face 26667c478bd9Sstevel@tonic-gate * of a CE storm. 26677c478bd9Sstevel@tonic-gate */ 26687c478bd9Sstevel@tonic-gate if (ch_flt->flt_trapped_ce & CE_CEEN_DEFER) { 26697c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, 2670f47a9c50Smathue (void *)(uintptr_t)aflt->flt_inst, 26717c478bd9Sstevel@tonic-gate drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 26727c478bd9Sstevel@tonic-gate } 26737c478bd9Sstevel@tonic-gate } 26747c478bd9Sstevel@tonic-gate 26757c478bd9Sstevel@tonic-gate /* 26767c478bd9Sstevel@tonic-gate * Invoked by error_init() early in startup and therefore before 26777c478bd9Sstevel@tonic-gate * startup_errorq() is called to drain any error Q - 26787c478bd9Sstevel@tonic-gate * 26797c478bd9Sstevel@tonic-gate * startup() 26807c478bd9Sstevel@tonic-gate * startup_end() 26817c478bd9Sstevel@tonic-gate * error_init() 26827c478bd9Sstevel@tonic-gate * cpu_error_init() 26837c478bd9Sstevel@tonic-gate * errorq_init() 26847c478bd9Sstevel@tonic-gate * errorq_drain() 26857c478bd9Sstevel@tonic-gate * start_other_cpus() 26867c478bd9Sstevel@tonic-gate * 26877c478bd9Sstevel@tonic-gate * The purpose of this routine is to create error-related taskqs. Taskqs 26887c478bd9Sstevel@tonic-gate * are used for this purpose because cpu_lock can't be grabbed from interrupt 26897c478bd9Sstevel@tonic-gate * context. 26907c478bd9Sstevel@tonic-gate */ 26917c478bd9Sstevel@tonic-gate void 26927c478bd9Sstevel@tonic-gate cpu_error_init(int items) 26937c478bd9Sstevel@tonic-gate { 26947c478bd9Sstevel@tonic-gate /* 26957c478bd9Sstevel@tonic-gate * Create taskq(s) to reenable CE 26967c478bd9Sstevel@tonic-gate */ 26977c478bd9Sstevel@tonic-gate ch_check_ce_tq = taskq_create("cheetah_check_ce", 1, minclsyspri, 26987c478bd9Sstevel@tonic-gate items, items, TASKQ_PREPOPULATE); 26997c478bd9Sstevel@tonic-gate } 27007c478bd9Sstevel@tonic-gate 27017c478bd9Sstevel@tonic-gate void 27027c478bd9Sstevel@tonic-gate cpu_ce_log_err(struct async_flt *aflt, errorq_elem_t *eqep) 27037c478bd9Sstevel@tonic-gate { 27047c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 27057c478bd9Sstevel@tonic-gate int len; 27067c478bd9Sstevel@tonic-gate 27077c478bd9Sstevel@tonic-gate switch (aflt->flt_class) { 27087c478bd9Sstevel@tonic-gate case CPU_FAULT: 27097c478bd9Sstevel@tonic-gate cpu_ereport_init(aflt); 27107c478bd9Sstevel@tonic-gate if (cpu_async_log_err(aflt, eqep)) 27117c478bd9Sstevel@tonic-gate cpu_ereport_post(aflt); 27127c478bd9Sstevel@tonic-gate break; 27137c478bd9Sstevel@tonic-gate 27147c478bd9Sstevel@tonic-gate case BUS_FAULT: 27157c478bd9Sstevel@tonic-gate if (aflt->flt_func != NULL) { 27167c478bd9Sstevel@tonic-gate (void) cpu_get_mem_unum_aflt(AFLT_STAT_VALID, aflt, 27177c478bd9Sstevel@tonic-gate unum, UNUM_NAMLEN, &len); 27187c478bd9Sstevel@tonic-gate aflt->flt_func(aflt, unum); 27197c478bd9Sstevel@tonic-gate } 27207c478bd9Sstevel@tonic-gate break; 27217c478bd9Sstevel@tonic-gate 27227c478bd9Sstevel@tonic-gate case RECIRC_CPU_FAULT: 27237c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 27247c478bd9Sstevel@tonic-gate cpu_log_err(aflt); 27257c478bd9Sstevel@tonic-gate cpu_ereport_post(aflt); 27267c478bd9Sstevel@tonic-gate break; 27277c478bd9Sstevel@tonic-gate 27287c478bd9Sstevel@tonic-gate case RECIRC_BUS_FAULT: 27297c478bd9Sstevel@tonic-gate ASSERT(aflt->flt_class != RECIRC_BUS_FAULT); 27307c478bd9Sstevel@tonic-gate /*FALLTHRU*/ 27317c478bd9Sstevel@tonic-gate default: 27327c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "discarding CE error 0x%p with invalid " 27337c478bd9Sstevel@tonic-gate "fault class (0x%x)", (void *)aflt, aflt->flt_class); 27347c478bd9Sstevel@tonic-gate return; 27357c478bd9Sstevel@tonic-gate } 27367c478bd9Sstevel@tonic-gate } 27377c478bd9Sstevel@tonic-gate 27387c478bd9Sstevel@tonic-gate /* 27397c478bd9Sstevel@tonic-gate * Scrub and classify a CE. This function must not modify the 27407c478bd9Sstevel@tonic-gate * fault structure passed to it but instead should return the classification 27417c478bd9Sstevel@tonic-gate * information. 27427c478bd9Sstevel@tonic-gate */ 27437c478bd9Sstevel@tonic-gate 27447c478bd9Sstevel@tonic-gate static uchar_t 27457c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err_common(struct async_flt *ecc, boolean_t logout_tried) 27467c478bd9Sstevel@tonic-gate { 27477c478bd9Sstevel@tonic-gate uchar_t disp = CE_XDIAG_EXTALG; 27487c478bd9Sstevel@tonic-gate on_trap_data_t otd; 27497c478bd9Sstevel@tonic-gate uint64_t orig_err; 27507c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 27517c478bd9Sstevel@tonic-gate 27527c478bd9Sstevel@tonic-gate /* 27537c478bd9Sstevel@tonic-gate * Clear CEEN. CPU CE TL > 0 trap handling will already have done 27547c478bd9Sstevel@tonic-gate * this, but our other callers have not. Disable preemption to 27557c478bd9Sstevel@tonic-gate * avoid CPU migration so that we restore CEEN on the correct 27567c478bd9Sstevel@tonic-gate * cpu later. 27577c478bd9Sstevel@tonic-gate * 27587c478bd9Sstevel@tonic-gate * CEEN is cleared so that further CEs that our instruction and 27597c478bd9Sstevel@tonic-gate * data footprint induce do not cause use to either creep down 27607c478bd9Sstevel@tonic-gate * kernel stack to the point of overflow, or do so much CE 27617c478bd9Sstevel@tonic-gate * notification as to make little real forward progress. 27627c478bd9Sstevel@tonic-gate * 27637c478bd9Sstevel@tonic-gate * NCEEN must not be cleared. However it is possible that 27647c478bd9Sstevel@tonic-gate * our accesses to the flt_addr may provoke a bus error or timeout 27657c478bd9Sstevel@tonic-gate * if the offending address has just been unconfigured as part of 27667c478bd9Sstevel@tonic-gate * a DR action. So we must operate under on_trap protection. 27677c478bd9Sstevel@tonic-gate */ 27687c478bd9Sstevel@tonic-gate kpreempt_disable(); 27697c478bd9Sstevel@tonic-gate orig_err = get_error_enable(); 27707c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 2771cbaac45eSkm set_error_enable(orig_err & ~EN_REG_CEEN); 27727c478bd9Sstevel@tonic-gate 27737c478bd9Sstevel@tonic-gate /* 27747c478bd9Sstevel@tonic-gate * Our classification algorithm includes the line state before 27757c478bd9Sstevel@tonic-gate * the scrub; we'd like this captured after the detection and 27767c478bd9Sstevel@tonic-gate * before the algorithm below - the earlier the better. 27777c478bd9Sstevel@tonic-gate * 27787c478bd9Sstevel@tonic-gate * If we've come from a cpu CE trap then this info already exists 27797c478bd9Sstevel@tonic-gate * in the cpu logout area. 27807c478bd9Sstevel@tonic-gate * 27817c478bd9Sstevel@tonic-gate * For a CE detected by memscrub for which there was no trap 27827c478bd9Sstevel@tonic-gate * (running with CEEN off) cpu_log_and_clear_ce has called 27837c478bd9Sstevel@tonic-gate * cpu_ce_delayed_ec_logout to capture some cache data, and 27847c478bd9Sstevel@tonic-gate * marked the fault structure as incomplete as a flag to later 27857c478bd9Sstevel@tonic-gate * logging code. 27867c478bd9Sstevel@tonic-gate * 27877c478bd9Sstevel@tonic-gate * If called directly from an IO detected CE there has been 27887c478bd9Sstevel@tonic-gate * no line data capture. In this case we logout to the cpu logout 27897c478bd9Sstevel@tonic-gate * area - that's appropriate since it's the cpu cache data we need 27907c478bd9Sstevel@tonic-gate * for classification. We thus borrow the cpu logout area for a 27917c478bd9Sstevel@tonic-gate * short time, and cpu_ce_delayed_ec_logout will mark it as busy in 27927c478bd9Sstevel@tonic-gate * this time (we will invalidate it again below). 27937c478bd9Sstevel@tonic-gate * 27947c478bd9Sstevel@tonic-gate * If called from the partner check xcall handler then this cpu 27957c478bd9Sstevel@tonic-gate * (the partner) has not necessarily experienced a CE at this 27967c478bd9Sstevel@tonic-gate * address. But we want to capture line state before its scrub 27977c478bd9Sstevel@tonic-gate * attempt since we use that in our classification. 27987c478bd9Sstevel@tonic-gate */ 27997c478bd9Sstevel@tonic-gate if (logout_tried == B_FALSE) { 28007c478bd9Sstevel@tonic-gate if (!cpu_ce_delayed_ec_logout(ecc->flt_addr)) 28017c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_NOLOGOUT; 28027c478bd9Sstevel@tonic-gate } 28037c478bd9Sstevel@tonic-gate 28047c478bd9Sstevel@tonic-gate /* 28057c478bd9Sstevel@tonic-gate * Scrub memory, then check AFSR for errors. The AFAR we scrub may 28067c478bd9Sstevel@tonic-gate * no longer be valid (if DR'd since the initial event) so we 28077c478bd9Sstevel@tonic-gate * perform this scrub under on_trap protection. If this access is 28087c478bd9Sstevel@tonic-gate * ok then further accesses below will also be ok - DR cannot 28097c478bd9Sstevel@tonic-gate * proceed while this thread is active (preemption is disabled); 28107c478bd9Sstevel@tonic-gate * to be safe we'll nonetheless use on_trap again below. 28117c478bd9Sstevel@tonic-gate */ 28127c478bd9Sstevel@tonic-gate if (!on_trap(&otd, OT_DATA_ACCESS)) { 28137c478bd9Sstevel@tonic-gate cpu_scrubphys(ecc); 28147c478bd9Sstevel@tonic-gate } else { 28157c478bd9Sstevel@tonic-gate no_trap(); 28167c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 2817cbaac45eSkm set_error_enable(orig_err); 28187c478bd9Sstevel@tonic-gate kpreempt_enable(); 28197c478bd9Sstevel@tonic-gate return (disp); 28207c478bd9Sstevel@tonic-gate } 28217c478bd9Sstevel@tonic-gate no_trap(); 28227c478bd9Sstevel@tonic-gate 28237c478bd9Sstevel@tonic-gate /* 28247c478bd9Sstevel@tonic-gate * Did the casx read of the scrub log a CE that matches the AFAR? 28257c478bd9Sstevel@tonic-gate * Note that it's quite possible that the read sourced the data from 28267c478bd9Sstevel@tonic-gate * another cpu. 28277c478bd9Sstevel@tonic-gate */ 28287c478bd9Sstevel@tonic-gate if (clear_ecc(ecc)) 28297c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_CE1; 28307c478bd9Sstevel@tonic-gate 28317c478bd9Sstevel@tonic-gate /* 28327c478bd9Sstevel@tonic-gate * Read the data again. This time the read is very likely to 28337c478bd9Sstevel@tonic-gate * come from memory since the scrub induced a writeback to memory. 28347c478bd9Sstevel@tonic-gate */ 28357c478bd9Sstevel@tonic-gate if (!on_trap(&otd, OT_DATA_ACCESS)) { 28367c478bd9Sstevel@tonic-gate (void) lddphys(P2ALIGN(ecc->flt_addr, 8)); 28377c478bd9Sstevel@tonic-gate } else { 28387c478bd9Sstevel@tonic-gate no_trap(); 28397c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 2840cbaac45eSkm set_error_enable(orig_err); 28417c478bd9Sstevel@tonic-gate kpreempt_enable(); 28427c478bd9Sstevel@tonic-gate return (disp); 28437c478bd9Sstevel@tonic-gate } 28447c478bd9Sstevel@tonic-gate no_trap(); 28457c478bd9Sstevel@tonic-gate 28467c478bd9Sstevel@tonic-gate /* Did that read induce a CE that matches the AFAR? */ 28477c478bd9Sstevel@tonic-gate if (clear_ecc(ecc)) 28487c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_CE2; 28497c478bd9Sstevel@tonic-gate 28507c478bd9Sstevel@tonic-gate /* 28517c478bd9Sstevel@tonic-gate * Look at the logout information and record whether we found the 28527c478bd9Sstevel@tonic-gate * line in l2/l3 cache. For Panther we are interested in whether 28537c478bd9Sstevel@tonic-gate * we found it in either cache (it won't reside in both but 28547c478bd9Sstevel@tonic-gate * it is possible to read it that way given the moving target). 28557c478bd9Sstevel@tonic-gate */ 28567c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE(CPU) ? CPU_PRIVATE_PTR(CPU, chpr_cecc_logout) : NULL; 28577c478bd9Sstevel@tonic-gate if (!(disp & CE_XDIAG_NOLOGOUT) && clop && 28587c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar != LOGOUT_INVALID) { 28597c478bd9Sstevel@tonic-gate int hit, level; 28607c478bd9Sstevel@tonic-gate int state; 28617c478bd9Sstevel@tonic-gate int totalsize; 28627c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp; 28637c478bd9Sstevel@tonic-gate 28647c478bd9Sstevel@tonic-gate /* 28657c478bd9Sstevel@tonic-gate * If hit is nonzero then a match was found and hit will 28667c478bd9Sstevel@tonic-gate * be one greater than the index which hit. For Panther we 28677c478bd9Sstevel@tonic-gate * also need to pay attention to level to see which of l2$ or 28687c478bd9Sstevel@tonic-gate * l3$ it hit in. 28697c478bd9Sstevel@tonic-gate */ 28707c478bd9Sstevel@tonic-gate hit = cpu_matching_ecache_line(ecc->flt_addr, &clop->clo_data, 28717c478bd9Sstevel@tonic-gate 0, &level); 28727c478bd9Sstevel@tonic-gate 28737c478bd9Sstevel@tonic-gate if (hit) { 28747c478bd9Sstevel@tonic-gate --hit; 28757c478bd9Sstevel@tonic-gate disp |= CE_XDIAG_AFARMATCH; 28767c478bd9Sstevel@tonic-gate 28777c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 28787c478bd9Sstevel@tonic-gate if (level == 2) 28797c478bd9Sstevel@tonic-gate ecp = &clop->clo_data.chd_l2_data[hit]; 28807c478bd9Sstevel@tonic-gate else 28817c478bd9Sstevel@tonic-gate ecp = &clop->clo_data.chd_ec_data[hit]; 28827c478bd9Sstevel@tonic-gate } else { 28837c478bd9Sstevel@tonic-gate ASSERT(level == 2); 28847c478bd9Sstevel@tonic-gate ecp = &clop->clo_data.chd_ec_data[hit]; 28857c478bd9Sstevel@tonic-gate } 28867c478bd9Sstevel@tonic-gate totalsize = cpunodes[CPU->cpu_id].ecache_size; 28877c478bd9Sstevel@tonic-gate state = cpu_ectag_pa_to_subblk_state(totalsize, 28887c478bd9Sstevel@tonic-gate ecc->flt_addr, ecp->ec_tag); 28897c478bd9Sstevel@tonic-gate 28907c478bd9Sstevel@tonic-gate /* 28917c478bd9Sstevel@tonic-gate * Cheetah variants use different state encodings - 28927c478bd9Sstevel@tonic-gate * the CH_ECSTATE_* defines vary depending on the 28937c478bd9Sstevel@tonic-gate * module we're compiled for. Translate into our 28947c478bd9Sstevel@tonic-gate * one true version. Conflate Owner-Shared state 28957c478bd9Sstevel@tonic-gate * of SSM mode with Owner as victimisation of such 28967c478bd9Sstevel@tonic-gate * lines may cause a writeback. 28977c478bd9Sstevel@tonic-gate */ 28987c478bd9Sstevel@tonic-gate switch (state) { 28997c478bd9Sstevel@tonic-gate case CH_ECSTATE_MOD: 29007c478bd9Sstevel@tonic-gate disp |= EC_STATE_M; 29017c478bd9Sstevel@tonic-gate break; 29027c478bd9Sstevel@tonic-gate 29037c478bd9Sstevel@tonic-gate case CH_ECSTATE_OWN: 29047c478bd9Sstevel@tonic-gate case CH_ECSTATE_OWS: 29057c478bd9Sstevel@tonic-gate disp |= EC_STATE_O; 29067c478bd9Sstevel@tonic-gate break; 29077c478bd9Sstevel@tonic-gate 29087c478bd9Sstevel@tonic-gate case CH_ECSTATE_EXL: 29097c478bd9Sstevel@tonic-gate disp |= EC_STATE_E; 29107c478bd9Sstevel@tonic-gate break; 29117c478bd9Sstevel@tonic-gate 29127c478bd9Sstevel@tonic-gate case CH_ECSTATE_SHR: 29137c478bd9Sstevel@tonic-gate disp |= EC_STATE_S; 29147c478bd9Sstevel@tonic-gate break; 29157c478bd9Sstevel@tonic-gate 29167c478bd9Sstevel@tonic-gate default: 29177c478bd9Sstevel@tonic-gate disp |= EC_STATE_I; 29187c478bd9Sstevel@tonic-gate break; 29197c478bd9Sstevel@tonic-gate } 29207c478bd9Sstevel@tonic-gate } 29217c478bd9Sstevel@tonic-gate 29227c478bd9Sstevel@tonic-gate /* 29237c478bd9Sstevel@tonic-gate * If we initiated the delayed logout then we are responsible 29247c478bd9Sstevel@tonic-gate * for invalidating the logout area. 29257c478bd9Sstevel@tonic-gate */ 29267c478bd9Sstevel@tonic-gate if (logout_tried == B_FALSE) { 29277c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 29287c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 29297c478bd9Sstevel@tonic-gate } 29307c478bd9Sstevel@tonic-gate } 29317c478bd9Sstevel@tonic-gate 29327c478bd9Sstevel@tonic-gate /* 29337c478bd9Sstevel@tonic-gate * Re-enable CEEN if we turned it off. 29347c478bd9Sstevel@tonic-gate */ 29357c478bd9Sstevel@tonic-gate if (orig_err & EN_REG_CEEN) 2936cbaac45eSkm set_error_enable(orig_err); 29377c478bd9Sstevel@tonic-gate kpreempt_enable(); 29387c478bd9Sstevel@tonic-gate 29397c478bd9Sstevel@tonic-gate return (disp); 29407c478bd9Sstevel@tonic-gate } 29417c478bd9Sstevel@tonic-gate 29427c478bd9Sstevel@tonic-gate /* 29437c478bd9Sstevel@tonic-gate * Scrub a correctable memory error and collect data for classification 29447c478bd9Sstevel@tonic-gate * of CE type. This function is called in the detection path, ie tl0 handling 29457c478bd9Sstevel@tonic-gate * of a correctable error trap (cpus) or interrupt (IO) at high PIL. 29467c478bd9Sstevel@tonic-gate */ 29477c478bd9Sstevel@tonic-gate void 29487c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err(struct async_flt *ecc, boolean_t logout_tried) 29497c478bd9Sstevel@tonic-gate { 29507c478bd9Sstevel@tonic-gate /* 29517c478bd9Sstevel@tonic-gate * Cheetah CE classification does not set any bits in flt_status. 29527c478bd9Sstevel@tonic-gate * Instead we will record classification datapoints in flt_disp. 29537c478bd9Sstevel@tonic-gate */ 29547c478bd9Sstevel@tonic-gate ecc->flt_status &= ~(ECC_INTERMITTENT | ECC_PERSISTENT | ECC_STICKY); 29557c478bd9Sstevel@tonic-gate 29567c478bd9Sstevel@tonic-gate /* 29577c478bd9Sstevel@tonic-gate * To check if the error detected by IO is persistent, sticky or 29587c478bd9Sstevel@tonic-gate * intermittent. This is noticed by clear_ecc(). 29597c478bd9Sstevel@tonic-gate */ 29607c478bd9Sstevel@tonic-gate if (ecc->flt_status & ECC_IOBUS) 29617c478bd9Sstevel@tonic-gate ecc->flt_stat = C_AFSR_MEMORY; 29627c478bd9Sstevel@tonic-gate 29637c478bd9Sstevel@tonic-gate /* 29647c478bd9Sstevel@tonic-gate * Record information from this first part of the algorithm in 29657c478bd9Sstevel@tonic-gate * flt_disp. 29667c478bd9Sstevel@tonic-gate */ 29677c478bd9Sstevel@tonic-gate ecc->flt_disp = cpu_ce_scrub_mem_err_common(ecc, logout_tried); 29687c478bd9Sstevel@tonic-gate } 29697c478bd9Sstevel@tonic-gate 29707c478bd9Sstevel@tonic-gate /* 29717c478bd9Sstevel@tonic-gate * Select a partner to perform a further CE classification check from. 29727c478bd9Sstevel@tonic-gate * Must be called with kernel preemption disabled (to stop the cpu list 29737c478bd9Sstevel@tonic-gate * from changing). The detecting cpu we are partnering has cpuid 29747c478bd9Sstevel@tonic-gate * aflt->flt_inst; we might not be running on the detecting cpu. 29757c478bd9Sstevel@tonic-gate * 29767c478bd9Sstevel@tonic-gate * Restrict choice to active cpus in the same cpu partition as ourselves in 29777c478bd9Sstevel@tonic-gate * an effort to stop bad cpus in one partition causing other partitions to 29787c478bd9Sstevel@tonic-gate * perform excessive diagnostic activity. Actually since the errorq drain 29797c478bd9Sstevel@tonic-gate * is run from a softint most of the time and that is a global mechanism 29807c478bd9Sstevel@tonic-gate * this isolation is only partial. Return NULL if we fail to find a 29817c478bd9Sstevel@tonic-gate * suitable partner. 29827c478bd9Sstevel@tonic-gate * 29837c478bd9Sstevel@tonic-gate * We prefer a partner that is in a different latency group to ourselves as 29847c478bd9Sstevel@tonic-gate * we will share fewer datapaths. If such a partner is unavailable then 29857c478bd9Sstevel@tonic-gate * choose one in the same lgroup but prefer a different chip and only allow 29867c478bd9Sstevel@tonic-gate * a sibling core if flags includes PTNR_SIBLINGOK. If all else fails and 29877c478bd9Sstevel@tonic-gate * flags includes PTNR_SELFOK then permit selection of the original detector. 29887c478bd9Sstevel@tonic-gate * 29897c478bd9Sstevel@tonic-gate * We keep a cache of the last partner selected for a cpu, and we'll try to 29907c478bd9Sstevel@tonic-gate * use that previous partner if no more than cpu_ce_ptnr_cachetime_sec seconds 29917c478bd9Sstevel@tonic-gate * have passed since that selection was made. This provides the benefit 29927c478bd9Sstevel@tonic-gate * of the point-of-view of different partners over time but without 29937c478bd9Sstevel@tonic-gate * requiring frequent cpu list traversals. 29947c478bd9Sstevel@tonic-gate */ 29957c478bd9Sstevel@tonic-gate 29967c478bd9Sstevel@tonic-gate #define PTNR_SIBLINGOK 0x1 /* Allow selection of sibling core */ 29977c478bd9Sstevel@tonic-gate #define PTNR_SELFOK 0x2 /* Allow selection of cpu to "partner" itself */ 29987c478bd9Sstevel@tonic-gate 29997c478bd9Sstevel@tonic-gate static cpu_t * 30007c478bd9Sstevel@tonic-gate ce_ptnr_select(struct async_flt *aflt, int flags, int *typep) 30017c478bd9Sstevel@tonic-gate { 30027c478bd9Sstevel@tonic-gate cpu_t *sp, *dtcr, *ptnr, *locptnr, *sibptnr; 30037c478bd9Sstevel@tonic-gate hrtime_t lasttime, thistime; 30047c478bd9Sstevel@tonic-gate 30057c478bd9Sstevel@tonic-gate ASSERT(curthread->t_preempt > 0 || getpil() >= DISP_LEVEL); 30067c478bd9Sstevel@tonic-gate 30077c478bd9Sstevel@tonic-gate dtcr = cpu[aflt->flt_inst]; 30087c478bd9Sstevel@tonic-gate 30097c478bd9Sstevel@tonic-gate /* 30107c478bd9Sstevel@tonic-gate * Short-circuit for the following cases: 30117c478bd9Sstevel@tonic-gate * . the dtcr is not flagged active 30127c478bd9Sstevel@tonic-gate * . there is just one cpu present 30137c478bd9Sstevel@tonic-gate * . the detector has disappeared 30147c478bd9Sstevel@tonic-gate * . we were given a bad flt_inst cpuid; this should not happen 30157c478bd9Sstevel@tonic-gate * (eg PCI code now fills flt_inst) but if it does it is no 30167c478bd9Sstevel@tonic-gate * reason to panic. 30177c478bd9Sstevel@tonic-gate * . there is just one cpu left online in the cpu partition 30187c478bd9Sstevel@tonic-gate * 30197c478bd9Sstevel@tonic-gate * If we return NULL after this point then we do not update the 30207c478bd9Sstevel@tonic-gate * chpr_ceptnr_seltime which will cause us to perform a full lookup 30217c478bd9Sstevel@tonic-gate * again next time; this is the case where the only other cpu online 30227c478bd9Sstevel@tonic-gate * in the detector's partition is on the same chip as the detector 30237c478bd9Sstevel@tonic-gate * and since CEEN re-enable is throttled even that case should not 30247c478bd9Sstevel@tonic-gate * hurt performance. 30257c478bd9Sstevel@tonic-gate */ 30267c478bd9Sstevel@tonic-gate if (dtcr == NULL || !cpu_flagged_active(dtcr->cpu_flags)) { 30277c478bd9Sstevel@tonic-gate return (NULL); 30287c478bd9Sstevel@tonic-gate } 30297c478bd9Sstevel@tonic-gate if (ncpus == 1 || dtcr->cpu_part->cp_ncpus == 1) { 30307c478bd9Sstevel@tonic-gate if (flags & PTNR_SELFOK) { 30317c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SELF; 30327c478bd9Sstevel@tonic-gate return (dtcr); 30337c478bd9Sstevel@tonic-gate } else { 30347c478bd9Sstevel@tonic-gate return (NULL); 30357c478bd9Sstevel@tonic-gate } 30367c478bd9Sstevel@tonic-gate } 30377c478bd9Sstevel@tonic-gate 30387c478bd9Sstevel@tonic-gate thistime = gethrtime(); 30397c478bd9Sstevel@tonic-gate lasttime = CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime); 30407c478bd9Sstevel@tonic-gate 30417c478bd9Sstevel@tonic-gate /* 30427c478bd9Sstevel@tonic-gate * Select a starting point. 30437c478bd9Sstevel@tonic-gate */ 30447c478bd9Sstevel@tonic-gate if (!lasttime) { 30457c478bd9Sstevel@tonic-gate /* 30467c478bd9Sstevel@tonic-gate * We've never selected a partner for this detector before. 30477c478bd9Sstevel@tonic-gate * Start the scan at the next online cpu in the same cpu 30487c478bd9Sstevel@tonic-gate * partition. 30497c478bd9Sstevel@tonic-gate */ 30507c478bd9Sstevel@tonic-gate sp = dtcr->cpu_next_part; 30517c478bd9Sstevel@tonic-gate } else if (thistime - lasttime < cpu_ce_ptnr_cachetime_sec * NANOSEC) { 30527c478bd9Sstevel@tonic-gate /* 30537c478bd9Sstevel@tonic-gate * Our last selection has not aged yet. If this partner: 30547c478bd9Sstevel@tonic-gate * . is still a valid cpu, 30557c478bd9Sstevel@tonic-gate * . is still in the same partition as the detector 30567c478bd9Sstevel@tonic-gate * . is still marked active 30577c478bd9Sstevel@tonic-gate * . satisfies the 'flags' argument criteria 30587c478bd9Sstevel@tonic-gate * then select it again without updating the timestamp. 30597c478bd9Sstevel@tonic-gate */ 30607c478bd9Sstevel@tonic-gate sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 30617c478bd9Sstevel@tonic-gate if (sp == NULL || sp->cpu_part != dtcr->cpu_part || 30627c478bd9Sstevel@tonic-gate !cpu_flagged_active(sp->cpu_flags) || 30637c478bd9Sstevel@tonic-gate (sp == dtcr && !(flags & PTNR_SELFOK)) || 3064fb2f18f8Sesaxe (pg_plat_cpus_share(sp, dtcr, PGHW_CHIP) && 30657c478bd9Sstevel@tonic-gate !(flags & PTNR_SIBLINGOK))) { 30667c478bd9Sstevel@tonic-gate sp = dtcr->cpu_next_part; 30677c478bd9Sstevel@tonic-gate } else { 30687c478bd9Sstevel@tonic-gate if (sp->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 30697c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_REMOTE; 30707c478bd9Sstevel@tonic-gate } else if (sp == dtcr) { 30717c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SELF; 3072fb2f18f8Sesaxe } else if (pg_plat_cpus_share(sp, dtcr, PGHW_CHIP)) { 30737c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SIBLING; 30747c478bd9Sstevel@tonic-gate } else { 30757c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_LOCAL; 30767c478bd9Sstevel@tonic-gate } 30777c478bd9Sstevel@tonic-gate return (sp); 30787c478bd9Sstevel@tonic-gate } 30797c478bd9Sstevel@tonic-gate } else { 30807c478bd9Sstevel@tonic-gate /* 30817c478bd9Sstevel@tonic-gate * Our last selection has aged. If it is nonetheless still a 30827c478bd9Sstevel@tonic-gate * valid cpu then start the scan at the next cpu in the 30837c478bd9Sstevel@tonic-gate * partition after our last partner. If the last selection 30847c478bd9Sstevel@tonic-gate * is no longer a valid cpu then go with our default. In 30857c478bd9Sstevel@tonic-gate * this way we slowly cycle through possible partners to 30867c478bd9Sstevel@tonic-gate * obtain multiple viewpoints over time. 30877c478bd9Sstevel@tonic-gate */ 30887c478bd9Sstevel@tonic-gate sp = cpu[CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id)]; 30897c478bd9Sstevel@tonic-gate if (sp == NULL) { 30907c478bd9Sstevel@tonic-gate sp = dtcr->cpu_next_part; 30917c478bd9Sstevel@tonic-gate } else { 30927c478bd9Sstevel@tonic-gate sp = sp->cpu_next_part; /* may be dtcr */ 30937c478bd9Sstevel@tonic-gate if (sp->cpu_part != dtcr->cpu_part) 30947c478bd9Sstevel@tonic-gate sp = dtcr; 30957c478bd9Sstevel@tonic-gate } 30967c478bd9Sstevel@tonic-gate } 30977c478bd9Sstevel@tonic-gate 30987c478bd9Sstevel@tonic-gate /* 30997c478bd9Sstevel@tonic-gate * We have a proposed starting point for our search, but if this 31007c478bd9Sstevel@tonic-gate * cpu is offline then its cpu_next_part will point to itself 31017c478bd9Sstevel@tonic-gate * so we can't use that to iterate over cpus in this partition in 31027c478bd9Sstevel@tonic-gate * the loop below. We still want to avoid iterating over cpus not 31037c478bd9Sstevel@tonic-gate * in our partition, so in the case that our starting point is offline 31047c478bd9Sstevel@tonic-gate * we will repoint it to be the detector itself; and if the detector 31057c478bd9Sstevel@tonic-gate * happens to be offline we'll return NULL from the following loop. 31067c478bd9Sstevel@tonic-gate */ 31077c478bd9Sstevel@tonic-gate if (!cpu_flagged_active(sp->cpu_flags)) { 31087c478bd9Sstevel@tonic-gate sp = dtcr; 31097c478bd9Sstevel@tonic-gate } 31107c478bd9Sstevel@tonic-gate 31117c478bd9Sstevel@tonic-gate ptnr = sp; 31127c478bd9Sstevel@tonic-gate locptnr = NULL; 31137c478bd9Sstevel@tonic-gate sibptnr = NULL; 31147c478bd9Sstevel@tonic-gate do { 31157c478bd9Sstevel@tonic-gate if (ptnr == dtcr || !cpu_flagged_active(ptnr->cpu_flags)) 31167c478bd9Sstevel@tonic-gate continue; 31177c478bd9Sstevel@tonic-gate if (ptnr->cpu_lpl->lpl_lgrp != dtcr->cpu_lpl->lpl_lgrp) { 31187c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = ptnr->cpu_id; 31197c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 31207c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_REMOTE; 31217c478bd9Sstevel@tonic-gate return (ptnr); 31227c478bd9Sstevel@tonic-gate } 3123fb2f18f8Sesaxe if (pg_plat_cpus_share(ptnr, dtcr, PGHW_CHIP)) { 31247c478bd9Sstevel@tonic-gate if (sibptnr == NULL) 31257c478bd9Sstevel@tonic-gate sibptnr = ptnr; 31267c478bd9Sstevel@tonic-gate continue; 31277c478bd9Sstevel@tonic-gate } 31287c478bd9Sstevel@tonic-gate if (locptnr == NULL) 31297c478bd9Sstevel@tonic-gate locptnr = ptnr; 31307c478bd9Sstevel@tonic-gate } while ((ptnr = ptnr->cpu_next_part) != sp); 31317c478bd9Sstevel@tonic-gate 31327c478bd9Sstevel@tonic-gate /* 31337c478bd9Sstevel@tonic-gate * A foreign partner has already been returned if one was available. 31347c478bd9Sstevel@tonic-gate * 31357c478bd9Sstevel@tonic-gate * If locptnr is not NULL it is a cpu in the same lgroup as the 31367c478bd9Sstevel@tonic-gate * detector, is active, and is not a sibling of the detector. 31377c478bd9Sstevel@tonic-gate * 31387c478bd9Sstevel@tonic-gate * If sibptnr is not NULL it is a sibling of the detector, and is 31397c478bd9Sstevel@tonic-gate * active. 31407c478bd9Sstevel@tonic-gate * 31417c478bd9Sstevel@tonic-gate * If we have to resort to using the detector itself we have already 31427c478bd9Sstevel@tonic-gate * checked that it is active. 31437c478bd9Sstevel@tonic-gate */ 31447c478bd9Sstevel@tonic-gate if (locptnr) { 31457c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = locptnr->cpu_id; 31467c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 31477c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_LOCAL; 31487c478bd9Sstevel@tonic-gate return (locptnr); 31497c478bd9Sstevel@tonic-gate } else if (sibptnr && flags & PTNR_SIBLINGOK) { 31507c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = sibptnr->cpu_id; 31517c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 31527c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SIBLING; 31537c478bd9Sstevel@tonic-gate return (sibptnr); 31547c478bd9Sstevel@tonic-gate } else if (flags & PTNR_SELFOK) { 31557c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_id) = dtcr->cpu_id; 31567c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(dtcr, chpr_ceptnr_seltime) = thistime; 31577c478bd9Sstevel@tonic-gate *typep = CE_XDIAG_PTNR_SELF; 31587c478bd9Sstevel@tonic-gate return (dtcr); 31597c478bd9Sstevel@tonic-gate } 31607c478bd9Sstevel@tonic-gate 31617c478bd9Sstevel@tonic-gate return (NULL); 31627c478bd9Sstevel@tonic-gate } 31637c478bd9Sstevel@tonic-gate 31647c478bd9Sstevel@tonic-gate /* 31657c478bd9Sstevel@tonic-gate * Cross call handler that is requested to run on the designated partner of 31667c478bd9Sstevel@tonic-gate * a cpu that experienced a possibly sticky or possibly persistnet CE. 31677c478bd9Sstevel@tonic-gate */ 31687c478bd9Sstevel@tonic-gate static void 31697c478bd9Sstevel@tonic-gate ce_ptnrchk_xc(struct async_flt *aflt, uchar_t *dispp) 31707c478bd9Sstevel@tonic-gate { 31717c478bd9Sstevel@tonic-gate *dispp = cpu_ce_scrub_mem_err_common(aflt, B_FALSE); 31727c478bd9Sstevel@tonic-gate } 31737c478bd9Sstevel@tonic-gate 31747c478bd9Sstevel@tonic-gate /* 31757c478bd9Sstevel@tonic-gate * The associated errorqs are never destroyed so we do not need to deal with 31767c478bd9Sstevel@tonic-gate * them disappearing before this timeout fires. If the affected memory 31777c478bd9Sstevel@tonic-gate * has been DR'd out since the original event the scrub algrithm will catch 31787c478bd9Sstevel@tonic-gate * any errors and return null disposition info. If the original detecting 31797c478bd9Sstevel@tonic-gate * cpu has been DR'd out then ereport detector info will not be able to 31807c478bd9Sstevel@tonic-gate * lookup CPU type; with a small timeout this is unlikely. 31817c478bd9Sstevel@tonic-gate */ 31827c478bd9Sstevel@tonic-gate static void 31837c478bd9Sstevel@tonic-gate ce_lkychk_cb(ce_lkychk_cb_t *cbarg) 31847c478bd9Sstevel@tonic-gate { 31857c478bd9Sstevel@tonic-gate struct async_flt *aflt = cbarg->lkycb_aflt; 31867c478bd9Sstevel@tonic-gate uchar_t disp; 31877c478bd9Sstevel@tonic-gate cpu_t *cp; 31887c478bd9Sstevel@tonic-gate int ptnrtype; 31897c478bd9Sstevel@tonic-gate 31907c478bd9Sstevel@tonic-gate kpreempt_disable(); 31917c478bd9Sstevel@tonic-gate if (cp = ce_ptnr_select(aflt, PTNR_SIBLINGOK | PTNR_SELFOK, 31927c478bd9Sstevel@tonic-gate &ptnrtype)) { 31937c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, (uint64_t)aflt, 31947c478bd9Sstevel@tonic-gate (uint64_t)&disp); 31957c478bd9Sstevel@tonic-gate CE_XDIAG_SETLKYINFO(aflt->flt_disp, disp); 31967c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 31977c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 31987c478bd9Sstevel@tonic-gate } else { 31997c478bd9Sstevel@tonic-gate ce_xdiag_lkydrops++; 32007c478bd9Sstevel@tonic-gate if (ncpus > 1) 32017c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 32027c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_NOPTNR); 32037c478bd9Sstevel@tonic-gate } 32047c478bd9Sstevel@tonic-gate kpreempt_enable(); 32057c478bd9Sstevel@tonic-gate 32067c478bd9Sstevel@tonic-gate errorq_commit(cbarg->lkycb_eqp, cbarg->lkycb_eqep, ERRORQ_ASYNC); 32077c478bd9Sstevel@tonic-gate kmem_free(cbarg, sizeof (ce_lkychk_cb_t)); 32087c478bd9Sstevel@tonic-gate } 32097c478bd9Sstevel@tonic-gate 32107c478bd9Sstevel@tonic-gate /* 32117c478bd9Sstevel@tonic-gate * Called from errorq drain code when processing a CE error, both from 32127c478bd9Sstevel@tonic-gate * CPU and PCI drain functions. Decide what further classification actions, 32137c478bd9Sstevel@tonic-gate * if any, we will perform. Perform immediate actions now, and schedule 32147c478bd9Sstevel@tonic-gate * delayed actions as required. Note that we are no longer necessarily running 32157c478bd9Sstevel@tonic-gate * on the detecting cpu, and that the async_flt structure will not persist on 32167c478bd9Sstevel@tonic-gate * return from this function. 32177c478bd9Sstevel@tonic-gate * 32187c478bd9Sstevel@tonic-gate * Calls to this function should aim to be self-throtlling in some way. With 32197c478bd9Sstevel@tonic-gate * the delayed re-enable of CEEN the absolute rate of calls should not 32207c478bd9Sstevel@tonic-gate * be excessive. Callers should also avoid performing in-depth classification 32217c478bd9Sstevel@tonic-gate * for events in pages that are already known to be suspect. 32227c478bd9Sstevel@tonic-gate * 32237c478bd9Sstevel@tonic-gate * We return nonzero to indicate that the event has been copied and 32247c478bd9Sstevel@tonic-gate * recirculated for further testing. The caller should not log the event 32257c478bd9Sstevel@tonic-gate * in this case - it will be logged when further test results are available. 32267c478bd9Sstevel@tonic-gate * 32277c478bd9Sstevel@tonic-gate * Our possible contexts are that of errorq_drain: below lock level or from 32287c478bd9Sstevel@tonic-gate * panic context. We can assume that the cpu we are running on is online. 32297c478bd9Sstevel@tonic-gate */ 32307c478bd9Sstevel@tonic-gate 32317c478bd9Sstevel@tonic-gate 32327c478bd9Sstevel@tonic-gate #ifdef DEBUG 32337c478bd9Sstevel@tonic-gate static int ce_xdiag_forceaction; 32347c478bd9Sstevel@tonic-gate #endif 32357c478bd9Sstevel@tonic-gate 32367c478bd9Sstevel@tonic-gate int 32377c478bd9Sstevel@tonic-gate ce_scrub_xdiag_recirc(struct async_flt *aflt, errorq_t *eqp, 32387c478bd9Sstevel@tonic-gate errorq_elem_t *eqep, size_t afltoffset) 32397c478bd9Sstevel@tonic-gate { 32407c478bd9Sstevel@tonic-gate ce_dispact_t dispact, action; 32417c478bd9Sstevel@tonic-gate cpu_t *cp; 32427c478bd9Sstevel@tonic-gate uchar_t dtcrinfo, disp; 32437c478bd9Sstevel@tonic-gate int ptnrtype; 32447c478bd9Sstevel@tonic-gate 32457c478bd9Sstevel@tonic-gate if (!ce_disp_inited || panicstr || ce_xdiag_off) { 32467c478bd9Sstevel@tonic-gate ce_xdiag_drops++; 32477c478bd9Sstevel@tonic-gate return (0); 32487c478bd9Sstevel@tonic-gate } else if (!aflt->flt_in_memory) { 32497c478bd9Sstevel@tonic-gate ce_xdiag_drops++; 32507c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOTMEM); 32517c478bd9Sstevel@tonic-gate return (0); 32527c478bd9Sstevel@tonic-gate } 32537c478bd9Sstevel@tonic-gate 32547c478bd9Sstevel@tonic-gate dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 32557c478bd9Sstevel@tonic-gate 32567c478bd9Sstevel@tonic-gate /* 32577c478bd9Sstevel@tonic-gate * Some correctable events are not scrubbed/classified, such as those 32587c478bd9Sstevel@tonic-gate * noticed at the tail of cpu_deferred_error. So if there is no 32597c478bd9Sstevel@tonic-gate * initial detector classification go no further. 32607c478bd9Sstevel@tonic-gate */ 32617c478bd9Sstevel@tonic-gate if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) { 32627c478bd9Sstevel@tonic-gate ce_xdiag_drops++; 32637c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_NOSCRUB); 32647c478bd9Sstevel@tonic-gate return (0); 32657c478bd9Sstevel@tonic-gate } 32667c478bd9Sstevel@tonic-gate 32677c478bd9Sstevel@tonic-gate dispact = CE_DISPACT(ce_disp_table, 32687c478bd9Sstevel@tonic-gate CE_XDIAG_AFARMATCHED(dtcrinfo), 32697c478bd9Sstevel@tonic-gate CE_XDIAG_STATE(dtcrinfo), 32707c478bd9Sstevel@tonic-gate CE_XDIAG_CE1SEEN(dtcrinfo), 32717c478bd9Sstevel@tonic-gate CE_XDIAG_CE2SEEN(dtcrinfo)); 32727c478bd9Sstevel@tonic-gate 32737c478bd9Sstevel@tonic-gate 32747c478bd9Sstevel@tonic-gate action = CE_ACT(dispact); /* bad lookup caught below */ 32757c478bd9Sstevel@tonic-gate #ifdef DEBUG 32767c478bd9Sstevel@tonic-gate if (ce_xdiag_forceaction != 0) 32777c478bd9Sstevel@tonic-gate action = ce_xdiag_forceaction; 32787c478bd9Sstevel@tonic-gate #endif 32797c478bd9Sstevel@tonic-gate 32807c478bd9Sstevel@tonic-gate switch (action) { 32817c478bd9Sstevel@tonic-gate case CE_ACT_LKYCHK: { 32827c478bd9Sstevel@tonic-gate caddr_t ndata; 32837c478bd9Sstevel@tonic-gate errorq_elem_t *neqep; 32847c478bd9Sstevel@tonic-gate struct async_flt *ecc; 32857c478bd9Sstevel@tonic-gate ce_lkychk_cb_t *cbargp; 32867c478bd9Sstevel@tonic-gate 32877c478bd9Sstevel@tonic-gate if ((ndata = errorq_elem_dup(eqp, eqep, &neqep)) == NULL) { 32887c478bd9Sstevel@tonic-gate ce_xdiag_lkydrops++; 32897c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 32907c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_DUPFAIL); 32917c478bd9Sstevel@tonic-gate break; 32927c478bd9Sstevel@tonic-gate } 32937c478bd9Sstevel@tonic-gate ecc = (struct async_flt *)(ndata + afltoffset); 32947c478bd9Sstevel@tonic-gate 32957c478bd9Sstevel@tonic-gate ASSERT(ecc->flt_class == CPU_FAULT || 32967c478bd9Sstevel@tonic-gate ecc->flt_class == BUS_FAULT); 32977c478bd9Sstevel@tonic-gate ecc->flt_class = (ecc->flt_class == CPU_FAULT) ? 32987c478bd9Sstevel@tonic-gate RECIRC_CPU_FAULT : RECIRC_BUS_FAULT; 32997c478bd9Sstevel@tonic-gate 33007c478bd9Sstevel@tonic-gate cbargp = kmem_alloc(sizeof (ce_lkychk_cb_t), KM_SLEEP); 33017c478bd9Sstevel@tonic-gate cbargp->lkycb_aflt = ecc; 33027c478bd9Sstevel@tonic-gate cbargp->lkycb_eqp = eqp; 33037c478bd9Sstevel@tonic-gate cbargp->lkycb_eqep = neqep; 33047c478bd9Sstevel@tonic-gate 33057c478bd9Sstevel@tonic-gate (void) timeout((void (*)(void *))ce_lkychk_cb, 33067c478bd9Sstevel@tonic-gate (void *)cbargp, drv_usectohz(cpu_ce_lkychk_timeout_usec)); 33077c478bd9Sstevel@tonic-gate return (1); 33087c478bd9Sstevel@tonic-gate } 33097c478bd9Sstevel@tonic-gate 33107c478bd9Sstevel@tonic-gate case CE_ACT_PTNRCHK: 33117c478bd9Sstevel@tonic-gate kpreempt_disable(); /* stop cpu list changing */ 33127c478bd9Sstevel@tonic-gate if ((cp = ce_ptnr_select(aflt, 0, &ptnrtype)) != NULL) { 33137c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)ce_ptnrchk_xc, 33147c478bd9Sstevel@tonic-gate (uint64_t)aflt, (uint64_t)&disp); 33157c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRINFO(aflt->flt_disp, disp); 33167c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRID(aflt->flt_disp, cp->cpu_id); 33177c478bd9Sstevel@tonic-gate CE_XDIAG_SETPTNRTYPE(aflt->flt_disp, ptnrtype); 33187c478bd9Sstevel@tonic-gate } else if (ncpus > 1) { 33197c478bd9Sstevel@tonic-gate ce_xdiag_ptnrdrops++; 33207c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 33217c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_NOPTNR); 33227c478bd9Sstevel@tonic-gate } else { 33237c478bd9Sstevel@tonic-gate ce_xdiag_ptnrdrops++; 33247c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, 33257c478bd9Sstevel@tonic-gate CE_XDIAG_SKIP_UNIPROC); 33267c478bd9Sstevel@tonic-gate } 33277c478bd9Sstevel@tonic-gate kpreempt_enable(); 33287c478bd9Sstevel@tonic-gate break; 33297c478bd9Sstevel@tonic-gate 33307c478bd9Sstevel@tonic-gate case CE_ACT_DONE: 33317c478bd9Sstevel@tonic-gate break; 33327c478bd9Sstevel@tonic-gate 33337c478bd9Sstevel@tonic-gate case CE_ACT(CE_DISP_BAD): 33347c478bd9Sstevel@tonic-gate default: 33357c478bd9Sstevel@tonic-gate #ifdef DEBUG 33367c478bd9Sstevel@tonic-gate cmn_err(CE_PANIC, "ce_scrub_post: Bad action '%d'", action); 33377c478bd9Sstevel@tonic-gate #endif 33387c478bd9Sstevel@tonic-gate ce_xdiag_bad++; 33397c478bd9Sstevel@tonic-gate CE_XDIAG_SETSKIPCODE(aflt->flt_disp, CE_XDIAG_SKIP_ACTBAD); 33407c478bd9Sstevel@tonic-gate break; 33417c478bd9Sstevel@tonic-gate } 33427c478bd9Sstevel@tonic-gate 33437c478bd9Sstevel@tonic-gate return (0); 33447c478bd9Sstevel@tonic-gate } 33457c478bd9Sstevel@tonic-gate 33467c478bd9Sstevel@tonic-gate /* 33477c478bd9Sstevel@tonic-gate * We route all errors through a single switch statement. 33487c478bd9Sstevel@tonic-gate */ 33497c478bd9Sstevel@tonic-gate void 33507c478bd9Sstevel@tonic-gate cpu_ue_log_err(struct async_flt *aflt) 33517c478bd9Sstevel@tonic-gate { 33527c478bd9Sstevel@tonic-gate switch (aflt->flt_class) { 33537c478bd9Sstevel@tonic-gate case CPU_FAULT: 33547c478bd9Sstevel@tonic-gate cpu_ereport_init(aflt); 33557c478bd9Sstevel@tonic-gate if (cpu_async_log_err(aflt, NULL)) 33567c478bd9Sstevel@tonic-gate cpu_ereport_post(aflt); 33577c478bd9Sstevel@tonic-gate break; 33587c478bd9Sstevel@tonic-gate 33597c478bd9Sstevel@tonic-gate case BUS_FAULT: 33607c478bd9Sstevel@tonic-gate bus_async_log_err(aflt); 33617c478bd9Sstevel@tonic-gate break; 33627c478bd9Sstevel@tonic-gate 33637c478bd9Sstevel@tonic-gate default: 33647c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "discarding async error %p with invalid " 33657c478bd9Sstevel@tonic-gate "fault class (0x%x)", (void *)aflt, aflt->flt_class); 33667c478bd9Sstevel@tonic-gate return; 33677c478bd9Sstevel@tonic-gate } 33687c478bd9Sstevel@tonic-gate } 33697c478bd9Sstevel@tonic-gate 33707c478bd9Sstevel@tonic-gate /* 33717c478bd9Sstevel@tonic-gate * Routine for panic hook callback from panic_idle(). 33727c478bd9Sstevel@tonic-gate */ 33737c478bd9Sstevel@tonic-gate void 33747c478bd9Sstevel@tonic-gate cpu_async_panic_callb(void) 33757c478bd9Sstevel@tonic-gate { 33767c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 33777c478bd9Sstevel@tonic-gate struct async_flt *aflt; 33787c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 33797c478bd9Sstevel@tonic-gate uint64_t afsr_errs; 33807c478bd9Sstevel@tonic-gate 33817c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 33827c478bd9Sstevel@tonic-gate 33837c478bd9Sstevel@tonic-gate afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 338438e9bdffSmikechr (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS); 33857c478bd9Sstevel@tonic-gate 33867c478bd9Sstevel@tonic-gate if (afsr_errs) { 33877c478bd9Sstevel@tonic-gate 33887c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 33897c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 33907c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 33917c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 33927c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 33937c478bd9Sstevel@tonic-gate aflt->flt_stat = cpu_error_regs.afsr; 33947c478bd9Sstevel@tonic-gate aflt->flt_addr = cpu_error_regs.afar; 33957c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 33967c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 33977c478bd9Sstevel@tonic-gate aflt->flt_priv = ((cpu_error_regs.afsr & C_AFSR_PRIV) != 0); 33987c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 33997c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = cpu_error_regs.afsr_ext; 34007c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = afsr_errs; 34017c478bd9Sstevel@tonic-gate #if defined(SERRANO) 34027c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs.afar2; 34037c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 34047c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, NULL, afsr_errs, NULL); 34057c478bd9Sstevel@tonic-gate } 34067c478bd9Sstevel@tonic-gate } 34077c478bd9Sstevel@tonic-gate 34087c478bd9Sstevel@tonic-gate /* 34097c478bd9Sstevel@tonic-gate * Routine to convert a syndrome into a syndrome code. 34107c478bd9Sstevel@tonic-gate */ 34117c478bd9Sstevel@tonic-gate static int 34127c478bd9Sstevel@tonic-gate synd_to_synd_code(int synd_status, ushort_t synd, uint64_t afsr_bit) 34137c478bd9Sstevel@tonic-gate { 34147c478bd9Sstevel@tonic-gate if (synd_status == AFLT_STAT_INVALID) 34157c478bd9Sstevel@tonic-gate return (-1); 34167c478bd9Sstevel@tonic-gate 34177c478bd9Sstevel@tonic-gate /* 34187c478bd9Sstevel@tonic-gate * Use the syndrome to index the appropriate syndrome table, 34197c478bd9Sstevel@tonic-gate * to get the code indicating which bit(s) is(are) bad. 34207c478bd9Sstevel@tonic-gate */ 34217c478bd9Sstevel@tonic-gate if (afsr_bit & 34227c478bd9Sstevel@tonic-gate (C_AFSR_MSYND_ERRS | C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 34237c478bd9Sstevel@tonic-gate if (afsr_bit & C_AFSR_MSYND_ERRS) { 34247c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 34257c478bd9Sstevel@tonic-gate if ((synd == 0) || (synd >= BSYND_TBL_SIZE)) 34267c478bd9Sstevel@tonic-gate return (-1); 34277c478bd9Sstevel@tonic-gate else 34287c478bd9Sstevel@tonic-gate return (BPAR0 + synd); 34297c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 34307c478bd9Sstevel@tonic-gate if ((synd == 0) || (synd >= MSYND_TBL_SIZE)) 34317c478bd9Sstevel@tonic-gate return (-1); 34327c478bd9Sstevel@tonic-gate else 34337c478bd9Sstevel@tonic-gate return (mtag_syndrome_tab[synd]); 34347c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 34357c478bd9Sstevel@tonic-gate } else { 34367c478bd9Sstevel@tonic-gate if ((synd == 0) || (synd >= ESYND_TBL_SIZE)) 34377c478bd9Sstevel@tonic-gate return (-1); 34387c478bd9Sstevel@tonic-gate else 34397c478bd9Sstevel@tonic-gate return (ecc_syndrome_tab[synd]); 34407c478bd9Sstevel@tonic-gate } 34417c478bd9Sstevel@tonic-gate } else { 34427c478bd9Sstevel@tonic-gate return (-1); 34437c478bd9Sstevel@tonic-gate } 34447c478bd9Sstevel@tonic-gate } 34457c478bd9Sstevel@tonic-gate 3446d00f0155Sayznaga int 3447d00f0155Sayznaga cpu_get_mem_sid(char *unum, char *buf, int buflen, int *lenp) 3448d00f0155Sayznaga { 3449d00f0155Sayznaga if (&plat_get_mem_sid) 3450d00f0155Sayznaga return (plat_get_mem_sid(unum, buf, buflen, lenp)); 3451d00f0155Sayznaga else 3452d00f0155Sayznaga return (ENOTSUP); 3453d00f0155Sayznaga } 3454d00f0155Sayznaga 3455d00f0155Sayznaga int 3456d00f0155Sayznaga cpu_get_mem_offset(uint64_t flt_addr, uint64_t *offp) 3457d00f0155Sayznaga { 3458d00f0155Sayznaga if (&plat_get_mem_offset) 3459d00f0155Sayznaga return (plat_get_mem_offset(flt_addr, offp)); 3460d00f0155Sayznaga else 3461d00f0155Sayznaga return (ENOTSUP); 3462d00f0155Sayznaga } 3463d00f0155Sayznaga 3464d00f0155Sayznaga int 3465d00f0155Sayznaga cpu_get_mem_addr(char *unum, char *sid, uint64_t offset, uint64_t *addrp) 3466d00f0155Sayznaga { 3467d00f0155Sayznaga if (&plat_get_mem_addr) 3468d00f0155Sayznaga return (plat_get_mem_addr(unum, sid, offset, addrp)); 3469d00f0155Sayznaga else 3470d00f0155Sayznaga return (ENOTSUP); 3471d00f0155Sayznaga } 3472d00f0155Sayznaga 34737c478bd9Sstevel@tonic-gate /* 34747c478bd9Sstevel@tonic-gate * Routine to return a string identifying the physical name 34757c478bd9Sstevel@tonic-gate * associated with a memory/cache error. 34767c478bd9Sstevel@tonic-gate */ 34777c478bd9Sstevel@tonic-gate int 34787c478bd9Sstevel@tonic-gate cpu_get_mem_unum(int synd_status, ushort_t flt_synd, uint64_t flt_stat, 34797c478bd9Sstevel@tonic-gate uint64_t flt_addr, int flt_bus_id, int flt_in_memory, 34807c478bd9Sstevel@tonic-gate ushort_t flt_status, char *buf, int buflen, int *lenp) 34817c478bd9Sstevel@tonic-gate { 34827c478bd9Sstevel@tonic-gate int synd_code; 34837c478bd9Sstevel@tonic-gate int ret; 34847c478bd9Sstevel@tonic-gate 34857c478bd9Sstevel@tonic-gate /* 34867c478bd9Sstevel@tonic-gate * An AFSR of -1 defaults to a memory syndrome. 34877c478bd9Sstevel@tonic-gate */ 34887c478bd9Sstevel@tonic-gate if (flt_stat == (uint64_t)-1) 34897c478bd9Sstevel@tonic-gate flt_stat = C_AFSR_CE; 34907c478bd9Sstevel@tonic-gate 34917c478bd9Sstevel@tonic-gate synd_code = synd_to_synd_code(synd_status, flt_synd, flt_stat); 34927c478bd9Sstevel@tonic-gate 34937c478bd9Sstevel@tonic-gate /* 34947c478bd9Sstevel@tonic-gate * Syndrome code must be either a single-bit error code 34957c478bd9Sstevel@tonic-gate * (0...143) or -1 for unum lookup. 34967c478bd9Sstevel@tonic-gate */ 34977c478bd9Sstevel@tonic-gate if (synd_code < 0 || synd_code >= M2) 34987c478bd9Sstevel@tonic-gate synd_code = -1; 34997c478bd9Sstevel@tonic-gate if (&plat_get_mem_unum) { 35007c478bd9Sstevel@tonic-gate if ((ret = plat_get_mem_unum(synd_code, flt_addr, flt_bus_id, 35017c478bd9Sstevel@tonic-gate flt_in_memory, flt_status, buf, buflen, lenp)) != 0) { 35027c478bd9Sstevel@tonic-gate buf[0] = '\0'; 35037c478bd9Sstevel@tonic-gate *lenp = 0; 35047c478bd9Sstevel@tonic-gate } 35057c478bd9Sstevel@tonic-gate 35067c478bd9Sstevel@tonic-gate return (ret); 35077c478bd9Sstevel@tonic-gate } 35087c478bd9Sstevel@tonic-gate 35097c478bd9Sstevel@tonic-gate return (ENOTSUP); 35107c478bd9Sstevel@tonic-gate } 35117c478bd9Sstevel@tonic-gate 35127c478bd9Sstevel@tonic-gate /* 35137c478bd9Sstevel@tonic-gate * Wrapper for cpu_get_mem_unum() routine that takes an 35147c478bd9Sstevel@tonic-gate * async_flt struct rather than explicit arguments. 35157c478bd9Sstevel@tonic-gate */ 35167c478bd9Sstevel@tonic-gate int 35177c478bd9Sstevel@tonic-gate cpu_get_mem_unum_aflt(int synd_status, struct async_flt *aflt, 35187c478bd9Sstevel@tonic-gate char *buf, int buflen, int *lenp) 35197c478bd9Sstevel@tonic-gate { 35207c478bd9Sstevel@tonic-gate /* 35217c478bd9Sstevel@tonic-gate * If we come thru here for an IO bus error aflt->flt_stat will 35227c478bd9Sstevel@tonic-gate * not be the CPU AFSR, and we pass in a -1 to cpu_get_mem_unum() 35237c478bd9Sstevel@tonic-gate * so it will interpret this as a memory error. 35247c478bd9Sstevel@tonic-gate */ 35257c478bd9Sstevel@tonic-gate return (cpu_get_mem_unum(synd_status, aflt->flt_synd, 35267c478bd9Sstevel@tonic-gate (aflt->flt_class == BUS_FAULT) ? 352738e9bdffSmikechr (uint64_t)-1 : ((ch_async_flt_t *)aflt)->flt_bit, 35287c478bd9Sstevel@tonic-gate aflt->flt_addr, aflt->flt_bus_id, aflt->flt_in_memory, 35297c478bd9Sstevel@tonic-gate aflt->flt_status, buf, buflen, lenp)); 35307c478bd9Sstevel@tonic-gate } 35317c478bd9Sstevel@tonic-gate 353293743541Smb /* 353393743541Smb * Return unum string given synd_code and async_flt into 353493743541Smb * the buf with size UNUM_NAMLEN 353593743541Smb */ 353693743541Smb static int 353793743541Smb cpu_get_mem_unum_synd(int synd_code, struct async_flt *aflt, char *buf) 353893743541Smb { 353993743541Smb int ret, len; 354093743541Smb 354193743541Smb /* 354293743541Smb * Syndrome code must be either a single-bit error code 354393743541Smb * (0...143) or -1 for unum lookup. 354493743541Smb */ 354593743541Smb if (synd_code < 0 || synd_code >= M2) 354693743541Smb synd_code = -1; 354793743541Smb if (&plat_get_mem_unum) { 354893743541Smb if ((ret = plat_get_mem_unum(synd_code, aflt->flt_addr, 354993743541Smb aflt->flt_bus_id, aflt->flt_in_memory, 355093743541Smb aflt->flt_status, buf, UNUM_NAMLEN, &len)) != 0) { 355193743541Smb buf[0] = '\0'; 355293743541Smb } 355393743541Smb return (ret); 355493743541Smb } 355593743541Smb 355693743541Smb buf[0] = '\0'; 355793743541Smb return (ENOTSUP); 355893743541Smb } 355993743541Smb 35607c478bd9Sstevel@tonic-gate /* 35617c478bd9Sstevel@tonic-gate * This routine is a more generic interface to cpu_get_mem_unum() 356238e9bdffSmikechr * that may be used by other modules (e.g. the 'mm' driver, through 356338e9bdffSmikechr * the 'MEM_NAME' ioctl, which is used by fmd to resolve unum's 356438e9bdffSmikechr * for Jalapeno/Serrano FRC/RCE or FRU/RUE paired events). 35657c478bd9Sstevel@tonic-gate */ 35667c478bd9Sstevel@tonic-gate int 35677c478bd9Sstevel@tonic-gate cpu_get_mem_name(uint64_t synd, uint64_t *afsr, uint64_t afar, 35687c478bd9Sstevel@tonic-gate char *buf, int buflen, int *lenp) 35697c478bd9Sstevel@tonic-gate { 35707c478bd9Sstevel@tonic-gate int synd_status, flt_in_memory, ret; 35717c478bd9Sstevel@tonic-gate ushort_t flt_status = 0; 35727c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 357338e9bdffSmikechr uint64_t t_afsr_errs; 35747c478bd9Sstevel@tonic-gate 35757c478bd9Sstevel@tonic-gate /* 35767c478bd9Sstevel@tonic-gate * Check for an invalid address. 35777c478bd9Sstevel@tonic-gate */ 35787c478bd9Sstevel@tonic-gate if (afar == (uint64_t)-1) 35797c478bd9Sstevel@tonic-gate return (ENXIO); 35807c478bd9Sstevel@tonic-gate 35817c478bd9Sstevel@tonic-gate if (synd == (uint64_t)-1) 35827c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_INVALID; 35837c478bd9Sstevel@tonic-gate else 35847c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_VALID; 35857c478bd9Sstevel@tonic-gate 35867c478bd9Sstevel@tonic-gate flt_in_memory = (*afsr & C_AFSR_MEMORY) && 35877c478bd9Sstevel@tonic-gate pf_is_memory(afar >> MMU_PAGESHIFT); 35887c478bd9Sstevel@tonic-gate 35897c478bd9Sstevel@tonic-gate /* 359038e9bdffSmikechr * Get aggregate AFSR for call to cpu_error_is_ecache_data. 35917c478bd9Sstevel@tonic-gate */ 359238e9bdffSmikechr if (*afsr == (uint64_t)-1) 359338e9bdffSmikechr t_afsr_errs = C_AFSR_CE; 359438e9bdffSmikechr else { 359538e9bdffSmikechr t_afsr_errs = (*afsr & C_AFSR_ALL_ERRS); 359638e9bdffSmikechr #if defined(CHEETAH_PLUS) 359738e9bdffSmikechr if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 359838e9bdffSmikechr t_afsr_errs |= (*(afsr + 1) & C_AFSR_EXT_ALL_ERRS); 359938e9bdffSmikechr #endif /* CHEETAH_PLUS */ 36007c478bd9Sstevel@tonic-gate } 36017c478bd9Sstevel@tonic-gate 360238e9bdffSmikechr /* 360338e9bdffSmikechr * Turn on ECC_ECACHE if error type is E$ Data. 360438e9bdffSmikechr */ 360538e9bdffSmikechr if (cpu_error_is_ecache_data(CPU->cpu_id, t_afsr_errs)) 360638e9bdffSmikechr flt_status |= ECC_ECACHE; 360738e9bdffSmikechr 360838e9bdffSmikechr ret = cpu_get_mem_unum(synd_status, (ushort_t)synd, t_afsr_errs, afar, 36097c478bd9Sstevel@tonic-gate CPU->cpu_id, flt_in_memory, flt_status, unum, UNUM_NAMLEN, lenp); 36107c478bd9Sstevel@tonic-gate if (ret != 0) 36117c478bd9Sstevel@tonic-gate return (ret); 36127c478bd9Sstevel@tonic-gate 36137c478bd9Sstevel@tonic-gate if (*lenp >= buflen) 36147c478bd9Sstevel@tonic-gate return (ENAMETOOLONG); 36157c478bd9Sstevel@tonic-gate 36167c478bd9Sstevel@tonic-gate (void) strncpy(buf, unum, buflen); 36177c478bd9Sstevel@tonic-gate 36187c478bd9Sstevel@tonic-gate return (0); 36197c478bd9Sstevel@tonic-gate } 36207c478bd9Sstevel@tonic-gate 36217c478bd9Sstevel@tonic-gate /* 36227c478bd9Sstevel@tonic-gate * Routine to return memory information associated 36237c478bd9Sstevel@tonic-gate * with a physical address and syndrome. 36247c478bd9Sstevel@tonic-gate */ 36257c478bd9Sstevel@tonic-gate int 36267c478bd9Sstevel@tonic-gate cpu_get_mem_info(uint64_t synd, uint64_t afar, 36277c478bd9Sstevel@tonic-gate uint64_t *mem_sizep, uint64_t *seg_sizep, uint64_t *bank_sizep, 36287c478bd9Sstevel@tonic-gate int *segsp, int *banksp, int *mcidp) 36297c478bd9Sstevel@tonic-gate { 36307c478bd9Sstevel@tonic-gate int synd_status, synd_code; 36317c478bd9Sstevel@tonic-gate 36327c478bd9Sstevel@tonic-gate if (afar == (uint64_t)-1) 36337c478bd9Sstevel@tonic-gate return (ENXIO); 36347c478bd9Sstevel@tonic-gate 36357c478bd9Sstevel@tonic-gate if (synd == (uint64_t)-1) 36367c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_INVALID; 36377c478bd9Sstevel@tonic-gate else 36387c478bd9Sstevel@tonic-gate synd_status = AFLT_STAT_VALID; 36397c478bd9Sstevel@tonic-gate 36407c478bd9Sstevel@tonic-gate synd_code = synd_to_synd_code(synd_status, synd, C_AFSR_CE); 36417c478bd9Sstevel@tonic-gate 36427c478bd9Sstevel@tonic-gate if (p2get_mem_info != NULL) 36437c478bd9Sstevel@tonic-gate return ((p2get_mem_info)(synd_code, afar, 3644cbaac45eSkm mem_sizep, seg_sizep, bank_sizep, 3645cbaac45eSkm segsp, banksp, mcidp)); 36467c478bd9Sstevel@tonic-gate else 36477c478bd9Sstevel@tonic-gate return (ENOTSUP); 36487c478bd9Sstevel@tonic-gate } 36497c478bd9Sstevel@tonic-gate 36507c478bd9Sstevel@tonic-gate /* 36517c478bd9Sstevel@tonic-gate * Routine to return a string identifying the physical 36527c478bd9Sstevel@tonic-gate * name associated with a cpuid. 36537c478bd9Sstevel@tonic-gate */ 36547c478bd9Sstevel@tonic-gate int 36557c478bd9Sstevel@tonic-gate cpu_get_cpu_unum(int cpuid, char *buf, int buflen, int *lenp) 36567c478bd9Sstevel@tonic-gate { 36577c478bd9Sstevel@tonic-gate int ret; 36587c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 36597c478bd9Sstevel@tonic-gate 36607c478bd9Sstevel@tonic-gate if (&plat_get_cpu_unum) { 36617c478bd9Sstevel@tonic-gate if ((ret = plat_get_cpu_unum(cpuid, unum, UNUM_NAMLEN, lenp)) 36627c478bd9Sstevel@tonic-gate != 0) 36637c478bd9Sstevel@tonic-gate return (ret); 36647c478bd9Sstevel@tonic-gate } else { 36657c478bd9Sstevel@tonic-gate return (ENOTSUP); 36667c478bd9Sstevel@tonic-gate } 36677c478bd9Sstevel@tonic-gate 36687c478bd9Sstevel@tonic-gate if (*lenp >= buflen) 36697c478bd9Sstevel@tonic-gate return (ENAMETOOLONG); 36707c478bd9Sstevel@tonic-gate 36717c478bd9Sstevel@tonic-gate (void) strncpy(buf, unum, buflen); 36727c478bd9Sstevel@tonic-gate 36737c478bd9Sstevel@tonic-gate return (0); 36747c478bd9Sstevel@tonic-gate } 36757c478bd9Sstevel@tonic-gate 36767c478bd9Sstevel@tonic-gate /* 36777c478bd9Sstevel@tonic-gate * This routine exports the name buffer size. 36787c478bd9Sstevel@tonic-gate */ 36797c478bd9Sstevel@tonic-gate size_t 36807c478bd9Sstevel@tonic-gate cpu_get_name_bufsize() 36817c478bd9Sstevel@tonic-gate { 36827c478bd9Sstevel@tonic-gate return (UNUM_NAMLEN); 36837c478bd9Sstevel@tonic-gate } 36847c478bd9Sstevel@tonic-gate 36857c478bd9Sstevel@tonic-gate /* 36867c478bd9Sstevel@tonic-gate * Historical function, apparantly not used. 36877c478bd9Sstevel@tonic-gate */ 36887c478bd9Sstevel@tonic-gate /* ARGSUSED */ 36897c478bd9Sstevel@tonic-gate void 36907c478bd9Sstevel@tonic-gate cpu_read_paddr(struct async_flt *ecc, short verbose, short ce_err) 36917c478bd9Sstevel@tonic-gate {} 36927c478bd9Sstevel@tonic-gate 36937c478bd9Sstevel@tonic-gate /* 36947c478bd9Sstevel@tonic-gate * Historical function only called for SBus errors in debugging. 36957c478bd9Sstevel@tonic-gate */ 36967c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 36977c478bd9Sstevel@tonic-gate void 36987c478bd9Sstevel@tonic-gate read_ecc_data(struct async_flt *aflt, short verbose, short ce_err) 36997c478bd9Sstevel@tonic-gate {} 37007c478bd9Sstevel@tonic-gate 37017c478bd9Sstevel@tonic-gate /* 37027c478bd9Sstevel@tonic-gate * Clear the AFSR sticky bits. The routine returns a non-zero value if 37037c478bd9Sstevel@tonic-gate * any of the AFSR's sticky errors are detected. If a non-null pointer to 37047c478bd9Sstevel@tonic-gate * an async fault structure argument is passed in, the captured error state 37057c478bd9Sstevel@tonic-gate * (AFSR, AFAR) info will be returned in the structure. 37067c478bd9Sstevel@tonic-gate */ 37077c478bd9Sstevel@tonic-gate int 37087c478bd9Sstevel@tonic-gate clear_errors(ch_async_flt_t *ch_flt) 37097c478bd9Sstevel@tonic-gate { 37107c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 37117c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 37127c478bd9Sstevel@tonic-gate 37137c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 37147c478bd9Sstevel@tonic-gate 37157c478bd9Sstevel@tonic-gate if (ch_flt != NULL) { 37167c478bd9Sstevel@tonic-gate aflt->flt_stat = cpu_error_regs.afsr & C_AFSR_MASK; 37177c478bd9Sstevel@tonic-gate aflt->flt_addr = cpu_error_regs.afar; 37187c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = cpu_error_regs.afsr_ext; 37197c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = (cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 37207c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS); 37217c478bd9Sstevel@tonic-gate #if defined(SERRANO) 37227c478bd9Sstevel@tonic-gate ch_flt->afar2 = cpu_error_regs.afar2; 37237c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 37247c478bd9Sstevel@tonic-gate } 37257c478bd9Sstevel@tonic-gate 37267c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 37277c478bd9Sstevel@tonic-gate 37287c478bd9Sstevel@tonic-gate return (((cpu_error_regs.afsr & C_AFSR_ALL_ERRS) | 37297c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr_ext & C_AFSR_EXT_ALL_ERRS)) != 0); 37307c478bd9Sstevel@tonic-gate } 37317c478bd9Sstevel@tonic-gate 37327c478bd9Sstevel@tonic-gate /* 37337c478bd9Sstevel@tonic-gate * Clear any AFSR error bits, and check for persistence. 37347c478bd9Sstevel@tonic-gate * 37357c478bd9Sstevel@tonic-gate * It would be desirable to also insist that syndrome match. PCI handling 37367c478bd9Sstevel@tonic-gate * has already filled flt_synd. For errors trapped by CPU we only fill 37377c478bd9Sstevel@tonic-gate * flt_synd when we queue the event, so we do not have a valid flt_synd 37387c478bd9Sstevel@tonic-gate * during initial classification (it is valid if we're called as part of 37397c478bd9Sstevel@tonic-gate * subsequent low-pil additional classification attempts). We could try 37407c478bd9Sstevel@tonic-gate * to determine which syndrome to use: we know we're only called for 37417c478bd9Sstevel@tonic-gate * CE/RCE (Jalapeno & Serrano) and CE/EMC (others) so the syndrome to use 37427c478bd9Sstevel@tonic-gate * would be esynd/none and esynd/msynd, respectively. If that is 37437c478bd9Sstevel@tonic-gate * implemented then what do we do in the case that we do experience an 37447c478bd9Sstevel@tonic-gate * error on the same afar but with different syndrome? At the very least 37457c478bd9Sstevel@tonic-gate * we should count such occurences. Anyway, for now, we'll leave it as 37467c478bd9Sstevel@tonic-gate * it has been for ages. 37477c478bd9Sstevel@tonic-gate */ 37487c478bd9Sstevel@tonic-gate static int 37497c478bd9Sstevel@tonic-gate clear_ecc(struct async_flt *aflt) 37507c478bd9Sstevel@tonic-gate { 37517c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 37527c478bd9Sstevel@tonic-gate 37537c478bd9Sstevel@tonic-gate /* 37547c478bd9Sstevel@tonic-gate * Snapshot the AFSR and AFAR and clear any errors 37557c478bd9Sstevel@tonic-gate */ 37567c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 37577c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 37587c478bd9Sstevel@tonic-gate 37597c478bd9Sstevel@tonic-gate /* 37607c478bd9Sstevel@tonic-gate * If any of the same memory access error bits are still on and 37617c478bd9Sstevel@tonic-gate * the AFAR matches, return that the error is persistent. 37627c478bd9Sstevel@tonic-gate */ 37637c478bd9Sstevel@tonic-gate return ((cpu_error_regs.afsr & (C_AFSR_MEMORY & aflt->flt_stat)) != 0 && 37647c478bd9Sstevel@tonic-gate cpu_error_regs.afar == aflt->flt_addr); 37657c478bd9Sstevel@tonic-gate } 37667c478bd9Sstevel@tonic-gate 37677c478bd9Sstevel@tonic-gate /* 37687c478bd9Sstevel@tonic-gate * Turn off all cpu error detection, normally only used for panics. 37697c478bd9Sstevel@tonic-gate */ 37707c478bd9Sstevel@tonic-gate void 37717c478bd9Sstevel@tonic-gate cpu_disable_errors(void) 37727c478bd9Sstevel@tonic-gate { 37737c478bd9Sstevel@tonic-gate xt_all(set_error_enable_tl1, EN_REG_DISABLE, EER_SET_ABSOLUTE); 3774db7840cdSrscott 3775db7840cdSrscott /* 3776db7840cdSrscott * With error detection now turned off, check the other cpus 3777db7840cdSrscott * logout areas for any unlogged errors. 3778db7840cdSrscott */ 3779db7840cdSrscott if (enable_check_other_cpus_logout) { 3780db7840cdSrscott cpu_check_other_cpus_logout(); 3781db7840cdSrscott /* 3782db7840cdSrscott * Make a second pass over the logout areas, in case 3783db7840cdSrscott * there is a failing CPU in an error-trap loop which 3784db7840cdSrscott * will write to the logout area once it is emptied. 3785db7840cdSrscott */ 3786db7840cdSrscott cpu_check_other_cpus_logout(); 3787db7840cdSrscott } 37887c478bd9Sstevel@tonic-gate } 37897c478bd9Sstevel@tonic-gate 37907c478bd9Sstevel@tonic-gate /* 37917c478bd9Sstevel@tonic-gate * Enable errors. 37927c478bd9Sstevel@tonic-gate */ 37937c478bd9Sstevel@tonic-gate void 37947c478bd9Sstevel@tonic-gate cpu_enable_errors(void) 37957c478bd9Sstevel@tonic-gate { 37967c478bd9Sstevel@tonic-gate xt_all(set_error_enable_tl1, EN_REG_ENABLE, EER_SET_ABSOLUTE); 37977c478bd9Sstevel@tonic-gate } 37987c478bd9Sstevel@tonic-gate 37997c478bd9Sstevel@tonic-gate /* 38007c478bd9Sstevel@tonic-gate * Flush the entire ecache using displacement flush by reading through a 38017c478bd9Sstevel@tonic-gate * physical address range twice as large as the Ecache. 38027c478bd9Sstevel@tonic-gate */ 38037c478bd9Sstevel@tonic-gate void 38047c478bd9Sstevel@tonic-gate cpu_flush_ecache(void) 38057c478bd9Sstevel@tonic-gate { 38067c478bd9Sstevel@tonic-gate flush_ecache(ecache_flushaddr, cpunodes[CPU->cpu_id].ecache_size, 38077c478bd9Sstevel@tonic-gate cpunodes[CPU->cpu_id].ecache_linesize); 38087c478bd9Sstevel@tonic-gate } 38097c478bd9Sstevel@tonic-gate 38107c478bd9Sstevel@tonic-gate /* 38117c478bd9Sstevel@tonic-gate * Return CPU E$ set size - E$ size divided by the associativity. 38127c478bd9Sstevel@tonic-gate * We use this function in places where the CPU_PRIVATE ptr may not be 38137c478bd9Sstevel@tonic-gate * initialized yet. Note that for send_mondo and in the Ecache scrubber, 38147c478bd9Sstevel@tonic-gate * we're guaranteed that CPU_PRIVATE is initialized. Also, cpunodes is set 38157c478bd9Sstevel@tonic-gate * up before the kernel switches from OBP's to the kernel's trap table, so 38167c478bd9Sstevel@tonic-gate * we don't have to worry about cpunodes being unitialized. 38177c478bd9Sstevel@tonic-gate */ 38187c478bd9Sstevel@tonic-gate int 38197c478bd9Sstevel@tonic-gate cpu_ecache_set_size(struct cpu *cp) 38207c478bd9Sstevel@tonic-gate { 38217c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(cp)) 38227c478bd9Sstevel@tonic-gate return (CPU_PRIVATE_VAL(cp, chpr_ec_set_size)); 38237c478bd9Sstevel@tonic-gate 38247c478bd9Sstevel@tonic-gate return (cpunodes[cp->cpu_id].ecache_size / cpu_ecache_nway()); 38257c478bd9Sstevel@tonic-gate } 38267c478bd9Sstevel@tonic-gate 38277c478bd9Sstevel@tonic-gate /* 38287c478bd9Sstevel@tonic-gate * Flush Ecache line. 38297c478bd9Sstevel@tonic-gate * Uses ASI_EC_DIAG for Cheetah+ and Jalapeno. 38307c478bd9Sstevel@tonic-gate * Uses normal displacement flush for Cheetah. 38317c478bd9Sstevel@tonic-gate */ 38327c478bd9Sstevel@tonic-gate static void 38337c478bd9Sstevel@tonic-gate cpu_flush_ecache_line(ch_async_flt_t *ch_flt) 38347c478bd9Sstevel@tonic-gate { 38357c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 38367c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 38377c478bd9Sstevel@tonic-gate 38387c478bd9Sstevel@tonic-gate ecache_flush_line(aflt->flt_addr, ec_set_size); 38397c478bd9Sstevel@tonic-gate } 38407c478bd9Sstevel@tonic-gate 38417c478bd9Sstevel@tonic-gate /* 38427c478bd9Sstevel@tonic-gate * Scrub physical address. 38437c478bd9Sstevel@tonic-gate * Scrub code is different depending upon whether this a Cheetah+ with 2-way 38447c478bd9Sstevel@tonic-gate * Ecache or direct-mapped Ecache. 38457c478bd9Sstevel@tonic-gate */ 38467c478bd9Sstevel@tonic-gate static void 38477c478bd9Sstevel@tonic-gate cpu_scrubphys(struct async_flt *aflt) 38487c478bd9Sstevel@tonic-gate { 38497c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 38507c478bd9Sstevel@tonic-gate 38517c478bd9Sstevel@tonic-gate scrubphys(aflt->flt_addr, ec_set_size); 38527c478bd9Sstevel@tonic-gate } 38537c478bd9Sstevel@tonic-gate 38547c478bd9Sstevel@tonic-gate /* 38557c478bd9Sstevel@tonic-gate * Clear physical address. 38567c478bd9Sstevel@tonic-gate * Scrub code is different depending upon whether this a Cheetah+ with 2-way 38577c478bd9Sstevel@tonic-gate * Ecache or direct-mapped Ecache. 38587c478bd9Sstevel@tonic-gate */ 38597c478bd9Sstevel@tonic-gate void 38607c478bd9Sstevel@tonic-gate cpu_clearphys(struct async_flt *aflt) 38617c478bd9Sstevel@tonic-gate { 38627c478bd9Sstevel@tonic-gate int lsize = cpunodes[CPU->cpu_id].ecache_linesize; 38637c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 38647c478bd9Sstevel@tonic-gate 38657c478bd9Sstevel@tonic-gate 3866750ba224Sanbui clearphys(aflt->flt_addr, ec_set_size, lsize); 38677c478bd9Sstevel@tonic-gate } 38687c478bd9Sstevel@tonic-gate 38697c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 38707c478bd9Sstevel@tonic-gate /* 38717c478bd9Sstevel@tonic-gate * Check for a matching valid line in all the sets. 38727c478bd9Sstevel@tonic-gate * If found, return set# + 1. Otherwise return 0. 38737c478bd9Sstevel@tonic-gate */ 38747c478bd9Sstevel@tonic-gate static int 38757c478bd9Sstevel@tonic-gate cpu_ecache_line_valid(ch_async_flt_t *ch_flt) 38767c478bd9Sstevel@tonic-gate { 38777c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 38787c478bd9Sstevel@tonic-gate int totalsize = cpunodes[CPU->cpu_id].ecache_size; 38797c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 38807c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 38817c478bd9Sstevel@tonic-gate int nway = cpu_ecache_nway(); 38827c478bd9Sstevel@tonic-gate int i; 38837c478bd9Sstevel@tonic-gate 38847c478bd9Sstevel@tonic-gate for (i = 0; i < nway; i++, ecp++) { 38857c478bd9Sstevel@tonic-gate if (!cpu_ectag_line_invalid(totalsize, ecp->ec_tag) && 38867c478bd9Sstevel@tonic-gate (aflt->flt_addr & P2ALIGN(C_AFAR_PA, ec_set_size)) == 38877c478bd9Sstevel@tonic-gate cpu_ectag_to_pa(ec_set_size, ecp->ec_tag)) 38887c478bd9Sstevel@tonic-gate return (i+1); 38897c478bd9Sstevel@tonic-gate } 38907c478bd9Sstevel@tonic-gate return (0); 38917c478bd9Sstevel@tonic-gate } 38927c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_ECACHE_ASSOC */ 38937c478bd9Sstevel@tonic-gate 38947c478bd9Sstevel@tonic-gate /* 38957c478bd9Sstevel@tonic-gate * Check whether a line in the given logout info matches the specified 38967c478bd9Sstevel@tonic-gate * fault address. If reqval is set then the line must not be Invalid. 38977c478bd9Sstevel@tonic-gate * Returns 0 on failure; on success (way + 1) is returned an *level is 38987c478bd9Sstevel@tonic-gate * set to 2 for l2$ or 3 for l3$. 38997c478bd9Sstevel@tonic-gate */ 39007c478bd9Sstevel@tonic-gate static int 39017c478bd9Sstevel@tonic-gate cpu_matching_ecache_line(uint64_t faddr, void *data, int reqval, int *level) 39027c478bd9Sstevel@tonic-gate { 39037c478bd9Sstevel@tonic-gate ch_diag_data_t *cdp = data; 39047c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp; 39057c478bd9Sstevel@tonic-gate int totalsize, ec_set_size; 39067c478bd9Sstevel@tonic-gate int i, ways; 39077c478bd9Sstevel@tonic-gate int match = 0; 39087c478bd9Sstevel@tonic-gate int tagvalid; 39097c478bd9Sstevel@tonic-gate uint64_t addr, tagpa; 39107c478bd9Sstevel@tonic-gate int ispanther = IS_PANTHER(cpunodes[CPU->cpu_id].implementation); 39117c478bd9Sstevel@tonic-gate 39127c478bd9Sstevel@tonic-gate /* 39137c478bd9Sstevel@tonic-gate * Check the l2$ logout data 39147c478bd9Sstevel@tonic-gate */ 39157c478bd9Sstevel@tonic-gate if (ispanther) { 39167c478bd9Sstevel@tonic-gate ecp = &cdp->chd_l2_data[0]; 39177c478bd9Sstevel@tonic-gate ec_set_size = PN_L2_SET_SIZE; 39187c478bd9Sstevel@tonic-gate ways = PN_L2_NWAYS; 39197c478bd9Sstevel@tonic-gate } else { 39207c478bd9Sstevel@tonic-gate ecp = &cdp->chd_ec_data[0]; 39217c478bd9Sstevel@tonic-gate ec_set_size = cpu_ecache_set_size(CPU); 39227c478bd9Sstevel@tonic-gate ways = cpu_ecache_nway(); 39237c478bd9Sstevel@tonic-gate totalsize = cpunodes[CPU->cpu_id].ecache_size; 39247c478bd9Sstevel@tonic-gate } 39257c478bd9Sstevel@tonic-gate /* remove low order PA bits from fault address not used in PA tag */ 39267c478bd9Sstevel@tonic-gate addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 39277c478bd9Sstevel@tonic-gate for (i = 0; i < ways; i++, ecp++) { 39287c478bd9Sstevel@tonic-gate if (ispanther) { 39297c478bd9Sstevel@tonic-gate tagpa = PN_L2TAG_TO_PA(ecp->ec_tag); 39307c478bd9Sstevel@tonic-gate tagvalid = !PN_L2_LINE_INVALID(ecp->ec_tag); 39317c478bd9Sstevel@tonic-gate } else { 39327c478bd9Sstevel@tonic-gate tagpa = cpu_ectag_to_pa(ec_set_size, ecp->ec_tag); 39337c478bd9Sstevel@tonic-gate tagvalid = !cpu_ectag_line_invalid(totalsize, 39347c478bd9Sstevel@tonic-gate ecp->ec_tag); 39357c478bd9Sstevel@tonic-gate } 39367c478bd9Sstevel@tonic-gate if (tagpa == addr && (!reqval || tagvalid)) { 39377c478bd9Sstevel@tonic-gate match = i + 1; 39387c478bd9Sstevel@tonic-gate *level = 2; 39397c478bd9Sstevel@tonic-gate break; 39407c478bd9Sstevel@tonic-gate } 39417c478bd9Sstevel@tonic-gate } 39427c478bd9Sstevel@tonic-gate 39437c478bd9Sstevel@tonic-gate if (match || !ispanther) 39447c478bd9Sstevel@tonic-gate return (match); 39457c478bd9Sstevel@tonic-gate 39467c478bd9Sstevel@tonic-gate /* For Panther we also check the l3$ */ 39477c478bd9Sstevel@tonic-gate ecp = &cdp->chd_ec_data[0]; 39487c478bd9Sstevel@tonic-gate ec_set_size = PN_L3_SET_SIZE; 39497c478bd9Sstevel@tonic-gate ways = PN_L3_NWAYS; 39507c478bd9Sstevel@tonic-gate addr = faddr & P2ALIGN(C_AFAR_PA, ec_set_size); 39517c478bd9Sstevel@tonic-gate 39527c478bd9Sstevel@tonic-gate for (i = 0; i < ways; i++, ecp++) { 39537c478bd9Sstevel@tonic-gate if (PN_L3TAG_TO_PA(ecp->ec_tag) == addr && (!reqval || 39547c478bd9Sstevel@tonic-gate !PN_L3_LINE_INVALID(ecp->ec_tag))) { 39557c478bd9Sstevel@tonic-gate match = i + 1; 39567c478bd9Sstevel@tonic-gate *level = 3; 39577c478bd9Sstevel@tonic-gate break; 39587c478bd9Sstevel@tonic-gate } 39597c478bd9Sstevel@tonic-gate } 39607c478bd9Sstevel@tonic-gate 39617c478bd9Sstevel@tonic-gate return (match); 39627c478bd9Sstevel@tonic-gate } 39637c478bd9Sstevel@tonic-gate 39647c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 39657c478bd9Sstevel@tonic-gate /* 39667c478bd9Sstevel@tonic-gate * Record information related to the source of an Dcache Parity Error. 39677c478bd9Sstevel@tonic-gate */ 39687c478bd9Sstevel@tonic-gate static void 39697c478bd9Sstevel@tonic-gate cpu_dcache_parity_info(ch_async_flt_t *ch_flt) 39707c478bd9Sstevel@tonic-gate { 39717c478bd9Sstevel@tonic-gate int dc_set_size = dcache_size / CH_DCACHE_NWAY; 39727c478bd9Sstevel@tonic-gate int index; 39737c478bd9Sstevel@tonic-gate 39747c478bd9Sstevel@tonic-gate /* 39757c478bd9Sstevel@tonic-gate * Since instruction decode cannot be done at high PIL 39767c478bd9Sstevel@tonic-gate * just examine the entire Dcache to locate the error. 39777c478bd9Sstevel@tonic-gate */ 39787c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 39797c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = -1; 39807c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = -1; 39817c478bd9Sstevel@tonic-gate } 39827c478bd9Sstevel@tonic-gate for (index = 0; index < dc_set_size; index += dcache_linesize) 39837c478bd9Sstevel@tonic-gate cpu_dcache_parity_check(ch_flt, index); 39847c478bd9Sstevel@tonic-gate } 39857c478bd9Sstevel@tonic-gate 39867c478bd9Sstevel@tonic-gate /* 39877c478bd9Sstevel@tonic-gate * Check all ways of the Dcache at a specified index for good parity. 39887c478bd9Sstevel@tonic-gate */ 39897c478bd9Sstevel@tonic-gate static void 39907c478bd9Sstevel@tonic-gate cpu_dcache_parity_check(ch_async_flt_t *ch_flt, int index) 39917c478bd9Sstevel@tonic-gate { 39927c478bd9Sstevel@tonic-gate int dc_set_size = dcache_size / CH_DCACHE_NWAY; 39937c478bd9Sstevel@tonic-gate uint64_t parity_bits, pbits, data_word; 39947c478bd9Sstevel@tonic-gate static int parity_bits_popc[] = { 0, 1, 1, 0 }; 39957c478bd9Sstevel@tonic-gate int way, word, data_byte; 39967c478bd9Sstevel@tonic-gate ch_dc_data_t *dcp = &ch_flt->parity_data.dpe.cpl_dc[0]; 39977c478bd9Sstevel@tonic-gate ch_dc_data_t tmp_dcp; 39987c478bd9Sstevel@tonic-gate 39997c478bd9Sstevel@tonic-gate for (way = 0; way < CH_DCACHE_NWAY; way++, dcp++) { 40007c478bd9Sstevel@tonic-gate /* 40017c478bd9Sstevel@tonic-gate * Perform diagnostic read. 40027c478bd9Sstevel@tonic-gate */ 40037c478bd9Sstevel@tonic-gate get_dcache_dtag(index + way * dc_set_size, 4004cbaac45eSkm (uint64_t *)&tmp_dcp); 40057c478bd9Sstevel@tonic-gate 40067c478bd9Sstevel@tonic-gate /* 40077c478bd9Sstevel@tonic-gate * Check tag for even parity. 40087c478bd9Sstevel@tonic-gate * Sum of 1 bits (including parity bit) should be even. 40097c478bd9Sstevel@tonic-gate */ 40107c478bd9Sstevel@tonic-gate if (popc64(tmp_dcp.dc_tag & CHP_DCTAG_PARMASK) & 1) { 40117c478bd9Sstevel@tonic-gate /* 40127c478bd9Sstevel@tonic-gate * If this is the first error log detailed information 40137c478bd9Sstevel@tonic-gate * about it and check the snoop tag. Otherwise just 40147c478bd9Sstevel@tonic-gate * record the fact that we found another error. 40157c478bd9Sstevel@tonic-gate */ 40167c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 40177c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = way; 40187c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_cache = 40197c478bd9Sstevel@tonic-gate CPU_DC_PARITY; 40207c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_tag |= CHP_DC_TAG; 40217c478bd9Sstevel@tonic-gate 40227c478bd9Sstevel@tonic-gate if (popc64(tmp_dcp.dc_sntag & 4023cbaac45eSkm CHP_DCSNTAG_PARMASK) & 1) { 40247c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_tag |= 4025cbaac45eSkm CHP_DC_SNTAG; 40267c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 40277c478bd9Sstevel@tonic-gate } 40287c478bd9Sstevel@tonic-gate 40297c478bd9Sstevel@tonic-gate bcopy(&tmp_dcp, dcp, sizeof (ch_dc_data_t)); 40307c478bd9Sstevel@tonic-gate } 40317c478bd9Sstevel@tonic-gate 40327c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 40337c478bd9Sstevel@tonic-gate } 40347c478bd9Sstevel@tonic-gate 40357c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 40367c478bd9Sstevel@tonic-gate /* 40377c478bd9Sstevel@tonic-gate * Panther has more parity bits than the other 40387c478bd9Sstevel@tonic-gate * processors for covering dcache data and so each 40397c478bd9Sstevel@tonic-gate * byte of data in each word has its own parity bit. 40407c478bd9Sstevel@tonic-gate */ 40417c478bd9Sstevel@tonic-gate parity_bits = tmp_dcp.dc_pn_data_parity; 40427c478bd9Sstevel@tonic-gate for (word = 0; word < 4; word++) { 40437c478bd9Sstevel@tonic-gate data_word = tmp_dcp.dc_data[word]; 40447c478bd9Sstevel@tonic-gate pbits = parity_bits & PN_DC_DATA_PARITY_MASK; 40457c478bd9Sstevel@tonic-gate for (data_byte = 0; data_byte < 8; 40467c478bd9Sstevel@tonic-gate data_byte++) { 40477c478bd9Sstevel@tonic-gate if (((popc64(data_word & 40487c478bd9Sstevel@tonic-gate PN_DC_DATA_PARITY_MASK)) & 1) ^ 40497c478bd9Sstevel@tonic-gate (pbits & 1)) { 40507c478bd9Sstevel@tonic-gate cpu_record_dc_data_parity( 4051cbaac45eSkm ch_flt, dcp, &tmp_dcp, way, 4052cbaac45eSkm word); 40537c478bd9Sstevel@tonic-gate } 40547c478bd9Sstevel@tonic-gate pbits >>= 1; 40557c478bd9Sstevel@tonic-gate data_word >>= 8; 40567c478bd9Sstevel@tonic-gate } 40577c478bd9Sstevel@tonic-gate parity_bits >>= 8; 40587c478bd9Sstevel@tonic-gate } 40597c478bd9Sstevel@tonic-gate } else { 40607c478bd9Sstevel@tonic-gate /* 40617c478bd9Sstevel@tonic-gate * Check data array for even parity. 40627c478bd9Sstevel@tonic-gate * The 8 parity bits are grouped into 4 pairs each 40637c478bd9Sstevel@tonic-gate * of which covers a 64-bit word. The endianness is 40647c478bd9Sstevel@tonic-gate * reversed -- the low-order parity bits cover the 40657c478bd9Sstevel@tonic-gate * high-order data words. 40667c478bd9Sstevel@tonic-gate */ 40677c478bd9Sstevel@tonic-gate parity_bits = tmp_dcp.dc_utag >> 8; 40687c478bd9Sstevel@tonic-gate for (word = 0; word < 4; word++) { 40697c478bd9Sstevel@tonic-gate pbits = (parity_bits >> (6 - word * 2)) & 3; 40707c478bd9Sstevel@tonic-gate if ((popc64(tmp_dcp.dc_data[word]) + 40717c478bd9Sstevel@tonic-gate parity_bits_popc[pbits]) & 1) { 40727c478bd9Sstevel@tonic-gate cpu_record_dc_data_parity(ch_flt, dcp, 40737c478bd9Sstevel@tonic-gate &tmp_dcp, way, word); 40747c478bd9Sstevel@tonic-gate } 40757c478bd9Sstevel@tonic-gate } 40767c478bd9Sstevel@tonic-gate } 40777c478bd9Sstevel@tonic-gate } 40787c478bd9Sstevel@tonic-gate } 40797c478bd9Sstevel@tonic-gate 40807c478bd9Sstevel@tonic-gate static void 40817c478bd9Sstevel@tonic-gate cpu_record_dc_data_parity(ch_async_flt_t *ch_flt, 40827c478bd9Sstevel@tonic-gate ch_dc_data_t *dest_dcp, ch_dc_data_t *src_dcp, int way, int word) 40837c478bd9Sstevel@tonic-gate { 40847c478bd9Sstevel@tonic-gate /* 40857c478bd9Sstevel@tonic-gate * If this is the first error log detailed information about it. 40867c478bd9Sstevel@tonic-gate * Otherwise just record the fact that we found another error. 40877c478bd9Sstevel@tonic-gate */ 40887c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 40897c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = way; 40907c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_cache = CPU_DC_PARITY; 40917c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = word * 8; 40927c478bd9Sstevel@tonic-gate bcopy(src_dcp, dest_dcp, sizeof (ch_dc_data_t)); 40937c478bd9Sstevel@tonic-gate } 40947c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 40957c478bd9Sstevel@tonic-gate } 40967c478bd9Sstevel@tonic-gate 40977c478bd9Sstevel@tonic-gate /* 40987c478bd9Sstevel@tonic-gate * Record information related to the source of an Icache Parity Error. 40997c478bd9Sstevel@tonic-gate * 41007c478bd9Sstevel@tonic-gate * Called with the Icache disabled so any diagnostic accesses are safe. 41017c478bd9Sstevel@tonic-gate */ 41027c478bd9Sstevel@tonic-gate static void 41037c478bd9Sstevel@tonic-gate cpu_icache_parity_info(ch_async_flt_t *ch_flt) 41047c478bd9Sstevel@tonic-gate { 41057c478bd9Sstevel@tonic-gate int ic_set_size; 41067c478bd9Sstevel@tonic-gate int ic_linesize; 41077c478bd9Sstevel@tonic-gate int index; 41087c478bd9Sstevel@tonic-gate 41097c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU)) { 41107c478bd9Sstevel@tonic-gate ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 41117c478bd9Sstevel@tonic-gate CH_ICACHE_NWAY; 41127c478bd9Sstevel@tonic-gate ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 41137c478bd9Sstevel@tonic-gate } else { 41147c478bd9Sstevel@tonic-gate ic_set_size = icache_size / CH_ICACHE_NWAY; 41157c478bd9Sstevel@tonic-gate ic_linesize = icache_linesize; 41167c478bd9Sstevel@tonic-gate } 41177c478bd9Sstevel@tonic-gate 41187c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_way = -1; 41197c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_off = -1; 41207c478bd9Sstevel@tonic-gate 41217c478bd9Sstevel@tonic-gate for (index = 0; index < ic_set_size; index += ic_linesize) 41227c478bd9Sstevel@tonic-gate cpu_icache_parity_check(ch_flt, index); 41237c478bd9Sstevel@tonic-gate } 41247c478bd9Sstevel@tonic-gate 41257c478bd9Sstevel@tonic-gate /* 41267c478bd9Sstevel@tonic-gate * Check all ways of the Icache at a specified index for good parity. 41277c478bd9Sstevel@tonic-gate */ 41287c478bd9Sstevel@tonic-gate static void 41297c478bd9Sstevel@tonic-gate cpu_icache_parity_check(ch_async_flt_t *ch_flt, int index) 41307c478bd9Sstevel@tonic-gate { 41317c478bd9Sstevel@tonic-gate uint64_t parmask, pn_inst_parity; 41327c478bd9Sstevel@tonic-gate int ic_set_size; 41337c478bd9Sstevel@tonic-gate int ic_linesize; 41347c478bd9Sstevel@tonic-gate int flt_index, way, instr, num_instr; 41357c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 41367c478bd9Sstevel@tonic-gate ch_ic_data_t *icp = &ch_flt->parity_data.ipe.cpl_ic[0]; 41377c478bd9Sstevel@tonic-gate ch_ic_data_t tmp_icp; 41387c478bd9Sstevel@tonic-gate 41397c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU)) { 41407c478bd9Sstevel@tonic-gate ic_set_size = CPU_PRIVATE_VAL(CPU, chpr_icache_size) / 41417c478bd9Sstevel@tonic-gate CH_ICACHE_NWAY; 41427c478bd9Sstevel@tonic-gate ic_linesize = CPU_PRIVATE_VAL(CPU, chpr_icache_linesize); 41437c478bd9Sstevel@tonic-gate } else { 41447c478bd9Sstevel@tonic-gate ic_set_size = icache_size / CH_ICACHE_NWAY; 41457c478bd9Sstevel@tonic-gate ic_linesize = icache_linesize; 41467c478bd9Sstevel@tonic-gate } 41477c478bd9Sstevel@tonic-gate 41487c478bd9Sstevel@tonic-gate /* 41497c478bd9Sstevel@tonic-gate * Panther has twice as many instructions per icache line and the 41507c478bd9Sstevel@tonic-gate * instruction parity bit is in a different location. 41517c478bd9Sstevel@tonic-gate */ 41527c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 41537c478bd9Sstevel@tonic-gate num_instr = PN_IC_DATA_REG_SIZE / sizeof (uint64_t); 41547c478bd9Sstevel@tonic-gate pn_inst_parity = PN_ICDATA_PARITY_BIT_MASK; 41557c478bd9Sstevel@tonic-gate } else { 41567c478bd9Sstevel@tonic-gate num_instr = CH_IC_DATA_REG_SIZE / sizeof (uint64_t); 41577c478bd9Sstevel@tonic-gate pn_inst_parity = 0; 41587c478bd9Sstevel@tonic-gate } 41597c478bd9Sstevel@tonic-gate 41607c478bd9Sstevel@tonic-gate /* 41617c478bd9Sstevel@tonic-gate * Index at which we expect to find the parity error. 41627c478bd9Sstevel@tonic-gate */ 41637c478bd9Sstevel@tonic-gate flt_index = P2ALIGN(aflt->flt_addr % ic_set_size, ic_linesize); 41647c478bd9Sstevel@tonic-gate 41657c478bd9Sstevel@tonic-gate for (way = 0; way < CH_ICACHE_NWAY; way++, icp++) { 41667c478bd9Sstevel@tonic-gate /* 41677c478bd9Sstevel@tonic-gate * Diagnostic reads expect address argument in ASI format. 41687c478bd9Sstevel@tonic-gate */ 41697c478bd9Sstevel@tonic-gate get_icache_dtag(2 * (index + way * ic_set_size), 4170cbaac45eSkm (uint64_t *)&tmp_icp); 41717c478bd9Sstevel@tonic-gate 41727c478bd9Sstevel@tonic-gate /* 41737c478bd9Sstevel@tonic-gate * If this is the index in which we expect to find the 41747c478bd9Sstevel@tonic-gate * error log detailed information about each of the ways. 41757c478bd9Sstevel@tonic-gate * This information will be displayed later if we can't 41767c478bd9Sstevel@tonic-gate * determine the exact way in which the error is located. 41777c478bd9Sstevel@tonic-gate */ 41787c478bd9Sstevel@tonic-gate if (flt_index == index) 41797c478bd9Sstevel@tonic-gate bcopy(&tmp_icp, icp, sizeof (ch_ic_data_t)); 41807c478bd9Sstevel@tonic-gate 41817c478bd9Sstevel@tonic-gate /* 41827c478bd9Sstevel@tonic-gate * Check tag for even parity. 41837c478bd9Sstevel@tonic-gate * Sum of 1 bits (including parity bit) should be even. 41847c478bd9Sstevel@tonic-gate */ 41857c478bd9Sstevel@tonic-gate if (popc64(tmp_icp.ic_patag & CHP_ICPATAG_PARMASK) & 1) { 41867c478bd9Sstevel@tonic-gate /* 41877c478bd9Sstevel@tonic-gate * If this way is the one in which we expected 41887c478bd9Sstevel@tonic-gate * to find the error record the way and check the 41897c478bd9Sstevel@tonic-gate * snoop tag. Otherwise just record the fact we 41907c478bd9Sstevel@tonic-gate * found another error. 41917c478bd9Sstevel@tonic-gate */ 41927c478bd9Sstevel@tonic-gate if (flt_index == index) { 41937c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_way = way; 41947c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_tag |= CHP_IC_TAG; 41957c478bd9Sstevel@tonic-gate 41967c478bd9Sstevel@tonic-gate if (popc64(tmp_icp.ic_sntag & 4197cbaac45eSkm CHP_ICSNTAG_PARMASK) & 1) { 41987c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_tag |= 4199cbaac45eSkm CHP_IC_SNTAG; 42007c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_lcnt++; 42017c478bd9Sstevel@tonic-gate } 42027c478bd9Sstevel@tonic-gate 42037c478bd9Sstevel@tonic-gate } 42047c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_lcnt++; 42057c478bd9Sstevel@tonic-gate continue; 42067c478bd9Sstevel@tonic-gate } 42077c478bd9Sstevel@tonic-gate 42087c478bd9Sstevel@tonic-gate /* 42097c478bd9Sstevel@tonic-gate * Check instruction data for even parity. 42107c478bd9Sstevel@tonic-gate * Bits participating in parity differ for PC-relative 42117c478bd9Sstevel@tonic-gate * versus non-PC-relative instructions. 42127c478bd9Sstevel@tonic-gate */ 42137c478bd9Sstevel@tonic-gate for (instr = 0; instr < num_instr; instr++) { 42147c478bd9Sstevel@tonic-gate parmask = (tmp_icp.ic_data[instr] & 4215cbaac45eSkm CH_ICDATA_PRED_ISPCREL) ? 4216cbaac45eSkm (CHP_ICDATA_PCREL_PARMASK | pn_inst_parity) : 4217cbaac45eSkm (CHP_ICDATA_NPCREL_PARMASK | pn_inst_parity); 42187c478bd9Sstevel@tonic-gate if (popc64(tmp_icp.ic_data[instr] & parmask) & 1) { 42197c478bd9Sstevel@tonic-gate /* 42207c478bd9Sstevel@tonic-gate * If this way is the one in which we expected 42217c478bd9Sstevel@tonic-gate * to find the error record the way and offset. 42227c478bd9Sstevel@tonic-gate * Otherwise just log the fact we found another 42237c478bd9Sstevel@tonic-gate * error. 42247c478bd9Sstevel@tonic-gate */ 42257c478bd9Sstevel@tonic-gate if (flt_index == index) { 42267c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_way = way; 42277c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_off = 4228cbaac45eSkm instr * 4; 42297c478bd9Sstevel@tonic-gate } 42307c478bd9Sstevel@tonic-gate ch_flt->parity_data.ipe.cpl_lcnt++; 42317c478bd9Sstevel@tonic-gate continue; 42327c478bd9Sstevel@tonic-gate } 42337c478bd9Sstevel@tonic-gate } 42347c478bd9Sstevel@tonic-gate } 42357c478bd9Sstevel@tonic-gate } 42367c478bd9Sstevel@tonic-gate 42377c478bd9Sstevel@tonic-gate /* 42387c478bd9Sstevel@tonic-gate * Record information related to the source of an Pcache Parity Error. 42397c478bd9Sstevel@tonic-gate */ 42407c478bd9Sstevel@tonic-gate static void 42417c478bd9Sstevel@tonic-gate cpu_pcache_parity_info(ch_async_flt_t *ch_flt) 42427c478bd9Sstevel@tonic-gate { 42437c478bd9Sstevel@tonic-gate int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 42447c478bd9Sstevel@tonic-gate int index; 42457c478bd9Sstevel@tonic-gate 42467c478bd9Sstevel@tonic-gate /* 42477c478bd9Sstevel@tonic-gate * Since instruction decode cannot be done at high PIL just 42487c478bd9Sstevel@tonic-gate * examine the entire Pcache to check for any parity errors. 42497c478bd9Sstevel@tonic-gate */ 42507c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 42517c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = -1; 42527c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = -1; 42537c478bd9Sstevel@tonic-gate } 42547c478bd9Sstevel@tonic-gate for (index = 0; index < pc_set_size; index += CH_PCACHE_LSIZE) 42557c478bd9Sstevel@tonic-gate cpu_pcache_parity_check(ch_flt, index); 42567c478bd9Sstevel@tonic-gate } 42577c478bd9Sstevel@tonic-gate 42587c478bd9Sstevel@tonic-gate /* 42597c478bd9Sstevel@tonic-gate * Check all ways of the Pcache at a specified index for good parity. 42607c478bd9Sstevel@tonic-gate */ 42617c478bd9Sstevel@tonic-gate static void 42627c478bd9Sstevel@tonic-gate cpu_pcache_parity_check(ch_async_flt_t *ch_flt, int index) 42637c478bd9Sstevel@tonic-gate { 42647c478bd9Sstevel@tonic-gate int pc_set_size = CH_PCACHE_SIZE / CH_PCACHE_NWAY; 42657c478bd9Sstevel@tonic-gate int pc_data_words = CH_PC_DATA_REG_SIZE / sizeof (uint64_t); 42667c478bd9Sstevel@tonic-gate int way, word, pbit, parity_bits; 42677c478bd9Sstevel@tonic-gate ch_pc_data_t *pcp = &ch_flt->parity_data.dpe.cpl_pc[0]; 42687c478bd9Sstevel@tonic-gate ch_pc_data_t tmp_pcp; 42697c478bd9Sstevel@tonic-gate 42707c478bd9Sstevel@tonic-gate for (way = 0; way < CH_PCACHE_NWAY; way++, pcp++) { 42717c478bd9Sstevel@tonic-gate /* 42727c478bd9Sstevel@tonic-gate * Perform diagnostic read. 42737c478bd9Sstevel@tonic-gate */ 42747c478bd9Sstevel@tonic-gate get_pcache_dtag(index + way * pc_set_size, 4275cbaac45eSkm (uint64_t *)&tmp_pcp); 42767c478bd9Sstevel@tonic-gate /* 42777c478bd9Sstevel@tonic-gate * Check data array for odd parity. There are 8 parity 42787c478bd9Sstevel@tonic-gate * bits (bits 57:50 of ASI_PCACHE_STATUS_DATA) and each 42797c478bd9Sstevel@tonic-gate * of those bits covers exactly 8 bytes of the data 42807c478bd9Sstevel@tonic-gate * array: 42817c478bd9Sstevel@tonic-gate * 42827c478bd9Sstevel@tonic-gate * parity bit P$ data bytes covered 42837c478bd9Sstevel@tonic-gate * ---------- --------------------- 42847c478bd9Sstevel@tonic-gate * 50 63:56 42857c478bd9Sstevel@tonic-gate * 51 55:48 42867c478bd9Sstevel@tonic-gate * 52 47:40 42877c478bd9Sstevel@tonic-gate * 53 39:32 42887c478bd9Sstevel@tonic-gate * 54 31:24 42897c478bd9Sstevel@tonic-gate * 55 23:16 42907c478bd9Sstevel@tonic-gate * 56 15:8 42917c478bd9Sstevel@tonic-gate * 57 7:0 42927c478bd9Sstevel@tonic-gate */ 42937c478bd9Sstevel@tonic-gate parity_bits = PN_PC_PARITY_BITS(tmp_pcp.pc_status); 42947c478bd9Sstevel@tonic-gate for (word = 0; word < pc_data_words; word++) { 42957c478bd9Sstevel@tonic-gate pbit = (parity_bits >> (pc_data_words - word - 1)) & 1; 42967c478bd9Sstevel@tonic-gate if ((popc64(tmp_pcp.pc_data[word]) & 1) ^ pbit) { 42977c478bd9Sstevel@tonic-gate /* 42987c478bd9Sstevel@tonic-gate * If this is the first error log detailed 42997c478bd9Sstevel@tonic-gate * information about it. Otherwise just record 43007c478bd9Sstevel@tonic-gate * the fact that we found another error. 43017c478bd9Sstevel@tonic-gate */ 43027c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_lcnt == 0) { 43037c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_way = way; 43047c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_cache = 43057c478bd9Sstevel@tonic-gate CPU_PC_PARITY; 43067c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_off = 43077c478bd9Sstevel@tonic-gate word * sizeof (uint64_t); 43087c478bd9Sstevel@tonic-gate bcopy(&tmp_pcp, pcp, 4309cbaac45eSkm sizeof (ch_pc_data_t)); 43107c478bd9Sstevel@tonic-gate } 43117c478bd9Sstevel@tonic-gate ch_flt->parity_data.dpe.cpl_lcnt++; 43127c478bd9Sstevel@tonic-gate } 43137c478bd9Sstevel@tonic-gate } 43147c478bd9Sstevel@tonic-gate } 43157c478bd9Sstevel@tonic-gate } 43167c478bd9Sstevel@tonic-gate 43177c478bd9Sstevel@tonic-gate 43187c478bd9Sstevel@tonic-gate /* 43197c478bd9Sstevel@tonic-gate * Add L1 Data cache data to the ereport payload. 43207c478bd9Sstevel@tonic-gate */ 43217c478bd9Sstevel@tonic-gate static void 43227c478bd9Sstevel@tonic-gate cpu_payload_add_dcache(struct async_flt *aflt, nvlist_t *nvl) 43237c478bd9Sstevel@tonic-gate { 43247c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 43257c478bd9Sstevel@tonic-gate ch_dc_data_t *dcp; 43267c478bd9Sstevel@tonic-gate ch_dc_data_t dcdata[CH_DCACHE_NWAY]; 43277c478bd9Sstevel@tonic-gate uint_t nelem; 43287c478bd9Sstevel@tonic-gate int i, ways_to_check, ways_logged = 0; 43297c478bd9Sstevel@tonic-gate 43307c478bd9Sstevel@tonic-gate /* 43317c478bd9Sstevel@tonic-gate * If this is an D$ fault then there may be multiple 43327c478bd9Sstevel@tonic-gate * ways captured in the ch_parity_log_t structure. 43337c478bd9Sstevel@tonic-gate * Otherwise, there will be at most one way captured 43347c478bd9Sstevel@tonic-gate * in the ch_diag_data_t struct. 43357c478bd9Sstevel@tonic-gate * Check each way to see if it should be encoded. 43367c478bd9Sstevel@tonic-gate */ 43377c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_DC_PARITY) 43387c478bd9Sstevel@tonic-gate ways_to_check = CH_DCACHE_NWAY; 43397c478bd9Sstevel@tonic-gate else 43407c478bd9Sstevel@tonic-gate ways_to_check = 1; 43417c478bd9Sstevel@tonic-gate for (i = 0; i < ways_to_check; i++) { 43427c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_DC_PARITY) 43437c478bd9Sstevel@tonic-gate dcp = &ch_flt->parity_data.dpe.cpl_dc[i]; 43447c478bd9Sstevel@tonic-gate else 43457c478bd9Sstevel@tonic-gate dcp = &ch_flt->flt_diag_data.chd_dc_data; 43467c478bd9Sstevel@tonic-gate if (dcp->dc_logflag == DC_LOGFLAG_MAGIC) { 43477c478bd9Sstevel@tonic-gate bcopy(dcp, &dcdata[ways_logged], 4348cbaac45eSkm sizeof (ch_dc_data_t)); 43497c478bd9Sstevel@tonic-gate ways_logged++; 43507c478bd9Sstevel@tonic-gate } 43517c478bd9Sstevel@tonic-gate } 43527c478bd9Sstevel@tonic-gate 43537c478bd9Sstevel@tonic-gate /* 43547c478bd9Sstevel@tonic-gate * Add the dcache data to the payload. 43557c478bd9Sstevel@tonic-gate */ 43567c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_WAYS, 43577c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 43587c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 43597c478bd9Sstevel@tonic-gate nelem = sizeof (ch_dc_data_t) / sizeof (uint64_t) * ways_logged; 43607c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1D_DATA, 43617c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)dcdata, NULL); 43627c478bd9Sstevel@tonic-gate } 43637c478bd9Sstevel@tonic-gate } 43647c478bd9Sstevel@tonic-gate 43657c478bd9Sstevel@tonic-gate /* 43667c478bd9Sstevel@tonic-gate * Add L1 Instruction cache data to the ereport payload. 43677c478bd9Sstevel@tonic-gate */ 43687c478bd9Sstevel@tonic-gate static void 43697c478bd9Sstevel@tonic-gate cpu_payload_add_icache(struct async_flt *aflt, nvlist_t *nvl) 43707c478bd9Sstevel@tonic-gate { 43717c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 43727c478bd9Sstevel@tonic-gate ch_ic_data_t *icp; 43737c478bd9Sstevel@tonic-gate ch_ic_data_t icdata[CH_ICACHE_NWAY]; 43747c478bd9Sstevel@tonic-gate uint_t nelem; 43757c478bd9Sstevel@tonic-gate int i, ways_to_check, ways_logged = 0; 43767c478bd9Sstevel@tonic-gate 43777c478bd9Sstevel@tonic-gate /* 43787c478bd9Sstevel@tonic-gate * If this is an I$ fault then there may be multiple 43797c478bd9Sstevel@tonic-gate * ways captured in the ch_parity_log_t structure. 43807c478bd9Sstevel@tonic-gate * Otherwise, there will be at most one way captured 43817c478bd9Sstevel@tonic-gate * in the ch_diag_data_t struct. 43827c478bd9Sstevel@tonic-gate * Check each way to see if it should be encoded. 43837c478bd9Sstevel@tonic-gate */ 43847c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_IC_PARITY) 43857c478bd9Sstevel@tonic-gate ways_to_check = CH_ICACHE_NWAY; 43867c478bd9Sstevel@tonic-gate else 43877c478bd9Sstevel@tonic-gate ways_to_check = 1; 43887c478bd9Sstevel@tonic-gate for (i = 0; i < ways_to_check; i++) { 43897c478bd9Sstevel@tonic-gate if (ch_flt->flt_type == CPU_IC_PARITY) 43907c478bd9Sstevel@tonic-gate icp = &ch_flt->parity_data.ipe.cpl_ic[i]; 43917c478bd9Sstevel@tonic-gate else 43927c478bd9Sstevel@tonic-gate icp = &ch_flt->flt_diag_data.chd_ic_data; 43937c478bd9Sstevel@tonic-gate if (icp->ic_logflag == IC_LOGFLAG_MAGIC) { 43947c478bd9Sstevel@tonic-gate bcopy(icp, &icdata[ways_logged], 4395cbaac45eSkm sizeof (ch_ic_data_t)); 43967c478bd9Sstevel@tonic-gate ways_logged++; 43977c478bd9Sstevel@tonic-gate } 43987c478bd9Sstevel@tonic-gate } 43997c478bd9Sstevel@tonic-gate 44007c478bd9Sstevel@tonic-gate /* 44017c478bd9Sstevel@tonic-gate * Add the icache data to the payload. 44027c478bd9Sstevel@tonic-gate */ 44037c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_WAYS, 44047c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 44057c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 44067c478bd9Sstevel@tonic-gate nelem = sizeof (ch_ic_data_t) / sizeof (uint64_t) * ways_logged; 44077c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L1I_DATA, 44087c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)icdata, NULL); 44097c478bd9Sstevel@tonic-gate } 44107c478bd9Sstevel@tonic-gate } 44117c478bd9Sstevel@tonic-gate 44127c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 44137c478bd9Sstevel@tonic-gate 44147c478bd9Sstevel@tonic-gate /* 44157c478bd9Sstevel@tonic-gate * Add ecache data to payload. 44167c478bd9Sstevel@tonic-gate */ 44177c478bd9Sstevel@tonic-gate static void 44187c478bd9Sstevel@tonic-gate cpu_payload_add_ecache(struct async_flt *aflt, nvlist_t *nvl) 44197c478bd9Sstevel@tonic-gate { 44207c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 44217c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp; 44227c478bd9Sstevel@tonic-gate ch_ec_data_t ecdata[CHD_EC_DATA_SETS]; 44237c478bd9Sstevel@tonic-gate uint_t nelem; 44247c478bd9Sstevel@tonic-gate int i, ways_logged = 0; 44257c478bd9Sstevel@tonic-gate 44267c478bd9Sstevel@tonic-gate /* 44277c478bd9Sstevel@tonic-gate * Check each way to see if it should be encoded 44287c478bd9Sstevel@tonic-gate * and concatinate it into a temporary buffer. 44297c478bd9Sstevel@tonic-gate */ 44307c478bd9Sstevel@tonic-gate for (i = 0; i < CHD_EC_DATA_SETS; i++) { 44317c478bd9Sstevel@tonic-gate ecp = &ch_flt->flt_diag_data.chd_ec_data[i]; 44327c478bd9Sstevel@tonic-gate if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 44337c478bd9Sstevel@tonic-gate bcopy(ecp, &ecdata[ways_logged], 4434cbaac45eSkm sizeof (ch_ec_data_t)); 44357c478bd9Sstevel@tonic-gate ways_logged++; 44367c478bd9Sstevel@tonic-gate } 44377c478bd9Sstevel@tonic-gate } 44387c478bd9Sstevel@tonic-gate 44397c478bd9Sstevel@tonic-gate /* 44407c478bd9Sstevel@tonic-gate * Panther CPUs have an additional level of cache and so 44417c478bd9Sstevel@tonic-gate * what we just collected was the L3 (ecache) and not the 44427c478bd9Sstevel@tonic-gate * L2 cache. 44437c478bd9Sstevel@tonic-gate */ 44447c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 44457c478bd9Sstevel@tonic-gate /* 44467c478bd9Sstevel@tonic-gate * Add the L3 (ecache) data to the payload. 44477c478bd9Sstevel@tonic-gate */ 44487c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_WAYS, 44497c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 44507c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 44517c478bd9Sstevel@tonic-gate nelem = sizeof (ch_ec_data_t) / 44527c478bd9Sstevel@tonic-gate sizeof (uint64_t) * ways_logged; 44537c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L3_DATA, 44547c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, 44557c478bd9Sstevel@tonic-gate (uint64_t *)ecdata, NULL); 44567c478bd9Sstevel@tonic-gate } 44577c478bd9Sstevel@tonic-gate 44587c478bd9Sstevel@tonic-gate /* 44597c478bd9Sstevel@tonic-gate * Now collect the L2 cache. 44607c478bd9Sstevel@tonic-gate */ 44617c478bd9Sstevel@tonic-gate ways_logged = 0; 44627c478bd9Sstevel@tonic-gate for (i = 0; i < PN_L2_NWAYS; i++) { 44637c478bd9Sstevel@tonic-gate ecp = &ch_flt->flt_diag_data.chd_l2_data[i]; 44647c478bd9Sstevel@tonic-gate if (ecp->ec_logflag == EC_LOGFLAG_MAGIC) { 44657c478bd9Sstevel@tonic-gate bcopy(ecp, &ecdata[ways_logged], 44667c478bd9Sstevel@tonic-gate sizeof (ch_ec_data_t)); 44677c478bd9Sstevel@tonic-gate ways_logged++; 44687c478bd9Sstevel@tonic-gate } 44697c478bd9Sstevel@tonic-gate } 44707c478bd9Sstevel@tonic-gate } 44717c478bd9Sstevel@tonic-gate 44727c478bd9Sstevel@tonic-gate /* 44737c478bd9Sstevel@tonic-gate * Add the L2 cache data to the payload. 44747c478bd9Sstevel@tonic-gate */ 44757c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_WAYS, 44767c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)ways_logged, NULL); 44777c478bd9Sstevel@tonic-gate if (ways_logged != 0) { 44787c478bd9Sstevel@tonic-gate nelem = sizeof (ch_ec_data_t) / 4479cbaac45eSkm sizeof (uint64_t) * ways_logged; 44807c478bd9Sstevel@tonic-gate fm_payload_set(nvl, FM_EREPORT_PAYLOAD_NAME_L2_DATA, 44817c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64_ARRAY, nelem, (uint64_t *)ecdata, NULL); 44827c478bd9Sstevel@tonic-gate } 44837c478bd9Sstevel@tonic-gate } 44847c478bd9Sstevel@tonic-gate 448538e9bdffSmikechr /* 448638e9bdffSmikechr * Initialize cpu scheme for specified cpu. 448738e9bdffSmikechr */ 448838e9bdffSmikechr static void 448938e9bdffSmikechr cpu_fmri_cpu_set(nvlist_t *cpu_fmri, int cpuid) 449038e9bdffSmikechr { 449138e9bdffSmikechr char sbuf[21]; /* sizeof (UINT64_MAX) + '\0' */ 449238e9bdffSmikechr uint8_t mask; 449338e9bdffSmikechr 449438e9bdffSmikechr mask = cpunodes[cpuid].version; 449538e9bdffSmikechr (void) snprintf(sbuf, sizeof (sbuf), "%llX", 449638e9bdffSmikechr (u_longlong_t)cpunodes[cpuid].device_id); 449738e9bdffSmikechr (void) fm_fmri_cpu_set(cpu_fmri, FM_CPU_SCHEME_VERSION, NULL, 449838e9bdffSmikechr cpuid, &mask, (const char *)sbuf); 449938e9bdffSmikechr } 450038e9bdffSmikechr 450138e9bdffSmikechr /* 450238e9bdffSmikechr * Returns ereport resource type. 450338e9bdffSmikechr */ 450438e9bdffSmikechr static int 450538e9bdffSmikechr cpu_error_to_resource_type(struct async_flt *aflt) 450638e9bdffSmikechr { 450738e9bdffSmikechr ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 450838e9bdffSmikechr 450938e9bdffSmikechr switch (ch_flt->flt_type) { 451038e9bdffSmikechr 451138e9bdffSmikechr case CPU_CE_ECACHE: 451238e9bdffSmikechr case CPU_UE_ECACHE: 451338e9bdffSmikechr case CPU_UE_ECACHE_RETIRE: 451438e9bdffSmikechr case CPU_ORPH: 451538e9bdffSmikechr /* 451638e9bdffSmikechr * If AFSR error bit indicates L2$ Data for Cheetah, 451738e9bdffSmikechr * Cheetah+ or Jaguar, or L3$ Data for Panther, return 451838e9bdffSmikechr * E$ Data type, otherwise, return CPU type. 451938e9bdffSmikechr */ 452038e9bdffSmikechr if (cpu_error_is_ecache_data(aflt->flt_inst, 452138e9bdffSmikechr ch_flt->flt_bit)) 452238e9bdffSmikechr return (ERRTYPE_ECACHE_DATA); 452338e9bdffSmikechr return (ERRTYPE_CPU); 452438e9bdffSmikechr 452538e9bdffSmikechr case CPU_CE: 452638e9bdffSmikechr case CPU_UE: 452738e9bdffSmikechr case CPU_EMC: 452838e9bdffSmikechr case CPU_DUE: 452938e9bdffSmikechr case CPU_RCE: 453038e9bdffSmikechr case CPU_RUE: 453138e9bdffSmikechr case CPU_FRC: 453238e9bdffSmikechr case CPU_FRU: 453338e9bdffSmikechr return (ERRTYPE_MEMORY); 453438e9bdffSmikechr 453538e9bdffSmikechr case CPU_IC_PARITY: 453638e9bdffSmikechr case CPU_DC_PARITY: 453738e9bdffSmikechr case CPU_FPUERR: 453838e9bdffSmikechr case CPU_PC_PARITY: 453938e9bdffSmikechr case CPU_ITLB_PARITY: 454038e9bdffSmikechr case CPU_DTLB_PARITY: 454138e9bdffSmikechr return (ERRTYPE_CPU); 454238e9bdffSmikechr } 454338e9bdffSmikechr return (ERRTYPE_UNKNOWN); 454438e9bdffSmikechr } 454538e9bdffSmikechr 45467c478bd9Sstevel@tonic-gate /* 45477c478bd9Sstevel@tonic-gate * Encode the data saved in the ch_async_flt_t struct into 45487c478bd9Sstevel@tonic-gate * the FM ereport payload. 45497c478bd9Sstevel@tonic-gate */ 45507c478bd9Sstevel@tonic-gate static void 45517c478bd9Sstevel@tonic-gate cpu_payload_add_aflt(struct async_flt *aflt, nvlist_t *payload, 45527c478bd9Sstevel@tonic-gate nvlist_t *resource, int *afar_status, int *synd_status) 45537c478bd9Sstevel@tonic-gate { 45547c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 45557c478bd9Sstevel@tonic-gate *synd_status = AFLT_STAT_INVALID; 45567c478bd9Sstevel@tonic-gate *afar_status = AFLT_STAT_INVALID; 45577c478bd9Sstevel@tonic-gate 45587c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR) { 45597c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR, 45607c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, aflt->flt_stat, NULL); 45617c478bd9Sstevel@tonic-gate } 45627c478bd9Sstevel@tonic-gate 45637c478bd9Sstevel@tonic-gate if ((aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFSR_EXT) && 45647c478bd9Sstevel@tonic-gate IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) { 45657c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFSR_EXT, 45667c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, ch_flt->afsr_ext, NULL); 45677c478bd9Sstevel@tonic-gate } 45687c478bd9Sstevel@tonic-gate 45697c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR_STATUS) { 45707c478bd9Sstevel@tonic-gate *afar_status = afsr_to_afar_status(ch_flt->afsr_errs, 45717c478bd9Sstevel@tonic-gate ch_flt->flt_bit); 45727c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR_STATUS, 45737c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)*afar_status, NULL); 45747c478bd9Sstevel@tonic-gate } 45757c478bd9Sstevel@tonic-gate 45767c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_AFAR) { 45777c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_AFAR, 45787c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, aflt->flt_addr, NULL); 45797c478bd9Sstevel@tonic-gate } 45807c478bd9Sstevel@tonic-gate 45817c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PC) { 45827c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PC, 45837c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, (uint64_t)aflt->flt_pc, NULL); 45847c478bd9Sstevel@tonic-gate } 45857c478bd9Sstevel@tonic-gate 45867c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TL) { 45877c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TL, 45887c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)aflt->flt_tl, NULL); 45897c478bd9Sstevel@tonic-gate } 45907c478bd9Sstevel@tonic-gate 45917c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_TT) { 45927c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_TT, 45937c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, flt_to_trap_type(aflt), NULL); 45947c478bd9Sstevel@tonic-gate } 45957c478bd9Sstevel@tonic-gate 45967c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_PRIV) { 45977c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_PRIV, 45987c478bd9Sstevel@tonic-gate DATA_TYPE_BOOLEAN_VALUE, 45997c478bd9Sstevel@tonic-gate (aflt->flt_priv ? B_TRUE : B_FALSE), NULL); 46007c478bd9Sstevel@tonic-gate } 46017c478bd9Sstevel@tonic-gate 46027c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ME) { 46037c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ME, 46047c478bd9Sstevel@tonic-gate DATA_TYPE_BOOLEAN_VALUE, 46057c478bd9Sstevel@tonic-gate (aflt->flt_stat & C_AFSR_ME) ? B_TRUE : B_FALSE, NULL); 46067c478bd9Sstevel@tonic-gate } 46077c478bd9Sstevel@tonic-gate 46087c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND_STATUS) { 46097c478bd9Sstevel@tonic-gate *synd_status = afsr_to_synd_status(aflt->flt_inst, 46107c478bd9Sstevel@tonic-gate ch_flt->afsr_errs, ch_flt->flt_bit); 46117c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND_STATUS, 46127c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)*synd_status, NULL); 46137c478bd9Sstevel@tonic-gate } 46147c478bd9Sstevel@tonic-gate 46157c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_SYND) { 46167c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_SYND, 46177c478bd9Sstevel@tonic-gate DATA_TYPE_UINT16, (uint16_t)aflt->flt_synd, NULL); 46187c478bd9Sstevel@tonic-gate } 46197c478bd9Sstevel@tonic-gate 46207c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_TYPE) { 46217c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_TYPE, 46227c478bd9Sstevel@tonic-gate DATA_TYPE_STRING, flt_to_error_type(aflt), NULL); 46237c478bd9Sstevel@tonic-gate } 46247c478bd9Sstevel@tonic-gate 46257c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_ERR_DISP) { 46267c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_ERR_DISP, 46277c478bd9Sstevel@tonic-gate DATA_TYPE_UINT64, aflt->flt_disp, NULL); 46287c478bd9Sstevel@tonic-gate } 46297c478bd9Sstevel@tonic-gate 46307c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L2) 46317c478bd9Sstevel@tonic-gate cpu_payload_add_ecache(aflt, payload); 46327c478bd9Sstevel@tonic-gate 46337c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_COPYFUNCTION) { 46347c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_COPYFUNCTION, 46357c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)aflt->flt_status & 0xff, NULL); 46367c478bd9Sstevel@tonic-gate } 46377c478bd9Sstevel@tonic-gate 46387c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_HOWDETECTED) { 46397c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_HOWDETECTED, 46407c478bd9Sstevel@tonic-gate DATA_TYPE_UINT8, (uint8_t)(aflt->flt_status >> 8), NULL); 46417c478bd9Sstevel@tonic-gate } 46427c478bd9Sstevel@tonic-gate 46437c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_INSTRBLOCK) { 46447c478bd9Sstevel@tonic-gate fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_INSTRBLOCK, 46457c478bd9Sstevel@tonic-gate DATA_TYPE_UINT32_ARRAY, 16, 46467c478bd9Sstevel@tonic-gate (uint32_t *)&ch_flt->flt_fpdata, NULL); 46477c478bd9Sstevel@tonic-gate } 46487c478bd9Sstevel@tonic-gate 46497c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 46507c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1D) 46517c478bd9Sstevel@tonic-gate cpu_payload_add_dcache(aflt, payload); 46527c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1I) 46537c478bd9Sstevel@tonic-gate cpu_payload_add_icache(aflt, payload); 46547c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 46557c478bd9Sstevel@tonic-gate 46567c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 46577c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_L1P) 46587c478bd9Sstevel@tonic-gate cpu_payload_add_pcache(aflt, payload); 46597c478bd9Sstevel@tonic-gate if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAGS_TLB) 46607c478bd9Sstevel@tonic-gate cpu_payload_add_tlb(aflt, payload); 46617c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 46627c478bd9Sstevel@tonic-gate /* 46637c478bd9Sstevel@tonic-gate * Create the FMRI that goes into the payload 46647c478bd9Sstevel@tonic-gate * and contains the unum info if necessary. 46657c478bd9Sstevel@tonic-gate */ 466638e9bdffSmikechr if (aflt->flt_payload & FM_EREPORT_PAYLOAD_FLAG_RESOURCE) { 4667d00f0155Sayznaga char unum[UNUM_NAMLEN] = ""; 4668d00f0155Sayznaga char sid[DIMM_SERIAL_ID_LEN] = ""; 466993743541Smb int len, ret, rtype, synd_code; 467038e9bdffSmikechr uint64_t offset = (uint64_t)-1; 467138e9bdffSmikechr 467238e9bdffSmikechr rtype = cpu_error_to_resource_type(aflt); 467338e9bdffSmikechr switch (rtype) { 46747c478bd9Sstevel@tonic-gate 467538e9bdffSmikechr case ERRTYPE_MEMORY: 467638e9bdffSmikechr case ERRTYPE_ECACHE_DATA: 467738e9bdffSmikechr 467838e9bdffSmikechr /* 467938e9bdffSmikechr * Memory errors, do unum lookup 468038e9bdffSmikechr */ 468138e9bdffSmikechr if (*afar_status == AFLT_STAT_INVALID) 468238e9bdffSmikechr break; 468338e9bdffSmikechr 468438e9bdffSmikechr if (rtype == ERRTYPE_ECACHE_DATA) 468538e9bdffSmikechr aflt->flt_status |= ECC_ECACHE; 468638e9bdffSmikechr else 468738e9bdffSmikechr aflt->flt_status &= ~ECC_ECACHE; 468838e9bdffSmikechr 468993743541Smb synd_code = synd_to_synd_code(*synd_status, 469093743541Smb aflt->flt_synd, ch_flt->flt_bit); 469193743541Smb 469293743541Smb if (cpu_get_mem_unum_synd(synd_code, aflt, unum) != 0) 469338e9bdffSmikechr break; 4694d00f0155Sayznaga 4695d00f0155Sayznaga ret = cpu_get_mem_sid(unum, sid, DIMM_SERIAL_ID_LEN, 4696d00f0155Sayznaga &len); 4697d00f0155Sayznaga 4698d00f0155Sayznaga if (ret == 0) { 4699d00f0155Sayznaga (void) cpu_get_mem_offset(aflt->flt_addr, 4700d00f0155Sayznaga &offset); 4701d00f0155Sayznaga } 4702d00f0155Sayznaga 47037c478bd9Sstevel@tonic-gate fm_fmri_mem_set(resource, FM_MEM_SCHEME_VERSION, 4704d00f0155Sayznaga NULL, unum, (ret == 0) ? sid : NULL, offset); 47057c478bd9Sstevel@tonic-gate fm_payload_set(payload, 47067c478bd9Sstevel@tonic-gate FM_EREPORT_PAYLOAD_NAME_RESOURCE, 47077c478bd9Sstevel@tonic-gate DATA_TYPE_NVLIST, resource, NULL); 470838e9bdffSmikechr break; 470938e9bdffSmikechr 471038e9bdffSmikechr case ERRTYPE_CPU: 471138e9bdffSmikechr /* 471238e9bdffSmikechr * On-board processor array error, add cpu resource. 471338e9bdffSmikechr */ 471438e9bdffSmikechr cpu_fmri_cpu_set(resource, aflt->flt_inst); 471538e9bdffSmikechr fm_payload_set(payload, 471638e9bdffSmikechr FM_EREPORT_PAYLOAD_NAME_RESOURCE, 471738e9bdffSmikechr DATA_TYPE_NVLIST, resource, NULL); 471838e9bdffSmikechr break; 47197c478bd9Sstevel@tonic-gate } 47207c478bd9Sstevel@tonic-gate } 47217c478bd9Sstevel@tonic-gate } 47227c478bd9Sstevel@tonic-gate 47237c478bd9Sstevel@tonic-gate /* 47247c478bd9Sstevel@tonic-gate * Initialize the way info if necessary. 47257c478bd9Sstevel@tonic-gate */ 47267c478bd9Sstevel@tonic-gate void 47277c478bd9Sstevel@tonic-gate cpu_ereport_init(struct async_flt *aflt) 47287c478bd9Sstevel@tonic-gate { 47297c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 47307c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 47317c478bd9Sstevel@tonic-gate ch_ec_data_t *l2p = &ch_flt->flt_diag_data.chd_l2_data[0]; 47327c478bd9Sstevel@tonic-gate int i; 47337c478bd9Sstevel@tonic-gate 47347c478bd9Sstevel@tonic-gate /* 47357c478bd9Sstevel@tonic-gate * Initialize the info in the CPU logout structure. 47367c478bd9Sstevel@tonic-gate * The I$/D$ way information is not initialized here 47377c478bd9Sstevel@tonic-gate * since it is captured in the logout assembly code. 47387c478bd9Sstevel@tonic-gate */ 47397c478bd9Sstevel@tonic-gate for (i = 0; i < CHD_EC_DATA_SETS; i++) 47407c478bd9Sstevel@tonic-gate (ecp + i)->ec_way = i; 47417c478bd9Sstevel@tonic-gate 47427c478bd9Sstevel@tonic-gate for (i = 0; i < PN_L2_NWAYS; i++) 47437c478bd9Sstevel@tonic-gate (l2p + i)->ec_way = i; 47447c478bd9Sstevel@tonic-gate } 47457c478bd9Sstevel@tonic-gate 47467c478bd9Sstevel@tonic-gate /* 47477c478bd9Sstevel@tonic-gate * Returns whether fault address is valid for this error bit and 47487c478bd9Sstevel@tonic-gate * whether the address is "in memory" (i.e. pf_is_memory returns 1). 47497c478bd9Sstevel@tonic-gate */ 47507c478bd9Sstevel@tonic-gate int 47517c478bd9Sstevel@tonic-gate cpu_flt_in_memory(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit) 47527c478bd9Sstevel@tonic-gate { 47537c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 47547c478bd9Sstevel@tonic-gate 475538e9bdffSmikechr return ((t_afsr_bit & C_AFSR_MEMORY) && 47567c478bd9Sstevel@tonic-gate afsr_to_afar_status(ch_flt->afsr_errs, t_afsr_bit) == 47577c478bd9Sstevel@tonic-gate AFLT_STAT_VALID && 47587c478bd9Sstevel@tonic-gate pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)); 47597c478bd9Sstevel@tonic-gate } 47607c478bd9Sstevel@tonic-gate 476193743541Smb /* 476293743541Smb * Returns whether fault address is valid based on the error bit for the 476393743541Smb * one event being queued and whether the address is "in memory". 476493743541Smb */ 476593743541Smb static int 476693743541Smb cpu_flt_in_memory_one_event(ch_async_flt_t *ch_flt, uint64_t t_afsr_bit) 476793743541Smb { 476893743541Smb struct async_flt *aflt = (struct async_flt *)ch_flt; 476993743541Smb int afar_status; 477093743541Smb uint64_t afsr_errs, afsr_ow, *ow_bits; 477193743541Smb 477293743541Smb if (!(t_afsr_bit & C_AFSR_MEMORY) || 477393743541Smb !pf_is_memory(aflt->flt_addr >> MMU_PAGESHIFT)) 477493743541Smb return (0); 477593743541Smb 477693743541Smb afsr_errs = ch_flt->afsr_errs; 477793743541Smb afar_status = afsr_to_afar_status(afsr_errs, t_afsr_bit); 477893743541Smb 477993743541Smb switch (afar_status) { 478093743541Smb case AFLT_STAT_VALID: 478193743541Smb return (1); 478293743541Smb 478393743541Smb case AFLT_STAT_AMBIGUOUS: 478493743541Smb /* 478593743541Smb * Status is ambiguous since another error bit (or bits) 478693743541Smb * of equal priority to the specified bit on in the afsr, 478793743541Smb * so check those bits. Return 1 only if the bits on in the 478893743541Smb * same class as the t_afsr_bit are also C_AFSR_MEMORY bits. 478993743541Smb * Otherwise not all the equal priority bits are for memory 479093743541Smb * errors, so return 0. 479193743541Smb */ 479293743541Smb ow_bits = afar_overwrite; 479393743541Smb while ((afsr_ow = *ow_bits++) != 0) { 479493743541Smb /* 479593743541Smb * Get other bits that are on in t_afsr_bit's priority 479693743541Smb * class to check for Memory Error bits only. 479793743541Smb */ 479893743541Smb if (afsr_ow & t_afsr_bit) { 479993743541Smb if ((afsr_errs & afsr_ow) & ~C_AFSR_MEMORY) 480093743541Smb return (0); 480193743541Smb else 480293743541Smb return (1); 480393743541Smb } 480493743541Smb } 480593743541Smb /*FALLTHRU*/ 480693743541Smb 480793743541Smb default: 480893743541Smb return (0); 480993743541Smb } 481093743541Smb } 481193743541Smb 48127c478bd9Sstevel@tonic-gate static void 48137c478bd9Sstevel@tonic-gate cpu_log_diag_info(ch_async_flt_t *ch_flt) 48147c478bd9Sstevel@tonic-gate { 48157c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 48167c478bd9Sstevel@tonic-gate ch_dc_data_t *dcp = &ch_flt->flt_diag_data.chd_dc_data; 48177c478bd9Sstevel@tonic-gate ch_ic_data_t *icp = &ch_flt->flt_diag_data.chd_ic_data; 48187c478bd9Sstevel@tonic-gate ch_ec_data_t *ecp = &ch_flt->flt_diag_data.chd_ec_data[0]; 48197c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 48207c478bd9Sstevel@tonic-gate int i, nway; 48217c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_ECACHE_ASSOC */ 48227c478bd9Sstevel@tonic-gate 48237c478bd9Sstevel@tonic-gate /* 48247c478bd9Sstevel@tonic-gate * Check if the CPU log out captured was valid. 48257c478bd9Sstevel@tonic-gate */ 48267c478bd9Sstevel@tonic-gate if (ch_flt->flt_diag_data.chd_afar == LOGOUT_INVALID || 48277c478bd9Sstevel@tonic-gate ch_flt->flt_data_incomplete) 48287c478bd9Sstevel@tonic-gate return; 48297c478bd9Sstevel@tonic-gate 48307c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_ECACHE_ASSOC) 48317c478bd9Sstevel@tonic-gate nway = cpu_ecache_nway(); 48327c478bd9Sstevel@tonic-gate i = cpu_ecache_line_valid(ch_flt); 48337c478bd9Sstevel@tonic-gate if (i == 0 || i > nway) { 48347c478bd9Sstevel@tonic-gate for (i = 0; i < nway; i++) 48357c478bd9Sstevel@tonic-gate ecp[i].ec_logflag = EC_LOGFLAG_MAGIC; 48367c478bd9Sstevel@tonic-gate } else 48377c478bd9Sstevel@tonic-gate ecp[i - 1].ec_logflag = EC_LOGFLAG_MAGIC; 48387c478bd9Sstevel@tonic-gate #else /* CPU_IMP_ECACHE_ASSOC */ 48397c478bd9Sstevel@tonic-gate ecp->ec_logflag = EC_LOGFLAG_MAGIC; 48407c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_ECACHE_ASSOC */ 48417c478bd9Sstevel@tonic-gate 48427c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 48437c478bd9Sstevel@tonic-gate pn_cpu_log_diag_l2_info(ch_flt); 48447c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 48457c478bd9Sstevel@tonic-gate 48467c478bd9Sstevel@tonic-gate if (CH_DCTAG_MATCH(dcp->dc_tag, aflt->flt_addr)) { 48477c478bd9Sstevel@tonic-gate dcp->dc_way = CH_DCIDX_TO_WAY(dcp->dc_idx); 48487c478bd9Sstevel@tonic-gate dcp->dc_logflag = DC_LOGFLAG_MAGIC; 48497c478bd9Sstevel@tonic-gate } 48507c478bd9Sstevel@tonic-gate 48517c478bd9Sstevel@tonic-gate if (CH_ICTAG_MATCH(icp, aflt->flt_addr)) { 48527c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[aflt->flt_inst].implementation)) 48537c478bd9Sstevel@tonic-gate icp->ic_way = PN_ICIDX_TO_WAY(icp->ic_idx); 48547c478bd9Sstevel@tonic-gate else 48557c478bd9Sstevel@tonic-gate icp->ic_way = CH_ICIDX_TO_WAY(icp->ic_idx); 48567c478bd9Sstevel@tonic-gate icp->ic_logflag = IC_LOGFLAG_MAGIC; 48577c478bd9Sstevel@tonic-gate } 48587c478bd9Sstevel@tonic-gate } 48597c478bd9Sstevel@tonic-gate 48607c478bd9Sstevel@tonic-gate /* 48617c478bd9Sstevel@tonic-gate * Cheetah ECC calculation. 48627c478bd9Sstevel@tonic-gate * 48637c478bd9Sstevel@tonic-gate * We only need to do the calculation on the data bits and can ignore check 48647c478bd9Sstevel@tonic-gate * bit and Mtag bit terms in the calculation. 48657c478bd9Sstevel@tonic-gate */ 48667c478bd9Sstevel@tonic-gate static uint64_t ch_ecc_table[9][2] = { 48677c478bd9Sstevel@tonic-gate /* 48687c478bd9Sstevel@tonic-gate * low order 64-bits high-order 64-bits 48697c478bd9Sstevel@tonic-gate */ 48707c478bd9Sstevel@tonic-gate { 0x46bffffeccd1177f, 0x488800022100014c }, 48717c478bd9Sstevel@tonic-gate { 0x42fccc81331ff77f, 0x14424f1010249184 }, 48727c478bd9Sstevel@tonic-gate { 0x8898827c222f1ffe, 0x22c1222808184aaf }, 48737c478bd9Sstevel@tonic-gate { 0xf7632203e131ccf1, 0xe1241121848292b8 }, 48747c478bd9Sstevel@tonic-gate { 0x7f5511421b113809, 0x901c88d84288aafe }, 48757c478bd9Sstevel@tonic-gate { 0x1d49412184882487, 0x8f338c87c044c6ef }, 48767c478bd9Sstevel@tonic-gate { 0xf552181014448344, 0x7ff8f4443e411911 }, 48777c478bd9Sstevel@tonic-gate { 0x2189240808f24228, 0xfeeff8cc81333f42 }, 48787c478bd9Sstevel@tonic-gate { 0x3280008440001112, 0xfee88b337ffffd62 }, 48797c478bd9Sstevel@tonic-gate }; 48807c478bd9Sstevel@tonic-gate 48817c478bd9Sstevel@tonic-gate /* 48827c478bd9Sstevel@tonic-gate * 64-bit population count, use well-known popcnt trick. 48837c478bd9Sstevel@tonic-gate * We could use the UltraSPARC V9 POPC instruction, but some 48847c478bd9Sstevel@tonic-gate * CPUs including Cheetahplus and Jaguar do not support that 48857c478bd9Sstevel@tonic-gate * instruction. 48867c478bd9Sstevel@tonic-gate */ 48877c478bd9Sstevel@tonic-gate int 48887c478bd9Sstevel@tonic-gate popc64(uint64_t val) 48897c478bd9Sstevel@tonic-gate { 48907c478bd9Sstevel@tonic-gate int cnt; 48917c478bd9Sstevel@tonic-gate 48927c478bd9Sstevel@tonic-gate for (cnt = 0; val != 0; val &= val - 1) 48937c478bd9Sstevel@tonic-gate cnt++; 48947c478bd9Sstevel@tonic-gate return (cnt); 48957c478bd9Sstevel@tonic-gate } 48967c478bd9Sstevel@tonic-gate 48977c478bd9Sstevel@tonic-gate /* 48987c478bd9Sstevel@tonic-gate * Generate the 9 ECC bits for the 128-bit chunk based on the table above. 48997c478bd9Sstevel@tonic-gate * Note that xor'ing an odd number of 1 bits == 1 and xor'ing an even number 49007c478bd9Sstevel@tonic-gate * of 1 bits == 0, so we can just use the least significant bit of the popcnt 49017c478bd9Sstevel@tonic-gate * instead of doing all the xor's. 49027c478bd9Sstevel@tonic-gate */ 49037c478bd9Sstevel@tonic-gate uint32_t 49047c478bd9Sstevel@tonic-gate us3_gen_ecc(uint64_t data_low, uint64_t data_high) 49057c478bd9Sstevel@tonic-gate { 49067c478bd9Sstevel@tonic-gate int bitno, s; 49077c478bd9Sstevel@tonic-gate int synd = 0; 49087c478bd9Sstevel@tonic-gate 49097c478bd9Sstevel@tonic-gate for (bitno = 0; bitno < 9; bitno++) { 49107c478bd9Sstevel@tonic-gate s = (popc64(data_low & ch_ecc_table[bitno][0]) + 49117c478bd9Sstevel@tonic-gate popc64(data_high & ch_ecc_table[bitno][1])) & 1; 49127c478bd9Sstevel@tonic-gate synd |= (s << bitno); 49137c478bd9Sstevel@tonic-gate } 49147c478bd9Sstevel@tonic-gate return (synd); 49157c478bd9Sstevel@tonic-gate 49167c478bd9Sstevel@tonic-gate } 49177c478bd9Sstevel@tonic-gate 49187c478bd9Sstevel@tonic-gate /* 49197c478bd9Sstevel@tonic-gate * Queue one event based on ecc_type_to_info entry. If the event has an AFT1 49207c478bd9Sstevel@tonic-gate * tag associated with it or is a fatal event (aflt_panic set), it is sent to 49217c478bd9Sstevel@tonic-gate * the UE event queue. Otherwise it is dispatched to the CE event queue. 49227c478bd9Sstevel@tonic-gate */ 49237c478bd9Sstevel@tonic-gate static void 49247c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_async_flt_t *ch_flt, char *reason, 49257c478bd9Sstevel@tonic-gate ecc_type_to_info_t *eccp, ch_diag_data_t *cdp) 49267c478bd9Sstevel@tonic-gate { 49277c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 49287c478bd9Sstevel@tonic-gate 49297c478bd9Sstevel@tonic-gate if (reason && 49307c478bd9Sstevel@tonic-gate strlen(reason) + strlen(eccp->ec_reason) < MAX_REASON_STRING) { 49317c478bd9Sstevel@tonic-gate (void) strcat(reason, eccp->ec_reason); 49327c478bd9Sstevel@tonic-gate } 49337c478bd9Sstevel@tonic-gate 49347c478bd9Sstevel@tonic-gate ch_flt->flt_bit = eccp->ec_afsr_bit; 49357c478bd9Sstevel@tonic-gate ch_flt->flt_type = eccp->ec_flt_type; 49367c478bd9Sstevel@tonic-gate if (cdp != NULL && cdp->chd_afar != LOGOUT_INVALID) 49377c478bd9Sstevel@tonic-gate ch_flt->flt_diag_data = *cdp; 49387c478bd9Sstevel@tonic-gate else 49397c478bd9Sstevel@tonic-gate ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 494093743541Smb aflt->flt_in_memory = 494193743541Smb cpu_flt_in_memory_one_event(ch_flt, ch_flt->flt_bit); 49427c478bd9Sstevel@tonic-gate 49437c478bd9Sstevel@tonic-gate if (ch_flt->flt_bit & C_AFSR_MSYND_ERRS) 49447c478bd9Sstevel@tonic-gate aflt->flt_synd = GET_M_SYND(aflt->flt_stat); 49457c478bd9Sstevel@tonic-gate else if (ch_flt->flt_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) 49467c478bd9Sstevel@tonic-gate aflt->flt_synd = GET_E_SYND(aflt->flt_stat); 49477c478bd9Sstevel@tonic-gate else 49487c478bd9Sstevel@tonic-gate aflt->flt_synd = 0; 49497c478bd9Sstevel@tonic-gate 49507c478bd9Sstevel@tonic-gate aflt->flt_payload = eccp->ec_err_payload; 49517c478bd9Sstevel@tonic-gate 49527c478bd9Sstevel@tonic-gate if (aflt->flt_panic || (eccp->ec_afsr_bit & 49537c478bd9Sstevel@tonic-gate (C_AFSR_LEVEL1 | C_AFSR_EXT_LEVEL1))) 49547c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(eccp->ec_err_class, 49557c478bd9Sstevel@tonic-gate (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 49567c478bd9Sstevel@tonic-gate aflt->flt_panic); 49577c478bd9Sstevel@tonic-gate else 49587c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(eccp->ec_err_class, 49597c478bd9Sstevel@tonic-gate (void *)ch_flt, sizeof (ch_async_flt_t), ce_queue, 49607c478bd9Sstevel@tonic-gate aflt->flt_panic); 49617c478bd9Sstevel@tonic-gate } 49627c478bd9Sstevel@tonic-gate 49637c478bd9Sstevel@tonic-gate /* 49647c478bd9Sstevel@tonic-gate * Queue events on async event queue one event per error bit. First we 49657c478bd9Sstevel@tonic-gate * queue the events that we "expect" for the given trap, then we queue events 49667c478bd9Sstevel@tonic-gate * that we may not expect. Return number of events queued. 49677c478bd9Sstevel@tonic-gate */ 49687c478bd9Sstevel@tonic-gate int 49697c478bd9Sstevel@tonic-gate cpu_queue_events(ch_async_flt_t *ch_flt, char *reason, uint64_t t_afsr_errs, 49707c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop) 49717c478bd9Sstevel@tonic-gate { 49727c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 49737c478bd9Sstevel@tonic-gate ecc_type_to_info_t *eccp; 49747c478bd9Sstevel@tonic-gate int nevents = 0; 49757c478bd9Sstevel@tonic-gate uint64_t primary_afar = aflt->flt_addr, primary_afsr = aflt->flt_stat; 49767c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 49777c478bd9Sstevel@tonic-gate uint64_t orig_t_afsr_errs; 49787c478bd9Sstevel@tonic-gate #endif 49797c478bd9Sstevel@tonic-gate uint64_t primary_afsr_ext = ch_flt->afsr_ext; 49807c478bd9Sstevel@tonic-gate uint64_t primary_afsr_errs = ch_flt->afsr_errs; 49817c478bd9Sstevel@tonic-gate ch_diag_data_t *cdp = NULL; 49827c478bd9Sstevel@tonic-gate 49837c478bd9Sstevel@tonic-gate t_afsr_errs &= ((C_AFSR_ALL_ERRS & ~C_AFSR_ME) | C_AFSR_EXT_ALL_ERRS); 49847c478bd9Sstevel@tonic-gate 49857c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 49867c478bd9Sstevel@tonic-gate orig_t_afsr_errs = t_afsr_errs; 49877c478bd9Sstevel@tonic-gate 49887c478bd9Sstevel@tonic-gate /* 49897c478bd9Sstevel@tonic-gate * For Cheetah+, log the shadow AFSR/AFAR bits first. 49907c478bd9Sstevel@tonic-gate */ 49917c478bd9Sstevel@tonic-gate if (clop != NULL) { 49927c478bd9Sstevel@tonic-gate /* 49937c478bd9Sstevel@tonic-gate * Set the AFSR and AFAR fields to the shadow registers. The 49947c478bd9Sstevel@tonic-gate * flt_addr and flt_stat fields will be reset to the primaries 49957c478bd9Sstevel@tonic-gate * below, but the sdw_addr and sdw_stat will stay as the 49967c478bd9Sstevel@tonic-gate * secondaries. 49977c478bd9Sstevel@tonic-gate */ 49987c478bd9Sstevel@tonic-gate cdp = &clop->clo_sdw_data; 49997c478bd9Sstevel@tonic-gate aflt->flt_addr = ch_flt->flt_sdw_afar = cdp->chd_afar; 50007c478bd9Sstevel@tonic-gate aflt->flt_stat = ch_flt->flt_sdw_afsr = cdp->chd_afsr; 50017c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = ch_flt->flt_sdw_afsr_ext = cdp->chd_afsr_ext; 50027c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = (cdp->chd_afsr_ext & C_AFSR_EXT_ALL_ERRS) | 50037c478bd9Sstevel@tonic-gate (cdp->chd_afsr & C_AFSR_ALL_ERRS); 50047c478bd9Sstevel@tonic-gate 50057c478bd9Sstevel@tonic-gate /* 50067c478bd9Sstevel@tonic-gate * If the primary and shadow AFSR differ, tag the shadow as 50077c478bd9Sstevel@tonic-gate * the first fault. 50087c478bd9Sstevel@tonic-gate */ 50097c478bd9Sstevel@tonic-gate if ((primary_afar != cdp->chd_afar) || 50107c478bd9Sstevel@tonic-gate (primary_afsr_errs != ch_flt->afsr_errs)) { 50117c478bd9Sstevel@tonic-gate aflt->flt_stat |= (1ull << C_AFSR_FIRSTFLT_SHIFT); 50127c478bd9Sstevel@tonic-gate } 50137c478bd9Sstevel@tonic-gate 50147c478bd9Sstevel@tonic-gate /* 50157c478bd9Sstevel@tonic-gate * Check AFSR bits as well as AFSR_EXT bits in order of 50167c478bd9Sstevel@tonic-gate * the AFAR overwrite priority. Our stored AFSR_EXT value 50177c478bd9Sstevel@tonic-gate * is expected to be zero for those CPUs which do not have 50187c478bd9Sstevel@tonic-gate * an AFSR_EXT register. 50197c478bd9Sstevel@tonic-gate */ 50207c478bd9Sstevel@tonic-gate for (eccp = ecc_type_to_info; eccp->ec_desc != NULL; eccp++) { 50217c478bd9Sstevel@tonic-gate if ((eccp->ec_afsr_bit & 50227c478bd9Sstevel@tonic-gate (ch_flt->afsr_errs & t_afsr_errs)) && 50237c478bd9Sstevel@tonic-gate ((eccp->ec_flags & aflt->flt_status) != 0)) { 50247c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_flt, reason, eccp, cdp); 50257c478bd9Sstevel@tonic-gate cdp = NULL; 50267c478bd9Sstevel@tonic-gate t_afsr_errs &= ~eccp->ec_afsr_bit; 50277c478bd9Sstevel@tonic-gate nevents++; 50287c478bd9Sstevel@tonic-gate } 50297c478bd9Sstevel@tonic-gate } 50307c478bd9Sstevel@tonic-gate 50317c478bd9Sstevel@tonic-gate /* 50327c478bd9Sstevel@tonic-gate * If the ME bit is on in the primary AFSR turn all the 50337c478bd9Sstevel@tonic-gate * error bits on again that may set the ME bit to make 50347c478bd9Sstevel@tonic-gate * sure we see the ME AFSR error logs. 50357c478bd9Sstevel@tonic-gate */ 50367c478bd9Sstevel@tonic-gate if ((primary_afsr & C_AFSR_ME) != 0) 50377c478bd9Sstevel@tonic-gate t_afsr_errs = (orig_t_afsr_errs & C_AFSR_ALL_ME_ERRS); 50387c478bd9Sstevel@tonic-gate } 50397c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 50407c478bd9Sstevel@tonic-gate 50417c478bd9Sstevel@tonic-gate if (clop != NULL) 50427c478bd9Sstevel@tonic-gate cdp = &clop->clo_data; 50437c478bd9Sstevel@tonic-gate 50447c478bd9Sstevel@tonic-gate /* 50457c478bd9Sstevel@tonic-gate * Queue expected errors, error bit and fault type must match 50467c478bd9Sstevel@tonic-gate * in the ecc_type_to_info table. 50477c478bd9Sstevel@tonic-gate */ 50487c478bd9Sstevel@tonic-gate for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 50497c478bd9Sstevel@tonic-gate eccp++) { 50507c478bd9Sstevel@tonic-gate if ((eccp->ec_afsr_bit & t_afsr_errs) != 0 && 50517c478bd9Sstevel@tonic-gate (eccp->ec_flags & aflt->flt_status) != 0) { 50527c478bd9Sstevel@tonic-gate #if defined(SERRANO) 50537c478bd9Sstevel@tonic-gate /* 50547c478bd9Sstevel@tonic-gate * For FRC/FRU errors on Serrano the afar2 captures 50557c478bd9Sstevel@tonic-gate * the address and the associated data is 50567c478bd9Sstevel@tonic-gate * in the shadow logout area. 50577c478bd9Sstevel@tonic-gate */ 50587c478bd9Sstevel@tonic-gate if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 50597c478bd9Sstevel@tonic-gate if (clop != NULL) 50607c478bd9Sstevel@tonic-gate cdp = &clop->clo_sdw_data; 50617c478bd9Sstevel@tonic-gate aflt->flt_addr = ch_flt->afar2; 50627c478bd9Sstevel@tonic-gate } else { 50637c478bd9Sstevel@tonic-gate if (clop != NULL) 50647c478bd9Sstevel@tonic-gate cdp = &clop->clo_data; 50657c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 50667c478bd9Sstevel@tonic-gate } 50677c478bd9Sstevel@tonic-gate #else /* SERRANO */ 50687c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 50697c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 50707c478bd9Sstevel@tonic-gate aflt->flt_stat = primary_afsr; 50717c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = primary_afsr_ext; 50727c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = primary_afsr_errs; 50737c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_flt, reason, eccp, cdp); 50747c478bd9Sstevel@tonic-gate cdp = NULL; 50757c478bd9Sstevel@tonic-gate t_afsr_errs &= ~eccp->ec_afsr_bit; 50767c478bd9Sstevel@tonic-gate nevents++; 50777c478bd9Sstevel@tonic-gate } 50787c478bd9Sstevel@tonic-gate } 50797c478bd9Sstevel@tonic-gate 50807c478bd9Sstevel@tonic-gate /* 50817c478bd9Sstevel@tonic-gate * Queue unexpected errors, error bit only match. 50827c478bd9Sstevel@tonic-gate */ 50837c478bd9Sstevel@tonic-gate for (eccp = ecc_type_to_info; t_afsr_errs != 0 && eccp->ec_desc != NULL; 50847c478bd9Sstevel@tonic-gate eccp++) { 50857c478bd9Sstevel@tonic-gate if (eccp->ec_afsr_bit & t_afsr_errs) { 50867c478bd9Sstevel@tonic-gate #if defined(SERRANO) 50877c478bd9Sstevel@tonic-gate /* 50887c478bd9Sstevel@tonic-gate * For FRC/FRU errors on Serrano the afar2 captures 50897c478bd9Sstevel@tonic-gate * the address and the associated data is 50907c478bd9Sstevel@tonic-gate * in the shadow logout area. 50917c478bd9Sstevel@tonic-gate */ 50927c478bd9Sstevel@tonic-gate if (eccp->ec_afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) { 50937c478bd9Sstevel@tonic-gate if (clop != NULL) 50947c478bd9Sstevel@tonic-gate cdp = &clop->clo_sdw_data; 50957c478bd9Sstevel@tonic-gate aflt->flt_addr = ch_flt->afar2; 50967c478bd9Sstevel@tonic-gate } else { 50977c478bd9Sstevel@tonic-gate if (clop != NULL) 50987c478bd9Sstevel@tonic-gate cdp = &clop->clo_data; 50997c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 51007c478bd9Sstevel@tonic-gate } 51017c478bd9Sstevel@tonic-gate #else /* SERRANO */ 51027c478bd9Sstevel@tonic-gate aflt->flt_addr = primary_afar; 51037c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 51047c478bd9Sstevel@tonic-gate aflt->flt_stat = primary_afsr; 51057c478bd9Sstevel@tonic-gate ch_flt->afsr_ext = primary_afsr_ext; 51067c478bd9Sstevel@tonic-gate ch_flt->afsr_errs = primary_afsr_errs; 51077c478bd9Sstevel@tonic-gate cpu_queue_one_event(ch_flt, reason, eccp, cdp); 51087c478bd9Sstevel@tonic-gate cdp = NULL; 51097c478bd9Sstevel@tonic-gate t_afsr_errs &= ~eccp->ec_afsr_bit; 51107c478bd9Sstevel@tonic-gate nevents++; 51117c478bd9Sstevel@tonic-gate } 51127c478bd9Sstevel@tonic-gate } 51137c478bd9Sstevel@tonic-gate return (nevents); 51147c478bd9Sstevel@tonic-gate } 51157c478bd9Sstevel@tonic-gate 51167c478bd9Sstevel@tonic-gate /* 51177c478bd9Sstevel@tonic-gate * Return trap type number. 51187c478bd9Sstevel@tonic-gate */ 51197c478bd9Sstevel@tonic-gate uint8_t 51207c478bd9Sstevel@tonic-gate flt_to_trap_type(struct async_flt *aflt) 51217c478bd9Sstevel@tonic-gate { 51227c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_I_TRAP) 51237c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_I); 51247c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_D_TRAP) 51257c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_D); 51267c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_F_TRAP) 51277c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_F); 51287c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_C_TRAP) 51297c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_C); 51307c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_DP_TRAP) 51317c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_DP); 51327c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_IP_TRAP) 51337c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_IP); 51347c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_ITLB_TRAP) 51357c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_ITLB); 51367c478bd9Sstevel@tonic-gate if (aflt->flt_status & ECC_DTLB_TRAP) 51377c478bd9Sstevel@tonic-gate return (TRAP_TYPE_ECC_DTLB); 51387c478bd9Sstevel@tonic-gate return (TRAP_TYPE_UNKNOWN); 51397c478bd9Sstevel@tonic-gate } 51407c478bd9Sstevel@tonic-gate 51417c478bd9Sstevel@tonic-gate /* 51427c478bd9Sstevel@tonic-gate * Decide an error type based on detector and leaky/partner tests. 51437c478bd9Sstevel@tonic-gate * The following array is used for quick translation - it must 51447c478bd9Sstevel@tonic-gate * stay in sync with ce_dispact_t. 51457c478bd9Sstevel@tonic-gate */ 51467c478bd9Sstevel@tonic-gate 51477c478bd9Sstevel@tonic-gate static char *cetypes[] = { 51487c478bd9Sstevel@tonic-gate CE_DISP_DESC_U, 51497c478bd9Sstevel@tonic-gate CE_DISP_DESC_I, 51507c478bd9Sstevel@tonic-gate CE_DISP_DESC_PP, 51517c478bd9Sstevel@tonic-gate CE_DISP_DESC_P, 51527c478bd9Sstevel@tonic-gate CE_DISP_DESC_L, 51537c478bd9Sstevel@tonic-gate CE_DISP_DESC_PS, 51547c478bd9Sstevel@tonic-gate CE_DISP_DESC_S 51557c478bd9Sstevel@tonic-gate }; 51567c478bd9Sstevel@tonic-gate 51577c478bd9Sstevel@tonic-gate char * 51587c478bd9Sstevel@tonic-gate flt_to_error_type(struct async_flt *aflt) 51597c478bd9Sstevel@tonic-gate { 51607c478bd9Sstevel@tonic-gate ce_dispact_t dispact, disp; 51617c478bd9Sstevel@tonic-gate uchar_t dtcrinfo, ptnrinfo, lkyinfo; 51627c478bd9Sstevel@tonic-gate 51637c478bd9Sstevel@tonic-gate /* 51647c478bd9Sstevel@tonic-gate * The memory payload bundle is shared by some events that do 51657c478bd9Sstevel@tonic-gate * not perform any classification. For those flt_disp will be 51667c478bd9Sstevel@tonic-gate * 0 and we will return "unknown". 51677c478bd9Sstevel@tonic-gate */ 51687c478bd9Sstevel@tonic-gate if (!ce_disp_inited || !aflt->flt_in_memory || aflt->flt_disp == 0) 51697c478bd9Sstevel@tonic-gate return (cetypes[CE_DISP_UNKNOWN]); 51707c478bd9Sstevel@tonic-gate 51717c478bd9Sstevel@tonic-gate dtcrinfo = CE_XDIAG_DTCRINFO(aflt->flt_disp); 51727c478bd9Sstevel@tonic-gate 51737c478bd9Sstevel@tonic-gate /* 51747c478bd9Sstevel@tonic-gate * It is also possible that no scrub/classification was performed 51757c478bd9Sstevel@tonic-gate * by the detector, for instance where a disrupting error logged 51767c478bd9Sstevel@tonic-gate * in the AFSR while CEEN was off in cpu_deferred_error. 51777c478bd9Sstevel@tonic-gate */ 51787c478bd9Sstevel@tonic-gate if (!CE_XDIAG_EXT_ALG_APPLIED(dtcrinfo)) 51797c478bd9Sstevel@tonic-gate return (cetypes[CE_DISP_UNKNOWN]); 51807c478bd9Sstevel@tonic-gate 51817c478bd9Sstevel@tonic-gate /* 51827c478bd9Sstevel@tonic-gate * Lookup type in initial classification/action table 51837c478bd9Sstevel@tonic-gate */ 51847c478bd9Sstevel@tonic-gate dispact = CE_DISPACT(ce_disp_table, 51857c478bd9Sstevel@tonic-gate CE_XDIAG_AFARMATCHED(dtcrinfo), 51867c478bd9Sstevel@tonic-gate CE_XDIAG_STATE(dtcrinfo), 51877c478bd9Sstevel@tonic-gate CE_XDIAG_CE1SEEN(dtcrinfo), 51887c478bd9Sstevel@tonic-gate CE_XDIAG_CE2SEEN(dtcrinfo)); 51897c478bd9Sstevel@tonic-gate 51907c478bd9Sstevel@tonic-gate /* 51917c478bd9Sstevel@tonic-gate * A bad lookup is not something to panic production systems for. 51927c478bd9Sstevel@tonic-gate */ 51937c478bd9Sstevel@tonic-gate ASSERT(dispact != CE_DISP_BAD); 51947c478bd9Sstevel@tonic-gate if (dispact == CE_DISP_BAD) 51957c478bd9Sstevel@tonic-gate return (cetypes[CE_DISP_UNKNOWN]); 51967c478bd9Sstevel@tonic-gate 51977c478bd9Sstevel@tonic-gate disp = CE_DISP(dispact); 51987c478bd9Sstevel@tonic-gate 51997c478bd9Sstevel@tonic-gate switch (disp) { 52007c478bd9Sstevel@tonic-gate case CE_DISP_UNKNOWN: 52017c478bd9Sstevel@tonic-gate case CE_DISP_INTERMITTENT: 52027c478bd9Sstevel@tonic-gate break; 52037c478bd9Sstevel@tonic-gate 52047c478bd9Sstevel@tonic-gate case CE_DISP_POSS_PERS: 52057c478bd9Sstevel@tonic-gate /* 52067c478bd9Sstevel@tonic-gate * "Possible persistent" errors to which we have applied a valid 52077c478bd9Sstevel@tonic-gate * leaky test can be separated into "persistent" or "leaky". 52087c478bd9Sstevel@tonic-gate */ 52097c478bd9Sstevel@tonic-gate lkyinfo = CE_XDIAG_LKYINFO(aflt->flt_disp); 52107c478bd9Sstevel@tonic-gate if (CE_XDIAG_TESTVALID(lkyinfo)) { 52117c478bd9Sstevel@tonic-gate if (CE_XDIAG_CE1SEEN(lkyinfo) || 52127c478bd9Sstevel@tonic-gate CE_XDIAG_CE2SEEN(lkyinfo)) 52137c478bd9Sstevel@tonic-gate disp = CE_DISP_LEAKY; 52147c478bd9Sstevel@tonic-gate else 52157c478bd9Sstevel@tonic-gate disp = CE_DISP_PERS; 52167c478bd9Sstevel@tonic-gate } 52177c478bd9Sstevel@tonic-gate break; 52187c478bd9Sstevel@tonic-gate 52197c478bd9Sstevel@tonic-gate case CE_DISP_POSS_STICKY: 52207c478bd9Sstevel@tonic-gate /* 52217c478bd9Sstevel@tonic-gate * Promote "possible sticky" results that have been 52227c478bd9Sstevel@tonic-gate * confirmed by a partner test to "sticky". Unconfirmed 52237c478bd9Sstevel@tonic-gate * "possible sticky" events are left at that status - we do not 52247c478bd9Sstevel@tonic-gate * guess at any bad reader/writer etc status here. 52257c478bd9Sstevel@tonic-gate */ 52267c478bd9Sstevel@tonic-gate ptnrinfo = CE_XDIAG_PTNRINFO(aflt->flt_disp); 52277c478bd9Sstevel@tonic-gate if (CE_XDIAG_TESTVALID(ptnrinfo) && 52287c478bd9Sstevel@tonic-gate CE_XDIAG_CE1SEEN(ptnrinfo) && CE_XDIAG_CE2SEEN(ptnrinfo)) 52297c478bd9Sstevel@tonic-gate disp = CE_DISP_STICKY; 52307c478bd9Sstevel@tonic-gate 52317c478bd9Sstevel@tonic-gate /* 52327c478bd9Sstevel@tonic-gate * Promote "possible sticky" results on a uniprocessor 52337c478bd9Sstevel@tonic-gate * to "sticky" 52347c478bd9Sstevel@tonic-gate */ 52357c478bd9Sstevel@tonic-gate if (disp == CE_DISP_POSS_STICKY && 52367c478bd9Sstevel@tonic-gate CE_XDIAG_SKIPCODE(disp) == CE_XDIAG_SKIP_UNIPROC) 52377c478bd9Sstevel@tonic-gate disp = CE_DISP_STICKY; 52387c478bd9Sstevel@tonic-gate break; 52397c478bd9Sstevel@tonic-gate 52407c478bd9Sstevel@tonic-gate default: 52417c478bd9Sstevel@tonic-gate disp = CE_DISP_UNKNOWN; 52427c478bd9Sstevel@tonic-gate break; 52437c478bd9Sstevel@tonic-gate } 52447c478bd9Sstevel@tonic-gate 52457c478bd9Sstevel@tonic-gate return (cetypes[disp]); 52467c478bd9Sstevel@tonic-gate } 52477c478bd9Sstevel@tonic-gate 52487c478bd9Sstevel@tonic-gate /* 52497c478bd9Sstevel@tonic-gate * Given the entire afsr, the specific bit to check and a prioritized list of 52507c478bd9Sstevel@tonic-gate * error bits, determine the validity of the various overwrite priority 52517c478bd9Sstevel@tonic-gate * features of the AFSR/AFAR: AFAR, ESYND and MSYND, each of which have 52527c478bd9Sstevel@tonic-gate * different overwrite priorities. 52537c478bd9Sstevel@tonic-gate * 52547c478bd9Sstevel@tonic-gate * Given a specific afsr error bit and the entire afsr, there are three cases: 52557c478bd9Sstevel@tonic-gate * INVALID: The specified bit is lower overwrite priority than some other 52567c478bd9Sstevel@tonic-gate * error bit which is on in the afsr (or IVU/IVC). 52577c478bd9Sstevel@tonic-gate * VALID: The specified bit is higher priority than all other error bits 52587c478bd9Sstevel@tonic-gate * which are on in the afsr. 52597c478bd9Sstevel@tonic-gate * AMBIGUOUS: Another error bit (or bits) of equal priority to the specified 52607c478bd9Sstevel@tonic-gate * bit is on in the afsr. 52617c478bd9Sstevel@tonic-gate */ 52627c478bd9Sstevel@tonic-gate int 52637c478bd9Sstevel@tonic-gate afsr_to_overw_status(uint64_t afsr, uint64_t afsr_bit, uint64_t *ow_bits) 52647c478bd9Sstevel@tonic-gate { 52657c478bd9Sstevel@tonic-gate uint64_t afsr_ow; 52667c478bd9Sstevel@tonic-gate 52677c478bd9Sstevel@tonic-gate while ((afsr_ow = *ow_bits++) != 0) { 52687c478bd9Sstevel@tonic-gate /* 52697c478bd9Sstevel@tonic-gate * If bit is in the priority class, check to see if another 52707c478bd9Sstevel@tonic-gate * bit in the same class is on => ambiguous. Otherwise, 52717c478bd9Sstevel@tonic-gate * the value is valid. If the bit is not on at this priority 52727c478bd9Sstevel@tonic-gate * class, but a higher priority bit is on, then the value is 52737c478bd9Sstevel@tonic-gate * invalid. 52747c478bd9Sstevel@tonic-gate */ 52757c478bd9Sstevel@tonic-gate if (afsr_ow & afsr_bit) { 52767c478bd9Sstevel@tonic-gate /* 52777c478bd9Sstevel@tonic-gate * If equal pri bit is on, ambiguous. 52787c478bd9Sstevel@tonic-gate */ 52797c478bd9Sstevel@tonic-gate if (afsr & (afsr_ow & ~afsr_bit)) 52807c478bd9Sstevel@tonic-gate return (AFLT_STAT_AMBIGUOUS); 52817c478bd9Sstevel@tonic-gate return (AFLT_STAT_VALID); 52827c478bd9Sstevel@tonic-gate } else if (afsr & afsr_ow) 52837c478bd9Sstevel@tonic-gate break; 52847c478bd9Sstevel@tonic-gate } 52857c478bd9Sstevel@tonic-gate 52867c478bd9Sstevel@tonic-gate /* 52877c478bd9Sstevel@tonic-gate * We didn't find a match or a higher priority bit was on. Not 52887c478bd9Sstevel@tonic-gate * finding a match handles the case of invalid AFAR for IVC, IVU. 52897c478bd9Sstevel@tonic-gate */ 52907c478bd9Sstevel@tonic-gate return (AFLT_STAT_INVALID); 52917c478bd9Sstevel@tonic-gate } 52927c478bd9Sstevel@tonic-gate 52937c478bd9Sstevel@tonic-gate static int 52947c478bd9Sstevel@tonic-gate afsr_to_afar_status(uint64_t afsr, uint64_t afsr_bit) 52957c478bd9Sstevel@tonic-gate { 52967c478bd9Sstevel@tonic-gate #if defined(SERRANO) 52977c478bd9Sstevel@tonic-gate if (afsr_bit & (C_AFSR_FRC | C_AFSR_FRU)) 52987c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, afar2_overwrite)); 52997c478bd9Sstevel@tonic-gate else 53007c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 53017c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, afar_overwrite)); 53027c478bd9Sstevel@tonic-gate } 53037c478bd9Sstevel@tonic-gate 53047c478bd9Sstevel@tonic-gate static int 53057c478bd9Sstevel@tonic-gate afsr_to_esynd_status(uint64_t afsr, uint64_t afsr_bit) 53067c478bd9Sstevel@tonic-gate { 53077c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, esynd_overwrite)); 53087c478bd9Sstevel@tonic-gate } 53097c478bd9Sstevel@tonic-gate 53107c478bd9Sstevel@tonic-gate static int 53117c478bd9Sstevel@tonic-gate afsr_to_msynd_status(uint64_t afsr, uint64_t afsr_bit) 53127c478bd9Sstevel@tonic-gate { 53137c478bd9Sstevel@tonic-gate return (afsr_to_overw_status(afsr, afsr_bit, msynd_overwrite)); 53147c478bd9Sstevel@tonic-gate } 53157c478bd9Sstevel@tonic-gate 53167c478bd9Sstevel@tonic-gate static int 53177c478bd9Sstevel@tonic-gate afsr_to_synd_status(uint_t cpuid, uint64_t afsr, uint64_t afsr_bit) 53187c478bd9Sstevel@tonic-gate { 53197c478bd9Sstevel@tonic-gate #ifdef lint 53207c478bd9Sstevel@tonic-gate cpuid = cpuid; 53217c478bd9Sstevel@tonic-gate #endif 532293743541Smb #if defined(CHEETAH_PLUS) 532393743541Smb /* 532493743541Smb * The M_SYND overwrite policy is combined with the E_SYND overwrite 532593743541Smb * policy for Cheetah+ and separate for Panther CPUs. 532693743541Smb */ 53277c478bd9Sstevel@tonic-gate if (afsr_bit & C_AFSR_MSYND_ERRS) { 532893743541Smb if (IS_PANTHER(cpunodes[cpuid].implementation)) 532993743541Smb return (afsr_to_msynd_status(afsr, afsr_bit)); 533093743541Smb else 533193743541Smb return (afsr_to_esynd_status(afsr, afsr_bit)); 53327c478bd9Sstevel@tonic-gate } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 53337c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[cpuid].implementation)) 53347c478bd9Sstevel@tonic-gate return (afsr_to_pn_esynd_status(afsr, afsr_bit)); 53357c478bd9Sstevel@tonic-gate else 53367c478bd9Sstevel@tonic-gate return (afsr_to_esynd_status(afsr, afsr_bit)); 53377c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */ 533893743541Smb if (afsr_bit & C_AFSR_MSYND_ERRS) { 533993743541Smb return (afsr_to_msynd_status(afsr, afsr_bit)); 534093743541Smb } else if (afsr_bit & (C_AFSR_ESYND_ERRS | C_AFSR_EXT_ESYND_ERRS)) { 53417c478bd9Sstevel@tonic-gate return (afsr_to_esynd_status(afsr, afsr_bit)); 53427c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 53437c478bd9Sstevel@tonic-gate } else { 53447c478bd9Sstevel@tonic-gate return (AFLT_STAT_INVALID); 53457c478bd9Sstevel@tonic-gate } 53467c478bd9Sstevel@tonic-gate } 53477c478bd9Sstevel@tonic-gate 53487c478bd9Sstevel@tonic-gate /* 53497c478bd9Sstevel@tonic-gate * Slave CPU stick synchronization. 53507c478bd9Sstevel@tonic-gate */ 53517c478bd9Sstevel@tonic-gate void 53527c478bd9Sstevel@tonic-gate sticksync_slave(void) 53537c478bd9Sstevel@tonic-gate { 53547c478bd9Sstevel@tonic-gate int i; 53557c478bd9Sstevel@tonic-gate int tries = 0; 53567c478bd9Sstevel@tonic-gate int64_t tskew; 53577c478bd9Sstevel@tonic-gate int64_t av_tskew; 53587c478bd9Sstevel@tonic-gate 53597c478bd9Sstevel@tonic-gate kpreempt_disable(); 53607c478bd9Sstevel@tonic-gate /* wait for the master side */ 53617c478bd9Sstevel@tonic-gate while (stick_sync_cmd != SLAVE_START) 53627c478bd9Sstevel@tonic-gate ; 53637c478bd9Sstevel@tonic-gate /* 53647c478bd9Sstevel@tonic-gate * Synchronization should only take a few tries at most. But in the 53657c478bd9Sstevel@tonic-gate * odd case where the cpu isn't cooperating we'll keep trying. A cpu 53667c478bd9Sstevel@tonic-gate * without it's stick synchronized wouldn't be a good citizen. 53677c478bd9Sstevel@tonic-gate */ 53687c478bd9Sstevel@tonic-gate while (slave_done == 0) { 53697c478bd9Sstevel@tonic-gate /* 53707c478bd9Sstevel@tonic-gate * Time skew calculation. 53717c478bd9Sstevel@tonic-gate */ 53727c478bd9Sstevel@tonic-gate av_tskew = tskew = 0; 53737c478bd9Sstevel@tonic-gate 53747c478bd9Sstevel@tonic-gate for (i = 0; i < stick_iter; i++) { 53757c478bd9Sstevel@tonic-gate /* make location hot */ 53767c478bd9Sstevel@tonic-gate timestamp[EV_A_START] = 0; 53777c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_A_START]); 53787c478bd9Sstevel@tonic-gate 53797c478bd9Sstevel@tonic-gate /* tell the master we're ready */ 53807c478bd9Sstevel@tonic-gate stick_sync_cmd = MASTER_START; 53817c478bd9Sstevel@tonic-gate 53827c478bd9Sstevel@tonic-gate /* and wait */ 53837c478bd9Sstevel@tonic-gate while (stick_sync_cmd != SLAVE_CONT) 53847c478bd9Sstevel@tonic-gate ; 53857c478bd9Sstevel@tonic-gate /* Event B end */ 53867c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_B_END]); 53877c478bd9Sstevel@tonic-gate 53887c478bd9Sstevel@tonic-gate /* calculate time skew */ 53897c478bd9Sstevel@tonic-gate tskew = ((timestamp[EV_B_END] - timestamp[EV_B_START]) 5390cbaac45eSkm - (timestamp[EV_A_END] - timestamp[EV_A_START])) 5391cbaac45eSkm / 2; 53927c478bd9Sstevel@tonic-gate 53937c478bd9Sstevel@tonic-gate /* keep running count */ 53947c478bd9Sstevel@tonic-gate av_tskew += tskew; 53957c478bd9Sstevel@tonic-gate } /* for */ 53967c478bd9Sstevel@tonic-gate 53977c478bd9Sstevel@tonic-gate /* 53987c478bd9Sstevel@tonic-gate * Adjust stick for time skew if not within the max allowed; 53997c478bd9Sstevel@tonic-gate * otherwise we're all done. 54007c478bd9Sstevel@tonic-gate */ 54017c478bd9Sstevel@tonic-gate if (stick_iter != 0) 54027c478bd9Sstevel@tonic-gate av_tskew = av_tskew/stick_iter; 54037c478bd9Sstevel@tonic-gate if (ABS(av_tskew) > stick_tsk) { 54047c478bd9Sstevel@tonic-gate /* 54057c478bd9Sstevel@tonic-gate * If the skew is 1 (the slave's STICK register 54067c478bd9Sstevel@tonic-gate * is 1 STICK ahead of the master's), stick_adj 54077c478bd9Sstevel@tonic-gate * could fail to adjust the slave's STICK register 54087c478bd9Sstevel@tonic-gate * if the STICK read on the slave happens to 54097c478bd9Sstevel@tonic-gate * align with the increment of the STICK. 54107c478bd9Sstevel@tonic-gate * Therefore, we increment the skew to 2. 54117c478bd9Sstevel@tonic-gate */ 54127c478bd9Sstevel@tonic-gate if (av_tskew == 1) 54137c478bd9Sstevel@tonic-gate av_tskew++; 54147c478bd9Sstevel@tonic-gate stick_adj(-av_tskew); 54157c478bd9Sstevel@tonic-gate } else 54167c478bd9Sstevel@tonic-gate slave_done = 1; 54177c478bd9Sstevel@tonic-gate #ifdef DEBUG 54187c478bd9Sstevel@tonic-gate if (tries < DSYNC_ATTEMPTS) 54197c478bd9Sstevel@tonic-gate stick_sync_stats[CPU->cpu_id].skew_val[tries] = 5420cbaac45eSkm av_tskew; 54217c478bd9Sstevel@tonic-gate ++tries; 54227c478bd9Sstevel@tonic-gate #endif /* DEBUG */ 54237c478bd9Sstevel@tonic-gate #ifdef lint 54247c478bd9Sstevel@tonic-gate tries = tries; 54257c478bd9Sstevel@tonic-gate #endif 54267c478bd9Sstevel@tonic-gate 54277c478bd9Sstevel@tonic-gate } /* while */ 54287c478bd9Sstevel@tonic-gate 54297c478bd9Sstevel@tonic-gate /* allow the master to finish */ 54307c478bd9Sstevel@tonic-gate stick_sync_cmd = EVENT_NULL; 54317c478bd9Sstevel@tonic-gate kpreempt_enable(); 54327c478bd9Sstevel@tonic-gate } 54337c478bd9Sstevel@tonic-gate 54347c478bd9Sstevel@tonic-gate /* 54357c478bd9Sstevel@tonic-gate * Master CPU side of stick synchronization. 54367c478bd9Sstevel@tonic-gate * - timestamp end of Event A 54377c478bd9Sstevel@tonic-gate * - timestamp beginning of Event B 54387c478bd9Sstevel@tonic-gate */ 54397c478bd9Sstevel@tonic-gate void 54407c478bd9Sstevel@tonic-gate sticksync_master(void) 54417c478bd9Sstevel@tonic-gate { 54427c478bd9Sstevel@tonic-gate int i; 54437c478bd9Sstevel@tonic-gate 54447c478bd9Sstevel@tonic-gate kpreempt_disable(); 54457c478bd9Sstevel@tonic-gate /* tell the slave we've started */ 54467c478bd9Sstevel@tonic-gate slave_done = 0; 54477c478bd9Sstevel@tonic-gate stick_sync_cmd = SLAVE_START; 54487c478bd9Sstevel@tonic-gate 54497c478bd9Sstevel@tonic-gate while (slave_done == 0) { 54507c478bd9Sstevel@tonic-gate for (i = 0; i < stick_iter; i++) { 54517c478bd9Sstevel@tonic-gate /* wait for the slave */ 54527c478bd9Sstevel@tonic-gate while (stick_sync_cmd != MASTER_START) 54537c478bd9Sstevel@tonic-gate ; 54547c478bd9Sstevel@tonic-gate /* Event A end */ 54557c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_A_END]); 54567c478bd9Sstevel@tonic-gate 54577c478bd9Sstevel@tonic-gate /* make location hot */ 54587c478bd9Sstevel@tonic-gate timestamp[EV_B_START] = 0; 54597c478bd9Sstevel@tonic-gate stick_timestamp(×tamp[EV_B_START]); 54607c478bd9Sstevel@tonic-gate 54617c478bd9Sstevel@tonic-gate /* tell the slave to continue */ 54627c478bd9Sstevel@tonic-gate stick_sync_cmd = SLAVE_CONT; 54637c478bd9Sstevel@tonic-gate } /* for */ 54647c478bd9Sstevel@tonic-gate 54657c478bd9Sstevel@tonic-gate /* wait while slave calculates time skew */ 54667c478bd9Sstevel@tonic-gate while (stick_sync_cmd == SLAVE_CONT) 54677c478bd9Sstevel@tonic-gate ; 54687c478bd9Sstevel@tonic-gate } /* while */ 54697c478bd9Sstevel@tonic-gate kpreempt_enable(); 54707c478bd9Sstevel@tonic-gate } 54717c478bd9Sstevel@tonic-gate 54727c478bd9Sstevel@tonic-gate /* 54737c478bd9Sstevel@tonic-gate * Cheetah/Cheetah+ have disrupting error for copyback's, so we don't need to 54747c478bd9Sstevel@tonic-gate * do Spitfire hack of xcall'ing all the cpus to ask to check for them. Also, 54757c478bd9Sstevel@tonic-gate * in cpu_async_panic_callb, each cpu checks for CPU events on its way to 54767c478bd9Sstevel@tonic-gate * panic idle. 54777c478bd9Sstevel@tonic-gate */ 54787c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 54797c478bd9Sstevel@tonic-gate void 54807c478bd9Sstevel@tonic-gate cpu_check_allcpus(struct async_flt *aflt) 54817c478bd9Sstevel@tonic-gate {} 54827c478bd9Sstevel@tonic-gate 54837c478bd9Sstevel@tonic-gate struct kmem_cache *ch_private_cache; 54847c478bd9Sstevel@tonic-gate 54857c478bd9Sstevel@tonic-gate /* 54867c478bd9Sstevel@tonic-gate * Cpu private unitialization. Uninitialize the Ecache scrubber and 54877c478bd9Sstevel@tonic-gate * deallocate the scrubber data structures and cpu_private data structure. 54887c478bd9Sstevel@tonic-gate */ 54897c478bd9Sstevel@tonic-gate void 54907c478bd9Sstevel@tonic-gate cpu_uninit_private(struct cpu *cp) 54917c478bd9Sstevel@tonic-gate { 54927c478bd9Sstevel@tonic-gate cheetah_private_t *chprp = CPU_PRIVATE(cp); 54937c478bd9Sstevel@tonic-gate 54947c478bd9Sstevel@tonic-gate ASSERT(chprp); 54957c478bd9Sstevel@tonic-gate cpu_uninit_ecache_scrub_dr(cp); 54967c478bd9Sstevel@tonic-gate CPU_PRIVATE(cp) = NULL; 54977c478bd9Sstevel@tonic-gate ch_err_tl1_paddrs[cp->cpu_id] = NULL; 54987c478bd9Sstevel@tonic-gate kmem_cache_free(ch_private_cache, chprp); 54997c478bd9Sstevel@tonic-gate cmp_delete_cpu(cp->cpu_id); 55007c478bd9Sstevel@tonic-gate 55017c478bd9Sstevel@tonic-gate } 55027c478bd9Sstevel@tonic-gate 55037c478bd9Sstevel@tonic-gate /* 55047c478bd9Sstevel@tonic-gate * Cheetah Cache Scrubbing 55057c478bd9Sstevel@tonic-gate * 55067c478bd9Sstevel@tonic-gate * The primary purpose of Cheetah cache scrubbing is to reduce the exposure 55077c478bd9Sstevel@tonic-gate * of E$ tags, D$ data, and I$ data to cosmic ray events since they are not 55087c478bd9Sstevel@tonic-gate * protected by either parity or ECC. 55097c478bd9Sstevel@tonic-gate * 55107c478bd9Sstevel@tonic-gate * We currently default the E$ and D$ scan rate to 100 (scan 10% of the 55117c478bd9Sstevel@tonic-gate * cache per second). Due to the the specifics of how the I$ control 55127c478bd9Sstevel@tonic-gate * logic works with respect to the ASI used to scrub I$ lines, the entire 55137c478bd9Sstevel@tonic-gate * I$ is scanned at once. 55147c478bd9Sstevel@tonic-gate */ 55157c478bd9Sstevel@tonic-gate 55167c478bd9Sstevel@tonic-gate /* 55177c478bd9Sstevel@tonic-gate * Tuneables to enable and disable the scrubbing of the caches, and to tune 55187c478bd9Sstevel@tonic-gate * scrubbing behavior. These may be changed via /etc/system or using mdb 55197c478bd9Sstevel@tonic-gate * on a running system. 55207c478bd9Sstevel@tonic-gate */ 55217c478bd9Sstevel@tonic-gate int dcache_scrub_enable = 1; /* D$ scrubbing is on by default */ 55227c478bd9Sstevel@tonic-gate 55237c478bd9Sstevel@tonic-gate /* 55247c478bd9Sstevel@tonic-gate * The following are the PIL levels that the softints/cross traps will fire at. 55257c478bd9Sstevel@tonic-gate */ 55267c478bd9Sstevel@tonic-gate uint_t ecache_scrub_pil = PIL_9; /* E$ scrub PIL for cross traps */ 55277c478bd9Sstevel@tonic-gate uint_t dcache_scrub_pil = PIL_9; /* D$ scrub PIL for cross traps */ 55287c478bd9Sstevel@tonic-gate uint_t icache_scrub_pil = PIL_9; /* I$ scrub PIL for cross traps */ 55297c478bd9Sstevel@tonic-gate 55307c478bd9Sstevel@tonic-gate #if defined(JALAPENO) 55317c478bd9Sstevel@tonic-gate 55327c478bd9Sstevel@tonic-gate /* 55337c478bd9Sstevel@tonic-gate * Due to several errata (82, 85, 86), we don't enable the L2$ scrubber 55347c478bd9Sstevel@tonic-gate * on Jalapeno. 55357c478bd9Sstevel@tonic-gate */ 55367c478bd9Sstevel@tonic-gate int ecache_scrub_enable = 0; 55377c478bd9Sstevel@tonic-gate 55387c478bd9Sstevel@tonic-gate #else /* JALAPENO */ 55397c478bd9Sstevel@tonic-gate 55407c478bd9Sstevel@tonic-gate /* 55417c478bd9Sstevel@tonic-gate * With all other cpu types, E$ scrubbing is on by default 55427c478bd9Sstevel@tonic-gate */ 55437c478bd9Sstevel@tonic-gate int ecache_scrub_enable = 1; 55447c478bd9Sstevel@tonic-gate 55457c478bd9Sstevel@tonic-gate #endif /* JALAPENO */ 55467c478bd9Sstevel@tonic-gate 55477c478bd9Sstevel@tonic-gate 55487c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) || defined(JALAPENO) || defined(SERRANO) 55497c478bd9Sstevel@tonic-gate 55507c478bd9Sstevel@tonic-gate /* 55517c478bd9Sstevel@tonic-gate * The I$ scrubber tends to cause latency problems for real-time SW, so it 55527c478bd9Sstevel@tonic-gate * is disabled by default on non-Cheetah systems 55537c478bd9Sstevel@tonic-gate */ 55547c478bd9Sstevel@tonic-gate int icache_scrub_enable = 0; 55557c478bd9Sstevel@tonic-gate 55567c478bd9Sstevel@tonic-gate /* 55577c478bd9Sstevel@tonic-gate * Tuneables specifying the scrub calls per second and the scan rate 55587c478bd9Sstevel@tonic-gate * for each cache 55597c478bd9Sstevel@tonic-gate * 55607c478bd9Sstevel@tonic-gate * The cyclic times are set during boot based on the following values. 55617c478bd9Sstevel@tonic-gate * Changing these values in mdb after this time will have no effect. If 55627c478bd9Sstevel@tonic-gate * a different value is desired, it must be set in /etc/system before a 55637c478bd9Sstevel@tonic-gate * reboot. 55647c478bd9Sstevel@tonic-gate */ 55657c478bd9Sstevel@tonic-gate int ecache_calls_a_sec = 1; 55667c478bd9Sstevel@tonic-gate int dcache_calls_a_sec = 2; 55677c478bd9Sstevel@tonic-gate int icache_calls_a_sec = 2; 55687c478bd9Sstevel@tonic-gate 55697c478bd9Sstevel@tonic-gate int ecache_scan_rate_idle = 1; 55707c478bd9Sstevel@tonic-gate int ecache_scan_rate_busy = 1; 55717c478bd9Sstevel@tonic-gate int dcache_scan_rate_idle = 1; 55727c478bd9Sstevel@tonic-gate int dcache_scan_rate_busy = 1; 55737c478bd9Sstevel@tonic-gate int icache_scan_rate_idle = 1; 55747c478bd9Sstevel@tonic-gate int icache_scan_rate_busy = 1; 55757c478bd9Sstevel@tonic-gate 55767c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS || JALAPENO || SERRANO */ 55777c478bd9Sstevel@tonic-gate 55787c478bd9Sstevel@tonic-gate int icache_scrub_enable = 1; /* I$ scrubbing is on by default */ 55797c478bd9Sstevel@tonic-gate 55807c478bd9Sstevel@tonic-gate int ecache_calls_a_sec = 100; /* E$ scrub calls per seconds */ 55817c478bd9Sstevel@tonic-gate int dcache_calls_a_sec = 100; /* D$ scrub calls per seconds */ 55827c478bd9Sstevel@tonic-gate int icache_calls_a_sec = 100; /* I$ scrub calls per seconds */ 55837c478bd9Sstevel@tonic-gate 55847c478bd9Sstevel@tonic-gate int ecache_scan_rate_idle = 100; /* E$ scan rate (in tenths of a %) */ 55857c478bd9Sstevel@tonic-gate int ecache_scan_rate_busy = 100; /* E$ scan rate (in tenths of a %) */ 55867c478bd9Sstevel@tonic-gate int dcache_scan_rate_idle = 100; /* D$ scan rate (in tenths of a %) */ 55877c478bd9Sstevel@tonic-gate int dcache_scan_rate_busy = 100; /* D$ scan rate (in tenths of a %) */ 55887c478bd9Sstevel@tonic-gate int icache_scan_rate_idle = 100; /* I$ scan rate (in tenths of a %) */ 55897c478bd9Sstevel@tonic-gate int icache_scan_rate_busy = 100; /* I$ scan rate (in tenths of a %) */ 55907c478bd9Sstevel@tonic-gate 55917c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS || JALAPENO || SERRANO */ 55927c478bd9Sstevel@tonic-gate 55937c478bd9Sstevel@tonic-gate /* 55947c478bd9Sstevel@tonic-gate * In order to scrub on offline cpus, a cross trap is sent. The handler will 55957c478bd9Sstevel@tonic-gate * increment the outstanding request counter and schedule a softint to run 55967c478bd9Sstevel@tonic-gate * the scrubber. 55977c478bd9Sstevel@tonic-gate */ 55987c478bd9Sstevel@tonic-gate extern xcfunc_t cache_scrubreq_tl1; 55997c478bd9Sstevel@tonic-gate 56007c478bd9Sstevel@tonic-gate /* 56017c478bd9Sstevel@tonic-gate * These are the softint functions for each cache scrubber 56027c478bd9Sstevel@tonic-gate */ 56037c478bd9Sstevel@tonic-gate static uint_t scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2); 56047c478bd9Sstevel@tonic-gate static uint_t scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2); 56057c478bd9Sstevel@tonic-gate static uint_t scrub_icache_line_intr(caddr_t arg1, caddr_t arg2); 56067c478bd9Sstevel@tonic-gate 56077c478bd9Sstevel@tonic-gate /* 56087c478bd9Sstevel@tonic-gate * The cache scrub info table contains cache specific information 56097c478bd9Sstevel@tonic-gate * and allows for some of the scrub code to be table driven, reducing 56107c478bd9Sstevel@tonic-gate * duplication of cache similar code. 56117c478bd9Sstevel@tonic-gate * 56127c478bd9Sstevel@tonic-gate * This table keeps a copy of the value in the calls per second variable 56137c478bd9Sstevel@tonic-gate * (?cache_calls_a_sec). This makes it much more difficult for someone 56147c478bd9Sstevel@tonic-gate * to cause us problems (for example, by setting ecache_calls_a_sec to 0 in 56157c478bd9Sstevel@tonic-gate * mdb in a misguided attempt to disable the scrubber). 56167c478bd9Sstevel@tonic-gate */ 56177c478bd9Sstevel@tonic-gate struct scrub_info { 56187c478bd9Sstevel@tonic-gate int *csi_enable; /* scrubber enable flag */ 56197c478bd9Sstevel@tonic-gate int csi_freq; /* scrubber calls per second */ 56207c478bd9Sstevel@tonic-gate int csi_index; /* index to chsm_outstanding[] */ 5621b0fc0e77Sgovinda uint64_t csi_inum; /* scrubber interrupt number */ 56227c478bd9Sstevel@tonic-gate cyclic_id_t csi_omni_cyc_id; /* omni cyclic ID */ 56237c478bd9Sstevel@tonic-gate cyclic_id_t csi_offline_cyc_id; /* offline cyclic ID */ 56247c478bd9Sstevel@tonic-gate char csi_name[3]; /* cache name for this scrub entry */ 56257c478bd9Sstevel@tonic-gate } cache_scrub_info[] = { 56267c478bd9Sstevel@tonic-gate { &ecache_scrub_enable, 0, CACHE_SCRUBBER_INFO_E, 0, 0, 0, "E$"}, 56277c478bd9Sstevel@tonic-gate { &dcache_scrub_enable, 0, CACHE_SCRUBBER_INFO_D, 0, 0, 0, "D$"}, 56287c478bd9Sstevel@tonic-gate { &icache_scrub_enable, 0, CACHE_SCRUBBER_INFO_I, 0, 0, 0, "I$"} 56297c478bd9Sstevel@tonic-gate }; 56307c478bd9Sstevel@tonic-gate 56317c478bd9Sstevel@tonic-gate /* 56327c478bd9Sstevel@tonic-gate * If scrubbing is enabled, increment the outstanding request counter. If it 56337c478bd9Sstevel@tonic-gate * is 1 (meaning there were no previous requests outstanding), call 56347c478bd9Sstevel@tonic-gate * setsoftint_tl1 through xt_one_unchecked, which eventually ends up doing 56357c478bd9Sstevel@tonic-gate * a self trap. 56367c478bd9Sstevel@tonic-gate */ 56377c478bd9Sstevel@tonic-gate static void 56387c478bd9Sstevel@tonic-gate do_scrub(struct scrub_info *csi) 56397c478bd9Sstevel@tonic-gate { 56407c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 56417c478bd9Sstevel@tonic-gate int index = csi->csi_index; 56427c478bd9Sstevel@tonic-gate uint32_t *outstanding = &csmp->chsm_outstanding[index]; 56437c478bd9Sstevel@tonic-gate 56447c478bd9Sstevel@tonic-gate if (*(csi->csi_enable) && (csmp->chsm_enable[index])) { 56457c478bd9Sstevel@tonic-gate if (atomic_add_32_nv(outstanding, 1) == 1) { 56467c478bd9Sstevel@tonic-gate xt_one_unchecked(CPU->cpu_id, setsoftint_tl1, 56477c478bd9Sstevel@tonic-gate csi->csi_inum, 0); 56487c478bd9Sstevel@tonic-gate } 56497c478bd9Sstevel@tonic-gate } 56507c478bd9Sstevel@tonic-gate } 56517c478bd9Sstevel@tonic-gate 56527c478bd9Sstevel@tonic-gate /* 56537c478bd9Sstevel@tonic-gate * Omni cyclics don't fire on offline cpus, so we use another cyclic to 56547c478bd9Sstevel@tonic-gate * cross-trap the offline cpus. 56557c478bd9Sstevel@tonic-gate */ 56567c478bd9Sstevel@tonic-gate static void 56577c478bd9Sstevel@tonic-gate do_scrub_offline(struct scrub_info *csi) 56587c478bd9Sstevel@tonic-gate { 56597c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 56607c478bd9Sstevel@tonic-gate 56617c478bd9Sstevel@tonic-gate if (CPUSET_ISNULL(cpu_offline_set)) { 56627c478bd9Sstevel@tonic-gate /* 56637c478bd9Sstevel@tonic-gate * No offline cpus - nothing to do 56647c478bd9Sstevel@tonic-gate */ 56657c478bd9Sstevel@tonic-gate return; 56667c478bd9Sstevel@tonic-gate } 56677c478bd9Sstevel@tonic-gate 56687c478bd9Sstevel@tonic-gate if (*(csi->csi_enable) && (csmp->chsm_enable[csi->csi_index])) { 56697c478bd9Sstevel@tonic-gate xt_some(cpu_offline_set, cache_scrubreq_tl1, csi->csi_inum, 56707c478bd9Sstevel@tonic-gate csi->csi_index); 56717c478bd9Sstevel@tonic-gate } 56727c478bd9Sstevel@tonic-gate } 56737c478bd9Sstevel@tonic-gate 56747c478bd9Sstevel@tonic-gate /* 56757c478bd9Sstevel@tonic-gate * This is the initial setup for the scrubber cyclics - it sets the 56767c478bd9Sstevel@tonic-gate * interrupt level, frequency, and function to call. 56777c478bd9Sstevel@tonic-gate */ 56787c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 56797c478bd9Sstevel@tonic-gate static void 56807c478bd9Sstevel@tonic-gate cpu_scrub_cyclic_setup(void *arg, cpu_t *cpu, cyc_handler_t *hdlr, 56817c478bd9Sstevel@tonic-gate cyc_time_t *when) 56827c478bd9Sstevel@tonic-gate { 56837c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg; 56847c478bd9Sstevel@tonic-gate 56857c478bd9Sstevel@tonic-gate ASSERT(csi != NULL); 56867c478bd9Sstevel@tonic-gate hdlr->cyh_func = (cyc_func_t)do_scrub; 56877c478bd9Sstevel@tonic-gate hdlr->cyh_level = CY_LOW_LEVEL; 56887c478bd9Sstevel@tonic-gate hdlr->cyh_arg = arg; 56897c478bd9Sstevel@tonic-gate 56907c478bd9Sstevel@tonic-gate when->cyt_when = 0; /* Start immediately */ 56917c478bd9Sstevel@tonic-gate when->cyt_interval = NANOSEC / csi->csi_freq; 56927c478bd9Sstevel@tonic-gate } 56937c478bd9Sstevel@tonic-gate 56947c478bd9Sstevel@tonic-gate /* 56957c478bd9Sstevel@tonic-gate * Initialization for cache scrubbing. 56967c478bd9Sstevel@tonic-gate * This routine is called AFTER all cpus have had cpu_init_private called 56977c478bd9Sstevel@tonic-gate * to initialize their private data areas. 56987c478bd9Sstevel@tonic-gate */ 56997c478bd9Sstevel@tonic-gate void 57007c478bd9Sstevel@tonic-gate cpu_init_cache_scrub(void) 57017c478bd9Sstevel@tonic-gate { 57027c478bd9Sstevel@tonic-gate int i; 57037c478bd9Sstevel@tonic-gate struct scrub_info *csi; 57047c478bd9Sstevel@tonic-gate cyc_omni_handler_t omni_hdlr; 57057c478bd9Sstevel@tonic-gate cyc_handler_t offline_hdlr; 57067c478bd9Sstevel@tonic-gate cyc_time_t when; 57077c478bd9Sstevel@tonic-gate 57087c478bd9Sstevel@tonic-gate /* 57097c478bd9Sstevel@tonic-gate * save away the maximum number of lines for the D$ 57107c478bd9Sstevel@tonic-gate */ 57117c478bd9Sstevel@tonic-gate dcache_nlines = dcache_size / dcache_linesize; 57127c478bd9Sstevel@tonic-gate 57137c478bd9Sstevel@tonic-gate /* 57147c478bd9Sstevel@tonic-gate * register the softints for the cache scrubbing 57157c478bd9Sstevel@tonic-gate */ 57167c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_inum = 57177c478bd9Sstevel@tonic-gate add_softintr(ecache_scrub_pil, scrub_ecache_line_intr, 5718b0fc0e77Sgovinda (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_E], SOFTINT_MT); 57197c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_E].csi_freq = ecache_calls_a_sec; 57207c478bd9Sstevel@tonic-gate 57217c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_inum = 57227c478bd9Sstevel@tonic-gate add_softintr(dcache_scrub_pil, scrub_dcache_line_intr, 5723b0fc0e77Sgovinda (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_D], SOFTINT_MT); 57247c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_D].csi_freq = dcache_calls_a_sec; 57257c478bd9Sstevel@tonic-gate 57267c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_inum = 57277c478bd9Sstevel@tonic-gate add_softintr(icache_scrub_pil, scrub_icache_line_intr, 5728b0fc0e77Sgovinda (caddr_t)&cache_scrub_info[CACHE_SCRUBBER_INFO_I], SOFTINT_MT); 57297c478bd9Sstevel@tonic-gate cache_scrub_info[CACHE_SCRUBBER_INFO_I].csi_freq = icache_calls_a_sec; 57307c478bd9Sstevel@tonic-gate 57317c478bd9Sstevel@tonic-gate /* 57327c478bd9Sstevel@tonic-gate * start the scrubbing for all the caches 57337c478bd9Sstevel@tonic-gate */ 57347c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 57357c478bd9Sstevel@tonic-gate for (i = 0; i < CACHE_SCRUBBER_COUNT; i++) { 57367c478bd9Sstevel@tonic-gate 57377c478bd9Sstevel@tonic-gate csi = &cache_scrub_info[i]; 57387c478bd9Sstevel@tonic-gate 57397c478bd9Sstevel@tonic-gate if (!(*csi->csi_enable)) 57407c478bd9Sstevel@tonic-gate continue; 57417c478bd9Sstevel@tonic-gate 57427c478bd9Sstevel@tonic-gate /* 57437c478bd9Sstevel@tonic-gate * force the following to be true: 57447c478bd9Sstevel@tonic-gate * 1 <= calls_a_sec <= hz 57457c478bd9Sstevel@tonic-gate */ 57467c478bd9Sstevel@tonic-gate if (csi->csi_freq > hz) { 57477c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "%s scrub calls_a_sec set too high " 5748cbaac45eSkm "(%d); resetting to hz (%d)", csi->csi_name, 5749cbaac45eSkm csi->csi_freq, hz); 57507c478bd9Sstevel@tonic-gate csi->csi_freq = hz; 57517c478bd9Sstevel@tonic-gate } else if (csi->csi_freq < 1) { 57527c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "%s scrub calls_a_sec set too low " 5753cbaac45eSkm "(%d); resetting to 1", csi->csi_name, 5754cbaac45eSkm csi->csi_freq); 57557c478bd9Sstevel@tonic-gate csi->csi_freq = 1; 57567c478bd9Sstevel@tonic-gate } 57577c478bd9Sstevel@tonic-gate 57587c478bd9Sstevel@tonic-gate omni_hdlr.cyo_online = cpu_scrub_cyclic_setup; 57597c478bd9Sstevel@tonic-gate omni_hdlr.cyo_offline = NULL; 57607c478bd9Sstevel@tonic-gate omni_hdlr.cyo_arg = (void *)csi; 57617c478bd9Sstevel@tonic-gate 57627c478bd9Sstevel@tonic-gate offline_hdlr.cyh_func = (cyc_func_t)do_scrub_offline; 57637c478bd9Sstevel@tonic-gate offline_hdlr.cyh_arg = (void *)csi; 57647c478bd9Sstevel@tonic-gate offline_hdlr.cyh_level = CY_LOW_LEVEL; 57657c478bd9Sstevel@tonic-gate 57667c478bd9Sstevel@tonic-gate when.cyt_when = 0; /* Start immediately */ 57677c478bd9Sstevel@tonic-gate when.cyt_interval = NANOSEC / csi->csi_freq; 57687c478bd9Sstevel@tonic-gate 57697c478bd9Sstevel@tonic-gate csi->csi_omni_cyc_id = cyclic_add_omni(&omni_hdlr); 57707c478bd9Sstevel@tonic-gate csi->csi_offline_cyc_id = cyclic_add(&offline_hdlr, &when); 57717c478bd9Sstevel@tonic-gate } 57727c478bd9Sstevel@tonic-gate register_cpu_setup_func(cpu_scrub_cpu_setup, NULL); 57737c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 57747c478bd9Sstevel@tonic-gate } 57757c478bd9Sstevel@tonic-gate 57767c478bd9Sstevel@tonic-gate /* 57777c478bd9Sstevel@tonic-gate * Indicate that the specified cpu is idle. 57787c478bd9Sstevel@tonic-gate */ 57797c478bd9Sstevel@tonic-gate void 57807c478bd9Sstevel@tonic-gate cpu_idle_ecache_scrub(struct cpu *cp) 57817c478bd9Sstevel@tonic-gate { 57827c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(cp) != NULL) { 57837c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 57847c478bd9Sstevel@tonic-gate csmp->chsm_ecache_busy = ECACHE_CPU_IDLE; 57857c478bd9Sstevel@tonic-gate } 57867c478bd9Sstevel@tonic-gate } 57877c478bd9Sstevel@tonic-gate 57887c478bd9Sstevel@tonic-gate /* 57897c478bd9Sstevel@tonic-gate * Indicate that the specified cpu is busy. 57907c478bd9Sstevel@tonic-gate */ 57917c478bd9Sstevel@tonic-gate void 57927c478bd9Sstevel@tonic-gate cpu_busy_ecache_scrub(struct cpu *cp) 57937c478bd9Sstevel@tonic-gate { 57947c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(cp) != NULL) { 57957c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 57967c478bd9Sstevel@tonic-gate csmp->chsm_ecache_busy = ECACHE_CPU_BUSY; 57977c478bd9Sstevel@tonic-gate } 57987c478bd9Sstevel@tonic-gate } 57997c478bd9Sstevel@tonic-gate 58007c478bd9Sstevel@tonic-gate /* 58017c478bd9Sstevel@tonic-gate * Initialization for cache scrubbing for the specified cpu. 58027c478bd9Sstevel@tonic-gate */ 58037c478bd9Sstevel@tonic-gate void 58047c478bd9Sstevel@tonic-gate cpu_init_ecache_scrub_dr(struct cpu *cp) 58057c478bd9Sstevel@tonic-gate { 58067c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 58077c478bd9Sstevel@tonic-gate int cpuid = cp->cpu_id; 58087c478bd9Sstevel@tonic-gate 58097c478bd9Sstevel@tonic-gate /* initialize the number of lines in the caches */ 58107c478bd9Sstevel@tonic-gate csmp->chsm_ecache_nlines = cpunodes[cpuid].ecache_size / 58117c478bd9Sstevel@tonic-gate cpunodes[cpuid].ecache_linesize; 58127c478bd9Sstevel@tonic-gate csmp->chsm_icache_nlines = CPU_PRIVATE_VAL(cp, chpr_icache_size) / 58137c478bd9Sstevel@tonic-gate CPU_PRIVATE_VAL(cp, chpr_icache_linesize); 58147c478bd9Sstevel@tonic-gate 58157c478bd9Sstevel@tonic-gate /* 58167c478bd9Sstevel@tonic-gate * do_scrub() and do_scrub_offline() check both the global 58177c478bd9Sstevel@tonic-gate * ?cache_scrub_enable and this per-cpu enable variable. All scrubbers 58187c478bd9Sstevel@tonic-gate * check this value before scrubbing. Currently, we use it to 58197c478bd9Sstevel@tonic-gate * disable the E$ scrubber on multi-core cpus or while running at 58207c478bd9Sstevel@tonic-gate * slowed speed. For now, just turn everything on and allow 58217c478bd9Sstevel@tonic-gate * cpu_init_private() to change it if necessary. 58227c478bd9Sstevel@tonic-gate */ 58237c478bd9Sstevel@tonic-gate csmp->chsm_enable[CACHE_SCRUBBER_INFO_E] = 1; 58247c478bd9Sstevel@tonic-gate csmp->chsm_enable[CACHE_SCRUBBER_INFO_D] = 1; 58257c478bd9Sstevel@tonic-gate csmp->chsm_enable[CACHE_SCRUBBER_INFO_I] = 1; 58267c478bd9Sstevel@tonic-gate 58277c478bd9Sstevel@tonic-gate cpu_busy_ecache_scrub(cp); 58287c478bd9Sstevel@tonic-gate } 58297c478bd9Sstevel@tonic-gate 58307c478bd9Sstevel@tonic-gate /* 58317c478bd9Sstevel@tonic-gate * Un-initialization for cache scrubbing for the specified cpu. 58327c478bd9Sstevel@tonic-gate */ 58337c478bd9Sstevel@tonic-gate static void 58347c478bd9Sstevel@tonic-gate cpu_uninit_ecache_scrub_dr(struct cpu *cp) 58357c478bd9Sstevel@tonic-gate { 58367c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(cp, chpr_scrub_misc); 58377c478bd9Sstevel@tonic-gate 58387c478bd9Sstevel@tonic-gate /* 58397c478bd9Sstevel@tonic-gate * un-initialize bookkeeping for cache scrubbing 58407c478bd9Sstevel@tonic-gate */ 58417c478bd9Sstevel@tonic-gate bzero(csmp, sizeof (ch_scrub_misc_t)); 58427c478bd9Sstevel@tonic-gate 58437c478bd9Sstevel@tonic-gate cpu_idle_ecache_scrub(cp); 58447c478bd9Sstevel@tonic-gate } 58457c478bd9Sstevel@tonic-gate 58467c478bd9Sstevel@tonic-gate /* 58477c478bd9Sstevel@tonic-gate * Called periodically on each CPU to scrub the D$. 58487c478bd9Sstevel@tonic-gate */ 58497c478bd9Sstevel@tonic-gate static void 58507c478bd9Sstevel@tonic-gate scrub_dcache(int how_many) 58517c478bd9Sstevel@tonic-gate { 58527c478bd9Sstevel@tonic-gate int i; 58537c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 58547c478bd9Sstevel@tonic-gate int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D]; 58557c478bd9Sstevel@tonic-gate 58567c478bd9Sstevel@tonic-gate /* 58577c478bd9Sstevel@tonic-gate * scrub the desired number of lines 58587c478bd9Sstevel@tonic-gate */ 58597c478bd9Sstevel@tonic-gate for (i = 0; i < how_many; i++) { 58607c478bd9Sstevel@tonic-gate /* 58617c478bd9Sstevel@tonic-gate * scrub a D$ line 58627c478bd9Sstevel@tonic-gate */ 58637c478bd9Sstevel@tonic-gate dcache_inval_line(index); 58647c478bd9Sstevel@tonic-gate 58657c478bd9Sstevel@tonic-gate /* 58667c478bd9Sstevel@tonic-gate * calculate the next D$ line to scrub, assumes 58677c478bd9Sstevel@tonic-gate * that dcache_nlines is a power of 2 58687c478bd9Sstevel@tonic-gate */ 58697c478bd9Sstevel@tonic-gate index = (index + 1) & (dcache_nlines - 1); 58707c478bd9Sstevel@tonic-gate } 58717c478bd9Sstevel@tonic-gate 58727c478bd9Sstevel@tonic-gate /* 58737c478bd9Sstevel@tonic-gate * set the scrub index for the next visit 58747c478bd9Sstevel@tonic-gate */ 58757c478bd9Sstevel@tonic-gate csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_D] = index; 58767c478bd9Sstevel@tonic-gate } 58777c478bd9Sstevel@tonic-gate 58787c478bd9Sstevel@tonic-gate /* 58797c478bd9Sstevel@tonic-gate * Handler for D$ scrub inum softint. Call scrub_dcache until 58807c478bd9Sstevel@tonic-gate * we decrement the outstanding request count to zero. 58817c478bd9Sstevel@tonic-gate */ 58827c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 58837c478bd9Sstevel@tonic-gate static uint_t 58847c478bd9Sstevel@tonic-gate scrub_dcache_line_intr(caddr_t arg1, caddr_t arg2) 58857c478bd9Sstevel@tonic-gate { 58867c478bd9Sstevel@tonic-gate int i; 58877c478bd9Sstevel@tonic-gate int how_many; 58887c478bd9Sstevel@tonic-gate int outstanding; 58897c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 58907c478bd9Sstevel@tonic-gate uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_D]; 58917c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg1; 58927c478bd9Sstevel@tonic-gate int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 5893cbaac45eSkm dcache_scan_rate_idle : dcache_scan_rate_busy; 58947c478bd9Sstevel@tonic-gate 58957c478bd9Sstevel@tonic-gate /* 58967c478bd9Sstevel@tonic-gate * The scan rates are expressed in units of tenths of a 58977c478bd9Sstevel@tonic-gate * percent. A scan rate of 1000 (100%) means the whole 58987c478bd9Sstevel@tonic-gate * cache is scanned every second. 58997c478bd9Sstevel@tonic-gate */ 59007c478bd9Sstevel@tonic-gate how_many = (dcache_nlines * scan_rate) / (1000 * csi->csi_freq); 59017c478bd9Sstevel@tonic-gate 59027c478bd9Sstevel@tonic-gate do { 59037c478bd9Sstevel@tonic-gate outstanding = *countp; 59047c478bd9Sstevel@tonic-gate for (i = 0; i < outstanding; i++) { 59057c478bd9Sstevel@tonic-gate scrub_dcache(how_many); 59067c478bd9Sstevel@tonic-gate } 59077c478bd9Sstevel@tonic-gate } while (atomic_add_32_nv(countp, -outstanding)); 59087c478bd9Sstevel@tonic-gate 59097c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 59107c478bd9Sstevel@tonic-gate } 59117c478bd9Sstevel@tonic-gate 59127c478bd9Sstevel@tonic-gate /* 59137c478bd9Sstevel@tonic-gate * Called periodically on each CPU to scrub the I$. The I$ is scrubbed 59147c478bd9Sstevel@tonic-gate * by invalidating lines. Due to the characteristics of the ASI which 59157c478bd9Sstevel@tonic-gate * is used to invalidate an I$ line, the entire I$ must be invalidated 59167c478bd9Sstevel@tonic-gate * vs. an individual I$ line. 59177c478bd9Sstevel@tonic-gate */ 59187c478bd9Sstevel@tonic-gate static void 59197c478bd9Sstevel@tonic-gate scrub_icache(int how_many) 59207c478bd9Sstevel@tonic-gate { 59217c478bd9Sstevel@tonic-gate int i; 59227c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 59237c478bd9Sstevel@tonic-gate int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I]; 59247c478bd9Sstevel@tonic-gate int icache_nlines = csmp->chsm_icache_nlines; 59257c478bd9Sstevel@tonic-gate 59267c478bd9Sstevel@tonic-gate /* 59277c478bd9Sstevel@tonic-gate * scrub the desired number of lines 59287c478bd9Sstevel@tonic-gate */ 59297c478bd9Sstevel@tonic-gate for (i = 0; i < how_many; i++) { 59307c478bd9Sstevel@tonic-gate /* 59317c478bd9Sstevel@tonic-gate * since the entire I$ must be scrubbed at once, 59327c478bd9Sstevel@tonic-gate * wait until the index wraps to zero to invalidate 59337c478bd9Sstevel@tonic-gate * the entire I$ 59347c478bd9Sstevel@tonic-gate */ 59357c478bd9Sstevel@tonic-gate if (index == 0) { 59367c478bd9Sstevel@tonic-gate icache_inval_all(); 59377c478bd9Sstevel@tonic-gate } 59387c478bd9Sstevel@tonic-gate 59397c478bd9Sstevel@tonic-gate /* 59407c478bd9Sstevel@tonic-gate * calculate the next I$ line to scrub, assumes 59417c478bd9Sstevel@tonic-gate * that chsm_icache_nlines is a power of 2 59427c478bd9Sstevel@tonic-gate */ 59437c478bd9Sstevel@tonic-gate index = (index + 1) & (icache_nlines - 1); 59447c478bd9Sstevel@tonic-gate } 59457c478bd9Sstevel@tonic-gate 59467c478bd9Sstevel@tonic-gate /* 59477c478bd9Sstevel@tonic-gate * set the scrub index for the next visit 59487c478bd9Sstevel@tonic-gate */ 59497c478bd9Sstevel@tonic-gate csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_I] = index; 59507c478bd9Sstevel@tonic-gate } 59517c478bd9Sstevel@tonic-gate 59527c478bd9Sstevel@tonic-gate /* 59537c478bd9Sstevel@tonic-gate * Handler for I$ scrub inum softint. Call scrub_icache until 59547c478bd9Sstevel@tonic-gate * we decrement the outstanding request count to zero. 59557c478bd9Sstevel@tonic-gate */ 59567c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 59577c478bd9Sstevel@tonic-gate static uint_t 59587c478bd9Sstevel@tonic-gate scrub_icache_line_intr(caddr_t arg1, caddr_t arg2) 59597c478bd9Sstevel@tonic-gate { 59607c478bd9Sstevel@tonic-gate int i; 59617c478bd9Sstevel@tonic-gate int how_many; 59627c478bd9Sstevel@tonic-gate int outstanding; 59637c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 59647c478bd9Sstevel@tonic-gate uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_I]; 59657c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg1; 59667c478bd9Sstevel@tonic-gate int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 59677c478bd9Sstevel@tonic-gate icache_scan_rate_idle : icache_scan_rate_busy; 59687c478bd9Sstevel@tonic-gate int icache_nlines = csmp->chsm_icache_nlines; 59697c478bd9Sstevel@tonic-gate 59707c478bd9Sstevel@tonic-gate /* 59717c478bd9Sstevel@tonic-gate * The scan rates are expressed in units of tenths of a 59727c478bd9Sstevel@tonic-gate * percent. A scan rate of 1000 (100%) means the whole 59737c478bd9Sstevel@tonic-gate * cache is scanned every second. 59747c478bd9Sstevel@tonic-gate */ 59757c478bd9Sstevel@tonic-gate how_many = (icache_nlines * scan_rate) / (1000 * csi->csi_freq); 59767c478bd9Sstevel@tonic-gate 59777c478bd9Sstevel@tonic-gate do { 59787c478bd9Sstevel@tonic-gate outstanding = *countp; 59797c478bd9Sstevel@tonic-gate for (i = 0; i < outstanding; i++) { 59807c478bd9Sstevel@tonic-gate scrub_icache(how_many); 59817c478bd9Sstevel@tonic-gate } 59827c478bd9Sstevel@tonic-gate } while (atomic_add_32_nv(countp, -outstanding)); 59837c478bd9Sstevel@tonic-gate 59847c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 59857c478bd9Sstevel@tonic-gate } 59867c478bd9Sstevel@tonic-gate 59877c478bd9Sstevel@tonic-gate /* 59887c478bd9Sstevel@tonic-gate * Called periodically on each CPU to scrub the E$. 59897c478bd9Sstevel@tonic-gate */ 59907c478bd9Sstevel@tonic-gate static void 59917c478bd9Sstevel@tonic-gate scrub_ecache(int how_many) 59927c478bd9Sstevel@tonic-gate { 59937c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 59947c478bd9Sstevel@tonic-gate int i; 59957c478bd9Sstevel@tonic-gate int cpuid = CPU->cpu_id; 59967c478bd9Sstevel@tonic-gate int index = csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E]; 59977c478bd9Sstevel@tonic-gate int nlines = csmp->chsm_ecache_nlines; 59987c478bd9Sstevel@tonic-gate int linesize = cpunodes[cpuid].ecache_linesize; 59997c478bd9Sstevel@tonic-gate int ec_set_size = cpu_ecache_set_size(CPU); 60007c478bd9Sstevel@tonic-gate 60017c478bd9Sstevel@tonic-gate /* 60027c478bd9Sstevel@tonic-gate * scrub the desired number of lines 60037c478bd9Sstevel@tonic-gate */ 60047c478bd9Sstevel@tonic-gate for (i = 0; i < how_many; i++) { 60057c478bd9Sstevel@tonic-gate /* 60067c478bd9Sstevel@tonic-gate * scrub the E$ line 60077c478bd9Sstevel@tonic-gate */ 60087c478bd9Sstevel@tonic-gate ecache_flush_line(ecache_flushaddr + (index * linesize), 60097c478bd9Sstevel@tonic-gate ec_set_size); 60107c478bd9Sstevel@tonic-gate 60117c478bd9Sstevel@tonic-gate /* 60127c478bd9Sstevel@tonic-gate * calculate the next E$ line to scrub based on twice 60137c478bd9Sstevel@tonic-gate * the number of E$ lines (to displace lines containing 60147c478bd9Sstevel@tonic-gate * flush area data), assumes that the number of lines 60157c478bd9Sstevel@tonic-gate * is a power of 2 60167c478bd9Sstevel@tonic-gate */ 60177c478bd9Sstevel@tonic-gate index = (index + 1) & ((nlines << 1) - 1); 60187c478bd9Sstevel@tonic-gate } 60197c478bd9Sstevel@tonic-gate 60207c478bd9Sstevel@tonic-gate /* 60217c478bd9Sstevel@tonic-gate * set the ecache scrub index for the next visit 60227c478bd9Sstevel@tonic-gate */ 60237c478bd9Sstevel@tonic-gate csmp->chsm_flush_index[CACHE_SCRUBBER_INFO_E] = index; 60247c478bd9Sstevel@tonic-gate } 60257c478bd9Sstevel@tonic-gate 60267c478bd9Sstevel@tonic-gate /* 60277c478bd9Sstevel@tonic-gate * Handler for E$ scrub inum softint. Call the E$ scrubber until 60287c478bd9Sstevel@tonic-gate * we decrement the outstanding request count to zero. 60296319f043Srscott * 60306319f043Srscott * Due to interactions with cpu_scrub_cpu_setup(), the outstanding count may 60316319f043Srscott * become negative after the atomic_add_32_nv(). This is not a problem, as 60326319f043Srscott * the next trip around the loop won't scrub anything, and the next add will 60336319f043Srscott * reset the count back to zero. 60347c478bd9Sstevel@tonic-gate */ 60357c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 60367c478bd9Sstevel@tonic-gate static uint_t 60377c478bd9Sstevel@tonic-gate scrub_ecache_line_intr(caddr_t arg1, caddr_t arg2) 60387c478bd9Sstevel@tonic-gate { 60397c478bd9Sstevel@tonic-gate int i; 60407c478bd9Sstevel@tonic-gate int how_many; 60417c478bd9Sstevel@tonic-gate int outstanding; 60427c478bd9Sstevel@tonic-gate ch_scrub_misc_t *csmp = CPU_PRIVATE_PTR(CPU, chpr_scrub_misc); 60437c478bd9Sstevel@tonic-gate uint32_t *countp = &csmp->chsm_outstanding[CACHE_SCRUBBER_INFO_E]; 60447c478bd9Sstevel@tonic-gate struct scrub_info *csi = (struct scrub_info *)arg1; 60457c478bd9Sstevel@tonic-gate int scan_rate = (csmp->chsm_ecache_busy == ECACHE_CPU_IDLE) ? 6046cbaac45eSkm ecache_scan_rate_idle : ecache_scan_rate_busy; 60477c478bd9Sstevel@tonic-gate int ecache_nlines = csmp->chsm_ecache_nlines; 60487c478bd9Sstevel@tonic-gate 60497c478bd9Sstevel@tonic-gate /* 60507c478bd9Sstevel@tonic-gate * The scan rates are expressed in units of tenths of a 60517c478bd9Sstevel@tonic-gate * percent. A scan rate of 1000 (100%) means the whole 60527c478bd9Sstevel@tonic-gate * cache is scanned every second. 60537c478bd9Sstevel@tonic-gate */ 60547c478bd9Sstevel@tonic-gate how_many = (ecache_nlines * scan_rate) / (1000 * csi->csi_freq); 60557c478bd9Sstevel@tonic-gate 60567c478bd9Sstevel@tonic-gate do { 60577c478bd9Sstevel@tonic-gate outstanding = *countp; 60587c478bd9Sstevel@tonic-gate for (i = 0; i < outstanding; i++) { 60597c478bd9Sstevel@tonic-gate scrub_ecache(how_many); 60607c478bd9Sstevel@tonic-gate } 60617c478bd9Sstevel@tonic-gate } while (atomic_add_32_nv(countp, -outstanding)); 60627c478bd9Sstevel@tonic-gate 60637c478bd9Sstevel@tonic-gate return (DDI_INTR_CLAIMED); 60647c478bd9Sstevel@tonic-gate } 60657c478bd9Sstevel@tonic-gate 60667c478bd9Sstevel@tonic-gate /* 60677c478bd9Sstevel@tonic-gate * Timeout function to reenable CE 60687c478bd9Sstevel@tonic-gate */ 60697c478bd9Sstevel@tonic-gate static void 60707c478bd9Sstevel@tonic-gate cpu_delayed_check_ce_errors(void *arg) 60717c478bd9Sstevel@tonic-gate { 60727c478bd9Sstevel@tonic-gate if (!taskq_dispatch(ch_check_ce_tq, cpu_check_ce_errors, arg, 60737c478bd9Sstevel@tonic-gate TQ_NOSLEEP)) { 60747c478bd9Sstevel@tonic-gate (void) timeout(cpu_delayed_check_ce_errors, arg, 60757c478bd9Sstevel@tonic-gate drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 60767c478bd9Sstevel@tonic-gate } 60777c478bd9Sstevel@tonic-gate } 60787c478bd9Sstevel@tonic-gate 60797c478bd9Sstevel@tonic-gate /* 60807c478bd9Sstevel@tonic-gate * CE Deferred Re-enable after trap. 60817c478bd9Sstevel@tonic-gate * 60827c478bd9Sstevel@tonic-gate * When the CPU gets a disrupting trap for any of the errors 60837c478bd9Sstevel@tonic-gate * controlled by the CEEN bit, CEEN is disabled in the trap handler 60847c478bd9Sstevel@tonic-gate * immediately. To eliminate the possibility of multiple CEs causing 60857c478bd9Sstevel@tonic-gate * recursive stack overflow in the trap handler, we cannot 60867c478bd9Sstevel@tonic-gate * reenable CEEN while still running in the trap handler. Instead, 60877c478bd9Sstevel@tonic-gate * after a CE is logged on a CPU, we schedule a timeout function, 60887c478bd9Sstevel@tonic-gate * cpu_check_ce_errors(), to trigger after cpu_ceen_delay_secs 60897c478bd9Sstevel@tonic-gate * seconds. This function will check whether any further CEs 60907c478bd9Sstevel@tonic-gate * have occurred on that CPU, and if none have, will reenable CEEN. 60917c478bd9Sstevel@tonic-gate * 60927c478bd9Sstevel@tonic-gate * If further CEs have occurred while CEEN is disabled, another 60937c478bd9Sstevel@tonic-gate * timeout will be scheduled. This is to ensure that the CPU can 60947c478bd9Sstevel@tonic-gate * make progress in the face of CE 'storms', and that it does not 60957c478bd9Sstevel@tonic-gate * spend all its time logging CE errors. 60967c478bd9Sstevel@tonic-gate */ 60977c478bd9Sstevel@tonic-gate static void 60987c478bd9Sstevel@tonic-gate cpu_check_ce_errors(void *arg) 60997c478bd9Sstevel@tonic-gate { 6100f47a9c50Smathue int cpuid = (int)(uintptr_t)arg; 61017c478bd9Sstevel@tonic-gate cpu_t *cp; 61027c478bd9Sstevel@tonic-gate 61037c478bd9Sstevel@tonic-gate /* 61047c478bd9Sstevel@tonic-gate * We acquire cpu_lock. 61057c478bd9Sstevel@tonic-gate */ 61067c478bd9Sstevel@tonic-gate ASSERT(curthread->t_pil == 0); 61077c478bd9Sstevel@tonic-gate 61087c478bd9Sstevel@tonic-gate /* 61097c478bd9Sstevel@tonic-gate * verify that the cpu is still around, DR 61107c478bd9Sstevel@tonic-gate * could have got there first ... 61117c478bd9Sstevel@tonic-gate */ 61127c478bd9Sstevel@tonic-gate mutex_enter(&cpu_lock); 61137c478bd9Sstevel@tonic-gate cp = cpu_get(cpuid); 61147c478bd9Sstevel@tonic-gate if (cp == NULL) { 61157c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 61167c478bd9Sstevel@tonic-gate return; 61177c478bd9Sstevel@tonic-gate } 61187c478bd9Sstevel@tonic-gate /* 61197c478bd9Sstevel@tonic-gate * make sure we don't migrate across CPUs 61207c478bd9Sstevel@tonic-gate * while checking our CE status. 61217c478bd9Sstevel@tonic-gate */ 61227c478bd9Sstevel@tonic-gate kpreempt_disable(); 61237c478bd9Sstevel@tonic-gate 61247c478bd9Sstevel@tonic-gate /* 61257c478bd9Sstevel@tonic-gate * If we are running on the CPU that got the 61267c478bd9Sstevel@tonic-gate * CE, we can do the checks directly. 61277c478bd9Sstevel@tonic-gate */ 61287c478bd9Sstevel@tonic-gate if (cp->cpu_id == CPU->cpu_id) { 61297c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 61307c478bd9Sstevel@tonic-gate cpu_check_ce(TIMEOUT_CEEN_CHECK, 0, 0, 0); 61317c478bd9Sstevel@tonic-gate kpreempt_enable(); 61327c478bd9Sstevel@tonic-gate return; 61337c478bd9Sstevel@tonic-gate } 61347c478bd9Sstevel@tonic-gate kpreempt_enable(); 61357c478bd9Sstevel@tonic-gate 61367c478bd9Sstevel@tonic-gate /* 61377c478bd9Sstevel@tonic-gate * send an x-call to get the CPU that originally 61387c478bd9Sstevel@tonic-gate * got the CE to do the necessary checks. If we can't 61397c478bd9Sstevel@tonic-gate * send the x-call, reschedule the timeout, otherwise we 61407c478bd9Sstevel@tonic-gate * lose CEEN forever on that CPU. 61417c478bd9Sstevel@tonic-gate */ 61427c478bd9Sstevel@tonic-gate if (CPU_XCALL_READY(cp->cpu_id) && (!(cp->cpu_flags & CPU_QUIESCED))) { 61437c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)cpu_check_ce, 61447c478bd9Sstevel@tonic-gate TIMEOUT_CEEN_CHECK, 0); 61457c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 61467c478bd9Sstevel@tonic-gate } else { 61477c478bd9Sstevel@tonic-gate /* 61487c478bd9Sstevel@tonic-gate * When the CPU is not accepting xcalls, or 61497c478bd9Sstevel@tonic-gate * the processor is offlined, we don't want to 61507c478bd9Sstevel@tonic-gate * incur the extra overhead of trying to schedule the 61517c478bd9Sstevel@tonic-gate * CE timeout indefinitely. However, we don't want to lose 61527c478bd9Sstevel@tonic-gate * CE checking forever. 61537c478bd9Sstevel@tonic-gate * 61547c478bd9Sstevel@tonic-gate * Keep rescheduling the timeout, accepting the additional 61557c478bd9Sstevel@tonic-gate * overhead as the cost of correctness in the case where we get 61567c478bd9Sstevel@tonic-gate * a CE, disable CEEN, offline the CPU during the 61577c478bd9Sstevel@tonic-gate * the timeout interval, and then online it at some 61587c478bd9Sstevel@tonic-gate * point in the future. This is unlikely given the short 61597c478bd9Sstevel@tonic-gate * cpu_ceen_delay_secs. 61607c478bd9Sstevel@tonic-gate */ 61617c478bd9Sstevel@tonic-gate mutex_exit(&cpu_lock); 6162f47a9c50Smathue (void) timeout(cpu_delayed_check_ce_errors, 6163f47a9c50Smathue (void *)(uintptr_t)cp->cpu_id, 61647c478bd9Sstevel@tonic-gate drv_usectohz((clock_t)cpu_ceen_delay_secs * MICROSEC)); 61657c478bd9Sstevel@tonic-gate } 61667c478bd9Sstevel@tonic-gate } 61677c478bd9Sstevel@tonic-gate 61687c478bd9Sstevel@tonic-gate /* 61697c478bd9Sstevel@tonic-gate * This routine will check whether CEs have occurred while 61707c478bd9Sstevel@tonic-gate * CEEN is disabled. Any CEs detected will be logged and, if 61717c478bd9Sstevel@tonic-gate * possible, scrubbed. 61727c478bd9Sstevel@tonic-gate * 61737c478bd9Sstevel@tonic-gate * The memscrubber will also use this routine to clear any errors 61747c478bd9Sstevel@tonic-gate * caused by its scrubbing with CEEN disabled. 61757c478bd9Sstevel@tonic-gate * 61767c478bd9Sstevel@tonic-gate * flag == SCRUBBER_CEEN_CHECK 61777c478bd9Sstevel@tonic-gate * called from memscrubber, just check/scrub, no reset 61787c478bd9Sstevel@tonic-gate * paddr physical addr. for start of scrub pages 61797c478bd9Sstevel@tonic-gate * vaddr virtual addr. for scrub area 61807c478bd9Sstevel@tonic-gate * psz page size of area to be scrubbed 61817c478bd9Sstevel@tonic-gate * 61827c478bd9Sstevel@tonic-gate * flag == TIMEOUT_CEEN_CHECK 61837c478bd9Sstevel@tonic-gate * timeout function has triggered, reset timeout or CEEN 61847c478bd9Sstevel@tonic-gate * 61857c478bd9Sstevel@tonic-gate * Note: We must not migrate cpus during this function. This can be 61867c478bd9Sstevel@tonic-gate * achieved by one of: 61877c478bd9Sstevel@tonic-gate * - invoking as target of an x-call in which case we're at XCALL_PIL 61887c478bd9Sstevel@tonic-gate * The flag value must be first xcall argument. 61897c478bd9Sstevel@tonic-gate * - disabling kernel preemption. This should be done for very short 61907c478bd9Sstevel@tonic-gate * periods so is not suitable for SCRUBBER_CEEN_CHECK where we might 61917c478bd9Sstevel@tonic-gate * scrub an extended area with cpu_check_block. The call for 61927c478bd9Sstevel@tonic-gate * TIMEOUT_CEEN_CHECK uses this so cpu_check_ce must be kept 61937c478bd9Sstevel@tonic-gate * brief for this case. 61947c478bd9Sstevel@tonic-gate * - binding to a cpu, eg with thread_affinity_set(). This is used 61957c478bd9Sstevel@tonic-gate * in the SCRUBBER_CEEN_CHECK case, but is not practical for 61967c478bd9Sstevel@tonic-gate * the TIMEOUT_CEEN_CHECK because both need cpu_lock. 61977c478bd9Sstevel@tonic-gate */ 61987c478bd9Sstevel@tonic-gate void 61997c478bd9Sstevel@tonic-gate cpu_check_ce(int flag, uint64_t pa, caddr_t va, uint_t psz) 62007c478bd9Sstevel@tonic-gate { 62017c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 62027c478bd9Sstevel@tonic-gate uint64_t ec_err_enable; 62037c478bd9Sstevel@tonic-gate uint64_t page_offset; 62047c478bd9Sstevel@tonic-gate 62057c478bd9Sstevel@tonic-gate /* Read AFSR */ 62067c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 62077c478bd9Sstevel@tonic-gate 62087c478bd9Sstevel@tonic-gate /* 62097c478bd9Sstevel@tonic-gate * If no CEEN errors have occurred during the timeout 62107c478bd9Sstevel@tonic-gate * interval, it is safe to re-enable CEEN and exit. 62117c478bd9Sstevel@tonic-gate */ 6212cbaac45eSkm if (((cpu_error_regs.afsr & C_AFSR_CECC_ERRS) | 6213cbaac45eSkm (cpu_error_regs.afsr_ext & C_AFSR_EXT_CECC_ERRS)) == 0) { 62147c478bd9Sstevel@tonic-gate if (flag == TIMEOUT_CEEN_CHECK && 62157c478bd9Sstevel@tonic-gate !((ec_err_enable = get_error_enable()) & EN_REG_CEEN)) 62167c478bd9Sstevel@tonic-gate set_error_enable(ec_err_enable | EN_REG_CEEN); 62177c478bd9Sstevel@tonic-gate return; 62187c478bd9Sstevel@tonic-gate } 62197c478bd9Sstevel@tonic-gate 62207c478bd9Sstevel@tonic-gate /* 62217c478bd9Sstevel@tonic-gate * Ensure that CEEN was not reenabled (maybe by DR) before 62227c478bd9Sstevel@tonic-gate * we log/clear the error. 62237c478bd9Sstevel@tonic-gate */ 62247c478bd9Sstevel@tonic-gate if ((ec_err_enable = get_error_enable()) & EN_REG_CEEN) 6225cbaac45eSkm set_error_enable(ec_err_enable & ~EN_REG_CEEN); 62267c478bd9Sstevel@tonic-gate 62277c478bd9Sstevel@tonic-gate /* 62287c478bd9Sstevel@tonic-gate * log/clear the CE. If CE_CEEN_DEFER is passed, the 62297c478bd9Sstevel@tonic-gate * timeout will be rescheduled when the error is logged. 62307c478bd9Sstevel@tonic-gate */ 6231cbaac45eSkm if (!((cpu_error_regs.afsr & cpu_ce_not_deferred) | 6232cbaac45eSkm (cpu_error_regs.afsr_ext & cpu_ce_not_deferred_ext))) 6233cbaac45eSkm cpu_ce_detected(&cpu_error_regs, 6234cbaac45eSkm CE_CEEN_DEFER | CE_CEEN_TIMEOUT); 62357c478bd9Sstevel@tonic-gate else 6236cbaac45eSkm cpu_ce_detected(&cpu_error_regs, CE_CEEN_TIMEOUT); 62377c478bd9Sstevel@tonic-gate 62387c478bd9Sstevel@tonic-gate /* 62397c478bd9Sstevel@tonic-gate * If the memory scrubber runs while CEEN is 62407c478bd9Sstevel@tonic-gate * disabled, (or if CEEN is disabled during the 62417c478bd9Sstevel@tonic-gate * scrub as a result of a CE being triggered by 62427c478bd9Sstevel@tonic-gate * it), the range being scrubbed will not be 62437c478bd9Sstevel@tonic-gate * completely cleaned. If there are multiple CEs 62447c478bd9Sstevel@tonic-gate * in the range at most two of these will be dealt 62457c478bd9Sstevel@tonic-gate * with, (one by the trap handler and one by the 62467c478bd9Sstevel@tonic-gate * timeout). It is also possible that none are dealt 62477c478bd9Sstevel@tonic-gate * with, (CEEN disabled and another CE occurs before 62487c478bd9Sstevel@tonic-gate * the timeout triggers). So to ensure that the 62497c478bd9Sstevel@tonic-gate * memory is actually scrubbed, we have to access each 62507c478bd9Sstevel@tonic-gate * memory location in the range and then check whether 62517c478bd9Sstevel@tonic-gate * that access causes a CE. 62527c478bd9Sstevel@tonic-gate */ 62537c478bd9Sstevel@tonic-gate if (flag == SCRUBBER_CEEN_CHECK && va) { 62547c478bd9Sstevel@tonic-gate if ((cpu_error_regs.afar >= pa) && 62557c478bd9Sstevel@tonic-gate (cpu_error_regs.afar < (pa + psz))) { 62567c478bd9Sstevel@tonic-gate /* 62577c478bd9Sstevel@tonic-gate * Force a load from physical memory for each 62587c478bd9Sstevel@tonic-gate * 64-byte block, then check AFSR to determine 62597c478bd9Sstevel@tonic-gate * whether this access caused an error. 62607c478bd9Sstevel@tonic-gate * 62617c478bd9Sstevel@tonic-gate * This is a slow way to do a scrub, but as it will 62627c478bd9Sstevel@tonic-gate * only be invoked when the memory scrubber actually 62637c478bd9Sstevel@tonic-gate * triggered a CE, it should not happen too 62647c478bd9Sstevel@tonic-gate * frequently. 62657c478bd9Sstevel@tonic-gate * 62667c478bd9Sstevel@tonic-gate * cut down what we need to check as the scrubber 62677c478bd9Sstevel@tonic-gate * has verified up to AFAR, so get it's offset 62687c478bd9Sstevel@tonic-gate * into the page and start there. 62697c478bd9Sstevel@tonic-gate */ 62707c478bd9Sstevel@tonic-gate page_offset = (uint64_t)(cpu_error_regs.afar & 62717c478bd9Sstevel@tonic-gate (psz - 1)); 62727c478bd9Sstevel@tonic-gate va = (caddr_t)(va + (P2ALIGN(page_offset, 64))); 62737c478bd9Sstevel@tonic-gate psz -= (uint_t)(P2ALIGN(page_offset, 64)); 62747c478bd9Sstevel@tonic-gate cpu_check_block((caddr_t)(P2ALIGN((uint64_t)va, 64)), 62757c478bd9Sstevel@tonic-gate psz); 62767c478bd9Sstevel@tonic-gate } 62777c478bd9Sstevel@tonic-gate } 62787c478bd9Sstevel@tonic-gate 62797c478bd9Sstevel@tonic-gate /* 62807c478bd9Sstevel@tonic-gate * Reset error enable if this CE is not masked. 62817c478bd9Sstevel@tonic-gate */ 62827c478bd9Sstevel@tonic-gate if ((flag == TIMEOUT_CEEN_CHECK) && 62837c478bd9Sstevel@tonic-gate (cpu_error_regs.afsr & cpu_ce_not_deferred)) 6284cbaac45eSkm set_error_enable(ec_err_enable | EN_REG_CEEN); 62857c478bd9Sstevel@tonic-gate 62867c478bd9Sstevel@tonic-gate } 62877c478bd9Sstevel@tonic-gate 62887c478bd9Sstevel@tonic-gate /* 62897c478bd9Sstevel@tonic-gate * Attempt a cpu logout for an error that we did not trap for, such 62907c478bd9Sstevel@tonic-gate * as a CE noticed with CEEN off. It is assumed that we are still running 62917c478bd9Sstevel@tonic-gate * on the cpu that took the error and that we cannot migrate. Returns 62927c478bd9Sstevel@tonic-gate * 0 on success, otherwise nonzero. 62937c478bd9Sstevel@tonic-gate */ 62947c478bd9Sstevel@tonic-gate static int 62957c478bd9Sstevel@tonic-gate cpu_ce_delayed_ec_logout(uint64_t afar) 62967c478bd9Sstevel@tonic-gate { 62977c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 62987c478bd9Sstevel@tonic-gate 62997c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) 63007c478bd9Sstevel@tonic-gate return (0); 63017c478bd9Sstevel@tonic-gate 63027c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 63037c478bd9Sstevel@tonic-gate if (cas64(&clop->clo_data.chd_afar, LOGOUT_INVALID, afar) != 63047c478bd9Sstevel@tonic-gate LOGOUT_INVALID) 63057c478bd9Sstevel@tonic-gate return (0); 63067c478bd9Sstevel@tonic-gate 63077c478bd9Sstevel@tonic-gate cpu_delayed_logout(afar, clop); 63087c478bd9Sstevel@tonic-gate return (1); 63097c478bd9Sstevel@tonic-gate } 63107c478bd9Sstevel@tonic-gate 63117c478bd9Sstevel@tonic-gate /* 63127c478bd9Sstevel@tonic-gate * We got an error while CEEN was disabled. We 63137c478bd9Sstevel@tonic-gate * need to clean up after it and log whatever 63147c478bd9Sstevel@tonic-gate * information we have on the CE. 63157c478bd9Sstevel@tonic-gate */ 63167c478bd9Sstevel@tonic-gate void 63177c478bd9Sstevel@tonic-gate cpu_ce_detected(ch_cpu_errors_t *cpu_error_regs, int flag) 63187c478bd9Sstevel@tonic-gate { 63197c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 63207c478bd9Sstevel@tonic-gate struct async_flt *aflt; 63217c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 63227c478bd9Sstevel@tonic-gate 63237c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 63247c478bd9Sstevel@tonic-gate ch_flt.flt_trapped_ce = flag; 63257c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)&ch_flt; 63267c478bd9Sstevel@tonic-gate aflt->flt_stat = cpu_error_regs->afsr & C_AFSR_MASK; 63277c478bd9Sstevel@tonic-gate ch_flt.afsr_ext = cpu_error_regs->afsr_ext; 63287c478bd9Sstevel@tonic-gate ch_flt.afsr_errs = (cpu_error_regs->afsr_ext & C_AFSR_EXT_ALL_ERRS) | 63297c478bd9Sstevel@tonic-gate (cpu_error_regs->afsr & C_AFSR_ALL_ERRS); 63307c478bd9Sstevel@tonic-gate aflt->flt_addr = cpu_error_regs->afar; 63317c478bd9Sstevel@tonic-gate #if defined(SERRANO) 63327c478bd9Sstevel@tonic-gate ch_flt.afar2 = cpu_error_regs->afar2; 63337c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 63347c478bd9Sstevel@tonic-gate aflt->flt_pc = NULL; 63357c478bd9Sstevel@tonic-gate aflt->flt_priv = ((cpu_error_regs->afsr & C_AFSR_PRIV) != 0); 63367c478bd9Sstevel@tonic-gate aflt->flt_tl = 0; 63377c478bd9Sstevel@tonic-gate aflt->flt_panic = 0; 63387c478bd9Sstevel@tonic-gate cpu_log_and_clear_ce(&ch_flt); 63397c478bd9Sstevel@tonic-gate 63407c478bd9Sstevel@tonic-gate /* 63417c478bd9Sstevel@tonic-gate * check if we caused any errors during cleanup 63427c478bd9Sstevel@tonic-gate */ 63437c478bd9Sstevel@tonic-gate if (clear_errors(&ch_flt)) { 63447c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 63457c478bd9Sstevel@tonic-gate (void) cpu_queue_events(&ch_flt, pr_reason, ch_flt.afsr_errs, 63467c478bd9Sstevel@tonic-gate NULL); 63477c478bd9Sstevel@tonic-gate } 63487c478bd9Sstevel@tonic-gate } 63497c478bd9Sstevel@tonic-gate 63507c478bd9Sstevel@tonic-gate /* 63517c478bd9Sstevel@tonic-gate * Log/clear CEEN-controlled disrupting errors 63527c478bd9Sstevel@tonic-gate */ 63537c478bd9Sstevel@tonic-gate static void 63547c478bd9Sstevel@tonic-gate cpu_log_and_clear_ce(ch_async_flt_t *ch_flt) 63557c478bd9Sstevel@tonic-gate { 63567c478bd9Sstevel@tonic-gate struct async_flt *aflt; 63577c478bd9Sstevel@tonic-gate uint64_t afsr, afsr_errs; 63587c478bd9Sstevel@tonic-gate ch_cpu_logout_t *clop; 63597c478bd9Sstevel@tonic-gate char pr_reason[MAX_REASON_STRING]; 63607c478bd9Sstevel@tonic-gate on_trap_data_t *otp = curthread->t_ontrap; 63617c478bd9Sstevel@tonic-gate 63627c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)ch_flt; 63637c478bd9Sstevel@tonic-gate afsr = aflt->flt_stat; 63647c478bd9Sstevel@tonic-gate afsr_errs = ch_flt->afsr_errs; 63657c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 63667c478bd9Sstevel@tonic-gate aflt->flt_bus_id = getprocessorid(); 63677c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 63687c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_NONE; 63697c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 63707c478bd9Sstevel@tonic-gate aflt->flt_status = ECC_C_TRAP; 63717c478bd9Sstevel@tonic-gate 63727c478bd9Sstevel@tonic-gate pr_reason[0] = '\0'; 63737c478bd9Sstevel@tonic-gate /* 63747c478bd9Sstevel@tonic-gate * Get the CPU log out info for Disrupting Trap. 63757c478bd9Sstevel@tonic-gate */ 63767c478bd9Sstevel@tonic-gate if (CPU_PRIVATE(CPU) == NULL) { 63777c478bd9Sstevel@tonic-gate clop = NULL; 63787c478bd9Sstevel@tonic-gate ch_flt->flt_diag_data.chd_afar = LOGOUT_INVALID; 63797c478bd9Sstevel@tonic-gate } else { 63807c478bd9Sstevel@tonic-gate clop = CPU_PRIVATE_PTR(CPU, chpr_cecc_logout); 63817c478bd9Sstevel@tonic-gate } 63827c478bd9Sstevel@tonic-gate 63837c478bd9Sstevel@tonic-gate if (clop && ch_flt->flt_trapped_ce & CE_CEEN_TIMEOUT) { 63847c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 63857c478bd9Sstevel@tonic-gate 63867c478bd9Sstevel@tonic-gate get_cpu_error_state(&cpu_error_regs); 63877c478bd9Sstevel@tonic-gate (void) cpu_ce_delayed_ec_logout(cpu_error_regs.afar); 63887c478bd9Sstevel@tonic-gate clop->clo_data.chd_afsr = cpu_error_regs.afsr; 63897c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = cpu_error_regs.afar; 63907c478bd9Sstevel@tonic-gate clop->clo_data.chd_afsr_ext = cpu_error_regs.afsr_ext; 63917c478bd9Sstevel@tonic-gate clop->clo_sdw_data.chd_afsr = cpu_error_regs.shadow_afsr; 63927c478bd9Sstevel@tonic-gate clop->clo_sdw_data.chd_afar = cpu_error_regs.shadow_afar; 63937c478bd9Sstevel@tonic-gate clop->clo_sdw_data.chd_afsr_ext = 63947c478bd9Sstevel@tonic-gate cpu_error_regs.shadow_afsr_ext; 63957c478bd9Sstevel@tonic-gate #if defined(SERRANO) 63967c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar2 = cpu_error_regs.afar2; 63977c478bd9Sstevel@tonic-gate #endif /* SERRANO */ 63987c478bd9Sstevel@tonic-gate ch_flt->flt_data_incomplete = 1; 63997c478bd9Sstevel@tonic-gate 64007c478bd9Sstevel@tonic-gate /* 64017c478bd9Sstevel@tonic-gate * The logging/clear code expects AFSR/AFAR to be cleared. 64027c478bd9Sstevel@tonic-gate * The trap handler does it for CEEN enabled errors 64037c478bd9Sstevel@tonic-gate * so we need to do it here. 64047c478bd9Sstevel@tonic-gate */ 64057c478bd9Sstevel@tonic-gate set_cpu_error_state(&cpu_error_regs); 64067c478bd9Sstevel@tonic-gate } 64077c478bd9Sstevel@tonic-gate 64087c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 64097c478bd9Sstevel@tonic-gate /* 64107c478bd9Sstevel@tonic-gate * FRC: Can't scrub memory as we don't have AFAR for Jalapeno. 64117c478bd9Sstevel@tonic-gate * For Serrano, even thou we do have the AFAR, we still do the 64127c478bd9Sstevel@tonic-gate * scrub on the RCE side since that's where the error type can 64137c478bd9Sstevel@tonic-gate * be properly classified as intermittent, persistent, etc. 64147c478bd9Sstevel@tonic-gate * 64157c478bd9Sstevel@tonic-gate * CE/RCE: If error is in memory and AFAR is valid, scrub the memory. 64167c478bd9Sstevel@tonic-gate * Must scrub memory before cpu_queue_events, as scrubbing memory sets 64177c478bd9Sstevel@tonic-gate * the flt_status bits. 64187c478bd9Sstevel@tonic-gate */ 64197c478bd9Sstevel@tonic-gate if ((afsr & (C_AFSR_CE|C_AFSR_RCE)) && 64207c478bd9Sstevel@tonic-gate (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 64217c478bd9Sstevel@tonic-gate cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_RCE)))) { 64227c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err(aflt, B_TRUE); 64237c478bd9Sstevel@tonic-gate } 64247c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 64257c478bd9Sstevel@tonic-gate /* 64267c478bd9Sstevel@tonic-gate * CE/EMC: If error is in memory and AFAR is valid, scrub the memory. 64277c478bd9Sstevel@tonic-gate * Must scrub memory before cpu_queue_events, as scrubbing memory sets 64287c478bd9Sstevel@tonic-gate * the flt_status bits. 64297c478bd9Sstevel@tonic-gate */ 64307c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_CE|C_AFSR_EMC)) { 64317c478bd9Sstevel@tonic-gate if (cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_CE)) || 64327c478bd9Sstevel@tonic-gate cpu_flt_in_memory(ch_flt, (afsr & C_AFSR_EMC))) { 64337c478bd9Sstevel@tonic-gate cpu_ce_scrub_mem_err(aflt, B_TRUE); 64347c478bd9Sstevel@tonic-gate } 64357c478bd9Sstevel@tonic-gate } 64367c478bd9Sstevel@tonic-gate 64377c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 64387c478bd9Sstevel@tonic-gate 64397c478bd9Sstevel@tonic-gate /* 64407c478bd9Sstevel@tonic-gate * Update flt_prot if this error occurred under on_trap protection. 64417c478bd9Sstevel@tonic-gate */ 64427c478bd9Sstevel@tonic-gate if (otp != NULL && (otp->ot_prot & OT_DATA_EC)) 64437c478bd9Sstevel@tonic-gate aflt->flt_prot = AFLT_PROT_EC; 64447c478bd9Sstevel@tonic-gate 64457c478bd9Sstevel@tonic-gate /* 64467c478bd9Sstevel@tonic-gate * Queue events on the async event queue, one event per error bit. 64477c478bd9Sstevel@tonic-gate */ 64487c478bd9Sstevel@tonic-gate if (cpu_queue_events(ch_flt, pr_reason, afsr_errs, clop) == 0 || 64497c478bd9Sstevel@tonic-gate (afsr_errs & (C_AFSR_CECC_ERRS | C_AFSR_EXT_CECC_ERRS)) == 0) { 64507c478bd9Sstevel@tonic-gate ch_flt->flt_type = CPU_INV_AFSR; 64517c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_INVALID_AFSR, 64527c478bd9Sstevel@tonic-gate (void *)ch_flt, sizeof (ch_async_flt_t), ue_queue, 64537c478bd9Sstevel@tonic-gate aflt->flt_panic); 64547c478bd9Sstevel@tonic-gate } 64557c478bd9Sstevel@tonic-gate 64567c478bd9Sstevel@tonic-gate /* 64577c478bd9Sstevel@tonic-gate * Zero out + invalidate CPU logout. 64587c478bd9Sstevel@tonic-gate */ 64597c478bd9Sstevel@tonic-gate if (clop) { 64607c478bd9Sstevel@tonic-gate bzero(clop, sizeof (ch_cpu_logout_t)); 64617c478bd9Sstevel@tonic-gate clop->clo_data.chd_afar = LOGOUT_INVALID; 64627c478bd9Sstevel@tonic-gate } 64637c478bd9Sstevel@tonic-gate 64647c478bd9Sstevel@tonic-gate /* 64657c478bd9Sstevel@tonic-gate * If either a CPC, WDC or EDC error has occurred while CEEN 64667c478bd9Sstevel@tonic-gate * was disabled, we need to flush either the entire 64677c478bd9Sstevel@tonic-gate * E$ or an E$ line. 64687c478bd9Sstevel@tonic-gate */ 64697c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 64707c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_CPU | C_AFSR_WDC)) 64717c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 64727c478bd9Sstevel@tonic-gate if (afsr_errs & (C_AFSR_EDC | C_AFSR_CPC | C_AFSR_WDC | C_AFSR_L3_EDC | 64737c478bd9Sstevel@tonic-gate C_AFSR_L3_CPC | C_AFSR_L3_WDC)) 64747c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 64757c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(ch_flt); 64767c478bd9Sstevel@tonic-gate 64777c478bd9Sstevel@tonic-gate } 64787c478bd9Sstevel@tonic-gate 64797c478bd9Sstevel@tonic-gate /* 64807c478bd9Sstevel@tonic-gate * depending on the error type, we determine whether we 64817c478bd9Sstevel@tonic-gate * need to flush the entire ecache or just a line. 64827c478bd9Sstevel@tonic-gate */ 64837c478bd9Sstevel@tonic-gate static int 64847c478bd9Sstevel@tonic-gate cpu_error_ecache_flush_required(ch_async_flt_t *ch_flt) 64857c478bd9Sstevel@tonic-gate { 64867c478bd9Sstevel@tonic-gate struct async_flt *aflt; 64877c478bd9Sstevel@tonic-gate uint64_t afsr; 64887c478bd9Sstevel@tonic-gate uint64_t afsr_errs = ch_flt->afsr_errs; 64897c478bd9Sstevel@tonic-gate 64907c478bd9Sstevel@tonic-gate aflt = (struct async_flt *)ch_flt; 64917c478bd9Sstevel@tonic-gate afsr = aflt->flt_stat; 64927c478bd9Sstevel@tonic-gate 64937c478bd9Sstevel@tonic-gate /* 64947c478bd9Sstevel@tonic-gate * If we got multiple errors, no point in trying 64957c478bd9Sstevel@tonic-gate * the individual cases, just flush the whole cache 64967c478bd9Sstevel@tonic-gate */ 64977c478bd9Sstevel@tonic-gate if (afsr & C_AFSR_ME) { 64987c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 64997c478bd9Sstevel@tonic-gate } 65007c478bd9Sstevel@tonic-gate 65017c478bd9Sstevel@tonic-gate /* 65027c478bd9Sstevel@tonic-gate * If either a CPC, WDC or EDC error has occurred while CEEN 65037c478bd9Sstevel@tonic-gate * was disabled, we need to flush entire E$. We can't just 65047c478bd9Sstevel@tonic-gate * flush the cache line affected as the ME bit 65057c478bd9Sstevel@tonic-gate * is not set when multiple correctable errors of the same 65067c478bd9Sstevel@tonic-gate * type occur, so we might have multiple CPC or EDC errors, 65077c478bd9Sstevel@tonic-gate * with only the first recorded. 65087c478bd9Sstevel@tonic-gate */ 65097c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 65107c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_CPC | C_AFSR_CPU | C_AFSR_EDC | C_AFSR_WDC)) { 65117c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 65127c478bd9Sstevel@tonic-gate if (afsr_errs & (C_AFSR_CPC | C_AFSR_EDC | C_AFSR_WDC | C_AFSR_L3_CPC | 65137c478bd9Sstevel@tonic-gate C_AFSR_L3_EDC | C_AFSR_L3_WDC)) { 65147c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 65157c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 65167c478bd9Sstevel@tonic-gate } 65177c478bd9Sstevel@tonic-gate 65187c478bd9Sstevel@tonic-gate #if defined(JALAPENO) || defined(SERRANO) 65197c478bd9Sstevel@tonic-gate /* 65207c478bd9Sstevel@tonic-gate * If only UE or RUE is set, flush the Ecache line, otherwise 65217c478bd9Sstevel@tonic-gate * flush the entire Ecache. 65227c478bd9Sstevel@tonic-gate */ 65237c478bd9Sstevel@tonic-gate if (afsr & (C_AFSR_UE|C_AFSR_RUE)) { 65247c478bd9Sstevel@tonic-gate if ((afsr & C_AFSR_ALL_ERRS) == C_AFSR_UE || 65257c478bd9Sstevel@tonic-gate (afsr & C_AFSR_ALL_ERRS) == C_AFSR_RUE) { 65267c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 65277c478bd9Sstevel@tonic-gate } else { 65287c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 65297c478bd9Sstevel@tonic-gate } 65307c478bd9Sstevel@tonic-gate } 65317c478bd9Sstevel@tonic-gate #else /* JALAPENO || SERRANO */ 65327c478bd9Sstevel@tonic-gate /* 65337c478bd9Sstevel@tonic-gate * If UE only is set, flush the Ecache line, otherwise 65347c478bd9Sstevel@tonic-gate * flush the entire Ecache. 65357c478bd9Sstevel@tonic-gate */ 65367c478bd9Sstevel@tonic-gate if (afsr_errs & C_AFSR_UE) { 65377c478bd9Sstevel@tonic-gate if ((afsr_errs & (C_AFSR_ALL_ERRS | C_AFSR_EXT_ALL_ERRS)) == 65387c478bd9Sstevel@tonic-gate C_AFSR_UE) { 65397c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 65407c478bd9Sstevel@tonic-gate } else { 65417c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 65427c478bd9Sstevel@tonic-gate } 65437c478bd9Sstevel@tonic-gate } 65447c478bd9Sstevel@tonic-gate #endif /* JALAPENO || SERRANO */ 65457c478bd9Sstevel@tonic-gate 65467c478bd9Sstevel@tonic-gate /* 65477c478bd9Sstevel@tonic-gate * EDU: If EDU only is set, flush the ecache line, otherwise 65487c478bd9Sstevel@tonic-gate * flush the entire Ecache. 65497c478bd9Sstevel@tonic-gate */ 65507c478bd9Sstevel@tonic-gate if (afsr_errs & (C_AFSR_EDU | C_AFSR_L3_EDU)) { 65517c478bd9Sstevel@tonic-gate if (((afsr_errs & ~C_AFSR_EDU) == 0) || 65527c478bd9Sstevel@tonic-gate ((afsr_errs & ~C_AFSR_L3_EDU) == 0)) { 65537c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 65547c478bd9Sstevel@tonic-gate } else { 65557c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 65567c478bd9Sstevel@tonic-gate } 65577c478bd9Sstevel@tonic-gate } 65587c478bd9Sstevel@tonic-gate 65597c478bd9Sstevel@tonic-gate /* 65607c478bd9Sstevel@tonic-gate * BERR: If BERR only is set, flush the Ecache line, otherwise 65617c478bd9Sstevel@tonic-gate * flush the entire Ecache. 65627c478bd9Sstevel@tonic-gate */ 65637c478bd9Sstevel@tonic-gate if (afsr_errs & C_AFSR_BERR) { 65647c478bd9Sstevel@tonic-gate if ((afsr_errs & ~C_AFSR_BERR) == 0) { 65657c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_LINE); 65667c478bd9Sstevel@tonic-gate } else { 65677c478bd9Sstevel@tonic-gate return (ECACHE_FLUSH_ALL); 65687c478bd9Sstevel@tonic-gate } 65697c478bd9Sstevel@tonic-gate } 65707c478bd9Sstevel@tonic-gate 65717c478bd9Sstevel@tonic-gate return (0); 65727c478bd9Sstevel@tonic-gate } 65737c478bd9Sstevel@tonic-gate 65747c478bd9Sstevel@tonic-gate void 65757c478bd9Sstevel@tonic-gate cpu_error_ecache_flush(ch_async_flt_t *ch_flt) 65767c478bd9Sstevel@tonic-gate { 65777c478bd9Sstevel@tonic-gate int ecache_flush_flag = 65787c478bd9Sstevel@tonic-gate cpu_error_ecache_flush_required(ch_flt); 65797c478bd9Sstevel@tonic-gate 65807c478bd9Sstevel@tonic-gate /* 65817c478bd9Sstevel@tonic-gate * Flush Ecache line or entire Ecache based on above checks. 65827c478bd9Sstevel@tonic-gate */ 65837c478bd9Sstevel@tonic-gate if (ecache_flush_flag == ECACHE_FLUSH_ALL) 65847c478bd9Sstevel@tonic-gate cpu_flush_ecache(); 65857c478bd9Sstevel@tonic-gate else if (ecache_flush_flag == ECACHE_FLUSH_LINE) { 65867c478bd9Sstevel@tonic-gate cpu_flush_ecache_line(ch_flt); 65877c478bd9Sstevel@tonic-gate } 65887c478bd9Sstevel@tonic-gate 65897c478bd9Sstevel@tonic-gate } 65907c478bd9Sstevel@tonic-gate 65917c478bd9Sstevel@tonic-gate /* 65927c478bd9Sstevel@tonic-gate * Extract the PA portion from the E$ tag. 65937c478bd9Sstevel@tonic-gate */ 65947c478bd9Sstevel@tonic-gate uint64_t 65957c478bd9Sstevel@tonic-gate cpu_ectag_to_pa(int setsize, uint64_t tag) 65967c478bd9Sstevel@tonic-gate { 65977c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 65987c478bd9Sstevel@tonic-gate return (JG_ECTAG_TO_PA(setsize, tag)); 65997c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 66007c478bd9Sstevel@tonic-gate return (PN_L3TAG_TO_PA(tag)); 66017c478bd9Sstevel@tonic-gate else 66027c478bd9Sstevel@tonic-gate return (CH_ECTAG_TO_PA(setsize, tag)); 66037c478bd9Sstevel@tonic-gate } 66047c478bd9Sstevel@tonic-gate 66057c478bd9Sstevel@tonic-gate /* 66067c478bd9Sstevel@tonic-gate * Convert the E$ tag PA into an E$ subblock index. 66077c478bd9Sstevel@tonic-gate */ 66087bebe46cSjc int 66097c478bd9Sstevel@tonic-gate cpu_ectag_pa_to_subblk(int cachesize, uint64_t subaddr) 66107c478bd9Sstevel@tonic-gate { 66117c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 66127c478bd9Sstevel@tonic-gate return (JG_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 66137c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 66147c478bd9Sstevel@tonic-gate /* Panther has only one subblock per line */ 66157c478bd9Sstevel@tonic-gate return (0); 66167c478bd9Sstevel@tonic-gate else 66177c478bd9Sstevel@tonic-gate return (CH_ECTAG_PA_TO_SUBBLK(cachesize, subaddr)); 66187c478bd9Sstevel@tonic-gate } 66197c478bd9Sstevel@tonic-gate 66207c478bd9Sstevel@tonic-gate /* 66217c478bd9Sstevel@tonic-gate * All subblocks in an E$ line must be invalid for 66227c478bd9Sstevel@tonic-gate * the line to be invalid. 66237c478bd9Sstevel@tonic-gate */ 66247c478bd9Sstevel@tonic-gate int 66257c478bd9Sstevel@tonic-gate cpu_ectag_line_invalid(int cachesize, uint64_t tag) 66267c478bd9Sstevel@tonic-gate { 66277c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 66287c478bd9Sstevel@tonic-gate return (JG_ECTAG_LINE_INVALID(cachesize, tag)); 66297c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 66307c478bd9Sstevel@tonic-gate return (PN_L3_LINE_INVALID(tag)); 66317c478bd9Sstevel@tonic-gate else 66327c478bd9Sstevel@tonic-gate return (CH_ECTAG_LINE_INVALID(cachesize, tag)); 66337c478bd9Sstevel@tonic-gate } 66347c478bd9Sstevel@tonic-gate 66357c478bd9Sstevel@tonic-gate /* 66367c478bd9Sstevel@tonic-gate * Extract state bits for a subblock given the tag. Note that for Panther 66377c478bd9Sstevel@tonic-gate * this works on both l2 and l3 tags. 66387c478bd9Sstevel@tonic-gate */ 66397bebe46cSjc int 66407c478bd9Sstevel@tonic-gate cpu_ectag_pa_to_subblk_state(int cachesize, uint64_t subaddr, uint64_t tag) 66417c478bd9Sstevel@tonic-gate { 66427c478bd9Sstevel@tonic-gate if (IS_JAGUAR(cpunodes[CPU->cpu_id].implementation)) 66437c478bd9Sstevel@tonic-gate return (JG_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 66447c478bd9Sstevel@tonic-gate else if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) 66457c478bd9Sstevel@tonic-gate return (tag & CH_ECSTATE_MASK); 66467c478bd9Sstevel@tonic-gate else 66477c478bd9Sstevel@tonic-gate return (CH_ECTAG_PA_TO_SUBBLK_STATE(cachesize, subaddr, tag)); 66487c478bd9Sstevel@tonic-gate } 66497c478bd9Sstevel@tonic-gate 66507c478bd9Sstevel@tonic-gate /* 66517c478bd9Sstevel@tonic-gate * Cpu specific initialization. 66527c478bd9Sstevel@tonic-gate */ 66537c478bd9Sstevel@tonic-gate void 66547c478bd9Sstevel@tonic-gate cpu_mp_init(void) 66557c478bd9Sstevel@tonic-gate { 66567c478bd9Sstevel@tonic-gate #ifdef CHEETAHPLUS_ERRATUM_25 66577c478bd9Sstevel@tonic-gate if (cheetah_sendmondo_recover) { 66587c478bd9Sstevel@tonic-gate cheetah_nudge_init(); 66597c478bd9Sstevel@tonic-gate } 66607c478bd9Sstevel@tonic-gate #endif 66617c478bd9Sstevel@tonic-gate } 66627c478bd9Sstevel@tonic-gate 66637c478bd9Sstevel@tonic-gate void 66647c478bd9Sstevel@tonic-gate cpu_ereport_post(struct async_flt *aflt) 66657c478bd9Sstevel@tonic-gate { 66667c478bd9Sstevel@tonic-gate char *cpu_type, buf[FM_MAX_CLASS]; 66677c478bd9Sstevel@tonic-gate nv_alloc_t *nva = NULL; 66687c478bd9Sstevel@tonic-gate nvlist_t *ereport, *detector, *resource; 66697c478bd9Sstevel@tonic-gate errorq_elem_t *eqep; 66707c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 66717c478bd9Sstevel@tonic-gate char unum[UNUM_NAMLEN]; 667293743541Smb int synd_code; 667338e9bdffSmikechr uint8_t msg_type; 66747c478bd9Sstevel@tonic-gate plat_ecc_ch_async_flt_t plat_ecc_ch_flt; 66757c478bd9Sstevel@tonic-gate 66767c478bd9Sstevel@tonic-gate if (aflt->flt_panic || panicstr) { 66777c478bd9Sstevel@tonic-gate eqep = errorq_reserve(ereport_errorq); 66787c478bd9Sstevel@tonic-gate if (eqep == NULL) 66797c478bd9Sstevel@tonic-gate return; 66807c478bd9Sstevel@tonic-gate ereport = errorq_elem_nvl(ereport_errorq, eqep); 66817c478bd9Sstevel@tonic-gate nva = errorq_elem_nva(ereport_errorq, eqep); 66827c478bd9Sstevel@tonic-gate } else { 66837c478bd9Sstevel@tonic-gate ereport = fm_nvlist_create(nva); 66847c478bd9Sstevel@tonic-gate } 66857c478bd9Sstevel@tonic-gate 66867c478bd9Sstevel@tonic-gate /* 66877c478bd9Sstevel@tonic-gate * Create the scheme "cpu" FMRI. 66887c478bd9Sstevel@tonic-gate */ 66897c478bd9Sstevel@tonic-gate detector = fm_nvlist_create(nva); 66907c478bd9Sstevel@tonic-gate resource = fm_nvlist_create(nva); 66917c478bd9Sstevel@tonic-gate switch (cpunodes[aflt->flt_inst].implementation) { 66927c478bd9Sstevel@tonic-gate case CHEETAH_IMPL: 66937c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIII; 66947c478bd9Sstevel@tonic-gate break; 66957c478bd9Sstevel@tonic-gate case CHEETAH_PLUS_IMPL: 66967c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIIIplus; 66977c478bd9Sstevel@tonic-gate break; 66987c478bd9Sstevel@tonic-gate case JALAPENO_IMPL: 66997c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIIIi; 67007c478bd9Sstevel@tonic-gate break; 67017c478bd9Sstevel@tonic-gate case SERRANO_IMPL: 67027c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIIIiplus; 67037c478bd9Sstevel@tonic-gate break; 67047c478bd9Sstevel@tonic-gate case JAGUAR_IMPL: 67057c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIV; 67067c478bd9Sstevel@tonic-gate break; 67077c478bd9Sstevel@tonic-gate case PANTHER_IMPL: 67087c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_USIVplus; 67097c478bd9Sstevel@tonic-gate break; 67107c478bd9Sstevel@tonic-gate default: 67117c478bd9Sstevel@tonic-gate cpu_type = FM_EREPORT_CPU_UNSUPPORTED; 67127c478bd9Sstevel@tonic-gate break; 67137c478bd9Sstevel@tonic-gate } 671438e9bdffSmikechr 671538e9bdffSmikechr cpu_fmri_cpu_set(detector, aflt->flt_inst); 67167c478bd9Sstevel@tonic-gate 67177c478bd9Sstevel@tonic-gate /* 67187c478bd9Sstevel@tonic-gate * Encode all the common data into the ereport. 67197c478bd9Sstevel@tonic-gate */ 67207c478bd9Sstevel@tonic-gate (void) snprintf(buf, FM_MAX_CLASS, "%s.%s.%s", 6721cbaac45eSkm FM_ERROR_CPU, cpu_type, aflt->flt_erpt_class); 67227c478bd9Sstevel@tonic-gate 67237c478bd9Sstevel@tonic-gate fm_ereport_set(ereport, FM_EREPORT_VERSION, buf, 67247c478bd9Sstevel@tonic-gate fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, FM_ENA_FMT1), 67257c478bd9Sstevel@tonic-gate detector, NULL); 67267c478bd9Sstevel@tonic-gate 67277c478bd9Sstevel@tonic-gate /* 67287c478bd9Sstevel@tonic-gate * Encode the error specific data that was saved in 67297c478bd9Sstevel@tonic-gate * the async_flt structure into the ereport. 67307c478bd9Sstevel@tonic-gate */ 67317c478bd9Sstevel@tonic-gate cpu_payload_add_aflt(aflt, ereport, resource, 67327c478bd9Sstevel@tonic-gate &plat_ecc_ch_flt.ecaf_afar_status, 67337c478bd9Sstevel@tonic-gate &plat_ecc_ch_flt.ecaf_synd_status); 67347c478bd9Sstevel@tonic-gate 67357c478bd9Sstevel@tonic-gate if (aflt->flt_panic || panicstr) { 67367c478bd9Sstevel@tonic-gate errorq_commit(ereport_errorq, eqep, ERRORQ_SYNC); 67377c478bd9Sstevel@tonic-gate } else { 67387c478bd9Sstevel@tonic-gate (void) fm_ereport_post(ereport, EVCH_TRYHARD); 67397c478bd9Sstevel@tonic-gate fm_nvlist_destroy(ereport, FM_NVA_FREE); 67407c478bd9Sstevel@tonic-gate fm_nvlist_destroy(detector, FM_NVA_FREE); 67417c478bd9Sstevel@tonic-gate fm_nvlist_destroy(resource, FM_NVA_FREE); 67427c478bd9Sstevel@tonic-gate } 67437c478bd9Sstevel@tonic-gate /* 67447c478bd9Sstevel@tonic-gate * Send the enhanced error information (plat_ecc_error2_data_t) 67457c478bd9Sstevel@tonic-gate * to the SC olny if it can process it. 67467c478bd9Sstevel@tonic-gate */ 67477c478bd9Sstevel@tonic-gate 67487c478bd9Sstevel@tonic-gate if (&plat_ecc_capability_sc_get && 67497c478bd9Sstevel@tonic-gate plat_ecc_capability_sc_get(PLAT_ECC_ERROR2_MESSAGE)) { 67507c478bd9Sstevel@tonic-gate msg_type = cpu_flt_bit_to_plat_error(aflt); 67517c478bd9Sstevel@tonic-gate if (msg_type != PLAT_ECC_ERROR2_NONE) { 67527c478bd9Sstevel@tonic-gate /* 67537c478bd9Sstevel@tonic-gate * If afar status is not invalid do a unum lookup. 67547c478bd9Sstevel@tonic-gate */ 67557c478bd9Sstevel@tonic-gate if (plat_ecc_ch_flt.ecaf_afar_status != 67567c478bd9Sstevel@tonic-gate AFLT_STAT_INVALID) { 675793743541Smb synd_code = synd_to_synd_code( 675893743541Smb plat_ecc_ch_flt.ecaf_synd_status, 675993743541Smb aflt->flt_synd, ch_flt->flt_bit); 676093743541Smb (void) cpu_get_mem_unum_synd(synd_code, 676193743541Smb aflt, unum); 67627c478bd9Sstevel@tonic-gate } else { 67637c478bd9Sstevel@tonic-gate unum[0] = '\0'; 67647c478bd9Sstevel@tonic-gate } 67657c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_sdw_afar = ch_flt->flt_sdw_afar; 67667c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_sdw_afsr = ch_flt->flt_sdw_afsr; 67677c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_afsr_ext = ch_flt->afsr_ext; 67687c478bd9Sstevel@tonic-gate plat_ecc_ch_flt.ecaf_sdw_afsr_ext = 67697c478bd9Sstevel@tonic-gate ch_flt->flt_sdw_afsr_ext; 67707c478bd9Sstevel@tonic-gate 67717c478bd9Sstevel@tonic-gate if (&plat_log_fruid_error2) 67727c478bd9Sstevel@tonic-gate plat_log_fruid_error2(msg_type, unum, aflt, 67737c478bd9Sstevel@tonic-gate &plat_ecc_ch_flt); 67747c478bd9Sstevel@tonic-gate } 67757c478bd9Sstevel@tonic-gate } 67767c478bd9Sstevel@tonic-gate } 67777c478bd9Sstevel@tonic-gate 67787c478bd9Sstevel@tonic-gate void 67797c478bd9Sstevel@tonic-gate cpu_run_bus_error_handlers(struct async_flt *aflt, int expected) 67807c478bd9Sstevel@tonic-gate { 67817c478bd9Sstevel@tonic-gate int status; 67827c478bd9Sstevel@tonic-gate ddi_fm_error_t de; 67837c478bd9Sstevel@tonic-gate 67847c478bd9Sstevel@tonic-gate bzero(&de, sizeof (ddi_fm_error_t)); 67857c478bd9Sstevel@tonic-gate 67867c478bd9Sstevel@tonic-gate de.fme_version = DDI_FME_VERSION; 67877c478bd9Sstevel@tonic-gate de.fme_ena = fm_ena_generate_cpu(aflt->flt_id, aflt->flt_inst, 67887c478bd9Sstevel@tonic-gate FM_ENA_FMT1); 67897c478bd9Sstevel@tonic-gate de.fme_flag = expected; 67907c478bd9Sstevel@tonic-gate de.fme_bus_specific = (void *)aflt->flt_addr; 67917c478bd9Sstevel@tonic-gate status = ndi_fm_handler_dispatch(ddi_root_node(), NULL, &de); 67927c478bd9Sstevel@tonic-gate if ((aflt->flt_prot == AFLT_PROT_NONE) && (status == DDI_FM_FATAL)) 67937c478bd9Sstevel@tonic-gate aflt->flt_panic = 1; 67947c478bd9Sstevel@tonic-gate } 67957c478bd9Sstevel@tonic-gate 67967c478bd9Sstevel@tonic-gate void 67977c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(char *error_class, void *payload, size_t payload_sz, 67987c478bd9Sstevel@tonic-gate errorq_t *eqp, uint_t flag) 67997c478bd9Sstevel@tonic-gate { 68007c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)payload; 68017c478bd9Sstevel@tonic-gate 68027c478bd9Sstevel@tonic-gate aflt->flt_erpt_class = error_class; 68037c478bd9Sstevel@tonic-gate errorq_dispatch(eqp, payload, payload_sz, flag); 68047c478bd9Sstevel@tonic-gate } 68057c478bd9Sstevel@tonic-gate 68067c478bd9Sstevel@tonic-gate /* 68077c478bd9Sstevel@tonic-gate * This routine may be called by the IO module, but does not do 68087c478bd9Sstevel@tonic-gate * anything in this cpu module. The SERD algorithm is handled by 68097c478bd9Sstevel@tonic-gate * cpumem-diagnosis engine instead. 68107c478bd9Sstevel@tonic-gate */ 68117c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 68127c478bd9Sstevel@tonic-gate void 68137c478bd9Sstevel@tonic-gate cpu_ce_count_unum(struct async_flt *ecc, int len, char *unum) 68147c478bd9Sstevel@tonic-gate {} 68157c478bd9Sstevel@tonic-gate 68167c478bd9Sstevel@tonic-gate void 68177c478bd9Sstevel@tonic-gate adjust_hw_copy_limits(int ecache_size) 68187c478bd9Sstevel@tonic-gate { 68197c478bd9Sstevel@tonic-gate /* 68207c478bd9Sstevel@tonic-gate * Set hw copy limits. 68217c478bd9Sstevel@tonic-gate * 68227c478bd9Sstevel@tonic-gate * /etc/system will be parsed later and can override one or more 68237c478bd9Sstevel@tonic-gate * of these settings. 68247c478bd9Sstevel@tonic-gate * 68257c478bd9Sstevel@tonic-gate * At this time, ecache size seems only mildly relevant. 68267c478bd9Sstevel@tonic-gate * We seem to run into issues with the d-cache and stalls 68277c478bd9Sstevel@tonic-gate * we see on misses. 68287c478bd9Sstevel@tonic-gate * 68297c478bd9Sstevel@tonic-gate * Cycle measurement indicates that 2 byte aligned copies fare 68307c478bd9Sstevel@tonic-gate * little better than doing things with VIS at around 512 bytes. 68317c478bd9Sstevel@tonic-gate * 4 byte aligned shows promise until around 1024 bytes. 8 Byte 68327c478bd9Sstevel@tonic-gate * aligned is faster whenever the source and destination data 68337c478bd9Sstevel@tonic-gate * in cache and the total size is less than 2 Kbytes. The 2K 68347c478bd9Sstevel@tonic-gate * limit seems to be driven by the 2K write cache. 68357c478bd9Sstevel@tonic-gate * When more than 2K of copies are done in non-VIS mode, stores 68367c478bd9Sstevel@tonic-gate * backup in the write cache. In VIS mode, the write cache is 68377c478bd9Sstevel@tonic-gate * bypassed, allowing faster cache-line writes aligned on cache 68387c478bd9Sstevel@tonic-gate * boundaries. 68397c478bd9Sstevel@tonic-gate * 68407c478bd9Sstevel@tonic-gate * In addition, in non-VIS mode, there is no prefetching, so 68417c478bd9Sstevel@tonic-gate * for larger copies, the advantage of prefetching to avoid even 68427c478bd9Sstevel@tonic-gate * occasional cache misses is enough to justify using the VIS code. 68437c478bd9Sstevel@tonic-gate * 68447c478bd9Sstevel@tonic-gate * During testing, it was discovered that netbench ran 3% slower 68457c478bd9Sstevel@tonic-gate * when hw_copy_limit_8 was 2K or larger. Apparently for server 68467c478bd9Sstevel@tonic-gate * applications, data is only used once (copied to the output 68477c478bd9Sstevel@tonic-gate * buffer, then copied by the network device off the system). Using 68487c478bd9Sstevel@tonic-gate * the VIS copy saves more L2 cache state. Network copies are 68497c478bd9Sstevel@tonic-gate * around 1.3K to 1.5K in size for historical reasons. 68507c478bd9Sstevel@tonic-gate * 68517c478bd9Sstevel@tonic-gate * Therefore, a limit of 1K bytes will be used for the 8 byte 68527c478bd9Sstevel@tonic-gate * aligned copy even for large caches and 8 MB ecache. The 68537c478bd9Sstevel@tonic-gate * infrastructure to allow different limits for different sized 68547c478bd9Sstevel@tonic-gate * caches is kept to allow further tuning in later releases. 68557c478bd9Sstevel@tonic-gate */ 68567c478bd9Sstevel@tonic-gate 68577c478bd9Sstevel@tonic-gate if (min_ecache_size == 0 && use_hw_bcopy) { 68587c478bd9Sstevel@tonic-gate /* 68597c478bd9Sstevel@tonic-gate * First time through - should be before /etc/system 68607c478bd9Sstevel@tonic-gate * is read. 68617c478bd9Sstevel@tonic-gate * Could skip the checks for zero but this lets us 68627c478bd9Sstevel@tonic-gate * preserve any debugger rewrites. 68637c478bd9Sstevel@tonic-gate */ 68647c478bd9Sstevel@tonic-gate if (hw_copy_limit_1 == 0) { 68657c478bd9Sstevel@tonic-gate hw_copy_limit_1 = VIS_COPY_THRESHOLD; 68667c478bd9Sstevel@tonic-gate priv_hcl_1 = hw_copy_limit_1; 68677c478bd9Sstevel@tonic-gate } 68687c478bd9Sstevel@tonic-gate if (hw_copy_limit_2 == 0) { 68697c478bd9Sstevel@tonic-gate hw_copy_limit_2 = 2 * VIS_COPY_THRESHOLD; 68707c478bd9Sstevel@tonic-gate priv_hcl_2 = hw_copy_limit_2; 68717c478bd9Sstevel@tonic-gate } 68727c478bd9Sstevel@tonic-gate if (hw_copy_limit_4 == 0) { 68737c478bd9Sstevel@tonic-gate hw_copy_limit_4 = 4 * VIS_COPY_THRESHOLD; 68747c478bd9Sstevel@tonic-gate priv_hcl_4 = hw_copy_limit_4; 68757c478bd9Sstevel@tonic-gate } 68767c478bd9Sstevel@tonic-gate if (hw_copy_limit_8 == 0) { 68777c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * VIS_COPY_THRESHOLD; 68787c478bd9Sstevel@tonic-gate priv_hcl_8 = hw_copy_limit_8; 68797c478bd9Sstevel@tonic-gate } 68807c478bd9Sstevel@tonic-gate min_ecache_size = ecache_size; 68817c478bd9Sstevel@tonic-gate } else { 68827c478bd9Sstevel@tonic-gate /* 68837c478bd9Sstevel@tonic-gate * MP initialization. Called *after* /etc/system has 68847c478bd9Sstevel@tonic-gate * been parsed. One CPU has already been initialized. 68857c478bd9Sstevel@tonic-gate * Need to cater for /etc/system having scragged one 68867c478bd9Sstevel@tonic-gate * of our values. 68877c478bd9Sstevel@tonic-gate */ 68887c478bd9Sstevel@tonic-gate if (ecache_size == min_ecache_size) { 68897c478bd9Sstevel@tonic-gate /* 68907c478bd9Sstevel@tonic-gate * Same size ecache. We do nothing unless we 68917c478bd9Sstevel@tonic-gate * have a pessimistic ecache setting. In that 68927c478bd9Sstevel@tonic-gate * case we become more optimistic (if the cache is 68937c478bd9Sstevel@tonic-gate * large enough). 68947c478bd9Sstevel@tonic-gate */ 68957c478bd9Sstevel@tonic-gate if (hw_copy_limit_8 == 4 * VIS_COPY_THRESHOLD) { 68967c478bd9Sstevel@tonic-gate /* 68977c478bd9Sstevel@tonic-gate * Need to adjust hw_copy_limit* from our 68987c478bd9Sstevel@tonic-gate * pessimistic uniprocessor value to a more 68997c478bd9Sstevel@tonic-gate * optimistic UP value *iff* it hasn't been 69007c478bd9Sstevel@tonic-gate * reset. 69017c478bd9Sstevel@tonic-gate */ 69027c478bd9Sstevel@tonic-gate if ((ecache_size > 1048576) && 69037c478bd9Sstevel@tonic-gate (priv_hcl_8 == hw_copy_limit_8)) { 69047c478bd9Sstevel@tonic-gate if (ecache_size <= 2097152) 69057c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * 69067c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 69077c478bd9Sstevel@tonic-gate else if (ecache_size <= 4194304) 69087c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * 69097c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 69107c478bd9Sstevel@tonic-gate else 69117c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 4 * 69127c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 69137c478bd9Sstevel@tonic-gate priv_hcl_8 = hw_copy_limit_8; 69147c478bd9Sstevel@tonic-gate } 69157c478bd9Sstevel@tonic-gate } 69167c478bd9Sstevel@tonic-gate } else if (ecache_size < min_ecache_size) { 69177c478bd9Sstevel@tonic-gate /* 69187c478bd9Sstevel@tonic-gate * A different ecache size. Can this even happen? 69197c478bd9Sstevel@tonic-gate */ 69207c478bd9Sstevel@tonic-gate if (priv_hcl_8 == hw_copy_limit_8) { 69217c478bd9Sstevel@tonic-gate /* 69227c478bd9Sstevel@tonic-gate * The previous value that we set 69237c478bd9Sstevel@tonic-gate * is unchanged (i.e., it hasn't been 69247c478bd9Sstevel@tonic-gate * scragged by /etc/system). Rewrite it. 69257c478bd9Sstevel@tonic-gate */ 69267c478bd9Sstevel@tonic-gate if (ecache_size <= 1048576) 69277c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 8 * 69287c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 69297c478bd9Sstevel@tonic-gate else if (ecache_size <= 2097152) 69307c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 8 * 69317c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 69327c478bd9Sstevel@tonic-gate else if (ecache_size <= 4194304) 69337c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 8 * 69347c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 69357c478bd9Sstevel@tonic-gate else 69367c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 10 * 69377c478bd9Sstevel@tonic-gate VIS_COPY_THRESHOLD; 69387c478bd9Sstevel@tonic-gate priv_hcl_8 = hw_copy_limit_8; 69397c478bd9Sstevel@tonic-gate min_ecache_size = ecache_size; 69407c478bd9Sstevel@tonic-gate } 69417c478bd9Sstevel@tonic-gate } 69427c478bd9Sstevel@tonic-gate } 69437c478bd9Sstevel@tonic-gate } 69447c478bd9Sstevel@tonic-gate 69457c478bd9Sstevel@tonic-gate /* 69467c478bd9Sstevel@tonic-gate * Called from illegal instruction trap handler to see if we can attribute 69477c478bd9Sstevel@tonic-gate * the trap to a fpras check. 69487c478bd9Sstevel@tonic-gate */ 69497c478bd9Sstevel@tonic-gate int 69507c478bd9Sstevel@tonic-gate fpras_chktrap(struct regs *rp) 69517c478bd9Sstevel@tonic-gate { 69527c478bd9Sstevel@tonic-gate int op; 69537c478bd9Sstevel@tonic-gate struct fpras_chkfngrp *cgp; 69547c478bd9Sstevel@tonic-gate uintptr_t tpc = (uintptr_t)rp->r_pc; 69557c478bd9Sstevel@tonic-gate 69567c478bd9Sstevel@tonic-gate if (fpras_chkfngrps == NULL) 69577c478bd9Sstevel@tonic-gate return (0); 69587c478bd9Sstevel@tonic-gate 69597c478bd9Sstevel@tonic-gate cgp = &fpras_chkfngrps[CPU->cpu_id]; 69607c478bd9Sstevel@tonic-gate for (op = 0; op < FPRAS_NCOPYOPS; ++op) { 69617c478bd9Sstevel@tonic-gate if (tpc >= (uintptr_t)&cgp->fpras_fn[op].fpras_blk0 && 69627c478bd9Sstevel@tonic-gate tpc < (uintptr_t)&cgp->fpras_fn[op].fpras_chkresult) 69637c478bd9Sstevel@tonic-gate break; 69647c478bd9Sstevel@tonic-gate } 69657c478bd9Sstevel@tonic-gate if (op == FPRAS_NCOPYOPS) 69667c478bd9Sstevel@tonic-gate return (0); 69677c478bd9Sstevel@tonic-gate 69687c478bd9Sstevel@tonic-gate /* 69697c478bd9Sstevel@tonic-gate * This is an fpRAS failure caught through an illegal 69707c478bd9Sstevel@tonic-gate * instruction - trampoline. 69717c478bd9Sstevel@tonic-gate */ 69727c478bd9Sstevel@tonic-gate rp->r_pc = (uintptr_t)&cgp->fpras_fn[op].fpras_trampoline; 69737c478bd9Sstevel@tonic-gate rp->r_npc = rp->r_pc + 4; 69747c478bd9Sstevel@tonic-gate return (1); 69757c478bd9Sstevel@tonic-gate } 69767c478bd9Sstevel@tonic-gate 69777c478bd9Sstevel@tonic-gate /* 69787c478bd9Sstevel@tonic-gate * fpras_failure is called when a fpras check detects a bad calculation 69797c478bd9Sstevel@tonic-gate * result or an illegal instruction trap is attributed to an fpras 69807c478bd9Sstevel@tonic-gate * check. In all cases we are still bound to CPU. 69817c478bd9Sstevel@tonic-gate */ 69827c478bd9Sstevel@tonic-gate int 69837c478bd9Sstevel@tonic-gate fpras_failure(int op, int how) 69847c478bd9Sstevel@tonic-gate { 69857c478bd9Sstevel@tonic-gate int use_hw_bcopy_orig, use_hw_bzero_orig; 69867c478bd9Sstevel@tonic-gate uint_t hcl1_orig, hcl2_orig, hcl4_orig, hcl8_orig; 69877c478bd9Sstevel@tonic-gate ch_async_flt_t ch_flt; 69887c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)&ch_flt; 69897c478bd9Sstevel@tonic-gate struct fpras_chkfn *sfp, *cfp; 69907c478bd9Sstevel@tonic-gate uint32_t *sip, *cip; 69917c478bd9Sstevel@tonic-gate int i; 69927c478bd9Sstevel@tonic-gate 69937c478bd9Sstevel@tonic-gate /* 69947c478bd9Sstevel@tonic-gate * We're running on a sick CPU. Avoid further FPU use at least for 69957c478bd9Sstevel@tonic-gate * the time in which we dispatch an ereport and (if applicable) panic. 69967c478bd9Sstevel@tonic-gate */ 69977c478bd9Sstevel@tonic-gate use_hw_bcopy_orig = use_hw_bcopy; 69987c478bd9Sstevel@tonic-gate use_hw_bzero_orig = use_hw_bzero; 69997c478bd9Sstevel@tonic-gate hcl1_orig = hw_copy_limit_1; 70007c478bd9Sstevel@tonic-gate hcl2_orig = hw_copy_limit_2; 70017c478bd9Sstevel@tonic-gate hcl4_orig = hw_copy_limit_4; 70027c478bd9Sstevel@tonic-gate hcl8_orig = hw_copy_limit_8; 70037c478bd9Sstevel@tonic-gate use_hw_bcopy = use_hw_bzero = 0; 70047c478bd9Sstevel@tonic-gate hw_copy_limit_1 = hw_copy_limit_2 = hw_copy_limit_4 = 70057c478bd9Sstevel@tonic-gate hw_copy_limit_8 = 0; 70067c478bd9Sstevel@tonic-gate 70077c478bd9Sstevel@tonic-gate bzero(&ch_flt, sizeof (ch_async_flt_t)); 70087c478bd9Sstevel@tonic-gate aflt->flt_id = gethrtime_waitfree(); 70097c478bd9Sstevel@tonic-gate aflt->flt_class = CPU_FAULT; 70107c478bd9Sstevel@tonic-gate aflt->flt_inst = CPU->cpu_id; 70117c478bd9Sstevel@tonic-gate aflt->flt_status = (how << 8) | op; 70127c478bd9Sstevel@tonic-gate aflt->flt_payload = FM_EREPORT_PAYLOAD_FPU_HWCOPY; 70137c478bd9Sstevel@tonic-gate ch_flt.flt_type = CPU_FPUERR; 70147c478bd9Sstevel@tonic-gate 70157c478bd9Sstevel@tonic-gate /* 70167c478bd9Sstevel@tonic-gate * We must panic if the copy operation had no lofault protection - 70177c478bd9Sstevel@tonic-gate * ie, don't panic for copyin, copyout, kcopy and bcopy called 70187c478bd9Sstevel@tonic-gate * under on_fault and do panic for unprotected bcopy and hwblkpagecopy. 70197c478bd9Sstevel@tonic-gate */ 70207c478bd9Sstevel@tonic-gate aflt->flt_panic = (curthread->t_lofault == NULL); 70217c478bd9Sstevel@tonic-gate 70227c478bd9Sstevel@tonic-gate /* 70237c478bd9Sstevel@tonic-gate * XOR the source instruction block with the copied instruction 70247c478bd9Sstevel@tonic-gate * block - this will show us which bit(s) are corrupted. 70257c478bd9Sstevel@tonic-gate */ 70267c478bd9Sstevel@tonic-gate sfp = (struct fpras_chkfn *)fpras_chkfn_type1; 70277c478bd9Sstevel@tonic-gate cfp = &fpras_chkfngrps[CPU->cpu_id].fpras_fn[op]; 70287c478bd9Sstevel@tonic-gate if (op == FPRAS_BCOPY || op == FPRAS_COPYOUT) { 70297c478bd9Sstevel@tonic-gate sip = &sfp->fpras_blk0[0]; 70307c478bd9Sstevel@tonic-gate cip = &cfp->fpras_blk0[0]; 70317c478bd9Sstevel@tonic-gate } else { 70327c478bd9Sstevel@tonic-gate sip = &sfp->fpras_blk1[0]; 70337c478bd9Sstevel@tonic-gate cip = &cfp->fpras_blk1[0]; 70347c478bd9Sstevel@tonic-gate } 70357c478bd9Sstevel@tonic-gate for (i = 0; i < 16; ++i, ++sip, ++cip) 70367c478bd9Sstevel@tonic-gate ch_flt.flt_fpdata[i] = *sip ^ *cip; 70377c478bd9Sstevel@tonic-gate 70387c478bd9Sstevel@tonic-gate cpu_errorq_dispatch(FM_EREPORT_CPU_USIII_FPU_HWCOPY, (void *)&ch_flt, 70397c478bd9Sstevel@tonic-gate sizeof (ch_async_flt_t), ue_queue, aflt->flt_panic); 70407c478bd9Sstevel@tonic-gate 70417c478bd9Sstevel@tonic-gate if (aflt->flt_panic) 70427c478bd9Sstevel@tonic-gate fm_panic("FPU failure on CPU %d", CPU->cpu_id); 70437c478bd9Sstevel@tonic-gate 70447c478bd9Sstevel@tonic-gate /* 70457c478bd9Sstevel@tonic-gate * We get here for copyin/copyout and kcopy or bcopy where the 70467c478bd9Sstevel@tonic-gate * caller has used on_fault. We will flag the error so that 70477c478bd9Sstevel@tonic-gate * the process may be killed The trap_async_hwerr mechanism will 70487c478bd9Sstevel@tonic-gate * take appropriate further action (such as a reboot, contract 70497c478bd9Sstevel@tonic-gate * notification etc). Since we may be continuing we will 70507c478bd9Sstevel@tonic-gate * restore the global hardware copy acceleration switches. 70517c478bd9Sstevel@tonic-gate * 70527c478bd9Sstevel@tonic-gate * When we return from this function to the copy function we want to 70537c478bd9Sstevel@tonic-gate * avoid potentially bad data being used, ie we want the affected 70547c478bd9Sstevel@tonic-gate * copy function to return an error. The caller should therefore 70557c478bd9Sstevel@tonic-gate * invoke its lofault handler (which always exists for these functions) 70567c478bd9Sstevel@tonic-gate * which will return the appropriate error. 70577c478bd9Sstevel@tonic-gate */ 70587c478bd9Sstevel@tonic-gate ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR; 70597c478bd9Sstevel@tonic-gate aston(curthread); 70607c478bd9Sstevel@tonic-gate 70617c478bd9Sstevel@tonic-gate use_hw_bcopy = use_hw_bcopy_orig; 70627c478bd9Sstevel@tonic-gate use_hw_bzero = use_hw_bzero_orig; 70637c478bd9Sstevel@tonic-gate hw_copy_limit_1 = hcl1_orig; 70647c478bd9Sstevel@tonic-gate hw_copy_limit_2 = hcl2_orig; 70657c478bd9Sstevel@tonic-gate hw_copy_limit_4 = hcl4_orig; 70667c478bd9Sstevel@tonic-gate hw_copy_limit_8 = hcl8_orig; 70677c478bd9Sstevel@tonic-gate 70687c478bd9Sstevel@tonic-gate return (1); 70697c478bd9Sstevel@tonic-gate } 70707c478bd9Sstevel@tonic-gate 70717c478bd9Sstevel@tonic-gate #define VIS_BLOCKSIZE 64 70727c478bd9Sstevel@tonic-gate 70737c478bd9Sstevel@tonic-gate int 70747c478bd9Sstevel@tonic-gate dtrace_blksuword32_err(uintptr_t addr, uint32_t *data) 70757c478bd9Sstevel@tonic-gate { 70767c478bd9Sstevel@tonic-gate int ret, watched; 70777c478bd9Sstevel@tonic-gate 70787c478bd9Sstevel@tonic-gate watched = watch_disable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 70797c478bd9Sstevel@tonic-gate ret = dtrace_blksuword32(addr, data, 0); 70807c478bd9Sstevel@tonic-gate if (watched) 70817c478bd9Sstevel@tonic-gate watch_enable_addr((void *)addr, VIS_BLOCKSIZE, S_WRITE); 70827c478bd9Sstevel@tonic-gate 70837c478bd9Sstevel@tonic-gate return (ret); 70847c478bd9Sstevel@tonic-gate } 70857c478bd9Sstevel@tonic-gate 70867c478bd9Sstevel@tonic-gate /* 70877c478bd9Sstevel@tonic-gate * Called when a cpu enters the CPU_FAULTED state (by the cpu placing the 70887c478bd9Sstevel@tonic-gate * faulted cpu into that state). Cross-trap to the faulted cpu to clear 70897c478bd9Sstevel@tonic-gate * CEEN from the EER to disable traps for further disrupting error types 70907c478bd9Sstevel@tonic-gate * on that cpu. We could cross-call instead, but that has a larger 70917c478bd9Sstevel@tonic-gate * instruction and data footprint than cross-trapping, and the cpu is known 70927c478bd9Sstevel@tonic-gate * to be faulted. 70937c478bd9Sstevel@tonic-gate */ 70947c478bd9Sstevel@tonic-gate 70957c478bd9Sstevel@tonic-gate void 70967c478bd9Sstevel@tonic-gate cpu_faulted_enter(struct cpu *cp) 70977c478bd9Sstevel@tonic-gate { 70987c478bd9Sstevel@tonic-gate xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_CLRBITS); 70997c478bd9Sstevel@tonic-gate } 71007c478bd9Sstevel@tonic-gate 71017c478bd9Sstevel@tonic-gate /* 71027c478bd9Sstevel@tonic-gate * Called when a cpu leaves the CPU_FAULTED state to return to one of 71037c478bd9Sstevel@tonic-gate * offline, spare, or online (by the cpu requesting this state change). 71047c478bd9Sstevel@tonic-gate * First we cross-call to clear the AFSR (and AFSR_EXT on Panther) of 71057c478bd9Sstevel@tonic-gate * disrupting error bits that have accumulated without trapping, then 71067c478bd9Sstevel@tonic-gate * we cross-trap to re-enable CEEN controlled traps. 71077c478bd9Sstevel@tonic-gate */ 71087c478bd9Sstevel@tonic-gate void 71097c478bd9Sstevel@tonic-gate cpu_faulted_exit(struct cpu *cp) 71107c478bd9Sstevel@tonic-gate { 71117c478bd9Sstevel@tonic-gate ch_cpu_errors_t cpu_error_regs; 71127c478bd9Sstevel@tonic-gate 71137c478bd9Sstevel@tonic-gate cpu_error_regs.afsr = C_AFSR_CECC_ERRS; 71147c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[cp->cpu_id].implementation)) 71157c478bd9Sstevel@tonic-gate cpu_error_regs.afsr_ext &= C_AFSR_EXT_CECC_ERRS; 71167c478bd9Sstevel@tonic-gate xc_one(cp->cpu_id, (xcfunc_t *)set_cpu_error_state, 71177c478bd9Sstevel@tonic-gate (uint64_t)&cpu_error_regs, 0); 71187c478bd9Sstevel@tonic-gate 71197c478bd9Sstevel@tonic-gate xt_one(cp->cpu_id, set_error_enable_tl1, EN_REG_CEEN, EER_SET_SETBITS); 71207c478bd9Sstevel@tonic-gate } 71217c478bd9Sstevel@tonic-gate 71227c478bd9Sstevel@tonic-gate /* 71237c478bd9Sstevel@tonic-gate * Return 1 if the errors in ch_flt's AFSR are secondary errors caused by 71247c478bd9Sstevel@tonic-gate * the errors in the original AFSR, 0 otherwise. 71257c478bd9Sstevel@tonic-gate * 71267c478bd9Sstevel@tonic-gate * For all procs if the initial error was a BERR or TO, then it is possible 71277c478bd9Sstevel@tonic-gate * that we may have caused a secondary BERR or TO in the process of logging the 71287c478bd9Sstevel@tonic-gate * inital error via cpu_run_bus_error_handlers(). If this is the case then 71297c478bd9Sstevel@tonic-gate * if the request was protected then a panic is still not necessary, if not 71307c478bd9Sstevel@tonic-gate * protected then aft_panic is already set - so either way there's no need 71317c478bd9Sstevel@tonic-gate * to set aft_panic for the secondary error. 71327c478bd9Sstevel@tonic-gate * 71337c478bd9Sstevel@tonic-gate * For Cheetah and Jalapeno, if the original error was a UE which occurred on 71347c478bd9Sstevel@tonic-gate * a store merge, then the error handling code will call cpu_deferred_error(). 71357c478bd9Sstevel@tonic-gate * When clear_errors() is called, it will determine that secondary errors have 71367c478bd9Sstevel@tonic-gate * occurred - in particular, the store merge also caused a EDU and WDU that 71377c478bd9Sstevel@tonic-gate * weren't discovered until this point. 71387c478bd9Sstevel@tonic-gate * 71397c478bd9Sstevel@tonic-gate * We do three checks to verify that we are in this case. If we pass all three 71407c478bd9Sstevel@tonic-gate * checks, we return 1 to indicate that we should not panic. If any unexpected 71417c478bd9Sstevel@tonic-gate * errors occur, we return 0. 71427c478bd9Sstevel@tonic-gate * 71437c478bd9Sstevel@tonic-gate * For Cheetah+ and derivative procs, the store merge causes a DUE, which is 71447c478bd9Sstevel@tonic-gate * handled in cpu_disrupting_errors(). Since this function is not even called 71457c478bd9Sstevel@tonic-gate * in the case we are interested in, we just return 0 for these processors. 71467c478bd9Sstevel@tonic-gate */ 71477c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 71487c478bd9Sstevel@tonic-gate static int 71497c478bd9Sstevel@tonic-gate cpu_check_secondary_errors(ch_async_flt_t *ch_flt, uint64_t t_afsr_errs, 71507c478bd9Sstevel@tonic-gate uint64_t t_afar) 71517c478bd9Sstevel@tonic-gate { 71527c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 71537c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */ 71547c478bd9Sstevel@tonic-gate struct async_flt *aflt = (struct async_flt *)ch_flt; 71557c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 71567c478bd9Sstevel@tonic-gate 71577c478bd9Sstevel@tonic-gate /* 71587c478bd9Sstevel@tonic-gate * Was the original error a BERR or TO and only a BERR or TO 71597c478bd9Sstevel@tonic-gate * (multiple errors are also OK) 71607c478bd9Sstevel@tonic-gate */ 71617c478bd9Sstevel@tonic-gate if ((t_afsr_errs & ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) { 71627c478bd9Sstevel@tonic-gate /* 71637c478bd9Sstevel@tonic-gate * Is the new error a BERR or TO and only a BERR or TO 71647c478bd9Sstevel@tonic-gate * (multiple errors are also OK) 71657c478bd9Sstevel@tonic-gate */ 71667c478bd9Sstevel@tonic-gate if ((ch_flt->afsr_errs & 71677c478bd9Sstevel@tonic-gate ~(C_AFSR_BERR | C_AFSR_TO | C_AFSR_ME)) == 0) 71687c478bd9Sstevel@tonic-gate return (1); 71697c478bd9Sstevel@tonic-gate } 71707c478bd9Sstevel@tonic-gate 71717c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 71727c478bd9Sstevel@tonic-gate return (0); 71737c478bd9Sstevel@tonic-gate #else /* CHEETAH_PLUS */ 71747c478bd9Sstevel@tonic-gate /* 71757c478bd9Sstevel@tonic-gate * Now look for secondary effects of a UE on cheetah/jalapeno 71767c478bd9Sstevel@tonic-gate * 71777c478bd9Sstevel@tonic-gate * Check the original error was a UE, and only a UE. Note that 71787c478bd9Sstevel@tonic-gate * the ME bit will cause us to fail this check. 71797c478bd9Sstevel@tonic-gate */ 71807c478bd9Sstevel@tonic-gate if (t_afsr_errs != C_AFSR_UE) 71817c478bd9Sstevel@tonic-gate return (0); 71827c478bd9Sstevel@tonic-gate 71837c478bd9Sstevel@tonic-gate /* 71847c478bd9Sstevel@tonic-gate * Check the secondary errors were exclusively an EDU and/or WDU. 71857c478bd9Sstevel@tonic-gate */ 71867c478bd9Sstevel@tonic-gate if ((ch_flt->afsr_errs & ~(C_AFSR_EDU|C_AFSR_WDU)) != 0) 71877c478bd9Sstevel@tonic-gate return (0); 71887c478bd9Sstevel@tonic-gate 71897c478bd9Sstevel@tonic-gate /* 71907c478bd9Sstevel@tonic-gate * Check the AFAR of the original error and secondary errors 71917c478bd9Sstevel@tonic-gate * match to the 64-byte boundary 71927c478bd9Sstevel@tonic-gate */ 71937c478bd9Sstevel@tonic-gate if (P2ALIGN(aflt->flt_addr, 64) != P2ALIGN(t_afar, 64)) 71947c478bd9Sstevel@tonic-gate return (0); 71957c478bd9Sstevel@tonic-gate 71967c478bd9Sstevel@tonic-gate /* 71977c478bd9Sstevel@tonic-gate * We've passed all the checks, so it's a secondary error! 71987c478bd9Sstevel@tonic-gate */ 71997c478bd9Sstevel@tonic-gate return (1); 72007c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 72017c478bd9Sstevel@tonic-gate } 72027c478bd9Sstevel@tonic-gate 72037c478bd9Sstevel@tonic-gate /* 72047c478bd9Sstevel@tonic-gate * Translate the flt_bit or flt_type into an error type. First, flt_bit 72057c478bd9Sstevel@tonic-gate * is checked for any valid errors. If found, the error type is 72067c478bd9Sstevel@tonic-gate * returned. If not found, the flt_type is checked for L1$ parity errors. 72077c478bd9Sstevel@tonic-gate */ 72087c478bd9Sstevel@tonic-gate /*ARGSUSED*/ 72097c478bd9Sstevel@tonic-gate static uint8_t 72107c478bd9Sstevel@tonic-gate cpu_flt_bit_to_plat_error(struct async_flt *aflt) 72117c478bd9Sstevel@tonic-gate { 72127c478bd9Sstevel@tonic-gate #if defined(JALAPENO) 72137c478bd9Sstevel@tonic-gate /* 72147c478bd9Sstevel@tonic-gate * Currently, logging errors to the SC is not supported on Jalapeno 72157c478bd9Sstevel@tonic-gate */ 72167c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_NONE); 72177c478bd9Sstevel@tonic-gate #else 72187c478bd9Sstevel@tonic-gate ch_async_flt_t *ch_flt = (ch_async_flt_t *)aflt; 72197c478bd9Sstevel@tonic-gate 72207c478bd9Sstevel@tonic-gate switch (ch_flt->flt_bit) { 72217c478bd9Sstevel@tonic-gate case C_AFSR_CE: 72227c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_CE); 72237c478bd9Sstevel@tonic-gate case C_AFSR_UCC: 72247c478bd9Sstevel@tonic-gate case C_AFSR_EDC: 72257c478bd9Sstevel@tonic-gate case C_AFSR_WDC: 72267c478bd9Sstevel@tonic-gate case C_AFSR_CPC: 72277c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_CE); 72287c478bd9Sstevel@tonic-gate case C_AFSR_EMC: 72297c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_EMC); 72307c478bd9Sstevel@tonic-gate case C_AFSR_IVC: 72317c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IVC); 72327c478bd9Sstevel@tonic-gate case C_AFSR_UE: 72337c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_UE); 72347c478bd9Sstevel@tonic-gate case C_AFSR_UCU: 72357c478bd9Sstevel@tonic-gate case C_AFSR_EDU: 72367c478bd9Sstevel@tonic-gate case C_AFSR_WDU: 72377c478bd9Sstevel@tonic-gate case C_AFSR_CPU: 72387c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_UE); 72397c478bd9Sstevel@tonic-gate case C_AFSR_IVU: 72407c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IVU); 72417c478bd9Sstevel@tonic-gate case C_AFSR_TO: 72427c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_TO); 72437c478bd9Sstevel@tonic-gate case C_AFSR_BERR: 72447c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_BERR); 72457c478bd9Sstevel@tonic-gate #if defined(CHEETAH_PLUS) 72467c478bd9Sstevel@tonic-gate case C_AFSR_L3_EDC: 72477c478bd9Sstevel@tonic-gate case C_AFSR_L3_UCC: 72487c478bd9Sstevel@tonic-gate case C_AFSR_L3_CPC: 72497c478bd9Sstevel@tonic-gate case C_AFSR_L3_WDC: 72507c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_CE); 72517c478bd9Sstevel@tonic-gate case C_AFSR_IMC: 72527c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IMC); 72537c478bd9Sstevel@tonic-gate case C_AFSR_TSCE: 72547c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_TSCE); 72557c478bd9Sstevel@tonic-gate case C_AFSR_THCE: 72567c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L2_THCE); 72577c478bd9Sstevel@tonic-gate case C_AFSR_L3_MECC: 72587c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_MECC); 72597c478bd9Sstevel@tonic-gate case C_AFSR_L3_THCE: 72607c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_THCE); 72617c478bd9Sstevel@tonic-gate case C_AFSR_L3_CPU: 72627c478bd9Sstevel@tonic-gate case C_AFSR_L3_EDU: 72637c478bd9Sstevel@tonic-gate case C_AFSR_L3_UCU: 72647c478bd9Sstevel@tonic-gate case C_AFSR_L3_WDU: 72657c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_L3_UE); 72667c478bd9Sstevel@tonic-gate case C_AFSR_DUE: 72677c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DUE); 72687c478bd9Sstevel@tonic-gate case C_AFSR_DTO: 72697c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DTO); 72707c478bd9Sstevel@tonic-gate case C_AFSR_DBERR: 72717c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DBERR); 72727c478bd9Sstevel@tonic-gate #endif /* CHEETAH_PLUS */ 72737c478bd9Sstevel@tonic-gate default: 72747c478bd9Sstevel@tonic-gate switch (ch_flt->flt_type) { 72757c478bd9Sstevel@tonic-gate #if defined(CPU_IMP_L1_CACHE_PARITY) 72767c478bd9Sstevel@tonic-gate case CPU_IC_PARITY: 72777c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_IPE); 72787c478bd9Sstevel@tonic-gate case CPU_DC_PARITY: 72797c478bd9Sstevel@tonic-gate if (IS_PANTHER(cpunodes[CPU->cpu_id].implementation)) { 72807c478bd9Sstevel@tonic-gate if (ch_flt->parity_data.dpe.cpl_cache == 72817c478bd9Sstevel@tonic-gate CPU_PC_PARITY) { 72827c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_PCACHE); 72837c478bd9Sstevel@tonic-gate } 72847c478bd9Sstevel@tonic-gate } 72857c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DPE); 72867c478bd9Sstevel@tonic-gate #endif /* CPU_IMP_L1_CACHE_PARITY */ 72877c478bd9Sstevel@tonic-gate case CPU_ITLB_PARITY: 72887c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_ITLB); 72897c478bd9Sstevel@tonic-gate case CPU_DTLB_PARITY: 72907c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_DTLB); 72917c478bd9Sstevel@tonic-gate default: 72927c478bd9Sstevel@tonic-gate return (PLAT_ECC_ERROR2_NONE); 72937c478bd9Sstevel@tonic-gate } 72947c478bd9Sstevel@tonic-gate } 72957c478bd9Sstevel@tonic-gate #endif /* JALAPENO */ 72967c478bd9Sstevel@tonic-gate } 7297