xref: /illumos-gate/usr/src/uts/i86pc/cpu/amd_opteron/ao_mca.c (revision c84b7bbef5ecc2a27799422588073deefd9db715)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/regset.h>
28 #include <sys/privregs.h>
29 #include <sys/pci_impl.h>
30 #include <sys/cpuvar.h>
31 #include <sys/x86_archext.h>
32 #include <sys/cmn_err.h>
33 #include <sys/systm.h>
34 #include <sys/sysmacros.h>
35 #include <sys/pghw.h>
36 #include <sys/cyclic.h>
37 #include <sys/sysevent.h>
38 #include <sys/smbios.h>
39 #include <sys/mca_x86.h>
40 #include <sys/mca_amd.h>
41 #include <sys/mc.h>
42 #include <sys/mc_amd.h>
43 #include <sys/psw.h>
44 #include <sys/ddi.h>
45 #include <sys/sunddi.h>
46 #include <sys/sdt.h>
47 #include <sys/fm/util.h>
48 #include <sys/fm/protocol.h>
49 #include <sys/fm/cpu/AMD.h>
50 #include <sys/fm/smb/fmsmb.h>
51 #include <sys/acpi/acpi.h>
52 #include <sys/acpi/acpi_pci.h>
53 #include <sys/acpica.h>
54 #include <sys/cpu_module.h>
55 
56 #include "ao.h"
57 #include "ao_mca_disp.h"
58 
59 #define	AO_F_REVS_FG (X86_CHIPREV_AMD_F_REV_F | X86_CHIPREV_AMD_F_REV_G)
60 
61 int ao_mca_smi_disable = 1;		/* attempt to disable SMI polling */
62 
63 extern int x86gentopo_legacy;	/* x86 generic topology support */
64 
65 struct ao_ctl_init {
66 	uint32_t ctl_revmask;	/* rev(s) to which this applies */
67 	uint64_t ctl_bits;	/* mca ctl reg bitmask to set */
68 };
69 
70 /*
71  * Additional NB MCA ctl initialization for revs F and G
72  */
73 static const struct ao_ctl_init ao_nb_ctl_init[] = {
74 	{ AO_F_REVS_FG, AMD_NB_CTL_INIT_REV_FG },
75 	{ X86_CHIPREV_UNKNOWN, 0 }
76 };
77 
78 typedef struct ao_bank_cfg {
79 	uint64_t bank_ctl_init_cmn;			/* Common init value */
80 	const struct ao_ctl_init *bank_ctl_init_extra;	/* Extra for each rev */
81 	void (*bank_misc_initfunc)(cmi_hdl_t, ao_ms_data_t *, uint32_t);
82 	uint_t bank_ctl_mask;
83 } ao_bank_cfg_t;
84 
85 static void nb_mcamisc_init(cmi_hdl_t, ao_ms_data_t *, uint32_t);
86 
87 static const ao_bank_cfg_t ao_bank_cfgs[] = {
88 	{ AMD_DC_CTL_INIT_CMN, NULL, NULL, AMD_MSR_DC_MASK },
89 	{ AMD_IC_CTL_INIT_CMN, NULL, NULL, AMD_MSR_IC_MASK },
90 	{ AMD_BU_CTL_INIT_CMN, NULL, NULL, AMD_MSR_BU_MASK },
91 	{ AMD_LS_CTL_INIT_CMN, NULL, NULL, AMD_MSR_LS_MASK },
92 	{ AMD_NB_CTL_INIT_CMN, &ao_nb_ctl_init[0], nb_mcamisc_init,
93 		AMD_MSR_NB_MASK },
94 };
95 
96 static int ao_nbanks = sizeof (ao_bank_cfgs) / sizeof (ao_bank_cfgs[0]);
97 
98 /*
99  * This is quite awful but necessary to work around x86 system vendor's view of
100  * the world.  Other operating systems (you know who you are) don't understand
101  * Opteron-specific error handling, so BIOS and system vendors often hide these
102  * conditions from them by using SMI polling to copy out any errors from the
103  * machine-check registers.  When Solaris runs on a system with this feature,
104  * we want to disable the SMI polling so we can use FMA instead.  Sadly, there
105  * isn't even a standard self-describing way to express the whole situation,
106  * so we have to resort to hard-coded values.  This should all be changed to
107  * be a self-describing vendor-specific SMBIOS structure in the future.
108  */
109 static const struct ao_smi_disable {
110 	const char *asd_sys_vendor;	/* SMB_TYPE_SYSTEM vendor prefix */
111 	const char *asd_sys_product;	/* SMB_TYPE_SYSTEM product prefix */
112 	const char *asd_bios_vendor;	/* SMB_TYPE_BIOS vendor prefix */
113 	uint8_t asd_code;		/* output code for SMI disable */
114 } ao_smi_disable[] = {
115 	{ "Sun Microsystems", "Galaxy12",
116 	    "American Megatrends", 0x59 },
117 	{ "Sun Microsystems", "Sun Fire X4100 Server",
118 	    "American Megatrends", 0x59 },
119 	{ "Sun Microsystems", "Sun Fire X4200 Server",
120 	    "American Megatrends", 0x59 },
121 	{ NULL, NULL, NULL, 0 }
122 };
123 
124 static int
125 ao_disp_match_r4(uint16_t ref, uint8_t r4)
126 {
127 	static const uint16_t ao_r4_map[] = {
128 		AO_MCA_R4_BIT_ERR,	/* MCAX86_ERRCODE_RRRR_ERR */
129 		AO_MCA_R4_BIT_RD,	/* MCAX86_ERRCODE_RRRR_RD */
130 		AO_MCA_R4_BIT_WR,	/* MCAX86_ERRCODE_RRRR_WR */
131 		AO_MCA_R4_BIT_DRD,	/* MCAX86_ERRCODE_RRRR_DRD */
132 		AO_MCA_R4_BIT_DWR,	/* MCAX86_ERRCODE_RRRR_DWR */
133 		AO_MCA_R4_BIT_IRD,	/* MCAX86_ERRCODE_RRRR_IRD */
134 		AO_MCA_R4_BIT_PREFETCH,	/* MCAX86_ERRCODE_RRRR_PREFETCH */
135 		AO_MCA_R4_BIT_EVICT,	/* MCAX86_ERRCODE_RRRR_EVICT */
136 		AO_MCA_R4_BIT_SNOOP	/* MCAX86_ERRCODE_RRRR_SNOOP */
137 	};
138 
139 	ASSERT(r4 < sizeof (ao_r4_map) / sizeof (uint16_t));
140 
141 	return ((ref & ao_r4_map[r4]) != 0);
142 }
143 
144 static int
145 ao_disp_match_pp(uint8_t ref, uint8_t pp)
146 {
147 	static const uint8_t ao_pp_map[] = {
148 		AO_MCA_PP_BIT_SRC,	/* MCAX86_ERRCODE_PP_SRC */
149 		AO_MCA_PP_BIT_RES,	/* MCAX86_ERRCODE_PP_RES */
150 		AO_MCA_PP_BIT_OBS,	/* MCAX86_ERRCODE_PP_OBS */
151 		AO_MCA_PP_BIT_GEN	/* MCAX86_ERRCODE_PP_GEN */
152 	};
153 
154 	ASSERT(pp < sizeof (ao_pp_map) / sizeof (uint8_t));
155 
156 	return ((ref & ao_pp_map[pp]) != 0);
157 }
158 
159 static int
160 ao_disp_match_ii(uint8_t ref, uint8_t ii)
161 {
162 	static const uint8_t ao_ii_map[] = {
163 		AO_MCA_II_BIT_MEM,	/* MCAX86_ERRCODE_II_MEM */
164 		0,
165 		AO_MCA_II_BIT_IO,	/* MCAX86_ERRCODE_II_IO */
166 		AO_MCA_II_BIT_GEN	/* MCAX86_ERRCODE_II_GEN */
167 	};
168 
169 	ASSERT(ii < sizeof (ao_ii_map) / sizeof (uint8_t));
170 
171 	return ((ref & ao_ii_map[ii]) != 0);
172 }
173 
174 static uint8_t
175 bit_strip(uint16_t *codep, uint16_t mask, uint16_t shift)
176 {
177 	uint8_t val = (*codep & mask) >> shift;
178 	*codep &= ~mask;
179 	return (val);
180 }
181 
182 #define	BIT_STRIP(codep, name) \
183 	bit_strip(codep, MCAX86_ERRCODE_##name##_MASK, \
184 	MCAX86_ERRCODE_##name##_SHIFT)
185 
186 /*ARGSUSED*/
187 static int
188 ao_disp_match_one(const ao_error_disp_t *aed, uint64_t status, uint32_t rev,
189     int bankno)
190 {
191 	uint16_t code = MCAX86_ERRCODE(status);
192 	uint8_t extcode = AMD_EXT_ERRCODE(status);
193 	uint64_t stat_mask = aed->aed_stat_mask;
194 	uint64_t stat_mask_res = aed->aed_stat_mask_res;
195 
196 	/*
197 	 * If the bank's status register indicates overflow, then we can no
198 	 * longer rely on the value of CECC: our experience with actual fault
199 	 * injection has shown that multiple CE's overwriting each other shows
200 	 * AMD_BANK_STAT_CECC and AMD_BANK_STAT_UECC both set to zero.  This
201 	 * should be clarified in a future BKDG or by the Revision Guide.
202 	 * This behaviour is fixed in revision F.
203 	 */
204 	if (bankno == AMD_MCA_BANK_NB &&
205 	    !X86_CHIPREV_ATLEAST(rev, X86_CHIPREV_AMD_F_REV_F) &&
206 	    status & MSR_MC_STATUS_OVER) {
207 		stat_mask &= ~AMD_BANK_STAT_CECC;
208 		stat_mask_res &= ~AMD_BANK_STAT_CECC;
209 	}
210 
211 	if ((status & stat_mask) != stat_mask_res)
212 		return (0);
213 
214 	/*
215 	 * r4 and pp bits are stored separately, so we mask off and compare them
216 	 * for the code types that use them.  Once we've taken the r4 and pp
217 	 * bits out of the equation, we can directly compare the resulting code
218 	 * with the one stored in the ao_error_disp_t.
219 	 */
220 	if (AMD_ERRCODE_ISMEM(code)) {
221 		uint8_t r4 = BIT_STRIP(&code, RRRR);
222 
223 		if (!ao_disp_match_r4(aed->aed_stat_r4_bits, r4))
224 			return (0);
225 
226 	} else if (AMD_ERRCODE_ISBUS(code)) {
227 		uint8_t r4 = BIT_STRIP(&code, RRRR);
228 		uint8_t pp = BIT_STRIP(&code, PP);
229 		uint8_t ii = BIT_STRIP(&code, II);
230 
231 		if (!ao_disp_match_r4(aed->aed_stat_r4_bits, r4) ||
232 		    !ao_disp_match_pp(aed->aed_stat_pp_bits, pp) ||
233 		    !ao_disp_match_ii(aed->aed_stat_ii_bits, ii))
234 			return (0);
235 	}
236 
237 	return (code == aed->aed_stat_code && extcode == aed->aed_stat_extcode);
238 }
239 
240 /*ARGSUSED*/
241 cms_cookie_t
242 ao_ms_disp_match(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status,
243     uint64_t addr, uint64_t misc, void *mslogout)
244 {
245 	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
246 	uint32_t rev = ao->ao_ms_shared->aos_chiprev;
247 	const ao_error_disp_t *aed;
248 
249 	for (aed = ao_error_disp[banknum]; aed->aed_stat_mask != 0; aed++) {
250 		if (ao_disp_match_one(aed, status, rev, banknum))
251 			return ((cms_cookie_t)aed);
252 	}
253 
254 	return (NULL);
255 }
256 
257 /*ARGSUSED*/
258 void
259 ao_ms_ereport_class(cmi_hdl_t hdl, cms_cookie_t mscookie,
260     const char **cpuclsp, const char **leafclsp)
261 {
262 	const ao_error_disp_t *aed = mscookie;
263 
264 	if (aed != NULL) {
265 		*cpuclsp = FM_EREPORT_CPU_AMD;
266 		*leafclsp = aed->aed_class;
267 	}
268 }
269 
270 static int
271 ao_chip_once(ao_ms_data_t *ao, enum ao_cfgonce_bitnum what)
272 {
273 	return (atomic_set_long_excl(&ao->ao_ms_shared->aos_cfgonce,
274 	    what) == 0 ?  B_TRUE : B_FALSE);
275 }
276 
277 /*
278  * This knob exists in case any platform has a problem with our default
279  * policy of disabling any interrupt registered in the NB MC4_MISC
280  * register.  Setting this may cause Solaris and external entities
281  * who also have an interest in this register to argue over available
282  * telemetry (so setting it is generally not recommended).
283  */
284 int ao_nb_cfg_mc4misc_noseize = 0;
285 
286 /*
287  * The BIOS may have setup to receive SMI on counter overflow.  It may also
288  * have locked various fields or made them read-only.  We will clear any
289  * SMI request and leave the register locked.  We will also clear the
290  * counter and enable counting - while we don't use the counter it is nice
291  * to have it enabled for verification and debug work.
292  */
293 static void
294 nb_mcamisc_init(cmi_hdl_t hdl, ao_ms_data_t *ao, uint32_t rev)
295 {
296 	uint64_t val, nval;
297 
298 	if (!X86_CHIPREV_MATCH(rev, AO_F_REVS_FG))
299 		return;
300 
301 	if (cmi_hdl_rdmsr(hdl, AMD_MSR_NB_MISC, &val) != CMI_SUCCESS)
302 		return;
303 
304 	ao->ao_ms_shared->aos_bcfg_nb_misc = val;
305 
306 	if (ao_nb_cfg_mc4misc_noseize)
307 		return;		/* stash BIOS value, but no changes */
308 
309 
310 	/*
311 	 * The Valid bit tells us whether the CtrP bit is defined; if it
312 	 * is the CtrP bit tells us whether an ErrCount field is present.
313 	 * If not then there is nothing for us to do.
314 	 */
315 	if (!(val & AMD_NB_MISC_VALID) || !(val & AMD_NB_MISC_CTRP))
316 		return;
317 
318 
319 	nval = val;
320 	nval |= AMD_NB_MISC_CNTEN;		/* enable ECC error counting */
321 	nval &= ~AMD_NB_MISC_ERRCOUNT_MASK;	/* clear ErrCount */
322 	nval &= ~AMD_NB_MISC_OVRFLW;		/* clear Ovrflw */
323 	nval &= ~AMD_NB_MISC_INTTYPE_MASK;	/* no interrupt on overflow */
324 	nval |= AMD_NB_MISC_LOCKED;
325 
326 	if (nval != val) {
327 		uint64_t locked = val & AMD_NB_MISC_LOCKED;
328 
329 		if (locked)
330 			ao_bankstatus_prewrite(hdl, ao);
331 
332 		(void) cmi_hdl_wrmsr(hdl, AMD_MSR_NB_MISC, nval);
333 
334 		if (locked)
335 			ao_bankstatus_postwrite(hdl, ao);
336 	}
337 }
338 
339 /*
340  * NorthBridge (NB) MCA Configuration.
341  *
342  * We add and remove bits from the BIOS-configured value, rather than
343  * writing an absolute value.  The variables ao_nb_cfg_{add,remove}_cmn and
344  * ap_nb_cfg_{add,remove}_revFG are available for modification via kmdb
345  * and /etc/system.  The revision-specific adds and removes are applied
346  * after the common changes, and one write is made to the config register.
347  * These are not intended for watchdog configuration via these variables -
348  * use the watchdog policy below.
349  */
350 
351 /*
352  * Bits to be added to the NB configuration register - all revs.
353  */
354 uint32_t ao_nb_cfg_add_cmn = AMD_NB_CFG_ADD_CMN;
355 
356 /*
357  * Bits to be cleared from the NB configuration register - all revs.
358  */
359 uint32_t ao_nb_cfg_remove_cmn = AMD_NB_CFG_REMOVE_CMN;
360 
361 /*
362  * Bits to be added to the NB configuration register - revs F and G.
363  */
364 uint32_t ao_nb_cfg_add_revFG = AMD_NB_CFG_ADD_REV_FG;
365 
366 /*
367  * Bits to be cleared from the NB configuration register - revs F and G.
368  */
369 uint32_t ao_nb_cfg_remove_revFG = AMD_NB_CFG_REMOVE_REV_FG;
370 
371 struct ao_nb_cfg {
372 	uint32_t cfg_revmask;
373 	uint32_t *cfg_add_p;
374 	uint32_t *cfg_remove_p;
375 };
376 
377 static const struct ao_nb_cfg ao_cfg_extra[] = {
378 	{ AO_F_REVS_FG, &ao_nb_cfg_add_revFG, &ao_nb_cfg_remove_revFG },
379 	{ X86_CHIPREV_UNKNOWN, NULL, NULL }
380 };
381 
382 /*
383  * Bits to be used if we configure the NorthBridge (NB) Watchdog.  The watchdog
384  * triggers a machine check exception when no response to an NB system access
385  * occurs within a specified time interval.
386  */
387 uint32_t ao_nb_cfg_wdog =
388     AMD_NB_CFG_WDOGTMRCNTSEL_4095 |
389     AMD_NB_CFG_WDOGTMRBASESEL_1MS;
390 
391 /*
392  * The default watchdog policy is to enable it (at the above rate) if it
393  * is disabled;  if it is enabled then we leave it enabled at the rate
394  * chosen by the BIOS.
395  */
396 enum {
397 	AO_NB_WDOG_LEAVEALONE,		/* Don't touch watchdog config */
398 	AO_NB_WDOG_DISABLE,		/* Always disable watchdog */
399 	AO_NB_WDOG_ENABLE_IF_DISABLED,	/* If disabled, enable at our rate */
400 	AO_NB_WDOG_ENABLE_FORCE_RATE	/* Enable and set our rate */
401 } ao_nb_watchdog_policy = AO_NB_WDOG_ENABLE_IF_DISABLED;
402 
403 static void
404 ao_nb_cfg(ao_ms_data_t *ao, uint32_t rev)
405 {
406 	const struct ao_nb_cfg *nbcp = &ao_cfg_extra[0];
407 	uint_t procnodeid = pg_plat_hw_instance_id(CPU, PGHW_PROCNODE);
408 	uint32_t val;
409 
410 	/*
411 	 * Read the NorthBridge (NB) configuration register in PCI space,
412 	 * modify the settings accordingly, and store the new value back.
413 	 * Note that the stashed BIOS config value aos_bcfg_nb_cfg is used
414 	 * in ereport payload population to determine ECC syndrome type for
415 	 * memory errors.
416 	 */
417 	ao->ao_ms_shared->aos_bcfg_nb_cfg = val =
418 	    ao_pcicfg_read(procnodeid, MC_FUNC_MISCCTL, MC_CTL_REG_NBCFG);
419 
420 	switch (ao_nb_watchdog_policy) {
421 	case AO_NB_WDOG_LEAVEALONE:
422 		break;
423 
424 	case AO_NB_WDOG_DISABLE:
425 		val &= ~AMD_NB_CFG_WDOGTMRBASESEL_MASK;
426 		val &= ~AMD_NB_CFG_WDOGTMRCNTSEL_MASK;
427 		val |= AMD_NB_CFG_WDOGTMRDIS;
428 		break;
429 
430 	default:
431 		cmn_err(CE_NOTE, "ao_nb_watchdog_policy=%d unrecognised, "
432 		    "using default policy", ao_nb_watchdog_policy);
433 		/*FALLTHRU*/
434 
435 	case AO_NB_WDOG_ENABLE_IF_DISABLED:
436 		if (!(val & AMD_NB_CFG_WDOGTMRDIS))
437 			break;	/* if enabled leave rate intact */
438 		/*FALLTHRU*/
439 
440 	case AO_NB_WDOG_ENABLE_FORCE_RATE:
441 		val &= ~AMD_NB_CFG_WDOGTMRBASESEL_MASK;
442 		val &= ~AMD_NB_CFG_WDOGTMRCNTSEL_MASK;
443 		val &= ~AMD_NB_CFG_WDOGTMRDIS;
444 		val |= ao_nb_cfg_wdog;
445 		break;
446 	}
447 
448 	/*
449 	 * Now apply bit adds and removes, first those common to all revs
450 	 * and then the revision-specific ones.
451 	 */
452 	val &= ~ao_nb_cfg_remove_cmn;
453 	val |= ao_nb_cfg_add_cmn;
454 
455 	while (nbcp->cfg_revmask != X86_CHIPREV_UNKNOWN) {
456 		if (X86_CHIPREV_MATCH(rev, nbcp->cfg_revmask)) {
457 			val &= ~(*nbcp->cfg_remove_p);
458 			val |= *nbcp->cfg_add_p;
459 		}
460 		nbcp++;
461 	}
462 
463 	ao_pcicfg_write(procnodeid, MC_FUNC_MISCCTL, MC_CTL_REG_NBCFG, val);
464 }
465 
466 static void
467 ao_dram_cfg(ao_ms_data_t *ao, uint32_t rev)
468 {
469 	uint_t procnodeid = pg_plat_hw_instance_id(CPU, PGHW_PROCNODE);
470 	union mcreg_dramcfg_lo dcfglo;
471 
472 	ao->ao_ms_shared->aos_bcfg_dcfg_lo = MCREG_VAL32(&dcfglo) =
473 	    ao_pcicfg_read(procnodeid, MC_FUNC_DRAMCTL, MC_DC_REG_DRAMCFGLO);
474 	ao->ao_ms_shared->aos_bcfg_dcfg_hi =
475 	    ao_pcicfg_read(procnodeid, MC_FUNC_DRAMCTL, MC_DC_REG_DRAMCFGHI);
476 #ifdef OPTERON_ERRATUM_172
477 	if (X86_CHIPREV_MATCH(rev, AO_F_REVS_FG) &&
478 	    MCREG_FIELD_F_revFG(&dcfglo, ParEn)) {
479 		MCREG_FIELD_F_revFG(&dcfglo, ParEn) = 0;
480 		ao_pcicfg_write(procnodeid, MC_FUNC_DRAMCTL,
481 		    MC_DC_REG_DRAMCFGLO, MCREG_VAL32(&dcfglo));
482 	}
483 #endif
484 }
485 
486 /*
487  * This knob exists in case any platform has a problem with our default
488  * policy of disabling any interrupt registered in the online spare
489  * control register.  Setting this may cause Solaris and external entities
490  * who also have an interest in this register to argue over available
491  * telemetry (so setting it is generally not recommended).
492  */
493 int ao_nb_cfg_sparectl_noseize = 0;
494 
495 /*
496  * Setup the online spare control register (revs F and G).  We disable
497  * any interrupt registered by the BIOS and zero all error counts.
498  */
499 static void
500 ao_sparectl_cfg(ao_ms_data_t *ao)
501 {
502 	uint_t procnodeid = pg_plat_hw_instance_id(CPU, PGHW_PROCNODE);
503 	union mcreg_sparectl sparectl;
504 	int chan, cs;
505 
506 	ao->ao_ms_shared->aos_bcfg_nb_sparectl = MCREG_VAL32(&sparectl) =
507 	    ao_pcicfg_read(procnodeid, MC_FUNC_MISCCTL, MC_CTL_REG_SPARECTL);
508 
509 	if (ao_nb_cfg_sparectl_noseize)
510 		return;	/* stash BIOS value, but no changes */
511 
512 	/*
513 	 * If the BIOS has requested SMI interrupt type for ECC count
514 	 * overflow for a chip-select or channel force those off.
515 	 */
516 	MCREG_FIELD_F_revFG(&sparectl, EccErrInt) = 0;
517 	MCREG_FIELD_F_revFG(&sparectl, SwapDoneInt) = 0;
518 
519 	/*
520 	 * Zero EccErrCnt and write this back to all chan/cs combinations.
521 	 */
522 	MCREG_FIELD_F_revFG(&sparectl, EccErrCntWrEn) = 1;
523 	MCREG_FIELD_F_revFG(&sparectl, EccErrCnt) = 0;
524 	for (chan = 0; chan < MC_CHIP_NDRAMCHAN; chan++) {
525 		MCREG_FIELD_F_revFG(&sparectl, EccErrCntDramChan) = chan;
526 
527 		for (cs = 0; cs < MC_CHIP_NCS; cs++) {
528 			MCREG_FIELD_F_revFG(&sparectl, EccErrCntDramCs) = cs;
529 			ao_pcicfg_write(procnodeid, MC_FUNC_MISCCTL,
530 			    MC_CTL_REG_SPARECTL, MCREG_VAL32(&sparectl));
531 		}
532 	}
533 }
534 
535 int ao_forgive_uc = 0;		/* For test/debug only */
536 int ao_forgive_pcc = 0;		/* For test/debug only */
537 int ao_fake_poison = 0;		/* For test/debug only */
538 
539 uint32_t
540 ao_ms_error_action(cmi_hdl_t hdl, int ismc, int banknum,
541     uint64_t status, uint64_t addr, uint64_t misc, void *mslogout)
542 {
543 	const ao_error_disp_t *aed;
544 	uint32_t retval = 0;
545 	uint8_t when;
546 	int en;
547 
548 	if (ao_forgive_uc)
549 		retval |= CMS_ERRSCOPE_CLEARED_UC;
550 
551 	if (ao_forgive_pcc)
552 		retval |= CMS_ERRSCOPE_CURCONTEXT_OK;
553 
554 	if (ao_fake_poison && status & MSR_MC_STATUS_UC)
555 		retval |= CMS_ERRSCOPE_POISONED;
556 
557 	if (retval)
558 		return (retval);
559 
560 	aed = ao_ms_disp_match(hdl, ismc, banknum, status, addr, misc,
561 	    mslogout);
562 
563 	/*
564 	 * If we do not recognise the error let the cpu module apply
565 	 * the generic criteria to decide how to react.
566 	 */
567 	if (aed == NULL)
568 		return (0);
569 
570 	en = (status & MSR_MC_STATUS_EN) != 0;
571 
572 	if ((when = aed->aed_panic_when) == AO_AED_PANIC_NEVER)
573 		retval |= CMS_ERRSCOPE_IGNORE_ERR;
574 
575 	if ((when & AO_AED_PANIC_ALWAYS) ||
576 	    ((when & AO_AED_PANIC_IFMCE) && (en || ismc)))
577 		retval |= CMS_ERRSCOPE_FORCE_FATAL;
578 
579 	/*
580 	 * The original AMD implementation would panic on a machine check
581 	 * (not a poll) if the status overflow bit was set, with an
582 	 * exception for the case of rev F or later with an NB error
583 	 * indicating CECC.  This came from the perception that the
584 	 * overflow bit was not correctly managed on rev E and earlier, for
585 	 * example that repeated correctable memeory errors did not set
586 	 * OVER but somehow clear CECC.
587 	 *
588 	 * We will leave the generic support to evaluate overflow errors
589 	 * and decide to panic on their individual merits, e.g., if PCC
590 	 * is set and so on.  The AMD docs do say (as Intel does) that
591 	 * the status information is *all* from the higher-priority
592 	 * error in the case of an overflow, so it is at least as serious
593 	 * as the original and we can decide panic etc based on it.
594 	 */
595 
596 	return (retval);
597 }
598 
599 /*
600  * Will need to change for family 0x10
601  */
602 static uint_t
603 ao_ereport_synd(ao_ms_data_t *ao, uint64_t status, uint_t *typep,
604     int is_nb)
605 {
606 	if (is_nb) {
607 		if (ao->ao_ms_shared->aos_bcfg_nb_cfg &
608 		    AMD_NB_CFG_CHIPKILLECCEN) {
609 			*typep = AMD_SYNDTYPE_CHIPKILL;
610 			return (AMD_NB_STAT_CKSYND(status));
611 		} else {
612 			*typep = AMD_SYNDTYPE_ECC;
613 			return (AMD_BANK_SYND(status));
614 		}
615 	} else {
616 		*typep = AMD_SYNDTYPE_ECC;
617 		return (AMD_BANK_SYND(status));
618 	}
619 }
620 
621 static nvlist_t *
622 ao_ereport_create_resource_elem(cmi_hdl_t hdl, nv_alloc_t *nva,
623     mc_unum_t *unump, int dimmnum)
624 {
625 	nvlist_t *nvl, *snvl;
626 	nvlist_t *board_list = NULL;
627 
628 	if ((nvl = fm_nvlist_create(nva)) == NULL)	/* freed by caller */
629 		return (NULL);
630 
631 	if ((snvl = fm_nvlist_create(nva)) == NULL) {
632 		fm_nvlist_destroy(nvl, nva ? FM_NVA_RETAIN : FM_NVA_FREE);
633 		return (NULL);
634 	}
635 
636 	(void) nvlist_add_uint64(snvl, FM_FMRI_HC_SPECIFIC_OFFSET,
637 	    unump->unum_offset);
638 
639 	if (!x86gentopo_legacy) {
640 		board_list = cmi_hdl_smb_bboard(hdl);
641 
642 		if (board_list == NULL) {
643 			fm_nvlist_destroy(nvl,
644 			    nva ? FM_NVA_RETAIN : FM_NVA_FREE);
645 			fm_nvlist_destroy(snvl,
646 			    nva ? FM_NVA_RETAIN : FM_NVA_FREE);
647 			return (NULL);
648 		}
649 
650 		fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION, NULL, snvl,
651 		    board_list, 4,
652 		    "chip", cmi_hdl_smb_chipid(hdl),
653 		    "memory-controller", unump->unum_mc,
654 		    "dimm", unump->unum_dimms[dimmnum],
655 		    "rank", unump->unum_rank);
656 	} else {
657 		fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, snvl, 5,
658 		    "motherboard", unump->unum_board,
659 		    "chip", unump->unum_chip,
660 		    "memory-controller", unump->unum_mc,
661 		    "dimm", unump->unum_dimms[dimmnum],
662 		    "rank", unump->unum_rank);
663 	}
664 
665 	fm_nvlist_destroy(snvl, nva ? FM_NVA_RETAIN : FM_NVA_FREE);
666 
667 	return (nvl);
668 }
669 
670 static void
671 ao_ereport_add_resource(cmi_hdl_t hdl, nvlist_t *payload, nv_alloc_t *nva,
672     mc_unum_t *unump)
673 {
674 
675 	nvlist_t *elems[MC_UNUM_NDIMM];
676 	int nelems = 0;
677 	int i;
678 
679 	for (i = 0; i < MC_UNUM_NDIMM; i++) {
680 		if (unump->unum_dimms[i] == MC_INVALNUM)
681 			break;
682 
683 		if ((elems[nelems] = ao_ereport_create_resource_elem(hdl, nva,
684 		    unump, i)) == NULL)
685 			break;
686 
687 		nelems++;
688 	}
689 
690 	if (nelems == 0)
691 		return;
692 
693 	fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_RESOURCE,
694 	    DATA_TYPE_NVLIST_ARRAY, nelems, elems, NULL);
695 
696 	for (i = 0; i < nelems; i++)
697 		fm_nvlist_destroy(elems[i], nva ? FM_NVA_RETAIN : FM_NVA_FREE);
698 }
699 
700 /*ARGSUSED*/
701 void
702 ao_ms_ereport_add_logout(cmi_hdl_t hdl, nvlist_t *ereport,
703     nv_alloc_t *nva, int banknum, uint64_t status, uint64_t addr,
704     uint64_t misc, void *mslogout, cms_cookie_t mscookie)
705 {
706 	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
707 	const ao_error_disp_t *aed = mscookie;
708 	uint_t synd, syndtype;
709 	uint64_t members;
710 
711 	if (aed == NULL)
712 		return;
713 
714 	members = aed->aed_ereport_members;
715 
716 	synd = ao_ereport_synd(ao, status, &syndtype,
717 	    banknum == AMD_MCA_BANK_NB);
718 
719 	if (members & FM_EREPORT_PAYLOAD_FLAG_SYND) {
720 		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_SYND,
721 		    DATA_TYPE_UINT16, synd, NULL);
722 	}
723 
724 	if (members & FM_EREPORT_PAYLOAD_FLAG_SYND_TYPE) {
725 		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_SYND_TYPE,
726 		    DATA_TYPE_STRING, (syndtype == AMD_SYNDTYPE_CHIPKILL ?
727 		    "C4" : "E"), NULL);
728 	}
729 
730 	if (members & FM_EREPORT_PAYLOAD_FLAG_RESOURCE) {
731 		mc_unum_t unum;
732 
733 		if (((aed->aed_flags & AO_AED_FLAGS_ADDRTYPE) ==
734 		    AO_AED_F_PHYSICAL) && (status & MSR_MC_STATUS_ADDRV) &&
735 		    cmi_mc_patounum(addr, aed->aed_addrvalid_hi,
736 		    aed->aed_addrvalid_lo, synd, syndtype, &unum) ==
737 		    CMI_SUCCESS)
738 			ao_ereport_add_resource(hdl, ereport, nva, &unum);
739 	}
740 }
741 
742 /*ARGSUSED*/
743 boolean_t
744 ao_ms_ereport_includestack(cmi_hdl_t hdl, cms_cookie_t mscookie)
745 {
746 	const ao_error_disp_t *aed = mscookie;
747 
748 	if (aed == NULL)
749 		return (0);
750 
751 	return ((aed->aed_ereport_members &
752 	    FM_EREPORT_PAYLOAD_FLAG_STACK) != 0);
753 }
754 
755 cms_errno_t
756 ao_ms_msrinject(cmi_hdl_t hdl, uint_t msr, uint64_t val)
757 {
758 	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
759 	cms_errno_t rv = CMSERR_BADMSRWRITE;
760 
761 	ao_bankstatus_prewrite(hdl, ao);
762 	if (cmi_hdl_wrmsr(hdl, msr, val) == CMI_SUCCESS)
763 		rv = CMS_SUCCESS;
764 	ao_bankstatus_postwrite(hdl, ao);
765 
766 	return (rv);
767 }
768 
769 /*ARGSUSED*/
770 uint64_t
771 ao_ms_mcgctl_val(cmi_hdl_t hdl, int nbanks, uint64_t def)
772 {
773 	return ((1ULL << nbanks) - 1);
774 }
775 
776 boolean_t
777 ao_ms_bankctl_skipinit(cmi_hdl_t hdl, int banknum)
778 {
779 	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
780 
781 	if (banknum != AMD_MCA_BANK_NB)
782 		return (B_FALSE);
783 
784 	/*
785 	 * If we are the first to atomically set the "I'll do it" bit
786 	 * then return B_FALSE (do not skip), otherwise skip with B_TRUE.
787 	 */
788 	return (ao_chip_once(ao, AO_CFGONCE_NBMCA) == B_TRUE ?
789 	    B_FALSE : B_TRUE);
790 }
791 
792 uint64_t
793 ao_ms_bankctl_val(cmi_hdl_t hdl, int banknum, uint64_t def)
794 {
795 	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
796 	const struct ao_ctl_init *extrap;
797 	const ao_bank_cfg_t *bankcfg;
798 	uint64_t mcictl;
799 	uint32_t rev = ao->ao_ms_shared->aos_chiprev;
800 
801 	if (banknum >= sizeof (ao_bank_cfgs) / sizeof (ao_bank_cfgs[0]))
802 		return (def);
803 
804 	bankcfg = &ao_bank_cfgs[banknum];
805 	extrap = bankcfg->bank_ctl_init_extra;
806 
807 	mcictl = bankcfg->bank_ctl_init_cmn;
808 
809 	while (extrap != NULL && extrap->ctl_revmask != X86_CHIPREV_UNKNOWN) {
810 		if (X86_CHIPREV_MATCH(rev, extrap->ctl_revmask))
811 			mcictl |= extrap->ctl_bits;
812 		extrap++;
813 	}
814 
815 	return (mcictl);
816 }
817 
818 /*ARGSUSED*/
819 void
820 ao_bankstatus_prewrite(cmi_hdl_t hdl, ao_ms_data_t *ao)
821 {
822 #ifndef __xpv
823 	uint64_t hwcr;
824 
825 	if (cmi_hdl_rdmsr(hdl, MSR_AMD_HWCR, &hwcr) != CMI_SUCCESS)
826 		return;
827 
828 	ao->ao_ms_hwcr_val = hwcr;
829 
830 	if (!(hwcr & AMD_HWCR_MCI_STATUS_WREN)) {
831 		hwcr |= AMD_HWCR_MCI_STATUS_WREN;
832 		(void) cmi_hdl_wrmsr(hdl, MSR_AMD_HWCR, hwcr);
833 	}
834 #endif
835 }
836 
837 /*ARGSUSED*/
838 void
839 ao_bankstatus_postwrite(cmi_hdl_t hdl, ao_ms_data_t *ao)
840 {
841 #ifndef __xpv
842 	uint64_t hwcr = ao->ao_ms_hwcr_val;
843 
844 	if (!(hwcr & AMD_HWCR_MCI_STATUS_WREN)) {
845 		hwcr &= ~AMD_HWCR_MCI_STATUS_WREN;
846 		(void) cmi_hdl_wrmsr(hdl, MSR_AMD_HWCR, hwcr);
847 	}
848 #endif
849 }
850 
851 void
852 ao_ms_mca_init(cmi_hdl_t hdl, int nbanks)
853 {
854 	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
855 	uint32_t rev = ao->ao_ms_shared->aos_chiprev;
856 	ao_ms_mca_t *mca = &ao->ao_ms_mca;
857 	uint64_t *maskp;
858 	int i;
859 
860 	maskp = mca->ao_mca_bios_cfg.bcfg_bank_mask = kmem_zalloc(nbanks *
861 	    sizeof (uint64_t), KM_SLEEP);
862 
863 	/*
864 	 * Read the bank ctl mask MSRs, but only as many as we know
865 	 * certainly exist - don't calculate the register address.
866 	 * Also initialize the MCi_MISC register where required.
867 	 */
868 	for (i = 0; i < MIN(nbanks, ao_nbanks); i++) {
869 		(void) cmi_hdl_rdmsr(hdl, ao_bank_cfgs[i].bank_ctl_mask,
870 		    maskp++);
871 		if (ao_bank_cfgs[i].bank_misc_initfunc != NULL)
872 			ao_bank_cfgs[i].bank_misc_initfunc(hdl, ao, rev);
873 
874 	}
875 
876 	if (ao_chip_once(ao, AO_CFGONCE_NBCFG) == B_TRUE) {
877 		ao_nb_cfg(ao, rev);
878 
879 		if (X86_CHIPREV_MATCH(rev, AO_F_REVS_FG))
880 			ao_sparectl_cfg(ao);
881 	}
882 
883 	if (ao_chip_once(ao, AO_CFGONCE_DRAMCFG) == B_TRUE)
884 		ao_dram_cfg(ao, rev);
885 
886 	ao_procnode_scrubber_enable(hdl, ao);
887 }
888 
889 /*
890  * Note that although this cpu module is loaded before the PSMs are
891  * loaded (and hence before acpica is loaded), this function is
892  * called from post_startup(), after PSMs are initialized and acpica
893  * is loaded.
894  */
895 static int
896 ao_acpi_find_smicmd(int *asd_port)
897 {
898 	ACPI_TABLE_FADT *fadt = NULL;
899 
900 	/*
901 	 * AcpiGetTable works even if ACPI is disabled, so a failure
902 	 * here means we weren't able to retreive a pointer to the FADT.
903 	 */
904 	if (AcpiGetTable(ACPI_SIG_FADT, 1, (ACPI_TABLE_HEADER **)&fadt) !=
905 	    AE_OK)
906 		return (-1);
907 
908 	ASSERT(fadt != NULL);
909 
910 	*asd_port = fadt->SmiCommand;
911 	return (0);
912 }
913 
914 /*ARGSUSED*/
915 void
916 ao_ms_post_startup(cmi_hdl_t hdl)
917 {
918 	const struct ao_smi_disable *asd;
919 	id_t id;
920 	int rv = -1, asd_port;
921 
922 	smbios_system_t sy;
923 	smbios_bios_t sb;
924 	smbios_info_t si;
925 
926 	/*
927 	 * Fetch the System and BIOS vendor strings from SMBIOS and see if they
928 	 * match a value in our table.  If so, disable SMI error polling.  This
929 	 * is grotesque and should be replaced by self-describing vendor-
930 	 * specific SMBIOS data or a specification enhancement instead.
931 	 */
932 	if (ao_mca_smi_disable && ksmbios != NULL &&
933 	    smbios_info_bios(ksmbios, &sb) != SMB_ERR &&
934 	    (id = smbios_info_system(ksmbios, &sy)) != SMB_ERR &&
935 	    smbios_info_common(ksmbios, id, &si) != SMB_ERR) {
936 
937 		for (asd = ao_smi_disable; asd->asd_sys_vendor != NULL; asd++) {
938 			if (strncmp(asd->asd_sys_vendor, si.smbi_manufacturer,
939 			    strlen(asd->asd_sys_vendor)) != 0 ||
940 			    strncmp(asd->asd_sys_product, si.smbi_product,
941 			    strlen(asd->asd_sys_product)) != 0 ||
942 			    strncmp(asd->asd_bios_vendor, sb.smbb_vendor,
943 			    strlen(asd->asd_bios_vendor)) != 0)
944 				continue;
945 
946 			/*
947 			 * Look for the SMI_CMD port in the ACPI FADT,
948 			 * if the port is 0, this platform doesn't support
949 			 * SMM, so there is no SMI error polling to disable.
950 			 */
951 			if ((rv = ao_acpi_find_smicmd(&asd_port)) == 0 &&
952 			    asd_port != 0) {
953 				cmn_err(CE_CONT, "?SMI polling disabled in "
954 				    "favor of Solaris Fault Management for "
955 				    "AMD Processors\n");
956 
957 				outb(asd_port, asd->asd_code);
958 
959 			} else if (rv < 0) {
960 				cmn_err(CE_CONT, "?Solaris Fault Management "
961 				    "for AMD Processors could not disable SMI "
962 				    "polling because an error occurred while "
963 				    "trying to determine the SMI command port "
964 				    "from the ACPI FADT table\n");
965 			}
966 			break;
967 		}
968 	}
969 }
970