xref: /illumos-gate/usr/src/uts/i86pc/os/cmi.c (revision 4da99751)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  * Copyright (c) 2018, Joyent, Inc.
26  */
27 
28 /*
29  * Public interface to routines implemented by CPU modules
30  */
31 
32 #include <sys/types.h>
33 #include <sys/atomic.h>
34 #include <sys/x86_archext.h>
35 #include <sys/cpu_module_impl.h>
36 #include <sys/cpu_module_ms.h>
37 #include <sys/fm/util.h>
38 #include <sys/reboot.h>
39 #include <sys/modctl.h>
40 #include <sys/param.h>
41 #include <sys/cmn_err.h>
42 #include <sys/systm.h>
43 #include <sys/fm/protocol.h>
44 #include <sys/pcb.h>
45 #include <sys/ontrap.h>
46 #include <sys/psw.h>
47 #include <sys/privregs.h>
48 #include <sys/machsystm.h>
49 
50 /*
51  * Set to force cmi_init to fail.
52  */
53 int cmi_no_init = 0;
54 
55 /*
56  * Set to avoid MCA initialization.
57  */
58 int cmi_no_mca_init = 0;
59 
60 /*
61  * If cleared for debugging we will not attempt to load a model-specific
62  * cpu module but will load the generic cpu module instead.
63  */
64 int cmi_force_generic = 0;
65 
66 /*
67  * If cleared for debugging, we will suppress panicking on fatal hardware
68  * errors.  This should *only* be used for debugging; it use can and will
69  * cause data corruption if actual hardware errors are detected by the system.
70  */
71 int cmi_panic_on_uncorrectable_error = 1;
72 
73 /*
74  * Subdirectory (relative to the module search path) in which we will
75  * look for cpu modules.
76  */
77 #define	CPUMOD_SUBDIR	"cpu"
78 
79 /*
80  * CPU modules have a filenames such as "cpu.AuthenticAMD.15" and
81  * "cpu.generic" - the "cpu" prefix is specified by the following.
82  */
83 #define	CPUMOD_PREFIX	"cpu"
84 
85 /*
86  * Structure used to keep track of cpu modules we have loaded and their ops
87  */
88 typedef struct cmi {
89 	struct cmi *cmi_next;
90 	struct cmi *cmi_prev;
91 	const cmi_ops_t *cmi_ops;
92 	struct modctl *cmi_modp;
93 	uint_t cmi_refcnt;
94 } cmi_t;
95 
96 static cmi_t *cmi_list;
97 static const cmi_mc_ops_t *cmi_mc_global_ops;
98 static void *cmi_mc_global_data;
99 static kmutex_t cmi_load_lock;
100 
101 /*
102  * Functions we need from cmi_hw.c that are not part of the cpu_module.h
103  * interface.
104  */
105 extern cmi_hdl_t cmi_hdl_create(enum cmi_hdl_class, uint_t, uint_t, uint_t);
106 extern void cmi_hdl_destroy(cmi_hdl_t ophdl);
107 extern void cmi_hdl_setcmi(cmi_hdl_t, void *, void *);
108 extern void *cmi_hdl_getcmi(cmi_hdl_t);
109 extern void cmi_hdl_setmc(cmi_hdl_t, const struct cmi_mc_ops *, void *);
110 extern void cmi_hdl_inj_begin(cmi_hdl_t);
111 extern void cmi_hdl_inj_end(cmi_hdl_t);
112 extern void cmi_read_smbios(cmi_hdl_t);
113 
114 #define	HDL2CMI(hdl)		cmi_hdl_getcmi(hdl)
115 
116 #define	CMI_OPS(cmi)		(cmi)->cmi_ops
117 #define	CMI_OP_PRESENT(cmi, op)	((cmi) && CMI_OPS(cmi)->op != NULL)
118 
119 #define	CMI_MATCH_VENDOR	0	/* Just match on vendor */
120 #define	CMI_MATCH_FAMILY	1	/* Match down to family */
121 #define	CMI_MATCH_MODEL		2	/* Match down to model */
122 #define	CMI_MATCH_STEPPING	3	/* Match down to stepping */
123 
124 static void
cmi_link(cmi_t * cmi)125 cmi_link(cmi_t *cmi)
126 {
127 	ASSERT(MUTEX_HELD(&cmi_load_lock));
128 
129 	cmi->cmi_prev = NULL;
130 	cmi->cmi_next = cmi_list;
131 	if (cmi_list != NULL)
132 		cmi_list->cmi_prev = cmi;
133 	cmi_list = cmi;
134 }
135 
136 static void
cmi_unlink(cmi_t * cmi)137 cmi_unlink(cmi_t *cmi)
138 {
139 	ASSERT(MUTEX_HELD(&cmi_load_lock));
140 	ASSERT(cmi->cmi_refcnt == 0);
141 
142 	if (cmi->cmi_prev != NULL)
143 		cmi->cmi_prev = cmi->cmi_next;
144 
145 	if (cmi->cmi_next != NULL)
146 		cmi->cmi_next->cmi_prev = cmi->cmi_prev;
147 
148 	if (cmi_list == cmi)
149 		cmi_list = cmi->cmi_next;
150 }
151 
152 /*
153  * Hold the module in memory.  We call to CPU modules without using the
154  * stubs mechanism, so these modules must be manually held in memory.
155  * The mod_ref acts as if another loaded module has a dependency on us.
156  */
157 static void
cmi_hold(cmi_t * cmi)158 cmi_hold(cmi_t *cmi)
159 {
160 	ASSERT(MUTEX_HELD(&cmi_load_lock));
161 
162 	mutex_enter(&mod_lock);
163 	cmi->cmi_modp->mod_ref++;
164 	mutex_exit(&mod_lock);
165 	cmi->cmi_refcnt++;
166 }
167 
168 static void
cmi_rele(cmi_t * cmi)169 cmi_rele(cmi_t *cmi)
170 {
171 	ASSERT(MUTEX_HELD(&cmi_load_lock));
172 
173 	mutex_enter(&mod_lock);
174 	cmi->cmi_modp->mod_ref--;
175 	mutex_exit(&mod_lock);
176 
177 	if (--cmi->cmi_refcnt == 0) {
178 		cmi_unlink(cmi);
179 		kmem_free(cmi, sizeof (cmi_t));
180 	}
181 }
182 
183 static cmi_ops_t *
cmi_getops(modctl_t * modp)184 cmi_getops(modctl_t *modp)
185 {
186 	cmi_ops_t *ops;
187 
188 	if ((ops = (cmi_ops_t *)modlookup_by_modctl(modp, "_cmi_ops")) ==
189 	    NULL) {
190 		cmn_err(CE_WARN, "cpu module '%s' is invalid: no _cmi_ops "
191 		    "found", modp->mod_modname);
192 		return (NULL);
193 	}
194 
195 	if (ops->cmi_init == NULL) {
196 		cmn_err(CE_WARN, "cpu module '%s' is invalid: no cmi_init "
197 		    "entry point", modp->mod_modname);
198 		return (NULL);
199 	}
200 
201 	return (ops);
202 }
203 
204 static cmi_t *
cmi_load_modctl(modctl_t * modp)205 cmi_load_modctl(modctl_t *modp)
206 {
207 	cmi_ops_t *ops;
208 	uintptr_t ver;
209 	cmi_t *cmi;
210 	cmi_api_ver_t apiver;
211 
212 	ASSERT(MUTEX_HELD(&cmi_load_lock));
213 
214 	for (cmi = cmi_list; cmi != NULL; cmi = cmi->cmi_next) {
215 		if (cmi->cmi_modp == modp)
216 			return (cmi);
217 	}
218 
219 	if ((ver = modlookup_by_modctl(modp, "_cmi_api_version")) == 0) {
220 		/*
221 		 * Apparently a cpu module before versioning was introduced -
222 		 * we call this version 0.
223 		 */
224 		apiver = CMI_API_VERSION_0;
225 	} else {
226 		apiver = *((cmi_api_ver_t *)ver);
227 		if (!CMI_API_VERSION_CHKMAGIC(apiver)) {
228 			cmn_err(CE_WARN, "cpu module '%s' is invalid: "
229 			    "_cmi_api_version 0x%x has bad magic",
230 			    modp->mod_modname, apiver);
231 			return (NULL);
232 		}
233 	}
234 
235 	if (apiver != CMI_API_VERSION) {
236 		cmn_err(CE_WARN, "cpu module '%s' has API version %d, "
237 		    "kernel requires API version %d", modp->mod_modname,
238 		    CMI_API_VERSION_TOPRINT(apiver),
239 		    CMI_API_VERSION_TOPRINT(CMI_API_VERSION));
240 		return (NULL);
241 	}
242 
243 	if ((ops = cmi_getops(modp)) == NULL)
244 		return (NULL);
245 
246 	cmi = kmem_zalloc(sizeof (*cmi), KM_SLEEP);
247 	cmi->cmi_ops = ops;
248 	cmi->cmi_modp = modp;
249 
250 	cmi_link(cmi);
251 
252 	return (cmi);
253 }
254 
255 static int
cmi_cpu_match(cmi_hdl_t hdl1,cmi_hdl_t hdl2,int match)256 cmi_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match)
257 {
258 	if (match >= CMI_MATCH_VENDOR &&
259 	    cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2))
260 		return (0);
261 
262 	if (match >= CMI_MATCH_FAMILY &&
263 	    cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2))
264 		return (0);
265 
266 	if (match >= CMI_MATCH_MODEL &&
267 	    cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2))
268 		return (0);
269 
270 	if (match >= CMI_MATCH_STEPPING &&
271 	    cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2))
272 		return (0);
273 
274 	return (1);
275 }
276 
277 static int
cmi_search_list_cb(cmi_hdl_t whdl,void * arg1,void * arg2,void * arg3)278 cmi_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3)
279 {
280 	cmi_hdl_t thdl = (cmi_hdl_t)arg1;
281 	int match = *((int *)arg2);
282 	cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3;
283 
284 	if (cmi_cpu_match(thdl, whdl, match)) {
285 		cmi_hdl_hold(whdl);	/* short-term hold */
286 		*rsltp = whdl;
287 		return (CMI_HDL_WALK_DONE);
288 	} else {
289 		return (CMI_HDL_WALK_NEXT);
290 	}
291 }
292 
293 static cmi_t *
cmi_search_list(cmi_hdl_t hdl,int match)294 cmi_search_list(cmi_hdl_t hdl, int match)
295 {
296 	cmi_hdl_t dhdl = NULL;
297 	cmi_t *cmi = NULL;
298 
299 	ASSERT(MUTEX_HELD(&cmi_load_lock));
300 
301 	cmi_hdl_walk(cmi_search_list_cb, (void *)hdl, (void *)&match, &dhdl);
302 	if (dhdl) {
303 		cmi = HDL2CMI(dhdl);
304 		cmi_hdl_rele(dhdl);	/* held in cmi_search_list_cb */
305 	}
306 
307 	return (cmi);
308 }
309 
310 static cmi_t *
cmi_load_module(cmi_hdl_t hdl,int match,int * chosenp)311 cmi_load_module(cmi_hdl_t hdl, int match, int *chosenp)
312 {
313 	modctl_t *modp;
314 	cmi_t *cmi;
315 	int modid;
316 	uint_t s[3];
317 
318 	ASSERT(MUTEX_HELD(&cmi_load_lock));
319 	ASSERT(match == CMI_MATCH_STEPPING || match == CMI_MATCH_MODEL ||
320 	    match == CMI_MATCH_FAMILY || match == CMI_MATCH_VENDOR);
321 
322 	/*
323 	 * Have we already loaded a module for a cpu with the same
324 	 * vendor/family/model/stepping?
325 	 */
326 	if ((cmi = cmi_search_list(hdl, match)) != NULL) {
327 		cmi_hold(cmi);
328 		return (cmi);
329 	}
330 
331 	s[0] = cmi_hdl_family(hdl);
332 	s[1] = cmi_hdl_model(hdl);
333 	s[2] = cmi_hdl_stepping(hdl);
334 	modid = modload_qualified(CPUMOD_SUBDIR, CPUMOD_PREFIX,
335 	    cmi_hdl_vendorstr(hdl), ".", s, match, chosenp);
336 
337 	if (modid == -1)
338 		return (NULL);
339 
340 	modp = mod_hold_by_id(modid);
341 	cmi = cmi_load_modctl(modp);
342 	if (cmi)
343 		cmi_hold(cmi);
344 	mod_release_mod(modp);
345 
346 	return (cmi);
347 }
348 
349 /*
350  * Try to load a cpu module with specific support for this chip type.
351  */
352 static cmi_t *
cmi_load_specific(cmi_hdl_t hdl,void ** datap)353 cmi_load_specific(cmi_hdl_t hdl, void **datap)
354 {
355 	cmi_t *cmi;
356 	int err;
357 	int i;
358 
359 	ASSERT(MUTEX_HELD(&cmi_load_lock));
360 
361 	for (i = CMI_MATCH_STEPPING; i >= CMI_MATCH_VENDOR; i--) {
362 		int suffixlevel;
363 
364 		if ((cmi = cmi_load_module(hdl, i, &suffixlevel)) == NULL)
365 			return (NULL);
366 
367 		/*
368 		 * A module has loaded and has a _cmi_ops structure, and the
369 		 * module has been held for this instance.  Call its cmi_init
370 		 * entry point - we expect success (0) or ENOTSUP.
371 		 */
372 		if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) == 0) {
373 			if (boothowto & RB_VERBOSE) {
374 				printf("initialized cpu module '%s' on "
375 				    "chip %d core %d strand %d\n",
376 				    cmi->cmi_modp->mod_modname,
377 				    cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
378 				    cmi_hdl_strandid(hdl));
379 			}
380 			return (cmi);
381 		} else if (err != ENOTSUP) {
382 			cmn_err(CE_WARN, "failed to init cpu module '%s' on "
383 			    "chip %d core %d strand %d: err=%d\n",
384 			    cmi->cmi_modp->mod_modname,
385 			    cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
386 			    cmi_hdl_strandid(hdl), err);
387 		}
388 
389 		/*
390 		 * The module failed or declined to init, so release
391 		 * it and update i to be equal to the number
392 		 * of suffices actually used in the last module path.
393 		 */
394 		cmi_rele(cmi);
395 		i = suffixlevel;
396 	}
397 
398 	return (NULL);
399 }
400 
401 /*
402  * Load the generic IA32 MCA cpu module, which may still supplement
403  * itself with model-specific support through cpu model-specific modules.
404  */
405 static cmi_t *
cmi_load_generic(cmi_hdl_t hdl,void ** datap)406 cmi_load_generic(cmi_hdl_t hdl, void **datap)
407 {
408 	modctl_t *modp;
409 	cmi_t *cmi;
410 	int modid;
411 	int err;
412 
413 	ASSERT(MUTEX_HELD(&cmi_load_lock));
414 
415 	if ((modid = modload(CPUMOD_SUBDIR, CPUMOD_PREFIX ".generic")) == -1)
416 		return (NULL);
417 
418 	modp = mod_hold_by_id(modid);
419 	cmi = cmi_load_modctl(modp);
420 	if (cmi)
421 		cmi_hold(cmi);
422 	mod_release_mod(modp);
423 
424 	if (cmi == NULL)
425 		return (NULL);
426 
427 	if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) != 0) {
428 		if (err != ENOTSUP)
429 			cmn_err(CE_WARN, CPUMOD_PREFIX ".generic failed to "
430 			    "init: err=%d", err);
431 		cmi_rele(cmi);
432 		return (NULL);
433 	}
434 
435 	return (cmi);
436 }
437 
438 cmi_hdl_t
cmi_init(enum cmi_hdl_class class,uint_t chipid,uint_t coreid,uint_t strandid)439 cmi_init(enum cmi_hdl_class class, uint_t chipid, uint_t coreid,
440     uint_t strandid)
441 {
442 	cmi_t *cmi = NULL;
443 	cmi_hdl_t hdl;
444 	void *data;
445 
446 	if (cmi_no_init) {
447 		cmi_no_mca_init = 1;
448 		return (NULL);
449 	}
450 
451 	mutex_enter(&cmi_load_lock);
452 
453 	if ((hdl = cmi_hdl_create(class, chipid, coreid, strandid)) == NULL) {
454 		mutex_exit(&cmi_load_lock);
455 		cmn_err(CE_WARN, "There will be no MCA support on chip %d "
456 		    "core %d strand %d (cmi_hdl_create returned NULL)\n",
457 		    chipid, coreid, strandid);
458 		return (NULL);
459 	}
460 
461 	if (!cmi_force_generic)
462 		cmi = cmi_load_specific(hdl, &data);
463 
464 	if (cmi == NULL && (cmi = cmi_load_generic(hdl, &data)) == NULL) {
465 		cmn_err(CE_WARN, "There will be no MCA support on chip %d "
466 		    "core %d strand %d\n", chipid, coreid, strandid);
467 		cmi_hdl_rele(hdl);
468 		mutex_exit(&cmi_load_lock);
469 		return (NULL);
470 	}
471 
472 	cmi_hdl_setcmi(hdl, cmi, data);
473 
474 	cms_init(hdl);
475 
476 	cmi_read_smbios(hdl);
477 
478 	mutex_exit(&cmi_load_lock);
479 
480 	return (hdl);
481 }
482 
483 /*
484  * cmi_fini is called on DR deconfigure of a cpu resource.
485  * It should not be called at simple offline of a cpu.
486  */
487 void
cmi_fini(cmi_hdl_t hdl)488 cmi_fini(cmi_hdl_t hdl)
489 {
490 	cmi_t *cmi = HDL2CMI(hdl);
491 
492 	if (cms_present(hdl))
493 		cms_fini(hdl);
494 
495 	if (CMI_OP_PRESENT(cmi, cmi_fini))
496 		CMI_OPS(cmi)->cmi_fini(hdl);
497 
498 	cmi_hdl_destroy(hdl);
499 }
500 
501 /*
502  * cmi_post_startup is called from post_startup for the boot cpu only (no
503  * other cpus are started yet).
504  */
505 void
cmi_post_startup(void)506 cmi_post_startup(void)
507 {
508 	cmi_hdl_t hdl;
509 	cmi_t *cmi;
510 
511 	if (cmi_no_mca_init != 0 ||
512 	    (hdl = cmi_hdl_any()) == NULL)	/* short-term hold */
513 		return;
514 
515 	cmi = HDL2CMI(hdl);
516 
517 	if (CMI_OP_PRESENT(cmi, cmi_post_startup))
518 		CMI_OPS(cmi)->cmi_post_startup(hdl);
519 
520 	cmi_hdl_rele(hdl);
521 }
522 
523 /*
524  * Called just once from start_other_cpus when all processors are started.
525  * This will not be called for each cpu, so the registered op must not
526  * assume it is called as such.  We are not necessarily executing on
527  * the boot cpu.
528  */
529 void
cmi_post_mpstartup(void)530 cmi_post_mpstartup(void)
531 {
532 	cmi_hdl_t hdl;
533 	cmi_t *cmi;
534 
535 	if (cmi_no_mca_init != 0 ||
536 	    (hdl = cmi_hdl_any()) == NULL)	/* short-term hold */
537 		return;
538 
539 	cmi = HDL2CMI(hdl);
540 
541 	if (CMI_OP_PRESENT(cmi, cmi_post_mpstartup))
542 		CMI_OPS(cmi)->cmi_post_mpstartup(hdl);
543 
544 	cmi_hdl_rele(hdl);
545 }
546 
547 void
cmi_faulted_enter(cmi_hdl_t hdl)548 cmi_faulted_enter(cmi_hdl_t hdl)
549 {
550 	cmi_t *cmi = HDL2CMI(hdl);
551 
552 	if (cmi_no_mca_init != 0)
553 		return;
554 
555 	if (CMI_OP_PRESENT(cmi, cmi_faulted_enter))
556 		CMI_OPS(cmi)->cmi_faulted_enter(hdl);
557 }
558 
559 void
cmi_faulted_exit(cmi_hdl_t hdl)560 cmi_faulted_exit(cmi_hdl_t hdl)
561 {
562 	cmi_t *cmi = HDL2CMI(hdl);
563 
564 	if (cmi_no_mca_init != 0)
565 		return;
566 
567 	if (CMI_OP_PRESENT(cmi, cmi_faulted_exit))
568 		CMI_OPS(cmi)->cmi_faulted_exit(hdl);
569 }
570 
571 void
cmi_mca_init(cmi_hdl_t hdl)572 cmi_mca_init(cmi_hdl_t hdl)
573 {
574 	cmi_t *cmi;
575 
576 	if (cmi_no_mca_init != 0)
577 		return;
578 
579 	cmi = HDL2CMI(hdl);
580 
581 	if (CMI_OP_PRESENT(cmi, cmi_mca_init))
582 		CMI_OPS(cmi)->cmi_mca_init(hdl);
583 }
584 
585 #define	CMI_RESPONSE_PANIC		0x0	/* panic must have value 0 */
586 #define	CMI_RESPONSE_NONE		0x1
587 #define	CMI_RESPONSE_CKILL		0x2
588 #define	CMI_RESPONSE_REBOOT		0x3	/* not implemented */
589 #define	CMI_RESPONSE_ONTRAP_PROT	0x4
590 #define	CMI_RESPONSE_LOFAULT_PROT	0x5
591 
592 /*
593  * Return 0 if we will panic in response to this machine check, otherwise
594  * non-zero.  If the caller is cmi_mca_trap in this file then the nonzero
595  * return values are to be interpreted from CMI_RESPONSE_* above.
596  *
597  * This function must just return what will be done without actually
598  * doing anything; this includes not changing the regs.
599  */
600 int
cmi_mce_response(struct regs * rp,uint64_t disp)601 cmi_mce_response(struct regs *rp, uint64_t disp)
602 {
603 	int panicrsp = cmi_panic_on_uncorrectable_error ? CMI_RESPONSE_PANIC :
604 	    CMI_RESPONSE_NONE;
605 	on_trap_data_t *otp;
606 
607 	ASSERT(rp != NULL);	/* don't call for polling, only on #MC */
608 
609 	/*
610 	 * If no bits are set in the disposition then there is nothing to
611 	 * worry about and we do not need to trampoline to ontrap or
612 	 * lofault handlers.
613 	 */
614 	if (disp == 0)
615 		return (CMI_RESPONSE_NONE);
616 
617 	/*
618 	 * Unconstrained errors cannot be forgiven, even by ontrap or
619 	 * lofault protection.  The data is not poisoned and may not
620 	 * even belong to the trapped context - eg a writeback of
621 	 * data that is found to be bad.
622 	 */
623 	if (disp & CMI_ERRDISP_UC_UNCONSTRAINED)
624 		return (panicrsp);
625 
626 	/*
627 	 * ontrap OT_DATA_EC and lofault protection forgive any disposition
628 	 * other than unconstrained, even those normally forced fatal.
629 	 */
630 	if ((otp = curthread->t_ontrap) != NULL && otp->ot_prot & OT_DATA_EC)
631 		return (CMI_RESPONSE_ONTRAP_PROT);
632 	else if (curthread->t_lofault)
633 		return (CMI_RESPONSE_LOFAULT_PROT);
634 
635 	/*
636 	 * Forced-fatal errors are terminal even in user mode.
637 	 */
638 	if (disp & CMI_ERRDISP_FORCEFATAL)
639 		return (panicrsp);
640 
641 	/*
642 	 * If the trapped context is corrupt or we have no instruction pointer
643 	 * to resume at (and aren't trampolining to a fault handler)
644 	 * then in the kernel case we must panic and in usermode we
645 	 * kill the affected contract.
646 	 */
647 	if (disp & (CMI_ERRDISP_CURCTXBAD | CMI_ERRDISP_RIPV_INVALID))
648 		return (USERMODE(rp->r_cs) ?  CMI_RESPONSE_CKILL : panicrsp);
649 
650 	/*
651 	 * Anything else is harmless
652 	 */
653 	return (CMI_RESPONSE_NONE);
654 }
655 
656 int cma_mca_trap_panic_suppressed = 0;
657 
658 static void
cmi_mca_panic(void)659 cmi_mca_panic(void)
660 {
661 	if (cmi_panic_on_uncorrectable_error) {
662 		fm_panic("Unrecoverable Machine-Check Exception");
663 	} else {
664 		cmn_err(CE_WARN, "suppressing panic from fatal #mc");
665 		cma_mca_trap_panic_suppressed++;
666 	}
667 }
668 
669 
670 int cma_mca_trap_contract_kills = 0;
671 int cma_mca_trap_ontrap_forgiven = 0;
672 int cma_mca_trap_lofault_forgiven = 0;
673 
674 /*
675  * Native #MC handler - we branch to here from mcetrap
676  */
677 /*ARGSUSED*/
678 void
cmi_mca_trap(struct regs * rp)679 cmi_mca_trap(struct regs *rp)
680 {
681 #ifndef	__xpv
682 	cmi_hdl_t hdl = NULL;
683 	uint64_t disp;
684 	cmi_t *cmi;
685 	int s;
686 
687 	if (cmi_no_mca_init != 0)
688 		return;
689 
690 	/*
691 	 * This function can call cmn_err, and the cpu module cmi_mca_trap
692 	 * entry point may also elect to call cmn_err (e.g., if it can't
693 	 * log the error onto an errorq, say very early in boot).
694 	 * We need to let cprintf know that we must not block.
695 	 */
696 	s = spl8();
697 
698 	if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
699 	    cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL ||
700 	    (cmi = HDL2CMI(hdl)) == NULL ||
701 	    !CMI_OP_PRESENT(cmi, cmi_mca_trap)) {
702 
703 		cmn_err(CE_WARN, "#MC exception on cpuid %d: %s",
704 		    CPU->cpu_id,
705 		    hdl ? "handle lookup ok but no #MC handler found" :
706 		    "handle lookup failed");
707 
708 		if (hdl != NULL)
709 			cmi_hdl_rele(hdl);
710 
711 		splx(s);
712 		return;
713 	}
714 
715 	disp = CMI_OPS(cmi)->cmi_mca_trap(hdl, rp);
716 
717 	switch (cmi_mce_response(rp, disp)) {
718 	default:
719 		cmn_err(CE_WARN, "Invalid response from cmi_mce_response");
720 		/*FALLTHRU*/
721 
722 	case CMI_RESPONSE_PANIC:
723 		cmi_mca_panic();
724 		break;
725 
726 	case CMI_RESPONSE_NONE:
727 		break;
728 
729 	case CMI_RESPONSE_CKILL:
730 		ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR;
731 		aston(curthread);
732 		cma_mca_trap_contract_kills++;
733 		break;
734 
735 	case CMI_RESPONSE_ONTRAP_PROT: {
736 		on_trap_data_t *otp = curthread->t_ontrap;
737 		otp->ot_trap = OT_DATA_EC;
738 		rp->r_pc = otp->ot_trampoline;
739 		cma_mca_trap_ontrap_forgiven++;
740 		break;
741 	}
742 
743 	case CMI_RESPONSE_LOFAULT_PROT:
744 		rp->r_r0 = EFAULT;
745 		rp->r_pc = curthread->t_lofault;
746 		cma_mca_trap_lofault_forgiven++;
747 		break;
748 	}
749 
750 	cmi_hdl_rele(hdl);
751 	splx(s);
752 #endif	/* __xpv */
753 }
754 
755 void
cmi_hdl_poke(cmi_hdl_t hdl)756 cmi_hdl_poke(cmi_hdl_t hdl)
757 {
758 	cmi_t *cmi = HDL2CMI(hdl);
759 
760 	if (!CMI_OP_PRESENT(cmi, cmi_hdl_poke))
761 		return;
762 
763 	CMI_OPS(cmi)->cmi_hdl_poke(hdl);
764 }
765 
766 #ifndef	__xpv
767 void
cmi_cmci_trap()768 cmi_cmci_trap()
769 {
770 	cmi_hdl_t hdl = NULL;
771 	cmi_t *cmi;
772 
773 	if (cmi_no_mca_init != 0)
774 		return;
775 
776 	if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
777 	    cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL ||
778 	    (cmi = HDL2CMI(hdl)) == NULL ||
779 	    !CMI_OP_PRESENT(cmi, cmi_cmci_trap)) {
780 
781 		cmn_err(CE_WARN, "CMCI interrupt on cpuid %d: %s",
782 		    CPU->cpu_id,
783 		    hdl ? "handle lookup ok but no CMCI handler found" :
784 		    "handle lookup failed");
785 
786 		if (hdl != NULL)
787 			cmi_hdl_rele(hdl);
788 
789 		return;
790 	}
791 
792 	CMI_OPS(cmi)->cmi_cmci_trap(hdl);
793 
794 	cmi_hdl_rele(hdl);
795 }
796 #endif	/* __xpv */
797 
798 void
cmi_mc_register(cmi_hdl_t hdl,const cmi_mc_ops_t * mcops,void * mcdata)799 cmi_mc_register(cmi_hdl_t hdl, const cmi_mc_ops_t *mcops, void *mcdata)
800 {
801 	if (!cmi_no_mca_init)
802 		cmi_hdl_setmc(hdl, mcops, mcdata);
803 }
804 
805 cmi_errno_t
cmi_mc_register_global(const cmi_mc_ops_t * mcops,void * mcdata)806 cmi_mc_register_global(const cmi_mc_ops_t *mcops, void *mcdata)
807 {
808 	if (!cmi_no_mca_init) {
809 		if (cmi_mc_global_ops != NULL || cmi_mc_global_data != NULL ||
810 		    mcops == NULL || mcops->cmi_mc_patounum == NULL ||
811 		    mcops->cmi_mc_unumtopa == NULL) {
812 			return (CMIERR_UNKNOWN);
813 		}
814 		cmi_mc_global_data = mcdata;
815 		cmi_mc_global_ops = mcops;
816 	}
817 	return (CMI_SUCCESS);
818 }
819 
820 void
cmi_mc_sw_memscrub_disable(void)821 cmi_mc_sw_memscrub_disable(void)
822 {
823 	memscrub_disable();
824 }
825 
826 cmi_errno_t
cmi_mc_patounum(uint64_t pa,uint8_t valid_hi,uint8_t valid_lo,uint32_t synd,int syndtype,mc_unum_t * up)827 cmi_mc_patounum(uint64_t pa, uint8_t valid_hi, uint8_t valid_lo, uint32_t synd,
828     int syndtype, mc_unum_t *up)
829 {
830 	const struct cmi_mc_ops *mcops;
831 	cmi_hdl_t hdl;
832 	cmi_errno_t rv;
833 
834 	if (cmi_no_mca_init)
835 		return (CMIERR_MC_ABSENT);
836 
837 	if (cmi_mc_global_ops != NULL) {
838 		if (cmi_mc_global_ops->cmi_mc_patounum == NULL)
839 			return (CMIERR_MC_NOTSUP);
840 		return (cmi_mc_global_ops->cmi_mc_patounum(cmi_mc_global_data,
841 		    pa, valid_hi, valid_lo, synd, syndtype, up));
842 	}
843 
844 	if ((hdl = cmi_hdl_any()) == NULL)	/* short-term hold */
845 		return (CMIERR_MC_ABSENT);
846 
847 	if ((mcops = cmi_hdl_getmcops(hdl)) == NULL ||
848 	    mcops->cmi_mc_patounum == NULL) {
849 		cmi_hdl_rele(hdl);
850 		return (CMIERR_MC_NOTSUP);
851 	}
852 
853 	rv = mcops->cmi_mc_patounum(cmi_hdl_getmcdata(hdl), pa, valid_hi,
854 	    valid_lo, synd, syndtype, up);
855 
856 	cmi_hdl_rele(hdl);
857 
858 	return (rv);
859 }
860 
861 cmi_errno_t
cmi_mc_unumtopa(mc_unum_t * up,nvlist_t * nvl,uint64_t * pap)862 cmi_mc_unumtopa(mc_unum_t *up, nvlist_t *nvl, uint64_t *pap)
863 {
864 	const struct cmi_mc_ops *mcops;
865 	cmi_hdl_t hdl;
866 	cmi_errno_t rv;
867 	nvlist_t *hcsp;
868 
869 	if (up != NULL && nvl != NULL)
870 		return (CMIERR_API);	/* convert from just one form */
871 
872 	if (cmi_no_mca_init)
873 		return (CMIERR_MC_ABSENT);
874 
875 	if (cmi_mc_global_ops != NULL) {
876 		if (cmi_mc_global_ops->cmi_mc_unumtopa == NULL)
877 			return (CMIERR_MC_NOTSUP);
878 		return (cmi_mc_global_ops->cmi_mc_unumtopa(cmi_mc_global_data,
879 		    up, nvl, pap));
880 	}
881 
882 	if ((hdl = cmi_hdl_any()) == NULL)	/* short-term hold */
883 		return (CMIERR_MC_ABSENT);
884 
885 	if ((mcops = cmi_hdl_getmcops(hdl)) == NULL ||
886 	    mcops->cmi_mc_unumtopa == NULL) {
887 		cmi_hdl_rele(hdl);
888 
889 		if (nvl != NULL && nvlist_lookup_nvlist(nvl,
890 		    FM_FMRI_HC_SPECIFIC, &hcsp) == 0 &&
891 		    (nvlist_lookup_uint64(hcsp,
892 		    "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR, pap) == 0 ||
893 		    nvlist_lookup_uint64(hcsp, FM_FMRI_HC_SPECIFIC_PHYSADDR,
894 		    pap) == 0)) {
895 			return (CMIERR_MC_PARTIALUNUMTOPA);
896 		} else {
897 			return (mcops && mcops->cmi_mc_unumtopa == NULL ?
898 			    CMIERR_MC_NOTSUP : CMIERR_MC_ABSENT);
899 		}
900 	}
901 
902 	rv = mcops->cmi_mc_unumtopa(cmi_hdl_getmcdata(hdl), up, nvl, pap);
903 
904 	cmi_hdl_rele(hdl);
905 
906 	return (rv);
907 }
908 
909 void
cmi_mc_logout(cmi_hdl_t hdl,boolean_t ismc,boolean_t sync)910 cmi_mc_logout(cmi_hdl_t hdl, boolean_t ismc, boolean_t sync)
911 {
912 	const struct cmi_mc_ops *mcops;
913 
914 	if (cmi_no_mca_init)
915 		return;
916 
917 	if (cmi_mc_global_ops != NULL)
918 		mcops = cmi_mc_global_ops;
919 	else
920 		mcops = cmi_hdl_getmcops(hdl);
921 
922 	if (mcops != NULL && mcops->cmi_mc_logout != NULL)
923 		mcops->cmi_mc_logout(hdl, ismc, sync);
924 }
925 
926 cmi_errno_t
cmi_hdl_msrinject(cmi_hdl_t hdl,cmi_mca_regs_t * regs,uint_t nregs,int force)927 cmi_hdl_msrinject(cmi_hdl_t hdl, cmi_mca_regs_t *regs, uint_t nregs,
928     int force)
929 {
930 	cmi_t *cmi = cmi_hdl_getcmi(hdl);
931 	cmi_errno_t rc;
932 
933 	if (!CMI_OP_PRESENT(cmi, cmi_msrinject))
934 		return (CMIERR_NOTSUP);
935 
936 	cmi_hdl_inj_begin(hdl);
937 	rc = CMI_OPS(cmi)->cmi_msrinject(hdl, regs, nregs, force);
938 	cmi_hdl_inj_end(hdl);
939 
940 	return (rc);
941 }
942 
943 boolean_t
cmi_panic_on_ue(void)944 cmi_panic_on_ue(void)
945 {
946 	return (cmi_panic_on_uncorrectable_error ? B_TRUE : B_FALSE);
947 }
948 
949 void
cmi_panic_callback(void)950 cmi_panic_callback(void)
951 {
952 	cmi_hdl_t hdl;
953 	cmi_t *cmi;
954 
955 	if (cmi_no_mca_init || (hdl = cmi_hdl_any()) == NULL)
956 		return;
957 
958 	cmi = cmi_hdl_getcmi(hdl);
959 	if (CMI_OP_PRESENT(cmi, cmi_panic_callback))
960 		CMI_OPS(cmi)->cmi_panic_callback();
961 
962 	cmi_hdl_rele(hdl);
963 }
964 
965 
966 const char *
cmi_hdl_chipident(cmi_hdl_t hdl)967 cmi_hdl_chipident(cmi_hdl_t hdl)
968 {
969 	cmi_t *cmi = cmi_hdl_getcmi(hdl);
970 
971 	if (!CMI_OP_PRESENT(cmi, cmi_ident))
972 		return (NULL);
973 
974 	return (CMI_OPS(cmi)->cmi_ident(hdl));
975 }
976