1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 * Copyright (c) 2018, Joyent, Inc.
26 */
27
28/*
29 * Public interface to routines implemented by CPU modules
30 */
31
32#include <sys/types.h>
33#include <sys/atomic.h>
34#include <sys/x86_archext.h>
35#include <sys/cpu_module_impl.h>
36#include <sys/cpu_module_ms.h>
37#include <sys/fm/util.h>
38#include <sys/reboot.h>
39#include <sys/modctl.h>
40#include <sys/param.h>
41#include <sys/cmn_err.h>
42#include <sys/systm.h>
43#include <sys/fm/protocol.h>
44#include <sys/pcb.h>
45#include <sys/ontrap.h>
46#include <sys/psw.h>
47#include <sys/privregs.h>
48#include <sys/machsystm.h>
49
50/*
51 * Set to force cmi_init to fail.
52 */
53int cmi_no_init = 0;
54
55/*
56 * Set to avoid MCA initialization.
57 */
58int cmi_no_mca_init = 0;
59
60/*
61 * If cleared for debugging we will not attempt to load a model-specific
62 * cpu module but will load the generic cpu module instead.
63 */
64int cmi_force_generic = 0;
65
66/*
67 * If cleared for debugging, we will suppress panicking on fatal hardware
68 * errors.  This should *only* be used for debugging; it use can and will
69 * cause data corruption if actual hardware errors are detected by the system.
70 */
71int cmi_panic_on_uncorrectable_error = 1;
72
73/*
74 * Subdirectory (relative to the module search path) in which we will
75 * look for cpu modules.
76 */
77#define	CPUMOD_SUBDIR	"cpu"
78
79/*
80 * CPU modules have a filenames such as "cpu.AuthenticAMD.15" and
81 * "cpu.generic" - the "cpu" prefix is specified by the following.
82 */
83#define	CPUMOD_PREFIX	"cpu"
84
85/*
86 * Structure used to keep track of cpu modules we have loaded and their ops
87 */
88typedef struct cmi {
89	struct cmi *cmi_next;
90	struct cmi *cmi_prev;
91	const cmi_ops_t *cmi_ops;
92	struct modctl *cmi_modp;
93	uint_t cmi_refcnt;
94} cmi_t;
95
96static cmi_t *cmi_list;
97static const cmi_mc_ops_t *cmi_mc_global_ops;
98static void *cmi_mc_global_data;
99static kmutex_t cmi_load_lock;
100
101/*
102 * Functions we need from cmi_hw.c that are not part of the cpu_module.h
103 * interface.
104 */
105extern cmi_hdl_t cmi_hdl_create(enum cmi_hdl_class, uint_t, uint_t, uint_t);
106extern void cmi_hdl_destroy(cmi_hdl_t ophdl);
107extern void cmi_hdl_setcmi(cmi_hdl_t, void *, void *);
108extern void *cmi_hdl_getcmi(cmi_hdl_t);
109extern void cmi_hdl_setmc(cmi_hdl_t, const struct cmi_mc_ops *, void *);
110extern void cmi_hdl_inj_begin(cmi_hdl_t);
111extern void cmi_hdl_inj_end(cmi_hdl_t);
112extern void cmi_read_smbios(cmi_hdl_t);
113
114#define	HDL2CMI(hdl)		cmi_hdl_getcmi(hdl)
115
116#define	CMI_OPS(cmi)		(cmi)->cmi_ops
117#define	CMI_OP_PRESENT(cmi, op)	((cmi) && CMI_OPS(cmi)->op != NULL)
118
119#define	CMI_MATCH_VENDOR	0	/* Just match on vendor */
120#define	CMI_MATCH_FAMILY	1	/* Match down to family */
121#define	CMI_MATCH_MODEL		2	/* Match down to model */
122#define	CMI_MATCH_STEPPING	3	/* Match down to stepping */
123
124static void
125cmi_link(cmi_t *cmi)
126{
127	ASSERT(MUTEX_HELD(&cmi_load_lock));
128
129	cmi->cmi_prev = NULL;
130	cmi->cmi_next = cmi_list;
131	if (cmi_list != NULL)
132		cmi_list->cmi_prev = cmi;
133	cmi_list = cmi;
134}
135
136static void
137cmi_unlink(cmi_t *cmi)
138{
139	ASSERT(MUTEX_HELD(&cmi_load_lock));
140	ASSERT(cmi->cmi_refcnt == 0);
141
142	if (cmi->cmi_prev != NULL)
143		cmi->cmi_prev = cmi->cmi_next;
144
145	if (cmi->cmi_next != NULL)
146		cmi->cmi_next->cmi_prev = cmi->cmi_prev;
147
148	if (cmi_list == cmi)
149		cmi_list = cmi->cmi_next;
150}
151
152/*
153 * Hold the module in memory.  We call to CPU modules without using the
154 * stubs mechanism, so these modules must be manually held in memory.
155 * The mod_ref acts as if another loaded module has a dependency on us.
156 */
157static void
158cmi_hold(cmi_t *cmi)
159{
160	ASSERT(MUTEX_HELD(&cmi_load_lock));
161
162	mutex_enter(&mod_lock);
163	cmi->cmi_modp->mod_ref++;
164	mutex_exit(&mod_lock);
165	cmi->cmi_refcnt++;
166}
167
168static void
169cmi_rele(cmi_t *cmi)
170{
171	ASSERT(MUTEX_HELD(&cmi_load_lock));
172
173	mutex_enter(&mod_lock);
174	cmi->cmi_modp->mod_ref--;
175	mutex_exit(&mod_lock);
176
177	if (--cmi->cmi_refcnt == 0) {
178		cmi_unlink(cmi);
179		kmem_free(cmi, sizeof (cmi_t));
180	}
181}
182
183static cmi_ops_t *
184cmi_getops(modctl_t *modp)
185{
186	cmi_ops_t *ops;
187
188	if ((ops = (cmi_ops_t *)modlookup_by_modctl(modp, "_cmi_ops")) ==
189	    NULL) {
190		cmn_err(CE_WARN, "cpu module '%s' is invalid: no _cmi_ops "
191		    "found", modp->mod_modname);
192		return (NULL);
193	}
194
195	if (ops->cmi_init == NULL) {
196		cmn_err(CE_WARN, "cpu module '%s' is invalid: no cmi_init "
197		    "entry point", modp->mod_modname);
198		return (NULL);
199	}
200
201	return (ops);
202}
203
204static cmi_t *
205cmi_load_modctl(modctl_t *modp)
206{
207	cmi_ops_t *ops;
208	uintptr_t ver;
209	cmi_t *cmi;
210	cmi_api_ver_t apiver;
211
212	ASSERT(MUTEX_HELD(&cmi_load_lock));
213
214	for (cmi = cmi_list; cmi != NULL; cmi = cmi->cmi_next) {
215		if (cmi->cmi_modp == modp)
216			return (cmi);
217	}
218
219	if ((ver = modlookup_by_modctl(modp, "_cmi_api_version")) == 0) {
220		/*
221		 * Apparently a cpu module before versioning was introduced -
222		 * we call this version 0.
223		 */
224		apiver = CMI_API_VERSION_0;
225	} else {
226		apiver = *((cmi_api_ver_t *)ver);
227		if (!CMI_API_VERSION_CHKMAGIC(apiver)) {
228			cmn_err(CE_WARN, "cpu module '%s' is invalid: "
229			    "_cmi_api_version 0x%x has bad magic",
230			    modp->mod_modname, apiver);
231			return (NULL);
232		}
233	}
234
235	if (apiver != CMI_API_VERSION) {
236		cmn_err(CE_WARN, "cpu module '%s' has API version %d, "
237		    "kernel requires API version %d", modp->mod_modname,
238		    CMI_API_VERSION_TOPRINT(apiver),
239		    CMI_API_VERSION_TOPRINT(CMI_API_VERSION));
240		return (NULL);
241	}
242
243	if ((ops = cmi_getops(modp)) == NULL)
244		return (NULL);
245
246	cmi = kmem_zalloc(sizeof (*cmi), KM_SLEEP);
247	cmi->cmi_ops = ops;
248	cmi->cmi_modp = modp;
249
250	cmi_link(cmi);
251
252	return (cmi);
253}
254
255static int
256cmi_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match)
257{
258	if (match >= CMI_MATCH_VENDOR &&
259	    cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2))
260		return (0);
261
262	if (match >= CMI_MATCH_FAMILY &&
263	    cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2))
264		return (0);
265
266	if (match >= CMI_MATCH_MODEL &&
267	    cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2))
268		return (0);
269
270	if (match >= CMI_MATCH_STEPPING &&
271	    cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2))
272		return (0);
273
274	return (1);
275}
276
277static int
278cmi_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3)
279{
280	cmi_hdl_t thdl = (cmi_hdl_t)arg1;
281	int match = *((int *)arg2);
282	cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3;
283
284	if (cmi_cpu_match(thdl, whdl, match)) {
285		cmi_hdl_hold(whdl);	/* short-term hold */
286		*rsltp = whdl;
287		return (CMI_HDL_WALK_DONE);
288	} else {
289		return (CMI_HDL_WALK_NEXT);
290	}
291}
292
293static cmi_t *
294cmi_search_list(cmi_hdl_t hdl, int match)
295{
296	cmi_hdl_t dhdl = NULL;
297	cmi_t *cmi = NULL;
298
299	ASSERT(MUTEX_HELD(&cmi_load_lock));
300
301	cmi_hdl_walk(cmi_search_list_cb, (void *)hdl, (void *)&match, &dhdl);
302	if (dhdl) {
303		cmi = HDL2CMI(dhdl);
304		cmi_hdl_rele(dhdl);	/* held in cmi_search_list_cb */
305	}
306
307	return (cmi);
308}
309
310static cmi_t *
311cmi_load_module(cmi_hdl_t hdl, int match, int *chosenp)
312{
313	modctl_t *modp;
314	cmi_t *cmi;
315	int modid;
316	uint_t s[3];
317
318	ASSERT(MUTEX_HELD(&cmi_load_lock));
319	ASSERT(match == CMI_MATCH_STEPPING || match == CMI_MATCH_MODEL ||
320	    match == CMI_MATCH_FAMILY || match == CMI_MATCH_VENDOR);
321
322	/*
323	 * Have we already loaded a module for a cpu with the same
324	 * vendor/family/model/stepping?
325	 */
326	if ((cmi = cmi_search_list(hdl, match)) != NULL) {
327		cmi_hold(cmi);
328		return (cmi);
329	}
330
331	s[0] = cmi_hdl_family(hdl);
332	s[1] = cmi_hdl_model(hdl);
333	s[2] = cmi_hdl_stepping(hdl);
334	modid = modload_qualified(CPUMOD_SUBDIR, CPUMOD_PREFIX,
335	    cmi_hdl_vendorstr(hdl), ".", s, match, chosenp);
336
337	if (modid == -1)
338		return (NULL);
339
340	modp = mod_hold_by_id(modid);
341	cmi = cmi_load_modctl(modp);
342	if (cmi)
343		cmi_hold(cmi);
344	mod_release_mod(modp);
345
346	return (cmi);
347}
348
349/*
350 * Try to load a cpu module with specific support for this chip type.
351 */
352static cmi_t *
353cmi_load_specific(cmi_hdl_t hdl, void **datap)
354{
355	cmi_t *cmi;
356	int err;
357	int i;
358
359	ASSERT(MUTEX_HELD(&cmi_load_lock));
360
361	for (i = CMI_MATCH_STEPPING; i >= CMI_MATCH_VENDOR; i--) {
362		int suffixlevel;
363
364		if ((cmi = cmi_load_module(hdl, i, &suffixlevel)) == NULL)
365			return (NULL);
366
367		/*
368		 * A module has loaded and has a _cmi_ops structure, and the
369		 * module has been held for this instance.  Call its cmi_init
370		 * entry point - we expect success (0) or ENOTSUP.
371		 */
372		if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) == 0) {
373			if (boothowto & RB_VERBOSE) {
374				printf("initialized cpu module '%s' on "
375				    "chip %d core %d strand %d\n",
376				    cmi->cmi_modp->mod_modname,
377				    cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
378				    cmi_hdl_strandid(hdl));
379			}
380			return (cmi);
381		} else if (err != ENOTSUP) {
382			cmn_err(CE_WARN, "failed to init cpu module '%s' on "
383			    "chip %d core %d strand %d: err=%d\n",
384			    cmi->cmi_modp->mod_modname,
385			    cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
386			    cmi_hdl_strandid(hdl), err);
387		}
388
389		/*
390		 * The module failed or declined to init, so release
391		 * it and update i to be equal to the number
392		 * of suffices actually used in the last module path.
393		 */
394		cmi_rele(cmi);
395		i = suffixlevel;
396	}
397
398	return (NULL);
399}
400
401/*
402 * Load the generic IA32 MCA cpu module, which may still supplement
403 * itself with model-specific support through cpu model-specific modules.
404 */
405static cmi_t *
406cmi_load_generic(cmi_hdl_t hdl, void **datap)
407{
408	modctl_t *modp;
409	cmi_t *cmi;
410	int modid;
411	int err;
412
413	ASSERT(MUTEX_HELD(&cmi_load_lock));
414
415	if ((modid = modload(CPUMOD_SUBDIR, CPUMOD_PREFIX ".generic")) == -1)
416		return (NULL);
417
418	modp = mod_hold_by_id(modid);
419	cmi = cmi_load_modctl(modp);
420	if (cmi)
421		cmi_hold(cmi);
422	mod_release_mod(modp);
423
424	if (cmi == NULL)
425		return (NULL);
426
427	if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) != 0) {
428		if (err != ENOTSUP)
429			cmn_err(CE_WARN, CPUMOD_PREFIX ".generic failed to "
430			    "init: err=%d", err);
431		cmi_rele(cmi);
432		return (NULL);
433	}
434
435	return (cmi);
436}
437
438cmi_hdl_t
439cmi_init(enum cmi_hdl_class class, uint_t chipid, uint_t coreid,
440    uint_t strandid)
441{
442	cmi_t *cmi = NULL;
443	cmi_hdl_t hdl;
444	void *data;
445
446	if (cmi_no_init) {
447		cmi_no_mca_init = 1;
448		return (NULL);
449	}
450
451	mutex_enter(&cmi_load_lock);
452
453	if ((hdl = cmi_hdl_create(class, chipid, coreid, strandid)) == NULL) {
454		mutex_exit(&cmi_load_lock);
455		cmn_err(CE_WARN, "There will be no MCA support on chip %d "
456		    "core %d strand %d (cmi_hdl_create returned NULL)\n",
457		    chipid, coreid, strandid);
458		return (NULL);
459	}
460
461	if (!cmi_force_generic)
462		cmi = cmi_load_specific(hdl, &data);
463
464	if (cmi == NULL && (cmi = cmi_load_generic(hdl, &data)) == NULL) {
465		cmn_err(CE_WARN, "There will be no MCA support on chip %d "
466		    "core %d strand %d\n", chipid, coreid, strandid);
467		cmi_hdl_rele(hdl);
468		mutex_exit(&cmi_load_lock);
469		return (NULL);
470	}
471
472	cmi_hdl_setcmi(hdl, cmi, data);
473
474	cms_init(hdl);
475
476	cmi_read_smbios(hdl);
477
478	mutex_exit(&cmi_load_lock);
479
480	return (hdl);
481}
482
483/*
484 * cmi_fini is called on DR deconfigure of a cpu resource.
485 * It should not be called at simple offline of a cpu.
486 */
487void
488cmi_fini(cmi_hdl_t hdl)
489{
490	cmi_t *cmi = HDL2CMI(hdl);
491
492	if (cms_present(hdl))
493		cms_fini(hdl);
494
495	if (CMI_OP_PRESENT(cmi, cmi_fini))
496		CMI_OPS(cmi)->cmi_fini(hdl);
497
498	cmi_hdl_destroy(hdl);
499}
500
501/*
502 * cmi_post_startup is called from post_startup for the boot cpu only (no
503 * other cpus are started yet).
504 */
505void
506cmi_post_startup(void)
507{
508	cmi_hdl_t hdl;
509	cmi_t *cmi;
510
511	if (cmi_no_mca_init != 0 ||
512	    (hdl = cmi_hdl_any()) == NULL)	/* short-term hold */
513		return;
514
515	cmi = HDL2CMI(hdl);
516
517	if (CMI_OP_PRESENT(cmi, cmi_post_startup))
518		CMI_OPS(cmi)->cmi_post_startup(hdl);
519
520	cmi_hdl_rele(hdl);
521}
522
523/*
524 * Called just once from start_other_cpus when all processors are started.
525 * This will not be called for each cpu, so the registered op must not
526 * assume it is called as such.  We are not necessarily executing on
527 * the boot cpu.
528 */
529void
530cmi_post_mpstartup(void)
531{
532	cmi_hdl_t hdl;
533	cmi_t *cmi;
534
535	if (cmi_no_mca_init != 0 ||
536	    (hdl = cmi_hdl_any()) == NULL)	/* short-term hold */
537		return;
538
539	cmi = HDL2CMI(hdl);
540
541	if (CMI_OP_PRESENT(cmi, cmi_post_mpstartup))
542		CMI_OPS(cmi)->cmi_post_mpstartup(hdl);
543
544	cmi_hdl_rele(hdl);
545}
546
547void
548cmi_faulted_enter(cmi_hdl_t hdl)
549{
550	cmi_t *cmi = HDL2CMI(hdl);
551
552	if (cmi_no_mca_init != 0)
553		return;
554
555	if (CMI_OP_PRESENT(cmi, cmi_faulted_enter))
556		CMI_OPS(cmi)->cmi_faulted_enter(hdl);
557}
558
559void
560cmi_faulted_exit(cmi_hdl_t hdl)
561{
562	cmi_t *cmi = HDL2CMI(hdl);
563
564	if (cmi_no_mca_init != 0)
565		return;
566
567	if (CMI_OP_PRESENT(cmi, cmi_faulted_exit))
568		CMI_OPS(cmi)->cmi_faulted_exit(hdl);
569}
570
571void
572cmi_mca_init(cmi_hdl_t hdl)
573{
574	cmi_t *cmi;
575
576	if (cmi_no_mca_init != 0)
577		return;
578
579	cmi = HDL2CMI(hdl);
580
581	if (CMI_OP_PRESENT(cmi, cmi_mca_init))
582		CMI_OPS(cmi)->cmi_mca_init(hdl);
583}
584
585#define	CMI_RESPONSE_PANIC		0x0	/* panic must have value 0 */
586#define	CMI_RESPONSE_NONE		0x1
587#define	CMI_RESPONSE_CKILL		0x2
588#define	CMI_RESPONSE_REBOOT		0x3	/* not implemented */
589#define	CMI_RESPONSE_ONTRAP_PROT	0x4
590#define	CMI_RESPONSE_LOFAULT_PROT	0x5
591
592/*
593 * Return 0 if we will panic in response to this machine check, otherwise
594 * non-zero.  If the caller is cmi_mca_trap in this file then the nonzero
595 * return values are to be interpreted from CMI_RESPONSE_* above.
596 *
597 * This function must just return what will be done without actually
598 * doing anything; this includes not changing the regs.
599 */
600int
601cmi_mce_response(struct regs *rp, uint64_t disp)
602{
603	int panicrsp = cmi_panic_on_uncorrectable_error ? CMI_RESPONSE_PANIC :
604	    CMI_RESPONSE_NONE;
605	on_trap_data_t *otp;
606
607	ASSERT(rp != NULL);	/* don't call for polling, only on #MC */
608
609	/*
610	 * If no bits are set in the disposition then there is nothing to
611	 * worry about and we do not need to trampoline to ontrap or
612	 * lofault handlers.
613	 */
614	if (disp == 0)
615		return (CMI_RESPONSE_NONE);
616
617	/*
618	 * Unconstrained errors cannot be forgiven, even by ontrap or
619	 * lofault protection.  The data is not poisoned and may not
620	 * even belong to the trapped context - eg a writeback of
621	 * data that is found to be bad.
622	 */
623	if (disp & CMI_ERRDISP_UC_UNCONSTRAINED)
624		return (panicrsp);
625
626	/*
627	 * ontrap OT_DATA_EC and lofault protection forgive any disposition
628	 * other than unconstrained, even those normally forced fatal.
629	 */
630	if ((otp = curthread->t_ontrap) != NULL && otp->ot_prot & OT_DATA_EC)
631		return (CMI_RESPONSE_ONTRAP_PROT);
632	else if (curthread->t_lofault)
633		return (CMI_RESPONSE_LOFAULT_PROT);
634
635	/*
636	 * Forced-fatal errors are terminal even in user mode.
637	 */
638	if (disp & CMI_ERRDISP_FORCEFATAL)
639		return (panicrsp);
640
641	/*
642	 * If the trapped context is corrupt or we have no instruction pointer
643	 * to resume at (and aren't trampolining to a fault handler)
644	 * then in the kernel case we must panic and in usermode we
645	 * kill the affected contract.
646	 */
647	if (disp & (CMI_ERRDISP_CURCTXBAD | CMI_ERRDISP_RIPV_INVALID))
648		return (USERMODE(rp->r_cs) ?  CMI_RESPONSE_CKILL : panicrsp);
649
650	/*
651	 * Anything else is harmless
652	 */
653	return (CMI_RESPONSE_NONE);
654}
655
656int cma_mca_trap_panic_suppressed = 0;
657
658static void
659cmi_mca_panic(void)
660{
661	if (cmi_panic_on_uncorrectable_error) {
662		fm_panic("Unrecoverable Machine-Check Exception");
663	} else {
664		cmn_err(CE_WARN, "suppressing panic from fatal #mc");
665		cma_mca_trap_panic_suppressed++;
666	}
667}
668
669
670int cma_mca_trap_contract_kills = 0;
671int cma_mca_trap_ontrap_forgiven = 0;
672int cma_mca_trap_lofault_forgiven = 0;
673
674/*
675 * Native #MC handler - we branch to here from mcetrap
676 */
677/*ARGSUSED*/
678void
679cmi_mca_trap(struct regs *rp)
680{
681#ifndef	__xpv
682	cmi_hdl_t hdl = NULL;
683	uint64_t disp;
684	cmi_t *cmi;
685	int s;
686
687	if (cmi_no_mca_init != 0)
688		return;
689
690	/*
691	 * This function can call cmn_err, and the cpu module cmi_mca_trap
692	 * entry point may also elect to call cmn_err (e.g., if it can't
693	 * log the error onto an errorq, say very early in boot).
694	 * We need to let cprintf know that we must not block.
695	 */
696	s = spl8();
697
698	if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
699	    cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL ||
700	    (cmi = HDL2CMI(hdl)) == NULL ||
701	    !CMI_OP_PRESENT(cmi, cmi_mca_trap)) {
702
703		cmn_err(CE_WARN, "#MC exception on cpuid %d: %s",
704		    CPU->cpu_id,
705		    hdl ? "handle lookup ok but no #MC handler found" :
706		    "handle lookup failed");
707
708		if (hdl != NULL)
709			cmi_hdl_rele(hdl);
710
711		splx(s);
712		return;
713	}
714
715	disp = CMI_OPS(cmi)->cmi_mca_trap(hdl, rp);
716
717	switch (cmi_mce_response(rp, disp)) {
718	default:
719		cmn_err(CE_WARN, "Invalid response from cmi_mce_response");
720		/*FALLTHRU*/
721
722	case CMI_RESPONSE_PANIC:
723		cmi_mca_panic();
724		break;
725
726	case CMI_RESPONSE_NONE:
727		break;
728
729	case CMI_RESPONSE_CKILL:
730		ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR;
731		aston(curthread);
732		cma_mca_trap_contract_kills++;
733		break;
734
735	case CMI_RESPONSE_ONTRAP_PROT: {
736		on_trap_data_t *otp = curthread->t_ontrap;
737		otp->ot_trap = OT_DATA_EC;
738		rp->r_pc = otp->ot_trampoline;
739		cma_mca_trap_ontrap_forgiven++;
740		break;
741	}
742
743	case CMI_RESPONSE_LOFAULT_PROT:
744		rp->r_r0 = EFAULT;
745		rp->r_pc = curthread->t_lofault;
746		cma_mca_trap_lofault_forgiven++;
747		break;
748	}
749
750	cmi_hdl_rele(hdl);
751	splx(s);
752#endif	/* __xpv */
753}
754
755void
756cmi_hdl_poke(cmi_hdl_t hdl)
757{
758	cmi_t *cmi = HDL2CMI(hdl);
759
760	if (!CMI_OP_PRESENT(cmi, cmi_hdl_poke))
761		return;
762
763	CMI_OPS(cmi)->cmi_hdl_poke(hdl);
764}
765
766#ifndef	__xpv
767void
768cmi_cmci_trap()
769{
770	cmi_hdl_t hdl = NULL;
771	cmi_t *cmi;
772
773	if (cmi_no_mca_init != 0)
774		return;
775
776	if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU),
777	    cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL ||
778	    (cmi = HDL2CMI(hdl)) == NULL ||
779	    !CMI_OP_PRESENT(cmi, cmi_cmci_trap)) {
780
781		cmn_err(CE_WARN, "CMCI interrupt on cpuid %d: %s",
782		    CPU->cpu_id,
783		    hdl ? "handle lookup ok but no CMCI handler found" :
784		    "handle lookup failed");
785
786		if (hdl != NULL)
787			cmi_hdl_rele(hdl);
788
789		return;
790	}
791
792	CMI_OPS(cmi)->cmi_cmci_trap(hdl);
793
794	cmi_hdl_rele(hdl);
795}
796#endif	/* __xpv */
797
798void
799cmi_mc_register(cmi_hdl_t hdl, const cmi_mc_ops_t *mcops, void *mcdata)
800{
801	if (!cmi_no_mca_init)
802		cmi_hdl_setmc(hdl, mcops, mcdata);
803}
804
805cmi_errno_t
806cmi_mc_register_global(const cmi_mc_ops_t *mcops, void *mcdata)
807{
808	if (!cmi_no_mca_init) {
809		if (cmi_mc_global_ops != NULL || cmi_mc_global_data != NULL ||
810		    mcops == NULL || mcops->cmi_mc_patounum == NULL ||
811		    mcops->cmi_mc_unumtopa == NULL) {
812			return (CMIERR_UNKNOWN);
813		}
814		cmi_mc_global_data = mcdata;
815		cmi_mc_global_ops = mcops;
816	}
817	return (CMI_SUCCESS);
818}
819
820void
821cmi_mc_sw_memscrub_disable(void)
822{
823	memscrub_disable();
824}
825
826cmi_errno_t
827cmi_mc_patounum(uint64_t pa, uint8_t valid_hi, uint8_t valid_lo, uint32_t synd,
828    int syndtype, mc_unum_t *up)
829{
830	const struct cmi_mc_ops *mcops;
831	cmi_hdl_t hdl;
832	cmi_errno_t rv;
833
834	if (cmi_no_mca_init)
835		return (CMIERR_MC_ABSENT);
836
837	if (cmi_mc_global_ops != NULL) {
838		if (cmi_mc_global_ops->cmi_mc_patounum == NULL)
839			return (CMIERR_MC_NOTSUP);
840		return (cmi_mc_global_ops->cmi_mc_patounum(cmi_mc_global_data,
841		    pa, valid_hi, valid_lo, synd, syndtype, up));
842	}
843
844	if ((hdl = cmi_hdl_any()) == NULL)	/* short-term hold */
845		return (CMIERR_MC_ABSENT);
846
847	if ((mcops = cmi_hdl_getmcops(hdl)) == NULL ||
848	    mcops->cmi_mc_patounum == NULL) {
849		cmi_hdl_rele(hdl);
850		return (CMIERR_MC_NOTSUP);
851	}
852
853	rv = mcops->cmi_mc_patounum(cmi_hdl_getmcdata(hdl), pa, valid_hi,
854	    valid_lo, synd, syndtype, up);
855
856	cmi_hdl_rele(hdl);
857
858	return (rv);
859}
860
861cmi_errno_t
862cmi_mc_unumtopa(mc_unum_t *up, nvlist_t *nvl, uint64_t *pap)
863{
864	const struct cmi_mc_ops *mcops;
865	cmi_hdl_t hdl;
866	cmi_errno_t rv;
867	nvlist_t *hcsp;
868
869	if (up != NULL && nvl != NULL)
870		return (CMIERR_API);	/* convert from just one form */
871
872	if (cmi_no_mca_init)
873		return (CMIERR_MC_ABSENT);
874
875	if (cmi_mc_global_ops != NULL) {
876		if (cmi_mc_global_ops->cmi_mc_unumtopa == NULL)
877			return (CMIERR_MC_NOTSUP);
878		return (cmi_mc_global_ops->cmi_mc_unumtopa(cmi_mc_global_data,
879		    up, nvl, pap));
880	}
881
882	if ((hdl = cmi_hdl_any()) == NULL)	/* short-term hold */
883		return (CMIERR_MC_ABSENT);
884
885	if ((mcops = cmi_hdl_getmcops(hdl)) == NULL ||
886	    mcops->cmi_mc_unumtopa == NULL) {
887		cmi_hdl_rele(hdl);
888
889		if (nvl != NULL && nvlist_lookup_nvlist(nvl,
890		    FM_FMRI_HC_SPECIFIC, &hcsp) == 0 &&
891		    (nvlist_lookup_uint64(hcsp,
892		    "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR, pap) == 0 ||
893		    nvlist_lookup_uint64(hcsp, FM_FMRI_HC_SPECIFIC_PHYSADDR,
894		    pap) == 0)) {
895			return (CMIERR_MC_PARTIALUNUMTOPA);
896		} else {
897			return (mcops && mcops->cmi_mc_unumtopa == NULL ?
898			    CMIERR_MC_NOTSUP : CMIERR_MC_ABSENT);
899		}
900	}
901
902	rv = mcops->cmi_mc_unumtopa(cmi_hdl_getmcdata(hdl), up, nvl, pap);
903
904	cmi_hdl_rele(hdl);
905
906	return (rv);
907}
908
909void
910cmi_mc_logout(cmi_hdl_t hdl, boolean_t ismc, boolean_t sync)
911{
912	const struct cmi_mc_ops *mcops;
913
914	if (cmi_no_mca_init)
915		return;
916
917	if (cmi_mc_global_ops != NULL)
918		mcops = cmi_mc_global_ops;
919	else
920		mcops = cmi_hdl_getmcops(hdl);
921
922	if (mcops != NULL && mcops->cmi_mc_logout != NULL)
923		mcops->cmi_mc_logout(hdl, ismc, sync);
924}
925
926cmi_errno_t
927cmi_hdl_msrinject(cmi_hdl_t hdl, cmi_mca_regs_t *regs, uint_t nregs,
928    int force)
929{
930	cmi_t *cmi = cmi_hdl_getcmi(hdl);
931	cmi_errno_t rc;
932
933	if (!CMI_OP_PRESENT(cmi, cmi_msrinject))
934		return (CMIERR_NOTSUP);
935
936	cmi_hdl_inj_begin(hdl);
937	rc = CMI_OPS(cmi)->cmi_msrinject(hdl, regs, nregs, force);
938	cmi_hdl_inj_end(hdl);
939
940	return (rc);
941}
942
943boolean_t
944cmi_panic_on_ue(void)
945{
946	return (cmi_panic_on_uncorrectable_error ? B_TRUE : B_FALSE);
947}
948
949void
950cmi_panic_callback(void)
951{
952	cmi_hdl_t hdl;
953	cmi_t *cmi;
954
955	if (cmi_no_mca_init || (hdl = cmi_hdl_any()) == NULL)
956		return;
957
958	cmi = cmi_hdl_getcmi(hdl);
959	if (CMI_OP_PRESENT(cmi, cmi_panic_callback))
960		CMI_OPS(cmi)->cmi_panic_callback();
961
962	cmi_hdl_rele(hdl);
963}
964
965
966const char *
967cmi_hdl_chipident(cmi_hdl_t hdl)
968{
969	cmi_t *cmi = cmi_hdl_getcmi(hdl);
970
971	if (!CMI_OP_PRESENT(cmi, cmi_ident))
972		return (NULL);
973
974	return (CMI_OPS(cmi)->cmi_ident(hdl));
975}
976