1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 */
26/*
27 * Copyright (c) 2010, Intel Corporation.
28 * All rights reserved.
29 */
30
31/*
32 * Copyright (c) 2018, Joyent, Inc.
33 */
34
35#include <sys/types.h>
36#include <sys/cpu_module_ms_impl.h>
37#include <sys/cpuvar.h>
38#include <sys/ksynch.h>
39#include <sys/modctl.h>
40#include <sys/x86_archext.h>
41#include <sys/systm.h>
42#include <sys/cmn_err.h>
43#include <sys/param.h>
44#include <sys/reboot.h>
45
46/*
47 * Set to prevent model-specific support from initialising.
48 */
49int cms_no_model_specific = 0;
50
51/*
52 * Subdirectory (relative to the module search path) in which we will
53 * look for model-specific modules.
54 */
55#define	CPUMOD_MS_SUBDIR	"cpu"
56
57/*
58 * Cpu model-specific modules have filenames beginning with the following.
59 */
60#define	CPUMOD_MS_PREFIX	"cpu_ms"
61
62#define	HDL2CMS(hdl)		cms_hdl_getcms(hdl)
63
64#define	CMS_OPS(cms)		(cms)->cms_ops
65#define	CMS_OP_PRESENT(cms, op)	((cms) && CMS_OPS(cms)->op != NULL)
66
67struct cms_cpuid {
68	const char *vendor;
69	uint_t family;
70	uint_t model;
71	uint_t stepping;
72};
73
74#define	CMS_MATCH_VENDOR	0	/* Just match on vendor */
75#define	CMS_MATCH_FAMILY	1	/* Match down to family */
76#define	CMS_MATCH_MODEL		2	/* Match down to model */
77#define	CMS_MATCH_STEPPING	3	/* Match down to stepping */
78
79/*
80 * Structure used to keep track of modules we have loaded.
81 */
82typedef struct cms {
83	struct cms *cms_next;
84	struct cms *cms_prev;
85	const cms_ops_t *cms_ops;
86	struct modctl *cms_modp;
87	uint_t cms_refcnt;
88} cms_t;
89
90static cms_t *cms_list;
91static kmutex_t cms_load_lock;
92
93/*
94 * We stash a cms_t and associated private data via cmi_hdl_setspecific.
95 */
96struct cms_ctl {
97	cms_t *cs_cms;
98	void *cs_cmsdata;
99};
100
101static cms_t *
102cms_hdl_getcms(cmi_hdl_t hdl)
103{
104	struct cms_ctl *cdp = cmi_hdl_getspecific(hdl);
105
106	return (cdp != NULL ? cdp->cs_cms : NULL);
107}
108
109void *
110cms_hdl_getcmsdata(cmi_hdl_t hdl)
111{
112	struct cms_ctl *cdp = cmi_hdl_getspecific(hdl);
113
114	return (cdp != NULL ? cdp->cs_cmsdata : NULL);
115}
116
117static void
118cms_link(cms_t *cms)
119{
120	ASSERT(MUTEX_HELD(&cms_load_lock));
121
122	cms->cms_prev = NULL;
123	cms->cms_next = cms_list;
124	if (cms_list != NULL)
125		cms_list->cms_prev = cms;
126	cms_list = cms;
127}
128
129static void
130cms_unlink(cms_t *cms)
131{
132	ASSERT(MUTEX_HELD(&cms_load_lock));
133	ASSERT(cms->cms_refcnt == 0);
134
135	if (cms->cms_prev != NULL)
136		cms->cms_prev->cms_next = cms->cms_next;
137
138	if (cms->cms_next != NULL)
139		cms->cms_next->cms_prev = cms->cms_prev;
140
141	if (cms_list == cms)
142		cms_list = cms->cms_next;
143}
144
145/*
146 * Hold the module in memory.  We call to CPU modules without using the
147 * stubs mechanism, so these modules must be manually held in memory.
148 * The mod_ref acts as if another loaded module has a dependency on us.
149 */
150static void
151cms_hold(cms_t *cms)
152{
153	ASSERT(MUTEX_HELD(&cms_load_lock));
154
155	mutex_enter(&mod_lock);
156	cms->cms_modp->mod_ref++;
157	mutex_exit(&mod_lock);
158	cms->cms_refcnt++;
159}
160
161static void
162cms_rele(cms_t *cms)
163{
164	ASSERT(MUTEX_HELD(&cms_load_lock));
165
166	mutex_enter(&mod_lock);
167	cms->cms_modp->mod_ref--;
168	mutex_exit(&mod_lock);
169
170	if (--cms->cms_refcnt == 0) {
171		cms_unlink(cms);
172		kmem_free(cms, sizeof (cms_t));
173	}
174}
175
176static cms_ops_t *
177cms_getops(modctl_t *modp)
178{
179	cms_ops_t *ops;
180
181	if ((ops = (cms_ops_t *)modlookup_by_modctl(modp, "_cms_ops")) ==
182	    NULL) {
183		cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no _cms_ops "
184		    "found", modp->mod_modname);
185		return (NULL);
186	}
187
188	if (ops->cms_init == NULL) {
189		cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no cms_init "
190		    "entry point", modp->mod_modname);
191		return (NULL);
192	}
193
194	return (ops);
195}
196
197static cms_t *
198cms_load_modctl(modctl_t *modp)
199{
200	cms_ops_t *ops;
201	uintptr_t ver;
202	cms_t *cms;
203	cms_api_ver_t apiver;
204
205	ASSERT(MUTEX_HELD(&cms_load_lock));
206
207	for (cms = cms_list; cms != NULL; cms = cms->cms_next) {
208		if (cms->cms_modp == modp)
209			return (cms);
210	}
211
212	if ((ver = modlookup_by_modctl(modp, "_cms_api_version")) == 0) {
213		cmn_err(CE_WARN, "cpu model-specific module '%s' is invalid:  "
214		    "no _cms_api_version", modp->mod_modname);
215		return (NULL);
216	} else {
217		apiver = *((cms_api_ver_t *)ver);
218		if (!CMS_API_VERSION_CHKMAGIC(apiver)) {
219			cmn_err(CE_WARN, "cpu model-specific module '%s' is "
220			    "invalid: _cms_api_version 0x%x has bad magic",
221			    modp->mod_modname, apiver);
222			return (NULL);
223		}
224	}
225
226	if (apiver != CMS_API_VERSION) {
227		cmn_err(CE_WARN, "cpu model-specific module '%s' has API "
228		    "version %d, kernel requires API version %d",
229		    modp->mod_modname, CMS_API_VERSION_TOPRINT(apiver),
230		    CMS_API_VERSION_TOPRINT(CMS_API_VERSION));
231		return (NULL);
232	}
233
234	if ((ops = cms_getops(modp)) == NULL)
235		return (NULL);
236
237	cms = kmem_zalloc(sizeof (cms_t), KM_SLEEP);
238	cms->cms_ops = ops;
239	cms->cms_modp = modp;
240
241	cms_link(cms);
242
243	return (cms);
244}
245
246static int
247cms_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match)
248{
249	if (match >= CMS_MATCH_VENDOR &&
250	    cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2))
251		return (0);
252
253	if (match >= CMS_MATCH_FAMILY &&
254	    cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2))
255		return (0);
256
257	if (match >= CMS_MATCH_MODEL &&
258	    cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2))
259		return (0);
260
261	if (match >= CMS_MATCH_STEPPING &&
262	    cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2))
263		return (0);
264
265	return (1);
266}
267
268static int
269cms_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3)
270{
271	cmi_hdl_t thdl = (cmi_hdl_t)arg1;
272	int match = *((int *)arg2);
273	cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3;
274
275	if (cms_cpu_match(thdl, whdl, match)) {
276		cmi_hdl_hold(whdl);	/* short-term hold */
277		*rsltp = whdl;
278		return (CMI_HDL_WALK_DONE);
279	} else {
280		return (CMI_HDL_WALK_NEXT);
281	}
282}
283
284/*
285 * Look to see if we've already got a module loaded for a CPU just
286 * like this one.  If we do, then we'll re-use it.
287 */
288static cms_t *
289cms_search_list(cmi_hdl_t hdl, int match)
290{
291	cmi_hdl_t dhdl = NULL;
292	cms_t *cms = NULL;
293
294	ASSERT(MUTEX_HELD(&cms_load_lock));
295
296	cmi_hdl_walk(cms_search_list_cb, (void *)hdl, (void *)&match, &dhdl);
297	if (dhdl) {
298		cms = HDL2CMS(dhdl);
299		cmi_hdl_rele(dhdl);	/* held in cms_search_list_cb */
300	}
301
302	return (cms);
303}
304
305/*
306 * Try to find or load a module that offers model-specific support for
307 * this vendor/family/model/stepping combination.  When attempting to load
308 * a module we look in CPUMOD_MS_SUBDIR first for a match on
309 * vendor/family/model/stepping, then on vendor/family/model (ignoring
310 * stepping), then on vendor/family (ignoring model and stepping), then
311 * on vendor alone.
312 */
313static cms_t *
314cms_load_module(cmi_hdl_t hdl, int match, int *chosenp)
315{
316	modctl_t *modp;
317	cms_t *cms;
318	int modid;
319	uint_t s[3];
320
321	ASSERT(MUTEX_HELD(&cms_load_lock));
322	ASSERT(match == CMS_MATCH_STEPPING || match == CMS_MATCH_MODEL ||
323	    match == CMS_MATCH_FAMILY || match == CMS_MATCH_VENDOR);
324
325	s[0] = cmi_hdl_family(hdl);
326	s[1] = cmi_hdl_model(hdl);
327	s[2] = cmi_hdl_stepping(hdl);
328
329	/*
330	 * Have we already loaded a module for a cpu with the same
331	 * vendor/family/model/stepping?
332	 */
333	if ((cms = cms_search_list(hdl, match)) != NULL) {
334		cms_hold(cms);
335		return (cms);
336	}
337
338	modid = modload_qualified(CPUMOD_MS_SUBDIR, CPUMOD_MS_PREFIX,
339	    cmi_hdl_vendorstr(hdl), ".", s, match, chosenp);
340
341	if (modid == -1)
342		return (NULL);
343
344	modp = mod_hold_by_id(modid);
345	cms = cms_load_modctl(modp);
346	if (cms)
347		cms_hold(cms);
348	mod_release_mod(modp);
349
350	return (cms);
351}
352
353static cms_t *
354cms_load_specific(cmi_hdl_t hdl, void **datap)
355{
356	cms_t *cms;
357	int err;
358	int i;
359
360	ASSERT(MUTEX_HELD(&cms_load_lock));
361
362	for (i = CMS_MATCH_STEPPING; i >= CMS_MATCH_VENDOR; i--) {
363		int suffixlevel;
364
365		if ((cms = cms_load_module(hdl, i, &suffixlevel)) == NULL)
366			return (NULL);
367
368		/*
369		 * A module has loaded and has a _cms_ops structure, and the
370		 * module has been held for this instance.  Call the cms_init
371		 * entry point - we expect success (0) or ENOTSUP.
372		 */
373		if ((err = cms->cms_ops->cms_init(hdl, datap)) == 0) {
374			if (boothowto & RB_VERBOSE) {
375				printf("initialized model-specific "
376				    "module '%s' on chip %d core %d "
377				    "strand %d\n",
378				    cms->cms_modp->mod_modname,
379				    cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
380				    cmi_hdl_strandid(hdl));
381			}
382			return (cms);
383		} else if (err != ENOTSUP) {
384			cmn_err(CE_WARN, "failed to init model-specific "
385			    "module '%s' on chip %d core %d strand %d: err=%d",
386			    cms->cms_modp->mod_modname,
387			    cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
388			    cmi_hdl_strandid(hdl), err);
389		}
390
391		/*
392		 * The module failed or declined to init, so release
393		 * it and potentially change i to be equal to the number
394		 * of suffices actually used in the last module path.
395		 */
396		cms_rele(cms);
397		i = suffixlevel;
398	}
399
400	return (NULL);
401}
402
403void
404cms_init(cmi_hdl_t hdl)
405{
406	cms_t *cms;
407	void *data;
408
409	if (cms_no_model_specific != 0)
410		return;
411
412	mutex_enter(&cms_load_lock);
413
414	if ((cms = cms_load_specific(hdl, &data)) != NULL) {
415		struct cms_ctl *cdp;
416
417		ASSERT(cmi_hdl_getspecific(hdl) == NULL);
418
419		cdp = kmem_alloc(sizeof (*cdp), KM_SLEEP);
420		cdp->cs_cms = cms;
421		cdp->cs_cmsdata = data;
422		cmi_hdl_setspecific(hdl, cdp);
423	}
424
425	mutex_exit(&cms_load_lock);
426}
427
428void
429cms_fini(cmi_hdl_t hdl)
430{
431	cms_t *cms = HDL2CMS(hdl);
432	struct cms_ctl *cdp;
433
434	if (CMS_OP_PRESENT(cms, cms_fini))
435		CMS_OPS(cms)->cms_fini(hdl);
436
437	mutex_enter(&cms_load_lock);
438	cdp = (struct cms_ctl *)cmi_hdl_getspecific(hdl);
439	if (cdp != NULL) {
440		if (cdp->cs_cms != NULL)
441			cms_rele(cdp->cs_cms);
442		kmem_free(cdp, sizeof (*cdp));
443	}
444	mutex_exit(&cms_load_lock);
445}
446
447boolean_t
448cms_present(cmi_hdl_t hdl)
449{
450	return (HDL2CMS(hdl) != NULL ? B_TRUE : B_FALSE);
451}
452
453void
454cms_post_startup(cmi_hdl_t hdl)
455{
456	cms_t *cms = HDL2CMS(hdl);
457
458	if (CMS_OP_PRESENT(cms, cms_post_startup))
459		CMS_OPS(cms)->cms_post_startup(hdl);
460}
461
462void
463cms_post_mpstartup(cmi_hdl_t hdl)
464{
465	cms_t *cms = HDL2CMS(hdl);
466
467	if (CMS_OP_PRESENT(cms, cms_post_mpstartup))
468		CMS_OPS(cms)->cms_post_mpstartup(hdl);
469}
470
471size_t
472cms_logout_size(cmi_hdl_t hdl)
473{
474	cms_t *cms = HDL2CMS(hdl);
475
476	if (!CMS_OP_PRESENT(cms, cms_logout_size))
477		return (0);
478
479	return (CMS_OPS(cms)->cms_logout_size(hdl));
480}
481
482uint64_t
483cms_mcgctl_val(cmi_hdl_t hdl, int nbanks, uint64_t def)
484{
485	cms_t *cms = HDL2CMS(hdl);
486
487	if (!CMS_OP_PRESENT(cms, cms_mcgctl_val))
488		return (def);
489
490	return (CMS_OPS(cms)->cms_mcgctl_val(hdl, nbanks, def));
491}
492
493boolean_t
494cms_bankctl_skipinit(cmi_hdl_t hdl, int banknum)
495{
496	cms_t *cms = HDL2CMS(hdl);
497
498	if (!CMS_OP_PRESENT(cms, cms_bankctl_skipinit))
499		return (B_FALSE);
500
501	return (CMS_OPS(cms)->cms_bankctl_skipinit(hdl, banknum));
502}
503
504uint64_t
505cms_bankctl_val(cmi_hdl_t hdl, int banknum, uint64_t def)
506{
507	cms_t *cms = HDL2CMS(hdl);
508
509	if (!CMS_OP_PRESENT(cms, cms_bankctl_val))
510		return (def);
511
512	return (CMS_OPS(cms)->cms_bankctl_val(hdl, banknum, def));
513}
514
515boolean_t
516cms_bankstatus_skipinit(cmi_hdl_t hdl, int banknum)
517{
518	cms_t *cms = HDL2CMS(hdl);
519
520	if (!CMS_OP_PRESENT(cms, cms_bankstatus_skipinit))
521		return (B_FALSE);
522
523	return (CMS_OPS(cms)->cms_bankstatus_skipinit(hdl, banknum));
524}
525
526uint64_t
527cms_bankstatus_val(cmi_hdl_t hdl, int banknum, uint64_t def)
528{
529	cms_t *cms = HDL2CMS(hdl);
530
531	if (!CMS_OP_PRESENT(cms, cms_bankstatus_val))
532		return (def);
533
534	return (CMS_OPS(cms)->cms_bankstatus_val(hdl, banknum, def));
535}
536
537void
538cms_mca_init(cmi_hdl_t hdl, int nbanks)
539{
540	cms_t *cms = HDL2CMS(hdl);
541
542	if (CMS_OP_PRESENT(cms, cms_mca_init))
543		CMS_OPS(cms)->cms_mca_init(hdl, nbanks);
544}
545
546uint64_t
547cms_poll_ownermask(cmi_hdl_t hdl, hrtime_t poll_interval)
548{
549	cms_t *cms = HDL2CMS(hdl);
550
551	if (CMS_OP_PRESENT(cms, cms_poll_ownermask))
552		return (CMS_OPS(cms)->cms_poll_ownermask(hdl, poll_interval));
553	else
554		return (-1ULL);		/* poll all banks by default */
555}
556
557void
558cms_bank_logout(cmi_hdl_t hdl, int banknum, uint64_t status, uint64_t addr,
559    uint64_t misc, void *mslogout)
560{
561	cms_t *cms = HDL2CMS(hdl);
562
563	if (mslogout != NULL && CMS_OP_PRESENT(cms, cms_bank_logout))
564		CMS_OPS(cms)->cms_bank_logout(hdl, banknum, status, addr,
565		    misc, mslogout);
566}
567
568cms_errno_t
569cms_msrinject(cmi_hdl_t hdl, uint_t msr, uint64_t val)
570{
571	cms_t *cms = HDL2CMS(hdl);
572
573	if (CMS_OP_PRESENT(cms, cms_msrinject))
574		return (CMS_OPS(cms)->cms_msrinject(hdl, msr, val));
575	else
576		return (CMSERR_NOTSUP);
577}
578
579uint32_t
580cms_error_action(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status,
581    uint64_t addr, uint64_t misc, void *mslogout)
582{
583	cms_t *cms = HDL2CMS(hdl);
584
585	if (CMS_OP_PRESENT(cms, cms_error_action))
586		return (CMS_OPS(cms)->cms_error_action(hdl, ismc, banknum,
587		    status, addr, misc, mslogout));
588	else
589		return (0);
590}
591
592cms_cookie_t
593cms_disp_match(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status,
594    uint64_t addr, uint64_t misc, void *mslogout)
595{
596	cms_t *cms = HDL2CMS(hdl);
597
598	if (CMS_OP_PRESENT(cms, cms_disp_match))
599		return (CMS_OPS(cms)->cms_disp_match(hdl, ismc, banknum,
600		    status, addr, misc, mslogout));
601	else
602		return (NULL);
603
604}
605
606void
607cms_ereport_class(cmi_hdl_t hdl, cms_cookie_t mscookie, const char **cpuclsp,
608    const char **leafclsp)
609{
610	cms_t *cms = HDL2CMS(hdl);
611
612	if (cpuclsp == NULL || leafclsp == NULL)
613		return;
614
615	*cpuclsp = *leafclsp = NULL;
616	if (CMS_OP_PRESENT(cms, cms_ereport_class)) {
617		CMS_OPS(cms)->cms_ereport_class(hdl, mscookie, cpuclsp,
618		    leafclsp);
619	}
620}
621
622nvlist_t *
623cms_ereport_detector(cmi_hdl_t hdl, int bankno, cms_cookie_t mscookie,
624    nv_alloc_t *nva)
625{
626	cms_t *cms = HDL2CMS(hdl);
627
628	if (CMS_OP_PRESENT(cms, cms_ereport_detector))
629		return (CMS_OPS(cms)->cms_ereport_detector(hdl, bankno,
630		    mscookie, nva));
631	else
632		return (NULL);
633
634}
635
636boolean_t
637cms_ereport_includestack(cmi_hdl_t hdl, cms_cookie_t mscookie)
638{
639	cms_t *cms = HDL2CMS(hdl);
640
641	if (CMS_OP_PRESENT(cms, cms_ereport_includestack)) {
642		return (CMS_OPS(cms)->cms_ereport_includestack(hdl, mscookie));
643	} else {
644		return (B_FALSE);
645	}
646}
647
648void
649cms_ereport_add_logout(cmi_hdl_t hdl, nvlist_t *nvl, nv_alloc_t *nva,
650    int banknum, uint64_t status, uint64_t addr, uint64_t misc, void *mslogout,
651    cms_cookie_t mscookie)
652{
653	cms_t *cms = HDL2CMS(hdl);
654
655	if (CMS_OP_PRESENT(cms, cms_ereport_add_logout))
656		CMS_OPS(cms)->cms_ereport_add_logout(hdl, nvl, nva, banknum,
657		    status, addr, misc, mslogout, mscookie);
658
659}
660