1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26#include <sys/conf.h>
27#include <sys/ddi.h>
28#include <sys/ddifm.h>
29#include <sys/sunddi.h>
30#include <sys/sunndi.h>
31#include <sys/stat.h>
32#include <sys/modctl.h>
33#include <sys/types.h>
34#include <sys/cpuvar.h>
35#include <sys/cmn_err.h>
36#include <sys/kmem.h>
37#include <sys/cred.h>
38#include <sys/ksynch.h>
39#include <sys/rwlock.h>
40#include <sys/pghw.h>
41#include <sys/open.h>
42#include <sys/policy.h>
43#include <sys/x86_archext.h>
44#include <sys/cpu_module.h>
45#include <qsort.h>
46#include <sys/pci_cfgspace.h>
47#include <sys/mc.h>
48#include <sys/mc_amd.h>
49#include <sys/smbios.h>
50#include <sys/pci.h>
51#include <mcamd.h>
52#include <mcamd_dimmcfg.h>
53#include <mcamd_pcicfg.h>
54#include <mcamd_api.h>
55#include <sys/fm/cpu/AMD.h>
56#include <sys/fm/smb/fmsmb.h>
57#include <sys/fm/protocol.h>
58#include <sys/fm/util.h>
59
60/*
61 * Set to prevent mc-amd from attaching.
62 */
63int mc_no_attach = 0;
64
65/*
66 * Of the 754/939/940 packages, only socket 940 supports quadrank registered
67 * dimms.  Unfortunately, no memory-controller register indicates the
68 * presence of quadrank dimm support or presence (i.e., in terms of number
69 * of slots per cpu, and chip-select lines per slot,  The following may be set
70 * in /etc/system to indicate the presence of quadrank support on a motherboard.
71 *
72 * There is no need to set this for F(1207) and S1g1.
73 */
74int mc_quadranksupport = 0;
75
76mc_t *mc_list, *mc_last;
77krwlock_t mc_lock;
78int mc_hold_attached = 1;
79
80#define	MAX(m, n) ((m) >= (n) ? (m) : (n))
81#define	MIN(m, n) ((m) <= (n) ? (m) : (n))
82
83/*
84 * The following tuneable is used to determine the DRAM scrubbing rate.
85 * The values range from 0x00-0x16 as described in the BKDG.  Zero
86 * disables DRAM scrubbing.  Values above zero indicate rates in descending
87 * order.
88 *
89 * The default value below is used on several Sun systems.  In the future
90 * this code should assign values dynamically based on memory sizing.
91 */
92uint32_t mc_scrub_rate_dram = 0xd;	/* 64B every 163.8 us; 1GB per 45 min */
93
94enum {
95	MC_SCRUB_BIOSDEFAULT,	/* retain system default value */
96	MC_SCRUB_FIXED,		/* assign mc_scrub_rate_* values */
97	MC_SCRUB_MAX		/* assign max of system and tunables */
98} mc_scrub_policy = MC_SCRUB_MAX;
99
100static void
101mc_snapshot_destroy(mc_t *mc)
102{
103	ASSERT(RW_LOCK_HELD(&mc_lock));
104
105	if (mc->mc_snapshot == NULL)
106		return;
107
108	kmem_free(mc->mc_snapshot, mc->mc_snapshotsz);
109	mc->mc_snapshot = NULL;
110	mc->mc_snapshotsz = 0;
111	mc->mc_snapshotgen++;
112}
113
114static int
115mc_snapshot_update(mc_t *mc)
116{
117	ASSERT(RW_LOCK_HELD(&mc_lock));
118
119	if (mc->mc_snapshot != NULL)
120		return (0);
121
122	if (nvlist_pack(mc->mc_nvl, &mc->mc_snapshot, &mc->mc_snapshotsz,
123	    NV_ENCODE_XDR, KM_SLEEP) != 0)
124		return (-1);
125
126	return (0);
127}
128
129static mc_t *
130mc_lookup_by_chipid(int chipid)
131{
132	mc_t *mc;
133
134	ASSERT(RW_LOCK_HELD(&mc_lock));
135
136	for (mc = mc_list; mc != NULL; mc = mc->mc_next) {
137		if (mc->mc_props.mcp_num  == chipid)
138			return (mc);
139	}
140
141	return (NULL);
142}
143
144/*
145 * Read config register pairs into the two arrays provided on the given
146 * handle and at offsets as follows:
147 *
148 *	Index	Array r1 offset			Array r2 offset
149 *	0	r1addr				r2addr
150 *	1	r1addr + incr			r2addr + incr
151 *	2	r1addr + 2 * incr		r2addr + 2 * incr
152 *	...
153 *	n - 1	r1addr + (n - 1) * incr		r2addr + (n - 1) * incr
154 *
155 * The number of registers to read into the r1 array is r1n; the number
156 * for the r2 array is r2n.
157 */
158static void
159mc_prop_read_pair(mc_pcicfg_hdl_t cfghdl, uint32_t *r1, off_t r1addr,
160    int r1n, uint32_t *r2, off_t r2addr, int r2n, off_t incr)
161{
162	int i;
163
164	for (i = 0; i < MAX(r1n, r2n); i++, r1addr += incr, r2addr += incr) {
165		if (i < r1n)
166			r1[i] = mc_pcicfg_get32(cfghdl, r1addr);
167		if (i < r2n)
168			r2[i] = mc_pcicfg_get32(cfghdl, r2addr);
169	}
170}
171
172/*ARGSUSED*/
173static int
174mc_nvl_add_socket_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3)
175{
176	uint32_t skt = *((uint32_t *)arg1);
177	cmi_hdl_t *hdlp = (cmi_hdl_t *)arg2;
178
179	if (cmi_hdl_getsockettype(whdl) == skt) {
180		cmi_hdl_hold(whdl);	/* short-term hold */
181		*hdlp = whdl;
182		return (CMI_HDL_WALK_DONE);
183	} else {
184		return (CMI_HDL_WALK_NEXT);
185	}
186}
187
188static void
189mc_nvl_add_socket(nvlist_t *nvl, mc_t *mc)
190{
191	cmi_hdl_t hdl = NULL;
192	const char *s;
193
194	cmi_hdl_walk(mc_nvl_add_socket_cb, (void *)&mc->mc_socket,
195	    (void *)&hdl, NULL);
196	if (hdl == NULL)
197		s = "Unknown";  /* no cpu for this chipid found */
198	else
199		s = cmi_hdl_getsocketstr(hdl);
200
201	(void) nvlist_add_string(nvl, "socket", s);
202
203	if (hdl != NULL)
204		cmi_hdl_rele(hdl);
205}
206
207static uint32_t
208mc_ecc_enabled(mc_t *mc)
209{
210	uint32_t rev = mc->mc_props.mcp_rev;
211	union mcreg_nbcfg nbcfg;
212
213	MCREG_VAL32(&nbcfg) = mc->mc_cfgregs.mcr_nbcfg;
214
215	return (MC_REV_MATCH(rev, MC_F_REVS_BCDE) ?
216	    MCREG_FIELD_F_preF(&nbcfg, EccEn) :
217	    MCREG_FIELD_F_revFG(&nbcfg, EccEn));
218}
219
220static uint32_t
221mc_ck_enabled(mc_t *mc)
222{
223	uint32_t rev = mc->mc_props.mcp_rev;
224	union mcreg_nbcfg nbcfg;
225
226	MCREG_VAL32(&nbcfg) = mc->mc_cfgregs.mcr_nbcfg;
227
228	return (MC_REV_MATCH(rev, MC_F_REVS_BCDE) ?
229	    MCREG_FIELD_F_preF(&nbcfg, ChipKillEccEn) :
230	    MCREG_FIELD_F_revFG(&nbcfg, ChipKillEccEn));
231}
232
233static void
234mc_nvl_add_ecctype(nvlist_t *nvl, mc_t *mc)
235{
236	(void) nvlist_add_string(nvl, "ecc-type", mc_ecc_enabled(mc) ?
237	    (mc_ck_enabled(mc) ? "ChipKill 128/16" : "Normal 64/8") : "None");
238}
239
240static void
241mc_nvl_add_prop(nvlist_t *nvl, void *node, mcamd_propcode_t code, int reqval)
242{
243	int valfound;
244	uint64_t value;
245	const char *name = mcamd_get_propname(code);
246
247	valfound = mcamd_get_numprop(NULL, (mcamd_node_t *)node, code, &value);
248
249	ASSERT(name != NULL && valfound);
250	if (name != NULL && valfound && (!reqval || value != MC_INVALNUM))
251		(void) nvlist_add_uint64(nvl, name, value);
252}
253
254static void
255mc_nvl_add_cslist(nvlist_t *mcnvl, mc_t *mc)
256{
257	mc_cs_t *mccs = mc->mc_cslist;
258	nvlist_t *cslist[MC_CHIP_NCS];
259	int nelem, i;
260
261	for (nelem = 0; mccs != NULL; mccs = mccs->mccs_next, nelem++) {
262		nvlist_t **csp = &cslist[nelem];
263		char csname[MCDCFG_CSNAMELEN];
264
265		(void) nvlist_alloc(csp, NV_UNIQUE_NAME, KM_SLEEP);
266		mc_nvl_add_prop(*csp, mccs, MCAMD_PROP_NUM, 0);
267		mc_nvl_add_prop(*csp, mccs, MCAMD_PROP_BASE_ADDR, 0);
268		mc_nvl_add_prop(*csp, mccs, MCAMD_PROP_MASK, 0);
269		mc_nvl_add_prop(*csp, mccs, MCAMD_PROP_SIZE, 0);
270
271		/*
272		 * It is possible for an mc_cs_t not to have associated
273		 * DIMM info if mcdcfg_lookup failed.
274		 */
275		if (mccs->mccs_csl[0] != NULL) {
276			mc_nvl_add_prop(*csp, mccs, MCAMD_PROP_CSDIMM1, 1);
277			mcdcfg_csname(mc->mc_socket, mccs->mccs_csl[0], csname,
278			    sizeof (csname));
279			(void) nvlist_add_string(*csp, "dimm1-csname", csname);
280		}
281
282		if (mccs->mccs_csl[1] != NULL) {
283			mc_nvl_add_prop(*csp, mccs, MCAMD_PROP_CSDIMM2, 1);
284			mcdcfg_csname(mc->mc_socket, mccs->mccs_csl[1], csname,
285			    sizeof (csname));
286			(void) nvlist_add_string(*csp, "dimm2-csname", csname);
287		}
288	}
289
290	/* Add cslist nvlist array even if zero members */
291	(void) nvlist_add_nvlist_array(mcnvl, "cslist", cslist, nelem);
292	for (i = 0; i < nelem; i++)
293		nvlist_free(cslist[i]);
294}
295
296static void
297mc_nvl_add_dimmlist(nvlist_t *mcnvl, mc_t *mc)
298{
299	nvlist_t *dimmlist[MC_CHIP_NDIMM];
300	mc_dimm_t *mcd;
301	int nelem, i;
302
303	for (nelem = 0, mcd = mc->mc_dimmlist; mcd != NULL;
304	    mcd = mcd->mcd_next, nelem++) {
305		nvlist_t **dimmp = &dimmlist[nelem];
306		uint64_t csnums[MC_CHIP_DIMMRANKMAX];
307		char csname[4][MCDCFG_CSNAMELEN];
308		char *csnamep[4];
309		int ncs = 0;
310
311		(void) nvlist_alloc(dimmp, NV_UNIQUE_NAME, KM_SLEEP);
312
313		mc_nvl_add_prop(*dimmp, mcd, MCAMD_PROP_NUM, 1);
314		mc_nvl_add_prop(*dimmp, mcd, MCAMD_PROP_SIZE, 1);
315
316		for (i = 0; i < MC_CHIP_DIMMRANKMAX; i++) {
317			if (mcd->mcd_cs[i] != NULL) {
318				csnums[ncs] =
319				    mcd->mcd_cs[i]->mccs_props.csp_num;
320				mcdcfg_csname(mc->mc_socket, mcd->mcd_csl[i],
321				    csname[ncs], MCDCFG_CSNAMELEN);
322				csnamep[ncs] = csname[ncs];
323				ncs++;
324			}
325		}
326
327		(void) nvlist_add_uint64_array(*dimmp, "csnums", csnums, ncs);
328		(void) nvlist_add_string_array(*dimmp, "csnames", csnamep, ncs);
329	}
330
331	/* Add dimmlist nvlist array even if zero members */
332	(void) nvlist_add_nvlist_array(mcnvl, "dimmlist", dimmlist, nelem);
333	for (i = 0; i < nelem; i++)
334		nvlist_free(dimmlist[i]);
335}
336
337static void
338mc_nvl_add_htconfig(nvlist_t *mcnvl, mc_t *mc)
339{
340	mc_cfgregs_t *mcr = &mc->mc_cfgregs;
341	union mcreg_htroute *htrp = (union mcreg_htroute *)&mcr->mcr_htroute[0];
342	union mcreg_nodeid *nip = (union mcreg_nodeid *)&mcr->mcr_htnodeid;
343	union mcreg_unitid *uip = (union mcreg_unitid *)&mcr->mcr_htunitid;
344	int ndcnt = HT_COHERENTNODES(nip);
345	uint32_t BCRte[MC_CHIP_MAXNODES];
346	uint32_t RPRte[MC_CHIP_MAXNODES];
347	uint32_t RQRte[MC_CHIP_MAXNODES];
348	nvlist_t *nvl;
349	int i;
350
351	(void) nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP);
352
353	(void) nvlist_add_uint32(nvl, "NodeId", MCREG_FIELD_CMN(nip, NodeId));
354	(void) nvlist_add_uint32(nvl, "CoherentNodes", HT_COHERENTNODES(nip));
355	(void) nvlist_add_uint32(nvl, "SbNode", MCREG_FIELD_CMN(nip, SbNode));
356	(void) nvlist_add_uint32(nvl, "LkNode", MCREG_FIELD_CMN(nip, LkNode));
357	(void) nvlist_add_uint32(nvl, "SystemCoreCount",
358	    HT_SYSTEMCORECOUNT(nip));
359
360	(void) nvlist_add_uint32(nvl, "C0Unit", MCREG_FIELD_CMN(uip, C0Unit));
361	(void) nvlist_add_uint32(nvl, "C1Unit", MCREG_FIELD_CMN(uip, C1Unit));
362	(void) nvlist_add_uint32(nvl, "McUnit", MCREG_FIELD_CMN(uip, McUnit));
363	(void) nvlist_add_uint32(nvl, "HbUnit", MCREG_FIELD_CMN(uip, HbUnit));
364	(void) nvlist_add_uint32(nvl, "SbLink", MCREG_FIELD_CMN(uip, SbLink));
365
366	if (ndcnt <= MC_CHIP_MAXNODES) {
367		for (i = 0; i < ndcnt; i++, htrp++) {
368			BCRte[i] = MCREG_FIELD_CMN(htrp, BCRte);
369			RPRte[i] = MCREG_FIELD_CMN(htrp, RPRte);
370			RQRte[i] = MCREG_FIELD_CMN(htrp, RQRte);
371		}
372
373		(void) nvlist_add_uint32_array(nvl, "BroadcastRoutes",
374		    &BCRte[0], ndcnt);
375		(void) nvlist_add_uint32_array(nvl, "ResponseRoutes",
376		    &RPRte[0], ndcnt);
377		(void) nvlist_add_uint32_array(nvl, "RequestRoutes",
378		    &RQRte[0], ndcnt);
379	}
380
381	(void) nvlist_add_nvlist(mcnvl, "htconfig", nvl);
382	nvlist_free(nvl);
383}
384
385static nvlist_t *
386mc_nvl_create(mc_t *mc)
387{
388	nvlist_t *mcnvl;
389
390	(void) nvlist_alloc(&mcnvl, NV_UNIQUE_NAME, KM_SLEEP);
391
392	/*
393	 * Since this nvlist is used in populating the topo tree changes
394	 * made here may propogate through to changed property names etc
395	 * in the topo tree.  Some properties in the topo tree will be
396	 * contracted via ARC, so be careful what you change here.
397	 */
398	(void) nvlist_add_uint8(mcnvl, MC_NVLIST_VERSTR, MC_NVLIST_VERS1);
399
400	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_NUM, 0);
401	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_REV, 0);
402	(void) nvlist_add_string(mcnvl, "revname", mc->mc_revname);
403	mc_nvl_add_socket(mcnvl, mc);
404	mc_nvl_add_ecctype(mcnvl, mc);
405
406	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_BASE_ADDR, 0);
407	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_LIM_ADDR, 0);
408	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_ILEN, 0);
409	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_ILSEL, 0);
410	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_CSINTLVFCTR, 0);
411	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_DRAMHOLE_SIZE, 0);
412	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_ACCESS_WIDTH, 0);
413	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_CSBANKMAPREG, 0);
414	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_BANKSWZL, 0);
415	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_MOD64MUX, 0);
416	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_SPARECS, 1);
417	mc_nvl_add_prop(mcnvl, mc, MCAMD_PROP_BADCS, 1);
418
419	mc_nvl_add_cslist(mcnvl, mc);
420	mc_nvl_add_dimmlist(mcnvl, mc);
421	mc_nvl_add_htconfig(mcnvl, mc);
422
423	return (mcnvl);
424}
425
426/*
427 * Link a dimm to its associated chip-selects and chip-select lines.
428 * Total the size of all ranks of this dimm.
429 */
430static void
431mc_dimm_csadd(mc_t *mc, mc_dimm_t *mcd, mc_cs_t *mccs, const mcdcfg_csl_t *csl)
432{
433	int factor = (mc->mc_props.mcp_accwidth == 128) ? 2 : 1;
434	uint64_t sz = 0;
435	int i;
436
437	/* Skip to first unused rank slot */
438	for (i = 0; i < MC_CHIP_DIMMRANKMAX; i++) {
439		if (mcd->mcd_cs[i] == NULL) {
440			mcd->mcd_cs[i] = mccs;
441			mcd->mcd_csl[i] = csl;
442			sz += mccs->mccs_props.csp_size / factor;
443			break;
444		} else {
445			sz += mcd->mcd_cs[i]->mccs_props.csp_size / factor;
446		}
447	}
448
449	ASSERT(i != MC_CHIP_DIMMRANKMAX);
450
451	mcd->mcd_size = sz;
452}
453
454/*
455 * Create a dimm structure and call to link it to its associated chip-selects.
456 */
457static mc_dimm_t *
458mc_dimm_create(mc_t *mc, uint_t num)
459{
460	mc_dimm_t *mcd = kmem_zalloc(sizeof (mc_dimm_t), KM_SLEEP);
461
462	mcd->mcd_hdr.mch_type = MC_NT_DIMM;
463	mcd->mcd_mc = mc;
464	mcd->mcd_num = num;
465
466	return (mcd);
467}
468
469/*
470 * The chip-select structure includes an array of dimms associated with
471 * that chip-select.  This function fills that array, and also builds
472 * the list of all dimms on this memory controller mc_dimmlist.  The
473 * caller has filled a structure with all there is to know about the
474 * associated dimm(s).
475 */
476static void
477mc_csdimms_create(mc_t *mc, mc_cs_t *mccs, mcdcfg_rslt_t *rsltp)
478{
479	mc_dimm_t *found[MC_CHIP_DIMMPERCS];
480	mc_dimm_t *mcd;
481	int nfound = 0;
482	int i;
483
484	/*
485	 * Has some other chip-select already created this dimm or dimms?
486	 * If so then link to the dimm(s) from the mccs_dimm array,
487	 * record their topo numbers in the csp_dimmnums array, and link
488	 * the dimm(s) to the additional chip-select.
489	 */
490	for (mcd = mc->mc_dimmlist; mcd != NULL; mcd = mcd->mcd_next) {
491		for (i = 0; i < rsltp->ndimm; i++) {
492			if (mcd->mcd_num == rsltp->dimm[i].toponum)
493				found[nfound++] = mcd;
494		}
495	}
496	ASSERT(nfound == 0 || nfound == rsltp->ndimm);
497
498	for (i = 0; i < rsltp->ndimm; i++) {
499		if (nfound == 0) {
500			mcd = mc_dimm_create(mc, rsltp->dimm[i].toponum);
501			if (mc->mc_dimmlist == NULL)
502				mc->mc_dimmlist = mcd;
503			else
504				mc->mc_dimmlast->mcd_next = mcd;
505			mc->mc_dimmlast = mcd;
506		} else {
507			mcd = found[i];
508		}
509
510		mccs->mccs_dimm[i] = mcd;
511		mccs->mccs_csl[i] = rsltp->dimm[i].cslp;
512		mccs->mccs_props.csp_dimmnums[i] = mcd->mcd_num;
513		mc_dimm_csadd(mc, mcd, mccs, rsltp->dimm[i].cslp);
514
515	}
516
517	/* The rank number is constant across all constituent dimm(s) */
518	mccs->mccs_props.csp_dimmrank = rsltp->dimm[0].cslp->csl_rank;
519}
520
521/*
522 * mc_dimmlist_create is called after we have discovered all enabled
523 * (and spare or testfailed on revs F and G) chip-selects on the
524 * given memory controller.  For each chip-select we must derive
525 * the associated dimms, remembering that a chip-select csbase/csmask
526 * pair may be associated with up to 2 chip-select lines (in 128 bit mode)
527 * and that any one dimm may be associated with 1, 2, or 4 chip-selects
528 * depending on whether it is single, dual or quadrank.
529 */
530static void
531mc_dimmlist_create(mc_t *mc)
532{
533	union mcreg_dramcfg_hi *drcfghip =
534	    (union mcreg_dramcfg_hi *)(&mc->mc_cfgregs.mcr_dramcfghi);
535	mc_props_t *mcp = &mc->mc_props;
536	uint32_t rev = mcp->mcp_rev;
537	mc_cs_t *mccs;
538	int r4 = 0, s4 = 0;
539
540	/*
541	 * Are we dealing with quadrank registered dimms?
542	 *
543	 * For socket 940 we can't tell and we'll assume we're not.
544	 * This can be over-ridden by the admin in /etc/system by setting
545	 * mc_quadranksupport nonzero.  A possible optimisation in systems
546	 * that export an SMBIOS table would be to count the number of
547	 * dimm slots per cpu - more than 4 would indicate no quadrank support
548	 * and 4 or fewer would indicate that if we see any of the upper
549	 * chip-selects enabled then a quadrank dimm is present.
550	 *
551	 * For socket F(1207) we can check a bit in the dram config high reg.
552	 *
553	 * Other socket types do not support registered dimms.
554	 */
555	if (mc->mc_socket == X86_SOCKET_940)
556		r4 = mc_quadranksupport != 0;
557	else if (mc->mc_socket == X86_SOCKET_F1207)
558		r4 = MCREG_FIELD_F_revFG(drcfghip, FourRankRDimm);
559
560	/*
561	 * Are we dealing with quadrank SO-DIMMs?  These are supported
562	 * in AM2 and S1g1 packages only, but in all rev F/G cases we
563	 * can detect their presence via a bit in the dram config high reg.
564	 */
565	if (MC_REV_MATCH(rev, MC_F_REVS_FG))
566		s4 = MCREG_FIELD_F_revFG(drcfghip, FourRankSODimm);
567
568	for (mccs = mc->mc_cslist; mccs != NULL; mccs = mccs->mccs_next) {
569		mcdcfg_rslt_t rslt;
570
571		/*
572		 * If lookup fails we will not create dimm structures for
573		 * this chip-select.  In the mc_cs_t we will have both
574		 * csp_dimmnum members set to MC_INVALNUM and patounum
575		 * code will see from those that we do not have dimm info
576		 * for this chip-select.
577		 */
578		if (mcdcfg_lookup(rev, mcp->mcp_mod64mux, mcp->mcp_accwidth,
579		    mccs->mccs_props.csp_num, mc->mc_socket,
580		    r4, s4, &rslt) < 0)
581			continue;
582
583		mc_csdimms_create(mc, mccs, &rslt);
584	}
585}
586
587static mc_cs_t *
588mc_cs_create(mc_t *mc, uint_t num, uint64_t base, uint64_t mask, size_t sz,
589    int csbe, int spare, int testfail)
590{
591	mc_cs_t *mccs = kmem_zalloc(sizeof (mc_cs_t), KM_SLEEP);
592	mccs_props_t *csp = &mccs->mccs_props;
593	int i;
594
595	mccs->mccs_hdr.mch_type = MC_NT_CS;
596	mccs->mccs_mc = mc;
597	csp->csp_num = num;
598	csp->csp_base = base;
599	csp->csp_mask = mask;
600	csp->csp_size = sz;
601	csp->csp_csbe = csbe;
602	csp->csp_spare = spare;
603	csp->csp_testfail = testfail;
604
605	for (i = 0; i < MC_CHIP_DIMMPERCS; i++)
606		csp->csp_dimmnums[i] = MC_INVALNUM;
607
608	if (spare)
609		mc->mc_props.mcp_sparecs = num;
610
611	return (mccs);
612}
613
614/*
615 * For any cs# of this mc marked TestFail generate an ereport with
616 * resource identifying the associated dimm(s).
617 */
618static void
619mc_report_testfails(mc_t *mc)
620{
621	mc_unum_t unum;
622	mc_cs_t *mccs;
623	int i;
624
625	for (mccs = mc->mc_cslist; mccs != NULL; mccs = mccs->mccs_next) {
626		if (mccs->mccs_props.csp_testfail) {
627			unum.unum_board = 0;
628			unum.unum_chip = mc->mc_props.mcp_num;
629			unum.unum_mc = 0;
630			unum.unum_chan = MC_INVALNUM;
631			unum.unum_cs = mccs->mccs_props.csp_num;
632			unum.unum_rank = mccs->mccs_props.csp_dimmrank;
633			unum.unum_offset = MCAMD_RC_INVALID_OFFSET;
634			for (i = 0; i < MC_CHIP_DIMMPERCS; i++)
635				unum.unum_dimms[i] = MC_INVALNUM;
636
637			mcamd_ereport_post(mc, FM_EREPORT_CPU_AMD_MC_TESTFAIL,
638			    &unum,
639			    FM_EREPORT_PAYLOAD_FLAGS_CPU_AMD_MC_TESTFAIL);
640		}
641	}
642}
643
644/*
645 * Function 0 - HyperTransport Technology Configuration
646 */
647static void
648mc_mkprops_htcfg(mc_pcicfg_hdl_t cfghdl, mc_t *mc)
649{
650	union mcreg_nodeid nodeid;
651	off_t offset;
652	int i;
653
654	mc->mc_cfgregs.mcr_htnodeid = MCREG_VAL32(&nodeid) =
655	    mc_pcicfg_get32(cfghdl, MC_HT_REG_NODEID);
656
657	mc->mc_cfgregs.mcr_htunitid = mc_pcicfg_get32(cfghdl, MC_HT_REG_UNITID);
658
659	for (i = 0, offset = MC_HT_REG_RTBL_NODE_0;
660	    i < HT_COHERENTNODES(&nodeid);
661	    i++, offset += MC_HT_REG_RTBL_INCR)
662		mc->mc_cfgregs.mcr_htroute[i] = mc_pcicfg_get32(cfghdl, offset);
663}
664
665/*
666 * Function 1 Configuration - Address Map (see BKDG 3.4.4 DRAM Address Map)
667 *
668 * Read the Function 1 Address Map for each potential DRAM node.  The Base
669 * Address for a node gives the starting system address mapped at that node,
670 * and the limit gives the last valid address mapped at that node.  Regions for
671 * different nodes should not overlap, unless node-interleaving is enabled.
672 * The base register also indicates the node-interleaving settings (IntlvEn).
673 * The limit register includes IntlvSel which determines which 4K blocks will
674 * be routed to this node and the destination node ID for addresses that fall
675 * within the [base, limit] range - this must match the pair number.
676 */
677static void
678mc_mkprops_addrmap(mc_pcicfg_hdl_t cfghdl, mc_t *mc)
679{
680	union mcreg_drambase basereg;
681	union mcreg_dramlimit limreg;
682	mc_props_t *mcp = &mc->mc_props;
683	mc_cfgregs_t *mcr = &mc->mc_cfgregs;
684	union mcreg_dramhole hole;
685	int nodeid = mc->mc_props.mcp_num;
686
687	mcr->mcr_drambase = MCREG_VAL32(&basereg) = mc_pcicfg_get32(cfghdl,
688	    MC_AM_REG_DRAMBASE_0 + nodeid * MC_AM_REG_DRAM_INCR);
689
690	mcr->mcr_dramlimit = MCREG_VAL32(&limreg) = mc_pcicfg_get32(cfghdl,
691	    MC_AM_REG_DRAMLIM_0 + nodeid * MC_AM_REG_DRAM_INCR);
692
693	/*
694	 * Derive some "cooked" properties for nodes that have a range of
695	 * physical addresses that are read or write enabled and for which
696	 * the DstNode matches the node we are attaching.
697	 */
698	if (MCREG_FIELD_CMN(&limreg, DRAMLimiti) != 0 &&
699	    MCREG_FIELD_CMN(&limreg, DstNode) == nodeid &&
700	    (MCREG_FIELD_CMN(&basereg, WE) || MCREG_FIELD_CMN(&basereg, RE))) {
701		mcp->mcp_base = MC_DRAMBASE(&basereg);
702		mcp->mcp_lim = MC_DRAMLIM(&limreg);
703		mcp->mcp_ilen = MCREG_FIELD_CMN(&basereg, IntlvEn);
704		mcp->mcp_ilsel = MCREG_FIELD_CMN(&limreg, IntlvSel);
705	}
706
707	/*
708	 * The Function 1 DRAM Hole Address Register tells us which node(s)
709	 * own the DRAM space that is hoisted above 4GB, together with the
710	 * hole base and offset for this node.  This was introduced in
711	 * revision E.
712	 */
713	if (MC_REV_ATLEAST(mc->mc_props.mcp_rev, MC_F_REV_E)) {
714		mcr->mcr_dramhole = MCREG_VAL32(&hole) =
715		    mc_pcicfg_get32(cfghdl, MC_AM_REG_HOLEADDR);
716
717		if (MCREG_FIELD_CMN(&hole, DramHoleValid))
718			mcp->mcp_dramhole_size = MC_DRAMHOLE_SIZE(&hole);
719	}
720}
721
722/*
723 * Read some function 3 parameters via PCI Mechanism 1 accesses (which
724 * will serialize any NB accesses).
725 */
726static void
727mc_getmiscctl(mc_t *mc)
728{
729	uint32_t rev = mc->mc_props.mcp_rev;
730	union mcreg_nbcfg nbcfg;
731	union mcreg_sparectl sparectl;
732
733	mc->mc_cfgregs.mcr_nbcfg = MCREG_VAL32(&nbcfg) =
734	    mc_pcicfg_get32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_NBCFG);
735
736	if (MC_REV_MATCH(rev, MC_F_REVS_FG)) {
737		mc->mc_cfgregs.mcr_sparectl = MCREG_VAL32(&sparectl) =
738		    mc_pcicfg_get32_nohdl(mc, MC_FUNC_MISCCTL,
739		    MC_CTL_REG_SPARECTL);
740
741		if (MCREG_FIELD_F_revFG(&sparectl, SwapDone)) {
742			mc->mc_props.mcp_badcs =
743			    MCREG_FIELD_F_revFG(&sparectl, BadDramCs);
744		}
745	}
746}
747
748static int
749csbasecmp(mc_cs_t **csapp, mc_cs_t **csbpp)
750{
751	uint64_t basea = (*csapp)->mccs_props.csp_base;
752	uint64_t baseb = (*csbpp)->mccs_props.csp_base;
753
754	if (basea == baseb)
755		return (0);
756	else if (basea < baseb)
757		return (-1);
758	else
759		return (1);
760}
761
762/*
763 * The following are for use in simulating TestFail for a chip-select
764 * without poking at the hardware (which tends to get upset if you do
765 * since the BIOS needs to restart to map a failed cs out).  For internal
766 * testing only!  Note that setting these does not give the full experience -
767 * the select chip-select *is* enabled and can give errors etc and the
768 * patounum logic will get confused.
769 */
770int testfail_mcnum = -1;
771int testfail_csnum = -1;
772
773/*
774 * Function 2 configuration - DRAM Controller
775 */
776static void
777mc_mkprops_dramctl(mc_pcicfg_hdl_t cfghdl, mc_t *mc)
778{
779	union mcreg_csbase base[MC_CHIP_NCS];
780	union mcreg_csmask mask[MC_CHIP_NCS];
781	union mcreg_dramcfg_lo drcfg_lo;
782	union mcreg_dramcfg_hi drcfg_hi;
783	union mcreg_drammisc drmisc;
784	union mcreg_bankaddrmap baddrmap;
785	mc_props_t *mcp = &mc->mc_props;
786	mc_cfgregs_t *mcr = &mc->mc_cfgregs;
787	int maskdivisor;
788	int wide = 0;
789	uint32_t rev = mc->mc_props.mcp_rev;
790	int i;
791	mcamd_hdl_t hdl;
792
793	mcamd_mkhdl(&hdl);	/* to call into common code */
794
795	/*
796	 * Read Function 2 DRAM Configuration High and Low registers.  The High
797	 * part is mostly concerned with memory clocks etc and we'll not have
798	 * any use for that.  The Low component tells us if ECC is enabled,
799	 * if we're in 64- or 128-bit MC mode, how the upper chip-selects
800	 * are mapped, which chip-select pairs are using x4 parts, etc.
801	 */
802	MCREG_VAL32(&drcfg_lo) = mc_pcicfg_get32(cfghdl, MC_DC_REG_DRAMCFGLO);
803	MCREG_VAL32(&drcfg_hi) = mc_pcicfg_get32(cfghdl, MC_DC_REG_DRAMCFGHI);
804	mcr->mcr_dramcfglo = MCREG_VAL32(&drcfg_lo);
805	mcr->mcr_dramcfghi = MCREG_VAL32(&drcfg_hi);
806
807	/*
808	 * Note the DRAM controller width.  The 64/128 bit is in a different
809	 * bit position for revision F and G.
810	 */
811	if (MC_REV_MATCH(rev, MC_F_REVS_FG)) {
812		wide = MCREG_FIELD_F_revFG(&drcfg_lo, Width128);
813	} else {
814		wide = MCREG_FIELD_F_preF(&drcfg_lo, Width128);
815	}
816	mcp->mcp_accwidth = wide ? 128 : 64;
817
818	/*
819	 * Read Function 2 DRAM Controller Miscellaenous Regsiter for those
820	 * revs that support it.  This include the Mod64Mux indication on
821	 * these revs - for rev E it is in DRAM config low.
822	 */
823	if (MC_REV_MATCH(rev, MC_F_REVS_FG)) {
824		mcr->mcr_drammisc = MCREG_VAL32(&drmisc) =
825		    mc_pcicfg_get32(cfghdl, MC_DC_REG_DRAMMISC);
826		mcp->mcp_mod64mux = MCREG_FIELD_F_revFG(&drmisc, Mod64Mux);
827	} else if (MC_REV_MATCH(rev, MC_F_REV_E)) {
828		mcp->mcp_mod64mux = MCREG_FIELD_F_preF(&drcfg_lo, Mod64BitMux);
829	}
830
831	/*
832	 * Read Function 2 DRAM Bank Address Mapping.  This encodes the
833	 * type of DIMM module in use for each chip-select pair.
834	 * Prior ro revision F it also tells us whether BankSwizzle mode
835	 * is enabled - in rev F that has moved to dram config hi register.
836	 */
837	mcp->mcp_csbankmapreg = MCREG_VAL32(&baddrmap) =
838	    mc_pcicfg_get32(cfghdl, MC_DC_REG_BANKADDRMAP);
839
840	/*
841	 * Determine whether bank swizzle mode is active.  Bank swizzling was
842	 * introduced as an option in rev E,  but the bit that indicates it
843	 * is enabled has moved in revs F/G.
844	 */
845	if (MC_REV_MATCH(rev, MC_F_REV_E)) {
846		mcp->mcp_bnkswzl =
847		    MCREG_FIELD_F_preF(&baddrmap, BankSwizzleMode);
848	} else if (MC_REV_MATCH(rev, MC_F_REVS_FG)) {
849		mcp->mcp_bnkswzl = MCREG_FIELD_F_revFG(&drcfg_hi,
850		    BankSwizzleMode);
851	}
852
853	/*
854	 * Read the DRAM CS Base and DRAM CS Mask registers.  Revisions prior
855	 * to F have an equal number of base and mask registers; revision F
856	 * has twice as many base registers as masks.
857	 */
858	maskdivisor = MC_REV_MATCH(rev, MC_F_REVS_FG) ? 2 : 1;
859
860	mc_prop_read_pair(cfghdl,
861	    (uint32_t *)base, MC_DC_REG_CSBASE_0, MC_CHIP_NCS,
862	    (uint32_t *)mask, MC_DC_REG_CSMASK_0, MC_CHIP_NCS / maskdivisor,
863	    MC_DC_REG_CS_INCR);
864
865	/*
866	 * Create a cs node for each enabled chip-select as well as
867	 * any appointed online spare chip-selects and for any that have
868	 * failed test.
869	 */
870	for (i = 0; i < MC_CHIP_NCS; i++) {
871		mc_cs_t *mccs;
872		uint64_t csbase, csmask;
873		size_t sz;
874		int csbe, spare, testfail;
875
876		if (MC_REV_MATCH(rev, MC_F_REVS_FG)) {
877			csbe = MCREG_FIELD_F_revFG(&base[i], CSEnable);
878			spare = MCREG_FIELD_F_revFG(&base[i], Spare);
879			testfail = MCREG_FIELD_F_revFG(&base[i], TestFail);
880		} else {
881			csbe = MCREG_FIELD_F_preF(&base[i], CSEnable);
882			spare = 0;
883			testfail = 0;
884		}
885
886		/* Testing hook */
887		if (testfail_mcnum != -1 && testfail_csnum != -1 &&
888		    mcp->mcp_num == testfail_mcnum && i == testfail_csnum) {
889			csbe = spare = 0;
890			testfail = 1;
891			cmn_err(CE_NOTE, "Pretending MC %d CS %d failed test",
892			    testfail_mcnum, testfail_csnum);
893		}
894
895		/*
896		 * If the chip-select is not enabled then skip it unless
897		 * it is a designated online spare or is marked with TestFail.
898		 */
899		if (!csbe && !(spare || testfail))
900			continue;
901
902		/*
903		 * For an enabled or spare chip-select the Bank Address Mapping
904		 * register will be valid as will the chip-select mask.  The
905		 * base will not be valid but we'll read and store it anyway.
906		 * We will not know whether the spare is already swapped in
907		 * until MC function 3 attaches.
908		 */
909		if (csbe || spare) {
910			if (mcamd_cs_size(&hdl, (mcamd_node_t *)mc, i, &sz) < 0)
911				continue;
912			csbase = MC_CSBASE(&base[i], rev);
913			csmask = MC_CSMASK(&mask[i / maskdivisor], rev);
914		} else {
915			sz = 0;
916			csbase = csmask = 0;
917		}
918
919		mccs = mc_cs_create(mc, i, csbase, csmask, sz,
920		    csbe, spare, testfail);
921
922		if (mc->mc_cslist == NULL)
923			mc->mc_cslist = mccs;
924		else
925			mc->mc_cslast->mccs_next = mccs;
926		mc->mc_cslast = mccs;
927
928		mccs->mccs_cfgregs.csr_csbase = MCREG_VAL32(&base[i]);
929		mccs->mccs_cfgregs.csr_csmask =
930		    MCREG_VAL32(&mask[i / maskdivisor]);
931
932		/*
933		 * Check for cs bank interleaving - some bits clear in the
934		 * lower mask.  All banks must/will have the same lomask bits
935		 * if cs interleaving is active.
936		 */
937		if (csbe && !mcp->mcp_csintlvfctr) {
938			int bitno, ibits = 0;
939			for (bitno = MC_CSMASKLO_LOBIT(rev);
940			    bitno <= MC_CSMASKLO_HIBIT(rev); bitno++) {
941				if (!(csmask & (1 << bitno)))
942					ibits++;
943			}
944			mcp->mcp_csintlvfctr = 1 << ibits;
945		}
946	}
947
948	/*
949	 * If there is no chip-select interleave on this node determine
950	 * whether the chip-select ranks are contiguous or if there
951	 * is a hole.
952	 */
953	if (mcp->mcp_csintlvfctr == 1) {
954		mc_cs_t *csp[MC_CHIP_NCS];
955		mc_cs_t *mccs;
956		int ncsbe = 0;
957
958		for (mccs = mc->mc_cslist; mccs != NULL;
959		    mccs = mccs->mccs_next) {
960			if (mccs->mccs_props.csp_csbe)
961				csp[ncsbe++] = mccs;
962		}
963
964		if (ncsbe != 0) {
965			qsort((void *)csp, ncsbe, sizeof (mc_cs_t *),
966			    (int (*)(const void *, const void *))csbasecmp);
967
968			for (i = 1; i < ncsbe; i++) {
969				if (csp[i]->mccs_props.csp_base !=
970				    csp[i - 1]->mccs_props.csp_base +
971				    csp[i - 1]->mccs_props.csp_size)
972					mc->mc_csdiscontig = 1;
973			}
974		}
975	}
976
977
978	/*
979	 * Since we do not attach to MC function 3 go ahead and read some
980	 * config parameters from it now.
981	 */
982	mc_getmiscctl(mc);
983
984	/*
985	 * Now that we have discovered all enabled/spare/testfail chip-selects
986	 * we divine the associated DIMM configuration.
987	 */
988	mc_dimmlist_create(mc);
989}
990
991typedef struct mc_bind_map {
992	const char *bm_bindnm;	 /* attachment binding name */
993	enum mc_funcnum bm_func; /* PCI config space function number for bind */
994	const char *bm_model;	 /* value for device node model property */
995	void (*bm_mkprops)(mc_pcicfg_hdl_t, mc_t *);
996} mc_bind_map_t;
997
998/*
999 * Do not attach to MC function 3 - agpgart already attaches to that.
1000 * Function 3 may be a good candidate for a nexus driver to fan it out
1001 * into virtual devices by functionality.  We will use pci_mech1_getl
1002 * to retrieve the function 3 parameters we require.
1003 */
1004
1005static const mc_bind_map_t mc_bind_map[] = {
1006	{ MC_FUNC_HTCONFIG_BINDNM, MC_FUNC_HTCONFIG,
1007	    "AMD Memory Controller (HT Configuration)", mc_mkprops_htcfg },
1008	{ MC_FUNC_ADDRMAP_BINDNM, MC_FUNC_ADDRMAP,
1009	    "AMD Memory Controller (Address Map)", mc_mkprops_addrmap },
1010	{ MC_FUNC_DRAMCTL_BINDNM, MC_FUNC_DRAMCTL,
1011	    "AMD Memory Controller (DRAM Controller & HT Trace)",
1012	    mc_mkprops_dramctl },
1013	NULL
1014};
1015
1016/*ARGSUSED*/
1017static int
1018mc_open(dev_t *devp, int flag, int otyp, cred_t *credp)
1019{
1020	if (otyp != OTYP_CHR)
1021		return (EINVAL);
1022
1023	rw_enter(&mc_lock, RW_READER);
1024	if (mc_lookup_by_chipid(getminor(*devp)) == NULL) {
1025		rw_exit(&mc_lock);
1026		return (EINVAL);
1027	}
1028	rw_exit(&mc_lock);
1029
1030	return (0);
1031}
1032
1033/*ARGSUSED*/
1034static int
1035mc_close(dev_t dev, int flag, int otyp, cred_t *credp)
1036{
1037	return (0);
1038}
1039
1040/*
1041 * Enable swap from chip-select csnum to the spare chip-select on this
1042 * memory controller (if any).
1043 */
1044
1045int mc_swapdonetime = 30;	/* max number of seconds to wait for SwapDone */
1046
1047static int
1048mc_onlinespare(mc_t *mc, int csnum)
1049{
1050	mc_props_t *mcp = &mc->mc_props;
1051	union mcreg_sparectl sparectl;
1052	union mcreg_scrubctl scrubctl;
1053	mc_cs_t *mccs;
1054	hrtime_t tmax;
1055	int i = 0;
1056
1057	ASSERT(RW_WRITE_HELD(&mc_lock));
1058
1059	if (!MC_REV_MATCH(mcp->mcp_rev, MC_F_REVS_FG))
1060		return (ENOTSUP);	/* MC rev does not offer online spare */
1061	else if (mcp->mcp_sparecs == MC_INVALNUM)
1062		return (ENODEV);	/* Supported, but no spare configured */
1063	else if (mcp->mcp_badcs != MC_INVALNUM)
1064		return (EBUSY);		/* Spare already swapped in */
1065	else if (csnum == mcp->mcp_sparecs)
1066		return (EINVAL);	/* Can't spare the spare! */
1067
1068	for (mccs = mc->mc_cslist; mccs != NULL; mccs = mccs->mccs_next) {
1069		if (mccs->mccs_props.csp_num == csnum)
1070			break;
1071	}
1072	if (mccs == NULL)
1073		return (EINVAL);	/* nominated bad CS does not exist */
1074
1075	/*
1076	 * If the DRAM Scrubber is not enabled then the swap cannot succeed.
1077	 */
1078	MCREG_VAL32(&scrubctl) = mc_pcicfg_get32_nohdl(mc, MC_FUNC_MISCCTL,
1079	    MC_CTL_REG_SCRUBCTL);
1080	if (MCREG_FIELD_CMN(&scrubctl, DramScrub) == 0)
1081		return (ENODEV);	/* DRAM scrubber not enabled */
1082
1083	/*
1084	 * Read Online Spare Comtrol Register again, just in case our
1085	 * state does not reflect reality.
1086	 */
1087	MCREG_VAL32(&sparectl) = mc_pcicfg_get32_nohdl(mc, MC_FUNC_MISCCTL,
1088	    MC_CTL_REG_SPARECTL);
1089
1090	if (MCREG_FIELD_F_revFG(&sparectl, SwapDone))
1091		return (EBUSY);
1092
1093	/* Write to the BadDramCs field */
1094	MCREG_FIELD_F_revFG(&sparectl, BadDramCs) = csnum;
1095	mc_pcicfg_put32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_SPARECTL,
1096	    MCREG_VAL32(&sparectl));
1097
1098	/* And request that the swap to the spare start */
1099	MCREG_FIELD_F_revFG(&sparectl, SwapEn) = 1;
1100	mc_pcicfg_put32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_SPARECTL,
1101	    MCREG_VAL32(&sparectl));
1102
1103	/*
1104	 * Poll for SwapDone - we have disabled notification by interrupt.
1105	 * Swap takes "several CPU cycles, depending on the DRAM speed, but
1106	 * is performed in the background" (Family 0Fh Bios Porting Guide).
1107	 * We're in a slow ioctl path so there is no harm in waiting around
1108	 * a bit - consumers of the ioctl must be aware that it may take
1109	 * a moment.  We will poll for up to mc_swapdonetime seconds,
1110	 * limiting that to 120s.
1111	 *
1112	 * The swap is performed by the DRAM scrubber (which must be enabled)
1113	 * whose scrub rate is accelerated for the duration of the swap.
1114	 * The maximum swap rate is 40.0ns per 64 bytes, so the maximum
1115	 * supported cs size of 16GB would take 10.7s at that max rate
1116	 * of 25000000 scrubs/second.
1117	 */
1118	tmax = gethrtime() + MIN(mc_swapdonetime, 120) * 1000000000ULL;
1119	do {
1120		if (i++ < 20)
1121			delay(drv_usectohz(100000));	/* 0.1s for up to 2s */
1122		else
1123			delay(drv_usectohz(500000));	/* 0.5s */
1124
1125		MCREG_VAL32(&sparectl) = mc_pcicfg_get32_nohdl(mc,
1126		    MC_FUNC_MISCCTL, MC_CTL_REG_SPARECTL);
1127	} while (!MCREG_FIELD_F_revFG(&sparectl, SwapDone) &&
1128	    gethrtime() < tmax);
1129
1130	if (!MCREG_FIELD_F_revFG(&sparectl, SwapDone))
1131		return (ETIME);		/* Operation timed out */
1132
1133	mcp->mcp_badcs = csnum;
1134	mc->mc_cfgregs.mcr_sparectl = MCREG_VAL32(&sparectl);
1135	mc->mc_spareswaptime = gethrtime();
1136
1137	return (0);
1138}
1139
1140/*ARGSUSED*/
1141static int
1142mc_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
1143{
1144	int rc = 0;
1145	mc_t *mc;
1146
1147	if (cmd != MC_IOC_SNAPSHOT_INFO && cmd != MC_IOC_SNAPSHOT &&
1148	    cmd != MC_IOC_ONLINESPARE_EN)
1149		return (EINVAL);
1150
1151	rw_enter(&mc_lock, RW_READER);
1152
1153	if ((mc = mc_lookup_by_chipid(getminor(dev))) == NULL) {
1154		rw_exit(&mc_lock);
1155		return (EINVAL);
1156	}
1157
1158	switch (cmd) {
1159	case MC_IOC_SNAPSHOT_INFO: {
1160		mc_snapshot_info_t mcs;
1161
1162		if (mc_snapshot_update(mc) < 0) {
1163			rw_exit(&mc_lock);
1164			return (EIO);
1165		}
1166
1167		mcs.mcs_size = mc->mc_snapshotsz;
1168		mcs.mcs_gen = mc->mc_snapshotgen;
1169
1170		if (ddi_copyout(&mcs, (void *)arg, sizeof (mc_snapshot_info_t),
1171		    mode) < 0)
1172			rc = EFAULT;
1173		break;
1174	}
1175
1176	case MC_IOC_SNAPSHOT:
1177		if (mc_snapshot_update(mc) < 0) {
1178			rw_exit(&mc_lock);
1179			return (EIO);
1180		}
1181
1182		if (ddi_copyout(mc->mc_snapshot, (void *)arg, mc->mc_snapshotsz,
1183		    mode) < 0)
1184			rc = EFAULT;
1185		break;
1186
1187	case MC_IOC_ONLINESPARE_EN:
1188		if (drv_priv(credp) != 0) {
1189			rw_exit(&mc_lock);
1190			return (EPERM);
1191		}
1192
1193		if (!rw_tryupgrade(&mc_lock)) {
1194			rw_exit(&mc_lock);
1195			return (EAGAIN);
1196		}
1197
1198		if ((rc = mc_onlinespare(mc, (int)arg)) == 0) {
1199			mc_snapshot_destroy(mc);
1200			nvlist_free(mc->mc_nvl);
1201			mc->mc_nvl = mc_nvl_create(mc);
1202		}
1203
1204		break;
1205	}
1206
1207	rw_exit(&mc_lock);
1208
1209	return (rc);
1210}
1211
1212static struct cb_ops mc_cb_ops = {
1213	mc_open,
1214	mc_close,
1215	nodev,		/* not a block driver */
1216	nodev,		/* no print routine */
1217	nodev,		/* no dump routine */
1218	nodev,		/* no read routine */
1219	nodev,		/* no write routine */
1220	mc_ioctl,
1221	nodev,		/* no devmap routine */
1222	nodev,		/* no mmap routine */
1223	nodev,		/* no segmap routine */
1224	nochpoll,	/* no chpoll routine */
1225	ddi_prop_op,
1226	0,		/* not a STREAMS driver */
1227	D_NEW | D_MP,	/* safe for multi-thread/multi-processor */
1228};
1229
1230/*ARGSUSED*/
1231static int
1232mc_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1233{
1234	int rc = DDI_SUCCESS;
1235	mc_t *mc;
1236
1237	if (infocmd != DDI_INFO_DEVT2DEVINFO &&
1238	    infocmd != DDI_INFO_DEVT2INSTANCE) {
1239		*result = NULL;
1240		return (DDI_FAILURE);
1241	}
1242
1243	rw_enter(&mc_lock, RW_READER);
1244
1245	if ((mc = mc_lookup_by_chipid(getminor((dev_t)arg))) == NULL ||
1246	    mc->mc_funcs[MC_FUNC_DEVIMAP].mcf_devi == NULL) {
1247		rc = DDI_FAILURE;
1248	} else if (infocmd == DDI_INFO_DEVT2DEVINFO) {
1249		*result = mc->mc_funcs[MC_FUNC_DEVIMAP].mcf_devi;
1250	} else {
1251		*result = (void *)(uintptr_t)
1252		    mc->mc_funcs[MC_FUNC_DEVIMAP].mcf_instance;
1253	}
1254
1255	rw_exit(&mc_lock);
1256
1257	return (rc);
1258}
1259
1260/*ARGSUSED2*/
1261static int
1262mc_fm_handle(dev_info_t *dip, ddi_fm_error_t *fmerr, const void *arg)
1263{
1264	pci_ereport_post(dip, fmerr, NULL);
1265	return (fmerr->fme_status);
1266}
1267
1268static void
1269mc_fm_init(dev_info_t *dip)
1270{
1271	int fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE;
1272	ddi_fm_init(dip, &fmcap, NULL);
1273	pci_ereport_setup(dip);
1274	ddi_fm_handler_register(dip, mc_fm_handle, NULL);
1275}
1276
1277static void
1278mc_read_smbios(mc_t *mc, dev_info_t *dip)
1279{
1280
1281	uint16_t bdf;
1282	pci_regspec_t *pci_rp = NULL;
1283	uint32_t phys_hi;
1284	int m = 0;
1285	uint_t chip_inst;
1286	int rc = 0;
1287
1288	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg",
1289	    (caddr_t)&pci_rp, &m) == DDI_SUCCESS) {
1290		phys_hi = pci_rp->pci_phys_hi;
1291		bdf = (uint16_t)(PCI_REG_BDFR_G(phys_hi) >>
1292		    PCI_REG_FUNC_SHIFT);
1293		kmem_free(pci_rp, m);
1294		pci_rp = NULL;
1295
1296		rc = fm_smb_mc_chipinst(bdf, &chip_inst);
1297		if (rc == 0) {
1298			mc->smb_chipid = chip_inst;
1299		} else {
1300#ifdef DEBUG
1301			cmn_err(CE_NOTE, "!mc read smbios chip info failed");
1302#endif /* DEBUG */
1303			return;
1304		}
1305		mc->smb_bboard = fm_smb_mc_bboards(bdf);
1306#ifdef DEBUG
1307		if (mc->smb_bboard == NULL)
1308			cmn_err(CE_NOTE,
1309			    "!mc read smbios base boards info failed");
1310#endif /* DEBUG */
1311	}
1312
1313	if (pci_rp != NULL)
1314		kmem_free(pci_rp, m);
1315}
1316
1317/*ARGSUSED*/
1318static int
1319mc_create_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3)
1320{
1321	chipid_t chipid = *((chipid_t *)arg1);
1322	cmi_hdl_t *hdlp = (cmi_hdl_t *)arg2;
1323
1324	if (cmi_hdl_chipid(whdl) == chipid) {
1325		cmi_hdl_hold(whdl);	/* short-term hold */
1326		*hdlp = whdl;
1327		return (CMI_HDL_WALK_DONE);
1328	} else {
1329		return (CMI_HDL_WALK_NEXT);
1330	}
1331}
1332
1333static mc_t *
1334mc_create(chipid_t chipid, dev_info_t *dip)
1335{
1336	mc_t *mc;
1337	cmi_hdl_t hdl = NULL;
1338
1339	ASSERT(RW_WRITE_HELD(&mc_lock));
1340
1341	/*
1342	 * Find a handle for one of a chip's CPU.
1343	 *
1344	 * We can use one of the chip's CPUs since all cores
1345	 * of a chip share the same revision and socket type.
1346	 */
1347	cmi_hdl_walk(mc_create_cb, (void *)&chipid, (void *)&hdl, NULL);
1348	if (hdl == NULL)
1349		return (NULL);	/* no cpu for this chipid found! */
1350
1351	mc = kmem_zalloc(sizeof (mc_t), KM_SLEEP);
1352
1353	mc->mc_hdr.mch_type = MC_NT_MC;
1354	mc->mc_props.mcp_num = chipid;
1355	mc->mc_props.mcp_sparecs = MC_INVALNUM;
1356	mc->mc_props.mcp_badcs = MC_INVALNUM;
1357
1358	mc->mc_props.mcp_rev = cmi_hdl_chiprev(hdl);
1359	mc->mc_revname = cmi_hdl_chiprevstr(hdl);
1360	mc->mc_socket = cmi_hdl_getsockettype(hdl);
1361
1362	mc_read_smbios(mc, dip);
1363
1364	if (mc_list == NULL)
1365		mc_list = mc;
1366	if (mc_last != NULL)
1367		mc_last->mc_next = mc;
1368
1369	mc->mc_next = NULL;
1370	mc_last = mc;
1371
1372	cmi_hdl_rele(hdl);
1373
1374	return (mc);
1375}
1376
1377/*
1378 * Return the maximum scrubbing rate between r1 and r2, where r2 is extracted
1379 * from the specified 'cfg' register value using 'mask' and 'shift'.  If a
1380 * value is zero, scrubbing is off so return the opposite value.  Otherwise
1381 * the maximum rate is the smallest non-zero value of the two values.
1382 */
1383static uint32_t
1384mc_scrubber_max(uint32_t r1, uint32_t cfg, uint32_t mask, uint32_t shift)
1385{
1386	uint32_t r2 = (cfg & mask) >> shift;
1387
1388	if (r1 != 0 && r2 != 0)
1389		return (MIN(r1, r2));
1390
1391	return (r1 ? r1 : r2);
1392}
1393
1394
1395/*
1396 * Enable the memory scrubber.  We must use the mc_pcicfg_{get32,put32}_nohdl
1397 * interfaces since we do not bind to function 3.
1398 */
1399cmi_errno_t
1400mc_scrubber_enable(mc_t *mc)
1401{
1402	mc_props_t *mcp = &mc->mc_props;
1403	chipid_t chipid = (chipid_t)mcp->mcp_num;
1404	uint32_t rev = (uint32_t)mcp->mcp_rev;
1405	mc_cfgregs_t *mcr = &mc->mc_cfgregs;
1406	union mcreg_scrubctl scrubctl;
1407	union mcreg_dramscrublo dalo;
1408	union mcreg_dramscrubhi dahi;
1409
1410	mcr->mcr_scrubctl = MCREG_VAL32(&scrubctl) =
1411	    mc_pcicfg_get32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_SCRUBCTL);
1412
1413	mcr->mcr_scrubaddrlo = MCREG_VAL32(&dalo) =
1414	    mc_pcicfg_get32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_SCRUBADDR_LO);
1415
1416	mcr->mcr_scrubaddrhi = MCREG_VAL32(&dahi) =
1417	    mc_pcicfg_get32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_SCRUBADDR_HI);
1418
1419	if (mc_scrub_policy == MC_SCRUB_BIOSDEFAULT)
1420		return (MCREG_FIELD_CMN(&scrubctl, DramScrub) !=
1421		    AMD_NB_SCRUBCTL_RATE_NONE ?
1422		    CMI_SUCCESS : CMIERR_MC_NOMEMSCRUB);
1423
1424	/*
1425	 * Disable DRAM scrubbing while we fiddle.
1426	 */
1427	MCREG_FIELD_CMN(&scrubctl, DramScrub) = AMD_NB_SCRUBCTL_RATE_NONE;
1428	mc_pcicfg_put32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_SCRUBCTL,
1429	    MCREG_VAL32(&scrubctl));
1430
1431	/*
1432	 * Setup DRAM Scrub Address Low and High registers for the
1433	 * base address of this node, and to select srubber redirect.
1434	 */
1435	MCREG_FIELD_CMN(&dalo, ScrubReDirEn) = 1;
1436	MCREG_FIELD_CMN(&dalo, ScrubAddrLo) =
1437	    AMD_NB_SCRUBADDR_MKLO(mcp->mcp_base);
1438
1439	MCREG_FIELD_CMN(&dahi, ScrubAddrHi) =
1440	    AMD_NB_SCRUBADDR_MKHI(mcp->mcp_base);
1441
1442	mc_pcicfg_put32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_SCRUBADDR_LO,
1443	    MCREG_VAL32(&dalo));
1444	mc_pcicfg_put32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_SCRUBADDR_HI,
1445	    MCREG_VAL32(&dahi));
1446
1447	if (mc_scrub_rate_dram > AMD_NB_SCRUBCTL_RATE_MAX) {
1448		cmn_err(CE_WARN, "mc_scrub_rate_dram is too large; "
1449		    "resetting to 0x%x\n", AMD_NB_SCRUBCTL_RATE_MAX);
1450		mc_scrub_rate_dram = AMD_NB_SCRUBCTL_RATE_MAX;
1451	}
1452
1453	switch (mc_scrub_policy) {
1454	case MC_SCRUB_FIXED:
1455		/* Use the system value checked above */
1456		break;
1457
1458	default:
1459		cmn_err(CE_WARN, "Unknown mc_scrub_policy value %d - "
1460		    "using default policy of MC_SCRUB_MAX", mc_scrub_policy);
1461		/*FALLTHRU*/
1462
1463	case MC_SCRUB_MAX:
1464		mc_scrub_rate_dram = mc_scrubber_max(mc_scrub_rate_dram,
1465		    mcr->mcr_scrubctl, AMD_NB_SCRUBCTL_DRAM_MASK,
1466		    AMD_NB_SCRUBCTL_DRAM_SHIFT);
1467		break;
1468	}
1469
1470	/*
1471	 * OPTERON_ERRATUM_99:
1472	 * This erratum applies on revisions D and earlier.
1473	 * This erratum also applies on revisions E and later,
1474	 * if BIOS uses chip-select hoisting instead of DRAM hole
1475	 * mapping.
1476	 *
1477	 * Do not enable the dram scrubber if the chip-select ranges
1478	 * for the node are not contiguous.
1479	 */
1480	if (mc_scrub_rate_dram != AMD_NB_SCRUBCTL_RATE_NONE &&
1481	    mc->mc_csdiscontig) {
1482		cmn_err(CE_CONT, "?Opteron DRAM scrubber disabled on revision "
1483		    "%s chip %d because DRAM hole is present on this node",
1484		    mc->mc_revname, chipid);
1485		mc_scrub_rate_dram = AMD_NB_SCRUBCTL_RATE_NONE;
1486	}
1487
1488	/*
1489	 * OPTERON_ERRATUM_101:
1490	 * This erratum applies on revisions D and earlier.
1491	 *
1492	 * If the DRAM Base Address register's IntlvEn field indicates that
1493	 * node interleaving is enabled, we must disable the DRAM scrubber
1494	 * and return zero to indicate that Solaris should use s/w instead.
1495	 */
1496	if (mc_scrub_rate_dram != AMD_NB_SCRUBCTL_RATE_NONE &&
1497	    mcp->mcp_ilen != 0 &&
1498	    !X86_CHIPREV_ATLEAST(rev, X86_CHIPREV_AMD_F_REV_E)) {
1499		cmn_err(CE_CONT, "?Opteron DRAM scrubber disabled on revision "
1500		    "%s chip %d because DRAM memory is node-interleaved",
1501		    mc->mc_revname, chipid);
1502		mc_scrub_rate_dram = AMD_NB_SCRUBCTL_RATE_NONE;
1503	}
1504
1505	if (mc_scrub_rate_dram != AMD_NB_SCRUBCTL_RATE_NONE) {
1506		MCREG_FIELD_CMN(&scrubctl, DramScrub) = mc_scrub_rate_dram;
1507		mc_pcicfg_put32_nohdl(mc, MC_FUNC_MISCCTL, MC_CTL_REG_SCRUBCTL,
1508		    MCREG_VAL32(&scrubctl));
1509	}
1510
1511	return (mc_scrub_rate_dram != AMD_NB_SCRUBCTL_RATE_NONE ?
1512	    CMI_SUCCESS : CMIERR_MC_NOMEMSCRUB);
1513}
1514
1515/*ARGSUSED*/
1516static int
1517mc_attach_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3)
1518{
1519	mc_t *mc = (mc_t *)arg1;
1520	mcamd_prop_t chipid = *((mcamd_prop_t *)arg2);
1521
1522	if (cmi_hdl_chipid(whdl) == chipid) {
1523		mcamd_mc_register(whdl, mc);
1524	}
1525
1526	return (CMI_HDL_WALK_NEXT);
1527}
1528
1529static int mc_sw_scrub_disabled = 0;
1530
1531static int
1532mc_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
1533{
1534	mc_pcicfg_hdl_t cfghdl;
1535	const mc_bind_map_t *bm;
1536	const char *bindnm;
1537	char *unitstr = NULL;
1538	enum mc_funcnum func;
1539	long unitaddr;
1540	int chipid, rc;
1541	mc_t *mc;
1542
1543	/*
1544	 * This driver has no hardware state, but does
1545	 * claim to have a reg property, so it will be
1546	 * called on suspend.  It is probably better to
1547	 * make sure it doesn't get called on suspend,
1548	 * but it is just as easy to make sure we just
1549	 * return DDI_SUCCESS if called.
1550	 */
1551	if (cmd == DDI_RESUME)
1552		return (DDI_SUCCESS);
1553
1554	if (cmd != DDI_ATTACH || mc_no_attach != 0)
1555		return (DDI_FAILURE);
1556
1557	bindnm = ddi_binding_name(dip);
1558	for (bm = mc_bind_map; bm->bm_bindnm != NULL; bm++) {
1559		if (strcmp(bindnm, bm->bm_bindnm) == 0) {
1560			func = bm->bm_func;
1561			break;
1562		}
1563	}
1564
1565	if (bm->bm_bindnm == NULL)
1566		return (DDI_FAILURE);
1567
1568	/*
1569	 * We need the device number, which corresponds to the processor node
1570	 * number plus 24.  The node number can then be used to associate this
1571	 * memory controller device with a given processor chip.
1572	 */
1573	if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip,
1574	    DDI_PROP_DONTPASS, "unit-address", &unitstr) != DDI_PROP_SUCCESS) {
1575		cmn_err(CE_WARN, "failed to find unit-address for %s", bindnm);
1576		return (DDI_FAILURE);
1577	}
1578
1579	rc = ddi_strtol(unitstr, NULL, 16, &unitaddr);
1580	ASSERT(rc == 0 && unitaddr >= MC_AMD_DEV_OFFSET);
1581
1582	if (rc != 0 || unitaddr < MC_AMD_DEV_OFFSET) {
1583		cmn_err(CE_WARN, "failed to parse unit address %s for %s\n",
1584		    unitstr, bindnm);
1585		ddi_prop_free(unitstr);
1586		return (DDI_FAILURE);
1587	}
1588	ddi_prop_free(unitstr);
1589
1590	chipid = unitaddr - MC_AMD_DEV_OFFSET;
1591
1592	rw_enter(&mc_lock, RW_WRITER);
1593
1594	for (mc = mc_list; mc != NULL; mc = mc->mc_next) {
1595		if (mc->mc_props.mcp_num == chipid)
1596			break;
1597	}
1598
1599	/* Integrate this memory controller device into existing set */
1600	if (mc == NULL) {
1601		mc = mc_create(chipid, dip);
1602
1603		if (mc == NULL) {
1604			/*
1605			 * We don't complain here because this is a legitimate
1606			 * path for MP systems.  On those machines, we'll attach
1607			 * before all CPUs have been initialized, and thus the
1608			 * chip verification in mc_create will fail.  We'll be
1609			 * reattached later for those CPUs.
1610			 */
1611			rw_exit(&mc_lock);
1612			return (DDI_FAILURE);
1613		}
1614	} else {
1615		mc_snapshot_destroy(mc);
1616	}
1617
1618	/* Beyond this point, we're committed to creating this node */
1619
1620	mc_fm_init(dip);
1621
1622	ASSERT(mc->mc_funcs[func].mcf_devi == NULL);
1623	mc->mc_funcs[func].mcf_devi = dip;
1624	mc->mc_funcs[func].mcf_instance = ddi_get_instance(dip);
1625
1626	mc->mc_ref++;
1627
1628	/*
1629	 * Add the common properties to this node, and then add any properties
1630	 * that are specific to this node based upon its configuration space.
1631	 */
1632	(void) ddi_prop_update_string(DDI_DEV_T_NONE,
1633	    dip, "model", (char *)bm->bm_model);
1634
1635	(void) ddi_prop_update_int(DDI_DEV_T_NONE,
1636	    dip, "chip-id", mc->mc_props.mcp_num);
1637
1638	if (bm->bm_mkprops != NULL &&
1639	    mc_pcicfg_setup(mc, bm->bm_func, &cfghdl) == DDI_SUCCESS) {
1640		bm->bm_mkprops(cfghdl, mc);
1641		mc_pcicfg_teardown(cfghdl);
1642	}
1643
1644	/*
1645	 * If this is the last node to be attached for this memory controller,
1646	 * then create the minor node, enable scrubbers, and register with
1647	 * cpu module(s) for this chip.
1648	 */
1649	if (func == MC_FUNC_DEVIMAP) {
1650		mc_props_t *mcp = &mc->mc_props;
1651		int dram_present = 0;
1652
1653		if (ddi_create_minor_node(dip, "mc-amd", S_IFCHR,
1654		    mcp->mcp_num, "ddi_mem_ctrl",
1655		    0) != DDI_SUCCESS) {
1656			cmn_err(CE_WARN, "failed to create minor node for chip "
1657			    "%d memory controller\n",
1658			    (chipid_t)mcp->mcp_num);
1659		}
1660
1661		/*
1662		 * Register the memory controller for every CPU of this chip.
1663		 *
1664		 * If there is memory present on this node and ECC is enabled
1665		 * attempt to enable h/w memory scrubbers for this node.
1666		 * If we are successful in enabling *any* hardware scrubbers,
1667		 * disable the software memory scrubber.
1668		 */
1669		cmi_hdl_walk(mc_attach_cb, (void *)mc, (void *)&mcp->mcp_num,
1670		    NULL);
1671
1672		if (mcp->mcp_lim != mcp->mcp_base) {
1673			/*
1674			 * This node may map non-dram memory alone, so we
1675			 * must check for an enabled chip-select to be
1676			 * sure there is dram present.
1677			 */
1678			mc_cs_t *mccs;
1679
1680			for (mccs = mc->mc_cslist; mccs != NULL;
1681			    mccs = mccs->mccs_next) {
1682				if (mccs->mccs_props.csp_csbe) {
1683					dram_present = 1;
1684					break;
1685				}
1686			}
1687		}
1688
1689		if (dram_present && !mc_ecc_enabled(mc)) {
1690			/*
1691			 * On a single chip system there is no point in
1692			 * scrubbing if there is no ECC on the single node.
1693			 * On a multichip system, necessarily Opteron using
1694			 * registered ECC-capable DIMMs, if there is memory
1695			 * present on a node but no ECC there then we'll assume
1696			 * ECC is disabled for all nodes and we will not enable
1697			 * the scrubber and wll also disable the software
1698			 * memscrub thread.
1699			 */
1700			rc = 1;
1701		} else if (!dram_present) {
1702			/* No memory on this node - others decide memscrub */
1703			rc = 0;
1704		} else {
1705			/*
1706			 * There is memory on this node and ECC is enabled.
1707			 * Call via the cpu module to enable memory scrubbing
1708			 * on this node - we could call directly but then
1709			 * we may overlap with a request to enable chip-cache
1710			 * scrubbing.
1711			 */
1712			rc = mc_scrubber_enable(mc);
1713		}
1714
1715		if (rc == CMI_SUCCESS && !mc_sw_scrub_disabled++)
1716			cmi_mc_sw_memscrub_disable();
1717
1718		mc_report_testfails(mc);
1719	}
1720
1721	/*
1722	 * Update nvlist for as far as we have gotten in attach/init.
1723	 */
1724	nvlist_free(mc->mc_nvl);
1725	mc->mc_nvl = mc_nvl_create(mc);
1726
1727	rw_exit(&mc_lock);
1728	return (DDI_SUCCESS);
1729}
1730
1731/*ARGSUSED*/
1732static int
1733mc_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1734{
1735	/*
1736	 * See the comment about suspend in
1737	 * mc_attach().
1738	 */
1739	if (cmd == DDI_SUSPEND)
1740		return (DDI_SUCCESS);
1741	else
1742		return (DDI_FAILURE);
1743}
1744
1745
1746static struct dev_ops mc_ops = {
1747	DEVO_REV,		/* devo_rev */
1748	0,			/* devo_refcnt */
1749	mc_getinfo,		/* devo_getinfo */
1750	nulldev,		/* devo_identify */
1751	nulldev,		/* devo_probe */
1752	mc_attach,		/* devo_attach */
1753	mc_detach,		/* devo_detach */
1754	nodev,			/* devo_reset */
1755	&mc_cb_ops,		/* devo_cb_ops */
1756	NULL,			/* devo_bus_ops */
1757	NULL,			/* devo_power */
1758	ddi_quiesce_not_needed,		/* devo_quiesce */
1759};
1760
1761static struct modldrv modldrv = {
1762	&mod_driverops,
1763	"Memory Controller for AMD processors",
1764	&mc_ops
1765};
1766
1767static struct modlinkage modlinkage = {
1768	MODREV_1,
1769	(void *)&modldrv,
1770	NULL
1771};
1772
1773int
1774_init(void)
1775{
1776	/*
1777	 * Refuse to load if there is no PCI config space support.
1778	 */
1779	if (pci_getl_func == NULL)
1780		return (ENOTSUP);
1781
1782	rw_init(&mc_lock, NULL, RW_DRIVER, NULL);
1783	return (mod_install(&modlinkage));
1784}
1785
1786int
1787_info(struct modinfo *modinfop)
1788{
1789	return (mod_info(&modlinkage, modinfop));
1790}
1791
1792int
1793_fini(void)
1794{
1795	int rc;
1796
1797	if ((rc = mod_remove(&modlinkage)) != 0)
1798		return (rc);
1799
1800	rw_destroy(&mc_lock);
1801	return (0);
1802}
1803