xref: /illumos-gate/usr/src/uts/intel/io/amdzen/amdzen.c (revision 1e0464b8)
1047043c2SRobert Mustacchi /*
2047043c2SRobert Mustacchi  * This file and its contents are supplied under the terms of the
3047043c2SRobert Mustacchi  * Common Development and Distribution License ("CDDL"), version 1.0.
4047043c2SRobert Mustacchi  * You may only use this file in accordance with the terms of version
5047043c2SRobert Mustacchi  * 1.0 of the CDDL.
6047043c2SRobert Mustacchi  *
7047043c2SRobert Mustacchi  * A full copy of the text of the CDDL should have accompanied this
8047043c2SRobert Mustacchi  * source.  A copy of the CDDL is also available via the Internet at
9047043c2SRobert Mustacchi  * http://www.illumos.org/license/CDDL.
10047043c2SRobert Mustacchi  */
11047043c2SRobert Mustacchi 
12047043c2SRobert Mustacchi /*
13047043c2SRobert Mustacchi  * Copyright 2019, Joyent, Inc.
14*1e0464b8SRobert Mustacchi  * Copyright 2024 Oxide Computer Company
15047043c2SRobert Mustacchi  */
16047043c2SRobert Mustacchi 
17047043c2SRobert Mustacchi /*
18047043c2SRobert Mustacchi  * Nexus Driver for AMD Zen family systems. The purpose of this driver is to
19047043c2SRobert Mustacchi  * provide access to the following resources in a single, centralized fashion:
20047043c2SRobert Mustacchi  *
21047043c2SRobert Mustacchi  *  - The per-chip Data Fabric
22047043c2SRobert Mustacchi  *  - The North Bridge
23047043c2SRobert Mustacchi  *  - The System Management Network (SMN)
24047043c2SRobert Mustacchi  *
25047043c2SRobert Mustacchi  * This is a nexus driver as once we have attached to all the requisite
26047043c2SRobert Mustacchi  * components, we will enumerate child devices which consume this functionality.
27047043c2SRobert Mustacchi  *
28047043c2SRobert Mustacchi  * ------------------------
29047043c2SRobert Mustacchi  * Mapping Devices Together
30047043c2SRobert Mustacchi  * ------------------------
31047043c2SRobert Mustacchi  *
32047043c2SRobert Mustacchi  * The operating system needs to expose things like temperature sensors and DRAM
3319040b41SDan Cross  * configuration registers in terms of things that are meaningful to the system
3419040b41SDan Cross  * such as logical CPUs, cores, etc. This driver attaches to the PCI devices
3519040b41SDan Cross  * that represent the northbridge, data fabrics, and dies. Note that there are
3619040b41SDan Cross  * multiple northbridge and DF devices (one each per die) and this driver maps
3719040b41SDan Cross  * all of these three things together. Unfortunately, this requires some
3819040b41SDan Cross  * acrobatics as there is no direct way to map a northbridge to its
3919040b41SDan Cross  * corresponding die. Instead, we map a CPU die to a data fabric PCI device and
4019040b41SDan Cross  * a data fabric PCI device to a corresponding northbridge PCI device. This
4119040b41SDan Cross  * transitive relationship allows us to map from between northbridge and die.
4219040b41SDan Cross  *
4319040b41SDan Cross  * As each data fabric device is attached, based on vendor and device portions
4419040b41SDan Cross  * of the PCI ID, we add it to the DF stubs list in the global amdzen_t
4519040b41SDan Cross  * structure, amdzen_data->azn_df_stubs. We must now map these to logical CPUs.
46047043c2SRobert Mustacchi  *
47047043c2SRobert Mustacchi  * In current Zen based products, there is a direct mapping between processor
4819040b41SDan Cross  * nodes and a data fabric PCI device: all of the devices are on PCI Bus 0 and
4919040b41SDan Cross  * start from Device 0x18, so device 0x18 maps to processor node 0, 0x19 to
50047043c2SRobert Mustacchi  * processor node 1, etc. This means that to map a logical CPU to a data fabric
51047043c2SRobert Mustacchi  * device, we take its processor node id, add it to 0x18 and find the PCI device
5219040b41SDan Cross  * that is on bus 0 with that ID number. We already discovered the DF devices as
5319040b41SDan Cross  * described above.
5419040b41SDan Cross  *
5519040b41SDan Cross  * The northbridge PCI device has a well-defined device and function, but the
5619040b41SDan Cross  * bus that it is on varies. Each die has its own set of assigned PCI buses and
5719040b41SDan Cross  * its northbridge device is on the first die-specific bus. This implies that
5819040b41SDan Cross  * the northbridges do not show up on PCI bus 0, as that is the PCI bus that all
5919040b41SDan Cross  * of the data fabric devices are on and is not assigned to any particular die.
6019040b41SDan Cross  * Additionally, while the northbridge on the lowest-numbered PCI bus
6119040b41SDan Cross  * intuitively corresponds to processor node zero, hardware does not guarantee
6219040b41SDan Cross  * this. Because we don't want to be at the mercy of firmware, we don't rely on
6319040b41SDan Cross  * this ordering assumption, though we have yet to find a system that deviates
6419040b41SDan Cross  * from it, either.
65047043c2SRobert Mustacchi  *
66047043c2SRobert Mustacchi  * One of the registers in the data fabric device's function 0
6719040b41SDan Cross  * (AMDZEN_DF_F0_CFG_ADDR_CTL) happens to identify the first PCI bus that is
68047043c2SRobert Mustacchi  * associated with the processor node. This means that we can map a data fabric
6919040b41SDan Cross  * device to a northbridge by finding the northbridge whose PCI bus ID matches
7019040b41SDan Cross  * the value in the corresponding data fabric's AMDZEN_DF_F0_CFG_ADDR_CTL.
7119040b41SDan Cross  *
7219040b41SDan Cross  * Given all of the above, we can map a northbridge to a data fabric device and
7319040b41SDan Cross  * a die to a data fabric device. Because these are 1:1 mappings, there is a
7419040b41SDan Cross  * transitive relationship from northbridge to die. and therefore we know which
7519040b41SDan Cross  * northbridge is associated with which processor die. This is summarized in the
7619040b41SDan Cross  * following image:
7719040b41SDan Cross  *
7819040b41SDan Cross  *  +-------+     +------------------------------------+     +--------------+
7919040b41SDan Cross  *  | Die 0 |---->| Data Fabric PCI BDF 0/18/0         |---->| Northbridge  |
8019040b41SDan Cross  *  +-------+     | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 10  |     | PCI  10/0/0  |
8119040b41SDan Cross  *     ...        +------------------------------------+     +--------------+
8219040b41SDan Cross  *  +-------+     +------------------------------------+     +--------------+
8319040b41SDan Cross  *  | Die n |---->| Data Fabric PCI BDF 0/18+n/0       |---->| Northbridge  |
8419040b41SDan Cross  *  +-------+     | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 133 |     | PCI 133/0/0  |
8519040b41SDan Cross  *                +------------------------------------+     +--------------+
8619040b41SDan Cross  *
8719040b41SDan Cross  * Note, the PCI buses used by the northbridges here are arbitrary examples that
8819040b41SDan Cross  * do not necessarily reflect actual hardware values; however, the
8919040b41SDan Cross  * bus/device/function (BDF) of the data fabric accurately models hardware. All
9019040b41SDan Cross  * BDF values are in hex.
91047043c2SRobert Mustacchi  *
92047043c2SRobert Mustacchi  * Starting with the Rome generation of processors (Family 17h Model 30-3Fh),
9319040b41SDan Cross  * AMD has multiple northbridges on a given die. All of these northbridges share
9419040b41SDan Cross  * the same data fabric and system management network port. From our perspective
9519040b41SDan Cross  * this means that some of the northbridge devices will be redundant and that we
9619040b41SDan Cross  * no longer have a 1:1 mapping between the northbridge and the data fabric
9719040b41SDan Cross  * devices. Every data fabric will have a northbridge, but not every northbridge
9819040b41SDan Cross  * will have a data fabric device mapped. Because we're always trying to map
9919040b41SDan Cross  * from a die to a northbridge and not the reverse, the fact that there are
10019040b41SDan Cross  * extra northbridge devices hanging around that we don't know about shouldn't
10119040b41SDan Cross  * be a problem.
102047043c2SRobert Mustacchi  *
103047043c2SRobert Mustacchi  * -------------------------------
104047043c2SRobert Mustacchi  * Attach and Detach Complications
105047043c2SRobert Mustacchi  * -------------------------------
106047043c2SRobert Mustacchi  *
10719040b41SDan Cross  * We need to map different PCI devices together. Each device is attached to a
10819040b41SDan Cross  * amdzen_stub driver to facilitate integration with the rest of the kernel PCI
10919040b41SDan Cross  * machinery and so we have to manage multiple dev_info_t structures, each of
11019040b41SDan Cross  * which may be independently attached and detached.
11119040b41SDan Cross  *
11219040b41SDan Cross  * This is not particularly complex for attach: our _init routine allocates the
11319040b41SDan Cross  * necessary mutex and list structures at module load time, and as each stub is
11419040b41SDan Cross  * attached, it calls into this code to be added to the appropriate list. When
11519040b41SDan Cross  * the nexus itself is attached, we walk the PCI device tree accumulating a
11619040b41SDan Cross  * counter for all devices we expect to be attached. Once the scan is complete
11719040b41SDan Cross  * and all such devices are accounted for (stub registration may be happening
11819040b41SDan Cross  * asynchronously with respect to nexus attach), we initialize the nexus device
11919040b41SDan Cross  * and the attach is complete.
120047043c2SRobert Mustacchi  *
12119040b41SDan Cross  * Most other device drivers support instances that can be brought back after
12219040b41SDan Cross  * detach, provided they are associated with an active minor node in the
12319040b41SDan Cross  * /devices file system. This driver is different. Once a stub device has been
12419040b41SDan Cross  * attached, we do not permit detaching the nexus driver instance, as the kernel
12519040b41SDan Cross  * does not give us interlocking guarantees between nexus and stub driver attach
12619040b41SDan Cross  * and detach. It is simplest to just unconditionally fail detach once a stub
12719040b41SDan Cross  * has attached.
128047043c2SRobert Mustacchi  *
129047043c2SRobert Mustacchi  * ---------------
130047043c2SRobert Mustacchi  * Exposed Devices
131047043c2SRobert Mustacchi  * ---------------
132047043c2SRobert Mustacchi  *
133047043c2SRobert Mustacchi  * Rather than try and have all of the different functions that could be
13419040b41SDan Cross  * provided in one driver, we have a nexus driver that tries to load child
13519040b41SDan Cross  * pseudo-device drivers that provide specific pieces of functionality.
13671815ce7SRobert Mustacchi  *
13771815ce7SRobert Mustacchi  * -------
13871815ce7SRobert Mustacchi  * Locking
13971815ce7SRobert Mustacchi  * -------
14071815ce7SRobert Mustacchi  *
14119040b41SDan Cross  * The amdzen_data structure contains a single lock, azn_mutex.
14219040b41SDan Cross  *
14319040b41SDan Cross  * The various client functions here are intended for our nexus's direct
14419040b41SDan Cross  * children, but have been designed in case someone else should depends on this
14519040b41SDan Cross  * driver. Once a DF has been discovered, the set of entities inside of it
14619040b41SDan Cross  * (adf_nents, adf_ents[]) is considered static, constant data, and iteration
14719040b41SDan Cross  * over them does not require locking. However, the discovery of the amd_df_t
14819040b41SDan Cross  * does. In addition, locking is required whenever performing register accesses
14919040b41SDan Cross  * to the DF or SMN.
15019040b41SDan Cross  *
15119040b41SDan Cross  * To summarize, one must hold the lock in the following circumstances:
15271815ce7SRobert Mustacchi  *
15319040b41SDan Cross  *  - Looking up DF structures
15419040b41SDan Cross  *  - Reading or writing to DF registers
15519040b41SDan Cross  *  - Reading or writing to SMN registers
15671815ce7SRobert Mustacchi  *
15771815ce7SRobert Mustacchi  * In general, it is preferred that the lock be held across an entire client
15871815ce7SRobert Mustacchi  * operation if possible. The only time this becomes an issue are when we have
15919040b41SDan Cross  * callbacks into our callers (ala amdzen_c_df_iter()) as they may recursively
16019040b41SDan Cross  * call into us.
161047043c2SRobert Mustacchi  */
162047043c2SRobert Mustacchi 
163047043c2SRobert Mustacchi #include <sys/modctl.h>
164047043c2SRobert Mustacchi #include <sys/conf.h>
165047043c2SRobert Mustacchi #include <sys/devops.h>
166047043c2SRobert Mustacchi #include <sys/ddi.h>
167047043c2SRobert Mustacchi #include <sys/sunddi.h>
168047043c2SRobert Mustacchi #include <sys/pci.h>
169047043c2SRobert Mustacchi #include <sys/sysmacros.h>
170047043c2SRobert Mustacchi #include <sys/sunndi.h>
171047043c2SRobert Mustacchi #include <sys/x86_archext.h>
172047043c2SRobert Mustacchi #include <sys/cpuvar.h>
173dd23d762SRobert Mustacchi #include <sys/policy.h>
174dd23d762SRobert Mustacchi #include <sys/stat.h>
175dd23d762SRobert Mustacchi #include <sys/sunddi.h>
176dd23d762SRobert Mustacchi #include <sys/bitmap.h>
177*1e0464b8SRobert Mustacchi #include <sys/stdbool.h>
178047043c2SRobert Mustacchi 
17971815ce7SRobert Mustacchi #include <sys/amdzen/df.h>
180dd23d762SRobert Mustacchi #include <sys/amdzen/ccd.h>
181047043c2SRobert Mustacchi #include "amdzen.h"
182dd23d762SRobert Mustacchi #include "amdzen_client.h"
183dd23d762SRobert Mustacchi #include "amdzen_topo.h"
184047043c2SRobert Mustacchi 
185047043c2SRobert Mustacchi amdzen_t *amdzen_data;
186047043c2SRobert Mustacchi 
187dd23d762SRobert Mustacchi /*
188dd23d762SRobert Mustacchi  * Internal minor nodes for devices that the nexus provides itself.
189dd23d762SRobert Mustacchi  */
190dd23d762SRobert Mustacchi #define	AMDZEN_MINOR_TOPO	0
191dd23d762SRobert Mustacchi 
192047043c2SRobert Mustacchi /*
193047043c2SRobert Mustacchi  * Array of northbridge IDs that we care about.
194047043c2SRobert Mustacchi  */
195047043c2SRobert Mustacchi static const uint16_t amdzen_nb_ids[] = {
196047043c2SRobert Mustacchi 	/* Family 17h Ryzen, Epyc Models 00h-0fh (Zen uarch) */
197047043c2SRobert Mustacchi 	0x1450,
198becd642cSRobert Mustacchi 	/* Family 17h Raven Ridge, Kestrel, Dali Models 10h-2fh (Zen uarch) */
199047043c2SRobert Mustacchi 	0x15d0,
200e9abe9d6SRobert Mustacchi 	/* Family 17h/19h Rome, Milan, Matisse, Vermeer Zen 2/Zen 3 uarch */
201becd642cSRobert Mustacchi 	0x1480,
202f8e9c7b3SRobert Mustacchi 	/* Family 17h/19h Renoir, Cezanne, Van Gogh Zen 2/3 uarch */
203f8e9c7b3SRobert Mustacchi 	0x1630,
204e6f89c3aSRobert Mustacchi 	/* Family 19h Genoa and Bergamo */
205f8e9c7b3SRobert Mustacchi 	0x14a4,
206f8e9c7b3SRobert Mustacchi 	/* Family 17h Mendocino, Family 19h Rembrandt */
207f8e9c7b3SRobert Mustacchi 	0x14b5,
208f8e9c7b3SRobert Mustacchi 	/* Family 19h Raphael */
209e6f89c3aSRobert Mustacchi 	0x14d8,
210e6f89c3aSRobert Mustacchi 	/* Family 19h Phoenix */
211e6f89c3aSRobert Mustacchi 	0x14e8
212047043c2SRobert Mustacchi };
213047043c2SRobert Mustacchi 
214047043c2SRobert Mustacchi typedef struct {
215047043c2SRobert Mustacchi 	char *acd_name;
216047043c2SRobert Mustacchi 	amdzen_child_t acd_addr;
217*1e0464b8SRobert Mustacchi 	/*
218*1e0464b8SRobert Mustacchi 	 * This indicates whether or not we should issue warnings to users when
219*1e0464b8SRobert Mustacchi 	 * something happens specific to this instance. The main reason we don't
220*1e0464b8SRobert Mustacchi 	 * want to is for optional devices that may not be installed as they are
221*1e0464b8SRobert Mustacchi 	 * for development purposes (e.g. usmn, zen_udf); however, if there is
222*1e0464b8SRobert Mustacchi 	 * an issue with the others we still want to know.
223*1e0464b8SRobert Mustacchi 	 */
224*1e0464b8SRobert Mustacchi 	bool acd_warn;
225047043c2SRobert Mustacchi } amdzen_child_data_t;
226047043c2SRobert Mustacchi 
227047043c2SRobert Mustacchi static const amdzen_child_data_t amdzen_children[] = {
228*1e0464b8SRobert Mustacchi 	{ "smntemp", AMDZEN_C_SMNTEMP, true },
229*1e0464b8SRobert Mustacchi 	{ "usmn", AMDZEN_C_USMN, false },
230*1e0464b8SRobert Mustacchi 	{ "zen_udf", AMDZEN_C_ZEN_UDF, false },
231*1e0464b8SRobert Mustacchi 	{ "zen_umc", AMDZEN_C_ZEN_UMC, true }
232047043c2SRobert Mustacchi };
233047043c2SRobert Mustacchi 
2344adf43b0SKeith M Wesolowski static uint8_t
amdzen_stub_get8(amdzen_stub_t * stub,off_t reg)2354adf43b0SKeith M Wesolowski amdzen_stub_get8(amdzen_stub_t *stub, off_t reg)
2364adf43b0SKeith M Wesolowski {
2374adf43b0SKeith M Wesolowski 	return (pci_config_get8(stub->azns_cfgspace, reg));
2384adf43b0SKeith M Wesolowski }
2394adf43b0SKeith M Wesolowski 
2404adf43b0SKeith M Wesolowski static uint16_t
amdzen_stub_get16(amdzen_stub_t * stub,off_t reg)2414adf43b0SKeith M Wesolowski amdzen_stub_get16(amdzen_stub_t *stub, off_t reg)
2424adf43b0SKeith M Wesolowski {
2434adf43b0SKeith M Wesolowski 	return (pci_config_get16(stub->azns_cfgspace, reg));
2444adf43b0SKeith M Wesolowski }
2454adf43b0SKeith M Wesolowski 
246047043c2SRobert Mustacchi static uint32_t
amdzen_stub_get32(amdzen_stub_t * stub,off_t reg)247047043c2SRobert Mustacchi amdzen_stub_get32(amdzen_stub_t *stub, off_t reg)
248047043c2SRobert Mustacchi {
249047043c2SRobert Mustacchi 	return (pci_config_get32(stub->azns_cfgspace, reg));
250047043c2SRobert Mustacchi }
251047043c2SRobert Mustacchi 
252549e0fd3SRobert Mustacchi static uint64_t
amdzen_stub_get64(amdzen_stub_t * stub,off_t reg)253549e0fd3SRobert Mustacchi amdzen_stub_get64(amdzen_stub_t *stub, off_t reg)
254549e0fd3SRobert Mustacchi {
255549e0fd3SRobert Mustacchi 	return (pci_config_get64(stub->azns_cfgspace, reg));
256549e0fd3SRobert Mustacchi }
257549e0fd3SRobert Mustacchi 
2584adf43b0SKeith M Wesolowski static void
amdzen_stub_put8(amdzen_stub_t * stub,off_t reg,uint8_t val)2594adf43b0SKeith M Wesolowski amdzen_stub_put8(amdzen_stub_t *stub, off_t reg, uint8_t val)
2604adf43b0SKeith M Wesolowski {
2614adf43b0SKeith M Wesolowski 	pci_config_put8(stub->azns_cfgspace, reg, val);
2624adf43b0SKeith M Wesolowski }
2634adf43b0SKeith M Wesolowski 
2644adf43b0SKeith M Wesolowski static void
amdzen_stub_put16(amdzen_stub_t * stub,off_t reg,uint16_t val)2654adf43b0SKeith M Wesolowski amdzen_stub_put16(amdzen_stub_t *stub, off_t reg, uint16_t val)
2664adf43b0SKeith M Wesolowski {
2674adf43b0SKeith M Wesolowski 	pci_config_put16(stub->azns_cfgspace, reg, val);
2684adf43b0SKeith M Wesolowski }
2694adf43b0SKeith M Wesolowski 
270047043c2SRobert Mustacchi static void
amdzen_stub_put32(amdzen_stub_t * stub,off_t reg,uint32_t val)271047043c2SRobert Mustacchi amdzen_stub_put32(amdzen_stub_t *stub, off_t reg, uint32_t val)
272047043c2SRobert Mustacchi {
273047043c2SRobert Mustacchi 	pci_config_put32(stub->azns_cfgspace, reg, val);
274047043c2SRobert Mustacchi }
275047043c2SRobert Mustacchi 
27671815ce7SRobert Mustacchi static uint64_t
amdzen_df_read_regdef(amdzen_t * azn,amdzen_df_t * df,const df_reg_def_t def,uint8_t inst,boolean_t do_64)27771815ce7SRobert Mustacchi amdzen_df_read_regdef(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def,
27871815ce7SRobert Mustacchi     uint8_t inst, boolean_t do_64)
27971815ce7SRobert Mustacchi {
28071815ce7SRobert Mustacchi 	df_reg_def_t ficaa;
28171815ce7SRobert Mustacchi 	df_reg_def_t ficad;
28271815ce7SRobert Mustacchi 	uint32_t val = 0;
28371815ce7SRobert Mustacchi 	df_rev_t df_rev = azn->azn_dfs[0].adf_rev;
28471815ce7SRobert Mustacchi 
28571815ce7SRobert Mustacchi 	VERIFY(MUTEX_HELD(&azn->azn_mutex));
28671815ce7SRobert Mustacchi 	ASSERT3U(def.drd_gens & df_rev, ==, df_rev);
28771815ce7SRobert Mustacchi 	val = DF_FICAA_V2_SET_TARG_INST(val, 1);
28871815ce7SRobert Mustacchi 	val = DF_FICAA_V2_SET_FUNC(val, def.drd_func);
28971815ce7SRobert Mustacchi 	val = DF_FICAA_V2_SET_INST(val, inst);
29071815ce7SRobert Mustacchi 	val = DF_FICAA_V2_SET_64B(val, do_64 ? 1 : 0);
29171815ce7SRobert Mustacchi 
29271815ce7SRobert Mustacchi 	switch (df_rev) {
29371815ce7SRobert Mustacchi 	case DF_REV_2:
29471815ce7SRobert Mustacchi 	case DF_REV_3:
29571815ce7SRobert Mustacchi 	case DF_REV_3P5:
29671815ce7SRobert Mustacchi 		ficaa = DF_FICAA_V2;
29771815ce7SRobert Mustacchi 		ficad = DF_FICAD_LO_V2;
29871815ce7SRobert Mustacchi 		/*
29971815ce7SRobert Mustacchi 		 * Both here and in the DFv4 case, the register ignores the
30071815ce7SRobert Mustacchi 		 * lower 2 bits. That is we can only address and encode things
30171815ce7SRobert Mustacchi 		 * in units of 4 bytes.
30271815ce7SRobert Mustacchi 		 */
30371815ce7SRobert Mustacchi 		val = DF_FICAA_V2_SET_REG(val, def.drd_reg >> 2);
30471815ce7SRobert Mustacchi 		break;
30571815ce7SRobert Mustacchi 	case DF_REV_4:
30671815ce7SRobert Mustacchi 		ficaa = DF_FICAA_V4;
30771815ce7SRobert Mustacchi 		ficad = DF_FICAD_LO_V4;
30871815ce7SRobert Mustacchi 		val = DF_FICAA_V4_SET_REG(val, def.drd_reg >> 2);
30971815ce7SRobert Mustacchi 		break;
31071815ce7SRobert Mustacchi 	default:
31171815ce7SRobert Mustacchi 		panic("encountered unexpected DF rev: %u", df_rev);
31271815ce7SRobert Mustacchi 	}
31371815ce7SRobert Mustacchi 
31471815ce7SRobert Mustacchi 	amdzen_stub_put32(df->adf_funcs[ficaa.drd_func], ficaa.drd_reg, val);
31571815ce7SRobert Mustacchi 	if (do_64) {
31671815ce7SRobert Mustacchi 		return (amdzen_stub_get64(df->adf_funcs[ficad.drd_func],
31771815ce7SRobert Mustacchi 		    ficad.drd_reg));
31871815ce7SRobert Mustacchi 	} else {
31971815ce7SRobert Mustacchi 		return (amdzen_stub_get32(df->adf_funcs[ficad.drd_func],
32071815ce7SRobert Mustacchi 		    ficad.drd_reg));
32171815ce7SRobert Mustacchi 	}
32271815ce7SRobert Mustacchi }
32371815ce7SRobert Mustacchi 
324047043c2SRobert Mustacchi /*
325047043c2SRobert Mustacchi  * Perform a targeted 32-bit indirect read to a specific instance and function.
326047043c2SRobert Mustacchi  */
327047043c2SRobert Mustacchi static uint32_t
amdzen_df_read32(amdzen_t * azn,amdzen_df_t * df,uint8_t inst,const df_reg_def_t def)32871815ce7SRobert Mustacchi amdzen_df_read32(amdzen_t *azn, amdzen_df_t *df, uint8_t inst,
32971815ce7SRobert Mustacchi     const df_reg_def_t def)
330047043c2SRobert Mustacchi {
33171815ce7SRobert Mustacchi 	return (amdzen_df_read_regdef(azn, df, def, inst, B_FALSE));
332047043c2SRobert Mustacchi }
333047043c2SRobert Mustacchi 
334549e0fd3SRobert Mustacchi /*
33571815ce7SRobert Mustacchi  * For a broadcast read, just go to the underlying PCI function and perform a
33671815ce7SRobert Mustacchi  * read. At this point in time, we don't believe we need to use the FICAA/FICAD
33771815ce7SRobert Mustacchi  * to access it (though it does have a broadcast mode).
338549e0fd3SRobert Mustacchi  */
33971815ce7SRobert Mustacchi static uint32_t
amdzen_df_read32_bcast(amdzen_t * azn,amdzen_df_t * df,const df_reg_def_t def)34071815ce7SRobert Mustacchi amdzen_df_read32_bcast(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def)
341549e0fd3SRobert Mustacchi {
342549e0fd3SRobert Mustacchi 	VERIFY(MUTEX_HELD(&azn->azn_mutex));
34371815ce7SRobert Mustacchi 	return (amdzen_stub_get32(df->adf_funcs[def.drd_func], def.drd_reg));
344549e0fd3SRobert Mustacchi }
345549e0fd3SRobert Mustacchi 
346047043c2SRobert Mustacchi static uint32_t
amdzen_smn_read(amdzen_t * azn,amdzen_df_t * df,const smn_reg_t reg)3474adf43b0SKeith M Wesolowski amdzen_smn_read(amdzen_t *azn, amdzen_df_t *df, const smn_reg_t reg)
348047043c2SRobert Mustacchi {
3494adf43b0SKeith M Wesolowski 	const uint32_t base_addr = SMN_REG_ADDR_BASE(reg);
3504adf43b0SKeith M Wesolowski 	const uint32_t addr_off = SMN_REG_ADDR_OFF(reg);
3514adf43b0SKeith M Wesolowski 
3524adf43b0SKeith M Wesolowski 	VERIFY(SMN_REG_IS_NATURALLY_ALIGNED(reg));
353047043c2SRobert Mustacchi 	VERIFY(MUTEX_HELD(&azn->azn_mutex));
3544adf43b0SKeith M Wesolowski 	amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, base_addr);
3554adf43b0SKeith M Wesolowski 
3564adf43b0SKeith M Wesolowski 	switch (SMN_REG_SIZE(reg)) {
3574adf43b0SKeith M Wesolowski 	case 1:
3584adf43b0SKeith M Wesolowski 		return ((uint32_t)amdzen_stub_get8(df->adf_nb,
3594adf43b0SKeith M Wesolowski 		    AMDZEN_NB_SMN_DATA + addr_off));
3604adf43b0SKeith M Wesolowski 	case 2:
3614adf43b0SKeith M Wesolowski 		return ((uint32_t)amdzen_stub_get16(df->adf_nb,
3624adf43b0SKeith M Wesolowski 		    AMDZEN_NB_SMN_DATA + addr_off));
3634adf43b0SKeith M Wesolowski 	case 4:
3644adf43b0SKeith M Wesolowski 		return (amdzen_stub_get32(df->adf_nb, AMDZEN_NB_SMN_DATA));
3654adf43b0SKeith M Wesolowski 	default:
3664adf43b0SKeith M Wesolowski 		panic("unreachable invalid SMN register size %u",
3674adf43b0SKeith M Wesolowski 		    SMN_REG_SIZE(reg));
3684adf43b0SKeith M Wesolowski 	}
369047043c2SRobert Mustacchi }
370047043c2SRobert Mustacchi 
371f198607dSRobert Mustacchi static void
amdzen_smn_write(amdzen_t * azn,amdzen_df_t * df,const smn_reg_t reg,const uint32_t val)3724adf43b0SKeith M Wesolowski amdzen_smn_write(amdzen_t *azn, amdzen_df_t *df, const smn_reg_t reg,
373ba215efeSKeith M Wesolowski     const uint32_t val)
374f198607dSRobert Mustacchi {
3754adf43b0SKeith M Wesolowski 	const uint32_t base_addr = SMN_REG_ADDR_BASE(reg);
3764adf43b0SKeith M Wesolowski 	const uint32_t addr_off = SMN_REG_ADDR_OFF(reg);
3774adf43b0SKeith M Wesolowski 
3784adf43b0SKeith M Wesolowski 	VERIFY(SMN_REG_IS_NATURALLY_ALIGNED(reg));
3794adf43b0SKeith M Wesolowski 	VERIFY(SMN_REG_VALUE_FITS(reg, val));
380f198607dSRobert Mustacchi 	VERIFY(MUTEX_HELD(&azn->azn_mutex));
3814adf43b0SKeith M Wesolowski 	amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, base_addr);
3824adf43b0SKeith M Wesolowski 
3834adf43b0SKeith M Wesolowski 	switch (SMN_REG_SIZE(reg)) {
3844adf43b0SKeith M Wesolowski 	case 1:
3854adf43b0SKeith M Wesolowski 		amdzen_stub_put8(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off,
3864adf43b0SKeith M Wesolowski 		    (uint8_t)val);
3874adf43b0SKeith M Wesolowski 		break;
3884adf43b0SKeith M Wesolowski 	case 2:
3894adf43b0SKeith M Wesolowski 		amdzen_stub_put16(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off,
3904adf43b0SKeith M Wesolowski 		    (uint16_t)val);
3914adf43b0SKeith M Wesolowski 		break;
3924adf43b0SKeith M Wesolowski 	case 4:
3934adf43b0SKeith M Wesolowski 		amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_DATA, val);
3944adf43b0SKeith M Wesolowski 		break;
3954adf43b0SKeith M Wesolowski 	default:
3964adf43b0SKeith M Wesolowski 		panic("unreachable invalid SMN register size %u",
3974adf43b0SKeith M Wesolowski 		    SMN_REG_SIZE(reg));
3984adf43b0SKeith M Wesolowski 	}
399f198607dSRobert Mustacchi }
400f198607dSRobert Mustacchi 
401047043c2SRobert Mustacchi static amdzen_df_t *
amdzen_df_find(amdzen_t * azn,uint_t dfno)402047043c2SRobert Mustacchi amdzen_df_find(amdzen_t *azn, uint_t dfno)
403047043c2SRobert Mustacchi {
404047043c2SRobert Mustacchi 	uint_t i;
405047043c2SRobert Mustacchi 
406047043c2SRobert Mustacchi 	ASSERT(MUTEX_HELD(&azn->azn_mutex));
407047043c2SRobert Mustacchi 	if (dfno >= azn->azn_ndfs) {
408047043c2SRobert Mustacchi 		return (NULL);
409047043c2SRobert Mustacchi 	}
410047043c2SRobert Mustacchi 
411047043c2SRobert Mustacchi 	for (i = 0; i < azn->azn_ndfs; i++) {
412047043c2SRobert Mustacchi 		amdzen_df_t *df = &azn->azn_dfs[i];
413047043c2SRobert Mustacchi 		if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) {
414047043c2SRobert Mustacchi 			continue;
415047043c2SRobert Mustacchi 		}
416047043c2SRobert Mustacchi 
417047043c2SRobert Mustacchi 		if (dfno == 0) {
418047043c2SRobert Mustacchi 			return (df);
419047043c2SRobert Mustacchi 		}
420047043c2SRobert Mustacchi 		dfno--;
421047043c2SRobert Mustacchi 	}
422047043c2SRobert Mustacchi 
423047043c2SRobert Mustacchi 	return (NULL);
424047043c2SRobert Mustacchi }
425047043c2SRobert Mustacchi 
426dd23d762SRobert Mustacchi static amdzen_df_ent_t *
amdzen_df_ent_find_by_instid(amdzen_df_t * df,uint8_t instid)427dd23d762SRobert Mustacchi amdzen_df_ent_find_by_instid(amdzen_df_t *df, uint8_t instid)
428dd23d762SRobert Mustacchi {
429dd23d762SRobert Mustacchi 	for (uint_t i = 0; i < df->adf_nents; i++) {
430dd23d762SRobert Mustacchi 		amdzen_df_ent_t *ent = &df->adf_ents[i];
431dd23d762SRobert Mustacchi 
432dd23d762SRobert Mustacchi 		if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) {
433dd23d762SRobert Mustacchi 			continue;
434dd23d762SRobert Mustacchi 		}
435dd23d762SRobert Mustacchi 
436dd23d762SRobert Mustacchi 		if (ent->adfe_inst_id == instid) {
437dd23d762SRobert Mustacchi 			return (ent);
438dd23d762SRobert Mustacchi 		}
439dd23d762SRobert Mustacchi 	}
440dd23d762SRobert Mustacchi 
441dd23d762SRobert Mustacchi 	return (NULL);
442dd23d762SRobert Mustacchi }
443dd23d762SRobert Mustacchi 
444047043c2SRobert Mustacchi /*
445047043c2SRobert Mustacchi  * Client functions that are used by nexus children.
446047043c2SRobert Mustacchi  */
447047043c2SRobert Mustacchi int
amdzen_c_smn_read(uint_t dfno,const smn_reg_t reg,uint32_t * valp)4484adf43b0SKeith M Wesolowski amdzen_c_smn_read(uint_t dfno, const smn_reg_t reg, uint32_t *valp)
449047043c2SRobert Mustacchi {
450047043c2SRobert Mustacchi 	amdzen_df_t *df;
451047043c2SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
452047043c2SRobert Mustacchi 
4534adf43b0SKeith M Wesolowski 	if (!SMN_REG_SIZE_IS_VALID(reg))
4544adf43b0SKeith M Wesolowski 		return (EINVAL);
4554adf43b0SKeith M Wesolowski 	if (!SMN_REG_IS_NATURALLY_ALIGNED(reg))
4564adf43b0SKeith M Wesolowski 		return (EINVAL);
4574adf43b0SKeith M Wesolowski 
458047043c2SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
459047043c2SRobert Mustacchi 	df = amdzen_df_find(azn, dfno);
460047043c2SRobert Mustacchi 	if (df == NULL) {
461047043c2SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
462047043c2SRobert Mustacchi 		return (ENOENT);
463047043c2SRobert Mustacchi 	}
464047043c2SRobert Mustacchi 
465047043c2SRobert Mustacchi 	if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) {
466047043c2SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
467047043c2SRobert Mustacchi 		return (ENXIO);
468047043c2SRobert Mustacchi 	}
469047043c2SRobert Mustacchi 
4704adf43b0SKeith M Wesolowski 	*valp = amdzen_smn_read(azn, df, reg);
471047043c2SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
472047043c2SRobert Mustacchi 	return (0);
473047043c2SRobert Mustacchi }
474047043c2SRobert Mustacchi 
475f198607dSRobert Mustacchi int
amdzen_c_smn_write(uint_t dfno,const smn_reg_t reg,const uint32_t val)4764adf43b0SKeith M Wesolowski amdzen_c_smn_write(uint_t dfno, const smn_reg_t reg, const uint32_t val)
477f198607dSRobert Mustacchi {
478f198607dSRobert Mustacchi 	amdzen_df_t *df;
479f198607dSRobert Mustacchi 	amdzen_t *azn = amdzen_data;
480f198607dSRobert Mustacchi 
4814adf43b0SKeith M Wesolowski 	if (!SMN_REG_SIZE_IS_VALID(reg))
4824adf43b0SKeith M Wesolowski 		return (EINVAL);
4834adf43b0SKeith M Wesolowski 	if (!SMN_REG_IS_NATURALLY_ALIGNED(reg))
4844adf43b0SKeith M Wesolowski 		return (EINVAL);
4854adf43b0SKeith M Wesolowski 	if (!SMN_REG_VALUE_FITS(reg, val))
4864adf43b0SKeith M Wesolowski 		return (EOVERFLOW);
4874adf43b0SKeith M Wesolowski 
488f198607dSRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
489f198607dSRobert Mustacchi 	df = amdzen_df_find(azn, dfno);
490f198607dSRobert Mustacchi 	if (df == NULL) {
491f198607dSRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
492f198607dSRobert Mustacchi 		return (ENOENT);
493f198607dSRobert Mustacchi 	}
494f198607dSRobert Mustacchi 
495f198607dSRobert Mustacchi 	if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) {
496f198607dSRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
497f198607dSRobert Mustacchi 		return (ENXIO);
498f198607dSRobert Mustacchi 	}
499f198607dSRobert Mustacchi 
5004adf43b0SKeith M Wesolowski 	amdzen_smn_write(azn, df, reg, val);
501f198607dSRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
502f198607dSRobert Mustacchi 	return (0);
503f198607dSRobert Mustacchi }
504f198607dSRobert Mustacchi 
505047043c2SRobert Mustacchi uint_t
amdzen_c_df_count(void)506047043c2SRobert Mustacchi amdzen_c_df_count(void)
507047043c2SRobert Mustacchi {
508047043c2SRobert Mustacchi 	uint_t ret;
509047043c2SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
510047043c2SRobert Mustacchi 
511047043c2SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
512047043c2SRobert Mustacchi 	ret = azn->azn_ndfs;
513047043c2SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
514047043c2SRobert Mustacchi 	return (ret);
515047043c2SRobert Mustacchi }
516047043c2SRobert Mustacchi 
51771815ce7SRobert Mustacchi df_rev_t
amdzen_c_df_rev(void)51871815ce7SRobert Mustacchi amdzen_c_df_rev(void)
51971815ce7SRobert Mustacchi {
52071815ce7SRobert Mustacchi 	amdzen_df_t *df;
52171815ce7SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
52271815ce7SRobert Mustacchi 	df_rev_t rev;
52371815ce7SRobert Mustacchi 
52471815ce7SRobert Mustacchi 	/*
52571815ce7SRobert Mustacchi 	 * Always use the first DF instance to determine what we're using. Our
52671815ce7SRobert Mustacchi 	 * current assumption, which seems to generally be true, is that the
52771815ce7SRobert Mustacchi 	 * given DF revisions are the same in a given system when the DFs are
52871815ce7SRobert Mustacchi 	 * directly connected.
52971815ce7SRobert Mustacchi 	 */
53071815ce7SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
53171815ce7SRobert Mustacchi 	df = amdzen_df_find(azn, 0);
53271815ce7SRobert Mustacchi 	if (df == NULL) {
53371815ce7SRobert Mustacchi 		rev = DF_REV_UNKNOWN;
53471815ce7SRobert Mustacchi 	} else {
53571815ce7SRobert Mustacchi 		rev = df->adf_rev;
53671815ce7SRobert Mustacchi 	}
53771815ce7SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
53871815ce7SRobert Mustacchi 
53971815ce7SRobert Mustacchi 	return (rev);
54071815ce7SRobert Mustacchi }
54171815ce7SRobert Mustacchi 
542549e0fd3SRobert Mustacchi int
amdzen_c_df_read32(uint_t dfno,uint8_t inst,const df_reg_def_t def,uint32_t * valp)54371815ce7SRobert Mustacchi amdzen_c_df_read32(uint_t dfno, uint8_t inst, const df_reg_def_t def,
54471815ce7SRobert Mustacchi     uint32_t *valp)
545549e0fd3SRobert Mustacchi {
546549e0fd3SRobert Mustacchi 	amdzen_df_t *df;
547549e0fd3SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
548549e0fd3SRobert Mustacchi 
549549e0fd3SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
550549e0fd3SRobert Mustacchi 	df = amdzen_df_find(azn, dfno);
551549e0fd3SRobert Mustacchi 	if (df == NULL) {
552549e0fd3SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
553549e0fd3SRobert Mustacchi 		return (ENOENT);
554549e0fd3SRobert Mustacchi 	}
555549e0fd3SRobert Mustacchi 
55671815ce7SRobert Mustacchi 	*valp = amdzen_df_read_regdef(azn, df, def, inst, B_FALSE);
557549e0fd3SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
558549e0fd3SRobert Mustacchi 
559549e0fd3SRobert Mustacchi 	return (0);
560549e0fd3SRobert Mustacchi }
561549e0fd3SRobert Mustacchi 
562549e0fd3SRobert Mustacchi int
amdzen_c_df_read64(uint_t dfno,uint8_t inst,const df_reg_def_t def,uint64_t * valp)56371815ce7SRobert Mustacchi amdzen_c_df_read64(uint_t dfno, uint8_t inst, const df_reg_def_t def,
56471815ce7SRobert Mustacchi     uint64_t *valp)
565549e0fd3SRobert Mustacchi {
566549e0fd3SRobert Mustacchi 	amdzen_df_t *df;
567549e0fd3SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
568549e0fd3SRobert Mustacchi 
569549e0fd3SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
570549e0fd3SRobert Mustacchi 	df = amdzen_df_find(azn, dfno);
571549e0fd3SRobert Mustacchi 	if (df == NULL) {
572549e0fd3SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
573549e0fd3SRobert Mustacchi 		return (ENOENT);
574549e0fd3SRobert Mustacchi 	}
575549e0fd3SRobert Mustacchi 
57671815ce7SRobert Mustacchi 	*valp = amdzen_df_read_regdef(azn, df, def, inst, B_TRUE);
57771815ce7SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
57871815ce7SRobert Mustacchi 
57971815ce7SRobert Mustacchi 	return (0);
58071815ce7SRobert Mustacchi }
58171815ce7SRobert Mustacchi 
58271815ce7SRobert Mustacchi int
amdzen_c_df_iter(uint_t dfno,zen_df_type_t type,amdzen_c_iter_f func,void * arg)58371815ce7SRobert Mustacchi amdzen_c_df_iter(uint_t dfno, zen_df_type_t type, amdzen_c_iter_f func,
58471815ce7SRobert Mustacchi     void *arg)
58571815ce7SRobert Mustacchi {
58671815ce7SRobert Mustacchi 	amdzen_df_t *df;
58771815ce7SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
58871815ce7SRobert Mustacchi 	df_type_t df_type;
58971815ce7SRobert Mustacchi 	uint8_t df_subtype;
59071815ce7SRobert Mustacchi 
59171815ce7SRobert Mustacchi 	/*
59271815ce7SRobert Mustacchi 	 * Unlike other calls here, we hold our lock only to find the DF here.
59371815ce7SRobert Mustacchi 	 * The main reason for this is the nature of the callback function.
59471815ce7SRobert Mustacchi 	 * Folks are iterating over instances so they can call back into us. If
59571815ce7SRobert Mustacchi 	 * you look at the locking statement, the thing that is most volatile
59671815ce7SRobert Mustacchi 	 * right here and what we need to protect is the DF itself and
59771815ce7SRobert Mustacchi 	 * subsequent register accesses to it. The actual data about which
59871815ce7SRobert Mustacchi 	 * entities exist is static and so once we have found a DF we should
59971815ce7SRobert Mustacchi 	 * hopefully be in good shape as they only come, but don't go.
60071815ce7SRobert Mustacchi 	 */
60171815ce7SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
60271815ce7SRobert Mustacchi 	df = amdzen_df_find(azn, dfno);
60371815ce7SRobert Mustacchi 	if (df == NULL) {
60471815ce7SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
60571815ce7SRobert Mustacchi 		return (ENOENT);
60671815ce7SRobert Mustacchi 	}
607549e0fd3SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
608549e0fd3SRobert Mustacchi 
60971815ce7SRobert Mustacchi 	switch (type) {
61071815ce7SRobert Mustacchi 	case ZEN_DF_TYPE_CS_UMC:
61171815ce7SRobert Mustacchi 		df_type = DF_TYPE_CS;
61271815ce7SRobert Mustacchi 		/*
61371815ce7SRobert Mustacchi 		 * In the original Zeppelin DFv2 die there was no subtype field
61471815ce7SRobert Mustacchi 		 * used for the CS. The UMC is the only type and has a subtype
61571815ce7SRobert Mustacchi 		 * of zero.
61671815ce7SRobert Mustacchi 		 */
61771815ce7SRobert Mustacchi 		if (df->adf_rev != DF_REV_2) {
61871815ce7SRobert Mustacchi 			df_subtype = DF_CS_SUBTYPE_UMC;
61971815ce7SRobert Mustacchi 		} else {
62071815ce7SRobert Mustacchi 			df_subtype = 0;
62171815ce7SRobert Mustacchi 		}
62271815ce7SRobert Mustacchi 		break;
62371815ce7SRobert Mustacchi 	case ZEN_DF_TYPE_CCM_CPU:
62471815ce7SRobert Mustacchi 		/*
625dd23d762SRobert Mustacchi 		 * Because the CCM CPU subtype has always remained zero, we can
626dd23d762SRobert Mustacchi 		 * use that regardless of the generation.
62771815ce7SRobert Mustacchi 		 */
628f8e9c7b3SRobert Mustacchi 		df_type = DF_TYPE_CCM;
629dd23d762SRobert Mustacchi 		df_subtype = DF_CCM_SUBTYPE_CPU;
63071815ce7SRobert Mustacchi 		break;
63171815ce7SRobert Mustacchi 	default:
63271815ce7SRobert Mustacchi 		return (EINVAL);
63371815ce7SRobert Mustacchi 	}
63471815ce7SRobert Mustacchi 
63571815ce7SRobert Mustacchi 	for (uint_t i = 0; i < df->adf_nents; i++) {
63671815ce7SRobert Mustacchi 		amdzen_df_ent_t *ent = &df->adf_ents[i];
63771815ce7SRobert Mustacchi 
63871815ce7SRobert Mustacchi 		/*
63971815ce7SRobert Mustacchi 		 * Some DF components are not considered enabled and therefore
64071815ce7SRobert Mustacchi 		 * will end up having bogus values in their ID fields. If we do
64171815ce7SRobert Mustacchi 		 * not have an enable flag set, we must skip this node.
64271815ce7SRobert Mustacchi 		 */
64371815ce7SRobert Mustacchi 		if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0)
64471815ce7SRobert Mustacchi 			continue;
64571815ce7SRobert Mustacchi 
64671815ce7SRobert Mustacchi 		if (ent->adfe_type == df_type &&
64771815ce7SRobert Mustacchi 		    ent->adfe_subtype == df_subtype) {
64871815ce7SRobert Mustacchi 			int ret = func(dfno, ent->adfe_fabric_id,
64971815ce7SRobert Mustacchi 			    ent->adfe_inst_id, arg);
65071815ce7SRobert Mustacchi 			if (ret != 0) {
65171815ce7SRobert Mustacchi 				return (ret);
65271815ce7SRobert Mustacchi 			}
65371815ce7SRobert Mustacchi 		}
65471815ce7SRobert Mustacchi 	}
65571815ce7SRobert Mustacchi 
65671815ce7SRobert Mustacchi 	return (0);
65771815ce7SRobert Mustacchi }
65871815ce7SRobert Mustacchi 
65971815ce7SRobert Mustacchi int
amdzen_c_df_fabric_decomp(df_fabric_decomp_t * decomp)66071815ce7SRobert Mustacchi amdzen_c_df_fabric_decomp(df_fabric_decomp_t *decomp)
66171815ce7SRobert Mustacchi {
66271815ce7SRobert Mustacchi 	const amdzen_df_t *df;
66371815ce7SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
66471815ce7SRobert Mustacchi 
66571815ce7SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
66671815ce7SRobert Mustacchi 	df = amdzen_df_find(azn, 0);
66771815ce7SRobert Mustacchi 	if (df == NULL) {
66871815ce7SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
66971815ce7SRobert Mustacchi 		return (ENOENT);
67071815ce7SRobert Mustacchi 	}
67171815ce7SRobert Mustacchi 
67271815ce7SRobert Mustacchi 	*decomp = df->adf_decomp;
67371815ce7SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
674549e0fd3SRobert Mustacchi 	return (0);
675549e0fd3SRobert Mustacchi }
676047043c2SRobert Mustacchi 
677047043c2SRobert Mustacchi static boolean_t
amdzen_create_child(amdzen_t * azn,const amdzen_child_data_t * acd)678047043c2SRobert Mustacchi amdzen_create_child(amdzen_t *azn, const amdzen_child_data_t *acd)
679047043c2SRobert Mustacchi {
680047043c2SRobert Mustacchi 	int ret;
681047043c2SRobert Mustacchi 	dev_info_t *child;
682047043c2SRobert Mustacchi 
683047043c2SRobert Mustacchi 	if (ndi_devi_alloc(azn->azn_dip, acd->acd_name,
684047043c2SRobert Mustacchi 	    (pnode_t)DEVI_SID_NODEID, &child) != NDI_SUCCESS) {
685549e0fd3SRobert Mustacchi 		dev_err(azn->azn_dip, CE_WARN, "!failed to allocate child "
686047043c2SRobert Mustacchi 		    "dip for %s", acd->acd_name);
687047043c2SRobert Mustacchi 		return (B_FALSE);
688047043c2SRobert Mustacchi 	}
689047043c2SRobert Mustacchi 
690047043c2SRobert Mustacchi 	ddi_set_parent_data(child, (void *)acd);
691047043c2SRobert Mustacchi 	if ((ret = ndi_devi_online(child, 0)) != NDI_SUCCESS) {
692*1e0464b8SRobert Mustacchi 		if (acd->acd_warn) {
693*1e0464b8SRobert Mustacchi 			dev_err(azn->azn_dip, CE_WARN, "!failed to online "
694*1e0464b8SRobert Mustacchi 			    "child dip %s: %d", acd->acd_name, ret);
695*1e0464b8SRobert Mustacchi 		}
696047043c2SRobert Mustacchi 		return (B_FALSE);
697047043c2SRobert Mustacchi 	}
698047043c2SRobert Mustacchi 
699047043c2SRobert Mustacchi 	return (B_TRUE);
700047043c2SRobert Mustacchi }
701047043c2SRobert Mustacchi 
702047043c2SRobert Mustacchi static boolean_t
amdzen_map_dfs(amdzen_t * azn)703047043c2SRobert Mustacchi amdzen_map_dfs(amdzen_t *azn)
704047043c2SRobert Mustacchi {
705047043c2SRobert Mustacchi 	amdzen_stub_t *stub;
706047043c2SRobert Mustacchi 
707047043c2SRobert Mustacchi 	ASSERT(MUTEX_HELD(&azn->azn_mutex));
708047043c2SRobert Mustacchi 
709047043c2SRobert Mustacchi 	for (stub = list_head(&azn->azn_df_stubs); stub != NULL;
710047043c2SRobert Mustacchi 	    stub = list_next(&azn->azn_df_stubs, stub)) {
711047043c2SRobert Mustacchi 		amdzen_df_t *df;
712047043c2SRobert Mustacchi 		uint_t dfno;
713047043c2SRobert Mustacchi 
714047043c2SRobert Mustacchi 		dfno = stub->azns_dev - AMDZEN_DF_FIRST_DEVICE;
715047043c2SRobert Mustacchi 		if (dfno > AMDZEN_MAX_DFS) {
716047043c2SRobert Mustacchi 			dev_err(stub->azns_dip, CE_WARN, "encountered df "
717047043c2SRobert Mustacchi 			    "device with illegal DF PCI b/d/f: 0x%x/%x/%x",
718047043c2SRobert Mustacchi 			    stub->azns_bus, stub->azns_dev, stub->azns_func);
719047043c2SRobert Mustacchi 			goto err;
720047043c2SRobert Mustacchi 		}
721047043c2SRobert Mustacchi 
722047043c2SRobert Mustacchi 		df = &azn->azn_dfs[dfno];
723047043c2SRobert Mustacchi 
724047043c2SRobert Mustacchi 		if (stub->azns_func >= AMDZEN_MAX_DF_FUNCS) {
725047043c2SRobert Mustacchi 			dev_err(stub->azns_dip, CE_WARN, "encountered df "
726047043c2SRobert Mustacchi 			    "device with illegal DF PCI b/d/f: 0x%x/%x/%x",
727047043c2SRobert Mustacchi 			    stub->azns_bus, stub->azns_dev, stub->azns_func);
728047043c2SRobert Mustacchi 			goto err;
729047043c2SRobert Mustacchi 		}
730047043c2SRobert Mustacchi 
731047043c2SRobert Mustacchi 		if (df->adf_funcs[stub->azns_func] != NULL) {
732047043c2SRobert Mustacchi 			dev_err(stub->azns_dip, CE_WARN, "encountered "
733047043c2SRobert Mustacchi 			    "duplicate df device with DF PCI b/d/f: 0x%x/%x/%x",
734047043c2SRobert Mustacchi 			    stub->azns_bus, stub->azns_dev, stub->azns_func);
735047043c2SRobert Mustacchi 			goto err;
736047043c2SRobert Mustacchi 		}
737047043c2SRobert Mustacchi 		df->adf_funcs[stub->azns_func] = stub;
738047043c2SRobert Mustacchi 	}
739047043c2SRobert Mustacchi 
740047043c2SRobert Mustacchi 	return (B_TRUE);
741047043c2SRobert Mustacchi 
742047043c2SRobert Mustacchi err:
743047043c2SRobert Mustacchi 	azn->azn_flags |= AMDZEN_F_DEVICE_ERROR;
744047043c2SRobert Mustacchi 	return (B_FALSE);
745047043c2SRobert Mustacchi }
746047043c2SRobert Mustacchi 
747047043c2SRobert Mustacchi static boolean_t
amdzen_check_dfs(amdzen_t * azn)748047043c2SRobert Mustacchi amdzen_check_dfs(amdzen_t *azn)
749047043c2SRobert Mustacchi {
750047043c2SRobert Mustacchi 	uint_t i;
751047043c2SRobert Mustacchi 	boolean_t ret = B_TRUE;
752047043c2SRobert Mustacchi 
753047043c2SRobert Mustacchi 	for (i = 0; i < AMDZEN_MAX_DFS; i++) {
754047043c2SRobert Mustacchi 		amdzen_df_t *df = &azn->azn_dfs[i];
755047043c2SRobert Mustacchi 		uint_t count = 0;
756047043c2SRobert Mustacchi 
757047043c2SRobert Mustacchi 		/*
758047043c2SRobert Mustacchi 		 * We require all platforms to have DFs functions 0-6. Not all
759047043c2SRobert Mustacchi 		 * platforms have DF function 7.
760047043c2SRobert Mustacchi 		 */
761047043c2SRobert Mustacchi 		for (uint_t func = 0; func < AMDZEN_MAX_DF_FUNCS - 1; func++) {
762047043c2SRobert Mustacchi 			if (df->adf_funcs[func] != NULL) {
763047043c2SRobert Mustacchi 				count++;
764047043c2SRobert Mustacchi 			}
765047043c2SRobert Mustacchi 		}
766047043c2SRobert Mustacchi 
767047043c2SRobert Mustacchi 		if (count == 0)
768047043c2SRobert Mustacchi 			continue;
769047043c2SRobert Mustacchi 
770047043c2SRobert Mustacchi 		if (count != 7) {
771047043c2SRobert Mustacchi 			ret = B_FALSE;
772047043c2SRobert Mustacchi 			dev_err(azn->azn_dip, CE_WARN, "df %u devices "
773047043c2SRobert Mustacchi 			    "incomplete", i);
774047043c2SRobert Mustacchi 		} else {
775047043c2SRobert Mustacchi 			df->adf_flags |= AMDZEN_DF_F_VALID;
776047043c2SRobert Mustacchi 			azn->azn_ndfs++;
777047043c2SRobert Mustacchi 		}
778047043c2SRobert Mustacchi 	}
779047043c2SRobert Mustacchi 
780047043c2SRobert Mustacchi 	return (ret);
781047043c2SRobert Mustacchi }
782047043c2SRobert Mustacchi 
783047043c2SRobert Mustacchi static const uint8_t amdzen_df_rome_ids[0x2b] = {
784047043c2SRobert Mustacchi 	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19, 20, 21, 22, 23,
785047043c2SRobert Mustacchi 	24, 25, 26, 27, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43,
786047043c2SRobert Mustacchi 	44, 45, 46, 47, 48
787047043c2SRobert Mustacchi };
788047043c2SRobert Mustacchi 
789e9abe9d6SRobert Mustacchi /*
790e9abe9d6SRobert Mustacchi  * Check the first df entry to see if it belongs to Rome or Milan. If so, then
791e9abe9d6SRobert Mustacchi  * it uses the disjoint ID space.
792e9abe9d6SRobert Mustacchi  */
793e9abe9d6SRobert Mustacchi static boolean_t
amdzen_is_rome_style(uint_t id)794e9abe9d6SRobert Mustacchi amdzen_is_rome_style(uint_t id)
795e9abe9d6SRobert Mustacchi {
796e9abe9d6SRobert Mustacchi 	return (id == 0x1490 || id == 0x1650);
797e9abe9d6SRobert Mustacchi }
798e9abe9d6SRobert Mustacchi 
79971815ce7SRobert Mustacchi /*
80071815ce7SRobert Mustacchi  * To be able to do most other things we want to do, we must first determine
80171815ce7SRobert Mustacchi  * what revision of the DF (data fabric) that we're using.
80271815ce7SRobert Mustacchi  *
80371815ce7SRobert Mustacchi  * Snapshot the df version. This was added explicitly in DFv4.0, around the Zen
80471815ce7SRobert Mustacchi  * 4 timeframe and allows us to tell apart different version of the DF register
80571815ce7SRobert Mustacchi  * set, most usefully when various subtypes were added.
80671815ce7SRobert Mustacchi  *
80771815ce7SRobert Mustacchi  * Older versions can theoretically be told apart based on usage of reserved
80871815ce7SRobert Mustacchi  * registers. We walk these in the following order, starting with the newest rev
80971815ce7SRobert Mustacchi  * and walking backwards to tell things apart:
81071815ce7SRobert Mustacchi  *
81171815ce7SRobert Mustacchi  *   o v3.5 -> Check function 1, register 0x150. This was reserved prior
81271815ce7SRobert Mustacchi  *             to this point. This is actually DF_FIDMASK0_V3P5. We are supposed
81371815ce7SRobert Mustacchi  *             to check bits [7:0].
81471815ce7SRobert Mustacchi  *
81571815ce7SRobert Mustacchi  *   o v3.0 -> Check function 1, register 0x208. The low byte (7:0) was
81671815ce7SRobert Mustacchi  *             changed to indicate a component mask. This is non-zero
81771815ce7SRobert Mustacchi  *             in the 3.0 generation. This is actually DF_FIDMASK_V2.
81871815ce7SRobert Mustacchi  *
81971815ce7SRobert Mustacchi  *   o v2.0 -> This is just the not that case. Presumably v1 wasn't part
82071815ce7SRobert Mustacchi  *             of the Zen generation.
82171815ce7SRobert Mustacchi  *
82271815ce7SRobert Mustacchi  * Because we don't know what version we are yet, we do not use the normal
82371815ce7SRobert Mustacchi  * versioned register accesses which would check what DF version we are and
82471815ce7SRobert Mustacchi  * would want to use the normal indirect register accesses (which also require
82571815ce7SRobert Mustacchi  * us to know the version). We instead do direct broadcast reads.
82671815ce7SRobert Mustacchi  */
82771815ce7SRobert Mustacchi static void
amdzen_determine_df_vers(amdzen_t * azn,amdzen_df_t * df)82871815ce7SRobert Mustacchi amdzen_determine_df_vers(amdzen_t *azn, amdzen_df_t *df)
82971815ce7SRobert Mustacchi {
83071815ce7SRobert Mustacchi 	uint32_t val;
83171815ce7SRobert Mustacchi 	df_reg_def_t rd = DF_FBICNT;
83271815ce7SRobert Mustacchi 
83371815ce7SRobert Mustacchi 	val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg);
83471815ce7SRobert Mustacchi 	df->adf_major = DF_FBICNT_V4_GET_MAJOR(val);
83571815ce7SRobert Mustacchi 	df->adf_minor = DF_FBICNT_V4_GET_MINOR(val);
83671815ce7SRobert Mustacchi 	if (df->adf_major == 0 && df->adf_minor == 0) {
83771815ce7SRobert Mustacchi 		rd = DF_FIDMASK0_V3P5;
83871815ce7SRobert Mustacchi 		val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg);
83971815ce7SRobert Mustacchi 		if (bitx32(val, 7, 0) != 0) {
84071815ce7SRobert Mustacchi 			df->adf_major = 3;
84171815ce7SRobert Mustacchi 			df->adf_minor = 5;
84271815ce7SRobert Mustacchi 			df->adf_rev = DF_REV_3P5;
84371815ce7SRobert Mustacchi 		} else {
84471815ce7SRobert Mustacchi 			rd = DF_FIDMASK_V2;
84571815ce7SRobert Mustacchi 			val = amdzen_stub_get32(df->adf_funcs[rd.drd_func],
84671815ce7SRobert Mustacchi 			    rd.drd_reg);
84771815ce7SRobert Mustacchi 			if (bitx32(val, 7, 0) != 0) {
84871815ce7SRobert Mustacchi 				df->adf_major = 3;
84971815ce7SRobert Mustacchi 				df->adf_minor = 0;
85071815ce7SRobert Mustacchi 				df->adf_rev = DF_REV_3;
85171815ce7SRobert Mustacchi 			} else {
85271815ce7SRobert Mustacchi 				df->adf_major = 2;
85371815ce7SRobert Mustacchi 				df->adf_minor = 0;
85471815ce7SRobert Mustacchi 				df->adf_rev = DF_REV_2;
85571815ce7SRobert Mustacchi 			}
85671815ce7SRobert Mustacchi 		}
85771815ce7SRobert Mustacchi 	} else if (df->adf_major == 4 && df->adf_minor == 0) {
85871815ce7SRobert Mustacchi 		df->adf_rev = DF_REV_4;
85971815ce7SRobert Mustacchi 	} else {
86071815ce7SRobert Mustacchi 		df->adf_rev = DF_REV_UNKNOWN;
86171815ce7SRobert Mustacchi 	}
86271815ce7SRobert Mustacchi }
86371815ce7SRobert Mustacchi 
86471815ce7SRobert Mustacchi /*
86571815ce7SRobert Mustacchi  * All of the different versions of the DF have different ways of getting at and
86671815ce7SRobert Mustacchi  * answering the question of how do I break a fabric ID into a corresponding
86771815ce7SRobert Mustacchi  * socket, die, and component. Importantly the goal here is to obtain, cache,
86871815ce7SRobert Mustacchi  * and normalize:
86971815ce7SRobert Mustacchi  *
87071815ce7SRobert Mustacchi  *  o The DF System Configuration
87171815ce7SRobert Mustacchi  *  o The various Mask registers
87271815ce7SRobert Mustacchi  *  o The Node ID
87371815ce7SRobert Mustacchi  */
87471815ce7SRobert Mustacchi static void
amdzen_determine_fabric_decomp(amdzen_t * azn,amdzen_df_t * df)87571815ce7SRobert Mustacchi amdzen_determine_fabric_decomp(amdzen_t *azn, amdzen_df_t *df)
87671815ce7SRobert Mustacchi {
87771815ce7SRobert Mustacchi 	uint32_t mask;
87871815ce7SRobert Mustacchi 	df_fabric_decomp_t *decomp = &df->adf_decomp;
87971815ce7SRobert Mustacchi 
88071815ce7SRobert Mustacchi 	switch (df->adf_rev) {
88171815ce7SRobert Mustacchi 	case DF_REV_2:
88271815ce7SRobert Mustacchi 		df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V2);
88371815ce7SRobert Mustacchi 		switch (DF_SYSCFG_V2_GET_MY_TYPE(df->adf_syscfg)) {
88471815ce7SRobert Mustacchi 		case DF_DIE_TYPE_CPU:
88571815ce7SRobert Mustacchi 			mask = amdzen_df_read32_bcast(azn, df,
88671815ce7SRobert Mustacchi 			    DF_DIEMASK_CPU_V2);
88771815ce7SRobert Mustacchi 			break;
88871815ce7SRobert Mustacchi 		case DF_DIE_TYPE_APU:
88971815ce7SRobert Mustacchi 			mask = amdzen_df_read32_bcast(azn, df,
89071815ce7SRobert Mustacchi 			    DF_DIEMASK_APU_V2);
89171815ce7SRobert Mustacchi 			break;
89271815ce7SRobert Mustacchi 		default:
89371815ce7SRobert Mustacchi 			panic("DF thinks we're not on a CPU!");
89471815ce7SRobert Mustacchi 		}
89571815ce7SRobert Mustacchi 		df->adf_mask0 = mask;
89671815ce7SRobert Mustacchi 
89771815ce7SRobert Mustacchi 		/*
89871815ce7SRobert Mustacchi 		 * DFv2 is a bit different in how the fabric mask register is
89971815ce7SRobert Mustacchi 		 * phrased. Logically a fabric ID is broken into something that
90071815ce7SRobert Mustacchi 		 * uniquely identifies a "node" (a particular die on a socket)
90171815ce7SRobert Mustacchi 		 * and something that identifies a "component", e.g. a memory
90271815ce7SRobert Mustacchi 		 * controller.
90371815ce7SRobert Mustacchi 		 *
90471815ce7SRobert Mustacchi 		 * Starting with DFv3, these registers logically called out how
90571815ce7SRobert Mustacchi 		 * to separate the fabric ID first into a node and a component.
90671815ce7SRobert Mustacchi 		 * Then the node was then broken down into a socket and die. In
90771815ce7SRobert Mustacchi 		 * DFv2, there is no separate mask and shift of a node. Instead
90871815ce7SRobert Mustacchi 		 * the socket and die are absolute offsets into the fabric ID
90971815ce7SRobert Mustacchi 		 * rather than relative offsets into the node ID. As such, when
91071815ce7SRobert Mustacchi 		 * we encounter DFv2, we fake up a node mask and shift and make
91171815ce7SRobert Mustacchi 		 * it look like DFv3+.
91271815ce7SRobert Mustacchi 		 */
91371815ce7SRobert Mustacchi 		decomp->dfd_node_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) |
91471815ce7SRobert Mustacchi 		    DF_DIEMASK_V2_GET_DIE_MASK(mask);
91571815ce7SRobert Mustacchi 		decomp->dfd_node_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask);
91671815ce7SRobert Mustacchi 		decomp->dfd_comp_mask = DF_DIEMASK_V2_GET_COMP_MASK(mask);
91771815ce7SRobert Mustacchi 		decomp->dfd_comp_shift = 0;
91871815ce7SRobert Mustacchi 
91971815ce7SRobert Mustacchi 		decomp->dfd_sock_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) >>
92071815ce7SRobert Mustacchi 		    decomp->dfd_node_shift;
92171815ce7SRobert Mustacchi 		decomp->dfd_die_mask = DF_DIEMASK_V2_GET_DIE_MASK(mask) >>
92271815ce7SRobert Mustacchi 		    decomp->dfd_node_shift;
92371815ce7SRobert Mustacchi 		decomp->dfd_sock_shift = DF_DIEMASK_V2_GET_SOCK_SHIFT(mask) -
92471815ce7SRobert Mustacchi 		    decomp->dfd_node_shift;
92571815ce7SRobert Mustacchi 		decomp->dfd_die_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask) -
92671815ce7SRobert Mustacchi 		    decomp->dfd_node_shift;
92771815ce7SRobert Mustacchi 		ASSERT3U(decomp->dfd_die_shift, ==, 0);
928dd23d762SRobert Mustacchi 
929dd23d762SRobert Mustacchi 		/*
930dd23d762SRobert Mustacchi 		 * There is no register in the actual data fabric with the node
931dd23d762SRobert Mustacchi 		 * ID in DFv2 that we have found. Instead we take the first
932dd23d762SRobert Mustacchi 		 * entity's fabric ID and transform it into the node id.
933dd23d762SRobert Mustacchi 		 */
934dd23d762SRobert Mustacchi 		df->adf_nodeid = (df->adf_ents[0].adfe_fabric_id &
935dd23d762SRobert Mustacchi 		    decomp->dfd_node_mask) >> decomp->dfd_node_shift;
93671815ce7SRobert Mustacchi 		break;
93771815ce7SRobert Mustacchi 	case DF_REV_3:
93871815ce7SRobert Mustacchi 		df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V3);
93971815ce7SRobert Mustacchi 		df->adf_mask0 =  amdzen_df_read32_bcast(azn, df,
94071815ce7SRobert Mustacchi 		    DF_FIDMASK0_V3);
94171815ce7SRobert Mustacchi 		df->adf_mask1 =  amdzen_df_read32_bcast(azn, df,
94271815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3);
94371815ce7SRobert Mustacchi 
94471815ce7SRobert Mustacchi 		decomp->dfd_sock_mask =
94571815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3_GET_SOCK_MASK(df->adf_mask1);
94671815ce7SRobert Mustacchi 		decomp->dfd_sock_shift =
94771815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3_GET_SOCK_SHIFT(df->adf_mask1);
94871815ce7SRobert Mustacchi 		decomp->dfd_die_mask =
94971815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3_GET_DIE_MASK(df->adf_mask1);
95071815ce7SRobert Mustacchi 		decomp->dfd_die_shift = 0;
95171815ce7SRobert Mustacchi 		decomp->dfd_node_mask =
95271815ce7SRobert Mustacchi 		    DF_FIDMASK0_V3_GET_NODE_MASK(df->adf_mask0);
95371815ce7SRobert Mustacchi 		decomp->dfd_node_shift =
95471815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3_GET_NODE_SHIFT(df->adf_mask1);
95571815ce7SRobert Mustacchi 		decomp->dfd_comp_mask =
95671815ce7SRobert Mustacchi 		    DF_FIDMASK0_V3_GET_COMP_MASK(df->adf_mask0);
95771815ce7SRobert Mustacchi 		decomp->dfd_comp_shift = 0;
958dd23d762SRobert Mustacchi 
959dd23d762SRobert Mustacchi 		df->adf_nodeid = DF_SYSCFG_V3_GET_NODE_ID(df->adf_syscfg);
96071815ce7SRobert Mustacchi 		break;
96171815ce7SRobert Mustacchi 	case DF_REV_3P5:
96271815ce7SRobert Mustacchi 		df->adf_syscfg = amdzen_df_read32_bcast(azn, df,
96371815ce7SRobert Mustacchi 		    DF_SYSCFG_V3P5);
96471815ce7SRobert Mustacchi 		df->adf_mask0 =  amdzen_df_read32_bcast(azn, df,
96571815ce7SRobert Mustacchi 		    DF_FIDMASK0_V3P5);
96671815ce7SRobert Mustacchi 		df->adf_mask1 =  amdzen_df_read32_bcast(azn, df,
96771815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3P5);
96871815ce7SRobert Mustacchi 		df->adf_mask2 =  amdzen_df_read32_bcast(azn, df,
96971815ce7SRobert Mustacchi 		    DF_FIDMASK2_V3P5);
97071815ce7SRobert Mustacchi 
97171815ce7SRobert Mustacchi 		decomp->dfd_sock_mask =
97271815ce7SRobert Mustacchi 		    DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2);
97371815ce7SRobert Mustacchi 		decomp->dfd_sock_shift =
97471815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1);
97571815ce7SRobert Mustacchi 		decomp->dfd_die_mask =
97671815ce7SRobert Mustacchi 		    DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2);
97771815ce7SRobert Mustacchi 		decomp->dfd_die_shift = 0;
97871815ce7SRobert Mustacchi 		decomp->dfd_node_mask =
97971815ce7SRobert Mustacchi 		    DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0);
98071815ce7SRobert Mustacchi 		decomp->dfd_node_shift =
98171815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1);
98271815ce7SRobert Mustacchi 		decomp->dfd_comp_mask =
98371815ce7SRobert Mustacchi 		    DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0);
98471815ce7SRobert Mustacchi 		decomp->dfd_comp_shift = 0;
985dd23d762SRobert Mustacchi 
986dd23d762SRobert Mustacchi 		df->adf_nodeid = DF_SYSCFG_V3P5_GET_NODE_ID(df->adf_syscfg);
98771815ce7SRobert Mustacchi 		break;
98871815ce7SRobert Mustacchi 	case DF_REV_4:
98971815ce7SRobert Mustacchi 		df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V4);
99071815ce7SRobert Mustacchi 		df->adf_mask0 =  amdzen_df_read32_bcast(azn, df,
99171815ce7SRobert Mustacchi 		    DF_FIDMASK0_V4);
99271815ce7SRobert Mustacchi 		df->adf_mask1 =  amdzen_df_read32_bcast(azn, df,
99371815ce7SRobert Mustacchi 		    DF_FIDMASK1_V4);
99471815ce7SRobert Mustacchi 		df->adf_mask2 =  amdzen_df_read32_bcast(azn, df,
99571815ce7SRobert Mustacchi 		    DF_FIDMASK2_V4);
99671815ce7SRobert Mustacchi 
99771815ce7SRobert Mustacchi 		/*
99871815ce7SRobert Mustacchi 		 * The DFv4 registers are at a different location in the DF;
99971815ce7SRobert Mustacchi 		 * however, the actual layout of fields is the same as DFv3.5.
100071815ce7SRobert Mustacchi 		 * This is why you see V3P5 below.
100171815ce7SRobert Mustacchi 		 */
100271815ce7SRobert Mustacchi 		decomp->dfd_sock_mask =
100371815ce7SRobert Mustacchi 		    DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2);
100471815ce7SRobert Mustacchi 		decomp->dfd_sock_shift =
100571815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1);
100671815ce7SRobert Mustacchi 		decomp->dfd_die_mask =
100771815ce7SRobert Mustacchi 		    DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2);
100871815ce7SRobert Mustacchi 		decomp->dfd_die_shift = 0;
100971815ce7SRobert Mustacchi 		decomp->dfd_node_mask =
101071815ce7SRobert Mustacchi 		    DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0);
101171815ce7SRobert Mustacchi 		decomp->dfd_node_shift =
101271815ce7SRobert Mustacchi 		    DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1);
101371815ce7SRobert Mustacchi 		decomp->dfd_comp_mask =
101471815ce7SRobert Mustacchi 		    DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0);
101571815ce7SRobert Mustacchi 		decomp->dfd_comp_shift = 0;
1016dd23d762SRobert Mustacchi 
1017dd23d762SRobert Mustacchi 		df->adf_nodeid = DF_SYSCFG_V4_GET_NODE_ID(df->adf_syscfg);
101871815ce7SRobert Mustacchi 		break;
101971815ce7SRobert Mustacchi 	default:
102071815ce7SRobert Mustacchi 		panic("encountered suspicious, previously rejected DF "
102171815ce7SRobert Mustacchi 		    "rev: 0x%x", df->adf_rev);
102271815ce7SRobert Mustacchi 	}
102371815ce7SRobert Mustacchi }
102471815ce7SRobert Mustacchi 
1025dd23d762SRobert Mustacchi /*
1026dd23d762SRobert Mustacchi  * The purpose of this function is to map CCMs to the corresponding CCDs that
1027dd23d762SRobert Mustacchi  * exist. This is not an obvious thing as there is no direct mapping in the data
1028dd23d762SRobert Mustacchi  * fabric between these IDs.
1029dd23d762SRobert Mustacchi  *
1030dd23d762SRobert Mustacchi  * Prior to DFv4, a given CCM was only ever connected to at most one CCD.
1031dd23d762SRobert Mustacchi  * Starting in DFv4 a given CCM may have one or two SDP (scalable data ports)
1032dd23d762SRobert Mustacchi  * that connect to CCDs. These may be connected to the same CCD or a different
1033dd23d762SRobert Mustacchi  * one. When both ports are enabled we must check whether or not the port is
1034dd23d762SRobert Mustacchi  * considered to be in wide mode. When wide mode is enabled then the two ports
1035dd23d762SRobert Mustacchi  * are connected to a single CCD. If wide mode is disabled then the two ports
1036dd23d762SRobert Mustacchi  * are connected to separate CCDs.
1037dd23d762SRobert Mustacchi  *
1038dd23d762SRobert Mustacchi  * The physical number of a CCD, which is how we determine the SMN aperture to
1039dd23d762SRobert Mustacchi  * use, is based on the CCM ID. In most sockets we have seen up to a maximum of
1040dd23d762SRobert Mustacchi  * 8 CCMs. When a CCM is connected to more than one CCD we have determined based
1041dd23d762SRobert Mustacchi  * on some hints from AMD's ACPI information that the numbering is assumed to be
1042dd23d762SRobert Mustacchi  * that CCM's number plus the total number of CCMs.
1043dd23d762SRobert Mustacchi  *
1044dd23d762SRobert Mustacchi  * More concretely, the SP5 Genoa/Bergamo Zen 4 platform has 8 CCMs. When there
1045dd23d762SRobert Mustacchi  * are more than 8 CCDs installed then CCM 0 maps to CCDs 0 and 8. CCM 1 to CCDs
1046dd23d762SRobert Mustacchi  * 1 and 9, etc. CCMs 4-7 map 1:1 to CCDs 4-7. However, the placement of CCDs
1047dd23d762SRobert Mustacchi  * within the package has changed across generations.
1048dd23d762SRobert Mustacchi  *
1049dd23d762SRobert Mustacchi  * Notably in Rome and Milan (Zen 2/3) it appears that each quadrant had an
1050dd23d762SRobert Mustacchi  * increasing number of CCDs. So CCDs 0/1 were together, 2/3, 4/5, and 6/7. This
1051dd23d762SRobert Mustacchi  * meant that in cases where only a subset of CCDs were populated it'd forcibly
1052dd23d762SRobert Mustacchi  * disable the higher CCD in a group (but with DFv3 the CCM would still be
1053dd23d762SRobert Mustacchi  * enabled). So a 4 CCD config would generally enable CCDs 0, 2, 4, and 6 say.
1054dd23d762SRobert Mustacchi  * This was almost certainly done to balance the NUMA config.
1055dd23d762SRobert Mustacchi  *
1056dd23d762SRobert Mustacchi  * Instead, starting in Genoa (Zen 4) the CCMs are round-robined around the
1057dd23d762SRobert Mustacchi  * quadrants so CCMs (CCDs) 0 (0/8) and 4 (4) are together, 1 (1/9) and 5 (5),
1058dd23d762SRobert Mustacchi  * etc. This is also why we more often see disabled CCMs in Genoa, but not in
1059dd23d762SRobert Mustacchi  * Rome/Milan.
1060dd23d762SRobert Mustacchi  *
1061dd23d762SRobert Mustacchi  * When we're operating in wide mode and therefore both SDPs are connected to a
1062dd23d762SRobert Mustacchi  * single CCD, we've always found that the lower CCD index will be used by the
1063dd23d762SRobert Mustacchi  * system and the higher one is not considered present. Therefore, when
1064dd23d762SRobert Mustacchi  * operating in wide mode, we need to make sure that whenever we have a non-zero
1065dd23d762SRobert Mustacchi  * value for SDPs being connected that we rewrite this to only appear as a
1066dd23d762SRobert Mustacchi  * single CCD is present. It's conceivable (though hard to imagine) that we
1067dd23d762SRobert Mustacchi  * could get a value of 0b10 indicating that only the upper SDP link is active
1068dd23d762SRobert Mustacchi  * for some reason.
1069dd23d762SRobert Mustacchi  */
1070dd23d762SRobert Mustacchi 
1071dd23d762SRobert Mustacchi static void
amdzen_setup_df_ccm(amdzen_t * azn,amdzen_df_t * df,amdzen_df_ent_t * dfe,uint32_t ccmno)1072dd23d762SRobert Mustacchi amdzen_setup_df_ccm(amdzen_t *azn, amdzen_df_t *df, amdzen_df_ent_t *dfe,
1073dd23d762SRobert Mustacchi     uint32_t ccmno)
1074dd23d762SRobert Mustacchi {
1075dd23d762SRobert Mustacchi 	amdzen_ccm_data_t *ccm = &dfe->adfe_data.aded_ccm;
1076dd23d762SRobert Mustacchi 	uint32_t ccd_en;
1077dd23d762SRobert Mustacchi 
1078dd23d762SRobert Mustacchi 	if (df->adf_rev >= DF_REV_4) {
1079dd23d762SRobert Mustacchi 		uint32_t val = amdzen_df_read32(azn, df, dfe->adfe_inst_id,
1080dd23d762SRobert Mustacchi 		    DF_CCD_EN_V4);
1081dd23d762SRobert Mustacchi 		ccd_en = DF_CCD_EN_V4_GET_CCD_EN(val);
1082dd23d762SRobert Mustacchi 
1083dd23d762SRobert Mustacchi 		val = amdzen_df_read32(azn, df, dfe->adfe_inst_id,
1084dd23d762SRobert Mustacchi 		    DF_CCMCFG4_V4);
1085dd23d762SRobert Mustacchi 		if (DF_CCMCFG4_V4_GET_WIDE_EN(val) != 0 && ccd_en != 0) {
1086dd23d762SRobert Mustacchi 			ccd_en = 0x1;
1087dd23d762SRobert Mustacchi 		}
1088dd23d762SRobert Mustacchi 	} else {
1089dd23d762SRobert Mustacchi 		ccd_en = 0x1;
1090dd23d762SRobert Mustacchi 	}
1091dd23d762SRobert Mustacchi 
1092dd23d762SRobert Mustacchi 	for (uint32_t i = 0; i < DF_MAX_CCDS_PER_CCM; i++) {
1093dd23d762SRobert Mustacchi 		ccm->acd_ccd_en[i] = (ccd_en & (1 << i)) != 0;
1094dd23d762SRobert Mustacchi 		if (ccm->acd_ccd_en[i] == 0)
1095dd23d762SRobert Mustacchi 			continue;
1096dd23d762SRobert Mustacchi 		ccm->acd_ccd_id[i] = ccmno + i * df->adf_nccm;
1097dd23d762SRobert Mustacchi 		ccm->acd_nccds++;
1098dd23d762SRobert Mustacchi 	}
1099dd23d762SRobert Mustacchi }
1100dd23d762SRobert Mustacchi 
1101047043c2SRobert Mustacchi /*
1102047043c2SRobert Mustacchi  * Initialize our knowledge about a given series of nodes on the data fabric.
1103047043c2SRobert Mustacchi  */
1104047043c2SRobert Mustacchi static void
amdzen_setup_df(amdzen_t * azn,amdzen_df_t * df)1105047043c2SRobert Mustacchi amdzen_setup_df(amdzen_t *azn, amdzen_df_t *df)
1106047043c2SRobert Mustacchi {
1107047043c2SRobert Mustacchi 	uint_t i;
1108dd23d762SRobert Mustacchi 	uint32_t val, ccmno;
1109047043c2SRobert Mustacchi 
111071815ce7SRobert Mustacchi 	amdzen_determine_df_vers(azn, df);
111171815ce7SRobert Mustacchi 
111271815ce7SRobert Mustacchi 	switch (df->adf_rev) {
111371815ce7SRobert Mustacchi 	case DF_REV_2:
111471815ce7SRobert Mustacchi 	case DF_REV_3:
111571815ce7SRobert Mustacchi 	case DF_REV_3P5:
111671815ce7SRobert Mustacchi 		val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V2);
111771815ce7SRobert Mustacchi 		break;
111871815ce7SRobert Mustacchi 	case DF_REV_4:
111971815ce7SRobert Mustacchi 		val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V4);
112071815ce7SRobert Mustacchi 		break;
112171815ce7SRobert Mustacchi 	default:
112271815ce7SRobert Mustacchi 		dev_err(azn->azn_dip, CE_WARN, "encountered unsupported DF "
112371815ce7SRobert Mustacchi 		    "revision: 0x%x", df->adf_rev);
112471815ce7SRobert Mustacchi 		return;
112571815ce7SRobert Mustacchi 	}
112671815ce7SRobert Mustacchi 	df->adf_nb_busno = DF_CFG_ADDR_CTL_GET_BUS_NUM(val);
112771815ce7SRobert Mustacchi 	val = amdzen_df_read32_bcast(azn, df, DF_FBICNT);
112871815ce7SRobert Mustacchi 	df->adf_nents = DF_FBICNT_GET_COUNT(val);
1129047043c2SRobert Mustacchi 	if (df->adf_nents == 0)
1130047043c2SRobert Mustacchi 		return;
1131047043c2SRobert Mustacchi 	df->adf_ents = kmem_zalloc(sizeof (amdzen_df_ent_t) * df->adf_nents,
1132047043c2SRobert Mustacchi 	    KM_SLEEP);
1133047043c2SRobert Mustacchi 
1134047043c2SRobert Mustacchi 	for (i = 0; i < df->adf_nents; i++) {
1135047043c2SRobert Mustacchi 		amdzen_df_ent_t *dfe = &df->adf_ents[i];
1136047043c2SRobert Mustacchi 		uint8_t inst = i;
1137047043c2SRobert Mustacchi 
1138047043c2SRobert Mustacchi 		/*
1139047043c2SRobert Mustacchi 		 * Unfortunately, Rome uses a discontinuous instance ID pattern
1140047043c2SRobert Mustacchi 		 * while everything else we can find uses a contiguous instance
114119040b41SDan Cross 		 * ID pattern. This means that for Rome, we need to adjust the
1142047043c2SRobert Mustacchi 		 * indexes that we iterate over, though the total number of
114371815ce7SRobert Mustacchi 		 * entries is right. This was carried over into Milan, but not
114471815ce7SRobert Mustacchi 		 * Genoa.
1145047043c2SRobert Mustacchi 		 */
1146e9abe9d6SRobert Mustacchi 		if (amdzen_is_rome_style(df->adf_funcs[0]->azns_did)) {
11477c3513e0SRobert Mustacchi 			if (inst >= ARRAY_SIZE(amdzen_df_rome_ids)) {
1148047043c2SRobert Mustacchi 				dev_err(azn->azn_dip, CE_WARN, "Rome family "
1149047043c2SRobert Mustacchi 				    "processor reported more ids than the PPR, "
1150e9abe9d6SRobert Mustacchi 				    "resetting %u to instance zero", inst);
1151047043c2SRobert Mustacchi 				inst = 0;
1152047043c2SRobert Mustacchi 			} else {
1153047043c2SRobert Mustacchi 				inst = amdzen_df_rome_ids[inst];
1154047043c2SRobert Mustacchi 			}
1155047043c2SRobert Mustacchi 		}
1156047043c2SRobert Mustacchi 
1157047043c2SRobert Mustacchi 		dfe->adfe_drvid = inst;
115871815ce7SRobert Mustacchi 		dfe->adfe_info0 = amdzen_df_read32(azn, df, inst, DF_FBIINFO0);
115971815ce7SRobert Mustacchi 		dfe->adfe_info1 = amdzen_df_read32(azn, df, inst, DF_FBIINFO1);
116071815ce7SRobert Mustacchi 		dfe->adfe_info2 = amdzen_df_read32(azn, df, inst, DF_FBIINFO2);
116171815ce7SRobert Mustacchi 		dfe->adfe_info3 = amdzen_df_read32(azn, df, inst, DF_FBIINFO3);
116271815ce7SRobert Mustacchi 
116371815ce7SRobert Mustacchi 		dfe->adfe_type = DF_FBIINFO0_GET_TYPE(dfe->adfe_info0);
116471815ce7SRobert Mustacchi 		dfe->adfe_subtype = DF_FBIINFO0_GET_SUBTYPE(dfe->adfe_info0);
116571815ce7SRobert Mustacchi 
116671815ce7SRobert Mustacchi 		/*
116771815ce7SRobert Mustacchi 		 * The enabled flag was not present in Zen 1. Simulate it by
116871815ce7SRobert Mustacchi 		 * checking for a non-zero register instead.
116971815ce7SRobert Mustacchi 		 */
117071815ce7SRobert Mustacchi 		if (DF_FBIINFO0_V3_GET_ENABLED(dfe->adfe_info0) ||
117171815ce7SRobert Mustacchi 		    (df->adf_rev == DF_REV_2 && dfe->adfe_info0 != 0)) {
1172047043c2SRobert Mustacchi 			dfe->adfe_flags |= AMDZEN_DFE_F_ENABLED;
1173047043c2SRobert Mustacchi 		}
117471815ce7SRobert Mustacchi 		if (DF_FBIINFO0_GET_HAS_MCA(dfe->adfe_info0)) {
1175047043c2SRobert Mustacchi 			dfe->adfe_flags |= AMDZEN_DFE_F_MCA;
1176047043c2SRobert Mustacchi 		}
117771815ce7SRobert Mustacchi 		dfe->adfe_inst_id = DF_FBIINFO3_GET_INSTID(dfe->adfe_info3);
117871815ce7SRobert Mustacchi 		switch (df->adf_rev) {
117971815ce7SRobert Mustacchi 		case DF_REV_2:
118071815ce7SRobert Mustacchi 			dfe->adfe_fabric_id =
118171815ce7SRobert Mustacchi 			    DF_FBIINFO3_V2_GET_BLOCKID(dfe->adfe_info3);
118271815ce7SRobert Mustacchi 			break;
118371815ce7SRobert Mustacchi 		case DF_REV_3:
118471815ce7SRobert Mustacchi 			dfe->adfe_fabric_id =
118571815ce7SRobert Mustacchi 			    DF_FBIINFO3_V3_GET_BLOCKID(dfe->adfe_info3);
118671815ce7SRobert Mustacchi 			break;
118771815ce7SRobert Mustacchi 		case DF_REV_3P5:
118871815ce7SRobert Mustacchi 			dfe->adfe_fabric_id =
118971815ce7SRobert Mustacchi 			    DF_FBIINFO3_V3P5_GET_BLOCKID(dfe->adfe_info3);
119071815ce7SRobert Mustacchi 			break;
119171815ce7SRobert Mustacchi 		case DF_REV_4:
119271815ce7SRobert Mustacchi 			dfe->adfe_fabric_id =
119371815ce7SRobert Mustacchi 			    DF_FBIINFO3_V4_GET_BLOCKID(dfe->adfe_info3);
119471815ce7SRobert Mustacchi 			break;
119571815ce7SRobert Mustacchi 		default:
119671815ce7SRobert Mustacchi 			panic("encountered suspicious, previously rejected DF "
119771815ce7SRobert Mustacchi 			    "rev: 0x%x", df->adf_rev);
119871815ce7SRobert Mustacchi 		}
1199dd23d762SRobert Mustacchi 
1200dd23d762SRobert Mustacchi 		/*
1201dd23d762SRobert Mustacchi 		 * Record information about a subset of DF entities that we've
1202dd23d762SRobert Mustacchi 		 * found. Currently we're tracking this only for CCMs.
1203dd23d762SRobert Mustacchi 		 */
1204dd23d762SRobert Mustacchi 		if ((dfe->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0)
1205dd23d762SRobert Mustacchi 			continue;
1206dd23d762SRobert Mustacchi 
1207dd23d762SRobert Mustacchi 		if (dfe->adfe_type == DF_TYPE_CCM &&
1208dd23d762SRobert Mustacchi 		    dfe->adfe_subtype == DF_CCM_SUBTYPE_CPU) {
1209dd23d762SRobert Mustacchi 			df->adf_nccm++;
1210dd23d762SRobert Mustacchi 		}
1211dd23d762SRobert Mustacchi 	}
1212dd23d762SRobert Mustacchi 
1213dd23d762SRobert Mustacchi 	/*
1214dd23d762SRobert Mustacchi 	 * Now that we have filled in all of our info, attempt to fill in
1215dd23d762SRobert Mustacchi 	 * specific information about different types of instances.
1216dd23d762SRobert Mustacchi 	 */
1217dd23d762SRobert Mustacchi 	ccmno = 0;
1218dd23d762SRobert Mustacchi 	for (uint_t i = 0; i < df->adf_nents; i++) {
1219dd23d762SRobert Mustacchi 		amdzen_df_ent_t *dfe = &df->adf_ents[i];
1220dd23d762SRobert Mustacchi 
1221dd23d762SRobert Mustacchi 		if ((dfe->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0)
1222dd23d762SRobert Mustacchi 			continue;
1223dd23d762SRobert Mustacchi 
1224dd23d762SRobert Mustacchi 		/*
1225dd23d762SRobert Mustacchi 		 * Perform type and sub-type specific initialization. Currently
1226dd23d762SRobert Mustacchi 		 * limited to CCMs.
1227dd23d762SRobert Mustacchi 		 */
1228dd23d762SRobert Mustacchi 		switch (dfe->adfe_type) {
1229dd23d762SRobert Mustacchi 		case DF_TYPE_CCM:
1230dd23d762SRobert Mustacchi 			amdzen_setup_df_ccm(azn, df, dfe, ccmno);
1231dd23d762SRobert Mustacchi 			ccmno++;
1232dd23d762SRobert Mustacchi 			break;
1233dd23d762SRobert Mustacchi 		default:
1234dd23d762SRobert Mustacchi 			break;
1235dd23d762SRobert Mustacchi 		}
1236047043c2SRobert Mustacchi 	}
1237047043c2SRobert Mustacchi 
123871815ce7SRobert Mustacchi 	amdzen_determine_fabric_decomp(azn, df);
1239047043c2SRobert Mustacchi }
1240047043c2SRobert Mustacchi 
1241047043c2SRobert Mustacchi static void
amdzen_find_nb(amdzen_t * azn,amdzen_df_t * df)1242047043c2SRobert Mustacchi amdzen_find_nb(amdzen_t *azn, amdzen_df_t *df)
1243047043c2SRobert Mustacchi {
1244047043c2SRobert Mustacchi 	amdzen_stub_t *stub;
1245047043c2SRobert Mustacchi 
1246047043c2SRobert Mustacchi 	for (stub = list_head(&azn->azn_nb_stubs); stub != NULL;
1247047043c2SRobert Mustacchi 	    stub = list_next(&azn->azn_nb_stubs, stub)) {
1248047043c2SRobert Mustacchi 		if (stub->azns_bus == df->adf_nb_busno) {
1249047043c2SRobert Mustacchi 			df->adf_flags |= AMDZEN_DF_F_FOUND_NB;
1250047043c2SRobert Mustacchi 			df->adf_nb = stub;
1251047043c2SRobert Mustacchi 			return;
1252047043c2SRobert Mustacchi 		}
1253047043c2SRobert Mustacchi 	}
1254047043c2SRobert Mustacchi }
1255047043c2SRobert Mustacchi 
1256dd23d762SRobert Mustacchi static void
amdzen_initpkg_to_apic(amdzen_t * azn,const uint32_t pkg0,const uint32_t pkg7)1257dd23d762SRobert Mustacchi amdzen_initpkg_to_apic(amdzen_t *azn, const uint32_t pkg0, const uint32_t pkg7)
1258dd23d762SRobert Mustacchi {
1259dd23d762SRobert Mustacchi 	uint32_t nsock, nccd, nccx, ncore, nthr, extccx;
1260dd23d762SRobert Mustacchi 	uint32_t nsock_bits, nccd_bits, nccx_bits, ncore_bits, nthr_bits;
1261dd23d762SRobert Mustacchi 	amdzen_apic_decomp_t *apic = &azn->azn_apic_decomp;
1262dd23d762SRobert Mustacchi 
1263dd23d762SRobert Mustacchi 	/*
1264dd23d762SRobert Mustacchi 	 * These are all 0 based values, meaning that we need to add one to each
1265dd23d762SRobert Mustacchi 	 * of them. However, we skip this because to calculate the number of
1266dd23d762SRobert Mustacchi 	 * bits to cover an entity we would subtract one.
1267dd23d762SRobert Mustacchi 	 */
1268dd23d762SRobert Mustacchi 	nthr = SCFCTP_PMREG_INITPKG0_GET_SMTEN(pkg0);
1269dd23d762SRobert Mustacchi 	ncore = SCFCTP_PMREG_INITPKG7_GET_N_CORES(pkg7);
1270dd23d762SRobert Mustacchi 	nccx = SCFCTP_PMREG_INITPKG7_GET_N_CCXS(pkg7);
1271dd23d762SRobert Mustacchi 	nccd = SCFCTP_PMREG_INITPKG7_GET_N_DIES(pkg7);
1272dd23d762SRobert Mustacchi 	nsock = SCFCTP_PMREG_INITPKG7_GET_N_SOCKETS(pkg7);
1273dd23d762SRobert Mustacchi 
1274dd23d762SRobert Mustacchi 	if (uarchrev_uarch(cpuid_getuarchrev(CPU)) >= X86_UARCH_AMD_ZEN4) {
1275dd23d762SRobert Mustacchi 		extccx = SCFCTP_PMREG_INITPKG7_ZEN4_GET_16TAPIC(pkg7);
1276dd23d762SRobert Mustacchi 	} else {
1277dd23d762SRobert Mustacchi 		extccx = 0;
1278dd23d762SRobert Mustacchi 	}
1279dd23d762SRobert Mustacchi 
1280dd23d762SRobert Mustacchi 	nthr_bits = highbit(nthr);
1281dd23d762SRobert Mustacchi 	ncore_bits = highbit(ncore);
1282dd23d762SRobert Mustacchi 	nccx_bits = highbit(nccx);
1283dd23d762SRobert Mustacchi 	nccd_bits = highbit(nccd);
1284dd23d762SRobert Mustacchi 	nsock_bits = highbit(nsock);
1285dd23d762SRobert Mustacchi 
1286dd23d762SRobert Mustacchi 	apic->aad_thread_shift = 0;
1287dd23d762SRobert Mustacchi 	apic->aad_thread_mask = (1 << nthr_bits) - 1;
1288dd23d762SRobert Mustacchi 
1289dd23d762SRobert Mustacchi 	apic->aad_core_shift = nthr_bits;
1290dd23d762SRobert Mustacchi 	if (ncore_bits > 0) {
1291dd23d762SRobert Mustacchi 		apic->aad_core_mask = (1 << ncore_bits) - 1;
1292dd23d762SRobert Mustacchi 		apic->aad_core_mask <<= apic->aad_core_shift;
1293dd23d762SRobert Mustacchi 	} else {
1294dd23d762SRobert Mustacchi 		apic->aad_core_mask = 0;
1295dd23d762SRobert Mustacchi 	}
1296dd23d762SRobert Mustacchi 
1297dd23d762SRobert Mustacchi 	/*
1298dd23d762SRobert Mustacchi 	 * The APIC_16T_MODE bit indicates that the total shift to start the CCX
1299dd23d762SRobert Mustacchi 	 * should be at 4 bits if it's not. It doesn't mean that the CCX portion
1300dd23d762SRobert Mustacchi 	 * of the value should take up four bits. In the common Genoa case,
1301dd23d762SRobert Mustacchi 	 * nccx_bits will be zero.
1302dd23d762SRobert Mustacchi 	 */
1303dd23d762SRobert Mustacchi 	apic->aad_ccx_shift = apic->aad_core_shift + ncore_bits;
1304dd23d762SRobert Mustacchi 	if (extccx != 0 && apic->aad_ccx_shift < 4) {
1305dd23d762SRobert Mustacchi 		apic->aad_ccx_shift = 4;
1306dd23d762SRobert Mustacchi 	}
1307dd23d762SRobert Mustacchi 	if (nccx_bits > 0) {
1308dd23d762SRobert Mustacchi 		apic->aad_ccx_mask = (1 << nccx_bits) - 1;
1309dd23d762SRobert Mustacchi 		apic->aad_ccx_mask <<= apic->aad_ccx_shift;
1310dd23d762SRobert Mustacchi 	} else {
1311dd23d762SRobert Mustacchi 		apic->aad_ccx_mask = 0;
1312dd23d762SRobert Mustacchi 	}
1313dd23d762SRobert Mustacchi 
1314dd23d762SRobert Mustacchi 	apic->aad_ccd_shift = apic->aad_ccx_shift + nccx_bits;
1315dd23d762SRobert Mustacchi 	if (nccd_bits > 0) {
1316dd23d762SRobert Mustacchi 		apic->aad_ccd_mask = (1 << nccd_bits) - 1;
1317dd23d762SRobert Mustacchi 		apic->aad_ccd_mask <<= apic->aad_ccd_shift;
1318dd23d762SRobert Mustacchi 	} else {
1319dd23d762SRobert Mustacchi 		apic->aad_ccd_mask = 0;
1320dd23d762SRobert Mustacchi 	}
1321dd23d762SRobert Mustacchi 
1322dd23d762SRobert Mustacchi 	apic->aad_sock_shift = apic->aad_ccd_shift + nccd_bits;
1323dd23d762SRobert Mustacchi 	if (nsock_bits > 0) {
1324dd23d762SRobert Mustacchi 		apic->aad_sock_mask = (1 << nsock_bits) - 1;
1325dd23d762SRobert Mustacchi 		apic->aad_sock_mask <<= apic->aad_sock_shift;
1326dd23d762SRobert Mustacchi 	} else {
1327dd23d762SRobert Mustacchi 		apic->aad_sock_mask = 0;
1328dd23d762SRobert Mustacchi 	}
1329dd23d762SRobert Mustacchi 
1330dd23d762SRobert Mustacchi 	/*
1331dd23d762SRobert Mustacchi 	 * Currently all supported Zen 2+ platforms only have a single die per
1332dd23d762SRobert Mustacchi 	 * socket as compared to Zen 1. So this is always kept at zero.
1333dd23d762SRobert Mustacchi 	 */
1334dd23d762SRobert Mustacchi 	apic->aad_die_mask = 0;
1335dd23d762SRobert Mustacchi 	apic->aad_die_shift = 0;
1336dd23d762SRobert Mustacchi }
1337dd23d762SRobert Mustacchi 
1338dd23d762SRobert Mustacchi /*
1339dd23d762SRobert Mustacchi  * We would like to determine what the logical APIC decomposition is on Zen 3
1340dd23d762SRobert Mustacchi  * and newer family parts. While there is information added to CPUID in the form
1341dd23d762SRobert Mustacchi  * of leaf 8X26, that isn't present in Zen 3, so instead we go to what we
1342dd23d762SRobert Mustacchi  * believe is the underlying source of the CPUID data.
1343dd23d762SRobert Mustacchi  *
1344dd23d762SRobert Mustacchi  * Fundamentally there are a series of registers in SMN space that relate to the
1345dd23d762SRobert Mustacchi  * SCFCTP. Coincidentally, there is one of these for each core and there are a
1346dd23d762SRobert Mustacchi  * pair of related SMN registers. L3::SCFCTP::PMREG_INITPKG0 contains
1347dd23d762SRobert Mustacchi  * information about a given's core logical and physical IDs. More interestingly
1348dd23d762SRobert Mustacchi  * for this particular case, L3::SCFCTP::PMREG_INITPKG7, contains the overall
1349dd23d762SRobert Mustacchi  * total number of logical entities. We've been promised that this has to be
1350dd23d762SRobert Mustacchi  * the same across the fabric. That's all well and good, but this begs the
1351dd23d762SRobert Mustacchi  * question of how do we actually get there. The above is a core-specific
1352dd23d762SRobert Mustacchi  * register and requires that we understand information about which CCDs and
1353dd23d762SRobert Mustacchi  * CCXs are actually present.
1354dd23d762SRobert Mustacchi  *
1355dd23d762SRobert Mustacchi  * So we are starting with a data fabric that has some CCM present. The CCM
1356dd23d762SRobert Mustacchi  * entries in the data fabric may be tagged with our ENABLED flag.
1357dd23d762SRobert Mustacchi  * Unfortunately, that can be true regardless of whether or not it's actually
1358dd23d762SRobert Mustacchi  * present or not. As a result, we go to another chunk of SMN space registers,
1359dd23d762SRobert Mustacchi  * SMU::PWR. These contain information about the CCDs, the physical cores that
1360dd23d762SRobert Mustacchi  * are enabled, and related. So we will first walk the DF entities and see if we
1361dd23d762SRobert Mustacchi  * can read its SMN::PWR::CCD_DIE_ID. If we get back a value of all 1s then
1362dd23d762SRobert Mustacchi  * there is nothing present. Otherwise, we should get back something that
1363dd23d762SRobert Mustacchi  * matches information in the data fabric.
1364dd23d762SRobert Mustacchi  *
1365dd23d762SRobert Mustacchi  * With that in hand, we can read the SMU::PWR::CORE_ENABLE register to
1366dd23d762SRobert Mustacchi  * determine which physical cores are enabled in the CCD/CCX. That will finally
1367dd23d762SRobert Mustacchi  * give us an index to get to our friend INITPKG7.
1368dd23d762SRobert Mustacchi  */
1369dd23d762SRobert Mustacchi static boolean_t
amdzen_determine_apic_decomp_initpkg(amdzen_t * azn)1370dd23d762SRobert Mustacchi amdzen_determine_apic_decomp_initpkg(amdzen_t *azn)
1371dd23d762SRobert Mustacchi {
1372dd23d762SRobert Mustacchi 	amdzen_df_t *df = &azn->azn_dfs[0];
1373dd23d762SRobert Mustacchi 	uint32_t ccdno = 0;
1374dd23d762SRobert Mustacchi 
1375dd23d762SRobert Mustacchi 	for (uint_t i = 0; i < df->adf_nents; i++) {
1376dd23d762SRobert Mustacchi 		const amdzen_df_ent_t *ent = &df->adf_ents[i];
1377dd23d762SRobert Mustacchi 		if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0)
1378dd23d762SRobert Mustacchi 			continue;
1379dd23d762SRobert Mustacchi 
1380dd23d762SRobert Mustacchi 		if (ent->adfe_type == DF_TYPE_CCM &&
1381dd23d762SRobert Mustacchi 		    ent->adfe_subtype == DF_CCM_SUBTYPE_CPU) {
1382dd23d762SRobert Mustacchi 			uint32_t val, nccx, pkg7, pkg0;
1383dd23d762SRobert Mustacchi 			smn_reg_t die_reg, thrcfg_reg, core_reg;
1384dd23d762SRobert Mustacchi 			smn_reg_t pkg7_reg, pkg0_reg;
1385dd23d762SRobert Mustacchi 			int core_bit;
1386dd23d762SRobert Mustacchi 			uint8_t pccxno, pcoreno;
1387dd23d762SRobert Mustacchi 
1388dd23d762SRobert Mustacchi 			die_reg = SMUPWR_CCD_DIE_ID(ccdno);
1389dd23d762SRobert Mustacchi 			val = amdzen_smn_read(azn, df, die_reg);
1390dd23d762SRobert Mustacchi 			if (val == SMN_EINVAL32) {
1391dd23d762SRobert Mustacchi 				ccdno++;
1392dd23d762SRobert Mustacchi 				continue;
1393dd23d762SRobert Mustacchi 			}
1394dd23d762SRobert Mustacchi 
1395dd23d762SRobert Mustacchi 			ASSERT3U(SMUPWR_CCD_DIE_ID_GET(val), ==, ccdno);
1396dd23d762SRobert Mustacchi 
1397dd23d762SRobert Mustacchi 			/*
1398dd23d762SRobert Mustacchi 			 * This die actually exists. Switch over to the core
1399dd23d762SRobert Mustacchi 			 * enable register to find one to ask about physically.
1400dd23d762SRobert Mustacchi 			 */
1401dd23d762SRobert Mustacchi 			thrcfg_reg = SMUPWR_THREAD_CFG(ccdno);
1402dd23d762SRobert Mustacchi 			val = amdzen_smn_read(azn, df, thrcfg_reg);
1403dd23d762SRobert Mustacchi 			nccx = SMUPWR_THREAD_CFG_GET_COMPLEX_COUNT(val) + 1;
1404dd23d762SRobert Mustacchi 			core_reg = SMUPWR_CORE_EN(ccdno);
1405dd23d762SRobert Mustacchi 			val = amdzen_smn_read(azn, df, core_reg);
1406dd23d762SRobert Mustacchi 			if (val == 0) {
1407dd23d762SRobert Mustacchi 				ccdno++;
1408dd23d762SRobert Mustacchi 				continue;
1409dd23d762SRobert Mustacchi 			}
1410dd23d762SRobert Mustacchi 
1411dd23d762SRobert Mustacchi 			/*
1412dd23d762SRobert Mustacchi 			 * There exists an enabled physical core. Find the first
1413dd23d762SRobert Mustacchi 			 * index of it and map it to the corresponding CCD and
1414dd23d762SRobert Mustacchi 			 * CCX. ddi_ffs is the bit index, but we want the
1415dd23d762SRobert Mustacchi 			 * physical core number, hence the -1.
1416dd23d762SRobert Mustacchi 			 */
1417dd23d762SRobert Mustacchi 			core_bit = ddi_ffs(val);
1418dd23d762SRobert Mustacchi 			ASSERT3S(core_bit, !=, 0);
1419dd23d762SRobert Mustacchi 			pcoreno = core_bit - 1;
1420dd23d762SRobert Mustacchi 
1421dd23d762SRobert Mustacchi 			/*
1422dd23d762SRobert Mustacchi 			 * Unfortunately SMU::PWR::THREAD_CONFIGURATION gives us
1423dd23d762SRobert Mustacchi 			 * the Number of logical cores that are present in the
1424dd23d762SRobert Mustacchi 			 * complex, not the total number of physical cores. So
1425dd23d762SRobert Mustacchi 			 * here we need to encode that in Zen 3+ the number of
1426dd23d762SRobert Mustacchi 			 * cores per CCX is a maximum of 8. Right now we do
1427dd23d762SRobert Mustacchi 			 * assume that the physical and logical ccx numbering is
1428dd23d762SRobert Mustacchi 			 * equivalent (we have no other way of knowing if it is
1429dd23d762SRobert Mustacchi 			 * or isn't right now) and that we'd always have CCX0
1430dd23d762SRobert Mustacchi 			 * before CCX1. AMD seems to suggest we can assume this,
1431dd23d762SRobert Mustacchi 			 * though it is a worrisome assumption.
1432dd23d762SRobert Mustacchi 			 */
1433dd23d762SRobert Mustacchi 			pccxno = pcoreno / 8;
1434dd23d762SRobert Mustacchi 			ASSERT3U(pccxno, <, nccx);
1435dd23d762SRobert Mustacchi 			pkg7_reg = SCFCTP_PMREG_INITPKG7(ccdno, pccxno,
1436dd23d762SRobert Mustacchi 			    pcoreno);
1437dd23d762SRobert Mustacchi 			pkg7 = amdzen_smn_read(azn, df, pkg7_reg);
1438dd23d762SRobert Mustacchi 			pkg0_reg = SCFCTP_PMREG_INITPKG0(ccdno, pccxno,
1439dd23d762SRobert Mustacchi 			    pcoreno);
1440dd23d762SRobert Mustacchi 			pkg0 = amdzen_smn_read(azn, df, pkg0_reg);
1441dd23d762SRobert Mustacchi 			amdzen_initpkg_to_apic(azn, pkg0, pkg7);
1442dd23d762SRobert Mustacchi 			return (B_TRUE);
1443dd23d762SRobert Mustacchi 		}
1444dd23d762SRobert Mustacchi 	}
1445dd23d762SRobert Mustacchi 
1446dd23d762SRobert Mustacchi 	return (B_FALSE);
1447dd23d762SRobert Mustacchi }
1448dd23d762SRobert Mustacchi 
1449dd23d762SRobert Mustacchi /*
1450dd23d762SRobert Mustacchi  * We have the fun job of trying to figure out what the correct form of the APIC
1451dd23d762SRobert Mustacchi  * decomposition should be and how to break that into its logical components.
1452dd23d762SRobert Mustacchi  * The way that we get at this is generation-specific unfortunately. Here's how
1453dd23d762SRobert Mustacchi  * it works out:
1454dd23d762SRobert Mustacchi  *
1455dd23d762SRobert Mustacchi  * Zen 1-2	This era of CPUs are deceptively simple. The PPR for a given
1456dd23d762SRobert Mustacchi  *		family defines exactly how the APIC ID is broken into logical
1457dd23d762SRobert Mustacchi  *		components and it's fixed. That is, depending on whether or
1458dd23d762SRobert Mustacchi  *		not SMT is enabled. Zen 1 and Zen 2 use different schemes for
1459dd23d762SRobert Mustacchi  *		constructing this. The way that we're supposed to check if SMT
1460dd23d762SRobert Mustacchi  *		is enabled is to use AMD leaf 8X1E and ask how many threads per
1461dd23d762SRobert Mustacchi  *		core there are. We use the x86 feature set to determine that
1462dd23d762SRobert Mustacchi  *		instead.
1463dd23d762SRobert Mustacchi  *
1464dd23d762SRobert Mustacchi  *		More specifically the Zen 1 scheme is 7 bits long. The bits have
1465dd23d762SRobert Mustacchi  *		the following meanings.
1466dd23d762SRobert Mustacchi  *
1467dd23d762SRobert Mustacchi  *		[6]   Socket ID
1468dd23d762SRobert Mustacchi  *		[5:4] Node ID
1469dd23d762SRobert Mustacchi  *		[3]   Logical CCX ID
1470dd23d762SRobert Mustacchi  *		With SMT		Without SMT
1471dd23d762SRobert Mustacchi  *		[2:1] Logical Core ID	[2]   hardcoded to zero
1472dd23d762SRobert Mustacchi  *		[0] Thread ID		[1:0] Logical Core ID
1473dd23d762SRobert Mustacchi  *
1474dd23d762SRobert Mustacchi  *		The following is the Zen 2 scheme assuming SMT. The Zen 2 scheme
1475dd23d762SRobert Mustacchi  *		without SMT shifts everything to the right by one bit.
1476dd23d762SRobert Mustacchi  *
1477dd23d762SRobert Mustacchi  *		[7]   Socket ID
1478dd23d762SRobert Mustacchi  *		[6:4] Logical CCD ID
1479dd23d762SRobert Mustacchi  *		[3]   Logical CCX ID
1480dd23d762SRobert Mustacchi  *		[2:1] Logical Core ID
1481dd23d762SRobert Mustacchi  *		[0]   Thread ID
1482dd23d762SRobert Mustacchi  *
1483dd23d762SRobert Mustacchi  * Zen 3	Zen 3 CPUs moved past the fixed APIC ID format that Zen 1 and
1484dd23d762SRobert Mustacchi  *		Zen 2 had, but also don't give us the nice way of discovering
1485dd23d762SRobert Mustacchi  *		this via CPUID that Zen 4 did. The APIC ID id uses a given
1486dd23d762SRobert Mustacchi  *		number of bits for each logical component that exists, but the
1487dd23d762SRobert Mustacchi  *		exact number varies based on what's actually present. To get at
1488dd23d762SRobert Mustacchi  *		this we use a piece of data that is embedded in the SCFCTP
1489dd23d762SRobert Mustacchi  *		(Scalable Control Fabric, Clocks, Test, Power Gating). This can
1490dd23d762SRobert Mustacchi  *		be used to determine how many logical entities of each kind the
1491dd23d762SRobert Mustacchi  *		system thinks exist. While we could use the various CPUID
1492dd23d762SRobert Mustacchi  *		topology items to try to speed this up, they don't tell us the
1493dd23d762SRobert Mustacchi  *		die information that we need to do this.
1494dd23d762SRobert Mustacchi  *
1495dd23d762SRobert Mustacchi  * Zen 4+	Zen 4 introduced CPUID leaf 8000_0026h which gives us a means
1496dd23d762SRobert Mustacchi  *		for determining how to extract the CCD, CCX, and related pieces
1497dd23d762SRobert Mustacchi  *		out of the device. One thing we have to be aware of is that when
1498dd23d762SRobert Mustacchi  *		the CCD and CCX shift are the same, that means that there is
1499dd23d762SRobert Mustacchi  *		only a single CCX and therefore have to take that into account
1500dd23d762SRobert Mustacchi  *		appropriately. This is the case generally on Zen 4 platforms,
1501dd23d762SRobert Mustacchi  *		but not on Bergamo. Until we can confirm the actual CPUID leaf
1502dd23d762SRobert Mustacchi  *		values that we receive in the cases of Bergamo and others, we
1503dd23d762SRobert Mustacchi  *		opt instead to use the same SCFCTP scheme as Zen 3.
1504dd23d762SRobert Mustacchi  */
1505dd23d762SRobert Mustacchi static boolean_t
amdzen_determine_apic_decomp(amdzen_t * azn)1506dd23d762SRobert Mustacchi amdzen_determine_apic_decomp(amdzen_t *azn)
1507dd23d762SRobert Mustacchi {
1508dd23d762SRobert Mustacchi 	x86_uarchrev_t uarchrev = cpuid_getuarchrev(CPU);
1509dd23d762SRobert Mustacchi 	amdzen_apic_decomp_t *apic = &azn->azn_apic_decomp;
1510dd23d762SRobert Mustacchi 	boolean_t smt = is_x86_feature(x86_featureset, X86FSET_HTT);
1511dd23d762SRobert Mustacchi 
1512dd23d762SRobert Mustacchi 	switch (uarchrev_uarch(uarchrev)) {
1513dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZEN1:
1514dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZENPLUS:
1515dd23d762SRobert Mustacchi 		apic->aad_sock_mask = 0x40;
1516dd23d762SRobert Mustacchi 		apic->aad_sock_shift = 6;
1517dd23d762SRobert Mustacchi 		apic->aad_die_mask = 0x30;
1518dd23d762SRobert Mustacchi 		apic->aad_die_shift = 4;
1519dd23d762SRobert Mustacchi 		apic->aad_ccd_mask = 0;
1520dd23d762SRobert Mustacchi 		apic->aad_ccd_shift = 0;
1521dd23d762SRobert Mustacchi 		apic->aad_ccx_mask = 0x08;
1522dd23d762SRobert Mustacchi 		apic->aad_ccx_shift = 3;
1523dd23d762SRobert Mustacchi 
1524dd23d762SRobert Mustacchi 		if (smt) {
1525dd23d762SRobert Mustacchi 			apic->aad_core_mask = 0x06;
1526dd23d762SRobert Mustacchi 			apic->aad_core_shift = 1;
1527dd23d762SRobert Mustacchi 			apic->aad_thread_mask = 0x1;
1528dd23d762SRobert Mustacchi 			apic->aad_thread_shift = 0;
1529dd23d762SRobert Mustacchi 		} else {
1530dd23d762SRobert Mustacchi 			apic->aad_core_mask = 0x03;
1531dd23d762SRobert Mustacchi 			apic->aad_core_shift = 0;
1532dd23d762SRobert Mustacchi 			apic->aad_thread_mask = 0;
1533dd23d762SRobert Mustacchi 			apic->aad_thread_shift = 0;
1534dd23d762SRobert Mustacchi 		}
1535dd23d762SRobert Mustacchi 		break;
1536dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZEN2:
1537dd23d762SRobert Mustacchi 		if (smt) {
1538dd23d762SRobert Mustacchi 			apic->aad_sock_mask = 0x80;
1539dd23d762SRobert Mustacchi 			apic->aad_sock_shift = 7;
1540dd23d762SRobert Mustacchi 			apic->aad_die_mask = 0;
1541dd23d762SRobert Mustacchi 			apic->aad_die_shift = 0;
1542dd23d762SRobert Mustacchi 			apic->aad_ccd_mask = 0x70;
1543dd23d762SRobert Mustacchi 			apic->aad_ccd_shift = 4;
1544dd23d762SRobert Mustacchi 			apic->aad_ccx_mask = 0x08;
1545dd23d762SRobert Mustacchi 			apic->aad_ccx_shift = 3;
1546dd23d762SRobert Mustacchi 			apic->aad_core_mask = 0x06;
1547dd23d762SRobert Mustacchi 			apic->aad_core_shift = 1;
1548dd23d762SRobert Mustacchi 			apic->aad_thread_mask = 0x01;
1549dd23d762SRobert Mustacchi 			apic->aad_thread_shift = 0;
1550dd23d762SRobert Mustacchi 		} else {
1551dd23d762SRobert Mustacchi 			apic->aad_sock_mask = 0x40;
1552dd23d762SRobert Mustacchi 			apic->aad_sock_shift = 6;
1553dd23d762SRobert Mustacchi 			apic->aad_die_mask = 0;
1554dd23d762SRobert Mustacchi 			apic->aad_die_shift = 0;
1555dd23d762SRobert Mustacchi 			apic->aad_ccd_mask = 0x38;
1556dd23d762SRobert Mustacchi 			apic->aad_ccd_shift = 3;
1557dd23d762SRobert Mustacchi 			apic->aad_ccx_mask = 0x04;
1558dd23d762SRobert Mustacchi 			apic->aad_ccx_shift = 2;
1559dd23d762SRobert Mustacchi 			apic->aad_core_mask = 0x3;
1560dd23d762SRobert Mustacchi 			apic->aad_core_shift = 0;
1561dd23d762SRobert Mustacchi 			apic->aad_thread_mask = 0;
1562dd23d762SRobert Mustacchi 			apic->aad_thread_shift = 0;
1563dd23d762SRobert Mustacchi 		}
1564dd23d762SRobert Mustacchi 		break;
1565dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZEN3:
1566dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZEN4:
1567dd23d762SRobert Mustacchi 		return (amdzen_determine_apic_decomp_initpkg(azn));
1568dd23d762SRobert Mustacchi 	default:
1569dd23d762SRobert Mustacchi 		return (B_FALSE);
1570dd23d762SRobert Mustacchi 	}
1571dd23d762SRobert Mustacchi 	return (B_TRUE);
1572dd23d762SRobert Mustacchi }
1573dd23d762SRobert Mustacchi 
1574dd23d762SRobert Mustacchi /*
1575dd23d762SRobert Mustacchi  * Snapshot the number of cores that can exist in a CCX based on the Zen
1576dd23d762SRobert Mustacchi  * microarchitecture revision. In Zen 1-4 this has been a constant number
1577dd23d762SRobert Mustacchi  * regardless of the actual CPU Family.
1578dd23d762SRobert Mustacchi  */
1579dd23d762SRobert Mustacchi static void
amdzen_determine_ncore_per_ccx(amdzen_t * azn)1580dd23d762SRobert Mustacchi amdzen_determine_ncore_per_ccx(amdzen_t *azn)
1581dd23d762SRobert Mustacchi {
1582dd23d762SRobert Mustacchi 	x86_uarchrev_t uarchrev = cpuid_getuarchrev(CPU);
1583dd23d762SRobert Mustacchi 
1584dd23d762SRobert Mustacchi 	switch (uarchrev_uarch(uarchrev)) {
1585dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZEN1:
1586dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZENPLUS:
1587dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZEN2:
1588dd23d762SRobert Mustacchi 		azn->azn_ncore_per_ccx = 4;
1589dd23d762SRobert Mustacchi 		break;
1590dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZEN3:
1591dd23d762SRobert Mustacchi 	case X86_UARCH_AMD_ZEN4:
1592dd23d762SRobert Mustacchi 		azn->azn_ncore_per_ccx = 8;
1593dd23d762SRobert Mustacchi 		break;
1594dd23d762SRobert Mustacchi 	default:
1595dd23d762SRobert Mustacchi 		panic("asked about non-Zen uarch");
1596dd23d762SRobert Mustacchi 	}
1597dd23d762SRobert Mustacchi }
1598dd23d762SRobert Mustacchi 
1599dd23d762SRobert Mustacchi /*
1600dd23d762SRobert Mustacchi  * We need to be careful using this function as different AMD generations have
1601dd23d762SRobert Mustacchi  * acted in different ways when there is a missing CCD. We've found that in
1602dd23d762SRobert Mustacchi  * hardware where the CCM is enabled but there is no CCD attached, it generally
1603dd23d762SRobert Mustacchi  * is safe (i.e. DFv3 on Rome), but on DFv4 if we ask for a CCD that would
1604dd23d762SRobert Mustacchi  * correspond to a disabled CCM then the firmware may inject a fatal error
1605dd23d762SRobert Mustacchi  * (which is hopefully something missing in our RAS/MCA-X enablement).
1606dd23d762SRobert Mustacchi  *
1607dd23d762SRobert Mustacchi  * Put differently if this doesn't correspond to an Enabled CCM and you know the
1608dd23d762SRobert Mustacchi  * number of valid CCDs on this, don't use it.
1609dd23d762SRobert Mustacchi  */
1610dd23d762SRobert Mustacchi static boolean_t
amdzen_ccd_present(amdzen_t * azn,amdzen_df_t * df,uint32_t ccdno)1611dd23d762SRobert Mustacchi amdzen_ccd_present(amdzen_t *azn, amdzen_df_t *df, uint32_t ccdno)
1612dd23d762SRobert Mustacchi {
1613dd23d762SRobert Mustacchi 	smn_reg_t die_reg = SMUPWR_CCD_DIE_ID(ccdno);
1614dd23d762SRobert Mustacchi 	uint32_t val = amdzen_smn_read(azn, df, die_reg);
1615dd23d762SRobert Mustacchi 	if (val == SMN_EINVAL32) {
1616dd23d762SRobert Mustacchi 		return (B_FALSE);
1617dd23d762SRobert Mustacchi 	}
1618dd23d762SRobert Mustacchi 
1619dd23d762SRobert Mustacchi 	ASSERT3U(ccdno, ==, SMUPWR_CCD_DIE_ID_GET(val));
1620dd23d762SRobert Mustacchi 	return (B_TRUE);
1621dd23d762SRobert Mustacchi }
1622dd23d762SRobert Mustacchi 
1623dd23d762SRobert Mustacchi /*
1624dd23d762SRobert Mustacchi  * Attempt to determine a logical CCD number of a given CCD where we don't have
1625dd23d762SRobert Mustacchi  * hardware support for L3::SCFCTP::PMREG_INITPKG* (e.g. pre-Zen 3 systems).
1626dd23d762SRobert Mustacchi  * The CCD numbers that we have are the in the physical space. Likely beacuse of
1627dd23d762SRobert Mustacchi  * how the orientation of CCM numbers map to physical locations and the layout
1628dd23d762SRobert Mustacchi  * of them within the pacakge, we haven't found a good way using the core DFv3
1629dd23d762SRobert Mustacchi  * registers to determine if a given CCD is actually present or not as generally
1630dd23d762SRobert Mustacchi  * all the CCMs are left enabled. Instead we use SMU::PWR::DIE_ID as a proxy to
1631dd23d762SRobert Mustacchi  * determine CCD presence.
1632dd23d762SRobert Mustacchi  */
1633dd23d762SRobert Mustacchi static uint32_t
amdzen_ccd_log_id_zen2(amdzen_t * azn,amdzen_df_t * df,const amdzen_df_ent_t * targ)1634dd23d762SRobert Mustacchi amdzen_ccd_log_id_zen2(amdzen_t *azn, amdzen_df_t *df,
1635dd23d762SRobert Mustacchi     const amdzen_df_ent_t *targ)
1636dd23d762SRobert Mustacchi {
1637dd23d762SRobert Mustacchi 	uint32_t smnid = 0;
1638dd23d762SRobert Mustacchi 	uint32_t logid = 0;
1639dd23d762SRobert Mustacchi 
1640dd23d762SRobert Mustacchi 	for (uint_t i = 0; i < df->adf_nents; i++) {
1641dd23d762SRobert Mustacchi 		const amdzen_df_ent_t *ent = &df->adf_ents[i];
1642dd23d762SRobert Mustacchi 
1643dd23d762SRobert Mustacchi 		if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) {
1644dd23d762SRobert Mustacchi 			continue;
1645dd23d762SRobert Mustacchi 		}
1646dd23d762SRobert Mustacchi 
1647dd23d762SRobert Mustacchi 		if (ent->adfe_inst_id == targ->adfe_inst_id) {
1648dd23d762SRobert Mustacchi 			return (logid);
1649dd23d762SRobert Mustacchi 		}
1650dd23d762SRobert Mustacchi 
1651dd23d762SRobert Mustacchi 		if (ent->adfe_type == targ->adfe_type &&
1652dd23d762SRobert Mustacchi 		    ent->adfe_subtype == targ->adfe_subtype) {
1653dd23d762SRobert Mustacchi 			boolean_t present = amdzen_ccd_present(azn, df, smnid);
1654dd23d762SRobert Mustacchi 			smnid++;
1655dd23d762SRobert Mustacchi 			if (present) {
1656dd23d762SRobert Mustacchi 				logid++;
1657dd23d762SRobert Mustacchi 			}
1658dd23d762SRobert Mustacchi 		}
1659dd23d762SRobert Mustacchi 	}
1660dd23d762SRobert Mustacchi 
1661dd23d762SRobert Mustacchi 	panic("asked to match against invalid DF entity %p in df %p", targ, df);
1662dd23d762SRobert Mustacchi }
1663dd23d762SRobert Mustacchi 
1664dd23d762SRobert Mustacchi static void
amdzen_ccd_fill_core_initpkg0(amdzen_t * azn,amdzen_df_t * df,amdzen_topo_ccd_t * ccd,amdzen_topo_ccx_t * ccx,amdzen_topo_core_t * core,boolean_t * ccd_set,boolean_t * ccx_set)1665dd23d762SRobert Mustacchi amdzen_ccd_fill_core_initpkg0(amdzen_t *azn, amdzen_df_t *df,
1666dd23d762SRobert Mustacchi     amdzen_topo_ccd_t *ccd, amdzen_topo_ccx_t *ccx, amdzen_topo_core_t *core,
1667dd23d762SRobert Mustacchi     boolean_t *ccd_set, boolean_t *ccx_set)
1668dd23d762SRobert Mustacchi {
1669dd23d762SRobert Mustacchi 	smn_reg_t pkg0_reg;
1670dd23d762SRobert Mustacchi 	uint32_t pkg0;
1671dd23d762SRobert Mustacchi 
1672dd23d762SRobert Mustacchi 	pkg0_reg = SCFCTP_PMREG_INITPKG0(ccd->atccd_phys_no, ccx->atccx_phys_no,
1673dd23d762SRobert Mustacchi 	    core->atcore_phys_no);
1674dd23d762SRobert Mustacchi 	pkg0 = amdzen_smn_read(azn, df, pkg0_reg);
1675dd23d762SRobert Mustacchi 	core->atcore_log_no = SCFCTP_PMREG_INITPKG0_GET_LOG_CORE(pkg0);
1676dd23d762SRobert Mustacchi 
1677dd23d762SRobert Mustacchi 	if (!*ccx_set) {
1678dd23d762SRobert Mustacchi 		ccx->atccx_log_no = SCFCTP_PMREG_INITPKG0_GET_LOG_CCX(pkg0);
1679dd23d762SRobert Mustacchi 		*ccx_set = B_TRUE;
1680dd23d762SRobert Mustacchi 	}
1681dd23d762SRobert Mustacchi 
1682dd23d762SRobert Mustacchi 	if (!*ccd_set) {
1683dd23d762SRobert Mustacchi 		ccd->atccd_log_no = SCFCTP_PMREG_INITPKG0_GET_LOG_DIE(pkg0);
1684dd23d762SRobert Mustacchi 		*ccd_set = B_TRUE;
1685dd23d762SRobert Mustacchi 	}
1686dd23d762SRobert Mustacchi }
1687dd23d762SRobert Mustacchi 
1688dd23d762SRobert Mustacchi /*
1689dd23d762SRobert Mustacchi  * Attempt to fill in the physical topology information for this given CCD.
1690dd23d762SRobert Mustacchi  * There are a few steps to this that we undertake to perform this as follows:
1691dd23d762SRobert Mustacchi  *
1692dd23d762SRobert Mustacchi  * 1) First we determine whether the CCD is actually present or not by reading
1693dd23d762SRobert Mustacchi  * SMU::PWR::DIE_ID. CCDs that are not installed will still have an enabled DF
1694dd23d762SRobert Mustacchi  * entry it appears, but the request for the die ID will returns an invalid
1695dd23d762SRobert Mustacchi  * read (all 1s). This die ID should match what we think of as the SMN number
1696dd23d762SRobert Mustacchi  * below. If not, we're in trouble and the rest of this is in question.
1697dd23d762SRobert Mustacchi  *
1698dd23d762SRobert Mustacchi  * 2) We use the SMU::PWR registers to determine how many logical and physical
1699dd23d762SRobert Mustacchi  * cores are present in this CCD and how they are split amongst the CCX. Here we
1700dd23d762SRobert Mustacchi  * need to encode the CPU to CCX core size rankings. Through this process we
1701dd23d762SRobert Mustacchi  * determine and fill out which threads and cores are enabled.
1702dd23d762SRobert Mustacchi  *
1703dd23d762SRobert Mustacchi  * 3) In Zen 3+ we then will read each core's INITPK0 values to ensure that we
1704dd23d762SRobert Mustacchi  * have a proper physical to logical mapping, at which point we can fill in the
1705dd23d762SRobert Mustacchi  * APIC IDs. For Zen 2, we will set the AMDZEN_TOPO_CCD_F_CORE_PHYS_UNKNOWN to
1706dd23d762SRobert Mustacchi  * indicate that we just mapped the first logical processor to the first enabled
1707dd23d762SRobert Mustacchi  * core.
1708dd23d762SRobert Mustacchi  *
1709dd23d762SRobert Mustacchi  * 4) Once we have the logical IDs determined we will construct the APIC ID that
1710dd23d762SRobert Mustacchi  * we expect this to have.
1711dd23d762SRobert Mustacchi  *
1712dd23d762SRobert Mustacchi  * Steps (2) - (4) are intertwined and done together.
1713dd23d762SRobert Mustacchi  */
1714dd23d762SRobert Mustacchi static void
amdzen_ccd_fill_topo(amdzen_t * azn,amdzen_df_t * df,amdzen_df_ent_t * ent,amdzen_topo_ccd_t * ccd)1715dd23d762SRobert Mustacchi amdzen_ccd_fill_topo(amdzen_t *azn, amdzen_df_t *df, amdzen_df_ent_t *ent,
1716dd23d762SRobert Mustacchi     amdzen_topo_ccd_t *ccd)
1717dd23d762SRobert Mustacchi {
1718dd23d762SRobert Mustacchi 	uint32_t val, nccx, core_en, thread_en;
1719dd23d762SRobert Mustacchi 	uint32_t nlcore_per_ccx, nthreads_per_core;
1720dd23d762SRobert Mustacchi 	uint32_t sockid, dieid, compid;
1721dd23d762SRobert Mustacchi 	const uint32_t ccdno = ccd->atccd_phys_no;
1722dd23d762SRobert Mustacchi 	const x86_uarch_t uarch = uarchrev_uarch(cpuid_getuarchrev(CPU));
1723dd23d762SRobert Mustacchi 	boolean_t smt, pkg0_ids, logccd_set = B_FALSE;
1724dd23d762SRobert Mustacchi 	smn_reg_t reg;
1725dd23d762SRobert Mustacchi 
1726dd23d762SRobert Mustacchi 	ASSERT(MUTEX_HELD(&azn->azn_mutex));
1727dd23d762SRobert Mustacchi 	if (!amdzen_ccd_present(azn, df, ccdno)) {
1728dd23d762SRobert Mustacchi 		ccd->atccd_err = AMDZEN_TOPO_CCD_E_CCD_MISSING;
1729dd23d762SRobert Mustacchi 		return;
1730dd23d762SRobert Mustacchi 	}
1731dd23d762SRobert Mustacchi 
1732dd23d762SRobert Mustacchi 	reg = SMUPWR_THREAD_CFG(ccdno);
1733dd23d762SRobert Mustacchi 	val = amdzen_smn_read(azn, df, reg);
1734dd23d762SRobert Mustacchi 	nccx = SMUPWR_THREAD_CFG_GET_COMPLEX_COUNT(val) + 1;
1735dd23d762SRobert Mustacchi 	nlcore_per_ccx = SMUPWR_THREAD_CFG_GET_COMPLEX_COUNT(val) + 1;
1736dd23d762SRobert Mustacchi 	smt = SMUPWR_THREAD_CFG_GET_SMT_MODE(val);
1737dd23d762SRobert Mustacchi 	ASSERT3U(nccx, <=, AMDZEN_TOPO_CCD_MAX_CCX);
1738dd23d762SRobert Mustacchi 	if (smt == SMUPWR_THREAD_CFG_SMT_MODE_SMT) {
1739dd23d762SRobert Mustacchi 		nthreads_per_core = 2;
1740dd23d762SRobert Mustacchi 	} else {
1741dd23d762SRobert Mustacchi 		nthreads_per_core = 1;
1742dd23d762SRobert Mustacchi 	}
1743dd23d762SRobert Mustacchi 
1744dd23d762SRobert Mustacchi 	reg = SMUPWR_CORE_EN(ccdno);
1745dd23d762SRobert Mustacchi 	core_en = amdzen_smn_read(azn, df, reg);
1746dd23d762SRobert Mustacchi 	reg = SMUPWR_THREAD_EN(ccdno);
1747dd23d762SRobert Mustacchi 	thread_en = amdzen_smn_read(azn, df, reg);
1748dd23d762SRobert Mustacchi 
1749dd23d762SRobert Mustacchi 	/*
1750dd23d762SRobert Mustacchi 	 * The BSP is never enabled in a conventional sense and therefore the
1751dd23d762SRobert Mustacchi 	 * bit is reserved and left as 0. As the BSP should be in the first CCD,
1752dd23d762SRobert Mustacchi 	 * we go through and OR back in the bit lest we think the thread isn't
1753dd23d762SRobert Mustacchi 	 * enabled.
1754dd23d762SRobert Mustacchi 	 */
1755dd23d762SRobert Mustacchi 	if (ccdno == 0) {
1756dd23d762SRobert Mustacchi 		thread_en |= 1;
1757dd23d762SRobert Mustacchi 	}
1758dd23d762SRobert Mustacchi 
1759dd23d762SRobert Mustacchi 	ccd->atccd_phys_no = ccdno;
1760dd23d762SRobert Mustacchi 	if (uarch >= X86_UARCH_AMD_ZEN3) {
1761dd23d762SRobert Mustacchi 		pkg0_ids = B_TRUE;
1762dd23d762SRobert Mustacchi 	} else {
1763dd23d762SRobert Mustacchi 		ccd->atccd_flags |= AMDZEN_TOPO_CCD_F_CORE_PHYS_UNKNOWN;
1764dd23d762SRobert Mustacchi 		pkg0_ids = B_FALSE;
1765dd23d762SRobert Mustacchi 
1766dd23d762SRobert Mustacchi 		/*
1767dd23d762SRobert Mustacchi 		 * Determine the CCD logical ID for Zen 2 now since this doesn't
1768dd23d762SRobert Mustacchi 		 * rely upon needing a valid physical core.
1769dd23d762SRobert Mustacchi 		 */
1770dd23d762SRobert Mustacchi 		ccd->atccd_log_no = amdzen_ccd_log_id_zen2(azn, df, ent);
1771dd23d762SRobert Mustacchi 		logccd_set = B_TRUE;
1772dd23d762SRobert Mustacchi 	}
1773dd23d762SRobert Mustacchi 
1774dd23d762SRobert Mustacchi 	/*
1775dd23d762SRobert Mustacchi 	 * To construct the APIC ID we need to know the socket and die (not CCD)
1776dd23d762SRobert Mustacchi 	 * this is on. We deconstruct the CCD's fabric ID to determine that.
1777dd23d762SRobert Mustacchi 	 */
1778dd23d762SRobert Mustacchi 	zen_fabric_id_decompose(&df->adf_decomp, ent->adfe_fabric_id, &sockid,
1779dd23d762SRobert Mustacchi 	    &dieid, &compid);
1780dd23d762SRobert Mustacchi 
1781dd23d762SRobert Mustacchi 	/*
1782dd23d762SRobert Mustacchi 	 * At this point we have all the information about the CCD, the number
1783dd23d762SRobert Mustacchi 	 * of CCX instances, and which physical cores and threads are enabled.
1784dd23d762SRobert Mustacchi 	 * Currently we assume that if we have one CCX enabled, then it is
1785dd23d762SRobert Mustacchi 	 * always CCX0. We cannot find evidence of a two CCX supporting part
1786dd23d762SRobert Mustacchi 	 * that doesn't always ship with both CCXs present and enabled.
1787dd23d762SRobert Mustacchi 	 */
1788dd23d762SRobert Mustacchi 	ccd->atccd_nlog_ccx = ccd->atccd_nphys_ccx = nccx;
1789dd23d762SRobert Mustacchi 	for (uint32_t ccxno = 0; ccxno < nccx; ccxno++) {
1790dd23d762SRobert Mustacchi 		amdzen_topo_ccx_t *ccx = &ccd->atccd_ccx[ccxno];
1791dd23d762SRobert Mustacchi 		const uint32_t core_mask = (1 << azn->azn_ncore_per_ccx) - 1;
1792dd23d762SRobert Mustacchi 		const uint32_t core_shift = ccxno * azn->azn_ncore_per_ccx;
1793dd23d762SRobert Mustacchi 		const uint32_t ccx_core_en = (core_en >> core_shift) &
1794dd23d762SRobert Mustacchi 		    core_mask;
1795dd23d762SRobert Mustacchi 		boolean_t logccx_set = B_FALSE;
1796dd23d762SRobert Mustacchi 
1797dd23d762SRobert Mustacchi 		ccd->atccd_ccx_en[ccxno] = 1;
1798dd23d762SRobert Mustacchi 		ccx->atccx_phys_no = ccxno;
1799dd23d762SRobert Mustacchi 		ccx->atccx_nphys_cores = azn->azn_ncore_per_ccx;
1800dd23d762SRobert Mustacchi 		ccx->atccx_nlog_cores = nlcore_per_ccx;
1801dd23d762SRobert Mustacchi 
1802dd23d762SRobert Mustacchi 		if (!pkg0_ids) {
1803dd23d762SRobert Mustacchi 			ccx->atccx_log_no = ccx->atccx_phys_no;
1804dd23d762SRobert Mustacchi 			logccx_set = B_TRUE;
1805dd23d762SRobert Mustacchi 		}
1806dd23d762SRobert Mustacchi 
1807dd23d762SRobert Mustacchi 		for (uint32_t coreno = 0, logcorezen2 = 0;
1808dd23d762SRobert Mustacchi 		    coreno < azn->azn_ncore_per_ccx; coreno++) {
1809dd23d762SRobert Mustacchi 			amdzen_topo_core_t *core = &ccx->atccx_cores[coreno];
1810dd23d762SRobert Mustacchi 
1811dd23d762SRobert Mustacchi 			if ((ccx_core_en & (1 << coreno)) == 0) {
1812dd23d762SRobert Mustacchi 				continue;
1813dd23d762SRobert Mustacchi 			}
1814dd23d762SRobert Mustacchi 
1815dd23d762SRobert Mustacchi 			ccx->atccx_core_en[coreno] = 1;
1816dd23d762SRobert Mustacchi 			core->atcore_phys_no = coreno;
1817dd23d762SRobert Mustacchi 
1818dd23d762SRobert Mustacchi 			/*
1819dd23d762SRobert Mustacchi 			 * Now that we have the physical core number present, we
1820dd23d762SRobert Mustacchi 			 * must determine the logical core number and fill out
1821dd23d762SRobert Mustacchi 			 * the logical CCX/CCD if it has not been set. We must
1822dd23d762SRobert Mustacchi 			 * do this before we attempt to look at which threads
1823dd23d762SRobert Mustacchi 			 * are enabled, because that operates based upon logical
1824dd23d762SRobert Mustacchi 			 * core number.
1825dd23d762SRobert Mustacchi 			 *
1826dd23d762SRobert Mustacchi 			 * For Zen 2 we do not have INITPKG0 at our disposal. We
1827dd23d762SRobert Mustacchi 			 * currently assume (and tag for userland with the
1828dd23d762SRobert Mustacchi 			 * AMDZEN_TOPO_CCD_F_CORE_PHYS_UNKNOWN flag) that we are
1829dd23d762SRobert Mustacchi 			 * mapping logical cores to physicals in the order of
1830dd23d762SRobert Mustacchi 			 * appearance.
1831dd23d762SRobert Mustacchi 			 */
1832dd23d762SRobert Mustacchi 			if (pkg0_ids) {
1833dd23d762SRobert Mustacchi 				amdzen_ccd_fill_core_initpkg0(azn, df, ccd, ccx,
1834dd23d762SRobert Mustacchi 				    core, &logccd_set, &logccx_set);
1835dd23d762SRobert Mustacchi 			} else {
1836dd23d762SRobert Mustacchi 				core->atcore_log_no = logcorezen2;
1837dd23d762SRobert Mustacchi 				logcorezen2++;
1838dd23d762SRobert Mustacchi 			}
1839dd23d762SRobert Mustacchi 
1840dd23d762SRobert Mustacchi 			/*
1841dd23d762SRobert Mustacchi 			 * Determining which bits to use for the thread is a bit
1842dd23d762SRobert Mustacchi 			 * weird here. Thread IDs within a CCX are logical, but
1843dd23d762SRobert Mustacchi 			 * there are always physically spaced CCX sizes. See the
1844dd23d762SRobert Mustacchi 			 * comment at the definition for SMU::PWR::THREAD_ENABLE
1845dd23d762SRobert Mustacchi 			 * for more information.
1846dd23d762SRobert Mustacchi 			 */
1847dd23d762SRobert Mustacchi 			const uint32_t thread_shift = (ccx->atccx_nphys_cores *
1848dd23d762SRobert Mustacchi 			    ccx->atccx_log_no + core->atcore_log_no) *
1849dd23d762SRobert Mustacchi 			    nthreads_per_core;
1850dd23d762SRobert Mustacchi 			const uint32_t thread_mask = (nthreads_per_core << 1) -
1851dd23d762SRobert Mustacchi 			    1;
1852dd23d762SRobert Mustacchi 			const uint32_t core_thread_en = (thread_en >>
1853dd23d762SRobert Mustacchi 			    thread_shift) & thread_mask;
1854dd23d762SRobert Mustacchi 			core->atcore_nthreads = nthreads_per_core;
1855dd23d762SRobert Mustacchi 			core->atcore_thr_en[0] = core_thread_en & 0x01;
1856dd23d762SRobert Mustacchi 			core->atcore_thr_en[1] = core_thread_en & 0x02;
1857dd23d762SRobert Mustacchi #ifdef	DEBUG
1858dd23d762SRobert Mustacchi 			if (nthreads_per_core == 1) {
1859dd23d762SRobert Mustacchi 				VERIFY0(core->atcore_thr_en[1]);
1860dd23d762SRobert Mustacchi 			}
1861dd23d762SRobert Mustacchi #endif
1862dd23d762SRobert Mustacchi 			for (uint32_t thrno = 0; thrno < core->atcore_nthreads;
1863dd23d762SRobert Mustacchi 			    thrno++) {
1864dd23d762SRobert Mustacchi 				ASSERT3U(core->atcore_thr_en[thrno], !=, 0);
1865dd23d762SRobert Mustacchi 
1866dd23d762SRobert Mustacchi 				zen_apic_id_compose(&azn->azn_apic_decomp,
1867dd23d762SRobert Mustacchi 				    sockid, dieid, ccd->atccd_log_no,
1868dd23d762SRobert Mustacchi 				    ccx->atccx_log_no, core->atcore_log_no,
1869dd23d762SRobert Mustacchi 				    thrno, &core->atcore_apicids[thrno]);
1870dd23d762SRobert Mustacchi 
1871dd23d762SRobert Mustacchi 			}
1872dd23d762SRobert Mustacchi 		}
1873dd23d762SRobert Mustacchi 
1874dd23d762SRobert Mustacchi 		ASSERT3U(logccx_set, ==, B_TRUE);
1875dd23d762SRobert Mustacchi 		ASSERT3U(logccd_set, ==, B_TRUE);
1876dd23d762SRobert Mustacchi 	}
1877dd23d762SRobert Mustacchi }
1878dd23d762SRobert Mustacchi 
1879047043c2SRobert Mustacchi static void
amdzen_nexus_init(void * arg)1880047043c2SRobert Mustacchi amdzen_nexus_init(void *arg)
1881047043c2SRobert Mustacchi {
1882047043c2SRobert Mustacchi 	uint_t i;
1883047043c2SRobert Mustacchi 	amdzen_t *azn = arg;
1884047043c2SRobert Mustacchi 
1885047043c2SRobert Mustacchi 	/*
1886047043c2SRobert Mustacchi 	 * First go through all of the stubs and assign the DF entries.
1887047043c2SRobert Mustacchi 	 */
1888047043c2SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
1889047043c2SRobert Mustacchi 	if (!amdzen_map_dfs(azn) || !amdzen_check_dfs(azn)) {
1890047043c2SRobert Mustacchi 		azn->azn_flags |= AMDZEN_F_MAP_ERROR;
1891047043c2SRobert Mustacchi 		goto done;
1892047043c2SRobert Mustacchi 	}
1893047043c2SRobert Mustacchi 
1894047043c2SRobert Mustacchi 	for (i = 0; i < AMDZEN_MAX_DFS; i++) {
1895047043c2SRobert Mustacchi 		amdzen_df_t *df = &azn->azn_dfs[i];
1896047043c2SRobert Mustacchi 
1897047043c2SRobert Mustacchi 		if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0)
1898047043c2SRobert Mustacchi 			continue;
1899047043c2SRobert Mustacchi 		amdzen_setup_df(azn, df);
1900047043c2SRobert Mustacchi 		amdzen_find_nb(azn, df);
1901047043c2SRobert Mustacchi 	}
1902047043c2SRobert Mustacchi 
1903dd23d762SRobert Mustacchi 	if (amdzen_determine_apic_decomp(azn)) {
1904dd23d762SRobert Mustacchi 		azn->azn_flags |= AMDZEN_F_APIC_DECOMP_VALID;
1905dd23d762SRobert Mustacchi 	}
1906dd23d762SRobert Mustacchi 
1907dd23d762SRobert Mustacchi 	amdzen_determine_ncore_per_ccx(azn);
1908dd23d762SRobert Mustacchi 
1909047043c2SRobert Mustacchi 	/*
1910047043c2SRobert Mustacchi 	 * Not all children may be installed. As such, we do not treat the
1911047043c2SRobert Mustacchi 	 * failure of a child as fatal to the driver.
1912047043c2SRobert Mustacchi 	 */
1913047043c2SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
1914047043c2SRobert Mustacchi 	for (i = 0; i < ARRAY_SIZE(amdzen_children); i++) {
1915047043c2SRobert Mustacchi 		(void) amdzen_create_child(azn, &amdzen_children[i]);
1916047043c2SRobert Mustacchi 	}
1917047043c2SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
1918047043c2SRobert Mustacchi 
1919047043c2SRobert Mustacchi done:
1920047043c2SRobert Mustacchi 	azn->azn_flags &= ~AMDZEN_F_ATTACH_DISPATCHED;
1921047043c2SRobert Mustacchi 	azn->azn_flags |= AMDZEN_F_ATTACH_COMPLETE;
1922047043c2SRobert Mustacchi 	azn->azn_taskqid = TASKQID_INVALID;
1923047043c2SRobert Mustacchi 	cv_broadcast(&azn->azn_cv);
1924047043c2SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
1925047043c2SRobert Mustacchi }
1926047043c2SRobert Mustacchi 
1927047043c2SRobert Mustacchi static int
amdzen_stub_scan_cb(dev_info_t * dip,void * arg)1928047043c2SRobert Mustacchi amdzen_stub_scan_cb(dev_info_t *dip, void *arg)
1929047043c2SRobert Mustacchi {
1930047043c2SRobert Mustacchi 	amdzen_t *azn = arg;
1931047043c2SRobert Mustacchi 	uint16_t vid, did;
1932047043c2SRobert Mustacchi 	int *regs;
1933047043c2SRobert Mustacchi 	uint_t nregs, i;
1934047043c2SRobert Mustacchi 	boolean_t match = B_FALSE;
1935047043c2SRobert Mustacchi 
1936047043c2SRobert Mustacchi 	if (dip == ddi_root_node()) {
1937047043c2SRobert Mustacchi 		return (DDI_WALK_CONTINUE);
1938047043c2SRobert Mustacchi 	}
1939047043c2SRobert Mustacchi 
1940047043c2SRobert Mustacchi 	/*
1941047043c2SRobert Mustacchi 	 * If a node in question is not a pci node, then we have no interest in
1942047043c2SRobert Mustacchi 	 * it as all the stubs that we care about are related to pci devices.
1943047043c2SRobert Mustacchi 	 */
1944047043c2SRobert Mustacchi 	if (strncmp("pci", ddi_get_name(dip), 3) != 0) {
1945047043c2SRobert Mustacchi 		return (DDI_WALK_PRUNECHILD);
1946047043c2SRobert Mustacchi 	}
1947047043c2SRobert Mustacchi 
1948047043c2SRobert Mustacchi 	/*
1949047043c2SRobert Mustacchi 	 * If we can't get a device or vendor ID and prove that this is an AMD
1950047043c2SRobert Mustacchi 	 * part, then we don't care about it.
1951047043c2SRobert Mustacchi 	 */
1952047043c2SRobert Mustacchi 	vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1953047043c2SRobert Mustacchi 	    "vendor-id", PCI_EINVAL16);
1954047043c2SRobert Mustacchi 	did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1955047043c2SRobert Mustacchi 	    "device-id", PCI_EINVAL16);
1956047043c2SRobert Mustacchi 	if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) {
1957047043c2SRobert Mustacchi 		return (DDI_WALK_CONTINUE);
1958047043c2SRobert Mustacchi 	}
1959047043c2SRobert Mustacchi 
19609b0429a1SPu Wen 	if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) {
1961047043c2SRobert Mustacchi 		return (DDI_WALK_CONTINUE);
1962047043c2SRobert Mustacchi 	}
1963047043c2SRobert Mustacchi 
1964047043c2SRobert Mustacchi 	for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) {
1965047043c2SRobert Mustacchi 		if (amdzen_nb_ids[i] == did) {
1966047043c2SRobert Mustacchi 			match = B_TRUE;
1967047043c2SRobert Mustacchi 		}
1968047043c2SRobert Mustacchi 	}
1969047043c2SRobert Mustacchi 
1970047043c2SRobert Mustacchi 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1971047043c2SRobert Mustacchi 	    "reg", &regs, &nregs) != DDI_PROP_SUCCESS) {
1972047043c2SRobert Mustacchi 		return (DDI_WALK_CONTINUE);
1973047043c2SRobert Mustacchi 	}
1974047043c2SRobert Mustacchi 
1975047043c2SRobert Mustacchi 	if (nregs == 0) {
1976047043c2SRobert Mustacchi 		ddi_prop_free(regs);
1977047043c2SRobert Mustacchi 		return (DDI_WALK_CONTINUE);
1978047043c2SRobert Mustacchi 	}
1979047043c2SRobert Mustacchi 
1980047043c2SRobert Mustacchi 	if (PCI_REG_BUS_G(regs[0]) == AMDZEN_DF_BUSNO &&
1981047043c2SRobert Mustacchi 	    PCI_REG_DEV_G(regs[0]) >= AMDZEN_DF_FIRST_DEVICE) {
1982047043c2SRobert Mustacchi 		match = B_TRUE;
1983047043c2SRobert Mustacchi 	}
1984047043c2SRobert Mustacchi 
1985047043c2SRobert Mustacchi 	ddi_prop_free(regs);
1986047043c2SRobert Mustacchi 	if (match) {
1987047043c2SRobert Mustacchi 		mutex_enter(&azn->azn_mutex);
1988047043c2SRobert Mustacchi 		azn->azn_nscanned++;
1989047043c2SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
1990047043c2SRobert Mustacchi 	}
1991047043c2SRobert Mustacchi 
1992047043c2SRobert Mustacchi 	return (DDI_WALK_CONTINUE);
1993047043c2SRobert Mustacchi }
1994047043c2SRobert Mustacchi 
1995047043c2SRobert Mustacchi static void
amdzen_stub_scan(void * arg)1996047043c2SRobert Mustacchi amdzen_stub_scan(void *arg)
1997047043c2SRobert Mustacchi {
1998047043c2SRobert Mustacchi 	amdzen_t *azn = arg;
1999047043c2SRobert Mustacchi 
2000047043c2SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
2001047043c2SRobert Mustacchi 	azn->azn_nscanned = 0;
2002047043c2SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
2003047043c2SRobert Mustacchi 
2004047043c2SRobert Mustacchi 	ddi_walk_devs(ddi_root_node(), amdzen_stub_scan_cb, azn);
2005047043c2SRobert Mustacchi 
2006047043c2SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
2007047043c2SRobert Mustacchi 	azn->azn_flags &= ~AMDZEN_F_SCAN_DISPATCHED;
2008047043c2SRobert Mustacchi 	azn->azn_flags |= AMDZEN_F_SCAN_COMPLETE;
2009047043c2SRobert Mustacchi 
2010047043c2SRobert Mustacchi 	if (azn->azn_nscanned == 0) {
2011047043c2SRobert Mustacchi 		azn->azn_flags |= AMDZEN_F_UNSUPPORTED;
2012047043c2SRobert Mustacchi 		azn->azn_taskqid = TASKQID_INVALID;
2013047043c2SRobert Mustacchi 		cv_broadcast(&azn->azn_cv);
2014047043c2SRobert Mustacchi 	} else if (azn->azn_npresent == azn->azn_nscanned) {
2015047043c2SRobert Mustacchi 		azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED;
2016047043c2SRobert Mustacchi 		azn->azn_taskqid = taskq_dispatch(system_taskq,
2017047043c2SRobert Mustacchi 		    amdzen_nexus_init, azn, TQ_SLEEP);
2018047043c2SRobert Mustacchi 	}
2019047043c2SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
2020047043c2SRobert Mustacchi }
2021047043c2SRobert Mustacchi 
2022047043c2SRobert Mustacchi /*
2023047043c2SRobert Mustacchi  * Unfortunately we can't really let the stubs detach as we may need them to be
2024047043c2SRobert Mustacchi  * available for client operations. We may be able to improve this if we know
2025047043c2SRobert Mustacchi  * that the actual nexus is going away. However, as long as it's active, we need
2026047043c2SRobert Mustacchi  * all the stubs.
2027047043c2SRobert Mustacchi  */
2028047043c2SRobert Mustacchi int
amdzen_detach_stub(dev_info_t * dip,ddi_detach_cmd_t cmd)2029047043c2SRobert Mustacchi amdzen_detach_stub(dev_info_t *dip, ddi_detach_cmd_t cmd)
2030047043c2SRobert Mustacchi {
2031047043c2SRobert Mustacchi 	if (cmd == DDI_SUSPEND) {
2032047043c2SRobert Mustacchi 		return (DDI_SUCCESS);
2033047043c2SRobert Mustacchi 	}
2034047043c2SRobert Mustacchi 
2035047043c2SRobert Mustacchi 	return (DDI_FAILURE);
2036047043c2SRobert Mustacchi }
2037047043c2SRobert Mustacchi 
2038047043c2SRobert Mustacchi int
amdzen_attach_stub(dev_info_t * dip,ddi_attach_cmd_t cmd)2039047043c2SRobert Mustacchi amdzen_attach_stub(dev_info_t *dip, ddi_attach_cmd_t cmd)
2040047043c2SRobert Mustacchi {
2041047043c2SRobert Mustacchi 	int *regs, reg;
2042047043c2SRobert Mustacchi 	uint_t nregs, i;
2043047043c2SRobert Mustacchi 	uint16_t vid, did;
2044047043c2SRobert Mustacchi 	amdzen_stub_t *stub;
2045047043c2SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
2046047043c2SRobert Mustacchi 	boolean_t valid = B_FALSE;
2047047043c2SRobert Mustacchi 	boolean_t nb = B_FALSE;
2048047043c2SRobert Mustacchi 
2049047043c2SRobert Mustacchi 	if (cmd == DDI_RESUME) {
2050047043c2SRobert Mustacchi 		return (DDI_SUCCESS);
2051047043c2SRobert Mustacchi 	} else if (cmd != DDI_ATTACH) {
2052047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2053047043c2SRobert Mustacchi 	}
2054047043c2SRobert Mustacchi 
2055047043c2SRobert Mustacchi 	/*
2056047043c2SRobert Mustacchi 	 * Make sure that the stub that we've been asked to attach is a pci type
2057047043c2SRobert Mustacchi 	 * device. If not, then there is no reason for us to proceed.
2058047043c2SRobert Mustacchi 	 */
2059047043c2SRobert Mustacchi 	if (strncmp("pci", ddi_get_name(dip), 3) != 0) {
2060047043c2SRobert Mustacchi 		dev_err(dip, CE_WARN, "asked to attach a bad AMD Zen nexus "
2061047043c2SRobert Mustacchi 		    "stub: %s", ddi_get_name(dip));
2062047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2063047043c2SRobert Mustacchi 	}
2064047043c2SRobert Mustacchi 	vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2065047043c2SRobert Mustacchi 	    "vendor-id", PCI_EINVAL16);
2066047043c2SRobert Mustacchi 	did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2067047043c2SRobert Mustacchi 	    "device-id", PCI_EINVAL16);
2068047043c2SRobert Mustacchi 	if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) {
2069047043c2SRobert Mustacchi 		dev_err(dip, CE_WARN, "failed to get PCI ID properties");
2070047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2071047043c2SRobert Mustacchi 	}
2072047043c2SRobert Mustacchi 
20739b0429a1SPu Wen 	if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) {
20749b0429a1SPu Wen 		dev_err(dip, CE_WARN, "expected vendor ID (0x%x), found 0x%x",
20759b0429a1SPu Wen 		    cpuid_getvendor(CPU) == X86_VENDOR_HYGON ?
20769b0429a1SPu Wen 		    AMDZEN_PCI_VID_HYGON : AMDZEN_PCI_VID_AMD, vid);
2077047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2078047043c2SRobert Mustacchi 	}
2079047043c2SRobert Mustacchi 
2080047043c2SRobert Mustacchi 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2081047043c2SRobert Mustacchi 	    "reg", &regs, &nregs) != DDI_PROP_SUCCESS) {
2082047043c2SRobert Mustacchi 		dev_err(dip, CE_WARN, "failed to get 'reg' property");
2083047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2084047043c2SRobert Mustacchi 	}
2085047043c2SRobert Mustacchi 
2086047043c2SRobert Mustacchi 	if (nregs == 0) {
2087047043c2SRobert Mustacchi 		ddi_prop_free(regs);
2088047043c2SRobert Mustacchi 		dev_err(dip, CE_WARN, "missing 'reg' property values");
2089047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2090047043c2SRobert Mustacchi 	}
2091047043c2SRobert Mustacchi 	reg = *regs;
2092047043c2SRobert Mustacchi 	ddi_prop_free(regs);
2093047043c2SRobert Mustacchi 
2094047043c2SRobert Mustacchi 	for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) {
2095047043c2SRobert Mustacchi 		if (amdzen_nb_ids[i] == did) {
2096047043c2SRobert Mustacchi 			valid = B_TRUE;
2097047043c2SRobert Mustacchi 			nb = B_TRUE;
2098047043c2SRobert Mustacchi 		}
2099047043c2SRobert Mustacchi 	}
2100047043c2SRobert Mustacchi 
2101047043c2SRobert Mustacchi 	if (!valid && PCI_REG_BUS_G(reg) == AMDZEN_DF_BUSNO &&
2102047043c2SRobert Mustacchi 	    PCI_REG_DEV_G(reg) >= AMDZEN_DF_FIRST_DEVICE) {
2103047043c2SRobert Mustacchi 		valid = B_TRUE;
2104047043c2SRobert Mustacchi 		nb = B_FALSE;
2105047043c2SRobert Mustacchi 	}
2106047043c2SRobert Mustacchi 
2107047043c2SRobert Mustacchi 	if (!valid) {
2108047043c2SRobert Mustacchi 		dev_err(dip, CE_WARN, "device %s didn't match the nexus list",
2109047043c2SRobert Mustacchi 		    ddi_get_name(dip));
2110047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2111047043c2SRobert Mustacchi 	}
2112047043c2SRobert Mustacchi 
2113047043c2SRobert Mustacchi 	stub = kmem_alloc(sizeof (amdzen_stub_t), KM_SLEEP);
2114047043c2SRobert Mustacchi 	if (pci_config_setup(dip, &stub->azns_cfgspace) != DDI_SUCCESS) {
2115047043c2SRobert Mustacchi 		dev_err(dip, CE_WARN, "failed to set up config space");
2116047043c2SRobert Mustacchi 		kmem_free(stub, sizeof (amdzen_stub_t));
2117047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2118047043c2SRobert Mustacchi 	}
2119047043c2SRobert Mustacchi 
2120047043c2SRobert Mustacchi 	stub->azns_dip = dip;
2121047043c2SRobert Mustacchi 	stub->azns_vid = vid;
2122047043c2SRobert Mustacchi 	stub->azns_did = did;
2123047043c2SRobert Mustacchi 	stub->azns_bus = PCI_REG_BUS_G(reg);
2124047043c2SRobert Mustacchi 	stub->azns_dev = PCI_REG_DEV_G(reg);
2125047043c2SRobert Mustacchi 	stub->azns_func = PCI_REG_FUNC_G(reg);
2126047043c2SRobert Mustacchi 	ddi_set_driver_private(dip, stub);
2127047043c2SRobert Mustacchi 
2128047043c2SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
2129047043c2SRobert Mustacchi 	azn->azn_npresent++;
2130047043c2SRobert Mustacchi 	if (nb) {
2131047043c2SRobert Mustacchi 		list_insert_tail(&azn->azn_nb_stubs, stub);
2132047043c2SRobert Mustacchi 	} else {
2133047043c2SRobert Mustacchi 		list_insert_tail(&azn->azn_df_stubs, stub);
2134047043c2SRobert Mustacchi 	}
2135047043c2SRobert Mustacchi 
2136047043c2SRobert Mustacchi 	if ((azn->azn_flags & AMDZEN_F_TASKQ_MASK) == AMDZEN_F_SCAN_COMPLETE &&
2137047043c2SRobert Mustacchi 	    azn->azn_nscanned == azn->azn_npresent) {
2138047043c2SRobert Mustacchi 		azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED;
2139047043c2SRobert Mustacchi 		azn->azn_taskqid = taskq_dispatch(system_taskq,
2140047043c2SRobert Mustacchi 		    amdzen_nexus_init, azn, TQ_SLEEP);
2141047043c2SRobert Mustacchi 	}
2142047043c2SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
2143047043c2SRobert Mustacchi 
2144047043c2SRobert Mustacchi 	return (DDI_SUCCESS);
2145047043c2SRobert Mustacchi }
2146047043c2SRobert Mustacchi 
2147047043c2SRobert Mustacchi static int
amdzen_bus_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_ctl_enum_t ctlop,void * arg,void * result)2148047043c2SRobert Mustacchi amdzen_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
2149047043c2SRobert Mustacchi     void *arg, void *result)
2150047043c2SRobert Mustacchi {
2151047043c2SRobert Mustacchi 	char buf[32];
2152047043c2SRobert Mustacchi 	dev_info_t *child;
2153047043c2SRobert Mustacchi 	const amdzen_child_data_t *acd;
2154047043c2SRobert Mustacchi 
2155047043c2SRobert Mustacchi 	switch (ctlop) {
2156047043c2SRobert Mustacchi 	case DDI_CTLOPS_REPORTDEV:
2157047043c2SRobert Mustacchi 		if (rdip == NULL) {
2158047043c2SRobert Mustacchi 			return (DDI_FAILURE);
2159047043c2SRobert Mustacchi 		}
2160047043c2SRobert Mustacchi 		cmn_err(CE_CONT, "amdzen nexus: %s@%s, %s%d\n",
2161047043c2SRobert Mustacchi 		    ddi_node_name(rdip), ddi_get_name_addr(rdip),
2162047043c2SRobert Mustacchi 		    ddi_driver_name(rdip), ddi_get_instance(rdip));
2163047043c2SRobert Mustacchi 		break;
2164047043c2SRobert Mustacchi 	case DDI_CTLOPS_INITCHILD:
2165047043c2SRobert Mustacchi 		child = arg;
2166047043c2SRobert Mustacchi 		if (child == NULL) {
2167047043c2SRobert Mustacchi 			dev_err(dip, CE_WARN, "!no child passed for "
2168047043c2SRobert Mustacchi 			    "DDI_CTLOPS_INITCHILD");
2169047043c2SRobert Mustacchi 		}
2170047043c2SRobert Mustacchi 
2171047043c2SRobert Mustacchi 		acd = ddi_get_parent_data(child);
2172047043c2SRobert Mustacchi 		if (acd == NULL) {
2173047043c2SRobert Mustacchi 			dev_err(dip, CE_WARN, "!missing child parent data");
2174047043c2SRobert Mustacchi 			return (DDI_FAILURE);
2175047043c2SRobert Mustacchi 		}
2176047043c2SRobert Mustacchi 
2177047043c2SRobert Mustacchi 		if (snprintf(buf, sizeof (buf), "%d", acd->acd_addr) >=
2178047043c2SRobert Mustacchi 		    sizeof (buf)) {
2179047043c2SRobert Mustacchi 			dev_err(dip, CE_WARN, "!failed to construct device "
2180047043c2SRobert Mustacchi 			    "addr due to overflow");
2181047043c2SRobert Mustacchi 			return (DDI_FAILURE);
2182047043c2SRobert Mustacchi 		}
2183047043c2SRobert Mustacchi 
2184047043c2SRobert Mustacchi 		ddi_set_name_addr(child, buf);
2185047043c2SRobert Mustacchi 		break;
2186047043c2SRobert Mustacchi 	case DDI_CTLOPS_UNINITCHILD:
2187047043c2SRobert Mustacchi 		child = arg;
2188047043c2SRobert Mustacchi 		if (child == NULL) {
2189047043c2SRobert Mustacchi 			dev_err(dip, CE_WARN, "!no child passed for "
2190047043c2SRobert Mustacchi 			    "DDI_CTLOPS_UNINITCHILD");
2191047043c2SRobert Mustacchi 		}
2192047043c2SRobert Mustacchi 
2193047043c2SRobert Mustacchi 		ddi_set_name_addr(child, NULL);
2194047043c2SRobert Mustacchi 		break;
2195047043c2SRobert Mustacchi 	default:
2196047043c2SRobert Mustacchi 		return (ddi_ctlops(dip, rdip, ctlop, arg, result));
2197047043c2SRobert Mustacchi 	}
2198047043c2SRobert Mustacchi 	return (DDI_SUCCESS);
2199047043c2SRobert Mustacchi }
2200047043c2SRobert Mustacchi 
2201dd23d762SRobert Mustacchi static int
amdzen_topo_open(dev_t * devp,int flag,int otyp,cred_t * credp)2202dd23d762SRobert Mustacchi amdzen_topo_open(dev_t *devp, int flag, int otyp, cred_t *credp)
2203dd23d762SRobert Mustacchi {
2204dd23d762SRobert Mustacchi 	minor_t m;
2205dd23d762SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
2206dd23d762SRobert Mustacchi 
2207dd23d762SRobert Mustacchi 	if (crgetzoneid(credp) != GLOBAL_ZONEID ||
2208dd23d762SRobert Mustacchi 	    secpolicy_sys_config(credp, B_FALSE) != 0) {
2209dd23d762SRobert Mustacchi 		return (EPERM);
2210dd23d762SRobert Mustacchi 	}
2211dd23d762SRobert Mustacchi 
2212dd23d762SRobert Mustacchi 	if ((flag & (FEXCL | FNDELAY | FNONBLOCK)) != 0) {
2213dd23d762SRobert Mustacchi 		return (EINVAL);
2214dd23d762SRobert Mustacchi 	}
2215dd23d762SRobert Mustacchi 
2216dd23d762SRobert Mustacchi 	if (otyp != OTYP_CHR) {
2217dd23d762SRobert Mustacchi 		return (EINVAL);
2218dd23d762SRobert Mustacchi 	}
2219dd23d762SRobert Mustacchi 
2220dd23d762SRobert Mustacchi 	m = getminor(*devp);
2221dd23d762SRobert Mustacchi 	if (m != AMDZEN_MINOR_TOPO) {
2222dd23d762SRobert Mustacchi 		return (ENXIO);
2223dd23d762SRobert Mustacchi 	}
2224dd23d762SRobert Mustacchi 
2225dd23d762SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
2226dd23d762SRobert Mustacchi 	if ((azn->azn_flags & AMDZEN_F_IOCTL_MASK) !=
2227dd23d762SRobert Mustacchi 	    AMDZEN_F_ATTACH_COMPLETE) {
2228dd23d762SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
2229dd23d762SRobert Mustacchi 		return (ENOTSUP);
2230dd23d762SRobert Mustacchi 	}
2231dd23d762SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
2232dd23d762SRobert Mustacchi 
2233dd23d762SRobert Mustacchi 	return (0);
2234dd23d762SRobert Mustacchi }
2235dd23d762SRobert Mustacchi 
2236dd23d762SRobert Mustacchi static int
amdzen_topo_ioctl_base(amdzen_t * azn,intptr_t arg,int mode)2237dd23d762SRobert Mustacchi amdzen_topo_ioctl_base(amdzen_t *azn, intptr_t arg, int mode)
2238dd23d762SRobert Mustacchi {
2239dd23d762SRobert Mustacchi 	amdzen_topo_base_t base;
2240dd23d762SRobert Mustacchi 
2241dd23d762SRobert Mustacchi 	bzero(&base, sizeof (base));
2242dd23d762SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
2243dd23d762SRobert Mustacchi 	base.atb_ndf = azn->azn_ndfs;
2244dd23d762SRobert Mustacchi 
2245dd23d762SRobert Mustacchi 	if ((azn->azn_flags & AMDZEN_F_APIC_DECOMP_VALID) == 0) {
2246dd23d762SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
2247dd23d762SRobert Mustacchi 		return (ENOTSUP);
2248dd23d762SRobert Mustacchi 	}
2249dd23d762SRobert Mustacchi 
2250dd23d762SRobert Mustacchi 	base.atb_apic_decomp = azn->azn_apic_decomp;
2251dd23d762SRobert Mustacchi 	for (uint_t i = 0; i < azn->azn_ndfs; i++) {
2252dd23d762SRobert Mustacchi 		const amdzen_df_t *df = &azn->azn_dfs[i];
2253dd23d762SRobert Mustacchi 
2254dd23d762SRobert Mustacchi 		base.atb_maxdfent = MAX(base.atb_maxdfent, df->adf_nents);
2255dd23d762SRobert Mustacchi 		if (i == 0) {
2256dd23d762SRobert Mustacchi 			base.atb_rev = df->adf_rev;
2257dd23d762SRobert Mustacchi 			base.atb_df_decomp = df->adf_decomp;
2258dd23d762SRobert Mustacchi 		}
2259dd23d762SRobert Mustacchi 	}
2260dd23d762SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
2261dd23d762SRobert Mustacchi 
2262dd23d762SRobert Mustacchi 	if (ddi_copyout(&base, (void *)(uintptr_t)arg, sizeof (base),
2263dd23d762SRobert Mustacchi 	    mode & FKIOCTL) != 0) {
2264dd23d762SRobert Mustacchi 		return (EFAULT);
2265dd23d762SRobert Mustacchi 	}
2266dd23d762SRobert Mustacchi 
2267dd23d762SRobert Mustacchi 	return (0);
2268dd23d762SRobert Mustacchi }
2269dd23d762SRobert Mustacchi 
2270dd23d762SRobert Mustacchi /*
2271dd23d762SRobert Mustacchi  * Fill in the peers. The way we do is this is to just fill in all the entries
2272dd23d762SRobert Mustacchi  * and then zero out the ones that aren't valid.
2273dd23d762SRobert Mustacchi  */
2274dd23d762SRobert Mustacchi static void
amdzen_topo_ioctl_df_fill_peers(const amdzen_df_ent_t * ent,amdzen_topo_df_ent_t * topo_ent)2275dd23d762SRobert Mustacchi amdzen_topo_ioctl_df_fill_peers(const amdzen_df_ent_t *ent,
2276dd23d762SRobert Mustacchi     amdzen_topo_df_ent_t *topo_ent)
2277dd23d762SRobert Mustacchi {
2278dd23d762SRobert Mustacchi 	topo_ent->atde_npeers = DF_FBIINFO0_GET_FTI_PCNT(ent->adfe_info0);
2279dd23d762SRobert Mustacchi 	topo_ent->atde_peers[0] = DF_FBINFO1_GET_FTI0_NINSTID(ent->adfe_info1);
2280dd23d762SRobert Mustacchi 	topo_ent->atde_peers[1] = DF_FBINFO1_GET_FTI1_NINSTID(ent->adfe_info1);
2281dd23d762SRobert Mustacchi 	topo_ent->atde_peers[2] = DF_FBINFO1_GET_FTI2_NINSTID(ent->adfe_info1);
2282dd23d762SRobert Mustacchi 	topo_ent->atde_peers[3] = DF_FBINFO1_GET_FTI3_NINSTID(ent->adfe_info1);
2283dd23d762SRobert Mustacchi 	topo_ent->atde_peers[4] = DF_FBINFO2_GET_FTI4_NINSTID(ent->adfe_info2);
2284dd23d762SRobert Mustacchi 	topo_ent->atde_peers[5] = DF_FBINFO2_GET_FTI5_NINSTID(ent->adfe_info2);
2285dd23d762SRobert Mustacchi 
2286dd23d762SRobert Mustacchi 	for (uint32_t i = topo_ent->atde_npeers; i < AMDZEN_TOPO_DF_MAX_PEERS;
2287dd23d762SRobert Mustacchi 	    i++) {
2288dd23d762SRobert Mustacchi 		topo_ent->atde_peers[i] = 0;
2289dd23d762SRobert Mustacchi 	}
2290dd23d762SRobert Mustacchi }
2291dd23d762SRobert Mustacchi 
2292dd23d762SRobert Mustacchi static void
amdzen_topo_ioctl_df_fill_ccm(const amdzen_df_ent_t * ent,amdzen_topo_df_ent_t * topo_ent)2293dd23d762SRobert Mustacchi amdzen_topo_ioctl_df_fill_ccm(const amdzen_df_ent_t *ent,
2294dd23d762SRobert Mustacchi     amdzen_topo_df_ent_t *topo_ent)
2295dd23d762SRobert Mustacchi {
2296dd23d762SRobert Mustacchi 	const amdzen_ccm_data_t *ccm = &ent->adfe_data.aded_ccm;
2297dd23d762SRobert Mustacchi 	amdzen_topo_ccm_data_t *topo_ccm = &topo_ent->atde_data.atded_ccm;
2298dd23d762SRobert Mustacchi 
2299dd23d762SRobert Mustacchi 	topo_ccm->atcd_nccds = ccm->acd_nccds;
2300dd23d762SRobert Mustacchi 	for (uint32_t i = 0; i < DF_MAX_CCDS_PER_CCM; i++) {
2301dd23d762SRobert Mustacchi 		topo_ccm->atcd_ccd_en[i] = ccm->acd_ccd_en[i];
2302dd23d762SRobert Mustacchi 		topo_ccm->atcd_ccd_ids[i] = ccm->acd_ccd_id[i];
2303dd23d762SRobert Mustacchi 	}
2304dd23d762SRobert Mustacchi }
2305dd23d762SRobert Mustacchi 
2306dd23d762SRobert Mustacchi static int
amdzen_topo_ioctl_df(amdzen_t * azn,intptr_t arg,int mode)2307dd23d762SRobert Mustacchi amdzen_topo_ioctl_df(amdzen_t *azn, intptr_t arg, int mode)
2308dd23d762SRobert Mustacchi {
2309dd23d762SRobert Mustacchi 	uint_t model;
2310dd23d762SRobert Mustacchi 	uint32_t max_ents, nwritten;
2311dd23d762SRobert Mustacchi 	const amdzen_df_t *df;
2312dd23d762SRobert Mustacchi 	amdzen_topo_df_t topo_df;
2313dd23d762SRobert Mustacchi #ifdef	_MULTI_DATAMODEL
2314dd23d762SRobert Mustacchi 	amdzen_topo_df32_t topo_df32;
2315dd23d762SRobert Mustacchi #endif
2316dd23d762SRobert Mustacchi 
2317dd23d762SRobert Mustacchi 	model = ddi_model_convert_from(mode);
2318dd23d762SRobert Mustacchi 	switch (model) {
2319dd23d762SRobert Mustacchi #ifdef	_MULTI_DATAMODEL
2320dd23d762SRobert Mustacchi 	case DDI_MODEL_ILP32:
2321dd23d762SRobert Mustacchi 		if (ddi_copyin((void *)(uintptr_t)arg, &topo_df32,
2322dd23d762SRobert Mustacchi 		    sizeof (topo_df32), mode & FKIOCTL) != 0) {
2323dd23d762SRobert Mustacchi 			return (EFAULT);
2324dd23d762SRobert Mustacchi 		}
2325dd23d762SRobert Mustacchi 		bzero(&topo_df, sizeof (topo_df));
2326dd23d762SRobert Mustacchi 		topo_df.atd_dfno = topo_df32.atd_dfno;
2327dd23d762SRobert Mustacchi 		topo_df.atd_df_buf_nents = topo_df32.atd_df_buf_nents;
2328dd23d762SRobert Mustacchi 		topo_df.atd_df_ents = (void *)(uintptr_t)topo_df32.atd_df_ents;
2329dd23d762SRobert Mustacchi 		break;
2330dd23d762SRobert Mustacchi #endif
2331dd23d762SRobert Mustacchi 	case DDI_MODEL_NONE:
2332dd23d762SRobert Mustacchi 		if (ddi_copyin((void *)(uintptr_t)arg, &topo_df,
2333dd23d762SRobert Mustacchi 		    sizeof (topo_df), mode & FKIOCTL) != 0) {
2334dd23d762SRobert Mustacchi 			return (EFAULT);
2335dd23d762SRobert Mustacchi 		}
2336dd23d762SRobert Mustacchi 		break;
2337dd23d762SRobert Mustacchi 	default:
2338dd23d762SRobert Mustacchi 		return (ENOTSUP);
2339dd23d762SRobert Mustacchi 	}
2340dd23d762SRobert Mustacchi 
2341dd23d762SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
2342dd23d762SRobert Mustacchi 	if (topo_df.atd_dfno >= azn->azn_ndfs) {
2343dd23d762SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
2344dd23d762SRobert Mustacchi 		return (EINVAL);
2345dd23d762SRobert Mustacchi 	}
2346dd23d762SRobert Mustacchi 
2347dd23d762SRobert Mustacchi 	df = &azn->azn_dfs[topo_df.atd_dfno];
2348dd23d762SRobert Mustacchi 	topo_df.atd_nodeid = df->adf_nodeid;
2349dd23d762SRobert Mustacchi 	topo_df.atd_sockid = (df->adf_nodeid & df->adf_decomp.dfd_sock_mask) >>
2350dd23d762SRobert Mustacchi 	    df->adf_decomp.dfd_sock_shift;
2351dd23d762SRobert Mustacchi 	topo_df.atd_dieid = (df->adf_nodeid & df->adf_decomp.dfd_die_mask) >>
2352dd23d762SRobert Mustacchi 	    df->adf_decomp.dfd_die_shift;
2353dd23d762SRobert Mustacchi 	topo_df.atd_rev = df->adf_rev;
2354dd23d762SRobert Mustacchi 	topo_df.atd_df_act_nents = df->adf_nents;
2355dd23d762SRobert Mustacchi 	max_ents = MIN(topo_df.atd_df_buf_nents, df->adf_nents);
2356dd23d762SRobert Mustacchi 
2357dd23d762SRobert Mustacchi 	if (topo_df.atd_df_ents == NULL) {
2358dd23d762SRobert Mustacchi 		topo_df.atd_df_buf_nvalid = 0;
2359dd23d762SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
2360dd23d762SRobert Mustacchi 		goto copyout;
2361dd23d762SRobert Mustacchi 	}
2362dd23d762SRobert Mustacchi 
2363dd23d762SRobert Mustacchi 	nwritten = 0;
2364dd23d762SRobert Mustacchi 	for (uint32_t i = 0; i < max_ents; i++) {
2365dd23d762SRobert Mustacchi 		amdzen_topo_df_ent_t topo_ent;
2366dd23d762SRobert Mustacchi 		const amdzen_df_ent_t *ent = &df->adf_ents[i];
2367dd23d762SRobert Mustacchi 
2368dd23d762SRobert Mustacchi 		/*
2369dd23d762SRobert Mustacchi 		 * We opt not to include disabled elements right now. They
2370dd23d762SRobert Mustacchi 		 * generally don't have a valid type and there isn't much useful
2371dd23d762SRobert Mustacchi 		 * information we can get from them. This can be changed if we
2372dd23d762SRobert Mustacchi 		 * find a use case for them for userland topo.
2373dd23d762SRobert Mustacchi 		 */
2374dd23d762SRobert Mustacchi 		if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0)
2375dd23d762SRobert Mustacchi 			continue;
2376dd23d762SRobert Mustacchi 
2377dd23d762SRobert Mustacchi 		bzero(&topo_ent, sizeof (topo_ent));
2378dd23d762SRobert Mustacchi 		topo_ent.atde_type = ent->adfe_type;
2379dd23d762SRobert Mustacchi 		topo_ent.atde_subtype = ent->adfe_subtype;
2380dd23d762SRobert Mustacchi 		topo_ent.atde_fabric_id = ent->adfe_fabric_id;
2381dd23d762SRobert Mustacchi 		topo_ent.atde_inst_id = ent->adfe_inst_id;
2382dd23d762SRobert Mustacchi 		amdzen_topo_ioctl_df_fill_peers(ent, &topo_ent);
2383dd23d762SRobert Mustacchi 
2384dd23d762SRobert Mustacchi 		if (ent->adfe_type == DF_TYPE_CCM &&
2385dd23d762SRobert Mustacchi 		    ent->adfe_subtype == DF_CCM_SUBTYPE_CPU) {
2386dd23d762SRobert Mustacchi 			amdzen_topo_ioctl_df_fill_ccm(ent, &topo_ent);
2387dd23d762SRobert Mustacchi 		}
2388dd23d762SRobert Mustacchi 
2389dd23d762SRobert Mustacchi 		if (ddi_copyout(&topo_ent, &topo_df.atd_df_ents[nwritten],
2390dd23d762SRobert Mustacchi 		    sizeof (topo_ent), mode & FKIOCTL) != 0) {
2391dd23d762SRobert Mustacchi 			mutex_exit(&azn->azn_mutex);
2392dd23d762SRobert Mustacchi 			return (EFAULT);
2393dd23d762SRobert Mustacchi 		}
2394dd23d762SRobert Mustacchi 		nwritten++;
2395dd23d762SRobert Mustacchi 	}
2396dd23d762SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
2397dd23d762SRobert Mustacchi 
2398dd23d762SRobert Mustacchi 	topo_df.atd_df_buf_nvalid = nwritten;
2399dd23d762SRobert Mustacchi copyout:
2400dd23d762SRobert Mustacchi 	switch (model) {
2401dd23d762SRobert Mustacchi #ifdef	_MULTI_DATAMODEL
2402dd23d762SRobert Mustacchi 	case DDI_MODEL_ILP32:
2403dd23d762SRobert Mustacchi 		topo_df32.atd_nodeid = topo_df.atd_nodeid;
2404dd23d762SRobert Mustacchi 		topo_df32.atd_sockid = topo_df.atd_sockid;
2405dd23d762SRobert Mustacchi 		topo_df32.atd_dieid = topo_df.atd_dieid;
2406dd23d762SRobert Mustacchi 		topo_df32.atd_rev = topo_df.atd_rev;
2407dd23d762SRobert Mustacchi 		topo_df32.atd_df_buf_nvalid = topo_df.atd_df_buf_nvalid;
2408dd23d762SRobert Mustacchi 		topo_df32.atd_df_act_nents = topo_df.atd_df_act_nents;
2409dd23d762SRobert Mustacchi 
2410dd23d762SRobert Mustacchi 		if (ddi_copyout(&topo_df32, (void *)(uintptr_t)arg,
2411dd23d762SRobert Mustacchi 		    sizeof (topo_df32), mode & FKIOCTL) != 0) {
2412dd23d762SRobert Mustacchi 			return (EFAULT);
2413dd23d762SRobert Mustacchi 		}
2414dd23d762SRobert Mustacchi 		break;
2415dd23d762SRobert Mustacchi #endif
2416dd23d762SRobert Mustacchi 	case DDI_MODEL_NONE:
2417dd23d762SRobert Mustacchi 		if (ddi_copyout(&topo_df, (void *)(uintptr_t)arg,
2418dd23d762SRobert Mustacchi 		    sizeof (topo_df), mode & FKIOCTL) != 0) {
2419dd23d762SRobert Mustacchi 			return (EFAULT);
2420dd23d762SRobert Mustacchi 		}
2421dd23d762SRobert Mustacchi 		break;
2422dd23d762SRobert Mustacchi 	default:
2423dd23d762SRobert Mustacchi 		break;
2424dd23d762SRobert Mustacchi 	}
2425dd23d762SRobert Mustacchi 
2426dd23d762SRobert Mustacchi 
2427dd23d762SRobert Mustacchi 	return (0);
2428dd23d762SRobert Mustacchi }
2429dd23d762SRobert Mustacchi 
2430dd23d762SRobert Mustacchi static int
amdzen_topo_ioctl_ccd(amdzen_t * azn,intptr_t arg,int mode)2431dd23d762SRobert Mustacchi amdzen_topo_ioctl_ccd(amdzen_t *azn, intptr_t arg, int mode)
2432dd23d762SRobert Mustacchi {
2433dd23d762SRobert Mustacchi 	amdzen_topo_ccd_t ccd, *ccdp;
2434dd23d762SRobert Mustacchi 	amdzen_df_t *df;
2435dd23d762SRobert Mustacchi 	amdzen_df_ent_t *ent;
2436dd23d762SRobert Mustacchi 	amdzen_ccm_data_t *ccm;
2437dd23d762SRobert Mustacchi 	uint32_t ccdno;
2438dd23d762SRobert Mustacchi 	size_t copyin_size = offsetof(amdzen_topo_ccd_t, atccd_err);
2439dd23d762SRobert Mustacchi 
2440dd23d762SRobert Mustacchi 	/*
2441dd23d762SRobert Mustacchi 	 * Only copy in the identifying information so that way we can ensure
2442dd23d762SRobert Mustacchi 	 * the rest of the structure we return to the user doesn't contain
2443dd23d762SRobert Mustacchi 	 * anything unexpected in it.
2444dd23d762SRobert Mustacchi 	 */
2445dd23d762SRobert Mustacchi 	bzero(&ccd, sizeof (ccd));
2446dd23d762SRobert Mustacchi 	if (ddi_copyin((void *)(uintptr_t)arg, &ccd, copyin_size,
2447dd23d762SRobert Mustacchi 	    mode & FKIOCTL) != 0) {
2448dd23d762SRobert Mustacchi 		return (EFAULT);
2449dd23d762SRobert Mustacchi 	}
2450dd23d762SRobert Mustacchi 
2451dd23d762SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
2452dd23d762SRobert Mustacchi 	if ((azn->azn_flags & AMDZEN_F_APIC_DECOMP_VALID) == 0) {
2453dd23d762SRobert Mustacchi 		ccd.atccd_err = AMDZEN_TOPO_CCD_E_NO_APIC_DECOMP;
2454dd23d762SRobert Mustacchi 		goto copyout;
2455dd23d762SRobert Mustacchi 	}
2456dd23d762SRobert Mustacchi 
2457dd23d762SRobert Mustacchi 	df = amdzen_df_find(azn, ccd.atccd_dfno);
2458dd23d762SRobert Mustacchi 	if (df == NULL) {
2459dd23d762SRobert Mustacchi 		ccd.atccd_err = AMDZEN_TOPO_CCD_E_BAD_DFNO;
2460dd23d762SRobert Mustacchi 		goto copyout;
2461dd23d762SRobert Mustacchi 	}
2462dd23d762SRobert Mustacchi 
2463dd23d762SRobert Mustacchi 	/*
2464dd23d762SRobert Mustacchi 	 * We don't have enough information to know how to construct this
2465dd23d762SRobert Mustacchi 	 * information in Zen 1 at this time, so refuse.
2466dd23d762SRobert Mustacchi 	 */
2467dd23d762SRobert Mustacchi 	if (df->adf_rev <= DF_REV_2) {
2468dd23d762SRobert Mustacchi 		ccd.atccd_err = AMDZEN_TOPO_CCD_E_SOC_UNSUPPORTED;
2469dd23d762SRobert Mustacchi 		goto copyout;
2470dd23d762SRobert Mustacchi 	}
2471dd23d762SRobert Mustacchi 
2472dd23d762SRobert Mustacchi 	ent = amdzen_df_ent_find_by_instid(df, ccd.atccd_instid);
2473dd23d762SRobert Mustacchi 	if (ent == NULL) {
2474dd23d762SRobert Mustacchi 		ccd.atccd_err = AMDZEN_TOPO_CCD_E_BAD_INSTID;
2475dd23d762SRobert Mustacchi 		goto copyout;
2476dd23d762SRobert Mustacchi 	}
2477dd23d762SRobert Mustacchi 
2478dd23d762SRobert Mustacchi 	if (ent->adfe_type != DF_TYPE_CCM ||
2479dd23d762SRobert Mustacchi 	    ent->adfe_subtype != DF_CCM_SUBTYPE_CPU) {
2480dd23d762SRobert Mustacchi 		ccd.atccd_err = AMDZEN_TOPO_CCD_E_NOT_A_CCD;
2481dd23d762SRobert Mustacchi 		goto copyout;
2482dd23d762SRobert Mustacchi 	}
2483dd23d762SRobert Mustacchi 
2484dd23d762SRobert Mustacchi 	ccm = &ent->adfe_data.aded_ccm;
2485dd23d762SRobert Mustacchi 	for (ccdno = 0; ccdno < DF_MAX_CCDS_PER_CCM; ccdno++) {
2486dd23d762SRobert Mustacchi 		if (ccm->acd_ccd_en[ccdno] != 0 &&
2487dd23d762SRobert Mustacchi 		    ccm->acd_ccd_id[ccdno] == ccd.atccd_phys_no) {
2488dd23d762SRobert Mustacchi 			break;
2489dd23d762SRobert Mustacchi 		}
2490dd23d762SRobert Mustacchi 	}
2491dd23d762SRobert Mustacchi 
2492dd23d762SRobert Mustacchi 	if (ccdno == DF_MAX_CCDS_PER_CCM) {
2493dd23d762SRobert Mustacchi 		ccd.atccd_err = AMDZEN_TOPO_CCD_E_NOT_A_CCD;
2494dd23d762SRobert Mustacchi 		goto copyout;
2495dd23d762SRobert Mustacchi 	}
2496dd23d762SRobert Mustacchi 
2497dd23d762SRobert Mustacchi 	if (ccm->acd_ccd_data[ccdno] == NULL) {
2498dd23d762SRobert Mustacchi 		/*
2499dd23d762SRobert Mustacchi 		 * We don't actually have this data. Go fill it out and save it
2500dd23d762SRobert Mustacchi 		 * for future use.
2501dd23d762SRobert Mustacchi 		 */
2502dd23d762SRobert Mustacchi 		ccdp = kmem_zalloc(sizeof (amdzen_topo_ccd_t), KM_NOSLEEP_LAZY);
2503dd23d762SRobert Mustacchi 		if (ccdp == NULL) {
2504dd23d762SRobert Mustacchi 			mutex_exit(&azn->azn_mutex);
2505dd23d762SRobert Mustacchi 			return (ENOMEM);
2506dd23d762SRobert Mustacchi 		}
2507dd23d762SRobert Mustacchi 
2508dd23d762SRobert Mustacchi 		ccdp->atccd_dfno = ccd.atccd_dfno;
2509dd23d762SRobert Mustacchi 		ccdp->atccd_instid = ccd.atccd_instid;
2510dd23d762SRobert Mustacchi 		ccdp->atccd_phys_no = ccd.atccd_phys_no;
2511dd23d762SRobert Mustacchi 		amdzen_ccd_fill_topo(azn, df, ent, ccdp);
2512dd23d762SRobert Mustacchi 		ccm->acd_ccd_data[ccdno] = ccdp;
2513dd23d762SRobert Mustacchi 	}
2514dd23d762SRobert Mustacchi 	ASSERT3P(ccm->acd_ccd_data[ccdno], !=, NULL);
2515dd23d762SRobert Mustacchi 	bcopy(ccm->acd_ccd_data[ccdno], &ccd, sizeof (ccd));
2516dd23d762SRobert Mustacchi 
2517dd23d762SRobert Mustacchi copyout:
2518dd23d762SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
2519dd23d762SRobert Mustacchi 	if (ddi_copyout(&ccd, (void *)(uintptr_t)arg, sizeof (ccd),
2520dd23d762SRobert Mustacchi 	    mode & FKIOCTL) != 0) {
2521dd23d762SRobert Mustacchi 		return (EFAULT);
2522dd23d762SRobert Mustacchi 	}
2523dd23d762SRobert Mustacchi 
2524dd23d762SRobert Mustacchi 	return (0);
2525dd23d762SRobert Mustacchi }
2526dd23d762SRobert Mustacchi 
2527dd23d762SRobert Mustacchi static int
amdzen_topo_ioctl(dev_t dev,int cmd,intptr_t arg,int mode,cred_t * credp,int * rvalp)2528dd23d762SRobert Mustacchi amdzen_topo_ioctl(dev_t dev, int cmd, intptr_t arg, int mode,
2529dd23d762SRobert Mustacchi     cred_t *credp, int *rvalp)
2530dd23d762SRobert Mustacchi {
2531dd23d762SRobert Mustacchi 	int ret;
2532dd23d762SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
2533dd23d762SRobert Mustacchi 
2534dd23d762SRobert Mustacchi 	if (getminor(dev) != AMDZEN_MINOR_TOPO) {
2535dd23d762SRobert Mustacchi 		return (ENXIO);
2536dd23d762SRobert Mustacchi 	}
2537dd23d762SRobert Mustacchi 
2538dd23d762SRobert Mustacchi 	if ((mode & FREAD) == 0) {
2539dd23d762SRobert Mustacchi 		return (EBADF);
2540dd23d762SRobert Mustacchi 	}
2541dd23d762SRobert Mustacchi 
2542dd23d762SRobert Mustacchi 	switch (cmd) {
2543dd23d762SRobert Mustacchi 	case AMDZEN_TOPO_IOCTL_BASE:
2544dd23d762SRobert Mustacchi 		ret = amdzen_topo_ioctl_base(azn, arg, mode);
2545dd23d762SRobert Mustacchi 		break;
2546dd23d762SRobert Mustacchi 	case AMDZEN_TOPO_IOCTL_DF:
2547dd23d762SRobert Mustacchi 		ret = amdzen_topo_ioctl_df(azn, arg, mode);
2548dd23d762SRobert Mustacchi 		break;
2549dd23d762SRobert Mustacchi 	case AMDZEN_TOPO_IOCTL_CCD:
2550dd23d762SRobert Mustacchi 		ret = amdzen_topo_ioctl_ccd(azn, arg, mode);
2551dd23d762SRobert Mustacchi 		break;
2552dd23d762SRobert Mustacchi 	default:
2553dd23d762SRobert Mustacchi 		ret = ENOTTY;
2554dd23d762SRobert Mustacchi 		break;
2555dd23d762SRobert Mustacchi 	}
2556dd23d762SRobert Mustacchi 
2557dd23d762SRobert Mustacchi 	return (ret);
2558dd23d762SRobert Mustacchi }
2559dd23d762SRobert Mustacchi 
2560dd23d762SRobert Mustacchi static int
amdzen_topo_close(dev_t dev,int flag,int otyp,cred_t * credp)2561dd23d762SRobert Mustacchi amdzen_topo_close(dev_t dev, int flag, int otyp, cred_t *credp)
2562dd23d762SRobert Mustacchi {
2563dd23d762SRobert Mustacchi 	if (otyp != OTYP_CHR) {
2564dd23d762SRobert Mustacchi 		return (EINVAL);
2565dd23d762SRobert Mustacchi 	}
2566dd23d762SRobert Mustacchi 
2567dd23d762SRobert Mustacchi 	if (getminor(dev) != AMDZEN_MINOR_TOPO) {
2568dd23d762SRobert Mustacchi 		return (ENXIO);
2569dd23d762SRobert Mustacchi 	}
2570dd23d762SRobert Mustacchi 
2571dd23d762SRobert Mustacchi 	return (0);
2572dd23d762SRobert Mustacchi }
2573dd23d762SRobert Mustacchi 
2574047043c2SRobert Mustacchi static int
amdzen_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)2575047043c2SRobert Mustacchi amdzen_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2576047043c2SRobert Mustacchi {
2577047043c2SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
2578047043c2SRobert Mustacchi 
2579047043c2SRobert Mustacchi 	if (cmd == DDI_RESUME) {
2580047043c2SRobert Mustacchi 		return (DDI_SUCCESS);
2581047043c2SRobert Mustacchi 	} else if (cmd != DDI_ATTACH) {
2582047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2583047043c2SRobert Mustacchi 	}
2584047043c2SRobert Mustacchi 
2585047043c2SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
2586047043c2SRobert Mustacchi 	if (azn->azn_dip != NULL) {
2587047043c2SRobert Mustacchi 		dev_err(dip, CE_WARN, "driver is already attached!");
2588047043c2SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
2589047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2590047043c2SRobert Mustacchi 	}
2591047043c2SRobert Mustacchi 
2592dd23d762SRobert Mustacchi 	if (ddi_create_minor_node(dip, "topo", S_IFCHR, AMDZEN_MINOR_TOPO,
2593dd23d762SRobert Mustacchi 	    DDI_PSEUDO, 0) != 0) {
2594dd23d762SRobert Mustacchi 		dev_err(dip, CE_WARN, "failed to create topo minor node!");
2595dd23d762SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
2596dd23d762SRobert Mustacchi 		return (DDI_FAILURE);
2597dd23d762SRobert Mustacchi 	}
2598dd23d762SRobert Mustacchi 
2599047043c2SRobert Mustacchi 	azn->azn_dip = dip;
2600047043c2SRobert Mustacchi 	azn->azn_taskqid = taskq_dispatch(system_taskq, amdzen_stub_scan,
2601047043c2SRobert Mustacchi 	    azn, TQ_SLEEP);
2602047043c2SRobert Mustacchi 	azn->azn_flags |= AMDZEN_F_SCAN_DISPATCHED;
2603047043c2SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
2604047043c2SRobert Mustacchi 
2605047043c2SRobert Mustacchi 	return (DDI_SUCCESS);
2606047043c2SRobert Mustacchi }
2607047043c2SRobert Mustacchi 
2608047043c2SRobert Mustacchi static int
amdzen_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)2609047043c2SRobert Mustacchi amdzen_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2610047043c2SRobert Mustacchi {
2611047043c2SRobert Mustacchi 	amdzen_t *azn = amdzen_data;
2612047043c2SRobert Mustacchi 
2613047043c2SRobert Mustacchi 	if (cmd == DDI_SUSPEND) {
2614047043c2SRobert Mustacchi 		return (DDI_SUCCESS);
2615047043c2SRobert Mustacchi 	} else if (cmd != DDI_DETACH) {
2616047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2617047043c2SRobert Mustacchi 	}
2618047043c2SRobert Mustacchi 
2619047043c2SRobert Mustacchi 	mutex_enter(&azn->azn_mutex);
2620047043c2SRobert Mustacchi 	while (azn->azn_taskqid != TASKQID_INVALID) {
2621047043c2SRobert Mustacchi 		cv_wait(&azn->azn_cv, &azn->azn_mutex);
2622047043c2SRobert Mustacchi 	}
2623047043c2SRobert Mustacchi 
2624047043c2SRobert Mustacchi 	/*
2625047043c2SRobert Mustacchi 	 * If we've attached any stub drivers, e.g. this platform is important
2626047043c2SRobert Mustacchi 	 * for us, then we fail detach.
2627047043c2SRobert Mustacchi 	 */
2628047043c2SRobert Mustacchi 	if (!list_is_empty(&azn->azn_df_stubs) ||
2629047043c2SRobert Mustacchi 	    !list_is_empty(&azn->azn_nb_stubs)) {
2630047043c2SRobert Mustacchi 		mutex_exit(&azn->azn_mutex);
2631047043c2SRobert Mustacchi 		return (DDI_FAILURE);
2632047043c2SRobert Mustacchi 	}
2633047043c2SRobert Mustacchi 
2634dd23d762SRobert Mustacchi 	ddi_remove_minor_node(azn->azn_dip, NULL);
2635047043c2SRobert Mustacchi 	azn->azn_dip = NULL;
2636047043c2SRobert Mustacchi 	mutex_exit(&azn->azn_mutex);
2637047043c2SRobert Mustacchi 
2638047043c2SRobert Mustacchi 	return (DDI_SUCCESS);
2639047043c2SRobert Mustacchi }
2640047043c2SRobert Mustacchi 
2641047043c2SRobert Mustacchi static void
amdzen_free(void)2642047043c2SRobert Mustacchi amdzen_free(void)
2643047043c2SRobert Mustacchi {
2644047043c2SRobert Mustacchi 	if (amdzen_data == NULL) {
2645047043c2SRobert Mustacchi 		return;
2646047043c2SRobert Mustacchi 	}
2647047043c2SRobert Mustacchi 
2648047043c2SRobert Mustacchi 	VERIFY(list_is_empty(&amdzen_data->azn_df_stubs));
2649047043c2SRobert Mustacchi 	list_destroy(&amdzen_data->azn_df_stubs);
2650047043c2SRobert Mustacchi 	VERIFY(list_is_empty(&amdzen_data->azn_nb_stubs));
2651047043c2SRobert Mustacchi 	list_destroy(&amdzen_data->azn_nb_stubs);
2652047043c2SRobert Mustacchi 	cv_destroy(&amdzen_data->azn_cv);
2653047043c2SRobert Mustacchi 	mutex_destroy(&amdzen_data->azn_mutex);
2654047043c2SRobert Mustacchi 	kmem_free(amdzen_data, sizeof (amdzen_t));
2655047043c2SRobert Mustacchi 	amdzen_data = NULL;
2656047043c2SRobert Mustacchi }
2657047043c2SRobert Mustacchi 
2658047043c2SRobert Mustacchi static void
amdzen_alloc(void)2659047043c2SRobert Mustacchi amdzen_alloc(void)
2660047043c2SRobert Mustacchi {
2661047043c2SRobert Mustacchi 	amdzen_data = kmem_zalloc(sizeof (amdzen_t), KM_SLEEP);
2662047043c2SRobert Mustacchi 	mutex_init(&amdzen_data->azn_mutex, NULL, MUTEX_DRIVER, NULL);
2663047043c2SRobert Mustacchi 	list_create(&amdzen_data->azn_df_stubs, sizeof (amdzen_stub_t),
2664047043c2SRobert Mustacchi 	    offsetof(amdzen_stub_t, azns_link));
2665047043c2SRobert Mustacchi 	list_create(&amdzen_data->azn_nb_stubs, sizeof (amdzen_stub_t),
2666047043c2SRobert Mustacchi 	    offsetof(amdzen_stub_t, azns_link));
2667047043c2SRobert Mustacchi 	cv_init(&amdzen_data->azn_cv, NULL, CV_DRIVER, NULL);
2668047043c2SRobert Mustacchi }
2669047043c2SRobert Mustacchi 
2670dd23d762SRobert Mustacchi static struct cb_ops amdzen_topo_cb_ops = {
2671dd23d762SRobert Mustacchi 	.cb_open = amdzen_topo_open,
2672dd23d762SRobert Mustacchi 	.cb_close = amdzen_topo_close,
2673dd23d762SRobert Mustacchi 	.cb_strategy = nodev,
2674dd23d762SRobert Mustacchi 	.cb_print = nodev,
2675dd23d762SRobert Mustacchi 	.cb_dump = nodev,
2676dd23d762SRobert Mustacchi 	.cb_read = nodev,
2677dd23d762SRobert Mustacchi 	.cb_write = nodev,
2678dd23d762SRobert Mustacchi 	.cb_ioctl = amdzen_topo_ioctl,
2679dd23d762SRobert Mustacchi 	.cb_devmap = nodev,
2680dd23d762SRobert Mustacchi 	.cb_mmap = nodev,
2681dd23d762SRobert Mustacchi 	.cb_segmap = nodev,
2682dd23d762SRobert Mustacchi 	.cb_chpoll = nochpoll,
2683dd23d762SRobert Mustacchi 	.cb_prop_op = ddi_prop_op,
2684dd23d762SRobert Mustacchi 	.cb_flag = D_MP,
2685dd23d762SRobert Mustacchi 	.cb_rev = CB_REV,
2686dd23d762SRobert Mustacchi 	.cb_aread = nodev,
2687dd23d762SRobert Mustacchi 	.cb_awrite = nodev
2688dd23d762SRobert Mustacchi };
2689dd23d762SRobert Mustacchi 
2690047043c2SRobert Mustacchi struct bus_ops amdzen_bus_ops = {
2691047043c2SRobert Mustacchi 	.busops_rev = BUSO_REV,
2692047043c2SRobert Mustacchi 	.bus_map = nullbusmap,
2693047043c2SRobert Mustacchi 	.bus_dma_map = ddi_no_dma_map,
2694047043c2SRobert Mustacchi 	.bus_dma_allochdl = ddi_no_dma_allochdl,
2695047043c2SRobert Mustacchi 	.bus_dma_freehdl = ddi_no_dma_freehdl,
2696047043c2SRobert Mustacchi 	.bus_dma_bindhdl = ddi_no_dma_bindhdl,
2697047043c2SRobert Mustacchi 	.bus_dma_unbindhdl = ddi_no_dma_unbindhdl,
2698047043c2SRobert Mustacchi 	.bus_dma_flush = ddi_no_dma_flush,
2699047043c2SRobert Mustacchi 	.bus_dma_win = ddi_no_dma_win,
2700047043c2SRobert Mustacchi 	.bus_dma_ctl = ddi_no_dma_mctl,
2701047043c2SRobert Mustacchi 	.bus_prop_op = ddi_bus_prop_op,
2702047043c2SRobert Mustacchi 	.bus_ctl = amdzen_bus_ctl
2703047043c2SRobert Mustacchi };
2704047043c2SRobert Mustacchi 
2705047043c2SRobert Mustacchi static struct dev_ops amdzen_dev_ops = {
2706047043c2SRobert Mustacchi 	.devo_rev = DEVO_REV,
2707047043c2SRobert Mustacchi 	.devo_refcnt = 0,
2708047043c2SRobert Mustacchi 	.devo_getinfo = nodev,
2709047043c2SRobert Mustacchi 	.devo_identify = nulldev,
2710047043c2SRobert Mustacchi 	.devo_probe = nulldev,
2711047043c2SRobert Mustacchi 	.devo_attach = amdzen_attach,
2712047043c2SRobert Mustacchi 	.devo_detach = amdzen_detach,
2713047043c2SRobert Mustacchi 	.devo_reset = nodev,
2714047043c2SRobert Mustacchi 	.devo_quiesce = ddi_quiesce_not_needed,
2715dd23d762SRobert Mustacchi 	.devo_bus_ops = &amdzen_bus_ops,
2716dd23d762SRobert Mustacchi 	.devo_cb_ops = &amdzen_topo_cb_ops
2717047043c2SRobert Mustacchi };
2718047043c2SRobert Mustacchi 
2719047043c2SRobert Mustacchi static struct modldrv amdzen_modldrv = {
2720047043c2SRobert Mustacchi 	.drv_modops = &mod_driverops,
2721047043c2SRobert Mustacchi 	.drv_linkinfo = "AMD Zen Nexus Driver",
2722047043c2SRobert Mustacchi 	.drv_dev_ops = &amdzen_dev_ops
2723047043c2SRobert Mustacchi };
2724047043c2SRobert Mustacchi 
2725047043c2SRobert Mustacchi static struct modlinkage amdzen_modlinkage = {
2726047043c2SRobert Mustacchi 	.ml_rev = MODREV_1,
2727047043c2SRobert Mustacchi 	.ml_linkage = { &amdzen_modldrv, NULL }
2728047043c2SRobert Mustacchi };
2729047043c2SRobert Mustacchi 
2730047043c2SRobert Mustacchi int
_init(void)2731047043c2SRobert Mustacchi _init(void)
2732047043c2SRobert Mustacchi {
2733047043c2SRobert Mustacchi 	int ret;
2734047043c2SRobert Mustacchi 
27359b0429a1SPu Wen 	if (cpuid_getvendor(CPU) != X86_VENDOR_AMD &&
27369b0429a1SPu Wen 	    cpuid_getvendor(CPU) != X86_VENDOR_HYGON) {
2737047043c2SRobert Mustacchi 		return (ENOTSUP);
2738047043c2SRobert Mustacchi 	}
2739047043c2SRobert Mustacchi 
2740047043c2SRobert Mustacchi 	if ((ret = mod_install(&amdzen_modlinkage)) == 0) {
2741047043c2SRobert Mustacchi 		amdzen_alloc();
2742047043c2SRobert Mustacchi 	}
2743047043c2SRobert Mustacchi 
2744047043c2SRobert Mustacchi 	return (ret);
2745047043c2SRobert Mustacchi }
2746047043c2SRobert Mustacchi 
2747047043c2SRobert Mustacchi int
_info(struct modinfo * modinfop)2748047043c2SRobert Mustacchi _info(struct modinfo *modinfop)
2749047043c2SRobert Mustacchi {
2750047043c2SRobert Mustacchi 	return (mod_info(&amdzen_modlinkage, modinfop));
2751047043c2SRobert Mustacchi }
2752047043c2SRobert Mustacchi 
2753047043c2SRobert Mustacchi int
_fini(void)2754047043c2SRobert Mustacchi _fini(void)
2755047043c2SRobert Mustacchi {
2756047043c2SRobert Mustacchi 	int ret;
2757047043c2SRobert Mustacchi 
2758047043c2SRobert Mustacchi 	if ((ret = mod_remove(&amdzen_modlinkage)) == 0) {
2759047043c2SRobert Mustacchi 		amdzen_free();
2760047043c2SRobert Mustacchi 	}
2761047043c2SRobert Mustacchi 
2762047043c2SRobert Mustacchi 	return (ret);
2763047043c2SRobert Mustacchi }
2764