1eb00b1c8SRobert Mustacchi /*
2eb00b1c8SRobert Mustacchi * This file and its contents are supplied under the terms of the
3eb00b1c8SRobert Mustacchi * Common Development and Distribution License ("CDDL"), version 1.0.
4eb00b1c8SRobert Mustacchi * You may only use this file in accordance with the terms of version
5eb00b1c8SRobert Mustacchi * 1.0 of the CDDL.
6eb00b1c8SRobert Mustacchi *
7eb00b1c8SRobert Mustacchi * A full copy of the text of the CDDL should have accompanied this
8eb00b1c8SRobert Mustacchi * source. A copy of the CDDL is also available via the Internet at
9eb00b1c8SRobert Mustacchi * http://www.illumos.org/license/CDDL.
10eb00b1c8SRobert Mustacchi */
11eb00b1c8SRobert Mustacchi
12eb00b1c8SRobert Mustacchi /*
13eb00b1c8SRobert Mustacchi * Copyright 2019 Joyent, Inc.
14*71815ce7SRobert Mustacchi * Copyright 2022 Oxide Computer Company
15eb00b1c8SRobert Mustacchi */
16eb00b1c8SRobert Mustacchi
17eb00b1c8SRobert Mustacchi /*
18eb00b1c8SRobert Mustacchi * Generic Intel Integrated Memory Controller (IMC) Driver
19eb00b1c8SRobert Mustacchi *
20eb00b1c8SRobert Mustacchi * This driver talks to the CPU's IMC to understand the detailed topology of the
21eb00b1c8SRobert Mustacchi * processor and to determine how to map between physical addresses to the
22eb00b1c8SRobert Mustacchi * corresponding DIMM. This driver supports the following generations of Intel
23eb00b1c8SRobert Mustacchi * chips:
24eb00b1c8SRobert Mustacchi *
25eb00b1c8SRobert Mustacchi * - Sandy Bridge
26eb00b1c8SRobert Mustacchi * - Ivy Bridge
27eb00b1c8SRobert Mustacchi * - Haswell
28eb00b1c8SRobert Mustacchi * - Broadwell
29eb00b1c8SRobert Mustacchi * - Skylake / Cascade Lake
30eb00b1c8SRobert Mustacchi *
31eb00b1c8SRobert Mustacchi * Memory Decoding
32eb00b1c8SRobert Mustacchi * ---------------
33eb00b1c8SRobert Mustacchi *
34eb00b1c8SRobert Mustacchi * For more detailed summaries of the memory decoding process, please refer to
35eb00b1c8SRobert Mustacchi * the Intel External Design Specifications for the corresponding processor.
36eb00b1c8SRobert Mustacchi * What follows is a rough overview of how the memory decoding system works.
37eb00b1c8SRobert Mustacchi *
38eb00b1c8SRobert Mustacchi * First, we'd like to define the following concepts:
39eb00b1c8SRobert Mustacchi *
40eb00b1c8SRobert Mustacchi * SYSTEM ADDRESS
41eb00b1c8SRobert Mustacchi *
42eb00b1c8SRobert Mustacchi * This is a physical address that the operating system normally uses. This
43eb00b1c8SRobert Mustacchi * address may refer to DRAM, it may refer to memory mapped PCI
44eb00b1c8SRobert Mustacchi * configuration space or device registers, or it may refer to other parts
45eb00b1c8SRobert Mustacchi * of the system's memory map, such as the extended advanced programmable
46eb00b1c8SRobert Mustacchi * interrupt controller (xAPIC), etc.
47eb00b1c8SRobert Mustacchi *
48eb00b1c8SRobert Mustacchi * DIMM
49eb00b1c8SRobert Mustacchi *
50eb00b1c8SRobert Mustacchi * Dual-inline memory module. This refers to a physical stick of volatile
51eb00b1c8SRobert Mustacchi * memory that is inserted into a slot on the motherboard.
52eb00b1c8SRobert Mustacchi *
53eb00b1c8SRobert Mustacchi * RANK
54eb00b1c8SRobert Mustacchi *
55eb00b1c8SRobert Mustacchi * A potential sub-division of a DIMM. A DIMM's memory capacity is divided
56eb00b1c8SRobert Mustacchi * into a number of equal sized ranks. For example, an 8 GiB DIMM, may have
57eb00b1c8SRobert Mustacchi * 1 8 GiB rank, 2 4 GiB ranks, or 4 2 GiB ranks.
58eb00b1c8SRobert Mustacchi *
59eb00b1c8SRobert Mustacchi * RANK ADDRESS
60eb00b1c8SRobert Mustacchi *
61eb00b1c8SRobert Mustacchi * An address that exists in the context of a given rank on a DIMM. All
62eb00b1c8SRobert Mustacchi * ranks have overlapping addresses, so the address 0x400 exists on all
63eb00b1c8SRobert Mustacchi * ranks on a given DIMM.
64eb00b1c8SRobert Mustacchi *
65eb00b1c8SRobert Mustacchi * CHANNEL
66eb00b1c8SRobert Mustacchi *
67eb00b1c8SRobert Mustacchi * Multiple DIMMs may be combined into a single channel. The channel
68eb00b1c8SRobert Mustacchi * represents the combined memory of all the DIMMs. A given channel only
69eb00b1c8SRobert Mustacchi * ever exists on a socket and is bound to a single memory controller.
70eb00b1c8SRobert Mustacchi *
71eb00b1c8SRobert Mustacchi * CHANNEL ADDRESS
72eb00b1c8SRobert Mustacchi *
73eb00b1c8SRobert Mustacchi * This is an address that exists logically on a channel. Each address on a
74eb00b1c8SRobert Mustacchi * channel maps to a corresponding DIMM that exists on that channel. The
75eb00b1c8SRobert Mustacchi * address space on one channel is independent from that on another. This
76eb00b1c8SRobert Mustacchi * means that address 0x1000 can exist on each memory channel in the
77eb00b1c8SRobert Mustacchi * system.
78eb00b1c8SRobert Mustacchi *
79eb00b1c8SRobert Mustacchi * INTERLEAVE
80eb00b1c8SRobert Mustacchi *
81eb00b1c8SRobert Mustacchi * There are several different cases where interleaving occurs on the
82eb00b1c8SRobert Mustacchi * system. For example, addresses may be interleaved across sockets,
83eb00b1c8SRobert Mustacchi * memory channels, or DIMM ranks. When addresses are interleaved, then
84eb00b1c8SRobert Mustacchi * some number of bits in an address are used to select which target to go
85eb00b1c8SRobert Mustacchi * to (usually through a look up table). The effect of interleaving is that
86eb00b1c8SRobert Mustacchi * addresses that are next to one another may not all go to the same
87eb00b1c8SRobert Mustacchi * device. The following image shows a non-interleaving case.
88eb00b1c8SRobert Mustacchi *
89eb00b1c8SRobert Mustacchi * 0x0fff +-----+ +-----+ 0x7ff
90eb00b1c8SRobert Mustacchi * | |\___________/| |
91eb00b1c8SRobert Mustacchi * | | __________ | (b) |
92eb00b1c8SRobert Mustacchi * | | / \| |
93eb00b1c8SRobert Mustacchi * 0x0800 |=====|= +-----+ 0x000 +-----+ 0x7ff
94eb00b1c8SRobert Mustacchi * | | \______________________________/| |
95eb00b1c8SRobert Mustacchi * | | _______________________________ | (a) |
96eb00b1c8SRobert Mustacchi * | |/ \| |
97eb00b1c8SRobert Mustacchi * 0x0000 +-----+ +-----+ 0x000
98eb00b1c8SRobert Mustacchi *
99eb00b1c8SRobert Mustacchi * In this example of non-interleaving, addresses 0x0000 to 0x07ff go to
100eb00b1c8SRobert Mustacchi * device (a). While, addresses 0x08000 to 0xfff, go to device (b).
101eb00b1c8SRobert Mustacchi * However, each range is divided into the same number of components.
102eb00b1c8SRobert Mustacchi *
103eb00b1c8SRobert Mustacchi * If instead, we were to look at that with interleaving, what we might say
104eb00b1c8SRobert Mustacchi * is that rather than splitting the range in half, we might say that if
105eb00b1c8SRobert Mustacchi * the address has bit 8 set (0x100), then it goes to (b), otherwise it
106eb00b1c8SRobert Mustacchi * goes to (a). This means that addresses 0x000 to 0x0ff, would go to (a).
107eb00b1c8SRobert Mustacchi * 0x100 to 0x1ff would go to (b). 0x200 to 0x2ff would go back to (a)
108eb00b1c8SRobert Mustacchi * again, and then 0x300 to 0x2ff would go back to (b). This would continue
109eb00b1c8SRobert Mustacchi * for a while. This would instead look something more like:
110eb00b1c8SRobert Mustacchi *
111eb00b1c8SRobert Mustacchi *
112eb00b1c8SRobert Mustacchi * 0x0fff +-----+ A: 0x7ff +---------+ B: 0x7ff +---------+
113eb00b1c8SRobert Mustacchi * | (b) | | e00-eff | | f00-fff |
114eb00b1c8SRobert Mustacchi * 0x0f00 |-----| 0x700 +---------+ 0x700 +---------+
115eb00b1c8SRobert Mustacchi * | (a) | | c00-cff | | d00-dff |
116eb00b1c8SRobert Mustacchi * 0x0e00 ~~~~~~~ 0x600 +---------+ 0x600 +---------+
117eb00b1c8SRobert Mustacchi * *** | a00-aff | | b00-bff |
118eb00b1c8SRobert Mustacchi * 0x0400 ~~~~~~~ 0x500 +---------+ 0x500 +---------+
119eb00b1c8SRobert Mustacchi * | (b) | | 800-8ff | | 900-9ff |
120eb00b1c8SRobert Mustacchi * 0x0300 |-----| 0x400 +---------+ 0x400 +---------+
121eb00b1c8SRobert Mustacchi * | (a) | | 600-6ff | | 700-7ff |
122eb00b1c8SRobert Mustacchi * 0x0200 |-----| 0x300 +---------+ 0x300 +---------+
123eb00b1c8SRobert Mustacchi * | (b) | | 400-4ff | | 500-5ff |
124eb00b1c8SRobert Mustacchi * 0x0100 |-----| 0x200 +---------+ 0x200 +---------+
125eb00b1c8SRobert Mustacchi * | (a) | | 200-2ff | | 300-3ff |
126eb00b1c8SRobert Mustacchi * 0x0000 +-----+ 0x100 +---------+ 0x100 +---------+
127eb00b1c8SRobert Mustacchi * | 000-0ff | | 100-1ff |
128eb00b1c8SRobert Mustacchi * 0x000 +---------+ 0x000 +---------+
129eb00b1c8SRobert Mustacchi *
130eb00b1c8SRobert Mustacchi * In this example we've performed two-way interleaving. The number of ways
131eb00b1c8SRobert Mustacchi * that something can interleave varies based on what we're interleaving
132eb00b1c8SRobert Mustacchi * between.
133eb00b1c8SRobert Mustacchi *
134eb00b1c8SRobert Mustacchi * MEMORY CONTROLLER
135eb00b1c8SRobert Mustacchi *
136eb00b1c8SRobert Mustacchi * A given processor die (see uts/i86pc/os/cpuid.c) contains a number of
137eb00b1c8SRobert Mustacchi * memory controllers. Usually 1 or two. Each memory controller supports a
138eb00b1c8SRobert Mustacchi * given number of DIMMs, which are divided across multiple channels.
139eb00b1c8SRobert Mustacchi *
140eb00b1c8SRobert Mustacchi * TARGET ADDRESS DECODER
141eb00b1c8SRobert Mustacchi *
142eb00b1c8SRobert Mustacchi * The target address decoder (TAD) is responsible for taking a system
143eb00b1c8SRobert Mustacchi * address and transforming it into a channel address based on the rules
144eb00b1c8SRobert Mustacchi * that are present. Each memory controller has a corresponding TAD. The
145eb00b1c8SRobert Mustacchi * TAD is often contained in a device called a 'Home Agent'.
146eb00b1c8SRobert Mustacchi *
147eb00b1c8SRobert Mustacchi * SYSTEM ADDRESS DECODER
148eb00b1c8SRobert Mustacchi *
149eb00b1c8SRobert Mustacchi * The system address decoder (SAD) is responsible for taking a system
150eb00b1c8SRobert Mustacchi * address and directing it to the right place, whether this be memory or
151eb00b1c8SRobert Mustacchi * otherwise. There is a single memory controller per socket (see
152eb00b1c8SRobert Mustacchi * uts/i86pc/os/cpuid.c) that is shared between all the cores currently.
153eb00b1c8SRobert Mustacchi *
154eb00b1c8SRobert Mustacchi * NODE IDENTIFIER
155eb00b1c8SRobert Mustacchi *
156eb00b1c8SRobert Mustacchi * The node identifier is used to uniquely identify an element in the
157eb00b1c8SRobert Mustacchi * various routing topologies on the die (see uts/i86pc/os/cpuid.c for the
158eb00b1c8SRobert Mustacchi * definition of 'die'). One can roughly think about this as a unique
159eb00b1c8SRobert Mustacchi * identifier for the socket itself. In general, the primary node ID for a
160eb00b1c8SRobert Mustacchi * socket should map to the socket APIC ID.
161eb00b1c8SRobert Mustacchi *
162eb00b1c8SRobert Mustacchi * Finding Devices
163eb00b1c8SRobert Mustacchi * ---------------
164eb00b1c8SRobert Mustacchi *
165eb00b1c8SRobert Mustacchi * There is a bit of a chicken and egg problem on Intel systems and in the
166eb00b1c8SRobert Mustacchi * device driver interface. The information that we need in the system is spread
167eb00b1c8SRobert Mustacchi * out amongst a large number of different PCI devices that the processor
168eb00b1c8SRobert Mustacchi * exposes. The number of such devices can vary based on the processor
169eb00b1c8SRobert Mustacchi * generation and the specific SKU in the processor. To deal with this, we break
170eb00b1c8SRobert Mustacchi * the driver into two different components: a stub driver and the full driver.
171eb00b1c8SRobert Mustacchi *
172eb00b1c8SRobert Mustacchi * The stub driver has aliases for all known PCI devices that we might attach to
173eb00b1c8SRobert Mustacchi * in a given generation on the system. This driver is called 'imcstub'. When a
174eb00b1c8SRobert Mustacchi * stub attaches, it just registers itself with the main driver, upon which it
175eb00b1c8SRobert Mustacchi * has a module dependency.
176eb00b1c8SRobert Mustacchi *
177eb00b1c8SRobert Mustacchi * The main driver, 'imc', is a pseudo-device driver. When it first attaches, it
178eb00b1c8SRobert Mustacchi * kicks off a scan of the device tree which takes place in a task queue. Once
179eb00b1c8SRobert Mustacchi * there, it determines the number of devices that it expects to exist by
180eb00b1c8SRobert Mustacchi * walking the tree and comparing it against the generation-specific table.
181eb00b1c8SRobert Mustacchi *
182eb00b1c8SRobert Mustacchi * If all devices are found, we'll go ahead and read through all the devices and
183eb00b1c8SRobert Mustacchi * build a map of all the information we need to understand the topology of the
184eb00b1c8SRobert Mustacchi * system and to be able to decode addresses. We do this here, because we can be
185eb00b1c8SRobert Mustacchi * asked to perform decoding in dangerous contexts (after taking an MCE, panic,
186eb00b1c8SRobert Mustacchi * etc) where we don't want to have to rely on the broader kernel functioning at
187eb00b1c8SRobert Mustacchi * this point in time.
188eb00b1c8SRobert Mustacchi *
189eb00b1c8SRobert Mustacchi * Once our topology is built, we'll create minor nodes which are used by the
190eb00b1c8SRobert Mustacchi * fault management architecture to query for information and register our
191eb00b1c8SRobert Mustacchi * decoding functionality with the kernel.
192eb00b1c8SRobert Mustacchi *
193eb00b1c8SRobert Mustacchi * PCI Numbering
194eb00b1c8SRobert Mustacchi * -------------
195eb00b1c8SRobert Mustacchi *
196eb00b1c8SRobert Mustacchi * For each device that we care about, Intel defines the device and function
197eb00b1c8SRobert Mustacchi * that we can expect to find the information and PCI configuration space
198eb00b1c8SRobert Mustacchi * registers that we care about at. However, the PCI bus is not well defined.
199eb00b1c8SRobert Mustacchi * Devices that are on the same socket use the same set of bus numbers; however,
200eb00b1c8SRobert Mustacchi * some sockets have multiple device numbers that they'll use to represent
201eb00b1c8SRobert Mustacchi * different classes. These bus numbers are programmed by systems firmware as
202eb00b1c8SRobert Mustacchi * part of powering on the system. This means, that we need the ability to
203eb00b1c8SRobert Mustacchi * map together these disparate ranges ourselves.
204eb00b1c8SRobert Mustacchi *
205eb00b1c8SRobert Mustacchi * There is a device called a utility box (UBOX), which exists per-socket and
206eb00b1c8SRobert Mustacchi * maps the different sockets together. We use this to determine which devices
207eb00b1c8SRobert Mustacchi * correspond to which sockets.
208eb00b1c8SRobert Mustacchi *
209eb00b1c8SRobert Mustacchi * Mapping Sockets
210eb00b1c8SRobert Mustacchi * ---------------
211eb00b1c8SRobert Mustacchi *
212eb00b1c8SRobert Mustacchi * Another wrinkle is that the way that the OS sees the numbering of the CPUs is
213eb00b1c8SRobert Mustacchi * generally based on the APIC ID (see uts/i86pc/os/cpuid.c for more
214eb00b1c8SRobert Mustacchi * information). However, to map to the corresponding socket, we need to look at
215eb00b1c8SRobert Mustacchi * the socket's node ID. The order of PCI buses in the system is not required to
216eb00b1c8SRobert Mustacchi * have any relation to the socket ID. Therefore, we have to have yet another
217eb00b1c8SRobert Mustacchi * indirection table in the imc_t.
218eb00b1c8SRobert Mustacchi *
219eb00b1c8SRobert Mustacchi * Exposing Data
220eb00b1c8SRobert Mustacchi * -------------
221eb00b1c8SRobert Mustacchi *
222eb00b1c8SRobert Mustacchi * We expose topology data to FMA using the OS-private memory controller
223eb00b1c8SRobert Mustacchi * interfaces. By creating minor nodes of the type, 'ddi_mem_ctrl', there are a
224eb00b1c8SRobert Mustacchi * number of specific interfaces that we can then implement. The ioctl API asks
225eb00b1c8SRobert Mustacchi * us for a snapshot of data, which basically has us go through and send an
226eb00b1c8SRobert Mustacchi * nvlist_t to userland. This nvlist_t is constructed as part of the scan
227eb00b1c8SRobert Mustacchi * process. This nvlist uses the version 1 format, which more explicitly encodes
228eb00b1c8SRobert Mustacchi * the topology in a series of nested nvlists.
229eb00b1c8SRobert Mustacchi *
230eb00b1c8SRobert Mustacchi * In addition, the tool /usr/lib/fm/fmd/mcdecode can be used to query the
231eb00b1c8SRobert Mustacchi * decoder and ask it to perform decoding.
232eb00b1c8SRobert Mustacchi *
233eb00b1c8SRobert Mustacchi * Decoding Addresses
234eb00b1c8SRobert Mustacchi * ------------------
235eb00b1c8SRobert Mustacchi *
236eb00b1c8SRobert Mustacchi * The decoding logic can be found in common/imc/imc_decode.c. This file is
237eb00b1c8SRobert Mustacchi * shared between the kernel and userland to allow for easier testing and
238eb00b1c8SRobert Mustacchi * additional flexibility in operation. The decoding process happens in a few
239eb00b1c8SRobert Mustacchi * different phases.
240eb00b1c8SRobert Mustacchi *
241eb00b1c8SRobert Mustacchi * The first phase, is to determine which memory controller on which socket is
242eb00b1c8SRobert Mustacchi * responsible for this data. To determine this, we use the system address
243eb00b1c8SRobert Mustacchi * decoder and walk the rules, looking for the correct target. There are various
244eb00b1c8SRobert Mustacchi * manipulations to the address that exist which are used to determine which
245eb00b1c8SRobert Mustacchi * index we use. The way that we interpret the output of the rule varies
246eb00b1c8SRobert Mustacchi * somewhat based on the generation. Sandy Bridge just has a node ID which
247eb00b1c8SRobert Mustacchi * points us to the socket with its single IMC. On Ivy Bridge through Broadwell,
248eb00b1c8SRobert Mustacchi * the memory controller to use is also encoded in part of the node ID. Finally,
249eb00b1c8SRobert Mustacchi * on Skylake, the SAD tells us which socket to look at. The socket in question
250eb00b1c8SRobert Mustacchi * then has a routing table which tells us which channel on which memory
251eb00b1c8SRobert Mustacchi * controller that is local to that socket.
252eb00b1c8SRobert Mustacchi *
253eb00b1c8SRobert Mustacchi * Once we have the target memory controller, we walk the list of target address
254eb00b1c8SRobert Mustacchi * decoder rules. These rules can help tell us which channel we care about
255eb00b1c8SRobert Mustacchi * (which is required on Sandy Bridge through Broadwell) and then describe some
256eb00b1c8SRobert Mustacchi * amount of the interleaving rules which are used to turn the system address
257eb00b1c8SRobert Mustacchi * into a channel address.
258eb00b1c8SRobert Mustacchi *
259eb00b1c8SRobert Mustacchi * Once we know the channel and the channel address, we walk the rank interleave
260eb00b1c8SRobert Mustacchi * rules which help us determine which DIMM and the corresponding rank on it
261eb00b1c8SRobert Mustacchi * that the corresponding channel address is on. It also has logic that we need
262eb00b1c8SRobert Mustacchi * to use to determine how to transform a channel address into an address on
263eb00b1c8SRobert Mustacchi * that specific rank. Once we have that, then the initial decoding is done.
264eb00b1c8SRobert Mustacchi *
265eb00b1c8SRobert Mustacchi * The logic in imc_decode.c is abstracted away from the broader kernel CMI
266eb00b1c8SRobert Mustacchi * logic. This is on purpose and allows us not only an easier time unit testing
267eb00b1c8SRobert Mustacchi * the logic, but also allows us to express more high fidelity errors that are
268eb00b1c8SRobert Mustacchi * translated into a much smaller subset. This logic is exercised in the
269eb00b1c8SRobert Mustacchi * 'imc_test' program which is built in 'test/os-tests/tests/imc'.
270eb00b1c8SRobert Mustacchi *
271eb00b1c8SRobert Mustacchi * Limitations
272eb00b1c8SRobert Mustacchi * -----------
273eb00b1c8SRobert Mustacchi *
274eb00b1c8SRobert Mustacchi * Currently, this driver has the following limitations:
275eb00b1c8SRobert Mustacchi *
276eb00b1c8SRobert Mustacchi * o It doesn't decode the row and column addresses.
277eb00b1c8SRobert Mustacchi * o It doesn't encode from a DIMM address to a system address.
278eb00b1c8SRobert Mustacchi * o It doesn't properly support lockstep and mirroring modes on Sandy Bridge -
279eb00b1c8SRobert Mustacchi * Broadwell platforms.
280eb00b1c8SRobert Mustacchi * o It doesn't support virtual lockstep and adaptive mirroring on Purley
281eb00b1c8SRobert Mustacchi * platforms.
282eb00b1c8SRobert Mustacchi * o It doesn't properly handle Intel Optane (3D-X Point) NVDIMMs.
283eb00b1c8SRobert Mustacchi * o It doesn't know how to decode three way channel interleaving.
284eb00b1c8SRobert Mustacchi *
285eb00b1c8SRobert Mustacchi * None of these are intrinsic problems to the driver, it's mostly a matter of
286eb00b1c8SRobert Mustacchi * having proper documentation and testing.
287eb00b1c8SRobert Mustacchi */
288eb00b1c8SRobert Mustacchi
289eb00b1c8SRobert Mustacchi #include <sys/modctl.h>
290eb00b1c8SRobert Mustacchi #include <sys/conf.h>
291eb00b1c8SRobert Mustacchi #include <sys/devops.h>
292eb00b1c8SRobert Mustacchi #include <sys/ddi.h>
293eb00b1c8SRobert Mustacchi #include <sys/sunddi.h>
294eb00b1c8SRobert Mustacchi #include <sys/types.h>
295eb00b1c8SRobert Mustacchi #include <sys/file.h>
296eb00b1c8SRobert Mustacchi #include <sys/errno.h>
297eb00b1c8SRobert Mustacchi #include <sys/open.h>
298eb00b1c8SRobert Mustacchi #include <sys/cred.h>
299eb00b1c8SRobert Mustacchi #include <sys/pci.h>
300eb00b1c8SRobert Mustacchi #include <sys/sysmacros.h>
301eb00b1c8SRobert Mustacchi #include <sys/avl.h>
302eb00b1c8SRobert Mustacchi #include <sys/stat.h>
303eb00b1c8SRobert Mustacchi #include <sys/policy.h>
304eb00b1c8SRobert Mustacchi
305eb00b1c8SRobert Mustacchi #include <sys/cpu_module.h>
306eb00b1c8SRobert Mustacchi #include <sys/mc.h>
307eb00b1c8SRobert Mustacchi #include <sys/mc_intel.h>
308eb00b1c8SRobert Mustacchi
309eb00b1c8SRobert Mustacchi #include "imc.h"
310eb00b1c8SRobert Mustacchi
311eb00b1c8SRobert Mustacchi /*
312eb00b1c8SRobert Mustacchi * These tables contain generational data that varies between processor
313eb00b1c8SRobert Mustacchi * generation such as the maximum number of sockets, memory controllers, and the
314eb00b1c8SRobert Mustacchi * offsets of the various registers.
315eb00b1c8SRobert Mustacchi */
316eb00b1c8SRobert Mustacchi
317eb00b1c8SRobert Mustacchi static const imc_gen_data_t imc_gen_data_snb = {
318eb00b1c8SRobert Mustacchi .igd_max_sockets = 4,
319eb00b1c8SRobert Mustacchi .igd_max_imcs = 2,
320eb00b1c8SRobert Mustacchi .igd_max_channels = 4,
321eb00b1c8SRobert Mustacchi .igd_max_dimms = 3,
322eb00b1c8SRobert Mustacchi .igd_max_ranks = IMC_MTR_DDR_RANKS_MAX,
323eb00b1c8SRobert Mustacchi .igd_mtr_offsets = { IMC_REG_MC_MTR0, IMC_REG_MC_MTR1,
324eb00b1c8SRobert Mustacchi IMC_REG_MC_MTR2 },
325eb00b1c8SRobert Mustacchi .igd_mcmtr_offset = 0x7c,
326eb00b1c8SRobert Mustacchi .igd_tolm_offset = 0x80,
327eb00b1c8SRobert Mustacchi .igd_tohm_low_offset = 0x84,
328eb00b1c8SRobert Mustacchi .igd_sad_dram_offset = 0x80,
329eb00b1c8SRobert Mustacchi .igd_sad_ndram_rules = 10,
330eb00b1c8SRobert Mustacchi .igd_sad_nodeid_offset = 0x40,
331eb00b1c8SRobert Mustacchi .igd_tad_nrules = 12,
332eb00b1c8SRobert Mustacchi .igd_tad_rule_offset = 0x40,
333eb00b1c8SRobert Mustacchi .igd_tad_chan_offset = 0x90,
334eb00b1c8SRobert Mustacchi .igd_tad_sysdef = 0x80,
335eb00b1c8SRobert Mustacchi .igd_tad_sysdef2 = 0x84,
336eb00b1c8SRobert Mustacchi .igd_mc_mirror = 0xac,
337eb00b1c8SRobert Mustacchi .igd_rir_nways = 5,
338eb00b1c8SRobert Mustacchi .igd_rir_way_offset = 0x108,
339eb00b1c8SRobert Mustacchi .igd_rir_nileaves = 8,
340eb00b1c8SRobert Mustacchi .igd_rir_ileave_offset = 0x120,
341eb00b1c8SRobert Mustacchi .igd_ubox_cpubusno_offset = 0xd0,
342eb00b1c8SRobert Mustacchi };
343eb00b1c8SRobert Mustacchi
344eb00b1c8SRobert Mustacchi static const imc_gen_data_t imc_gen_data_ivb = {
345eb00b1c8SRobert Mustacchi .igd_max_sockets = 4,
346eb00b1c8SRobert Mustacchi .igd_max_imcs = 2,
347eb00b1c8SRobert Mustacchi .igd_max_channels = 4,
348eb00b1c8SRobert Mustacchi .igd_max_dimms = 3,
349eb00b1c8SRobert Mustacchi .igd_max_ranks = IMC_MTR_DDR_RANKS_MAX,
350eb00b1c8SRobert Mustacchi .igd_mtr_offsets = { IMC_REG_MC_MTR0, IMC_REG_MC_MTR1,
351eb00b1c8SRobert Mustacchi IMC_REG_MC_MTR2 },
352eb00b1c8SRobert Mustacchi .igd_mcmtr_offset = 0x7c,
353eb00b1c8SRobert Mustacchi .igd_tolm_offset = 0x80,
354eb00b1c8SRobert Mustacchi .igd_tohm_low_offset = 0x84,
355eb00b1c8SRobert Mustacchi .igd_sad_dram_offset = 0x60,
356eb00b1c8SRobert Mustacchi .igd_sad_ndram_rules = 20,
357eb00b1c8SRobert Mustacchi .igd_sad_nodeid_offset = 0x40,
358eb00b1c8SRobert Mustacchi .igd_tad_nrules = 12,
359eb00b1c8SRobert Mustacchi .igd_tad_rule_offset = 0x40,
360eb00b1c8SRobert Mustacchi .igd_tad_chan_offset = 0x90,
361eb00b1c8SRobert Mustacchi .igd_tad_sysdef = 0x80,
362eb00b1c8SRobert Mustacchi .igd_tad_sysdef2 = 0x84,
363eb00b1c8SRobert Mustacchi .igd_mc_mirror = 0xac,
364eb00b1c8SRobert Mustacchi .igd_rir_nways = 5,
365eb00b1c8SRobert Mustacchi .igd_rir_way_offset = 0x108,
366eb00b1c8SRobert Mustacchi .igd_rir_nileaves = 8,
367eb00b1c8SRobert Mustacchi .igd_rir_ileave_offset = 0x120,
368eb00b1c8SRobert Mustacchi .igd_ubox_cpubusno_offset = 0xd0,
369eb00b1c8SRobert Mustacchi };
370eb00b1c8SRobert Mustacchi
371eb00b1c8SRobert Mustacchi static const imc_gen_data_t imc_gen_data_has_brd = {
372eb00b1c8SRobert Mustacchi .igd_max_sockets = 4,
373eb00b1c8SRobert Mustacchi .igd_max_imcs = 2,
374eb00b1c8SRobert Mustacchi .igd_max_channels = 4,
375eb00b1c8SRobert Mustacchi .igd_max_dimms = 3,
376eb00b1c8SRobert Mustacchi .igd_max_ranks = IMC_MTR_DDR_RANKS_MAX_HAS_SKX,
377eb00b1c8SRobert Mustacchi .igd_mtr_offsets = { IMC_REG_MC_MTR0, IMC_REG_MC_MTR1,
378eb00b1c8SRobert Mustacchi IMC_REG_MC_MTR2 },
379eb00b1c8SRobert Mustacchi .igd_mcmtr_offset = 0x7c,
380eb00b1c8SRobert Mustacchi .igd_tolm_offset = 0xd0,
381eb00b1c8SRobert Mustacchi .igd_tohm_low_offset = 0xd4,
382eb00b1c8SRobert Mustacchi .igd_tohm_hi_offset = 0xd8,
383eb00b1c8SRobert Mustacchi .igd_sad_dram_offset = 0x60,
384eb00b1c8SRobert Mustacchi .igd_sad_ndram_rules = 20,
385eb00b1c8SRobert Mustacchi .igd_sad_nodeid_offset = 0x40,
386eb00b1c8SRobert Mustacchi .igd_tad_nrules = 12,
387eb00b1c8SRobert Mustacchi .igd_tad_rule_offset = 0x40,
388eb00b1c8SRobert Mustacchi .igd_tad_chan_offset = 0x90,
389eb00b1c8SRobert Mustacchi .igd_tad_sysdef = 0x80,
390eb00b1c8SRobert Mustacchi .igd_tad_sysdef2 = 0x84,
391eb00b1c8SRobert Mustacchi .igd_mc_mirror = 0xac,
392eb00b1c8SRobert Mustacchi .igd_rir_nways = 5,
393eb00b1c8SRobert Mustacchi .igd_rir_way_offset = 0x108,
394eb00b1c8SRobert Mustacchi .igd_rir_nileaves = 8,
395eb00b1c8SRobert Mustacchi .igd_rir_ileave_offset = 0x120,
396eb00b1c8SRobert Mustacchi .igd_ubox_cpubusno_offset = 0xd0,
397eb00b1c8SRobert Mustacchi };
398eb00b1c8SRobert Mustacchi
399eb00b1c8SRobert Mustacchi static const imc_gen_data_t imc_gen_data_skx = {
400eb00b1c8SRobert Mustacchi .igd_max_sockets = 8,
401eb00b1c8SRobert Mustacchi .igd_max_imcs = 2,
402eb00b1c8SRobert Mustacchi .igd_max_channels = 3,
403eb00b1c8SRobert Mustacchi .igd_max_dimms = 2,
404eb00b1c8SRobert Mustacchi .igd_max_ranks = IMC_MTR_DDR_RANKS_MAX,
405eb00b1c8SRobert Mustacchi .igd_mtr_offsets = { IMC_REG_MC_MTR0, IMC_REG_MC_MTR1 },
406eb00b1c8SRobert Mustacchi .igd_mcmtr_offset = 0x87c,
407eb00b1c8SRobert Mustacchi .igd_topo_offset = 0x88,
408eb00b1c8SRobert Mustacchi .igd_tolm_offset = 0xd0,
409eb00b1c8SRobert Mustacchi .igd_tohm_low_offset = 0xd4,
410eb00b1c8SRobert Mustacchi .igd_tohm_hi_offset = 0xd8,
411eb00b1c8SRobert Mustacchi .igd_sad_dram_offset = 0x60,
412eb00b1c8SRobert Mustacchi .igd_sad_ndram_rules = 24,
413eb00b1c8SRobert Mustacchi .igd_sad_nodeid_offset = 0xc0,
414eb00b1c8SRobert Mustacchi .igd_tad_nrules = 8,
415eb00b1c8SRobert Mustacchi .igd_tad_rule_offset = 0x850,
416eb00b1c8SRobert Mustacchi .igd_tad_chan_offset = 0x90,
417eb00b1c8SRobert Mustacchi .igd_rir_nways = 4,
418eb00b1c8SRobert Mustacchi .igd_rir_way_offset = 0x108,
419eb00b1c8SRobert Mustacchi .igd_rir_nileaves = 4,
420eb00b1c8SRobert Mustacchi .igd_rir_ileave_offset = 0x120,
421eb00b1c8SRobert Mustacchi .igd_ubox_cpubusno_offset = 0xcc,
422eb00b1c8SRobert Mustacchi };
423eb00b1c8SRobert Mustacchi
424eb00b1c8SRobert Mustacchi /*
425eb00b1c8SRobert Mustacchi * This table contains all of the devices that we're looking for from a stub
426eb00b1c8SRobert Mustacchi * perspective. These are organized by generation. Different generations behave
427eb00b1c8SRobert Mustacchi * in slightly different ways. For example, Sandy Bridge through Broadwell use
428eb00b1c8SRobert Mustacchi * unique PCI IDs for each PCI device/function combination that appears. Whereas
429eb00b1c8SRobert Mustacchi * Skylake based systems use the same PCI ID; however, different device/function
430eb00b1c8SRobert Mustacchi * values indicate that the IDs are used for different purposes.
431eb00b1c8SRobert Mustacchi */
432eb00b1c8SRobert Mustacchi /* BEGIN CSTYLED */
433eb00b1c8SRobert Mustacchi static const imc_stub_table_t imc_stub_table[] = {
434eb00b1c8SRobert Mustacchi /* Sandy Bridge */
435eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_MC0_MAIN0, 0x3ca8, 15, 0, "IMC 0 Main 0" },
436eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_MC0_MAIN1, 0x3c71, 15, 1, "IMC 0 Main 0" },
437eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_MC0_CHANNEL0, 0x3caa, 15, 2, "IMC 0 Channel 0 Info" },
438eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_MC0_CHANNEL1, 0x3cab, 15, 3, "IMC 0 Channel 1 Info" },
439eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_MC0_CHANNEL2, 0x3cac, 15, 4, "IMC 0 Channel 2 Info" },
440eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_MC0_CHANNEL3, 0x3cad, 15, 5, "IMC 0 Channel 3 Info" },
441eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_SAD_DRAM, 0x3cf4, 12, 6, "SAD DRAM Rules" },
442eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_SAD_MMIO, 0x3cf5, 13, 6, "SAD MMIO Rules" },
443eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_SAD_MISC, 0x3cf6, 12, 7, "SAD Memory Map" },
444eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_UBOX, 0x3ce0, 11, 0, "UBox" },
445eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_UBOX_CPUBUSNO, 0x3ce3, 11, 3, "UBox Scratch" },
446eb00b1c8SRobert Mustacchi { IMC_GEN_SANDY, IMC_TYPE_HA0, 0x3ca0, 14, 0, "Home Agent" },
447eb00b1c8SRobert Mustacchi /* Ivy Bridge */
448eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC0_MAIN0, 0x0ea8, 15, 0, "IMC 0 Main 0" },
449eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC0_MAIN1, 0x0e71, 15, 1, "IMC 0 Main 1" },
450eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC0_CHANNEL0, 0x0eaa, 15, 2, "IMC 0 Channel 0 Info" },
451eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC0_CHANNEL1, 0x0eab, 15, 3, "IMC 0 Channel 1 Info" },
452eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC0_CHANNEL2, 0x0eac, 15, 4, "IMC 0 Channel 2 Info" },
453eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC0_CHANNEL3, 0x0ead, 15, 5, "IMC 0 Channel 3 Info" },
454eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC1_MAIN0, 0x0e68, 29, 0, "IMC 1 Main 0" },
455eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC1_MAIN1, 0x0e79, 29, 1, "IMC 1 Main 1" },
456eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC1_CHANNEL0, 0x0e6a, 15, 2, "IMC 1 Channel 0 Info" },
457eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC1_CHANNEL1, 0x0e6b, 15, 3, "IMC 1 Channel 1 Info" },
458eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC1_CHANNEL2, 0x0e6c, 15, 4, "IMC 1 Channel 2 Info" },
459eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_MC1_CHANNEL3, 0x0e6d, 15, 5, "IMC 1 Channel 3 Info" },
460eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_SAD_DRAM, 0x0ec8, 22, 0, "SAD DRAM Rules" },
461eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_SAD_MMIO, 0x0ec9, 22, 1, "SAD MMIO Rules" },
462eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_SAD_MISC, 0x0eca, 22, 2, "SAD Memory Map" },
463eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_UBOX, 0x0e1e, 11, 0, "UBox" },
464eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_UBOX_CPUBUSNO, 0x0e1f, 11, 3, "UBox Scratch" },
465eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_HA0, 0x0ea0, 14, 0, "Home Agent 0" },
466eb00b1c8SRobert Mustacchi { IMC_GEN_IVY, IMC_TYPE_HA1, 0x0e60, 28, 0, "Home Agent 1" },
467eb00b1c8SRobert Mustacchi /* Haswell */
468eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC0_MAIN0, 0x2fa8, 19, 0, "IMC 0 Main 0" },
469eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC0_MAIN1, 0x2f71, 19, 1, "IMC 0 Main 1" },
470eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC0_CHANNEL0, 0x2faa, 19, 2, "IMC 0 Channel 0 Info" },
471eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC0_CHANNEL1, 0x2fab, 19, 3, "IMC 0 Channel 1 Info" },
472eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC0_CHANNEL2, 0x2fac, 19, 4, "IMC 0 Channel 2 Info" },
473eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC0_CHANNEL3, 0x2fad, 19, 5, "IMC 0 Channel 3 Info" },
474eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC1_MAIN0, 0x2f68, 22, 0, "IMC 1 Main 0" },
475eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC1_MAIN1, 0x2f79, 22, 1, "IMC 1 Main 1" },
476eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC1_CHANNEL0, 0x2f6a, 22, 2, "IMC 1 Channel 0 Info" },
477eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC1_CHANNEL1, 0x2f6b, 22, 3, "IMC 1 Channel 1 Info" },
478eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC1_CHANNEL2, 0x2f6c, 22, 4, "IMC 1 Channel 2 Info" },
479eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_MC1_CHANNEL3, 0x2f6d, 22, 5, "IMC 1 Channel 3 Info" },
480eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_SAD_DRAM, 0x2ffc, 15, 4, "SAD DRAM Rules" },
481eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_SAD_MMIO, 0x2ffd, 15, 5, "SAD MMIO Rules" },
482eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_VTD_MISC, 0x2f28, 5, 0, "Misc. Vritualization" },
483eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_UBOX, 0x2f1e, 16, 5, "UBox" },
484eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_UBOX_CPUBUSNO, 0x2f1f, 16, 7, "UBox Scratch" },
485eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_HA0, 0x2fa0, 18, 0, "Home Agent 0" },
486eb00b1c8SRobert Mustacchi { IMC_GEN_HASWELL, IMC_TYPE_HA1, 0x2f60, 18, 4, "Home Agent 1" },
487eb00b1c8SRobert Mustacchi /* Broadwell Devices */
488eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC0_MAIN0, 0x6fa8, 19, 0, "IMC 0 Main 0" },
489eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC0_MAIN1, 0x6f71, 19, 1, "IMC 0 Main 1" },
490eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC0_CHANNEL0, 0x6faa, 19, 2, "IMC 0 Channel 0 Info" },
491eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC0_CHANNEL1, 0x6fab, 19, 3, "IMC 0 Channel 1 Info" },
492eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC0_CHANNEL2, 0x6fac, 19, 4, "IMC 0 Channel 2 Info" },
493eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC0_CHANNEL3, 0x6fad, 19, 5, "IMC 0 Channel 3 Info" },
494eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC1_MAIN0, 0x6f68, 22, 0, "IMC 1 Main 0" },
495eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC1_MAIN1, 0x6f79, 22, 1, "IMC 1 Main 1" },
496eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC1_CHANNEL0, 0x6f6a, 22, 2, "IMC 1 Channel 0 Info" },
497eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC1_CHANNEL1, 0x6f6b, 22, 3, "IMC 1 Channel 1 Info" },
498eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC1_CHANNEL2, 0x6f6c, 22, 4, "IMC 1 Channel 2 Info" },
499eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_MC1_CHANNEL3, 0x6f6d, 22, 5, "IMC 1 Channel 3 Info" },
500eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_SAD_DRAM, 0x6ffc, 15, 4, "SAD DRAM Rules" },
501eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_SAD_MMIO, 0x6ffd, 15, 5, "SAD MMIO Rules" },
502eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_VTD_MISC, 0x6f28, 5, 0, "Misc. Vritualization" },
503eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_UBOX, 0x6f1e, 16, 5, "UBox" },
504eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_UBOX_CPUBUSNO, 0x6f1f, 16, 7, "UBox Scratch" },
505eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_HA0, 0x6fa0, 18, 0, "Home Agent 0" },
506eb00b1c8SRobert Mustacchi { IMC_GEN_BROADWELL, IMC_TYPE_HA1, 0x6f60, 18, 4, "Home Agent 1" },
507eb00b1c8SRobert Mustacchi /* Skylake and Cascade Lake Devices */
508eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_MC0_M2M, 0x2066, 8, 0, "IMC 0 M2M" },
509eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_MC1_M2M, 0x2066, 9, 0, "IMC 0 M2M" },
510eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_MC0_MAIN0, 0x2040, 10, 0, "IMC 0 Main / Channel 0" },
511eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_MC1_MAIN0, 0x2040, 12, 0, "IMC 0 Main / Channel 0" },
512eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_MC0_CHANNEL1, 0x2044, 10, 4, "IMC 0 Channel 1" },
513eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_MC0_CHANNEL2, 0x2048, 11, 0, "IMC 0 Channel 2" },
514eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_MC1_CHANNEL1, 0x2044, 12, 4, "IMC 1 Channel 1" },
515eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_MC1_CHANNEL2, 0x2048, 13, 0, "IMC 1 Channel 2" },
516eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_DRAM, 0x2054, 29, 0, "SAD DRAM Rules" },
517eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MMIO, 0x2055, 29, 1, "SAD MMIO Rules" },
518eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_VTD_MISC, 0x2024, 5, 0, "Misc. Virtualization" },
519eb00b1c8SRobert Mustacchi
520eb00b1c8SRobert Mustacchi /*
521eb00b1c8SRobert Mustacchi * There is one SAD MC Route type device per core! Because of this a
522eb00b1c8SRobert Mustacchi * wide array of device and functions are allocated. For now, we list
523eb00b1c8SRobert Mustacchi * all 28 of them out.
524eb00b1c8SRobert Mustacchi */
525eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 14, 0, "Per-Core SAD" },
526eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 14, 1, "Per-Core SAD" },
527eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 14, 2, "Per-Core SAD" },
528eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 14, 3, "Per-Core SAD" },
529eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 14, 4, "Per-Core SAD" },
530eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 14, 5, "Per-Core SAD" },
531eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 14, 6, "Per-Core SAD" },
532eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 14, 7, "Per-Core SAD" },
533eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 15, 0, "Per-Core SAD" },
534eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 15, 1, "Per-Core SAD" },
535eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 15, 2, "Per-Core SAD" },
536eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 15, 3, "Per-Core SAD" },
537eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 15, 4, "Per-Core SAD" },
538eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 15, 5, "Per-Core SAD" },
539eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 15, 6, "Per-Core SAD" },
540eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 15, 7, "Per-Core SAD" },
541eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 16, 0, "Per-Core SAD" },
542eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 16, 1, "Per-Core SAD" },
543eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 16, 2, "Per-Core SAD" },
544eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 16, 3, "Per-Core SAD" },
545eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 16, 4, "Per-Core SAD" },
546eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 16, 5, "Per-Core SAD" },
547eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 16, 6, "Per-Core SAD" },
548eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 16, 7, "Per-Core SAD" },
549eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 17, 0, "Per-Core SAD" },
550eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 17, 1, "Per-Core SAD" },
551eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 17, 2, "Per-Core SAD" },
552eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 17, 3, "Per-Core SAD" },
553eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 17, 4, "Per-Core SAD" },
554eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 17, 5, "Per-Core SAD" },
555eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 17, 6, "Per-Core SAD" },
556eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_SAD_MCROUTE, 0x208e, 17, 7, "Per-Core SAD" },
557eb00b1c8SRobert Mustacchi
558eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_UBOX, 0x2014, 8, 0, "UBox" },
559eb00b1c8SRobert Mustacchi { IMC_GEN_SKYLAKE, IMC_TYPE_UBOX_CPUBUSNO, 0x2016, 8, 2, "DECS" },
560eb00b1c8SRobert Mustacchi };
561eb00b1c8SRobert Mustacchi /* END CSTYLED */
562eb00b1c8SRobert Mustacchi
563eb00b1c8SRobert Mustacchi #define IMC_PCI_VENDOR_INTC 0x8086
564eb00b1c8SRobert Mustacchi
565eb00b1c8SRobert Mustacchi /*
566eb00b1c8SRobert Mustacchi * Our IMC data is global and statically set up during a combination of
567eb00b1c8SRobert Mustacchi * _init(9E) and attach(9E). While we have a module dependency between the PCI
568eb00b1c8SRobert Mustacchi * stub driver, imcstub, and this pseudo-driver, imc, the dependencies don't
569eb00b1c8SRobert Mustacchi * guarantee that the imc driver has finished attaching. As such we make sure
570eb00b1c8SRobert Mustacchi * that it can operate without it being attached in any way.
571eb00b1c8SRobert Mustacchi */
572eb00b1c8SRobert Mustacchi static imc_t *imc_data = NULL;
573eb00b1c8SRobert Mustacchi
574eb00b1c8SRobert Mustacchi /*
575eb00b1c8SRobert Mustacchi * By default we should not allow the stubs to detach as we don't have a good
576eb00b1c8SRobert Mustacchi * way of forcing them to attach again. This is provided in case someone does
577eb00b1c8SRobert Mustacchi * want to allow the driver to unload.
578eb00b1c8SRobert Mustacchi */
579eb00b1c8SRobert Mustacchi int imc_allow_detach = 0;
580eb00b1c8SRobert Mustacchi
581eb00b1c8SRobert Mustacchi static void
imc_set_gen_data(imc_t * imc)582eb00b1c8SRobert Mustacchi imc_set_gen_data(imc_t *imc)
583eb00b1c8SRobert Mustacchi {
584eb00b1c8SRobert Mustacchi switch (imc->imc_gen) {
585eb00b1c8SRobert Mustacchi case IMC_GEN_SANDY:
586eb00b1c8SRobert Mustacchi imc->imc_gen_data = &imc_gen_data_snb;
587eb00b1c8SRobert Mustacchi break;
588eb00b1c8SRobert Mustacchi case IMC_GEN_IVY:
589eb00b1c8SRobert Mustacchi imc->imc_gen_data = &imc_gen_data_ivb;
590eb00b1c8SRobert Mustacchi break;
591eb00b1c8SRobert Mustacchi case IMC_GEN_HASWELL:
592eb00b1c8SRobert Mustacchi case IMC_GEN_BROADWELL:
593eb00b1c8SRobert Mustacchi imc->imc_gen_data = &imc_gen_data_has_brd;
594eb00b1c8SRobert Mustacchi break;
595eb00b1c8SRobert Mustacchi case IMC_GEN_SKYLAKE:
596eb00b1c8SRobert Mustacchi imc->imc_gen_data = &imc_gen_data_skx;
597eb00b1c8SRobert Mustacchi break;
598eb00b1c8SRobert Mustacchi default:
599eb00b1c8SRobert Mustacchi dev_err(imc->imc_dip, CE_PANIC, "imc driver programmer error: "
600eb00b1c8SRobert Mustacchi "set to unknown generation: %u", imc->imc_gen);
601eb00b1c8SRobert Mustacchi }
602eb00b1c8SRobert Mustacchi }
603eb00b1c8SRobert Mustacchi
604eb00b1c8SRobert Mustacchi /*
605eb00b1c8SRobert Mustacchi * If our device (dev_info_t) does not have a non-zero unit address, then
606eb00b1c8SRobert Mustacchi * devfsadmd will not pay attention to us at all. Therefore we need to set the
607eb00b1c8SRobert Mustacchi * unit address below, before we create minor nodes.
608eb00b1c8SRobert Mustacchi *
609eb00b1c8SRobert Mustacchi * The rest of the system expects us to have one minor node per socket. The
610eb00b1c8SRobert Mustacchi * minor node ID should be the ID of the socket.
611eb00b1c8SRobert Mustacchi */
612eb00b1c8SRobert Mustacchi static boolean_t
imc_create_minors(imc_t * imc)613eb00b1c8SRobert Mustacchi imc_create_minors(imc_t *imc)
614eb00b1c8SRobert Mustacchi {
615eb00b1c8SRobert Mustacchi uint_t i;
616eb00b1c8SRobert Mustacchi
617eb00b1c8SRobert Mustacchi ddi_set_name_addr(imc->imc_dip, "1");
618eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
619eb00b1c8SRobert Mustacchi char buf[MAXNAMELEN];
620eb00b1c8SRobert Mustacchi
621eb00b1c8SRobert Mustacchi if (snprintf(buf, sizeof (buf), "mc-imc-%u", i) >=
622eb00b1c8SRobert Mustacchi sizeof (buf)) {
623eb00b1c8SRobert Mustacchi goto fail;
624eb00b1c8SRobert Mustacchi }
625eb00b1c8SRobert Mustacchi
626eb00b1c8SRobert Mustacchi if (ddi_create_minor_node(imc->imc_dip, buf, S_IFCHR, i,
627eb00b1c8SRobert Mustacchi "ddi_mem_ctrl", 0) != DDI_SUCCESS) {
628eb00b1c8SRobert Mustacchi dev_err(imc->imc_dip, CE_WARN, "failed to create "
629eb00b1c8SRobert Mustacchi "minor node %u: %s", i, buf);
630eb00b1c8SRobert Mustacchi goto fail;
631eb00b1c8SRobert Mustacchi }
632eb00b1c8SRobert Mustacchi }
633eb00b1c8SRobert Mustacchi return (B_TRUE);
634eb00b1c8SRobert Mustacchi
635eb00b1c8SRobert Mustacchi fail:
636eb00b1c8SRobert Mustacchi ddi_remove_minor_node(imc->imc_dip, NULL);
637eb00b1c8SRobert Mustacchi return (B_FALSE);
638eb00b1c8SRobert Mustacchi }
639eb00b1c8SRobert Mustacchi
640eb00b1c8SRobert Mustacchi /*
641eb00b1c8SRobert Mustacchi * Check the current MC route value for this SAD. On Skylake systems there is
642eb00b1c8SRobert Mustacchi * one per core. Every core should agree. If not, we will not trust the SAD
643eb00b1c8SRobert Mustacchi * MCROUTE values and this will cause system address decoding to fail on
644eb00b1c8SRobert Mustacchi * skylake.
645eb00b1c8SRobert Mustacchi */
646eb00b1c8SRobert Mustacchi static void
imc_mcroute_check(imc_t * imc,imc_sad_t * sad,imc_stub_t * stub)647eb00b1c8SRobert Mustacchi imc_mcroute_check(imc_t *imc, imc_sad_t *sad, imc_stub_t *stub)
648eb00b1c8SRobert Mustacchi {
649eb00b1c8SRobert Mustacchi uint32_t val;
650eb00b1c8SRobert Mustacchi
651eb00b1c8SRobert Mustacchi val = pci_config_get32(stub->istub_cfgspace,
652eb00b1c8SRobert Mustacchi IMC_REG_SKX_SAD_MC_ROUTE_TABLE);
653eb00b1c8SRobert Mustacchi if (val == PCI_EINVAL32) {
654eb00b1c8SRobert Mustacchi sad->isad_valid |= IMC_SAD_V_BAD_PCI_READ;
655eb00b1c8SRobert Mustacchi return;
656eb00b1c8SRobert Mustacchi }
657eb00b1c8SRobert Mustacchi
658eb00b1c8SRobert Mustacchi if ((sad->isad_flags & IMC_SAD_MCROUTE_VALID) == 0 && val != 0) {
659eb00b1c8SRobert Mustacchi sad->isad_flags |= IMC_SAD_MCROUTE_VALID;
660eb00b1c8SRobert Mustacchi sad->isad_mcroute.ismc_raw_mcroute = val;
661eb00b1c8SRobert Mustacchi return;
662eb00b1c8SRobert Mustacchi }
663eb00b1c8SRobert Mustacchi
664eb00b1c8SRobert Mustacchi /*
665eb00b1c8SRobert Mustacchi * Occasionally we see MC ROUTE table entries with a value of zero.
666eb00b1c8SRobert Mustacchi * We should ignore those for now.
667eb00b1c8SRobert Mustacchi */
668eb00b1c8SRobert Mustacchi if (val != sad->isad_mcroute.ismc_raw_mcroute && val != 0) {
669eb00b1c8SRobert Mustacchi dev_err(imc->imc_dip, CE_WARN, "SAD MC_ROUTE_TABLE mismatch "
670eb00b1c8SRobert Mustacchi "with socket. SAD has val 0x%x, system has %x\n",
671eb00b1c8SRobert Mustacchi val, sad->isad_mcroute.ismc_raw_mcroute);
672eb00b1c8SRobert Mustacchi sad->isad_valid |= IMC_SAD_V_BAD_MCROUTE;
673eb00b1c8SRobert Mustacchi }
674eb00b1c8SRobert Mustacchi }
675eb00b1c8SRobert Mustacchi
676eb00b1c8SRobert Mustacchi /*
677eb00b1c8SRobert Mustacchi * On Skylake, many of the devices that we care about are on separate PCI Buses.
678eb00b1c8SRobert Mustacchi * These can be mapped together by the DECS register. However, we need to know
679eb00b1c8SRobert Mustacchi * how to map different buses together so that we can more usefully associate
680eb00b1c8SRobert Mustacchi * information. The set of buses is all present in the DECS register. We'll
681eb00b1c8SRobert Mustacchi * effectively assign sockets to buses. This is also still something that comes
682eb00b1c8SRobert Mustacchi * up on pre-Skylake systems as well.
683eb00b1c8SRobert Mustacchi */
684eb00b1c8SRobert Mustacchi static boolean_t
imc_map_buses(imc_t * imc)685eb00b1c8SRobert Mustacchi imc_map_buses(imc_t *imc)
686eb00b1c8SRobert Mustacchi {
687eb00b1c8SRobert Mustacchi imc_stub_t *stub;
688eb00b1c8SRobert Mustacchi uint_t nsock;
689eb00b1c8SRobert Mustacchi
690eb00b1c8SRobert Mustacchi /*
691eb00b1c8SRobert Mustacchi * Find the UBOX_DECS registers so we can establish socket mappings. On
692eb00b1c8SRobert Mustacchi * Skylake, there are three different sets of buses that we need to
693eb00b1c8SRobert Mustacchi * cover all of our devices, while there are only two before that.
694eb00b1c8SRobert Mustacchi */
695eb00b1c8SRobert Mustacchi for (nsock = 0, stub = avl_first(&imc->imc_stubs); stub != NULL;
696eb00b1c8SRobert Mustacchi stub = AVL_NEXT(&imc->imc_stubs, stub)) {
697eb00b1c8SRobert Mustacchi uint32_t busno;
698eb00b1c8SRobert Mustacchi
699eb00b1c8SRobert Mustacchi if (stub->istub_table->imcs_type != IMC_TYPE_UBOX_CPUBUSNO) {
700eb00b1c8SRobert Mustacchi continue;
701eb00b1c8SRobert Mustacchi }
702eb00b1c8SRobert Mustacchi
703eb00b1c8SRobert Mustacchi busno = pci_config_get32(stub->istub_cfgspace,
704eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_ubox_cpubusno_offset);
705eb00b1c8SRobert Mustacchi if (busno == PCI_EINVAL32) {
706eb00b1c8SRobert Mustacchi dev_err(imc->imc_dip, CE_WARN, "failed to read "
707eb00b1c8SRobert Mustacchi "UBOX_DECS CPUBUSNO0: invalid PCI read");
708eb00b1c8SRobert Mustacchi return (B_FALSE);
709eb00b1c8SRobert Mustacchi }
710eb00b1c8SRobert Mustacchi
711eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE) {
712eb00b1c8SRobert Mustacchi imc->imc_sockets[nsock].isock_nbus = 3;
713eb00b1c8SRobert Mustacchi imc->imc_sockets[nsock].isock_bus[0] =
714eb00b1c8SRobert Mustacchi IMC_UBOX_CPUBUSNO_0(busno);
715eb00b1c8SRobert Mustacchi imc->imc_sockets[nsock].isock_bus[1] =
716eb00b1c8SRobert Mustacchi IMC_UBOX_CPUBUSNO_1(busno);
717eb00b1c8SRobert Mustacchi imc->imc_sockets[nsock].isock_bus[2] =
718eb00b1c8SRobert Mustacchi IMC_UBOX_CPUBUSNO_2(busno);
719eb00b1c8SRobert Mustacchi } else {
720eb00b1c8SRobert Mustacchi imc->imc_sockets[nsock].isock_bus[0] =
721eb00b1c8SRobert Mustacchi IMC_UBOX_CPUBUSNO_0(busno);
722eb00b1c8SRobert Mustacchi imc->imc_sockets[nsock].isock_bus[1] =
723eb00b1c8SRobert Mustacchi IMC_UBOX_CPUBUSNO_1(busno);
724eb00b1c8SRobert Mustacchi imc->imc_sockets[nsock].isock_nbus = 2;
725eb00b1c8SRobert Mustacchi }
726eb00b1c8SRobert Mustacchi nsock++;
727eb00b1c8SRobert Mustacchi }
728eb00b1c8SRobert Mustacchi imc->imc_nsockets = nsock;
729eb00b1c8SRobert Mustacchi
730eb00b1c8SRobert Mustacchi return (B_TRUE);
731eb00b1c8SRobert Mustacchi }
732eb00b1c8SRobert Mustacchi
733eb00b1c8SRobert Mustacchi /*
734eb00b1c8SRobert Mustacchi * For a given stub that we've found, map it to its corresponding socket based
735eb00b1c8SRobert Mustacchi * on the PCI bus that it has.
736eb00b1c8SRobert Mustacchi */
737eb00b1c8SRobert Mustacchi static imc_socket_t *
imc_map_find_socket(imc_t * imc,imc_stub_t * stub)738eb00b1c8SRobert Mustacchi imc_map_find_socket(imc_t *imc, imc_stub_t *stub)
739eb00b1c8SRobert Mustacchi {
740eb00b1c8SRobert Mustacchi uint_t i;
741eb00b1c8SRobert Mustacchi
742eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
743eb00b1c8SRobert Mustacchi uint_t bus;
744eb00b1c8SRobert Mustacchi
745eb00b1c8SRobert Mustacchi for (bus = 0; bus < imc->imc_sockets[i].isock_nbus; bus++) {
746eb00b1c8SRobert Mustacchi if (imc->imc_sockets[i].isock_bus[bus] ==
747eb00b1c8SRobert Mustacchi stub->istub_bus) {
748eb00b1c8SRobert Mustacchi return (&imc->imc_sockets[i]);
749eb00b1c8SRobert Mustacchi }
750eb00b1c8SRobert Mustacchi }
751eb00b1c8SRobert Mustacchi }
752eb00b1c8SRobert Mustacchi
753eb00b1c8SRobert Mustacchi return (NULL);
754eb00b1c8SRobert Mustacchi }
755eb00b1c8SRobert Mustacchi
756eb00b1c8SRobert Mustacchi static boolean_t
imc_map_stubs(imc_t * imc)757eb00b1c8SRobert Mustacchi imc_map_stubs(imc_t *imc)
758eb00b1c8SRobert Mustacchi {
759eb00b1c8SRobert Mustacchi imc_stub_t *stub;
760eb00b1c8SRobert Mustacchi
761eb00b1c8SRobert Mustacchi if (!imc_map_buses(imc)) {
762eb00b1c8SRobert Mustacchi return (B_FALSE);
763eb00b1c8SRobert Mustacchi }
764eb00b1c8SRobert Mustacchi
765eb00b1c8SRobert Mustacchi stub = avl_first(&imc->imc_stubs);
766eb00b1c8SRobert Mustacchi for (stub = avl_first(&imc->imc_stubs); stub != NULL;
767eb00b1c8SRobert Mustacchi stub = AVL_NEXT(&imc->imc_stubs, stub)) {
768eb00b1c8SRobert Mustacchi imc_socket_t *sock = imc_map_find_socket(imc, stub);
769eb00b1c8SRobert Mustacchi
770eb00b1c8SRobert Mustacchi if (sock == NULL) {
771eb00b1c8SRobert Mustacchi dev_err(imc->imc_dip, CE_WARN, "found stub type %u "
772eb00b1c8SRobert Mustacchi "PCI%x,%x with bdf %u/%u/%u that does not match a "
773eb00b1c8SRobert Mustacchi "known PCI bus for any of %u sockets",
774eb00b1c8SRobert Mustacchi stub->istub_table->imcs_type, stub->istub_vid,
775eb00b1c8SRobert Mustacchi stub->istub_did, stub->istub_bus, stub->istub_dev,
776eb00b1c8SRobert Mustacchi stub->istub_func, imc->imc_nsockets);
777eb00b1c8SRobert Mustacchi continue;
778eb00b1c8SRobert Mustacchi }
779eb00b1c8SRobert Mustacchi
780eb00b1c8SRobert Mustacchi /*
781eb00b1c8SRobert Mustacchi * We don't have to worry about duplicates here. We check to
782eb00b1c8SRobert Mustacchi * make sure that we have unique bdfs here.
783eb00b1c8SRobert Mustacchi */
784eb00b1c8SRobert Mustacchi switch (stub->istub_table->imcs_type) {
785eb00b1c8SRobert Mustacchi case IMC_TYPE_MC0_M2M:
786eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_m2m = stub;
787eb00b1c8SRobert Mustacchi break;
788eb00b1c8SRobert Mustacchi case IMC_TYPE_MC1_M2M:
789eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_m2m = stub;
790eb00b1c8SRobert Mustacchi break;
791eb00b1c8SRobert Mustacchi case IMC_TYPE_MC0_MAIN0:
792eb00b1c8SRobert Mustacchi sock->isock_nimc++;
793eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_main0 = stub;
794eb00b1c8SRobert Mustacchi
795eb00b1c8SRobert Mustacchi /*
796eb00b1c8SRobert Mustacchi * On Skylake, the MAIN0 does double duty as channel
797eb00b1c8SRobert Mustacchi * zero and as the TAD.
798eb00b1c8SRobert Mustacchi */
799eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE) {
800eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_nchannels++;
801eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_channels[0].ich_desc =
802eb00b1c8SRobert Mustacchi stub;
803eb00b1c8SRobert Mustacchi sock->isock_tad[0].itad_stub = stub;
804eb00b1c8SRobert Mustacchi sock->isock_ntad++;
805eb00b1c8SRobert Mustacchi }
806eb00b1c8SRobert Mustacchi break;
807eb00b1c8SRobert Mustacchi case IMC_TYPE_MC0_MAIN1:
808eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_main1 = stub;
809eb00b1c8SRobert Mustacchi break;
810eb00b1c8SRobert Mustacchi case IMC_TYPE_MC1_MAIN0:
811eb00b1c8SRobert Mustacchi sock->isock_nimc++;
812eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_main0 = stub;
813eb00b1c8SRobert Mustacchi
814eb00b1c8SRobert Mustacchi /*
815eb00b1c8SRobert Mustacchi * On Skylake, the MAIN0 does double duty as channel
816eb00b1c8SRobert Mustacchi * zero and as the TAD.
817eb00b1c8SRobert Mustacchi */
818eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE) {
819eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_nchannels++;
820eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_channels[0].ich_desc =
821eb00b1c8SRobert Mustacchi stub;
822eb00b1c8SRobert Mustacchi sock->isock_tad[1].itad_stub = stub;
823eb00b1c8SRobert Mustacchi sock->isock_ntad++;
824eb00b1c8SRobert Mustacchi }
825eb00b1c8SRobert Mustacchi break;
826eb00b1c8SRobert Mustacchi case IMC_TYPE_MC1_MAIN1:
827eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_main1 = stub;
828eb00b1c8SRobert Mustacchi break;
829eb00b1c8SRobert Mustacchi case IMC_TYPE_MC0_CHANNEL0:
830eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_nchannels++;
831eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_channels[0].ich_desc = stub;
832eb00b1c8SRobert Mustacchi break;
833eb00b1c8SRobert Mustacchi case IMC_TYPE_MC0_CHANNEL1:
834eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_nchannels++;
835eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_channels[1].ich_desc = stub;
836eb00b1c8SRobert Mustacchi break;
837eb00b1c8SRobert Mustacchi case IMC_TYPE_MC0_CHANNEL2:
838eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_nchannels++;
839eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_channels[2].ich_desc = stub;
840eb00b1c8SRobert Mustacchi break;
841eb00b1c8SRobert Mustacchi case IMC_TYPE_MC0_CHANNEL3:
842eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_nchannels++;
843eb00b1c8SRobert Mustacchi sock->isock_imcs[0].icn_channels[3].ich_desc = stub;
844eb00b1c8SRobert Mustacchi break;
845eb00b1c8SRobert Mustacchi case IMC_TYPE_MC1_CHANNEL0:
846eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_nchannels++;
847eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_channels[0].ich_desc = stub;
848eb00b1c8SRobert Mustacchi break;
849eb00b1c8SRobert Mustacchi case IMC_TYPE_MC1_CHANNEL1:
850eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_nchannels++;
851eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_channels[1].ich_desc = stub;
852eb00b1c8SRobert Mustacchi break;
853eb00b1c8SRobert Mustacchi case IMC_TYPE_MC1_CHANNEL2:
854eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_nchannels++;
855eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_channels[2].ich_desc = stub;
856eb00b1c8SRobert Mustacchi break;
857eb00b1c8SRobert Mustacchi case IMC_TYPE_MC1_CHANNEL3:
858eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_nchannels++;
859eb00b1c8SRobert Mustacchi sock->isock_imcs[1].icn_channels[3].ich_desc = stub;
860eb00b1c8SRobert Mustacchi break;
861eb00b1c8SRobert Mustacchi case IMC_TYPE_SAD_DRAM:
862eb00b1c8SRobert Mustacchi sock->isock_sad.isad_dram = stub;
863eb00b1c8SRobert Mustacchi break;
864eb00b1c8SRobert Mustacchi case IMC_TYPE_SAD_MMIO:
865eb00b1c8SRobert Mustacchi sock->isock_sad.isad_mmio = stub;
866eb00b1c8SRobert Mustacchi break;
867eb00b1c8SRobert Mustacchi case IMC_TYPE_SAD_MISC:
868eb00b1c8SRobert Mustacchi sock->isock_sad.isad_tolh = stub;
869eb00b1c8SRobert Mustacchi break;
870eb00b1c8SRobert Mustacchi case IMC_TYPE_VTD_MISC:
871eb00b1c8SRobert Mustacchi /*
872eb00b1c8SRobert Mustacchi * Some systems have multiple VT-D Misc. entry points
873eb00b1c8SRobert Mustacchi * in the system. In this case, only use the first one
874eb00b1c8SRobert Mustacchi * we find.
875eb00b1c8SRobert Mustacchi */
876eb00b1c8SRobert Mustacchi if (imc->imc_gvtd_misc == NULL) {
877eb00b1c8SRobert Mustacchi imc->imc_gvtd_misc = stub;
878eb00b1c8SRobert Mustacchi }
879eb00b1c8SRobert Mustacchi break;
880eb00b1c8SRobert Mustacchi case IMC_TYPE_SAD_MCROUTE:
881eb00b1c8SRobert Mustacchi ASSERT3U(imc->imc_gen, >=, IMC_GEN_SKYLAKE);
882eb00b1c8SRobert Mustacchi imc_mcroute_check(imc, &sock->isock_sad, stub);
883eb00b1c8SRobert Mustacchi break;
884eb00b1c8SRobert Mustacchi case IMC_TYPE_UBOX:
885eb00b1c8SRobert Mustacchi sock->isock_ubox = stub;
886eb00b1c8SRobert Mustacchi break;
887eb00b1c8SRobert Mustacchi case IMC_TYPE_HA0:
888eb00b1c8SRobert Mustacchi sock->isock_ntad++;
889eb00b1c8SRobert Mustacchi sock->isock_tad[0].itad_stub = stub;
890eb00b1c8SRobert Mustacchi break;
891eb00b1c8SRobert Mustacchi case IMC_TYPE_HA1:
892eb00b1c8SRobert Mustacchi sock->isock_ntad++;
893eb00b1c8SRobert Mustacchi sock->isock_tad[1].itad_stub = stub;
894eb00b1c8SRobert Mustacchi break;
895eb00b1c8SRobert Mustacchi case IMC_TYPE_UBOX_CPUBUSNO:
896eb00b1c8SRobert Mustacchi sock->isock_cpubusno = stub;
897eb00b1c8SRobert Mustacchi break;
898eb00b1c8SRobert Mustacchi default:
899eb00b1c8SRobert Mustacchi /*
900eb00b1c8SRobert Mustacchi * Attempt to still attach if we can.
901eb00b1c8SRobert Mustacchi */
902eb00b1c8SRobert Mustacchi dev_err(imc->imc_dip, CE_WARN, "Encountered unknown "
903eb00b1c8SRobert Mustacchi "IMC type (%u) on PCI %x,%x",
904eb00b1c8SRobert Mustacchi stub->istub_table->imcs_type,
905eb00b1c8SRobert Mustacchi stub->istub_vid, stub->istub_did);
906eb00b1c8SRobert Mustacchi break;
907eb00b1c8SRobert Mustacchi }
908eb00b1c8SRobert Mustacchi }
909eb00b1c8SRobert Mustacchi
910eb00b1c8SRobert Mustacchi return (B_TRUE);
911eb00b1c8SRobert Mustacchi }
912eb00b1c8SRobert Mustacchi
913eb00b1c8SRobert Mustacchi /*
914eb00b1c8SRobert Mustacchi * Go through and fix up various aspects of the stubs mappings on systems. The
915eb00b1c8SRobert Mustacchi * following are a list of what we need to fix up:
916eb00b1c8SRobert Mustacchi *
917eb00b1c8SRobert Mustacchi * 1. On Haswell and newer systems, there is only one global VT-d device. We
918eb00b1c8SRobert Mustacchi * need to go back and map that to all of the per-socket imc_sad_t entries.
919eb00b1c8SRobert Mustacchi */
920eb00b1c8SRobert Mustacchi static void
imc_fixup_stubs(imc_t * imc)921eb00b1c8SRobert Mustacchi imc_fixup_stubs(imc_t *imc)
922eb00b1c8SRobert Mustacchi {
923eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_HASWELL) {
924eb00b1c8SRobert Mustacchi uint_t i;
925eb00b1c8SRobert Mustacchi
926eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
927eb00b1c8SRobert Mustacchi ASSERT3P(imc->imc_sockets[i].isock_sad.isad_tolh,
928eb00b1c8SRobert Mustacchi ==, NULL);
929eb00b1c8SRobert Mustacchi imc->imc_sockets[i].isock_sad.isad_tolh =
930eb00b1c8SRobert Mustacchi imc->imc_gvtd_misc;
931eb00b1c8SRobert Mustacchi }
932eb00b1c8SRobert Mustacchi }
933eb00b1c8SRobert Mustacchi }
934eb00b1c8SRobert Mustacchi
9357d916034SRobert Mustacchi /*
9367d916034SRobert Mustacchi * In the wild we've hit a few odd cases where not all devices are exposed that
9377d916034SRobert Mustacchi * we might expect by firmware. In particular we've seen and validate the
9387d916034SRobert Mustacchi * following cases:
9397d916034SRobert Mustacchi *
9407d916034SRobert Mustacchi * o We don't find all of the channel devices that we expect, e.g. we have the
9417d916034SRobert Mustacchi * stubs for channels 1-3, but not 0. That has been seen on an Intel S2600CW
9427d916034SRobert Mustacchi * with an E5-2630v3.
9437d916034SRobert Mustacchi */
9447d916034SRobert Mustacchi static boolean_t
imc_validate_stubs(imc_t * imc)9457d916034SRobert Mustacchi imc_validate_stubs(imc_t *imc)
9467d916034SRobert Mustacchi {
9477d916034SRobert Mustacchi for (uint_t sock = 0; sock < imc->imc_nsockets; sock++) {
9487d916034SRobert Mustacchi imc_socket_t *socket = &imc->imc_sockets[sock];
9497d916034SRobert Mustacchi
9507d916034SRobert Mustacchi for (uint_t mc = 0; mc < socket->isock_nimc; mc++) {
9517d916034SRobert Mustacchi imc_mc_t *mcp = &socket->isock_imcs[mc];
9527d916034SRobert Mustacchi
9537d916034SRobert Mustacchi for (uint_t chan = 0; chan < mcp->icn_nchannels;
9547d916034SRobert Mustacchi chan++) {
9557d916034SRobert Mustacchi if (mcp->icn_channels[chan].ich_desc == NULL) {
9567d916034SRobert Mustacchi dev_err(imc->imc_dip, CE_WARN,
9577d916034SRobert Mustacchi "!missing device for socket %u/"
9587d916034SRobert Mustacchi "imc %u/channel %u", sock, mc,
9597d916034SRobert Mustacchi chan);
9607d916034SRobert Mustacchi return (B_FALSE);
9617d916034SRobert Mustacchi }
9627d916034SRobert Mustacchi }
9637d916034SRobert Mustacchi }
9647d916034SRobert Mustacchi }
9657d916034SRobert Mustacchi
9667d916034SRobert Mustacchi return (B_TRUE);
9677d916034SRobert Mustacchi }
9687d916034SRobert Mustacchi
969eb00b1c8SRobert Mustacchi /*
970eb00b1c8SRobert Mustacchi * Attempt to map all of the discovered sockets to the corresponding APIC based
971eb00b1c8SRobert Mustacchi * socket. We do these mappings by getting the node id of the socket and
972eb00b1c8SRobert Mustacchi * adjusting it to make sure that no home agent is present in it. We use the
973eb00b1c8SRobert Mustacchi * UBOX to avoid any home agent related bits that are present in other
974eb00b1c8SRobert Mustacchi * registers.
975eb00b1c8SRobert Mustacchi */
976eb00b1c8SRobert Mustacchi static void
imc_map_sockets(imc_t * imc)977eb00b1c8SRobert Mustacchi imc_map_sockets(imc_t *imc)
978eb00b1c8SRobert Mustacchi {
979eb00b1c8SRobert Mustacchi uint_t i;
980eb00b1c8SRobert Mustacchi
981eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
982eb00b1c8SRobert Mustacchi uint32_t nodeid;
983eb00b1c8SRobert Mustacchi ddi_acc_handle_t h;
984eb00b1c8SRobert Mustacchi
985eb00b1c8SRobert Mustacchi h = imc->imc_sockets[i].isock_ubox->istub_cfgspace;
986eb00b1c8SRobert Mustacchi nodeid = pci_config_get32(h,
987eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_sad_nodeid_offset);
988eb00b1c8SRobert Mustacchi if (nodeid == PCI_EINVAL32) {
989eb00b1c8SRobert Mustacchi imc->imc_sockets[i].isock_valid |=
990eb00b1c8SRobert Mustacchi IMC_SOCKET_V_BAD_NODEID;
991eb00b1c8SRobert Mustacchi continue;
992eb00b1c8SRobert Mustacchi }
993eb00b1c8SRobert Mustacchi
994eb00b1c8SRobert Mustacchi imc->imc_sockets[i].isock_nodeid = IMC_NODEID_UBOX_MASK(nodeid);
995eb00b1c8SRobert Mustacchi imc->imc_spointers[nodeid] = &imc->imc_sockets[i];
996eb00b1c8SRobert Mustacchi }
997eb00b1c8SRobert Mustacchi }
998eb00b1c8SRobert Mustacchi
999eb00b1c8SRobert Mustacchi /*
1000eb00b1c8SRobert Mustacchi * Decode the MTR, accounting for variances between processor generations.
1001eb00b1c8SRobert Mustacchi */
1002eb00b1c8SRobert Mustacchi static void
imc_decode_mtr(imc_t * imc,imc_mc_t * icn,imc_dimm_t * dimm,uint32_t mtr)1003eb00b1c8SRobert Mustacchi imc_decode_mtr(imc_t *imc, imc_mc_t *icn, imc_dimm_t *dimm, uint32_t mtr)
1004eb00b1c8SRobert Mustacchi {
1005eb00b1c8SRobert Mustacchi uint8_t disable;
1006eb00b1c8SRobert Mustacchi
1007eb00b1c8SRobert Mustacchi /*
1008eb00b1c8SRobert Mustacchi * Check present first, before worrying about anything else.
1009eb00b1c8SRobert Mustacchi */
1010eb00b1c8SRobert Mustacchi if (imc->imc_gen < IMC_GEN_SKYLAKE &&
1011eb00b1c8SRobert Mustacchi IMC_MTR_PRESENT_SNB_BRD(mtr) == 0) {
1012eb00b1c8SRobert Mustacchi dimm->idimm_present = B_FALSE;
1013eb00b1c8SRobert Mustacchi return;
1014eb00b1c8SRobert Mustacchi } else if (imc->imc_gen >= IMC_GEN_SKYLAKE &&
1015eb00b1c8SRobert Mustacchi IMC_MTR_PRESENT_SKYLAKE(mtr) == 0) {
1016eb00b1c8SRobert Mustacchi dimm->idimm_present = B_FALSE;
1017eb00b1c8SRobert Mustacchi return;
1018eb00b1c8SRobert Mustacchi }
1019eb00b1c8SRobert Mustacchi
1020eb00b1c8SRobert Mustacchi dimm->idimm_present = B_TRUE;
1021eb00b1c8SRobert Mustacchi dimm->idimm_ncolumns = IMC_MTR_CA_WIDTH(mtr) + IMC_MTR_CA_BASE;
1022eb00b1c8SRobert Mustacchi if (dimm->idimm_ncolumns < IMC_MTR_CA_MIN ||
1023eb00b1c8SRobert Mustacchi dimm->idimm_ncolumns > IMC_MTR_CA_MAX) {
1024eb00b1c8SRobert Mustacchi dimm->idimm_valid |= IMC_DIMM_V_BAD_COLUMNS;
1025eb00b1c8SRobert Mustacchi }
1026eb00b1c8SRobert Mustacchi
1027eb00b1c8SRobert Mustacchi dimm->idimm_nrows = IMC_MTR_RA_WIDTH(mtr) + IMC_MTR_RA_BASE;
1028eb00b1c8SRobert Mustacchi if (dimm->idimm_nrows < IMC_MTR_RA_MIN ||
1029eb00b1c8SRobert Mustacchi dimm->idimm_nrows > IMC_MTR_RA_MAX) {
1030eb00b1c8SRobert Mustacchi dimm->idimm_valid |= IMC_DIMM_V_BAD_ROWS;
1031eb00b1c8SRobert Mustacchi }
1032eb00b1c8SRobert Mustacchi
1033eb00b1c8SRobert Mustacchi /*
1034eb00b1c8SRobert Mustacchi * Determine Density, this information is not present on Sandy Bridge.
1035eb00b1c8SRobert Mustacchi */
1036eb00b1c8SRobert Mustacchi switch (imc->imc_gen) {
1037eb00b1c8SRobert Mustacchi case IMC_GEN_IVY:
1038eb00b1c8SRobert Mustacchi dimm->idimm_density = 1U << IMC_MTR_DENSITY_IVY_BRD(mtr);
1039eb00b1c8SRobert Mustacchi break;
1040eb00b1c8SRobert Mustacchi case IMC_GEN_HASWELL:
1041eb00b1c8SRobert Mustacchi case IMC_GEN_BROADWELL:
1042eb00b1c8SRobert Mustacchi switch (IMC_MTR_DENSITY_IVY_BRD(mtr)) {
1043eb00b1c8SRobert Mustacchi case 0:
1044eb00b1c8SRobert Mustacchi default:
1045eb00b1c8SRobert Mustacchi dimm->idimm_density = 0;
1046eb00b1c8SRobert Mustacchi dimm->idimm_valid |= IMC_DIMM_V_BAD_DENSITY;
1047eb00b1c8SRobert Mustacchi break;
1048eb00b1c8SRobert Mustacchi case 1:
1049eb00b1c8SRobert Mustacchi dimm->idimm_density = 2;
1050eb00b1c8SRobert Mustacchi break;
1051eb00b1c8SRobert Mustacchi case 2:
1052eb00b1c8SRobert Mustacchi dimm->idimm_density = 4;
1053eb00b1c8SRobert Mustacchi break;
1054eb00b1c8SRobert Mustacchi case 3:
1055eb00b1c8SRobert Mustacchi dimm->idimm_density = 8;
1056eb00b1c8SRobert Mustacchi break;
1057eb00b1c8SRobert Mustacchi }
1058eb00b1c8SRobert Mustacchi break;
1059eb00b1c8SRobert Mustacchi case IMC_GEN_SKYLAKE:
1060eb00b1c8SRobert Mustacchi switch (IMC_MTR_DENSITY_SKX(mtr)) {
1061eb00b1c8SRobert Mustacchi case 0:
1062eb00b1c8SRobert Mustacchi default:
1063eb00b1c8SRobert Mustacchi dimm->idimm_density = 0;
1064eb00b1c8SRobert Mustacchi dimm->idimm_valid |= IMC_DIMM_V_BAD_DENSITY;
1065eb00b1c8SRobert Mustacchi break;
1066eb00b1c8SRobert Mustacchi case 1:
1067eb00b1c8SRobert Mustacchi dimm->idimm_density = 2;
1068eb00b1c8SRobert Mustacchi break;
1069eb00b1c8SRobert Mustacchi case 2:
1070eb00b1c8SRobert Mustacchi dimm->idimm_density = 4;
1071eb00b1c8SRobert Mustacchi break;
1072eb00b1c8SRobert Mustacchi case 3:
1073eb00b1c8SRobert Mustacchi dimm->idimm_density = 8;
1074eb00b1c8SRobert Mustacchi break;
1075eb00b1c8SRobert Mustacchi case 4:
1076eb00b1c8SRobert Mustacchi dimm->idimm_density = 16;
1077eb00b1c8SRobert Mustacchi break;
1078eb00b1c8SRobert Mustacchi case 5:
1079eb00b1c8SRobert Mustacchi dimm->idimm_density = 12;
1080eb00b1c8SRobert Mustacchi break;
1081eb00b1c8SRobert Mustacchi }
1082eb00b1c8SRobert Mustacchi break;
1083eb00b1c8SRobert Mustacchi case IMC_GEN_UNKNOWN:
1084eb00b1c8SRobert Mustacchi case IMC_GEN_SANDY:
1085eb00b1c8SRobert Mustacchi dimm->idimm_density = 0;
1086eb00b1c8SRobert Mustacchi break;
1087eb00b1c8SRobert Mustacchi }
1088eb00b1c8SRobert Mustacchi
1089eb00b1c8SRobert Mustacchi /*
1090eb00b1c8SRobert Mustacchi * The values of width are the same on IVY->SKX, but the bits are
1091eb00b1c8SRobert Mustacchi * different. This doesn't exist on SNB.
1092eb00b1c8SRobert Mustacchi */
1093eb00b1c8SRobert Mustacchi if (imc->imc_gen > IMC_GEN_SANDY) {
1094eb00b1c8SRobert Mustacchi uint8_t width;
1095eb00b1c8SRobert Mustacchi
1096eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_BROADWELL) {
1097eb00b1c8SRobert Mustacchi width = IMC_MTR_WIDTH_BRD_SKX(mtr);
1098eb00b1c8SRobert Mustacchi } else {
1099eb00b1c8SRobert Mustacchi width = IMC_MTR_WIDTH_IVB_HAS(mtr);
1100eb00b1c8SRobert Mustacchi }
1101eb00b1c8SRobert Mustacchi switch (width) {
1102eb00b1c8SRobert Mustacchi case 0:
1103eb00b1c8SRobert Mustacchi dimm->idimm_width = 4;
1104eb00b1c8SRobert Mustacchi break;
1105eb00b1c8SRobert Mustacchi case 1:
1106eb00b1c8SRobert Mustacchi dimm->idimm_width = 8;
1107eb00b1c8SRobert Mustacchi break;
1108eb00b1c8SRobert Mustacchi case 2:
1109eb00b1c8SRobert Mustacchi dimm->idimm_width = 16;
1110eb00b1c8SRobert Mustacchi break;
1111eb00b1c8SRobert Mustacchi default:
1112eb00b1c8SRobert Mustacchi dimm->idimm_width = 0;
1113eb00b1c8SRobert Mustacchi dimm->idimm_valid |= IMC_DIMM_V_BAD_WIDTH;
1114eb00b1c8SRobert Mustacchi break;
1115eb00b1c8SRobert Mustacchi }
1116eb00b1c8SRobert Mustacchi } else {
1117eb00b1c8SRobert Mustacchi dimm->idimm_width = 0;
1118eb00b1c8SRobert Mustacchi }
1119eb00b1c8SRobert Mustacchi
1120eb00b1c8SRobert Mustacchi dimm->idimm_nranks = 1 << IMC_MTR_DDR_RANKS(mtr);
1121eb00b1c8SRobert Mustacchi switch (imc->imc_gen) {
1122eb00b1c8SRobert Mustacchi case IMC_GEN_HASWELL:
1123eb00b1c8SRobert Mustacchi case IMC_GEN_BROADWELL:
1124eb00b1c8SRobert Mustacchi case IMC_GEN_SKYLAKE:
1125eb00b1c8SRobert Mustacchi if (dimm->idimm_nranks > IMC_MTR_DDR_RANKS_MAX_HAS_SKX) {
1126eb00b1c8SRobert Mustacchi dimm->idimm_nranks = 0;
1127eb00b1c8SRobert Mustacchi dimm->idimm_valid |= IMC_DIMM_V_BAD_RANKS;
1128eb00b1c8SRobert Mustacchi }
1129eb00b1c8SRobert Mustacchi break;
1130eb00b1c8SRobert Mustacchi default:
1131eb00b1c8SRobert Mustacchi if (dimm->idimm_nranks > IMC_MTR_DDR_RANKS_MAX) {
1132eb00b1c8SRobert Mustacchi dimm->idimm_nranks = 0;
1133eb00b1c8SRobert Mustacchi dimm->idimm_valid |= IMC_DIMM_V_BAD_RANKS;
1134eb00b1c8SRobert Mustacchi }
1135eb00b1c8SRobert Mustacchi }
1136eb00b1c8SRobert Mustacchi
1137eb00b1c8SRobert Mustacchi disable = IMC_MTR_RANK_DISABLE(mtr);
1138eb00b1c8SRobert Mustacchi dimm->idimm_ranks_disabled[0] = (disable & 0x1) != 0;
1139eb00b1c8SRobert Mustacchi dimm->idimm_ranks_disabled[1] = (disable & 0x2) != 0;
1140eb00b1c8SRobert Mustacchi dimm->idimm_ranks_disabled[2] = (disable & 0x4) != 0;
1141eb00b1c8SRobert Mustacchi dimm->idimm_ranks_disabled[3] = (disable & 0x8) != 0;
1142eb00b1c8SRobert Mustacchi
1143eb00b1c8SRobert Mustacchi /*
1144eb00b1c8SRobert Mustacchi * Only Haswell and later have this information.
1145eb00b1c8SRobert Mustacchi */
1146eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_HASWELL) {
1147eb00b1c8SRobert Mustacchi dimm->idimm_hdrl = IMC_MTR_HDRL_HAS_SKX(mtr) != 0;
1148eb00b1c8SRobert Mustacchi dimm->idimm_hdrl_parity = IMC_MTR_HDRL_PARITY_HAS_SKX(mtr) != 0;
1149eb00b1c8SRobert Mustacchi dimm->idimm_3dsranks = IMC_MTR_3DSRANKS_HAS_SKX(mtr);
1150eb00b1c8SRobert Mustacchi if (dimm->idimm_3dsranks != 0) {
1151eb00b1c8SRobert Mustacchi dimm->idimm_3dsranks = 1 << dimm->idimm_3dsranks;
1152eb00b1c8SRobert Mustacchi }
1153eb00b1c8SRobert Mustacchi }
1154eb00b1c8SRobert Mustacchi
1155eb00b1c8SRobert Mustacchi
1156eb00b1c8SRobert Mustacchi if (icn->icn_dimm_type == IMC_DIMM_DDR4) {
1157eb00b1c8SRobert Mustacchi dimm->idimm_nbanks = 16;
1158eb00b1c8SRobert Mustacchi } else {
1159eb00b1c8SRobert Mustacchi dimm->idimm_nbanks = 8;
1160eb00b1c8SRobert Mustacchi }
1161eb00b1c8SRobert Mustacchi
1162eb00b1c8SRobert Mustacchi /*
1163eb00b1c8SRobert Mustacchi * To calculate the DIMM size we need first take the number of rows and
1164eb00b1c8SRobert Mustacchi * columns. This gives us the number of slots per chip. In a given rank
1165eb00b1c8SRobert Mustacchi * there are nbanks of these. There are nrank entries of those. Each of
1166eb00b1c8SRobert Mustacchi * these slots can fit a byte.
1167eb00b1c8SRobert Mustacchi */
1168eb00b1c8SRobert Mustacchi dimm->idimm_size = dimm->idimm_nbanks * dimm->idimm_nranks * 8 *
1169eb00b1c8SRobert Mustacchi (1ULL << (dimm->idimm_ncolumns + dimm->idimm_nrows));
1170eb00b1c8SRobert Mustacchi }
1171eb00b1c8SRobert Mustacchi
1172eb00b1c8SRobert Mustacchi static void
imc_fill_dimms(imc_t * imc,imc_mc_t * icn,imc_channel_t * chan)1173eb00b1c8SRobert Mustacchi imc_fill_dimms(imc_t *imc, imc_mc_t *icn, imc_channel_t *chan)
1174eb00b1c8SRobert Mustacchi {
1175eb00b1c8SRobert Mustacchi uint_t i;
1176eb00b1c8SRobert Mustacchi
1177eb00b1c8SRobert Mustacchi /*
1178eb00b1c8SRobert Mustacchi * There's one register for each DIMM that might be present, we always
1179eb00b1c8SRobert Mustacchi * read that information to determine information about the DIMMs.
1180eb00b1c8SRobert Mustacchi */
1181eb00b1c8SRobert Mustacchi chan->ich_ndimms = imc->imc_gen_data->igd_max_dimms;
1182eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_gen_data->igd_max_dimms; i++) {
1183eb00b1c8SRobert Mustacchi uint32_t mtr;
1184eb00b1c8SRobert Mustacchi imc_dimm_t *dimm = &chan->ich_dimms[i];
1185eb00b1c8SRobert Mustacchi
1186eb00b1c8SRobert Mustacchi bzero(dimm, sizeof (imc_dimm_t));
1187eb00b1c8SRobert Mustacchi mtr = pci_config_get32(chan->ich_desc->istub_cfgspace,
1188eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_mtr_offsets[i]);
1189eb00b1c8SRobert Mustacchi dimm->idimm_mtr = mtr;
1190eb00b1c8SRobert Mustacchi /*
1191eb00b1c8SRobert Mustacchi * We don't really expect to get a bad PCIe read. However, if we
1192eb00b1c8SRobert Mustacchi * do, treat that for the moment as though the DIMM is bad.
1193eb00b1c8SRobert Mustacchi */
1194eb00b1c8SRobert Mustacchi if (mtr == PCI_EINVAL32) {
1195eb00b1c8SRobert Mustacchi dimm->idimm_valid |= IMC_DIMM_V_BAD_PCI_READ;
1196eb00b1c8SRobert Mustacchi continue;
1197eb00b1c8SRobert Mustacchi }
1198eb00b1c8SRobert Mustacchi
1199eb00b1c8SRobert Mustacchi imc_decode_mtr(imc, icn, dimm, mtr);
1200eb00b1c8SRobert Mustacchi }
1201eb00b1c8SRobert Mustacchi }
1202eb00b1c8SRobert Mustacchi
1203eb00b1c8SRobert Mustacchi static boolean_t
imc_fill_controller(imc_t * imc,imc_mc_t * icn)1204eb00b1c8SRobert Mustacchi imc_fill_controller(imc_t *imc, imc_mc_t *icn)
1205eb00b1c8SRobert Mustacchi {
1206eb00b1c8SRobert Mustacchi uint32_t mcmtr;
1207eb00b1c8SRobert Mustacchi
1208eb00b1c8SRobert Mustacchi mcmtr = pci_config_get32(icn->icn_main0->istub_cfgspace,
1209eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_mcmtr_offset);
1210eb00b1c8SRobert Mustacchi if (mcmtr == PCI_EINVAL32) {
1211eb00b1c8SRobert Mustacchi icn->icn_invalid = B_TRUE;
1212eb00b1c8SRobert Mustacchi return (B_FALSE);
1213eb00b1c8SRobert Mustacchi }
1214eb00b1c8SRobert Mustacchi
1215eb00b1c8SRobert Mustacchi icn->icn_closed = IMC_MCMTR_CLOSED_PAGE(mcmtr) != 0;
1216eb00b1c8SRobert Mustacchi if (imc->imc_gen < IMC_GEN_SKYLAKE) {
1217eb00b1c8SRobert Mustacchi icn->icn_lockstep = IMC_MCMTR_LOCKSTEP(mcmtr) != 0;
1218eb00b1c8SRobert Mustacchi } else {
1219eb00b1c8SRobert Mustacchi icn->icn_lockstep = B_FALSE;
1220eb00b1c8SRobert Mustacchi }
1221eb00b1c8SRobert Mustacchi
1222eb00b1c8SRobert Mustacchi icn->icn_ecc = IMC_MCMTR_ECC_ENABLED(mcmtr) != 0;
1223eb00b1c8SRobert Mustacchi
1224eb00b1c8SRobert Mustacchi /*
1225eb00b1c8SRobert Mustacchi * SNB and IVB only support DDR3. Haswell and Broadwell may support
1226eb00b1c8SRobert Mustacchi * DDR4, depends on the SKU. Skylake only supports DDR4.
1227eb00b1c8SRobert Mustacchi */
1228eb00b1c8SRobert Mustacchi switch (imc->imc_gen) {
1229eb00b1c8SRobert Mustacchi case IMC_GEN_SANDY:
1230eb00b1c8SRobert Mustacchi case IMC_GEN_IVY:
1231eb00b1c8SRobert Mustacchi icn->icn_dimm_type = IMC_DIMM_DDR3;
1232eb00b1c8SRobert Mustacchi break;
1233eb00b1c8SRobert Mustacchi case IMC_GEN_HASWELL:
1234eb00b1c8SRobert Mustacchi case IMC_GEN_BROADWELL:
1235eb00b1c8SRobert Mustacchi if (IMC_MCMTR_DDR4_HAS_BRD(mcmtr)) {
1236eb00b1c8SRobert Mustacchi icn->icn_dimm_type = IMC_DIMM_DDR4;
1237eb00b1c8SRobert Mustacchi } else {
1238eb00b1c8SRobert Mustacchi icn->icn_dimm_type = IMC_DIMM_DDR3;
1239eb00b1c8SRobert Mustacchi }
1240eb00b1c8SRobert Mustacchi break;
1241eb00b1c8SRobert Mustacchi default:
1242eb00b1c8SRobert Mustacchi /*
1243eb00b1c8SRobert Mustacchi * Skylake and on are all DDR4.
1244eb00b1c8SRobert Mustacchi */
1245eb00b1c8SRobert Mustacchi icn->icn_dimm_type = IMC_DIMM_DDR4;
1246eb00b1c8SRobert Mustacchi break;
1247eb00b1c8SRobert Mustacchi }
1248eb00b1c8SRobert Mustacchi
1249eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE && icn->icn_m2m != NULL) {
1250eb00b1c8SRobert Mustacchi icn->icn_topo = pci_config_get32(icn->icn_m2m->istub_cfgspace,
1251eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_topo_offset);
1252eb00b1c8SRobert Mustacchi }
1253eb00b1c8SRobert Mustacchi
1254eb00b1c8SRobert Mustacchi return (B_TRUE);
1255eb00b1c8SRobert Mustacchi }
1256eb00b1c8SRobert Mustacchi
1257eb00b1c8SRobert Mustacchi /*
1258eb00b1c8SRobert Mustacchi * Walk the IMC data and fill in the information on DIMMs and the memory
1259eb00b1c8SRobert Mustacchi * controller configurations.
1260eb00b1c8SRobert Mustacchi */
1261eb00b1c8SRobert Mustacchi static void
imc_fill_data(imc_t * imc)1262eb00b1c8SRobert Mustacchi imc_fill_data(imc_t *imc)
1263eb00b1c8SRobert Mustacchi {
1264eb00b1c8SRobert Mustacchi uint_t csock, cmc, cchan;
1265eb00b1c8SRobert Mustacchi
1266eb00b1c8SRobert Mustacchi for (csock = 0; csock < imc->imc_nsockets; csock++) {
1267eb00b1c8SRobert Mustacchi imc_socket_t *sock = &imc->imc_sockets[csock];
1268eb00b1c8SRobert Mustacchi
1269eb00b1c8SRobert Mustacchi for (cmc = 0; cmc < sock->isock_nimc; cmc++) {
1270eb00b1c8SRobert Mustacchi imc_mc_t *icn = &sock->isock_imcs[cmc];
1271eb00b1c8SRobert Mustacchi
1272eb00b1c8SRobert Mustacchi if (!imc_fill_controller(imc, icn))
1273eb00b1c8SRobert Mustacchi continue;
1274eb00b1c8SRobert Mustacchi
1275eb00b1c8SRobert Mustacchi for (cchan = 0; cchan < icn->icn_nchannels; cchan++) {
1276eb00b1c8SRobert Mustacchi imc_fill_dimms(imc, icn,
1277eb00b1c8SRobert Mustacchi &icn->icn_channels[cchan]);
1278eb00b1c8SRobert Mustacchi }
1279eb00b1c8SRobert Mustacchi }
1280eb00b1c8SRobert Mustacchi }
1281eb00b1c8SRobert Mustacchi }
1282eb00b1c8SRobert Mustacchi
1283eb00b1c8SRobert Mustacchi static nvlist_t *
imc_nvl_create_dimm(imc_t * imc,imc_dimm_t * dimm)1284eb00b1c8SRobert Mustacchi imc_nvl_create_dimm(imc_t *imc, imc_dimm_t *dimm)
1285eb00b1c8SRobert Mustacchi {
1286eb00b1c8SRobert Mustacchi nvlist_t *nvl;
1287eb00b1c8SRobert Mustacchi
1288eb00b1c8SRobert Mustacchi nvl = fnvlist_alloc();
1289eb00b1c8SRobert Mustacchi fnvlist_add_boolean_value(nvl, MCINTEL_NVLIST_V1_DIMM_PRESENT,
1290eb00b1c8SRobert Mustacchi dimm->idimm_present);
1291eb00b1c8SRobert Mustacchi if (!dimm->idimm_present) {
1292eb00b1c8SRobert Mustacchi return (nvl);
1293eb00b1c8SRobert Mustacchi }
1294eb00b1c8SRobert Mustacchi
1295eb00b1c8SRobert Mustacchi fnvlist_add_uint64(nvl, MCINTEL_NVLIST_V1_DIMM_SIZE, dimm->idimm_size);
1296eb00b1c8SRobert Mustacchi fnvlist_add_uint32(nvl, MCINTEL_NVLIST_V1_DIMM_NCOLS,
1297eb00b1c8SRobert Mustacchi dimm->idimm_ncolumns);
1298eb00b1c8SRobert Mustacchi fnvlist_add_uint32(nvl, MCINTEL_NVLIST_V1_DIMM_NROWS,
1299eb00b1c8SRobert Mustacchi dimm->idimm_nrows);
1300eb00b1c8SRobert Mustacchi
1301eb00b1c8SRobert Mustacchi if (imc->imc_gen > IMC_GEN_SANDY) {
1302eb00b1c8SRobert Mustacchi fnvlist_add_uint64(nvl, MCINTEL_NVLIST_V1_DIMM_DENSITY,
1303eb00b1c8SRobert Mustacchi dimm->idimm_density * (1ULL << 30));
1304eb00b1c8SRobert Mustacchi fnvlist_add_uint32(nvl, MCINTEL_NVLIST_V1_DIMM_WIDTH,
1305eb00b1c8SRobert Mustacchi dimm->idimm_width);
1306eb00b1c8SRobert Mustacchi }
1307eb00b1c8SRobert Mustacchi fnvlist_add_uint32(nvl, MCINTEL_NVLIST_V1_DIMM_RANKS,
1308eb00b1c8SRobert Mustacchi dimm->idimm_nranks);
1309eb00b1c8SRobert Mustacchi fnvlist_add_uint32(nvl, MCINTEL_NVLIST_V1_DIMM_BANKS,
1310eb00b1c8SRobert Mustacchi dimm->idimm_nbanks);
1311eb00b1c8SRobert Mustacchi fnvlist_add_boolean_array(nvl, MCINTEL_NVLIST_V1_DIMM_RDIS,
1312eb00b1c8SRobert Mustacchi dimm->idimm_ranks_disabled, IMC_MAX_RANK_DISABLE);
1313eb00b1c8SRobert Mustacchi
1314eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_HASWELL) {
1315eb00b1c8SRobert Mustacchi fnvlist_add_boolean_value(nvl, MCINTEL_NVLIST_V1_DIMM_HDRL,
1316eb00b1c8SRobert Mustacchi dimm->idimm_hdrl);
1317eb00b1c8SRobert Mustacchi fnvlist_add_boolean_value(nvl, MCINTEL_NVLIST_V1_DIMM_HDRLP,
1318eb00b1c8SRobert Mustacchi dimm->idimm_hdrl_parity);
1319eb00b1c8SRobert Mustacchi if (dimm->idimm_3dsranks > 0) {
1320eb00b1c8SRobert Mustacchi fnvlist_add_uint32(nvl, MCINTEL_NVLIST_V1_DIMM_3DRANK,
1321eb00b1c8SRobert Mustacchi dimm->idimm_3dsranks);
1322eb00b1c8SRobert Mustacchi }
1323eb00b1c8SRobert Mustacchi }
1324eb00b1c8SRobert Mustacchi
1325eb00b1c8SRobert Mustacchi return (nvl);
1326eb00b1c8SRobert Mustacchi }
1327eb00b1c8SRobert Mustacchi
1328eb00b1c8SRobert Mustacchi static nvlist_t *
imc_nvl_create_channel(imc_t * imc,imc_channel_t * chan)1329eb00b1c8SRobert Mustacchi imc_nvl_create_channel(imc_t *imc, imc_channel_t *chan)
1330eb00b1c8SRobert Mustacchi {
1331eb00b1c8SRobert Mustacchi nvlist_t *nvl;
1332eb00b1c8SRobert Mustacchi nvlist_t *dimms[IMC_MAX_DIMMPERCHAN];
1333eb00b1c8SRobert Mustacchi uint_t i;
1334eb00b1c8SRobert Mustacchi
1335eb00b1c8SRobert Mustacchi nvl = fnvlist_alloc();
1336eb00b1c8SRobert Mustacchi fnvlist_add_uint32(nvl, MCINTEL_NVLIST_V1_CHAN_NDPC,
1337eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_max_dimms);
1338eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_gen_data->igd_max_dimms; i++) {
1339eb00b1c8SRobert Mustacchi dimms[i] = imc_nvl_create_dimm(imc, &chan->ich_dimms[i]);
1340eb00b1c8SRobert Mustacchi }
1341eb00b1c8SRobert Mustacchi
1342eb00b1c8SRobert Mustacchi fnvlist_add_nvlist_array(nvl, MCINTEL_NVLIST_V1_CHAN_DIMMS,
1343eb00b1c8SRobert Mustacchi dimms, i);
1344eb00b1c8SRobert Mustacchi
1345eb00b1c8SRobert Mustacchi for (; i > 0; i--) {
1346eb00b1c8SRobert Mustacchi nvlist_free(dimms[i-1]);
1347eb00b1c8SRobert Mustacchi }
1348eb00b1c8SRobert Mustacchi
1349eb00b1c8SRobert Mustacchi return (nvl);
1350eb00b1c8SRobert Mustacchi }
1351eb00b1c8SRobert Mustacchi
1352eb00b1c8SRobert Mustacchi static nvlist_t *
imc_nvl_create_mc(imc_t * imc,imc_mc_t * icn)1353eb00b1c8SRobert Mustacchi imc_nvl_create_mc(imc_t *imc, imc_mc_t *icn)
1354eb00b1c8SRobert Mustacchi {
1355eb00b1c8SRobert Mustacchi nvlist_t *nvl;
1356eb00b1c8SRobert Mustacchi nvlist_t *channels[IMC_MAX_CHANPERMC];
1357eb00b1c8SRobert Mustacchi uint_t i;
1358eb00b1c8SRobert Mustacchi
1359eb00b1c8SRobert Mustacchi nvl = fnvlist_alloc();
1360eb00b1c8SRobert Mustacchi fnvlist_add_uint32(nvl, MCINTEL_NVLIST_V1_MC_NCHAN, icn->icn_nchannels);
1361eb00b1c8SRobert Mustacchi fnvlist_add_boolean_value(nvl, MCINTEL_NVLIST_V1_MC_ECC,
1362eb00b1c8SRobert Mustacchi icn->icn_ecc);
1363eb00b1c8SRobert Mustacchi if (icn->icn_lockstep) {
1364eb00b1c8SRobert Mustacchi fnvlist_add_string(nvl, MCINTEL_NVLIST_V1_MC_CHAN_MODE,
1365eb00b1c8SRobert Mustacchi MCINTEL_NVLIST_V1_MC_CHAN_MODE_LOCK);
1366eb00b1c8SRobert Mustacchi } else {
1367eb00b1c8SRobert Mustacchi fnvlist_add_string(nvl, MCINTEL_NVLIST_V1_MC_CHAN_MODE,
1368eb00b1c8SRobert Mustacchi MCINTEL_NVLIST_V1_MC_CHAN_MODE_INDEP);
1369eb00b1c8SRobert Mustacchi
1370eb00b1c8SRobert Mustacchi }
1371eb00b1c8SRobert Mustacchi
1372eb00b1c8SRobert Mustacchi if (icn->icn_closed) {
1373eb00b1c8SRobert Mustacchi fnvlist_add_string(nvl, MCINTEL_NVLIST_V1_MC_POLICY,
1374eb00b1c8SRobert Mustacchi MCINTEL_NVLIST_V1_MC_POLICY_CLOSED);
1375eb00b1c8SRobert Mustacchi } else {
1376eb00b1c8SRobert Mustacchi fnvlist_add_string(nvl, MCINTEL_NVLIST_V1_MC_POLICY,
1377eb00b1c8SRobert Mustacchi MCINTEL_NVLIST_V1_MC_POLICY_OPEN);
1378eb00b1c8SRobert Mustacchi }
1379eb00b1c8SRobert Mustacchi
1380eb00b1c8SRobert Mustacchi for (i = 0; i < icn->icn_nchannels; i++) {
1381eb00b1c8SRobert Mustacchi channels[i] = imc_nvl_create_channel(imc,
1382eb00b1c8SRobert Mustacchi &icn->icn_channels[i]);
1383eb00b1c8SRobert Mustacchi }
1384eb00b1c8SRobert Mustacchi fnvlist_add_nvlist_array(nvl, MCINTEL_NVLIST_V1_MC_CHANNELS,
1385eb00b1c8SRobert Mustacchi channels, icn->icn_nchannels);
1386eb00b1c8SRobert Mustacchi for (i = 0; i < icn->icn_nchannels; i++) {
1387eb00b1c8SRobert Mustacchi nvlist_free(channels[i]);
1388eb00b1c8SRobert Mustacchi }
1389eb00b1c8SRobert Mustacchi
1390eb00b1c8SRobert Mustacchi return (nvl);
1391eb00b1c8SRobert Mustacchi }
1392eb00b1c8SRobert Mustacchi
1393eb00b1c8SRobert Mustacchi static void
imc_nvl_pack(imc_socket_t * sock,boolean_t sleep)1394eb00b1c8SRobert Mustacchi imc_nvl_pack(imc_socket_t *sock, boolean_t sleep)
1395eb00b1c8SRobert Mustacchi {
1396eb00b1c8SRobert Mustacchi char *buf = NULL;
1397eb00b1c8SRobert Mustacchi size_t len = 0;
1398eb00b1c8SRobert Mustacchi int kmflag;
1399eb00b1c8SRobert Mustacchi
1400eb00b1c8SRobert Mustacchi if (sock->isock_nvl == NULL)
1401eb00b1c8SRobert Mustacchi return;
1402eb00b1c8SRobert Mustacchi
1403eb00b1c8SRobert Mustacchi if (sock->isock_buf != NULL)
1404eb00b1c8SRobert Mustacchi return;
1405eb00b1c8SRobert Mustacchi
1406eb00b1c8SRobert Mustacchi if (sleep) {
1407eb00b1c8SRobert Mustacchi kmflag = KM_SLEEP;
1408eb00b1c8SRobert Mustacchi } else {
1409ca783257SDan McDonald kmflag = KM_NOSLEEP_LAZY;
1410eb00b1c8SRobert Mustacchi }
1411eb00b1c8SRobert Mustacchi
1412eb00b1c8SRobert Mustacchi if (nvlist_pack(sock->isock_nvl, &buf, &len, NV_ENCODE_XDR,
1413eb00b1c8SRobert Mustacchi kmflag) != 0) {
1414eb00b1c8SRobert Mustacchi return;
1415eb00b1c8SRobert Mustacchi }
1416eb00b1c8SRobert Mustacchi
1417eb00b1c8SRobert Mustacchi sock->isock_buf = buf;
1418eb00b1c8SRobert Mustacchi sock->isock_buflen = len;
1419eb00b1c8SRobert Mustacchi sock->isock_gen++;
1420eb00b1c8SRobert Mustacchi }
1421eb00b1c8SRobert Mustacchi
1422eb00b1c8SRobert Mustacchi static void
imc_decoder_pack(imc_t * imc)1423eb00b1c8SRobert Mustacchi imc_decoder_pack(imc_t *imc)
1424eb00b1c8SRobert Mustacchi {
1425eb00b1c8SRobert Mustacchi char *buf = NULL;
1426eb00b1c8SRobert Mustacchi size_t len = 0;
1427eb00b1c8SRobert Mustacchi
1428eb00b1c8SRobert Mustacchi if (imc->imc_decoder_buf != NULL)
1429eb00b1c8SRobert Mustacchi return;
1430eb00b1c8SRobert Mustacchi
1431eb00b1c8SRobert Mustacchi if (imc->imc_decoder_dump == NULL) {
1432eb00b1c8SRobert Mustacchi imc->imc_decoder_dump = imc_dump_decoder(imc);
1433eb00b1c8SRobert Mustacchi }
1434eb00b1c8SRobert Mustacchi
1435eb00b1c8SRobert Mustacchi if (nvlist_pack(imc->imc_decoder_dump, &buf, &len, NV_ENCODE_XDR,
1436ca783257SDan McDonald KM_NOSLEEP_LAZY) != 0) {
1437eb00b1c8SRobert Mustacchi return;
1438eb00b1c8SRobert Mustacchi }
1439eb00b1c8SRobert Mustacchi
1440eb00b1c8SRobert Mustacchi imc->imc_decoder_buf = buf;
1441eb00b1c8SRobert Mustacchi imc->imc_decoder_len = len;
1442eb00b1c8SRobert Mustacchi }
1443eb00b1c8SRobert Mustacchi
1444eb00b1c8SRobert Mustacchi static void
imc_nvl_create(imc_t * imc)1445eb00b1c8SRobert Mustacchi imc_nvl_create(imc_t *imc)
1446eb00b1c8SRobert Mustacchi {
1447eb00b1c8SRobert Mustacchi uint_t csock;
1448eb00b1c8SRobert Mustacchi for (csock = 0; csock < imc->imc_nsockets; csock++) {
1449eb00b1c8SRobert Mustacchi uint_t i;
1450eb00b1c8SRobert Mustacchi nvlist_t *nvl;
1451eb00b1c8SRobert Mustacchi nvlist_t *mcs[IMC_MAX_IMCPERSOCK];
1452eb00b1c8SRobert Mustacchi imc_socket_t *sock = &imc->imc_sockets[csock];
1453eb00b1c8SRobert Mustacchi
1454eb00b1c8SRobert Mustacchi nvl = fnvlist_alloc();
1455eb00b1c8SRobert Mustacchi fnvlist_add_uint8(nvl, MCINTEL_NVLIST_VERSTR,
1456eb00b1c8SRobert Mustacchi MCINTEL_NVLIST_VERS1);
1457eb00b1c8SRobert Mustacchi fnvlist_add_uint8(nvl, MCINTEL_NVLIST_V1_NMC,
1458eb00b1c8SRobert Mustacchi sock->isock_nimc);
1459eb00b1c8SRobert Mustacchi
1460eb00b1c8SRobert Mustacchi for (i = 0; i < sock->isock_nimc; i++) {
1461eb00b1c8SRobert Mustacchi mcs[i] = imc_nvl_create_mc(imc, &sock->isock_imcs[i]);
1462eb00b1c8SRobert Mustacchi }
1463eb00b1c8SRobert Mustacchi
1464eb00b1c8SRobert Mustacchi fnvlist_add_nvlist_array(nvl, MCINTEL_NVLIST_V1_MCS,
1465eb00b1c8SRobert Mustacchi mcs, sock->isock_nimc);
1466eb00b1c8SRobert Mustacchi
1467eb00b1c8SRobert Mustacchi for (i = 0; i < sock->isock_nimc; i++) {
1468eb00b1c8SRobert Mustacchi nvlist_free(mcs[i]);
1469eb00b1c8SRobert Mustacchi }
1470eb00b1c8SRobert Mustacchi
1471eb00b1c8SRobert Mustacchi sock->isock_nvl = nvl;
1472eb00b1c8SRobert Mustacchi imc_nvl_pack(sock, B_TRUE);
1473eb00b1c8SRobert Mustacchi }
1474eb00b1c8SRobert Mustacchi }
1475eb00b1c8SRobert Mustacchi
1476eb00b1c8SRobert Mustacchi /*
1477eb00b1c8SRobert Mustacchi * Determine the top of low and high memory. These determine whether transaction
1478eb00b1c8SRobert Mustacchi * addresses target main memory or not. Unfortunately, the way that these are
1479eb00b1c8SRobert Mustacchi * stored and fetched changes with different generations.
1480eb00b1c8SRobert Mustacchi */
1481eb00b1c8SRobert Mustacchi static void
imc_sad_read_tohm(imc_t * imc,imc_sad_t * sad)1482eb00b1c8SRobert Mustacchi imc_sad_read_tohm(imc_t *imc, imc_sad_t *sad)
1483eb00b1c8SRobert Mustacchi {
1484eb00b1c8SRobert Mustacchi uint32_t tolm, tohm_low, tohm_hi;
1485eb00b1c8SRobert Mustacchi
1486eb00b1c8SRobert Mustacchi tolm = pci_config_get32(sad->isad_tolh->istub_cfgspace,
1487eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_tolm_offset);
1488eb00b1c8SRobert Mustacchi tohm_low = pci_config_get32(sad->isad_tolh->istub_cfgspace,
1489eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_tohm_low_offset);
1490eb00b1c8SRobert Mustacchi if (imc->imc_gen_data->igd_tohm_hi_offset != 0) {
1491eb00b1c8SRobert Mustacchi tohm_hi = pci_config_get32(sad->isad_tolh->istub_cfgspace,
1492eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_tohm_hi_offset);
1493eb00b1c8SRobert Mustacchi } else {
1494eb00b1c8SRobert Mustacchi tohm_hi = 0;
1495eb00b1c8SRobert Mustacchi }
1496eb00b1c8SRobert Mustacchi
1497eb00b1c8SRobert Mustacchi if (tolm == PCI_EINVAL32 || tohm_low == PCI_EINVAL32 ||
1498eb00b1c8SRobert Mustacchi tohm_hi == PCI_EINVAL32) {
1499eb00b1c8SRobert Mustacchi sad->isad_valid |= IMC_SAD_V_BAD_PCI_READ;
1500eb00b1c8SRobert Mustacchi return;
1501eb00b1c8SRobert Mustacchi }
1502eb00b1c8SRobert Mustacchi
1503eb00b1c8SRobert Mustacchi switch (imc->imc_gen) {
1504eb00b1c8SRobert Mustacchi case IMC_GEN_SANDY:
1505eb00b1c8SRobert Mustacchi case IMC_GEN_IVY:
1506eb00b1c8SRobert Mustacchi sad->isad_tolm = ((uint64_t)tolm & IMC_TOLM_SNB_IVY_MASK) <<
1507eb00b1c8SRobert Mustacchi IMC_TOLM_SNB_IVY_SHIFT;
1508eb00b1c8SRobert Mustacchi sad->isad_tohm = ((uint64_t)tohm_low & IMC_TOHM_SNB_IVY_MASK) <<
1509eb00b1c8SRobert Mustacchi IMC_TOLM_SNB_IVY_SHIFT;
1510eb00b1c8SRobert Mustacchi break;
1511eb00b1c8SRobert Mustacchi case IMC_GEN_HASWELL:
1512eb00b1c8SRobert Mustacchi case IMC_GEN_BROADWELL:
1513eb00b1c8SRobert Mustacchi case IMC_GEN_SKYLAKE:
1514eb00b1c8SRobert Mustacchi sad->isad_tolm = (uint64_t)tolm & IMC_TOLM_HAS_SKX_MASK;
1515eb00b1c8SRobert Mustacchi sad->isad_tohm = ((uint64_t)tohm_low &
1516eb00b1c8SRobert Mustacchi IMC_TOHM_LOW_HAS_SKX_MASK) | ((uint64_t)tohm_hi << 32);
1517eb00b1c8SRobert Mustacchi
1518eb00b1c8SRobert Mustacchi /*
1519eb00b1c8SRobert Mustacchi * Adjust the values to turn them into an exclusive range.
1520eb00b1c8SRobert Mustacchi */
1521eb00b1c8SRobert Mustacchi sad->isad_tolm += IMC_TOLM_HAS_SKY_EXCL;
1522eb00b1c8SRobert Mustacchi sad->isad_tohm += IMC_TOHM_HAS_SKY_EXCL;
1523eb00b1c8SRobert Mustacchi break;
1524eb00b1c8SRobert Mustacchi default:
1525eb00b1c8SRobert Mustacchi dev_err(imc->imc_dip, CE_PANIC, "imc driver programmer error: "
1526eb00b1c8SRobert Mustacchi "set to unknown generation: %u", imc->imc_gen);
1527eb00b1c8SRobert Mustacchi return;
1528eb00b1c8SRobert Mustacchi }
1529eb00b1c8SRobert Mustacchi }
1530eb00b1c8SRobert Mustacchi
1531eb00b1c8SRobert Mustacchi static void
imc_sad_fill_rule(imc_t * imc,imc_sad_t * sad,imc_sad_rule_t * rule,uint32_t raw)1532eb00b1c8SRobert Mustacchi imc_sad_fill_rule(imc_t *imc, imc_sad_t *sad, imc_sad_rule_t *rule,
1533eb00b1c8SRobert Mustacchi uint32_t raw)
1534eb00b1c8SRobert Mustacchi {
1535eb00b1c8SRobert Mustacchi uint_t attr;
1536eb00b1c8SRobert Mustacchi uint64_t limit;
1537eb00b1c8SRobert Mustacchi bzero(rule, sizeof (imc_sad_rule_t));
1538eb00b1c8SRobert Mustacchi
1539eb00b1c8SRobert Mustacchi rule->isr_raw_dram = raw;
1540eb00b1c8SRobert Mustacchi rule->isr_enable = IMC_SAD_DRAM_RULE_ENABLE(raw) != 0;
1541eb00b1c8SRobert Mustacchi if (imc->imc_gen < IMC_GEN_SKYLAKE) {
1542eb00b1c8SRobert Mustacchi switch (IMC_SAD_DRAM_INTERLEAVE_SNB_BRD(raw)) {
1543eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_INTERLEAVE_SNB_BRD_8t6:
1544eb00b1c8SRobert Mustacchi rule->isr_imode = IMC_SAD_IMODE_8t6;
1545eb00b1c8SRobert Mustacchi break;
1546eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_INTERLEAVE_SNB_BRD_8t6XOR:
1547eb00b1c8SRobert Mustacchi rule->isr_imode = IMC_SAD_IMODE_8t6XOR;
1548eb00b1c8SRobert Mustacchi break;
1549eb00b1c8SRobert Mustacchi }
1550eb00b1c8SRobert Mustacchi } else {
1551eb00b1c8SRobert Mustacchi switch (IMC_SAD_DRAM_INTERLEAVE_SKX(raw)) {
1552eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_INTERLEAVE_SKX_8t6:
1553eb00b1c8SRobert Mustacchi rule->isr_imode = IMC_SAD_IMODE_8t6;
1554eb00b1c8SRobert Mustacchi break;
1555eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_INTERLEAVE_SKX_10t8:
1556eb00b1c8SRobert Mustacchi rule->isr_imode = IMC_SAD_IMODE_10t8;
1557eb00b1c8SRobert Mustacchi break;
1558eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_INTERLEAVE_SKX_14t12:
1559eb00b1c8SRobert Mustacchi rule->isr_imode = IMC_SAD_IMODE_14t12;
1560eb00b1c8SRobert Mustacchi break;
1561eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_INTERLEAVE_SKX_32t30:
1562eb00b1c8SRobert Mustacchi rule->isr_imode = IMC_SAD_IMODE_32t30;
1563eb00b1c8SRobert Mustacchi break;
1564eb00b1c8SRobert Mustacchi }
1565eb00b1c8SRobert Mustacchi }
1566eb00b1c8SRobert Mustacchi
1567eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE) {
1568eb00b1c8SRobert Mustacchi attr = IMC_SAD_DRAM_ATTR_SKX(raw);
1569eb00b1c8SRobert Mustacchi } else {
1570eb00b1c8SRobert Mustacchi attr = IMC_SAD_DRAM_ATTR_SNB_BRD(raw);
1571eb00b1c8SRobert Mustacchi }
1572eb00b1c8SRobert Mustacchi
1573eb00b1c8SRobert Mustacchi switch (attr) {
1574eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_ATTR_DRAM:
1575eb00b1c8SRobert Mustacchi rule->isr_type = IMC_SAD_TYPE_DRAM;
1576eb00b1c8SRobert Mustacchi break;
1577eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_ATTR_MMCFG:
1578eb00b1c8SRobert Mustacchi rule->isr_type = IMC_SAD_TYPE_MMCFG;
1579eb00b1c8SRobert Mustacchi break;
1580eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_ATTR_NXM:
1581eb00b1c8SRobert Mustacchi if (imc->imc_gen < IMC_GEN_SKYLAKE) {
1582eb00b1c8SRobert Mustacchi sad->isad_valid |= IMC_SAD_V_BAD_DRAM_ATTR;
1583eb00b1c8SRobert Mustacchi }
1584eb00b1c8SRobert Mustacchi rule->isr_type = IMC_SAD_TYPE_NXM;
1585eb00b1c8SRobert Mustacchi break;
1586eb00b1c8SRobert Mustacchi default:
1587eb00b1c8SRobert Mustacchi sad->isad_valid |= IMC_SAD_V_BAD_DRAM_ATTR;
1588eb00b1c8SRobert Mustacchi break;
1589eb00b1c8SRobert Mustacchi }
1590eb00b1c8SRobert Mustacchi
1591eb00b1c8SRobert Mustacchi /*
1592eb00b1c8SRobert Mustacchi * Fetch the limit which represents bits 45:26 and then adjust this so
1593eb00b1c8SRobert Mustacchi * that it is exclusive.
1594eb00b1c8SRobert Mustacchi */
1595eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE) {
1596eb00b1c8SRobert Mustacchi limit = IMC_SAD_DRAM_LIMIT_SKX(raw);
1597eb00b1c8SRobert Mustacchi } else {
1598eb00b1c8SRobert Mustacchi limit = IMC_SAD_DRAM_LIMIT_SNB_BRD(raw);
1599eb00b1c8SRobert Mustacchi }
1600eb00b1c8SRobert Mustacchi rule->isr_limit = (limit << IMC_SAD_DRAM_LIMIT_SHIFT) +
1601eb00b1c8SRobert Mustacchi IMC_SAD_DRAM_LIMIT_EXCLUSIVE;
1602eb00b1c8SRobert Mustacchi
1603eb00b1c8SRobert Mustacchi /*
1604eb00b1c8SRobert Mustacchi * The rest of this does not apply to Sandy Bridge.
1605eb00b1c8SRobert Mustacchi */
1606eb00b1c8SRobert Mustacchi if (imc->imc_gen == IMC_GEN_SANDY)
1607eb00b1c8SRobert Mustacchi return;
1608eb00b1c8SRobert Mustacchi
1609eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_IVY && imc->imc_gen < IMC_GEN_SKYLAKE) {
1610eb00b1c8SRobert Mustacchi rule->isr_a7mode = IMC_SAD_DRAM_A7_IVB_BRD(raw) != 0;
1611eb00b1c8SRobert Mustacchi return;
1612eb00b1c8SRobert Mustacchi }
1613eb00b1c8SRobert Mustacchi
1614eb00b1c8SRobert Mustacchi switch (IMC_SAD_DRAM_MOD23_SKX(raw)) {
1615eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_MOD23_MOD3:
1616eb00b1c8SRobert Mustacchi rule->isr_mod_type = IMC_SAD_MOD_TYPE_MOD3;
1617eb00b1c8SRobert Mustacchi break;
1618eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_MOD23_MOD2_C01:
1619eb00b1c8SRobert Mustacchi rule->isr_mod_type = IMC_SAD_MOD_TYPE_MOD2_01;
1620eb00b1c8SRobert Mustacchi break;
1621eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_MOD23_MOD2_C12:
1622eb00b1c8SRobert Mustacchi rule->isr_mod_type = IMC_SAD_MOD_TYPE_MOD2_12;
1623eb00b1c8SRobert Mustacchi break;
1624eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_MOD23_MOD2_C02:
1625eb00b1c8SRobert Mustacchi rule->isr_mod_type = IMC_SAD_MOD_TYPE_MOD2_02;
1626eb00b1c8SRobert Mustacchi break;
1627eb00b1c8SRobert Mustacchi }
1628eb00b1c8SRobert Mustacchi
1629eb00b1c8SRobert Mustacchi rule->isr_need_mod3 = IMC_SAD_DRAM_MOD3_SKX(raw) != 0;
1630eb00b1c8SRobert Mustacchi switch (IMC_SAD_DRAM_MOD3_SKX(raw)) {
1631eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_MOD3_MODE_45t6:
1632eb00b1c8SRobert Mustacchi rule->isr_mod_mode = IMC_SAD_MOD_MODE_45t6;
1633eb00b1c8SRobert Mustacchi break;
1634eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_MOD3_MODE_45t8:
1635eb00b1c8SRobert Mustacchi rule->isr_mod_mode = IMC_SAD_MOD_MODE_45t8;
1636eb00b1c8SRobert Mustacchi break;
1637eb00b1c8SRobert Mustacchi case IMC_SAD_DRAM_MOD3_MODE_45t12:
1638eb00b1c8SRobert Mustacchi rule->isr_mod_mode = IMC_SAD_MOD_MODE_45t12;
1639eb00b1c8SRobert Mustacchi break;
1640eb00b1c8SRobert Mustacchi default:
1641eb00b1c8SRobert Mustacchi sad->isad_valid |= IMC_SAD_V_BAD_MOD3;
1642eb00b1c8SRobert Mustacchi break;
1643eb00b1c8SRobert Mustacchi }
1644eb00b1c8SRobert Mustacchi }
1645eb00b1c8SRobert Mustacchi
1646eb00b1c8SRobert Mustacchi static void
imc_sad_fill_rule_interleave(imc_t * imc,imc_sad_rule_t * rule,uint32_t raw)1647eb00b1c8SRobert Mustacchi imc_sad_fill_rule_interleave(imc_t *imc, imc_sad_rule_t *rule, uint32_t raw)
1648eb00b1c8SRobert Mustacchi {
1649eb00b1c8SRobert Mustacchi uint_t i;
1650eb00b1c8SRobert Mustacchi uint32_t mlen, mbase, skipbits, skipafter;
1651eb00b1c8SRobert Mustacchi
1652eb00b1c8SRobert Mustacchi rule->isr_raw_interleave = raw;
1653eb00b1c8SRobert Mustacchi
1654eb00b1c8SRobert Mustacchi /*
1655eb00b1c8SRobert Mustacchi * Right now all architectures always have the maximum number of SAD
1656eb00b1c8SRobert Mustacchi * interleave targets.
1657eb00b1c8SRobert Mustacchi */
1658eb00b1c8SRobert Mustacchi rule->isr_ntargets = IMC_MAX_SAD_INTERLEAVE;
1659eb00b1c8SRobert Mustacchi
1660eb00b1c8SRobert Mustacchi /*
1661eb00b1c8SRobert Mustacchi * Sandy Bridge has a gap in the interleave list due to the fact that it
1662eb00b1c8SRobert Mustacchi * uses a smaller length.
1663eb00b1c8SRobert Mustacchi */
1664eb00b1c8SRobert Mustacchi if (imc->imc_gen > IMC_GEN_SANDY) {
1665eb00b1c8SRobert Mustacchi mlen = IMC_SAD_ILEAVE_IVB_SKX_LEN;
1666eb00b1c8SRobert Mustacchi mbase = IMC_SAD_ILEAVE_IVB_SKX_MASK;
1667eb00b1c8SRobert Mustacchi skipbits = skipafter = 0;
1668eb00b1c8SRobert Mustacchi } else {
1669eb00b1c8SRobert Mustacchi mlen = IMC_SAD_ILEAVE_SNB_LEN;
1670eb00b1c8SRobert Mustacchi mbase = IMC_SAD_ILEAVE_SNB_MASK;
1671eb00b1c8SRobert Mustacchi skipbits = 2;
1672eb00b1c8SRobert Mustacchi skipafter = 4;
1673eb00b1c8SRobert Mustacchi }
1674eb00b1c8SRobert Mustacchi
1675eb00b1c8SRobert Mustacchi for (i = 0; i < rule->isr_ntargets; i++) {
1676eb00b1c8SRobert Mustacchi uint32_t mask, shift;
1677eb00b1c8SRobert Mustacchi
1678eb00b1c8SRobert Mustacchi shift = i * mlen;
1679eb00b1c8SRobert Mustacchi if (i >= skipafter)
1680eb00b1c8SRobert Mustacchi shift += skipbits;
1681eb00b1c8SRobert Mustacchi mask = mbase << shift;
1682eb00b1c8SRobert Mustacchi rule->isr_targets[i] = (raw & mask) >> shift;
1683eb00b1c8SRobert Mustacchi }
1684eb00b1c8SRobert Mustacchi }
1685eb00b1c8SRobert Mustacchi
1686eb00b1c8SRobert Mustacchi static void
imc_sad_read_dram_rules(imc_t * imc,imc_sad_t * sad)1687eb00b1c8SRobert Mustacchi imc_sad_read_dram_rules(imc_t *imc, imc_sad_t *sad)
1688eb00b1c8SRobert Mustacchi {
1689eb00b1c8SRobert Mustacchi uint_t i;
1690eb00b1c8SRobert Mustacchi off_t off;
1691eb00b1c8SRobert Mustacchi
1692eb00b1c8SRobert Mustacchi sad->isad_nrules = imc->imc_gen_data->igd_sad_ndram_rules;
1693eb00b1c8SRobert Mustacchi for (i = 0, off = imc->imc_gen_data->igd_sad_dram_offset;
1694eb00b1c8SRobert Mustacchi i < sad->isad_nrules; i++, off += sizeof (uint64_t)) {
1695eb00b1c8SRobert Mustacchi uint32_t dram, interleave;
1696eb00b1c8SRobert Mustacchi imc_sad_rule_t *rule = &sad->isad_rules[i];
1697eb00b1c8SRobert Mustacchi
1698eb00b1c8SRobert Mustacchi dram = pci_config_get32(sad->isad_dram->istub_cfgspace, off);
1699eb00b1c8SRobert Mustacchi interleave = pci_config_get32(sad->isad_dram->istub_cfgspace,
1700eb00b1c8SRobert Mustacchi off + 4);
1701eb00b1c8SRobert Mustacchi
1702eb00b1c8SRobert Mustacchi if (dram == PCI_EINVAL32 || interleave == PCI_EINVAL32) {
1703eb00b1c8SRobert Mustacchi sad->isad_valid |= IMC_SAD_V_BAD_PCI_READ;
1704eb00b1c8SRobert Mustacchi return;
1705eb00b1c8SRobert Mustacchi }
1706eb00b1c8SRobert Mustacchi
1707eb00b1c8SRobert Mustacchi imc_sad_fill_rule(imc, sad, rule, dram);
1708eb00b1c8SRobert Mustacchi imc_sad_fill_rule_interleave(imc, rule, interleave);
1709eb00b1c8SRobert Mustacchi }
1710eb00b1c8SRobert Mustacchi }
1711eb00b1c8SRobert Mustacchi
1712eb00b1c8SRobert Mustacchi static void
imc_sad_decode_mcroute(imc_t * imc,imc_sad_t * sad)1713eb00b1c8SRobert Mustacchi imc_sad_decode_mcroute(imc_t *imc, imc_sad_t *sad)
1714eb00b1c8SRobert Mustacchi {
1715eb00b1c8SRobert Mustacchi uint_t i;
1716eb00b1c8SRobert Mustacchi imc_sad_mcroute_table_t *mc = &sad->isad_mcroute;
1717eb00b1c8SRobert Mustacchi
1718eb00b1c8SRobert Mustacchi if (imc->imc_gen < IMC_GEN_SKYLAKE)
1719eb00b1c8SRobert Mustacchi return;
1720eb00b1c8SRobert Mustacchi if (sad->isad_valid != 0)
1721eb00b1c8SRobert Mustacchi return;
1722eb00b1c8SRobert Mustacchi
1723eb00b1c8SRobert Mustacchi mc->ismc_nroutes = IMC_MAX_SAD_MCROUTES;
1724eb00b1c8SRobert Mustacchi for (i = 0; i < IMC_MAX_SAD_MCROUTES; i++) {
1725eb00b1c8SRobert Mustacchi uint_t chanoff, ringoff;
1726eb00b1c8SRobert Mustacchi
1727eb00b1c8SRobert Mustacchi ringoff = i * IMC_MC_ROUTE_RING_BITS;
1728eb00b1c8SRobert Mustacchi chanoff = i * IMC_MC_ROUTE_CHAN_BITS + IMC_MC_ROUTE_CHAN_OFFSET;
1729eb00b1c8SRobert Mustacchi
1730eb00b1c8SRobert Mustacchi mc->ismc_mcroutes[i].ismce_imc = (mc->ismc_raw_mcroute >>
1731eb00b1c8SRobert Mustacchi ringoff) & IMC_MC_ROUTE_RING_MASK;
1732eb00b1c8SRobert Mustacchi mc->ismc_mcroutes[i].ismce_pchannel = (mc->ismc_raw_mcroute >>
1733eb00b1c8SRobert Mustacchi chanoff) & IMC_MC_ROUTE_CHAN_MASK;
1734eb00b1c8SRobert Mustacchi }
1735eb00b1c8SRobert Mustacchi }
1736eb00b1c8SRobert Mustacchi
1737eb00b1c8SRobert Mustacchi /*
1738eb00b1c8SRobert Mustacchi * Initialize the SAD. To do this we have to do a few different things:
1739eb00b1c8SRobert Mustacchi *
1740eb00b1c8SRobert Mustacchi * 1. Determine where the top of low and high memory is.
1741eb00b1c8SRobert Mustacchi * 2. Read and decode all of the rules for the SAD
1742eb00b1c8SRobert Mustacchi * 3. On systems with a route table, decode the raw routes
1743eb00b1c8SRobert Mustacchi *
1744eb00b1c8SRobert Mustacchi * At this point in time, we treat TOLM and TOHM as a per-socket construct, even
1745eb00b1c8SRobert Mustacchi * though it really should be global, this just makes life a bit simpler.
1746eb00b1c8SRobert Mustacchi */
1747eb00b1c8SRobert Mustacchi static void
imc_decoder_init_sad(imc_t * imc)1748eb00b1c8SRobert Mustacchi imc_decoder_init_sad(imc_t *imc)
1749eb00b1c8SRobert Mustacchi {
1750eb00b1c8SRobert Mustacchi uint_t i;
1751eb00b1c8SRobert Mustacchi
1752eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
1753eb00b1c8SRobert Mustacchi imc_sad_read_tohm(imc, &imc->imc_sockets[i].isock_sad);
1754eb00b1c8SRobert Mustacchi imc_sad_read_dram_rules(imc, &imc->imc_sockets[i].isock_sad);
1755eb00b1c8SRobert Mustacchi imc_sad_decode_mcroute(imc, &imc->imc_sockets[i].isock_sad);
1756eb00b1c8SRobert Mustacchi }
1757eb00b1c8SRobert Mustacchi }
1758eb00b1c8SRobert Mustacchi
1759eb00b1c8SRobert Mustacchi static void
imc_tad_fill_rule(imc_t * imc,imc_tad_t * tad,imc_tad_rule_t * prev,imc_tad_rule_t * rule,uint32_t val)1760eb00b1c8SRobert Mustacchi imc_tad_fill_rule(imc_t *imc, imc_tad_t *tad, imc_tad_rule_t *prev,
1761eb00b1c8SRobert Mustacchi imc_tad_rule_t *rule, uint32_t val)
1762eb00b1c8SRobert Mustacchi {
1763eb00b1c8SRobert Mustacchi uint64_t limit;
1764eb00b1c8SRobert Mustacchi
1765eb00b1c8SRobert Mustacchi limit = IMC_TAD_LIMIT(val);
1766eb00b1c8SRobert Mustacchi rule->itr_limit = (limit << IMC_TAD_LIMIT_SHIFT) +
1767eb00b1c8SRobert Mustacchi IMC_TAD_LIMIT_EXCLUSIVE;
1768eb00b1c8SRobert Mustacchi rule->itr_raw = val;
1769eb00b1c8SRobert Mustacchi
1770eb00b1c8SRobert Mustacchi switch (IMC_TAD_SOCK_WAY(val)) {
1771eb00b1c8SRobert Mustacchi case IMC_TAD_SOCK_WAY_1:
1772eb00b1c8SRobert Mustacchi rule->itr_sock_way = 1;
1773eb00b1c8SRobert Mustacchi break;
1774eb00b1c8SRobert Mustacchi case IMC_TAD_SOCK_WAY_2:
1775eb00b1c8SRobert Mustacchi rule->itr_sock_way = 2;
1776eb00b1c8SRobert Mustacchi break;
1777eb00b1c8SRobert Mustacchi case IMC_TAD_SOCK_WAY_4:
1778eb00b1c8SRobert Mustacchi rule->itr_sock_way = 4;
1779eb00b1c8SRobert Mustacchi break;
1780eb00b1c8SRobert Mustacchi case IMC_TAD_SOCK_WAY_8:
1781eb00b1c8SRobert Mustacchi rule->itr_sock_way = 8;
1782eb00b1c8SRobert Mustacchi break;
1783eb00b1c8SRobert Mustacchi }
1784eb00b1c8SRobert Mustacchi
1785eb00b1c8SRobert Mustacchi rule->itr_chan_way = IMC_TAD_CHAN_WAY(val) + 1;
1786eb00b1c8SRobert Mustacchi rule->itr_sock_gran = IMC_TAD_GRAN_64B;
1787eb00b1c8SRobert Mustacchi rule->itr_chan_gran = IMC_TAD_GRAN_64B;
1788eb00b1c8SRobert Mustacchi
1789eb00b1c8SRobert Mustacchi /*
1790eb00b1c8SRobert Mustacchi * Starting with Skylake the targets that are used are no longer part of
1791eb00b1c8SRobert Mustacchi * the TAD. Those come from the IMC route table.
1792eb00b1c8SRobert Mustacchi */
1793eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE) {
1794eb00b1c8SRobert Mustacchi rule->itr_ntargets = 0;
1795eb00b1c8SRobert Mustacchi return;
1796eb00b1c8SRobert Mustacchi }
1797eb00b1c8SRobert Mustacchi
1798eb00b1c8SRobert Mustacchi rule->itr_ntargets = IMC_TAD_SNB_BRD_NTARGETS;
1799eb00b1c8SRobert Mustacchi rule->itr_targets[0] = IMC_TAD_TARG0(val);
1800eb00b1c8SRobert Mustacchi rule->itr_targets[1] = IMC_TAD_TARG1(val);
1801eb00b1c8SRobert Mustacchi rule->itr_targets[2] = IMC_TAD_TARG2(val);
1802eb00b1c8SRobert Mustacchi rule->itr_targets[3] = IMC_TAD_TARG3(val);
1803eb00b1c8SRobert Mustacchi
1804eb00b1c8SRobert Mustacchi if (prev == NULL) {
1805eb00b1c8SRobert Mustacchi rule->itr_base = 0;
1806eb00b1c8SRobert Mustacchi } else {
1807eb00b1c8SRobert Mustacchi rule->itr_base = prev->itr_limit + 1;
1808eb00b1c8SRobert Mustacchi }
1809eb00b1c8SRobert Mustacchi }
1810eb00b1c8SRobert Mustacchi
1811eb00b1c8SRobert Mustacchi static void
imc_tad_fill_skx(imc_t * imc,imc_tad_t * tad,imc_tad_rule_t * rule,uint32_t val)1812eb00b1c8SRobert Mustacchi imc_tad_fill_skx(imc_t *imc, imc_tad_t *tad, imc_tad_rule_t *rule,
1813eb00b1c8SRobert Mustacchi uint32_t val)
1814eb00b1c8SRobert Mustacchi {
1815eb00b1c8SRobert Mustacchi uint64_t base;
1816eb00b1c8SRobert Mustacchi
1817eb00b1c8SRobert Mustacchi rule->itr_raw_gran = val;
1818eb00b1c8SRobert Mustacchi base = IMC_TAD_BASE_BASE(val);
1819eb00b1c8SRobert Mustacchi rule->itr_base = base << IMC_TAD_BASE_SHIFT;
1820eb00b1c8SRobert Mustacchi
1821eb00b1c8SRobert Mustacchi switch (IMC_TAD_BASE_CHAN_GRAN(val)) {
1822eb00b1c8SRobert Mustacchi case IMC_TAD_BASE_CHAN_GRAN_64B:
1823eb00b1c8SRobert Mustacchi rule->itr_sock_gran = IMC_TAD_GRAN_64B;
1824eb00b1c8SRobert Mustacchi break;
1825eb00b1c8SRobert Mustacchi case IMC_TAD_BASE_CHAN_GRAN_256B:
1826eb00b1c8SRobert Mustacchi rule->itr_sock_gran = IMC_TAD_GRAN_256B;
1827eb00b1c8SRobert Mustacchi break;
1828eb00b1c8SRobert Mustacchi case IMC_TAD_BASE_CHAN_GRAN_4KB:
1829eb00b1c8SRobert Mustacchi rule->itr_sock_gran = IMC_TAD_GRAN_4KB;
1830eb00b1c8SRobert Mustacchi break;
1831eb00b1c8SRobert Mustacchi default:
1832eb00b1c8SRobert Mustacchi tad->itad_valid |= IMC_TAD_V_BAD_CHAN_GRAN;
1833eb00b1c8SRobert Mustacchi return;
1834eb00b1c8SRobert Mustacchi }
1835eb00b1c8SRobert Mustacchi
1836eb00b1c8SRobert Mustacchi switch (IMC_TAD_BASE_SOCK_GRAN(val)) {
1837eb00b1c8SRobert Mustacchi case IMC_TAD_BASE_SOCK_GRAN_64B:
1838eb00b1c8SRobert Mustacchi rule->itr_sock_gran = IMC_TAD_GRAN_64B;
1839eb00b1c8SRobert Mustacchi break;
1840eb00b1c8SRobert Mustacchi case IMC_TAD_BASE_SOCK_GRAN_256B:
1841eb00b1c8SRobert Mustacchi rule->itr_sock_gran = IMC_TAD_GRAN_256B;
1842eb00b1c8SRobert Mustacchi break;
1843eb00b1c8SRobert Mustacchi case IMC_TAD_BASE_SOCK_GRAN_4KB:
1844eb00b1c8SRobert Mustacchi rule->itr_sock_gran = IMC_TAD_GRAN_4KB;
1845eb00b1c8SRobert Mustacchi break;
1846eb00b1c8SRobert Mustacchi case IMC_TAD_BASE_SOCK_GRAN_1GB:
1847eb00b1c8SRobert Mustacchi rule->itr_sock_gran = IMC_TAD_GRAN_1GB;
1848eb00b1c8SRobert Mustacchi break;
1849eb00b1c8SRobert Mustacchi }
1850eb00b1c8SRobert Mustacchi }
1851eb00b1c8SRobert Mustacchi
1852eb00b1c8SRobert Mustacchi /*
1853eb00b1c8SRobert Mustacchi * When mirroring is enabled, at least in Sandy Bridge to Broadwell, it's
1854eb00b1c8SRobert Mustacchi * suggested that the channel wayness will take this into account and therefore
1855eb00b1c8SRobert Mustacchi * should be accurately reflected.
1856eb00b1c8SRobert Mustacchi */
1857eb00b1c8SRobert Mustacchi static void
imc_tad_read_rules(imc_t * imc,imc_tad_t * tad)1858eb00b1c8SRobert Mustacchi imc_tad_read_rules(imc_t *imc, imc_tad_t *tad)
1859eb00b1c8SRobert Mustacchi {
1860eb00b1c8SRobert Mustacchi uint_t i;
1861eb00b1c8SRobert Mustacchi off_t baseoff;
1862eb00b1c8SRobert Mustacchi imc_tad_rule_t *prev;
1863eb00b1c8SRobert Mustacchi
1864eb00b1c8SRobert Mustacchi tad->itad_nrules = imc->imc_gen_data->igd_tad_nrules;
1865eb00b1c8SRobert Mustacchi for (i = 0, baseoff = imc->imc_gen_data->igd_tad_rule_offset,
1866eb00b1c8SRobert Mustacchi prev = NULL; i < tad->itad_nrules;
1867eb00b1c8SRobert Mustacchi i++, baseoff += sizeof (uint32_t)) {
1868eb00b1c8SRobert Mustacchi uint32_t val;
1869eb00b1c8SRobert Mustacchi off_t off;
1870eb00b1c8SRobert Mustacchi imc_tad_rule_t *rule = &tad->itad_rules[i];
1871eb00b1c8SRobert Mustacchi
1872eb00b1c8SRobert Mustacchi /*
1873eb00b1c8SRobert Mustacchi * On Skylake, the TAD rules are split among two registers. The
1874eb00b1c8SRobert Mustacchi * latter set mimics what exists on pre-Skylake.
1875eb00b1c8SRobert Mustacchi */
1876eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE) {
1877eb00b1c8SRobert Mustacchi off = baseoff + IMC_SKX_WAYNESS_OFFSET;
1878eb00b1c8SRobert Mustacchi } else {
1879eb00b1c8SRobert Mustacchi off = baseoff;
1880eb00b1c8SRobert Mustacchi }
1881eb00b1c8SRobert Mustacchi
1882eb00b1c8SRobert Mustacchi val = pci_config_get32(tad->itad_stub->istub_cfgspace, off);
1883eb00b1c8SRobert Mustacchi if (val == PCI_EINVAL32) {
1884eb00b1c8SRobert Mustacchi tad->itad_valid |= IMC_TAD_V_BAD_PCI_READ;
1885eb00b1c8SRobert Mustacchi return;
1886eb00b1c8SRobert Mustacchi }
1887eb00b1c8SRobert Mustacchi
1888eb00b1c8SRobert Mustacchi imc_tad_fill_rule(imc, tad, prev, rule, val);
1889eb00b1c8SRobert Mustacchi prev = rule;
1890eb00b1c8SRobert Mustacchi if (imc->imc_gen < IMC_GEN_SKYLAKE)
1891eb00b1c8SRobert Mustacchi continue;
1892eb00b1c8SRobert Mustacchi
1893eb00b1c8SRobert Mustacchi val = pci_config_get32(tad->itad_stub->istub_cfgspace, baseoff);
1894eb00b1c8SRobert Mustacchi if (val == PCI_EINVAL32) {
1895eb00b1c8SRobert Mustacchi tad->itad_valid |= IMC_TAD_V_BAD_PCI_READ;
1896eb00b1c8SRobert Mustacchi return;
1897eb00b1c8SRobert Mustacchi }
1898eb00b1c8SRobert Mustacchi
1899eb00b1c8SRobert Mustacchi imc_tad_fill_skx(imc, tad, rule, val);
1900eb00b1c8SRobert Mustacchi }
1901eb00b1c8SRobert Mustacchi }
1902eb00b1c8SRobert Mustacchi
1903eb00b1c8SRobert Mustacchi /*
1904eb00b1c8SRobert Mustacchi * Check for features which change how decoding works.
1905eb00b1c8SRobert Mustacchi */
1906eb00b1c8SRobert Mustacchi static void
imc_tad_read_features(imc_t * imc,imc_tad_t * tad,imc_mc_t * mc)1907eb00b1c8SRobert Mustacchi imc_tad_read_features(imc_t *imc, imc_tad_t *tad, imc_mc_t *mc)
1908eb00b1c8SRobert Mustacchi {
1909eb00b1c8SRobert Mustacchi uint32_t val;
1910eb00b1c8SRobert Mustacchi
1911eb00b1c8SRobert Mustacchi /*
1912eb00b1c8SRobert Mustacchi * Determine whether or not lockstep mode or mirroring are enabled.
1913eb00b1c8SRobert Mustacchi * These change the behavior of how we're supposed to interpret channel
1914eb00b1c8SRobert Mustacchi * wayness. Lockstep is available in the TAD's features. Mirroring is
1915eb00b1c8SRobert Mustacchi * available on the IMC's features. This isn't present in Skylake+. On
1916eb00b1c8SRobert Mustacchi * Skylake Mirorring is a property of the SAD rule and there is no
1917eb00b1c8SRobert Mustacchi * lockstep.
1918eb00b1c8SRobert Mustacchi */
1919eb00b1c8SRobert Mustacchi switch (imc->imc_gen) {
1920eb00b1c8SRobert Mustacchi case IMC_GEN_SANDY:
1921eb00b1c8SRobert Mustacchi case IMC_GEN_IVY:
1922eb00b1c8SRobert Mustacchi case IMC_GEN_HASWELL:
1923eb00b1c8SRobert Mustacchi case IMC_GEN_BROADWELL:
1924eb00b1c8SRobert Mustacchi val = pci_config_get32(tad->itad_stub->istub_cfgspace,
1925eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_tad_sysdef);
1926eb00b1c8SRobert Mustacchi if (val == PCI_EINVAL32) {
1927eb00b1c8SRobert Mustacchi tad->itad_valid |= IMC_TAD_V_BAD_PCI_READ;
1928eb00b1c8SRobert Mustacchi return;
1929eb00b1c8SRobert Mustacchi }
1930eb00b1c8SRobert Mustacchi if (IMC_TAD_SYSDEF_LOCKSTEP(val)) {
1931eb00b1c8SRobert Mustacchi tad->itad_flags |= IMC_TAD_FLAG_LOCKSTEP;
1932eb00b1c8SRobert Mustacchi }
1933eb00b1c8SRobert Mustacchi
1934eb00b1c8SRobert Mustacchi val = pci_config_get32(mc->icn_main1->istub_cfgspace,
1935eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_mc_mirror);
1936eb00b1c8SRobert Mustacchi if (val == PCI_EINVAL32) {
1937eb00b1c8SRobert Mustacchi tad->itad_valid |= IMC_TAD_V_BAD_PCI_READ;
1938eb00b1c8SRobert Mustacchi return;
1939eb00b1c8SRobert Mustacchi }
1940eb00b1c8SRobert Mustacchi if (IMC_MC_MIRROR_SNB_BRD(val)) {
1941eb00b1c8SRobert Mustacchi tad->itad_flags |= IMC_TAD_FLAG_MIRROR;
1942eb00b1c8SRobert Mustacchi }
1943eb00b1c8SRobert Mustacchi break;
1944eb00b1c8SRobert Mustacchi default:
1945eb00b1c8SRobert Mustacchi break;
1946eb00b1c8SRobert Mustacchi }
1947eb00b1c8SRobert Mustacchi
1948eb00b1c8SRobert Mustacchi /*
1949eb00b1c8SRobert Mustacchi * Now, go through and look at values that'll change how we do the
1950eb00b1c8SRobert Mustacchi * channel index and adddress calculation. These are only present
1951eb00b1c8SRobert Mustacchi * between Ivy Bridge and Broadwell. They don't exist on Sandy Bridge
1952eb00b1c8SRobert Mustacchi * and they don't exist on Skylake+.
1953eb00b1c8SRobert Mustacchi */
1954eb00b1c8SRobert Mustacchi switch (imc->imc_gen) {
1955eb00b1c8SRobert Mustacchi case IMC_GEN_IVY:
1956eb00b1c8SRobert Mustacchi case IMC_GEN_HASWELL:
1957eb00b1c8SRobert Mustacchi case IMC_GEN_BROADWELL:
1958eb00b1c8SRobert Mustacchi val = pci_config_get32(tad->itad_stub->istub_cfgspace,
1959eb00b1c8SRobert Mustacchi imc->imc_gen_data->igd_tad_sysdef2);
1960eb00b1c8SRobert Mustacchi if (val == PCI_EINVAL32) {
1961eb00b1c8SRobert Mustacchi tad->itad_valid |= IMC_TAD_V_BAD_PCI_READ;
1962eb00b1c8SRobert Mustacchi return;
1963eb00b1c8SRobert Mustacchi }
1964eb00b1c8SRobert Mustacchi if (IMC_TAD_SYSDEF2_SHIFTUP(val)) {
1965eb00b1c8SRobert Mustacchi tad->itad_flags |= IMC_TAD_FLAG_CHANSHIFT;
1966eb00b1c8SRobert Mustacchi }
1967eb00b1c8SRobert Mustacchi if (IMC_TAD_SYSDEF2_SHIFTUP(val)) {
1968eb00b1c8SRobert Mustacchi tad->itad_flags |= IMC_TAD_FLAG_CHANHASH;
1969eb00b1c8SRobert Mustacchi }
1970eb00b1c8SRobert Mustacchi break;
1971eb00b1c8SRobert Mustacchi default:
1972eb00b1c8SRobert Mustacchi break;
1973eb00b1c8SRobert Mustacchi }
1974eb00b1c8SRobert Mustacchi }
1975eb00b1c8SRobert Mustacchi
1976eb00b1c8SRobert Mustacchi /*
1977eb00b1c8SRobert Mustacchi * Read the IMC channel interleave records
1978eb00b1c8SRobert Mustacchi */
1979eb00b1c8SRobert Mustacchi static void
imc_tad_read_interleave(imc_t * imc,imc_channel_t * chan)1980eb00b1c8SRobert Mustacchi imc_tad_read_interleave(imc_t *imc, imc_channel_t *chan)
1981eb00b1c8SRobert Mustacchi {
1982eb00b1c8SRobert Mustacchi uint_t i;
1983eb00b1c8SRobert Mustacchi off_t off;
1984eb00b1c8SRobert Mustacchi
1985eb00b1c8SRobert Mustacchi chan->ich_ntad_offsets = imc->imc_gen_data->igd_tad_nrules;
1986eb00b1c8SRobert Mustacchi for (i = 0, off = imc->imc_gen_data->igd_tad_chan_offset;
1987eb00b1c8SRobert Mustacchi i < chan->ich_ntad_offsets; i++, off += sizeof (uint32_t)) {
1988eb00b1c8SRobert Mustacchi uint32_t val;
1989eb00b1c8SRobert Mustacchi uint64_t offset;
1990eb00b1c8SRobert Mustacchi
1991eb00b1c8SRobert Mustacchi val = pci_config_get32(chan->ich_desc->istub_cfgspace,
1992eb00b1c8SRobert Mustacchi off);
1993eb00b1c8SRobert Mustacchi if (val == PCI_EINVAL32) {
1994eb00b1c8SRobert Mustacchi chan->ich_valid |= IMC_CHANNEL_V_BAD_PCI_READ;
1995eb00b1c8SRobert Mustacchi return;
1996eb00b1c8SRobert Mustacchi }
1997eb00b1c8SRobert Mustacchi
1998eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE) {
1999eb00b1c8SRobert Mustacchi offset = IMC_TADCHAN_OFFSET_SKX(val);
2000eb00b1c8SRobert Mustacchi } else {
2001eb00b1c8SRobert Mustacchi offset = IMC_TADCHAN_OFFSET_SNB_BRD(val);
2002eb00b1c8SRobert Mustacchi }
2003eb00b1c8SRobert Mustacchi
2004eb00b1c8SRobert Mustacchi chan->ich_tad_offsets[i] = offset << IMC_TADCHAN_OFFSET_SHIFT;
2005eb00b1c8SRobert Mustacchi chan->ich_tad_offsets_raw[i] = val;
2006eb00b1c8SRobert Mustacchi }
2007eb00b1c8SRobert Mustacchi }
2008eb00b1c8SRobert Mustacchi
2009eb00b1c8SRobert Mustacchi static void
imc_decoder_init_tad(imc_t * imc)2010eb00b1c8SRobert Mustacchi imc_decoder_init_tad(imc_t *imc)
2011eb00b1c8SRobert Mustacchi {
2012eb00b1c8SRobert Mustacchi uint_t i;
2013eb00b1c8SRobert Mustacchi
2014eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
2015eb00b1c8SRobert Mustacchi uint_t j;
2016eb00b1c8SRobert Mustacchi
2017eb00b1c8SRobert Mustacchi for (j = 0; j < imc->imc_sockets[i].isock_ntad; j++) {
2018eb00b1c8SRobert Mustacchi imc_tad_read_features(imc,
2019eb00b1c8SRobert Mustacchi &imc->imc_sockets[i].isock_tad[j],
2020eb00b1c8SRobert Mustacchi &imc->imc_sockets[i].isock_imcs[j]);
2021eb00b1c8SRobert Mustacchi imc_tad_read_rules(imc,
2022eb00b1c8SRobert Mustacchi &imc->imc_sockets[i].isock_tad[j]);
2023eb00b1c8SRobert Mustacchi }
2024eb00b1c8SRobert Mustacchi }
2025eb00b1c8SRobert Mustacchi
2026eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
2027eb00b1c8SRobert Mustacchi uint_t j;
2028eb00b1c8SRobert Mustacchi imc_socket_t *sock = &imc->imc_sockets[i];
2029eb00b1c8SRobert Mustacchi
2030eb00b1c8SRobert Mustacchi for (j = 0; j < imc->imc_sockets[i].isock_nimc; j++) {
2031eb00b1c8SRobert Mustacchi uint_t k;
2032eb00b1c8SRobert Mustacchi imc_mc_t *mc = &sock->isock_imcs[j];
2033eb00b1c8SRobert Mustacchi
2034eb00b1c8SRobert Mustacchi for (k = 0; k < mc->icn_nchannels; k++) {
2035eb00b1c8SRobert Mustacchi imc_channel_t *chan = &mc->icn_channels[k];
2036eb00b1c8SRobert Mustacchi imc_tad_read_interleave(imc, chan);
2037eb00b1c8SRobert Mustacchi }
2038eb00b1c8SRobert Mustacchi }
2039eb00b1c8SRobert Mustacchi }
2040eb00b1c8SRobert Mustacchi }
2041eb00b1c8SRobert Mustacchi
2042eb00b1c8SRobert Mustacchi static void
imc_rir_read_ileave_offsets(imc_t * imc,imc_channel_t * chan,imc_rank_ileave_t * rank,uint_t rirno,boolean_t contig)2043eb00b1c8SRobert Mustacchi imc_rir_read_ileave_offsets(imc_t *imc, imc_channel_t *chan,
2044eb00b1c8SRobert Mustacchi imc_rank_ileave_t *rank, uint_t rirno, boolean_t contig)
2045eb00b1c8SRobert Mustacchi {
2046eb00b1c8SRobert Mustacchi uint_t i;
2047eb00b1c8SRobert Mustacchi off_t off, incr;
2048eb00b1c8SRobert Mustacchi
2049eb00b1c8SRobert Mustacchi /*
2050eb00b1c8SRobert Mustacchi * Rank interleave offset registers come in two forms. Either they are
2051eb00b1c8SRobert Mustacchi * contiguous for a given wayness, meaning that all of the entries for
2052eb00b1c8SRobert Mustacchi * wayness zero are contiguous, or they are sparse, meaning that there
2053eb00b1c8SRobert Mustacchi * is a bank for entry zero for all wayness, then entry one for all
2054eb00b1c8SRobert Mustacchi * wayness, etc.
2055eb00b1c8SRobert Mustacchi */
2056eb00b1c8SRobert Mustacchi if (contig) {
2057eb00b1c8SRobert Mustacchi off = imc->imc_gen_data->igd_rir_ileave_offset +
2058eb00b1c8SRobert Mustacchi (rirno * imc->imc_gen_data->igd_rir_nileaves *
2059eb00b1c8SRobert Mustacchi sizeof (uint32_t));
2060eb00b1c8SRobert Mustacchi incr = sizeof (uint32_t);
2061eb00b1c8SRobert Mustacchi } else {
2062eb00b1c8SRobert Mustacchi off = imc->imc_gen_data->igd_rir_ileave_offset +
2063eb00b1c8SRobert Mustacchi (rirno * sizeof (uint32_t));
2064eb00b1c8SRobert Mustacchi incr = imc->imc_gen_data->igd_rir_nileaves * sizeof (uint32_t);
2065eb00b1c8SRobert Mustacchi }
2066eb00b1c8SRobert Mustacchi for (i = 0; i < rank->irle_nentries; i++, off += incr) {
2067eb00b1c8SRobert Mustacchi uint32_t val;
2068eb00b1c8SRobert Mustacchi uint64_t offset;
2069eb00b1c8SRobert Mustacchi imc_rank_ileave_entry_t *ent = &rank->irle_entries[i];
2070eb00b1c8SRobert Mustacchi
2071eb00b1c8SRobert Mustacchi val = pci_config_get32(chan->ich_desc->istub_cfgspace, off);
2072eb00b1c8SRobert Mustacchi if (val == PCI_EINVAL32) {
2073eb00b1c8SRobert Mustacchi chan->ich_valid |= IMC_CHANNEL_V_BAD_PCI_READ;
2074eb00b1c8SRobert Mustacchi return;
2075eb00b1c8SRobert Mustacchi }
2076eb00b1c8SRobert Mustacchi
2077eb00b1c8SRobert Mustacchi switch (imc->imc_gen) {
2078eb00b1c8SRobert Mustacchi case IMC_GEN_BROADWELL:
2079eb00b1c8SRobert Mustacchi ent->irle_target = IMC_RIR_OFFSET_TARGET_BRD(val);
2080eb00b1c8SRobert Mustacchi break;
2081eb00b1c8SRobert Mustacchi default:
2082eb00b1c8SRobert Mustacchi ent->irle_target = IMC_RIR_OFFSET_TARGET(val);
2083eb00b1c8SRobert Mustacchi break;
2084eb00b1c8SRobert Mustacchi }
2085eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_HASWELL) {
2086eb00b1c8SRobert Mustacchi offset = IMC_RIR_OFFSET_OFFSET_HAS_SKX(val);
2087eb00b1c8SRobert Mustacchi } else {
2088eb00b1c8SRobert Mustacchi offset = IMC_RIR_OFFSET_OFFSET_SNB_IVB(val);
2089eb00b1c8SRobert Mustacchi }
2090eb00b1c8SRobert Mustacchi ent->irle_offset = offset << IMC_RIR_OFFSET_SHIFT;
2091eb00b1c8SRobert Mustacchi }
2092eb00b1c8SRobert Mustacchi }
2093eb00b1c8SRobert Mustacchi
2094eb00b1c8SRobert Mustacchi static void
imc_rir_read_wayness(imc_t * imc,imc_channel_t * chan)2095eb00b1c8SRobert Mustacchi imc_rir_read_wayness(imc_t *imc, imc_channel_t *chan)
2096eb00b1c8SRobert Mustacchi {
2097eb00b1c8SRobert Mustacchi uint_t i;
2098eb00b1c8SRobert Mustacchi off_t off;
2099eb00b1c8SRobert Mustacchi
2100eb00b1c8SRobert Mustacchi chan->ich_nrankileaves = imc->imc_gen_data->igd_rir_nways;
2101eb00b1c8SRobert Mustacchi for (i = 0, off = imc->imc_gen_data->igd_rir_way_offset;
2102eb00b1c8SRobert Mustacchi i < chan->ich_nrankileaves; i++, off += sizeof (uint32_t)) {
2103eb00b1c8SRobert Mustacchi uint32_t val;
2104eb00b1c8SRobert Mustacchi uint64_t lim;
2105eb00b1c8SRobert Mustacchi imc_rank_ileave_t *ent = &chan->ich_rankileaves[i];
2106eb00b1c8SRobert Mustacchi
2107eb00b1c8SRobert Mustacchi val = pci_config_get32(chan->ich_desc->istub_cfgspace, off);
2108eb00b1c8SRobert Mustacchi if (val == PCI_EINVAL32) {
2109eb00b1c8SRobert Mustacchi chan->ich_valid |= IMC_CHANNEL_V_BAD_PCI_READ;
2110eb00b1c8SRobert Mustacchi return;
2111eb00b1c8SRobert Mustacchi }
2112eb00b1c8SRobert Mustacchi
2113eb00b1c8SRobert Mustacchi ent->irle_raw = val;
2114eb00b1c8SRobert Mustacchi ent->irle_enabled = IMC_RIR_WAYNESS_ENABLED(val) != 0;
2115eb00b1c8SRobert Mustacchi ent->irle_nways = 1 << IMC_RIR_WAYNESS_WAY(val);
2116eb00b1c8SRobert Mustacchi ent->irle_nwaysbits = IMC_RIR_WAYNESS_WAY(val);
2117eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_HASWELL) {
2118eb00b1c8SRobert Mustacchi lim = IMC_RIR_LIMIT_HAS_SKX(val);
2119eb00b1c8SRobert Mustacchi } else {
2120eb00b1c8SRobert Mustacchi lim = IMC_RIR_LIMIT_SNB_IVB(val);
2121eb00b1c8SRobert Mustacchi }
2122eb00b1c8SRobert Mustacchi
2123eb00b1c8SRobert Mustacchi ent->irle_limit = (lim << IMC_RIR_LIMIT_SHIFT) +
2124eb00b1c8SRobert Mustacchi IMC_RIR_LIMIT_EXCLUSIVE;
2125eb00b1c8SRobert Mustacchi
2126eb00b1c8SRobert Mustacchi ent->irle_nentries = imc->imc_gen_data->igd_rir_nileaves;
2127eb00b1c8SRobert Mustacchi if (imc->imc_gen >= IMC_GEN_SKYLAKE) {
2128eb00b1c8SRobert Mustacchi imc_rir_read_ileave_offsets(imc, chan, ent, i, B_FALSE);
2129eb00b1c8SRobert Mustacchi } else {
2130eb00b1c8SRobert Mustacchi imc_rir_read_ileave_offsets(imc, chan, ent, i, B_TRUE);
2131eb00b1c8SRobert Mustacchi }
2132eb00b1c8SRobert Mustacchi }
2133eb00b1c8SRobert Mustacchi }
2134eb00b1c8SRobert Mustacchi
2135eb00b1c8SRobert Mustacchi static void
imc_decoder_init_rir(imc_t * imc)2136eb00b1c8SRobert Mustacchi imc_decoder_init_rir(imc_t *imc)
2137eb00b1c8SRobert Mustacchi {
2138eb00b1c8SRobert Mustacchi uint_t i;
2139eb00b1c8SRobert Mustacchi
2140eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
2141eb00b1c8SRobert Mustacchi uint_t j;
2142eb00b1c8SRobert Mustacchi imc_socket_t *sock = &imc->imc_sockets[i];
2143eb00b1c8SRobert Mustacchi
2144eb00b1c8SRobert Mustacchi for (j = 0; j < imc->imc_sockets[i].isock_nimc; j++) {
2145eb00b1c8SRobert Mustacchi uint_t k;
2146eb00b1c8SRobert Mustacchi imc_mc_t *mc = &sock->isock_imcs[j];
2147eb00b1c8SRobert Mustacchi
2148eb00b1c8SRobert Mustacchi for (k = 0; k < mc->icn_nchannels; k++) {
2149eb00b1c8SRobert Mustacchi imc_channel_t *chan = &mc->icn_channels[k];
2150eb00b1c8SRobert Mustacchi imc_rir_read_wayness(imc, chan);
2151eb00b1c8SRobert Mustacchi }
2152eb00b1c8SRobert Mustacchi }
2153eb00b1c8SRobert Mustacchi }
2154eb00b1c8SRobert Mustacchi }
2155eb00b1c8SRobert Mustacchi
2156eb00b1c8SRobert Mustacchi static cmi_errno_t
imc_mc_patounum(void * arg,uint64_t pa,uint8_t valid_hi,uint8_t valid_lo,uint32_t synd,int syndtype,mc_unum_t * unump)2157eb00b1c8SRobert Mustacchi imc_mc_patounum(void *arg, uint64_t pa, uint8_t valid_hi, uint8_t valid_lo,
2158eb00b1c8SRobert Mustacchi uint32_t synd, int syndtype, mc_unum_t *unump)
2159eb00b1c8SRobert Mustacchi {
2160eb00b1c8SRobert Mustacchi imc_t *imc = arg;
2161eb00b1c8SRobert Mustacchi uint_t i;
2162eb00b1c8SRobert Mustacchi imc_decode_state_t dec;
2163eb00b1c8SRobert Mustacchi
2164eb00b1c8SRobert Mustacchi bzero(&dec, sizeof (dec));
2165eb00b1c8SRobert Mustacchi if (!imc_decode_pa(imc, pa, &dec)) {
2166eb00b1c8SRobert Mustacchi switch (dec.ids_fail) {
2167eb00b1c8SRobert Mustacchi case IMC_DECODE_F_LEGACY_RANGE:
2168eb00b1c8SRobert Mustacchi case IMC_DECODE_F_OUTSIDE_DRAM:
2169eb00b1c8SRobert Mustacchi return (CMIERR_MC_NOTDIMMADDR);
2170eb00b1c8SRobert Mustacchi default:
2171eb00b1c8SRobert Mustacchi return (CMIERR_MC_BADSTATE);
2172eb00b1c8SRobert Mustacchi }
2173eb00b1c8SRobert Mustacchi }
2174eb00b1c8SRobert Mustacchi
2175eb00b1c8SRobert Mustacchi unump->unum_board = 0;
2176eb00b1c8SRobert Mustacchi /*
2177eb00b1c8SRobert Mustacchi * The chip id needs to be in the order that the OS expects it, which
2178eb00b1c8SRobert Mustacchi * may not be our order.
2179eb00b1c8SRobert Mustacchi */
2180eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
2181eb00b1c8SRobert Mustacchi if (imc->imc_spointers[i] == dec.ids_socket)
2182eb00b1c8SRobert Mustacchi break;
2183eb00b1c8SRobert Mustacchi }
2184eb00b1c8SRobert Mustacchi if (i == imc->imc_nsockets) {
2185eb00b1c8SRobert Mustacchi return (CMIERR_MC_BADSTATE);
2186eb00b1c8SRobert Mustacchi }
2187eb00b1c8SRobert Mustacchi unump->unum_chip = i;
2188eb00b1c8SRobert Mustacchi unump->unum_mc = dec.ids_tadid;
2189eb00b1c8SRobert Mustacchi unump->unum_chan = dec.ids_channelid;
2190eb00b1c8SRobert Mustacchi unump->unum_cs = dec.ids_dimmid;
2191eb00b1c8SRobert Mustacchi unump->unum_rank = dec.ids_rankid;
2192eb00b1c8SRobert Mustacchi unump->unum_offset = dec.ids_rankaddr;
2193eb00b1c8SRobert Mustacchi for (i = 0; i < MC_UNUM_NDIMM; i++) {
2194eb00b1c8SRobert Mustacchi unump->unum_dimms[i] = MC_INVALNUM;
2195eb00b1c8SRobert Mustacchi }
2196eb00b1c8SRobert Mustacchi
2197eb00b1c8SRobert Mustacchi return (CMI_SUCCESS);
2198eb00b1c8SRobert Mustacchi }
2199eb00b1c8SRobert Mustacchi
2200eb00b1c8SRobert Mustacchi static cmi_errno_t
imc_mc_unumtopa(void * arg,mc_unum_t * unum,nvlist_t * nvl,uint64_t * pa)2201eb00b1c8SRobert Mustacchi imc_mc_unumtopa(void *arg, mc_unum_t *unum, nvlist_t *nvl, uint64_t *pa)
2202eb00b1c8SRobert Mustacchi {
2203eb00b1c8SRobert Mustacchi return (CMIERR_UNKNOWN);
2204eb00b1c8SRobert Mustacchi }
2205eb00b1c8SRobert Mustacchi
2206eb00b1c8SRobert Mustacchi static const cmi_mc_ops_t imc_mc_ops = {
2207eb00b1c8SRobert Mustacchi .cmi_mc_patounum = imc_mc_patounum,
2208eb00b1c8SRobert Mustacchi .cmi_mc_unumtopa = imc_mc_unumtopa
2209eb00b1c8SRobert Mustacchi };
2210eb00b1c8SRobert Mustacchi
2211eb00b1c8SRobert Mustacchi /*
2212eb00b1c8SRobert Mustacchi * This is where we really finish attaching and become open for business. This
2213eb00b1c8SRobert Mustacchi * occurs once we have all of the expected stubs attached. Here's where all of
2214eb00b1c8SRobert Mustacchi * the real fun begins.
2215eb00b1c8SRobert Mustacchi */
2216eb00b1c8SRobert Mustacchi static void
imc_attach_complete(void * arg)2217eb00b1c8SRobert Mustacchi imc_attach_complete(void *arg)
2218eb00b1c8SRobert Mustacchi {
2219eb00b1c8SRobert Mustacchi imc_t *imc = arg;
2220eb00b1c8SRobert Mustacchi cmi_errno_t err;
2221eb00b1c8SRobert Mustacchi
2222eb00b1c8SRobert Mustacchi imc_set_gen_data(imc);
2223eb00b1c8SRobert Mustacchi
2224eb00b1c8SRobert Mustacchi /*
2225eb00b1c8SRobert Mustacchi * On SKX and newer, we can fail to map PCI buses at this point due to
2226eb00b1c8SRobert Mustacchi * bad PCIe reads.
2227eb00b1c8SRobert Mustacchi */
2228eb00b1c8SRobert Mustacchi if (!imc_map_stubs(imc)) {
2229eb00b1c8SRobert Mustacchi goto done;
2230eb00b1c8SRobert Mustacchi }
2231eb00b1c8SRobert Mustacchi
22327d916034SRobert Mustacchi if (!imc_validate_stubs(imc)) {
22337d916034SRobert Mustacchi imc->imc_flags |= IMC_F_VALIDATE_FAILED;
22347d916034SRobert Mustacchi goto done;
22357d916034SRobert Mustacchi }
22367d916034SRobert Mustacchi
2237eb00b1c8SRobert Mustacchi imc_fixup_stubs(imc);
2238eb00b1c8SRobert Mustacchi imc_map_sockets(imc);
2239eb00b1c8SRobert Mustacchi
2240eb00b1c8SRobert Mustacchi if (!imc_create_minors(imc)) {
2241eb00b1c8SRobert Mustacchi goto done;
2242eb00b1c8SRobert Mustacchi }
2243eb00b1c8SRobert Mustacchi
2244eb00b1c8SRobert Mustacchi imc_fill_data(imc);
2245eb00b1c8SRobert Mustacchi imc_nvl_create(imc);
2246eb00b1c8SRobert Mustacchi
2247eb00b1c8SRobert Mustacchi /*
2248eb00b1c8SRobert Mustacchi * Gather additional information that we need so that we can properly
2249eb00b1c8SRobert Mustacchi * initialize the memory decoder and encoder.
2250eb00b1c8SRobert Mustacchi */
2251eb00b1c8SRobert Mustacchi imc_decoder_init_sad(imc);
2252eb00b1c8SRobert Mustacchi imc_decoder_init_tad(imc);
2253eb00b1c8SRobert Mustacchi imc_decoder_init_rir(imc);
2254eb00b1c8SRobert Mustacchi
2255eb00b1c8SRobert Mustacchi /*
2256eb00b1c8SRobert Mustacchi * Register decoder functions. This may fail. If so, try and complain
2257eb00b1c8SRobert Mustacchi * loudly, but stay active to allow other data to be useful. Register a
2258eb00b1c8SRobert Mustacchi * global handle.
2259eb00b1c8SRobert Mustacchi */
2260eb00b1c8SRobert Mustacchi if ((err = cmi_mc_register_global(&imc_mc_ops, imc)) != CMI_SUCCESS) {
2261eb00b1c8SRobert Mustacchi imc->imc_flags |= IMC_F_MCREG_FAILED;
2262eb00b1c8SRobert Mustacchi dev_err(imc->imc_dip, CE_WARN, "failed to register memory "
2263eb00b1c8SRobert Mustacchi "decoding operations: 0x%x", err);
2264eb00b1c8SRobert Mustacchi }
2265eb00b1c8SRobert Mustacchi
2266eb00b1c8SRobert Mustacchi done:
2267eb00b1c8SRobert Mustacchi mutex_enter(&imc->imc_lock);
2268eb00b1c8SRobert Mustacchi imc->imc_flags &= IMC_F_ATTACH_DISPATCHED;
2269eb00b1c8SRobert Mustacchi imc->imc_flags |= IMC_F_ATTACH_COMPLETE;
2270eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2271eb00b1c8SRobert Mustacchi }
2272eb00b1c8SRobert Mustacchi
2273eb00b1c8SRobert Mustacchi static int
imc_stub_comparator(const void * l,const void * r)2274eb00b1c8SRobert Mustacchi imc_stub_comparator(const void *l, const void *r)
2275eb00b1c8SRobert Mustacchi {
2276eb00b1c8SRobert Mustacchi const imc_stub_t *sl = l, *sr = r;
2277eb00b1c8SRobert Mustacchi if (sl->istub_bus > sr->istub_bus)
2278eb00b1c8SRobert Mustacchi return (1);
2279eb00b1c8SRobert Mustacchi if (sl->istub_bus < sr->istub_bus)
2280eb00b1c8SRobert Mustacchi return (-1);
2281eb00b1c8SRobert Mustacchi if (sl->istub_dev > sr->istub_dev)
2282eb00b1c8SRobert Mustacchi return (1);
2283eb00b1c8SRobert Mustacchi if (sl->istub_dev < sr->istub_dev)
2284eb00b1c8SRobert Mustacchi return (-1);
2285eb00b1c8SRobert Mustacchi if (sl->istub_func > sr->istub_func)
2286eb00b1c8SRobert Mustacchi return (1);
2287eb00b1c8SRobert Mustacchi if (sl->istub_func < sr->istub_func)
2288eb00b1c8SRobert Mustacchi return (-1);
2289eb00b1c8SRobert Mustacchi return (0);
2290eb00b1c8SRobert Mustacchi }
2291eb00b1c8SRobert Mustacchi
2292eb00b1c8SRobert Mustacchi static int
imc_stub_scan_cb(dev_info_t * dip,void * arg)2293eb00b1c8SRobert Mustacchi imc_stub_scan_cb(dev_info_t *dip, void *arg)
2294eb00b1c8SRobert Mustacchi {
2295eb00b1c8SRobert Mustacchi int vid, did;
2296eb00b1c8SRobert Mustacchi const imc_stub_table_t *table;
2297eb00b1c8SRobert Mustacchi imc_t *imc = arg;
2298eb00b1c8SRobert Mustacchi int *regs;
2299eb00b1c8SRobert Mustacchi uint_t i, nregs;
2300eb00b1c8SRobert Mustacchi
2301eb00b1c8SRobert Mustacchi if (dip == ddi_root_node()) {
2302eb00b1c8SRobert Mustacchi return (DDI_WALK_CONTINUE);
2303eb00b1c8SRobert Mustacchi }
2304eb00b1c8SRobert Mustacchi
2305eb00b1c8SRobert Mustacchi /*
2306eb00b1c8SRobert Mustacchi * Get the dev info name. PCI devices will always be children of PCI
2307eb00b1c8SRobert Mustacchi * devices today on x86. If we reach something that has a device name
2308eb00b1c8SRobert Mustacchi * that's not PCI, then we can prune it's children.
2309eb00b1c8SRobert Mustacchi */
2310eb00b1c8SRobert Mustacchi if (strncmp("pci", ddi_get_name(dip), 3) != 0) {
2311eb00b1c8SRobert Mustacchi return (DDI_WALK_PRUNECHILD);
2312eb00b1c8SRobert Mustacchi }
2313eb00b1c8SRobert Mustacchi
2314eb00b1c8SRobert Mustacchi /*
2315eb00b1c8SRobert Mustacchi * Get the device and vendor ID and see if this is something the imc
2316eb00b1c8SRobert Mustacchi * knows about or cares about.
2317eb00b1c8SRobert Mustacchi */
2318eb00b1c8SRobert Mustacchi vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2319eb00b1c8SRobert Mustacchi "vendor-id", PCI_EINVAL16);
2320eb00b1c8SRobert Mustacchi did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2321eb00b1c8SRobert Mustacchi "device-id", PCI_EINVAL16);
2322eb00b1c8SRobert Mustacchi if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) {
2323eb00b1c8SRobert Mustacchi return (DDI_WALK_CONTINUE);
2324eb00b1c8SRobert Mustacchi }
2325eb00b1c8SRobert Mustacchi
2326eb00b1c8SRobert Mustacchi if (vid != IMC_PCI_VENDOR_INTC) {
2327eb00b1c8SRobert Mustacchi return (DDI_WALK_PRUNECHILD);
2328eb00b1c8SRobert Mustacchi }
2329eb00b1c8SRobert Mustacchi
2330eb00b1c8SRobert Mustacchi if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2331eb00b1c8SRobert Mustacchi "reg", ®s, &nregs) != DDI_PROP_SUCCESS) {
2332eb00b1c8SRobert Mustacchi return (DDI_WALK_CONTINUE);
2333eb00b1c8SRobert Mustacchi }
2334eb00b1c8SRobert Mustacchi
2335eb00b1c8SRobert Mustacchi if (nregs == 0) {
2336eb00b1c8SRobert Mustacchi ddi_prop_free(regs);
2337eb00b1c8SRobert Mustacchi return (DDI_WALK_CONTINUE);
2338eb00b1c8SRobert Mustacchi }
2339eb00b1c8SRobert Mustacchi
2340eb00b1c8SRobert Mustacchi
2341eb00b1c8SRobert Mustacchi table = NULL;
2342eb00b1c8SRobert Mustacchi for (i = 0; i < ARRAY_SIZE(imc_stub_table); i++) {
2343eb00b1c8SRobert Mustacchi if (imc_stub_table[i].imcs_devid == did &&
2344eb00b1c8SRobert Mustacchi imc_stub_table[i].imcs_pcidev == PCI_REG_DEV_G(regs[0]) &&
2345eb00b1c8SRobert Mustacchi imc_stub_table[i].imcs_pcifunc == PCI_REG_FUNC_G(regs[0])) {
2346eb00b1c8SRobert Mustacchi table = &imc_stub_table[i];
2347eb00b1c8SRobert Mustacchi break;
2348eb00b1c8SRobert Mustacchi }
2349eb00b1c8SRobert Mustacchi }
2350eb00b1c8SRobert Mustacchi ddi_prop_free(regs);
2351eb00b1c8SRobert Mustacchi
2352eb00b1c8SRobert Mustacchi /*
2353eb00b1c8SRobert Mustacchi * Not a match, not interesting.
2354eb00b1c8SRobert Mustacchi */
2355eb00b1c8SRobert Mustacchi if (table == NULL) {
2356eb00b1c8SRobert Mustacchi return (DDI_WALK_CONTINUE);
2357eb00b1c8SRobert Mustacchi }
2358eb00b1c8SRobert Mustacchi
2359eb00b1c8SRobert Mustacchi mutex_enter(&imc->imc_lock);
2360eb00b1c8SRobert Mustacchi imc->imc_nscanned++;
2361eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2362eb00b1c8SRobert Mustacchi
2363eb00b1c8SRobert Mustacchi return (DDI_WALK_CONTINUE);
2364eb00b1c8SRobert Mustacchi }
2365eb00b1c8SRobert Mustacchi
2366eb00b1c8SRobert Mustacchi /*
2367eb00b1c8SRobert Mustacchi * From here, go through and see how many of the devices that we know about.
2368eb00b1c8SRobert Mustacchi */
2369eb00b1c8SRobert Mustacchi static void
imc_stub_scan(void * arg)2370eb00b1c8SRobert Mustacchi imc_stub_scan(void *arg)
2371eb00b1c8SRobert Mustacchi {
2372eb00b1c8SRobert Mustacchi imc_t *imc = arg;
2373eb00b1c8SRobert Mustacchi boolean_t dispatch = B_FALSE;
2374eb00b1c8SRobert Mustacchi
2375eb00b1c8SRobert Mustacchi /*
2376eb00b1c8SRobert Mustacchi * Zero out the scan results in case we've been detached and reattached.
2377eb00b1c8SRobert Mustacchi */
2378eb00b1c8SRobert Mustacchi mutex_enter(&imc->imc_lock);
2379eb00b1c8SRobert Mustacchi imc->imc_nscanned = 0;
2380eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2381eb00b1c8SRobert Mustacchi
2382eb00b1c8SRobert Mustacchi ddi_walk_devs(ddi_root_node(), imc_stub_scan_cb, imc);
2383eb00b1c8SRobert Mustacchi
2384eb00b1c8SRobert Mustacchi mutex_enter(&imc->imc_lock);
2385eb00b1c8SRobert Mustacchi imc->imc_flags |= IMC_F_SCAN_COMPLETE;
2386eb00b1c8SRobert Mustacchi imc->imc_flags &= ~IMC_F_SCAN_DISPATCHED;
2387eb00b1c8SRobert Mustacchi
2388eb00b1c8SRobert Mustacchi /*
2389eb00b1c8SRobert Mustacchi * If the scan found no nodes, then that means that we're on a hardware
2390eb00b1c8SRobert Mustacchi * platform that we don't support. Therefore, there's no reason to do
2391eb00b1c8SRobert Mustacchi * anything here.
2392eb00b1c8SRobert Mustacchi */
2393eb00b1c8SRobert Mustacchi if (imc->imc_nscanned == 0) {
2394eb00b1c8SRobert Mustacchi imc->imc_flags |= IMC_F_UNSUP_PLATFORM;
2395eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2396eb00b1c8SRobert Mustacchi return;
2397eb00b1c8SRobert Mustacchi }
2398eb00b1c8SRobert Mustacchi
2399eb00b1c8SRobert Mustacchi if (avl_numnodes(&imc->imc_stubs) == imc->imc_nscanned) {
2400eb00b1c8SRobert Mustacchi imc->imc_flags |= IMC_F_ATTACH_DISPATCHED;
2401eb00b1c8SRobert Mustacchi dispatch = B_TRUE;
2402eb00b1c8SRobert Mustacchi }
2403eb00b1c8SRobert Mustacchi
2404eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2405eb00b1c8SRobert Mustacchi
2406eb00b1c8SRobert Mustacchi if (dispatch) {
2407eb00b1c8SRobert Mustacchi (void) ddi_taskq_dispatch(imc->imc_taskq, imc_attach_complete,
2408eb00b1c8SRobert Mustacchi imc, DDI_SLEEP);
2409eb00b1c8SRobert Mustacchi }
2410eb00b1c8SRobert Mustacchi }
2411eb00b1c8SRobert Mustacchi
2412eb00b1c8SRobert Mustacchi /*
2413eb00b1c8SRobert Mustacchi * By default, refuse to allow stubs to detach.
2414eb00b1c8SRobert Mustacchi */
2415eb00b1c8SRobert Mustacchi int
imc_detach_stub(dev_info_t * dip,ddi_detach_cmd_t cmd)2416eb00b1c8SRobert Mustacchi imc_detach_stub(dev_info_t *dip, ddi_detach_cmd_t cmd)
2417eb00b1c8SRobert Mustacchi {
2418eb00b1c8SRobert Mustacchi imc_stub_t *stub;
2419eb00b1c8SRobert Mustacchi imc_t *imc = imc_data;
2420eb00b1c8SRobert Mustacchi
2421eb00b1c8SRobert Mustacchi mutex_enter(&imc->imc_lock);
2422eb00b1c8SRobert Mustacchi
2423eb00b1c8SRobert Mustacchi /*
2424eb00b1c8SRobert Mustacchi * By default, we do not allow stubs to detach. However, if the driver
2425eb00b1c8SRobert Mustacchi * has attached to devices on a platform it doesn't recognize or
2426eb00b1c8SRobert Mustacchi * support or if the override flag has been set, then allow detach to
2427eb00b1c8SRobert Mustacchi * proceed.
2428eb00b1c8SRobert Mustacchi */
2429eb00b1c8SRobert Mustacchi if ((imc->imc_flags & IMC_F_UNSUP_PLATFORM) == 0 &&
2430eb00b1c8SRobert Mustacchi imc_allow_detach == 0) {
2431eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2432eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2433eb00b1c8SRobert Mustacchi }
2434eb00b1c8SRobert Mustacchi
2435eb00b1c8SRobert Mustacchi for (stub = avl_first(&imc->imc_stubs); stub != NULL;
2436eb00b1c8SRobert Mustacchi stub = AVL_NEXT(&imc->imc_stubs, stub)) {
2437eb00b1c8SRobert Mustacchi if (stub->istub_dip == dip) {
2438eb00b1c8SRobert Mustacchi break;
2439eb00b1c8SRobert Mustacchi }
2440eb00b1c8SRobert Mustacchi }
2441eb00b1c8SRobert Mustacchi
2442eb00b1c8SRobert Mustacchi /*
2443eb00b1c8SRobert Mustacchi * A device was attached to us that we somehow don't know about. Allow
2444eb00b1c8SRobert Mustacchi * this to proceed.
2445eb00b1c8SRobert Mustacchi */
2446eb00b1c8SRobert Mustacchi if (stub == NULL) {
2447eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2448eb00b1c8SRobert Mustacchi return (DDI_SUCCESS);
2449eb00b1c8SRobert Mustacchi }
2450eb00b1c8SRobert Mustacchi
2451eb00b1c8SRobert Mustacchi pci_config_teardown(&stub->istub_cfgspace);
2452eb00b1c8SRobert Mustacchi avl_remove(&imc->imc_stubs, stub);
2453eb00b1c8SRobert Mustacchi kmem_free(stub, sizeof (imc_stub_t));
2454eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2455eb00b1c8SRobert Mustacchi
2456eb00b1c8SRobert Mustacchi return (DDI_SUCCESS);
2457eb00b1c8SRobert Mustacchi }
2458eb00b1c8SRobert Mustacchi
2459eb00b1c8SRobert Mustacchi int
imc_attach_stub(dev_info_t * dip,ddi_attach_cmd_t cmd)2460eb00b1c8SRobert Mustacchi imc_attach_stub(dev_info_t *dip, ddi_attach_cmd_t cmd)
2461eb00b1c8SRobert Mustacchi {
2462eb00b1c8SRobert Mustacchi imc_stub_t *stub, *lookup;
2463eb00b1c8SRobert Mustacchi int did, vid, *regs;
2464eb00b1c8SRobert Mustacchi uint_t i, nregs;
2465eb00b1c8SRobert Mustacchi const imc_stub_table_t *table;
2466eb00b1c8SRobert Mustacchi avl_index_t idx;
2467eb00b1c8SRobert Mustacchi boolean_t dispatch = B_FALSE;
2468eb00b1c8SRobert Mustacchi imc_t *imc = imc_data;
2469eb00b1c8SRobert Mustacchi
2470eb00b1c8SRobert Mustacchi if (cmd != DDI_ATTACH) {
2471eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2472eb00b1c8SRobert Mustacchi }
2473eb00b1c8SRobert Mustacchi
2474eb00b1c8SRobert Mustacchi /*
2475eb00b1c8SRobert Mustacchi * We've been asked to attach a stub. First, determine if this is even a
2476eb00b1c8SRobert Mustacchi * PCI device that we should care about. Then, append it to our global
2477eb00b1c8SRobert Mustacchi * list and kick off the configuration task. Note that we do this
2478eb00b1c8SRobert Mustacchi * configuration task in a taskq so that we don't interfere with the
2479eb00b1c8SRobert Mustacchi * normal attach / detach path processing.
2480eb00b1c8SRobert Mustacchi */
2481eb00b1c8SRobert Mustacchi if (strncmp("pci", ddi_get_name(dip), 3) != 0) {
2482eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2483eb00b1c8SRobert Mustacchi }
2484eb00b1c8SRobert Mustacchi
2485eb00b1c8SRobert Mustacchi /*
2486eb00b1c8SRobert Mustacchi * Get the device and vendor ID and see if this is something the imc
2487eb00b1c8SRobert Mustacchi * knows about or cares about.
2488eb00b1c8SRobert Mustacchi */
2489eb00b1c8SRobert Mustacchi vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2490eb00b1c8SRobert Mustacchi "vendor-id", PCI_EINVAL16);
2491eb00b1c8SRobert Mustacchi did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2492eb00b1c8SRobert Mustacchi "device-id", PCI_EINVAL16);
2493eb00b1c8SRobert Mustacchi if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) {
2494eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2495eb00b1c8SRobert Mustacchi }
2496eb00b1c8SRobert Mustacchi
2497eb00b1c8SRobert Mustacchi /*
2498eb00b1c8SRobert Mustacchi * Only accept INTC parts on the imc driver.
2499eb00b1c8SRobert Mustacchi */
2500eb00b1c8SRobert Mustacchi if (vid != IMC_PCI_VENDOR_INTC) {
2501eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2502eb00b1c8SRobert Mustacchi }
2503eb00b1c8SRobert Mustacchi
2504eb00b1c8SRobert Mustacchi if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
2505eb00b1c8SRobert Mustacchi "reg", ®s, &nregs) != DDI_PROP_SUCCESS) {
2506eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2507eb00b1c8SRobert Mustacchi }
2508eb00b1c8SRobert Mustacchi
2509eb00b1c8SRobert Mustacchi if (nregs == 0) {
2510eb00b1c8SRobert Mustacchi ddi_prop_free(regs);
2511eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2512eb00b1c8SRobert Mustacchi }
2513eb00b1c8SRobert Mustacchi
2514eb00b1c8SRobert Mustacchi /*
2515eb00b1c8SRobert Mustacchi * Determine if this matches a known device.
2516eb00b1c8SRobert Mustacchi */
2517eb00b1c8SRobert Mustacchi table = NULL;
2518eb00b1c8SRobert Mustacchi for (i = 0; i < ARRAY_SIZE(imc_stub_table); i++) {
2519eb00b1c8SRobert Mustacchi if (imc_stub_table[i].imcs_devid == did &&
2520eb00b1c8SRobert Mustacchi imc_stub_table[i].imcs_pcidev == PCI_REG_DEV_G(regs[0]) &&
2521eb00b1c8SRobert Mustacchi imc_stub_table[i].imcs_pcifunc == PCI_REG_FUNC_G(regs[0])) {
2522eb00b1c8SRobert Mustacchi table = &imc_stub_table[i];
2523eb00b1c8SRobert Mustacchi break;
2524eb00b1c8SRobert Mustacchi }
2525eb00b1c8SRobert Mustacchi }
2526eb00b1c8SRobert Mustacchi
2527eb00b1c8SRobert Mustacchi if (i == ARRAY_SIZE(imc_stub_table)) {
2528eb00b1c8SRobert Mustacchi ddi_prop_free(regs);
2529eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2530eb00b1c8SRobert Mustacchi }
2531eb00b1c8SRobert Mustacchi
2532eb00b1c8SRobert Mustacchi /*
2533eb00b1c8SRobert Mustacchi * We've found something. Make sure the generation matches our current
2534eb00b1c8SRobert Mustacchi * one. If it does, construct the entry and append it to the list.
2535eb00b1c8SRobert Mustacchi */
2536eb00b1c8SRobert Mustacchi mutex_enter(&imc->imc_lock);
2537eb00b1c8SRobert Mustacchi if (imc->imc_gen != IMC_GEN_UNKNOWN && imc->imc_gen !=
2538eb00b1c8SRobert Mustacchi table->imcs_gen) {
2539eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2540eb00b1c8SRobert Mustacchi ddi_prop_free(regs);
2541eb00b1c8SRobert Mustacchi dev_err(dip, CE_WARN, "Encountered IMC stub device (%u/%u) "
2542eb00b1c8SRobert Mustacchi "that has different hardware generation (%u) from current "
2543eb00b1c8SRobert Mustacchi "generation (%u)", vid, did, table->imcs_gen, imc->imc_gen);
2544eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2545eb00b1c8SRobert Mustacchi } else {
2546eb00b1c8SRobert Mustacchi imc->imc_gen = table->imcs_gen;
2547eb00b1c8SRobert Mustacchi }
2548eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2549eb00b1c8SRobert Mustacchi
2550eb00b1c8SRobert Mustacchi stub = kmem_zalloc(sizeof (imc_stub_t), KM_SLEEP);
2551eb00b1c8SRobert Mustacchi stub->istub_dip = dip;
2552eb00b1c8SRobert Mustacchi stub->istub_vid = vid;
2553eb00b1c8SRobert Mustacchi stub->istub_did = did;
2554eb00b1c8SRobert Mustacchi stub->istub_bus = PCI_REG_BUS_G(regs[0]);
2555eb00b1c8SRobert Mustacchi stub->istub_dev = PCI_REG_DEV_G(regs[0]);
2556eb00b1c8SRobert Mustacchi stub->istub_func = PCI_REG_FUNC_G(regs[0]);
2557eb00b1c8SRobert Mustacchi ddi_prop_free(regs);
2558eb00b1c8SRobert Mustacchi stub->istub_table = table;
2559eb00b1c8SRobert Mustacchi
2560eb00b1c8SRobert Mustacchi if (pci_config_setup(dip, &stub->istub_cfgspace) != DDI_SUCCESS) {
2561eb00b1c8SRobert Mustacchi kmem_free(stub, sizeof (stub));
2562eb00b1c8SRobert Mustacchi dev_err(dip, CE_WARN, "Failed to set up PCI config space "
2563eb00b1c8SRobert Mustacchi "for IMC stub device %s (%u/%u)", ddi_node_name(dip),
2564eb00b1c8SRobert Mustacchi vid, did);
2565eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2566eb00b1c8SRobert Mustacchi }
2567eb00b1c8SRobert Mustacchi
2568eb00b1c8SRobert Mustacchi mutex_enter(&imc->imc_lock);
2569eb00b1c8SRobert Mustacchi if ((lookup = avl_find(&imc->imc_stubs, stub, &idx)) != NULL) {
2570eb00b1c8SRobert Mustacchi dev_err(dip, CE_WARN, "IMC stub %s (%u/%u) has duplicate "
2571eb00b1c8SRobert Mustacchi "bdf %u/%u/%u with %s (%u/%u), not attaching",
2572eb00b1c8SRobert Mustacchi ddi_node_name(imc->imc_dip), vid, did,
2573eb00b1c8SRobert Mustacchi stub->istub_bus, stub->istub_dev, stub->istub_func,
2574eb00b1c8SRobert Mustacchi ddi_node_name(lookup->istub_dip), lookup->istub_vid,
2575eb00b1c8SRobert Mustacchi lookup->istub_did);
2576eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2577eb00b1c8SRobert Mustacchi pci_config_teardown(&stub->istub_cfgspace);
2578eb00b1c8SRobert Mustacchi kmem_free(stub, sizeof (stub));
2579eb00b1c8SRobert Mustacchi
2580eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2581eb00b1c8SRobert Mustacchi }
2582eb00b1c8SRobert Mustacchi avl_insert(&imc->imc_stubs, stub, idx);
2583eb00b1c8SRobert Mustacchi
2584eb00b1c8SRobert Mustacchi if ((imc->imc_flags & IMC_F_ALL_FLAGS) == IMC_F_SCAN_COMPLETE &&
2585eb00b1c8SRobert Mustacchi avl_numnodes(&imc->imc_stubs) == imc->imc_nscanned) {
2586eb00b1c8SRobert Mustacchi imc->imc_flags |= IMC_F_ATTACH_DISPATCHED;
2587eb00b1c8SRobert Mustacchi dispatch = B_TRUE;
2588eb00b1c8SRobert Mustacchi }
2589eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2590eb00b1c8SRobert Mustacchi
2591eb00b1c8SRobert Mustacchi if (dispatch) {
2592eb00b1c8SRobert Mustacchi (void) ddi_taskq_dispatch(imc->imc_taskq, imc_attach_complete,
2593eb00b1c8SRobert Mustacchi imc, DDI_SLEEP);
2594eb00b1c8SRobert Mustacchi }
2595eb00b1c8SRobert Mustacchi
2596eb00b1c8SRobert Mustacchi return (DDI_SUCCESS);
2597eb00b1c8SRobert Mustacchi }
2598eb00b1c8SRobert Mustacchi
2599eb00b1c8SRobert Mustacchi static int
imc_open(dev_t * devp,int flag,int otyp,cred_t * credp)2600eb00b1c8SRobert Mustacchi imc_open(dev_t *devp, int flag, int otyp, cred_t *credp)
2601eb00b1c8SRobert Mustacchi {
2602eb00b1c8SRobert Mustacchi imc_t *imc = imc_data;
2603eb00b1c8SRobert Mustacchi
2604eb00b1c8SRobert Mustacchi if ((flag & (FEXCL | FNDELAY)) != 0)
2605eb00b1c8SRobert Mustacchi return (EINVAL);
2606eb00b1c8SRobert Mustacchi
2607eb00b1c8SRobert Mustacchi if (otyp != OTYP_CHR)
2608eb00b1c8SRobert Mustacchi return (EINVAL);
2609eb00b1c8SRobert Mustacchi
2610eb00b1c8SRobert Mustacchi mutex_enter(&imc->imc_lock);
2611eb00b1c8SRobert Mustacchi
2612eb00b1c8SRobert Mustacchi if ((imc->imc_flags & IMC_F_UNSUP_PLATFORM) != 0) {
2613eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2614eb00b1c8SRobert Mustacchi return (ENOTSUP);
2615eb00b1c8SRobert Mustacchi }
2616eb00b1c8SRobert Mustacchi
2617eb00b1c8SRobert Mustacchi /*
2618eb00b1c8SRobert Mustacchi * It's possible that someone has come in during the window between when
2619eb00b1c8SRobert Mustacchi * we've created the minor node and when we've finished doing work.
2620eb00b1c8SRobert Mustacchi */
2621eb00b1c8SRobert Mustacchi if ((imc->imc_flags & IMC_F_ATTACH_COMPLETE) == 0) {
2622eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2623eb00b1c8SRobert Mustacchi return (EAGAIN);
2624eb00b1c8SRobert Mustacchi }
2625eb00b1c8SRobert Mustacchi
2626eb00b1c8SRobert Mustacchi /*
2627eb00b1c8SRobert Mustacchi * It's not clear how someone would get a minor that we didn't create.
2628eb00b1c8SRobert Mustacchi * But be paranoid and make sure.
2629eb00b1c8SRobert Mustacchi */
2630eb00b1c8SRobert Mustacchi if (getminor(*devp) >= imc->imc_nsockets) {
2631eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2632eb00b1c8SRobert Mustacchi return (EINVAL);
2633eb00b1c8SRobert Mustacchi }
2634eb00b1c8SRobert Mustacchi
2635eb00b1c8SRobert Mustacchi /*
2636eb00b1c8SRobert Mustacchi * Make sure this socket entry has been filled in.
2637eb00b1c8SRobert Mustacchi */
2638eb00b1c8SRobert Mustacchi if (imc->imc_spointers[getminor(*devp)] == NULL) {
2639eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2640eb00b1c8SRobert Mustacchi return (EINVAL);
2641eb00b1c8SRobert Mustacchi }
2642eb00b1c8SRobert Mustacchi
2643eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2644eb00b1c8SRobert Mustacchi
2645eb00b1c8SRobert Mustacchi return (0);
2646eb00b1c8SRobert Mustacchi }
2647eb00b1c8SRobert Mustacchi
2648eb00b1c8SRobert Mustacchi static void
imc_ioctl_decode(imc_t * imc,mc_encode_ioc_t * encode)2649eb00b1c8SRobert Mustacchi imc_ioctl_decode(imc_t *imc, mc_encode_ioc_t *encode)
2650eb00b1c8SRobert Mustacchi {
2651eb00b1c8SRobert Mustacchi imc_decode_state_t dec;
2652eb00b1c8SRobert Mustacchi uint_t i;
2653eb00b1c8SRobert Mustacchi
2654eb00b1c8SRobert Mustacchi bzero(&dec, sizeof (dec));
2655eb00b1c8SRobert Mustacchi if (!imc_decode_pa(imc, encode->mcei_pa, &dec)) {
2656eb00b1c8SRobert Mustacchi encode->mcei_err = (uint32_t)dec.ids_fail;
2657eb00b1c8SRobert Mustacchi encode->mcei_errdata = dec.ids_fail_data;
2658eb00b1c8SRobert Mustacchi return;
2659eb00b1c8SRobert Mustacchi }
2660eb00b1c8SRobert Mustacchi
2661eb00b1c8SRobert Mustacchi encode->mcei_errdata = 0;
2662eb00b1c8SRobert Mustacchi encode->mcei_err = 0;
2663eb00b1c8SRobert Mustacchi encode->mcei_board = 0;
2664eb00b1c8SRobert Mustacchi for (i = 0; i < imc->imc_nsockets; i++) {
2665eb00b1c8SRobert Mustacchi if (imc->imc_spointers[i] == dec.ids_socket)
2666eb00b1c8SRobert Mustacchi break;
2667eb00b1c8SRobert Mustacchi }
2668eb00b1c8SRobert Mustacchi encode->mcei_chip = i;
2669*71815ce7SRobert Mustacchi /*
2670*71815ce7SRobert Mustacchi * These Intel platforms are all monolithic dies, so set the die to
2671*71815ce7SRobert Mustacchi * zero.
2672*71815ce7SRobert Mustacchi */
2673*71815ce7SRobert Mustacchi encode->mcei_die = 0;
2674eb00b1c8SRobert Mustacchi encode->mcei_mc = dec.ids_tadid;
2675*71815ce7SRobert Mustacchi encode->mcei_chan_addr = dec.ids_chanaddr;
2676eb00b1c8SRobert Mustacchi encode->mcei_chan = dec.ids_channelid;
2677eb00b1c8SRobert Mustacchi encode->mcei_dimm = dec.ids_dimmid;
2678eb00b1c8SRobert Mustacchi encode->mcei_rank_addr = dec.ids_rankaddr;
2679eb00b1c8SRobert Mustacchi encode->mcei_rank = dec.ids_rankid;
2680eb00b1c8SRobert Mustacchi encode->mcei_row = UINT32_MAX;
2681eb00b1c8SRobert Mustacchi encode->mcei_column = UINT32_MAX;
2682*71815ce7SRobert Mustacchi encode->mcei_cs = encode->mcei_rm = UINT8_MAX;
2683*71815ce7SRobert Mustacchi encode->mcei_bank = encode->mcei_bank_group = UINT8_MAX;
2684*71815ce7SRobert Mustacchi encode->mcei_subchan = UINT8_MAX;
2685eb00b1c8SRobert Mustacchi }
2686eb00b1c8SRobert Mustacchi
2687eb00b1c8SRobert Mustacchi static int
imc_ioctl(dev_t dev,int cmd,intptr_t arg,int mode,cred_t * credp,int * rvalp)2688eb00b1c8SRobert Mustacchi imc_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
2689eb00b1c8SRobert Mustacchi int *rvalp)
2690eb00b1c8SRobert Mustacchi {
2691eb00b1c8SRobert Mustacchi int ret;
2692eb00b1c8SRobert Mustacchi minor_t m;
2693eb00b1c8SRobert Mustacchi mc_snapshot_info_t info;
2694eb00b1c8SRobert Mustacchi mc_encode_ioc_t encode;
2695eb00b1c8SRobert Mustacchi imc_t *imc = imc_data;
2696eb00b1c8SRobert Mustacchi imc_socket_t *sock;
2697eb00b1c8SRobert Mustacchi
2698eb00b1c8SRobert Mustacchi mutex_enter(&imc->imc_lock);
2699eb00b1c8SRobert Mustacchi m = getminor(dev);
2700eb00b1c8SRobert Mustacchi if (m >= imc->imc_nsockets) {
2701eb00b1c8SRobert Mustacchi ret = EINVAL;
2702eb00b1c8SRobert Mustacchi goto done;
2703eb00b1c8SRobert Mustacchi }
2704eb00b1c8SRobert Mustacchi sock = imc->imc_spointers[m];
2705eb00b1c8SRobert Mustacchi if (sock == NULL) {
2706eb00b1c8SRobert Mustacchi ret = EINVAL;
2707eb00b1c8SRobert Mustacchi goto done;
2708eb00b1c8SRobert Mustacchi }
2709eb00b1c8SRobert Mustacchi
2710eb00b1c8SRobert Mustacchi /*
2711eb00b1c8SRobert Mustacchi * Note, other memory controller drivers don't check mode for reading
2712eb00b1c8SRobert Mustacchi * data nor do they care who can read it from a credential perspective.
2713eb00b1c8SRobert Mustacchi * As such we don't either at this time.
2714eb00b1c8SRobert Mustacchi */
2715eb00b1c8SRobert Mustacchi switch (cmd) {
2716eb00b1c8SRobert Mustacchi case MC_IOC_SNAPSHOT_INFO:
2717eb00b1c8SRobert Mustacchi imc_nvl_pack(sock, B_FALSE);
2718eb00b1c8SRobert Mustacchi if (sock->isock_buf == NULL) {
2719eb00b1c8SRobert Mustacchi ret = EIO;
2720eb00b1c8SRobert Mustacchi break;
2721eb00b1c8SRobert Mustacchi }
2722eb00b1c8SRobert Mustacchi
2723eb00b1c8SRobert Mustacchi info.mcs_size = sock->isock_buflen;
2724eb00b1c8SRobert Mustacchi info.mcs_gen = sock->isock_gen;
2725eb00b1c8SRobert Mustacchi
2726eb00b1c8SRobert Mustacchi if (ddi_copyout(&info, (void *)arg, sizeof (info), mode) != 0) {
2727eb00b1c8SRobert Mustacchi ret = EFAULT;
2728eb00b1c8SRobert Mustacchi break;
2729eb00b1c8SRobert Mustacchi }
2730eb00b1c8SRobert Mustacchi
2731eb00b1c8SRobert Mustacchi ret = 0;
2732eb00b1c8SRobert Mustacchi break;
2733eb00b1c8SRobert Mustacchi case MC_IOC_SNAPSHOT:
2734eb00b1c8SRobert Mustacchi imc_nvl_pack(sock, B_FALSE);
2735eb00b1c8SRobert Mustacchi if (sock->isock_buf == NULL) {
2736eb00b1c8SRobert Mustacchi ret = EIO;
2737eb00b1c8SRobert Mustacchi break;
2738eb00b1c8SRobert Mustacchi }
2739eb00b1c8SRobert Mustacchi
2740eb00b1c8SRobert Mustacchi if (ddi_copyout(sock->isock_buf, (void *)arg,
2741eb00b1c8SRobert Mustacchi sock->isock_buflen, mode) != 0) {
2742eb00b1c8SRobert Mustacchi ret = EFAULT;
2743eb00b1c8SRobert Mustacchi break;
2744eb00b1c8SRobert Mustacchi }
2745eb00b1c8SRobert Mustacchi
2746eb00b1c8SRobert Mustacchi ret = 0;
2747eb00b1c8SRobert Mustacchi break;
2748eb00b1c8SRobert Mustacchi case MC_IOC_DECODE_SNAPSHOT_INFO:
2749eb00b1c8SRobert Mustacchi imc_decoder_pack(imc);
2750eb00b1c8SRobert Mustacchi if (imc->imc_decoder_buf == NULL) {
2751eb00b1c8SRobert Mustacchi ret = EIO;
2752eb00b1c8SRobert Mustacchi break;
2753eb00b1c8SRobert Mustacchi }
2754eb00b1c8SRobert Mustacchi
2755eb00b1c8SRobert Mustacchi info.mcs_size = imc->imc_decoder_len;
2756eb00b1c8SRobert Mustacchi info.mcs_gen = imc->imc_spointers[0]->isock_gen;
2757eb00b1c8SRobert Mustacchi
2758eb00b1c8SRobert Mustacchi if (ddi_copyout(&info, (void *)arg, sizeof (info), mode) != 0) {
2759eb00b1c8SRobert Mustacchi ret = EFAULT;
2760eb00b1c8SRobert Mustacchi break;
2761eb00b1c8SRobert Mustacchi }
2762eb00b1c8SRobert Mustacchi
2763eb00b1c8SRobert Mustacchi ret = 0;
2764eb00b1c8SRobert Mustacchi break;
2765eb00b1c8SRobert Mustacchi case MC_IOC_DECODE_SNAPSHOT:
2766eb00b1c8SRobert Mustacchi imc_decoder_pack(imc);
2767eb00b1c8SRobert Mustacchi if (imc->imc_decoder_buf == NULL) {
2768eb00b1c8SRobert Mustacchi ret = EIO;
2769eb00b1c8SRobert Mustacchi break;
2770eb00b1c8SRobert Mustacchi }
2771eb00b1c8SRobert Mustacchi
2772eb00b1c8SRobert Mustacchi if (ddi_copyout(imc->imc_decoder_buf, (void *)arg,
2773eb00b1c8SRobert Mustacchi imc->imc_decoder_len, mode) != 0) {
2774eb00b1c8SRobert Mustacchi ret = EFAULT;
2775eb00b1c8SRobert Mustacchi break;
2776eb00b1c8SRobert Mustacchi }
2777eb00b1c8SRobert Mustacchi
2778eb00b1c8SRobert Mustacchi ret = 0;
2779eb00b1c8SRobert Mustacchi break;
2780eb00b1c8SRobert Mustacchi case MC_IOC_DECODE_PA:
2781eb00b1c8SRobert Mustacchi if (crgetzoneid(credp) != GLOBAL_ZONEID ||
2782eb00b1c8SRobert Mustacchi drv_priv(credp) != 0) {
2783eb00b1c8SRobert Mustacchi ret = EPERM;
2784eb00b1c8SRobert Mustacchi break;
2785eb00b1c8SRobert Mustacchi }
2786eb00b1c8SRobert Mustacchi
2787eb00b1c8SRobert Mustacchi if (ddi_copyin((void *)arg, &encode, sizeof (encode),
2788eb00b1c8SRobert Mustacchi mode & FKIOCTL) != 0) {
2789eb00b1c8SRobert Mustacchi ret = EPERM;
2790eb00b1c8SRobert Mustacchi break;
2791eb00b1c8SRobert Mustacchi }
2792eb00b1c8SRobert Mustacchi
2793eb00b1c8SRobert Mustacchi imc_ioctl_decode(imc, &encode);
2794eb00b1c8SRobert Mustacchi ret = 0;
2795eb00b1c8SRobert Mustacchi
2796eb00b1c8SRobert Mustacchi if (ddi_copyout(&encode, (void *)arg, sizeof (encode),
2797eb00b1c8SRobert Mustacchi mode & FKIOCTL) != 0) {
2798eb00b1c8SRobert Mustacchi ret = EPERM;
2799eb00b1c8SRobert Mustacchi break;
2800eb00b1c8SRobert Mustacchi }
2801eb00b1c8SRobert Mustacchi break;
2802eb00b1c8SRobert Mustacchi default:
2803eb00b1c8SRobert Mustacchi ret = EINVAL;
2804eb00b1c8SRobert Mustacchi goto done;
2805eb00b1c8SRobert Mustacchi }
2806eb00b1c8SRobert Mustacchi
2807eb00b1c8SRobert Mustacchi done:
2808eb00b1c8SRobert Mustacchi mutex_exit(&imc->imc_lock);
2809eb00b1c8SRobert Mustacchi return (ret);
2810eb00b1c8SRobert Mustacchi }
2811eb00b1c8SRobert Mustacchi
2812eb00b1c8SRobert Mustacchi static int
imc_close(dev_t dev,int flag,int otyp,cred_t * credp)2813eb00b1c8SRobert Mustacchi imc_close(dev_t dev, int flag, int otyp, cred_t *credp)
2814eb00b1c8SRobert Mustacchi {
2815eb00b1c8SRobert Mustacchi return (0);
2816eb00b1c8SRobert Mustacchi }
2817eb00b1c8SRobert Mustacchi
2818eb00b1c8SRobert Mustacchi static int
imc_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)2819eb00b1c8SRobert Mustacchi imc_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2820eb00b1c8SRobert Mustacchi {
2821eb00b1c8SRobert Mustacchi if (cmd != DDI_ATTACH) {
2822eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2823eb00b1c8SRobert Mustacchi }
2824eb00b1c8SRobert Mustacchi
2825eb00b1c8SRobert Mustacchi if (imc_data == NULL || imc_data->imc_dip != NULL) {
2826eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2827eb00b1c8SRobert Mustacchi }
2828eb00b1c8SRobert Mustacchi
2829eb00b1c8SRobert Mustacchi mutex_enter(&imc_data->imc_lock);
2830eb00b1c8SRobert Mustacchi if ((imc_data->imc_taskq = ddi_taskq_create(dip, "imc", 1,
2831eb00b1c8SRobert Mustacchi TASKQ_DEFAULTPRI, 0)) == NULL) {
2832eb00b1c8SRobert Mustacchi mutex_exit(&imc_data->imc_lock);
2833eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2834eb00b1c8SRobert Mustacchi }
2835eb00b1c8SRobert Mustacchi
2836eb00b1c8SRobert Mustacchi imc_data->imc_dip = dip;
2837eb00b1c8SRobert Mustacchi imc_data->imc_flags |= IMC_F_SCAN_DISPATCHED;
2838eb00b1c8SRobert Mustacchi mutex_exit(&imc_data->imc_lock);
2839eb00b1c8SRobert Mustacchi
2840eb00b1c8SRobert Mustacchi (void) ddi_taskq_dispatch(imc_data->imc_taskq, imc_stub_scan, imc_data,
2841eb00b1c8SRobert Mustacchi DDI_SLEEP);
2842eb00b1c8SRobert Mustacchi
2843eb00b1c8SRobert Mustacchi return (DDI_SUCCESS);
2844eb00b1c8SRobert Mustacchi }
2845eb00b1c8SRobert Mustacchi
2846eb00b1c8SRobert Mustacchi /*
2847eb00b1c8SRobert Mustacchi * We only export a single instance.
2848eb00b1c8SRobert Mustacchi */
2849eb00b1c8SRobert Mustacchi static int
imc_getinfo(dev_info_t * dip,ddi_info_cmd_t infocmd,void * arg,void ** resultp)2850eb00b1c8SRobert Mustacchi imc_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **resultp)
2851eb00b1c8SRobert Mustacchi {
2852eb00b1c8SRobert Mustacchi /*
2853eb00b1c8SRobert Mustacchi * getinfo(9E) shouldn't be called if we're not attached. But be
2854eb00b1c8SRobert Mustacchi * paranoid.
2855eb00b1c8SRobert Mustacchi */
2856eb00b1c8SRobert Mustacchi if (imc_data == NULL || imc_data->imc_dip == NULL) {
2857eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2858eb00b1c8SRobert Mustacchi }
2859eb00b1c8SRobert Mustacchi
2860eb00b1c8SRobert Mustacchi switch (infocmd) {
2861eb00b1c8SRobert Mustacchi case DDI_INFO_DEVT2DEVINFO:
2862eb00b1c8SRobert Mustacchi *resultp = imc_data->imc_dip;
2863eb00b1c8SRobert Mustacchi break;
2864eb00b1c8SRobert Mustacchi case DDI_INFO_DEVT2INSTANCE:
2865eb00b1c8SRobert Mustacchi *resultp = (void *)0;
2866eb00b1c8SRobert Mustacchi break;
2867eb00b1c8SRobert Mustacchi default:
2868eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2869eb00b1c8SRobert Mustacchi }
2870eb00b1c8SRobert Mustacchi
2871eb00b1c8SRobert Mustacchi return (DDI_SUCCESS);
2872eb00b1c8SRobert Mustacchi }
2873eb00b1c8SRobert Mustacchi
2874eb00b1c8SRobert Mustacchi static int
imc_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)2875eb00b1c8SRobert Mustacchi imc_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2876eb00b1c8SRobert Mustacchi {
2877eb00b1c8SRobert Mustacchi if (cmd != DDI_DETACH) {
2878eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2879eb00b1c8SRobert Mustacchi }
2880eb00b1c8SRobert Mustacchi
2881eb00b1c8SRobert Mustacchi if (imc_data == NULL || imc_data->imc_dip) {
2882eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2883eb00b1c8SRobert Mustacchi }
2884eb00b1c8SRobert Mustacchi
2885eb00b1c8SRobert Mustacchi mutex_enter(&imc_data->imc_lock);
2886eb00b1c8SRobert Mustacchi
2887eb00b1c8SRobert Mustacchi /*
2888eb00b1c8SRobert Mustacchi * While a scan or attach is outstanding, don't allow us to detach.
2889eb00b1c8SRobert Mustacchi */
2890eb00b1c8SRobert Mustacchi if ((imc_data->imc_flags &
2891eb00b1c8SRobert Mustacchi (IMC_F_SCAN_DISPATCHED | IMC_F_ATTACH_DISPATCHED)) != 0) {
2892eb00b1c8SRobert Mustacchi mutex_exit(&imc_data->imc_lock);
2893eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2894eb00b1c8SRobert Mustacchi }
2895eb00b1c8SRobert Mustacchi
2896eb00b1c8SRobert Mustacchi /*
2897eb00b1c8SRobert Mustacchi * Because the stub driver depends on the imc driver, we shouldn't be
2898eb00b1c8SRobert Mustacchi * able to have any entries in this list when we detach. However, we
2899eb00b1c8SRobert Mustacchi * check just to make sure.
2900eb00b1c8SRobert Mustacchi */
2901eb00b1c8SRobert Mustacchi if (!avl_is_empty(&imc_data->imc_stubs)) {
2902eb00b1c8SRobert Mustacchi mutex_exit(&imc_data->imc_lock);
2903eb00b1c8SRobert Mustacchi return (DDI_FAILURE);
2904eb00b1c8SRobert Mustacchi }
2905eb00b1c8SRobert Mustacchi
2906eb00b1c8SRobert Mustacchi nvlist_free(imc_data->imc_decoder_dump);
2907eb00b1c8SRobert Mustacchi imc_data->imc_decoder_dump = NULL;
2908eb00b1c8SRobert Mustacchi if (imc_data->imc_decoder_buf != NULL) {
2909eb00b1c8SRobert Mustacchi kmem_free(imc_data->imc_decoder_buf, imc_data->imc_decoder_len);
2910eb00b1c8SRobert Mustacchi imc_data->imc_decoder_buf = NULL;
2911eb00b1c8SRobert Mustacchi imc_data->imc_decoder_len = 0;
2912eb00b1c8SRobert Mustacchi }
2913eb00b1c8SRobert Mustacchi
2914eb00b1c8SRobert Mustacchi ddi_remove_minor_node(imc_data->imc_dip, NULL);
2915eb00b1c8SRobert Mustacchi imc_data->imc_dip = NULL;
2916eb00b1c8SRobert Mustacchi mutex_exit(&imc_data->imc_lock);
2917eb00b1c8SRobert Mustacchi
2918eb00b1c8SRobert Mustacchi ddi_taskq_wait(imc_data->imc_taskq);
2919eb00b1c8SRobert Mustacchi ddi_taskq_destroy(imc_data->imc_taskq);
2920eb00b1c8SRobert Mustacchi imc_data->imc_taskq = NULL;
2921eb00b1c8SRobert Mustacchi
2922eb00b1c8SRobert Mustacchi return (DDI_SUCCESS);
2923eb00b1c8SRobert Mustacchi }
2924eb00b1c8SRobert Mustacchi
2925eb00b1c8SRobert Mustacchi static void
imc_free(void)2926eb00b1c8SRobert Mustacchi imc_free(void)
2927eb00b1c8SRobert Mustacchi {
2928eb00b1c8SRobert Mustacchi if (imc_data == NULL) {
2929eb00b1c8SRobert Mustacchi return;
2930eb00b1c8SRobert Mustacchi }
2931eb00b1c8SRobert Mustacchi
2932eb00b1c8SRobert Mustacchi VERIFY(avl_is_empty(&imc_data->imc_stubs));
2933eb00b1c8SRobert Mustacchi avl_destroy(&imc_data->imc_stubs);
2934eb00b1c8SRobert Mustacchi mutex_destroy(&imc_data->imc_lock);
2935eb00b1c8SRobert Mustacchi kmem_free(imc_data, sizeof (imc_t));
2936eb00b1c8SRobert Mustacchi imc_data = NULL;
2937eb00b1c8SRobert Mustacchi }
2938eb00b1c8SRobert Mustacchi
2939eb00b1c8SRobert Mustacchi static void
imc_alloc(void)2940eb00b1c8SRobert Mustacchi imc_alloc(void)
2941eb00b1c8SRobert Mustacchi {
2942eb00b1c8SRobert Mustacchi imc_data = kmem_zalloc(sizeof (imc_t), KM_SLEEP);
2943eb00b1c8SRobert Mustacchi
2944eb00b1c8SRobert Mustacchi mutex_init(&imc_data->imc_lock, NULL, MUTEX_DRIVER, NULL);
2945eb00b1c8SRobert Mustacchi avl_create(&imc_data->imc_stubs, imc_stub_comparator,
2946eb00b1c8SRobert Mustacchi sizeof (imc_stub_t), offsetof(imc_stub_t, istub_link));
2947eb00b1c8SRobert Mustacchi }
2948eb00b1c8SRobert Mustacchi
2949eb00b1c8SRobert Mustacchi static struct cb_ops imc_cb_ops = {
2950eb00b1c8SRobert Mustacchi .cb_open = imc_open,
2951eb00b1c8SRobert Mustacchi .cb_close = imc_close,
2952eb00b1c8SRobert Mustacchi .cb_strategy = nodev,
2953eb00b1c8SRobert Mustacchi .cb_print = nodev,
2954eb00b1c8SRobert Mustacchi .cb_dump = nodev,
2955eb00b1c8SRobert Mustacchi .cb_read = nodev,
2956eb00b1c8SRobert Mustacchi .cb_write = nodev,
2957eb00b1c8SRobert Mustacchi .cb_ioctl = imc_ioctl,
2958eb00b1c8SRobert Mustacchi .cb_devmap = nodev,
2959eb00b1c8SRobert Mustacchi .cb_mmap = nodev,
2960eb00b1c8SRobert Mustacchi .cb_segmap = nodev,
2961eb00b1c8SRobert Mustacchi .cb_chpoll = nochpoll,
2962eb00b1c8SRobert Mustacchi .cb_prop_op = ddi_prop_op,
2963eb00b1c8SRobert Mustacchi .cb_flag = D_MP,
2964eb00b1c8SRobert Mustacchi .cb_rev = CB_REV,
2965eb00b1c8SRobert Mustacchi .cb_aread = nodev,
2966eb00b1c8SRobert Mustacchi .cb_awrite = nodev
2967eb00b1c8SRobert Mustacchi };
2968eb00b1c8SRobert Mustacchi
2969eb00b1c8SRobert Mustacchi static struct dev_ops imc_dev_ops = {
2970eb00b1c8SRobert Mustacchi .devo_rev = DEVO_REV,
2971eb00b1c8SRobert Mustacchi .devo_refcnt = 0,
2972eb00b1c8SRobert Mustacchi .devo_getinfo = imc_getinfo,
2973eb00b1c8SRobert Mustacchi .devo_identify = nulldev,
2974eb00b1c8SRobert Mustacchi .devo_probe = nulldev,
2975eb00b1c8SRobert Mustacchi .devo_attach = imc_attach,
2976eb00b1c8SRobert Mustacchi .devo_detach = imc_detach,
2977eb00b1c8SRobert Mustacchi .devo_reset = nodev,
2978eb00b1c8SRobert Mustacchi .devo_cb_ops = &imc_cb_ops,
2979eb00b1c8SRobert Mustacchi .devo_quiesce = ddi_quiesce_not_needed
2980eb00b1c8SRobert Mustacchi };
2981eb00b1c8SRobert Mustacchi
2982eb00b1c8SRobert Mustacchi static struct modldrv imc_modldrv = {
2983eb00b1c8SRobert Mustacchi .drv_modops = &mod_driverops,
2984eb00b1c8SRobert Mustacchi .drv_linkinfo = "Intel Integrated Memory Controller Driver",
2985eb00b1c8SRobert Mustacchi .drv_dev_ops = &imc_dev_ops
2986eb00b1c8SRobert Mustacchi };
2987eb00b1c8SRobert Mustacchi
2988eb00b1c8SRobert Mustacchi static struct modlinkage imc_modlinkage = {
2989eb00b1c8SRobert Mustacchi .ml_rev = MODREV_1,
2990eb00b1c8SRobert Mustacchi .ml_linkage = { &imc_modldrv, NULL }
2991eb00b1c8SRobert Mustacchi };
2992eb00b1c8SRobert Mustacchi
2993eb00b1c8SRobert Mustacchi int
_init(void)2994eb00b1c8SRobert Mustacchi _init(void)
2995eb00b1c8SRobert Mustacchi {
2996eb00b1c8SRobert Mustacchi int ret;
2997eb00b1c8SRobert Mustacchi
2998eb00b1c8SRobert Mustacchi if ((ret = mod_install(&imc_modlinkage)) == 0) {
2999eb00b1c8SRobert Mustacchi imc_alloc();
3000eb00b1c8SRobert Mustacchi }
3001eb00b1c8SRobert Mustacchi
3002eb00b1c8SRobert Mustacchi return (ret);
3003eb00b1c8SRobert Mustacchi }
3004eb00b1c8SRobert Mustacchi
3005eb00b1c8SRobert Mustacchi int
_info(struct modinfo * modinfop)3006eb00b1c8SRobert Mustacchi _info(struct modinfo *modinfop)
3007eb00b1c8SRobert Mustacchi {
3008eb00b1c8SRobert Mustacchi return (mod_info(&imc_modlinkage, modinfop));
3009eb00b1c8SRobert Mustacchi }
3010eb00b1c8SRobert Mustacchi
3011eb00b1c8SRobert Mustacchi int
_fini(void)3012eb00b1c8SRobert Mustacchi _fini(void)
3013eb00b1c8SRobert Mustacchi {
3014eb00b1c8SRobert Mustacchi int ret;
3015eb00b1c8SRobert Mustacchi
3016eb00b1c8SRobert Mustacchi if ((ret = mod_remove(&imc_modlinkage)) == 0) {
3017eb00b1c8SRobert Mustacchi imc_free();
3018eb00b1c8SRobert Mustacchi }
3019eb00b1c8SRobert Mustacchi return (ret);
3020eb00b1c8SRobert Mustacchi }
3021