xref: /illumos-gate/usr/src/uts/intel/io/imc/imc.h (revision 74079a53)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2019 Joyent, Inc.
14  */
15 
16 #ifndef _INTEL_IMC_H
17 #define	_INTEL_IMC_H
18 
19 #include <sys/types.h>
20 #include <sys/bitmap.h>
21 #include <sys/list.h>
22 #include <sys/sunddi.h>
23 
24 /*
25  * This header file contains the definitions used for the various generations of
26  * the Intel IMC driver.
27  */
28 
29 #ifdef __cplusplus
30 extern "C" {
31 #endif
32 
33 /*
34  * The maximum number of sockets that the IMC driver supports. This is currently
35  * determined by the Purley platforms (Skylake) which support up to 8 CPUs.
36  */
37 #define	IMC_MAX_SOCKETS		8
38 
39 /*
40  * The maximum number of memory controllers that exist per socket. Currently all
41  * supported platforms (Sandy Bridge -> Skylake) support at most two.
42  */
43 #define	IMC_MAX_IMCPERSOCK	2
44 
45 /*
46  * The maximum number of channels that exist per IMC. Currently Skylake supports
47  * 3 per IMC. On certain configurations of Haswell/Broadwell, there is only a
48  * single IMC which supports all 4 channels.
49  */
50 #define	IMC_MAX_CHANPERMC	4
51 
52 /*
53  * The maximum number of DIMMs that exist per channel. On Skylake this is two
54  * DIMMs. However, Sandy Bridge through Broadwell support three.
55  */
56 #define	IMC_MAX_DIMMPERCHAN	3
57 
58 /*
59  * The maximum number of rank disable bits per DIMM. This is currently
60  * consistent across all generations that have these bits.
61  */
62 #define	IMC_MAX_RANK_DISABLE	4
63 
64 /*
65  * The number of different PCI buses that we need to record for a given
66  * platform. Pre-Skylake there are only two that are required, one for the IIO
67  * and one for the non-IIO. On Skylake, more PCI buses are used.
68  */
69 #define	IMC_MAX_PCIBUSES	3
70 
71 /*
72  * Macros to take apart the node id for a given processor. These assume that
73  * we're reading the nodeid from the UBox and not from the SAD control.
74  */
75 #define	IMC_NODEID_UBOX_MASK(x)		((x) & 0x7)
76 
77 /*
78  * On Ivy Bridge through Broadwell, the node id that is found in the SAD targets
79  * has the HA indicator as NodeID[2]. This means that the actual target node of
80  * the socket is NodeID[3] | NodeID[1:0].
81  */
82 #define	IMC_NODEID_IVY_BRD_UPPER(x)	BITX(x, 3, 3)
83 #define	IMC_NODEID_IVY_BRD_LOWER(x)	BITX(x, 1, 0)
84 #define	IMC_NODEID_IVY_BRD_HA(x)	BITX(x, 2, 2)
85 
86 /*
87  * Macros to take apart the MCMTR register bits that we care about.
88  */
89 #define	IMC_MCMTR_CLOSED_PAGE(x)	BITX(x, 0, 0)
90 #define	IMC_MCMTR_LOCKSTEP(x)		BITX(x, 1, 1)
91 #define	IMC_MCMTR_ECC_ENABLED(x)	BITX(x, 2, 2)
92 
93 #define	IMC_MCMTR_DDR4_HAS_BRD(x)	BITX(x, 14, 14)
94 
95 /*
96  * Macros to take apart the dimmmtr_* registers in different generations. While
97  * there are similarities, these often end up different between generations and
98  * chips. These macros use a range of CPUs that they're valid for in the name.
99  * Macros with no suffix are valid for all currently supported CPUs.
100  */
101 
102 #define	IMC_REG_MC_MTR0		0x80
103 #define	IMC_REG_MC_MTR1		0x84
104 #define	IMC_REG_MC_MTR2		0x88
105 
106 #define	IMC_MTR_CA_WIDTH(x)	BITX(x, 1, 0)
107 #define	IMC_MTR_CA_BASE		10
108 #define	IMC_MTR_CA_MIN		10
109 #define	IMC_MTR_CA_MAX		12
110 
111 #define	IMC_MTR_RA_WIDTH(x)	BITX(x, 4, 2)
112 #define	IMC_MTR_RA_BASE		12
113 #define	IMC_MTR_RA_MIN		13
114 #define	IMC_MTR_RA_MAX		18
115 
116 #define	IMC_MTR_DENSITY_IVY_BRD(x)	BITX(x, 6, 5)
117 #define	IMC_MTR_DENSITY_SKX(x)		BITX(x, 7, 5)
118 
119 #define	IMC_MTR_WIDTH_IVB_HAS(x)	BITX(x, 8, 7)
120 #define	IMC_MTR_WIDTH_BRD_SKX(x)	BITX(x, 9, 8)
121 
122 #define	IMC_MTR_DDR_RANKS(x)		BITX(x, 13, 12)
123 #define	IMC_MTR_DDR_RANKS_MAX		4
124 #define	IMC_MTR_DDR_RANKS_MAX_HAS_SKX	8
125 
126 #define	IMC_MTR_PRESENT_SNB_BRD(x)	BITX(x, 14, 14)
127 #define	IMC_MTR_PRESENT_SKYLAKE(x)	BITX(x, 15, 15)
128 
129 #define	IMC_MTR_RANK_DISABLE(x)		BITX(x, 19, 16)
130 
131 #define	IMC_MTR_DDR4_ENABLE_HAS_BRD(x)	BITX(x, 20, 20)
132 #define	IMC_MTR_HDRL_HAS_SKX(x)		BITX(x, 21, 21)
133 #define	IMC_MTR_HDRL_PARITY_HAS_SKX(x)	BITX(x, 22, 22)
134 #define	IMC_MTR_3DSRANKS_HAS_SKX(x)	BITX(x, 24, 23)
135 
136 /*
137  * Data for the RASENABLES register.
138  */
139 #define	IMC_MC_MIRROR_SNB_BRD(x)	BITX(x, 0, 0)
140 
141 /*
142  * The maximum number of SAD rules that exist on all supported platforms.
143  */
144 #define	IMC_MAX_SAD_RULES	24
145 
146 /*
147  * The maximum number of targets that can be interleaved in a sad rule.
148  */
149 #define	IMC_MAX_SAD_INTERLEAVE	8
150 
151 /*
152  * The maximum number of route entries that exist in SAD. This is only used on
153  * SKX.
154  */
155 #define	IMC_MAX_SAD_MCROUTES	6
156 
157 /*
158  * Definitions used to decode the MC Route table. Note that at this time this is
159  * very Skylake specific (as it's the only platform it's supported on).
160  */
161 #define	IMC_REG_SKX_SAD_MC_ROUTE_TABLE	0xb4
162 #define	IMC_MC_ROUTE_RING_BITS		3
163 #define	IMC_MC_ROUTE_RING_MASK		0x7
164 #define	IMC_MC_ROUTE_CHAN_BITS		2
165 #define	IMC_MC_ROUTE_CHAN_MASK		0x3
166 #define	IMC_MC_ROUTE_CHAN_OFFSET	18
167 
168 /*
169  * Definitions to help decode TOLM (top of low memory) and TOHM (top of high
170  * memory). The way this is done varies based on generation. These regions are
171  * currently always 64-MByte aligned
172  *
173  * On Sandy Bridge and Ivy Bridge the low four bits of TOLM are bits 31:28. TOHM
174  * is a single register. Bits 20:0 map to bits 45:25. Both registers represent
175  * the upper limit (as in one higher than the max DRAM value).
176  *
177  * On Haswell through Skylake, TOLM is represented as a 32-bit quantity. No
178  * shifting is required. However, only bits 31:26 are present. TOHM is spread
179  * out among two registers. The lower 32-bits is masked in a similar fashion. In
180  * both cases, these registers represent an inclusive range where we don't care
181  * about other bits. To deal with this we'll increment the lowest bit we care
182  * about to make it an exclusive range.
183  *
184  * Based on the above, we have opted to make both ranges in the IMC driver
185  * normalized to an _exclusive_ value.
186  *
187  * Ivy Bridge has the values in both the CBo SAD and a VT-d section; however, we
188  * use the CBo SAD which is why it looks like Sandy Bridge and not Haswell.
189  */
190 
191 #define	IMC_TOLM_SNB_IVY_MASK		0xf
192 #define	IMC_TOLM_SNB_IVY_SHIFT		28
193 #define	IMC_TOHM_SNB_IVY_MASK		0x1fffff
194 #define	IMC_TOHM_SNB_IVY_SHIFT		25
195 
196 #define	IMC_TOLM_HAS_SKX_MASK		0xfc000000
197 #define	IMC_TOLM_HAS_SKY_EXCL		(1 << 26)
198 #define	IMC_TOHM_LOW_HAS_SKX_MASK	0xfc000000
199 #define	IMC_TOHM_HAS_SKY_EXCL		(1 << 26)
200 
201 /*
202  * Definitions to decode SAD values. These are sometimes subtlety different
203  * across generations.
204  */
205 #define	IMC_SAD_DRAM_RULE_ENABLE(x)		BITX(x, 0, 0)
206 
207 #define	IMC_SAD_DRAM_INTERLEAVE_SNB_BRD(x)	BITX(x, 1, 1)
208 #define	IMC_SAD_DRAM_INTERLEAVE_SNB_BRD_8t6XOR	0
209 #define	IMC_SAD_DRAM_INTERLEAVE_SNB_BRD_8t6	1
210 
211 #define	IMC_SAD_DRAM_INTERLEAVE_SKX(x)		BITX(x, 2, 1)
212 #define	IMC_SAD_DRAM_INTERLEAVE_SKX_8t6		0
213 #define	IMC_SAD_DRAM_INTERLEAVE_SKX_10t8	1
214 #define	IMC_SAD_DRAM_INTERLEAVE_SKX_14t12	2
215 #define	IMC_SAD_DRAM_INTERLEAVE_SKX_32t30	3
216 
217 #define	IMC_SAD_DRAM_ATTR_SNB_BRD(x)		BITX(x, 3, 2)
218 #define	IMC_SAD_DRAM_ATTR_SKX(x)		BITX(x, 4, 3)
219 #define	IMC_SAD_DRAM_ATTR_DRAM			0
220 #define	IMC_SAD_DRAM_ATTR_MMCFG			1
221 #define	IMC_SAD_DRAM_ATTR_NXM			2
222 
223 #define	IMC_SAD_DRAM_MOD23_SKX(x)		BITX(x, 6, 5)
224 #define	IMC_SAD_DRAM_MOD23_MOD3			0
225 #define	IMC_SAD_DRAM_MOD23_MOD2_C01		1
226 #define	IMC_SAD_DRAM_MOD23_MOD2_C12		2
227 #define	IMC_SAD_DRAM_MOD23_MOD2_C02		3
228 
229 #define	IMC_SAD_DRAM_LIMIT_SNB_BRD(x)		BITX(x, 25, 6)
230 #define	IMC_SAD_DRAM_LIMIT_SKX(x)		BITX(x, 26, 7)
231 #define	IMC_SAD_DRAM_LIMIT_SHIFT		26
232 #define	IMC_SAD_DRAM_LIMIT_EXCLUSIVE		(1 << IMC_SAD_DRAM_LIMIT_SHIFT)
233 
234 #define	IMC_SAD_DRAM_A7_IVB_BRD(x)		BITX(x, 26, 26)
235 #define	IMC_SAD_DRAM_MOD3_SKX(x)		BITX(x, 27, 27)
236 #define	IMC_SAD_DRAM_MOD3_MODE_SKX(x)		BITX(x, 31, 30)
237 #define	IMC_SAD_DRAM_MOD3_MODE_45t6		0
238 #define	IMC_SAD_DRAM_MOD3_MODE_45t8		1
239 #define	IMC_SAD_DRAM_MOD3_MODE_45t12		2
240 
241 #define	IMC_SAD_ILEAVE_SNB_MASK			0x7
242 #define	IMC_SAD_ILEAVE_SNB_LEN			3
243 #define	IMC_SAD_ILEAVE_IVB_SKX_MASK		0xf
244 #define	IMC_SAD_ILEAVE_IVB_SKX_LEN		4
245 
246 /*
247  * The interleave targets on Skylake use the upper bit to indicate whether it is
248  * referring to a local memory controller or if it actually refers to another
249  * node that is far away. The maximum value includes the upper bit which is used
250  * to indicate whether it is remote or far.
251  */
252 #define	IMC_SAD_ILEAVE_SKX_LOCAL(x)		BITX(x, 3, 3)
253 #define	IMC_SAD_ILEAVE_SKX_TARGET(x)		BITX(x, 2, 0)
254 #define	IMC_SAD_ILEAVE_SKX_MAX			0xf
255 
256 /*
257  * Maximum number of TAD tables that we need to consider. On Sandy Bridge
258  * through Broadwell this is based on the number of home agents that are present
259  * in the system. On Sandy Bridge there is one, on others, there are up to two.
260  * On Skylake, there is one TAD per IMC.
261  */
262 #define	IMC_MAX_TAD	2
263 
264 /*
265  * Maximum number of TAD rules on any of the supported processors.
266  */
267 #define	IMC_MAX_TAD_RULES	12
268 
269 /*
270  * Maximum number of interleave targets. Note, this only applies to Sandy Bridge
271  * through Broadwell. Skylake gets this information in another form.
272  */
273 #define	IMC_MAX_TAD_TARGETS	4
274 
275 /*
276  * Offset between the base TAD rule and the corresponding wayness rule on
277  * Skylake.
278  */
279 #define	IMC_SKX_WAYNESS_OFFSET	0x30
280 
281 /*
282  * Various macros to decode the TAD rules.
283  */
284 #define	IMC_TAD_LIMIT(x)		BITX(x, 31, 12)
285 #define	IMC_TAD_LIMIT_SHIFT		26
286 #define	IMC_TAD_LIMIT_EXCLUSIVE		(1 << IMC_TAD_LIMIT_SHIFT)
287 
288 #define	IMC_TAD_SOCK_WAY(x)		BITX(x, 11, 10)
289 #define	IMC_TAD_SOCK_WAY_1		0
290 #define	IMC_TAD_SOCK_WAY_2		1
291 #define	IMC_TAD_SOCK_WAY_4		2
292 #define	IMC_TAD_SOCK_WAY_8		3
293 #define	IMC_TAD_CHAN_WAY(x)		BITX(x, 9, 8)
294 #define	IMC_TAD_TARG3(x)		BITX(x, 7, 6)
295 #define	IMC_TAD_TARG2(x)		BITX(x, 5, 4)
296 #define	IMC_TAD_TARG1(x)		BITX(x, 3, 2)
297 #define	IMC_TAD_TARG0(x)		BITX(x, 1, 0)
298 
299 #define	IMC_TAD_SNB_BRD_NTARGETS	4
300 
301 /*
302  * These are registers specific to the Skylake and newer TAD BASE registers.
303  */
304 #define	IMC_TAD_BASE_BASE(x)		BITX(x, 31, 12)
305 #define	IMC_TAD_BASE_SHIFT		26
306 
307 #define	IMC_TAD_BASE_CHAN_GRAN(x)	BITX(x, 7, 6)
308 #define	IMC_TAD_BASE_CHAN_GRAN_64B	0
309 #define	IMC_TAD_BASE_CHAN_GRAN_256B	1
310 #define	IMC_TAD_BASE_CHAN_GRAN_4KB	2
311 
312 #define	IMC_TAD_BASE_SOCK_GRAN(x)	BITX(x, 5, 4)
313 #define	IMC_TAD_BASE_SOCK_GRAN_64B	0
314 #define	IMC_TAD_BASE_SOCK_GRAN_256B	1
315 #define	IMC_TAD_BASE_SOCK_GRAN_4KB	2
316 #define	IMC_TAD_BASE_SOCK_GRAN_1GB	3
317 
318 #define	IMC_TADCHAN_OFFSET_SNB_BRD(x)	BITX(x, 25, 6)
319 #define	IMC_TADCHAN_OFFSET_SKX(x)	BITX(x, 23, 4)
320 #define	IMC_TADCHAN_OFFSET_SHIFT	26
321 
322 /*
323  * Macros to get at various TAD features.
324  */
325 #define	IMC_TAD_SYSDEF_LOCKSTEP(x)	BITX(x, 7, 7)
326 #define	IMC_TAD_SYSDEF2_SHIFTUP(x)	BITX(x, 22, 22)
327 #define	IMC_TAD_SYSDEF2_CHANHASH(x)	BITX(x, 21, 21)
328 
329 /*
330  * Maximum number of different wayness entries that exist across the various IMC
331  * generations. Each wayness then has a maximum number of target entries.
332  */
333 #define	IMC_MAX_RANK_WAYS		5
334 #define	IMC_MAX_RANK_INTERLEAVES	8
335 
336 /*
337  * Macros to take apart the rank interleave wayness and offset registers.
338  */
339 #define	IMC_RIR_WAYNESS_ENABLED(x)	BITX(x, 31, 31)
340 #define	IMC_RIR_WAYNESS_WAY(x)		BITX(x, 29, 28)
341 #define	IMC_RIR_LIMIT_HAS_SKX(x)	BITX(x, 11, 1)
342 #define	IMC_RIR_LIMIT_SNB_IVB(x)	BITX(x, 10, 1)
343 #define	IMC_RIR_LIMIT_SHIFT		29
344 #define	IMC_RIR_LIMIT_EXCLUSIVE		(1 << IMC_RIR_LIMIT_SHIFT)
345 
346 /*
347  * Currently, everything other than Broadwell has the same value for the target
348  * offset.
349  */
350 #define	IMC_RIR_OFFSET_TARGET_BRD(x)		BITX(x, 23, 20)
351 #define	IMC_RIR_OFFSET_TARGET(x)		BITX(x, 19, 16)
352 #define	IMC_RIR_OFFSET_OFFSET_HAS_SKX(x)	BITX(x, 15, 2)
353 #define	IMC_RIR_OFFSET_OFFSET_SNB_IVB(x)	BITX(x, 14, 2)
354 #define	IMC_RIR_OFFSET_SHIFT			29
355 
356 /*
357  * Definitions to cover manipulations of open and closed pages.
358  */
359 #define	IMC_PAGE_BITS_CLOSED	6
360 #define	IMC_PAGE_BITS_OPEN	13
361 
362 /*
363  * Macros to decode and understand the CPUBUSNO registers in the UBOX_DECS.
364  */
365 #define	IMC_UBOX_CPUBUSNO_0(x)			BITX(x, 7, 0)
366 #define	IMC_UBOX_CPUBUSNO_1(x)			BITX(x, 15, 8)
367 #define	IMC_UBOX_CPUBUSNO_2(x)			BITX(x, 23, 16)
368 
369 /*
370  * Hardware generations supported by the IMC driver.
371  */
372 typedef enum {
373 	IMC_GEN_UNKNOWN = 0,
374 	IMC_GEN_SANDY,
375 	IMC_GEN_IVY,
376 	IMC_GEN_HASWELL,
377 	IMC_GEN_BROADWELL,
378 	/*
379 	 * IMC_GEN_SKYLAKE also covers Cascade Lake. The two are similar to the
380 	 * point of even having the same PCI IDs for all of the devices. The
381 	 * only difference in the cpuid signature between them is the stepping,
382 	 * hence we do not have a separate Cascade Lake target here, as it's
383 	 * really the same as Skylake.
384 	 */
385 	IMC_GEN_SKYLAKE
386 } imc_gen_t;
387 
388 /*
389  * Generation specific limits.
390  */
391 typedef struct imc_gen_data {
392 	uint_t	igd_max_sockets;
393 	uint_t	igd_max_imcs;
394 	uint_t	igd_max_channels;
395 	uint_t	igd_max_dimms;
396 	uint_t	igd_max_ranks;
397 	uint_t	igd_mtr_offsets[IMC_MAX_DIMMPERCHAN];
398 	uint_t	igd_mcmtr_offset;
399 	uint_t	igd_topo_offset;
400 	uint_t	igd_num_mcroutes;
401 	uint_t	igd_tolm_offset;
402 	uint_t	igd_tohm_low_offset;
403 	uint_t	igd_tohm_hi_offset;
404 	uint_t	igd_sad_dram_offset;
405 	uint_t	igd_sad_ndram_rules;
406 	uint_t	igd_sad_nodeid_offset;
407 	uint_t	igd_tad_nrules;
408 	uint_t	igd_tad_rule_offset;
409 	uint_t	igd_tad_chan_offset;
410 	uint_t	igd_tad_sysdef;
411 	uint_t	igd_tad_sysdef2;
412 	uint_t	igd_mc_mirror;
413 	uint_t	igd_rir_nways;
414 	uint_t	igd_rir_way_offset;
415 	uint_t	igd_rir_nileaves;
416 	uint_t	igd_rir_ileave_offset;
417 	uint_t	igd_ubox_cpubusno_offset;
418 } imc_gen_data_t;
419 
420 /*
421  * Different types of PCI devices that show up on the core that we may need to
422  * attach to.
423  */
424 typedef enum {
425 	IMC_TYPE_UNKNOWN = 0,
426 	IMC_TYPE_MC0_M2M,	/* SKX Only */
427 	IMC_TYPE_MC1_M2M,	/* SKX Only */
428 	IMC_TYPE_MC0_MAIN0,
429 	IMC_TYPE_MC0_MAIN1,
430 	IMC_TYPE_MC1_MAIN0,
431 	IMC_TYPE_MC1_MAIN1,
432 	IMC_TYPE_MC0_CHANNEL0,
433 	IMC_TYPE_MC0_CHANNEL1,
434 	IMC_TYPE_MC0_CHANNEL2,
435 	IMC_TYPE_MC0_CHANNEL3,
436 	IMC_TYPE_MC1_CHANNEL0,
437 	IMC_TYPE_MC1_CHANNEL1,
438 	IMC_TYPE_MC1_CHANNEL2,
439 	IMC_TYPE_MC1_CHANNEL3,
440 	IMC_TYPE_SAD_DRAM,
441 	IMC_TYPE_SAD_MMIO,
442 	/*
443 	 * We want to note which device has the TOLM and TOHM registers.
444 	 * Unfortunately this is a rather complicated affair. On Sandy Bridge
445 	 * they are a part of the IMC_TYPE_SAD_MMIO. On Ivy Bridge, it's on its
446 	 * own dedicated device on the CBo.
447 	 *
448 	 * On Haswell onward, these move to the VT-D misc. registers. On Haswell
449 	 * and Broadwell, only one of these exist in the system. However, on
450 	 * Skylake these exist per socket.
451 	 */
452 	IMC_TYPE_SAD_MISC,
453 	IMC_TYPE_VTD_MISC,
454 	/*
455 	 * On SKX this exists on a per-core basis. It contains the memory
456 	 * controller routing table.
457 	 */
458 	IMC_TYPE_SAD_MCROUTE,
459 	IMC_TYPE_UBOX,
460 	IMC_TYPE_UBOX_CPUBUSNO,
461 	IMC_TYPE_HA0,
462 	IMC_TYPE_HA1,
463 } imc_type_t;
464 
465 /*
466  * Each entry in the stub table represents a device that we might attach to in a
467  * given generation. This is only defined in the kernel to make it easier to
468  * build the imc decoder in userland for testing.
469  */
470 #ifdef	_KERNEL
471 typedef struct imc_stub_table {
472 	imc_gen_t	imcs_gen;
473 	imc_type_t	imcs_type;
474 	uint16_t	imcs_devid;
475 	uint16_t	imcs_pcidev;
476 	uint16_t	imcs_pcifunc;
477 	const char	*imcs_desc;
478 } imc_stub_table_t;
479 
480 typedef struct imc_stub {
481 	avl_node_t		istub_link;
482 	dev_info_t		*istub_dip;
483 	uint16_t		istub_vid;
484 	uint16_t		istub_did;
485 	uint16_t		istub_bus;
486 	uint16_t		istub_dev;
487 	uint16_t		istub_func;
488 	ddi_acc_handle_t	istub_cfgspace;
489 	const imc_stub_table_t	*istub_table;
490 } imc_stub_t;
491 #else
492 typedef struct imc_stub {
493 	void	*istub_unused;
494 } imc_stub_t;
495 #endif	/* _KERNEL */
496 
497 typedef enum {
498 	IMC_F_UNSUP_PLATFORM	= (1 << 0),
499 	IMC_F_SCAN_DISPATCHED	= (1 << 1),
500 	IMC_F_SCAN_COMPLETE	= (1 << 2),
501 	IMC_F_ATTACH_DISPATCHED	= (1 << 3),
502 	IMC_F_ATTACH_COMPLETE	= (1 << 4),
503 	IMC_F_MCREG_FAILED	= (1 << 5),
504 	IMC_F_VALIDATE_FAILED	= (1 << 6)
505 } imc_flags_t;
506 
507 #define	IMC_F_ALL_FLAGS	(IMC_F_UNSUP_PLATFORM | IMC_F_SCAN_DISPATCHED | \
508     IMC_F_SCAN_COMPLETE | IMC_F_ATTACH_DISPATCHED | IMC_F_ATTACH_COMPLETE | \
509     IMC_F_MCREG_FAILED | IMC_F_VALIDATE_FAILED)
510 
511 typedef enum imc_dimm_type {
512 	IMC_DIMM_UNKNOWN,
513 	IMC_DIMM_DDR3,
514 	IMC_DIMM_DDR4,
515 	IMC_DIMM_NVDIMM
516 } imc_dimm_type_t;
517 
518 typedef enum imc_dimm_valid {
519 	IMC_DIMM_V_VALID	= 0,
520 	IMC_DIMM_V_BAD_PCI_READ	= (1 << 0),
521 	IMC_DIMM_V_BAD_ROWS	= (1 << 1),
522 	IMC_DIMM_V_BAD_COLUMNS	= (1 << 2),
523 	IMC_DIMM_V_BAD_DENSITY	= (1 <<	3),
524 	IMC_DIMM_V_BAD_WIDTH	= (1 << 4),
525 	IMC_DIMM_V_BAD_RANKS	= (1 << 5)
526 } imc_dimm_valid_t;
527 
528 typedef struct imc_dimm {
529 	imc_dimm_valid_t	idimm_valid;
530 	boolean_t	idimm_present;
531 	uint8_t		idimm_3dsranks;
532 	boolean_t	idimm_hdrl_parity;
533 	boolean_t	idimm_hdrl;
534 	boolean_t	idimm_ranks_disabled[IMC_MAX_RANK_DISABLE];
535 	uint8_t		idimm_nbanks;
536 	uint8_t		idimm_nranks;
537 	uint8_t		idimm_width;
538 	uint8_t		idimm_density; /* In GiB */
539 	uint8_t		idimm_nrows;
540 	uint8_t		idimm_ncolumns;
541 	/* Synthesized */
542 	uint64_t	idimm_size;
543 	/* Raw data */
544 	uint32_t	idimm_mtr;
545 } imc_dimm_t;
546 
547 typedef struct imc_rank_ileave_entry {
548 	uint8_t		irle_target;
549 	uint64_t	irle_offset;
550 } imc_rank_ileave_entry_t;
551 
552 typedef struct imc_rank_ileave {
553 	boolean_t		irle_enabled;
554 	uint32_t		irle_raw;
555 	uint8_t			irle_nways;
556 	uint8_t			irle_nwaysbits;
557 	uint64_t		irle_limit;
558 	uint_t			irle_nentries;
559 	imc_rank_ileave_entry_t	irle_entries[IMC_MAX_RANK_INTERLEAVES];
560 } imc_rank_ileave_t;
561 
562 typedef enum imc_channel_valid {
563 	IMC_CHANNEL_V_VALID		= 0,
564 	IMC_CHANNEL_V_BAD_PCI_READ	= 1 << 0,
565 } imc_channel_valid_t;
566 
567 typedef struct imc_channel {
568 	imc_channel_valid_t	ich_valid;
569 	imc_stub_t		*ich_desc;
570 	uint_t			ich_ndimms;
571 	imc_dimm_t		ich_dimms[IMC_MAX_DIMMPERCHAN];
572 	uint_t			ich_ntad_offsets;
573 	uint32_t		ich_tad_offsets_raw[IMC_MAX_TAD_RULES];
574 	uint64_t		ich_tad_offsets[IMC_MAX_TAD_RULES];
575 	uint_t			ich_nrankileaves;
576 	imc_rank_ileave_t	ich_rankileaves[IMC_MAX_RANK_WAYS];
577 } imc_channel_t;
578 
579 typedef struct imc_controller {
580 	imc_stub_t	*icn_main0;
581 	imc_stub_t	*icn_main1;
582 	imc_stub_t	*icn_m2m;
583 	boolean_t	icn_invalid;
584 	imc_dimm_type_t	icn_dimm_type;
585 	boolean_t	icn_ecc;
586 	boolean_t	icn_lockstep;
587 	boolean_t	icn_closed;
588 	uint32_t	icn_topo;
589 	uint_t		icn_nchannels;
590 	imc_channel_t	icn_channels[IMC_MAX_CHANPERMC];
591 } imc_mc_t;
592 
593 typedef enum imc_sad_rule_type {
594 	IMC_SAD_TYPE_DRAM,
595 	IMC_SAD_TYPE_MMCFG,
596 	IMC_SAD_TYPE_NXM
597 } imc_sad_rule_type_t;
598 
599 typedef enum imc_sad_rule_imode {
600 	IMC_SAD_IMODE_8t6,
601 	IMC_SAD_IMODE_8t6XOR,
602 	IMC_SAD_IMODE_10t8,
603 	IMC_SAD_IMODE_14t12,
604 	IMC_SAD_IMODE_32t30
605 } imc_sad_rule_imode_t;
606 
607 typedef enum imc_sad_rule_mod_mode {
608 	IMC_SAD_MOD_MODE_NONE,
609 	IMC_SAD_MOD_MODE_45t6,
610 	IMC_SAD_MOD_MODE_45t8,
611 	IMC_SAD_MOD_MODE_45t12
612 } imc_sad_rule_mod_mode_t;
613 
614 typedef enum imc_sad_rule_mod_type {
615 	IMC_SAD_MOD_TYPE_NONE,
616 	IMC_SAD_MOD_TYPE_MOD3,
617 	IMC_SAD_MOD_TYPE_MOD2_01,
618 	IMC_SAD_MOD_TYPE_MOD2_12,
619 	IMC_SAD_MOD_TYPE_MOD2_02
620 } imc_sad_rule_mod_type_t;
621 
622 typedef struct imc_sad_mcroute_entry {
623 	uint8_t	ismce_imc;		/* ID of the target IMC */
624 	uint8_t	ismce_pchannel;		/* ID of the target physical channel */
625 } imc_sad_mcroute_entry_t;
626 
627 typedef struct imc_sad_mcroute_table {
628 	uint32_t		ismc_raw_mcroute;
629 	uint_t			ismc_nroutes;
630 	imc_sad_mcroute_entry_t	ismc_mcroutes[IMC_MAX_SAD_MCROUTES];
631 } imc_sad_mcroute_table_t;
632 
633 /*
634  * This rule represents a single SAD entry.
635  */
636 typedef struct imc_sad_rule {
637 	uint32_t		isr_raw_dram;
638 	uint32_t		isr_raw_interleave;
639 	boolean_t		isr_enable;
640 	boolean_t		isr_a7mode;
641 	boolean_t		isr_need_mod3;
642 	uint64_t		isr_limit;
643 	imc_sad_rule_type_t	isr_type;
644 	imc_sad_rule_imode_t	isr_imode;
645 	imc_sad_rule_mod_mode_t	isr_mod_mode;
646 	imc_sad_rule_mod_type_t	isr_mod_type;
647 	uint_t			isr_ntargets;
648 	uint8_t			isr_targets[IMC_MAX_SAD_INTERLEAVE];
649 } imc_sad_rule_t;
650 
651 typedef enum imc_sad_flags {
652 	IMC_SAD_MCROUTE_VALID	= 1 << 0,
653 } imc_sad_flags_t;
654 
655 typedef enum imc_sad_valid {
656 	IMC_SAD_V_VALID		= 0,
657 	IMC_SAD_V_BAD_PCI_READ	= 1 << 0,
658 	IMC_SAD_V_BAD_MCROUTE	= 1 << 1,
659 	IMC_SAD_V_BAD_DRAM_ATTR	= 1 << 2,
660 	IMC_SAD_V_BAD_MOD3	= 1 << 3,
661 } imc_sad_valid_t;
662 
663 typedef struct imc_sad {
664 	imc_sad_flags_t	isad_flags;
665 	imc_sad_valid_t	isad_valid;
666 	imc_stub_t	*isad_dram;
667 	imc_stub_t	*isad_mmio;
668 	imc_stub_t	*isad_tolh;
669 	uint64_t	isad_tolm;
670 	uint64_t	isad_tohm;
671 	uint_t		isad_nrules;
672 	imc_sad_rule_t	isad_rules[IMC_MAX_SAD_RULES];
673 	imc_sad_mcroute_table_t isad_mcroute;
674 } imc_sad_t;
675 
676 typedef enum imc_tad_gran {
677 	IMC_TAD_GRAN_64B = 0,
678 	IMC_TAD_GRAN_256B,
679 	IMC_TAD_GRAN_4KB,
680 	IMC_TAD_GRAN_1GB
681 } imc_tad_gran_t;
682 
683 typedef struct imc_tad_rule {
684 	uint64_t	itr_base;
685 	uint64_t	itr_limit;
686 	uint32_t	itr_raw;
687 	uint32_t	itr_raw_gran;
688 	uint8_t		itr_sock_way;
689 	uint8_t		itr_chan_way;
690 	imc_tad_gran_t	itr_sock_gran;
691 	imc_tad_gran_t	itr_chan_gran;
692 	uint_t		itr_ntargets;
693 	uint8_t		itr_targets[IMC_MAX_TAD_TARGETS];
694 } imc_tad_rule_t;
695 
696 typedef enum imc_tad_valid {
697 	IMC_TAD_V_VALID		= 1 << 0,
698 	IMC_TAD_V_BAD_PCI_READ	= 1 << 1,
699 	IMC_TAD_V_BAD_CHAN_GRAN	= 1 << 2
700 } imc_tad_valid_t;
701 
702 typedef enum imc_tad_flags {
703 	IMC_TAD_FLAG_CHANSHIFT	= 1 << 0,
704 	IMC_TAD_FLAG_CHANHASH	= 1 << 1,
705 	IMC_TAD_FLAG_MIRROR	= 1 << 2,
706 	IMC_TAD_FLAG_LOCKSTEP	= 1 << 3
707 } imc_tad_flags_t;
708 
709 typedef struct imc_tad {
710 	imc_tad_valid_t	itad_valid;
711 	imc_stub_t	*itad_stub;
712 	imc_tad_flags_t	itad_flags;
713 	uint_t		itad_nrules;
714 	imc_tad_rule_t	itad_rules[IMC_MAX_TAD_RULES];
715 } imc_tad_t;
716 
717 typedef enum imc_socket_valid {
718 	IMC_SOCKET_V_VALID	= 0,
719 	IMC_SOCKET_V_BAD_NODEID	= 1 << 0
720 } imc_socket_valid_t;
721 
722 typedef struct imc_socket {
723 	imc_socket_valid_t	isock_valid;
724 	uint_t			isock_bus[IMC_MAX_PCIBUSES];
725 	uint_t			isock_nbus;
726 	uint_t			isock_gen;
727 	nvlist_t		*isock_nvl;
728 	char			*isock_buf;
729 	size_t			isock_buflen;
730 	imc_sad_t		isock_sad;
731 	uint_t			isock_ntad;
732 	imc_tad_t		isock_tad[IMC_MAX_TAD];
733 	imc_stub_t		*isock_ubox;
734 	imc_stub_t		*isock_cpubusno;
735 	uint32_t		isock_nodeid;
736 	uint_t			isock_nimc;
737 	imc_mc_t		isock_imcs[IMC_MAX_IMCPERSOCK];
738 } imc_socket_t;
739 
740 typedef struct imc {
741 	/*
742 	 * The initial members here are only used in the kernel. This is done to
743 	 * make it easier for us to be able to define a version of this to use
744 	 * in testing.
745 	 */
746 #ifdef	_KERNEL
747 	dev_info_t	*imc_dip;
748 	kmutex_t	imc_lock;
749 	imc_flags_t	imc_flags;
750 	const imc_gen_data_t	*imc_gen_data;
751 	ddi_taskq_t	*imc_taskq;
752 	uint_t		imc_nscanned;
753 	avl_tree_t	imc_stubs;
754 	nvlist_t	*imc_decoder_dump;
755 	char		*imc_decoder_buf;
756 	size_t		imc_decoder_len;
757 #endif	/* _KERNEL */
758 	imc_gen_t	imc_gen;
759 
760 	/*
761 	 * Data about the memory in the system
762 	 */
763 	uint_t		imc_nsockets;
764 	imc_socket_t	imc_sockets[IMC_MAX_SOCKETS];
765 
766 #ifdef _KERNEL
767 	/*
768 	 * The imc_sockets[] array is organized based on increasing PCI Bus ID.
769 	 * This array maps the socket id that user land thinks of back to the
770 	 * actual underlying socket in case hardware does not put them in order.
771 	 */
772 	imc_socket_t	*imc_spointers[IMC_MAX_SOCKETS];
773 
774 	/*
775 	 * Store the IIO global VT-D misc. device. While there are sometimes
776 	 * multiple on the system, we only keep a single one around.
777 	 */
778 	imc_stub_t	*imc_gvtd_misc;
779 #endif
780 } imc_t;
781 
782 
783 /*
784  * Decoder failure reasons
785  */
786 typedef enum imc_decode_failure {
787 	IMC_DECODE_F_NONE = 0,
788 	/*
789 	 * Indicates that the memory address fell into a reserved legacy range.
790 	 * The legacy range index is stored in the failure data.
791 	 */
792 	IMC_DECODE_F_LEGACY_RANGE,
793 	/*
794 	 * Indicates that we had bad socket data. The socket in question is
795 	 * noted in the failure data.
796 	 */
797 	IMC_DECODE_F_BAD_SOCKET,
798 	/*
799 	 * Indicates that we had bad SAD data. The socket the SAD is associated
800 	 * with is noted in the failure data.
801 	 */
802 	IMC_DECODE_F_BAD_SAD,
803 	/*
804 	 * Indicates that the address was not contained in conventional, low,
805 	 * or high memory.
806 	 */
807 	IMC_DECODE_F_OUTSIDE_DRAM,
808 	/*
809 	 * Indicates that no valid SAD rule was found for the address.
810 	 */
811 	IMC_DECODE_F_NO_SAD_RULE,
812 	/*
813 	 * Indicates that the SAD interleave target was beyond the valid index.
814 	 */
815 	IMC_DECODE_F_BAD_SAD_INTERLEAVE,
816 	/*
817 	 * Indicates that the route suggested a remote processor we can't find.
818 	 */
819 	IMC_DECODE_F_BAD_REMOTE_MC_ROUTE,
820 	/*
821 	 * Indicates that we ended up in a loop trying to find the right socket
822 	 * to use.
823 	 */
824 	IMC_DECODE_F_SAD_SEARCH_LOOP,
825 	/*
826 	 * Indicates that we encountered a SAD rule that asked for inconsistent
827 	 * mod rules.
828 	 */
829 	IMC_DECODE_F_SAD_BAD_MOD,
830 	/*
831 	 * Indicates that the socket or tad rule we found doesn't actually point
832 	 * to something that we know about.
833 	 */
834 	IMC_DECODE_F_SAD_BAD_SOCKET,
835 	IMC_DECODE_F_SAD_BAD_TAD,
836 	/*
837 	 * Indicates that we could not find a matching tad rule.
838 	 */
839 	IMC_DECODE_F_NO_TAD_RULE,
840 	/*
841 	 * Indicates that we encountered the TAD channel 3-way interleave that
842 	 * we don't support.
843 	 */
844 	IMC_DECODE_F_TAD_3_ILEAVE,
845 	/*
846 	 * Indicates that we had a bad target index.
847 	 */
848 	IMC_DECODE_F_TAD_BAD_TARGET_INDEX,
849 	/*
850 	 * Indicates that we have a bad channel ID.
851 	 */
852 	IMC_DECODE_F_BAD_CHANNEL_ID,
853 	/*
854 	 * Indicates that the TAD rule offset in the channel interleave was
855 	 * incorrect.
856 	 */
857 	IMC_DECODE_F_BAD_CHANNEL_TAD_OFFSET,
858 	/*
859 	 * We couldn't find a valid rank interleave rule.
860 	 */
861 	IMC_DECODE_F_NO_RIR_RULE,
862 	/*
863 	 * Indicates that the index of the rank interleaving target was bad.
864 	 */
865 	IMC_DECODE_F_BAD_RIR_ILEAVE_TARGET,
866 	/*
867 	 * Indicates that the calculated DIMM represents an invalid DIMM that is
868 	 * beyond the number of supported DIMMS per channel on the platform.
869 	 */
870 	IMC_DECODE_F_BAD_DIMM_INDEX,
871 	/*
872 	 * Indicates that the specified DIMM is not preset; however, it is a
873 	 * valid DIMM number.
874 	 */
875 	IMC_DECODE_F_DIMM_NOT_PRESENT,
876 	/*
877 	 * Indicates that the specified rank on the DIMM is more than the number
878 	 * of ranks that the DIMM has.
879 	 */
880 	IMC_DECODE_F_BAD_DIMM_RANK,
881 	/*
882 	 * Indicates that the channel offset is larger than the system address,
883 	 * meaning that we would end up with an underflow if we continued. The
884 	 * equivalent is true for the rank address.
885 	 */
886 	IMC_DECODE_F_CHANOFF_UNDERFLOW,
887 	IMC_DECODE_F_RANKOFF_UNDERFLOW,
888 } imc_decode_failure_t;
889 
890 /*
891  * Decoder state tracking
892  */
893 typedef struct imc_decode_state {
894 	imc_decode_failure_t	ids_fail;
895 	uint64_t		ids_fail_data;
896 	uint64_t		ids_pa;
897 	uint64_t		ids_chanaddr;
898 	uint64_t		ids_rankaddr;
899 	uint32_t		ids_nodeid;
900 	uint32_t		ids_tadid;
901 	uint32_t		ids_channelid;
902 	uint32_t		ids_physrankid;
903 	uint32_t		ids_dimmid;
904 	uint32_t		ids_rankid;
905 	const imc_socket_t	*ids_socket;
906 	const imc_sad_t		*ids_sad;
907 	const imc_sad_rule_t	*ids_sad_rule;
908 	const imc_tad_t		*ids_tad;
909 	const imc_tad_rule_t	*ids_tad_rule;
910 	const imc_mc_t		*ids_mc;
911 	const imc_channel_t	*ids_chan;
912 	const imc_rank_ileave_t	*ids_rir;
913 	const imc_dimm_t	*ids_dimm;
914 } imc_decode_state_t;
915 
916 #ifdef	_KERNEL
917 
918 /*
919  * Functions needed for the stub drivers.
920  */
921 extern int imc_attach_stub(dev_info_t *, ddi_attach_cmd_t);
922 extern int imc_detach_stub(dev_info_t *, ddi_detach_cmd_t);
923 
924 /*
925  * Decoder related functions
926  */
927 extern void imc_decoder_init(imc_t *);
928 
929 extern nvlist_t *imc_dump_decoder(imc_t *);
930 #else	/* !_KERNEL */
931 extern boolean_t imc_restore_decoder(nvlist_t *, imc_t *);
932 #endif	/* _KERNEL */
933 
934 extern boolean_t imc_decode_pa(const imc_t *, uint64_t, imc_decode_state_t *);
935 
936 
937 #ifdef __cplusplus
938 }
939 #endif
940 
941 #endif /* _INTEL_IMC_H */
942