xref: /illumos-gate/usr/src/uts/i86pc/sys/immu.h (revision 7fd1b424)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Portions Copyright (c) 2010, Oracle and/or its affiliates.
23  * All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2008, Intel Corporation.
28  * All rights reserved.
29  */
30 
31 #ifndef	_SYS_INTEL_IOMMU_H
32 #define	_SYS_INTEL_IOMMU_H
33 
34 /*
35  * Intel IOMMU implementation specific state
36  */
37 
38 #ifdef	__cplusplus
39 extern "C" {
40 #endif
41 
42 #include <sys/types.h>
43 #include <sys/bitset.h>
44 #include <sys/kstat.h>
45 #include <sys/kmem.h>
46 #include <sys/vmem.h>
47 #include <sys/rootnex.h>
48 #include <sys/iommulib.h>
49 #include <sys/sdt.h>
50 
51 /*
52  * Some ON drivers have bugs. Keep this define until all such drivers
53  * have been fixed
54  */
55 #define	BUGGY_DRIVERS 1
56 
57 /* PD(T)E entries */
58 typedef uint64_t hw_pdte_t;
59 
60 #define	IMMU_MAXNAMELEN (64)
61 #define	IMMU_MAXSEG	(1)
62 #define	IMMU_REGSZ	(1UL << 12)
63 #define	IMMU_PAGESIZE   (4096)
64 #define	IMMU_PAGESHIFT	(12)
65 #define	IMMU_PAGEOFFSET	(IMMU_PAGESIZE - 1)
66 #define	IMMU_PAGEMASK	(~IMMU_PAGEOFFSET)
67 #define	IMMU_BTOP(b)	(((uint64_t)b) >> IMMU_PAGESHIFT)
68 #define	IMMU_PTOB(p)	(((uint64_t)p) << IMMU_PAGESHIFT)
69 #define	IMMU_BTOPR(x)	((((x) + IMMU_PAGEOFFSET) >> IMMU_PAGESHIFT))
70 #define	IMMU_PGTABLE_MAX_LEVELS	(6)
71 #define	IMMU_ROUNDUP(size) (((size) + IMMU_PAGEOFFSET) & ~IMMU_PAGEOFFSET)
72 #define	IMMU_ROUNDOWN(addr) ((addr) & ~IMMU_PAGEOFFSET)
73 #define	IMMU_PGTABLE_LEVEL_STRIDE	(9)
74 #define	IMMU_PGTABLE_LEVEL_MASK	((1<<IMMU_PGTABLE_LEVEL_STRIDE) - 1)
75 #define	IMMU_PGTABLE_OFFSHIFT  (IMMU_PAGESHIFT - IMMU_PGTABLE_LEVEL_STRIDE)
76 #define	IMMU_PGTABLE_MAXIDX  ((IMMU_PAGESIZE / sizeof (hw_pdte_t)) - 1)
77 
78 /*
79  * DMAR global defines
80  */
81 #define	DMAR_TABLE	"dmar-table"
82 #define	DMAR_INTRMAP_SUPPORT	(0x01)
83 
84 /* DMAR unit types */
85 #define	DMAR_DRHD	0
86 #define	DMAR_RMRR	1
87 #define	DMAR_ATSR	2
88 #define	DMAR_RHSA	3
89 
90 /* DRHD flag values */
91 #define	DMAR_INCLUDE_ALL	(0x01)
92 
93 /* Device scope types */
94 #define	DMAR_ENDPOINT	1
95 #define	DMAR_SUBTREE	2
96 #define	DMAR_IOAPIC	3
97 #define	DMAR_HPET	4
98 
99 
100 /* Forward declarations for IOMMU state structure and DVMA domain struct */
101 struct immu;
102 struct domain;
103 
104 /*
105  * The following structure describes the formate of DMAR ACPI table format.
106  * They are used to parse DMAR ACPI table. Read the spec for the meaning
107  * of each member.
108  */
109 
110 /* lengths of various strings */
111 #define	DMAR_SIG_LEN    (4)	/* table signature */
112 #define	DMAR_OEMID_LEN  (6)	/* OEM ID */
113 #define	DMAR_TBLID_LEN  (8)	/* OEM table ID */
114 #define	DMAR_ASL_LEN    (4)	/* ASL len */
115 
116 typedef struct dmar_table {
117 	kmutex_t	tbl_lock;
118 	uint8_t		tbl_haw;
119 	boolean_t	tbl_intrmap;
120 	list_t		tbl_drhd_list[IMMU_MAXSEG];
121 	list_t		tbl_rmrr_list[IMMU_MAXSEG];
122 	char		*tbl_oem_id;
123 	char		*tbl_oem_tblid;
124 	uint32_t	tbl_oem_rev;
125 	caddr_t		tbl_raw;
126 	int		tbl_rawlen;
127 } dmar_table_t;
128 
129 typedef struct drhd {
130 	kmutex_t	dr_lock;   /* protects the dmar field */
131 	struct immu	*dr_immu;
132 	dev_info_t	*dr_dip;
133 	uint16_t	dr_seg;
134 	uint64_t	dr_regs;
135 	boolean_t	dr_include_all;
136 	list_t		dr_scope_list;
137 	list_node_t	dr_node;
138 } drhd_t;
139 
140 typedef struct rmrr {
141 	kmutex_t	rm_lock;
142 	uint16_t	rm_seg;
143 	uint64_t	rm_base;
144 	uint64_t	rm_limit;
145 	list_t		rm_scope_list;
146 	list_node_t	rm_node;
147 } rmrr_t;
148 
149 #define	IMMU_UNIT_NAME	"iommu"
150 
151 /*
152  * Macros based on PCI spec
153  */
154 #define	IMMU_PCI_DEV(devfunc)    ((uint64_t)devfunc >> 3) /* from devfunc  */
155 #define	IMMU_PCI_FUNC(devfunc)   (devfunc & 7)  /* get func from devfunc */
156 #define	IMMU_PCI_DEVFUNC(d, f)   (((d) << 3) | (f))  /* create devfunc */
157 
158 typedef struct scope {
159 	uint8_t scp_type;
160 	uint8_t scp_enumid;
161 	uint8_t scp_bus;
162 	uint8_t scp_dev;
163 	uint8_t scp_func;
164 	list_node_t scp_node;
165 } scope_t;
166 
167 /*
168  * interrupt source id and drhd info for ioapic
169  */
170 typedef struct ioapic_drhd {
171 	uchar_t		ioapic_ioapicid;
172 	uint16_t	ioapic_sid;	/* ioapic source id */
173 	drhd_t		*ioapic_drhd;
174 	list_node_t	ioapic_node;
175 } ioapic_drhd_t;
176 
177 typedef struct memrng {
178 	uint64_t mrng_start;
179 	uint64_t mrng_npages;
180 } memrng_t;
181 
182 typedef enum immu_flags {
183 	IMMU_FLAGS_NONE = 0x1,
184 	IMMU_FLAGS_SLEEP = 0x1,
185 	IMMU_FLAGS_NOSLEEP = 0x2,
186 	IMMU_FLAGS_READ = 0x4,
187 	IMMU_FLAGS_WRITE = 0x8,
188 	IMMU_FLAGS_DONTPASS = 0x10,
189 	IMMU_FLAGS_ALLOC = 0x20,
190 	IMMU_FLAGS_MUST_MATCH = 0x40,
191 	IMMU_FLAGS_PAGE1 = 0x80,
192 	IMMU_FLAGS_UNITY = 0x100,
193 	IMMU_FLAGS_DMAHDL = 0x200,
194 	IMMU_FLAGS_MEMRNG = 0x400
195 } immu_flags_t;
196 
197 typedef enum cont_avail {
198 	IMMU_CONT_BAD = 0x0,
199 	IMMU_CONT_UNINITED = 0x1,
200 	IMMU_CONT_INITED = 0x2
201 } cont_avail_t;
202 
203 /* Size of root and context tables and their entries */
204 #define	IMMU_ROOT_TBLSZ		(4096)
205 #define	IMMU_CONT_TBLSZ		(4096)
206 #define	IMMU_ROOT_NUM		(256)
207 #define	IMMU_CONT_NUM		(256)
208 
209 /* register offset */
210 #define	IMMU_REG_VERSION	(0x00)  /* Version Rigister, 32 bit */
211 #define	IMMU_REG_CAP		(0x08)  /* Capability Register, 64 bit */
212 #define	IMMU_REG_EXCAP		(0x10)  /* Extended Capability Reg, 64 bit */
213 #define	IMMU_REG_GLOBAL_CMD	(0x18)  /* Global Command Register, 32 bit */
214 #define	IMMU_REG_GLOBAL_STS	(0x1C)  /* Global Status Register, 32 bit */
215 #define	IMMU_REG_ROOTENTRY	(0x20)  /* Root-Entry Table Addr Reg, 64 bit */
216 #define	IMMU_REG_CONTEXT_CMD	(0x28)  /* Context Comand Register, 64 bit */
217 #define	IMMU_REG_FAULT_STS	(0x34)  /* Fault Status Register, 32 bit */
218 #define	IMMU_REG_FEVNT_CON	(0x38)  /* Fault Event Control Reg, 32 bit */
219 #define	IMMU_REG_FEVNT_DATA	(0x3C)  /* Fault Event Data Register, 32 bit */
220 #define	IMMU_REG_FEVNT_ADDR	(0x40)  /* Fault Event Address Reg, 32 bit */
221 #define	IMMU_REG_FEVNT_UADDR	(0x44)  /* Fault Event Upper Addr Reg, 32 bit */
222 #define	IMMU_REG_AFAULT_LOG	(0x58)  /* Advanced Fault Log Reg, 64 bit */
223 #define	IMMU_REG_PMER		(0x64)  /* Protected Memory Enble Reg, 32 bit */
224 #define	IMMU_REG_PLMBR		(0x68)  /* Protected Low Mem Base Reg, 32 bit */
225 #define	IMMU_REG_PLMLR		(0x6C)  /* Protected Low Mem Lim Reg, 32 bit */
226 #define	IMMU_REG_PHMBR		(0X70)  /* Protectd High Mem Base Reg, 64 bit */
227 #define	IMMU_REG_PHMLR		(0x78)  /* Protected High Mem Lim Reg, 64 bit */
228 #define	IMMU_REG_INVAL_QH	(0x80)  /* Invalidation Queue Head, 64 bit */
229 #define	IMMU_REG_INVAL_QT	(0x88)  /* Invalidation Queue Tail, 64 bit */
230 #define	IMMU_REG_INVAL_QAR	(0x90)  /* Invalidtion Queue Addr Reg, 64 bit */
231 #define	IMMU_REG_INVAL_CSR	(0x9C)  /* Inval Compl Status Reg, 32 bit */
232 #define	IMMU_REG_INVAL_CECR	(0xA0)  /* Inval Compl Evnt Ctrl Reg, 32 bit */
233 #define	IMMU_REG_INVAL_CEDR	(0xA4)  /* Inval Compl Evnt Data Reg, 32 bit */
234 #define	IMMU_REG_INVAL_CEAR	(0xA8)  /* Inval Compl Event Addr Reg, 32 bit */
235 #define	IMMU_REG_INVAL_CEUAR	(0xAC)  /* Inval Comp Evnt Up Addr reg, 32bit */
236 #define	IMMU_REG_IRTAR		(0xB8)  /* INTR Remap Tbl Addr Reg, 64 bit */
237 
238 /* ioapic memory region */
239 #define	IOAPIC_REGION_START	(0xfee00000)
240 #define	IOAPIC_REGION_END	(0xfeefffff)
241 
242 /* fault register */
243 #define	IMMU_FAULT_STS_PPF		(2)
244 #define	IMMU_FAULT_STS_PFO		(1)
245 #define	IMMU_FAULT_STS_ITE		(1 << 6)
246 #define	IMMU_FAULT_STS_ICE		(1 << 5)
247 #define	IMMU_FAULT_STS_IQE		(1 << 4)
248 #define	IMMU_FAULT_GET_INDEX(x)		((((uint64_t)x) >> 8) & 0xff)
249 #define	IMMU_FRR_GET_F(x)		(((uint64_t)x) >> 63)
250 #define	IMMU_FRR_GET_FR(x)		((((uint64_t)x) >> 32) & 0xff)
251 #define	IMMU_FRR_GET_FT(x)		((((uint64_t)x) >> 62) & 0x1)
252 #define	IMMU_FRR_GET_SID(x)		((x) & 0xffff)
253 
254 /* (ex)capability register */
255 #define	IMMU_CAP_GET_NFR(x)		(((((uint64_t)x) >> 40) & 0xff) + 1)
256 #define	IMMU_CAP_GET_DWD(x)		((((uint64_t)x) >> 54) & 1)
257 #define	IMMU_CAP_GET_DRD(x)		((((uint64_t)x) >> 55) & 1)
258 #define	IMMU_CAP_GET_PSI(x)		((((uint64_t)x) >> 39) & 1)
259 #define	IMMU_CAP_GET_SPS(x)		((((uint64_t)x) >> 34) & 0xf)
260 #define	IMMU_CAP_GET_ISOCH(x)		((((uint64_t)x) >> 23) & 1)
261 #define	IMMU_CAP_GET_ZLR(x)		((((uint64_t)x) >> 22) & 1)
262 #define	IMMU_CAP_GET_MAMV(x)		((((uint64_t)x) >> 48) & 0x3f)
263 #define	IMMU_CAP_GET_CM(x)		((((uint64_t)x) >> 7) & 1)
264 #define	IMMU_CAP_GET_PHMR(x)		((((uint64_t)x) >> 6) & 1)
265 #define	IMMU_CAP_GET_PLMR(x)		((((uint64_t)x) >> 5) & 1)
266 #define	IMMU_CAP_GET_RWBF(x)		((((uint64_t)x) >> 4) & 1)
267 #define	IMMU_CAP_GET_AFL(x)		((((uint64_t)x) >> 3) & 1)
268 #define	IMMU_CAP_GET_FRO(x)		(((((uint64_t)x) >> 24) & 0x3ff) * 16)
269 #define	IMMU_CAP_MGAW(x)		(((((uint64_t)x) >> 16) & 0x3f) + 1)
270 #define	IMMU_CAP_SAGAW(x)		((((uint64_t)x) >> 8) & 0x1f)
271 #define	IMMU_CAP_ND(x)			(1 << (((x) & 0x7) *2 + 4)) -1
272 #define	IMMU_ECAP_GET_IRO(x)		(((((uint64_t)x) >> 8) & 0x3ff) << 4)
273 #define	IMMU_ECAP_GET_MHMV(x)		(((uint64_t)x >> 20) & 0xf)
274 #define	IMMU_ECAP_GET_SC(x)		((x) & 0x80)
275 #define	IMMU_ECAP_GET_PT(x)		((x) & 0x40)
276 #define	IMMU_ECAP_GET_CH(x)		((x) & 0x20)
277 #define	IMMU_ECAP_GET_EIM(x)		((x) & 0x10)
278 #define	IMMU_ECAP_GET_IR(x)		((x) & 0x8)
279 #define	IMMU_ECAP_GET_DI(x)		((x) & 0x4)
280 #define	IMMU_ECAP_GET_QI(x)		((x) & 0x2)
281 #define	IMMU_ECAP_GET_C(x)		((x) & 0x1)
282 
283 #define	IMMU_CAP_SET_RWBF(x)		((x) |= (1 << 4))
284 
285 
286 /* iotlb invalidation */
287 #define	TLB_INV_GLOBAL		(((uint64_t)1) << 60)
288 #define	TLB_INV_DOMAIN		(((uint64_t)2) << 60)
289 #define	TLB_INV_PAGE		(((uint64_t)3) << 60)
290 #define	TLB_INV_GET_IAIG(x)	((((uint64_t)x) >> 57) & 7)
291 #define	TLB_INV_DRAIN_READ	(((uint64_t)1) << 49)
292 #define	TLB_INV_DRAIN_WRITE	(((uint64_t)1) << 48)
293 #define	TLB_INV_DID(x)		(((uint64_t)((x) & 0xffff)) << 32)
294 #define	TLB_INV_IVT		(((uint64_t)1) << 63)
295 #define	TLB_IVA_HINT(x)		(((x) & 0x1) << 6)
296 #define	TLB_IVA_LEAF		1
297 #define	TLB_IVA_WHOLE		0
298 
299 /* dont use value 0 for  enums - to catch unit 8 */
300 typedef enum iotlb_inv {
301 	IOTLB_PSI = 1,
302 	IOTLB_DSI,
303 	IOTLB_GLOBAL
304 } immu_iotlb_inv_t;
305 
306 typedef enum context_inv {
307 	CONTEXT_FSI = 1,
308 	CONTEXT_DSI,
309 	CONTEXT_GLOBAL
310 } immu_context_inv_t;
311 
312 /* context invalidation */
313 #define	CCMD_INV_ICC		(((uint64_t)1) << 63)
314 #define	CCMD_INV_GLOBAL		(((uint64_t)1) << 61)
315 #define	CCMD_INV_DOMAIN		(((uint64_t)2) << 61)
316 #define	CCMD_INV_DEVICE		(((uint64_t)3) << 61)
317 #define	CCMD_INV_DID(x)		((uint64_t)((x) & 0xffff))
318 #define	CCMD_INV_SID(x)		(((uint64_t)((x) & 0xffff)) << 16)
319 #define	CCMD_INV_FM(x)		(((uint64_t)((x) & 0x3)) << 32)
320 
321 /* global command register */
322 #define	IMMU_GCMD_TE		(((uint32_t)1) << 31)
323 #define	IMMU_GCMD_SRTP		(((uint32_t)1) << 30)
324 #define	IMMU_GCMD_SFL		(((uint32_t)1) << 29)
325 #define	IMMU_GCMD_EAFL		(((uint32_t)1) << 28)
326 #define	IMMU_GCMD_WBF		(((uint32_t)1) << 27)
327 #define	IMMU_GCMD_QIE		(((uint32_t)1) << 26)
328 #define	IMMU_GCMD_IRE		(((uint32_t)1) << 25)
329 #define	IMMU_GCMD_SIRTP	(((uint32_t)1) << 24)
330 #define	IMMU_GCMD_CFI		(((uint32_t)1) << 23)
331 
332 /* global status register */
333 #define	IMMU_GSTS_TES		(((uint32_t)1) << 31)
334 #define	IMMU_GSTS_RTPS		(((uint32_t)1) << 30)
335 #define	IMMU_GSTS_FLS		(((uint32_t)1) << 29)
336 #define	IMMU_GSTS_AFLS		(((uint32_t)1) << 28)
337 #define	IMMU_GSTS_WBFS		(((uint32_t)1) << 27)
338 #define	IMMU_GSTS_QIES		(((uint32_t)1) << 26)
339 #define	IMMU_GSTS_IRES		(((uint32_t)1) << 25)
340 #define	IMMU_GSTS_IRTPS	(((uint32_t)1) << 24)
341 #define	IMMU_GSTS_CFIS		(((uint32_t)1) << 23)
342 
343 /* psi address mask */
344 #define	ADDR_AM_MAX(m)		(((uint_t)1) << (m))
345 #define	ADDR_AM_OFFSET(n, m)	((n) & (ADDR_AM_MAX(m) - 1))
346 
347 /* dmar fault event */
348 #define	IMMU_INTR_IPL			(4)
349 #define	IMMU_REG_FEVNT_CON_IM_SHIFT	(31)
350 
351 #define	IMMU_ALLOC_RESOURCE_DELAY    (drv_usectohz(5000))
352 
353 /* max value of Size field of Interrupt Remapping Table Address Register */
354 #define	INTRMAP_MAX_IRTA_SIZE	0xf
355 
356 /* interrupt remapping table entry size */
357 #define	INTRMAP_RTE_SIZE		0x10
358 
359 /* ioapic redirection table entry related shift of remappable interrupt */
360 #define	INTRMAP_IOAPIC_IDX_SHIFT		17
361 #define	INTRMAP_IOAPIC_FORMAT_SHIFT	16
362 #define	INTRMAP_IOAPIC_TM_SHIFT		15
363 #define	INTRMAP_IOAPIC_POL_SHIFT		13
364 #define	INTRMAP_IOAPIC_IDX15_SHIFT	11
365 
366 /* msi intr entry related shift of remappable interrupt */
367 #define	INTRMAP_MSI_IDX_SHIFT	5
368 #define	INTRMAP_MSI_FORMAT_SHIFT	4
369 #define	INTRMAP_MSI_SHV_SHIFT	3
370 #define	INTRMAP_MSI_IDX15_SHIFT	2
371 
372 #define	INTRMAP_IDX_FULL		(uint_t)-1
373 
374 #define	RDT_DLM(rdt)	BITX((rdt), 10, 8)
375 #define	RDT_DM(rdt)	BT_TEST(&(rdt), 11)
376 #define	RDT_POL(rdt)	BT_TEST(&(rdt), 13)
377 #define	RDT_TM(rdt)	BT_TEST(&(rdt), 15)
378 
379 #define	INTRMAP_DISABLE	(void *)-1
380 
381 /*
382  * invalidation granularity
383  */
384 typedef enum {
385 	TLB_INV_G_GLOBAL = 1,
386 	TLB_INV_G_DOMAIN,
387 	TLB_INV_G_PAGE
388 } tlb_inv_g_t;
389 
390 typedef enum {
391 	CTT_INV_G_GLOBAL = 1,
392 	CTT_INV_G_DOMAIN,
393 	CTT_INV_G_DEVICE
394 } ctt_inv_g_t;
395 
396 typedef enum {
397 	IEC_INV_GLOBAL = 0,
398 	IEC_INV_INDEX
399 } iec_inv_g_t;
400 
401 
402 struct inv_queue_state;
403 struct intrmap_tbl_state;
404 
405 /* A software page table structure */
406 typedef struct pgtable {
407 	krwlock_t swpg_rwlock;
408 	caddr_t hwpg_vaddr;   /* HW pgtable VA */
409 	paddr_t hwpg_paddr;   /* HW pgtable PA */
410 	ddi_dma_handle_t hwpg_dmahdl;
411 	ddi_acc_handle_t hwpg_memhdl;
412 	struct pgtable **swpg_next_array;
413 	list_node_t swpg_domain_node;  /* domain list of pgtables */
414 } pgtable_t;
415 
416 /* interrupt remapping table state info */
417 typedef struct intrmap {
418 	kmutex_t		intrmap_lock;
419 	ddi_dma_handle_t	intrmap_dma_hdl;
420 	ddi_acc_handle_t	intrmap_acc_hdl;
421 	caddr_t			intrmap_vaddr;
422 	paddr_t			intrmap_paddr;
423 	uint_t			intrmap_size;
424 	bitset_t		intrmap_map;
425 	uint_t			intrmap_free;
426 } intrmap_t;
427 
428 typedef struct hw_rce {
429 	uint64_t lo;
430 	uint64_t hi;
431 } hw_rce_t;
432 
433 
434 #define	ROOT_GET_P(hrent) ((hrent)->lo & 0x1)
435 #define	ROOT_SET_P(hrent) ((hrent)->lo |= 0x1)
436 
437 #define	ROOT_GET_CONT(hrent) ((hrent)->lo & ~(0xFFF))
438 #define	ROOT_SET_CONT(hrent, paddr) ((hrent)->lo |= (paddr & (~0xFFF)))
439 
440 #define	TTYPE_XLATE_ONLY  (0x0)
441 #define	TTYPE_XLATE_IOTLB (0x1)
442 #define	TTYPE_PASSTHRU    (0x2)
443 #define	TTYPE_RESERVED    (0x3)
444 
445 #define	CONT_GET_DID(hcent) ((((uint64_t)(hcent)->hi) >> 8) & 0xFFFF)
446 #define	CONT_SET_DID(hcent, did) ((hcent)->hi |= ((0xFFFF & (did)) << 8))
447 
448 #define	CONT_GET_AVAIL(hcent) ((((uint64_t)((hcent)->hi)) >> 0x3) & 0xF)
449 #define	CONT_SET_AVAIL(hcent, av) ((hcent)->hi |= ((0xF & (av)) << 0x3))
450 
451 #define	CONT_GET_LO_AW(hcent) (30 + 9 *((hcent)->hi & 0x7))
452 #define	CONT_GET_AW(hcent) \
453 	((CONT_GET_LO_AW(hcent) == 66) ? 64 : CONT_GET_LO_AW(hcent))
454 #define	CONT_SET_AW(hcent, aw) \
455 	((hcent)->hi |= (((((aw) + 2) - 30) / 9) & 0x7))
456 
457 #define	CONT_GET_ASR(hcent) ((hcent)->lo & ~(0xFFF))
458 #define	CONT_SET_ASR(hcent, paddr) ((hcent)->lo |= (paddr & (~0xFFF)))
459 
460 #define	CONT_GET_TTYPE(hcent) ((((uint64_t)(hcent)->lo) >> 0x2) & 0x3)
461 #define	CONT_SET_TTYPE(hcent, ttype) ((hcent)->lo |= (((ttype) & 0x3) << 0x2))
462 
463 #define	CONT_GET_P(hcent) ((hcent)->lo & 0x1)
464 #define	CONT_SET_P(hcent) ((hcent)->lo |= 0x1)
465 
466 #define	CONT_GET_ALH(hcent) ((hcent)->lo & 0x20)
467 #define	CONT_SET_ALH(hcent) ((hcent)->lo |= 0x20)
468 
469 #define	CONT_GET_EH(hcent) ((hcent)->lo & 0x10)
470 #define	CONT_SET_EH(hcent) ((hcent)->lo |= 0x10)
471 
472 
473 /* we use the bit 63 (available for system SW) as a present bit */
474 #define	PDTE_SW4(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63))
475 #define	PDTE_CLEAR_SW4(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63))
476 
477 #define	PDTE_P(hw_pdte) ((hw_pdte) & ((uint64_t)1<<63))
478 #define	PDTE_CLEAR_P(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<63))
479 #define	PDTE_SET_P(hw_pdte) ((hw_pdte) |= ((uint64_t)1<<63))
480 
481 #define	PDTE_TM(hw_pdte) ((hw_pdte) & ((uint64_t)1<<62))
482 #define	PDTE_CLEAR_TM(hw_pdte) ((hw_pdte) &= ~((uint64_t)1<<62))
483 
484 #define	PDTE_SW3(hw_pdte) \
485 	(((hw_pdte) & ~(((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1))) >> 52)
486 #define	PDTE_SW3_OVERFLOW(hw_pdte) \
487 	(PDTE_SW3(hw_pdte) == 0x3FF)
488 #define	PDTE_CLEAR_SW3(hw_pdte) \
489 	((hw_pdte) &= (((uint64_t)0x3<<62)|(((uint64_t)1<<52)-1)))
490 #define	PDTE_SET_SW3(hw_pdte, ref) \
491 	((hw_pdte) |= ((((uint64_t)(ref)) & 0x3FF) << 52))
492 
493 #define	PDTE_PADDR(hw_pdte) ((hw_pdte) & ~(((uint64_t)0xFFF<<52)|((1<<12)-1)))
494 #define	PDTE_CLEAR_PADDR(hw_pdte) \
495 		((hw_pdte) &= (((uint64_t)0xFFF<<52)|((1<<12)-1)))
496 #define	PDTE_SET_PADDR(hw_pdte, paddr) ((hw_pdte) |= PDTE_PADDR(paddr))
497 
498 #define	PDTE_SNP(hw_pdte) ((hw_pdte) & (1<<11))
499 #define	PDTE_CLEAR_SNP(hw_pdte) ((hw_pdte) &= ~(1<<11))
500 #define	PDTE_SET_SNP(hw_pdte) ((hw_pdte) |= (1<<11))
501 
502 #define	PDTE_SW2(hw_pdte) ((hw_pdte) & (0x700))
503 #define	PDTE_CLEAR_SW2(hw_pdte) ((hw_pdte) &= ~(0x700))
504 
505 #define	PDTE_SP(hw_pdte) ((hw_pdte) & (0x80))
506 #define	PDTE_CLEAR_SP(hw_pdte) ((hw_pdte) &= ~(0x80))
507 
508 #define	PDTE_SW1(hw_pdte) ((hw_pdte) & (0x7C))
509 #define	PDTE_CLEAR_SW1(hw_pdte) ((hw_pdte) &= ~(0x7C))
510 
511 #define	PDTE_WRITE(hw_pdte) ((hw_pdte) & (0x2))
512 #define	PDTE_CLEAR_WRITE(hw_pdte) ((hw_pdte) &= ~(0x2))
513 #define	PDTE_SET_WRITE(hw_pdte) ((hw_pdte) |= (0x2))
514 
515 #define	PDTE_READ(hw_pdte) ((hw_pdte) & (0x1))
516 #define	PDTE_CLEAR_READ(hw_pdte) ((hw_pdte) &= ~(0x1))
517 #define	PDTE_SET_READ(hw_pdte) ((hw_pdte) |= (0x1))
518 
519 #define	PDTE_MASK_R	((uint64_t)1 << 0)
520 #define	PDTE_MASK_W	((uint64_t)1 << 1)
521 #define	PDTE_MASK_SNP	((uint64_t)1 << 11)
522 #define	PDTE_MASK_TM	((uint64_t)1 << 62)
523 #define	PDTE_MASK_P	((uint64_t)1 << 63)
524 
525 struct immu_flushops;
526 
527 /*
528  * Used to wait for invalidation completion.
529  *     vstatus is the virtual address of the status word that will be written
530  *     pstatus is the physical addres
531  * If sync is true, then the the operation will be waited on for
532  * completion immediately. Else, the wait interface can be called
533  * to wait for completion later.
534  */
535 
536 #define	IMMU_INV_DATA_PENDING	1
537 #define	IMMU_INV_DATA_DONE	2
538 
539 typedef struct immu_inv_wait {
540 	volatile uint32_t iwp_vstatus;
541 	uint64_t iwp_pstatus;
542 	boolean_t iwp_sync;
543 	const char *iwp_name;		/* ID for debugging/statistics */
544 } immu_inv_wait_t;
545 
546 /*
547  * Used to batch IOMMU pagetable writes.
548  */
549 typedef struct immu_dcookie {
550 	paddr_t dck_paddr;
551 	uint64_t dck_npages;
552 } immu_dcookie_t;
553 
554 typedef struct immu {
555 	kmutex_t		immu_lock;
556 	char			*immu_name;
557 
558 	/* lock grabbed by interrupt handler */
559 	kmutex_t		immu_intr_lock;
560 
561 	/* ACPI/DMAR table related */
562 	void			*immu_dmar_unit;
563 	dev_info_t		*immu_dip;
564 	struct domain		*immu_unity_domain;
565 
566 	/* IOMMU register related */
567 	kmutex_t		immu_regs_lock;
568 	kcondvar_t		immu_regs_cv;
569 	boolean_t		immu_regs_busy;
570 	boolean_t		immu_regs_setup;
571 	boolean_t		immu_regs_running;
572 	boolean_t		immu_regs_quiesced;
573 	ddi_acc_handle_t	immu_regs_handle;
574 	caddr_t			immu_regs_addr;
575 	uint64_t		immu_regs_cap;
576 	uint64_t		immu_regs_excap;
577 	uint32_t		immu_regs_cmdval;
578 	uint32_t		immu_regs_intr_msi_addr;
579 	uint32_t		immu_regs_intr_msi_data;
580 	uint32_t		immu_regs_intr_uaddr;
581 
582 	/* DVMA related */
583 	kmutex_t		immu_dvma_lock;
584 	boolean_t		immu_dvma_setup;
585 	boolean_t		immu_dvma_running;
586 	int			immu_dvma_gaw;
587 	int			immu_dvma_agaw;
588 	int			immu_dvma_nlevels;
589 	boolean_t		immu_dvma_coherent;
590 	boolean_t		immu_TM_reserved;
591 	boolean_t		immu_SNP_reserved;
592 	uint64_t		immu_ptemask;
593 
594 	/* DVMA context related */
595 	krwlock_t		immu_ctx_rwlock;
596 	pgtable_t		*immu_ctx_root;
597 	immu_inv_wait_t		immu_ctx_inv_wait;
598 
599 	/* DVMA domain related */
600 	int			immu_max_domains;
601 	vmem_t			*immu_did_arena;
602 	char			immu_did_arena_name[IMMU_MAXNAMELEN];
603 	list_t			immu_domain_list;
604 
605 	/* DVMA special devices */
606 	boolean_t		immu_dvma_gfx_only;
607 	list_t			immu_dvma_lpc_list;
608 	list_t			immu_dvma_gfx_list;
609 
610 	/* interrupt remapping related */
611 	kmutex_t		immu_intrmap_lock;
612 	boolean_t		immu_intrmap_setup;
613 	boolean_t		immu_intrmap_running;
614 	intrmap_t		*immu_intrmap;
615 	uint64_t		immu_intrmap_irta_reg;
616 	immu_inv_wait_t		immu_intrmap_inv_wait;
617 
618 	/* queued invalidation related */
619 	kmutex_t		immu_qinv_lock;
620 	boolean_t		immu_qinv_setup;
621 	boolean_t		immu_qinv_running;
622 	boolean_t		immu_qinv_enabled;
623 	void			*immu_qinv;
624 	uint64_t		immu_qinv_reg_value;
625 
626 	/* list_node for system-wide list of DMAR units */
627 	list_node_t		immu_node;
628 
629 	struct immu_flushops	*immu_flushops;
630 
631 	kmem_cache_t		*immu_hdl_cache;
632 	kmem_cache_t		*immu_pgtable_cache;
633 
634 	iommulib_handle_t	immu_iommulib_handle;
635 } immu_t;
636 
637 /*
638  * Enough space to hold the decimal number of any device instance.
639  * Used for device/cache names.
640  */
641 #define	IMMU_ISTRLEN	11	/* log10(2^31)  + 1 */
642 
643 /* properties that control DVMA */
644 #define	DDI_DVMA_MAPTYPE_ROOTNEX_PROP	"immu-dvma-mapping"
645 
646 #define	DDI_DVMA_MAPTYPE_UNITY		"unity"
647 #define	DDI_DVMA_MAPTYPE_XLATE		"xlate"
648 
649 typedef enum immu_maptype {
650 	IMMU_MAPTYPE_BAD = 0,    /* 0 is always bad */
651 	IMMU_MAPTYPE_UNITY = 1,
652 	IMMU_MAPTYPE_XLATE
653 } immu_maptype_t;
654 
655 #define	IMMU_COOKIE_HASHSZ	(512)
656 
657 /*
658  * domain_t
659  *
660  */
661 typedef struct domain {
662 	/* the basics */
663 	uint_t			dom_did;
664 	immu_t			*dom_immu;
665 
666 	/* mapping related */
667 	immu_maptype_t		dom_maptype;
668 	vmem_t			*dom_dvma_arena;
669 	char			dom_dvma_arena_name[IMMU_MAXNAMELEN];
670 
671 	/* pgtables */
672 	pgtable_t		*dom_pgtable_root;
673 	krwlock_t		dom_pgtable_rwlock;
674 
675 	/* list node for list of domains (unity or xlate) */
676 	list_node_t		dom_maptype_node;
677 	/* list node for list of domains off immu */
678 	list_node_t		dom_immu_node;
679 
680 	mod_hash_t		*dom_cookie_hash;
681 
682 	/* topmost device in domain; usually the device itself (non-shared) */
683 	dev_info_t		*dom_dip;
684 } domain_t;
685 
686 typedef enum immu_pcib {
687 	IMMU_PCIB_BAD = 0,
688 	IMMU_PCIB_NOBDF,
689 	IMMU_PCIB_PCIE_PCIE,
690 	IMMU_PCIB_PCIE_PCI,
691 	IMMU_PCIB_PCI_PCI,
692 	IMMU_PCIB_ENDPOINT
693 } immu_pcib_t;
694 
695 /*
696  *  immu_devi_t
697  *      Intel IOMMU in devinfo node
698  */
699 typedef struct immu_devi {
700 	/* pci seg, bus, dev, func */
701 	int		imd_seg;
702 	int		imd_bus;
703 	int		imd_devfunc;
704 
705 	/* ppb information */
706 	immu_pcib_t	imd_pcib_type;
707 	int		imd_sec;
708 	int		imd_sub;
709 
710 	/* identifier for special devices */
711 	boolean_t	imd_display;
712 	boolean_t	imd_lpc;
713 
714 	/* set if premapped DVMA space is used */
715 	boolean_t	imd_use_premap;
716 
717 	/* dmar unit to which this dip belongs */
718 	immu_t		*imd_immu;
719 
720 	immu_flags_t	imd_dvma_flags;
721 
722 	/* domain ptr */
723 	domain_t	*imd_domain;
724 	dev_info_t	*imd_ddip;
725 
726 	/* my devinfo */
727 	dev_info_t	*imd_dip;
728 
729 	/*
730 	 * if we are a "special" devinfo
731 	 * the node for the special linked list
732 	 * off the DMAR unit structure
733 	 */
734 	list_node_t	imd_spc_node;
735 } immu_devi_t;
736 
737 #define	IMMU_DEVI(dip)		((immu_devi_t *)(DEVI(dip)->devi_iommu))
738 #define	IMMU_DEVI_SET(dip, imd)	(DEVI(dip)->devi_iommu = (void *)imd)
739 
740 /*
741  * struct dmar_arg
742  */
743 typedef struct immu_arg {
744 	int		ima_seg;
745 	int		ima_bus;
746 	int		ima_devfunc;
747 	dev_info_t	*ima_rdip;
748 	dev_info_t	*ima_ddip;
749 } immu_arg_t;
750 
751 #define	IMMU_NDVSEG	8
752 #define	IMMU_NDCK	64
753 #define	IMMU_NPREPTES	8
754 
755 typedef struct immu_hdl_private {
756 	immu_inv_wait_t ihp_inv_wait;
757 	size_t ihp_ndvseg;
758 	struct dvmaseg ihp_dvseg[IMMU_NDVSEG];
759 	immu_dcookie_t ihp_dcookies[IMMU_NDCK];
760 
761 	hw_pdte_t *ihp_preptes[IMMU_NPREPTES];
762 	uint64_t ihp_predvma;
763 	int ihp_npremapped;
764 } immu_hdl_priv_t;
765 
766 /*
767  * Invalidation operation function pointers for context and IOTLB.
768  * These will be set to either the register or the queue invalidation
769  * interface functions, since the hardware does not allow using them
770  * both at the same time.
771  */
772 struct immu_flushops {
773 	void (*imf_context_fsi)(immu_t *, uint8_t, uint16_t, uint_t,
774 	    immu_inv_wait_t *);
775 	void (*imf_context_dsi)(immu_t *, uint_t, immu_inv_wait_t *);
776 	void (*imf_context_gbl)(immu_t *, immu_inv_wait_t *);
777 
778 	void (*imf_iotlb_psi)(immu_t *, uint_t, uint64_t, uint_t, uint_t,
779 	    immu_inv_wait_t *);
780 	void (*imf_iotlb_dsi)(immu_t *, uint_t, immu_inv_wait_t *);
781 	void (*imf_iotlb_gbl)(immu_t *, immu_inv_wait_t *);
782 
783 	void (*imf_wait)(immu_inv_wait_t *);
784 };
785 
786 #define	immu_flush_context_fsi(i, f, s, d, w) \
787 	(i)->immu_flushops->imf_context_fsi(i, f, s, d, w)
788 #define	immu_flush_context_dsi(i, d, w) \
789 	(i)->immu_flushops->imf_context_dsi(i, d, w)
790 #define	immu_flush_context_gbl(i, w) \
791 	(i)->immu_flushops->imf_context_gbl(i, w)
792 
793 #define	immu_flush_iotlb_psi(i, d, v, c, h, w) \
794 	(i)->immu_flushops->imf_iotlb_psi(i, d, v, c, h, w)
795 #define	immu_flush_iotlb_dsi(i, d, w) \
796 	(i)->immu_flushops->imf_iotlb_dsi(i, d, w)
797 #define	immu_flush_iotlb_gbl(i, w) \
798 	(i)->immu_flushops->imf_iotlb_gbl(i, w)
799 
800 #define	immu_flush_wait(i, w) \
801 	(i)->immu_flushops->imf_wait(w)
802 
803 /*
804  * Globals used by IOMMU code
805  */
806 /* shared between IOMMU files */
807 extern dev_info_t *root_devinfo;
808 extern kmutex_t immu_lock;
809 extern list_t immu_list;
810 extern boolean_t immu_setup;
811 extern boolean_t immu_running;
812 extern kmutex_t ioapic_drhd_lock;
813 extern list_t ioapic_drhd_list;
814 extern struct iommulib_ops immulib_ops;
815 
816 /* switches */
817 
818 /* Various features */
819 extern boolean_t immu_enable;
820 extern boolean_t immu_gfxdvma_enable;
821 extern boolean_t immu_intrmap_enable;
822 extern boolean_t immu_qinv_enable;
823 
824 /* various quirks that need working around */
825 extern boolean_t immu_quirk_usbpage0;
826 extern boolean_t immu_quirk_usbfullpa;
827 extern boolean_t immu_quirk_usbrmrr;
828 extern boolean_t immu_quirk_mobile4;
829 
830 /* debug messages */
831 extern boolean_t immu_dmar_print;
832 
833 /* tunables */
834 extern int64_t immu_flush_gran;
835 
836 extern immu_flags_t immu_global_dvma_flags;
837 
838 extern int immu_use_tm;
839 extern int immu_use_alh;
840 
841 /* ################### Interfaces exported outside IOMMU code ############## */
842 void immu_init(void);
843 void immu_startup(void);
844 void immu_shutdown(void);
845 void immu_destroy(void);
846 int immu_map_sgl(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
847     int prealloc_count, dev_info_t *rdip);
848 int immu_unmap_sgl(ddi_dma_impl_t *hp, dev_info_t *rdip);
849 void immu_device_tree_changed(void);
850 void immu_physmem_update(uint64_t addr, uint64_t size);
851 int immu_quiesce(void);
852 int immu_unquiesce(void);
853 /* ######################################################################### */
854 
855 /* ################# Interfaces used within IOMMU code #################### */
856 /* immu_dmar.c interfaces */
857 int immu_dmar_setup(void);
858 int immu_dmar_parse(void);
859 void immu_dmar_startup(void);
860 void immu_dmar_shutdown(void);
861 void immu_dmar_destroy(void);
862 boolean_t immu_dmar_blacklisted(char **strings_array, uint_t nstrings);
863 immu_t *immu_dmar_get_immu(dev_info_t *rdip);
864 dev_info_t *immu_dmar_unit_dip(void *dmar_unit);
865 void immu_dmar_set_immu(void *dmar_unit, immu_t *immu);
866 void *immu_dmar_walk_units(int seg, void *dmar_unit);
867 boolean_t immu_dmar_intrmap_supported(void);
868 uint16_t immu_dmar_ioapic_sid(int ioapicid);
869 immu_t *immu_dmar_ioapic_immu(int ioapicid);
870 void immu_dmar_rmrr_map(void);
871 
872 /* immu.c interfaces */
873 int immu_walk_ancestor(dev_info_t *rdip, dev_info_t *ddip,
874     int (*func)(dev_info_t *, void *arg), void *arg,
875     int *level, immu_flags_t immu_flags);
876 void immu_init_inv_wait(immu_inv_wait_t *iwp, const char *s, boolean_t sync);
877 
878 /* immu_regs.c interfaces */
879 void immu_regs_setup(list_t *immu_list);
880 void immu_regs_startup(immu_t *immu);
881 int immu_regs_resume(immu_t *immu);
882 void immu_regs_suspend(immu_t *immu);
883 void immu_regs_shutdown(immu_t *immu);
884 void immu_regs_destroy(list_t *immu_list);
885 
886 void immu_regs_intr(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,
887     uint32_t uaddr);
888 
889 boolean_t immu_regs_passthru_supported(immu_t *immu);
890 boolean_t immu_regs_is_TM_reserved(immu_t *immu);
891 boolean_t immu_regs_is_SNP_reserved(immu_t *immu);
892 
893 void immu_regs_wbf_flush(immu_t *immu);
894 void immu_regs_cpu_flush(immu_t *immu, caddr_t addr, uint_t size);
895 
896 void immu_regs_context_fsi(immu_t *immu, uint8_t function_mask,
897     uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp);
898 void immu_regs_context_dsi(immu_t *immu, uint_t domain_id,
899     immu_inv_wait_t *iwp);
900 void immu_regs_context_gbl(immu_t *immu, immu_inv_wait_t *iwp);
901 void immu_regs_iotlb_psi(immu_t *immu, uint_t domain_id,
902     uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp);
903 void immu_regs_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp);
904 void immu_regs_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp);
905 
906 void immu_regs_set_root_table(immu_t *immu);
907 void immu_regs_qinv_enable(immu_t *immu, uint64_t qinv_reg_value);
908 void immu_regs_intr_enable(immu_t *immu, uint32_t msi_addr, uint32_t msi_data,
909     uint32_t uaddr);
910 void immu_regs_intrmap_enable(immu_t *immu, uint64_t irta_reg);
911 uint64_t immu_regs_get64(immu_t *immu, uint_t reg);
912 void immu_regs_put64(immu_t *immu, uint_t reg, uint64_t val);
913 uint32_t immu_regs_get32(immu_t *immu, uint_t reg);
914 void immu_regs_put32(immu_t *immu, uint_t reg, uint32_t val);
915 
916 /* immu_dvma.c interfaces */
917 void immu_dvma_setup(list_t *immu_list);
918 void immu_dvma_startup(immu_t *immu);
919 void immu_dvma_shutdown(immu_t *immu);
920 void immu_dvma_destroy(list_t *immu_list);
921 
922 void immu_dvma_physmem_update(uint64_t addr, uint64_t size);
923 int immu_map_memrange(dev_info_t *, memrng_t *);
924 int immu_dvma_map(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
925     uint_t prealloc_count, dev_info_t *rdip);
926 int immu_dvma_unmap(ddi_dma_impl_t *hp, dev_info_t *rdip);
927 int immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags);
928 immu_devi_t *immu_devi_get(dev_info_t *dip);
929 immu_t *immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags);
930 int pgtable_ctor(void *buf, void *arg, int kmflag);
931 void pgtable_dtor(void *buf, void *arg);
932 
933 int immu_hdl_priv_ctor(void *buf, void *arg, int kmf);
934 
935 int immu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags);
936 
937 void immu_print_fault_info(uint_t sid, uint64_t dvma);
938 
939 /* immu_intrmap.c interfaces */
940 void immu_intrmap_setup(list_t *immu_list);
941 void immu_intrmap_startup(immu_t *immu);
942 void immu_intrmap_shutdown(immu_t *immu);
943 void immu_intrmap_destroy(list_t *immu_list);
944 
945 /* registers interrupt handler for IOMMU unit */
946 void immu_intr_register(immu_t *immu);
947 uint_t immu_intr_handler(caddr_t, caddr_t);
948 
949 
950 /* immu_qinv.c interfaces */
951 int immu_qinv_setup(list_t *immu_list);
952 void immu_qinv_startup(immu_t *immu);
953 void immu_qinv_shutdown(immu_t *immu);
954 void immu_qinv_destroy(list_t *immu_list);
955 
956 void immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask,
957     uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp);
958 void immu_qinv_context_dsi(immu_t *immu, uint_t domain_id,
959     immu_inv_wait_t *iwp);
960 void immu_qinv_context_gbl(immu_t *immu, immu_inv_wait_t *iwp);
961 void immu_qinv_iotlb_psi(immu_t *immu, uint_t domain_id,
962     uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp);
963 void immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp);
964 void immu_qinv_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp);
965 
966 void immu_qinv_intr_global(immu_t *immu, immu_inv_wait_t *iwp);
967 void immu_qinv_intr_one_cache(immu_t *immu, uint_t idx, immu_inv_wait_t *iwp);
968 void immu_qinv_intr_caches(immu_t *immu, uint_t idx, uint_t cnt,
969     immu_inv_wait_t *);
970 void immu_qinv_report_fault(immu_t *immu);
971 
972 #ifdef DEBUG
973 #define	IMMU_DPROBE1(name, type1, arg1) \
974 	DTRACE_PROBE1(name, type1, arg1)
975 #define	IMMU_DPROBE2(name, type1, arg1, type2, arg2) \
976 	DTRACE_PROBE2(name, type1, arg1, type2, arg2)
977 #define	IMMU_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3) \
978 	DTRACE_PROBE3(name, type1, arg1, type2, arg2, type3, arg3)
979 #define	IMMU_DPROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4) \
980 	DTRACE_PROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4)
981 #else
982 #define	IMMU_DPROBE1(name, type1, arg1)
983 #define	IMMU_DPROBE2(name, type1, arg1, type2, arg2)
984 #define	IMMU_DPROBE3(name, type1, arg1, type2, arg2, type3, arg3)
985 #define	IMMU_DPROBE4(name, type1, arg1, type2, arg2, type3, arg3, type4, arg4)
986 #endif
987 
988 
989 #ifdef	__cplusplus
990 }
991 #endif
992 
993 #endif	/* _SYS_INTEL_IOMMU_H */
994