1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2016 Anish Gupta (anish@freebsd.org)
5  * Copyright (c) 2021 The FreeBSD Foundation
6  *
7  * Portions of this software were developed by Ka Ho Ng
8  * under sponsorship from the FreeBSD Foundation.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice unmodified, this list of conditions, and the following
15  *    disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
21  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
22  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
23  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
24  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
25  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
29  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 #ifndef _AMDVI_PRIV_H_
33 #define _AMDVI_PRIV_H_
34 
35 #include <contrib/dev/acpica/include/acpi.h>
36 
37 #define	BIT(n)			(1ULL << (n))
38 /* Return value of bits[n:m] where n and (n >= ) m are bit positions. */
39 #define REG_BITS(x, n, m)	(((x) >> (m)) &			\
40 				((1 << (((n) - (m)) + 1)) - 1))
41 
42 /*
43  * IOMMU PCI capability.
44  */
45 #define AMDVI_PCI_CAP_IOTLB	BIT(0)	/* IOTLB is supported. */
46 #define AMDVI_PCI_CAP_HT	BIT(1)	/* HyperTransport tunnel support. */
47 #define AMDVI_PCI_CAP_NPCACHE	BIT(2)	/* Not present page cached. */
48 #define AMDVI_PCI_CAP_EFR	BIT(3)	/* Extended features. */
49 #define AMDVI_PCI_CAP_EXT	BIT(4)	/* Miscellaneous information reg. */
50 
51 /*
52  * IOMMU extended features.
53  */
54 #define AMDVI_EX_FEA_PREFSUP	BIT(0)	/* Prefetch command support. */
55 #define AMDVI_EX_FEA_PPRSUP	BIT(1)	/* PPR support */
56 #define AMDVI_EX_FEA_XTSUP	BIT(2)	/* Reserved */
57 #define AMDVI_EX_FEA_NXSUP	BIT(3)	/* No-execute. */
58 #define AMDVI_EX_FEA_GTSUP	BIT(4)	/* Guest translation support. */
59 #define AMDVI_EX_FEA_EFRW	BIT(5)	/* Reserved */
60 #define AMDVI_EX_FEA_IASUP	BIT(6)	/* Invalidate all command supp. */
61 #define AMDVI_EX_FEA_GASUP	BIT(7)	/* Guest APIC or AVIC support. */
62 #define AMDVI_EX_FEA_HESUP	BIT(8)	/* Hardware Error. */
63 #define AMDVI_EX_FEA_PCSUP	BIT(9)	/* Performance counters support. */
64 /* XXX: add more EFER bits. */
65 
66 /*
67  * Device table entry or DTE
68  * NOTE: Must be 256-bits/32 bytes aligned.
69  */
70 struct amdvi_dte {
71 	uint32_t dt_valid:1;		/* Device Table valid. */
72 	uint32_t pt_valid:1;		/* Page translation valid. */
73 	uint16_t :7;			/* Reserved[8:2] */
74 	uint8_t	 pt_level:3;		/* Paging level, 0 to disable. */
75 	uint64_t pt_base:40;		/* Page table root pointer. */
76 	uint8_t  :3;			/* Reserved[54:52] */
77 	uint8_t	 gv_valid:1;		/* Revision 2, GVA to SPA. */
78 	uint8_t	 gv_level:2;		/* Revision 2, GLX level. */
79 	uint8_t	 gv_cr3_lsb:3;		/* Revision 2, GCR3[14:12] */
80 	uint8_t	 read_allow:1;		/* I/O read enabled. */
81 	uint8_t	 write_allow:1;		/* I/O write enabled. */
82 	uint8_t  :1;			/* Reserved[63] */
83 	uint16_t domain_id:16;		/* Domain ID */
84 	uint16_t gv_cr3_lsb2:16;	/* Revision 2, GCR3[30:15] */
85 	uint8_t	 iotlb_enable:1;	/* Device support IOTLB */
86 	uint8_t	 sup_second_io_fault:1;	/* Suppress subsequent I/O faults. */
87 	uint8_t	 sup_all_io_fault:1;	/* Suppress all I/O page faults. */
88 	uint8_t	 IOctl:2;		/* Port I/O control. */
89 	uint8_t	 iotlb_cache_disable:1;	/* IOTLB cache hints. */
90 	uint8_t	 snoop_disable:1;	/* Snoop disable. */
91 	uint8_t	 allow_ex:1;		/* Allow exclusion. */
92 	uint8_t	 sysmgmt:2;		/* System management message.*/
93 	uint8_t  :1;			/* Reserved[106] */
94 	uint32_t gv_cr3_msb:21;		/* Revision 2, GCR3[51:31] */
95 	uint8_t	 intmap_valid:1;	/* Interrupt map valid. */
96 	uint8_t	 intmap_len:4;		/* Interrupt map table length. */
97 	uint8_t	 intmap_ign:1;		/* Ignore unmapped interrupts. */
98 	uint64_t intmap_base:46;	/* IntMap base. */
99 	uint8_t  :4;			/* Reserved[183:180] */
100 	uint8_t	 init_pass:1;		/* INIT pass through or PT */
101 	uint8_t	 extintr_pass:1;	/* External Interrupt PT */
102 	uint8_t	 nmi_pass:1;		/* NMI PT */
103 	uint8_t  :1;			/* Reserved[187] */
104 	uint8_t	 intr_ctrl:2;		/* Interrupt control */
105 	uint8_t	 lint0_pass:1;		/* LINT0 PT */
106 	uint8_t	 lint1_pass:1;		/* LINT1 PT */
107 	uint64_t :64;			/* Reserved[255:192] */
108 } __attribute__((__packed__));
109 CTASSERT(sizeof(struct amdvi_dte) == 32);
110 
111 /*
112  * IOMMU command entry.
113  */
114 struct amdvi_cmd {
115 	uint32_t	word0;
116 	uint32_t	word1:28;
117 	uint8_t		opcode:4;
118 	uint64_t	addr;
119 } __attribute__((__packed__));
120 
121 /* Command opcodes. */
122 #define AMDVI_CMP_WAIT_OPCODE	0x1	/* Completion wait. */
123 #define AMDVI_INVD_DTE_OPCODE	0x2	/* Invalidate device table entry. */
124 #define AMDVI_INVD_PAGE_OPCODE	0x3	/* Invalidate pages. */
125 #define AMDVI_INVD_IOTLB_OPCODE	0x4	/* Invalidate IOTLB pages. */
126 #define AMDVI_INVD_INTR_OPCODE	0x5	/* Invalidate Interrupt table. */
127 #define AMDVI_PREFETCH_PAGES_OPCODE	0x6	/* Prefetch IOMMU pages. */
128 #define AMDVI_COMP_PPR_OPCODE	0x7	/* Complete PPR request. */
129 #define AMDVI_INV_ALL_OPCODE	0x8	/* Invalidate all. */
130 
131 /* Completion wait attributes. */
132 #define AMDVI_CMP_WAIT_STORE	BIT(0)	/* Write back data. */
133 #define AMDVI_CMP_WAIT_INTR	BIT(1)	/* Completion wait interrupt. */
134 #define AMDVI_CMP_WAIT_FLUSH	BIT(2)	/* Flush queue. */
135 
136 /* Invalidate page. */
137 #define AMDVI_INVD_PAGE_S	BIT(0)	/* Invalidation size. */
138 #define AMDVI_INVD_PAGE_PDE	BIT(1)	/* Invalidate PDE. */
139 #define AMDVI_INVD_PAGE_GN_GVA	BIT(2)	/* GPA or GVA. */
140 
141 #define AMDVI_INVD_PAGE_ALL_ADDR	(0x7FFFFFFFFFFFFULL << 12)
142 
143 /* Invalidate IOTLB. */
144 #define AMDVI_INVD_IOTLB_S	BIT(0)	/* Invalidation size 4k or addr */
145 #define AMDVI_INVD_IOTLB_GN_GVA	BIT(2)	/* GPA or GVA. */
146 
147 #define AMDVI_INVD_IOTLB_ALL_ADDR	(0x7FFFFFFFFFFFFULL << 12)
148 /* XXX: add more command entries. */
149 
150 /*
151  * IOMMU event entry.
152  */
153 struct amdvi_event {
154 	uint16_t	devid;
155 	uint16_t	pasid_hi;
156 	uint16_t	pasid_domid;	/* PASID low or DomainID */
157 	uint16_t	flag:12;
158 	uint8_t		opcode:4;
159 	uint64_t	addr;
160 } __attribute__((__packed__));
161 CTASSERT(sizeof(struct amdvi_event) == 16);
162 
163 /* Various event types. */
164 #define AMDVI_EVENT_INVALID_DTE		0x1
165 #define AMDVI_EVENT_PFAULT		0x2
166 #define AMDVI_EVENT_DTE_HW_ERROR	0x3
167 #define AMDVI_EVENT_PAGE_HW_ERROR	0x4
168 #define AMDVI_EVENT_ILLEGAL_CMD		0x5
169 #define AMDVI_EVENT_CMD_HW_ERROR	0x6
170 #define AMDVI_EVENT_IOTLB_TIMEOUT	0x7
171 #define AMDVI_EVENT_INVALID_DTE_REQ	0x8
172 #define AMDVI_EVENT_INVALID_PPR_REQ	0x9
173 #define AMDVI_EVENT_COUNTER_ZERO	0xA
174 
175 #define AMDVI_EVENT_FLAG_MASK           0x1FF	/* Mask for event flags. */
176 #define AMDVI_EVENT_FLAG_TYPE(x)        (((x) >> 9) & 0x3)
177 
178 /*
179  * IOMMU control block.
180  */
181 struct amdvi_ctrl {
182 	struct {
183 		uint16_t size:9;
184 		uint16_t :3;
185 		uint64_t base:40;	/* Devtable register base. */
186 		uint16_t :12;
187 	} dte;
188 	struct {
189 		uint16_t :12;
190 		uint64_t base:40;
191 		uint8_t  :4;
192 		uint8_t	 len:4;
193 		uint8_t  :4;
194 	} cmd;
195 	struct {
196 		uint16_t :12;
197 		uint64_t base:40;
198 		uint8_t  :4;
199 		uint8_t	 len:4;
200 		uint8_t  :4;
201 	} event;
202 	uint16_t control :13;
203 	uint64_t	 :51;
204 	struct {
205 		uint8_t	 enable:1;
206 		uint8_t	 allow:1;
207 		uint16_t :10;
208 		uint64_t base:40;
209 		uint16_t :12;
210 		uint16_t :12;
211 		uint64_t limit:40;
212 		uint16_t :12;
213 	} excl;
214 	/*
215 	 * Revision 2 only.
216 	 */
217 	uint64_t ex_feature;
218 	struct {
219 		uint16_t :12;
220 		uint64_t base:40;
221 		uint8_t  :4;
222 		uint8_t	 len:4;
223 		uint8_t  :4;
224 	} ppr;
225 	uint64_t first_event;
226 	uint64_t second_event;
227 	uint64_t event_status;
228 	/* Revision 2 only, end. */
229 	uint8_t	 pad1[0x1FA8];		/* Padding. */
230 	uint32_t cmd_head:19;
231 	uint64_t :45;
232 	uint32_t cmd_tail:19;
233 	uint64_t :45;
234 	uint32_t evt_head:19;
235 	uint64_t :45;
236 	uint32_t evt_tail:19;
237 	uint64_t :45;
238 	uint32_t status:19;
239 	uint64_t :45;
240 	uint64_t pad2;
241 	uint8_t  :4;
242 	uint16_t ppr_head:15;
243 	uint64_t :45;
244 	uint8_t  :4;
245 	uint16_t ppr_tail:15;
246 	uint64_t :45;
247 	uint8_t	 pad3[0x1FC0];		/* Padding. */
248 
249 	/* XXX: More for rev2. */
250 } __attribute__((__packed__));
251 CTASSERT(offsetof(struct amdvi_ctrl, pad1)== 0x58);
252 CTASSERT(offsetof(struct amdvi_ctrl, pad2)== 0x2028);
253 CTASSERT(offsetof(struct amdvi_ctrl, pad3)== 0x2040);
254 
255 #define AMDVI_MMIO_V1_SIZE	(4 * PAGE_SIZE)	/* v1 size */
256 /*
257  * AMF IOMMU v2 size including event counters
258  */
259 #define AMDVI_MMIO_V2_SIZE	(8 * PAGE_SIZE)
260 
261 CTASSERT(sizeof(struct amdvi_ctrl) == 0x4000);
262 CTASSERT(sizeof(struct amdvi_ctrl) == AMDVI_MMIO_V1_SIZE);
263 
264 /* IVHD flag */
265 #define IVHD_FLAG_HTT		BIT(0)	/* Hypertransport Tunnel. */
266 #define IVHD_FLAG_PPW		BIT(1)	/* Pass posted write. */
267 #define IVHD_FLAG_RPPW		BIT(2)	/* Response pass posted write. */
268 #define IVHD_FLAG_ISOC		BIT(3)	/* Isoc support. */
269 #define IVHD_FLAG_IOTLB		BIT(4)	/* IOTLB support. */
270 #define IVHD_FLAG_COH		BIT(5)	/* Coherent control, default 1 */
271 #define IVHD_FLAG_PFS		BIT(6)	/* Prefetch IOMMU pages. */
272 #define IVHD_FLAG_PPRS		BIT(7)	/* Peripheral page support. */
273 
274 /* IVHD device entry data setting. */
275 #define IVHD_DEV_LINT0_PASS	BIT(6)	/* LINT0 interrupts. */
276 #define IVHD_DEV_LINT1_PASS	BIT(7)	/* LINT1 interrupts. */
277 
278 /* Bit[5:4] for System Mgmt. Bit3 is reserved. */
279 #define IVHD_DEV_INIT_PASS	BIT(0)	/* INIT */
280 #define IVHD_DEV_EXTINTR_PASS	BIT(1)	/* ExtInt */
281 #define IVHD_DEV_NMI_PASS	BIT(2)	/* NMI */
282 
283 /* IVHD 8-byte extended data settings. */
284 #define IVHD_DEV_EXT_ATS_DISABLE	BIT(31)	/* Disable ATS */
285 
286 /* IOMMU control register. */
287 #define AMDVI_CTRL_EN		BIT(0)	/* IOMMU enable. */
288 #define AMDVI_CTRL_HTT		BIT(1)	/* Hypertransport tunnel enable. */
289 #define AMDVI_CTRL_ELOG		BIT(2)	/* Event log enable. */
290 #define AMDVI_CTRL_ELOGINT	BIT(3)	/* Event log interrupt. */
291 #define AMDVI_CTRL_COMINT	BIT(4)	/* Completion wait interrupt. */
292 #define AMDVI_CTRL_PPW		BIT(8)
293 #define AMDVI_CTRL_RPPW		BIT(9)
294 #define AMDVI_CTRL_COH		BIT(10)
295 #define AMDVI_CTRL_ISOC		BIT(11)
296 #define AMDVI_CTRL_CMD		BIT(12)	/* Command buffer enable. */
297 #define AMDVI_CTRL_PPRLOG	BIT(13)
298 #define AMDVI_CTRL_PPRINT	BIT(14)
299 #define AMDVI_CTRL_PPREN	BIT(15)
300 #define AMDVI_CTRL_GTE		BIT(16)	/* Guest translation enable. */
301 #define AMDVI_CTRL_GAE		BIT(17)	/* Guest APIC enable. */
302 
303 /* Invalidation timeout. */
304 #define AMDVI_CTRL_INV_NO_TO	0	/* No timeout. */
305 #define AMDVI_CTRL_INV_TO_1ms	1	/* 1 ms */
306 #define AMDVI_CTRL_INV_TO_10ms	2	/* 10 ms */
307 #define AMDVI_CTRL_INV_TO_100ms	3	/* 100 ms */
308 #define AMDVI_CTRL_INV_TO_1S	4	/* 1 second */
309 #define AMDVI_CTRL_INV_TO_10S	5	/* 10 second */
310 #define AMDVI_CTRL_INV_TO_100S	6	/* 100 second */
311 
312 /*
313  * Max number of PCI devices.
314  * 256 bus x 32 slot/devices x 8 functions.
315  */
316 #define PCI_NUM_DEV_MAX		0x10000
317 
318 /* Maximum number of domains supported by IOMMU. */
319 #define AMDVI_MAX_DOMAIN	(BIT(16) - 1)
320 
321 /*
322  * IOMMU Page Table attributes.
323  */
324 #define AMDVI_PT_PRESENT	BIT(0)
325 #define AMDVI_PT_COHERENT	BIT(60)
326 #define AMDVI_PT_READ		BIT(61)
327 #define AMDVI_PT_WRITE		BIT(62)
328 
329 #define AMDVI_PT_RW		(AMDVI_PT_READ | AMDVI_PT_WRITE)
330 #define AMDVI_PT_MASK		0xFFFFFFFFFF000UL /* Only [51:12] for PA */
331 
332 #define AMDVI_PD_LEVEL_SHIFT	9
333 #define AMDVI_PD_SUPER(x)	(((x) >> AMDVI_PD_LEVEL_SHIFT) == 7)
334 /*
335  * IOMMU Status, offset 0x2020
336  */
337 #define AMDVI_STATUS_EV_OF		BIT(0)	/* Event overflow. */
338 #define AMDVI_STATUS_EV_INTR		BIT(1)	/* Event interrupt. */
339 /* Completion wait command completed. */
340 #define AMDVI_STATUS_CMP		BIT(2)
341 
342 #define	IVRS_CTRL_RID			1	/* MMIO RID */
343 
344 /* ACPI IVHD */
345 struct ivhd_dev_cfg {
346 	uint32_t start_id;
347 	uint32_t end_id;
348 	uint8_t	 data;			/* Device configuration. */
349 	bool	 enable_ats;		/* ATS enabled for the device. */
350 	int	 ats_qlen;		/* ATS invalidation queue depth. */
351 };
352 
353 struct amdvi_domain {
354 	uint64_t *ptp;			/* Highest level page table */
355 	int	ptp_level;		/* Level of page tables */
356 	u_int	id;			/* Domain id */
357 	SLIST_ENTRY (amdvi_domain) next;
358 };
359 
360 /*
361  * Different type of IVHD.
362  * XXX: Use AcpiIvrsType once new IVHD types are available.
363 */
364 enum IvrsType
365 {
366 	IVRS_TYPE_HARDWARE_LEGACY = ACPI_IVRS_TYPE_HARDWARE1,
367 					/* Legacy without EFRi support. */
368 	IVRS_TYPE_HARDWARE_EFR	  = ACPI_IVRS_TYPE_HARDWARE2,
369 						/* With EFR support. */
370 	IVRS_TYPE_HARDWARE_MIXED  = 0x40, /* Mixed with EFR support. */
371 };
372 
373 /*
374  * AMD IOMMU softc.
375  */
376 struct amdvi_softc {
377 	struct amdvi_ctrl *ctrl;	/* Control area. */
378 	device_t	dev;		/* IOMMU device. */
379 	device_t	pci_dev;	/* IOMMU PCI function device. */
380 	enum IvrsType   ivhd_type;	/* IOMMU IVHD type. */
381 	bool		iotlb;		/* IOTLB supported by IOMMU */
382 	struct amdvi_cmd *cmd;		/* Command descriptor area. */
383 	int		cmd_max;	/* Max number of commands. */
384 	uint64_t	cmp_data;	/* Command completion write back. */
385 	struct amdvi_event *event;	/* Event descriptor area. */
386 	int		event_max;	/* Max number of events. */
387 	/* ACPI various flags. */
388 	uint32_t	ivhd_flag;	/* ACPI IVHD flag. */
389 	uint32_t	ivhd_feature;	/* ACPI v1 Reserved or v2 attribute. */
390 	uint64_t	ext_feature;	/* IVHD EFR */
391 	/* PCI related. */
392 	uint16_t	cap_off;	/* PCI Capability offset. */
393 	uint8_t		pci_cap;	/* PCI capability. */
394 	uint16_t	pci_seg;	/* IOMMU PCI domain/segment. */
395 	uint16_t	pci_rid;	/* PCI BDF of IOMMU */
396 	/* ACPI device configuration for end points. */
397 	struct		ivhd_dev_cfg *dev_cfg;
398 	int		dev_cfg_cnt;
399 	int		dev_cfg_cap;
400 
401 	/* Software statistics. */
402 	uint64_t	event_intr_cnt;	/* Total event INTR count. */
403 	uint64_t	total_cmd;	/* Total number of commands. */
404 };
405 
406 int	amdvi_setup_hw(struct amdvi_softc *softc);
407 int	amdvi_teardown_hw(struct amdvi_softc *softc);
408 #endif /* _AMDVI_PRIV_H_ */
409