1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2016 Anish Gupta (anish@freebsd.org)
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice unmodified, this list of conditions, and the following
12 *    disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * $FreeBSD$
29 */
30
31#ifndef _AMDVI_PRIV_H_
32#define _AMDVI_PRIV_H_
33
34#include <contrib/dev/acpica/include/acpi.h>
35
36#define	BIT(n)			(1ULL << (n))
37/* Return value of bits[n:m] where n and (n >= ) m are bit positions. */
38#define REG_BITS(x, n, m)	(((x) >> (m)) & 		\
39				((1 << (((n) - (m)) + 1)) - 1))
40
41/*
42 * IOMMU PCI capability.
43 */
44#define AMDVI_PCI_CAP_IOTLB	BIT(0)	/* IOTLB is supported. */
45#define AMDVI_PCI_CAP_HT	BIT(1)	/* HyperTransport tunnel support. */
46#define AMDVI_PCI_CAP_NPCACHE	BIT(2)	/* Not present page cached. */
47#define AMDVI_PCI_CAP_EFR	BIT(3)	/* Extended features. */
48#define AMDVI_PCI_CAP_EXT	BIT(4)	/* Miscellaneous information reg. */
49
50/*
51 * IOMMU extended features.
52 */
53#define AMDVI_EX_FEA_PREFSUP	BIT(0)	/* Prefetch command support. */
54#define AMDVI_EX_FEA_PPRSUP	BIT(1)	/* PPR support */
55#define AMDVI_EX_FEA_XTSUP	BIT(2)	/* Reserved */
56#define AMDVI_EX_FEA_NXSUP	BIT(3)	/* No-execute. */
57#define AMDVI_EX_FEA_GTSUP	BIT(4)	/* Guest translation support. */
58#define AMDVI_EX_FEA_EFRW	BIT(5)	/* Reserved */
59#define AMDVI_EX_FEA_IASUP	BIT(6)	/* Invalidate all command supp. */
60#define AMDVI_EX_FEA_GASUP	BIT(7)	/* Guest APIC or AVIC support. */
61#define AMDVI_EX_FEA_HESUP	BIT(8)	/* Hardware Error. */
62#define AMDVI_EX_FEA_PCSUP	BIT(9)	/* Performance counters support. */
63/* XXX: add more EFER bits. */
64
65/*
66 * Device table entry or DTE
67 * NOTE: Must be 256-bits/32 bytes aligned.
68 */
69struct amdvi_dte {
70	uint32_t dt_valid:1;		/* Device Table valid. */
71	uint32_t pt_valid:1;		/* Page translation valid. */
72	uint16_t :7;			/* Reserved[8:2] */
73	uint8_t	 pt_level:3;		/* Paging level, 0 to disable. */
74	uint64_t pt_base:40;		/* Page table root pointer. */
75	uint8_t  :3;			/* Reserved[54:52] */
76	uint8_t	 gv_valid:1;		/* Revision 2, GVA to SPA. */
77	uint8_t	 gv_level:2;		/* Revision 2, GLX level. */
78	uint8_t	 gv_cr3_lsb:3;		/* Revision 2, GCR3[14:12] */
79	uint8_t	 read_allow:1;		/* I/O read enabled. */
80	uint8_t	 write_allow:1;		/* I/O write enabled. */
81	uint8_t  :1;			/* Reserved[63] */
82	uint16_t domain_id:16;		/* Domain ID */
83	uint16_t gv_cr3_lsb2:16;	/* Revision 2, GCR3[30:15] */
84	uint8_t	 iotlb_enable:1;	/* Device support IOTLB */
85	uint8_t	 sup_second_io_fault:1;	/* Suppress subsequent I/O faults. */
86	uint8_t	 sup_all_io_fault:1;	/* Suppress all I/O page faults. */
87	uint8_t	 IOctl:2;		/* Port I/O control. */
88	uint8_t	 iotlb_cache_disable:1;	/* IOTLB cache hints. */
89	uint8_t	 snoop_disable:1;	/* Snoop disable. */
90	uint8_t	 allow_ex:1;		/* Allow exclusion. */
91	uint8_t	 sysmgmt:2;		/* System management message.*/
92	uint8_t  :1;			/* Reserved[106] */
93	uint32_t gv_cr3_msb:21;		/* Revision 2, GCR3[51:31] */
94	uint8_t	 intmap_valid:1;	/* Interrupt map valid. */
95	uint8_t	 intmap_len:4;		/* Interrupt map table length. */
96	uint8_t	 intmap_ign:1;		/* Ignore unmapped interrupts. */
97	uint64_t intmap_base:46;	/* IntMap base. */
98	uint8_t  :4;			/* Reserved[183:180] */
99	uint8_t	 init_pass:1;		/* INIT pass through or PT */
100	uint8_t	 extintr_pass:1;	/* External Interrupt PT */
101	uint8_t	 nmi_pass:1;		/* NMI PT */
102	uint8_t  :1;			/* Reserved[187] */
103	uint8_t	 intr_ctrl:2;		/* Interrupt control */
104	uint8_t	 lint0_pass:1;		/* LINT0 PT */
105	uint8_t	 lint1_pass:1;		/* LINT1 PT */
106	uint64_t :64;			/* Reserved[255:192] */
107} __attribute__((__packed__));
108CTASSERT(sizeof(struct amdvi_dte) == 32);
109
110/*
111 * IOMMU command entry.
112 */
113struct amdvi_cmd {
114	uint32_t 	word0;
115	uint32_t 	word1:28;
116	uint8_t		opcode:4;
117	uint64_t 	addr;
118} __attribute__((__packed__));
119
120/* Command opcodes. */
121#define AMDVI_CMP_WAIT_OPCODE	0x1	/* Completion wait. */
122#define AMDVI_INVD_DTE_OPCODE	0x2	/* Invalidate device table entry. */
123#define AMDVI_INVD_PAGE_OPCODE	0x3	/* Invalidate pages. */
124#define AMDVI_INVD_IOTLB_OPCODE	0x4	/* Invalidate IOTLB pages. */
125#define AMDVI_INVD_INTR_OPCODE	0x5	/* Invalidate Interrupt table. */
126#define AMDVI_PREFETCH_PAGES_OPCODE	0x6	/* Prefetch IOMMU pages. */
127#define AMDVI_COMP_PPR_OPCODE	0x7	/* Complete PPR request. */
128#define AMDVI_INV_ALL_OPCODE	0x8	/* Invalidate all. */
129
130/* Completion wait attributes. */
131#define AMDVI_CMP_WAIT_STORE	BIT(0)	/* Write back data. */
132#define AMDVI_CMP_WAIT_INTR	BIT(1)	/* Completion wait interrupt. */
133#define AMDVI_CMP_WAIT_FLUSH	BIT(2)	/* Flush queue. */
134
135/* Invalidate page. */
136#define AMDVI_INVD_PAGE_S	BIT(0)	/* Invalidation size. */
137#define AMDVI_INVD_PAGE_PDE	BIT(1)	/* Invalidate PDE. */
138#define AMDVI_INVD_PAGE_GN_GVA	BIT(2)	/* GPA or GVA. */
139
140#define AMDVI_INVD_PAGE_ALL_ADDR	(0x7FFFFFFFFFFFFULL << 12)
141
142/* Invalidate IOTLB. */
143#define AMDVI_INVD_IOTLB_S	BIT(0)	/* Invalidation size 4k or addr */
144#define AMDVI_INVD_IOTLB_GN_GVA	BIT(2)	/* GPA or GVA. */
145
146#define AMDVI_INVD_IOTLB_ALL_ADDR	(0x7FFFFFFFFFFFFULL << 12)
147/* XXX: add more command entries. */
148
149/*
150 * IOMMU event entry.
151 */
152struct amdvi_event {
153	uint16_t 	devid;
154	uint16_t 	pasid_hi;
155	uint16_t 	pasid_domid;	/* PASID low or DomainID */
156	uint16_t 	flag:12;
157	uint8_t		opcode:4;
158	uint64_t 	addr;
159} __attribute__((__packed__));
160CTASSERT(sizeof(struct amdvi_event) == 16);
161
162/* Various event types. */
163#define AMDVI_EVENT_INVALID_DTE		0x1
164#define AMDVI_EVENT_PFAULT		0x2
165#define AMDVI_EVENT_DTE_HW_ERROR	0x3
166#define AMDVI_EVENT_PAGE_HW_ERROR	0x4
167#define AMDVI_EVENT_ILLEGAL_CMD		0x5
168#define AMDVI_EVENT_CMD_HW_ERROR	0x6
169#define AMDVI_EVENT_IOTLB_TIMEOUT	0x7
170#define AMDVI_EVENT_INVALID_DTE_REQ	0x8
171#define AMDVI_EVENT_INVALID_PPR_REQ	0x9
172#define AMDVI_EVENT_COUNTER_ZERO	0xA
173
174#define AMDVI_EVENT_FLAG_MASK           0x1FF	/* Mask for event flags. */
175#define AMDVI_EVENT_FLAG_TYPE(x)        (((x) >> 9) & 0x3)
176
177/*
178 * IOMMU control block.
179 */
180struct amdvi_ctrl {
181	struct {
182		uint16_t size:9;
183		uint16_t :3;
184		uint64_t base:40;	/* Devtable register base. */
185		uint16_t :12;
186	} dte;
187	struct {
188		uint16_t :12;
189		uint64_t base:40;
190		uint8_t  :4;
191		uint8_t	 len:4;
192		uint8_t  :4;
193	} cmd;
194	struct {
195		uint16_t :12;
196		uint64_t base:40;
197		uint8_t  :4;
198		uint8_t	 len:4;
199		uint8_t  :4;
200	} event;
201	uint16_t control :13;
202	uint64_t	 :51;
203	struct {
204		uint8_t	 enable:1;
205		uint8_t	 allow:1;
206		uint16_t :10;
207		uint64_t base:40;
208		uint16_t :12;
209		uint16_t :12;
210		uint64_t limit:40;
211		uint16_t :12;
212	} excl;
213	/*
214	 * Revision 2 only.
215	 */
216	uint64_t ex_feature;
217	struct {
218		uint16_t :12;
219		uint64_t base:40;
220		uint8_t  :4;
221		uint8_t	 len:4;
222		uint8_t  :4;
223	} ppr;
224	uint64_t first_event;
225	uint64_t second_event;
226	uint64_t event_status;
227	/* Revision 2 only, end. */
228	uint8_t	 pad1[0x1FA8];		/* Padding. */
229	uint32_t cmd_head:19;
230	uint64_t :45;
231	uint32_t cmd_tail:19;
232	uint64_t :45;
233	uint32_t evt_head:19;
234	uint64_t :45;
235	uint32_t evt_tail:19;
236	uint64_t :45;
237	uint32_t status:19;
238	uint64_t :45;
239	uint64_t pad2;
240	uint8_t  :4;
241	uint16_t ppr_head:15;
242	uint64_t :45;
243	uint8_t  :4;
244	uint16_t ppr_tail:15;
245	uint64_t :45;
246	uint8_t	 pad3[0x1FC0];		/* Padding. */
247
248	/* XXX: More for rev2. */
249} __attribute__((__packed__));
250CTASSERT(offsetof(struct amdvi_ctrl, pad1)== 0x58);
251CTASSERT(offsetof(struct amdvi_ctrl, pad2)== 0x2028);
252CTASSERT(offsetof(struct amdvi_ctrl, pad3)== 0x2040);
253
254#define AMDVI_MMIO_V1_SIZE	(4 * PAGE_SIZE)	/* v1 size */
255/*
256 * AMF IOMMU v2 size including event counters
257 */
258#define AMDVI_MMIO_V2_SIZE	(8 * PAGE_SIZE)
259
260CTASSERT(sizeof(struct amdvi_ctrl) == 0x4000);
261CTASSERT(sizeof(struct amdvi_ctrl) == AMDVI_MMIO_V1_SIZE);
262
263/* IVHD flag */
264#define IVHD_FLAG_HTT		BIT(0)	/* Hypertransport Tunnel. */
265#define IVHD_FLAG_PPW		BIT(1)	/* Pass posted write. */
266#define IVHD_FLAG_RPPW		BIT(2)	/* Response pass posted write. */
267#define IVHD_FLAG_ISOC		BIT(3)	/* Isoc support. */
268#define IVHD_FLAG_IOTLB		BIT(4)	/* IOTLB support. */
269#define IVHD_FLAG_COH		BIT(5)	/* Coherent control, default 1 */
270#define IVHD_FLAG_PFS		BIT(6)	/* Prefetch IOMMU pages. */
271#define IVHD_FLAG_PPRS		BIT(7)	/* Peripheral page support. */
272
273/* IVHD device entry data setting. */
274#define IVHD_DEV_LINT0_PASS	BIT(6)	/* LINT0 interrupts. */
275#define IVHD_DEV_LINT1_PASS	BIT(7)	/* LINT1 interrupts. */
276
277/* Bit[5:4] for System Mgmt. Bit3 is reserved. */
278#define IVHD_DEV_INIT_PASS	BIT(0)	/* INIT */
279#define IVHD_DEV_EXTINTR_PASS	BIT(1)	/* ExtInt */
280#define IVHD_DEV_NMI_PASS	BIT(2)	/* NMI */
281
282/* IVHD 8-byte extended data settings. */
283#define IVHD_DEV_EXT_ATS_DISABLE	BIT(31)	/* Disable ATS */
284
285/* IOMMU control register. */
286#define AMDVI_CTRL_EN		BIT(0)	/* IOMMU enable. */
287#define AMDVI_CTRL_HTT		BIT(1)	/* Hypertransport tunnel enable. */
288#define AMDVI_CTRL_ELOG		BIT(2)	/* Event log enable. */
289#define AMDVI_CTRL_ELOGINT	BIT(3)	/* Event log interrupt. */
290#define AMDVI_CTRL_COMINT	BIT(4)	/* Completion wait interrupt. */
291#define AMDVI_CTRL_PPW		BIT(8)
292#define AMDVI_CTRL_RPPW		BIT(9)
293#define AMDVI_CTRL_COH		BIT(10)
294#define AMDVI_CTRL_ISOC		BIT(11)
295#define AMDVI_CTRL_CMD		BIT(12)	/* Command buffer enable. */
296#define AMDVI_CTRL_PPRLOG	BIT(13)
297#define AMDVI_CTRL_PPRINT	BIT(14)
298#define AMDVI_CTRL_PPREN	BIT(15)
299#define AMDVI_CTRL_GTE		BIT(16)	/* Guest translation enable. */
300#define AMDVI_CTRL_GAE		BIT(17)	/* Guest APIC enable. */
301
302/* Invalidation timeout. */
303#define AMDVI_CTRL_INV_NO_TO	0	/* No timeout. */
304#define AMDVI_CTRL_INV_TO_1ms	1	/* 1 ms */
305#define AMDVI_CTRL_INV_TO_10ms	2	/* 10 ms */
306#define AMDVI_CTRL_INV_TO_100ms	3	/* 100 ms */
307#define AMDVI_CTRL_INV_TO_1S	4	/* 1 second */
308#define AMDVI_CTRL_INV_TO_10S	5	/* 10 second */
309#define AMDVI_CTRL_INV_TO_100S	6	/* 100 second */
310
311/*
312 * Max number of PCI devices.
313 * 256 bus x 32 slot/devices x 8 functions.
314 */
315#define PCI_NUM_DEV_MAX		0x10000
316
317/* Maximum number of domains supported by IOMMU. */
318#define AMDVI_MAX_DOMAIN	(BIT(16) - 1)
319
320/*
321 * IOMMU Page Table attributes.
322 */
323#define AMDVI_PT_PRESENT	BIT(0)
324#define AMDVI_PT_COHERENT	BIT(60)
325#define AMDVI_PT_READ		BIT(61)
326#define AMDVI_PT_WRITE		BIT(62)
327
328#define AMDVI_PT_RW		(AMDVI_PT_READ | AMDVI_PT_WRITE)
329#define AMDVI_PT_MASK		0xFFFFFFFFFF000UL /* Only [51:12] for PA */
330
331#define AMDVI_PD_LEVEL_SHIFT	9
332#define AMDVI_PD_SUPER(x)	(((x) >> AMDVI_PD_LEVEL_SHIFT) == 7)
333/*
334 * IOMMU Status, offset 0x2020
335 */
336#define AMDVI_STATUS_EV_OF		BIT(0)	/* Event overflow. */
337#define AMDVI_STATUS_EV_INTR		BIT(1)	/* Event interrupt. */
338/* Completion wait command completed. */
339#define AMDVI_STATUS_CMP		BIT(2)
340
341#define	IVRS_CTRL_RID			1	/* MMIO RID */
342
343/* ACPI IVHD */
344struct ivhd_dev_cfg {
345	uint32_t start_id;
346	uint32_t end_id;
347	uint8_t	 data;			/* Device configuration. */
348	bool	 enable_ats;		/* ATS enabled for the device. */
349	int	 ats_qlen;		/* ATS invalidation queue depth. */
350};
351
352struct amdvi_domain {
353	uint64_t *ptp;			/* Highest level page table */
354	int	ptp_level;		/* Level of page tables */
355	u_int	id;			/* Domain id */
356	SLIST_ENTRY (amdvi_domain) next;
357};
358
359/*
360 * I/O Virtualization Hardware Definition Block (IVHD) type 0x10 (legacy)
361 * uses ACPI_IVRS_HARDWARE define in contrib/dev/acpica/include/actbl2.h
362 * New IVHD types 0x11 and 0x40 as defined in AMD IOMMU spec[48882] are missing in
363 * ACPI code. These new types add extra field EFR(Extended Feature Register).
364 * XXX : Use definition from ACPI when it is available.
365 */
366typedef struct acpi_ivrs_hardware_efr_sup
367{
368	ACPI_IVRS_HEADER Header;
369	UINT16 CapabilityOffset;   /* Offset for IOMMU control fields */
370	UINT64 BaseAddress;        /* IOMMU control registers */
371	UINT16 PciSegmentGroup;
372	UINT16 Info;               /* MSI number and unit ID */
373	UINT32 Attr;               /* IOMMU Feature */
374	UINT64 ExtFR;              /* IOMMU Extended Feature */
375	UINT64 Reserved;           /* v1 feature or v2 attribute */
376} __attribute__ ((__packed__)) ACPI_IVRS_HARDWARE_EFRSUP;
377CTASSERT(sizeof(ACPI_IVRS_HARDWARE_EFRSUP) == 40);
378
379/*
380 * Different type of IVHD.
381 * XXX: Use AcpiIvrsType once new IVHD types are available.
382*/
383enum IvrsType
384{
385	IVRS_TYPE_HARDWARE_LEGACY = 0x10, /* Legacy without EFRi support. */
386	IVRS_TYPE_HARDWARE_EFR 	  = 0x11, /* With EFR support. */
387	IVRS_TYPE_HARDWARE_MIXED  = 0x40, /* Mixed with EFR support. */
388};
389
390/*
391 * AMD IOMMU softc.
392 */
393struct amdvi_softc {
394	struct amdvi_ctrl *ctrl;	/* Control area. */
395	device_t 	dev;		/* IOMMU device. */
396	enum IvrsType   ivhd_type;	/* IOMMU IVHD type. */
397	bool		iotlb;		/* IOTLB supported by IOMMU */
398	struct amdvi_cmd *cmd;		/* Command descriptor area. */
399	int 		cmd_max;	/* Max number of commands. */
400	uint64_t	cmp_data;	/* Command completion write back. */
401	struct amdvi_event *event;	/* Event descriptor area. */
402	struct resource *event_res;	/* Event interrupt resource. */
403	void   		*event_tag;	/* Event interrupt tag. */
404	int		event_max;	/* Max number of events. */
405	int		event_irq;
406	int		event_rid;
407	/* ACPI various flags. */
408	uint32_t 	ivhd_flag;	/* ACPI IVHD flag. */
409	uint32_t 	ivhd_feature;	/* ACPI v1 Reserved or v2 attribute. */
410	uint64_t 	ext_feature;	/* IVHD EFR */
411	/* PCI related. */
412	uint16_t 	cap_off;	/* PCI Capability offset. */
413	uint8_t		pci_cap;	/* PCI capability. */
414	uint16_t 	pci_seg;	/* IOMMU PCI domain/segment. */
415	uint16_t 	pci_rid;	/* PCI BDF of IOMMU */
416	/* Device range under this IOMMU. */
417	uint16_t 	start_dev_rid;	/* First device under this IOMMU. */
418	uint16_t 	end_dev_rid;	/* Last device under this IOMMU. */
419
420	/* BIOS provided device configuration for end points. */
421	struct 		ivhd_dev_cfg dev_cfg[10];
422	int		dev_cfg_cnt;
423
424	/* Software statistics. */
425	uint64_t 	event_intr_cnt;	/* Total event INTR count. */
426	uint64_t 	total_cmd;	/* Total number of commands. */
427};
428
429int	amdvi_setup_hw(struct amdvi_softc *softc);
430int	amdvi_teardown_hw(struct amdvi_softc *softc);
431#endif /* _AMDVI_PRIV_H_ */
432