1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2016, Anish Gupta (anish@freebsd.org)
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice unmodified, this list of conditions, and the following
12 *    disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29#include <sys/cdefs.h>
30__FBSDID("$FreeBSD$");
31
32#include <sys/param.h>
33#include <sys/systm.h>
34#include <sys/bus.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/malloc.h>
38#include <sys/pcpu.h>
39#include <sys/rman.h>
40#include <sys/smp.h>
41#include <sys/sysctl.h>
42
43#include <vm/vm.h>
44#include <vm/pmap.h>
45
46#include <dev/pci/pcivar.h>
47#include <dev/pci/pcireg.h>
48
49#include <machine/resource.h>
50#include <machine/vmm.h>
51#include <machine/pmap.h>
52#include <machine/vmparam.h>
53#include <machine/pci_cfgreg.h>
54
55#include "pcib_if.h"
56
57#include "io/iommu.h"
58#include "amdvi_priv.h"
59
60SYSCTL_DECL(_hw_vmm);
61SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
62    NULL);
63
64#define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s)))
65#define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s)))
66
67/* Print RID or device ID in PCI string format. */
68#define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d)
69
70static void amdvi_dump_cmds(struct amdvi_softc *softc, int count);
71static void amdvi_print_dev_cap(struct amdvi_softc *softc);
72
73MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi");
74
75extern device_t *ivhd_devs;
76
77extern int ivhd_count;
78SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count,
79    0, NULL);
80
81static int amdvi_enable_user = 0;
82SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN,
83    &amdvi_enable_user, 0, NULL);
84TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user);
85
86#ifdef AMDVI_ATS_ENABLE
87/* XXX: ATS is not tested. */
88static int amdvi_enable_iotlb = 1;
89SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN,
90    &amdvi_enable_iotlb, 0, NULL);
91TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb);
92#endif
93
94static int amdvi_host_ptp = 1;	/* Use page tables for host. */
95SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN,
96    &amdvi_host_ptp, 0, NULL);
97TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp);
98
99/* Page table level used <= supported by h/w[v1=7]. */
100int amdvi_ptp_level = 4;
101SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN,
102    &amdvi_ptp_level, 0, NULL);
103TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level);
104
105/* Disable fault event reporting. */
106static int amdvi_disable_io_fault = 0;
107SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN,
108    &amdvi_disable_io_fault, 0, NULL);
109TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault);
110
111static uint32_t amdvi_dom_id = 0;	/* 0 is reserved for host. */
112SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD,
113    &amdvi_dom_id, 0, NULL);
114/*
115 * Device table entry.
116 * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes).
117 *	= 256 * 2 * PAGE_SIZE.
118 */
119static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE);
120CTASSERT(PCI_NUM_DEV_MAX == 0x10000);
121CTASSERT(sizeof(amdvi_dte) == 0x200000);
122
123static SLIST_HEAD (, amdvi_domain) dom_head;
124
125static inline uint32_t
126amdvi_pci_read(struct amdvi_softc *softc, int off)
127{
128
129	return (pci_cfgregread(PCI_RID2BUS(softc->pci_rid),
130	    PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid),
131	    off, 4));
132}
133
134#ifdef AMDVI_ATS_ENABLE
135/* XXX: Should be in pci.c */
136/*
137 * Check if device has ATS capability and its enabled.
138 * If ATS is absent or disabled, return (-1), otherwise ATS
139 * queue length.
140 */
141static int
142amdvi_find_ats_qlen(uint16_t devid)
143{
144	device_t dev;
145	uint32_t off, cap;
146	int qlen = -1;
147
148	dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid),
149			   PCI_RID2FUNC(devid));
150
151	if (!dev) {
152		return (-1);
153	}
154#define PCIM_ATS_EN	BIT(31)
155
156	if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) {
157		cap = pci_read_config(dev, off + 4, 4);
158		qlen = (cap & 0x1F);
159		qlen = qlen ? qlen : 32;
160		printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n",
161		       RID2PCI_STR(devid),
162		       (cap & PCIM_ATS_EN) ? "enabled" : "Disabled",
163		       qlen);
164		qlen = (cap & PCIM_ATS_EN) ? qlen : -1;
165	}
166
167	return (qlen);
168}
169
170/*
171 * Check if an endpoint device support device IOTLB or ATS.
172 */
173static inline bool
174amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid)
175{
176	struct ivhd_dev_cfg *cfg;
177	int qlen, i;
178	bool pci_ats, ivhd_ats;
179
180	qlen = amdvi_find_ats_qlen(devid);
181	if (qlen < 0)
182		return (false);
183
184	KASSERT(softc, ("softc is NULL"));
185	cfg = softc->dev_cfg;
186
187	ivhd_ats = false;
188	for (i = 0; i < softc->dev_cfg_cnt; i++) {
189		if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) {
190			ivhd_ats = cfg->enable_ats;
191			break;
192		}
193		cfg++;
194	}
195
196	pci_ats = (qlen < 0) ? false : true;
197	if (pci_ats != ivhd_ats)
198		device_printf(softc->dev,
199		    "BIOS bug: mismatch in ATS setting for %d.%d.%d,"
200		    "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen);
201
202	/* Ignore IVRS setting and respect PCI setting. */
203	return (pci_ats);
204}
205#endif
206
207/* Enable IOTLB support for IOMMU if its supported. */
208static inline void
209amdvi_hw_enable_iotlb(struct amdvi_softc *softc)
210{
211#ifndef AMDVI_ATS_ENABLE
212	softc->iotlb = false;
213#else
214	bool supported;
215
216	supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false;
217
218	if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) {
219		if (!supported)
220			device_printf(softc->dev, "IOTLB disabled by BIOS.\n");
221
222		if (supported && !amdvi_enable_iotlb) {
223			device_printf(softc->dev, "IOTLB disabled by user.\n");
224			supported = false;
225		}
226	} else
227		supported = false;
228
229	softc->iotlb = supported;
230
231#endif
232}
233
234static int
235amdvi_init_cmd(struct amdvi_softc *softc)
236{
237	struct amdvi_ctrl *ctrl = softc->ctrl;
238
239	ctrl->cmd.len = 8;	/* Use 256 command buffer entries. */
240	softc->cmd_max = 1 << ctrl->cmd.len;
241
242	softc->cmd = malloc(sizeof(struct amdvi_cmd) *
243	    softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO);
244
245	if ((uintptr_t)softc->cmd & PAGE_MASK)
246		panic("AMDVi: Command buffer not aligned on page boundary.");
247
248	ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE;
249	/*
250	 * XXX: Reset the h/w pointers in case IOMMU is restarting,
251	 * h/w doesn't clear these pointers based on empirical data.
252	 */
253	ctrl->cmd_tail = 0;
254	ctrl->cmd_head = 0;
255
256	return (0);
257}
258
259/*
260 * Note: Update tail pointer after we have written the command since tail
261 * pointer update cause h/w to execute new commands, see section 3.3
262 * of AMD IOMMU spec ver 2.0.
263 */
264/* Get the command tail pointer w/o updating it. */
265static struct amdvi_cmd *
266amdvi_get_cmd_tail(struct amdvi_softc *softc)
267{
268	struct amdvi_ctrl *ctrl;
269	struct amdvi_cmd *tail;
270
271	KASSERT(softc, ("softc is NULL"));
272	KASSERT(softc->cmd != NULL, ("cmd is NULL"));
273
274	ctrl = softc->ctrl;
275	KASSERT(ctrl != NULL, ("ctrl is NULL"));
276
277	tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd +
278	    ctrl->cmd_tail);
279
280	return (tail);
281}
282
283/*
284 * Update the command tail pointer which will start command execution.
285 */
286static void
287amdvi_update_cmd_tail(struct amdvi_softc *softc)
288{
289	struct amdvi_ctrl *ctrl;
290	int size;
291
292	size = sizeof(struct amdvi_cmd);
293	KASSERT(softc->cmd != NULL, ("cmd is NULL"));
294
295	ctrl = softc->ctrl;
296	KASSERT(ctrl != NULL, ("ctrl is NULL"));
297
298	ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max);
299	softc->total_cmd++;
300
301#ifdef AMDVI_DEBUG_CMD
302	device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n",
303	    ctrl->cmd_tail,
304	    ctrl->cmd_head);
305#endif
306
307}
308
309/*
310 * Various commands supported by IOMMU.
311 */
312
313/* Completion wait command. */
314static void
315amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data)
316{
317	struct amdvi_cmd *cmd;
318	uint64_t pa;
319
320	cmd = amdvi_get_cmd_tail(softc);
321	KASSERT(cmd != NULL, ("Cmd is NULL"));
322
323	pa = vtophys(&softc->cmp_data);
324	cmd->opcode = AMDVI_CMP_WAIT_OPCODE;
325	cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE;
326	cmd->word1 = (pa >> 32) & 0xFFFFF;
327	cmd->addr = data;
328
329	amdvi_update_cmd_tail(softc);
330}
331
332/* Invalidate device table entry. */
333static void
334amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid)
335{
336	struct amdvi_cmd *cmd;
337
338	cmd = amdvi_get_cmd_tail(softc);
339	KASSERT(cmd != NULL, ("Cmd is NULL"));
340	cmd->opcode = AMDVI_INVD_DTE_OPCODE;
341	cmd->word0 = devid;
342	amdvi_update_cmd_tail(softc);
343#ifdef AMDVI_DEBUG_CMD
344	device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid);
345#endif
346}
347
348/* Invalidate IOMMU page, use for invalidation of domain. */
349static void
350amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id,
351			  uint64_t addr, bool guest_nested,
352			  bool pde, bool page)
353{
354	struct amdvi_cmd *cmd;
355
356	cmd = amdvi_get_cmd_tail(softc);
357	KASSERT(cmd != NULL, ("Cmd is NULL"));
358
359
360	cmd->opcode = AMDVI_INVD_PAGE_OPCODE;
361	cmd->word1 = domain_id;
362	/*
363	 * Invalidate all addresses for this domain.
364	 */
365	cmd->addr = addr;
366	cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0;
367	cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0;
368
369	amdvi_update_cmd_tail(softc);
370}
371
372#ifdef AMDVI_ATS_ENABLE
373/* Invalidate device IOTLB. */
374static void
375amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid)
376{
377	struct amdvi_cmd *cmd;
378	int qlen;
379
380	if (!softc->iotlb)
381		return;
382
383	qlen = amdvi_find_ats_qlen(devid);
384	if (qlen < 0) {
385		panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n",
386		      qlen, RID2PCI_STR(devid));
387	}
388	cmd = amdvi_get_cmd_tail(softc);
389	KASSERT(cmd != NULL, ("Cmd is NULL"));
390
391#ifdef AMDVI_DEBUG_CMD
392	device_printf(softc->dev, "Invalidate IOTLB devID 0x%x"
393		      " Qlen:%d\n", devid, qlen);
394#endif
395	cmd->opcode = AMDVI_INVD_IOTLB_OPCODE;
396	cmd->word0 = devid;
397	cmd->word1 = qlen;
398	cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR |
399		AMDVI_INVD_IOTLB_S;
400	amdvi_update_cmd_tail(softc);
401}
402#endif
403
404#ifdef notyet				/* For Interrupt Remap. */
405static void
406amdvi_cmd_inv_intr_map(struct amdvi_softc *softc,
407		       uint16_t devid)
408{
409	struct amdvi_cmd *cmd;
410
411	cmd = amdvi_get_cmd_tail(softc);
412	KASSERT(cmd != NULL, ("Cmd is NULL"));
413	cmd->opcode = AMDVI_INVD_INTR_OPCODE;
414	cmd->word0 = devid;
415	amdvi_update_cmd_tail(softc);
416#ifdef AMDVI_DEBUG_CMD
417	device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid);
418#endif
419}
420#endif
421
422/* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */
423static void
424amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id)
425{
426	struct amdvi_cmd *cmd;
427
428	cmd = amdvi_get_cmd_tail(softc);
429	KASSERT(cmd != NULL, ("Cmd is NULL"));
430
431	/*
432	 * See section 3.3.3 of IOMMU spec rev 2.0, software note
433	 * for invalidating domain.
434	 */
435	amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR,
436				false, true, true);
437
438#ifdef AMDVI_DEBUG_CMD
439	device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id);
440
441#endif
442}
443
444static	bool
445amdvi_cmp_wait(struct amdvi_softc *softc)
446{
447	struct amdvi_ctrl *ctrl;
448	const uint64_t VERIFY = 0xA5A5;
449	volatile uint64_t *read;
450	int i;
451	bool status;
452
453	ctrl = softc->ctrl;
454	read = &softc->cmp_data;
455	*read = 0;
456	amdvi_cmd_cmp(softc, VERIFY);
457	/* Wait for h/w to update completion data. */
458	for (i = 0; i < 100 && (*read != VERIFY); i++) {
459		DELAY(1000);		/* 1 ms */
460	}
461	status = (VERIFY == softc->cmp_data) ? true : false;
462
463#ifdef AMDVI_DEBUG_CMD
464	if (status)
465		device_printf(softc->dev, "CMD completion DONE Tail:0x%x, "
466			      "Head:0x%x, loop:%d.\n", ctrl->cmd_tail,
467			      ctrl->cmd_head, loop);
468#endif
469	return (status);
470}
471
472static void
473amdvi_wait(struct amdvi_softc *softc)
474{
475	struct amdvi_ctrl *ctrl;
476	int i;
477
478	KASSERT(softc, ("softc is NULL"));
479
480	ctrl = softc->ctrl;
481	KASSERT(ctrl != NULL, ("ctrl is NULL"));
482	/* Don't wait if h/w is not enabled. */
483	if ((ctrl->control & AMDVI_CTRL_EN) == 0)
484		return;
485
486	for (i = 0; i < 10; i++) {
487		if (amdvi_cmp_wait(softc))
488			return;
489	}
490
491	device_printf(softc->dev, "Error: completion failed"
492		      " tail:0x%x, head:0x%x.\n",
493		      ctrl->cmd_tail, ctrl->cmd_head);
494	/* Dump the last command. */
495	amdvi_dump_cmds(softc, 1);
496}
497
498static void
499amdvi_dump_cmds(struct amdvi_softc *softc, int count)
500{
501	struct amdvi_ctrl *ctrl;
502	struct amdvi_cmd *cmd;
503	int off, i;
504
505	ctrl = softc->ctrl;
506	device_printf(softc->dev, "Dump last %d command(s):\n", count);
507	/*
508	 * If h/w is stuck in completion, it is the previous command,
509	 * start dumping from previous command onward.
510	 */
511	off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd),
512	    softc->cmd_max);
513	for (i = 0; off != ctrl->cmd_tail && i < count; i++) {
514		cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off);
515		printf("  [CMD%d, off:0x%x] opcode= 0x%x 0x%x"
516		    " 0x%x 0x%lx\n", i, off, cmd->opcode,
517		    cmd->word0, cmd->word1, cmd->addr);
518		off = (off + sizeof(struct amdvi_cmd)) %
519		    (softc->cmd_max * sizeof(struct amdvi_cmd));
520	}
521}
522
523static int
524amdvi_init_event(struct amdvi_softc *softc)
525{
526	struct amdvi_ctrl *ctrl;
527
528	ctrl = softc->ctrl;
529	ctrl->event.len = 8;
530	softc->event_max = 1 << ctrl->event.len;
531	softc->event = malloc(sizeof(struct amdvi_event) *
532	    softc->event_max, M_AMDVI, M_WAITOK | M_ZERO);
533	if ((uintptr_t)softc->event & PAGE_MASK) {
534		device_printf(softc->dev, "Event buffer not aligned on page.");
535		return (false);
536	}
537	ctrl->event.base = vtophys(softc->event) / PAGE_SIZE;
538
539	/* Reset the pointers. */
540	ctrl->evt_head = 0;
541	ctrl->evt_tail = 0;
542
543	return (0);
544}
545
546static inline void
547amdvi_decode_evt_flag(uint16_t flag)
548{
549
550	flag &= AMDVI_EVENT_FLAG_MASK;
551	printf(" 0x%b]\n", flag,
552		"\020"
553		"\001GN"
554		"\002NX"
555		"\003US"
556		"\004I"
557		"\005PR"
558		"\006RW"
559		"\007PE"
560		"\010RZ"
561		"\011TR"
562		);
563}
564
565/* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/
566static inline void
567amdvi_decode_evt_flag_type(uint8_t type)
568{
569
570	switch (AMDVI_EVENT_FLAG_TYPE(type)) {
571	case 0:
572		printf("RSVD\n");
573		break;
574	case 1:
575		printf("Master Abort\n");
576		break;
577	case 2:
578		printf("Target Abort\n");
579		break;
580	case 3:
581		printf("Data Err\n");
582		break;
583	default:
584		break;
585	}
586}
587
588static void
589amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr,
590    uint16_t flag)
591{
592
593	printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
594	    " Addr:0x%lx",
595	    devid, domid, addr);
596	amdvi_decode_evt_flag(flag);
597}
598
599static void
600amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr,
601    uint16_t flag)
602{
603
604	printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
605	    " Addr:0x%lx",
606	    devid, domid, addr);
607	amdvi_decode_evt_flag(flag);
608}
609
610static void
611amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid,
612    uint64_t addr, uint16_t flag)
613{
614
615	printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
616	    " Addr:0x%lx", devid, domid, addr);
617	amdvi_decode_evt_flag(flag);
618	amdvi_decode_evt_flag_type(flag);
619}
620
621static void
622amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr,
623    uint16_t flag)
624{
625
626	printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
627	    " Addr:0x%lx", devid, domid, addr);
628	amdvi_decode_evt_flag(flag);
629	amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag));
630}
631
632static void
633amdvi_decode_evt(struct amdvi_event *evt)
634{
635	struct amdvi_cmd *cmd;
636
637	switch (evt->opcode) {
638	case AMDVI_EVENT_INVALID_DTE:
639		amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid,
640		    evt->addr, evt->flag);
641		break;
642
643	case AMDVI_EVENT_PFAULT:
644		amdvi_decode_pf_evt(evt->devid, evt->pasid_domid,
645		    evt->addr, evt->flag);
646		break;
647
648	case AMDVI_EVENT_DTE_HW_ERROR:
649		amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid,
650		    evt->addr, evt->flag);
651		break;
652
653	case AMDVI_EVENT_PAGE_HW_ERROR:
654		amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid,
655		    evt->addr, evt->flag);
656		break;
657
658	case AMDVI_EVENT_ILLEGAL_CMD:
659		/* FALL THROUGH */
660	case AMDVI_EVENT_CMD_HW_ERROR:
661		printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ?
662		    "ILLEGAL CMD" : "CMD HW ERR");
663		cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr);
664		printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n",
665		    cmd->opcode, cmd->word0, cmd->word1, cmd->addr);
666		break;
667
668	case AMDVI_EVENT_IOTLB_TIMEOUT:
669		printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n",
670		    evt->devid, evt->addr);
671		break;
672
673	case AMDVI_EVENT_INVALID_DTE_REQ:
674		printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n",
675		    evt->devid, evt->addr, evt->flag >> 9,
676		    (evt->flag >> 8) & 1);
677		break;
678
679	case AMDVI_EVENT_INVALID_PPR_REQ:
680	case AMDVI_EVENT_COUNTER_ZERO:
681		printf("AMD-Vi: v2 events.\n");
682		break;
683
684	default:
685		printf("Unsupported AMD-Vi event:%d\n", evt->opcode);
686	}
687}
688
689static void
690amdvi_print_events(struct amdvi_softc *softc)
691{
692	struct amdvi_ctrl *ctrl;
693	struct amdvi_event *event;
694	int i, size;
695
696	ctrl = softc->ctrl;
697	size = sizeof(struct amdvi_event);
698	for (i = 0; i < softc->event_max; i++) {
699		event = &softc->event[ctrl->evt_head / size];
700		if (!event->opcode)
701			break;
702		device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n",
703		    i, ctrl->evt_head, ctrl->evt_tail);
704		amdvi_decode_evt(event);
705		ctrl->evt_head = MOD_INC(ctrl->evt_head, size,
706		    softc->event_max);
707	}
708}
709
710static int
711amdvi_init_dte(struct amdvi_softc *softc)
712{
713	struct amdvi_ctrl *ctrl;
714
715	ctrl = softc->ctrl;
716	ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE;
717	ctrl->dte.size = 0x1FF;		/* 2MB device table. */
718
719	return (0);
720}
721
722/*
723 * Not all capabilities of IOMMU are available in ACPI IVHD flag
724 * or EFR entry, read directly from device.
725 */
726static int
727amdvi_print_pci_cap(device_t dev)
728{
729	struct amdvi_softc *softc;
730	uint32_t off, cap;
731
732
733	softc = device_get_softc(dev);
734	off = softc->cap_off;
735
736	/*
737	 * Section 3.7.1 of IOMMU sepc rev 2.0.
738	 * Read capability from device.
739	 */
740	cap = amdvi_pci_read(softc, off);
741
742	/* Make sure capability type[18:16] is 3. */
743	KASSERT((((cap >> 16) & 0x7) == 0x3),
744	    ("Not a IOMMU capability 0x%x@0x%x", cap, off));
745
746	softc->pci_cap = cap >> 24;
747	device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n",
748	    cap, off, softc->pci_cap,
749	    "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt");
750
751	return (0);
752}
753
754static void
755amdvi_event_intr(void *arg)
756{
757	struct amdvi_softc *softc;
758	struct amdvi_ctrl *ctrl;
759
760	softc = (struct amdvi_softc *)arg;
761	ctrl = softc->ctrl;
762	device_printf(softc->dev, "EVT INTR %ld Status:0x%x"
763	    " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++,
764	    ctrl->status, ctrl->evt_head, ctrl->evt_tail);
765	printf("  [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n",
766	    softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head);
767
768	amdvi_print_events(softc);
769	ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
770}
771
772static void
773amdvi_free_evt_intr_res(device_t dev)
774{
775
776	struct amdvi_softc *softc;
777
778	softc = device_get_softc(dev);
779	if (softc->event_tag != NULL) {
780		bus_teardown_intr(dev, softc->event_res, softc->event_tag);
781	}
782	if (softc->event_res != NULL) {
783		bus_release_resource(dev, SYS_RES_IRQ, softc->event_rid,
784		    softc->event_res);
785	}
786	bus_delete_resource(dev, SYS_RES_IRQ, softc->event_rid);
787	PCIB_RELEASE_MSI(device_get_parent(device_get_parent(dev)),
788	    dev, 1, &softc->event_irq);
789}
790
791static bool
792amdvi_alloc_intr_resources(struct amdvi_softc *softc)
793{
794	struct amdvi_ctrl *ctrl;
795	device_t dev, pcib;
796	device_t mmio_dev;
797	uint64_t msi_addr;
798	uint32_t msi_data;
799	int err;
800
801	dev = softc->dev;
802	pcib = device_get_parent(device_get_parent(dev));
803	mmio_dev = pci_find_bsf(PCI_RID2BUS(softc->pci_rid),
804            PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid));
805	if (device_is_attached(mmio_dev)) {
806		device_printf(dev,
807		    "warning: IOMMU device is claimed by another driver %s\n",
808		    device_get_driver(mmio_dev)->name);
809	}
810
811	softc->event_irq = -1;
812	softc->event_rid = 0;
813
814	/*
815	 * Section 3.7.1 of IOMMU rev 2.0. With MSI, there is only one
816	 * interrupt. XXX: Enable MSI/X support.
817	 */
818	err = PCIB_ALLOC_MSI(pcib, dev, 1, 1, &softc->event_irq);
819	if (err) {
820		device_printf(dev,
821		    "Couldn't find event MSI IRQ resource.\n");
822		return (ENOENT);
823	}
824
825	err = bus_set_resource(dev, SYS_RES_IRQ, softc->event_rid,
826	    softc->event_irq, 1);
827	if (err) {
828		device_printf(dev, "Couldn't set event MSI resource.\n");
829		return (ENXIO);
830	}
831
832	softc->event_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
833	    &softc->event_rid, RF_ACTIVE);
834	if (!softc->event_res) {
835		device_printf(dev,
836		    "Unable to allocate event INTR resource.\n");
837		return (ENOMEM);
838	}
839
840	if (bus_setup_intr(dev, softc->event_res,
841	    INTR_TYPE_MISC | INTR_MPSAFE, NULL, amdvi_event_intr,
842	    softc, &softc->event_tag)) {
843		device_printf(dev, "Fail to setup event intr\n");
844		bus_release_resource(softc->dev, SYS_RES_IRQ,
845		    softc->event_rid, softc->event_res);
846		softc->event_res = NULL;
847		return (ENXIO);
848	}
849
850	bus_describe_intr(dev, softc->event_res, softc->event_tag,
851	    "fault");
852
853	err = PCIB_MAP_MSI(pcib, dev, softc->event_irq, &msi_addr,
854	    &msi_data);
855	if (err) {
856		device_printf(dev,
857		    "Event interrupt config failed, err=%d.\n",
858		    err);
859		amdvi_free_evt_intr_res(softc->dev);
860		return (err);
861	}
862
863	/* Clear interrupt status bits. */
864	ctrl = softc->ctrl;
865	ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
866
867	/* Now enable MSI interrupt. */
868	pci_enable_msi(mmio_dev, msi_addr, msi_data);
869	return (0);
870}
871
872
873static void
874amdvi_print_dev_cap(struct amdvi_softc *softc)
875{
876	struct ivhd_dev_cfg *cfg;
877	int i;
878
879	cfg = softc->dev_cfg;
880	for (i = 0; i < softc->dev_cfg_cnt; i++) {
881		device_printf(softc->dev, "device [0x%x - 0x%x]"
882		    "config:%b%s\n", cfg->start_id, cfg->end_id,
883		    cfg->data,
884		    "\020\001INIT\002ExtInt\003NMI"
885		    "\007LINT0\008LINT1",
886		    cfg->enable_ats ? "ATS enabled" : "");
887		cfg++;
888	}
889}
890
891static int
892amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS)
893{
894	struct amdvi_softc *softc;
895	int result, type, error = 0;
896
897	softc = (struct amdvi_softc *)arg1;
898	type = arg2;
899
900	switch (type) {
901	case 0:
902		result = softc->ctrl->cmd_head;
903		error = sysctl_handle_int(oidp, &result, 0,
904		    req);
905		break;
906	case 1:
907		result = softc->ctrl->cmd_tail;
908		error = sysctl_handle_int(oidp, &result, 0,
909		    req);
910		break;
911	case 2:
912		result = softc->ctrl->evt_head;
913		error = sysctl_handle_int(oidp, &result, 0,
914		    req);
915		break;
916	case 3:
917		result = softc->ctrl->evt_tail;
918		error = sysctl_handle_int(oidp, &result, 0,
919		    req);
920		break;
921
922	default:
923		device_printf(softc->dev, "Unknown sysctl:%d\n", type);
924	}
925
926	return (error);
927}
928
929static void
930amdvi_add_sysctl(struct amdvi_softc *softc)
931{
932	struct sysctl_oid_list *child;
933	struct sysctl_ctx_list *ctx;
934	device_t dev;
935
936	dev = softc->dev;
937	ctx = device_get_sysctl_ctx(dev);
938	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
939
940	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD,
941	    &softc->event_intr_cnt, "Event interrupt count");
942	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD,
943	    &softc->total_cmd, "Command submitted count");
944	SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD,
945	    &softc->pci_rid, 0, "IOMMU RID");
946	SYSCTL_ADD_U16(ctx, child, OID_AUTO, "start_dev_rid", CTLFLAG_RD,
947	    &softc->start_dev_rid, 0, "Start of device under this IOMMU");
948	SYSCTL_ADD_U16(ctx, child, OID_AUTO, "end_dev_rid", CTLFLAG_RD,
949	    &softc->end_dev_rid, 0, "End of device under this IOMMU");
950	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head",
951	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0,
952	    amdvi_handle_sysctl, "IU", "Command head");
953	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail",
954	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1,
955	    amdvi_handle_sysctl, "IU", "Command tail");
956	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head",
957	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2,
958	    amdvi_handle_sysctl, "IU", "Command head");
959	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail",
960	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3,
961	    amdvi_handle_sysctl, "IU", "Command tail");
962}
963
964int
965amdvi_setup_hw(struct amdvi_softc *softc)
966{
967	device_t dev;
968	int status;
969
970	dev = softc->dev;
971
972	amdvi_hw_enable_iotlb(softc);
973
974	amdvi_print_dev_cap(softc);
975
976	if ((status = amdvi_print_pci_cap(dev)) != 0) {
977		device_printf(dev, "PCI capability.\n");
978		return (status);
979	}
980	if ((status = amdvi_init_cmd(softc)) != 0) {
981		device_printf(dev, "Couldn't configure command buffer.\n");
982		return (status);
983	}
984	if ((status = amdvi_init_event(softc)) != 0) {
985		device_printf(dev, "Couldn't configure event buffer.\n");
986		return (status);
987	}
988	if ((status = amdvi_init_dte(softc)) != 0) {
989		device_printf(dev, "Couldn't configure device table.\n");
990		return (status);
991	}
992	if ((status = amdvi_alloc_intr_resources(softc)) != 0) {
993		return (status);
994	}
995	amdvi_add_sysctl(softc);
996	return (0);
997}
998
999int
1000amdvi_teardown_hw(struct amdvi_softc *softc)
1001{
1002	device_t dev;
1003
1004	dev = softc->dev;
1005
1006	/*
1007	 * Called after disable, h/w is stopped by now, free all the resources.
1008	 */
1009	amdvi_free_evt_intr_res(dev);
1010
1011	if (softc->cmd)
1012		free(softc->cmd, M_AMDVI);
1013
1014	if (softc->event)
1015		free(softc->event, M_AMDVI);
1016
1017	return (0);
1018}
1019
1020/*********** bhyve interfaces *********************/
1021static int
1022amdvi_init(void)
1023{
1024	if (!ivhd_count) {
1025		return (EIO);
1026	}
1027	if (!amdvi_enable_user && ivhd_count) {
1028		printf("bhyve: Found %d AMD-Vi/IOMMU device(s), "
1029		    "use hw.vmm.amdvi.enable=1 to enable pass-through.\n",
1030		    ivhd_count);
1031		return (EINVAL);
1032	}
1033	return (0);
1034}
1035
1036static void
1037amdvi_cleanup(void)
1038{
1039	/* Nothing. */
1040}
1041
1042static uint16_t
1043amdvi_domainId(void)
1044{
1045
1046	/*
1047	 * If we hit maximum domain limit, rollover leaving host
1048	 * domain(0).
1049	 * XXX: make sure that this domain is not used.
1050	 */
1051	if (amdvi_dom_id == AMDVI_MAX_DOMAIN)
1052		amdvi_dom_id = 1;
1053
1054	return ((uint16_t)amdvi_dom_id++);
1055}
1056
1057static void
1058amdvi_do_inv_domain(uint16_t domain_id, bool create)
1059{
1060	struct amdvi_softc *softc;
1061	int i;
1062
1063	for (i = 0; i < ivhd_count; i++) {
1064		softc = device_get_softc(ivhd_devs[i]);
1065		KASSERT(softc, ("softc is NULL"));
1066		/*
1067		 * If not present pages are cached, invalidate page after
1068		 * creating domain.
1069		 */
1070#if 0
1071		if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0))
1072			continue;
1073#endif
1074		amdvi_inv_domain(softc, domain_id);
1075		amdvi_wait(softc);
1076	}
1077}
1078
1079static void *
1080amdvi_create_domain(vm_paddr_t maxaddr)
1081{
1082	struct amdvi_domain *dom;
1083
1084	dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK);
1085	dom->id = amdvi_domainId();
1086	//dom->maxaddr = maxaddr;
1087#ifdef AMDVI_DEBUG_CMD
1088	printf("Created domain #%d\n", dom->id);
1089#endif
1090	/*
1091	 * Host domain(#0) don't create translation table.
1092	 */
1093	if (dom->id || amdvi_host_ptp)
1094		dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1095
1096	dom->ptp_level = amdvi_ptp_level;
1097
1098	amdvi_do_inv_domain(dom->id, true);
1099	SLIST_INSERT_HEAD(&dom_head, dom, next);
1100
1101	return (dom);
1102}
1103
1104static void
1105amdvi_free_ptp(uint64_t *ptp, int level)
1106{
1107	int i;
1108
1109	if (level < 1)
1110		return;
1111
1112	for (i = 0; i < NPTEPG ; i++) {
1113		if ((ptp[i] & AMDVI_PT_PRESENT) == 0)
1114			continue;
1115		/* XXX: Add super-page or PTE mapping > 4KB. */
1116#ifdef notyet
1117		/* Super-page mapping. */
1118		if (AMDVI_PD_SUPER(ptp[i]))
1119			continue;
1120#endif
1121
1122		amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i]
1123		    & AMDVI_PT_MASK), level - 1);
1124
1125	}
1126
1127	free(ptp, M_AMDVI);
1128}
1129
1130static void
1131amdvi_destroy_domain(void *arg)
1132{
1133	struct amdvi_domain *domain;
1134
1135	domain = (struct amdvi_domain *)arg;
1136	KASSERT(domain, ("domain is NULL"));
1137#ifdef AMDVI_DEBUG_CMD
1138	printf("Destroying domain %d\n", domain->id);
1139#endif
1140	if (domain->ptp)
1141		amdvi_free_ptp(domain->ptp, domain->ptp_level);
1142
1143	amdvi_do_inv_domain(domain->id, false);
1144	SLIST_REMOVE(&dom_head, domain, amdvi_domain, next);
1145	free(domain, M_AMDVI);
1146}
1147
1148static uint64_t
1149amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa,
1150    vm_paddr_t hpa, uint64_t pg_size, bool create)
1151{
1152	uint64_t *page, pa;
1153	int shift, index;
1154	const int PT_SHIFT = 9;
1155	const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1;	/* Based on PT_SHIFT */
1156
1157	if (!pg_size)
1158		return (0);
1159
1160	if (hpa & (pg_size - 1)) {
1161		printf("HPA is not size aligned.\n");
1162		return (0);
1163	}
1164	if (gpa & (pg_size - 1)) {
1165		printf("HPA is not size aligned.\n");
1166		return (0);
1167	}
1168	shift = PML4SHIFT;
1169	while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) {
1170		index = (gpa >> shift) & PT_INDEX_MASK;
1171
1172		if ((pt[index] == 0) && create) {
1173			page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1174			pa = vtophys(page);
1175			pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW |
1176			    ((level - 1) << AMDVI_PD_LEVEL_SHIFT);
1177		}
1178#ifdef AMDVI_DEBUG_PTE
1179		if ((gpa % 0x1000000) == 0)
1180			printf("[level%d, shift = %d]PTE:0x%lx\n",
1181			    level, shift, pt[index]);
1182#endif
1183#define PTE2PA(x)	((uint64_t)(x) & AMDVI_PT_MASK)
1184		pa = PTE2PA(pt[index]);
1185		pt = (uint64_t *)PHYS_TO_DMAP(pa);
1186		shift -= PT_SHIFT;
1187		level--;
1188	}
1189
1190	/* Leaf entry. */
1191	index = (gpa >> shift) & PT_INDEX_MASK;
1192
1193	if (create) {
1194		pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT;
1195	} else
1196		pt[index] = 0;
1197
1198#ifdef AMDVI_DEBUG_PTE
1199	if ((gpa % 0x1000000) == 0)
1200		printf("[Last level%d, shift = %d]PTE:0x%lx\n",
1201		    level, shift, pt[index]);
1202#endif
1203	return (1ULL << shift);
1204}
1205
1206static uint64_t
1207amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa,
1208    vm_paddr_t hpa, uint64_t size, bool create)
1209{
1210	uint64_t mapped, *ptp, len;
1211	int level;
1212
1213	KASSERT(domain, ("domain is NULL"));
1214	level = domain->ptp_level;
1215	KASSERT(level, ("Page table level is 0"));
1216
1217	ptp = domain->ptp;
1218	KASSERT(ptp, ("PTP is NULL"));
1219	mapped = 0;
1220	while (mapped < size) {
1221		len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped,
1222		    PAGE_SIZE, create);
1223		if (!len) {
1224			printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n",
1225			    hpa, gpa);
1226			return (0);
1227		}
1228		mapped += len;
1229	}
1230
1231	return (mapped);
1232}
1233
1234static uint64_t
1235amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa,
1236    uint64_t len)
1237{
1238	struct amdvi_domain *domain;
1239
1240	domain = (struct amdvi_domain *)arg;
1241
1242	if (domain->id && !domain->ptp) {
1243		printf("ptp is NULL");
1244		return (-1);
1245	}
1246
1247	/*
1248	 * If host domain is created w/o page table, skip IOMMU page
1249	 * table set-up.
1250	 */
1251	if (domain->ptp)
1252		return (amdvi_update_mapping(domain, gpa, hpa, len, true));
1253	else
1254		return (len);
1255}
1256
1257static uint64_t
1258amdvi_destroy_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
1259{
1260	struct amdvi_domain *domain;
1261
1262	domain = (struct amdvi_domain *)arg;
1263	/*
1264	 * If host domain is created w/o page table, skip IOMMU page
1265	 * table set-up.
1266	 */
1267	if (domain->ptp)
1268		return (amdvi_update_mapping(domain, gpa, 0, len, false));
1269	return
1270	    (len);
1271}
1272
1273static struct amdvi_softc *
1274amdvi_find_iommu(uint16_t devid)
1275{
1276	struct amdvi_softc *softc;
1277	int i;
1278
1279	for (i = 0; i < ivhd_count; i++) {
1280		softc = device_get_softc(ivhd_devs[i]);
1281		if ((devid >= softc->start_dev_rid) &&
1282		    (devid <= softc->end_dev_rid))
1283			return (softc);
1284	}
1285
1286	/*
1287	 * XXX: BIOS bug, device not in IVRS table, assume its from first IOMMU.
1288	 */
1289	printf("BIOS bug device(%d.%d.%d) doesn't have IVHD entry.\n",
1290	    RID2PCI_STR(devid));
1291
1292	return (device_get_softc(ivhd_devs[0]));
1293}
1294
1295/*
1296 * Set-up device table entry.
1297 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must
1298 * be set concurrently, e.g. read and write bits.
1299 */
1300static void
1301amdvi_set_dte(struct amdvi_domain *domain, uint16_t devid, bool enable)
1302{
1303	struct amdvi_softc *softc;
1304	struct amdvi_dte* temp;
1305
1306	KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid));
1307
1308	softc = amdvi_find_iommu(devid);
1309	KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid));
1310
1311	temp = &amdvi_dte[devid];
1312
1313#ifdef AMDVI_ATS_ENABLE
1314	/* If IOMMU and device support IOTLB, enable it. */
1315	if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb)
1316		temp->iotlb_enable = 1;
1317#endif
1318
1319	/* Avoid duplicate I/O faults. */
1320	temp->sup_second_io_fault = 1;
1321	temp->sup_all_io_fault = amdvi_disable_io_fault;
1322
1323	temp->dt_valid = 1;
1324	temp->domain_id = domain->id;
1325
1326	if (enable) {
1327		if (domain->ptp) {
1328			temp->pt_base = vtophys(domain->ptp) >> 12;
1329			temp->pt_level = amdvi_ptp_level;
1330		}
1331		/*
1332		 * XXX: Page table valid[TV] bit must be set even if host domain
1333		 * page tables are not enabled.
1334		 */
1335		temp->pt_valid = 1;
1336		temp->read_allow = 1;
1337		temp->write_allow = 1;
1338	}
1339}
1340
1341static void
1342amdvi_inv_device(uint16_t devid)
1343{
1344	struct amdvi_softc *softc;
1345
1346	softc = amdvi_find_iommu(devid);
1347	KASSERT(softc, ("softc is NULL"));
1348
1349	amdvi_cmd_inv_dte(softc, devid);
1350#ifdef AMDVI_ATS_ENABLE
1351	if (amdvi_dev_support_iotlb(softc, devid))
1352		amdvi_cmd_inv_iotlb(softc, devid);
1353#endif
1354	amdvi_wait(softc);
1355}
1356
1357static void
1358amdvi_add_device(void *arg, uint16_t devid)
1359{
1360	struct amdvi_domain *domain;
1361
1362	domain = (struct amdvi_domain *)arg;
1363	KASSERT(domain != NULL, ("domain is NULL"));
1364#ifdef AMDVI_DEBUG_CMD
1365	printf("Assigning device(%d.%d.%d) to domain:%d\n",
1366	    RID2PCI_STR(devid), domain->id);
1367#endif
1368	amdvi_set_dte(domain, devid, true);
1369	amdvi_inv_device(devid);
1370}
1371
1372static void
1373amdvi_remove_device(void *arg, uint16_t devid)
1374{
1375	struct amdvi_domain *domain;
1376
1377	domain = (struct amdvi_domain *)arg;
1378#ifdef AMDVI_DEBUG_CMD
1379	printf("Remove device(0x%x) from domain:%d\n",
1380	       devid, domain->id);
1381#endif
1382	amdvi_set_dte(domain, devid, false);
1383	amdvi_inv_device(devid);
1384}
1385
1386static void
1387amdvi_enable(void)
1388{
1389	struct amdvi_ctrl *ctrl;
1390	struct amdvi_softc *softc;
1391	uint64_t val;
1392	int i;
1393
1394	for (i = 0; i < ivhd_count; i++) {
1395		softc = device_get_softc(ivhd_devs[i]);
1396		KASSERT(softc, ("softc is NULL\n"));
1397		ctrl = softc->ctrl;
1398		KASSERT(ctrl, ("ctrl is NULL\n"));
1399
1400		val = (	AMDVI_CTRL_EN		|
1401			AMDVI_CTRL_CMD		|
1402			AMDVI_CTRL_ELOG		|
1403			AMDVI_CTRL_ELOGINT	|
1404			AMDVI_CTRL_INV_TO_1S);
1405
1406		if (softc->ivhd_flag & IVHD_FLAG_COH)
1407			val |= AMDVI_CTRL_COH;
1408		if (softc->ivhd_flag & IVHD_FLAG_HTT)
1409			val |= AMDVI_CTRL_HTT;
1410		if (softc->ivhd_flag & IVHD_FLAG_RPPW)
1411			val |= AMDVI_CTRL_RPPW;
1412		if (softc->ivhd_flag & IVHD_FLAG_PPW)
1413			val |= AMDVI_CTRL_PPW;
1414		if (softc->ivhd_flag & IVHD_FLAG_ISOC)
1415			val |= AMDVI_CTRL_ISOC;
1416
1417		ctrl->control = val;
1418	}
1419}
1420
1421static void
1422amdvi_disable(void)
1423{
1424	struct amdvi_ctrl *ctrl;
1425	struct amdvi_softc *softc;
1426	int i;
1427
1428	for (i = 0; i < ivhd_count; i++) {
1429		softc = device_get_softc(ivhd_devs[i]);
1430		KASSERT(softc, ("softc is NULL\n"));
1431		ctrl = softc->ctrl;
1432		KASSERT(ctrl, ("ctrl is NULL\n"));
1433
1434		ctrl->control = 0;
1435	}
1436}
1437
1438static void
1439amdvi_inv_tlb(void *arg)
1440{
1441	struct amdvi_domain *domain;
1442
1443	domain = (struct amdvi_domain *)arg;
1444	KASSERT(domain, ("domain is NULL"));
1445	amdvi_do_inv_domain(domain->id, false);
1446}
1447
1448struct iommu_ops iommu_ops_amd = {
1449	amdvi_init,
1450	amdvi_cleanup,
1451	amdvi_enable,
1452	amdvi_disable,
1453	amdvi_create_domain,
1454	amdvi_destroy_domain,
1455	amdvi_create_mapping,
1456	amdvi_destroy_mapping,
1457	amdvi_add_device,
1458	amdvi_remove_device,
1459	amdvi_inv_tlb
1460};
1461