14c87aefeSPatrick Mooney /*-
2*32640292SAndy Fiddaman * SPDX-License-Identifier: BSD-2-Clause
34c87aefeSPatrick Mooney *
44c87aefeSPatrick Mooney * Copyright (c) 2016, Anish Gupta (anish@freebsd.org)
54c87aefeSPatrick Mooney * All rights reserved.
64c87aefeSPatrick Mooney *
74c87aefeSPatrick Mooney * Redistribution and use in source and binary forms, with or without
84c87aefeSPatrick Mooney * modification, are permitted provided that the following conditions
94c87aefeSPatrick Mooney * are met:
104c87aefeSPatrick Mooney * 1. Redistributions of source code must retain the above copyright
114c87aefeSPatrick Mooney * notice unmodified, this list of conditions, and the following
124c87aefeSPatrick Mooney * disclaimer.
134c87aefeSPatrick Mooney * 2. Redistributions in binary form must reproduce the above copyright
144c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer in the
154c87aefeSPatrick Mooney * documentation and/or other materials provided with the distribution.
164c87aefeSPatrick Mooney *
174c87aefeSPatrick Mooney * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
184c87aefeSPatrick Mooney * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
194c87aefeSPatrick Mooney * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
204c87aefeSPatrick Mooney * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
214c87aefeSPatrick Mooney * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
224c87aefeSPatrick Mooney * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
234c87aefeSPatrick Mooney * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
244c87aefeSPatrick Mooney * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
254c87aefeSPatrick Mooney * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
264c87aefeSPatrick Mooney * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
274c87aefeSPatrick Mooney */
284c87aefeSPatrick Mooney
294c87aefeSPatrick Mooney #include <sys/cdefs.h>
304c87aefeSPatrick Mooney
314c87aefeSPatrick Mooney #include <sys/param.h>
324c87aefeSPatrick Mooney #include <sys/systm.h>
334c87aefeSPatrick Mooney #include <sys/bus.h>
344c87aefeSPatrick Mooney #include <sys/kernel.h>
354c87aefeSPatrick Mooney #include <sys/module.h>
364c87aefeSPatrick Mooney #include <sys/malloc.h>
374c87aefeSPatrick Mooney #include <sys/pcpu.h>
384c87aefeSPatrick Mooney #include <sys/rman.h>
394c87aefeSPatrick Mooney #include <sys/sysctl.h>
404c87aefeSPatrick Mooney
414c87aefeSPatrick Mooney #include <dev/pci/pcivar.h>
424c87aefeSPatrick Mooney #include <dev/pci/pcireg.h>
434c87aefeSPatrick Mooney
444c87aefeSPatrick Mooney #include <machine/resource.h>
454c87aefeSPatrick Mooney #include <machine/vmm.h>
464c87aefeSPatrick Mooney #include <machine/vmparam.h>
474c87aefeSPatrick Mooney #include <machine/pci_cfgreg.h>
484c87aefeSPatrick Mooney
492b948146SAndy Fiddaman #include "ivhd_if.h"
504c87aefeSPatrick Mooney #include "pcib_if.h"
514c87aefeSPatrick Mooney
524c87aefeSPatrick Mooney #include "io/iommu.h"
534c87aefeSPatrick Mooney #include "amdvi_priv.h"
544c87aefeSPatrick Mooney
554c87aefeSPatrick Mooney SYSCTL_DECL(_hw_vmm);
56154972afSPatrick Mooney SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
57154972afSPatrick Mooney NULL);
584c87aefeSPatrick Mooney
594c87aefeSPatrick Mooney #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s)))
604c87aefeSPatrick Mooney #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s)))
614c87aefeSPatrick Mooney
624c87aefeSPatrick Mooney /* Print RID or device ID in PCI string format. */
634c87aefeSPatrick Mooney #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d)
644c87aefeSPatrick Mooney
65154972afSPatrick Mooney static void amdvi_dump_cmds(struct amdvi_softc *softc, int count);
664c87aefeSPatrick Mooney static void amdvi_print_dev_cap(struct amdvi_softc *softc);
674c87aefeSPatrick Mooney
684c87aefeSPatrick Mooney MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi");
694c87aefeSPatrick Mooney
704c87aefeSPatrick Mooney extern device_t *ivhd_devs;
714c87aefeSPatrick Mooney
724c87aefeSPatrick Mooney extern int ivhd_count;
734c87aefeSPatrick Mooney SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count,
744c87aefeSPatrick Mooney 0, NULL);
754c87aefeSPatrick Mooney
764c87aefeSPatrick Mooney static int amdvi_enable_user = 0;
774c87aefeSPatrick Mooney SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN,
784c87aefeSPatrick Mooney &amdvi_enable_user, 0, NULL);
794c87aefeSPatrick Mooney TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user);
804c87aefeSPatrick Mooney
814c87aefeSPatrick Mooney #ifdef AMDVI_ATS_ENABLE
824c87aefeSPatrick Mooney /* XXX: ATS is not tested. */
834c87aefeSPatrick Mooney static int amdvi_enable_iotlb = 1;
844c87aefeSPatrick Mooney SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN,
854c87aefeSPatrick Mooney &amdvi_enable_iotlb, 0, NULL);
864c87aefeSPatrick Mooney TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb);
874c87aefeSPatrick Mooney #endif
884c87aefeSPatrick Mooney
894c87aefeSPatrick Mooney static int amdvi_host_ptp = 1; /* Use page tables for host. */
904c87aefeSPatrick Mooney SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN,
914c87aefeSPatrick Mooney &amdvi_host_ptp, 0, NULL);
924c87aefeSPatrick Mooney TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp);
934c87aefeSPatrick Mooney
944c87aefeSPatrick Mooney /* Page table level used <= supported by h/w[v1=7]. */
95154972afSPatrick Mooney int amdvi_ptp_level = 4;
964c87aefeSPatrick Mooney SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN,
974c87aefeSPatrick Mooney &amdvi_ptp_level, 0, NULL);
984c87aefeSPatrick Mooney TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level);
994c87aefeSPatrick Mooney
1004c87aefeSPatrick Mooney /* Disable fault event reporting. */
1014c87aefeSPatrick Mooney static int amdvi_disable_io_fault = 0;
1024c87aefeSPatrick Mooney SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN,
1034c87aefeSPatrick Mooney &amdvi_disable_io_fault, 0, NULL);
1044c87aefeSPatrick Mooney TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault);
1054c87aefeSPatrick Mooney
1064c87aefeSPatrick Mooney static uint32_t amdvi_dom_id = 0; /* 0 is reserved for host. */
1074c87aefeSPatrick Mooney SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD,
1084c87aefeSPatrick Mooney &amdvi_dom_id, 0, NULL);
1094c87aefeSPatrick Mooney /*
1104c87aefeSPatrick Mooney * Device table entry.
1114c87aefeSPatrick Mooney * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes).
1124c87aefeSPatrick Mooney * = 256 * 2 * PAGE_SIZE.
1134c87aefeSPatrick Mooney */
1144c87aefeSPatrick Mooney static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE);
1154c87aefeSPatrick Mooney CTASSERT(PCI_NUM_DEV_MAX == 0x10000);
1164c87aefeSPatrick Mooney CTASSERT(sizeof(amdvi_dte) == 0x200000);
1174c87aefeSPatrick Mooney
1184c87aefeSPatrick Mooney static SLIST_HEAD (, amdvi_domain) dom_head;
1194c87aefeSPatrick Mooney
1204c87aefeSPatrick Mooney static inline uint32_t
amdvi_pci_read(struct amdvi_softc * softc,int off)1214c87aefeSPatrick Mooney amdvi_pci_read(struct amdvi_softc *softc, int off)
1224c87aefeSPatrick Mooney {
1234c87aefeSPatrick Mooney
1244c87aefeSPatrick Mooney return (pci_cfgregread(PCI_RID2BUS(softc->pci_rid),
1254c87aefeSPatrick Mooney PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid),
1264c87aefeSPatrick Mooney off, 4));
1274c87aefeSPatrick Mooney }
1284c87aefeSPatrick Mooney
1294c87aefeSPatrick Mooney #ifdef AMDVI_ATS_ENABLE
1304c87aefeSPatrick Mooney /* XXX: Should be in pci.c */
1314c87aefeSPatrick Mooney /*
1324c87aefeSPatrick Mooney * Check if device has ATS capability and its enabled.
1334c87aefeSPatrick Mooney * If ATS is absent or disabled, return (-1), otherwise ATS
1344c87aefeSPatrick Mooney * queue length.
1354c87aefeSPatrick Mooney */
1364c87aefeSPatrick Mooney static int
amdvi_find_ats_qlen(uint16_t devid)1374c87aefeSPatrick Mooney amdvi_find_ats_qlen(uint16_t devid)
1384c87aefeSPatrick Mooney {
1394c87aefeSPatrick Mooney device_t dev;
1404c87aefeSPatrick Mooney uint32_t off, cap;
1414c87aefeSPatrick Mooney int qlen = -1;
1424c87aefeSPatrick Mooney
1434c87aefeSPatrick Mooney dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid),
1444c87aefeSPatrick Mooney PCI_RID2FUNC(devid));
1454c87aefeSPatrick Mooney
1464c87aefeSPatrick Mooney if (!dev) {
1474c87aefeSPatrick Mooney return (-1);
1484c87aefeSPatrick Mooney }
1494c87aefeSPatrick Mooney #define PCIM_ATS_EN BIT(31)
1504c87aefeSPatrick Mooney
1514c87aefeSPatrick Mooney if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) {
1524c87aefeSPatrick Mooney cap = pci_read_config(dev, off + 4, 4);
1534c87aefeSPatrick Mooney qlen = (cap & 0x1F);
1544c87aefeSPatrick Mooney qlen = qlen ? qlen : 32;
1554c87aefeSPatrick Mooney printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n",
1564c87aefeSPatrick Mooney RID2PCI_STR(devid),
1574c87aefeSPatrick Mooney (cap & PCIM_ATS_EN) ? "enabled" : "Disabled",
1584c87aefeSPatrick Mooney qlen);
1594c87aefeSPatrick Mooney qlen = (cap & PCIM_ATS_EN) ? qlen : -1;
1604c87aefeSPatrick Mooney }
1614c87aefeSPatrick Mooney
1624c87aefeSPatrick Mooney return (qlen);
1634c87aefeSPatrick Mooney }
1644c87aefeSPatrick Mooney
1654c87aefeSPatrick Mooney /*
1664c87aefeSPatrick Mooney * Check if an endpoint device support device IOTLB or ATS.
1674c87aefeSPatrick Mooney */
1684c87aefeSPatrick Mooney static inline bool
amdvi_dev_support_iotlb(struct amdvi_softc * softc,uint16_t devid)1694c87aefeSPatrick Mooney amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid)
1704c87aefeSPatrick Mooney {
1714c87aefeSPatrick Mooney struct ivhd_dev_cfg *cfg;
1724c87aefeSPatrick Mooney int qlen, i;
1734c87aefeSPatrick Mooney bool pci_ats, ivhd_ats;
1744c87aefeSPatrick Mooney
1754c87aefeSPatrick Mooney qlen = amdvi_find_ats_qlen(devid);
1764c87aefeSPatrick Mooney if (qlen < 0)
1774c87aefeSPatrick Mooney return (false);
1784c87aefeSPatrick Mooney
1794c87aefeSPatrick Mooney KASSERT(softc, ("softc is NULL"));
1804c87aefeSPatrick Mooney cfg = softc->dev_cfg;
1814c87aefeSPatrick Mooney
1824c87aefeSPatrick Mooney ivhd_ats = false;
1834c87aefeSPatrick Mooney for (i = 0; i < softc->dev_cfg_cnt; i++) {
1844c87aefeSPatrick Mooney if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) {
1854c87aefeSPatrick Mooney ivhd_ats = cfg->enable_ats;
1864c87aefeSPatrick Mooney break;
1874c87aefeSPatrick Mooney }
1884c87aefeSPatrick Mooney cfg++;
1894c87aefeSPatrick Mooney }
1904c87aefeSPatrick Mooney
1914c87aefeSPatrick Mooney pci_ats = (qlen < 0) ? false : true;
1924c87aefeSPatrick Mooney if (pci_ats != ivhd_ats)
1934c87aefeSPatrick Mooney device_printf(softc->dev,
1944c87aefeSPatrick Mooney "BIOS bug: mismatch in ATS setting for %d.%d.%d,"
1954c87aefeSPatrick Mooney "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen);
1964c87aefeSPatrick Mooney
1974c87aefeSPatrick Mooney /* Ignore IVRS setting and respect PCI setting. */
1984c87aefeSPatrick Mooney return (pci_ats);
1994c87aefeSPatrick Mooney }
2004c87aefeSPatrick Mooney #endif
2014c87aefeSPatrick Mooney
2024c87aefeSPatrick Mooney /* Enable IOTLB support for IOMMU if its supported. */
2034c87aefeSPatrick Mooney static inline void
amdvi_hw_enable_iotlb(struct amdvi_softc * softc)2044c87aefeSPatrick Mooney amdvi_hw_enable_iotlb(struct amdvi_softc *softc)
2054c87aefeSPatrick Mooney {
2064c87aefeSPatrick Mooney #ifndef AMDVI_ATS_ENABLE
2074c87aefeSPatrick Mooney softc->iotlb = false;
2084c87aefeSPatrick Mooney #else
2094c87aefeSPatrick Mooney bool supported;
2104c87aefeSPatrick Mooney
2114c87aefeSPatrick Mooney supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false;
2124c87aefeSPatrick Mooney
2134c87aefeSPatrick Mooney if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) {
2144c87aefeSPatrick Mooney if (!supported)
2154c87aefeSPatrick Mooney device_printf(softc->dev, "IOTLB disabled by BIOS.\n");
2164c87aefeSPatrick Mooney
2174c87aefeSPatrick Mooney if (supported && !amdvi_enable_iotlb) {
2184c87aefeSPatrick Mooney device_printf(softc->dev, "IOTLB disabled by user.\n");
2194c87aefeSPatrick Mooney supported = false;
2204c87aefeSPatrick Mooney }
2214c87aefeSPatrick Mooney } else
2224c87aefeSPatrick Mooney supported = false;
2234c87aefeSPatrick Mooney
2244c87aefeSPatrick Mooney softc->iotlb = supported;
2254c87aefeSPatrick Mooney
2264c87aefeSPatrick Mooney #endif
2274c87aefeSPatrick Mooney }
2284c87aefeSPatrick Mooney
2294c87aefeSPatrick Mooney static int
amdvi_init_cmd(struct amdvi_softc * softc)2304c87aefeSPatrick Mooney amdvi_init_cmd(struct amdvi_softc *softc)
2314c87aefeSPatrick Mooney {
2324c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl = softc->ctrl;
2334c87aefeSPatrick Mooney
2344c87aefeSPatrick Mooney ctrl->cmd.len = 8; /* Use 256 command buffer entries. */
2354c87aefeSPatrick Mooney softc->cmd_max = 1 << ctrl->cmd.len;
2364c87aefeSPatrick Mooney
2374c87aefeSPatrick Mooney softc->cmd = malloc(sizeof(struct amdvi_cmd) *
2384c87aefeSPatrick Mooney softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO);
2394c87aefeSPatrick Mooney
2404c87aefeSPatrick Mooney if ((uintptr_t)softc->cmd & PAGE_MASK)
2414c87aefeSPatrick Mooney panic("AMDVi: Command buffer not aligned on page boundary.");
2424c87aefeSPatrick Mooney
2434c87aefeSPatrick Mooney ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE;
2444c87aefeSPatrick Mooney /*
2454c87aefeSPatrick Mooney * XXX: Reset the h/w pointers in case IOMMU is restarting,
2464c87aefeSPatrick Mooney * h/w doesn't clear these pointers based on empirical data.
2474c87aefeSPatrick Mooney */
2484c87aefeSPatrick Mooney ctrl->cmd_tail = 0;
2494c87aefeSPatrick Mooney ctrl->cmd_head = 0;
2504c87aefeSPatrick Mooney
2514c87aefeSPatrick Mooney return (0);
2524c87aefeSPatrick Mooney }
2534c87aefeSPatrick Mooney
2544c87aefeSPatrick Mooney /*
2554c87aefeSPatrick Mooney * Note: Update tail pointer after we have written the command since tail
2564c87aefeSPatrick Mooney * pointer update cause h/w to execute new commands, see section 3.3
2574c87aefeSPatrick Mooney * of AMD IOMMU spec ver 2.0.
2584c87aefeSPatrick Mooney */
2594c87aefeSPatrick Mooney /* Get the command tail pointer w/o updating it. */
2604c87aefeSPatrick Mooney static struct amdvi_cmd *
amdvi_get_cmd_tail(struct amdvi_softc * softc)2614c87aefeSPatrick Mooney amdvi_get_cmd_tail(struct amdvi_softc *softc)
2624c87aefeSPatrick Mooney {
2634c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
2644c87aefeSPatrick Mooney struct amdvi_cmd *tail;
2654c87aefeSPatrick Mooney
2664c87aefeSPatrick Mooney KASSERT(softc, ("softc is NULL"));
2674c87aefeSPatrick Mooney KASSERT(softc->cmd != NULL, ("cmd is NULL"));
2684c87aefeSPatrick Mooney
2694c87aefeSPatrick Mooney ctrl = softc->ctrl;
2704c87aefeSPatrick Mooney KASSERT(ctrl != NULL, ("ctrl is NULL"));
2714c87aefeSPatrick Mooney
2724c87aefeSPatrick Mooney tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd +
2734c87aefeSPatrick Mooney ctrl->cmd_tail);
2744c87aefeSPatrick Mooney
2754c87aefeSPatrick Mooney return (tail);
2764c87aefeSPatrick Mooney }
2774c87aefeSPatrick Mooney
2784c87aefeSPatrick Mooney /*
2794c87aefeSPatrick Mooney * Update the command tail pointer which will start command execution.
2804c87aefeSPatrick Mooney */
2814c87aefeSPatrick Mooney static void
amdvi_update_cmd_tail(struct amdvi_softc * softc)2824c87aefeSPatrick Mooney amdvi_update_cmd_tail(struct amdvi_softc *softc)
2834c87aefeSPatrick Mooney {
2844c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
2854c87aefeSPatrick Mooney int size;
2864c87aefeSPatrick Mooney
2874c87aefeSPatrick Mooney size = sizeof(struct amdvi_cmd);
2884c87aefeSPatrick Mooney KASSERT(softc->cmd != NULL, ("cmd is NULL"));
2894c87aefeSPatrick Mooney
2904c87aefeSPatrick Mooney ctrl = softc->ctrl;
2914c87aefeSPatrick Mooney KASSERT(ctrl != NULL, ("ctrl is NULL"));
2924c87aefeSPatrick Mooney
2934c87aefeSPatrick Mooney ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max);
2944c87aefeSPatrick Mooney softc->total_cmd++;
2954c87aefeSPatrick Mooney
2964c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
2974c87aefeSPatrick Mooney device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n",
2984c87aefeSPatrick Mooney ctrl->cmd_tail,
2994c87aefeSPatrick Mooney ctrl->cmd_head);
3004c87aefeSPatrick Mooney #endif
3014c87aefeSPatrick Mooney
3024c87aefeSPatrick Mooney }
3034c87aefeSPatrick Mooney
3044c87aefeSPatrick Mooney /*
3054c87aefeSPatrick Mooney * Various commands supported by IOMMU.
3064c87aefeSPatrick Mooney */
3074c87aefeSPatrick Mooney
3084c87aefeSPatrick Mooney /* Completion wait command. */
3094c87aefeSPatrick Mooney static void
amdvi_cmd_cmp(struct amdvi_softc * softc,const uint64_t data)3104c87aefeSPatrick Mooney amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data)
3114c87aefeSPatrick Mooney {
3124c87aefeSPatrick Mooney struct amdvi_cmd *cmd;
3134c87aefeSPatrick Mooney uint64_t pa;
3144c87aefeSPatrick Mooney
3154c87aefeSPatrick Mooney cmd = amdvi_get_cmd_tail(softc);
3164c87aefeSPatrick Mooney KASSERT(cmd != NULL, ("Cmd is NULL"));
3174c87aefeSPatrick Mooney
3184c87aefeSPatrick Mooney pa = vtophys(&softc->cmp_data);
3194c87aefeSPatrick Mooney cmd->opcode = AMDVI_CMP_WAIT_OPCODE;
320154972afSPatrick Mooney cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE;
3214c87aefeSPatrick Mooney cmd->word1 = (pa >> 32) & 0xFFFFF;
3224c87aefeSPatrick Mooney cmd->addr = data;
3234c87aefeSPatrick Mooney
3244c87aefeSPatrick Mooney amdvi_update_cmd_tail(softc);
3254c87aefeSPatrick Mooney }
3264c87aefeSPatrick Mooney
3274c87aefeSPatrick Mooney /* Invalidate device table entry. */
3284c87aefeSPatrick Mooney static void
amdvi_cmd_inv_dte(struct amdvi_softc * softc,uint16_t devid)3294c87aefeSPatrick Mooney amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid)
3304c87aefeSPatrick Mooney {
3314c87aefeSPatrick Mooney struct amdvi_cmd *cmd;
3324c87aefeSPatrick Mooney
3334c87aefeSPatrick Mooney cmd = amdvi_get_cmd_tail(softc);
3344c87aefeSPatrick Mooney KASSERT(cmd != NULL, ("Cmd is NULL"));
3354c87aefeSPatrick Mooney cmd->opcode = AMDVI_INVD_DTE_OPCODE;
3364c87aefeSPatrick Mooney cmd->word0 = devid;
3374c87aefeSPatrick Mooney amdvi_update_cmd_tail(softc);
3384c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
3394c87aefeSPatrick Mooney device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid);
3404c87aefeSPatrick Mooney #endif
3414c87aefeSPatrick Mooney }
3424c87aefeSPatrick Mooney
3434c87aefeSPatrick Mooney /* Invalidate IOMMU page, use for invalidation of domain. */
3444c87aefeSPatrick Mooney static void
amdvi_cmd_inv_iommu_pages(struct amdvi_softc * softc,uint16_t domain_id,uint64_t addr,bool guest_nested,bool pde,bool page)3454c87aefeSPatrick Mooney amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id,
3464c87aefeSPatrick Mooney uint64_t addr, bool guest_nested,
3474c87aefeSPatrick Mooney bool pde, bool page)
3484c87aefeSPatrick Mooney {
3494c87aefeSPatrick Mooney struct amdvi_cmd *cmd;
3504c87aefeSPatrick Mooney
3514c87aefeSPatrick Mooney cmd = amdvi_get_cmd_tail(softc);
3524c87aefeSPatrick Mooney KASSERT(cmd != NULL, ("Cmd is NULL"));
3534c87aefeSPatrick Mooney
3544c87aefeSPatrick Mooney cmd->opcode = AMDVI_INVD_PAGE_OPCODE;
3554c87aefeSPatrick Mooney cmd->word1 = domain_id;
3564c87aefeSPatrick Mooney /*
3574c87aefeSPatrick Mooney * Invalidate all addresses for this domain.
3584c87aefeSPatrick Mooney */
3594c87aefeSPatrick Mooney cmd->addr = addr;
3604c87aefeSPatrick Mooney cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0;
3614c87aefeSPatrick Mooney cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0;
3624c87aefeSPatrick Mooney
3634c87aefeSPatrick Mooney amdvi_update_cmd_tail(softc);
3644c87aefeSPatrick Mooney }
3654c87aefeSPatrick Mooney
3664c87aefeSPatrick Mooney #ifdef AMDVI_ATS_ENABLE
3674c87aefeSPatrick Mooney /* Invalidate device IOTLB. */
3684c87aefeSPatrick Mooney static void
amdvi_cmd_inv_iotlb(struct amdvi_softc * softc,uint16_t devid)3694c87aefeSPatrick Mooney amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid)
3704c87aefeSPatrick Mooney {
3714c87aefeSPatrick Mooney struct amdvi_cmd *cmd;
3724c87aefeSPatrick Mooney int qlen;
3734c87aefeSPatrick Mooney
3744c87aefeSPatrick Mooney if (!softc->iotlb)
3754c87aefeSPatrick Mooney return;
3764c87aefeSPatrick Mooney
3774c87aefeSPatrick Mooney qlen = amdvi_find_ats_qlen(devid);
3784c87aefeSPatrick Mooney if (qlen < 0) {
3794c87aefeSPatrick Mooney panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n",
3804c87aefeSPatrick Mooney qlen, RID2PCI_STR(devid));
3814c87aefeSPatrick Mooney }
3824c87aefeSPatrick Mooney cmd = amdvi_get_cmd_tail(softc);
3834c87aefeSPatrick Mooney KASSERT(cmd != NULL, ("Cmd is NULL"));
3844c87aefeSPatrick Mooney
3854c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
3864c87aefeSPatrick Mooney device_printf(softc->dev, "Invalidate IOTLB devID 0x%x"
3874c87aefeSPatrick Mooney " Qlen:%d\n", devid, qlen);
3884c87aefeSPatrick Mooney #endif
3894c87aefeSPatrick Mooney cmd->opcode = AMDVI_INVD_IOTLB_OPCODE;
3904c87aefeSPatrick Mooney cmd->word0 = devid;
3914c87aefeSPatrick Mooney cmd->word1 = qlen;
3924c87aefeSPatrick Mooney cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR |
3934c87aefeSPatrick Mooney AMDVI_INVD_IOTLB_S;
3944c87aefeSPatrick Mooney amdvi_update_cmd_tail(softc);
3954c87aefeSPatrick Mooney }
3964c87aefeSPatrick Mooney #endif
3974c87aefeSPatrick Mooney
3984c87aefeSPatrick Mooney #ifdef notyet /* For Interrupt Remap. */
3994c87aefeSPatrick Mooney static void
amdvi_cmd_inv_intr_map(struct amdvi_softc * softc,uint16_t devid)4004c87aefeSPatrick Mooney amdvi_cmd_inv_intr_map(struct amdvi_softc *softc,
4014c87aefeSPatrick Mooney uint16_t devid)
4024c87aefeSPatrick Mooney {
4034c87aefeSPatrick Mooney struct amdvi_cmd *cmd;
4044c87aefeSPatrick Mooney
4054c87aefeSPatrick Mooney cmd = amdvi_get_cmd_tail(softc);
4064c87aefeSPatrick Mooney KASSERT(cmd != NULL, ("Cmd is NULL"));
4074c87aefeSPatrick Mooney cmd->opcode = AMDVI_INVD_INTR_OPCODE;
4084c87aefeSPatrick Mooney cmd->word0 = devid;
4094c87aefeSPatrick Mooney amdvi_update_cmd_tail(softc);
4104c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
4114c87aefeSPatrick Mooney device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid);
4124c87aefeSPatrick Mooney #endif
4134c87aefeSPatrick Mooney }
4144c87aefeSPatrick Mooney #endif
4154c87aefeSPatrick Mooney
4164c87aefeSPatrick Mooney /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */
4174c87aefeSPatrick Mooney static void
amdvi_inv_domain(struct amdvi_softc * softc,uint16_t domain_id)4184c87aefeSPatrick Mooney amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id)
4194c87aefeSPatrick Mooney {
4204f3f3e9aSAndy Fiddaman struct amdvi_cmd *cmd __diagused;
4214c87aefeSPatrick Mooney
4224c87aefeSPatrick Mooney cmd = amdvi_get_cmd_tail(softc);
4234c87aefeSPatrick Mooney KASSERT(cmd != NULL, ("Cmd is NULL"));
4244c87aefeSPatrick Mooney
4254c87aefeSPatrick Mooney /*
4264c87aefeSPatrick Mooney * See section 3.3.3 of IOMMU spec rev 2.0, software note
4274c87aefeSPatrick Mooney * for invalidating domain.
4284c87aefeSPatrick Mooney */
4294c87aefeSPatrick Mooney amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR,
4304c87aefeSPatrick Mooney false, true, true);
4314c87aefeSPatrick Mooney
4324c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
4334c87aefeSPatrick Mooney device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id);
4344c87aefeSPatrick Mooney
4354c87aefeSPatrick Mooney #endif
4364c87aefeSPatrick Mooney }
4374c87aefeSPatrick Mooney
4384c87aefeSPatrick Mooney static bool
amdvi_cmp_wait(struct amdvi_softc * softc)4394c87aefeSPatrick Mooney amdvi_cmp_wait(struct amdvi_softc *softc)
4404c87aefeSPatrick Mooney {
4414f3f3e9aSAndy Fiddaman #ifdef AMDVI_DEBUG_CMD
4424f3f3e9aSAndy Fiddaman struct amdvi_ctrl *ctrl = softc->ctrl;
4434f3f3e9aSAndy Fiddaman #endif
4444c87aefeSPatrick Mooney const uint64_t VERIFY = 0xA5A5;
4454c87aefeSPatrick Mooney volatile uint64_t *read;
4464c87aefeSPatrick Mooney int i;
4474c87aefeSPatrick Mooney bool status;
4484c87aefeSPatrick Mooney
4494c87aefeSPatrick Mooney read = &softc->cmp_data;
4504c87aefeSPatrick Mooney *read = 0;
4514c87aefeSPatrick Mooney amdvi_cmd_cmp(softc, VERIFY);
4524c87aefeSPatrick Mooney /* Wait for h/w to update completion data. */
4534c87aefeSPatrick Mooney for (i = 0; i < 100 && (*read != VERIFY); i++) {
4544c87aefeSPatrick Mooney DELAY(1000); /* 1 ms */
4554c87aefeSPatrick Mooney }
4564c87aefeSPatrick Mooney status = (VERIFY == softc->cmp_data) ? true : false;
4574c87aefeSPatrick Mooney
4584c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
4594c87aefeSPatrick Mooney if (status)
4604c87aefeSPatrick Mooney device_printf(softc->dev, "CMD completion DONE Tail:0x%x, "
4614c87aefeSPatrick Mooney "Head:0x%x, loop:%d.\n", ctrl->cmd_tail,
4624c87aefeSPatrick Mooney ctrl->cmd_head, loop);
4634c87aefeSPatrick Mooney #endif
4644c87aefeSPatrick Mooney return (status);
4654c87aefeSPatrick Mooney }
4664c87aefeSPatrick Mooney
4674c87aefeSPatrick Mooney static void
amdvi_wait(struct amdvi_softc * softc)4684c87aefeSPatrick Mooney amdvi_wait(struct amdvi_softc *softc)
4694c87aefeSPatrick Mooney {
4704c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
4714c87aefeSPatrick Mooney int i;
4724c87aefeSPatrick Mooney
4734c87aefeSPatrick Mooney KASSERT(softc, ("softc is NULL"));
4744c87aefeSPatrick Mooney
4754c87aefeSPatrick Mooney ctrl = softc->ctrl;
4764c87aefeSPatrick Mooney KASSERT(ctrl != NULL, ("ctrl is NULL"));
4774c87aefeSPatrick Mooney /* Don't wait if h/w is not enabled. */
4784c87aefeSPatrick Mooney if ((ctrl->control & AMDVI_CTRL_EN) == 0)
4794c87aefeSPatrick Mooney return;
4804c87aefeSPatrick Mooney
4814c87aefeSPatrick Mooney for (i = 0; i < 10; i++) {
4824c87aefeSPatrick Mooney if (amdvi_cmp_wait(softc))
4834c87aefeSPatrick Mooney return;
4844c87aefeSPatrick Mooney }
4854c87aefeSPatrick Mooney
4864c87aefeSPatrick Mooney device_printf(softc->dev, "Error: completion failed"
4874c87aefeSPatrick Mooney " tail:0x%x, head:0x%x.\n",
4884c87aefeSPatrick Mooney ctrl->cmd_tail, ctrl->cmd_head);
489154972afSPatrick Mooney /* Dump the last command. */
490154972afSPatrick Mooney amdvi_dump_cmds(softc, 1);
4914c87aefeSPatrick Mooney }
4924c87aefeSPatrick Mooney
4934c87aefeSPatrick Mooney static void
amdvi_dump_cmds(struct amdvi_softc * softc,int count)494154972afSPatrick Mooney amdvi_dump_cmds(struct amdvi_softc *softc, int count)
4954c87aefeSPatrick Mooney {
4964c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
4974c87aefeSPatrick Mooney struct amdvi_cmd *cmd;
4984c87aefeSPatrick Mooney int off, i;
4994c87aefeSPatrick Mooney
5004c87aefeSPatrick Mooney ctrl = softc->ctrl;
501154972afSPatrick Mooney device_printf(softc->dev, "Dump last %d command(s):\n", count);
5024c87aefeSPatrick Mooney /*
5034c87aefeSPatrick Mooney * If h/w is stuck in completion, it is the previous command,
5044c87aefeSPatrick Mooney * start dumping from previous command onward.
5054c87aefeSPatrick Mooney */
5064c87aefeSPatrick Mooney off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd),
5074c87aefeSPatrick Mooney softc->cmd_max);
508154972afSPatrick Mooney for (i = 0; off != ctrl->cmd_tail && i < count; i++) {
5094c87aefeSPatrick Mooney cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off);
5104c87aefeSPatrick Mooney printf(" [CMD%d, off:0x%x] opcode= 0x%x 0x%x"
5114c87aefeSPatrick Mooney " 0x%x 0x%lx\n", i, off, cmd->opcode,
5124c87aefeSPatrick Mooney cmd->word0, cmd->word1, cmd->addr);
5132b948146SAndy Fiddaman off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max);
5144c87aefeSPatrick Mooney }
5154c87aefeSPatrick Mooney }
5164c87aefeSPatrick Mooney
5174c87aefeSPatrick Mooney static int
amdvi_init_event(struct amdvi_softc * softc)5184c87aefeSPatrick Mooney amdvi_init_event(struct amdvi_softc *softc)
5194c87aefeSPatrick Mooney {
5204c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
5214c87aefeSPatrick Mooney
5224c87aefeSPatrick Mooney ctrl = softc->ctrl;
5234c87aefeSPatrick Mooney ctrl->event.len = 8;
5244c87aefeSPatrick Mooney softc->event_max = 1 << ctrl->event.len;
5254c87aefeSPatrick Mooney softc->event = malloc(sizeof(struct amdvi_event) *
5264c87aefeSPatrick Mooney softc->event_max, M_AMDVI, M_WAITOK | M_ZERO);
5274c87aefeSPatrick Mooney if ((uintptr_t)softc->event & PAGE_MASK) {
5284c87aefeSPatrick Mooney device_printf(softc->dev, "Event buffer not aligned on page.");
5294c87aefeSPatrick Mooney return (false);
5304c87aefeSPatrick Mooney }
5314c87aefeSPatrick Mooney ctrl->event.base = vtophys(softc->event) / PAGE_SIZE;
5324c87aefeSPatrick Mooney
5334c87aefeSPatrick Mooney /* Reset the pointers. */
5344c87aefeSPatrick Mooney ctrl->evt_head = 0;
5354c87aefeSPatrick Mooney ctrl->evt_tail = 0;
5364c87aefeSPatrick Mooney
5374c87aefeSPatrick Mooney return (0);
5384c87aefeSPatrick Mooney }
5394c87aefeSPatrick Mooney
5404c87aefeSPatrick Mooney static inline void
amdvi_decode_evt_flag(uint16_t flag)5414c87aefeSPatrick Mooney amdvi_decode_evt_flag(uint16_t flag)
5424c87aefeSPatrick Mooney {
5434c87aefeSPatrick Mooney
5444c87aefeSPatrick Mooney flag &= AMDVI_EVENT_FLAG_MASK;
5454c87aefeSPatrick Mooney printf(" 0x%b]\n", flag,
5464c87aefeSPatrick Mooney "\020"
5474c87aefeSPatrick Mooney "\001GN"
5484c87aefeSPatrick Mooney "\002NX"
5494c87aefeSPatrick Mooney "\003US"
5504c87aefeSPatrick Mooney "\004I"
5514c87aefeSPatrick Mooney "\005PR"
5524c87aefeSPatrick Mooney "\006RW"
5534c87aefeSPatrick Mooney "\007PE"
5544c87aefeSPatrick Mooney "\010RZ"
5554c87aefeSPatrick Mooney "\011TR"
5564c87aefeSPatrick Mooney );
5574c87aefeSPatrick Mooney }
5584c87aefeSPatrick Mooney
5594c87aefeSPatrick Mooney /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/
5604c87aefeSPatrick Mooney static inline void
amdvi_decode_evt_flag_type(uint8_t type)5614c87aefeSPatrick Mooney amdvi_decode_evt_flag_type(uint8_t type)
5624c87aefeSPatrick Mooney {
5634c87aefeSPatrick Mooney
5644c87aefeSPatrick Mooney switch (AMDVI_EVENT_FLAG_TYPE(type)) {
5654c87aefeSPatrick Mooney case 0:
5664c87aefeSPatrick Mooney printf("RSVD\n");
5674c87aefeSPatrick Mooney break;
5684c87aefeSPatrick Mooney case 1:
5694c87aefeSPatrick Mooney printf("Master Abort\n");
5704c87aefeSPatrick Mooney break;
5714c87aefeSPatrick Mooney case 2:
5724c87aefeSPatrick Mooney printf("Target Abort\n");
5734c87aefeSPatrick Mooney break;
5744c87aefeSPatrick Mooney case 3:
5754c87aefeSPatrick Mooney printf("Data Err\n");
5764c87aefeSPatrick Mooney break;
5774c87aefeSPatrick Mooney default:
5784c87aefeSPatrick Mooney break;
5794c87aefeSPatrick Mooney }
5804c87aefeSPatrick Mooney }
5814c87aefeSPatrick Mooney
5824c87aefeSPatrick Mooney static void
amdvi_decode_inv_dte_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)5834c87aefeSPatrick Mooney amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr,
5844c87aefeSPatrick Mooney uint16_t flag)
5854c87aefeSPatrick Mooney {
5864c87aefeSPatrick Mooney
5874c87aefeSPatrick Mooney printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
5884c87aefeSPatrick Mooney " Addr:0x%lx",
5894c87aefeSPatrick Mooney devid, domid, addr);
5904c87aefeSPatrick Mooney amdvi_decode_evt_flag(flag);
5914c87aefeSPatrick Mooney }
5924c87aefeSPatrick Mooney
5934c87aefeSPatrick Mooney static void
amdvi_decode_pf_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)5944c87aefeSPatrick Mooney amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr,
5954c87aefeSPatrick Mooney uint16_t flag)
5964c87aefeSPatrick Mooney {
5974c87aefeSPatrick Mooney
5984c87aefeSPatrick Mooney printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
5994c87aefeSPatrick Mooney " Addr:0x%lx",
6004c87aefeSPatrick Mooney devid, domid, addr);
6014c87aefeSPatrick Mooney amdvi_decode_evt_flag(flag);
6024c87aefeSPatrick Mooney }
6034c87aefeSPatrick Mooney
6044c87aefeSPatrick Mooney static void
amdvi_decode_dte_hwerr_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)6054c87aefeSPatrick Mooney amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid,
6064c87aefeSPatrick Mooney uint64_t addr, uint16_t flag)
6074c87aefeSPatrick Mooney {
6084c87aefeSPatrick Mooney
6094c87aefeSPatrick Mooney printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
6104c87aefeSPatrick Mooney " Addr:0x%lx", devid, domid, addr);
6114c87aefeSPatrick Mooney amdvi_decode_evt_flag(flag);
6124c87aefeSPatrick Mooney amdvi_decode_evt_flag_type(flag);
6134c87aefeSPatrick Mooney }
6144c87aefeSPatrick Mooney
6154c87aefeSPatrick Mooney static void
amdvi_decode_page_hwerr_evt(uint16_t devid,uint16_t domid,uint64_t addr,uint16_t flag)6164c87aefeSPatrick Mooney amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr,
6174c87aefeSPatrick Mooney uint16_t flag)
6184c87aefeSPatrick Mooney {
6194c87aefeSPatrick Mooney
6204c87aefeSPatrick Mooney printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
6214c87aefeSPatrick Mooney " Addr:0x%lx", devid, domid, addr);
6224c87aefeSPatrick Mooney amdvi_decode_evt_flag(flag);
6234c87aefeSPatrick Mooney amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag));
6244c87aefeSPatrick Mooney }
6254c87aefeSPatrick Mooney
6264c87aefeSPatrick Mooney static void
amdvi_decode_evt(struct amdvi_event * evt)6274c87aefeSPatrick Mooney amdvi_decode_evt(struct amdvi_event *evt)
6284c87aefeSPatrick Mooney {
6294c87aefeSPatrick Mooney struct amdvi_cmd *cmd;
6304c87aefeSPatrick Mooney
6314c87aefeSPatrick Mooney switch (evt->opcode) {
6324c87aefeSPatrick Mooney case AMDVI_EVENT_INVALID_DTE:
6334c87aefeSPatrick Mooney amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid,
6344c87aefeSPatrick Mooney evt->addr, evt->flag);
6354c87aefeSPatrick Mooney break;
6364c87aefeSPatrick Mooney
6374c87aefeSPatrick Mooney case AMDVI_EVENT_PFAULT:
6384c87aefeSPatrick Mooney amdvi_decode_pf_evt(evt->devid, evt->pasid_domid,
6394c87aefeSPatrick Mooney evt->addr, evt->flag);
6404c87aefeSPatrick Mooney break;
6414c87aefeSPatrick Mooney
6424c87aefeSPatrick Mooney case AMDVI_EVENT_DTE_HW_ERROR:
6434c87aefeSPatrick Mooney amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid,
6444c87aefeSPatrick Mooney evt->addr, evt->flag);
6454c87aefeSPatrick Mooney break;
6464c87aefeSPatrick Mooney
6474c87aefeSPatrick Mooney case AMDVI_EVENT_PAGE_HW_ERROR:
6484c87aefeSPatrick Mooney amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid,
6494c87aefeSPatrick Mooney evt->addr, evt->flag);
6504c87aefeSPatrick Mooney break;
6514c87aefeSPatrick Mooney
6524c87aefeSPatrick Mooney case AMDVI_EVENT_ILLEGAL_CMD:
6534c87aefeSPatrick Mooney /* FALL THROUGH */
6544c87aefeSPatrick Mooney case AMDVI_EVENT_CMD_HW_ERROR:
6554c87aefeSPatrick Mooney printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ?
6564c87aefeSPatrick Mooney "ILLEGAL CMD" : "CMD HW ERR");
6574c87aefeSPatrick Mooney cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr);
6584c87aefeSPatrick Mooney printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n",
6594c87aefeSPatrick Mooney cmd->opcode, cmd->word0, cmd->word1, cmd->addr);
6604c87aefeSPatrick Mooney break;
6614c87aefeSPatrick Mooney
6624c87aefeSPatrick Mooney case AMDVI_EVENT_IOTLB_TIMEOUT:
6634c87aefeSPatrick Mooney printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n",
6644c87aefeSPatrick Mooney evt->devid, evt->addr);
6654c87aefeSPatrick Mooney break;
6664c87aefeSPatrick Mooney
6674c87aefeSPatrick Mooney case AMDVI_EVENT_INVALID_DTE_REQ:
6684c87aefeSPatrick Mooney printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n",
6694c87aefeSPatrick Mooney evt->devid, evt->addr, evt->flag >> 9,
6704c87aefeSPatrick Mooney (evt->flag >> 8) & 1);
6714c87aefeSPatrick Mooney break;
6724c87aefeSPatrick Mooney
6734c87aefeSPatrick Mooney case AMDVI_EVENT_INVALID_PPR_REQ:
6744c87aefeSPatrick Mooney case AMDVI_EVENT_COUNTER_ZERO:
6754c87aefeSPatrick Mooney printf("AMD-Vi: v2 events.\n");
6764c87aefeSPatrick Mooney break;
6774c87aefeSPatrick Mooney
6784c87aefeSPatrick Mooney default:
6794c87aefeSPatrick Mooney printf("Unsupported AMD-Vi event:%d\n", evt->opcode);
6804c87aefeSPatrick Mooney }
6814c87aefeSPatrick Mooney }
6824c87aefeSPatrick Mooney
6834c87aefeSPatrick Mooney static void
amdvi_print_events(struct amdvi_softc * softc)6844c87aefeSPatrick Mooney amdvi_print_events(struct amdvi_softc *softc)
6854c87aefeSPatrick Mooney {
6864c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
6874c87aefeSPatrick Mooney struct amdvi_event *event;
6884c87aefeSPatrick Mooney int i, size;
6894c87aefeSPatrick Mooney
6904c87aefeSPatrick Mooney ctrl = softc->ctrl;
6914c87aefeSPatrick Mooney size = sizeof(struct amdvi_event);
6924c87aefeSPatrick Mooney for (i = 0; i < softc->event_max; i++) {
6934c87aefeSPatrick Mooney event = &softc->event[ctrl->evt_head / size];
6944c87aefeSPatrick Mooney if (!event->opcode)
6954c87aefeSPatrick Mooney break;
6964c87aefeSPatrick Mooney device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n",
6974c87aefeSPatrick Mooney i, ctrl->evt_head, ctrl->evt_tail);
6984c87aefeSPatrick Mooney amdvi_decode_evt(event);
6994c87aefeSPatrick Mooney ctrl->evt_head = MOD_INC(ctrl->evt_head, size,
7004c87aefeSPatrick Mooney softc->event_max);
7014c87aefeSPatrick Mooney }
7024c87aefeSPatrick Mooney }
7034c87aefeSPatrick Mooney
7044c87aefeSPatrick Mooney static int
amdvi_init_dte(struct amdvi_softc * softc)7054c87aefeSPatrick Mooney amdvi_init_dte(struct amdvi_softc *softc)
7064c87aefeSPatrick Mooney {
7074c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
7084c87aefeSPatrick Mooney
7094c87aefeSPatrick Mooney ctrl = softc->ctrl;
7104c87aefeSPatrick Mooney ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE;
7114c87aefeSPatrick Mooney ctrl->dte.size = 0x1FF; /* 2MB device table. */
7124c87aefeSPatrick Mooney
7134c87aefeSPatrick Mooney return (0);
7144c87aefeSPatrick Mooney }
7154c87aefeSPatrick Mooney
7164c87aefeSPatrick Mooney /*
7174c87aefeSPatrick Mooney * Not all capabilities of IOMMU are available in ACPI IVHD flag
7184c87aefeSPatrick Mooney * or EFR entry, read directly from device.
7194c87aefeSPatrick Mooney */
7204c87aefeSPatrick Mooney static int
amdvi_print_pci_cap(device_t dev)7214c87aefeSPatrick Mooney amdvi_print_pci_cap(device_t dev)
7224c87aefeSPatrick Mooney {
7234c87aefeSPatrick Mooney struct amdvi_softc *softc;
7244c87aefeSPatrick Mooney uint32_t off, cap;
7254c87aefeSPatrick Mooney
7264c87aefeSPatrick Mooney softc = device_get_softc(dev);
7274c87aefeSPatrick Mooney off = softc->cap_off;
7284c87aefeSPatrick Mooney
7294c87aefeSPatrick Mooney /*
7304c87aefeSPatrick Mooney * Section 3.7.1 of IOMMU sepc rev 2.0.
7314c87aefeSPatrick Mooney * Read capability from device.
7324c87aefeSPatrick Mooney */
7334c87aefeSPatrick Mooney cap = amdvi_pci_read(softc, off);
7344c87aefeSPatrick Mooney
7354c87aefeSPatrick Mooney /* Make sure capability type[18:16] is 3. */
7364c87aefeSPatrick Mooney KASSERT((((cap >> 16) & 0x7) == 0x3),
7374c87aefeSPatrick Mooney ("Not a IOMMU capability 0x%x@0x%x", cap, off));
7384c87aefeSPatrick Mooney
7394c87aefeSPatrick Mooney softc->pci_cap = cap >> 24;
7404c87aefeSPatrick Mooney device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n",
7414c87aefeSPatrick Mooney cap, off, softc->pci_cap,
7424c87aefeSPatrick Mooney "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt");
7434c87aefeSPatrick Mooney
7444c87aefeSPatrick Mooney return (0);
7454c87aefeSPatrick Mooney }
7464c87aefeSPatrick Mooney
7474c87aefeSPatrick Mooney static void
amdvi_event_intr(void * arg)7484c87aefeSPatrick Mooney amdvi_event_intr(void *arg)
7494c87aefeSPatrick Mooney {
7504c87aefeSPatrick Mooney struct amdvi_softc *softc;
7514c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
7524c87aefeSPatrick Mooney
7534c87aefeSPatrick Mooney softc = (struct amdvi_softc *)arg;
7544c87aefeSPatrick Mooney ctrl = softc->ctrl;
7554c87aefeSPatrick Mooney device_printf(softc->dev, "EVT INTR %ld Status:0x%x"
7564c87aefeSPatrick Mooney " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++,
7574c87aefeSPatrick Mooney ctrl->status, ctrl->evt_head, ctrl->evt_tail);
7584c87aefeSPatrick Mooney printf(" [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n",
7594c87aefeSPatrick Mooney softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head);
7604c87aefeSPatrick Mooney
7614c87aefeSPatrick Mooney amdvi_print_events(softc);
7624c87aefeSPatrick Mooney ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
7634c87aefeSPatrick Mooney }
7644c87aefeSPatrick Mooney
7654c87aefeSPatrick Mooney static void
amdvi_free_evt_intr_res(device_t dev)7664c87aefeSPatrick Mooney amdvi_free_evt_intr_res(device_t dev)
7674c87aefeSPatrick Mooney {
7684c87aefeSPatrick Mooney
7694c87aefeSPatrick Mooney struct amdvi_softc *softc;
7702b948146SAndy Fiddaman device_t mmio_dev;
7714c87aefeSPatrick Mooney
7724c87aefeSPatrick Mooney softc = device_get_softc(dev);
7732b948146SAndy Fiddaman mmio_dev = softc->pci_dev;
7742b948146SAndy Fiddaman
7752b948146SAndy Fiddaman IVHD_TEARDOWN_INTR(mmio_dev);
7764c87aefeSPatrick Mooney }
7774c87aefeSPatrick Mooney
7784c87aefeSPatrick Mooney static bool
amdvi_alloc_intr_resources(struct amdvi_softc * softc)7794c87aefeSPatrick Mooney amdvi_alloc_intr_resources(struct amdvi_softc *softc)
7804c87aefeSPatrick Mooney {
7814c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
7822b948146SAndy Fiddaman device_t dev, mmio_dev;
7834c87aefeSPatrick Mooney int err;
7844c87aefeSPatrick Mooney
7854c87aefeSPatrick Mooney dev = softc->dev;
7862b948146SAndy Fiddaman mmio_dev = softc->pci_dev;
7874c87aefeSPatrick Mooney
7884c87aefeSPatrick Mooney /* Clear interrupt status bits. */
7894c87aefeSPatrick Mooney ctrl = softc->ctrl;
7904c87aefeSPatrick Mooney ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
7914c87aefeSPatrick Mooney
7922b948146SAndy Fiddaman err = IVHD_SETUP_INTR(mmio_dev, amdvi_event_intr, softc, "fault");
7932b948146SAndy Fiddaman if (err)
7942b948146SAndy Fiddaman device_printf(dev, "Interrupt setup failed on %s\n",
7952b948146SAndy Fiddaman device_get_nameunit(mmio_dev));
7962b948146SAndy Fiddaman return (err);
7974c87aefeSPatrick Mooney }
7984c87aefeSPatrick Mooney
7994c87aefeSPatrick Mooney static void
amdvi_print_dev_cap(struct amdvi_softc * softc)8004c87aefeSPatrick Mooney amdvi_print_dev_cap(struct amdvi_softc *softc)
8014c87aefeSPatrick Mooney {
8024c87aefeSPatrick Mooney struct ivhd_dev_cfg *cfg;
8034c87aefeSPatrick Mooney int i;
8044c87aefeSPatrick Mooney
8054c87aefeSPatrick Mooney cfg = softc->dev_cfg;
8064c87aefeSPatrick Mooney for (i = 0; i < softc->dev_cfg_cnt; i++) {
807b0de25cbSAndy Fiddaman device_printf(softc->dev, "device [0x%x - 0x%x] "
8084c87aefeSPatrick Mooney "config:%b%s\n", cfg->start_id, cfg->end_id,
8094c87aefeSPatrick Mooney cfg->data,
8104c87aefeSPatrick Mooney "\020\001INIT\002ExtInt\003NMI"
811b0de25cbSAndy Fiddaman "\007LINT0\010LINT1",
8124c87aefeSPatrick Mooney cfg->enable_ats ? "ATS enabled" : "");
8134c87aefeSPatrick Mooney cfg++;
8144c87aefeSPatrick Mooney }
8154c87aefeSPatrick Mooney }
8164c87aefeSPatrick Mooney
8174c87aefeSPatrick Mooney static int
amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS)8184c87aefeSPatrick Mooney amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS)
8194c87aefeSPatrick Mooney {
8204c87aefeSPatrick Mooney struct amdvi_softc *softc;
8214c87aefeSPatrick Mooney int result, type, error = 0;
8224c87aefeSPatrick Mooney
8234c87aefeSPatrick Mooney softc = (struct amdvi_softc *)arg1;
8244c87aefeSPatrick Mooney type = arg2;
8254c87aefeSPatrick Mooney
8264c87aefeSPatrick Mooney switch (type) {
8274c87aefeSPatrick Mooney case 0:
8284c87aefeSPatrick Mooney result = softc->ctrl->cmd_head;
8294c87aefeSPatrick Mooney error = sysctl_handle_int(oidp, &result, 0,
8304c87aefeSPatrick Mooney req);
8314c87aefeSPatrick Mooney break;
8324c87aefeSPatrick Mooney case 1:
8334c87aefeSPatrick Mooney result = softc->ctrl->cmd_tail;
8344c87aefeSPatrick Mooney error = sysctl_handle_int(oidp, &result, 0,
8354c87aefeSPatrick Mooney req);
8364c87aefeSPatrick Mooney break;
8374c87aefeSPatrick Mooney case 2:
8384c87aefeSPatrick Mooney result = softc->ctrl->evt_head;
8394c87aefeSPatrick Mooney error = sysctl_handle_int(oidp, &result, 0,
8404c87aefeSPatrick Mooney req);
8414c87aefeSPatrick Mooney break;
8424c87aefeSPatrick Mooney case 3:
8434c87aefeSPatrick Mooney result = softc->ctrl->evt_tail;
8444c87aefeSPatrick Mooney error = sysctl_handle_int(oidp, &result, 0,
8454c87aefeSPatrick Mooney req);
8464c87aefeSPatrick Mooney break;
8474c87aefeSPatrick Mooney
8484c87aefeSPatrick Mooney default:
8494c87aefeSPatrick Mooney device_printf(softc->dev, "Unknown sysctl:%d\n", type);
8504c87aefeSPatrick Mooney }
8514c87aefeSPatrick Mooney
8524c87aefeSPatrick Mooney return (error);
8534c87aefeSPatrick Mooney }
8544c87aefeSPatrick Mooney
8554c87aefeSPatrick Mooney static void
amdvi_add_sysctl(struct amdvi_softc * softc)8564c87aefeSPatrick Mooney amdvi_add_sysctl(struct amdvi_softc *softc)
8574c87aefeSPatrick Mooney {
8584c87aefeSPatrick Mooney struct sysctl_oid_list *child;
8594c87aefeSPatrick Mooney struct sysctl_ctx_list *ctx;
8604c87aefeSPatrick Mooney device_t dev;
8614c87aefeSPatrick Mooney
8624c87aefeSPatrick Mooney dev = softc->dev;
8634c87aefeSPatrick Mooney ctx = device_get_sysctl_ctx(dev);
8644c87aefeSPatrick Mooney child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
8654c87aefeSPatrick Mooney
8664c87aefeSPatrick Mooney SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD,
8674c87aefeSPatrick Mooney &softc->event_intr_cnt, "Event interrupt count");
8684c87aefeSPatrick Mooney SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD,
8694c87aefeSPatrick Mooney &softc->total_cmd, "Command submitted count");
8704c87aefeSPatrick Mooney SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD,
8714c87aefeSPatrick Mooney &softc->pci_rid, 0, "IOMMU RID");
8724c87aefeSPatrick Mooney SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head",
873154972afSPatrick Mooney CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0,
8744c87aefeSPatrick Mooney amdvi_handle_sysctl, "IU", "Command head");
8754c87aefeSPatrick Mooney SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail",
876154972afSPatrick Mooney CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1,
8774c87aefeSPatrick Mooney amdvi_handle_sysctl, "IU", "Command tail");
8784c87aefeSPatrick Mooney SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head",
879154972afSPatrick Mooney CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2,
8804c87aefeSPatrick Mooney amdvi_handle_sysctl, "IU", "Command head");
8814c87aefeSPatrick Mooney SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail",
882154972afSPatrick Mooney CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3,
8834c87aefeSPatrick Mooney amdvi_handle_sysctl, "IU", "Command tail");
8844c87aefeSPatrick Mooney }
8854c87aefeSPatrick Mooney
8864c87aefeSPatrick Mooney int
amdvi_setup_hw(struct amdvi_softc * softc)8874c87aefeSPatrick Mooney amdvi_setup_hw(struct amdvi_softc *softc)
8884c87aefeSPatrick Mooney {
8894c87aefeSPatrick Mooney device_t dev;
8904c87aefeSPatrick Mooney int status;
8914c87aefeSPatrick Mooney
8924c87aefeSPatrick Mooney dev = softc->dev;
8934c87aefeSPatrick Mooney
8944c87aefeSPatrick Mooney amdvi_hw_enable_iotlb(softc);
8954c87aefeSPatrick Mooney
8964c87aefeSPatrick Mooney amdvi_print_dev_cap(softc);
8974c87aefeSPatrick Mooney
8984c87aefeSPatrick Mooney if ((status = amdvi_print_pci_cap(dev)) != 0) {
8994c87aefeSPatrick Mooney device_printf(dev, "PCI capability.\n");
9004c87aefeSPatrick Mooney return (status);
9014c87aefeSPatrick Mooney }
9024c87aefeSPatrick Mooney if ((status = amdvi_init_cmd(softc)) != 0) {
9034c87aefeSPatrick Mooney device_printf(dev, "Couldn't configure command buffer.\n");
9044c87aefeSPatrick Mooney return (status);
9054c87aefeSPatrick Mooney }
9064c87aefeSPatrick Mooney if ((status = amdvi_init_event(softc)) != 0) {
9074c87aefeSPatrick Mooney device_printf(dev, "Couldn't configure event buffer.\n");
9084c87aefeSPatrick Mooney return (status);
9094c87aefeSPatrick Mooney }
9104c87aefeSPatrick Mooney if ((status = amdvi_init_dte(softc)) != 0) {
9114c87aefeSPatrick Mooney device_printf(dev, "Couldn't configure device table.\n");
9124c87aefeSPatrick Mooney return (status);
9134c87aefeSPatrick Mooney }
9144c87aefeSPatrick Mooney if ((status = amdvi_alloc_intr_resources(softc)) != 0) {
9154c87aefeSPatrick Mooney return (status);
9164c87aefeSPatrick Mooney }
9174c87aefeSPatrick Mooney amdvi_add_sysctl(softc);
9184c87aefeSPatrick Mooney return (0);
9194c87aefeSPatrick Mooney }
9204c87aefeSPatrick Mooney
9214c87aefeSPatrick Mooney int
amdvi_teardown_hw(struct amdvi_softc * softc)9224c87aefeSPatrick Mooney amdvi_teardown_hw(struct amdvi_softc *softc)
9234c87aefeSPatrick Mooney {
9244c87aefeSPatrick Mooney device_t dev;
9254c87aefeSPatrick Mooney
9264c87aefeSPatrick Mooney dev = softc->dev;
9274c87aefeSPatrick Mooney
92884971882SPatrick Mooney /*
92984971882SPatrick Mooney * Called after disable, h/w is stopped by now, free all the resources.
9304c87aefeSPatrick Mooney */
9314c87aefeSPatrick Mooney amdvi_free_evt_intr_res(dev);
9324c87aefeSPatrick Mooney
9334c87aefeSPatrick Mooney if (softc->cmd)
9344c87aefeSPatrick Mooney free(softc->cmd, M_AMDVI);
9354c87aefeSPatrick Mooney
9364c87aefeSPatrick Mooney if (softc->event)
9374c87aefeSPatrick Mooney free(softc->event, M_AMDVI);
9384c87aefeSPatrick Mooney
9394c87aefeSPatrick Mooney return (0);
9404c87aefeSPatrick Mooney }
9414c87aefeSPatrick Mooney
9424c87aefeSPatrick Mooney /*********** bhyve interfaces *********************/
9434c87aefeSPatrick Mooney static int
amdvi_init(void)9444c87aefeSPatrick Mooney amdvi_init(void)
9454c87aefeSPatrick Mooney {
9464c87aefeSPatrick Mooney if (!ivhd_count) {
9474c87aefeSPatrick Mooney return (EIO);
9484c87aefeSPatrick Mooney }
9494c87aefeSPatrick Mooney if (!amdvi_enable_user && ivhd_count) {
9504c87aefeSPatrick Mooney printf("bhyve: Found %d AMD-Vi/IOMMU device(s), "
95184971882SPatrick Mooney "use hw.vmm.amdvi.enable=1 to enable pass-through.\n",
9524c87aefeSPatrick Mooney ivhd_count);
9534c87aefeSPatrick Mooney return (EINVAL);
9544c87aefeSPatrick Mooney }
9554c87aefeSPatrick Mooney return (0);
9564c87aefeSPatrick Mooney }
9574c87aefeSPatrick Mooney
9584c87aefeSPatrick Mooney static void
amdvi_cleanup(void)9594c87aefeSPatrick Mooney amdvi_cleanup(void)
9604c87aefeSPatrick Mooney {
9614c87aefeSPatrick Mooney /* Nothing. */
9624c87aefeSPatrick Mooney }
9634c87aefeSPatrick Mooney
9644c87aefeSPatrick Mooney static uint16_t
amdvi_domainId(void)9654c87aefeSPatrick Mooney amdvi_domainId(void)
9664c87aefeSPatrick Mooney {
9674c87aefeSPatrick Mooney
9684c87aefeSPatrick Mooney /*
9694c87aefeSPatrick Mooney * If we hit maximum domain limit, rollover leaving host
9704c87aefeSPatrick Mooney * domain(0).
9714c87aefeSPatrick Mooney * XXX: make sure that this domain is not used.
9724c87aefeSPatrick Mooney */
9734c87aefeSPatrick Mooney if (amdvi_dom_id == AMDVI_MAX_DOMAIN)
9744c87aefeSPatrick Mooney amdvi_dom_id = 1;
9754c87aefeSPatrick Mooney
9764c87aefeSPatrick Mooney return ((uint16_t)amdvi_dom_id++);
9774c87aefeSPatrick Mooney }
9784c87aefeSPatrick Mooney
9794c87aefeSPatrick Mooney static void
amdvi_do_inv_domain(uint16_t domain_id,bool create)9804c87aefeSPatrick Mooney amdvi_do_inv_domain(uint16_t domain_id, bool create)
9814c87aefeSPatrick Mooney {
9824c87aefeSPatrick Mooney struct amdvi_softc *softc;
9834c87aefeSPatrick Mooney int i;
9844c87aefeSPatrick Mooney
9854c87aefeSPatrick Mooney for (i = 0; i < ivhd_count; i++) {
9864c87aefeSPatrick Mooney softc = device_get_softc(ivhd_devs[i]);
9874c87aefeSPatrick Mooney KASSERT(softc, ("softc is NULL"));
9884c87aefeSPatrick Mooney /*
9894c87aefeSPatrick Mooney * If not present pages are cached, invalidate page after
9904c87aefeSPatrick Mooney * creating domain.
9914c87aefeSPatrick Mooney */
9924c87aefeSPatrick Mooney #if 0
9934c87aefeSPatrick Mooney if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0))
9944c87aefeSPatrick Mooney continue;
9954c87aefeSPatrick Mooney #endif
9964c87aefeSPatrick Mooney amdvi_inv_domain(softc, domain_id);
9974c87aefeSPatrick Mooney amdvi_wait(softc);
9984c87aefeSPatrick Mooney }
9994c87aefeSPatrick Mooney }
10004c87aefeSPatrick Mooney
10014c87aefeSPatrick Mooney static void *
amdvi_create_domain(vm_paddr_t maxaddr)10024c87aefeSPatrick Mooney amdvi_create_domain(vm_paddr_t maxaddr)
10034c87aefeSPatrick Mooney {
10044c87aefeSPatrick Mooney struct amdvi_domain *dom;
10054c87aefeSPatrick Mooney
10064c87aefeSPatrick Mooney dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK);
10074c87aefeSPatrick Mooney dom->id = amdvi_domainId();
10084c87aefeSPatrick Mooney //dom->maxaddr = maxaddr;
10094c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
10104c87aefeSPatrick Mooney printf("Created domain #%d\n", dom->id);
10114c87aefeSPatrick Mooney #endif
10124c87aefeSPatrick Mooney /*
10134c87aefeSPatrick Mooney * Host domain(#0) don't create translation table.
10144c87aefeSPatrick Mooney */
10154c87aefeSPatrick Mooney if (dom->id || amdvi_host_ptp)
10164c87aefeSPatrick Mooney dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
10174c87aefeSPatrick Mooney
10184c87aefeSPatrick Mooney dom->ptp_level = amdvi_ptp_level;
10194c87aefeSPatrick Mooney
10204c87aefeSPatrick Mooney amdvi_do_inv_domain(dom->id, true);
10214c87aefeSPatrick Mooney SLIST_INSERT_HEAD(&dom_head, dom, next);
10224c87aefeSPatrick Mooney
10234c87aefeSPatrick Mooney return (dom);
10244c87aefeSPatrick Mooney }
10254c87aefeSPatrick Mooney
10264c87aefeSPatrick Mooney static void
amdvi_free_ptp(uint64_t * ptp,int level)10274c87aefeSPatrick Mooney amdvi_free_ptp(uint64_t *ptp, int level)
10284c87aefeSPatrick Mooney {
10294c87aefeSPatrick Mooney int i;
10304c87aefeSPatrick Mooney
10314c87aefeSPatrick Mooney if (level < 1)
10324c87aefeSPatrick Mooney return;
10334c87aefeSPatrick Mooney
10344c87aefeSPatrick Mooney for (i = 0; i < NPTEPG ; i++) {
10354c87aefeSPatrick Mooney if ((ptp[i] & AMDVI_PT_PRESENT) == 0)
10364c87aefeSPatrick Mooney continue;
10374c87aefeSPatrick Mooney /* XXX: Add super-page or PTE mapping > 4KB. */
10384c87aefeSPatrick Mooney #ifdef notyet
10394c87aefeSPatrick Mooney /* Super-page mapping. */
10404c87aefeSPatrick Mooney if (AMDVI_PD_SUPER(ptp[i]))
10414c87aefeSPatrick Mooney continue;
10424c87aefeSPatrick Mooney #endif
10434c87aefeSPatrick Mooney
10444c87aefeSPatrick Mooney amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i]
10454c87aefeSPatrick Mooney & AMDVI_PT_MASK), level - 1);
10464c87aefeSPatrick Mooney }
10474c87aefeSPatrick Mooney
10484c87aefeSPatrick Mooney free(ptp, M_AMDVI);
10494c87aefeSPatrick Mooney }
10504c87aefeSPatrick Mooney
10514c87aefeSPatrick Mooney static void
amdvi_destroy_domain(void * arg)10524c87aefeSPatrick Mooney amdvi_destroy_domain(void *arg)
10534c87aefeSPatrick Mooney {
10544c87aefeSPatrick Mooney struct amdvi_domain *domain;
10554c87aefeSPatrick Mooney
10564c87aefeSPatrick Mooney domain = (struct amdvi_domain *)arg;
10574c87aefeSPatrick Mooney KASSERT(domain, ("domain is NULL"));
10584c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
10594c87aefeSPatrick Mooney printf("Destroying domain %d\n", domain->id);
10604c87aefeSPatrick Mooney #endif
10614c87aefeSPatrick Mooney if (domain->ptp)
10624c87aefeSPatrick Mooney amdvi_free_ptp(domain->ptp, domain->ptp_level);
10634c87aefeSPatrick Mooney
10644c87aefeSPatrick Mooney amdvi_do_inv_domain(domain->id, false);
10654c87aefeSPatrick Mooney SLIST_REMOVE(&dom_head, domain, amdvi_domain, next);
10664c87aefeSPatrick Mooney free(domain, M_AMDVI);
10674c87aefeSPatrick Mooney }
10684c87aefeSPatrick Mooney
10694c87aefeSPatrick Mooney static uint64_t
amdvi_set_pt(uint64_t * pt,int level,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t pg_size,bool create)10704c87aefeSPatrick Mooney amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa,
10714c87aefeSPatrick Mooney vm_paddr_t hpa, uint64_t pg_size, bool create)
10724c87aefeSPatrick Mooney {
10734c87aefeSPatrick Mooney uint64_t *page, pa;
10744c87aefeSPatrick Mooney int shift, index;
10754c87aefeSPatrick Mooney const int PT_SHIFT = 9;
10764c87aefeSPatrick Mooney const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */
10774c87aefeSPatrick Mooney
10784c87aefeSPatrick Mooney if (!pg_size)
10794c87aefeSPatrick Mooney return (0);
10804c87aefeSPatrick Mooney
10814c87aefeSPatrick Mooney if (hpa & (pg_size - 1)) {
10824c87aefeSPatrick Mooney printf("HPA is not size aligned.\n");
10834c87aefeSPatrick Mooney return (0);
10844c87aefeSPatrick Mooney }
10854c87aefeSPatrick Mooney if (gpa & (pg_size - 1)) {
10864c87aefeSPatrick Mooney printf("HPA is not size aligned.\n");
10874c87aefeSPatrick Mooney return (0);
10884c87aefeSPatrick Mooney }
10894c87aefeSPatrick Mooney shift = PML4SHIFT;
10904c87aefeSPatrick Mooney while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) {
10914c87aefeSPatrick Mooney index = (gpa >> shift) & PT_INDEX_MASK;
10924c87aefeSPatrick Mooney
10934c87aefeSPatrick Mooney if ((pt[index] == 0) && create) {
10944c87aefeSPatrick Mooney page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
10954c87aefeSPatrick Mooney pa = vtophys(page);
10964c87aefeSPatrick Mooney pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW |
10974c87aefeSPatrick Mooney ((level - 1) << AMDVI_PD_LEVEL_SHIFT);
10984c87aefeSPatrick Mooney }
10994c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_PTE
11004c87aefeSPatrick Mooney if ((gpa % 0x1000000) == 0)
11014c87aefeSPatrick Mooney printf("[level%d, shift = %d]PTE:0x%lx\n",
11024c87aefeSPatrick Mooney level, shift, pt[index]);
11034c87aefeSPatrick Mooney #endif
11044c87aefeSPatrick Mooney #define PTE2PA(x) ((uint64_t)(x) & AMDVI_PT_MASK)
11054c87aefeSPatrick Mooney pa = PTE2PA(pt[index]);
11064c87aefeSPatrick Mooney pt = (uint64_t *)PHYS_TO_DMAP(pa);
11074c87aefeSPatrick Mooney shift -= PT_SHIFT;
11084c87aefeSPatrick Mooney level--;
11094c87aefeSPatrick Mooney }
11104c87aefeSPatrick Mooney
11114c87aefeSPatrick Mooney /* Leaf entry. */
11124c87aefeSPatrick Mooney index = (gpa >> shift) & PT_INDEX_MASK;
11134c87aefeSPatrick Mooney
11144c87aefeSPatrick Mooney if (create) {
11154c87aefeSPatrick Mooney pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT;
11164c87aefeSPatrick Mooney } else
11174c87aefeSPatrick Mooney pt[index] = 0;
11184c87aefeSPatrick Mooney
11194c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_PTE
11204c87aefeSPatrick Mooney if ((gpa % 0x1000000) == 0)
11214c87aefeSPatrick Mooney printf("[Last level%d, shift = %d]PTE:0x%lx\n",
11224c87aefeSPatrick Mooney level, shift, pt[index]);
11234c87aefeSPatrick Mooney #endif
11244c87aefeSPatrick Mooney return (1ULL << shift);
11254c87aefeSPatrick Mooney }
11264c87aefeSPatrick Mooney
11274c87aefeSPatrick Mooney static uint64_t
amdvi_update_mapping(struct amdvi_domain * domain,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t size,bool create)11284c87aefeSPatrick Mooney amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa,
11294c87aefeSPatrick Mooney vm_paddr_t hpa, uint64_t size, bool create)
11304c87aefeSPatrick Mooney {
11314c87aefeSPatrick Mooney uint64_t mapped, *ptp, len;
11324c87aefeSPatrick Mooney int level;
11334c87aefeSPatrick Mooney
11344c87aefeSPatrick Mooney KASSERT(domain, ("domain is NULL"));
11354c87aefeSPatrick Mooney level = domain->ptp_level;
11364c87aefeSPatrick Mooney KASSERT(level, ("Page table level is 0"));
11374c87aefeSPatrick Mooney
11384c87aefeSPatrick Mooney ptp = domain->ptp;
11394c87aefeSPatrick Mooney KASSERT(ptp, ("PTP is NULL"));
11404c87aefeSPatrick Mooney mapped = 0;
11414c87aefeSPatrick Mooney while (mapped < size) {
11424c87aefeSPatrick Mooney len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped,
11434c87aefeSPatrick Mooney PAGE_SIZE, create);
11444c87aefeSPatrick Mooney if (!len) {
11454c87aefeSPatrick Mooney printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n",
11464c87aefeSPatrick Mooney hpa, gpa);
11474c87aefeSPatrick Mooney return (0);
11484c87aefeSPatrick Mooney }
11494c87aefeSPatrick Mooney mapped += len;
11504c87aefeSPatrick Mooney }
11514c87aefeSPatrick Mooney
11524c87aefeSPatrick Mooney return (mapped);
11534c87aefeSPatrick Mooney }
11544c87aefeSPatrick Mooney
11554c87aefeSPatrick Mooney static uint64_t
amdvi_create_mapping(void * arg,vm_paddr_t gpa,vm_paddr_t hpa,uint64_t len)11564c87aefeSPatrick Mooney amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa,
11574c87aefeSPatrick Mooney uint64_t len)
11584c87aefeSPatrick Mooney {
11594c87aefeSPatrick Mooney struct amdvi_domain *domain;
11604c87aefeSPatrick Mooney
11614c87aefeSPatrick Mooney domain = (struct amdvi_domain *)arg;
11624c87aefeSPatrick Mooney
11634c87aefeSPatrick Mooney if (domain->id && !domain->ptp) {
11644c87aefeSPatrick Mooney printf("ptp is NULL");
11654c87aefeSPatrick Mooney return (-1);
11664c87aefeSPatrick Mooney }
11674c87aefeSPatrick Mooney
11684c87aefeSPatrick Mooney /*
11694c87aefeSPatrick Mooney * If host domain is created w/o page table, skip IOMMU page
11704c87aefeSPatrick Mooney * table set-up.
11714c87aefeSPatrick Mooney */
11724c87aefeSPatrick Mooney if (domain->ptp)
11734c87aefeSPatrick Mooney return (amdvi_update_mapping(domain, gpa, hpa, len, true));
11744c87aefeSPatrick Mooney else
11754c87aefeSPatrick Mooney return (len);
11764c87aefeSPatrick Mooney }
11774c87aefeSPatrick Mooney
11784c87aefeSPatrick Mooney static uint64_t
amdvi_remove_mapping(void * arg,vm_paddr_t gpa,uint64_t len)1179b0de25cbSAndy Fiddaman amdvi_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
11804c87aefeSPatrick Mooney {
11814c87aefeSPatrick Mooney struct amdvi_domain *domain;
11824c87aefeSPatrick Mooney
11834c87aefeSPatrick Mooney domain = (struct amdvi_domain *)arg;
11844c87aefeSPatrick Mooney /*
11854c87aefeSPatrick Mooney * If host domain is created w/o page table, skip IOMMU page
11864c87aefeSPatrick Mooney * table set-up.
11874c87aefeSPatrick Mooney */
11884c87aefeSPatrick Mooney if (domain->ptp)
11894c87aefeSPatrick Mooney return (amdvi_update_mapping(domain, gpa, 0, len, false));
11904c87aefeSPatrick Mooney return
11914c87aefeSPatrick Mooney (len);
11924c87aefeSPatrick Mooney }
11934c87aefeSPatrick Mooney
11944c87aefeSPatrick Mooney static struct amdvi_softc *
amdvi_find_iommu(uint16_t devid)11954c87aefeSPatrick Mooney amdvi_find_iommu(uint16_t devid)
11964c87aefeSPatrick Mooney {
11974c87aefeSPatrick Mooney struct amdvi_softc *softc;
1198b0de25cbSAndy Fiddaman int i, j;
11994c87aefeSPatrick Mooney
12004c87aefeSPatrick Mooney for (i = 0; i < ivhd_count; i++) {
12014c87aefeSPatrick Mooney softc = device_get_softc(ivhd_devs[i]);
1202b0de25cbSAndy Fiddaman for (j = 0; j < softc->dev_cfg_cnt; j++)
1203b0de25cbSAndy Fiddaman if ((devid >= softc->dev_cfg[j].start_id) &&
1204b0de25cbSAndy Fiddaman (devid <= softc->dev_cfg[j].end_id))
1205b0de25cbSAndy Fiddaman return (softc);
12064c87aefeSPatrick Mooney }
12074c87aefeSPatrick Mooney
1208b0de25cbSAndy Fiddaman return (NULL);
12094c87aefeSPatrick Mooney }
12104c87aefeSPatrick Mooney
12114c87aefeSPatrick Mooney /*
12124c87aefeSPatrick Mooney * Set-up device table entry.
12134c87aefeSPatrick Mooney * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must
12144c87aefeSPatrick Mooney * be set concurrently, e.g. read and write bits.
12154c87aefeSPatrick Mooney */
12164c87aefeSPatrick Mooney static void
amdvi_set_dte(struct amdvi_domain * domain,struct amdvi_softc * softc,uint16_t devid,bool enable)1217b0de25cbSAndy Fiddaman amdvi_set_dte(struct amdvi_domain *domain, struct amdvi_softc *softc,
1218b0de25cbSAndy Fiddaman uint16_t devid, bool enable)
12194c87aefeSPatrick Mooney {
12204c87aefeSPatrick Mooney struct amdvi_dte* temp;
12214c87aefeSPatrick Mooney
12224c87aefeSPatrick Mooney KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid));
12234c87aefeSPatrick Mooney KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid));
12244c87aefeSPatrick Mooney
12254c87aefeSPatrick Mooney temp = &amdvi_dte[devid];
12264c87aefeSPatrick Mooney
12274c87aefeSPatrick Mooney #ifdef AMDVI_ATS_ENABLE
12284c87aefeSPatrick Mooney /* If IOMMU and device support IOTLB, enable it. */
12294c87aefeSPatrick Mooney if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb)
12304c87aefeSPatrick Mooney temp->iotlb_enable = 1;
12314c87aefeSPatrick Mooney #endif
12324c87aefeSPatrick Mooney
12334c87aefeSPatrick Mooney /* Avoid duplicate I/O faults. */
12344c87aefeSPatrick Mooney temp->sup_second_io_fault = 1;
12354c87aefeSPatrick Mooney temp->sup_all_io_fault = amdvi_disable_io_fault;
12364c87aefeSPatrick Mooney
12374c87aefeSPatrick Mooney temp->dt_valid = 1;
12384c87aefeSPatrick Mooney temp->domain_id = domain->id;
12394c87aefeSPatrick Mooney
12404c87aefeSPatrick Mooney if (enable) {
12414c87aefeSPatrick Mooney if (domain->ptp) {
12424c87aefeSPatrick Mooney temp->pt_base = vtophys(domain->ptp) >> 12;
12434c87aefeSPatrick Mooney temp->pt_level = amdvi_ptp_level;
12444c87aefeSPatrick Mooney }
12454c87aefeSPatrick Mooney /*
12464c87aefeSPatrick Mooney * XXX: Page table valid[TV] bit must be set even if host domain
12474c87aefeSPatrick Mooney * page tables are not enabled.
12484c87aefeSPatrick Mooney */
12494c87aefeSPatrick Mooney temp->pt_valid = 1;
12504c87aefeSPatrick Mooney temp->read_allow = 1;
12514c87aefeSPatrick Mooney temp->write_allow = 1;
12524c87aefeSPatrick Mooney }
12534c87aefeSPatrick Mooney }
12544c87aefeSPatrick Mooney
12554c87aefeSPatrick Mooney static void
amdvi_inv_device(struct amdvi_softc * softc,uint16_t devid)1256b0de25cbSAndy Fiddaman amdvi_inv_device(struct amdvi_softc *softc, uint16_t devid)
12574c87aefeSPatrick Mooney {
12584c87aefeSPatrick Mooney KASSERT(softc, ("softc is NULL"));
12594c87aefeSPatrick Mooney
12604c87aefeSPatrick Mooney amdvi_cmd_inv_dte(softc, devid);
12614c87aefeSPatrick Mooney #ifdef AMDVI_ATS_ENABLE
12624c87aefeSPatrick Mooney if (amdvi_dev_support_iotlb(softc, devid))
12634c87aefeSPatrick Mooney amdvi_cmd_inv_iotlb(softc, devid);
12644c87aefeSPatrick Mooney #endif
12654c87aefeSPatrick Mooney amdvi_wait(softc);
12664c87aefeSPatrick Mooney }
12674c87aefeSPatrick Mooney
12684c87aefeSPatrick Mooney static void
amdvi_add_device(void * arg,uint16_t devid)12694c87aefeSPatrick Mooney amdvi_add_device(void *arg, uint16_t devid)
12704c87aefeSPatrick Mooney {
12714c87aefeSPatrick Mooney struct amdvi_domain *domain;
1272b0de25cbSAndy Fiddaman struct amdvi_softc *softc;
12734c87aefeSPatrick Mooney
12744c87aefeSPatrick Mooney domain = (struct amdvi_domain *)arg;
12754c87aefeSPatrick Mooney KASSERT(domain != NULL, ("domain is NULL"));
12764c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
12774c87aefeSPatrick Mooney printf("Assigning device(%d.%d.%d) to domain:%d\n",
12784c87aefeSPatrick Mooney RID2PCI_STR(devid), domain->id);
12794c87aefeSPatrick Mooney #endif
1280b0de25cbSAndy Fiddaman softc = amdvi_find_iommu(devid);
1281b0de25cbSAndy Fiddaman if (softc == NULL)
1282b0de25cbSAndy Fiddaman return;
1283b0de25cbSAndy Fiddaman amdvi_set_dte(domain, softc, devid, true);
1284b0de25cbSAndy Fiddaman amdvi_inv_device(softc, devid);
12854c87aefeSPatrick Mooney }
12864c87aefeSPatrick Mooney
12874c87aefeSPatrick Mooney static void
amdvi_remove_device(void * arg,uint16_t devid)12884c87aefeSPatrick Mooney amdvi_remove_device(void *arg, uint16_t devid)
12894c87aefeSPatrick Mooney {
12904c87aefeSPatrick Mooney struct amdvi_domain *domain;
1291b0de25cbSAndy Fiddaman struct amdvi_softc *softc;
12924c87aefeSPatrick Mooney
12934c87aefeSPatrick Mooney domain = (struct amdvi_domain *)arg;
12944c87aefeSPatrick Mooney #ifdef AMDVI_DEBUG_CMD
12954c87aefeSPatrick Mooney printf("Remove device(0x%x) from domain:%d\n",
12964c87aefeSPatrick Mooney devid, domain->id);
12974c87aefeSPatrick Mooney #endif
1298b0de25cbSAndy Fiddaman softc = amdvi_find_iommu(devid);
1299b0de25cbSAndy Fiddaman if (softc == NULL)
1300b0de25cbSAndy Fiddaman return;
1301b0de25cbSAndy Fiddaman amdvi_set_dte(domain, softc, devid, false);
1302b0de25cbSAndy Fiddaman amdvi_inv_device(softc, devid);
13034c87aefeSPatrick Mooney }
13044c87aefeSPatrick Mooney
13054c87aefeSPatrick Mooney static void
amdvi_enable(void)13064c87aefeSPatrick Mooney amdvi_enable(void)
13074c87aefeSPatrick Mooney {
13084c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
13094c87aefeSPatrick Mooney struct amdvi_softc *softc;
13104c87aefeSPatrick Mooney uint64_t val;
13114c87aefeSPatrick Mooney int i;
13124c87aefeSPatrick Mooney
13134c87aefeSPatrick Mooney for (i = 0; i < ivhd_count; i++) {
13144c87aefeSPatrick Mooney softc = device_get_softc(ivhd_devs[i]);
13154c87aefeSPatrick Mooney KASSERT(softc, ("softc is NULL\n"));
13164c87aefeSPatrick Mooney ctrl = softc->ctrl;
13174c87aefeSPatrick Mooney KASSERT(ctrl, ("ctrl is NULL\n"));
13184c87aefeSPatrick Mooney
131984971882SPatrick Mooney val = ( AMDVI_CTRL_EN |
132084971882SPatrick Mooney AMDVI_CTRL_CMD |
132184971882SPatrick Mooney AMDVI_CTRL_ELOG |
132284971882SPatrick Mooney AMDVI_CTRL_ELOGINT |
132384971882SPatrick Mooney AMDVI_CTRL_INV_TO_1S);
13244c87aefeSPatrick Mooney
13254c87aefeSPatrick Mooney if (softc->ivhd_flag & IVHD_FLAG_COH)
13264c87aefeSPatrick Mooney val |= AMDVI_CTRL_COH;
13274c87aefeSPatrick Mooney if (softc->ivhd_flag & IVHD_FLAG_HTT)
13284c87aefeSPatrick Mooney val |= AMDVI_CTRL_HTT;
13294c87aefeSPatrick Mooney if (softc->ivhd_flag & IVHD_FLAG_RPPW)
13304c87aefeSPatrick Mooney val |= AMDVI_CTRL_RPPW;
13314c87aefeSPatrick Mooney if (softc->ivhd_flag & IVHD_FLAG_PPW)
13324c87aefeSPatrick Mooney val |= AMDVI_CTRL_PPW;
13334c87aefeSPatrick Mooney if (softc->ivhd_flag & IVHD_FLAG_ISOC)
13344c87aefeSPatrick Mooney val |= AMDVI_CTRL_ISOC;
13354c87aefeSPatrick Mooney
13364c87aefeSPatrick Mooney ctrl->control = val;
13374c87aefeSPatrick Mooney }
13384c87aefeSPatrick Mooney }
13394c87aefeSPatrick Mooney
13404c87aefeSPatrick Mooney static void
amdvi_disable(void)13414c87aefeSPatrick Mooney amdvi_disable(void)
13424c87aefeSPatrick Mooney {
13434c87aefeSPatrick Mooney struct amdvi_ctrl *ctrl;
13444c87aefeSPatrick Mooney struct amdvi_softc *softc;
13454c87aefeSPatrick Mooney int i;
13464c87aefeSPatrick Mooney
13474c87aefeSPatrick Mooney for (i = 0; i < ivhd_count; i++) {
13484c87aefeSPatrick Mooney softc = device_get_softc(ivhd_devs[i]);
13494c87aefeSPatrick Mooney KASSERT(softc, ("softc is NULL\n"));
13504c87aefeSPatrick Mooney ctrl = softc->ctrl;
13514c87aefeSPatrick Mooney KASSERT(ctrl, ("ctrl is NULL\n"));
13524c87aefeSPatrick Mooney
13534c87aefeSPatrick Mooney ctrl->control = 0;
13544c87aefeSPatrick Mooney }
13554c87aefeSPatrick Mooney }
13564c87aefeSPatrick Mooney
13574c87aefeSPatrick Mooney static void
amdvi_invalidate_tlb(void * arg)1358b0de25cbSAndy Fiddaman amdvi_invalidate_tlb(void *arg)
13594c87aefeSPatrick Mooney {
13604c87aefeSPatrick Mooney struct amdvi_domain *domain;
13614c87aefeSPatrick Mooney
13624c87aefeSPatrick Mooney domain = (struct amdvi_domain *)arg;
13634c87aefeSPatrick Mooney KASSERT(domain, ("domain is NULL"));
13644c87aefeSPatrick Mooney amdvi_do_inv_domain(domain->id, false);
13654c87aefeSPatrick Mooney }
13664c87aefeSPatrick Mooney
1367b0de25cbSAndy Fiddaman const struct iommu_ops iommu_ops_amd = {
1368b0de25cbSAndy Fiddaman .init = amdvi_init,
1369b0de25cbSAndy Fiddaman .cleanup = amdvi_cleanup,
1370b0de25cbSAndy Fiddaman .enable = amdvi_enable,
1371b0de25cbSAndy Fiddaman .disable = amdvi_disable,
1372b0de25cbSAndy Fiddaman .create_domain = amdvi_create_domain,
1373b0de25cbSAndy Fiddaman .destroy_domain = amdvi_destroy_domain,
1374b0de25cbSAndy Fiddaman .create_mapping = amdvi_create_mapping,
1375b0de25cbSAndy Fiddaman .remove_mapping = amdvi_remove_mapping,
1376b0de25cbSAndy Fiddaman .add_device = amdvi_add_device,
1377b0de25cbSAndy Fiddaman .remove_device = amdvi_remove_device,
1378b0de25cbSAndy Fiddaman .invalidate_tlb = amdvi_invalidate_tlb
13794c87aefeSPatrick Mooney };
1380