1bf21cd93STycho Nightingale /* 2bf21cd93STycho Nightingale * This file and its contents are supplied under the terms of the 3bf21cd93STycho Nightingale * Common Development and Distribution License ("CDDL"), version 1.0. 4bf21cd93STycho Nightingale * You may only use this file in accordance with the terms of version 5bf21cd93STycho Nightingale * 1.0 of the CDDL. 6bf21cd93STycho Nightingale * 7bf21cd93STycho Nightingale * A full copy of the text of the CDDL should have accompanied this 8bf21cd93STycho Nightingale * source. A copy of the CDDL is also available via the Internet at 9bf21cd93STycho Nightingale * http://www.illumos.org/license/CDDL. 10bf21cd93STycho Nightingale */ 111fa07ac7SMike Zeller /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ 12bf21cd93STycho Nightingale 13bf21cd93STycho Nightingale /* 14bf21cd93STycho Nightingale * Copyright 2015 Pluribus Networks Inc. 154c87aefeSPatrick Mooney * Copyright 2019 Joyent, Inc. 168e3a263eSAndy Fiddaman * Copyright 2020 OmniOS Community Edition (OmniOSce) Association. 1759460b49SPatrick Mooney * Copyright 2021 Oxide Computer Company 18bf21cd93STycho Nightingale */ 19bf21cd93STycho Nightingale 20bf21cd93STycho Nightingale #include <sys/types.h> 21bf21cd93STycho Nightingale #include <sys/conf.h> 22bf21cd93STycho Nightingale #include <sys/cpuvar.h> 23bf21cd93STycho Nightingale #include <sys/ioccom.h> 24bf21cd93STycho Nightingale #include <sys/stat.h> 25bf21cd93STycho Nightingale #include <sys/vmsystm.h> 26bf21cd93STycho Nightingale #include <sys/ddi.h> 274c87aefeSPatrick Mooney #include <sys/mkdev.h> 28bf21cd93STycho Nightingale #include <sys/sunddi.h> 29bf21cd93STycho Nightingale #include <sys/fs/dv_node.h> 304c87aefeSPatrick Mooney #include <sys/cpuset.h> 314c87aefeSPatrick Mooney #include <sys/id_space.h> 324c87aefeSPatrick Mooney #include <sys/fs/sdev_plugin.h> 334c87aefeSPatrick Mooney #include <sys/smt.h> 3459460b49SPatrick Mooney #include <sys/kstat.h> 354c87aefeSPatrick Mooney 364c87aefeSPatrick Mooney #include <sys/kernel.h> 374c87aefeSPatrick Mooney #include <sys/hma.h> 384c87aefeSPatrick Mooney #include <sys/x86_archext.h> 39154972afSPatrick Mooney #include <x86/apicreg.h> 40bf21cd93STycho Nightingale 41bf21cd93STycho Nightingale #include <sys/vmm.h> 42b58b977eSPatrick Mooney #include <sys/vmm_kernel.h> 43bf21cd93STycho Nightingale #include <sys/vmm_instruction_emul.h> 44bf21cd93STycho Nightingale #include <sys/vmm_dev.h> 45bf21cd93STycho Nightingale #include <sys/vmm_impl.h> 464c87aefeSPatrick Mooney #include <sys/vmm_drv.h> 47cf409e3fSDan Cross #include <sys/vmm_vm.h> 48b57f5d3eSPatrick Mooney #include <sys/vmm_reservoir.h> 49bf21cd93STycho Nightingale 50bf21cd93STycho Nightingale #include <vm/seg_dev.h> 51bf21cd93STycho Nightingale 52eb9a1df2SHans Rosenfeld #include "io/ppt.h" 53bf21cd93STycho Nightingale #include "io/vatpic.h" 54bf21cd93STycho Nightingale #include "io/vioapic.h" 554c87aefeSPatrick Mooney #include "io/vrtc.h" 564c87aefeSPatrick Mooney #include "io/vhpet.h" 570e1453c3SPatrick Mooney #include "io/vpmtmr.h" 58bf21cd93STycho Nightingale #include "vmm_lapic.h" 594c87aefeSPatrick Mooney #include "vmm_stat.h" 604c87aefeSPatrick Mooney #include "vmm_util.h" 61bf21cd93STycho Nightingale 624c87aefeSPatrick Mooney /* 634c87aefeSPatrick Mooney * Locking details: 644c87aefeSPatrick Mooney * 654c87aefeSPatrick Mooney * Driver-wide data (vmmdev_*) , including HMA and sdev registration, is 664c87aefeSPatrick Mooney * protected by vmmdev_mtx. The list of vmm_softc_t instances and related data 674c87aefeSPatrick Mooney * (vmm_*) are protected by vmm_mtx. Actions requiring both locks must acquire 684c87aefeSPatrick Mooney * vmmdev_mtx before vmm_mtx. The sdev plugin functions must not attempt to 694c87aefeSPatrick Mooney * acquire vmmdev_mtx, as they could deadlock with plugin unregistration. 704c87aefeSPatrick Mooney */ 71bf21cd93STycho Nightingale 724c87aefeSPatrick Mooney static kmutex_t vmmdev_mtx; 734c87aefeSPatrick Mooney static dev_info_t *vmmdev_dip; 744c87aefeSPatrick Mooney static hma_reg_t *vmmdev_hma_reg; 758e3a263eSAndy Fiddaman static uint_t vmmdev_hma_ref; 764c87aefeSPatrick Mooney static sdev_plugin_hdl_t vmmdev_sdev_hdl; 77bf21cd93STycho Nightingale 784c87aefeSPatrick Mooney static kmutex_t vmm_mtx; 794c87aefeSPatrick Mooney static list_t vmm_list; 804c87aefeSPatrick Mooney static list_t vmm_destroy_list; 814c87aefeSPatrick Mooney static id_space_t *vmm_minors; 824c87aefeSPatrick Mooney static void *vmm_statep; 83bf21cd93STycho Nightingale 844c87aefeSPatrick Mooney static const char *vmmdev_hvm_name = "bhyve"; 85bf21cd93STycho Nightingale 864c87aefeSPatrick Mooney /* For sdev plugin (/dev) */ 874c87aefeSPatrick Mooney #define VMM_SDEV_ROOT "/dev/vmm" 884c87aefeSPatrick Mooney 897c8c0b82SPatrick Mooney /* From uts/intel/io/vmm/intel/vmx.c */ 904c87aefeSPatrick Mooney extern int vmx_x86_supported(const char **); 914c87aefeSPatrick Mooney 924c87aefeSPatrick Mooney /* Holds and hooks from drivers external to vmm */ 934c87aefeSPatrick Mooney struct vmm_hold { 944c87aefeSPatrick Mooney list_node_t vmh_node; 954c87aefeSPatrick Mooney vmm_softc_t *vmh_sc; 964c87aefeSPatrick Mooney boolean_t vmh_release_req; 974c87aefeSPatrick Mooney uint_t vmh_ioport_hook_cnt; 984c87aefeSPatrick Mooney }; 994c87aefeSPatrick Mooney 1004c87aefeSPatrick Mooney struct vmm_lease { 1014c87aefeSPatrick Mooney list_node_t vml_node; 1024c87aefeSPatrick Mooney struct vm *vml_vm; 1030153d828SPatrick Mooney vm_client_t *vml_vmclient; 1044c87aefeSPatrick Mooney boolean_t vml_expired; 1056703a0e8SPatrick Mooney boolean_t vml_break_deferred; 1064c87aefeSPatrick Mooney boolean_t (*vml_expire_func)(void *); 1074c87aefeSPatrick Mooney void *vml_expire_arg; 1084c87aefeSPatrick Mooney struct vmm_hold *vml_hold; 1094c87aefeSPatrick Mooney }; 1104c87aefeSPatrick Mooney 1114c87aefeSPatrick Mooney static int vmm_drv_block_hook(vmm_softc_t *, boolean_t); 1126703a0e8SPatrick Mooney static void vmm_lease_block(vmm_softc_t *); 1136703a0e8SPatrick Mooney static void vmm_lease_unblock(vmm_softc_t *); 11459460b49SPatrick Mooney static int vmm_kstat_alloc(vmm_softc_t *, minor_t, const cred_t *); 11559460b49SPatrick Mooney static void vmm_kstat_init(vmm_softc_t *); 11659460b49SPatrick Mooney static void vmm_kstat_fini(vmm_softc_t *); 1174c87aefeSPatrick Mooney 1183d066281SPatrick Mooney /* 1193d066281SPatrick Mooney * The 'devmem' hack: 1203d066281SPatrick Mooney * 1213d066281SPatrick Mooney * On native FreeBSD, bhyve consumers are allowed to create 'devmem' segments 1223d066281SPatrick Mooney * in the vm which appear with their own name related to the vm under /dev. 1233d066281SPatrick Mooney * Since this would be a hassle from an sdev perspective and would require a 1243d066281SPatrick Mooney * new cdev interface (or complicate the existing one), we choose to implement 1253d066281SPatrick Mooney * this in a different manner. Direct access to the underlying vm memory 1263d066281SPatrick Mooney * segments is exposed by placing them in a range of offsets beyond the normal 1273d066281SPatrick Mooney * guest memory space. Userspace can query the appropriate offset to mmap() 1283d066281SPatrick Mooney * for a given segment-id with the VM_DEVMEM_GETOFFSET ioctl. 1293d066281SPatrick Mooney */ 1303d066281SPatrick Mooney 1313d066281SPatrick Mooney static vmm_devmem_entry_t * 1323d066281SPatrick Mooney vmmdev_devmem_find(vmm_softc_t *sc, int segid) 1333d066281SPatrick Mooney { 1343d066281SPatrick Mooney vmm_devmem_entry_t *ent = NULL; 1353d066281SPatrick Mooney list_t *dl = &sc->vmm_devmem_list; 1363d066281SPatrick Mooney 1373d066281SPatrick Mooney for (ent = list_head(dl); ent != NULL; ent = list_next(dl, ent)) { 1383d066281SPatrick Mooney if (ent->vde_segid == segid) { 1393d066281SPatrick Mooney return (ent); 1403d066281SPatrick Mooney } 1413d066281SPatrick Mooney } 1423d066281SPatrick Mooney return (NULL); 1433d066281SPatrick Mooney } 1443d066281SPatrick Mooney 1454c87aefeSPatrick Mooney static int 1464c87aefeSPatrick Mooney vmmdev_get_memseg(vmm_softc_t *sc, struct vm_memseg *mseg) 147bf21cd93STycho Nightingale { 1484c87aefeSPatrick Mooney int error; 1494c87aefeSPatrick Mooney bool sysmem; 150bf21cd93STycho Nightingale 1514c87aefeSPatrick Mooney error = vm_get_memseg(sc->vmm_vm, mseg->segid, &mseg->len, &sysmem, 1524c87aefeSPatrick Mooney NULL); 1534c87aefeSPatrick Mooney if (error || mseg->len == 0) 1544c87aefeSPatrick Mooney return (error); 155bf21cd93STycho Nightingale 1564c87aefeSPatrick Mooney if (!sysmem) { 1574c87aefeSPatrick Mooney vmm_devmem_entry_t *de; 158bf21cd93STycho Nightingale 1593d066281SPatrick Mooney de = vmmdev_devmem_find(sc, mseg->segid); 1604c87aefeSPatrick Mooney if (de != NULL) { 1614c87aefeSPatrick Mooney (void) strlcpy(mseg->name, de->vde_name, 1624c87aefeSPatrick Mooney sizeof (mseg->name)); 1634c87aefeSPatrick Mooney } 1644c87aefeSPatrick Mooney } else { 1654c87aefeSPatrick Mooney bzero(mseg->name, sizeof (mseg->name)); 166bf21cd93STycho Nightingale } 1674c87aefeSPatrick Mooney 1684c87aefeSPatrick Mooney return (error); 169bf21cd93STycho Nightingale } 170bf21cd93STycho Nightingale 1714c87aefeSPatrick Mooney static int 1724c87aefeSPatrick Mooney vmmdev_devmem_create(vmm_softc_t *sc, struct vm_memseg *mseg, const char *name) 1734c87aefeSPatrick Mooney { 1744c87aefeSPatrick Mooney off_t map_offset; 1754c87aefeSPatrick Mooney vmm_devmem_entry_t *entry; 176bf21cd93STycho Nightingale 1774c87aefeSPatrick Mooney if (list_is_empty(&sc->vmm_devmem_list)) { 1784c87aefeSPatrick Mooney map_offset = VM_DEVMEM_START; 1794c87aefeSPatrick Mooney } else { 1804c87aefeSPatrick Mooney entry = list_tail(&sc->vmm_devmem_list); 1814c87aefeSPatrick Mooney map_offset = entry->vde_off + entry->vde_len; 1824c87aefeSPatrick Mooney if (map_offset < entry->vde_off) { 1834c87aefeSPatrick Mooney /* Do not tolerate overflow */ 1844c87aefeSPatrick Mooney return (ERANGE); 1854c87aefeSPatrick Mooney } 186bf21cd93STycho Nightingale /* 1874c87aefeSPatrick Mooney * XXXJOY: We could choose to search the list for duplicate 1884c87aefeSPatrick Mooney * names and toss an error. Since we're using the offset 1894c87aefeSPatrick Mooney * method for now, it does not make much of a difference. 190bf21cd93STycho Nightingale */ 191bf21cd93STycho Nightingale } 1924c87aefeSPatrick Mooney 1934c87aefeSPatrick Mooney entry = kmem_zalloc(sizeof (*entry), KM_SLEEP); 1944c87aefeSPatrick Mooney entry->vde_segid = mseg->segid; 1954c87aefeSPatrick Mooney entry->vde_len = mseg->len; 1964c87aefeSPatrick Mooney entry->vde_off = map_offset; 1974c87aefeSPatrick Mooney (void) strlcpy(entry->vde_name, name, sizeof (entry->vde_name)); 1984c87aefeSPatrick Mooney list_insert_tail(&sc->vmm_devmem_list, entry); 1994c87aefeSPatrick Mooney 2004c87aefeSPatrick Mooney return (0); 201bf21cd93STycho Nightingale } 202bf21cd93STycho Nightingale 2034c87aefeSPatrick Mooney static boolean_t 204c3d209caSPatrick Mooney vmmdev_devmem_segid(vmm_softc_t *sc, off_t off, off_t len, int *segidp, 205c3d209caSPatrick Mooney off_t *map_offp) 206bf21cd93STycho Nightingale { 2074c87aefeSPatrick Mooney list_t *dl = &sc->vmm_devmem_list; 2084c87aefeSPatrick Mooney vmm_devmem_entry_t *de = NULL; 209c3d209caSPatrick Mooney const off_t map_end = off + len; 210bf21cd93STycho Nightingale 2114c87aefeSPatrick Mooney VERIFY(off >= VM_DEVMEM_START); 212bf21cd93STycho Nightingale 213c3d209caSPatrick Mooney if (map_end < off) { 214c3d209caSPatrick Mooney /* No match on overflow */ 2154c87aefeSPatrick Mooney return (B_FALSE); 216bf21cd93STycho Nightingale } 217bf21cd93STycho Nightingale 218c3d209caSPatrick Mooney for (de = list_head(dl); de != NULL; de = list_next(dl, de)) { 219c3d209caSPatrick Mooney const off_t item_end = de->vde_off + de->vde_len; 220c3d209caSPatrick Mooney 221c3d209caSPatrick Mooney if (de->vde_off <= off && item_end >= map_end) { 222c3d209caSPatrick Mooney *segidp = de->vde_segid; 223c3d209caSPatrick Mooney *map_offp = off - de->vde_off; 224c3d209caSPatrick Mooney return (B_TRUE); 225c3d209caSPatrick Mooney } 226c3d209caSPatrick Mooney } 227c3d209caSPatrick Mooney return (B_FALSE); 2284c87aefeSPatrick Mooney } 229bf21cd93STycho Nightingale 230bf21cd93STycho Nightingale static void 2314c87aefeSPatrick Mooney vmmdev_devmem_purge(vmm_softc_t *sc) 232bf21cd93STycho Nightingale { 2334c87aefeSPatrick Mooney vmm_devmem_entry_t *entry; 2344c87aefeSPatrick Mooney 2354c87aefeSPatrick Mooney while ((entry = list_remove_head(&sc->vmm_devmem_list)) != NULL) { 2364c87aefeSPatrick Mooney kmem_free(entry, sizeof (*entry)); 2374c87aefeSPatrick Mooney } 238bf21cd93STycho Nightingale } 239bf21cd93STycho Nightingale 2404c87aefeSPatrick Mooney static int 2414c87aefeSPatrick Mooney vmmdev_alloc_memseg(vmm_softc_t *sc, struct vm_memseg *mseg) 242bf21cd93STycho Nightingale { 2434c87aefeSPatrick Mooney int error; 2444c87aefeSPatrick Mooney bool sysmem = true; 245bf21cd93STycho Nightingale 2464c87aefeSPatrick Mooney if (VM_MEMSEG_NAME(mseg)) { 2474c87aefeSPatrick Mooney sysmem = false; 248bf21cd93STycho Nightingale } 2494c87aefeSPatrick Mooney error = vm_alloc_memseg(sc->vmm_vm, mseg->segid, mseg->len, sysmem); 250bf21cd93STycho Nightingale 2513d066281SPatrick Mooney if (error == 0) { 2524c87aefeSPatrick Mooney /* 2534c87aefeSPatrick Mooney * Rather than create a whole fresh device from which userspace 2544c87aefeSPatrick Mooney * can mmap this segment, instead make it available at an 2554c87aefeSPatrick Mooney * offset above where the main guest memory resides. 2564c87aefeSPatrick Mooney */ 2574c87aefeSPatrick Mooney error = vmmdev_devmem_create(sc, mseg, mseg->name); 2584c87aefeSPatrick Mooney if (error != 0) { 2594c87aefeSPatrick Mooney vm_free_memseg(sc->vmm_vm, mseg->segid); 2604c87aefeSPatrick Mooney } 261bf21cd93STycho Nightingale } 2624c87aefeSPatrick Mooney return (error); 2634c87aefeSPatrick Mooney } 264bf21cd93STycho Nightingale 2654c87aefeSPatrick Mooney /* 2664c87aefeSPatrick Mooney * Resource Locking and Exclusion 2674c87aefeSPatrick Mooney * 2684c87aefeSPatrick Mooney * Much of bhyve depends on key portions of VM state, such as the guest memory 2694c87aefeSPatrick Mooney * map, to remain unchanged while the guest is running. As ported from 2704c87aefeSPatrick Mooney * FreeBSD, the initial strategy for this resource exclusion hinged on gating 2714c87aefeSPatrick Mooney * access to the instance vCPUs. Threads acting on a single vCPU, like those 2724c87aefeSPatrick Mooney * performing the work of actually running the guest in VMX/SVM, would lock 2734c87aefeSPatrick Mooney * only that vCPU during ioctl() entry. For ioctls which would change VM-wide 2744c87aefeSPatrick Mooney * state, all of the vCPUs would be first locked, ensuring that the 2754c87aefeSPatrick Mooney * operation(s) could complete without any other threads stumbling into 2764c87aefeSPatrick Mooney * intermediate states. 2774c87aefeSPatrick Mooney * 2784c87aefeSPatrick Mooney * This approach is largely effective for bhyve. Common operations, such as 2794c87aefeSPatrick Mooney * running the vCPUs, steer clear of lock contention. The model begins to 2804c87aefeSPatrick Mooney * break down for operations which do not occur in the context of a specific 2814c87aefeSPatrick Mooney * vCPU. LAPIC MSI delivery, for example, may be initiated from a worker 2824c87aefeSPatrick Mooney * thread in the bhyve process. In order to properly protect those vCPU-less 2834c87aefeSPatrick Mooney * operations from encountering invalid states, additional locking is required. 2844c87aefeSPatrick Mooney * This was solved by forcing those operations to lock the VM_MAXCPU-1 vCPU. 2854c87aefeSPatrick Mooney * It does mean that class of operations will be serialized on locking the 2864c87aefeSPatrick Mooney * specific vCPU and that instances sized at VM_MAXCPU will potentially see 2874c87aefeSPatrick Mooney * undue contention on the VM_MAXCPU-1 vCPU. 2884c87aefeSPatrick Mooney * 2894c87aefeSPatrick Mooney * In order to address the shortcomings of this model, the concept of a 2904c87aefeSPatrick Mooney * read/write lock has been added to bhyve. Operations which change 2914c87aefeSPatrick Mooney * fundamental aspects of a VM (such as the memory map) must acquire the write 2924c87aefeSPatrick Mooney * lock, which also implies locking all of the vCPUs and waiting for all read 2934c87aefeSPatrick Mooney * lock holders to release. While it increases the cost and waiting time for 2944c87aefeSPatrick Mooney * those few operations, it allows most hot-path operations on the VM (which 2954c87aefeSPatrick Mooney * depend on its configuration remaining stable) to occur with minimal locking. 2964c87aefeSPatrick Mooney * 2974c87aefeSPatrick Mooney * Consumers of the Driver API (see below) are a special case when it comes to 2984c87aefeSPatrick Mooney * this locking, since they may hold a read lock via the drv_lease mechanism 2994c87aefeSPatrick Mooney * for an extended period of time. Rather than forcing those consumers to 3004c87aefeSPatrick Mooney * continuously poll for a write lock attempt, the lease system forces them to 3014c87aefeSPatrick Mooney * provide a release callback to trigger their clean-up (and potential later 3024c87aefeSPatrick Mooney * reacquisition) of the read lock. 3034c87aefeSPatrick Mooney */ 304bf21cd93STycho Nightingale 3054c87aefeSPatrick Mooney static void 3064c87aefeSPatrick Mooney vcpu_lock_one(vmm_softc_t *sc, int vcpu) 3074c87aefeSPatrick Mooney { 3084c87aefeSPatrick Mooney ASSERT(vcpu >= 0 && vcpu < VM_MAXCPU); 309bf21cd93STycho Nightingale 3104c87aefeSPatrick Mooney /* 3114c87aefeSPatrick Mooney * Since this state transition is utilizing from_idle=true, it should 3124c87aefeSPatrick Mooney * not fail, but rather block until it can be successful. 3134c87aefeSPatrick Mooney */ 3144c87aefeSPatrick Mooney VERIFY0(vcpu_set_state(sc->vmm_vm, vcpu, VCPU_FROZEN, true)); 3154c87aefeSPatrick Mooney } 316bf21cd93STycho Nightingale 3174c87aefeSPatrick Mooney static void 3184c87aefeSPatrick Mooney vcpu_unlock_one(vmm_softc_t *sc, int vcpu) 3194c87aefeSPatrick Mooney { 3204c87aefeSPatrick Mooney ASSERT(vcpu >= 0 && vcpu < VM_MAXCPU); 321bf21cd93STycho Nightingale 3224c87aefeSPatrick Mooney VERIFY3U(vcpu_get_state(sc->vmm_vm, vcpu, NULL), ==, VCPU_FROZEN); 323e0994bd2SPatrick Mooney VERIFY0(vcpu_set_state(sc->vmm_vm, vcpu, VCPU_IDLE, false)); 324bf21cd93STycho Nightingale } 325bf21cd93STycho Nightingale 3264c87aefeSPatrick Mooney static void 3274c87aefeSPatrick Mooney vmm_read_lock(vmm_softc_t *sc) 328bf21cd93STycho Nightingale { 3294c87aefeSPatrick Mooney rw_enter(&sc->vmm_rwlock, RW_READER); 330bf21cd93STycho Nightingale } 331bf21cd93STycho Nightingale 3324c87aefeSPatrick Mooney static void 3334c87aefeSPatrick Mooney vmm_read_unlock(vmm_softc_t *sc) 334bf21cd93STycho Nightingale { 3354c87aefeSPatrick Mooney rw_exit(&sc->vmm_rwlock); 336bf21cd93STycho Nightingale } 337bf21cd93STycho Nightingale 3384c87aefeSPatrick Mooney static void 3394c87aefeSPatrick Mooney vmm_write_lock(vmm_softc_t *sc) 340bf21cd93STycho Nightingale { 3414c87aefeSPatrick Mooney int maxcpus; 342bf21cd93STycho Nightingale 3434c87aefeSPatrick Mooney /* First lock all the vCPUs */ 3444c87aefeSPatrick Mooney maxcpus = vm_get_maxcpus(sc->vmm_vm); 3454c87aefeSPatrick Mooney for (int vcpu = 0; vcpu < maxcpus; vcpu++) { 3464c87aefeSPatrick Mooney vcpu_lock_one(sc, vcpu); 3474c87aefeSPatrick Mooney } 348bf21cd93STycho Nightingale 3496703a0e8SPatrick Mooney /* 3506703a0e8SPatrick Mooney * Block vmm_drv leases from being acquired or held while the VM write 3516703a0e8SPatrick Mooney * lock is held. 3526703a0e8SPatrick Mooney */ 3536703a0e8SPatrick Mooney vmm_lease_block(sc); 3544c87aefeSPatrick Mooney 3554c87aefeSPatrick Mooney rw_enter(&sc->vmm_rwlock, RW_WRITER); 3564c87aefeSPatrick Mooney /* 3574c87aefeSPatrick Mooney * For now, the 'maxcpus' value for an instance is fixed at the 3584c87aefeSPatrick Mooney * compile-time constant of VM_MAXCPU at creation. If this changes in 3594c87aefeSPatrick Mooney * the future, allowing for dynamic vCPU resource sizing, acquisition 3604c87aefeSPatrick Mooney * of the write lock will need to be wary of such changes. 3614c87aefeSPatrick Mooney */ 3624c87aefeSPatrick Mooney VERIFY(maxcpus == vm_get_maxcpus(sc->vmm_vm)); 363bf21cd93STycho Nightingale } 364bf21cd93STycho Nightingale 3654c87aefeSPatrick Mooney static void 3664c87aefeSPatrick Mooney vmm_write_unlock(vmm_softc_t *sc) 367bf21cd93STycho Nightingale { 3684c87aefeSPatrick Mooney int maxcpus; 3694c87aefeSPatrick Mooney 3706703a0e8SPatrick Mooney /* Allow vmm_drv leases to be acquired once write lock is dropped */ 3716703a0e8SPatrick Mooney vmm_lease_unblock(sc); 372bf21cd93STycho Nightingale 373bf21cd93STycho Nightingale /* 3744c87aefeSPatrick Mooney * The VM write lock _must_ be released from the same thread it was 3754c87aefeSPatrick Mooney * acquired in, unlike the read lock. 376bf21cd93STycho Nightingale */ 3774c87aefeSPatrick Mooney VERIFY(rw_write_held(&sc->vmm_rwlock)); 3784c87aefeSPatrick Mooney rw_exit(&sc->vmm_rwlock); 3794c87aefeSPatrick Mooney 3804c87aefeSPatrick Mooney /* Unlock all the vCPUs */ 3814c87aefeSPatrick Mooney maxcpus = vm_get_maxcpus(sc->vmm_vm); 3824c87aefeSPatrick Mooney for (int vcpu = 0; vcpu < maxcpus; vcpu++) { 3834c87aefeSPatrick Mooney vcpu_unlock_one(sc, vcpu); 3844c87aefeSPatrick Mooney } 3854c87aefeSPatrick Mooney } 3864c87aefeSPatrick Mooney 3874c87aefeSPatrick Mooney static int 3884c87aefeSPatrick Mooney vmmdev_do_ioctl(vmm_softc_t *sc, int cmd, intptr_t arg, int md, 3894c87aefeSPatrick Mooney cred_t *credp, int *rvalp) 3904c87aefeSPatrick Mooney { 3914c87aefeSPatrick Mooney int error = 0, vcpu = -1; 3924c87aefeSPatrick Mooney void *datap = (void *)arg; 3934c87aefeSPatrick Mooney enum vm_lock_type { 3944c87aefeSPatrick Mooney LOCK_NONE = 0, 3954c87aefeSPatrick Mooney LOCK_VCPU, 3964c87aefeSPatrick Mooney LOCK_READ_HOLD, 3974c87aefeSPatrick Mooney LOCK_WRITE_HOLD 3984c87aefeSPatrick Mooney } lock_type = LOCK_NONE; 3994c87aefeSPatrick Mooney 4004c87aefeSPatrick Mooney /* Acquire any exclusion resources needed for the operation. */ 401bf21cd93STycho Nightingale switch (cmd) { 402bf21cd93STycho Nightingale case VM_RUN: 403bf21cd93STycho Nightingale case VM_GET_REGISTER: 404bf21cd93STycho Nightingale case VM_SET_REGISTER: 405bf21cd93STycho Nightingale case VM_GET_SEGMENT_DESCRIPTOR: 406bf21cd93STycho Nightingale case VM_SET_SEGMENT_DESCRIPTOR: 4074c87aefeSPatrick Mooney case VM_GET_REGISTER_SET: 4084c87aefeSPatrick Mooney case VM_SET_REGISTER_SET: 409bf21cd93STycho Nightingale case VM_INJECT_EXCEPTION: 410bf21cd93STycho Nightingale case VM_GET_CAPABILITY: 411bf21cd93STycho Nightingale case VM_SET_CAPABILITY: 412bf21cd93STycho Nightingale case VM_PPTDEV_MSI: 413bf21cd93STycho Nightingale case VM_PPTDEV_MSIX: 414bf21cd93STycho Nightingale case VM_SET_X2APIC_STATE: 415bf21cd93STycho Nightingale case VM_GLA2GPA: 4164c87aefeSPatrick Mooney case VM_GLA2GPA_NOFAULT: 417bf21cd93STycho Nightingale case VM_ACTIVATE_CPU: 4184c87aefeSPatrick Mooney case VM_SET_INTINFO: 4194c87aefeSPatrick Mooney case VM_GET_INTINFO: 420bf21cd93STycho Nightingale case VM_RESTART_INSTRUCTION: 421154972afSPatrick Mooney case VM_SET_KERNEMU_DEV: 422154972afSPatrick Mooney case VM_GET_KERNEMU_DEV: 4232606939dSPatrick Mooney case VM_RESET_CPU: 4242606939dSPatrick Mooney case VM_GET_RUN_STATE: 4252606939dSPatrick Mooney case VM_SET_RUN_STATE: 426957246c9SPatrick Mooney case VM_GET_FPU: 427957246c9SPatrick Mooney case VM_SET_FPU: 428bf21cd93STycho Nightingale /* 4294c87aefeSPatrick Mooney * Copy in the ID of the vCPU chosen for this operation. 4304c87aefeSPatrick Mooney * Since a nefarious caller could update their struct between 4314c87aefeSPatrick Mooney * this locking and when the rest of the ioctl data is copied 4324c87aefeSPatrick Mooney * in, it is _critical_ that this local 'vcpu' variable be used 4334c87aefeSPatrick Mooney * rather than the in-struct one when performing the ioctl. 434bf21cd93STycho Nightingale */ 4354c87aefeSPatrick Mooney if (ddi_copyin(datap, &vcpu, sizeof (vcpu), md)) { 436bf21cd93STycho Nightingale return (EFAULT); 437bf21cd93STycho Nightingale } 4384c87aefeSPatrick Mooney if (vcpu < 0 || vcpu > vm_get_maxcpus(sc->vmm_vm)) { 4394c87aefeSPatrick Mooney return (EINVAL); 440bf21cd93STycho Nightingale } 4414c87aefeSPatrick Mooney vcpu_lock_one(sc, vcpu); 4424c87aefeSPatrick Mooney lock_type = LOCK_VCPU; 443bf21cd93STycho Nightingale break; 444bf21cd93STycho Nightingale 4454c87aefeSPatrick Mooney case VM_REINIT: 4464c87aefeSPatrick Mooney case VM_BIND_PPTDEV: 4474c87aefeSPatrick Mooney case VM_UNBIND_PPTDEV: 4484c87aefeSPatrick Mooney case VM_MAP_PPTDEV_MMIO: 4492b948146SAndy Fiddaman case VM_UNMAP_PPTDEV_MMIO: 4504c87aefeSPatrick Mooney case VM_ALLOC_MEMSEG: 4514c87aefeSPatrick Mooney case VM_MMAP_MEMSEG: 4522b948146SAndy Fiddaman case VM_MUNMAP_MEMSEG: 4534c87aefeSPatrick Mooney case VM_WRLOCK_CYCLE: 4540e1453c3SPatrick Mooney case VM_PMTMR_LOCATE: 4554c87aefeSPatrick Mooney vmm_write_lock(sc); 4564c87aefeSPatrick Mooney lock_type = LOCK_WRITE_HOLD; 4574c87aefeSPatrick Mooney break; 458bf21cd93STycho Nightingale 4594c87aefeSPatrick Mooney case VM_GET_MEMSEG: 4604c87aefeSPatrick Mooney case VM_MMAP_GETNEXT: 4614c87aefeSPatrick Mooney case VM_LAPIC_IRQ: 4624c87aefeSPatrick Mooney case VM_INJECT_NMI: 4634c87aefeSPatrick Mooney case VM_IOAPIC_ASSERT_IRQ: 4644c87aefeSPatrick Mooney case VM_IOAPIC_DEASSERT_IRQ: 4654c87aefeSPatrick Mooney case VM_IOAPIC_PULSE_IRQ: 4664c87aefeSPatrick Mooney case VM_LAPIC_MSI: 4674c87aefeSPatrick Mooney case VM_LAPIC_LOCAL_IRQ: 4684c87aefeSPatrick Mooney case VM_GET_X2APIC_STATE: 4694c87aefeSPatrick Mooney case VM_RTC_READ: 4704c87aefeSPatrick Mooney case VM_RTC_WRITE: 4714c87aefeSPatrick Mooney case VM_RTC_SETTIME: 4724c87aefeSPatrick Mooney case VM_RTC_GETTIME: 4736960cd89SAndy Fiddaman case VM_PPTDEV_DISABLE_MSIX: 4744c87aefeSPatrick Mooney case VM_DEVMEM_GETOFFSET: 4758779b448SDan Cross case VM_TRACK_DIRTY_PAGES: 4764c87aefeSPatrick Mooney vmm_read_lock(sc); 4774c87aefeSPatrick Mooney lock_type = LOCK_READ_HOLD; 478bf21cd93STycho Nightingale break; 479bf21cd93STycho Nightingale 4800153d828SPatrick Mooney case VM_GET_GPA_PMAP: 4814c87aefeSPatrick Mooney case VM_IOAPIC_PINCOUNT: 48252fac30eSPatrick Mooney case VM_SUSPEND: 483957246c9SPatrick Mooney case VM_DESC_FPU_AREA: 484bf21cd93STycho Nightingale default: 485bf21cd93STycho Nightingale break; 486bf21cd93STycho Nightingale } 487bf21cd93STycho Nightingale 4884c87aefeSPatrick Mooney /* Execute the primary logic for the ioctl. */ 4894c87aefeSPatrick Mooney switch (cmd) { 4904c87aefeSPatrick Mooney case VM_RUN: { 491e0c0d44eSPatrick Mooney struct vm_entry entry; 4924c87aefeSPatrick Mooney 493e0c0d44eSPatrick Mooney if (ddi_copyin(datap, &entry, sizeof (entry), md)) { 4944c87aefeSPatrick Mooney error = EFAULT; 4954c87aefeSPatrick Mooney break; 496bf21cd93STycho Nightingale } 4974c87aefeSPatrick Mooney 4984c87aefeSPatrick Mooney if (!(curthread->t_schedflag & TS_VCPU)) 4994c87aefeSPatrick Mooney smt_mark_as_vcpu(); 5004c87aefeSPatrick Mooney 501e0c0d44eSPatrick Mooney error = vm_run(sc->vmm_vm, vcpu, &entry); 502e0c0d44eSPatrick Mooney 5033e1c5f3aSPatrick Mooney /* 5043e1c5f3aSPatrick Mooney * Unexpected states in vm_run() are expressed through positive 5053e1c5f3aSPatrick Mooney * errno-oriented return values. VM states which expect further 5063e1c5f3aSPatrick Mooney * processing in userspace (necessary context via exitinfo) are 5073e1c5f3aSPatrick Mooney * expressed through negative return values. For the time being 5083e1c5f3aSPatrick Mooney * a return value of 0 is not expected from vm_run(). 5093e1c5f3aSPatrick Mooney */ 5103e1c5f3aSPatrick Mooney ASSERT(error != 0); 5113e1c5f3aSPatrick Mooney if (error < 0) { 512e0c0d44eSPatrick Mooney const struct vm_exit *vme; 513e0c0d44eSPatrick Mooney void *outp = entry.exit_data; 514e0c0d44eSPatrick Mooney 5153e1c5f3aSPatrick Mooney error = 0; 516e0c0d44eSPatrick Mooney vme = vm_exitinfo(sc->vmm_vm, vcpu); 517e0c0d44eSPatrick Mooney if (ddi_copyout(vme, outp, sizeof (*vme), md)) { 518e0c0d44eSPatrick Mooney error = EFAULT; 519e0c0d44eSPatrick Mooney } 520bf21cd93STycho Nightingale } 521bf21cd93STycho Nightingale break; 5224c87aefeSPatrick Mooney } 5234c87aefeSPatrick Mooney case VM_SUSPEND: { 5244c87aefeSPatrick Mooney struct vm_suspend vmsuspend; 5254c87aefeSPatrick Mooney 5264c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmsuspend, sizeof (vmsuspend), md)) { 5274c87aefeSPatrick Mooney error = EFAULT; 5284c87aefeSPatrick Mooney break; 529bf21cd93STycho Nightingale } 5304c87aefeSPatrick Mooney error = vm_suspend(sc->vmm_vm, vmsuspend.how); 531bf21cd93STycho Nightingale break; 5324c87aefeSPatrick Mooney } 53352fac30eSPatrick Mooney case VM_REINIT: { 53452fac30eSPatrick Mooney struct vm_reinit reinit; 53552fac30eSPatrick Mooney 53652fac30eSPatrick Mooney if (ddi_copyin(datap, &reinit, sizeof (reinit), md)) { 53752fac30eSPatrick Mooney error = EFAULT; 53852fac30eSPatrick Mooney break; 53952fac30eSPatrick Mooney } 5404c87aefeSPatrick Mooney if ((error = vmm_drv_block_hook(sc, B_TRUE)) != 0) { 5414c87aefeSPatrick Mooney /* 5424c87aefeSPatrick Mooney * The VM instance should be free of driver-attached 5434c87aefeSPatrick Mooney * hooks during the reinitialization process. 5444c87aefeSPatrick Mooney */ 5454c87aefeSPatrick Mooney break; 546bf21cd93STycho Nightingale } 54752fac30eSPatrick Mooney error = vm_reinit(sc->vmm_vm, reinit.flags); 5484c87aefeSPatrick Mooney (void) vmm_drv_block_hook(sc, B_FALSE); 549bf21cd93STycho Nightingale break; 55052fac30eSPatrick Mooney } 5514c87aefeSPatrick Mooney case VM_STAT_DESC: { 5524c87aefeSPatrick Mooney struct vm_stat_desc statdesc; 5534c87aefeSPatrick Mooney 5544c87aefeSPatrick Mooney if (ddi_copyin(datap, &statdesc, sizeof (statdesc), md)) { 5554c87aefeSPatrick Mooney error = EFAULT; 5564c87aefeSPatrick Mooney break; 557bf21cd93STycho Nightingale } 5584c87aefeSPatrick Mooney error = vmm_stat_desc_copy(statdesc.index, statdesc.desc, 5594c87aefeSPatrick Mooney sizeof (statdesc.desc)); 5604c87aefeSPatrick Mooney if (error == 0 && 5614c87aefeSPatrick Mooney ddi_copyout(&statdesc, datap, sizeof (statdesc), md)) { 5624c87aefeSPatrick Mooney error = EFAULT; 5634c87aefeSPatrick Mooney break; 564bf21cd93STycho Nightingale } 565bf21cd93STycho Nightingale break; 5664c87aefeSPatrick Mooney } 5674c87aefeSPatrick Mooney case VM_STATS_IOC: { 5684c87aefeSPatrick Mooney struct vm_stats vmstats; 5694c87aefeSPatrick Mooney 5704c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmstats, sizeof (vmstats), md)) { 5714c87aefeSPatrick Mooney error = EFAULT; 5724c87aefeSPatrick Mooney break; 573bf21cd93STycho Nightingale } 5744c87aefeSPatrick Mooney hrt2tv(gethrtime(), &vmstats.tv); 575*d7b72f7bSAndy Fiddaman error = vmm_stat_copy(sc->vmm_vm, vmstats.cpuid, vmstats.index, 576*d7b72f7bSAndy Fiddaman nitems(vmstats.statbuf), 5774c87aefeSPatrick Mooney &vmstats.num_entries, vmstats.statbuf); 5784c87aefeSPatrick Mooney if (error == 0 && 5794c87aefeSPatrick Mooney ddi_copyout(&vmstats, datap, sizeof (vmstats), md)) { 5804c87aefeSPatrick Mooney error = EFAULT; 5814c87aefeSPatrick Mooney break; 582bf21cd93STycho Nightingale } 583bf21cd93STycho Nightingale break; 5844c87aefeSPatrick Mooney } 5854c87aefeSPatrick Mooney 5864c87aefeSPatrick Mooney case VM_PPTDEV_MSI: { 5874c87aefeSPatrick Mooney struct vm_pptdev_msi pptmsi; 5884c87aefeSPatrick Mooney 5894c87aefeSPatrick Mooney if (ddi_copyin(datap, &pptmsi, sizeof (pptmsi), md)) { 5904c87aefeSPatrick Mooney error = EFAULT; 5914c87aefeSPatrick Mooney break; 592bf21cd93STycho Nightingale } 593eb9a1df2SHans Rosenfeld error = ppt_setup_msi(sc->vmm_vm, pptmsi.vcpu, pptmsi.pptfd, 594eb9a1df2SHans Rosenfeld pptmsi.addr, pptmsi.msg, pptmsi.numvec); 595eb9a1df2SHans Rosenfeld break; 5964c87aefeSPatrick Mooney } 5974c87aefeSPatrick Mooney case VM_PPTDEV_MSIX: { 5984c87aefeSPatrick Mooney struct vm_pptdev_msix pptmsix; 5994c87aefeSPatrick Mooney 6004c87aefeSPatrick Mooney if (ddi_copyin(datap, &pptmsix, sizeof (pptmsix), md)) { 6014c87aefeSPatrick Mooney error = EFAULT; 6024c87aefeSPatrick Mooney break; 603bf21cd93STycho Nightingale } 604eb9a1df2SHans Rosenfeld error = ppt_setup_msix(sc->vmm_vm, pptmsix.vcpu, pptmsix.pptfd, 605eb9a1df2SHans Rosenfeld pptmsix.idx, pptmsix.addr, pptmsix.msg, 606eb9a1df2SHans Rosenfeld pptmsix.vector_control); 607eb9a1df2SHans Rosenfeld break; 6084c87aefeSPatrick Mooney } 6096960cd89SAndy Fiddaman case VM_PPTDEV_DISABLE_MSIX: { 6106960cd89SAndy Fiddaman struct vm_pptdev pptdev; 6116960cd89SAndy Fiddaman 6126960cd89SAndy Fiddaman if (ddi_copyin(datap, &pptdev, sizeof (pptdev), md)) { 6136960cd89SAndy Fiddaman error = EFAULT; 6146960cd89SAndy Fiddaman break; 6156960cd89SAndy Fiddaman } 6166960cd89SAndy Fiddaman error = ppt_disable_msix(sc->vmm_vm, pptdev.pptfd); 6176960cd89SAndy Fiddaman break; 6186960cd89SAndy Fiddaman } 6194c87aefeSPatrick Mooney case VM_MAP_PPTDEV_MMIO: { 6204c87aefeSPatrick Mooney struct vm_pptdev_mmio pptmmio; 6214c87aefeSPatrick Mooney 6224c87aefeSPatrick Mooney if (ddi_copyin(datap, &pptmmio, sizeof (pptmmio), md)) { 6234c87aefeSPatrick Mooney error = EFAULT; 6244c87aefeSPatrick Mooney break; 625bf21cd93STycho Nightingale } 626eb9a1df2SHans Rosenfeld error = ppt_map_mmio(sc->vmm_vm, pptmmio.pptfd, pptmmio.gpa, 627eb9a1df2SHans Rosenfeld pptmmio.len, pptmmio.hpa); 628eb9a1df2SHans Rosenfeld break; 629eb9a1df2SHans Rosenfeld } 6302b948146SAndy Fiddaman case VM_UNMAP_PPTDEV_MMIO: { 6312b948146SAndy Fiddaman struct vm_pptdev_mmio pptmmio; 6322b948146SAndy Fiddaman 6332b948146SAndy Fiddaman if (ddi_copyin(datap, &pptmmio, sizeof (pptmmio), md)) { 6342b948146SAndy Fiddaman error = EFAULT; 6352b948146SAndy Fiddaman break; 6362b948146SAndy Fiddaman } 6372b948146SAndy Fiddaman error = ppt_unmap_mmio(sc->vmm_vm, pptmmio.pptfd, pptmmio.gpa, 6382b948146SAndy Fiddaman pptmmio.len); 6392b948146SAndy Fiddaman break; 6402b948146SAndy Fiddaman } 641eb9a1df2SHans Rosenfeld case VM_BIND_PPTDEV: { 642eb9a1df2SHans Rosenfeld struct vm_pptdev pptdev; 643eb9a1df2SHans Rosenfeld 644eb9a1df2SHans Rosenfeld if (ddi_copyin(datap, &pptdev, sizeof (pptdev), md)) { 645eb9a1df2SHans Rosenfeld error = EFAULT; 646eb9a1df2SHans Rosenfeld break; 647eb9a1df2SHans Rosenfeld } 648eb9a1df2SHans Rosenfeld error = vm_assign_pptdev(sc->vmm_vm, pptdev.pptfd); 649eb9a1df2SHans Rosenfeld break; 6504c87aefeSPatrick Mooney } 6514c87aefeSPatrick Mooney case VM_UNBIND_PPTDEV: { 6524c87aefeSPatrick Mooney struct vm_pptdev pptdev; 6534c87aefeSPatrick Mooney 6544c87aefeSPatrick Mooney if (ddi_copyin(datap, &pptdev, sizeof (pptdev), md)) { 6554c87aefeSPatrick Mooney error = EFAULT; 6564c87aefeSPatrick Mooney break; 657bf21cd93STycho Nightingale } 658eb9a1df2SHans Rosenfeld error = vm_unassign_pptdev(sc->vmm_vm, pptdev.pptfd); 659eb9a1df2SHans Rosenfeld break; 6604c87aefeSPatrick Mooney } 661eb9a1df2SHans Rosenfeld case VM_GET_PPTDEV_LIMITS: { 662eb9a1df2SHans Rosenfeld struct vm_pptdev_limits pptlimits; 6634c87aefeSPatrick Mooney 664eb9a1df2SHans Rosenfeld if (ddi_copyin(datap, &pptlimits, sizeof (pptlimits), md)) { 665eb9a1df2SHans Rosenfeld error = EFAULT; 666eb9a1df2SHans Rosenfeld break; 667eb9a1df2SHans Rosenfeld } 668eb9a1df2SHans Rosenfeld error = ppt_get_limits(sc->vmm_vm, pptlimits.pptfd, 669eb9a1df2SHans Rosenfeld &pptlimits.msi_limit, &pptlimits.msix_limit); 670eb9a1df2SHans Rosenfeld if (error == 0 && 671eb9a1df2SHans Rosenfeld ddi_copyout(&pptlimits, datap, sizeof (pptlimits), md)) { 672eb9a1df2SHans Rosenfeld error = EFAULT; 673eb9a1df2SHans Rosenfeld break; 674eb9a1df2SHans Rosenfeld } 675eb9a1df2SHans Rosenfeld break; 676eb9a1df2SHans Rosenfeld } 6774c87aefeSPatrick Mooney case VM_INJECT_EXCEPTION: { 6784c87aefeSPatrick Mooney struct vm_exception vmexc; 6794c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmexc, sizeof (vmexc), md)) { 6804c87aefeSPatrick Mooney error = EFAULT; 6814c87aefeSPatrick Mooney break; 682bf21cd93STycho Nightingale } 6834c87aefeSPatrick Mooney error = vm_inject_exception(sc->vmm_vm, vcpu, vmexc.vector, 6844c87aefeSPatrick Mooney vmexc.error_code_valid, vmexc.error_code, 6854c87aefeSPatrick Mooney vmexc.restart_instruction); 686bf21cd93STycho Nightingale break; 6874c87aefeSPatrick Mooney } 6884c87aefeSPatrick Mooney case VM_INJECT_NMI: { 6894c87aefeSPatrick Mooney struct vm_nmi vmnmi; 6904c87aefeSPatrick Mooney 6914c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmnmi, sizeof (vmnmi), md)) { 6924c87aefeSPatrick Mooney error = EFAULT; 6934c87aefeSPatrick Mooney break; 694bf21cd93STycho Nightingale } 6954c87aefeSPatrick Mooney error = vm_inject_nmi(sc->vmm_vm, vmnmi.cpuid); 696bf21cd93STycho Nightingale break; 6974c87aefeSPatrick Mooney } 6984c87aefeSPatrick Mooney case VM_LAPIC_IRQ: { 6994c87aefeSPatrick Mooney struct vm_lapic_irq vmirq; 7004c87aefeSPatrick Mooney 7014c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmirq, sizeof (vmirq), md)) { 7024c87aefeSPatrick Mooney error = EFAULT; 7034c87aefeSPatrick Mooney break; 704bf21cd93STycho Nightingale } 7054c87aefeSPatrick Mooney error = lapic_intr_edge(sc->vmm_vm, vmirq.cpuid, vmirq.vector); 706bf21cd93STycho Nightingale break; 7074c87aefeSPatrick Mooney } 7084c87aefeSPatrick Mooney case VM_LAPIC_LOCAL_IRQ: { 7094c87aefeSPatrick Mooney struct vm_lapic_irq vmirq; 7104c87aefeSPatrick Mooney 7114c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmirq, sizeof (vmirq), md)) { 7124c87aefeSPatrick Mooney error = EFAULT; 7134c87aefeSPatrick Mooney break; 714bf21cd93STycho Nightingale } 7154c87aefeSPatrick Mooney error = lapic_set_local_intr(sc->vmm_vm, vmirq.cpuid, 7164c87aefeSPatrick Mooney vmirq.vector); 717bf21cd93STycho Nightingale break; 7184c87aefeSPatrick Mooney } 7194c87aefeSPatrick Mooney case VM_LAPIC_MSI: { 7204c87aefeSPatrick Mooney struct vm_lapic_msi vmmsi; 7214c87aefeSPatrick Mooney 7224c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmmsi, sizeof (vmmsi), md)) { 7234c87aefeSPatrick Mooney error = EFAULT; 7244c87aefeSPatrick Mooney break; 725bf21cd93STycho Nightingale } 7264c87aefeSPatrick Mooney error = lapic_intr_msi(sc->vmm_vm, vmmsi.addr, vmmsi.msg); 727bf21cd93STycho Nightingale break; 7284c87aefeSPatrick Mooney } 7294c87aefeSPatrick Mooney 7304c87aefeSPatrick Mooney case VM_IOAPIC_ASSERT_IRQ: { 7314c87aefeSPatrick Mooney struct vm_ioapic_irq ioapic_irq; 7324c87aefeSPatrick Mooney 7334c87aefeSPatrick Mooney if (ddi_copyin(datap, &ioapic_irq, sizeof (ioapic_irq), md)) { 7344c87aefeSPatrick Mooney error = EFAULT; 7354c87aefeSPatrick Mooney break; 736bf21cd93STycho Nightingale } 7374c87aefeSPatrick Mooney error = vioapic_assert_irq(sc->vmm_vm, ioapic_irq.irq); 738bf21cd93STycho Nightingale break; 7394c87aefeSPatrick Mooney } 7404c87aefeSPatrick Mooney case VM_IOAPIC_DEASSERT_IRQ: { 7414c87aefeSPatrick Mooney struct vm_ioapic_irq ioapic_irq; 7424c87aefeSPatrick Mooney 7434c87aefeSPatrick Mooney if (ddi_copyin(datap, &ioapic_irq, sizeof (ioapic_irq), md)) { 7444c87aefeSPatrick Mooney error = EFAULT; 7454c87aefeSPatrick Mooney break; 746bf21cd93STycho Nightingale } 7474c87aefeSPatrick Mooney error = vioapic_deassert_irq(sc->vmm_vm, ioapic_irq.irq); 748bf21cd93STycho Nightingale break; 7494c87aefeSPatrick Mooney } 7504c87aefeSPatrick Mooney case VM_IOAPIC_PULSE_IRQ: { 7514c87aefeSPatrick Mooney struct vm_ioapic_irq ioapic_irq; 7524c87aefeSPatrick Mooney 7534c87aefeSPatrick Mooney if (ddi_copyin(datap, &ioapic_irq, sizeof (ioapic_irq), md)) { 7544c87aefeSPatrick Mooney error = EFAULT; 7554c87aefeSPatrick Mooney break; 756bf21cd93STycho Nightingale } 7574c87aefeSPatrick Mooney error = vioapic_pulse_irq(sc->vmm_vm, ioapic_irq.irq); 758bf21cd93STycho Nightingale break; 7594c87aefeSPatrick Mooney } 7604c87aefeSPatrick Mooney case VM_IOAPIC_PINCOUNT: { 7614c87aefeSPatrick Mooney int pincount; 7624c87aefeSPatrick Mooney 7634c87aefeSPatrick Mooney pincount = vioapic_pincount(sc->vmm_vm); 7644c87aefeSPatrick Mooney if (ddi_copyout(&pincount, datap, sizeof (int), md)) { 7654c87aefeSPatrick Mooney error = EFAULT; 7664c87aefeSPatrick Mooney break; 767bf21cd93STycho Nightingale } 7684c87aefeSPatrick Mooney break; 7694c87aefeSPatrick Mooney } 770957246c9SPatrick Mooney case VM_DESC_FPU_AREA: { 771957246c9SPatrick Mooney struct vm_fpu_desc desc; 772957246c9SPatrick Mooney void *buf = NULL; 773957246c9SPatrick Mooney 774957246c9SPatrick Mooney if (ddi_copyin(datap, &desc, sizeof (desc), md)) { 775957246c9SPatrick Mooney error = EFAULT; 776957246c9SPatrick Mooney break; 777957246c9SPatrick Mooney } 778957246c9SPatrick Mooney if (desc.vfd_num_entries > 64) { 779957246c9SPatrick Mooney error = EINVAL; 780957246c9SPatrick Mooney break; 781957246c9SPatrick Mooney } 782957246c9SPatrick Mooney const size_t buf_sz = sizeof (struct vm_fpu_desc_entry) * 783957246c9SPatrick Mooney desc.vfd_num_entries; 784957246c9SPatrick Mooney if (buf_sz != 0) { 785957246c9SPatrick Mooney buf = kmem_zalloc(buf_sz, KM_SLEEP); 786957246c9SPatrick Mooney } 787957246c9SPatrick Mooney 788957246c9SPatrick Mooney /* 789957246c9SPatrick Mooney * For now, we are depending on vm_fpu_desc_entry and 790957246c9SPatrick Mooney * hma_xsave_state_desc_t having the same format. 791957246c9SPatrick Mooney */ 792957246c9SPatrick Mooney CTASSERT(sizeof (struct vm_fpu_desc_entry) == 793957246c9SPatrick Mooney sizeof (hma_xsave_state_desc_t)); 794957246c9SPatrick Mooney 795957246c9SPatrick Mooney size_t req_size; 796957246c9SPatrick Mooney const uint_t max_entries = hma_fpu_describe_xsave_state( 797957246c9SPatrick Mooney (hma_xsave_state_desc_t *)buf, 798957246c9SPatrick Mooney desc.vfd_num_entries, 799957246c9SPatrick Mooney &req_size); 800957246c9SPatrick Mooney 801957246c9SPatrick Mooney desc.vfd_req_size = req_size; 802957246c9SPatrick Mooney desc.vfd_num_entries = max_entries; 803957246c9SPatrick Mooney if (buf_sz != 0) { 804957246c9SPatrick Mooney if (ddi_copyout(buf, desc.vfd_entry_data, buf_sz, md)) { 805957246c9SPatrick Mooney error = EFAULT; 806957246c9SPatrick Mooney } 807957246c9SPatrick Mooney kmem_free(buf, buf_sz); 808957246c9SPatrick Mooney } 809957246c9SPatrick Mooney 810957246c9SPatrick Mooney if (error == 0) { 811957246c9SPatrick Mooney if (ddi_copyout(&desc, datap, sizeof (desc), md)) { 812957246c9SPatrick Mooney error = EFAULT; 813957246c9SPatrick Mooney } 814957246c9SPatrick Mooney } 815957246c9SPatrick Mooney break; 816957246c9SPatrick Mooney } 8174c87aefeSPatrick Mooney 8184c87aefeSPatrick Mooney case VM_ISA_ASSERT_IRQ: { 8194c87aefeSPatrick Mooney struct vm_isa_irq isa_irq; 8204c87aefeSPatrick Mooney 8214c87aefeSPatrick Mooney if (ddi_copyin(datap, &isa_irq, sizeof (isa_irq), md)) { 8224c87aefeSPatrick Mooney error = EFAULT; 8234c87aefeSPatrick Mooney break; 8244c87aefeSPatrick Mooney } 8254c87aefeSPatrick Mooney error = vatpic_assert_irq(sc->vmm_vm, isa_irq.atpic_irq); 8264c87aefeSPatrick Mooney if (error == 0 && isa_irq.ioapic_irq != -1) { 8274c87aefeSPatrick Mooney error = vioapic_assert_irq(sc->vmm_vm, 8284c87aefeSPatrick Mooney isa_irq.ioapic_irq); 8294c87aefeSPatrick Mooney } 8304c87aefeSPatrick Mooney break; 8314c87aefeSPatrick Mooney } 8324c87aefeSPatrick Mooney case VM_ISA_DEASSERT_IRQ: { 8334c87aefeSPatrick Mooney struct vm_isa_irq isa_irq; 8344c87aefeSPatrick Mooney 8354c87aefeSPatrick Mooney if (ddi_copyin(datap, &isa_irq, sizeof (isa_irq), md)) { 8364c87aefeSPatrick Mooney error = EFAULT; 8374c87aefeSPatrick Mooney break; 8384c87aefeSPatrick Mooney } 8394c87aefeSPatrick Mooney error = vatpic_deassert_irq(sc->vmm_vm, isa_irq.atpic_irq); 8404c87aefeSPatrick Mooney if (error == 0 && isa_irq.ioapic_irq != -1) { 8414c87aefeSPatrick Mooney error = vioapic_deassert_irq(sc->vmm_vm, 8424c87aefeSPatrick Mooney isa_irq.ioapic_irq); 8434c87aefeSPatrick Mooney } 8444c87aefeSPatrick Mooney break; 8454c87aefeSPatrick Mooney } 8464c87aefeSPatrick Mooney case VM_ISA_PULSE_IRQ: { 8474c87aefeSPatrick Mooney struct vm_isa_irq isa_irq; 8484c87aefeSPatrick Mooney 8494c87aefeSPatrick Mooney if (ddi_copyin(datap, &isa_irq, sizeof (isa_irq), md)) { 8504c87aefeSPatrick Mooney error = EFAULT; 8514c87aefeSPatrick Mooney break; 8524c87aefeSPatrick Mooney } 8534c87aefeSPatrick Mooney error = vatpic_pulse_irq(sc->vmm_vm, isa_irq.atpic_irq); 8544c87aefeSPatrick Mooney if (error == 0 && isa_irq.ioapic_irq != -1) { 8554c87aefeSPatrick Mooney error = vioapic_pulse_irq(sc->vmm_vm, 8564c87aefeSPatrick Mooney isa_irq.ioapic_irq); 8574c87aefeSPatrick Mooney } 8584c87aefeSPatrick Mooney break; 8594c87aefeSPatrick Mooney } 8604c87aefeSPatrick Mooney case VM_ISA_SET_IRQ_TRIGGER: { 8614c87aefeSPatrick Mooney struct vm_isa_irq_trigger isa_irq_trigger; 8624c87aefeSPatrick Mooney 8634c87aefeSPatrick Mooney if (ddi_copyin(datap, &isa_irq_trigger, 8644c87aefeSPatrick Mooney sizeof (isa_irq_trigger), md)) { 8654c87aefeSPatrick Mooney error = EFAULT; 8664c87aefeSPatrick Mooney break; 8674c87aefeSPatrick Mooney } 8684c87aefeSPatrick Mooney error = vatpic_set_irq_trigger(sc->vmm_vm, 8694c87aefeSPatrick Mooney isa_irq_trigger.atpic_irq, isa_irq_trigger.trigger); 8704c87aefeSPatrick Mooney break; 8714c87aefeSPatrick Mooney } 8724c87aefeSPatrick Mooney 8734c87aefeSPatrick Mooney case VM_MMAP_GETNEXT: { 8744c87aefeSPatrick Mooney struct vm_memmap mm; 8754c87aefeSPatrick Mooney 8764c87aefeSPatrick Mooney if (ddi_copyin(datap, &mm, sizeof (mm), md)) { 8774c87aefeSPatrick Mooney error = EFAULT; 8784c87aefeSPatrick Mooney break; 8794c87aefeSPatrick Mooney } 8804c87aefeSPatrick Mooney error = vm_mmap_getnext(sc->vmm_vm, &mm.gpa, &mm.segid, 8814c87aefeSPatrick Mooney &mm.segoff, &mm.len, &mm.prot, &mm.flags); 8824c87aefeSPatrick Mooney if (error == 0 && ddi_copyout(&mm, datap, sizeof (mm), md)) { 8834c87aefeSPatrick Mooney error = EFAULT; 8844c87aefeSPatrick Mooney break; 8854c87aefeSPatrick Mooney } 8864c87aefeSPatrick Mooney break; 8874c87aefeSPatrick Mooney } 8884c87aefeSPatrick Mooney case VM_MMAP_MEMSEG: { 8894c87aefeSPatrick Mooney struct vm_memmap mm; 8904c87aefeSPatrick Mooney 8914c87aefeSPatrick Mooney if (ddi_copyin(datap, &mm, sizeof (mm), md)) { 8924c87aefeSPatrick Mooney error = EFAULT; 8934c87aefeSPatrick Mooney break; 8944c87aefeSPatrick Mooney } 8954c87aefeSPatrick Mooney error = vm_mmap_memseg(sc->vmm_vm, mm.gpa, mm.segid, mm.segoff, 8964c87aefeSPatrick Mooney mm.len, mm.prot, mm.flags); 8974c87aefeSPatrick Mooney break; 8984c87aefeSPatrick Mooney } 8992b948146SAndy Fiddaman case VM_MUNMAP_MEMSEG: { 9002b948146SAndy Fiddaman struct vm_munmap mu; 9012b948146SAndy Fiddaman 9022b948146SAndy Fiddaman if (ddi_copyin(datap, &mu, sizeof (mu), md)) { 9032b948146SAndy Fiddaman error = EFAULT; 9042b948146SAndy Fiddaman break; 9052b948146SAndy Fiddaman } 9062b948146SAndy Fiddaman error = vm_munmap_memseg(sc->vmm_vm, mu.gpa, mu.len); 9072b948146SAndy Fiddaman break; 9082b948146SAndy Fiddaman } 9094c87aefeSPatrick Mooney case VM_ALLOC_MEMSEG: { 9104c87aefeSPatrick Mooney struct vm_memseg vmseg; 9114c87aefeSPatrick Mooney 9124c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmseg, sizeof (vmseg), md)) { 9134c87aefeSPatrick Mooney error = EFAULT; 9144c87aefeSPatrick Mooney break; 9154c87aefeSPatrick Mooney } 9164c87aefeSPatrick Mooney error = vmmdev_alloc_memseg(sc, &vmseg); 9174c87aefeSPatrick Mooney break; 9184c87aefeSPatrick Mooney } 9194c87aefeSPatrick Mooney case VM_GET_MEMSEG: { 9204c87aefeSPatrick Mooney struct vm_memseg vmseg; 9214c87aefeSPatrick Mooney 9224c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmseg, sizeof (vmseg), md)) { 9234c87aefeSPatrick Mooney error = EFAULT; 9244c87aefeSPatrick Mooney break; 9254c87aefeSPatrick Mooney } 9264c87aefeSPatrick Mooney error = vmmdev_get_memseg(sc, &vmseg); 9274c87aefeSPatrick Mooney if (error == 0 && 9284c87aefeSPatrick Mooney ddi_copyout(&vmseg, datap, sizeof (vmseg), md)) { 9294c87aefeSPatrick Mooney error = EFAULT; 9304c87aefeSPatrick Mooney break; 9314c87aefeSPatrick Mooney } 9324c87aefeSPatrick Mooney break; 9334c87aefeSPatrick Mooney } 9344c87aefeSPatrick Mooney case VM_GET_REGISTER: { 9354c87aefeSPatrick Mooney struct vm_register vmreg; 9364c87aefeSPatrick Mooney 9374c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmreg, sizeof (vmreg), md)) { 9384c87aefeSPatrick Mooney error = EFAULT; 9394c87aefeSPatrick Mooney break; 9404c87aefeSPatrick Mooney } 9414c87aefeSPatrick Mooney error = vm_get_register(sc->vmm_vm, vcpu, vmreg.regnum, 9424c87aefeSPatrick Mooney &vmreg.regval); 9434c87aefeSPatrick Mooney if (error == 0 && 9444c87aefeSPatrick Mooney ddi_copyout(&vmreg, datap, sizeof (vmreg), md)) { 9454c87aefeSPatrick Mooney error = EFAULT; 9464c87aefeSPatrick Mooney break; 9474c87aefeSPatrick Mooney } 9484c87aefeSPatrick Mooney break; 9494c87aefeSPatrick Mooney } 9504c87aefeSPatrick Mooney case VM_SET_REGISTER: { 9514c87aefeSPatrick Mooney struct vm_register vmreg; 9524c87aefeSPatrick Mooney 9534c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmreg, sizeof (vmreg), md)) { 9544c87aefeSPatrick Mooney error = EFAULT; 9554c87aefeSPatrick Mooney break; 9564c87aefeSPatrick Mooney } 9574c87aefeSPatrick Mooney error = vm_set_register(sc->vmm_vm, vcpu, vmreg.regnum, 9584c87aefeSPatrick Mooney vmreg.regval); 9594c87aefeSPatrick Mooney break; 9604c87aefeSPatrick Mooney } 9614c87aefeSPatrick Mooney case VM_SET_SEGMENT_DESCRIPTOR: { 9624c87aefeSPatrick Mooney struct vm_seg_desc vmsegd; 9634c87aefeSPatrick Mooney 9644c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmsegd, sizeof (vmsegd), md)) { 9654c87aefeSPatrick Mooney error = EFAULT; 9664c87aefeSPatrick Mooney break; 9674c87aefeSPatrick Mooney } 9684c87aefeSPatrick Mooney error = vm_set_seg_desc(sc->vmm_vm, vcpu, vmsegd.regnum, 9694c87aefeSPatrick Mooney &vmsegd.desc); 9704c87aefeSPatrick Mooney break; 9714c87aefeSPatrick Mooney } 9724c87aefeSPatrick Mooney case VM_GET_SEGMENT_DESCRIPTOR: { 9734c87aefeSPatrick Mooney struct vm_seg_desc vmsegd; 9744c87aefeSPatrick Mooney 9754c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmsegd, sizeof (vmsegd), md)) { 9764c87aefeSPatrick Mooney error = EFAULT; 9774c87aefeSPatrick Mooney break; 9784c87aefeSPatrick Mooney } 9794c87aefeSPatrick Mooney error = vm_get_seg_desc(sc->vmm_vm, vcpu, vmsegd.regnum, 9804c87aefeSPatrick Mooney &vmsegd.desc); 9814c87aefeSPatrick Mooney if (error == 0 && 9824c87aefeSPatrick Mooney ddi_copyout(&vmsegd, datap, sizeof (vmsegd), md)) { 9834c87aefeSPatrick Mooney error = EFAULT; 9844c87aefeSPatrick Mooney break; 9854c87aefeSPatrick Mooney } 9864c87aefeSPatrick Mooney break; 9874c87aefeSPatrick Mooney } 9884c87aefeSPatrick Mooney case VM_GET_REGISTER_SET: { 9894c87aefeSPatrick Mooney struct vm_register_set vrs; 9904c87aefeSPatrick Mooney int regnums[VM_REG_LAST]; 9914c87aefeSPatrick Mooney uint64_t regvals[VM_REG_LAST]; 9924c87aefeSPatrick Mooney 9934c87aefeSPatrick Mooney if (ddi_copyin(datap, &vrs, sizeof (vrs), md)) { 9944c87aefeSPatrick Mooney error = EFAULT; 9954c87aefeSPatrick Mooney break; 9964c87aefeSPatrick Mooney } 9974c87aefeSPatrick Mooney if (vrs.count > VM_REG_LAST || vrs.count == 0) { 9984c87aefeSPatrick Mooney error = EINVAL; 9994c87aefeSPatrick Mooney break; 10004c87aefeSPatrick Mooney } 10014c87aefeSPatrick Mooney if (ddi_copyin(vrs.regnums, regnums, 10024c87aefeSPatrick Mooney sizeof (int) * vrs.count, md)) { 10034c87aefeSPatrick Mooney error = EFAULT; 10044c87aefeSPatrick Mooney break; 10054c87aefeSPatrick Mooney } 10064c87aefeSPatrick Mooney 10074c87aefeSPatrick Mooney error = 0; 10084c87aefeSPatrick Mooney for (uint_t i = 0; i < vrs.count && error == 0; i++) { 10094c87aefeSPatrick Mooney if (regnums[i] < 0) { 10104c87aefeSPatrick Mooney error = EINVAL; 10114c87aefeSPatrick Mooney break; 1012bf21cd93STycho Nightingale } 10134c87aefeSPatrick Mooney error = vm_get_register(sc->vmm_vm, vcpu, regnums[i], 10144c87aefeSPatrick Mooney ®vals[i]); 10154c87aefeSPatrick Mooney } 10164c87aefeSPatrick Mooney if (error == 0 && ddi_copyout(regvals, vrs.regvals, 10174c87aefeSPatrick Mooney sizeof (uint64_t) * vrs.count, md)) { 10184c87aefeSPatrick Mooney error = EFAULT; 1019bf21cd93STycho Nightingale } 1020bf21cd93STycho Nightingale break; 10214c87aefeSPatrick Mooney } 10224c87aefeSPatrick Mooney case VM_SET_REGISTER_SET: { 10234c87aefeSPatrick Mooney struct vm_register_set vrs; 10244c87aefeSPatrick Mooney int regnums[VM_REG_LAST]; 10254c87aefeSPatrick Mooney uint64_t regvals[VM_REG_LAST]; 10264c87aefeSPatrick Mooney 10274c87aefeSPatrick Mooney if (ddi_copyin(datap, &vrs, sizeof (vrs), md)) { 10284c87aefeSPatrick Mooney error = EFAULT; 10294c87aefeSPatrick Mooney break; 1030bf21cd93STycho Nightingale } 10314c87aefeSPatrick Mooney if (vrs.count > VM_REG_LAST || vrs.count == 0) { 10324c87aefeSPatrick Mooney error = EINVAL; 10334c87aefeSPatrick Mooney break; 10344c87aefeSPatrick Mooney } 10354c87aefeSPatrick Mooney if (ddi_copyin(vrs.regnums, regnums, 10364c87aefeSPatrick Mooney sizeof (int) * vrs.count, md)) { 10374c87aefeSPatrick Mooney error = EFAULT; 10384c87aefeSPatrick Mooney break; 10394c87aefeSPatrick Mooney } 10404c87aefeSPatrick Mooney if (ddi_copyin(vrs.regvals, regvals, 10414c87aefeSPatrick Mooney sizeof (uint64_t) * vrs.count, md)) { 10424c87aefeSPatrick Mooney error = EFAULT; 10434c87aefeSPatrick Mooney break; 10444c87aefeSPatrick Mooney } 10454c87aefeSPatrick Mooney 10464c87aefeSPatrick Mooney error = 0; 10474c87aefeSPatrick Mooney for (uint_t i = 0; i < vrs.count && error == 0; i++) { 10484c87aefeSPatrick Mooney /* 10494c87aefeSPatrick Mooney * Setting registers in a set is not atomic, since a 10504c87aefeSPatrick Mooney * failure in the middle of the set will cause a 10514c87aefeSPatrick Mooney * bail-out and inconsistent register state. Callers 10524c87aefeSPatrick Mooney * should be wary of this. 10534c87aefeSPatrick Mooney */ 10544c87aefeSPatrick Mooney if (regnums[i] < 0) { 10554c87aefeSPatrick Mooney error = EINVAL; 10564c87aefeSPatrick Mooney break; 1057bf21cd93STycho Nightingale } 10584c87aefeSPatrick Mooney error = vm_set_register(sc->vmm_vm, vcpu, regnums[i], 10594c87aefeSPatrick Mooney regvals[i]); 1060bf21cd93STycho Nightingale } 1061bf21cd93STycho Nightingale break; 10624c87aefeSPatrick Mooney } 10632606939dSPatrick Mooney case VM_RESET_CPU: { 10642606939dSPatrick Mooney struct vm_vcpu_reset vvr; 10652606939dSPatrick Mooney 10662606939dSPatrick Mooney if (ddi_copyin(datap, &vvr, sizeof (vvr), md)) { 10672606939dSPatrick Mooney error = EFAULT; 10682606939dSPatrick Mooney break; 10692606939dSPatrick Mooney } 10702606939dSPatrick Mooney if (vvr.kind != VRK_RESET && vvr.kind != VRK_INIT) { 10712606939dSPatrick Mooney error = EINVAL; 10722606939dSPatrick Mooney } 10732606939dSPatrick Mooney 10742606939dSPatrick Mooney error = vcpu_arch_reset(sc->vmm_vm, vcpu, vvr.kind == VRK_INIT); 10752606939dSPatrick Mooney break; 10762606939dSPatrick Mooney } 10772606939dSPatrick Mooney case VM_GET_RUN_STATE: { 10782606939dSPatrick Mooney struct vm_run_state vrs; 10792606939dSPatrick Mooney 10802606939dSPatrick Mooney bzero(&vrs, sizeof (vrs)); 10812606939dSPatrick Mooney error = vm_get_run_state(sc->vmm_vm, vcpu, &vrs.state, 10822606939dSPatrick Mooney &vrs.sipi_vector); 10832606939dSPatrick Mooney if (error == 0) { 10842606939dSPatrick Mooney if (ddi_copyout(&vrs, datap, sizeof (vrs), md)) { 10852606939dSPatrick Mooney error = EFAULT; 10862606939dSPatrick Mooney break; 10872606939dSPatrick Mooney } 10882606939dSPatrick Mooney } 10892606939dSPatrick Mooney break; 10902606939dSPatrick Mooney } 10912606939dSPatrick Mooney case VM_SET_RUN_STATE: { 10922606939dSPatrick Mooney struct vm_run_state vrs; 10932606939dSPatrick Mooney 10942606939dSPatrick Mooney if (ddi_copyin(datap, &vrs, sizeof (vrs), md)) { 10952606939dSPatrick Mooney error = EFAULT; 10962606939dSPatrick Mooney break; 10972606939dSPatrick Mooney } 10982606939dSPatrick Mooney error = vm_set_run_state(sc->vmm_vm, vcpu, vrs.state, 10992606939dSPatrick Mooney vrs.sipi_vector); 11002606939dSPatrick Mooney break; 11012606939dSPatrick Mooney } 1102957246c9SPatrick Mooney case VM_GET_FPU: { 1103957246c9SPatrick Mooney struct vm_fpu_state req; 1104957246c9SPatrick Mooney const size_t max_len = (PAGESIZE * 2); 1105957246c9SPatrick Mooney void *kbuf; 1106957246c9SPatrick Mooney 1107957246c9SPatrick Mooney if (ddi_copyin(datap, &req, sizeof (req), md)) { 1108957246c9SPatrick Mooney error = EFAULT; 1109957246c9SPatrick Mooney break; 1110957246c9SPatrick Mooney } 1111957246c9SPatrick Mooney if (req.len > max_len || req.len == 0) { 1112957246c9SPatrick Mooney error = EINVAL; 1113957246c9SPatrick Mooney break; 1114957246c9SPatrick Mooney } 1115957246c9SPatrick Mooney kbuf = kmem_zalloc(req.len, KM_SLEEP); 1116957246c9SPatrick Mooney error = vm_get_fpu(sc->vmm_vm, vcpu, kbuf, req.len); 1117957246c9SPatrick Mooney if (error == 0) { 1118957246c9SPatrick Mooney if (ddi_copyout(kbuf, req.buf, req.len, md)) { 1119957246c9SPatrick Mooney error = EFAULT; 1120957246c9SPatrick Mooney } 1121957246c9SPatrick Mooney } 1122957246c9SPatrick Mooney kmem_free(kbuf, req.len); 1123957246c9SPatrick Mooney break; 1124957246c9SPatrick Mooney } 1125957246c9SPatrick Mooney case VM_SET_FPU: { 1126957246c9SPatrick Mooney struct vm_fpu_state req; 1127957246c9SPatrick Mooney const size_t max_len = (PAGESIZE * 2); 1128957246c9SPatrick Mooney void *kbuf; 1129957246c9SPatrick Mooney 1130957246c9SPatrick Mooney if (ddi_copyin(datap, &req, sizeof (req), md)) { 1131957246c9SPatrick Mooney error = EFAULT; 1132957246c9SPatrick Mooney break; 1133957246c9SPatrick Mooney } 1134957246c9SPatrick Mooney if (req.len > max_len || req.len == 0) { 1135957246c9SPatrick Mooney error = EINVAL; 1136957246c9SPatrick Mooney break; 1137957246c9SPatrick Mooney } 1138957246c9SPatrick Mooney kbuf = kmem_alloc(req.len, KM_SLEEP); 1139957246c9SPatrick Mooney if (ddi_copyin(req.buf, kbuf, req.len, md)) { 1140957246c9SPatrick Mooney error = EFAULT; 1141957246c9SPatrick Mooney } else { 1142957246c9SPatrick Mooney error = vm_set_fpu(sc->vmm_vm, vcpu, kbuf, req.len); 1143957246c9SPatrick Mooney } 1144957246c9SPatrick Mooney kmem_free(kbuf, req.len); 1145957246c9SPatrick Mooney break; 1146957246c9SPatrick Mooney } 11474c87aefeSPatrick Mooney 1148154972afSPatrick Mooney case VM_SET_KERNEMU_DEV: 1149154972afSPatrick Mooney case VM_GET_KERNEMU_DEV: { 1150154972afSPatrick Mooney struct vm_readwrite_kernemu_device kemu; 1151154972afSPatrick Mooney size_t size = 0; 1152154972afSPatrick Mooney 1153154972afSPatrick Mooney if (ddi_copyin(datap, &kemu, sizeof (kemu), md)) { 1154154972afSPatrick Mooney error = EFAULT; 1155154972afSPatrick Mooney break; 1156154972afSPatrick Mooney } 1157154972afSPatrick Mooney 1158154972afSPatrick Mooney if (kemu.access_width > 3) { 1159154972afSPatrick Mooney error = EINVAL; 1160154972afSPatrick Mooney break; 1161154972afSPatrick Mooney } 1162154972afSPatrick Mooney size = (1 << kemu.access_width); 1163154972afSPatrick Mooney ASSERT(size >= 1 && size <= 8); 1164154972afSPatrick Mooney 1165154972afSPatrick Mooney if (cmd == VM_SET_KERNEMU_DEV) { 1166e0c0d44eSPatrick Mooney error = vm_service_mmio_write(sc->vmm_vm, vcpu, 1167e0c0d44eSPatrick Mooney kemu.gpa, kemu.value, size); 1168154972afSPatrick Mooney } else { 1169e0c0d44eSPatrick Mooney error = vm_service_mmio_read(sc->vmm_vm, vcpu, 1170e0c0d44eSPatrick Mooney kemu.gpa, &kemu.value, size); 1171154972afSPatrick Mooney } 1172154972afSPatrick Mooney 1173154972afSPatrick Mooney if (error == 0) { 1174154972afSPatrick Mooney if (ddi_copyout(&kemu, datap, sizeof (kemu), md)) { 1175154972afSPatrick Mooney error = EFAULT; 1176154972afSPatrick Mooney break; 1177154972afSPatrick Mooney } 1178154972afSPatrick Mooney } 1179154972afSPatrick Mooney break; 1180154972afSPatrick Mooney } 1181154972afSPatrick Mooney 11824c87aefeSPatrick Mooney case VM_GET_CAPABILITY: { 11834c87aefeSPatrick Mooney struct vm_capability vmcap; 11844c87aefeSPatrick Mooney 11854c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmcap, sizeof (vmcap), md)) { 11864c87aefeSPatrick Mooney error = EFAULT; 11874c87aefeSPatrick Mooney break; 11884c87aefeSPatrick Mooney } 11894c87aefeSPatrick Mooney error = vm_get_capability(sc->vmm_vm, vcpu, vmcap.captype, 11904c87aefeSPatrick Mooney &vmcap.capval); 11914c87aefeSPatrick Mooney if (error == 0 && 11924c87aefeSPatrick Mooney ddi_copyout(&vmcap, datap, sizeof (vmcap), md)) { 11934c87aefeSPatrick Mooney error = EFAULT; 11944c87aefeSPatrick Mooney break; 1195bf21cd93STycho Nightingale } 1196bf21cd93STycho Nightingale break; 11974c87aefeSPatrick Mooney } 11984c87aefeSPatrick Mooney case VM_SET_CAPABILITY: { 11994c87aefeSPatrick Mooney struct vm_capability vmcap; 12004c87aefeSPatrick Mooney 12014c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmcap, sizeof (vmcap), md)) { 12024c87aefeSPatrick Mooney error = EFAULT; 12034c87aefeSPatrick Mooney break; 1204bf21cd93STycho Nightingale } 12054c87aefeSPatrick Mooney error = vm_set_capability(sc->vmm_vm, vcpu, vmcap.captype, 12064c87aefeSPatrick Mooney vmcap.capval); 1207bf21cd93STycho Nightingale break; 12084c87aefeSPatrick Mooney } 12094c87aefeSPatrick Mooney case VM_SET_X2APIC_STATE: { 12104c87aefeSPatrick Mooney struct vm_x2apic x2apic; 12114c87aefeSPatrick Mooney 12124c87aefeSPatrick Mooney if (ddi_copyin(datap, &x2apic, sizeof (x2apic), md)) { 12134c87aefeSPatrick Mooney error = EFAULT; 12144c87aefeSPatrick Mooney break; 1215bf21cd93STycho Nightingale } 12164c87aefeSPatrick Mooney error = vm_set_x2apic_state(sc->vmm_vm, vcpu, x2apic.state); 12174c87aefeSPatrick Mooney break; 12184c87aefeSPatrick Mooney } 12194c87aefeSPatrick Mooney case VM_GET_X2APIC_STATE: { 12204c87aefeSPatrick Mooney struct vm_x2apic x2apic; 12214c87aefeSPatrick Mooney 12224c87aefeSPatrick Mooney if (ddi_copyin(datap, &x2apic, sizeof (x2apic), md)) { 12234c87aefeSPatrick Mooney error = EFAULT; 12244c87aefeSPatrick Mooney break; 12254c87aefeSPatrick Mooney } 12264c87aefeSPatrick Mooney error = vm_get_x2apic_state(sc->vmm_vm, x2apic.cpuid, 12274c87aefeSPatrick Mooney &x2apic.state); 12284c87aefeSPatrick Mooney if (error == 0 && 12294c87aefeSPatrick Mooney ddi_copyout(&x2apic, datap, sizeof (x2apic), md)) { 12304c87aefeSPatrick Mooney error = EFAULT; 12314c87aefeSPatrick Mooney break; 12324c87aefeSPatrick Mooney } 12334c87aefeSPatrick Mooney break; 12344c87aefeSPatrick Mooney } 12354c87aefeSPatrick Mooney case VM_GET_GPA_PMAP: { 12360153d828SPatrick Mooney /* 12370153d828SPatrick Mooney * Until there is a necessity to leak EPT/RVI PTE values to 12380153d828SPatrick Mooney * userspace, this will remain unimplemented 12390153d828SPatrick Mooney */ 12400153d828SPatrick Mooney error = EINVAL; 12414c87aefeSPatrick Mooney break; 12424c87aefeSPatrick Mooney } 12434c87aefeSPatrick Mooney case VM_GET_HPET_CAPABILITIES: { 12444c87aefeSPatrick Mooney struct vm_hpet_cap hpetcap; 12454c87aefeSPatrick Mooney 12464c87aefeSPatrick Mooney error = vhpet_getcap(&hpetcap); 12474c87aefeSPatrick Mooney if (error == 0 && 12484c87aefeSPatrick Mooney ddi_copyout(&hpetcap, datap, sizeof (hpetcap), md)) { 12494c87aefeSPatrick Mooney error = EFAULT; 12504c87aefeSPatrick Mooney break; 1251bf21cd93STycho Nightingale } 1252bf21cd93STycho Nightingale break; 12534c87aefeSPatrick Mooney } 1254bf21cd93STycho Nightingale case VM_GLA2GPA: { 12554c87aefeSPatrick Mooney struct vm_gla2gpa gg; 12564c87aefeSPatrick Mooney 12574c87aefeSPatrick Mooney if (ddi_copyin(datap, &gg, sizeof (gg), md)) { 12584c87aefeSPatrick Mooney error = EFAULT; 12594c87aefeSPatrick Mooney break; 1260bf21cd93STycho Nightingale } 12614c87aefeSPatrick Mooney gg.vcpuid = vcpu; 12624c87aefeSPatrick Mooney error = vm_gla2gpa(sc->vmm_vm, vcpu, &gg.paging, gg.gla, 12634c87aefeSPatrick Mooney gg.prot, &gg.gpa, &gg.fault); 12644c87aefeSPatrick Mooney if (error == 0 && ddi_copyout(&gg, datap, sizeof (gg), md)) { 12654c87aefeSPatrick Mooney error = EFAULT; 12664c87aefeSPatrick Mooney break; 12674c87aefeSPatrick Mooney } 12684c87aefeSPatrick Mooney break; 12694c87aefeSPatrick Mooney } 12704c87aefeSPatrick Mooney case VM_GLA2GPA_NOFAULT: { 12714c87aefeSPatrick Mooney struct vm_gla2gpa gg; 12724c87aefeSPatrick Mooney 12734c87aefeSPatrick Mooney if (ddi_copyin(datap, &gg, sizeof (gg), md)) { 12744c87aefeSPatrick Mooney error = EFAULT; 12754c87aefeSPatrick Mooney break; 12764c87aefeSPatrick Mooney } 12774c87aefeSPatrick Mooney gg.vcpuid = vcpu; 12784c87aefeSPatrick Mooney error = vm_gla2gpa_nofault(sc->vmm_vm, vcpu, &gg.paging, 12794c87aefeSPatrick Mooney gg.gla, gg.prot, &gg.gpa, &gg.fault); 12804c87aefeSPatrick Mooney if (error == 0 && ddi_copyout(&gg, datap, sizeof (gg), md)) { 12814c87aefeSPatrick Mooney error = EFAULT; 12824c87aefeSPatrick Mooney break; 12834c87aefeSPatrick Mooney } 12844c87aefeSPatrick Mooney break; 12854c87aefeSPatrick Mooney } 12864c87aefeSPatrick Mooney 12874c87aefeSPatrick Mooney case VM_ACTIVATE_CPU: 12884c87aefeSPatrick Mooney error = vm_activate_cpu(sc->vmm_vm, vcpu); 12894c87aefeSPatrick Mooney break; 12904c87aefeSPatrick Mooney 12914c87aefeSPatrick Mooney case VM_SUSPEND_CPU: 12924c87aefeSPatrick Mooney if (ddi_copyin(datap, &vcpu, sizeof (vcpu), md)) { 12934c87aefeSPatrick Mooney error = EFAULT; 12944c87aefeSPatrick Mooney } else { 12954c87aefeSPatrick Mooney error = vm_suspend_cpu(sc->vmm_vm, vcpu); 12964c87aefeSPatrick Mooney } 12974c87aefeSPatrick Mooney break; 12984c87aefeSPatrick Mooney 12994c87aefeSPatrick Mooney case VM_RESUME_CPU: 13004c87aefeSPatrick Mooney if (ddi_copyin(datap, &vcpu, sizeof (vcpu), md)) { 13014c87aefeSPatrick Mooney error = EFAULT; 13024c87aefeSPatrick Mooney } else { 13034c87aefeSPatrick Mooney error = vm_resume_cpu(sc->vmm_vm, vcpu); 13044c87aefeSPatrick Mooney } 13054c87aefeSPatrick Mooney break; 13064c87aefeSPatrick Mooney 13074c87aefeSPatrick Mooney case VM_GET_CPUS: { 13084c87aefeSPatrick Mooney struct vm_cpuset vm_cpuset; 13094c87aefeSPatrick Mooney cpuset_t tempset; 13104c87aefeSPatrick Mooney void *srcp = &tempset; 13114c87aefeSPatrick Mooney int size; 13124c87aefeSPatrick Mooney 13134c87aefeSPatrick Mooney if (ddi_copyin(datap, &vm_cpuset, sizeof (vm_cpuset), md)) { 13144c87aefeSPatrick Mooney error = EFAULT; 13154c87aefeSPatrick Mooney break; 13164c87aefeSPatrick Mooney } 13174c87aefeSPatrick Mooney 13184c87aefeSPatrick Mooney /* Be more generous about sizing since our cpuset_t is large. */ 13194c87aefeSPatrick Mooney size = vm_cpuset.cpusetsize; 13204c87aefeSPatrick Mooney if (size <= 0 || size > sizeof (cpuset_t)) { 13214c87aefeSPatrick Mooney error = ERANGE; 13224c87aefeSPatrick Mooney } 13234c87aefeSPatrick Mooney /* 13244c87aefeSPatrick Mooney * If they want a ulong_t or less, make sure they receive the 13254c87aefeSPatrick Mooney * low bits with all the useful information. 13264c87aefeSPatrick Mooney */ 13274c87aefeSPatrick Mooney if (size <= sizeof (tempset.cpub[0])) { 13284c87aefeSPatrick Mooney srcp = &tempset.cpub[0]; 13294c87aefeSPatrick Mooney } 13304c87aefeSPatrick Mooney 13314c87aefeSPatrick Mooney if (vm_cpuset.which == VM_ACTIVE_CPUS) { 13324c87aefeSPatrick Mooney tempset = vm_active_cpus(sc->vmm_vm); 13334c87aefeSPatrick Mooney } else if (vm_cpuset.which == VM_SUSPENDED_CPUS) { 13344c87aefeSPatrick Mooney tempset = vm_suspended_cpus(sc->vmm_vm); 13354c87aefeSPatrick Mooney } else if (vm_cpuset.which == VM_DEBUG_CPUS) { 13364c87aefeSPatrick Mooney tempset = vm_debug_cpus(sc->vmm_vm); 1337bf21cd93STycho Nightingale } else { 13384c87aefeSPatrick Mooney error = EINVAL; 13394c87aefeSPatrick Mooney } 13404c87aefeSPatrick Mooney 13414c87aefeSPatrick Mooney ASSERT(size > 0 && size <= sizeof (tempset)); 13424c87aefeSPatrick Mooney if (error == 0 && 13434c87aefeSPatrick Mooney ddi_copyout(srcp, vm_cpuset.cpus, size, md)) { 13444c87aefeSPatrick Mooney error = EFAULT; 13454c87aefeSPatrick Mooney break; 13464c87aefeSPatrick Mooney } 13474c87aefeSPatrick Mooney break; 13484c87aefeSPatrick Mooney } 13494c87aefeSPatrick Mooney case VM_SET_INTINFO: { 13504c87aefeSPatrick Mooney struct vm_intinfo vmii; 13514c87aefeSPatrick Mooney 13524c87aefeSPatrick Mooney if (ddi_copyin(datap, &vmii, sizeof (vmii), md)) { 13534c87aefeSPatrick Mooney error = EFAULT; 13544c87aefeSPatrick Mooney break; 13554c87aefeSPatrick Mooney } 13564c87aefeSPatrick Mooney error = vm_exit_intinfo(sc->vmm_vm, vcpu, vmii.info1); 13574c87aefeSPatrick Mooney break; 13584c87aefeSPatrick Mooney } 13594c87aefeSPatrick Mooney case VM_GET_INTINFO: { 13604c87aefeSPatrick Mooney struct vm_intinfo vmii; 13614c87aefeSPatrick Mooney 13624c87aefeSPatrick Mooney vmii.vcpuid = vcpu; 13634c87aefeSPatrick Mooney error = vm_get_intinfo(sc->vmm_vm, vcpu, &vmii.info1, 13644c87aefeSPatrick Mooney &vmii.info2); 13654c87aefeSPatrick Mooney if (error == 0 && 13664c87aefeSPatrick Mooney ddi_copyout(&vmii, datap, sizeof (vmii), md)) { 13674c87aefeSPatrick Mooney error = EFAULT; 13684c87aefeSPatrick Mooney break; 13694c87aefeSPatrick Mooney } 13704c87aefeSPatrick Mooney break; 13714c87aefeSPatrick Mooney } 13724c87aefeSPatrick Mooney case VM_RTC_WRITE: { 13734c87aefeSPatrick Mooney struct vm_rtc_data rtcdata; 13744c87aefeSPatrick Mooney 13754c87aefeSPatrick Mooney if (ddi_copyin(datap, &rtcdata, sizeof (rtcdata), md)) { 13764c87aefeSPatrick Mooney error = EFAULT; 13774c87aefeSPatrick Mooney break; 13784c87aefeSPatrick Mooney } 13794c87aefeSPatrick Mooney error = vrtc_nvram_write(sc->vmm_vm, rtcdata.offset, 13804c87aefeSPatrick Mooney rtcdata.value); 13814c87aefeSPatrick Mooney break; 13824c87aefeSPatrick Mooney } 13834c87aefeSPatrick Mooney case VM_RTC_READ: { 13844c87aefeSPatrick Mooney struct vm_rtc_data rtcdata; 13854c87aefeSPatrick Mooney 13864c87aefeSPatrick Mooney if (ddi_copyin(datap, &rtcdata, sizeof (rtcdata), md)) { 13874c87aefeSPatrick Mooney error = EFAULT; 13884c87aefeSPatrick Mooney break; 13894c87aefeSPatrick Mooney } 13904c87aefeSPatrick Mooney error = vrtc_nvram_read(sc->vmm_vm, rtcdata.offset, 13914c87aefeSPatrick Mooney &rtcdata.value); 13924c87aefeSPatrick Mooney if (error == 0 && 13934c87aefeSPatrick Mooney ddi_copyout(&rtcdata, datap, sizeof (rtcdata), md)) { 13944c87aefeSPatrick Mooney error = EFAULT; 13954c87aefeSPatrick Mooney break; 13964c87aefeSPatrick Mooney } 13974c87aefeSPatrick Mooney break; 13984c87aefeSPatrick Mooney } 13994c87aefeSPatrick Mooney case VM_RTC_SETTIME: { 14004c87aefeSPatrick Mooney struct vm_rtc_time rtctime; 14014c87aefeSPatrick Mooney 14024c87aefeSPatrick Mooney if (ddi_copyin(datap, &rtctime, sizeof (rtctime), md)) { 14034c87aefeSPatrick Mooney error = EFAULT; 14044c87aefeSPatrick Mooney break; 14054c87aefeSPatrick Mooney } 14064c87aefeSPatrick Mooney error = vrtc_set_time(sc->vmm_vm, rtctime.secs); 14074c87aefeSPatrick Mooney break; 14084c87aefeSPatrick Mooney } 14094c87aefeSPatrick Mooney case VM_RTC_GETTIME: { 14104c87aefeSPatrick Mooney struct vm_rtc_time rtctime; 14114c87aefeSPatrick Mooney 14124c87aefeSPatrick Mooney rtctime.secs = vrtc_get_time(sc->vmm_vm); 14134c87aefeSPatrick Mooney if (ddi_copyout(&rtctime, datap, sizeof (rtctime), md)) { 14144c87aefeSPatrick Mooney error = EFAULT; 14154c87aefeSPatrick Mooney break; 14164c87aefeSPatrick Mooney } 14174c87aefeSPatrick Mooney break; 14184c87aefeSPatrick Mooney } 14194c87aefeSPatrick Mooney 14200e1453c3SPatrick Mooney case VM_PMTMR_LOCATE: { 14210e1453c3SPatrick Mooney uint16_t port = arg; 14220e1453c3SPatrick Mooney error = vpmtmr_set_location(sc->vmm_vm, port); 14230e1453c3SPatrick Mooney break; 14240e1453c3SPatrick Mooney } 14250e1453c3SPatrick Mooney 14264c87aefeSPatrick Mooney case VM_RESTART_INSTRUCTION: 14274c87aefeSPatrick Mooney error = vm_restart_instruction(sc->vmm_vm, vcpu); 14284c87aefeSPatrick Mooney break; 14294c87aefeSPatrick Mooney 14304c87aefeSPatrick Mooney case VM_SET_TOPOLOGY: { 14314c87aefeSPatrick Mooney struct vm_cpu_topology topo; 14324c87aefeSPatrick Mooney 14334c87aefeSPatrick Mooney if (ddi_copyin(datap, &topo, sizeof (topo), md) != 0) { 14344c87aefeSPatrick Mooney error = EFAULT; 14354c87aefeSPatrick Mooney break; 14364c87aefeSPatrick Mooney } 14374c87aefeSPatrick Mooney error = vm_set_topology(sc->vmm_vm, topo.sockets, topo.cores, 14384c87aefeSPatrick Mooney topo.threads, topo.maxcpus); 14394c87aefeSPatrick Mooney break; 14404c87aefeSPatrick Mooney } 14414c87aefeSPatrick Mooney case VM_GET_TOPOLOGY: { 14424c87aefeSPatrick Mooney struct vm_cpu_topology topo; 14434c87aefeSPatrick Mooney 14444c87aefeSPatrick Mooney vm_get_topology(sc->vmm_vm, &topo.sockets, &topo.cores, 14454c87aefeSPatrick Mooney &topo.threads, &topo.maxcpus); 14464c87aefeSPatrick Mooney if (ddi_copyout(&topo, datap, sizeof (topo), md) != 0) { 14474c87aefeSPatrick Mooney error = EFAULT; 14484c87aefeSPatrick Mooney break; 14494c87aefeSPatrick Mooney } 14504c87aefeSPatrick Mooney break; 14514c87aefeSPatrick Mooney } 14524c87aefeSPatrick Mooney case VM_DEVMEM_GETOFFSET: { 14534c87aefeSPatrick Mooney struct vm_devmem_offset vdo; 14543d066281SPatrick Mooney vmm_devmem_entry_t *de; 14554c87aefeSPatrick Mooney 14564c87aefeSPatrick Mooney if (ddi_copyin(datap, &vdo, sizeof (vdo), md) != 0) { 1457bf21cd93STycho Nightingale error = EFAULT; 14584c87aefeSPatrick Mooney break; 14594c87aefeSPatrick Mooney } 14604c87aefeSPatrick Mooney 14613d066281SPatrick Mooney de = vmmdev_devmem_find(sc, vdo.segid); 14624c87aefeSPatrick Mooney if (de != NULL) { 14634c87aefeSPatrick Mooney vdo.offset = de->vde_off; 14644c87aefeSPatrick Mooney if (ddi_copyout(&vdo, datap, sizeof (vdo), md) != 0) { 14654c87aefeSPatrick Mooney error = EFAULT; 14664c87aefeSPatrick Mooney } 14674c87aefeSPatrick Mooney } else { 14684c87aefeSPatrick Mooney error = ENOENT; 1469bf21cd93STycho Nightingale } 1470bf21cd93STycho Nightingale break; 1471bf21cd93STycho Nightingale } 14728779b448SDan Cross case VM_TRACK_DIRTY_PAGES: { 14738779b448SDan Cross const size_t max_track_region_len = 8 * PAGESIZE * 8 * PAGESIZE; 14748779b448SDan Cross struct vmm_dirty_tracker tracker; 14758779b448SDan Cross uint8_t *bitmap; 14768779b448SDan Cross size_t len; 14778779b448SDan Cross 14788779b448SDan Cross if (ddi_copyin(datap, &tracker, sizeof (tracker), md) != 0) { 14798779b448SDan Cross error = EFAULT; 14808779b448SDan Cross break; 14818779b448SDan Cross } 14828779b448SDan Cross if ((tracker.vdt_start_gpa & PAGEOFFSET) != 0) { 14838779b448SDan Cross error = EINVAL; 14848779b448SDan Cross break; 14858779b448SDan Cross } 14868779b448SDan Cross if (tracker.vdt_len == 0) { 14878779b448SDan Cross break; 14888779b448SDan Cross } 14898779b448SDan Cross if ((tracker.vdt_len & PAGEOFFSET) != 0) { 14908779b448SDan Cross error = EINVAL; 14918779b448SDan Cross break; 14928779b448SDan Cross } 14938779b448SDan Cross if (tracker.vdt_len > max_track_region_len) { 14948779b448SDan Cross error = EINVAL; 14958779b448SDan Cross break; 14968779b448SDan Cross } 14978779b448SDan Cross len = roundup(tracker.vdt_len / PAGESIZE, 8) / 8; 14988779b448SDan Cross bitmap = kmem_zalloc(len, KM_SLEEP); 14998779b448SDan Cross vm_track_dirty_pages(sc->vmm_vm, tracker.vdt_start_gpa, 15008779b448SDan Cross tracker.vdt_len, bitmap); 15018779b448SDan Cross if (ddi_copyout(bitmap, tracker.vdt_pfns, len, md) != 0) { 15028779b448SDan Cross error = EFAULT; 15038779b448SDan Cross } 15048779b448SDan Cross kmem_free(bitmap, len); 15058779b448SDan Cross 15068779b448SDan Cross break; 15078779b448SDan Cross } 15084c87aefeSPatrick Mooney case VM_WRLOCK_CYCLE: { 15094c87aefeSPatrick Mooney /* 15104c87aefeSPatrick Mooney * Present a test mechanism to acquire/release the write lock 15114c87aefeSPatrick Mooney * on the VM without any other effects. 15124c87aefeSPatrick Mooney */ 15134c87aefeSPatrick Mooney break; 15144c87aefeSPatrick Mooney } 1515f703164bSPatrick Mooney 15164c87aefeSPatrick Mooney default: 15174c87aefeSPatrick Mooney error = ENOTTY; 15184c87aefeSPatrick Mooney break; 15194c87aefeSPatrick Mooney } 15204c87aefeSPatrick Mooney 15214c87aefeSPatrick Mooney /* Release exclusion resources */ 15224c87aefeSPatrick Mooney switch (lock_type) { 15234c87aefeSPatrick Mooney case LOCK_NONE: 15244c87aefeSPatrick Mooney break; 15254c87aefeSPatrick Mooney case LOCK_VCPU: 15264c87aefeSPatrick Mooney vcpu_unlock_one(sc, vcpu); 15274c87aefeSPatrick Mooney break; 15284c87aefeSPatrick Mooney case LOCK_READ_HOLD: 15294c87aefeSPatrick Mooney vmm_read_unlock(sc); 15304c87aefeSPatrick Mooney break; 15314c87aefeSPatrick Mooney case LOCK_WRITE_HOLD: 15324c87aefeSPatrick Mooney vmm_write_unlock(sc); 15334c87aefeSPatrick Mooney break; 15344c87aefeSPatrick Mooney default: 15354c87aefeSPatrick Mooney panic("unexpected lock type"); 15364c87aefeSPatrick Mooney break; 15374c87aefeSPatrick Mooney } 15384c87aefeSPatrick Mooney 15394c87aefeSPatrick Mooney return (error); 15404c87aefeSPatrick Mooney } 15414c87aefeSPatrick Mooney 15424c87aefeSPatrick Mooney static vmm_softc_t * 15434c87aefeSPatrick Mooney vmm_lookup(const char *name) 15444c87aefeSPatrick Mooney { 15454c87aefeSPatrick Mooney list_t *vml = &vmm_list; 15464c87aefeSPatrick Mooney vmm_softc_t *sc; 15474c87aefeSPatrick Mooney 15484c87aefeSPatrick Mooney ASSERT(MUTEX_HELD(&vmm_mtx)); 15494c87aefeSPatrick Mooney 15504c87aefeSPatrick Mooney for (sc = list_head(vml); sc != NULL; sc = list_next(vml, sc)) { 15514c87aefeSPatrick Mooney if (strcmp(sc->vmm_name, name) == 0) { 15524c87aefeSPatrick Mooney break; 15534c87aefeSPatrick Mooney } 15544c87aefeSPatrick Mooney } 15554c87aefeSPatrick Mooney 15564c87aefeSPatrick Mooney return (sc); 15574c87aefeSPatrick Mooney } 15584c87aefeSPatrick Mooney 15598e3a263eSAndy Fiddaman /* 15608e3a263eSAndy Fiddaman * Acquire an HMA registration if not already held. 15618e3a263eSAndy Fiddaman */ 15628e3a263eSAndy Fiddaman static boolean_t 15638e3a263eSAndy Fiddaman vmm_hma_acquire(void) 15648e3a263eSAndy Fiddaman { 15658e3a263eSAndy Fiddaman ASSERT(MUTEX_NOT_HELD(&vmm_mtx)); 15668e3a263eSAndy Fiddaman 15678e3a263eSAndy Fiddaman mutex_enter(&vmmdev_mtx); 15688e3a263eSAndy Fiddaman 15698e3a263eSAndy Fiddaman if (vmmdev_hma_reg == NULL) { 15708e3a263eSAndy Fiddaman VERIFY3U(vmmdev_hma_ref, ==, 0); 15718e3a263eSAndy Fiddaman vmmdev_hma_reg = hma_register(vmmdev_hvm_name); 15728e3a263eSAndy Fiddaman if (vmmdev_hma_reg == NULL) { 15738e3a263eSAndy Fiddaman cmn_err(CE_WARN, "%s HMA registration failed.", 15748e3a263eSAndy Fiddaman vmmdev_hvm_name); 15758e3a263eSAndy Fiddaman mutex_exit(&vmmdev_mtx); 15768e3a263eSAndy Fiddaman return (B_FALSE); 15778e3a263eSAndy Fiddaman } 15788e3a263eSAndy Fiddaman } 15798e3a263eSAndy Fiddaman 15808e3a263eSAndy Fiddaman vmmdev_hma_ref++; 15818e3a263eSAndy Fiddaman 15828e3a263eSAndy Fiddaman mutex_exit(&vmmdev_mtx); 15838e3a263eSAndy Fiddaman 15848e3a263eSAndy Fiddaman return (B_TRUE); 15858e3a263eSAndy Fiddaman } 15868e3a263eSAndy Fiddaman 15878e3a263eSAndy Fiddaman /* 15888e3a263eSAndy Fiddaman * Release the HMA registration if held and there are no remaining VMs. 15898e3a263eSAndy Fiddaman */ 15908e3a263eSAndy Fiddaman static void 15918e3a263eSAndy Fiddaman vmm_hma_release(void) 15928e3a263eSAndy Fiddaman { 15938e3a263eSAndy Fiddaman ASSERT(MUTEX_NOT_HELD(&vmm_mtx)); 15948e3a263eSAndy Fiddaman 15958e3a263eSAndy Fiddaman mutex_enter(&vmmdev_mtx); 15968e3a263eSAndy Fiddaman 15978e3a263eSAndy Fiddaman VERIFY3U(vmmdev_hma_ref, !=, 0); 15988e3a263eSAndy Fiddaman 15998e3a263eSAndy Fiddaman vmmdev_hma_ref--; 16008e3a263eSAndy Fiddaman 16018e3a263eSAndy Fiddaman if (vmmdev_hma_ref == 0) { 16028e3a263eSAndy Fiddaman VERIFY(vmmdev_hma_reg != NULL); 16038e3a263eSAndy Fiddaman hma_unregister(vmmdev_hma_reg); 16048e3a263eSAndy Fiddaman vmmdev_hma_reg = NULL; 16058e3a263eSAndy Fiddaman } 16068e3a263eSAndy Fiddaman mutex_exit(&vmmdev_mtx); 16078e3a263eSAndy Fiddaman } 16088e3a263eSAndy Fiddaman 16094c87aefeSPatrick Mooney static int 1610b57f5d3eSPatrick Mooney vmmdev_do_vm_create(const struct vm_create_req *req, cred_t *cr) 16114c87aefeSPatrick Mooney { 16124c87aefeSPatrick Mooney vmm_softc_t *sc = NULL; 16134c87aefeSPatrick Mooney minor_t minor; 16144c87aefeSPatrick Mooney int error = ENOMEM; 1615b57f5d3eSPatrick Mooney size_t len; 1616b57f5d3eSPatrick Mooney const char *name = req->name; 16174c87aefeSPatrick Mooney 1618b57f5d3eSPatrick Mooney len = strnlen(name, VM_MAX_NAMELEN); 1619b57f5d3eSPatrick Mooney if (len == 0) { 1620b57f5d3eSPatrick Mooney return (EINVAL); 1621b57f5d3eSPatrick Mooney } 1622b57f5d3eSPatrick Mooney if (len >= VM_MAX_NAMELEN) { 1623b57f5d3eSPatrick Mooney return (ENAMETOOLONG); 1624b57f5d3eSPatrick Mooney } 1625b57f5d3eSPatrick Mooney if (strchr(name, '/') != NULL) { 16264c87aefeSPatrick Mooney return (EINVAL); 16274c87aefeSPatrick Mooney } 16284c87aefeSPatrick Mooney 16298e3a263eSAndy Fiddaman if (!vmm_hma_acquire()) 16308e3a263eSAndy Fiddaman return (ENXIO); 16318e3a263eSAndy Fiddaman 16324c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 16334c87aefeSPatrick Mooney 16348e3a263eSAndy Fiddaman /* Look for duplicate names */ 16354c87aefeSPatrick Mooney if (vmm_lookup(name) != NULL) { 16364c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 16378e3a263eSAndy Fiddaman vmm_hma_release(); 16384c87aefeSPatrick Mooney return (EEXIST); 16394c87aefeSPatrick Mooney } 16404c87aefeSPatrick Mooney 16414c87aefeSPatrick Mooney /* Allow only one instance per non-global zone. */ 16424c87aefeSPatrick Mooney if (!INGLOBALZONE(curproc)) { 16434c87aefeSPatrick Mooney for (sc = list_head(&vmm_list); sc != NULL; 16444c87aefeSPatrick Mooney sc = list_next(&vmm_list, sc)) { 16454c87aefeSPatrick Mooney if (sc->vmm_zone == curzone) { 16464c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 16478e3a263eSAndy Fiddaman vmm_hma_release(); 16484c87aefeSPatrick Mooney return (EINVAL); 16494c87aefeSPatrick Mooney } 16504c87aefeSPatrick Mooney } 16514c87aefeSPatrick Mooney } 16524c87aefeSPatrick Mooney 16534c87aefeSPatrick Mooney minor = id_alloc(vmm_minors); 16544c87aefeSPatrick Mooney if (ddi_soft_state_zalloc(vmm_statep, minor) != DDI_SUCCESS) { 16554c87aefeSPatrick Mooney goto fail; 16564c87aefeSPatrick Mooney } else if ((sc = ddi_get_soft_state(vmm_statep, minor)) == NULL) { 16574c87aefeSPatrick Mooney ddi_soft_state_free(vmm_statep, minor); 16584c87aefeSPatrick Mooney goto fail; 16594c87aefeSPatrick Mooney } else if (ddi_create_minor_node(vmmdev_dip, name, S_IFCHR, minor, 16604c87aefeSPatrick Mooney DDI_PSEUDO, 0) != DDI_SUCCESS) { 16614c87aefeSPatrick Mooney goto fail; 16624c87aefeSPatrick Mooney } 16634c87aefeSPatrick Mooney 166459460b49SPatrick Mooney if (vmm_kstat_alloc(sc, minor, cr) != 0) { 166559460b49SPatrick Mooney goto fail; 166659460b49SPatrick Mooney } 166759460b49SPatrick Mooney 1668b57f5d3eSPatrick Mooney error = vm_create(req->name, req->flags, &sc->vmm_vm); 16694c87aefeSPatrick Mooney if (error == 0) { 16704c87aefeSPatrick Mooney /* Complete VM intialization and report success. */ 16714c87aefeSPatrick Mooney (void) strlcpy(sc->vmm_name, name, sizeof (sc->vmm_name)); 16724c87aefeSPatrick Mooney sc->vmm_minor = minor; 16734c87aefeSPatrick Mooney list_create(&sc->vmm_devmem_list, sizeof (vmm_devmem_entry_t), 16744c87aefeSPatrick Mooney offsetof(vmm_devmem_entry_t, vde_node)); 16754c87aefeSPatrick Mooney 16764c87aefeSPatrick Mooney list_create(&sc->vmm_holds, sizeof (vmm_hold_t), 16774c87aefeSPatrick Mooney offsetof(vmm_hold_t, vmh_node)); 16784c87aefeSPatrick Mooney cv_init(&sc->vmm_cv, NULL, CV_DEFAULT, NULL); 16794c87aefeSPatrick Mooney 16804c87aefeSPatrick Mooney mutex_init(&sc->vmm_lease_lock, NULL, MUTEX_DEFAULT, NULL); 16814c87aefeSPatrick Mooney list_create(&sc->vmm_lease_list, sizeof (vmm_lease_t), 16824c87aefeSPatrick Mooney offsetof(vmm_lease_t, vml_node)); 16834c87aefeSPatrick Mooney cv_init(&sc->vmm_lease_cv, NULL, CV_DEFAULT, NULL); 16844c87aefeSPatrick Mooney rw_init(&sc->vmm_rwlock, NULL, RW_DEFAULT, NULL); 16854c87aefeSPatrick Mooney 16864c87aefeSPatrick Mooney sc->vmm_zone = crgetzone(cr); 16874c87aefeSPatrick Mooney zone_hold(sc->vmm_zone); 16884c87aefeSPatrick Mooney vmm_zsd_add_vm(sc); 168959460b49SPatrick Mooney vmm_kstat_init(sc); 16904c87aefeSPatrick Mooney 16914c87aefeSPatrick Mooney list_insert_tail(&vmm_list, sc); 16924c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 16934c87aefeSPatrick Mooney return (0); 16944c87aefeSPatrick Mooney } 16954c87aefeSPatrick Mooney 169659460b49SPatrick Mooney vmm_kstat_fini(sc); 16974c87aefeSPatrick Mooney ddi_remove_minor_node(vmmdev_dip, name); 16984c87aefeSPatrick Mooney fail: 16994c87aefeSPatrick Mooney id_free(vmm_minors, minor); 17004c87aefeSPatrick Mooney if (sc != NULL) { 17014c87aefeSPatrick Mooney ddi_soft_state_free(vmm_statep, minor); 17024c87aefeSPatrick Mooney } 17034c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 17048e3a263eSAndy Fiddaman vmm_hma_release(); 17054c87aefeSPatrick Mooney 17064c87aefeSPatrick Mooney return (error); 17074c87aefeSPatrick Mooney } 17084c87aefeSPatrick Mooney 17094c87aefeSPatrick Mooney /* 17104c87aefeSPatrick Mooney * Bhyve 'Driver' Interface 17114c87aefeSPatrick Mooney * 17124c87aefeSPatrick Mooney * While many devices are emulated in the bhyve userspace process, there are 17134c87aefeSPatrick Mooney * others with performance constraints which require that they run mostly or 17144c87aefeSPatrick Mooney * entirely in-kernel. For those not integrated directly into bhyve, an API is 17154c87aefeSPatrick Mooney * needed so they can query/manipulate the portions of VM state needed to 17164c87aefeSPatrick Mooney * fulfill their purpose. 17174c87aefeSPatrick Mooney * 17184c87aefeSPatrick Mooney * This includes: 17194c87aefeSPatrick Mooney * - Translating guest-physical addresses to host-virtual pointers 17204c87aefeSPatrick Mooney * - Injecting MSIs 17214c87aefeSPatrick Mooney * - Hooking IO port addresses 17224c87aefeSPatrick Mooney * 17234c87aefeSPatrick Mooney * The vmm_drv interface exists to provide that functionality to its consumers. 17244c87aefeSPatrick Mooney * (At this time, 'viona' is the only user) 17254c87aefeSPatrick Mooney */ 17264c87aefeSPatrick Mooney int 17274c87aefeSPatrick Mooney vmm_drv_hold(file_t *fp, cred_t *cr, vmm_hold_t **holdp) 17284c87aefeSPatrick Mooney { 17294c87aefeSPatrick Mooney vnode_t *vp = fp->f_vnode; 17304c87aefeSPatrick Mooney const dev_t dev = vp->v_rdev; 17314c87aefeSPatrick Mooney vmm_softc_t *sc; 17324c87aefeSPatrick Mooney vmm_hold_t *hold; 17334c87aefeSPatrick Mooney int err = 0; 17344c87aefeSPatrick Mooney 17354c87aefeSPatrick Mooney if (vp->v_type != VCHR) { 17364c87aefeSPatrick Mooney return (ENXIO); 17374c87aefeSPatrick Mooney } 17384c87aefeSPatrick Mooney const major_t major = getmajor(dev); 17394c87aefeSPatrick Mooney const minor_t minor = getminor(dev); 17404c87aefeSPatrick Mooney 17414c87aefeSPatrick Mooney mutex_enter(&vmmdev_mtx); 17424c87aefeSPatrick Mooney if (vmmdev_dip == NULL || major != ddi_driver_major(vmmdev_dip)) { 17434c87aefeSPatrick Mooney mutex_exit(&vmmdev_mtx); 17444c87aefeSPatrick Mooney return (ENOENT); 1745bf21cd93STycho Nightingale } 17464c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 17474c87aefeSPatrick Mooney mutex_exit(&vmmdev_mtx); 1748bf21cd93STycho Nightingale 17494c87aefeSPatrick Mooney if ((sc = ddi_get_soft_state(vmm_statep, minor)) == NULL) { 17504c87aefeSPatrick Mooney err = ENOENT; 17514c87aefeSPatrick Mooney goto out; 1752bf21cd93STycho Nightingale } 17534c87aefeSPatrick Mooney /* XXXJOY: check cred permissions against instance */ 1754bf21cd93STycho Nightingale 17554c87aefeSPatrick Mooney if ((sc->vmm_flags & (VMM_CLEANUP|VMM_PURGED|VMM_DESTROY)) != 0) { 17564c87aefeSPatrick Mooney err = EBUSY; 17574c87aefeSPatrick Mooney goto out; 17584c87aefeSPatrick Mooney } 17594c87aefeSPatrick Mooney 17604c87aefeSPatrick Mooney hold = kmem_zalloc(sizeof (*hold), KM_SLEEP); 17614c87aefeSPatrick Mooney hold->vmh_sc = sc; 17624c87aefeSPatrick Mooney hold->vmh_release_req = B_FALSE; 17634c87aefeSPatrick Mooney 17644c87aefeSPatrick Mooney list_insert_tail(&sc->vmm_holds, hold); 17654c87aefeSPatrick Mooney sc->vmm_flags |= VMM_HELD; 17664c87aefeSPatrick Mooney *holdp = hold; 17674c87aefeSPatrick Mooney 17684c87aefeSPatrick Mooney out: 17694c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 17704c87aefeSPatrick Mooney return (err); 1771bf21cd93STycho Nightingale } 1772bf21cd93STycho Nightingale 17734c87aefeSPatrick Mooney void 17744c87aefeSPatrick Mooney vmm_drv_rele(vmm_hold_t *hold) 1775bf21cd93STycho Nightingale { 17764c87aefeSPatrick Mooney vmm_softc_t *sc; 17774c87aefeSPatrick Mooney 17784c87aefeSPatrick Mooney ASSERT(hold != NULL); 17794c87aefeSPatrick Mooney ASSERT(hold->vmh_sc != NULL); 17804c87aefeSPatrick Mooney VERIFY(hold->vmh_ioport_hook_cnt == 0); 17814c87aefeSPatrick Mooney 17824c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 17834c87aefeSPatrick Mooney sc = hold->vmh_sc; 17844c87aefeSPatrick Mooney list_remove(&sc->vmm_holds, hold); 17854c87aefeSPatrick Mooney if (list_is_empty(&sc->vmm_holds)) { 17864c87aefeSPatrick Mooney sc->vmm_flags &= ~VMM_HELD; 17874c87aefeSPatrick Mooney cv_broadcast(&sc->vmm_cv); 1788bf21cd93STycho Nightingale } 17894c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 17904c87aefeSPatrick Mooney kmem_free(hold, sizeof (*hold)); 17914c87aefeSPatrick Mooney } 17924c87aefeSPatrick Mooney 17934c87aefeSPatrick Mooney boolean_t 17944c87aefeSPatrick Mooney vmm_drv_release_reqd(vmm_hold_t *hold) 17954c87aefeSPatrick Mooney { 17964c87aefeSPatrick Mooney ASSERT(hold != NULL); 1797bf21cd93STycho Nightingale 17984c87aefeSPatrick Mooney return (hold->vmh_release_req); 1799bf21cd93STycho Nightingale } 1800bf21cd93STycho Nightingale 18014c87aefeSPatrick Mooney vmm_lease_t * 18024c87aefeSPatrick Mooney vmm_drv_lease_sign(vmm_hold_t *hold, boolean_t (*expiref)(void *), void *arg) 1803bf21cd93STycho Nightingale { 18044c87aefeSPatrick Mooney vmm_softc_t *sc = hold->vmh_sc; 18054c87aefeSPatrick Mooney vmm_lease_t *lease; 1806bf21cd93STycho Nightingale 18074c87aefeSPatrick Mooney ASSERT3P(expiref, !=, NULL); 1808bf21cd93STycho Nightingale 18094c87aefeSPatrick Mooney if (hold->vmh_release_req) { 18104c87aefeSPatrick Mooney return (NULL); 1811bf21cd93STycho Nightingale } 1812bf21cd93STycho Nightingale 18134c87aefeSPatrick Mooney lease = kmem_alloc(sizeof (*lease), KM_SLEEP); 18144c87aefeSPatrick Mooney list_link_init(&lease->vml_node); 18154c87aefeSPatrick Mooney lease->vml_expire_func = expiref; 18164c87aefeSPatrick Mooney lease->vml_expire_arg = arg; 18174c87aefeSPatrick Mooney lease->vml_expired = B_FALSE; 18186703a0e8SPatrick Mooney lease->vml_break_deferred = B_FALSE; 18194c87aefeSPatrick Mooney lease->vml_hold = hold; 18204c87aefeSPatrick Mooney /* cache the VM pointer for one less pointer chase */ 18214c87aefeSPatrick Mooney lease->vml_vm = sc->vmm_vm; 18220153d828SPatrick Mooney lease->vml_vmclient = vmspace_client_alloc(vm_get_vmspace(sc->vmm_vm)); 18234c87aefeSPatrick Mooney 18244c87aefeSPatrick Mooney mutex_enter(&sc->vmm_lease_lock); 18254c87aefeSPatrick Mooney while (sc->vmm_lease_blocker != 0) { 18264c87aefeSPatrick Mooney cv_wait(&sc->vmm_lease_cv, &sc->vmm_lease_lock); 1827bf21cd93STycho Nightingale } 18284c87aefeSPatrick Mooney list_insert_tail(&sc->vmm_lease_list, lease); 18294c87aefeSPatrick Mooney vmm_read_lock(sc); 18304c87aefeSPatrick Mooney mutex_exit(&sc->vmm_lease_lock); 1831bf21cd93STycho Nightingale 18324c87aefeSPatrick Mooney return (lease); 18334c87aefeSPatrick Mooney } 1834bf21cd93STycho Nightingale 18354c87aefeSPatrick Mooney static void 18364c87aefeSPatrick Mooney vmm_lease_break_locked(vmm_softc_t *sc, vmm_lease_t *lease) 18374c87aefeSPatrick Mooney { 18384c87aefeSPatrick Mooney ASSERT(MUTEX_HELD(&sc->vmm_lease_lock)); 1839bf21cd93STycho Nightingale 18404c87aefeSPatrick Mooney list_remove(&sc->vmm_lease_list, lease); 18414c87aefeSPatrick Mooney vmm_read_unlock(sc); 18420153d828SPatrick Mooney vmc_destroy(lease->vml_vmclient); 18434c87aefeSPatrick Mooney kmem_free(lease, sizeof (*lease)); 18444c87aefeSPatrick Mooney } 18454c87aefeSPatrick Mooney 18466703a0e8SPatrick Mooney static void 18476703a0e8SPatrick Mooney vmm_lease_block(vmm_softc_t *sc) 18486703a0e8SPatrick Mooney { 18496703a0e8SPatrick Mooney mutex_enter(&sc->vmm_lease_lock); 18506703a0e8SPatrick Mooney VERIFY3U(sc->vmm_lease_blocker, !=, UINT_MAX); 18516703a0e8SPatrick Mooney sc->vmm_lease_blocker++; 18526703a0e8SPatrick Mooney if (sc->vmm_lease_blocker == 1) { 18536703a0e8SPatrick Mooney list_t *list = &sc->vmm_lease_list; 18546703a0e8SPatrick Mooney vmm_lease_t *lease = list_head(list); 18556703a0e8SPatrick Mooney 18566703a0e8SPatrick Mooney while (lease != NULL) { 18576703a0e8SPatrick Mooney void *arg = lease->vml_expire_arg; 18586703a0e8SPatrick Mooney boolean_t (*expiref)(void *) = lease->vml_expire_func; 18596703a0e8SPatrick Mooney boolean_t sync_break = B_FALSE; 18606703a0e8SPatrick Mooney 18616703a0e8SPatrick Mooney /* 18626703a0e8SPatrick Mooney * Since the lease expiration notification may 18636703a0e8SPatrick Mooney * need to take locks which would deadlock with 18646703a0e8SPatrick Mooney * vmm_lease_lock, drop it across the call. 18656703a0e8SPatrick Mooney * 18666703a0e8SPatrick Mooney * We are the only one allowed to manipulate 18676703a0e8SPatrick Mooney * vmm_lease_list right now, so it is safe to 18686703a0e8SPatrick Mooney * continue iterating through it after 18696703a0e8SPatrick Mooney * reacquiring the lock. 18706703a0e8SPatrick Mooney */ 18716703a0e8SPatrick Mooney lease->vml_expired = B_TRUE; 18726703a0e8SPatrick Mooney mutex_exit(&sc->vmm_lease_lock); 18736703a0e8SPatrick Mooney sync_break = expiref(arg); 18746703a0e8SPatrick Mooney mutex_enter(&sc->vmm_lease_lock); 18756703a0e8SPatrick Mooney 18766703a0e8SPatrick Mooney if (sync_break) { 18776703a0e8SPatrick Mooney vmm_lease_t *next; 18786703a0e8SPatrick Mooney 18796703a0e8SPatrick Mooney /* 18806703a0e8SPatrick Mooney * These leases which are synchronously broken 18816703a0e8SPatrick Mooney * result in vmm_read_unlock() calls from a 18826703a0e8SPatrick Mooney * different thread than the corresponding 18836703a0e8SPatrick Mooney * vmm_read_lock(). This is acceptable, given 18846703a0e8SPatrick Mooney * that the rwlock underpinning the whole 18856703a0e8SPatrick Mooney * mechanism tolerates the behavior. This 18866703a0e8SPatrick Mooney * flexibility is _only_ afforded to VM read 18876703a0e8SPatrick Mooney * lock (RW_READER) holders. 18886703a0e8SPatrick Mooney */ 18896703a0e8SPatrick Mooney next = list_next(list, lease); 18906703a0e8SPatrick Mooney vmm_lease_break_locked(sc, lease); 18916703a0e8SPatrick Mooney lease = next; 18926703a0e8SPatrick Mooney } else { 18936703a0e8SPatrick Mooney lease = list_next(list, lease); 18946703a0e8SPatrick Mooney } 18956703a0e8SPatrick Mooney } 18966703a0e8SPatrick Mooney 18976703a0e8SPatrick Mooney /* Process leases which were not broken synchronously. */ 18986703a0e8SPatrick Mooney while (!list_is_empty(list)) { 18996703a0e8SPatrick Mooney /* 19006703a0e8SPatrick Mooney * Although the nested loops are quadratic, the number 19016703a0e8SPatrick Mooney * of leases is small. 19026703a0e8SPatrick Mooney */ 19036703a0e8SPatrick Mooney lease = list_head(list); 19046703a0e8SPatrick Mooney while (lease != NULL) { 19056703a0e8SPatrick Mooney vmm_lease_t *next = list_next(list, lease); 19066703a0e8SPatrick Mooney if (lease->vml_break_deferred) { 19076703a0e8SPatrick Mooney vmm_lease_break_locked(sc, lease); 19086703a0e8SPatrick Mooney } 19096703a0e8SPatrick Mooney lease = next; 19106703a0e8SPatrick Mooney } 19116703a0e8SPatrick Mooney if (list_is_empty(list)) { 19126703a0e8SPatrick Mooney break; 19136703a0e8SPatrick Mooney } 19146703a0e8SPatrick Mooney cv_wait(&sc->vmm_lease_cv, &sc->vmm_lease_lock); 19156703a0e8SPatrick Mooney } 19166703a0e8SPatrick Mooney /* Wake anyone else waiting for the lease list to be empty */ 19176703a0e8SPatrick Mooney cv_broadcast(&sc->vmm_lease_cv); 19186703a0e8SPatrick Mooney } else { 19196703a0e8SPatrick Mooney list_t *list = &sc->vmm_lease_list; 19206703a0e8SPatrick Mooney 19216703a0e8SPatrick Mooney /* 19226703a0e8SPatrick Mooney * Some other thread beat us to the duty of lease cleanup. 19236703a0e8SPatrick Mooney * Wait until that is complete. 19246703a0e8SPatrick Mooney */ 19256703a0e8SPatrick Mooney while (!list_is_empty(list)) { 19266703a0e8SPatrick Mooney cv_wait(&sc->vmm_lease_cv, &sc->vmm_lease_lock); 19276703a0e8SPatrick Mooney } 19286703a0e8SPatrick Mooney } 19296703a0e8SPatrick Mooney mutex_exit(&sc->vmm_lease_lock); 19306703a0e8SPatrick Mooney } 19316703a0e8SPatrick Mooney 19326703a0e8SPatrick Mooney static void 19336703a0e8SPatrick Mooney vmm_lease_unblock(vmm_softc_t *sc) 19346703a0e8SPatrick Mooney { 19356703a0e8SPatrick Mooney mutex_enter(&sc->vmm_lease_lock); 19366703a0e8SPatrick Mooney VERIFY3U(sc->vmm_lease_blocker, !=, 0); 19376703a0e8SPatrick Mooney sc->vmm_lease_blocker--; 19386703a0e8SPatrick Mooney if (sc->vmm_lease_blocker == 0) { 19396703a0e8SPatrick Mooney cv_broadcast(&sc->vmm_lease_cv); 19406703a0e8SPatrick Mooney } 19416703a0e8SPatrick Mooney mutex_exit(&sc->vmm_lease_lock); 19426703a0e8SPatrick Mooney } 19436703a0e8SPatrick Mooney 19444c87aefeSPatrick Mooney void 19454c87aefeSPatrick Mooney vmm_drv_lease_break(vmm_hold_t *hold, vmm_lease_t *lease) 19464c87aefeSPatrick Mooney { 19474c87aefeSPatrick Mooney vmm_softc_t *sc = hold->vmh_sc; 19484c87aefeSPatrick Mooney 19494c87aefeSPatrick Mooney VERIFY3P(hold, ==, lease->vml_hold); 19506703a0e8SPatrick Mooney VERIFY(!lease->vml_break_deferred); 19514c87aefeSPatrick Mooney 19524c87aefeSPatrick Mooney mutex_enter(&sc->vmm_lease_lock); 19536703a0e8SPatrick Mooney if (sc->vmm_lease_blocker == 0) { 19546703a0e8SPatrick Mooney vmm_lease_break_locked(sc, lease); 19556703a0e8SPatrick Mooney } else { 19566703a0e8SPatrick Mooney /* 19576703a0e8SPatrick Mooney * Defer the lease-breaking to whichever thread is currently 19586703a0e8SPatrick Mooney * cleaning up all leases as part of a vmm_lease_block() call. 19596703a0e8SPatrick Mooney */ 19606703a0e8SPatrick Mooney lease->vml_break_deferred = B_TRUE; 19616703a0e8SPatrick Mooney cv_broadcast(&sc->vmm_lease_cv); 19626703a0e8SPatrick Mooney } 19634c87aefeSPatrick Mooney mutex_exit(&sc->vmm_lease_lock); 19644c87aefeSPatrick Mooney } 19654c87aefeSPatrick Mooney 19664c87aefeSPatrick Mooney boolean_t 19674c87aefeSPatrick Mooney vmm_drv_lease_expired(vmm_lease_t *lease) 19684c87aefeSPatrick Mooney { 19694c87aefeSPatrick Mooney return (lease->vml_expired); 19704c87aefeSPatrick Mooney } 19714c87aefeSPatrick Mooney 1972db9aa506SPatrick Mooney vmm_page_t * 1973db9aa506SPatrick Mooney vmm_drv_page_hold(vmm_lease_t *lease, uintptr_t gpa, int prot) 19744c87aefeSPatrick Mooney { 19754c87aefeSPatrick Mooney ASSERT(lease != NULL); 19760153d828SPatrick Mooney ASSERT0(gpa & PAGEOFFSET); 19770153d828SPatrick Mooney 1978db9aa506SPatrick Mooney return ((vmm_page_t *)vmc_hold(lease->vml_vmclient, gpa, prot)); 1979db9aa506SPatrick Mooney } 1980db9aa506SPatrick Mooney 1981db9aa506SPatrick Mooney void 1982db9aa506SPatrick Mooney vmm_drv_page_release(vmm_page_t *vmmp) 1983db9aa506SPatrick Mooney { 1984e0994bd2SPatrick Mooney (void) vmp_release((vm_page_t *)vmmp); 1985db9aa506SPatrick Mooney } 1986db9aa506SPatrick Mooney 1987db9aa506SPatrick Mooney void 1988db9aa506SPatrick Mooney vmm_drv_page_release_chain(vmm_page_t *vmmp) 1989db9aa506SPatrick Mooney { 1990e0994bd2SPatrick Mooney (void) vmp_release_chain((vm_page_t *)vmmp); 1991db9aa506SPatrick Mooney } 1992db9aa506SPatrick Mooney 1993db9aa506SPatrick Mooney const void * 1994db9aa506SPatrick Mooney vmm_drv_page_readable(const vmm_page_t *vmmp) 1995db9aa506SPatrick Mooney { 1996db9aa506SPatrick Mooney return (vmp_get_readable((const vm_page_t *)vmmp)); 1997db9aa506SPatrick Mooney } 19984c87aefeSPatrick Mooney 1999db9aa506SPatrick Mooney void * 2000db9aa506SPatrick Mooney vmm_drv_page_writable(const vmm_page_t *vmmp) 2001db9aa506SPatrick Mooney { 2002db9aa506SPatrick Mooney return (vmp_get_writable((const vm_page_t *)vmmp)); 2003db9aa506SPatrick Mooney } 2004db9aa506SPatrick Mooney 2005db9aa506SPatrick Mooney void 2006db9aa506SPatrick Mooney vmm_drv_page_chain(vmm_page_t *vmmp, vmm_page_t *to_chain) 2007db9aa506SPatrick Mooney { 2008db9aa506SPatrick Mooney vmp_chain((vm_page_t *)vmmp, (vm_page_t *)to_chain); 2009db9aa506SPatrick Mooney } 2010db9aa506SPatrick Mooney 2011db9aa506SPatrick Mooney vmm_page_t * 2012db9aa506SPatrick Mooney vmm_drv_page_next(const vmm_page_t *vmmp) 2013db9aa506SPatrick Mooney { 2014db9aa506SPatrick Mooney return ((vmm_page_t *)vmp_next((vm_page_t *)vmmp)); 20154c87aefeSPatrick Mooney } 20164c87aefeSPatrick Mooney 20174c87aefeSPatrick Mooney int 20184c87aefeSPatrick Mooney vmm_drv_msi(vmm_lease_t *lease, uint64_t addr, uint64_t msg) 20194c87aefeSPatrick Mooney { 20204c87aefeSPatrick Mooney ASSERT(lease != NULL); 20214c87aefeSPatrick Mooney 20224c87aefeSPatrick Mooney return (lapic_intr_msi(lease->vml_vm, addr, msg)); 20234c87aefeSPatrick Mooney } 20244c87aefeSPatrick Mooney 20254c87aefeSPatrick Mooney int 20260e1453c3SPatrick Mooney vmm_drv_ioport_hook(vmm_hold_t *hold, uint16_t ioport, vmm_drv_iop_cb_t func, 20270e1453c3SPatrick Mooney void *arg, void **cookie) 20284c87aefeSPatrick Mooney { 20294c87aefeSPatrick Mooney vmm_softc_t *sc; 20304c87aefeSPatrick Mooney int err; 20314c87aefeSPatrick Mooney 20324c87aefeSPatrick Mooney ASSERT(hold != NULL); 20334c87aefeSPatrick Mooney ASSERT(cookie != NULL); 20344c87aefeSPatrick Mooney 20354c87aefeSPatrick Mooney sc = hold->vmh_sc; 20364c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 20374c87aefeSPatrick Mooney /* Confirm that hook installation is not blocked */ 20384c87aefeSPatrick Mooney if ((sc->vmm_flags & VMM_BLOCK_HOOK) != 0) { 20394c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 20404c87aefeSPatrick Mooney return (EBUSY); 20414c87aefeSPatrick Mooney } 20424c87aefeSPatrick Mooney /* 20434c87aefeSPatrick Mooney * Optimistically record an installed hook which will prevent a block 20444c87aefeSPatrick Mooney * from being asserted while the mutex is dropped. 20454c87aefeSPatrick Mooney */ 20464c87aefeSPatrick Mooney hold->vmh_ioport_hook_cnt++; 20474c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 20484c87aefeSPatrick Mooney 20494c87aefeSPatrick Mooney vmm_write_lock(sc); 20500e1453c3SPatrick Mooney err = vm_ioport_hook(sc->vmm_vm, ioport, (ioport_handler_t)func, 20510e1453c3SPatrick Mooney arg, cookie); 20524c87aefeSPatrick Mooney vmm_write_unlock(sc); 20534c87aefeSPatrick Mooney 20544c87aefeSPatrick Mooney if (err != 0) { 20554c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 20564c87aefeSPatrick Mooney /* Walk back optimism about the hook installation */ 20574c87aefeSPatrick Mooney hold->vmh_ioport_hook_cnt--; 20584c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 2059bf21cd93STycho Nightingale } 20604c87aefeSPatrick Mooney return (err); 20614c87aefeSPatrick Mooney } 2062bf21cd93STycho Nightingale 20634c87aefeSPatrick Mooney void 20644c87aefeSPatrick Mooney vmm_drv_ioport_unhook(vmm_hold_t *hold, void **cookie) 20654c87aefeSPatrick Mooney { 20664c87aefeSPatrick Mooney vmm_softc_t *sc; 2067bf21cd93STycho Nightingale 20684c87aefeSPatrick Mooney ASSERT(hold != NULL); 20694c87aefeSPatrick Mooney ASSERT(cookie != NULL); 20704c87aefeSPatrick Mooney ASSERT(hold->vmh_ioport_hook_cnt != 0); 20714c87aefeSPatrick Mooney 20724c87aefeSPatrick Mooney sc = hold->vmh_sc; 20734c87aefeSPatrick Mooney vmm_write_lock(sc); 20744c87aefeSPatrick Mooney vm_ioport_unhook(sc->vmm_vm, cookie); 20754c87aefeSPatrick Mooney vmm_write_unlock(sc); 20764c87aefeSPatrick Mooney 20774c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 20784c87aefeSPatrick Mooney hold->vmh_ioport_hook_cnt--; 20794c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 2080bf21cd93STycho Nightingale } 2081bf21cd93STycho Nightingale 20824c87aefeSPatrick Mooney static int 20834c87aefeSPatrick Mooney vmm_drv_purge(vmm_softc_t *sc) 2084bf21cd93STycho Nightingale { 20854c87aefeSPatrick Mooney ASSERT(MUTEX_HELD(&vmm_mtx)); 2086bf21cd93STycho Nightingale 20874c87aefeSPatrick Mooney if ((sc->vmm_flags & VMM_HELD) != 0) { 20884c87aefeSPatrick Mooney vmm_hold_t *hold; 20894c87aefeSPatrick Mooney 20904c87aefeSPatrick Mooney sc->vmm_flags |= VMM_CLEANUP; 20914c87aefeSPatrick Mooney for (hold = list_head(&sc->vmm_holds); hold != NULL; 20924c87aefeSPatrick Mooney hold = list_next(&sc->vmm_holds, hold)) { 20934c87aefeSPatrick Mooney hold->vmh_release_req = B_TRUE; 2094bf21cd93STycho Nightingale } 20954c87aefeSPatrick Mooney while ((sc->vmm_flags & VMM_HELD) != 0) { 20964c87aefeSPatrick Mooney if (cv_wait_sig(&sc->vmm_cv, &vmm_mtx) <= 0) { 20974c87aefeSPatrick Mooney return (EINTR); 20984c87aefeSPatrick Mooney } 20994c87aefeSPatrick Mooney } 21004c87aefeSPatrick Mooney sc->vmm_flags &= ~VMM_CLEANUP; 2101bf21cd93STycho Nightingale } 2102bf21cd93STycho Nightingale 21034c87aefeSPatrick Mooney VERIFY(list_is_empty(&sc->vmm_holds)); 21044c87aefeSPatrick Mooney sc->vmm_flags |= VMM_PURGED; 21054c87aefeSPatrick Mooney return (0); 2106bf21cd93STycho Nightingale } 2107bf21cd93STycho Nightingale 21084c87aefeSPatrick Mooney static int 21094c87aefeSPatrick Mooney vmm_drv_block_hook(vmm_softc_t *sc, boolean_t enable_block) 2110bf21cd93STycho Nightingale { 21114c87aefeSPatrick Mooney int err = 0; 2112bf21cd93STycho Nightingale 21134c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 21144c87aefeSPatrick Mooney if (!enable_block) { 21154c87aefeSPatrick Mooney VERIFY((sc->vmm_flags & VMM_BLOCK_HOOK) != 0); 2116bf21cd93STycho Nightingale 21174c87aefeSPatrick Mooney sc->vmm_flags &= ~VMM_BLOCK_HOOK; 21184c87aefeSPatrick Mooney goto done; 2119bf21cd93STycho Nightingale } 2120bf21cd93STycho Nightingale 21214c87aefeSPatrick Mooney /* If any holds have hooks installed, the block is a failure */ 21224c87aefeSPatrick Mooney if (!list_is_empty(&sc->vmm_holds)) { 21234c87aefeSPatrick Mooney vmm_hold_t *hold; 21244c87aefeSPatrick Mooney 21254c87aefeSPatrick Mooney for (hold = list_head(&sc->vmm_holds); hold != NULL; 21264c87aefeSPatrick Mooney hold = list_next(&sc->vmm_holds, hold)) { 21274c87aefeSPatrick Mooney if (hold->vmh_ioport_hook_cnt != 0) { 21284c87aefeSPatrick Mooney err = EBUSY; 21294c87aefeSPatrick Mooney goto done; 21304c87aefeSPatrick Mooney } 21314c87aefeSPatrick Mooney } 21324c87aefeSPatrick Mooney } 21334c87aefeSPatrick Mooney sc->vmm_flags |= VMM_BLOCK_HOOK; 2134bf21cd93STycho Nightingale 21354c87aefeSPatrick Mooney done: 21364c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 21374c87aefeSPatrick Mooney return (err); 2138bf21cd93STycho Nightingale } 2139bf21cd93STycho Nightingale 21404c87aefeSPatrick Mooney static int 21418e3a263eSAndy Fiddaman vmm_do_vm_destroy_locked(vmm_softc_t *sc, boolean_t clean_zsd, 21428e3a263eSAndy Fiddaman boolean_t *hma_release) 2143bf21cd93STycho Nightingale { 21444c87aefeSPatrick Mooney dev_info_t *pdip = ddi_get_parent(vmmdev_dip); 21454c87aefeSPatrick Mooney minor_t minor; 2146bf21cd93STycho Nightingale 21474c87aefeSPatrick Mooney ASSERT(MUTEX_HELD(&vmm_mtx)); 2148bf21cd93STycho Nightingale 21498e3a263eSAndy Fiddaman *hma_release = B_FALSE; 21508e3a263eSAndy Fiddaman 21514c87aefeSPatrick Mooney if (vmm_drv_purge(sc) != 0) { 21524c87aefeSPatrick Mooney return (EINTR); 2153bf21cd93STycho Nightingale } 2154bf21cd93STycho Nightingale 2155b4100263SAndy Fiddaman if (clean_zsd) { 2156b4100263SAndy Fiddaman vmm_zsd_rem_vm(sc); 2157b4100263SAndy Fiddaman } 2158b4100263SAndy Fiddaman 21594c87aefeSPatrick Mooney /* Clean up devmem entries */ 21604c87aefeSPatrick Mooney vmmdev_devmem_purge(sc); 2161bf21cd93STycho Nightingale 21624c87aefeSPatrick Mooney list_remove(&vmm_list, sc); 21634c87aefeSPatrick Mooney ddi_remove_minor_node(vmmdev_dip, sc->vmm_name); 21644c87aefeSPatrick Mooney minor = sc->vmm_minor; 21654c87aefeSPatrick Mooney zone_rele(sc->vmm_zone); 21664c87aefeSPatrick Mooney if (sc->vmm_is_open) { 21674c87aefeSPatrick Mooney list_insert_tail(&vmm_destroy_list, sc); 21684c87aefeSPatrick Mooney sc->vmm_flags |= VMM_DESTROY; 21694c87aefeSPatrick Mooney } else { 217059460b49SPatrick Mooney vmm_kstat_fini(sc); 21716884664dSPatrick Mooney vm_destroy(sc->vmm_vm); 21724c87aefeSPatrick Mooney ddi_soft_state_free(vmm_statep, minor); 21734c87aefeSPatrick Mooney id_free(vmm_minors, minor); 21748e3a263eSAndy Fiddaman *hma_release = B_TRUE; 21754c87aefeSPatrick Mooney } 21764c87aefeSPatrick Mooney (void) devfs_clean(pdip, NULL, DV_CLEAN_FORCE); 2177bf21cd93STycho Nightingale 2178bf21cd93STycho Nightingale return (0); 2179bf21cd93STycho Nightingale } 2180bf21cd93STycho Nightingale 2181bf21cd93STycho Nightingale int 21824c87aefeSPatrick Mooney vmm_do_vm_destroy(vmm_softc_t *sc, boolean_t clean_zsd) 2183bf21cd93STycho Nightingale { 21848e3a263eSAndy Fiddaman boolean_t hma_release = B_FALSE; 21854c87aefeSPatrick Mooney int err; 2186bf21cd93STycho Nightingale 21874c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 21888e3a263eSAndy Fiddaman err = vmm_do_vm_destroy_locked(sc, clean_zsd, &hma_release); 21894c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 2190bf21cd93STycho Nightingale 21918e3a263eSAndy Fiddaman if (hma_release) 21928e3a263eSAndy Fiddaman vmm_hma_release(); 21938e3a263eSAndy Fiddaman 21944c87aefeSPatrick Mooney return (err); 21954c87aefeSPatrick Mooney } 2196bf21cd93STycho Nightingale 21974c87aefeSPatrick Mooney /* ARGSUSED */ 21984c87aefeSPatrick Mooney static int 2199b57f5d3eSPatrick Mooney vmmdev_do_vm_destroy(const struct vm_destroy_req *req, cred_t *cr) 22004c87aefeSPatrick Mooney { 22018e3a263eSAndy Fiddaman boolean_t hma_release = B_FALSE; 22024c87aefeSPatrick Mooney vmm_softc_t *sc; 22034c87aefeSPatrick Mooney int err; 22044c87aefeSPatrick Mooney 22054c87aefeSPatrick Mooney if (crgetuid(cr) != 0) 22064c87aefeSPatrick Mooney return (EPERM); 22074c87aefeSPatrick Mooney 22084c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 22094c87aefeSPatrick Mooney 2210b57f5d3eSPatrick Mooney if ((sc = vmm_lookup(req->name)) == NULL) { 22114c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 22124c87aefeSPatrick Mooney return (ENOENT); 22134c87aefeSPatrick Mooney } 22144c87aefeSPatrick Mooney /* 22154c87aefeSPatrick Mooney * We don't check this in vmm_lookup() since that function is also used 22164c87aefeSPatrick Mooney * for validation during create and currently vmm names must be unique. 22174c87aefeSPatrick Mooney */ 22184c87aefeSPatrick Mooney if (!INGLOBALZONE(curproc) && sc->vmm_zone != curzone) { 22194c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 22204c87aefeSPatrick Mooney return (EPERM); 22214c87aefeSPatrick Mooney } 22228e3a263eSAndy Fiddaman err = vmm_do_vm_destroy_locked(sc, B_TRUE, &hma_release); 22238e3a263eSAndy Fiddaman 22244c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 2225bf21cd93STycho Nightingale 22268e3a263eSAndy Fiddaman if (hma_release) 22278e3a263eSAndy Fiddaman vmm_hma_release(); 22288e3a263eSAndy Fiddaman 22294c87aefeSPatrick Mooney return (err); 2230bf21cd93STycho Nightingale } 2231bf21cd93STycho Nightingale 223259460b49SPatrick Mooney #define VCPU_NAME_BUFLEN 32 223359460b49SPatrick Mooney 223459460b49SPatrick Mooney static int 223559460b49SPatrick Mooney vmm_kstat_alloc(vmm_softc_t *sc, minor_t minor, const cred_t *cr) 223659460b49SPatrick Mooney { 223759460b49SPatrick Mooney zoneid_t zid = crgetzoneid(cr); 223859460b49SPatrick Mooney int instance = minor; 223959460b49SPatrick Mooney kstat_t *ksp; 224059460b49SPatrick Mooney 224159460b49SPatrick Mooney ASSERT3P(sc->vmm_kstat_vm, ==, NULL); 224259460b49SPatrick Mooney 224359460b49SPatrick Mooney ksp = kstat_create_zone(VMM_MODULE_NAME, instance, "vm", 224459460b49SPatrick Mooney VMM_KSTAT_CLASS, KSTAT_TYPE_NAMED, 224559460b49SPatrick Mooney sizeof (vmm_kstats_t) / sizeof (kstat_named_t), 0, zid); 224659460b49SPatrick Mooney 224759460b49SPatrick Mooney if (ksp == NULL) { 224859460b49SPatrick Mooney return (-1); 224959460b49SPatrick Mooney } 225059460b49SPatrick Mooney sc->vmm_kstat_vm = ksp; 225159460b49SPatrick Mooney 225259460b49SPatrick Mooney for (uint_t i = 0; i < VM_MAXCPU; i++) { 225359460b49SPatrick Mooney char namebuf[VCPU_NAME_BUFLEN]; 225459460b49SPatrick Mooney 225559460b49SPatrick Mooney ASSERT3P(sc->vmm_kstat_vcpu[i], ==, NULL); 225659460b49SPatrick Mooney 225759460b49SPatrick Mooney (void) snprintf(namebuf, VCPU_NAME_BUFLEN, "vcpu%u", i); 225859460b49SPatrick Mooney ksp = kstat_create_zone(VMM_MODULE_NAME, instance, namebuf, 225959460b49SPatrick Mooney VMM_KSTAT_CLASS, KSTAT_TYPE_NAMED, 226059460b49SPatrick Mooney sizeof (vmm_vcpu_kstats_t) / sizeof (kstat_named_t), 226159460b49SPatrick Mooney 0, zid); 226259460b49SPatrick Mooney if (ksp == NULL) { 226359460b49SPatrick Mooney goto fail; 226459460b49SPatrick Mooney } 226559460b49SPatrick Mooney 226659460b49SPatrick Mooney sc->vmm_kstat_vcpu[i] = ksp; 226759460b49SPatrick Mooney } 226859460b49SPatrick Mooney 226959460b49SPatrick Mooney /* 227059460b49SPatrick Mooney * If this instance is associated with a non-global zone, make its 227159460b49SPatrick Mooney * kstats visible from the GZ. 227259460b49SPatrick Mooney */ 227359460b49SPatrick Mooney if (zid != GLOBAL_ZONEID) { 227459460b49SPatrick Mooney kstat_zone_add(sc->vmm_kstat_vm, GLOBAL_ZONEID); 227559460b49SPatrick Mooney for (uint_t i = 0; i < VM_MAXCPU; i++) { 227659460b49SPatrick Mooney kstat_zone_add(sc->vmm_kstat_vcpu[i], GLOBAL_ZONEID); 227759460b49SPatrick Mooney } 227859460b49SPatrick Mooney } 227959460b49SPatrick Mooney 228059460b49SPatrick Mooney return (0); 228159460b49SPatrick Mooney 228259460b49SPatrick Mooney fail: 228359460b49SPatrick Mooney for (uint_t i = 0; i < VM_MAXCPU; i++) { 228459460b49SPatrick Mooney if (sc->vmm_kstat_vcpu[i] != NULL) { 228559460b49SPatrick Mooney kstat_delete(sc->vmm_kstat_vcpu[i]); 228659460b49SPatrick Mooney sc->vmm_kstat_vcpu[i] = NULL; 228759460b49SPatrick Mooney } else { 228859460b49SPatrick Mooney break; 228959460b49SPatrick Mooney } 229059460b49SPatrick Mooney } 229159460b49SPatrick Mooney kstat_delete(sc->vmm_kstat_vm); 229259460b49SPatrick Mooney sc->vmm_kstat_vm = NULL; 229359460b49SPatrick Mooney return (-1); 229459460b49SPatrick Mooney } 229559460b49SPatrick Mooney 229659460b49SPatrick Mooney static void 229759460b49SPatrick Mooney vmm_kstat_init(vmm_softc_t *sc) 229859460b49SPatrick Mooney { 229959460b49SPatrick Mooney kstat_t *ksp; 230059460b49SPatrick Mooney 230159460b49SPatrick Mooney ASSERT3P(sc->vmm_vm, !=, NULL); 230259460b49SPatrick Mooney ASSERT3P(sc->vmm_kstat_vm, !=, NULL); 230359460b49SPatrick Mooney 230459460b49SPatrick Mooney ksp = sc->vmm_kstat_vm; 230559460b49SPatrick Mooney vmm_kstats_t *vk = ksp->ks_data; 230659460b49SPatrick Mooney ksp->ks_private = sc->vmm_vm; 230759460b49SPatrick Mooney kstat_named_init(&vk->vk_name, "vm_name", KSTAT_DATA_STRING); 230859460b49SPatrick Mooney kstat_named_setstr(&vk->vk_name, sc->vmm_name); 230959460b49SPatrick Mooney 231059460b49SPatrick Mooney for (uint_t i = 0; i < VM_MAXCPU; i++) { 231159460b49SPatrick Mooney ASSERT3P(sc->vmm_kstat_vcpu[i], !=, NULL); 231259460b49SPatrick Mooney 231359460b49SPatrick Mooney ksp = sc->vmm_kstat_vcpu[i]; 231459460b49SPatrick Mooney vmm_vcpu_kstats_t *vvk = ksp->ks_data; 231559460b49SPatrick Mooney 231659460b49SPatrick Mooney kstat_named_init(&vvk->vvk_vcpu, "vcpu", KSTAT_DATA_UINT32); 231759460b49SPatrick Mooney vvk->vvk_vcpu.value.ui32 = i; 231859460b49SPatrick Mooney kstat_named_init(&vvk->vvk_time_init, "time_init", 231959460b49SPatrick Mooney KSTAT_DATA_UINT64); 232059460b49SPatrick Mooney kstat_named_init(&vvk->vvk_time_run, "time_run", 232159460b49SPatrick Mooney KSTAT_DATA_UINT64); 232259460b49SPatrick Mooney kstat_named_init(&vvk->vvk_time_idle, "time_idle", 232359460b49SPatrick Mooney KSTAT_DATA_UINT64); 232459460b49SPatrick Mooney kstat_named_init(&vvk->vvk_time_emu_kern, "time_emu_kern", 232559460b49SPatrick Mooney KSTAT_DATA_UINT64); 232659460b49SPatrick Mooney kstat_named_init(&vvk->vvk_time_emu_user, "time_emu_user", 232759460b49SPatrick Mooney KSTAT_DATA_UINT64); 232859460b49SPatrick Mooney kstat_named_init(&vvk->vvk_time_sched, "time_sched", 232959460b49SPatrick Mooney KSTAT_DATA_UINT64); 233059460b49SPatrick Mooney ksp->ks_private = sc->vmm_vm; 233159460b49SPatrick Mooney ksp->ks_update = vmm_kstat_update_vcpu; 233259460b49SPatrick Mooney } 233359460b49SPatrick Mooney 233459460b49SPatrick Mooney kstat_install(sc->vmm_kstat_vm); 233559460b49SPatrick Mooney for (uint_t i = 0; i < VM_MAXCPU; i++) { 233659460b49SPatrick Mooney kstat_install(sc->vmm_kstat_vcpu[i]); 233759460b49SPatrick Mooney } 233859460b49SPatrick Mooney } 233959460b49SPatrick Mooney 234059460b49SPatrick Mooney static void 234159460b49SPatrick Mooney vmm_kstat_fini(vmm_softc_t *sc) 234259460b49SPatrick Mooney { 234359460b49SPatrick Mooney ASSERT(sc->vmm_kstat_vm != NULL); 234459460b49SPatrick Mooney 234559460b49SPatrick Mooney kstat_delete(sc->vmm_kstat_vm); 234659460b49SPatrick Mooney sc->vmm_kstat_vm = NULL; 234759460b49SPatrick Mooney 234859460b49SPatrick Mooney for (uint_t i = 0; i < VM_MAXCPU; i++) { 234959460b49SPatrick Mooney ASSERT3P(sc->vmm_kstat_vcpu[i], !=, NULL); 235059460b49SPatrick Mooney 235159460b49SPatrick Mooney kstat_delete(sc->vmm_kstat_vcpu[i]); 235259460b49SPatrick Mooney sc->vmm_kstat_vcpu[i] = NULL; 235359460b49SPatrick Mooney } 235459460b49SPatrick Mooney } 235559460b49SPatrick Mooney 2356bf21cd93STycho Nightingale static int 2357bf21cd93STycho Nightingale vmm_open(dev_t *devp, int flag, int otyp, cred_t *credp) 2358bf21cd93STycho Nightingale { 23594c87aefeSPatrick Mooney minor_t minor; 23604c87aefeSPatrick Mooney vmm_softc_t *sc; 2361bf21cd93STycho Nightingale 23620153d828SPatrick Mooney /* 23630153d828SPatrick Mooney * Forbid running bhyve in a 32-bit process until it has been tested and 23640153d828SPatrick Mooney * verified to be safe. 23650153d828SPatrick Mooney */ 23660153d828SPatrick Mooney if (curproc->p_model != DATAMODEL_LP64) { 23670153d828SPatrick Mooney return (EFBIG); 23680153d828SPatrick Mooney } 23690153d828SPatrick Mooney 2370bf21cd93STycho Nightingale minor = getminor(*devp); 2371bf21cd93STycho Nightingale if (minor == VMM_CTL_MINOR) { 2372bf21cd93STycho Nightingale /* 2373bf21cd93STycho Nightingale * Master control device must be opened exclusively. 2374bf21cd93STycho Nightingale */ 2375bf21cd93STycho Nightingale if ((flag & FEXCL) != FEXCL || otyp != OTYP_CHR) { 2376bf21cd93STycho Nightingale return (EINVAL); 2377bf21cd93STycho Nightingale } 2378bf21cd93STycho Nightingale 2379bf21cd93STycho Nightingale return (0); 2380bf21cd93STycho Nightingale } 2381bf21cd93STycho Nightingale 23824c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 2383bf21cd93STycho Nightingale sc = ddi_get_soft_state(vmm_statep, minor); 2384bf21cd93STycho Nightingale if (sc == NULL) { 23854c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 2386bf21cd93STycho Nightingale return (ENXIO); 2387bf21cd93STycho Nightingale } 2388bf21cd93STycho Nightingale 23894c87aefeSPatrick Mooney sc->vmm_is_open = B_TRUE; 23904c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 2391bf21cd93STycho Nightingale 2392bf21cd93STycho Nightingale return (0); 2393bf21cd93STycho Nightingale } 2394bf21cd93STycho Nightingale 2395bf21cd93STycho Nightingale static int 2396bf21cd93STycho Nightingale vmm_close(dev_t dev, int flag, int otyp, cred_t *credp) 2397bf21cd93STycho Nightingale { 23984c87aefeSPatrick Mooney minor_t minor; 23994c87aefeSPatrick Mooney vmm_softc_t *sc; 24008e3a263eSAndy Fiddaman boolean_t hma_release = B_FALSE; 2401bf21cd93STycho Nightingale 2402bf21cd93STycho Nightingale minor = getminor(dev); 2403bf21cd93STycho Nightingale if (minor == VMM_CTL_MINOR) 2404bf21cd93STycho Nightingale return (0); 2405bf21cd93STycho Nightingale 24064c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 2407bf21cd93STycho Nightingale sc = ddi_get_soft_state(vmm_statep, minor); 2408bf21cd93STycho Nightingale if (sc == NULL) { 24094c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 2410bf21cd93STycho Nightingale return (ENXIO); 2411bf21cd93STycho Nightingale } 2412bf21cd93STycho Nightingale 24134c87aefeSPatrick Mooney VERIFY(sc->vmm_is_open); 24144c87aefeSPatrick Mooney sc->vmm_is_open = B_FALSE; 24154c87aefeSPatrick Mooney 24168e3a263eSAndy Fiddaman /* 24178e3a263eSAndy Fiddaman * If this VM was destroyed while the vmm device was open, then 24188e3a263eSAndy Fiddaman * clean it up now that it is closed. 24198e3a263eSAndy Fiddaman */ 24204c87aefeSPatrick Mooney if (sc->vmm_flags & VMM_DESTROY) { 24214c87aefeSPatrick Mooney list_remove(&vmm_destroy_list, sc); 24226884664dSPatrick Mooney vmm_kstat_fini(sc); 24234c87aefeSPatrick Mooney vm_destroy(sc->vmm_vm); 24244c87aefeSPatrick Mooney ddi_soft_state_free(vmm_statep, minor); 24254c87aefeSPatrick Mooney id_free(vmm_minors, minor); 24268e3a263eSAndy Fiddaman hma_release = B_TRUE; 24274c87aefeSPatrick Mooney } 24284c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 2429bf21cd93STycho Nightingale 24308e3a263eSAndy Fiddaman if (hma_release) 24318e3a263eSAndy Fiddaman vmm_hma_release(); 24328e3a263eSAndy Fiddaman 2433bf21cd93STycho Nightingale return (0); 2434bf21cd93STycho Nightingale } 2435bf21cd93STycho Nightingale 24364c87aefeSPatrick Mooney static int 24374c87aefeSPatrick Mooney vmm_is_supported(intptr_t arg) 24384c87aefeSPatrick Mooney { 24394c87aefeSPatrick Mooney int r; 24404c87aefeSPatrick Mooney const char *msg; 24414c87aefeSPatrick Mooney 24424c87aefeSPatrick Mooney if (vmm_is_intel()) { 24434c87aefeSPatrick Mooney r = vmx_x86_supported(&msg); 2444154972afSPatrick Mooney } else if (vmm_is_svm()) { 24454c87aefeSPatrick Mooney /* 24464c87aefeSPatrick Mooney * HMA already ensured that the features necessary for SVM 24474c87aefeSPatrick Mooney * operation were present and online during vmm_attach(). 24484c87aefeSPatrick Mooney */ 24494c87aefeSPatrick Mooney r = 0; 24504c87aefeSPatrick Mooney } else { 24514c87aefeSPatrick Mooney r = ENXIO; 24524c87aefeSPatrick Mooney msg = "Unsupported CPU vendor"; 24534c87aefeSPatrick Mooney } 24544c87aefeSPatrick Mooney 24554c87aefeSPatrick Mooney if (r != 0 && arg != (intptr_t)NULL) { 24564c87aefeSPatrick Mooney if (copyoutstr(msg, (char *)arg, strlen(msg) + 1, NULL) != 0) 24574c87aefeSPatrick Mooney return (EFAULT); 24584c87aefeSPatrick Mooney } 24594c87aefeSPatrick Mooney return (r); 24604c87aefeSPatrick Mooney } 24614c87aefeSPatrick Mooney 2462b57f5d3eSPatrick Mooney static int 2463b57f5d3eSPatrick Mooney vmm_ctl_ioctl(int cmd, intptr_t arg, int md, cred_t *cr, int *rvalp) 2464b57f5d3eSPatrick Mooney { 2465b57f5d3eSPatrick Mooney void *argp = (void *)arg; 2466b57f5d3eSPatrick Mooney 2467b57f5d3eSPatrick Mooney switch (cmd) { 2468b57f5d3eSPatrick Mooney case VMM_CREATE_VM: { 2469b57f5d3eSPatrick Mooney struct vm_create_req req; 2470b57f5d3eSPatrick Mooney 2471b57f5d3eSPatrick Mooney if ((md & FWRITE) == 0) { 2472b57f5d3eSPatrick Mooney return (EPERM); 2473b57f5d3eSPatrick Mooney } 2474b57f5d3eSPatrick Mooney if (ddi_copyin(argp, &req, sizeof (req), md) != 0) { 2475b57f5d3eSPatrick Mooney return (EFAULT); 2476b57f5d3eSPatrick Mooney } 2477b57f5d3eSPatrick Mooney return (vmmdev_do_vm_create(&req, cr)); 2478b57f5d3eSPatrick Mooney } 2479b57f5d3eSPatrick Mooney case VMM_DESTROY_VM: { 2480b57f5d3eSPatrick Mooney struct vm_destroy_req req; 2481b57f5d3eSPatrick Mooney 2482b57f5d3eSPatrick Mooney if ((md & FWRITE) == 0) { 2483b57f5d3eSPatrick Mooney return (EPERM); 2484b57f5d3eSPatrick Mooney } 2485b57f5d3eSPatrick Mooney if (ddi_copyin(argp, &req, sizeof (req), md) != 0) { 2486b57f5d3eSPatrick Mooney return (EFAULT); 2487b57f5d3eSPatrick Mooney } 2488b57f5d3eSPatrick Mooney return (vmmdev_do_vm_destroy(&req, cr)); 2489b57f5d3eSPatrick Mooney } 2490b57f5d3eSPatrick Mooney case VMM_VM_SUPPORTED: 2491b57f5d3eSPatrick Mooney return (vmm_is_supported(arg)); 24923466010bSPatrick Mooney case VMM_INTERFACE_VERSION: 24933466010bSPatrick Mooney *rvalp = VMM_CURRENT_INTERFACE_VERSION; 24943466010bSPatrick Mooney return (0); 2495b57f5d3eSPatrick Mooney case VMM_RESV_QUERY: 2496b57f5d3eSPatrick Mooney case VMM_RESV_ADD: 2497b57f5d3eSPatrick Mooney case VMM_RESV_REMOVE: 2498b57f5d3eSPatrick Mooney return (vmmr_ioctl(cmd, arg, md, cr, rvalp)); 2499b57f5d3eSPatrick Mooney default: 2500b57f5d3eSPatrick Mooney break; 2501b57f5d3eSPatrick Mooney } 2502b57f5d3eSPatrick Mooney /* No other actions are legal on ctl device */ 2503b57f5d3eSPatrick Mooney return (ENOTTY); 2504b57f5d3eSPatrick Mooney } 2505b57f5d3eSPatrick Mooney 2506bf21cd93STycho Nightingale static int 2507bf21cd93STycho Nightingale vmm_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, 2508bf21cd93STycho Nightingale int *rvalp) 2509bf21cd93STycho Nightingale { 25104c87aefeSPatrick Mooney vmm_softc_t *sc; 25114c87aefeSPatrick Mooney minor_t minor; 2512bf21cd93STycho Nightingale 25130153d828SPatrick Mooney /* 25140153d828SPatrick Mooney * Forbid running bhyve in a 32-bit process until it has been tested and 25150153d828SPatrick Mooney * verified to be safe. 25160153d828SPatrick Mooney */ 25170153d828SPatrick Mooney if (curproc->p_model != DATAMODEL_LP64) { 25180153d828SPatrick Mooney return (EFBIG); 25190153d828SPatrick Mooney } 25200153d828SPatrick Mooney 2521e0c0d44eSPatrick Mooney /* The structs in bhyve ioctls assume a 64-bit datamodel */ 2522e0c0d44eSPatrick Mooney if (ddi_model_convert_from(mode & FMODELS) != DDI_MODEL_NONE) { 2523e0c0d44eSPatrick Mooney return (ENOTSUP); 2524e0c0d44eSPatrick Mooney } 2525e0c0d44eSPatrick Mooney 2526bf21cd93STycho Nightingale minor = getminor(dev); 2527bf21cd93STycho Nightingale 2528bf21cd93STycho Nightingale if (minor == VMM_CTL_MINOR) { 2529b57f5d3eSPatrick Mooney return (vmm_ctl_ioctl(cmd, arg, mode, credp, rvalp)); 2530bf21cd93STycho Nightingale } 2531bf21cd93STycho Nightingale 2532bf21cd93STycho Nightingale sc = ddi_get_soft_state(vmm_statep, minor); 2533bf21cd93STycho Nightingale ASSERT(sc); 2534bf21cd93STycho Nightingale 25354c87aefeSPatrick Mooney if (sc->vmm_flags & VMM_DESTROY) 25364c87aefeSPatrick Mooney return (ENXIO); 25374c87aefeSPatrick Mooney 2538bf21cd93STycho Nightingale return (vmmdev_do_ioctl(sc, cmd, arg, mode, credp, rvalp)); 2539bf21cd93STycho Nightingale } 2540bf21cd93STycho Nightingale 2541bf21cd93STycho Nightingale static int 25424c87aefeSPatrick Mooney vmm_segmap(dev_t dev, off_t off, struct as *as, caddr_t *addrp, off_t len, 25434c87aefeSPatrick Mooney unsigned int prot, unsigned int maxprot, unsigned int flags, cred_t *credp) 2544bf21cd93STycho Nightingale { 25454c87aefeSPatrick Mooney vmm_softc_t *sc; 25464c87aefeSPatrick Mooney const minor_t minor = getminor(dev); 25474c87aefeSPatrick Mooney int err; 25484c87aefeSPatrick Mooney 25494c87aefeSPatrick Mooney if (minor == VMM_CTL_MINOR) { 25504c87aefeSPatrick Mooney return (ENODEV); 25514c87aefeSPatrick Mooney } 25524c87aefeSPatrick Mooney if (off < 0 || (off + len) <= 0) { 25534c87aefeSPatrick Mooney return (EINVAL); 25544c87aefeSPatrick Mooney } 25554c87aefeSPatrick Mooney if ((prot & PROT_USER) == 0) { 25564c87aefeSPatrick Mooney return (EACCES); 25574c87aefeSPatrick Mooney } 2558bf21cd93STycho Nightingale 25594c87aefeSPatrick Mooney sc = ddi_get_soft_state(vmm_statep, minor); 2560bf21cd93STycho Nightingale ASSERT(sc); 2561bf21cd93STycho Nightingale 25624c87aefeSPatrick Mooney if (sc->vmm_flags & VMM_DESTROY) 25634c87aefeSPatrick Mooney return (ENXIO); 2564bf21cd93STycho Nightingale 25654c87aefeSPatrick Mooney /* Grab read lock on the VM to prevent any changes to the memory map */ 25664c87aefeSPatrick Mooney vmm_read_lock(sc); 2567bf21cd93STycho Nightingale 25684c87aefeSPatrick Mooney if (off >= VM_DEVMEM_START) { 25694c87aefeSPatrick Mooney int segid; 25700153d828SPatrick Mooney off_t segoff; 2571bf21cd93STycho Nightingale 25724c87aefeSPatrick Mooney /* Mapping a devmem "device" */ 25730153d828SPatrick Mooney if (!vmmdev_devmem_segid(sc, off, len, &segid, &segoff)) { 25744c87aefeSPatrick Mooney err = ENODEV; 25750153d828SPatrick Mooney } else { 25760153d828SPatrick Mooney err = vm_segmap_obj(sc->vmm_vm, segid, segoff, len, as, 25770153d828SPatrick Mooney addrp, prot, maxprot, flags); 25784c87aefeSPatrick Mooney } 25794c87aefeSPatrick Mooney } else { 25804c87aefeSPatrick Mooney /* Mapping a part of the guest physical space */ 25810153d828SPatrick Mooney err = vm_segmap_space(sc->vmm_vm, off, as, addrp, len, prot, 25820153d828SPatrick Mooney maxprot, flags); 2583bf21cd93STycho Nightingale } 2584bf21cd93STycho Nightingale 25854c87aefeSPatrick Mooney vmm_read_unlock(sc); 25864c87aefeSPatrick Mooney return (err); 2587bf21cd93STycho Nightingale } 2588bf21cd93STycho Nightingale 25894c87aefeSPatrick Mooney static sdev_plugin_validate_t 25904c87aefeSPatrick Mooney vmm_sdev_validate(sdev_ctx_t ctx) 2591bf21cd93STycho Nightingale { 25924c87aefeSPatrick Mooney const char *name = sdev_ctx_name(ctx); 25934c87aefeSPatrick Mooney vmm_softc_t *sc; 25944c87aefeSPatrick Mooney sdev_plugin_validate_t ret; 25954c87aefeSPatrick Mooney minor_t minor; 25964c87aefeSPatrick Mooney 25974c87aefeSPatrick Mooney if (sdev_ctx_vtype(ctx) != VCHR) 25984c87aefeSPatrick Mooney return (SDEV_VTOR_INVALID); 25994c87aefeSPatrick Mooney 26004c87aefeSPatrick Mooney VERIFY3S(sdev_ctx_minor(ctx, &minor), ==, 0); 26014c87aefeSPatrick Mooney 26024c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 26034c87aefeSPatrick Mooney if ((sc = vmm_lookup(name)) == NULL) 26044c87aefeSPatrick Mooney ret = SDEV_VTOR_INVALID; 26054c87aefeSPatrick Mooney else if (sc->vmm_minor != minor) 26064c87aefeSPatrick Mooney ret = SDEV_VTOR_STALE; 26074c87aefeSPatrick Mooney else 26084c87aefeSPatrick Mooney ret = SDEV_VTOR_VALID; 26094c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 26104c87aefeSPatrick Mooney 26114c87aefeSPatrick Mooney return (ret); 2612bf21cd93STycho Nightingale } 2613bf21cd93STycho Nightingale 2614bf21cd93STycho Nightingale static int 26154c87aefeSPatrick Mooney vmm_sdev_filldir(sdev_ctx_t ctx) 2616bf21cd93STycho Nightingale { 26174c87aefeSPatrick Mooney vmm_softc_t *sc; 26184c87aefeSPatrick Mooney int ret; 26194c87aefeSPatrick Mooney 26204c87aefeSPatrick Mooney if (strcmp(sdev_ctx_path(ctx), VMM_SDEV_ROOT) != 0) { 26214c87aefeSPatrick Mooney cmn_err(CE_WARN, "%s: bad path '%s' != '%s'\n", __func__, 26224c87aefeSPatrick Mooney sdev_ctx_path(ctx), VMM_SDEV_ROOT); 26234c87aefeSPatrick Mooney return (EINVAL); 26244c87aefeSPatrick Mooney } 26254c87aefeSPatrick Mooney 26264c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 26274c87aefeSPatrick Mooney ASSERT(vmmdev_dip != NULL); 26284c87aefeSPatrick Mooney for (sc = list_head(&vmm_list); sc != NULL; 26294c87aefeSPatrick Mooney sc = list_next(&vmm_list, sc)) { 26304c87aefeSPatrick Mooney if (INGLOBALZONE(curproc) || sc->vmm_zone == curzone) { 26314c87aefeSPatrick Mooney ret = sdev_plugin_mknod(ctx, sc->vmm_name, 26324c87aefeSPatrick Mooney S_IFCHR | 0600, 26334c87aefeSPatrick Mooney makedevice(ddi_driver_major(vmmdev_dip), 26344c87aefeSPatrick Mooney sc->vmm_minor)); 26354c87aefeSPatrick Mooney } else { 26364c87aefeSPatrick Mooney continue; 26374c87aefeSPatrick Mooney } 26384c87aefeSPatrick Mooney if (ret != 0 && ret != EEXIST) 26394c87aefeSPatrick Mooney goto out; 2640bf21cd93STycho Nightingale } 2641bf21cd93STycho Nightingale 26424c87aefeSPatrick Mooney ret = 0; 26434c87aefeSPatrick Mooney 26444c87aefeSPatrick Mooney out: 26454c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 26464c87aefeSPatrick Mooney return (ret); 26474c87aefeSPatrick Mooney } 26484c87aefeSPatrick Mooney 26494c87aefeSPatrick Mooney /* ARGSUSED */ 26504c87aefeSPatrick Mooney static void 26514c87aefeSPatrick Mooney vmm_sdev_inactive(sdev_ctx_t ctx) 26524c87aefeSPatrick Mooney { 2653bf21cd93STycho Nightingale } 2654bf21cd93STycho Nightingale 26554c87aefeSPatrick Mooney static sdev_plugin_ops_t vmm_sdev_ops = { 26564c87aefeSPatrick Mooney .spo_version = SDEV_PLUGIN_VERSION, 26574c87aefeSPatrick Mooney .spo_flags = SDEV_PLUGIN_SUBDIR, 26584c87aefeSPatrick Mooney .spo_validate = vmm_sdev_validate, 26594c87aefeSPatrick Mooney .spo_filldir = vmm_sdev_filldir, 26604c87aefeSPatrick Mooney .spo_inactive = vmm_sdev_inactive 26614c87aefeSPatrick Mooney }; 26624c87aefeSPatrick Mooney 26634c87aefeSPatrick Mooney /* ARGSUSED */ 2664bf21cd93STycho Nightingale static int 26654c87aefeSPatrick Mooney vmm_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) 2666bf21cd93STycho Nightingale { 26674c87aefeSPatrick Mooney int error; 26684c87aefeSPatrick Mooney 2669bf21cd93STycho Nightingale switch (cmd) { 26704c87aefeSPatrick Mooney case DDI_INFO_DEVT2DEVINFO: 26714c87aefeSPatrick Mooney *result = (void *)vmmdev_dip; 26724c87aefeSPatrick Mooney error = DDI_SUCCESS; 26734c87aefeSPatrick Mooney break; 26744c87aefeSPatrick Mooney case DDI_INFO_DEVT2INSTANCE: 26754c87aefeSPatrick Mooney *result = (void *)0; 26764c87aefeSPatrick Mooney error = DDI_SUCCESS; 2677bf21cd93STycho Nightingale break; 2678bf21cd93STycho Nightingale default: 26794c87aefeSPatrick Mooney error = DDI_FAILURE; 26804c87aefeSPatrick Mooney break; 26814c87aefeSPatrick Mooney } 26824c87aefeSPatrick Mooney return (error); 26834c87aefeSPatrick Mooney } 26844c87aefeSPatrick Mooney 26854c87aefeSPatrick Mooney static int 26864c87aefeSPatrick Mooney vmm_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 26874c87aefeSPatrick Mooney { 26884c87aefeSPatrick Mooney sdev_plugin_hdl_t sph; 26894c87aefeSPatrick Mooney hma_reg_t *reg = NULL; 26904c87aefeSPatrick Mooney boolean_t vmm_loaded = B_FALSE; 26914c87aefeSPatrick Mooney 26924c87aefeSPatrick Mooney if (cmd != DDI_ATTACH) { 2693bf21cd93STycho Nightingale return (DDI_FAILURE); 2694bf21cd93STycho Nightingale } 2695bf21cd93STycho Nightingale 26964c87aefeSPatrick Mooney mutex_enter(&vmmdev_mtx); 26974c87aefeSPatrick Mooney /* Ensure we are not already attached. */ 26984c87aefeSPatrick Mooney if (vmmdev_dip != NULL) { 26994c87aefeSPatrick Mooney mutex_exit(&vmmdev_mtx); 2700bf21cd93STycho Nightingale return (DDI_FAILURE); 2701bf21cd93STycho Nightingale } 2702bf21cd93STycho Nightingale 27034c87aefeSPatrick Mooney vmm_sol_glue_init(); 2704bf21cd93STycho Nightingale 27058e3a263eSAndy Fiddaman /* 27068e3a263eSAndy Fiddaman * Perform temporary HMA registration to determine if the system 27078e3a263eSAndy Fiddaman * is capable. 27088e3a263eSAndy Fiddaman */ 27094c87aefeSPatrick Mooney if ((reg = hma_register(vmmdev_hvm_name)) == NULL) { 27104c87aefeSPatrick Mooney goto fail; 27114c87aefeSPatrick Mooney } else if (vmm_mod_load() != 0) { 27124c87aefeSPatrick Mooney goto fail; 27134c87aefeSPatrick Mooney } 27144c87aefeSPatrick Mooney vmm_loaded = B_TRUE; 27158e3a263eSAndy Fiddaman hma_unregister(reg); 27168e3a263eSAndy Fiddaman reg = NULL; 27174c87aefeSPatrick Mooney 27184c87aefeSPatrick Mooney /* Create control node. Other nodes will be created on demand. */ 27194c87aefeSPatrick Mooney if (ddi_create_minor_node(dip, "ctl", S_IFCHR, 2720bf21cd93STycho Nightingale VMM_CTL_MINOR, DDI_PSEUDO, 0) != 0) { 27214c87aefeSPatrick Mooney goto fail; 2722bf21cd93STycho Nightingale } 2723bf21cd93STycho Nightingale 272459460b49SPatrick Mooney sph = sdev_plugin_register(VMM_MODULE_NAME, &vmm_sdev_ops, NULL); 272559460b49SPatrick Mooney if (sph == (sdev_plugin_hdl_t)NULL) { 27264c87aefeSPatrick Mooney ddi_remove_minor_node(dip, NULL); 27274c87aefeSPatrick Mooney goto fail; 27284c87aefeSPatrick Mooney } 2729bf21cd93STycho Nightingale 27304c87aefeSPatrick Mooney ddi_report_dev(dip); 27314c87aefeSPatrick Mooney vmmdev_sdev_hdl = sph; 27324c87aefeSPatrick Mooney vmmdev_dip = dip; 27334c87aefeSPatrick Mooney mutex_exit(&vmmdev_mtx); 2734bf21cd93STycho Nightingale return (DDI_SUCCESS); 27354c87aefeSPatrick Mooney 27364c87aefeSPatrick Mooney fail: 27374c87aefeSPatrick Mooney if (vmm_loaded) { 27384c87aefeSPatrick Mooney VERIFY0(vmm_mod_unload()); 27394c87aefeSPatrick Mooney } 27404c87aefeSPatrick Mooney if (reg != NULL) { 27414c87aefeSPatrick Mooney hma_unregister(reg); 27424c87aefeSPatrick Mooney } 27434c87aefeSPatrick Mooney vmm_sol_glue_cleanup(); 27444c87aefeSPatrick Mooney mutex_exit(&vmmdev_mtx); 27454c87aefeSPatrick Mooney return (DDI_FAILURE); 2746bf21cd93STycho Nightingale } 2747bf21cd93STycho Nightingale 2748bf21cd93STycho Nightingale static int 2749bf21cd93STycho Nightingale vmm_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 2750bf21cd93STycho Nightingale { 27514c87aefeSPatrick Mooney if (cmd != DDI_DETACH) { 27524c87aefeSPatrick Mooney return (DDI_FAILURE); 27534c87aefeSPatrick Mooney } 27544c87aefeSPatrick Mooney 2755eb9a1df2SHans Rosenfeld /* 2756eb9a1df2SHans Rosenfeld * Ensure that all resources have been cleaned up. 2757eb9a1df2SHans Rosenfeld * 2758eb9a1df2SHans Rosenfeld * To prevent a deadlock with iommu_cleanup() we'll fail the detach if 2759eb9a1df2SHans Rosenfeld * vmmdev_mtx is already held. We can't wait for vmmdev_mtx with our 2760eb9a1df2SHans Rosenfeld * devinfo locked as iommu_cleanup() tries to recursively lock each 2761eb9a1df2SHans Rosenfeld * devinfo, including our own, while holding vmmdev_mtx. 2762eb9a1df2SHans Rosenfeld */ 2763eb9a1df2SHans Rosenfeld if (mutex_tryenter(&vmmdev_mtx) == 0) 2764eb9a1df2SHans Rosenfeld return (DDI_FAILURE); 27654c87aefeSPatrick Mooney 27664c87aefeSPatrick Mooney mutex_enter(&vmm_mtx); 27674c87aefeSPatrick Mooney if (!list_is_empty(&vmm_list) || !list_is_empty(&vmm_destroy_list)) { 27684c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 27694c87aefeSPatrick Mooney mutex_exit(&vmmdev_mtx); 2770bf21cd93STycho Nightingale return (DDI_FAILURE); 2771bf21cd93STycho Nightingale } 27724c87aefeSPatrick Mooney mutex_exit(&vmm_mtx); 2773bf21cd93STycho Nightingale 2774b57f5d3eSPatrick Mooney if (!vmmr_is_empty()) { 2775b57f5d3eSPatrick Mooney mutex_exit(&vmmdev_mtx); 2776b57f5d3eSPatrick Mooney return (DDI_FAILURE); 2777b57f5d3eSPatrick Mooney } 2778b57f5d3eSPatrick Mooney 27794c87aefeSPatrick Mooney VERIFY(vmmdev_sdev_hdl != (sdev_plugin_hdl_t)NULL); 27804c87aefeSPatrick Mooney if (sdev_plugin_unregister(vmmdev_sdev_hdl) != 0) { 27814c87aefeSPatrick Mooney mutex_exit(&vmmdev_mtx); 2782bf21cd93STycho Nightingale return (DDI_FAILURE); 2783bf21cd93STycho Nightingale } 27844c87aefeSPatrick Mooney vmmdev_sdev_hdl = (sdev_plugin_hdl_t)NULL; 2785bf21cd93STycho Nightingale 27864c87aefeSPatrick Mooney /* Remove the control node. */ 27874c87aefeSPatrick Mooney ddi_remove_minor_node(dip, "ctl"); 27884c87aefeSPatrick Mooney vmmdev_dip = NULL; 27894c87aefeSPatrick Mooney 27904c87aefeSPatrick Mooney VERIFY0(vmm_mod_unload()); 27918e3a263eSAndy Fiddaman VERIFY3U(vmmdev_hma_reg, ==, NULL); 27924c87aefeSPatrick Mooney vmm_sol_glue_cleanup(); 27934c87aefeSPatrick Mooney 27944c87aefeSPatrick Mooney mutex_exit(&vmmdev_mtx); 2795bf21cd93STycho Nightingale 2796bf21cd93STycho Nightingale return (DDI_SUCCESS); 2797bf21cd93STycho Nightingale } 2798bf21cd93STycho Nightingale 2799bf21cd93STycho Nightingale static struct cb_ops vmm_cb_ops = { 2800bf21cd93STycho Nightingale vmm_open, 2801bf21cd93STycho Nightingale vmm_close, 2802bf21cd93STycho Nightingale nodev, /* strategy */ 2803bf21cd93STycho Nightingale nodev, /* print */ 2804bf21cd93STycho Nightingale nodev, /* dump */ 2805bf21cd93STycho Nightingale nodev, /* read */ 2806bf21cd93STycho Nightingale nodev, /* write */ 2807bf21cd93STycho Nightingale vmm_ioctl, 2808bf21cd93STycho Nightingale nodev, /* devmap */ 28094c87aefeSPatrick Mooney nodev, /* mmap */ 2810bf21cd93STycho Nightingale vmm_segmap, 2811bf21cd93STycho Nightingale nochpoll, /* poll */ 2812bf21cd93STycho Nightingale ddi_prop_op, 2813bf21cd93STycho Nightingale NULL, 2814bf21cd93STycho Nightingale D_NEW | D_MP | D_DEVMAP 2815bf21cd93STycho Nightingale }; 2816bf21cd93STycho Nightingale 2817bf21cd93STycho Nightingale static struct dev_ops vmm_ops = { 2818bf21cd93STycho Nightingale DEVO_REV, 2819bf21cd93STycho Nightingale 0, 28204c87aefeSPatrick Mooney vmm_info, 2821bf21cd93STycho Nightingale nulldev, /* identify */ 28224c87aefeSPatrick Mooney nulldev, /* probe */ 2823bf21cd93STycho Nightingale vmm_attach, 2824bf21cd93STycho Nightingale vmm_detach, 2825bf21cd93STycho Nightingale nodev, /* reset */ 2826bf21cd93STycho Nightingale &vmm_cb_ops, 2827bf21cd93STycho Nightingale (struct bus_ops *)NULL 2828bf21cd93STycho Nightingale }; 2829bf21cd93STycho Nightingale 2830bf21cd93STycho Nightingale static struct modldrv modldrv = { 2831bf21cd93STycho Nightingale &mod_driverops, 28324c87aefeSPatrick Mooney "bhyve vmm", 2833bf21cd93STycho Nightingale &vmm_ops 2834bf21cd93STycho Nightingale }; 2835bf21cd93STycho Nightingale 2836bf21cd93STycho Nightingale static struct modlinkage modlinkage = { 2837bf21cd93STycho Nightingale MODREV_1, 2838bf21cd93STycho Nightingale &modldrv, 2839bf21cd93STycho Nightingale NULL 2840bf21cd93STycho Nightingale }; 2841bf21cd93STycho Nightingale 2842bf21cd93STycho Nightingale int 2843bf21cd93STycho Nightingale _init(void) 2844bf21cd93STycho Nightingale { 2845bf21cd93STycho Nightingale int error; 2846bf21cd93STycho Nightingale 28474c87aefeSPatrick Mooney sysinit(); 2848bf21cd93STycho Nightingale 28494c87aefeSPatrick Mooney mutex_init(&vmmdev_mtx, NULL, MUTEX_DRIVER, NULL); 28504c87aefeSPatrick Mooney mutex_init(&vmm_mtx, NULL, MUTEX_DRIVER, NULL); 28514c87aefeSPatrick Mooney list_create(&vmm_list, sizeof (vmm_softc_t), 28524c87aefeSPatrick Mooney offsetof(vmm_softc_t, vmm_node)); 28534c87aefeSPatrick Mooney list_create(&vmm_destroy_list, sizeof (vmm_softc_t), 28544c87aefeSPatrick Mooney offsetof(vmm_softc_t, vmm_node)); 28554c87aefeSPatrick Mooney vmm_minors = id_space_create("vmm_minors", VMM_CTL_MINOR + 1, MAXMIN32); 28564c87aefeSPatrick Mooney 28574c87aefeSPatrick Mooney error = ddi_soft_state_init(&vmm_statep, sizeof (vmm_softc_t), 0); 2858bf21cd93STycho Nightingale if (error) { 2859bf21cd93STycho Nightingale return (error); 2860bf21cd93STycho Nightingale } 2861bf21cd93STycho Nightingale 28624c87aefeSPatrick Mooney vmm_zsd_init(); 2863b57f5d3eSPatrick Mooney vmmr_init(); 28644c87aefeSPatrick Mooney 2865bf21cd93STycho Nightingale error = mod_install(&modlinkage); 2866bf21cd93STycho Nightingale if (error) { 2867bf21cd93STycho Nightingale ddi_soft_state_fini(&vmm_statep); 28684c87aefeSPatrick Mooney vmm_zsd_fini(); 2869b57f5d3eSPatrick Mooney vmmr_fini(); 2870bf21cd93STycho Nightingale } 2871bf21cd93STycho Nightingale 2872bf21cd93STycho Nightingale return (error); 2873bf21cd93STycho Nightingale } 2874bf21cd93STycho Nightingale 2875bf21cd93STycho Nightingale int 2876bf21cd93STycho Nightingale _fini(void) 2877bf21cd93STycho Nightingale { 2878bf21cd93STycho Nightingale int error; 2879bf21cd93STycho Nightingale 2880bf21cd93STycho Nightingale error = mod_remove(&modlinkage); 2881bf21cd93STycho Nightingale if (error) { 2882bf21cd93STycho Nightingale return (error); 2883bf21cd93STycho Nightingale } 28844c87aefeSPatrick Mooney 28854c87aefeSPatrick Mooney vmm_zsd_fini(); 2886b57f5d3eSPatrick Mooney vmmr_fini(); 28874c87aefeSPatrick Mooney 2888bf21cd93STycho Nightingale ddi_soft_state_fini(&vmm_statep); 2889bf21cd93STycho Nightingale 2890bf21cd93STycho Nightingale return (0); 2891bf21cd93STycho Nightingale } 2892bf21cd93STycho Nightingale 2893bf21cd93STycho Nightingale int 2894bf21cd93STycho Nightingale _info(struct modinfo *modinfop) 2895bf21cd93STycho Nightingale { 2896bf21cd93STycho Nightingale return (mod_info(&modlinkage, modinfop)); 2897bf21cd93STycho Nightingale } 2898