xref: /illumos-gate/usr/src/uts/intel/io/vmm/vmm_sol_dev.c (revision 54cf5b63)
1bf21cd93STycho Nightingale /*
2bf21cd93STycho Nightingale  * This file and its contents are supplied under the terms of the
3bf21cd93STycho Nightingale  * Common Development and Distribution License ("CDDL"), version 1.0.
4bf21cd93STycho Nightingale  * You may only use this file in accordance with the terms of version
5bf21cd93STycho Nightingale  * 1.0 of the CDDL.
6bf21cd93STycho Nightingale  *
7bf21cd93STycho Nightingale  * A full copy of the text of the CDDL should have accompanied this
8bf21cd93STycho Nightingale  * source.  A copy of the CDDL is also available via the Internet at
9bf21cd93STycho Nightingale  * http://www.illumos.org/license/CDDL.
10bf21cd93STycho Nightingale  */
111fa07ac7SMike Zeller /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */
12bf21cd93STycho Nightingale 
13bf21cd93STycho Nightingale /*
14bf21cd93STycho Nightingale  * Copyright 2015 Pluribus Networks Inc.
154c87aefeSPatrick Mooney  * Copyright 2019 Joyent, Inc.
168e3a263eSAndy Fiddaman  * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
173d097f7dSPatrick Mooney  * Copyright 2022 Oxide Computer Company
18bf21cd93STycho Nightingale  */
19bf21cd93STycho Nightingale 
20bf21cd93STycho Nightingale #include <sys/types.h>
21bf21cd93STycho Nightingale #include <sys/conf.h>
22bf21cd93STycho Nightingale #include <sys/cpuvar.h>
23bf21cd93STycho Nightingale #include <sys/ioccom.h>
24bf21cd93STycho Nightingale #include <sys/stat.h>
25bf21cd93STycho Nightingale #include <sys/vmsystm.h>
26bf21cd93STycho Nightingale #include <sys/ddi.h>
274c87aefeSPatrick Mooney #include <sys/mkdev.h>
28bf21cd93STycho Nightingale #include <sys/sunddi.h>
29bf21cd93STycho Nightingale #include <sys/fs/dv_node.h>
304c87aefeSPatrick Mooney #include <sys/cpuset.h>
314c87aefeSPatrick Mooney #include <sys/id_space.h>
324c87aefeSPatrick Mooney #include <sys/fs/sdev_plugin.h>
334c87aefeSPatrick Mooney #include <sys/smt.h>
3459460b49SPatrick Mooney #include <sys/kstat.h>
354c87aefeSPatrick Mooney 
364c87aefeSPatrick Mooney #include <sys/kernel.h>
374c87aefeSPatrick Mooney #include <sys/hma.h>
384c87aefeSPatrick Mooney #include <sys/x86_archext.h>
39154972afSPatrick Mooney #include <x86/apicreg.h>
40bf21cd93STycho Nightingale 
41bf21cd93STycho Nightingale #include <sys/vmm.h>
42b58b977eSPatrick Mooney #include <sys/vmm_kernel.h>
43bf21cd93STycho Nightingale #include <sys/vmm_instruction_emul.h>
44bf21cd93STycho Nightingale #include <sys/vmm_dev.h>
45bf21cd93STycho Nightingale #include <sys/vmm_impl.h>
464c87aefeSPatrick Mooney #include <sys/vmm_drv.h>
47cf409e3fSDan Cross #include <sys/vmm_vm.h>
48b57f5d3eSPatrick Mooney #include <sys/vmm_reservoir.h>
49bf21cd93STycho Nightingale 
50bf21cd93STycho Nightingale #include <vm/seg_dev.h>
51bf21cd93STycho Nightingale 
52eb9a1df2SHans Rosenfeld #include "io/ppt.h"
53bf21cd93STycho Nightingale #include "io/vatpic.h"
54bf21cd93STycho Nightingale #include "io/vioapic.h"
554c87aefeSPatrick Mooney #include "io/vrtc.h"
564c87aefeSPatrick Mooney #include "io/vhpet.h"
570e1453c3SPatrick Mooney #include "io/vpmtmr.h"
58bf21cd93STycho Nightingale #include "vmm_lapic.h"
594c87aefeSPatrick Mooney #include "vmm_stat.h"
604c87aefeSPatrick Mooney #include "vmm_util.h"
61bf21cd93STycho Nightingale 
624c87aefeSPatrick Mooney /*
634c87aefeSPatrick Mooney  * Locking details:
644c87aefeSPatrick Mooney  *
654c87aefeSPatrick Mooney  * Driver-wide data (vmmdev_*) , including HMA and sdev registration, is
664c87aefeSPatrick Mooney  * protected by vmmdev_mtx.  The list of vmm_softc_t instances and related data
674c87aefeSPatrick Mooney  * (vmm_*) are protected by vmm_mtx.  Actions requiring both locks must acquire
684c87aefeSPatrick Mooney  * vmmdev_mtx before vmm_mtx.  The sdev plugin functions must not attempt to
694c87aefeSPatrick Mooney  * acquire vmmdev_mtx, as they could deadlock with plugin unregistration.
704c87aefeSPatrick Mooney  */
71bf21cd93STycho Nightingale 
724c87aefeSPatrick Mooney static kmutex_t		vmmdev_mtx;
734c87aefeSPatrick Mooney static dev_info_t	*vmmdev_dip;
744c87aefeSPatrick Mooney static hma_reg_t	*vmmdev_hma_reg;
758e3a263eSAndy Fiddaman static uint_t		vmmdev_hma_ref;
764c87aefeSPatrick Mooney static sdev_plugin_hdl_t vmmdev_sdev_hdl;
77bf21cd93STycho Nightingale 
784c87aefeSPatrick Mooney static kmutex_t		vmm_mtx;
794c87aefeSPatrick Mooney static list_t		vmm_list;
804c87aefeSPatrick Mooney static list_t		vmm_destroy_list;
814c87aefeSPatrick Mooney static id_space_t	*vmm_minors;
824c87aefeSPatrick Mooney static void		*vmm_statep;
83bf21cd93STycho Nightingale 
84d515dd77SPatrick Mooney /* temporary safety switch */
85d515dd77SPatrick Mooney int		vmm_allow_state_writes;
86d515dd77SPatrick Mooney 
874c87aefeSPatrick Mooney static const char *vmmdev_hvm_name = "bhyve";
88bf21cd93STycho Nightingale 
894c87aefeSPatrick Mooney /* For sdev plugin (/dev) */
904c87aefeSPatrick Mooney #define	VMM_SDEV_ROOT "/dev/vmm"
914c87aefeSPatrick Mooney 
927c8c0b82SPatrick Mooney /* From uts/intel/io/vmm/intel/vmx.c */
934c87aefeSPatrick Mooney extern int vmx_x86_supported(const char **);
944c87aefeSPatrick Mooney 
954c87aefeSPatrick Mooney /* Holds and hooks from drivers external to vmm */
964c87aefeSPatrick Mooney struct vmm_hold {
974c87aefeSPatrick Mooney 	list_node_t	vmh_node;
984c87aefeSPatrick Mooney 	vmm_softc_t	*vmh_sc;
994c87aefeSPatrick Mooney 	boolean_t	vmh_release_req;
1004c87aefeSPatrick Mooney 	uint_t		vmh_ioport_hook_cnt;
1014c87aefeSPatrick Mooney };
1024c87aefeSPatrick Mooney 
1034c87aefeSPatrick Mooney struct vmm_lease {
1044c87aefeSPatrick Mooney 	list_node_t		vml_node;
1054c87aefeSPatrick Mooney 	struct vm		*vml_vm;
1060153d828SPatrick Mooney 	vm_client_t		*vml_vmclient;
1074c87aefeSPatrick Mooney 	boolean_t		vml_expired;
1086703a0e8SPatrick Mooney 	boolean_t		vml_break_deferred;
1094c87aefeSPatrick Mooney 	boolean_t		(*vml_expire_func)(void *);
1104c87aefeSPatrick Mooney 	void			*vml_expire_arg;
1114c87aefeSPatrick Mooney 	struct vmm_hold		*vml_hold;
1124c87aefeSPatrick Mooney };
1134c87aefeSPatrick Mooney 
1144c87aefeSPatrick Mooney static int vmm_drv_block_hook(vmm_softc_t *, boolean_t);
1156703a0e8SPatrick Mooney static void vmm_lease_block(vmm_softc_t *);
1166703a0e8SPatrick Mooney static void vmm_lease_unblock(vmm_softc_t *);
11759460b49SPatrick Mooney static int vmm_kstat_alloc(vmm_softc_t *, minor_t, const cred_t *);
11859460b49SPatrick Mooney static void vmm_kstat_init(vmm_softc_t *);
11959460b49SPatrick Mooney static void vmm_kstat_fini(vmm_softc_t *);
1204c87aefeSPatrick Mooney 
1213d066281SPatrick Mooney /*
1223d066281SPatrick Mooney  * The 'devmem' hack:
1233d066281SPatrick Mooney  *
1243d066281SPatrick Mooney  * On native FreeBSD, bhyve consumers are allowed to create 'devmem' segments
1253d066281SPatrick Mooney  * in the vm which appear with their own name related to the vm under /dev.
1263d066281SPatrick Mooney  * Since this would be a hassle from an sdev perspective and would require a
1273d066281SPatrick Mooney  * new cdev interface (or complicate the existing one), we choose to implement
1283d066281SPatrick Mooney  * this in a different manner.  Direct access to the underlying vm memory
1293d066281SPatrick Mooney  * segments is exposed by placing them in a range of offsets beyond the normal
1303d066281SPatrick Mooney  * guest memory space.  Userspace can query the appropriate offset to mmap()
1313d066281SPatrick Mooney  * for a given segment-id with the VM_DEVMEM_GETOFFSET ioctl.
1323d066281SPatrick Mooney  */
1333d066281SPatrick Mooney 
1343d066281SPatrick Mooney static vmm_devmem_entry_t *
vmmdev_devmem_find(vmm_softc_t * sc,int segid)1353d066281SPatrick Mooney vmmdev_devmem_find(vmm_softc_t *sc, int segid)
1363d066281SPatrick Mooney {
1373d066281SPatrick Mooney 	vmm_devmem_entry_t *ent = NULL;
1383d066281SPatrick Mooney 	list_t *dl = &sc->vmm_devmem_list;
1393d066281SPatrick Mooney 
1403d066281SPatrick Mooney 	for (ent = list_head(dl); ent != NULL; ent = list_next(dl, ent)) {
1413d066281SPatrick Mooney 		if (ent->vde_segid == segid) {
1423d066281SPatrick Mooney 			return (ent);
1433d066281SPatrick Mooney 		}
1443d066281SPatrick Mooney 	}
1453d066281SPatrick Mooney 	return (NULL);
1463d066281SPatrick Mooney }
1473d066281SPatrick Mooney 
1484c87aefeSPatrick Mooney static int
vmmdev_get_memseg(vmm_softc_t * sc,struct vm_memseg * mseg)1494c87aefeSPatrick Mooney vmmdev_get_memseg(vmm_softc_t *sc, struct vm_memseg *mseg)
150bf21cd93STycho Nightingale {
1514c87aefeSPatrick Mooney 	int error;
1524c87aefeSPatrick Mooney 	bool sysmem;
153bf21cd93STycho Nightingale 
1544c87aefeSPatrick Mooney 	error = vm_get_memseg(sc->vmm_vm, mseg->segid, &mseg->len, &sysmem,
1554c87aefeSPatrick Mooney 	    NULL);
1564c87aefeSPatrick Mooney 	if (error || mseg->len == 0)
1574c87aefeSPatrick Mooney 		return (error);
158bf21cd93STycho Nightingale 
1594c87aefeSPatrick Mooney 	if (!sysmem) {
1604c87aefeSPatrick Mooney 		vmm_devmem_entry_t *de;
161bf21cd93STycho Nightingale 
1623d066281SPatrick Mooney 		de = vmmdev_devmem_find(sc, mseg->segid);
1634c87aefeSPatrick Mooney 		if (de != NULL) {
1644c87aefeSPatrick Mooney 			(void) strlcpy(mseg->name, de->vde_name,
1654c87aefeSPatrick Mooney 			    sizeof (mseg->name));
1664c87aefeSPatrick Mooney 		}
1674c87aefeSPatrick Mooney 	} else {
1684c87aefeSPatrick Mooney 		bzero(mseg->name, sizeof (mseg->name));
169bf21cd93STycho Nightingale 	}
1704c87aefeSPatrick Mooney 
1714c87aefeSPatrick Mooney 	return (error);
172bf21cd93STycho Nightingale }
173bf21cd93STycho Nightingale 
1744c87aefeSPatrick Mooney static int
vmmdev_devmem_create(vmm_softc_t * sc,struct vm_memseg * mseg,const char * name)1754c87aefeSPatrick Mooney vmmdev_devmem_create(vmm_softc_t *sc, struct vm_memseg *mseg, const char *name)
1764c87aefeSPatrick Mooney {
1774c87aefeSPatrick Mooney 	off_t map_offset;
1784c87aefeSPatrick Mooney 	vmm_devmem_entry_t *entry;
179bf21cd93STycho Nightingale 
1804c87aefeSPatrick Mooney 	if (list_is_empty(&sc->vmm_devmem_list)) {
1814c87aefeSPatrick Mooney 		map_offset = VM_DEVMEM_START;
1824c87aefeSPatrick Mooney 	} else {
1834c87aefeSPatrick Mooney 		entry = list_tail(&sc->vmm_devmem_list);
1844c87aefeSPatrick Mooney 		map_offset = entry->vde_off + entry->vde_len;
1854c87aefeSPatrick Mooney 		if (map_offset < entry->vde_off) {
1864c87aefeSPatrick Mooney 			/* Do not tolerate overflow */
1874c87aefeSPatrick Mooney 			return (ERANGE);
1884c87aefeSPatrick Mooney 		}
189bf21cd93STycho Nightingale 		/*
1904c87aefeSPatrick Mooney 		 * XXXJOY: We could choose to search the list for duplicate
1914c87aefeSPatrick Mooney 		 * names and toss an error.  Since we're using the offset
1924c87aefeSPatrick Mooney 		 * method for now, it does not make much of a difference.
193bf21cd93STycho Nightingale 		 */
194bf21cd93STycho Nightingale 	}
1954c87aefeSPatrick Mooney 
1964c87aefeSPatrick Mooney 	entry = kmem_zalloc(sizeof (*entry), KM_SLEEP);
1974c87aefeSPatrick Mooney 	entry->vde_segid = mseg->segid;
1984c87aefeSPatrick Mooney 	entry->vde_len = mseg->len;
1994c87aefeSPatrick Mooney 	entry->vde_off = map_offset;
2004c87aefeSPatrick Mooney 	(void) strlcpy(entry->vde_name, name, sizeof (entry->vde_name));
2014c87aefeSPatrick Mooney 	list_insert_tail(&sc->vmm_devmem_list, entry);
2024c87aefeSPatrick Mooney 
2034c87aefeSPatrick Mooney 	return (0);
204bf21cd93STycho Nightingale }
205bf21cd93STycho Nightingale 
2064c87aefeSPatrick Mooney static boolean_t
vmmdev_devmem_segid(vmm_softc_t * sc,off_t off,off_t len,int * segidp,off_t * map_offp)207c3d209caSPatrick Mooney vmmdev_devmem_segid(vmm_softc_t *sc, off_t off, off_t len, int *segidp,
208c3d209caSPatrick Mooney     off_t *map_offp)
209bf21cd93STycho Nightingale {
2104c87aefeSPatrick Mooney 	list_t *dl = &sc->vmm_devmem_list;
2114c87aefeSPatrick Mooney 	vmm_devmem_entry_t *de = NULL;
212c3d209caSPatrick Mooney 	const off_t map_end = off + len;
213bf21cd93STycho Nightingale 
2144c87aefeSPatrick Mooney 	VERIFY(off >= VM_DEVMEM_START);
215bf21cd93STycho Nightingale 
216c3d209caSPatrick Mooney 	if (map_end < off) {
217c3d209caSPatrick Mooney 		/* No match on overflow */
2184c87aefeSPatrick Mooney 		return (B_FALSE);
219bf21cd93STycho Nightingale 	}
220bf21cd93STycho Nightingale 
221c3d209caSPatrick Mooney 	for (de = list_head(dl); de != NULL; de = list_next(dl, de)) {
222c3d209caSPatrick Mooney 		const off_t item_end = de->vde_off + de->vde_len;
223c3d209caSPatrick Mooney 
224c3d209caSPatrick Mooney 		if (de->vde_off <= off && item_end >= map_end) {
225c3d209caSPatrick Mooney 			*segidp = de->vde_segid;
226c3d209caSPatrick Mooney 			*map_offp = off - de->vde_off;
227c3d209caSPatrick Mooney 			return (B_TRUE);
228c3d209caSPatrick Mooney 		}
229c3d209caSPatrick Mooney 	}
230c3d209caSPatrick Mooney 	return (B_FALSE);
2314c87aefeSPatrick Mooney }
232bf21cd93STycho Nightingale 
233bf21cd93STycho Nightingale static void
vmmdev_devmem_purge(vmm_softc_t * sc)2344c87aefeSPatrick Mooney vmmdev_devmem_purge(vmm_softc_t *sc)
235bf21cd93STycho Nightingale {
2364c87aefeSPatrick Mooney 	vmm_devmem_entry_t *entry;
2374c87aefeSPatrick Mooney 
2384c87aefeSPatrick Mooney 	while ((entry = list_remove_head(&sc->vmm_devmem_list)) != NULL) {
2394c87aefeSPatrick Mooney 		kmem_free(entry, sizeof (*entry));
2404c87aefeSPatrick Mooney 	}
241bf21cd93STycho Nightingale }
242bf21cd93STycho Nightingale 
2434c87aefeSPatrick Mooney static int
vmmdev_alloc_memseg(vmm_softc_t * sc,struct vm_memseg * mseg)2444c87aefeSPatrick Mooney vmmdev_alloc_memseg(vmm_softc_t *sc, struct vm_memseg *mseg)
245bf21cd93STycho Nightingale {
2464c87aefeSPatrick Mooney 	int error;
2474c87aefeSPatrick Mooney 	bool sysmem = true;
248bf21cd93STycho Nightingale 
2494c87aefeSPatrick Mooney 	if (VM_MEMSEG_NAME(mseg)) {
2504c87aefeSPatrick Mooney 		sysmem = false;
251bf21cd93STycho Nightingale 	}
2524c87aefeSPatrick Mooney 	error = vm_alloc_memseg(sc->vmm_vm, mseg->segid, mseg->len, sysmem);
253bf21cd93STycho Nightingale 
2543d066281SPatrick Mooney 	if (error == 0) {
2554c87aefeSPatrick Mooney 		/*
2564c87aefeSPatrick Mooney 		 * Rather than create a whole fresh device from which userspace
2574c87aefeSPatrick Mooney 		 * can mmap this segment, instead make it available at an
2584c87aefeSPatrick Mooney 		 * offset above where the main guest memory resides.
2594c87aefeSPatrick Mooney 		 */
2604c87aefeSPatrick Mooney 		error = vmmdev_devmem_create(sc, mseg, mseg->name);
2614c87aefeSPatrick Mooney 		if (error != 0) {
2624c87aefeSPatrick Mooney 			vm_free_memseg(sc->vmm_vm, mseg->segid);
2634c87aefeSPatrick Mooney 		}
264bf21cd93STycho Nightingale 	}
2654c87aefeSPatrick Mooney 	return (error);
2664c87aefeSPatrick Mooney }
267bf21cd93STycho Nightingale 
2684c87aefeSPatrick Mooney /*
2694c87aefeSPatrick Mooney  * Resource Locking and Exclusion
2704c87aefeSPatrick Mooney  *
2714c87aefeSPatrick Mooney  * Much of bhyve depends on key portions of VM state, such as the guest memory
2724c87aefeSPatrick Mooney  * map, to remain unchanged while the guest is running.  As ported from
2734c87aefeSPatrick Mooney  * FreeBSD, the initial strategy for this resource exclusion hinged on gating
2744c87aefeSPatrick Mooney  * access to the instance vCPUs.  Threads acting on a single vCPU, like those
2754c87aefeSPatrick Mooney  * performing the work of actually running the guest in VMX/SVM, would lock
2764c87aefeSPatrick Mooney  * only that vCPU during ioctl() entry.  For ioctls which would change VM-wide
2774c87aefeSPatrick Mooney  * state, all of the vCPUs would be first locked, ensuring that the
2784c87aefeSPatrick Mooney  * operation(s) could complete without any other threads stumbling into
2794c87aefeSPatrick Mooney  * intermediate states.
2804c87aefeSPatrick Mooney  *
2814c87aefeSPatrick Mooney  * This approach is largely effective for bhyve.  Common operations, such as
2824c87aefeSPatrick Mooney  * running the vCPUs, steer clear of lock contention.  The model begins to
2834c87aefeSPatrick Mooney  * break down for operations which do not occur in the context of a specific
2844c87aefeSPatrick Mooney  * vCPU.  LAPIC MSI delivery, for example, may be initiated from a worker
2854c87aefeSPatrick Mooney  * thread in the bhyve process.  In order to properly protect those vCPU-less
2864c87aefeSPatrick Mooney  * operations from encountering invalid states, additional locking is required.
2874c87aefeSPatrick Mooney  * This was solved by forcing those operations to lock the VM_MAXCPU-1 vCPU.
2884c87aefeSPatrick Mooney  * It does mean that class of operations will be serialized on locking the
2894c87aefeSPatrick Mooney  * specific vCPU and that instances sized at VM_MAXCPU will potentially see
2904c87aefeSPatrick Mooney  * undue contention on the VM_MAXCPU-1 vCPU.
2914c87aefeSPatrick Mooney  *
2924c87aefeSPatrick Mooney  * In order to address the shortcomings of this model, the concept of a
2934c87aefeSPatrick Mooney  * read/write lock has been added to bhyve.  Operations which change
2944c87aefeSPatrick Mooney  * fundamental aspects of a VM (such as the memory map) must acquire the write
2954c87aefeSPatrick Mooney  * lock, which also implies locking all of the vCPUs and waiting for all read
2964c87aefeSPatrick Mooney  * lock holders to release.  While it increases the cost and waiting time for
2974c87aefeSPatrick Mooney  * those few operations, it allows most hot-path operations on the VM (which
2984c87aefeSPatrick Mooney  * depend on its configuration remaining stable) to occur with minimal locking.
2994c87aefeSPatrick Mooney  *
3004c87aefeSPatrick Mooney  * Consumers of the Driver API (see below) are a special case when it comes to
3014c87aefeSPatrick Mooney  * this locking, since they may hold a read lock via the drv_lease mechanism
3024c87aefeSPatrick Mooney  * for an extended period of time.  Rather than forcing those consumers to
3034c87aefeSPatrick Mooney  * continuously poll for a write lock attempt, the lease system forces them to
3044c87aefeSPatrick Mooney  * provide a release callback to trigger their clean-up (and potential later
3054c87aefeSPatrick Mooney  * reacquisition) of the read lock.
3064c87aefeSPatrick Mooney  */
307bf21cd93STycho Nightingale 
3084c87aefeSPatrick Mooney static void
vcpu_lock_one(vmm_softc_t * sc,int vcpu)3094c87aefeSPatrick Mooney vcpu_lock_one(vmm_softc_t *sc, int vcpu)
3104c87aefeSPatrick Mooney {
3114c87aefeSPatrick Mooney 	ASSERT(vcpu >= 0 && vcpu < VM_MAXCPU);
312bf21cd93STycho Nightingale 
3134c87aefeSPatrick Mooney 	/*
3144c87aefeSPatrick Mooney 	 * Since this state transition is utilizing from_idle=true, it should
3154c87aefeSPatrick Mooney 	 * not fail, but rather block until it can be successful.
3164c87aefeSPatrick Mooney 	 */
3174c87aefeSPatrick Mooney 	VERIFY0(vcpu_set_state(sc->vmm_vm, vcpu, VCPU_FROZEN, true));
3184c87aefeSPatrick Mooney }
319bf21cd93STycho Nightingale 
3204c87aefeSPatrick Mooney static void
vcpu_unlock_one(vmm_softc_t * sc,int vcpu)3214c87aefeSPatrick Mooney vcpu_unlock_one(vmm_softc_t *sc, int vcpu)
3224c87aefeSPatrick Mooney {
3234c87aefeSPatrick Mooney 	ASSERT(vcpu >= 0 && vcpu < VM_MAXCPU);
324bf21cd93STycho Nightingale 
3254c87aefeSPatrick Mooney 	VERIFY3U(vcpu_get_state(sc->vmm_vm, vcpu, NULL), ==, VCPU_FROZEN);
326e0994bd2SPatrick Mooney 	VERIFY0(vcpu_set_state(sc->vmm_vm, vcpu, VCPU_IDLE, false));
327bf21cd93STycho Nightingale }
328bf21cd93STycho Nightingale 
3294c87aefeSPatrick Mooney static void
vmm_read_lock(vmm_softc_t * sc)3304c87aefeSPatrick Mooney vmm_read_lock(vmm_softc_t *sc)
331bf21cd93STycho Nightingale {
3324c87aefeSPatrick Mooney 	rw_enter(&sc->vmm_rwlock, RW_READER);
333bf21cd93STycho Nightingale }
334bf21cd93STycho Nightingale 
3354c87aefeSPatrick Mooney static void
vmm_read_unlock(vmm_softc_t * sc)3364c87aefeSPatrick Mooney vmm_read_unlock(vmm_softc_t *sc)
337bf21cd93STycho Nightingale {
3384c87aefeSPatrick Mooney 	rw_exit(&sc->vmm_rwlock);
339bf21cd93STycho Nightingale }
340bf21cd93STycho Nightingale 
3414c87aefeSPatrick Mooney static void
vmm_write_lock(vmm_softc_t * sc)3424c87aefeSPatrick Mooney vmm_write_lock(vmm_softc_t *sc)
343bf21cd93STycho Nightingale {
3444c87aefeSPatrick Mooney 	int maxcpus;
345bf21cd93STycho Nightingale 
3464c87aefeSPatrick Mooney 	/* First lock all the vCPUs */
3474c87aefeSPatrick Mooney 	maxcpus = vm_get_maxcpus(sc->vmm_vm);
3484c87aefeSPatrick Mooney 	for (int vcpu = 0; vcpu < maxcpus; vcpu++) {
3494c87aefeSPatrick Mooney 		vcpu_lock_one(sc, vcpu);
3504c87aefeSPatrick Mooney 	}
351bf21cd93STycho Nightingale 
3526703a0e8SPatrick Mooney 	/*
3536703a0e8SPatrick Mooney 	 * Block vmm_drv leases from being acquired or held while the VM write
3546703a0e8SPatrick Mooney 	 * lock is held.
3556703a0e8SPatrick Mooney 	 */
3566703a0e8SPatrick Mooney 	vmm_lease_block(sc);
3574c87aefeSPatrick Mooney 
3584c87aefeSPatrick Mooney 	rw_enter(&sc->vmm_rwlock, RW_WRITER);
3594c87aefeSPatrick Mooney 	/*
3604c87aefeSPatrick Mooney 	 * For now, the 'maxcpus' value for an instance is fixed at the
3614c87aefeSPatrick Mooney 	 * compile-time constant of VM_MAXCPU at creation.  If this changes in
3624c87aefeSPatrick Mooney 	 * the future, allowing for dynamic vCPU resource sizing, acquisition
3634c87aefeSPatrick Mooney 	 * of the write lock will need to be wary of such changes.
3644c87aefeSPatrick Mooney 	 */
3654c87aefeSPatrick Mooney 	VERIFY(maxcpus == vm_get_maxcpus(sc->vmm_vm));
366bf21cd93STycho Nightingale }
367bf21cd93STycho Nightingale 
3684c87aefeSPatrick Mooney static void
vmm_write_unlock(vmm_softc_t * sc)3694c87aefeSPatrick Mooney vmm_write_unlock(vmm_softc_t *sc)
370bf21cd93STycho Nightingale {
3714c87aefeSPatrick Mooney 	int maxcpus;
3724c87aefeSPatrick Mooney 
3736703a0e8SPatrick Mooney 	/* Allow vmm_drv leases to be acquired once write lock is dropped */
3746703a0e8SPatrick Mooney 	vmm_lease_unblock(sc);
375bf21cd93STycho Nightingale 
376bf21cd93STycho Nightingale 	/*
3774c87aefeSPatrick Mooney 	 * The VM write lock _must_ be released from the same thread it was
3784c87aefeSPatrick Mooney 	 * acquired in, unlike the read lock.
379bf21cd93STycho Nightingale 	 */
3804c87aefeSPatrick Mooney 	VERIFY(rw_write_held(&sc->vmm_rwlock));
3814c87aefeSPatrick Mooney 	rw_exit(&sc->vmm_rwlock);
3824c87aefeSPatrick Mooney 
3834c87aefeSPatrick Mooney 	/* Unlock all the vCPUs */
3844c87aefeSPatrick Mooney 	maxcpus = vm_get_maxcpus(sc->vmm_vm);
3854c87aefeSPatrick Mooney 	for (int vcpu = 0; vcpu < maxcpus; vcpu++) {
3864c87aefeSPatrick Mooney 		vcpu_unlock_one(sc, vcpu);
3874c87aefeSPatrick Mooney 	}
3884c87aefeSPatrick Mooney }
3894c87aefeSPatrick Mooney 
3904c87aefeSPatrick Mooney static int
vmmdev_do_ioctl(vmm_softc_t * sc,int cmd,intptr_t arg,int md,cred_t * credp,int * rvalp)3914c87aefeSPatrick Mooney vmmdev_do_ioctl(vmm_softc_t *sc, int cmd, intptr_t arg, int md,
3924c87aefeSPatrick Mooney     cred_t *credp, int *rvalp)
3934c87aefeSPatrick Mooney {
3944c87aefeSPatrick Mooney 	int error = 0, vcpu = -1;
3954c87aefeSPatrick Mooney 	void *datap = (void *)arg;
3964c87aefeSPatrick Mooney 	enum vm_lock_type {
3974c87aefeSPatrick Mooney 		LOCK_NONE = 0,
3984c87aefeSPatrick Mooney 		LOCK_VCPU,
3994c87aefeSPatrick Mooney 		LOCK_READ_HOLD,
4004c87aefeSPatrick Mooney 		LOCK_WRITE_HOLD
4014c87aefeSPatrick Mooney 	} lock_type = LOCK_NONE;
4024c87aefeSPatrick Mooney 
4034c87aefeSPatrick Mooney 	/* Acquire any exclusion resources needed for the operation. */
404bf21cd93STycho Nightingale 	switch (cmd) {
405bf21cd93STycho Nightingale 	case VM_RUN:
406bf21cd93STycho Nightingale 	case VM_GET_REGISTER:
407bf21cd93STycho Nightingale 	case VM_SET_REGISTER:
408bf21cd93STycho Nightingale 	case VM_GET_SEGMENT_DESCRIPTOR:
409bf21cd93STycho Nightingale 	case VM_SET_SEGMENT_DESCRIPTOR:
4104c87aefeSPatrick Mooney 	case VM_GET_REGISTER_SET:
4114c87aefeSPatrick Mooney 	case VM_SET_REGISTER_SET:
412bf21cd93STycho Nightingale 	case VM_INJECT_EXCEPTION:
413bf21cd93STycho Nightingale 	case VM_GET_CAPABILITY:
414bf21cd93STycho Nightingale 	case VM_SET_CAPABILITY:
415bf21cd93STycho Nightingale 	case VM_PPTDEV_MSI:
416bf21cd93STycho Nightingale 	case VM_PPTDEV_MSIX:
417bf21cd93STycho Nightingale 	case VM_SET_X2APIC_STATE:
418bf21cd93STycho Nightingale 	case VM_GLA2GPA:
4194c87aefeSPatrick Mooney 	case VM_GLA2GPA_NOFAULT:
420bf21cd93STycho Nightingale 	case VM_ACTIVATE_CPU:
4214c87aefeSPatrick Mooney 	case VM_SET_INTINFO:
4224c87aefeSPatrick Mooney 	case VM_GET_INTINFO:
423bf21cd93STycho Nightingale 	case VM_RESTART_INSTRUCTION:
424154972afSPatrick Mooney 	case VM_SET_KERNEMU_DEV:
425154972afSPatrick Mooney 	case VM_GET_KERNEMU_DEV:
4262606939dSPatrick Mooney 	case VM_RESET_CPU:
4272606939dSPatrick Mooney 	case VM_GET_RUN_STATE:
4282606939dSPatrick Mooney 	case VM_SET_RUN_STATE:
429957246c9SPatrick Mooney 	case VM_GET_FPU:
430957246c9SPatrick Mooney 	case VM_SET_FPU:
431bf21cd93STycho Nightingale 		/*
4324c87aefeSPatrick Mooney 		 * Copy in the ID of the vCPU chosen for this operation.
4334c87aefeSPatrick Mooney 		 * Since a nefarious caller could update their struct between
4344c87aefeSPatrick Mooney 		 * this locking and when the rest of the ioctl data is copied
4354c87aefeSPatrick Mooney 		 * in, it is _critical_ that this local 'vcpu' variable be used
4364c87aefeSPatrick Mooney 		 * rather than the in-struct one when performing the ioctl.
437bf21cd93STycho Nightingale 		 */
4384c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vcpu, sizeof (vcpu), md)) {
439bf21cd93STycho Nightingale 			return (EFAULT);
440bf21cd93STycho Nightingale 		}
4414c87aefeSPatrick Mooney 		if (vcpu < 0 || vcpu > vm_get_maxcpus(sc->vmm_vm)) {
4424c87aefeSPatrick Mooney 			return (EINVAL);
443bf21cd93STycho Nightingale 		}
4444c87aefeSPatrick Mooney 		vcpu_lock_one(sc, vcpu);
4454c87aefeSPatrick Mooney 		lock_type = LOCK_VCPU;
446bf21cd93STycho Nightingale 		break;
447bf21cd93STycho Nightingale 
4484c87aefeSPatrick Mooney 	case VM_REINIT:
4494c87aefeSPatrick Mooney 	case VM_BIND_PPTDEV:
4504c87aefeSPatrick Mooney 	case VM_UNBIND_PPTDEV:
4514c87aefeSPatrick Mooney 	case VM_MAP_PPTDEV_MMIO:
4522b948146SAndy Fiddaman 	case VM_UNMAP_PPTDEV_MMIO:
4534c87aefeSPatrick Mooney 	case VM_ALLOC_MEMSEG:
4544c87aefeSPatrick Mooney 	case VM_MMAP_MEMSEG:
4552b948146SAndy Fiddaman 	case VM_MUNMAP_MEMSEG:
4564c87aefeSPatrick Mooney 	case VM_WRLOCK_CYCLE:
4570e1453c3SPatrick Mooney 	case VM_PMTMR_LOCATE:
4584c87aefeSPatrick Mooney 		vmm_write_lock(sc);
4594c87aefeSPatrick Mooney 		lock_type = LOCK_WRITE_HOLD;
4604c87aefeSPatrick Mooney 		break;
461bf21cd93STycho Nightingale 
4624c87aefeSPatrick Mooney 	case VM_GET_MEMSEG:
4634c87aefeSPatrick Mooney 	case VM_MMAP_GETNEXT:
4644c87aefeSPatrick Mooney 	case VM_LAPIC_IRQ:
4654c87aefeSPatrick Mooney 	case VM_INJECT_NMI:
4664c87aefeSPatrick Mooney 	case VM_IOAPIC_ASSERT_IRQ:
4674c87aefeSPatrick Mooney 	case VM_IOAPIC_DEASSERT_IRQ:
4684c87aefeSPatrick Mooney 	case VM_IOAPIC_PULSE_IRQ:
4694c87aefeSPatrick Mooney 	case VM_LAPIC_MSI:
4704c87aefeSPatrick Mooney 	case VM_LAPIC_LOCAL_IRQ:
4714c87aefeSPatrick Mooney 	case VM_GET_X2APIC_STATE:
4724c87aefeSPatrick Mooney 	case VM_RTC_READ:
4734c87aefeSPatrick Mooney 	case VM_RTC_WRITE:
4744c87aefeSPatrick Mooney 	case VM_RTC_SETTIME:
4754c87aefeSPatrick Mooney 	case VM_RTC_GETTIME:
4766960cd89SAndy Fiddaman 	case VM_PPTDEV_DISABLE_MSIX:
4774c87aefeSPatrick Mooney 	case VM_DEVMEM_GETOFFSET:
4788779b448SDan Cross 	case VM_TRACK_DIRTY_PAGES:
4794c87aefeSPatrick Mooney 		vmm_read_lock(sc);
4804c87aefeSPatrick Mooney 		lock_type = LOCK_READ_HOLD;
481bf21cd93STycho Nightingale 		break;
482bf21cd93STycho Nightingale 
483d515dd77SPatrick Mooney 	case VM_DATA_READ:
484d515dd77SPatrick Mooney 	case VM_DATA_WRITE:
485d515dd77SPatrick Mooney 		if (ddi_copyin(datap, &vcpu, sizeof (vcpu), md)) {
486d515dd77SPatrick Mooney 			return (EFAULT);
487d515dd77SPatrick Mooney 		}
488d515dd77SPatrick Mooney 		if (vcpu == -1) {
489d515dd77SPatrick Mooney 			/* Access data for VM-wide devices */
490d515dd77SPatrick Mooney 			vmm_write_lock(sc);
491d515dd77SPatrick Mooney 			lock_type = LOCK_WRITE_HOLD;
492d515dd77SPatrick Mooney 		} else if (vcpu >= 0 && vcpu < vm_get_maxcpus(sc->vmm_vm)) {
493d515dd77SPatrick Mooney 			/* Access data associated with a specific vCPU */
494d515dd77SPatrick Mooney 			vcpu_lock_one(sc, vcpu);
495d515dd77SPatrick Mooney 			lock_type = LOCK_VCPU;
496d515dd77SPatrick Mooney 		} else {
497d515dd77SPatrick Mooney 			return (EINVAL);
498d515dd77SPatrick Mooney 		}
499d515dd77SPatrick Mooney 		break;
500d515dd77SPatrick Mooney 
5010153d828SPatrick Mooney 	case VM_GET_GPA_PMAP:
5024c87aefeSPatrick Mooney 	case VM_IOAPIC_PINCOUNT:
50352fac30eSPatrick Mooney 	case VM_SUSPEND:
504957246c9SPatrick Mooney 	case VM_DESC_FPU_AREA:
505bf21cd93STycho Nightingale 	default:
506bf21cd93STycho Nightingale 		break;
507bf21cd93STycho Nightingale 	}
508bf21cd93STycho Nightingale 
5094c87aefeSPatrick Mooney 	/* Execute the primary logic for the ioctl. */
5104c87aefeSPatrick Mooney 	switch (cmd) {
5114c87aefeSPatrick Mooney 	case VM_RUN: {
512e0c0d44eSPatrick Mooney 		struct vm_entry entry;
5134c87aefeSPatrick Mooney 
514e0c0d44eSPatrick Mooney 		if (ddi_copyin(datap, &entry, sizeof (entry), md)) {
5154c87aefeSPatrick Mooney 			error = EFAULT;
5164c87aefeSPatrick Mooney 			break;
517bf21cd93STycho Nightingale 		}
5184c87aefeSPatrick Mooney 
5194c87aefeSPatrick Mooney 		if (!(curthread->t_schedflag & TS_VCPU))
5204c87aefeSPatrick Mooney 			smt_mark_as_vcpu();
5214c87aefeSPatrick Mooney 
522e0c0d44eSPatrick Mooney 		error = vm_run(sc->vmm_vm, vcpu, &entry);
523e0c0d44eSPatrick Mooney 
5243e1c5f3aSPatrick Mooney 		/*
5253e1c5f3aSPatrick Mooney 		 * Unexpected states in vm_run() are expressed through positive
5263e1c5f3aSPatrick Mooney 		 * errno-oriented return values.  VM states which expect further
5273e1c5f3aSPatrick Mooney 		 * processing in userspace (necessary context via exitinfo) are
5283e1c5f3aSPatrick Mooney 		 * expressed through negative return values.  For the time being
5293e1c5f3aSPatrick Mooney 		 * a return value of 0 is not expected from vm_run().
5303e1c5f3aSPatrick Mooney 		 */
5313e1c5f3aSPatrick Mooney 		ASSERT(error != 0);
5323e1c5f3aSPatrick Mooney 		if (error < 0) {
533e0c0d44eSPatrick Mooney 			const struct vm_exit *vme;
534e0c0d44eSPatrick Mooney 			void *outp = entry.exit_data;
535e0c0d44eSPatrick Mooney 
5363e1c5f3aSPatrick Mooney 			error = 0;
537e0c0d44eSPatrick Mooney 			vme = vm_exitinfo(sc->vmm_vm, vcpu);
538e0c0d44eSPatrick Mooney 			if (ddi_copyout(vme, outp, sizeof (*vme), md)) {
539e0c0d44eSPatrick Mooney 				error = EFAULT;
540e0c0d44eSPatrick Mooney 			}
541bf21cd93STycho Nightingale 		}
542bf21cd93STycho Nightingale 		break;
5434c87aefeSPatrick Mooney 	}
5444c87aefeSPatrick Mooney 	case VM_SUSPEND: {
5454c87aefeSPatrick Mooney 		struct vm_suspend vmsuspend;
5464c87aefeSPatrick Mooney 
5474c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmsuspend, sizeof (vmsuspend), md)) {
5484c87aefeSPatrick Mooney 			error = EFAULT;
5494c87aefeSPatrick Mooney 			break;
550bf21cd93STycho Nightingale 		}
5514c87aefeSPatrick Mooney 		error = vm_suspend(sc->vmm_vm, vmsuspend.how);
552bf21cd93STycho Nightingale 		break;
5534c87aefeSPatrick Mooney 	}
55452fac30eSPatrick Mooney 	case VM_REINIT: {
55552fac30eSPatrick Mooney 		struct vm_reinit reinit;
55652fac30eSPatrick Mooney 
55752fac30eSPatrick Mooney 		if (ddi_copyin(datap, &reinit, sizeof (reinit), md)) {
55852fac30eSPatrick Mooney 			error = EFAULT;
55952fac30eSPatrick Mooney 			break;
56052fac30eSPatrick Mooney 		}
5614c87aefeSPatrick Mooney 		if ((error = vmm_drv_block_hook(sc, B_TRUE)) != 0) {
5624c87aefeSPatrick Mooney 			/*
5634c87aefeSPatrick Mooney 			 * The VM instance should be free of driver-attached
5644c87aefeSPatrick Mooney 			 * hooks during the reinitialization process.
5654c87aefeSPatrick Mooney 			 */
5664c87aefeSPatrick Mooney 			break;
567bf21cd93STycho Nightingale 		}
56852fac30eSPatrick Mooney 		error = vm_reinit(sc->vmm_vm, reinit.flags);
5694c87aefeSPatrick Mooney 		(void) vmm_drv_block_hook(sc, B_FALSE);
570bf21cd93STycho Nightingale 		break;
57152fac30eSPatrick Mooney 	}
5724c87aefeSPatrick Mooney 	case VM_STAT_DESC: {
5734c87aefeSPatrick Mooney 		struct vm_stat_desc statdesc;
5744c87aefeSPatrick Mooney 
5754c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &statdesc, sizeof (statdesc), md)) {
5764c87aefeSPatrick Mooney 			error = EFAULT;
5774c87aefeSPatrick Mooney 			break;
578bf21cd93STycho Nightingale 		}
5794c87aefeSPatrick Mooney 		error = vmm_stat_desc_copy(statdesc.index, statdesc.desc,
5804c87aefeSPatrick Mooney 		    sizeof (statdesc.desc));
5814c87aefeSPatrick Mooney 		if (error == 0 &&
5824c87aefeSPatrick Mooney 		    ddi_copyout(&statdesc, datap, sizeof (statdesc), md)) {
5834c87aefeSPatrick Mooney 			error = EFAULT;
5844c87aefeSPatrick Mooney 			break;
585bf21cd93STycho Nightingale 		}
586bf21cd93STycho Nightingale 		break;
5874c87aefeSPatrick Mooney 	}
5884c87aefeSPatrick Mooney 	case VM_STATS_IOC: {
5894c87aefeSPatrick Mooney 		struct vm_stats vmstats;
5904c87aefeSPatrick Mooney 
5914c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmstats, sizeof (vmstats), md)) {
5924c87aefeSPatrick Mooney 			error = EFAULT;
5934c87aefeSPatrick Mooney 			break;
594bf21cd93STycho Nightingale 		}
5954c87aefeSPatrick Mooney 		hrt2tv(gethrtime(), &vmstats.tv);
596d7b72f7bSAndy Fiddaman 		error = vmm_stat_copy(sc->vmm_vm, vmstats.cpuid, vmstats.index,
597d7b72f7bSAndy Fiddaman 		    nitems(vmstats.statbuf),
5984c87aefeSPatrick Mooney 		    &vmstats.num_entries, vmstats.statbuf);
5994c87aefeSPatrick Mooney 		if (error == 0 &&
6004c87aefeSPatrick Mooney 		    ddi_copyout(&vmstats, datap, sizeof (vmstats), md)) {
6014c87aefeSPatrick Mooney 			error = EFAULT;
6024c87aefeSPatrick Mooney 			break;
603bf21cd93STycho Nightingale 		}
604bf21cd93STycho Nightingale 		break;
6054c87aefeSPatrick Mooney 	}
6064c87aefeSPatrick Mooney 
6074c87aefeSPatrick Mooney 	case VM_PPTDEV_MSI: {
6084c87aefeSPatrick Mooney 		struct vm_pptdev_msi pptmsi;
6094c87aefeSPatrick Mooney 
6104c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &pptmsi, sizeof (pptmsi), md)) {
6114c87aefeSPatrick Mooney 			error = EFAULT;
6124c87aefeSPatrick Mooney 			break;
613bf21cd93STycho Nightingale 		}
614eb9a1df2SHans Rosenfeld 		error = ppt_setup_msi(sc->vmm_vm, pptmsi.vcpu, pptmsi.pptfd,
615eb9a1df2SHans Rosenfeld 		    pptmsi.addr, pptmsi.msg, pptmsi.numvec);
616eb9a1df2SHans Rosenfeld 		break;
6174c87aefeSPatrick Mooney 	}
6184c87aefeSPatrick Mooney 	case VM_PPTDEV_MSIX: {
6194c87aefeSPatrick Mooney 		struct vm_pptdev_msix pptmsix;
6204c87aefeSPatrick Mooney 
6214c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &pptmsix, sizeof (pptmsix), md)) {
6224c87aefeSPatrick Mooney 			error = EFAULT;
6234c87aefeSPatrick Mooney 			break;
624bf21cd93STycho Nightingale 		}
625eb9a1df2SHans Rosenfeld 		error = ppt_setup_msix(sc->vmm_vm, pptmsix.vcpu, pptmsix.pptfd,
626eb9a1df2SHans Rosenfeld 		    pptmsix.idx, pptmsix.addr, pptmsix.msg,
627eb9a1df2SHans Rosenfeld 		    pptmsix.vector_control);
628eb9a1df2SHans Rosenfeld 		break;
6294c87aefeSPatrick Mooney 	}
6306960cd89SAndy Fiddaman 	case VM_PPTDEV_DISABLE_MSIX: {
6316960cd89SAndy Fiddaman 		struct vm_pptdev pptdev;
6326960cd89SAndy Fiddaman 
6336960cd89SAndy Fiddaman 		if (ddi_copyin(datap, &pptdev, sizeof (pptdev), md)) {
6346960cd89SAndy Fiddaman 			error = EFAULT;
6356960cd89SAndy Fiddaman 			break;
6366960cd89SAndy Fiddaman 		}
6376960cd89SAndy Fiddaman 		error = ppt_disable_msix(sc->vmm_vm, pptdev.pptfd);
6386960cd89SAndy Fiddaman 		break;
6396960cd89SAndy Fiddaman 	}
6404c87aefeSPatrick Mooney 	case VM_MAP_PPTDEV_MMIO: {
6414c87aefeSPatrick Mooney 		struct vm_pptdev_mmio pptmmio;
6424c87aefeSPatrick Mooney 
6434c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &pptmmio, sizeof (pptmmio), md)) {
6444c87aefeSPatrick Mooney 			error = EFAULT;
6454c87aefeSPatrick Mooney 			break;
646bf21cd93STycho Nightingale 		}
647eb9a1df2SHans Rosenfeld 		error = ppt_map_mmio(sc->vmm_vm, pptmmio.pptfd, pptmmio.gpa,
648eb9a1df2SHans Rosenfeld 		    pptmmio.len, pptmmio.hpa);
649eb9a1df2SHans Rosenfeld 		break;
650eb9a1df2SHans Rosenfeld 	}
6512b948146SAndy Fiddaman 	case VM_UNMAP_PPTDEV_MMIO: {
6522b948146SAndy Fiddaman 		struct vm_pptdev_mmio pptmmio;
6532b948146SAndy Fiddaman 
6542b948146SAndy Fiddaman 		if (ddi_copyin(datap, &pptmmio, sizeof (pptmmio), md)) {
6552b948146SAndy Fiddaman 			error = EFAULT;
6562b948146SAndy Fiddaman 			break;
6572b948146SAndy Fiddaman 		}
6582b948146SAndy Fiddaman 		error = ppt_unmap_mmio(sc->vmm_vm, pptmmio.pptfd, pptmmio.gpa,
6592b948146SAndy Fiddaman 		    pptmmio.len);
6602b948146SAndy Fiddaman 		break;
6612b948146SAndy Fiddaman 	}
662eb9a1df2SHans Rosenfeld 	case VM_BIND_PPTDEV: {
663eb9a1df2SHans Rosenfeld 		struct vm_pptdev pptdev;
664eb9a1df2SHans Rosenfeld 
665eb9a1df2SHans Rosenfeld 		if (ddi_copyin(datap, &pptdev, sizeof (pptdev), md)) {
666eb9a1df2SHans Rosenfeld 			error = EFAULT;
667eb9a1df2SHans Rosenfeld 			break;
668eb9a1df2SHans Rosenfeld 		}
669eb9a1df2SHans Rosenfeld 		error = vm_assign_pptdev(sc->vmm_vm, pptdev.pptfd);
670eb9a1df2SHans Rosenfeld 		break;
6714c87aefeSPatrick Mooney 	}
6724c87aefeSPatrick Mooney 	case VM_UNBIND_PPTDEV: {
6734c87aefeSPatrick Mooney 		struct vm_pptdev pptdev;
6744c87aefeSPatrick Mooney 
6754c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &pptdev, sizeof (pptdev), md)) {
6764c87aefeSPatrick Mooney 			error = EFAULT;
6774c87aefeSPatrick Mooney 			break;
678bf21cd93STycho Nightingale 		}
679eb9a1df2SHans Rosenfeld 		error = vm_unassign_pptdev(sc->vmm_vm, pptdev.pptfd);
680eb9a1df2SHans Rosenfeld 		break;
6814c87aefeSPatrick Mooney 	}
682eb9a1df2SHans Rosenfeld 	case VM_GET_PPTDEV_LIMITS: {
683eb9a1df2SHans Rosenfeld 		struct vm_pptdev_limits pptlimits;
6844c87aefeSPatrick Mooney 
685eb9a1df2SHans Rosenfeld 		if (ddi_copyin(datap, &pptlimits, sizeof (pptlimits), md)) {
686eb9a1df2SHans Rosenfeld 			error = EFAULT;
687eb9a1df2SHans Rosenfeld 			break;
688eb9a1df2SHans Rosenfeld 		}
689eb9a1df2SHans Rosenfeld 		error = ppt_get_limits(sc->vmm_vm, pptlimits.pptfd,
690eb9a1df2SHans Rosenfeld 		    &pptlimits.msi_limit, &pptlimits.msix_limit);
691eb9a1df2SHans Rosenfeld 		if (error == 0 &&
692eb9a1df2SHans Rosenfeld 		    ddi_copyout(&pptlimits, datap, sizeof (pptlimits), md)) {
693eb9a1df2SHans Rosenfeld 			error = EFAULT;
694eb9a1df2SHans Rosenfeld 			break;
695eb9a1df2SHans Rosenfeld 		}
696eb9a1df2SHans Rosenfeld 		break;
697eb9a1df2SHans Rosenfeld 	}
6984c87aefeSPatrick Mooney 	case VM_INJECT_EXCEPTION: {
6994c87aefeSPatrick Mooney 		struct vm_exception vmexc;
7004c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmexc, sizeof (vmexc), md)) {
7014c87aefeSPatrick Mooney 			error = EFAULT;
7024c87aefeSPatrick Mooney 			break;
703bf21cd93STycho Nightingale 		}
7044c87aefeSPatrick Mooney 		error = vm_inject_exception(sc->vmm_vm, vcpu, vmexc.vector,
7053d097f7dSPatrick Mooney 		    vmexc.error_code_valid != 0, vmexc.error_code,
7063d097f7dSPatrick Mooney 		    vmexc.restart_instruction != 0);
707bf21cd93STycho Nightingale 		break;
7084c87aefeSPatrick Mooney 	}
7094c87aefeSPatrick Mooney 	case VM_INJECT_NMI: {
7104c87aefeSPatrick Mooney 		struct vm_nmi vmnmi;
7114c87aefeSPatrick Mooney 
7124c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmnmi, sizeof (vmnmi), md)) {
7134c87aefeSPatrick Mooney 			error = EFAULT;
7144c87aefeSPatrick Mooney 			break;
715bf21cd93STycho Nightingale 		}
7164c87aefeSPatrick Mooney 		error = vm_inject_nmi(sc->vmm_vm, vmnmi.cpuid);
717bf21cd93STycho Nightingale 		break;
7184c87aefeSPatrick Mooney 	}
7194c87aefeSPatrick Mooney 	case VM_LAPIC_IRQ: {
7204c87aefeSPatrick Mooney 		struct vm_lapic_irq vmirq;
7214c87aefeSPatrick Mooney 
7224c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmirq, sizeof (vmirq), md)) {
7234c87aefeSPatrick Mooney 			error = EFAULT;
7244c87aefeSPatrick Mooney 			break;
725bf21cd93STycho Nightingale 		}
7264c87aefeSPatrick Mooney 		error = lapic_intr_edge(sc->vmm_vm, vmirq.cpuid, vmirq.vector);
727bf21cd93STycho Nightingale 		break;
7284c87aefeSPatrick Mooney 	}
7294c87aefeSPatrick Mooney 	case VM_LAPIC_LOCAL_IRQ: {
7304c87aefeSPatrick Mooney 		struct vm_lapic_irq vmirq;
7314c87aefeSPatrick Mooney 
7324c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmirq, sizeof (vmirq), md)) {
7334c87aefeSPatrick Mooney 			error = EFAULT;
7344c87aefeSPatrick Mooney 			break;
735bf21cd93STycho Nightingale 		}
7364c87aefeSPatrick Mooney 		error = lapic_set_local_intr(sc->vmm_vm, vmirq.cpuid,
7374c87aefeSPatrick Mooney 		    vmirq.vector);
738bf21cd93STycho Nightingale 		break;
7394c87aefeSPatrick Mooney 	}
7404c87aefeSPatrick Mooney 	case VM_LAPIC_MSI: {
7414c87aefeSPatrick Mooney 		struct vm_lapic_msi vmmsi;
7424c87aefeSPatrick Mooney 
7434c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmmsi, sizeof (vmmsi), md)) {
7444c87aefeSPatrick Mooney 			error = EFAULT;
7454c87aefeSPatrick Mooney 			break;
746bf21cd93STycho Nightingale 		}
7474c87aefeSPatrick Mooney 		error = lapic_intr_msi(sc->vmm_vm, vmmsi.addr, vmmsi.msg);
748bf21cd93STycho Nightingale 		break;
7494c87aefeSPatrick Mooney 	}
7504c87aefeSPatrick Mooney 
7514c87aefeSPatrick Mooney 	case VM_IOAPIC_ASSERT_IRQ: {
7524c87aefeSPatrick Mooney 		struct vm_ioapic_irq ioapic_irq;
7534c87aefeSPatrick Mooney 
7544c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &ioapic_irq, sizeof (ioapic_irq), md)) {
7554c87aefeSPatrick Mooney 			error = EFAULT;
7564c87aefeSPatrick Mooney 			break;
757bf21cd93STycho Nightingale 		}
7584c87aefeSPatrick Mooney 		error = vioapic_assert_irq(sc->vmm_vm, ioapic_irq.irq);
759bf21cd93STycho Nightingale 		break;
7604c87aefeSPatrick Mooney 	}
7614c87aefeSPatrick Mooney 	case VM_IOAPIC_DEASSERT_IRQ: {
7624c87aefeSPatrick Mooney 		struct vm_ioapic_irq ioapic_irq;
7634c87aefeSPatrick Mooney 
7644c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &ioapic_irq, sizeof (ioapic_irq), md)) {
7654c87aefeSPatrick Mooney 			error = EFAULT;
7664c87aefeSPatrick Mooney 			break;
767bf21cd93STycho Nightingale 		}
7684c87aefeSPatrick Mooney 		error = vioapic_deassert_irq(sc->vmm_vm, ioapic_irq.irq);
769bf21cd93STycho Nightingale 		break;
7704c87aefeSPatrick Mooney 	}
7714c87aefeSPatrick Mooney 	case VM_IOAPIC_PULSE_IRQ: {
7724c87aefeSPatrick Mooney 		struct vm_ioapic_irq ioapic_irq;
7734c87aefeSPatrick Mooney 
7744c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &ioapic_irq, sizeof (ioapic_irq), md)) {
7754c87aefeSPatrick Mooney 			error = EFAULT;
7764c87aefeSPatrick Mooney 			break;
777bf21cd93STycho Nightingale 		}
7784c87aefeSPatrick Mooney 		error = vioapic_pulse_irq(sc->vmm_vm, ioapic_irq.irq);
779bf21cd93STycho Nightingale 		break;
7804c87aefeSPatrick Mooney 	}
7814c87aefeSPatrick Mooney 	case VM_IOAPIC_PINCOUNT: {
7824c87aefeSPatrick Mooney 		int pincount;
7834c87aefeSPatrick Mooney 
7844c87aefeSPatrick Mooney 		pincount = vioapic_pincount(sc->vmm_vm);
7854c87aefeSPatrick Mooney 		if (ddi_copyout(&pincount, datap, sizeof (int), md)) {
7864c87aefeSPatrick Mooney 			error = EFAULT;
7874c87aefeSPatrick Mooney 			break;
788bf21cd93STycho Nightingale 		}
7894c87aefeSPatrick Mooney 		break;
7904c87aefeSPatrick Mooney 	}
791957246c9SPatrick Mooney 	case VM_DESC_FPU_AREA: {
792957246c9SPatrick Mooney 		struct vm_fpu_desc desc;
793957246c9SPatrick Mooney 		void *buf = NULL;
794957246c9SPatrick Mooney 
795957246c9SPatrick Mooney 		if (ddi_copyin(datap, &desc, sizeof (desc), md)) {
796957246c9SPatrick Mooney 			error = EFAULT;
797957246c9SPatrick Mooney 			break;
798957246c9SPatrick Mooney 		}
799957246c9SPatrick Mooney 		if (desc.vfd_num_entries > 64) {
800957246c9SPatrick Mooney 			error = EINVAL;
801957246c9SPatrick Mooney 			break;
802957246c9SPatrick Mooney 		}
803957246c9SPatrick Mooney 		const size_t buf_sz = sizeof (struct vm_fpu_desc_entry) *
804957246c9SPatrick Mooney 		    desc.vfd_num_entries;
805957246c9SPatrick Mooney 		if (buf_sz != 0) {
806957246c9SPatrick Mooney 			buf = kmem_zalloc(buf_sz, KM_SLEEP);
807957246c9SPatrick Mooney 		}
808957246c9SPatrick Mooney 
809957246c9SPatrick Mooney 		/*
810957246c9SPatrick Mooney 		 * For now, we are depending on vm_fpu_desc_entry and
811957246c9SPatrick Mooney 		 * hma_xsave_state_desc_t having the same format.
812957246c9SPatrick Mooney 		 */
813957246c9SPatrick Mooney 		CTASSERT(sizeof (struct vm_fpu_desc_entry) ==
814957246c9SPatrick Mooney 		    sizeof (hma_xsave_state_desc_t));
815957246c9SPatrick Mooney 
816957246c9SPatrick Mooney 		size_t req_size;
817957246c9SPatrick Mooney 		const uint_t max_entries = hma_fpu_describe_xsave_state(
818957246c9SPatrick Mooney 		    (hma_xsave_state_desc_t *)buf,
819957246c9SPatrick Mooney 		    desc.vfd_num_entries,
820957246c9SPatrick Mooney 		    &req_size);
821957246c9SPatrick Mooney 
822957246c9SPatrick Mooney 		desc.vfd_req_size = req_size;
823957246c9SPatrick Mooney 		desc.vfd_num_entries = max_entries;
824957246c9SPatrick Mooney 		if (buf_sz != 0) {
825957246c9SPatrick Mooney 			if (ddi_copyout(buf, desc.vfd_entry_data, buf_sz, md)) {
826957246c9SPatrick Mooney 				error = EFAULT;
827957246c9SPatrick Mooney 			}
828957246c9SPatrick Mooney 			kmem_free(buf, buf_sz);
829957246c9SPatrick Mooney 		}
830957246c9SPatrick Mooney 
831957246c9SPatrick Mooney 		if (error == 0) {
832957246c9SPatrick Mooney 			if (ddi_copyout(&desc, datap, sizeof (desc), md)) {
833957246c9SPatrick Mooney 				error = EFAULT;
834957246c9SPatrick Mooney 			}
835957246c9SPatrick Mooney 		}
836957246c9SPatrick Mooney 		break;
837957246c9SPatrick Mooney 	}
8384c87aefeSPatrick Mooney 
8394c87aefeSPatrick Mooney 	case VM_ISA_ASSERT_IRQ: {
8404c87aefeSPatrick Mooney 		struct vm_isa_irq isa_irq;
8414c87aefeSPatrick Mooney 
8424c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &isa_irq, sizeof (isa_irq), md)) {
8434c87aefeSPatrick Mooney 			error = EFAULT;
8444c87aefeSPatrick Mooney 			break;
8454c87aefeSPatrick Mooney 		}
8464c87aefeSPatrick Mooney 		error = vatpic_assert_irq(sc->vmm_vm, isa_irq.atpic_irq);
8474c87aefeSPatrick Mooney 		if (error == 0 && isa_irq.ioapic_irq != -1) {
8484c87aefeSPatrick Mooney 			error = vioapic_assert_irq(sc->vmm_vm,
8494c87aefeSPatrick Mooney 			    isa_irq.ioapic_irq);
8504c87aefeSPatrick Mooney 		}
8514c87aefeSPatrick Mooney 		break;
8524c87aefeSPatrick Mooney 	}
8534c87aefeSPatrick Mooney 	case VM_ISA_DEASSERT_IRQ: {
8544c87aefeSPatrick Mooney 		struct vm_isa_irq isa_irq;
8554c87aefeSPatrick Mooney 
8564c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &isa_irq, sizeof (isa_irq), md)) {
8574c87aefeSPatrick Mooney 			error = EFAULT;
8584c87aefeSPatrick Mooney 			break;
8594c87aefeSPatrick Mooney 		}
8604c87aefeSPatrick Mooney 		error = vatpic_deassert_irq(sc->vmm_vm, isa_irq.atpic_irq);
8614c87aefeSPatrick Mooney 		if (error == 0 && isa_irq.ioapic_irq != -1) {
8624c87aefeSPatrick Mooney 			error = vioapic_deassert_irq(sc->vmm_vm,
8634c87aefeSPatrick Mooney 			    isa_irq.ioapic_irq);
8644c87aefeSPatrick Mooney 		}
8654c87aefeSPatrick Mooney 		break;
8664c87aefeSPatrick Mooney 	}
8674c87aefeSPatrick Mooney 	case VM_ISA_PULSE_IRQ: {
8684c87aefeSPatrick Mooney 		struct vm_isa_irq isa_irq;
8694c87aefeSPatrick Mooney 
8704c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &isa_irq, sizeof (isa_irq), md)) {
8714c87aefeSPatrick Mooney 			error = EFAULT;
8724c87aefeSPatrick Mooney 			break;
8734c87aefeSPatrick Mooney 		}
8744c87aefeSPatrick Mooney 		error = vatpic_pulse_irq(sc->vmm_vm, isa_irq.atpic_irq);
8754c87aefeSPatrick Mooney 		if (error == 0 && isa_irq.ioapic_irq != -1) {
8764c87aefeSPatrick Mooney 			error = vioapic_pulse_irq(sc->vmm_vm,
8774c87aefeSPatrick Mooney 			    isa_irq.ioapic_irq);
8784c87aefeSPatrick Mooney 		}
8794c87aefeSPatrick Mooney 		break;
8804c87aefeSPatrick Mooney 	}
8814c87aefeSPatrick Mooney 	case VM_ISA_SET_IRQ_TRIGGER: {
8824c87aefeSPatrick Mooney 		struct vm_isa_irq_trigger isa_irq_trigger;
8834c87aefeSPatrick Mooney 
8844c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &isa_irq_trigger,
8854c87aefeSPatrick Mooney 		    sizeof (isa_irq_trigger), md)) {
8864c87aefeSPatrick Mooney 			error = EFAULT;
8874c87aefeSPatrick Mooney 			break;
8884c87aefeSPatrick Mooney 		}
8894c87aefeSPatrick Mooney 		error = vatpic_set_irq_trigger(sc->vmm_vm,
8904c87aefeSPatrick Mooney 		    isa_irq_trigger.atpic_irq, isa_irq_trigger.trigger);
8914c87aefeSPatrick Mooney 		break;
8924c87aefeSPatrick Mooney 	}
8934c87aefeSPatrick Mooney 
8944c87aefeSPatrick Mooney 	case VM_MMAP_GETNEXT: {
8954c87aefeSPatrick Mooney 		struct vm_memmap mm;
8964c87aefeSPatrick Mooney 
8974c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &mm, sizeof (mm), md)) {
8984c87aefeSPatrick Mooney 			error = EFAULT;
8994c87aefeSPatrick Mooney 			break;
9004c87aefeSPatrick Mooney 		}
9014c87aefeSPatrick Mooney 		error = vm_mmap_getnext(sc->vmm_vm, &mm.gpa, &mm.segid,
9024c87aefeSPatrick Mooney 		    &mm.segoff, &mm.len, &mm.prot, &mm.flags);
9034c87aefeSPatrick Mooney 		if (error == 0 && ddi_copyout(&mm, datap, sizeof (mm), md)) {
9044c87aefeSPatrick Mooney 			error = EFAULT;
9054c87aefeSPatrick Mooney 			break;
9064c87aefeSPatrick Mooney 		}
9074c87aefeSPatrick Mooney 		break;
9084c87aefeSPatrick Mooney 	}
9094c87aefeSPatrick Mooney 	case VM_MMAP_MEMSEG: {
9104c87aefeSPatrick Mooney 		struct vm_memmap mm;
9114c87aefeSPatrick Mooney 
9124c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &mm, sizeof (mm), md)) {
9134c87aefeSPatrick Mooney 			error = EFAULT;
9144c87aefeSPatrick Mooney 			break;
9154c87aefeSPatrick Mooney 		}
9164c87aefeSPatrick Mooney 		error = vm_mmap_memseg(sc->vmm_vm, mm.gpa, mm.segid, mm.segoff,
9174c87aefeSPatrick Mooney 		    mm.len, mm.prot, mm.flags);
9184c87aefeSPatrick Mooney 		break;
9194c87aefeSPatrick Mooney 	}
9202b948146SAndy Fiddaman 	case VM_MUNMAP_MEMSEG: {
9212b948146SAndy Fiddaman 		struct vm_munmap mu;
9222b948146SAndy Fiddaman 
9232b948146SAndy Fiddaman 		if (ddi_copyin(datap, &mu, sizeof (mu), md)) {
9242b948146SAndy Fiddaman 			error = EFAULT;
9252b948146SAndy Fiddaman 			break;
9262b948146SAndy Fiddaman 		}
9272b948146SAndy Fiddaman 		error = vm_munmap_memseg(sc->vmm_vm, mu.gpa, mu.len);
9282b948146SAndy Fiddaman 		break;
9292b948146SAndy Fiddaman 	}
9304c87aefeSPatrick Mooney 	case VM_ALLOC_MEMSEG: {
9314c87aefeSPatrick Mooney 		struct vm_memseg vmseg;
9324c87aefeSPatrick Mooney 
9334c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmseg, sizeof (vmseg), md)) {
9344c87aefeSPatrick Mooney 			error = EFAULT;
9354c87aefeSPatrick Mooney 			break;
9364c87aefeSPatrick Mooney 		}
9374c87aefeSPatrick Mooney 		error = vmmdev_alloc_memseg(sc, &vmseg);
9384c87aefeSPatrick Mooney 		break;
9394c87aefeSPatrick Mooney 	}
9404c87aefeSPatrick Mooney 	case VM_GET_MEMSEG: {
9414c87aefeSPatrick Mooney 		struct vm_memseg vmseg;
9424c87aefeSPatrick Mooney 
9434c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmseg, sizeof (vmseg), md)) {
9444c87aefeSPatrick Mooney 			error = EFAULT;
9454c87aefeSPatrick Mooney 			break;
9464c87aefeSPatrick Mooney 		}
9474c87aefeSPatrick Mooney 		error = vmmdev_get_memseg(sc, &vmseg);
9484c87aefeSPatrick Mooney 		if (error == 0 &&
9494c87aefeSPatrick Mooney 		    ddi_copyout(&vmseg, datap, sizeof (vmseg), md)) {
9504c87aefeSPatrick Mooney 			error = EFAULT;
9514c87aefeSPatrick Mooney 			break;
9524c87aefeSPatrick Mooney 		}
9534c87aefeSPatrick Mooney 		break;
9544c87aefeSPatrick Mooney 	}
9554c87aefeSPatrick Mooney 	case VM_GET_REGISTER: {
9564c87aefeSPatrick Mooney 		struct vm_register vmreg;
9574c87aefeSPatrick Mooney 
9584c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmreg, sizeof (vmreg), md)) {
9594c87aefeSPatrick Mooney 			error = EFAULT;
9604c87aefeSPatrick Mooney 			break;
9614c87aefeSPatrick Mooney 		}
9624c87aefeSPatrick Mooney 		error = vm_get_register(sc->vmm_vm, vcpu, vmreg.regnum,
9634c87aefeSPatrick Mooney 		    &vmreg.regval);
9644c87aefeSPatrick Mooney 		if (error == 0 &&
9654c87aefeSPatrick Mooney 		    ddi_copyout(&vmreg, datap, sizeof (vmreg), md)) {
9664c87aefeSPatrick Mooney 			error = EFAULT;
9674c87aefeSPatrick Mooney 			break;
9684c87aefeSPatrick Mooney 		}
9694c87aefeSPatrick Mooney 		break;
9704c87aefeSPatrick Mooney 	}
9714c87aefeSPatrick Mooney 	case VM_SET_REGISTER: {
9724c87aefeSPatrick Mooney 		struct vm_register vmreg;
9734c87aefeSPatrick Mooney 
9744c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmreg, sizeof (vmreg), md)) {
9754c87aefeSPatrick Mooney 			error = EFAULT;
9764c87aefeSPatrick Mooney 			break;
9774c87aefeSPatrick Mooney 		}
9784c87aefeSPatrick Mooney 		error = vm_set_register(sc->vmm_vm, vcpu, vmreg.regnum,
9794c87aefeSPatrick Mooney 		    vmreg.regval);
9804c87aefeSPatrick Mooney 		break;
9814c87aefeSPatrick Mooney 	}
9824c87aefeSPatrick Mooney 	case VM_SET_SEGMENT_DESCRIPTOR: {
9834c87aefeSPatrick Mooney 		struct vm_seg_desc vmsegd;
9844c87aefeSPatrick Mooney 
9854c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmsegd, sizeof (vmsegd), md)) {
9864c87aefeSPatrick Mooney 			error = EFAULT;
9874c87aefeSPatrick Mooney 			break;
9884c87aefeSPatrick Mooney 		}
9894c87aefeSPatrick Mooney 		error = vm_set_seg_desc(sc->vmm_vm, vcpu, vmsegd.regnum,
9904c87aefeSPatrick Mooney 		    &vmsegd.desc);
9914c87aefeSPatrick Mooney 		break;
9924c87aefeSPatrick Mooney 	}
9934c87aefeSPatrick Mooney 	case VM_GET_SEGMENT_DESCRIPTOR: {
9944c87aefeSPatrick Mooney 		struct vm_seg_desc vmsegd;
9954c87aefeSPatrick Mooney 
9964c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmsegd, sizeof (vmsegd), md)) {
9974c87aefeSPatrick Mooney 			error = EFAULT;
9984c87aefeSPatrick Mooney 			break;
9994c87aefeSPatrick Mooney 		}
10004c87aefeSPatrick Mooney 		error = vm_get_seg_desc(sc->vmm_vm, vcpu, vmsegd.regnum,
10014c87aefeSPatrick Mooney 		    &vmsegd.desc);
10024c87aefeSPatrick Mooney 		if (error == 0 &&
10034c87aefeSPatrick Mooney 		    ddi_copyout(&vmsegd, datap, sizeof (vmsegd), md)) {
10044c87aefeSPatrick Mooney 			error = EFAULT;
10054c87aefeSPatrick Mooney 			break;
10064c87aefeSPatrick Mooney 		}
10074c87aefeSPatrick Mooney 		break;
10084c87aefeSPatrick Mooney 	}
10094c87aefeSPatrick Mooney 	case VM_GET_REGISTER_SET: {
10104c87aefeSPatrick Mooney 		struct vm_register_set vrs;
10114c87aefeSPatrick Mooney 		int regnums[VM_REG_LAST];
10124c87aefeSPatrick Mooney 		uint64_t regvals[VM_REG_LAST];
10134c87aefeSPatrick Mooney 
10144c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vrs, sizeof (vrs), md)) {
10154c87aefeSPatrick Mooney 			error = EFAULT;
10164c87aefeSPatrick Mooney 			break;
10174c87aefeSPatrick Mooney 		}
10184c87aefeSPatrick Mooney 		if (vrs.count > VM_REG_LAST || vrs.count == 0) {
10194c87aefeSPatrick Mooney 			error = EINVAL;
10204c87aefeSPatrick Mooney 			break;
10214c87aefeSPatrick Mooney 		}
10224c87aefeSPatrick Mooney 		if (ddi_copyin(vrs.regnums, regnums,
10234c87aefeSPatrick Mooney 		    sizeof (int) * vrs.count, md)) {
10244c87aefeSPatrick Mooney 			error = EFAULT;
10254c87aefeSPatrick Mooney 			break;
10264c87aefeSPatrick Mooney 		}
10274c87aefeSPatrick Mooney 
10284c87aefeSPatrick Mooney 		error = 0;
10294c87aefeSPatrick Mooney 		for (uint_t i = 0; i < vrs.count && error == 0; i++) {
10304c87aefeSPatrick Mooney 			if (regnums[i] < 0) {
10314c87aefeSPatrick Mooney 				error = EINVAL;
10324c87aefeSPatrick Mooney 				break;
1033bf21cd93STycho Nightingale 			}
10344c87aefeSPatrick Mooney 			error = vm_get_register(sc->vmm_vm, vcpu, regnums[i],
10354c87aefeSPatrick Mooney 			    &regvals[i]);
10364c87aefeSPatrick Mooney 		}
10374c87aefeSPatrick Mooney 		if (error == 0 && ddi_copyout(regvals, vrs.regvals,
10384c87aefeSPatrick Mooney 		    sizeof (uint64_t) * vrs.count, md)) {
10394c87aefeSPatrick Mooney 			error = EFAULT;
1040bf21cd93STycho Nightingale 		}
1041bf21cd93STycho Nightingale 		break;
10424c87aefeSPatrick Mooney 	}
10434c87aefeSPatrick Mooney 	case VM_SET_REGISTER_SET: {
10444c87aefeSPatrick Mooney 		struct vm_register_set vrs;
10454c87aefeSPatrick Mooney 		int regnums[VM_REG_LAST];
10464c87aefeSPatrick Mooney 		uint64_t regvals[VM_REG_LAST];
10474c87aefeSPatrick Mooney 
10484c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vrs, sizeof (vrs), md)) {
10494c87aefeSPatrick Mooney 			error = EFAULT;
10504c87aefeSPatrick Mooney 			break;
1051bf21cd93STycho Nightingale 		}
10524c87aefeSPatrick Mooney 		if (vrs.count > VM_REG_LAST || vrs.count == 0) {
10534c87aefeSPatrick Mooney 			error = EINVAL;
10544c87aefeSPatrick Mooney 			break;
10554c87aefeSPatrick Mooney 		}
10564c87aefeSPatrick Mooney 		if (ddi_copyin(vrs.regnums, regnums,
10574c87aefeSPatrick Mooney 		    sizeof (int) * vrs.count, md)) {
10584c87aefeSPatrick Mooney 			error = EFAULT;
10594c87aefeSPatrick Mooney 			break;
10604c87aefeSPatrick Mooney 		}
10614c87aefeSPatrick Mooney 		if (ddi_copyin(vrs.regvals, regvals,
10624c87aefeSPatrick Mooney 		    sizeof (uint64_t) * vrs.count, md)) {
10634c87aefeSPatrick Mooney 			error = EFAULT;
10644c87aefeSPatrick Mooney 			break;
10654c87aefeSPatrick Mooney 		}
10664c87aefeSPatrick Mooney 
10674c87aefeSPatrick Mooney 		error = 0;
10684c87aefeSPatrick Mooney 		for (uint_t i = 0; i < vrs.count && error == 0; i++) {
10694c87aefeSPatrick Mooney 			/*
10704c87aefeSPatrick Mooney 			 * Setting registers in a set is not atomic, since a
10714c87aefeSPatrick Mooney 			 * failure in the middle of the set will cause a
10724c87aefeSPatrick Mooney 			 * bail-out and inconsistent register state.  Callers
10734c87aefeSPatrick Mooney 			 * should be wary of this.
10744c87aefeSPatrick Mooney 			 */
10754c87aefeSPatrick Mooney 			if (regnums[i] < 0) {
10764c87aefeSPatrick Mooney 				error = EINVAL;
10774c87aefeSPatrick Mooney 				break;
1078bf21cd93STycho Nightingale 			}
10794c87aefeSPatrick Mooney 			error = vm_set_register(sc->vmm_vm, vcpu, regnums[i],
10804c87aefeSPatrick Mooney 			    regvals[i]);
1081bf21cd93STycho Nightingale 		}
1082bf21cd93STycho Nightingale 		break;
10834c87aefeSPatrick Mooney 	}
10842606939dSPatrick Mooney 	case VM_RESET_CPU: {
10852606939dSPatrick Mooney 		struct vm_vcpu_reset vvr;
10862606939dSPatrick Mooney 
10872606939dSPatrick Mooney 		if (ddi_copyin(datap, &vvr, sizeof (vvr), md)) {
10882606939dSPatrick Mooney 			error = EFAULT;
10892606939dSPatrick Mooney 			break;
10902606939dSPatrick Mooney 		}
10912606939dSPatrick Mooney 		if (vvr.kind != VRK_RESET && vvr.kind != VRK_INIT) {
10922606939dSPatrick Mooney 			error = EINVAL;
10932606939dSPatrick Mooney 		}
10942606939dSPatrick Mooney 
10952606939dSPatrick Mooney 		error = vcpu_arch_reset(sc->vmm_vm, vcpu, vvr.kind == VRK_INIT);
10962606939dSPatrick Mooney 		break;
10972606939dSPatrick Mooney 	}
10982606939dSPatrick Mooney 	case VM_GET_RUN_STATE: {
10992606939dSPatrick Mooney 		struct vm_run_state vrs;
11002606939dSPatrick Mooney 
11012606939dSPatrick Mooney 		bzero(&vrs, sizeof (vrs));
11022606939dSPatrick Mooney 		error = vm_get_run_state(sc->vmm_vm, vcpu, &vrs.state,
11032606939dSPatrick Mooney 		    &vrs.sipi_vector);
11042606939dSPatrick Mooney 		if (error == 0) {
11052606939dSPatrick Mooney 			if (ddi_copyout(&vrs, datap, sizeof (vrs), md)) {
11062606939dSPatrick Mooney 				error = EFAULT;
11072606939dSPatrick Mooney 				break;
11082606939dSPatrick Mooney 			}
11092606939dSPatrick Mooney 		}
11102606939dSPatrick Mooney 		break;
11112606939dSPatrick Mooney 	}
11122606939dSPatrick Mooney 	case VM_SET_RUN_STATE: {
11132606939dSPatrick Mooney 		struct vm_run_state vrs;
11142606939dSPatrick Mooney 
11152606939dSPatrick Mooney 		if (ddi_copyin(datap, &vrs, sizeof (vrs), md)) {
11162606939dSPatrick Mooney 			error = EFAULT;
11172606939dSPatrick Mooney 			break;
11182606939dSPatrick Mooney 		}
11192606939dSPatrick Mooney 		error = vm_set_run_state(sc->vmm_vm, vcpu, vrs.state,
11202606939dSPatrick Mooney 		    vrs.sipi_vector);
11212606939dSPatrick Mooney 		break;
11222606939dSPatrick Mooney 	}
1123957246c9SPatrick Mooney 	case VM_GET_FPU: {
1124957246c9SPatrick Mooney 		struct vm_fpu_state req;
1125957246c9SPatrick Mooney 		const size_t max_len = (PAGESIZE * 2);
1126957246c9SPatrick Mooney 		void *kbuf;
1127957246c9SPatrick Mooney 
1128957246c9SPatrick Mooney 		if (ddi_copyin(datap, &req, sizeof (req), md)) {
1129957246c9SPatrick Mooney 			error = EFAULT;
1130957246c9SPatrick Mooney 			break;
1131957246c9SPatrick Mooney 		}
1132957246c9SPatrick Mooney 		if (req.len > max_len || req.len == 0) {
1133957246c9SPatrick Mooney 			error = EINVAL;
1134957246c9SPatrick Mooney 			break;
1135957246c9SPatrick Mooney 		}
1136957246c9SPatrick Mooney 		kbuf = kmem_zalloc(req.len, KM_SLEEP);
1137957246c9SPatrick Mooney 		error = vm_get_fpu(sc->vmm_vm, vcpu, kbuf, req.len);
1138957246c9SPatrick Mooney 		if (error == 0) {
1139957246c9SPatrick Mooney 			if (ddi_copyout(kbuf, req.buf, req.len, md)) {
1140957246c9SPatrick Mooney 				error = EFAULT;
1141957246c9SPatrick Mooney 			}
1142957246c9SPatrick Mooney 		}
1143957246c9SPatrick Mooney 		kmem_free(kbuf, req.len);
1144957246c9SPatrick Mooney 		break;
1145957246c9SPatrick Mooney 	}
1146957246c9SPatrick Mooney 	case VM_SET_FPU: {
1147957246c9SPatrick Mooney 		struct vm_fpu_state req;
1148957246c9SPatrick Mooney 		const size_t max_len = (PAGESIZE * 2);
1149957246c9SPatrick Mooney 		void *kbuf;
1150957246c9SPatrick Mooney 
1151957246c9SPatrick Mooney 		if (ddi_copyin(datap, &req, sizeof (req), md)) {
1152957246c9SPatrick Mooney 			error = EFAULT;
1153957246c9SPatrick Mooney 			break;
1154957246c9SPatrick Mooney 		}
1155957246c9SPatrick Mooney 		if (req.len > max_len || req.len == 0) {
1156957246c9SPatrick Mooney 			error = EINVAL;
1157957246c9SPatrick Mooney 			break;
1158957246c9SPatrick Mooney 		}
1159957246c9SPatrick Mooney 		kbuf = kmem_alloc(req.len, KM_SLEEP);
1160957246c9SPatrick Mooney 		if (ddi_copyin(req.buf, kbuf, req.len, md)) {
1161957246c9SPatrick Mooney 			error = EFAULT;
1162957246c9SPatrick Mooney 		} else {
1163957246c9SPatrick Mooney 			error = vm_set_fpu(sc->vmm_vm, vcpu, kbuf, req.len);
1164957246c9SPatrick Mooney 		}
1165957246c9SPatrick Mooney 		kmem_free(kbuf, req.len);
1166957246c9SPatrick Mooney 		break;
1167957246c9SPatrick Mooney 	}
11684c87aefeSPatrick Mooney 
1169154972afSPatrick Mooney 	case VM_SET_KERNEMU_DEV:
1170154972afSPatrick Mooney 	case VM_GET_KERNEMU_DEV: {
1171154972afSPatrick Mooney 		struct vm_readwrite_kernemu_device kemu;
1172154972afSPatrick Mooney 		size_t size = 0;
1173154972afSPatrick Mooney 
1174154972afSPatrick Mooney 		if (ddi_copyin(datap, &kemu, sizeof (kemu), md)) {
1175154972afSPatrick Mooney 			error = EFAULT;
1176154972afSPatrick Mooney 			break;
1177154972afSPatrick Mooney 		}
1178154972afSPatrick Mooney 
1179154972afSPatrick Mooney 		if (kemu.access_width > 3) {
1180154972afSPatrick Mooney 			error = EINVAL;
1181154972afSPatrick Mooney 			break;
1182154972afSPatrick Mooney 		}
1183154972afSPatrick Mooney 		size = (1 << kemu.access_width);
1184154972afSPatrick Mooney 		ASSERT(size >= 1 && size <= 8);
1185154972afSPatrick Mooney 
1186154972afSPatrick Mooney 		if (cmd == VM_SET_KERNEMU_DEV) {
1187e0c0d44eSPatrick Mooney 			error = vm_service_mmio_write(sc->vmm_vm, vcpu,
1188e0c0d44eSPatrick Mooney 			    kemu.gpa, kemu.value, size);
1189154972afSPatrick Mooney 		} else {
1190e0c0d44eSPatrick Mooney 			error = vm_service_mmio_read(sc->vmm_vm, vcpu,
1191e0c0d44eSPatrick Mooney 			    kemu.gpa, &kemu.value, size);
1192154972afSPatrick Mooney 		}
1193154972afSPatrick Mooney 
1194154972afSPatrick Mooney 		if (error == 0) {
1195154972afSPatrick Mooney 			if (ddi_copyout(&kemu, datap, sizeof (kemu), md)) {
1196154972afSPatrick Mooney 				error = EFAULT;
1197154972afSPatrick Mooney 				break;
1198154972afSPatrick Mooney 			}
1199154972afSPatrick Mooney 		}
1200154972afSPatrick Mooney 		break;
1201154972afSPatrick Mooney 	}
1202154972afSPatrick Mooney 
12034c87aefeSPatrick Mooney 	case VM_GET_CAPABILITY: {
12044c87aefeSPatrick Mooney 		struct vm_capability vmcap;
12054c87aefeSPatrick Mooney 
12064c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmcap, sizeof (vmcap), md)) {
12074c87aefeSPatrick Mooney 			error = EFAULT;
12084c87aefeSPatrick Mooney 			break;
12094c87aefeSPatrick Mooney 		}
12104c87aefeSPatrick Mooney 		error = vm_get_capability(sc->vmm_vm, vcpu, vmcap.captype,
12114c87aefeSPatrick Mooney 		    &vmcap.capval);
12124c87aefeSPatrick Mooney 		if (error == 0 &&
12134c87aefeSPatrick Mooney 		    ddi_copyout(&vmcap, datap, sizeof (vmcap), md)) {
12144c87aefeSPatrick Mooney 			error = EFAULT;
12154c87aefeSPatrick Mooney 			break;
1216bf21cd93STycho Nightingale 		}
1217bf21cd93STycho Nightingale 		break;
12184c87aefeSPatrick Mooney 	}
12194c87aefeSPatrick Mooney 	case VM_SET_CAPABILITY: {
12204c87aefeSPatrick Mooney 		struct vm_capability vmcap;
12214c87aefeSPatrick Mooney 
12224c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmcap, sizeof (vmcap), md)) {
12234c87aefeSPatrick Mooney 			error = EFAULT;
12244c87aefeSPatrick Mooney 			break;
1225bf21cd93STycho Nightingale 		}
12264c87aefeSPatrick Mooney 		error = vm_set_capability(sc->vmm_vm, vcpu, vmcap.captype,
12274c87aefeSPatrick Mooney 		    vmcap.capval);
1228bf21cd93STycho Nightingale 		break;
12294c87aefeSPatrick Mooney 	}
12304c87aefeSPatrick Mooney 	case VM_SET_X2APIC_STATE: {
12314c87aefeSPatrick Mooney 		struct vm_x2apic x2apic;
12324c87aefeSPatrick Mooney 
12334c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &x2apic, sizeof (x2apic), md)) {
12344c87aefeSPatrick Mooney 			error = EFAULT;
12354c87aefeSPatrick Mooney 			break;
1236bf21cd93STycho Nightingale 		}
12374c87aefeSPatrick Mooney 		error = vm_set_x2apic_state(sc->vmm_vm, vcpu, x2apic.state);
12384c87aefeSPatrick Mooney 		break;
12394c87aefeSPatrick Mooney 	}
12404c87aefeSPatrick Mooney 	case VM_GET_X2APIC_STATE: {
12414c87aefeSPatrick Mooney 		struct vm_x2apic x2apic;
12424c87aefeSPatrick Mooney 
12434c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &x2apic, sizeof (x2apic), md)) {
12444c87aefeSPatrick Mooney 			error = EFAULT;
12454c87aefeSPatrick Mooney 			break;
12464c87aefeSPatrick Mooney 		}
12474c87aefeSPatrick Mooney 		error = vm_get_x2apic_state(sc->vmm_vm, x2apic.cpuid,
12484c87aefeSPatrick Mooney 		    &x2apic.state);
12494c87aefeSPatrick Mooney 		if (error == 0 &&
12504c87aefeSPatrick Mooney 		    ddi_copyout(&x2apic, datap, sizeof (x2apic), md)) {
12514c87aefeSPatrick Mooney 			error = EFAULT;
12524c87aefeSPatrick Mooney 			break;
12534c87aefeSPatrick Mooney 		}
12544c87aefeSPatrick Mooney 		break;
12554c87aefeSPatrick Mooney 	}
12564c87aefeSPatrick Mooney 	case VM_GET_GPA_PMAP: {
12570153d828SPatrick Mooney 		/*
12580153d828SPatrick Mooney 		 * Until there is a necessity to leak EPT/RVI PTE values to
12590153d828SPatrick Mooney 		 * userspace, this will remain unimplemented
12600153d828SPatrick Mooney 		 */
12610153d828SPatrick Mooney 		error = EINVAL;
12624c87aefeSPatrick Mooney 		break;
12634c87aefeSPatrick Mooney 	}
12644c87aefeSPatrick Mooney 	case VM_GET_HPET_CAPABILITIES: {
12654c87aefeSPatrick Mooney 		struct vm_hpet_cap hpetcap;
12664c87aefeSPatrick Mooney 
12674c87aefeSPatrick Mooney 		error = vhpet_getcap(&hpetcap);
12684c87aefeSPatrick Mooney 		if (error == 0 &&
12694c87aefeSPatrick Mooney 		    ddi_copyout(&hpetcap, datap, sizeof (hpetcap), md)) {
12704c87aefeSPatrick Mooney 			error = EFAULT;
12714c87aefeSPatrick Mooney 			break;
1272bf21cd93STycho Nightingale 		}
1273bf21cd93STycho Nightingale 		break;
12744c87aefeSPatrick Mooney 	}
1275bf21cd93STycho Nightingale 	case VM_GLA2GPA: {
12764c87aefeSPatrick Mooney 		struct vm_gla2gpa gg;
12774c87aefeSPatrick Mooney 
12784c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &gg, sizeof (gg), md)) {
12794c87aefeSPatrick Mooney 			error = EFAULT;
12804c87aefeSPatrick Mooney 			break;
1281bf21cd93STycho Nightingale 		}
12824c87aefeSPatrick Mooney 		gg.vcpuid = vcpu;
12834c87aefeSPatrick Mooney 		error = vm_gla2gpa(sc->vmm_vm, vcpu, &gg.paging, gg.gla,
12844c87aefeSPatrick Mooney 		    gg.prot, &gg.gpa, &gg.fault);
12854c87aefeSPatrick Mooney 		if (error == 0 && ddi_copyout(&gg, datap, sizeof (gg), md)) {
12864c87aefeSPatrick Mooney 			error = EFAULT;
12874c87aefeSPatrick Mooney 			break;
12884c87aefeSPatrick Mooney 		}
12894c87aefeSPatrick Mooney 		break;
12904c87aefeSPatrick Mooney 	}
12914c87aefeSPatrick Mooney 	case VM_GLA2GPA_NOFAULT: {
12924c87aefeSPatrick Mooney 		struct vm_gla2gpa gg;
12934c87aefeSPatrick Mooney 
12944c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &gg, sizeof (gg), md)) {
12954c87aefeSPatrick Mooney 			error = EFAULT;
12964c87aefeSPatrick Mooney 			break;
12974c87aefeSPatrick Mooney 		}
12984c87aefeSPatrick Mooney 		gg.vcpuid = vcpu;
12994c87aefeSPatrick Mooney 		error = vm_gla2gpa_nofault(sc->vmm_vm, vcpu, &gg.paging,
13004c87aefeSPatrick Mooney 		    gg.gla, gg.prot, &gg.gpa, &gg.fault);
13014c87aefeSPatrick Mooney 		if (error == 0 && ddi_copyout(&gg, datap, sizeof (gg), md)) {
13024c87aefeSPatrick Mooney 			error = EFAULT;
13034c87aefeSPatrick Mooney 			break;
13044c87aefeSPatrick Mooney 		}
13054c87aefeSPatrick Mooney 		break;
13064c87aefeSPatrick Mooney 	}
13074c87aefeSPatrick Mooney 
13084c87aefeSPatrick Mooney 	case VM_ACTIVATE_CPU:
13094c87aefeSPatrick Mooney 		error = vm_activate_cpu(sc->vmm_vm, vcpu);
13104c87aefeSPatrick Mooney 		break;
13114c87aefeSPatrick Mooney 
13124c87aefeSPatrick Mooney 	case VM_SUSPEND_CPU:
13134c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vcpu, sizeof (vcpu), md)) {
13144c87aefeSPatrick Mooney 			error = EFAULT;
13154c87aefeSPatrick Mooney 		} else {
13164c87aefeSPatrick Mooney 			error = vm_suspend_cpu(sc->vmm_vm, vcpu);
13174c87aefeSPatrick Mooney 		}
13184c87aefeSPatrick Mooney 		break;
13194c87aefeSPatrick Mooney 
13204c87aefeSPatrick Mooney 	case VM_RESUME_CPU:
13214c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vcpu, sizeof (vcpu), md)) {
13224c87aefeSPatrick Mooney 			error = EFAULT;
13234c87aefeSPatrick Mooney 		} else {
13244c87aefeSPatrick Mooney 			error = vm_resume_cpu(sc->vmm_vm, vcpu);
13254c87aefeSPatrick Mooney 		}
13264c87aefeSPatrick Mooney 		break;
13274c87aefeSPatrick Mooney 
13284c87aefeSPatrick Mooney 	case VM_GET_CPUS: {
13294c87aefeSPatrick Mooney 		struct vm_cpuset vm_cpuset;
13304c87aefeSPatrick Mooney 		cpuset_t tempset;
13314c87aefeSPatrick Mooney 		void *srcp = &tempset;
13324c87aefeSPatrick Mooney 		int size;
13334c87aefeSPatrick Mooney 
13344c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vm_cpuset, sizeof (vm_cpuset), md)) {
13354c87aefeSPatrick Mooney 			error = EFAULT;
13364c87aefeSPatrick Mooney 			break;
13374c87aefeSPatrick Mooney 		}
13384c87aefeSPatrick Mooney 
13394c87aefeSPatrick Mooney 		/* Be more generous about sizing since our cpuset_t is large. */
13404c87aefeSPatrick Mooney 		size = vm_cpuset.cpusetsize;
13414c87aefeSPatrick Mooney 		if (size <= 0 || size > sizeof (cpuset_t)) {
13424c87aefeSPatrick Mooney 			error = ERANGE;
13434c87aefeSPatrick Mooney 		}
13444c87aefeSPatrick Mooney 		/*
13454c87aefeSPatrick Mooney 		 * If they want a ulong_t or less, make sure they receive the
13464c87aefeSPatrick Mooney 		 * low bits with all the useful information.
13474c87aefeSPatrick Mooney 		 */
13484c87aefeSPatrick Mooney 		if (size <= sizeof (tempset.cpub[0])) {
13494c87aefeSPatrick Mooney 			srcp = &tempset.cpub[0];
13504c87aefeSPatrick Mooney 		}
13514c87aefeSPatrick Mooney 
13524c87aefeSPatrick Mooney 		if (vm_cpuset.which == VM_ACTIVE_CPUS) {
13534c87aefeSPatrick Mooney 			tempset = vm_active_cpus(sc->vmm_vm);
13544c87aefeSPatrick Mooney 		} else if (vm_cpuset.which == VM_SUSPENDED_CPUS) {
13554c87aefeSPatrick Mooney 			tempset = vm_suspended_cpus(sc->vmm_vm);
13564c87aefeSPatrick Mooney 		} else if (vm_cpuset.which == VM_DEBUG_CPUS) {
13574c87aefeSPatrick Mooney 			tempset = vm_debug_cpus(sc->vmm_vm);
1358bf21cd93STycho Nightingale 		} else {
13594c87aefeSPatrick Mooney 			error = EINVAL;
13604c87aefeSPatrick Mooney 		}
13614c87aefeSPatrick Mooney 
13624c87aefeSPatrick Mooney 		ASSERT(size > 0 && size <= sizeof (tempset));
13634c87aefeSPatrick Mooney 		if (error == 0 &&
13644c87aefeSPatrick Mooney 		    ddi_copyout(srcp, vm_cpuset.cpus, size, md)) {
13654c87aefeSPatrick Mooney 			error = EFAULT;
13664c87aefeSPatrick Mooney 			break;
13674c87aefeSPatrick Mooney 		}
13684c87aefeSPatrick Mooney 		break;
13694c87aefeSPatrick Mooney 	}
13704c87aefeSPatrick Mooney 	case VM_SET_INTINFO: {
13714c87aefeSPatrick Mooney 		struct vm_intinfo vmii;
13724c87aefeSPatrick Mooney 
13734c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vmii, sizeof (vmii), md)) {
13744c87aefeSPatrick Mooney 			error = EFAULT;
13754c87aefeSPatrick Mooney 			break;
13764c87aefeSPatrick Mooney 		}
13774c87aefeSPatrick Mooney 		error = vm_exit_intinfo(sc->vmm_vm, vcpu, vmii.info1);
13784c87aefeSPatrick Mooney 		break;
13794c87aefeSPatrick Mooney 	}
13804c87aefeSPatrick Mooney 	case VM_GET_INTINFO: {
13814c87aefeSPatrick Mooney 		struct vm_intinfo vmii;
13824c87aefeSPatrick Mooney 
13834c87aefeSPatrick Mooney 		vmii.vcpuid = vcpu;
13844c87aefeSPatrick Mooney 		error = vm_get_intinfo(sc->vmm_vm, vcpu, &vmii.info1,
13854c87aefeSPatrick Mooney 		    &vmii.info2);
13864c87aefeSPatrick Mooney 		if (error == 0 &&
13874c87aefeSPatrick Mooney 		    ddi_copyout(&vmii, datap, sizeof (vmii), md)) {
13884c87aefeSPatrick Mooney 			error = EFAULT;
13894c87aefeSPatrick Mooney 			break;
13904c87aefeSPatrick Mooney 		}
13914c87aefeSPatrick Mooney 		break;
13924c87aefeSPatrick Mooney 	}
13934c87aefeSPatrick Mooney 	case VM_RTC_WRITE: {
13944c87aefeSPatrick Mooney 		struct vm_rtc_data rtcdata;
13954c87aefeSPatrick Mooney 
13964c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &rtcdata, sizeof (rtcdata), md)) {
13974c87aefeSPatrick Mooney 			error = EFAULT;
13984c87aefeSPatrick Mooney 			break;
13994c87aefeSPatrick Mooney 		}
14004c87aefeSPatrick Mooney 		error = vrtc_nvram_write(sc->vmm_vm, rtcdata.offset,
14014c87aefeSPatrick Mooney 		    rtcdata.value);
14024c87aefeSPatrick Mooney 		break;
14034c87aefeSPatrick Mooney 	}
14044c87aefeSPatrick Mooney 	case VM_RTC_READ: {
14054c87aefeSPatrick Mooney 		struct vm_rtc_data rtcdata;
14064c87aefeSPatrick Mooney 
14074c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &rtcdata, sizeof (rtcdata), md)) {
14084c87aefeSPatrick Mooney 			error = EFAULT;
14094c87aefeSPatrick Mooney 			break;
14104c87aefeSPatrick Mooney 		}
14114c87aefeSPatrick Mooney 		error = vrtc_nvram_read(sc->vmm_vm, rtcdata.offset,
14124c87aefeSPatrick Mooney 		    &rtcdata.value);
14134c87aefeSPatrick Mooney 		if (error == 0 &&
14144c87aefeSPatrick Mooney 		    ddi_copyout(&rtcdata, datap, sizeof (rtcdata), md)) {
14154c87aefeSPatrick Mooney 			error = EFAULT;
14164c87aefeSPatrick Mooney 			break;
14174c87aefeSPatrick Mooney 		}
14184c87aefeSPatrick Mooney 		break;
14194c87aefeSPatrick Mooney 	}
14204c87aefeSPatrick Mooney 	case VM_RTC_SETTIME: {
14214c87aefeSPatrick Mooney 		struct vm_rtc_time rtctime;
14224c87aefeSPatrick Mooney 
14234c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &rtctime, sizeof (rtctime), md)) {
14244c87aefeSPatrick Mooney 			error = EFAULT;
14254c87aefeSPatrick Mooney 			break;
14264c87aefeSPatrick Mooney 		}
14274c87aefeSPatrick Mooney 		error = vrtc_set_time(sc->vmm_vm, rtctime.secs);
14284c87aefeSPatrick Mooney 		break;
14294c87aefeSPatrick Mooney 	}
14304c87aefeSPatrick Mooney 	case VM_RTC_GETTIME: {
14314c87aefeSPatrick Mooney 		struct vm_rtc_time rtctime;
14324c87aefeSPatrick Mooney 
14334c87aefeSPatrick Mooney 		rtctime.secs = vrtc_get_time(sc->vmm_vm);
14344c87aefeSPatrick Mooney 		if (ddi_copyout(&rtctime, datap, sizeof (rtctime), md)) {
14354c87aefeSPatrick Mooney 			error = EFAULT;
14364c87aefeSPatrick Mooney 			break;
14374c87aefeSPatrick Mooney 		}
14384c87aefeSPatrick Mooney 		break;
14394c87aefeSPatrick Mooney 	}
14404c87aefeSPatrick Mooney 
14410e1453c3SPatrick Mooney 	case VM_PMTMR_LOCATE: {
14420e1453c3SPatrick Mooney 		uint16_t port = arg;
14430e1453c3SPatrick Mooney 		error = vpmtmr_set_location(sc->vmm_vm, port);
14440e1453c3SPatrick Mooney 		break;
14450e1453c3SPatrick Mooney 	}
14460e1453c3SPatrick Mooney 
14474c87aefeSPatrick Mooney 	case VM_RESTART_INSTRUCTION:
14484c87aefeSPatrick Mooney 		error = vm_restart_instruction(sc->vmm_vm, vcpu);
14494c87aefeSPatrick Mooney 		break;
14504c87aefeSPatrick Mooney 
14514c87aefeSPatrick Mooney 	case VM_SET_TOPOLOGY: {
14524c87aefeSPatrick Mooney 		struct vm_cpu_topology topo;
14534c87aefeSPatrick Mooney 
14544c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &topo, sizeof (topo), md) != 0) {
14554c87aefeSPatrick Mooney 			error = EFAULT;
14564c87aefeSPatrick Mooney 			break;
14574c87aefeSPatrick Mooney 		}
14584c87aefeSPatrick Mooney 		error = vm_set_topology(sc->vmm_vm, topo.sockets, topo.cores,
14594c87aefeSPatrick Mooney 		    topo.threads, topo.maxcpus);
14604c87aefeSPatrick Mooney 		break;
14614c87aefeSPatrick Mooney 	}
14624c87aefeSPatrick Mooney 	case VM_GET_TOPOLOGY: {
14634c87aefeSPatrick Mooney 		struct vm_cpu_topology topo;
14644c87aefeSPatrick Mooney 
14654c87aefeSPatrick Mooney 		vm_get_topology(sc->vmm_vm, &topo.sockets, &topo.cores,
14664c87aefeSPatrick Mooney 		    &topo.threads, &topo.maxcpus);
14674c87aefeSPatrick Mooney 		if (ddi_copyout(&topo, datap, sizeof (topo), md) != 0) {
14684c87aefeSPatrick Mooney 			error = EFAULT;
14694c87aefeSPatrick Mooney 			break;
14704c87aefeSPatrick Mooney 		}
14714c87aefeSPatrick Mooney 		break;
14724c87aefeSPatrick Mooney 	}
14734c87aefeSPatrick Mooney 	case VM_DEVMEM_GETOFFSET: {
14744c87aefeSPatrick Mooney 		struct vm_devmem_offset vdo;
14753d066281SPatrick Mooney 		vmm_devmem_entry_t *de;
14764c87aefeSPatrick Mooney 
14774c87aefeSPatrick Mooney 		if (ddi_copyin(datap, &vdo, sizeof (vdo), md) != 0) {
1478bf21cd93STycho Nightingale 			error = EFAULT;
14794c87aefeSPatrick Mooney 			break;
14804c87aefeSPatrick Mooney 		}
14814c87aefeSPatrick Mooney 
14823d066281SPatrick Mooney 		de = vmmdev_devmem_find(sc, vdo.segid);
14834c87aefeSPatrick Mooney 		if (de != NULL) {
14844c87aefeSPatrick Mooney 			vdo.offset = de->vde_off;
14854c87aefeSPatrick Mooney 			if (ddi_copyout(&vdo, datap, sizeof (vdo), md) != 0) {
14864c87aefeSPatrick Mooney 				error = EFAULT;
14874c87aefeSPatrick Mooney 			}
14884c87aefeSPatrick Mooney 		} else {
14894c87aefeSPatrick Mooney 			error = ENOENT;
1490bf21cd93STycho Nightingale 		}
1491bf21cd93STycho Nightingale 		break;
1492bf21cd93STycho Nightingale 	}
14938779b448SDan Cross 	case VM_TRACK_DIRTY_PAGES: {
14948779b448SDan Cross 		const size_t max_track_region_len = 8 * PAGESIZE * 8 * PAGESIZE;
14958779b448SDan Cross 		struct vmm_dirty_tracker tracker;
14968779b448SDan Cross 		uint8_t *bitmap;
14978779b448SDan Cross 		size_t len;
14988779b448SDan Cross 
14998779b448SDan Cross 		if (ddi_copyin(datap, &tracker, sizeof (tracker), md) != 0) {
15008779b448SDan Cross 			error = EFAULT;
15018779b448SDan Cross 			break;
15028779b448SDan Cross 		}
15038779b448SDan Cross 		if ((tracker.vdt_start_gpa & PAGEOFFSET) != 0) {
15048779b448SDan Cross 			error = EINVAL;
15058779b448SDan Cross 			break;
15068779b448SDan Cross 		}
15078779b448SDan Cross 		if (tracker.vdt_len == 0) {
15088779b448SDan Cross 			break;
15098779b448SDan Cross 		}
15108779b448SDan Cross 		if ((tracker.vdt_len & PAGEOFFSET) != 0) {
15118779b448SDan Cross 			error = EINVAL;
15128779b448SDan Cross 			break;
15138779b448SDan Cross 		}
15148779b448SDan Cross 		if (tracker.vdt_len > max_track_region_len) {
15158779b448SDan Cross 			error = EINVAL;
15168779b448SDan Cross 			break;
15178779b448SDan Cross 		}
15188779b448SDan Cross 		len = roundup(tracker.vdt_len / PAGESIZE, 8) / 8;
15198779b448SDan Cross 		bitmap = kmem_zalloc(len, KM_SLEEP);
15208779b448SDan Cross 		vm_track_dirty_pages(sc->vmm_vm, tracker.vdt_start_gpa,
15218779b448SDan Cross 		    tracker.vdt_len, bitmap);
15228779b448SDan Cross 		if (ddi_copyout(bitmap, tracker.vdt_pfns, len, md) != 0) {
15238779b448SDan Cross 			error = EFAULT;
15248779b448SDan Cross 		}
15258779b448SDan Cross 		kmem_free(bitmap, len);
15268779b448SDan Cross 
15278779b448SDan Cross 		break;
15288779b448SDan Cross 	}
15294c87aefeSPatrick Mooney 	case