xref: /illumos-gate/usr/src/uts/intel/io/vmm/vmm.c (revision 4bd36be4)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2011 NetApp, Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 /*
29  * This file and its contents are supplied under the terms of the
30  * Common Development and Distribution License ("CDDL"), version 1.0.
31  * You may only use this file in accordance with the terms of version
32  * 1.0 of the CDDL.
33  *
34  * A full copy of the text of the CDDL should have accompanied this
35  * source.  A copy of the CDDL is also available via the Internet at
36  * http://www.illumos.org/license/CDDL.
37  *
38  * Copyright 2015 Pluribus Networks Inc.
39  * Copyright 2018 Joyent, Inc.
40  * Copyright 2024 Oxide Computer Company
41  * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
42  */
43 
44 
45 #include <sys/cdefs.h>
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
50 #include <sys/module.h>
51 #include <sys/sysctl.h>
52 #include <sys/kmem.h>
53 #include <sys/pcpu.h>
54 #include <sys/mutex.h>
55 #include <sys/proc.h>
56 #include <sys/rwlock.h>
57 #include <sys/sched.h>
58 #include <sys/systm.h>
59 #include <sys/sunddi.h>
60 #include <sys/hma.h>
61 #include <sys/archsystm.h>
62 
63 #include <machine/md_var.h>
64 #include <x86/psl.h>
65 #include <x86/apicreg.h>
66 
67 #include <machine/specialreg.h>
68 #include <machine/vmm.h>
69 #include <machine/vmm_dev.h>
70 #include <machine/vmparam.h>
71 #include <sys/vmm_instruction_emul.h>
72 #include <sys/vmm_vm.h>
73 #include <sys/vmm_gpt.h>
74 #include <sys/vmm_data.h>
75 
76 #include "vmm_ioport.h"
77 #include "vmm_host.h"
78 #include "vmm_util.h"
79 #include "vatpic.h"
80 #include "vatpit.h"
81 #include "vhpet.h"
82 #include "vioapic.h"
83 #include "vlapic.h"
84 #include "vpmtmr.h"
85 #include "vrtc.h"
86 #include "vmm_stat.h"
87 #include "vmm_lapic.h"
88 
89 #include "io/ppt.h"
90 #include "io/iommu.h"
91 
92 struct vlapic;
93 
94 /* Flags for vtc_status */
95 #define	VTCS_FPU_RESTORED	1 /* guest FPU restored, host FPU saved */
96 #define	VTCS_FPU_CTX_CRITICAL	2 /* in ctx where FPU restore cannot be lazy */
97 
98 typedef struct vm_thread_ctx {
99 	struct vm	*vtc_vm;
100 	int		vtc_vcpuid;
101 	uint_t		vtc_status;
102 	enum vcpu_ustate vtc_ustate;
103 } vm_thread_ctx_t;
104 
105 #define	VMM_MTRR_VAR_MAX 10
106 #define	VMM_MTRR_DEF_MASK \
107 	(MTRR_DEF_ENABLE | MTRR_DEF_FIXED_ENABLE | MTRR_DEF_TYPE)
108 #define	VMM_MTRR_PHYSBASE_MASK (MTRR_PHYSBASE_PHYSBASE | MTRR_PHYSBASE_TYPE)
109 #define	VMM_MTRR_PHYSMASK_MASK (MTRR_PHYSMASK_PHYSMASK | MTRR_PHYSMASK_VALID)
110 struct vm_mtrr {
111 	uint64_t def_type;
112 	uint64_t fixed4k[8];
113 	uint64_t fixed16k[2];
114 	uint64_t fixed64k;
115 	struct {
116 		uint64_t base;
117 		uint64_t mask;
118 	} var[VMM_MTRR_VAR_MAX];
119 };
120 
121 /*
122  * Initialization:
123  * (a) allocated when vcpu is created
124  * (i) initialized when vcpu is created and when it is reinitialized
125  * (o) initialized the first time the vcpu is created
126  * (x) initialized before use
127  */
128 struct vcpu {
129 	/* (o) protects state, run_state, hostcpu, sipi_vector */
130 	kmutex_t	lock;
131 
132 	enum vcpu_state	state;		/* (o) vcpu state */
133 	enum vcpu_run_state run_state;	/* (i) vcpu init/sipi/run state */
134 	kcondvar_t	vcpu_cv;	/* (o) cpu waiter cv */
135 	kcondvar_t	state_cv;	/* (o) IDLE-transition cv */
136 	int		hostcpu;	/* (o) vcpu's current host cpu */
137 	int		lastloccpu;	/* (o) last host cpu localized to */
138 	bool		reqidle;	/* (i) request vcpu to idle */
139 	bool		reqconsist;	/* (i) req. vcpu exit when consistent */
140 	bool		reqbarrier;	/* (i) request vcpu exit barrier */
141 	struct vlapic	*vlapic;	/* (i) APIC device model */
142 	enum x2apic_state x2apic_state;	/* (i) APIC mode */
143 	uint64_t	exit_intinfo;	/* (i) events pending at VM exit */
144 	uint64_t	exc_pending;	/* (i) exception pending */
145 	bool		nmi_pending;	/* (i) NMI pending */
146 	bool		extint_pending;	/* (i) INTR pending */
147 
148 	uint8_t		sipi_vector;	/* (i) SIPI vector */
149 	hma_fpu_t	*guestfpu;	/* (a,i) guest fpu state */
150 	uint64_t	guest_xcr0;	/* (i) guest %xcr0 register */
151 	void		*stats;		/* (a,i) statistics */
152 	struct vm_exit	exitinfo;	/* (x) exit reason and collateral */
153 	uint64_t	nextrip;	/* (x) next instruction to execute */
154 	struct vie	*vie_ctx;	/* (x) instruction emulation context */
155 	vm_client_t	*vmclient;	/* (a) VM-system client */
156 	uint64_t	tsc_offset;	/* (x) vCPU TSC offset */
157 	struct vm_mtrr	mtrr;		/* (i) vcpu's MTRR */
158 	vcpu_cpuid_config_t cpuid_cfg;	/* (x) cpuid configuration */
159 
160 	enum vcpu_ustate ustate;	/* (i) microstate for the vcpu */
161 	hrtime_t	ustate_when;	/* (i) time of last ustate change */
162 	uint64_t ustate_total[VU_MAX];	/* (o) total time spent in ustates */
163 	vm_thread_ctx_t	vtc;		/* (o) thread state for ctxops */
164 	struct ctxop	*ctxop;		/* (o) ctxop storage for vcpu */
165 };
166 
167 #define	vcpu_lock(v)		mutex_enter(&((v)->lock))
168 #define	vcpu_unlock(v)		mutex_exit(&((v)->lock))
169 #define	vcpu_assert_locked(v)	ASSERT(MUTEX_HELD(&((v)->lock)))
170 
171 struct mem_seg {
172 	size_t	len;
173 	bool	sysmem;
174 	vm_object_t *object;
175 };
176 #define	VM_MAX_MEMSEGS	5
177 
178 struct mem_map {
179 	vm_paddr_t	gpa;
180 	size_t		len;
181 	vm_ooffset_t	segoff;
182 	int		segid;
183 	int		prot;
184 	int		flags;
185 };
186 #define	VM_MAX_MEMMAPS	8
187 
188 /*
189  * Initialization:
190  * (o) initialized the first time the VM is created
191  * (i) initialized when VM is created and when it is reinitialized
192  * (x) initialized before use
193  */
194 struct vm {
195 	void		*cookie;		/* (i) cpu-specific data */
196 	void		*iommu;			/* (x) iommu-specific data */
197 	struct vhpet	*vhpet;			/* (i) virtual HPET */
198 	struct vioapic	*vioapic;		/* (i) virtual ioapic */
199 	struct vatpic	*vatpic;		/* (i) virtual atpic */
200 	struct vatpit	*vatpit;		/* (i) virtual atpit */
201 	struct vpmtmr	*vpmtmr;		/* (i) virtual ACPI PM timer */
202 	struct vrtc	*vrtc;			/* (o) virtual RTC */
203 	volatile cpuset_t active_cpus;		/* (i) active vcpus */
204 	volatile cpuset_t debug_cpus;		/* (i) vcpus stopped for dbg */
205 	volatile cpuset_t halted_cpus;		/* (x) cpus in a hard halt */
206 	int		suspend_how;		/* (i) stop VM execution */
207 	int		suspend_source;		/* (i) src vcpuid of suspend */
208 	hrtime_t	suspend_when;		/* (i) time suspend asserted */
209 	struct mem_map	mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */
210 	struct mem_seg	mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */
211 	struct vmspace	*vmspace;		/* (o) guest's address space */
212 	struct vcpu	vcpu[VM_MAXCPU];	/* (i) guest vcpus */
213 	/* The following describe the vm cpu topology */
214 	uint16_t	sockets;		/* (o) num of sockets */
215 	uint16_t	cores;			/* (o) num of cores/socket */
216 	uint16_t	threads;		/* (o) num of threads/core */
217 	uint16_t	maxcpus;		/* (o) max pluggable cpus */
218 
219 	hrtime_t	boot_hrtime;		/* (i) hrtime at VM boot */
220 
221 	/* TSC and TSC scaling related values */
222 	uint64_t	tsc_offset;		/* (i) VM-wide TSC offset */
223 	uint64_t	guest_freq;		/* (i) guest TSC Frequency */
224 	uint64_t	freq_multiplier;	/* (i) guest/host TSC Ratio */
225 
226 	struct ioport_config ioports;		/* (o) ioport handling */
227 
228 	bool		mem_transient;		/* (o) alloc transient memory */
229 	bool		is_paused;		/* (i) instance is paused */
230 };
231 
232 static int vmm_initialized;
233 static uint64_t vmm_host_freq;
234 
235 
236 static void
nullop_panic(void)237 nullop_panic(void)
238 {
239 	panic("null vmm operation call");
240 }
241 
242 /* Do not allow use of an un-set `ops` to do anything but panic */
243 static struct vmm_ops vmm_ops_null = {
244 	.init		= (vmm_init_func_t)nullop_panic,
245 	.cleanup	= (vmm_cleanup_func_t)nullop_panic,
246 	.resume		= (vmm_resume_func_t)nullop_panic,
247 	.vminit		= (vmi_init_func_t)nullop_panic,
248 	.vmrun		= (vmi_run_func_t)nullop_panic,
249 	.vmcleanup	= (vmi_cleanup_func_t)nullop_panic,
250 	.vmgetreg	= (vmi_get_register_t)nullop_panic,
251 	.vmsetreg	= (vmi_set_register_t)nullop_panic,
252 	.vmgetdesc	= (vmi_get_desc_t)nullop_panic,
253 	.vmsetdesc	= (vmi_set_desc_t)nullop_panic,
254 	.vmgetcap	= (vmi_get_cap_t)nullop_panic,
255 	.vmsetcap	= (vmi_set_cap_t)nullop_panic,
256 	.vlapic_init	= (vmi_vlapic_init)nullop_panic,
257 	.vlapic_cleanup	= (vmi_vlapic_cleanup)nullop_panic,
258 	.vmpause	= (vmi_pause_t)nullop_panic,
259 	.vmsavectx	= (vmi_savectx)nullop_panic,
260 	.vmrestorectx	= (vmi_restorectx)nullop_panic,
261 	.vmgetmsr	= (vmi_get_msr_t)nullop_panic,
262 	.vmsetmsr	= (vmi_set_msr_t)nullop_panic,
263 	.vmfreqratio	= (vmi_freqratio_t)nullop_panic,
264 	.fr_fracsize	= 0,
265 	.fr_intsize	= 0,
266 };
267 
268 static struct vmm_ops *ops = &vmm_ops_null;
269 static vmm_pte_ops_t *pte_ops = NULL;
270 
271 #define	VMM_INIT()			((*ops->init)())
272 #define	VMM_CLEANUP()			((*ops->cleanup)())
273 #define	VMM_RESUME()			((*ops->resume)())
274 
275 #define	VMINIT(vm)		((*ops->vminit)(vm))
276 #define	VMRUN(vmi, vcpu, rip)	((*ops->vmrun)(vmi, vcpu, rip))
277 #define	VMCLEANUP(vmi)			((*ops->vmcleanup)(vmi))
278 
279 #define	VMGETREG(vmi, vcpu, num, rv)	((*ops->vmgetreg)(vmi, vcpu, num, rv))
280 #define	VMSETREG(vmi, vcpu, num, val)	((*ops->vmsetreg)(vmi, vcpu, num, val))
281 #define	VMGETDESC(vmi, vcpu, num, dsc)	((*ops->vmgetdesc)(vmi, vcpu, num, dsc))
282 #define	VMSETDESC(vmi, vcpu, num, dsc)	((*ops->vmsetdesc)(vmi, vcpu, num, dsc))
283 #define	VMGETCAP(vmi, vcpu, num, rv)	((*ops->vmgetcap)(vmi, vcpu, num, rv))
284 #define	VMSETCAP(vmi, vcpu, num, val)	((*ops->vmsetcap)(vmi, vcpu, num, val))
285 #define	VLAPIC_INIT(vmi, vcpu)		((*ops->vlapic_init)(vmi, vcpu))
286 #define	VLAPIC_CLEANUP(vmi, vlapic)	((*ops->vlapic_cleanup)(vmi, vlapic))
287 
288 #define	fpu_start_emulating()	load_cr0(rcr0() | CR0_TS)
289 #define	fpu_stop_emulating()	clts()
290 
291 SDT_PROVIDER_DEFINE(vmm);
292 
293 SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
294     NULL);
295 
296 /*
297  * Halt the guest if all vcpus are executing a HLT instruction with
298  * interrupts disabled.
299  */
300 int halt_detection_enabled = 1;
301 
302 /* Trap into hypervisor on all guest exceptions and reflect them back */
303 int trace_guest_exceptions;
304 
305 /* Trap WBINVD and ignore it */
306 int trap_wbinvd = 1;
307 
308 static void vm_free_memmap(struct vm *vm, int ident);
309 static bool sysmem_mapping(struct vm *vm, struct mem_map *mm);
310 static void vcpu_notify_event_locked(struct vcpu *vcpu, vcpu_notify_t);
311 static bool vcpu_sleep_bailout_checks(struct vm *vm, int vcpuid);
312 static int vcpu_vector_sipi(struct vm *vm, int vcpuid, uint8_t vector);
313 static bool vm_is_suspended(struct vm *, struct vm_exit *);
314 
315 static void vmm_savectx(void *);
316 static void vmm_restorectx(void *);
317 static const struct ctxop_template vmm_ctxop_tpl = {
318 	.ct_rev		= CTXOP_TPL_REV,
319 	.ct_save	= vmm_savectx,
320 	.ct_restore	= vmm_restorectx,
321 };
322 
323 static uint64_t calc_tsc_offset(uint64_t base_host_tsc, uint64_t base_guest_tsc,
324     uint64_t mult);
325 static uint64_t calc_guest_tsc(uint64_t host_tsc, uint64_t mult,
326     uint64_t offset);
327 
328 /* functions implemented in vmm_time_support.S */
329 uint64_t calc_freq_multiplier(uint64_t guest_hz, uint64_t host_hz,
330     uint32_t frac_size);
331 uint64_t scale_tsc(uint64_t tsc, uint64_t multiplier, uint32_t frac_size);
332 
333 #ifdef KTR
334 static const char *
vcpu_state2str(enum vcpu_state state)335 vcpu_state2str(enum vcpu_state state)
336 {
337 
338 	switch (state) {
339 	case VCPU_IDLE:
340 		return ("idle");
341 	case VCPU_FROZEN:
342 		return ("frozen");
343 	case VCPU_RUNNING:
344 		return ("running");
345 	case VCPU_SLEEPING:
346 		return ("sleeping");
347 	default:
348 		return ("unknown");
349 	}
350 }
351 #endif
352 
353 static void
vcpu_cleanup(struct vm * vm,int i,bool destroy)354 vcpu_cleanup(struct vm *vm, int i, bool destroy)
355 {
356 	struct vcpu *vcpu = &vm->vcpu[i];
357 
358 	VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic);
359 	if (destroy) {
360 		vmm_stat_free(vcpu->stats);
361 
362 		vcpu_cpuid_cleanup(&vcpu->cpuid_cfg);
363 
364 		hma_fpu_free(vcpu->guestfpu);
365 		vcpu->guestfpu = NULL;
366 
367 		vie_free(vcpu->vie_ctx);
368 		vcpu->vie_ctx = NULL;
369 
370 		vmc_destroy(vcpu->vmclient);
371 		vcpu->vmclient = NULL;
372 
373 		ctxop_free(vcpu->ctxop);
374 		mutex_destroy(&vcpu->lock);
375 	}
376 }
377 
378 static void
vcpu_init(struct vm * vm,int vcpu_id,bool create)379 vcpu_init(struct vm *vm, int vcpu_id, bool create)
380 {
381 	struct vcpu *vcpu;
382 
383 	KASSERT(vcpu_id >= 0 && vcpu_id < vm->maxcpus,
384 	    ("vcpu_init: invalid vcpu %d", vcpu_id));
385 
386 	vcpu = &vm->vcpu[vcpu_id];
387 
388 	if (create) {
389 		mutex_init(&vcpu->lock, NULL, MUTEX_ADAPTIVE, NULL);
390 
391 		vcpu->state = VCPU_IDLE;
392 		vcpu->hostcpu = NOCPU;
393 		vcpu->lastloccpu = NOCPU;
394 		vcpu->guestfpu = hma_fpu_alloc(KM_SLEEP);
395 		vcpu->stats = vmm_stat_alloc();
396 		vcpu->vie_ctx = vie_alloc();
397 		vcpu_cpuid_init(&vcpu->cpuid_cfg);
398 
399 		vcpu->ustate = VU_INIT;
400 		vcpu->ustate_when = gethrtime();
401 
402 		vcpu->vtc.vtc_vm = vm;
403 		vcpu->vtc.vtc_vcpuid = vcpu_id;
404 		vcpu->ctxop = ctxop_allocate(&vmm_ctxop_tpl, &vcpu->vtc);
405 	} else {
406 		vie_reset(vcpu->vie_ctx);
407 		bzero(&vcpu->exitinfo, sizeof (vcpu->exitinfo));
408 		vcpu_ustate_change(vm, vcpu_id, VU_INIT);
409 		bzero(&vcpu->mtrr, sizeof (vcpu->mtrr));
410 	}
411 
412 	vcpu->run_state = VRS_HALT;
413 	vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);
414 	(void) vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
415 	vcpu->reqidle = false;
416 	vcpu->reqconsist = false;
417 	vcpu->reqbarrier = false;
418 	vcpu->exit_intinfo = 0;
419 	vcpu->nmi_pending = false;
420 	vcpu->extint_pending = false;
421 	vcpu->exc_pending = 0;
422 	vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
423 	(void) hma_fpu_init(vcpu->guestfpu);
424 	vmm_stat_init(vcpu->stats);
425 	vcpu->tsc_offset = 0;
426 }
427 
428 int
vcpu_trace_exceptions(struct vm * vm,int vcpuid)429 vcpu_trace_exceptions(struct vm *vm, int vcpuid)
430 {
431 	return (trace_guest_exceptions);
432 }
433 
434 int
vcpu_trap_wbinvd(struct vm * vm,int vcpuid)435 vcpu_trap_wbinvd(struct vm *vm, int vcpuid)
436 {
437 	return (trap_wbinvd);
438 }
439 
440 struct vm_exit *
vm_exitinfo(struct vm * vm,int cpuid)441 vm_exitinfo(struct vm *vm, int cpuid)
442 {
443 	struct vcpu *vcpu;
444 
445 	if (cpuid < 0 || cpuid >= vm->maxcpus)
446 		panic("vm_exitinfo: invalid cpuid %d", cpuid);
447 
448 	vcpu = &vm->vcpu[cpuid];
449 
450 	return (&vcpu->exitinfo);
451 }
452 
453 struct vie *
vm_vie_ctx(struct vm * vm,int cpuid)454 vm_vie_ctx(struct vm *vm, int cpuid)
455 {
456 	if (cpuid < 0 || cpuid >= vm->maxcpus)
457 		panic("vm_vie_ctx: invalid cpuid %d", cpuid);
458 
459 	return (vm->vcpu[cpuid].vie_ctx);
460 }
461 
462 static int
vmm_init(void)463 vmm_init(void)
464 {
465 	vmm_host_state_init();
466 	vmm_host_freq = unscalehrtime(NANOSEC);
467 
468 	if (vmm_is_intel()) {
469 		ops = &vmm_ops_intel;
470 		pte_ops = &ept_pte_ops;
471 	} else if (vmm_is_svm()) {
472 		ops = &vmm_ops_amd;
473 		pte_ops = &rvi_pte_ops;
474 	} else {
475 		return (ENXIO);
476 	}
477 
478 	return (VMM_INIT());
479 }
480 
481 int
vmm_mod_load()482 vmm_mod_load()
483 {
484 	int	error;
485 
486 	VERIFY(vmm_initialized == 0);
487 
488 	error = vmm_init();
489 	if (error == 0)
490 		vmm_initialized = 1;
491 
492 	return (error);
493 }
494 
495 int
vmm_mod_unload()496 vmm_mod_unload()
497 {
498 	int	error;
499 
500 	VERIFY(vmm_initialized == 1);
501 
502 	error = VMM_CLEANUP();
503 	if (error)
504 		return (error);
505 	vmm_initialized = 0;
506 
507 	return (0);
508 }
509 
510 /*
511  * Create a test IOMMU domain to see if the host system has necessary hardware
512  * and drivers to do so.
513  */
514 bool
vmm_check_iommu(void)515 vmm_check_iommu(void)
516 {
517 	void *domain;
518 	const size_t arb_test_sz = (1UL << 32);
519 
520 	domain = iommu_create_domain(arb_test_sz);
521 	if (domain == NULL) {
522 		return (false);
523 	}
524 	iommu_destroy_domain(domain);
525 	return (true);
526 }
527 
528 static void
vm_init(struct vm * vm,bool create)529 vm_init(struct vm *vm, bool create)
530 {
531 	int i;
532 
533 	vm->cookie = VMINIT(vm);
534 	vm->iommu = NULL;
535 	vm->vioapic = vioapic_init(vm);
536 	vm->vhpet = vhpet_init(vm);
537 	vm->vatpic = vatpic_init(vm);
538 	vm->vatpit = vatpit_init(vm);
539 	vm->vpmtmr = vpmtmr_init(vm);
540 	if (create)
541 		vm->vrtc = vrtc_init(vm);
542 
543 	vm_inout_init(vm, &vm->ioports);
544 
545 	CPU_ZERO(&vm->active_cpus);
546 	CPU_ZERO(&vm->debug_cpus);
547 
548 	vm->suspend_how = 0;
549 	vm->suspend_source = 0;
550 	vm->suspend_when = 0;
551 
552 	for (i = 0; i < vm->maxcpus; i++)
553 		vcpu_init(vm, i, create);
554 
555 	/*
556 	 * Configure VM time-related data, including:
557 	 * - VM-wide TSC offset
558 	 * - boot_hrtime
559 	 * - guest_freq (same as host at boot time)
560 	 * - freq_multiplier (used for scaling)
561 	 *
562 	 * This data is configured such that the call to vm_init() represents
563 	 * the boot time (when the TSC(s) read 0).  Each vCPU will have its own
564 	 * offset from this, which is altered if/when the guest writes to
565 	 * MSR_TSC.
566 	 *
567 	 * Further changes to this data may occur if userspace writes to the
568 	 * time data.
569 	 */
570 	const uint64_t boot_tsc = rdtsc_offset();
571 
572 	/* Convert the boot TSC reading to hrtime */
573 	vm->boot_hrtime = (hrtime_t)boot_tsc;
574 	scalehrtime(&vm->boot_hrtime);
575 
576 	/* Guest frequency is the same as the host at boot time */
577 	vm->guest_freq = vmm_host_freq;
578 
579 	/* no scaling needed if guest_freq == host_freq */
580 	vm->freq_multiplier = VM_TSCM_NOSCALE;
581 
582 	/* configure VM-wide offset: initial guest TSC is 0 at boot */
583 	vm->tsc_offset = calc_tsc_offset(boot_tsc, 0, vm->freq_multiplier);
584 }
585 
586 /*
587  * The default CPU topology is a single thread per package.
588  */
589 uint_t cores_per_package = 1;
590 uint_t threads_per_core = 1;
591 
592 int
vm_create(uint64_t flags,struct vm ** retvm)593 vm_create(uint64_t flags, struct vm **retvm)
594 {
595 	struct vm *vm;
596 	struct vmspace *vmspace;
597 
598 	/*
599 	 * If vmm.ko could not be successfully initialized then don't attempt
600 	 * to create the virtual machine.
601 	 */
602 	if (!vmm_initialized)
603 		return (ENXIO);
604 
605 	bool track_dirty = (flags & VCF_TRACK_DIRTY) != 0;
606 	if (track_dirty && !pte_ops->vpeo_hw_ad_supported())
607 		return (ENOTSUP);
608 
609 	vmspace = vmspace_alloc(VM_MAXUSER_ADDRESS, pte_ops, track_dirty);
610 	if (vmspace == NULL)
611 		return (ENOMEM);
612 
613 	vm = kmem_zalloc(sizeof (struct vm), KM_SLEEP);
614 
615 	vm->vmspace = vmspace;
616 	vm->mem_transient = (flags & VCF_RESERVOIR_MEM) == 0;
617 	for (uint_t i = 0; i < VM_MAXCPU; i++) {
618 		vm->vcpu[i].vmclient = vmspace_client_alloc(vmspace);
619 	}
620 
621 	vm->sockets = 1;
622 	vm->cores = cores_per_package;	/* XXX backwards compatibility */
623 	vm->threads = threads_per_core;	/* XXX backwards compatibility */
624 	vm->maxcpus = VM_MAXCPU;	/* XXX temp to keep code working */
625 
626 	vm_init(vm, true);
627 
628 	*retvm = vm;
629 	return (0);
630 }
631 
632 void
vm_get_topology(struct vm * vm,uint16_t * sockets,uint16_t * cores,uint16_t * threads,uint16_t * maxcpus)633 vm_get_topology(struct vm *vm, uint16_t *sockets, uint16_t *cores,
634     uint16_t *threads, uint16_t *maxcpus)
635 {
636 	*sockets = vm->sockets;
637 	*cores = vm->cores;
638 	*threads = vm->threads;
639 	*maxcpus = vm->maxcpus;
640 }
641 
642 uint16_t
vm_get_maxcpus(struct vm * vm)643 vm_get_maxcpus(struct vm *vm)
644 {
645 	return (vm->maxcpus);
646 }
647 
648 int
vm_set_topology(struct vm * vm,uint16_t sockets,uint16_t cores,uint16_t threads,uint16_t maxcpus)649 vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores,
650     uint16_t threads, uint16_t maxcpus)
651 {
652 	if (maxcpus != 0)
653 		return (EINVAL);	/* XXX remove when supported */
654 	if ((sockets * cores * threads) > vm->maxcpus)
655 		return (EINVAL);
656 	/* XXX need to check sockets * cores * threads == vCPU, how? */
657 	vm->sockets = sockets;
658 	vm->cores = cores;
659 	vm->threads = threads;
660 	vm->maxcpus = VM_MAXCPU;	/* XXX temp to keep code working */
661 	return (0);
662 }
663 
664 static void
vm_cleanup(struct vm * vm,bool destroy)665 vm_cleanup(struct vm *vm, bool destroy)
666 {
667 	struct mem_map *mm;
668 	int i;
669 
670 	ppt_unassign_all(vm);
671 
672 	if (vm->iommu != NULL)
673 		iommu_destroy_domain(vm->iommu);
674 
675 	/*
676 	 * Devices which attach their own ioport hooks should be cleaned up
677 	 * first so they can tear down those registrations.
678 	 */
679 	vpmtmr_cleanup(vm->vpmtmr);
680 
681 	vm_inout_cleanup(vm, &vm->ioports);
682 
683 	if (destroy)
684 		vrtc_cleanup(vm->vrtc);
685 	else
686 		vrtc_reset(vm->vrtc);
687 
688 	vatpit_cleanup(vm->vatpit);
689 	vhpet_cleanup(vm->vhpet);
690 	vatpic_cleanup(vm->vatpic);
691 	vioapic_cleanup(vm->vioapic);
692 
693 	for (i = 0; i < vm->maxcpus; i++)
694 		vcpu_cleanup(vm, i, destroy);
695 
696 	VMCLEANUP(vm->cookie);
697 
698 	/*
699 	 * System memory is removed from the guest address space only when
700 	 * the VM is destroyed. This is because the mapping remains the same
701 	 * across VM reset.
702 	 *
703 	 * Device memory can be relocated by the guest (e.g. using PCI BARs)
704 	 * so those mappings are removed on a VM reset.
705 	 */
706 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
707 		mm = &vm->mem_maps[i];
708 		if (destroy || !sysmem_mapping(vm, mm)) {
709 			vm_free_memmap(vm, i);
710 		} else {
711 			/*
712 			 * We need to reset the IOMMU flag so this mapping can
713 			 * be reused when a VM is rebooted. Since the IOMMU
714 			 * domain has already been destroyed we can just reset
715 			 * the flag here.
716 			 */
717 			mm->flags &= ~VM_MEMMAP_F_IOMMU;
718 		}
719 	}
720 
721 	if (destroy) {
722 		for (i = 0; i < VM_MAX_MEMSEGS; i++)
723 			vm_free_memseg(vm, i);
724 
725 		vmspace_destroy(vm->vmspace);
726 		vm->vmspace = NULL;
727 	}
728 }
729 
730 void
vm_destroy(struct vm * vm)731 vm_destroy(struct vm *vm)
732 {
733 	vm_cleanup(vm, true);
734 	kmem_free(vm, sizeof (*vm));
735 }
736 
737 int
vm_reinit(struct vm * vm,uint64_t flags)738 vm_reinit(struct vm *vm, uint64_t flags)
739 {
740 	vm_cleanup(vm, false);
741 	vm_init(vm, false);
742 	return (0);
743 }
744 
745 bool
vm_is_paused(struct vm * vm)746 vm_is_paused(struct vm *vm)
747 {
748 	return (vm->is_paused);
749 }
750 
751 int
vm_pause_instance(struct vm * vm)752 vm_pause_instance(struct vm *vm)
753 {
754 	if (vm->is_paused) {
755 		return (EALREADY);
756 	}
757 	vm->is_paused = true;
758 
759 	for (uint_t i = 0; i < vm->maxcpus; i++) {
760 		struct vcpu *vcpu = &vm->vcpu[i];
761 
762 		if (!CPU_ISSET(i, &vm->active_cpus)) {
763 			continue;
764 		}
765 		vlapic_pause(vcpu->vlapic);
766 
767 		/*
768 		 * vCPU-specific pause logic includes stashing any
769 		 * to-be-injected events in exit_intinfo where it can be
770 		 * accessed in a manner generic to the backend.
771 		 */
772 		ops->vmpause(vm->cookie, i);
773 	}
774 	vhpet_pause(vm->vhpet);
775 	vatpit_pause(vm->vatpit);
776 	vrtc_pause(vm->vrtc);
777 
778 	return (0);
779 }
780 
781 int
vm_resume_instance(struct vm * vm)782 vm_resume_instance(struct vm *vm)
783 {
784 	if (!vm->is_paused) {
785 		return (EALREADY);
786 	}
787 	vm->is_paused = false;
788 
789 	vrtc_resume(vm->vrtc);
790 	vatpit_resume(vm->vatpit);
791 	vhpet_resume(vm->vhpet);
792 	for (uint_t i = 0; i < vm->maxcpus; i++) {
793 		struct vcpu *vcpu = &vm->vcpu[i];
794 
795 		if (!CPU_ISSET(i, &vm->active_cpus)) {
796 			continue;
797 		}
798 		vlapic_resume(vcpu->vlapic);
799 	}
800 
801 	return (0);
802 }
803 
804 int
vm_map_mmio(struct vm * vm,vm_paddr_t gpa,size_t len,vm_paddr_t hpa)805 vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
806 {
807 	vm_object_t *obj;
808 
809 	if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
810 		return (ENOMEM);
811 	else
812 		return (0);
813 }
814 
815 int
vm_unmap_mmio(struct vm * vm,vm_paddr_t gpa,size_t len)816 vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
817 {
818 	return (vmspace_unmap(vm->vmspace, gpa, len));
819 }
820 
821 /*
822  * Return 'true' if 'gpa' is allocated in the guest address space.
823  *
824  * This function is called in the context of a running vcpu which acts as
825  * an implicit lock on 'vm->mem_maps[]'.
826  */
827 bool
vm_mem_allocated(struct vm * vm,int vcpuid,vm_paddr_t gpa)828 vm_mem_allocated(struct vm *vm, int vcpuid, vm_paddr_t gpa)
829 {
830 	struct mem_map *mm;
831 	int i;
832 
833 #ifdef INVARIANTS
834 	int hostcpu, state;
835 	state = vcpu_get_state(vm, vcpuid, &hostcpu);
836 	KASSERT(state == VCPU_RUNNING && hostcpu == curcpu,
837 	    ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu));
838 #endif
839 
840 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
841 		mm = &vm->mem_maps[i];
842 		if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len)
843 			return (true);		/* 'gpa' is sysmem or devmem */
844 	}
845 
846 	if (ppt_is_mmio(vm, gpa))
847 		return (true);			/* 'gpa' is pci passthru mmio */
848 
849 	return (false);
850 }
851 
852 int
vm_alloc_memseg(struct vm * vm,int ident,size_t len,bool sysmem)853 vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem)
854 {
855 	struct mem_seg *seg;
856 	vm_object_t *obj;
857 
858 	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
859 		return (EINVAL);
860 
861 	if (len == 0 || (len & PAGE_MASK))
862 		return (EINVAL);
863 
864 	seg = &vm->mem_segs[ident];
865 	if (seg->object != NULL) {
866 		if (seg->len == len && seg->sysmem == sysmem)
867 			return (EEXIST);
868 		else
869 			return (EINVAL);
870 	}
871 
872 	obj = vm_object_mem_allocate(len, vm->mem_transient);
873 	if (obj == NULL)
874 		return (ENOMEM);
875 
876 	seg->len = len;
877 	seg->object = obj;
878 	seg->sysmem = sysmem;
879 	return (0);
880 }
881 
882 int
vm_get_memseg(struct vm * vm,int ident,size_t * len,bool * sysmem,vm_object_t ** objptr)883 vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
884     vm_object_t **objptr)
885 {
886 	struct mem_seg *seg;
887 
888 	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
889 		return (EINVAL);
890 
891 	seg = &vm->mem_segs[ident];
892 	if (len)
893 		*len = seg->len;
894 	if (sysmem)
895 		*sysmem = seg->sysmem;
896 	if (objptr)
897 		*objptr = seg->object;
898 	return (0);
899 }
900 
901 void
vm_free_memseg(struct vm * vm,int ident)902 vm_free_memseg(struct vm *vm, int ident)
903 {
904 	struct mem_seg *seg;
905 
906 	KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS,
907 	    ("%s: invalid memseg ident %d", __func__, ident));
908 
909 	seg = &vm->mem_segs[ident];
910 	if (seg->object != NULL) {
911 		vm_object_release(seg->object);
912 		bzero(seg, sizeof (struct mem_seg));
913 	}
914 }
915 
916 int
vm_mmap_memseg(struct vm * vm,vm_paddr_t gpa,int segid,vm_ooffset_t first,size_t len,int prot,int flags)917 vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
918     size_t len, int prot, int flags)
919 {
920 	struct mem_seg *seg;
921 	struct mem_map *m, *map;
922 	vm_ooffset_t last;
923 	int i, error;
924 
925 	if (prot == 0 || (prot & ~(PROT_ALL)) != 0)
926 		return (EINVAL);
927 
928 	if (flags & ~VM_MEMMAP_F_WIRED)
929 		return (EINVAL);
930 
931 	if (segid < 0 || segid >= VM_MAX_MEMSEGS)
932 		return (EINVAL);
933 
934 	seg = &vm->mem_segs[segid];
935 	if (seg->object == NULL)
936 		return (EINVAL);
937 
938 	last = first + len;
939 	if (first < 0 || first >= last || last > seg->len)
940 		return (EINVAL);
941 
942 	if ((gpa | first | last) & PAGE_MASK)
943 		return (EINVAL);
944 
945 	map = NULL;
946 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
947 		m = &vm->mem_maps[i];
948 		if (m->len == 0) {
949 			map = m;
950 			break;
951 		}
952 	}
953 
954 	if (map == NULL)
955 		return (ENOSPC);
956 
957 	error = vmspace_map(vm->vmspace, seg->object, first, gpa, len, prot);
958 	if (error != 0)
959 		return (EFAULT);
960 
961 	vm_object_reference(seg->object);
962 
963 	if ((flags & VM_MEMMAP_F_WIRED) != 0) {
964 		error = vmspace_populate(vm->vmspace, gpa, len);
965 		if (error != 0) {
966 			VERIFY0(vmspace_unmap(vm->vmspace, gpa, len));
967 			return (EFAULT);
968 		}
969 	}
970 
971 	map->gpa = gpa;
972 	map->len = len;
973 	map->segoff = first;
974 	map->segid = segid;
975 	map->prot = prot;
976 	map->flags = flags;
977 	return (0);
978 }
979 
980 int
vm_munmap_memseg(struct vm * vm,vm_paddr_t gpa,size_t len)981 vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len)
982 {
983 	struct mem_map *m;
984 	int i;
985 
986 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
987 		m = &vm->mem_maps[i];
988 		if (m->gpa == gpa && m->len == len &&
989 		    (m->flags & VM_MEMMAP_F_IOMMU) == 0) {
990 			vm_free_memmap(vm, i);
991 			return (0);
992 		}
993 	}
994 
995 	return (EINVAL);
996 }
997 
998 int
vm_mmap_getnext(struct vm * vm,vm_paddr_t * gpa,int * segid,vm_ooffset_t * segoff,size_t * len,int * prot,int * flags)999 vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
1000     vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
1001 {
1002 	struct mem_map *mm, *mmnext;
1003 	int i;
1004 
1005 	mmnext = NULL;
1006 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1007 		mm = &vm->mem_maps[i];
1008 		if (mm->len == 0 || mm->gpa < *gpa)
1009 			continue;
1010 		if (mmnext == NULL || mm->gpa < mmnext->gpa)
1011 			mmnext = mm;
1012 	}
1013 
1014 	if (mmnext != NULL) {
1015 		*gpa = mmnext->gpa;
1016 		if (segid)
1017 			*segid = mmnext->segid;
1018 		if (segoff)
1019 			*segoff = mmnext->segoff;
1020 		if (len)
1021 			*len = mmnext->len;
1022 		if (prot)
1023 			*prot = mmnext->prot;
1024 		if (flags)
1025 			*flags = mmnext->flags;
1026 		return (0);
1027 	} else {
1028 		return (ENOENT);
1029 	}
1030 }
1031 
1032 static void
vm_free_memmap(struct vm * vm,int ident)1033 vm_free_memmap(struct vm *vm, int ident)
1034 {
1035 	struct mem_map *mm;
1036 	int error;
1037 
1038 	mm = &vm->mem_maps[ident];
1039 	if (mm->len) {
1040 		error = vmspace_unmap(vm->vmspace, mm->gpa, mm->len);
1041 		VERIFY0(error);
1042 		bzero(mm, sizeof (struct mem_map));
1043 	}
1044 }
1045 
1046 static __inline bool
sysmem_mapping(struct vm * vm,struct mem_map * mm)1047 sysmem_mapping(struct vm *vm, struct mem_map *mm)
1048 {
1049 
1050 	if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem)
1051 		return (true);
1052 	else
1053 		return (false);
1054 }
1055 
1056 vm_paddr_t
vmm_sysmem_maxaddr(struct vm * vm)1057 vmm_sysmem_maxaddr(struct vm *vm)
1058 {
1059 	struct mem_map *mm;
1060 	vm_paddr_t maxaddr;
1061 	int i;
1062 
1063 	maxaddr = 0;
1064 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1065 		mm = &vm->mem_maps[i];
1066 		if (sysmem_mapping(vm, mm)) {
1067 			if (maxaddr < mm->gpa + mm->len)
1068 				maxaddr = mm->gpa + mm->len;
1069 		}
1070 	}
1071 	return (maxaddr);
1072 }
1073 
1074 static void
vm_iommu_modify(struct vm * vm,bool map)1075 vm_iommu_modify(struct vm *vm, bool map)
1076 {
1077 	int i, sz;
1078 	vm_paddr_t gpa, hpa;
1079 	struct mem_map *mm;
1080 	vm_client_t *vmc;
1081 
1082 	sz = PAGE_SIZE;
1083 	vmc = vmspace_client_alloc(vm->vmspace);
1084 
1085 	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
1086 		mm = &vm->mem_maps[i];
1087 		if (!sysmem_mapping(vm, mm))
1088 			continue;
1089 
1090 		if (map) {
1091 			KASSERT((mm->flags & VM_MEMMAP_F_IOMMU) == 0,
1092 			    ("iommu map found invalid memmap %lx/%lx/%x",
1093 			    mm->gpa, mm->len, mm->flags));
1094 			if ((mm->flags & VM_MEMMAP_F_WIRED) == 0)
1095 				continue;
1096 			mm->flags |= VM_MEMMAP_F_IOMMU;
1097 		} else {
1098 			if ((mm->flags & VM_MEMMAP_F_IOMMU) == 0)
1099 				continue;
1100 			mm->flags &= ~VM_MEMMAP_F_IOMMU;
1101 			KASSERT((mm->flags & VM_MEMMAP_F_WIRED) != 0,
1102 			    ("iommu unmap found invalid memmap %lx/%lx/%x",
1103 			    mm->gpa, mm->len, mm->flags));
1104 		}
1105 
1106 		gpa = mm->gpa;
1107 		while (gpa < mm->gpa + mm->len) {
1108 			vm_page_t *vmp;
1109 
1110 			vmp = vmc_hold(vmc, gpa, PROT_WRITE);
1111 			ASSERT(vmp != NULL);
1112 			hpa = ((uintptr_t)vmp_get_pfn(vmp) << PAGESHIFT);
1113 			(void) vmp_release(vmp);
1114 
1115 			/*
1116 			 * When originally ported from FreeBSD, the logic for
1117 			 * adding memory to the guest domain would
1118 			 * simultaneously remove it from the host domain.  The
1119 			 * justification for that is not clear, and FreeBSD has
1120 			 * subsequently changed the behavior to not remove the
1121 			 * memory from the host domain.
1122 			 *
1123 			 * Leaving the guest memory in the host domain for the
1124 			 * life of the VM is necessary to make it available for
1125 			 * DMA, such as through viona in the TX path.
1126 			 */
1127 			if (map) {
1128 				iommu_create_mapping(vm->iommu, gpa, hpa, sz);
1129 			} else {
1130 				iommu_remove_mapping(vm->iommu, gpa, sz);
1131 			}
1132 
1133 			gpa += PAGE_SIZE;
1134 		}
1135 	}
1136 	vmc_destroy(vmc);
1137 
1138 	/*
1139 	 * Invalidate the cached translations associated with the domain
1140 	 * from which pages were removed.
1141 	 */
1142 	iommu_invalidate_tlb(vm->iommu);
1143 }
1144 
1145 int
vm_unassign_pptdev(struct vm * vm,int pptfd)1146 vm_unassign_pptdev(struct vm *vm, int pptfd)
1147 {
1148 	int error;
1149 
1150 	error = ppt_unassign_device(vm, pptfd);
1151 	if (error)
1152 		return (error);
1153 
1154 	if (ppt_assigned_devices(vm) == 0)
1155 		vm_iommu_modify(vm, false);
1156 
1157 	return (0);
1158 }
1159 
1160 int
vm_assign_pptdev(struct vm * vm,int pptfd)1161 vm_assign_pptdev(struct vm *vm, int pptfd)
1162 {
1163 	int error;
1164 	vm_paddr_t maxaddr;
1165 
1166 	/* Set up the IOMMU to do the 'gpa' to 'hpa' translation */
1167 	if (ppt_assigned_devices(vm) == 0) {
1168 		KASSERT(vm->iommu == NULL,
1169 		    ("vm_assign_pptdev: iommu must be NULL"));
1170 		maxaddr = vmm_sysmem_maxaddr(vm);
1171 		vm->iommu = iommu_create_domain(maxaddr);
1172 		if (vm->iommu == NULL)
1173 			return (ENXIO);
1174 		vm_iommu_modify(vm, true);
1175 	}
1176 
1177 	error = ppt_assign_device(vm, pptfd);
1178 	return (error);
1179 }
1180 
1181 int
vm_get_register(struct vm * vm,int vcpuid,int reg,uint64_t * retval)1182 vm_get_register(struct vm *vm, int vcpuid, int reg, uint64_t *retval)
1183 {
1184 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1185 		return (EINVAL);
1186 
1187 	if (reg >= VM_REG_LAST)
1188 		return (EINVAL);
1189 
1190 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1191 	switch (reg) {
1192 	case VM_REG_GUEST_XCR0:
1193 		*retval = vcpu->guest_xcr0;
1194 		return (0);
1195 	default:
1196 		return (VMGETREG(vm->cookie, vcpuid, reg, retval));
1197 	}
1198 }
1199 
1200 int
vm_set_register(struct vm * vm,int vcpuid,int reg,uint64_t val)1201 vm_set_register(struct vm *vm, int vcpuid, int reg, uint64_t val)
1202 {
1203 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1204 		return (EINVAL);
1205 
1206 	if (reg >= VM_REG_LAST)
1207 		return (EINVAL);
1208 
1209 	int error;
1210 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1211 	switch (reg) {
1212 	case VM_REG_GUEST_RIP:
1213 		error = VMSETREG(vm->cookie, vcpuid, reg, val);
1214 		if (error == 0) {
1215 			vcpu->nextrip = val;
1216 		}
1217 		return (error);
1218 	case VM_REG_GUEST_XCR0:
1219 		if (!validate_guest_xcr0(val, vmm_get_host_xcr0())) {
1220 			return (EINVAL);
1221 		}
1222 		vcpu->guest_xcr0 = val;
1223 		return (0);
1224 	default:
1225 		return (VMSETREG(vm->cookie, vcpuid, reg, val));
1226 	}
1227 }
1228 
1229 static bool
is_descriptor_table(int reg)1230 is_descriptor_table(int reg)
1231 {
1232 	switch (reg) {
1233 	case VM_REG_GUEST_IDTR:
1234 	case VM_REG_GUEST_GDTR:
1235 		return (true);
1236 	default:
1237 		return (false);
1238 	}
1239 }
1240 
1241 static bool
is_segment_register(int reg)1242 is_segment_register(int reg)
1243 {
1244 	switch (reg) {
1245 	case VM_REG_GUEST_ES:
1246 	case VM_REG_GUEST_CS:
1247 	case VM_REG_GUEST_SS:
1248 	case VM_REG_GUEST_DS:
1249 	case VM_REG_GUEST_FS:
1250 	case VM_REG_GUEST_GS:
1251 	case VM_REG_GUEST_TR:
1252 	case VM_REG_GUEST_LDTR:
1253 		return (true);
1254 	default:
1255 		return (false);
1256 	}
1257 }
1258 
1259 int
vm_get_seg_desc(struct vm * vm,int vcpu,int reg,struct seg_desc * desc)1260 vm_get_seg_desc(struct vm *vm, int vcpu, int reg, struct seg_desc *desc)
1261 {
1262 
1263 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1264 		return (EINVAL);
1265 
1266 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
1267 		return (EINVAL);
1268 
1269 	return (VMGETDESC(vm->cookie, vcpu, reg, desc));
1270 }
1271 
1272 int
vm_set_seg_desc(struct vm * vm,int vcpu,int reg,const struct seg_desc * desc)1273 vm_set_seg_desc(struct vm *vm, int vcpu, int reg, const struct seg_desc *desc)
1274 {
1275 	if (vcpu < 0 || vcpu >= vm->maxcpus)
1276 		return (EINVAL);
1277 
1278 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
1279 		return (EINVAL);
1280 
1281 	return (VMSETDESC(vm->cookie, vcpu, reg, desc));
1282 }
1283 
1284 static int
translate_hma_xsave_result(hma_fpu_xsave_result_t res)1285 translate_hma_xsave_result(hma_fpu_xsave_result_t res)
1286 {
1287 	switch (res) {
1288 	case HFXR_OK:
1289 		return (0);
1290 	case HFXR_NO_SPACE:
1291 		return (ENOSPC);
1292 	case HFXR_BAD_ALIGN:
1293 	case HFXR_UNSUP_FMT:
1294 	case HFXR_UNSUP_FEAT:
1295 	case HFXR_INVALID_DATA:
1296 		return (EINVAL);
1297 	default:
1298 		panic("unexpected xsave result");
1299 	}
1300 }
1301 
1302 int
vm_get_fpu(struct vm * vm,int vcpuid,void * buf,size_t len)1303 vm_get_fpu(struct vm *vm, int vcpuid, void *buf, size_t len)
1304 {
1305 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1306 		return (EINVAL);
1307 
1308 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1309 	hma_fpu_xsave_result_t res;
1310 
1311 	res = hma_fpu_get_xsave_state(vcpu->guestfpu, buf, len);
1312 	return (translate_hma_xsave_result(res));
1313 }
1314 
1315 int
vm_set_fpu(struct vm * vm,int vcpuid,void * buf,size_t len)1316 vm_set_fpu(struct vm *vm, int vcpuid, void *buf, size_t len)
1317 {
1318 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
1319 		return (EINVAL);
1320 
1321 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1322 	hma_fpu_xsave_result_t res;
1323 
1324 	res = hma_fpu_set_xsave_state(vcpu->guestfpu, buf, len);
1325 	return (translate_hma_xsave_result(res));
1326 }
1327 
1328 int
vm_get_run_state(struct vm * vm,int vcpuid,uint32_t * state,uint8_t * sipi_vec)1329 vm_get_run_state(struct vm *vm, int vcpuid, uint32_t *state, uint8_t *sipi_vec)
1330 {
1331 	struct vcpu *vcpu;
1332 
1333 	if (vcpuid < 0 || vcpuid >= vm->maxcpus) {
1334 		return (EINVAL);
1335 	}
1336 
1337 	vcpu = &vm->vcpu[vcpuid];
1338 
1339 	vcpu_lock(vcpu);
1340 	*state = vcpu->run_state;
1341 	*sipi_vec = vcpu->sipi_vector;
1342 	vcpu_unlock(vcpu);
1343 
1344 	return (0);
1345 }
1346 
1347 int
vm_set_run_state(struct vm * vm,int vcpuid,uint32_t state,uint8_t sipi_vec)1348 vm_set_run_state(struct vm *vm, int vcpuid, uint32_t state, uint8_t sipi_vec)
1349 {
1350 	struct vcpu *vcpu;
1351 
1352 	if (vcpuid < 0 || vcpuid >= vm->maxcpus) {
1353 		return (EINVAL);
1354 	}
1355 	if (!VRS_IS_VALID(state)) {
1356 		return (EINVAL);
1357 	}
1358 
1359 	vcpu = &vm->vcpu[vcpuid];
1360 
1361 	vcpu_lock(vcpu);
1362 	vcpu->run_state = state;
1363 	vcpu->sipi_vector = sipi_vec;
1364 	vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
1365 	vcpu_unlock(vcpu);
1366 
1367 	return (0);
1368 }
1369 
1370 int
vm_track_dirty_pages(struct vm * vm,uint64_t gpa,size_t len,uint8_t * bitmap)1371 vm_track_dirty_pages(struct vm *vm, uint64_t gpa, size_t len, uint8_t *bitmap)
1372 {
1373 	ASSERT0(gpa & PAGEOFFSET);
1374 	ASSERT0(len & PAGEOFFSET);
1375 
1376 	/*
1377 	 * The only difference in expectations between this legacy interface and
1378 	 * an equivalent call to vm_npt_do_operation() is the check for
1379 	 * dirty-page-tracking being enabled on the vmspace.
1380 	 */
1381 	if (!vmspace_get_tracking(vm->vmspace)) {
1382 		return (EPERM);
1383 	}
1384 
1385 	vmspace_bits_operate(vm->vmspace, gpa, len,
1386 	    VBO_RESET_DIRTY | VBO_FLAG_BITMAP_OUT, bitmap);
1387 	return (0);
1388 }
1389 
1390 int
vm_npt_do_operation(struct vm * vm,uint64_t gpa,size_t len,uint32_t oper,uint8_t * bitmap,int * rvalp)1391 vm_npt_do_operation(struct vm *vm, uint64_t gpa, size_t len, uint32_t oper,
1392     uint8_t *bitmap, int *rvalp)
1393 {
1394 	ASSERT0(gpa & PAGEOFFSET);
1395 	ASSERT0(len & PAGEOFFSET);
1396 
1397 	/*
1398 	 * For now, the bits defined in vmm_dev.h are meant to match up 1:1 with
1399 	 * those in vmm_vm.h
1400 	 */
1401 	CTASSERT(VNO_OP_RESET_DIRTY == VBO_RESET_DIRTY);
1402 	CTASSERT(VNO_OP_SET_DIRTY == VBO_SET_DIRTY);
1403 	CTASSERT(VNO_OP_GET_DIRTY == VBO_GET_DIRTY);
1404 	CTASSERT(VNO_FLAG_BITMAP_IN == VBO_FLAG_BITMAP_IN);
1405 	CTASSERT(VNO_FLAG_BITMAP_OUT == VBO_FLAG_BITMAP_OUT);
1406 
1407 	const uint32_t oper_only =
1408 	    oper & ~(VNO_FLAG_BITMAP_IN | VNO_FLAG_BITMAP_OUT);
1409 	switch (oper_only) {
1410 	case VNO_OP_RESET_DIRTY:
1411 	case VNO_OP_SET_DIRTY:
1412 	case VNO_OP_GET_DIRTY:
1413 		if (len == 0) {
1414 			break;
1415 		}
1416 		vmspace_bits_operate(vm->vmspace, gpa, len, oper, bitmap);
1417 		break;
1418 	case VNO_OP_GET_TRACK_DIRTY:
1419 		ASSERT3P(rvalp, !=, NULL);
1420 		*rvalp = vmspace_get_tracking(vm->vmspace) ? 1 : 0;
1421 		break;
1422 	case VNO_OP_EN_TRACK_DIRTY:
1423 		return (vmspace_set_tracking(vm->vmspace, true));
1424 	case VNO_OP_DIS_TRACK_DIRTY:
1425 		return (vmspace_set_tracking(vm->vmspace, false));
1426 	default:
1427 		return (EINVAL);
1428 	}
1429 	return (0);
1430 }
1431 
1432 static void
restore_guest_fpustate(struct vcpu * vcpu)1433 restore_guest_fpustate(struct vcpu *vcpu)
1434 {
1435 	/* Save host FPU and restore guest FPU */
1436 	fpu_stop_emulating();
1437 	hma_fpu_start_guest(vcpu->guestfpu);
1438 
1439 	/* restore guest XCR0 if XSAVE is enabled in the host */
1440 	if (rcr4() & CR4_XSAVE)
1441 		load_xcr(0, vcpu->guest_xcr0);
1442 
1443 	/*
1444 	 * The FPU is now "dirty" with the guest's state so turn on emulation
1445 	 * to trap any access to the FPU by the host.
1446 	 */
1447 	fpu_start_emulating();
1448 }
1449 
1450 static void
save_guest_fpustate(struct vcpu * vcpu)1451 save_guest_fpustate(struct vcpu *vcpu)
1452 {
1453 
1454 	if ((rcr0() & CR0_TS) == 0)
1455 		panic("fpu emulation not enabled in host!");
1456 
1457 	/* save guest XCR0 and restore host XCR0 */
1458 	if (rcr4() & CR4_XSAVE) {
1459 		vcpu->guest_xcr0 = rxcr(0);
1460 		load_xcr(0, vmm_get_host_xcr0());
1461 	}
1462 
1463 	/* save guest FPU and restore host FPU */
1464 	fpu_stop_emulating();
1465 	hma_fpu_stop_guest(vcpu->guestfpu);
1466 	/*
1467 	 * When the host state has been restored, we should not re-enable
1468 	 * CR0.TS on illumos for eager FPU.
1469 	 */
1470 }
1471 
1472 static int
vcpu_set_state_locked(struct vm * vm,int vcpuid,enum vcpu_state newstate,bool from_idle)1473 vcpu_set_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate,
1474     bool from_idle)
1475 {
1476 	struct vcpu *vcpu;
1477 	int error;
1478 
1479 	vcpu = &vm->vcpu[vcpuid];
1480 	vcpu_assert_locked(vcpu);
1481 
1482 	/*
1483 	 * State transitions from the vmmdev_ioctl() must always begin from
1484 	 * the VCPU_IDLE state. This guarantees that there is only a single
1485 	 * ioctl() operating on a vcpu at any point.
1486 	 */
1487 	if (from_idle) {
1488 		while (vcpu->state != VCPU_IDLE) {
1489 			vcpu->reqidle = true;
1490 			vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
1491 			cv_wait(&vcpu->state_cv, &vcpu->lock);
1492 			vcpu->reqidle = false;
1493 		}
1494 	} else {
1495 		KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
1496 		    "vcpu idle state"));
1497 	}
1498 
1499 	if (vcpu->state == VCPU_RUNNING) {
1500 		KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
1501 		    "mismatch for running vcpu", curcpu, vcpu->hostcpu));
1502 	} else {
1503 		KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
1504 		    "vcpu that is not running", vcpu->hostcpu));
1505 	}
1506 
1507 	/*
1508 	 * The following state transitions are allowed:
1509 	 * IDLE -> FROZEN -> IDLE
1510 	 * FROZEN -> RUNNING -> FROZEN
1511 	 * FROZEN -> SLEEPING -> FROZEN
1512 	 */
1513 	switch (vcpu->state) {
1514 	case VCPU_IDLE:
1515 	case VCPU_RUNNING:
1516 	case VCPU_SLEEPING:
1517 		error = (newstate != VCPU_FROZEN);
1518 		break;
1519 	case VCPU_FROZEN:
1520 		error = (newstate == VCPU_FROZEN);
1521 		break;
1522 	default:
1523 		error = 1;
1524 		break;
1525 	}
1526 
1527 	if (error)
1528 		return (EBUSY);
1529 
1530 	vcpu->state = newstate;
1531 	if (newstate == VCPU_RUNNING)
1532 		vcpu->hostcpu = curcpu;
1533 	else
1534 		vcpu->hostcpu = NOCPU;
1535 
1536 	if (newstate == VCPU_IDLE) {
1537 		cv_broadcast(&vcpu->state_cv);
1538 	}
1539 
1540 	return (0);
1541 }
1542 
1543 static void
vcpu_require_state(struct vm * vm,int vcpuid,enum vcpu_state newstate)1544 vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1545 {
1546 	int error;
1547 
1548 	if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0)
1549 		panic("Error %d setting state to %d\n", error, newstate);
1550 }
1551 
1552 static void
vcpu_require_state_locked(struct vm * vm,int vcpuid,enum vcpu_state newstate)1553 vcpu_require_state_locked(struct vm *vm, int vcpuid, enum vcpu_state newstate)
1554 {
1555 	int error;
1556 
1557 	if ((error = vcpu_set_state_locked(vm, vcpuid, newstate, false)) != 0)
1558 		panic("Error %d setting state to %d", error, newstate);
1559 }
1560 
1561 /*
1562  * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run.
1563  */
1564 static int
vm_handle_hlt(struct vm * vm,int vcpuid,bool intr_disabled)1565 vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled)
1566 {
1567 	struct vcpu *vcpu;
1568 	int vcpu_halted, vm_halted;
1569 	bool userspace_exit = false;
1570 
1571 	KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted"));
1572 
1573 	vcpu = &vm->vcpu[vcpuid];
1574 	vcpu_halted = 0;
1575 	vm_halted = 0;
1576 
1577 	vcpu_lock(vcpu);
1578 	while (1) {
1579 		/*
1580 		 * Do a final check for pending interrupts (including NMI and
1581 		 * INIT) before putting this thread to sleep.
1582 		 */
1583 		if (vm_nmi_pending(vm, vcpuid))
1584 			break;
1585 		if (vcpu_run_state_pending(vm, vcpuid))
1586 			break;
1587 		if (!intr_disabled) {
1588 			if (vm_extint_pending(vm, vcpuid) ||
1589 			    vlapic_pending_intr(vcpu->vlapic, NULL)) {
1590 				break;
1591 			}
1592 		}
1593 
1594 		/*
1595 		 * Also check for software events which would cause a wake-up.
1596 		 * This will set the appropriate exitcode directly, rather than
1597 		 * requiring a trip through VM_RUN().
1598 		 */
1599 		if (vcpu_sleep_bailout_checks(vm, vcpuid)) {
1600 			userspace_exit = true;
1601 			break;
1602 		}
1603 
1604 		/*
1605 		 * Some Linux guests implement "halt" by having all vcpus
1606 		 * execute HLT with interrupts disabled. 'halted_cpus' keeps
1607 		 * track of the vcpus that have entered this state. When all
1608 		 * vcpus enter the halted state the virtual machine is halted.
1609 		 */
1610 		if (intr_disabled) {
1611 			if (!vcpu_halted && halt_detection_enabled) {
1612 				vcpu_halted = 1;
1613 				CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus);
1614 			}
1615 			if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) {
1616 				vm_halted = 1;
1617 				break;
1618 			}
1619 		}
1620 
1621 		vcpu_ustate_change(vm, vcpuid, VU_IDLE);
1622 		vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1623 		(void) cv_wait_sig(&vcpu->vcpu_cv, &vcpu->lock);
1624 		vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
1625 		vcpu_ustate_change(vm, vcpuid, VU_EMU_KERN);
1626 	}
1627 
1628 	if (vcpu_halted)
1629 		CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus);
1630 
1631 	vcpu_unlock(vcpu);
1632 
1633 	if (vm_halted) {
1634 		(void) vm_suspend(vm, VM_SUSPEND_HALT, -1);
1635 	}
1636 
1637 	return (userspace_exit ? -1 : 0);
1638 }
1639 
1640 static int
vm_handle_paging(struct vm * vm,int vcpuid)1641 vm_handle_paging(struct vm *vm, int vcpuid)
1642 {
1643 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1644 	vm_client_t *vmc = vcpu->vmclient;
1645 	struct vm_exit *vme = &vcpu->exitinfo;
1646 	const int ftype = vme->u.paging.fault_type;
1647 
1648 	ASSERT0(vme->inst_length);
1649 	ASSERT(ftype == PROT_READ || ftype == PROT_WRITE || ftype == PROT_EXEC);
1650 
1651 	if (vmc_fault(vmc, vme->u.paging.gpa, ftype) != 0) {
1652 		/*
1653 		 * If the fault cannot be serviced, kick it out to userspace for
1654 		 * handling (or more likely, halting the instance).
1655 		 */
1656 		return (-1);
1657 	}
1658 
1659 	return (0);
1660 }
1661 
1662 int
vm_service_mmio_read(struct vm * vm,int cpuid,uint64_t gpa,uint64_t * rval,int rsize)1663 vm_service_mmio_read(struct vm *vm, int cpuid, uint64_t gpa, uint64_t *rval,
1664     int rsize)
1665 {
1666 	int err = ESRCH;
1667 
1668 	if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1669 		struct vlapic *vlapic = vm_lapic(vm, cpuid);
1670 
1671 		err = vlapic_mmio_read(vlapic, gpa, rval, rsize);
1672 	} else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1673 		err = vioapic_mmio_read(vm, cpuid, gpa, rval, rsize);
1674 	} else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
1675 		err = vhpet_mmio_read(vm, cpuid, gpa, rval, rsize);
1676 	}
1677 
1678 	return (err);
1679 }
1680 
1681 int
vm_service_mmio_write(struct vm * vm,int cpuid,uint64_t gpa,uint64_t wval,int wsize)1682 vm_service_mmio_write(struct vm *vm, int cpuid, uint64_t gpa, uint64_t wval,
1683     int wsize)
1684 {
1685 	int err = ESRCH;
1686 
1687 	if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
1688 		struct vlapic *vlapic = vm_lapic(vm, cpuid);
1689 
1690 		err = vlapic_mmio_write(vlapic, gpa, wval, wsize);
1691 	} else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
1692 		err = vioapic_mmio_write(vm, cpuid, gpa, wval, wsize);
1693 	} else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
1694 		err = vhpet_mmio_write(vm, cpuid, gpa, wval, wsize);
1695 	}
1696 
1697 	return (err);
1698 }
1699 
1700 static int
vm_handle_mmio_emul(struct vm * vm,int vcpuid)1701 vm_handle_mmio_emul(struct vm *vm, int vcpuid)
1702 {
1703 	struct vie *vie;
1704 	struct vcpu *vcpu;
1705 	struct vm_exit *vme;
1706 	uint64_t inst_addr;
1707 	int error, fault, cs_d;
1708 
1709 	vcpu = &vm->vcpu[vcpuid];
1710 	vme = &vcpu->exitinfo;
1711 	vie = vcpu->vie_ctx;
1712 
1713 	KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d",
1714 	    __func__, vme->inst_length));
1715 
1716 	inst_addr = vme->rip + vme->u.mmio_emul.cs_base;
1717 	cs_d = vme->u.mmio_emul.cs_d;
1718 
1719 	/* Fetch the faulting instruction */
1720 	if (vie_needs_fetch(vie)) {
1721 		error = vie_fetch_instruction(vie, vm, vcpuid, inst_addr,
1722 		    &fault);
1723 		if (error != 0) {
1724 			return (error);
1725 		} else if (fault) {
1726 			/*
1727 			 * If a fault during instruction fetch was encountered,
1728 			 * it will have asserted that the appropriate exception
1729 			 * be injected at next entry.
1730 			 * No further work is required.
1731 			 */
1732 			return (0);
1733 		}
1734 	}
1735 
1736 	if (vie_decode_instruction(vie, vm, vcpuid, cs_d) != 0) {
1737 		/* Dump (unrecognized) instruction bytes in userspace */
1738 		vie_fallback_exitinfo(vie, vme);
1739 		return (-1);
1740 	}
1741 	if (vme->u.mmio_emul.gla != VIE_INVALID_GLA &&
1742 	    vie_verify_gla(vie, vm, vcpuid, vme->u.mmio_emul.gla) != 0) {
1743 		/* Decoded GLA does not match GLA from VM exit state */
1744 		vie_fallback_exitinfo(vie, vme);
1745 		return (-1);
1746 	}
1747 
1748 repeat:
1749 	error = vie_emulate_mmio(vie, vm, vcpuid);
1750 	if (error < 0) {
1751 		/*
1752 		 * MMIO not handled by any of the in-kernel-emulated devices, so
1753 		 * make a trip out to userspace for it.
1754 		 */
1755 		vie_exitinfo(vie, vme);
1756 	} else if (error == EAGAIN) {
1757 		/*
1758 		 * Continue emulating the rep-prefixed instruction, which has
1759 		 * not completed its iterations.
1760 		 *
1761 		 * In case this can be emulated in-kernel and has a high
1762 		 * repetition count (causing a tight spin), it should be
1763 		 * deferential to yield conditions.
1764 		 */
1765 		if (!vcpu_should_yield(vm, vcpuid)) {
1766 			goto repeat;
1767 		} else {
1768 			/*
1769 			 * Defer to the contending load by making a trip to
1770 			 * userspace with a no-op (BOGUS) exit reason.
1771 			 */
1772 			vie_reset(vie);
1773 			vme->exitcode = VM_EXITCODE_BOGUS;
1774 			return (-1);
1775 		}
1776 	} else if (error == 0) {
1777 		/* Update %rip now that instruction has been emulated */
1778 		vie_advance_pc(vie, &vcpu->nextrip);
1779 	}
1780 	return (error);
1781 }
1782 
1783 static int
vm_handle_inout(struct vm * vm,int vcpuid,struct vm_exit * vme)1784 vm_handle_inout(struct vm *vm, int vcpuid, struct vm_exit *vme)
1785 {
1786 	struct vcpu *vcpu;
1787 	struct vie *vie;
1788 	int err;
1789 
1790 	vcpu = &vm->vcpu[vcpuid];
1791 	vie = vcpu->vie_ctx;
1792 
1793 repeat:
1794 	err = vie_emulate_inout(vie, vm, vcpuid);
1795 
1796 	if (err < 0) {
1797 		/*
1798 		 * In/out not handled by any of the in-kernel-emulated devices,
1799 		 * so make a trip out to userspace for it.
1800 		 */
1801 		vie_exitinfo(vie, vme);
1802 		return (err);
1803 	} else if (err == EAGAIN) {
1804 		/*
1805 		 * Continue emulating the rep-prefixed ins/outs, which has not
1806 		 * completed its iterations.
1807 		 *
1808 		 * In case this can be emulated in-kernel and has a high
1809 		 * repetition count (causing a tight spin), it should be
1810 		 * deferential to yield conditions.
1811 		 */
1812 		if (!vcpu_should_yield(vm, vcpuid)) {
1813 			goto repeat;
1814 		} else {
1815 			/*
1816 			 * Defer to the contending load by making a trip to
1817 			 * userspace with a no-op (BOGUS) exit reason.
1818 			 */
1819 			vie_reset(vie);
1820 			vme->exitcode = VM_EXITCODE_BOGUS;
1821 			return (-1);
1822 		}
1823 	} else if (err != 0) {
1824 		/* Emulation failure.  Bail all the way out to userspace. */
1825 		vme->exitcode = VM_EXITCODE_INST_EMUL;
1826 		bzero(&vme->u.inst_emul, sizeof (vme->u.inst_emul));
1827 		return (-1);
1828 	}
1829 
1830 	vie_advance_pc(vie, &vcpu->nextrip);
1831 	return (0);
1832 }
1833 
1834 static int
vm_handle_inst_emul(struct vm * vm,int vcpuid)1835 vm_handle_inst_emul(struct vm *vm, int vcpuid)
1836 {
1837 	struct vie *vie;
1838 	struct vcpu *vcpu;
1839 	struct vm_exit *vme;
1840 	uint64_t cs_base;
1841 	int error, fault, cs_d;
1842 
1843 	vcpu = &vm->vcpu[vcpuid];
1844 	vme = &vcpu->exitinfo;
1845 	vie = vcpu->vie_ctx;
1846 
1847 	vie_cs_info(vie, vm, vcpuid, &cs_base, &cs_d);
1848 
1849 	/* Fetch the faulting instruction */
1850 	ASSERT(vie_needs_fetch(vie));
1851 	error = vie_fetch_instruction(vie, vm, vcpuid, vme->rip + cs_base,
1852 	    &fault);
1853 	if (error != 0) {
1854 		return (error);
1855 	} else if (fault) {
1856 		/*
1857 		 * If a fault during instruction fetch was encounted, it will
1858 		 * have asserted that the appropriate exception be injected at
1859 		 * next entry.  No further work is required.
1860 		 */
1861 		return (0);
1862 	}
1863 
1864 	if (vie_decode_instruction(vie, vm, vcpuid, cs_d) != 0) {
1865 		/* Dump (unrecognized) instruction bytes in userspace */
1866 		vie_fallback_exitinfo(vie, vme);
1867 		return (-1);
1868 	}
1869 
1870 	error = vie_emulate_other(vie, vm, vcpuid);
1871 	if (error != 0) {
1872 		/*
1873 		 * Instruction emulation was unable to complete successfully, so
1874 		 * kick it out to userspace for handling.
1875 		 */
1876 		vie_fallback_exitinfo(vie, vme);
1877 	} else {
1878 		/* Update %rip now that instruction has been emulated */
1879 		vie_advance_pc(vie, &vcpu->nextrip);
1880 	}
1881 	return (error);
1882 }
1883 
1884 static int
vm_handle_run_state(struct vm * vm,int vcpuid)1885 vm_handle_run_state(struct vm *vm, int vcpuid)
1886 {
1887 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
1888 	bool handled = false;
1889 
1890 	vcpu_lock(vcpu);
1891 	while (1) {
1892 		if ((vcpu->run_state & VRS_PEND_INIT) != 0) {
1893 			vcpu_unlock(vcpu);
1894 			VERIFY0(vcpu_arch_reset(vm, vcpuid, true));
1895 			vcpu_lock(vcpu);
1896 
1897 			vcpu->run_state &= ~(VRS_RUN | VRS_PEND_INIT);
1898 			vcpu->run_state |= VRS_INIT;
1899 		}
1900 
1901 		if ((vcpu->run_state & (VRS_INIT | VRS_RUN | VRS_PEND_SIPI)) ==
1902 		    (VRS_INIT | VRS_PEND_SIPI)) {
1903 			const uint8_t vector = vcpu->sipi_vector;
1904 
1905 			vcpu_unlock(vcpu);
1906 			VERIFY0(vcpu_vector_sipi(vm, vcpuid, vector));
1907 			vcpu_lock(vcpu);
1908 
1909 			vcpu->run_state &= ~VRS_PEND_SIPI;
1910 			vcpu->run_state |= VRS_RUN;
1911 		}
1912 
1913 		/*
1914 		 * If the vCPU is now in the running state, there is no need to
1915 		 * wait for anything prior to re-entry.
1916 		 */
1917 		if ((vcpu->run_state & VRS_RUN) != 0) {
1918 			handled = true;
1919 			break;
1920 		}
1921 
1922 		/*
1923 		 * Also check for software events which would cause a wake-up.
1924 		 * This will set the appropriate exitcode directly, rather than
1925 		 * requiring a trip through VM_RUN().
1926 		 */
1927 		if (vcpu_sleep_bailout_checks(vm, vcpuid)) {
1928 			break;
1929 		}
1930 
1931 		vcpu_ustate_change(vm, vcpuid, VU_IDLE);
1932 		vcpu_require_state_locked(vm, vcpuid, VCPU_SLEEPING);
1933 		(void) cv_wait_sig(&vcpu->vcpu_cv, &vcpu->lock);
1934 		vcpu_require_state_locked(vm, vcpuid, VCPU_FROZEN);
1935 		vcpu_ustate_change(vm, vcpuid, VU_EMU_KERN);
1936 	}
1937 	vcpu_unlock(vcpu);
1938 
1939 	return (handled ? 0 : -1);
1940 }
1941 
1942 static int
vm_rdmtrr(const struct vm_mtrr * mtrr,uint32_t num,uint64_t * val)1943 vm_rdmtrr(const struct vm_mtrr *mtrr, uint32_t num, uint64_t *val)
1944 {
1945 	switch (num) {
1946 	case MSR_MTRRcap:
1947 		*val = MTRR_CAP_WC | MTRR_CAP_FIXED | VMM_MTRR_VAR_MAX;
1948 		break;
1949 	case MSR_MTRRdefType:
1950 		*val = mtrr->def_type;
1951 		break;
1952 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
1953 		*val = mtrr->fixed4k[num - MSR_MTRR4kBase];
1954 		break;
1955 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
1956 		*val = mtrr->fixed16k[num - MSR_MTRR16kBase];
1957 		break;
1958 	case MSR_MTRR64kBase:
1959 		*val = mtrr->fixed64k;
1960 		break;
1961 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: {
1962 		uint_t offset = num - MSR_MTRRVarBase;
1963 		if (offset % 2 == 0) {
1964 			*val = mtrr->var[offset / 2].base;
1965 		} else {
1966 			*val = mtrr->var[offset / 2].mask;
1967 		}
1968 		break;
1969 	}
1970 	default:
1971 		return (EINVAL);
1972 	}
1973 
1974 	return (0);
1975 }
1976 
1977 static int
vm_wrmtrr(struct vm_mtrr * mtrr,uint32_t num,uint64_t val)1978 vm_wrmtrr(struct vm_mtrr *mtrr, uint32_t num, uint64_t val)
1979 {
1980 	switch (num) {
1981 	case MSR_MTRRcap:
1982 		/* MTRRCAP is read only */
1983 		return (EPERM);
1984 	case MSR_MTRRdefType:
1985 		if (val & ~VMM_MTRR_DEF_MASK) {
1986 			/* generate #GP on writes to reserved fields */
1987 			return (EINVAL);
1988 		}
1989 		mtrr->def_type = val;
1990 		break;
1991 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
1992 		mtrr->fixed4k[num - MSR_MTRR4kBase] = val;
1993 		break;
1994 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
1995 		mtrr->fixed16k[num - MSR_MTRR16kBase] = val;
1996 		break;
1997 	case MSR_MTRR64kBase:
1998 		mtrr->fixed64k = val;
1999 		break;
2000 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1: {
2001 		uint_t offset = num - MSR_MTRRVarBase;
2002 		if (offset % 2 == 0) {
2003 			if (val & ~VMM_MTRR_PHYSBASE_MASK) {
2004 				/* generate #GP on writes to reserved fields */
2005 				return (EINVAL);
2006 			}
2007 			mtrr->var[offset / 2].base = val;
2008 		} else {
2009 			if (val & ~VMM_MTRR_PHYSMASK_MASK) {
2010 				/* generate #GP on writes to reserved fields */
2011 				return (EINVAL);
2012 			}
2013 			mtrr->var[offset / 2].mask = val;
2014 		}
2015 		break;
2016 	}
2017 	default:
2018 		return (EINVAL);
2019 	}
2020 
2021 	return (0);
2022 }
2023 
2024 static bool
is_mtrr_msr(uint32_t msr)2025 is_mtrr_msr(uint32_t msr)
2026 {
2027 	switch (msr) {
2028 	case MSR_MTRRcap:
2029 	case MSR_MTRRdefType:
2030 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
2031 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
2032 	case MSR_MTRR64kBase:
2033 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1:
2034 		return (true);
2035 	default:
2036 		return (false);
2037 	}
2038 }
2039 
2040 static int
vm_handle_rdmsr(struct vm * vm,int vcpuid,struct vm_exit * vme)2041 vm_handle_rdmsr(struct vm *vm, int vcpuid, struct vm_exit *vme)
2042 {
2043 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2044 	const uint32_t code = vme->u.msr.code;
2045 	uint64_t val = 0;
2046 
2047 	switch (code) {
2048 	case MSR_MCG_CAP:
2049 	case MSR_MCG_STATUS:
2050 		val = 0;
2051 		break;
2052 
2053 	case MSR_MTRRcap:
2054 	case MSR_MTRRdefType:
2055 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
2056 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
2057 	case MSR_MTRR64kBase:
2058 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1:
2059 		if (vm_rdmtrr(&vcpu->mtrr, code, &val) != 0)
2060 			vm_inject_gp(vm, vcpuid);
2061 		break;
2062 
2063 	case MSR_TSC:
2064 		/*
2065 		 * Get the guest TSC, applying necessary vCPU offsets.
2066 		 *
2067 		 * In all likelihood, this should always be handled in guest
2068 		 * context by VMX/SVM rather than taking an exit.  (Both VMX and
2069 		 * SVM pass through read-only access to MSR_TSC to the guest.)
2070 		 *
2071 		 * The VM-wide TSC offset and per-vCPU offset are included in
2072 		 * the calculations of vcpu_tsc_offset(), so this is sufficient
2073 		 * to use as the offset in our calculations.
2074 		 *
2075 		 * No physical offset is requested of vcpu_tsc_offset() since
2076 		 * rdtsc_offset() takes care of that instead.
2077 		 */
2078 		val = calc_guest_tsc(rdtsc_offset(), vm->freq_multiplier,
2079 		    vcpu_tsc_offset(vm, vcpuid, false));
2080 		break;
2081 
2082 	default:
2083 		/*
2084 		 * Anything not handled at this point will be kicked out to
2085 		 * userspace for attempted processing there.
2086 		 */
2087 		return (-1);
2088 	}
2089 
2090 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RAX,
2091 	    val & 0xffffffff));
2092 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RDX,
2093 	    val >> 32));
2094 	return (0);
2095 }
2096 
2097 static int
vm_handle_wrmsr(struct vm * vm,int vcpuid,struct vm_exit * vme)2098 vm_handle_wrmsr(struct vm *vm, int vcpuid, struct vm_exit *vme)
2099 {
2100 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2101 	const uint32_t code = vme->u.msr.code;
2102 	const uint64_t val = vme->u.msr.wval;
2103 
2104 	switch (code) {
2105 	case MSR_MCG_CAP:
2106 	case MSR_MCG_STATUS:
2107 		/* Ignore writes */
2108 		break;
2109 
2110 	case MSR_MTRRcap:
2111 	case MSR_MTRRdefType:
2112 	case MSR_MTRR4kBase ... MSR_MTRR4kBase + 7:
2113 	case MSR_MTRR16kBase ... MSR_MTRR16kBase + 1:
2114 	case MSR_MTRR64kBase:
2115 	case MSR_MTRRVarBase ... MSR_MTRRVarBase + (VMM_MTRR_VAR_MAX * 2) - 1:
2116 		if (vm_wrmtrr(&vcpu->mtrr, code, val) != 0)
2117 			vm_inject_gp(vm, vcpuid);
2118 		break;
2119 
2120 	case MSR_TSC:
2121 		/*
2122 		 * The effect of writing the TSC MSR is that a subsequent read
2123 		 * of the TSC would report that value written (plus any time
2124 		 * elapsed between the write and the read).
2125 		 *
2126 		 * To calculate that per-vCPU offset, we can work backwards from
2127 		 * the guest TSC at the time of write:
2128 		 *
2129 		 * value = current guest TSC + vCPU offset
2130 		 *
2131 		 * so therefore:
2132 		 *
2133 		 * value - current guest TSC = vCPU offset
2134 		 */
2135 		vcpu->tsc_offset = val - calc_guest_tsc(rdtsc_offset(),
2136 		    vm->freq_multiplier, vm->tsc_offset);
2137 		break;
2138 
2139 	default:
2140 		/*
2141 		 * Anything not handled at this point will be kicked out to
2142 		 * userspace for attempted processing there.
2143 		 */
2144 		return (-1);
2145 	}
2146 
2147 	return (0);
2148 }
2149 
2150 /*
2151  * Has a suspend event been asserted on the VM?
2152  *
2153  * The reason and (in the case of a triple-fault) source vcpuid are optionally
2154  * returned if such a state is present.
2155  */
2156 static bool
vm_is_suspended(struct vm * vm,struct vm_exit * vme)2157 vm_is_suspended(struct vm *vm, struct vm_exit *vme)
2158 {
2159 	const int val = vm->suspend_how;
2160 	if (val == 0) {
2161 		return (false);
2162 	} else {
2163 		if (vme != NULL) {
2164 			vme->exitcode = VM_EXITCODE_SUSPENDED;
2165 			vme->u.suspended.how = val;
2166 			vme->u.suspended.source = vm->suspend_source;
2167 			/*
2168 			 * Normalize suspend event time and, on the off chance
2169 			 * that it was recorded as occuring prior to VM boot,
2170 			 * clamp it to a minimum of 0.
2171 			 */
2172 			vme->u.suspended.when = (uint64_t)
2173 			    MAX(vm_normalize_hrtime(vm, vm->suspend_when), 0);
2174 		}
2175 		return (true);
2176 	}
2177 }
2178 
2179 int
vm_suspend(struct vm * vm,enum vm_suspend_how how,int source)2180 vm_suspend(struct vm *vm, enum vm_suspend_how how, int source)
2181 {
2182 	if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST) {
2183 		return (EINVAL);
2184 	}
2185 
2186 	/*
2187 	 * Although the common case of calling vm_suspend() is via
2188 	 * ioctl(VM_SUSPEND), where all the vCPUs will be held in the frozen
2189 	 * state, it can also be called by a running vCPU to indicate a
2190 	 * triple-fault.  In the latter case, there is no exclusion from a
2191 	 * racing vm_suspend() from a different vCPU, so assertion of the
2192 	 * suspended state must be performed carefully.
2193 	 *
2194 	 * The `suspend_when` is set first via atomic cmpset to pick a "winner"
2195 	 * of the suspension race, followed by population of 'suspend_source'.
2196 	 * Only after those are done, and a membar is emitted will 'suspend_how'
2197 	 * be set, which makes the suspended state visible to any vCPU checking
2198 	 * for it.  That order will prevent an incomplete suspend state (between
2199 	 * 'how', 'source', and 'when') from being observed.
2200 	 */
2201 	const hrtime_t now = gethrtime();
2202 	if (atomic_cmpset_long((ulong_t *)&vm->suspend_when, 0, now) == 0) {
2203 		return (EALREADY);
2204 	}
2205 	vm->suspend_source = source;
2206 	membar_producer();
2207 	vm->suspend_how = how;
2208 
2209 	/* Notify all active vcpus that they are now suspended. */
2210 	for (uint_t i = 0; i < vm->maxcpus; i++) {
2211 		struct vcpu *vcpu = &vm->vcpu[i];
2212 
2213 		vcpu_lock(vcpu);
2214 
2215 		if (!CPU_ISSET(i, &vm->active_cpus)) {
2216 			/*
2217 			 * vCPUs not already marked as active can be ignored,
2218 			 * since they cannot become marked as active unless the
2219 			 * VM is reinitialized, clearing the suspended state.
2220 			 */
2221 			vcpu_unlock(vcpu);
2222 			continue;
2223 		}
2224 
2225 		switch (vcpu->state) {
2226 		case VCPU_IDLE:
2227 		case VCPU_FROZEN:
2228 			/*
2229 			 * vCPUs not locked by in-kernel activity can be
2230 			 * immediately marked as suspended: The ustate is moved
2231 			 * back to VU_INIT, since no further guest work will
2232 			 * occur while the VM is in this state.
2233 			 *
2234 			 * A FROZEN vCPU may still change its ustate on the way
2235 			 * out of the kernel, but a subsequent check at the end
2236 			 * of vm_run() should be adequate to fix it up.
2237 			 */
2238 			vcpu_ustate_change(vm, i, VU_INIT);
2239 			break;
2240 		default:
2241 			/*
2242 			 * Any vCPUs which are running or waiting in-kernel
2243 			 * (such as in HLT) are notified to pick up the newly
2244 			 * suspended state.
2245 			 */
2246 			vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
2247 			break;
2248 		}
2249 		vcpu_unlock(vcpu);
2250 	}
2251 	return (0);
2252 }
2253 
2254 void
vm_exit_run_state(struct vm * vm,int vcpuid,uint64_t rip)2255 vm_exit_run_state(struct vm *vm, int vcpuid, uint64_t rip)
2256 {
2257 	struct vm_exit *vmexit;
2258 
2259 	vmexit = vm_exitinfo(vm, vcpuid);
2260 	vmexit->rip = rip;
2261 	vmexit->inst_length = 0;
2262 	vmexit->exitcode = VM_EXITCODE_RUN_STATE;
2263 	vmm_stat_incr(vm, vcpuid, VMEXIT_RUN_STATE, 1);
2264 }
2265 
2266 /*
2267  * Some vmm resources, such as the lapic, may have CPU-specific resources
2268  * allocated to them which would benefit from migration onto the host CPU which
2269  * is processing the vcpu state.
2270  */
2271 static void
vm_localize_resources(struct vm * vm,struct vcpu * vcpu)2272 vm_localize_resources(struct vm *vm, struct vcpu *vcpu)
2273 {
2274 	/*
2275 	 * Localizing cyclic resources requires acquisition of cpu_lock, and
2276 	 * doing so with kpreempt disabled is a recipe for deadlock disaster.
2277 	 */
2278 	VERIFY(curthread->t_preempt == 0);
2279 
2280 	/*
2281 	 * Do not bother with localization if this vCPU is about to return to
2282 	 * the host CPU it was last localized to.
2283 	 */
2284 	if (vcpu->lastloccpu == curcpu)
2285 		return;
2286 
2287 	/*
2288 	 * Localize system-wide resources to the primary boot vCPU.  While any
2289 	 * of the other vCPUs may access them, it keeps the potential interrupt
2290 	 * footprint constrained to CPUs involved with this instance.
2291 	 */
2292 	if (vcpu == &vm->vcpu[0]) {
2293 		vhpet_localize_resources(vm->vhpet);
2294 		vrtc_localize_resources(vm->vrtc);
2295 		vatpit_localize_resources(vm->vatpit);
2296 	}
2297 
2298 	vlapic_localize_resources(vcpu->vlapic);
2299 
2300 	vcpu->lastloccpu = curcpu;
2301 }
2302 
2303 static void
vmm_savectx(void * arg)2304 vmm_savectx(void *arg)
2305 {
2306 	vm_thread_ctx_t *vtc = arg;
2307 	struct vm *vm = vtc->vtc_vm;
2308 	const int vcpuid = vtc->vtc_vcpuid;
2309 
2310 	if (ops->vmsavectx != NULL) {
2311 		ops->vmsavectx(vm->cookie, vcpuid);
2312 	}
2313 
2314 	/*
2315 	 * Account for going off-cpu, unless the vCPU is idled, where being
2316 	 * off-cpu is the explicit point.
2317 	 */
2318 	if (vm->vcpu[vcpuid].ustate != VU_IDLE) {
2319 		vtc->vtc_ustate = vm->vcpu[vcpuid].ustate;
2320 		vcpu_ustate_change(vm, vcpuid, VU_SCHED);
2321 	}
2322 
2323 	/*
2324 	 * If the CPU holds the restored guest FPU state, save it and restore
2325 	 * the host FPU state before this thread goes off-cpu.
2326 	 */
2327 	if ((vtc->vtc_status & VTCS_FPU_RESTORED) != 0) {
2328 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
2329 
2330 		save_guest_fpustate(vcpu);
2331 		vtc->vtc_status &= ~VTCS_FPU_RESTORED;
2332 	}
2333 }
2334 
2335 static void
vmm_restorectx(void * arg)2336 vmm_restorectx(void *arg)
2337 {
2338 	vm_thread_ctx_t *vtc = arg;
2339 	struct vm *vm = vtc->vtc_vm;
2340 	const int vcpuid = vtc->vtc_vcpuid;
2341 
2342 	/* Complete microstate accounting for vCPU being off-cpu */
2343 	if (vm->vcpu[vcpuid].ustate != VU_IDLE) {
2344 		vcpu_ustate_change(vm, vcpuid, vtc->vtc_ustate);
2345 	}
2346 
2347 	/*
2348 	 * When coming back on-cpu, only restore the guest FPU status if the
2349 	 * thread is in a context marked as requiring it.  This should be rare,
2350 	 * occurring only when a future logic error results in a voluntary
2351 	 * sleep during the VMRUN critical section.
2352 	 *
2353 	 * The common case will result in elision of the guest FPU state
2354 	 * restoration, deferring that action until it is clearly necessary
2355 	 * during vm_run.
2356 	 */
2357 	VERIFY((vtc->vtc_status & VTCS_FPU_RESTORED) == 0);
2358 	if ((vtc->vtc_status & VTCS_FPU_CTX_CRITICAL) != 0) {
2359 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
2360 
2361 		restore_guest_fpustate(vcpu);
2362 		vtc->vtc_status |= VTCS_FPU_RESTORED;
2363 	}
2364 
2365 	if (ops->vmrestorectx != NULL) {
2366 		ops->vmrestorectx(vm->cookie, vcpuid);
2367 	}
2368 
2369 }
2370 
2371 /* Convenience defines for parsing vm_entry`cmd values */
2372 #define	VEC_MASK_FLAGS	(VEC_FLAG_EXIT_CONSISTENT)
2373 #define	VEC_MASK_CMD	(~VEC_MASK_FLAGS)
2374 
2375 static int
vm_entry_actions(struct vm * vm,int vcpuid,const struct vm_entry * entry,struct vm_exit * vme)2376 vm_entry_actions(struct vm *vm, int vcpuid, const struct vm_entry *entry,
2377     struct vm_exit *vme)
2378 {
2379 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2380 	struct vie *vie = vcpu->vie_ctx;
2381 	int err = 0;
2382 
2383 	const uint_t cmd = entry->cmd & VEC_MASK_CMD;
2384 	const uint_t flags = entry->cmd & VEC_MASK_FLAGS;
2385 
2386 	switch (cmd) {
2387 	case VEC_DEFAULT:
2388 		break;
2389 	case VEC_DISCARD_INSTR:
2390 		vie_reset(vie);
2391 		break;
2392 	case VEC_FULFILL_MMIO:
2393 		err = vie_fulfill_mmio(vie, &entry->u.mmio);
2394 		if (err == 0) {
2395 			err = vie_emulate_mmio(vie, vm, vcpuid);
2396 			if (err == 0) {
2397 				vie_advance_pc(vie, &vcpu->nextrip);
2398 			} else if (err < 0) {
2399 				vie_exitinfo(vie, vme);
2400 			} else if (err == EAGAIN) {
2401 				/*
2402 				 * Clear the instruction emulation state in
2403 				 * order to re-enter VM context and continue
2404 				 * this 'rep <instruction>'
2405 				 */
2406 				vie_reset(vie);
2407 				err = 0;
2408 			}
2409 		}
2410 		break;
2411 	case VEC_FULFILL_INOUT:
2412 		err = vie_fulfill_inout(vie, &entry->u.inout);
2413 		if (err == 0) {
2414 			err = vie_emulate_inout(vie, vm, vcpuid);
2415 			if (err == 0) {
2416 				vie_advance_pc(vie, &vcpu->nextrip);
2417 			} else if (err < 0) {
2418 				vie_exitinfo(vie, vme);
2419 			} else if (err == EAGAIN) {
2420 				/*
2421 				 * Clear the instruction emulation state in
2422 				 * order to re-enter VM context and continue
2423 				 * this 'rep ins/outs'
2424 				 */
2425 				vie_reset(vie);
2426 				err = 0;
2427 			}
2428 		}
2429 		break;
2430 	default:
2431 		return (EINVAL);
2432 	}
2433 
2434 	/*
2435 	 * Pay heed to requests for exit-when-vCPU-is-consistent requests, at
2436 	 * least when we are not immediately bound for another exit due to
2437 	 * multi-part instruction emulation or related causes.
2438 	 */
2439 	if ((flags & VEC_FLAG_EXIT_CONSISTENT) != 0 && err == 0) {
2440 		vcpu->reqconsist = true;
2441 	}
2442 
2443 	return (err);
2444 }
2445 
2446 static int
vm_loop_checks(struct vm * vm,int vcpuid,struct vm_exit * vme)2447 vm_loop_checks(struct vm *vm, int vcpuid, struct vm_exit *vme)
2448 {
2449 	struct vie *vie;
2450 
2451 	vie = vm->vcpu[vcpuid].vie_ctx;
2452 
2453 	if (vie_pending(vie)) {
2454 		/*
2455 		 * Userspace has not fulfilled the pending needs of the
2456 		 * instruction emulation, so bail back out.
2457 		 */
2458 		vie_exitinfo(vie, vme);
2459 		return (-1);
2460 	}
2461 
2462 	return (0);
2463 }
2464 
2465 int
vm_run(struct vm * vm,int vcpuid,const struct vm_entry * entry)2466 vm_run(struct vm *vm, int vcpuid, const struct vm_entry *entry)
2467 {
2468 	int error;
2469 	struct vcpu *vcpu;
2470 	struct vm_exit *vme;
2471 	bool intr_disabled;
2472 	int affinity_type = CPU_CURRENT;
2473 
2474 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2475 		return (EINVAL);
2476 	if (!CPU_ISSET(vcpuid, &vm->active_cpus))
2477 		return (EINVAL);
2478 	if (vm->is_paused) {
2479 		return (EBUSY);
2480 	}
2481 
2482 	vcpu = &vm->vcpu[vcpuid];
2483 	vme = &vcpu->exitinfo;
2484 
2485 	vcpu_ustate_change(vm, vcpuid, VU_EMU_KERN);
2486 
2487 	vcpu->vtc.vtc_status = 0;
2488 	ctxop_attach(curthread, vcpu->ctxop);
2489 
2490 	error = vm_entry_actions(vm, vcpuid, entry, vme);
2491 	if (error != 0) {
2492 		goto exit;
2493 	}
2494 
2495 restart:
2496 	error = vm_loop_checks(vm, vcpuid, vme);
2497 	if (error != 0) {
2498 		goto exit;
2499 	}
2500 
2501 	thread_affinity_set(curthread, affinity_type);
2502 	/*
2503 	 * Resource localization should happen after the CPU affinity for the
2504 	 * thread has been set to ensure that access from restricted contexts,
2505 	 * such as VMX-accelerated APIC operations, can occur without inducing
2506 	 * cyclic cross-calls.
2507 	 *
2508 	 * This must be done prior to disabling kpreempt via critical_enter().
2509 	 */
2510 	vm_localize_resources(vm, vcpu);
2511 	affinity_type = CPU_CURRENT;
2512 	critical_enter();
2513 
2514 	/* Force a trip through update_sregs to reload %fs/%gs and friends */
2515 	PCB_SET_UPDATE_SEGS(&ttolwp(curthread)->lwp_pcb);
2516 
2517 	if ((vcpu->vtc.vtc_status & VTCS_FPU_RESTORED) == 0) {
2518 		restore_guest_fpustate(vcpu);
2519 		vcpu->vtc.vtc_status |= VTCS_FPU_RESTORED;
2520 	}
2521 	vcpu->vtc.vtc_status |= VTCS_FPU_CTX_CRITICAL;
2522 
2523 	vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
2524 	error = VMRUN(vm->cookie, vcpuid, vcpu->nextrip);
2525 	vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
2526 
2527 	/*
2528 	 * Once clear of the delicate contexts comprising the VM_RUN handler,
2529 	 * thread CPU affinity can be loosened while other processing occurs.
2530 	 */
2531 	vcpu->vtc.vtc_status &= ~VTCS_FPU_CTX_CRITICAL;
2532 	thread_affinity_clear(curthread);
2533 	critical_exit();
2534 
2535 	if (error != 0) {
2536 		/* Communicate out any error from VMRUN() above */
2537 		goto exit;
2538 	}
2539 
2540 	vcpu->nextrip = vme->rip + vme->inst_length;
2541 	switch (vme->exitcode) {
2542 	case VM_EXITCODE_RUN_STATE:
2543 		error = vm_handle_run_state(vm, vcpuid);
2544 		break;
2545 	case VM_EXITCODE_IOAPIC_EOI:
2546 		vioapic_process_eoi(vm, vcpuid,
2547 		    vme->u.ioapic_eoi.vector);
2548 		break;
2549 	case VM_EXITCODE_HLT:
2550 		intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0);
2551 		error = vm_handle_hlt(vm, vcpuid, intr_disabled);
2552 		break;
2553 	case VM_EXITCODE_PAGING:
2554 		error = vm_handle_paging(vm, vcpuid);
2555 		break;
2556 	case VM_EXITCODE_MMIO_EMUL:
2557 		error = vm_handle_mmio_emul(vm, vcpuid);
2558 		break;
2559 	case VM_EXITCODE_INOUT:
2560 		error = vm_handle_inout(vm, vcpuid, vme);
2561 		break;
2562 	case VM_EXITCODE_INST_EMUL:
2563 		error = vm_handle_inst_emul(vm, vcpuid);
2564 		break;
2565 	case VM_EXITCODE_MONITOR:
2566 	case VM_EXITCODE_MWAIT:
2567 	case VM_EXITCODE_VMINSN:
2568 		vm_inject_ud(vm, vcpuid);
2569 		break;
2570 	case VM_EXITCODE_RDMSR:
2571 		error = vm_handle_rdmsr(vm, vcpuid, vme);
2572 		break;
2573 	case VM_EXITCODE_WRMSR:
2574 		error = vm_handle_wrmsr(vm, vcpuid, vme);
2575 		break;
2576 	case VM_EXITCODE_HT:
2577 		affinity_type = CPU_BEST;
2578 		break;
2579 	case VM_EXITCODE_MTRAP:
2580 		VERIFY0(vm_suspend_cpu(vm, vcpuid));
2581 		error = -1;
2582 		break;
2583 	default:
2584 		/* handled in userland */
2585 		error = -1;
2586 		break;
2587 	}
2588 
2589 	if (error == 0) {
2590 		/* VM exit conditions handled in-kernel, continue running */
2591 		goto restart;
2592 	}
2593 
2594 exit:
2595 	kpreempt_disable();
2596 	ctxop_detach(curthread, vcpu->ctxop);
2597 	/* Make sure all of the needed vCPU context state is saved */
2598 	vmm_savectx(&vcpu->vtc);
2599 	kpreempt_enable();
2600 
2601 	/*
2602 	 * Bill time in userspace against VU_EMU_USER, unless the VM is
2603 	 * suspended, in which case VU_INIT is the choice.
2604 	 */
2605 	vcpu_ustate_change(vm, vcpuid,
2606 	    vm_is_suspended(vm, NULL) ? VU_INIT : VU_EMU_USER);
2607 
2608 	return (error);
2609 }
2610 
2611 int
vm_restart_instruction(void * arg,int vcpuid)2612 vm_restart_instruction(void *arg, int vcpuid)
2613 {
2614 	struct vm *vm;
2615 	struct vcpu *vcpu;
2616 	enum vcpu_state state;
2617 	uint64_t rip;
2618 	int error;
2619 
2620 	vm = arg;
2621 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2622 		return (EINVAL);
2623 
2624 	vcpu = &vm->vcpu[vcpuid];
2625 	state = vcpu_get_state(vm, vcpuid, NULL);
2626 	if (state == VCPU_RUNNING) {
2627 		/*
2628 		 * When a vcpu is "running" the next instruction is determined
2629 		 * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'.
2630 		 * Thus setting 'inst_length' to zero will cause the current
2631 		 * instruction to be restarted.
2632 		 */
2633 		vcpu->exitinfo.inst_length = 0;
2634 	} else if (state == VCPU_FROZEN) {
2635 		/*
2636 		 * When a vcpu is "frozen" it is outside the critical section
2637 		 * around VMRUN() and 'nextrip' points to the next instruction.
2638 		 * Thus instruction restart is achieved by setting 'nextrip'
2639 		 * to the vcpu's %rip.
2640 		 */
2641 		error = vm_get_register(vm, vcpuid, VM_REG_GUEST_RIP, &rip);
2642 		KASSERT(!error, ("%s: error %d getting rip", __func__, error));
2643 		vcpu->nextrip = rip;
2644 	} else {
2645 		panic("%s: invalid state %d", __func__, state);
2646 	}
2647 	return (0);
2648 }
2649 
2650 int
vm_exit_intinfo(struct vm * vm,int vcpuid,uint64_t info)2651 vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info)
2652 {
2653 	struct vcpu *vcpu;
2654 
2655 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2656 		return (EINVAL);
2657 
2658 	vcpu = &vm->vcpu[vcpuid];
2659 
2660 	if (VM_INTINFO_PENDING(info)) {
2661 		const uint32_t type = VM_INTINFO_TYPE(info);
2662 		const uint8_t vector = VM_INTINFO_VECTOR(info);
2663 
2664 		if (type == VM_INTINFO_NMI && vector != IDT_NMI)
2665 			return (EINVAL);
2666 		if (type == VM_INTINFO_HWEXCP && vector >= 32)
2667 			return (EINVAL);
2668 		if (info & VM_INTINFO_MASK_RSVD)
2669 			return (EINVAL);
2670 	} else {
2671 		info = 0;
2672 	}
2673 	vcpu->exit_intinfo = info;
2674 	return (0);
2675 }
2676 
2677 enum exc_class {
2678 	EXC_BENIGN,
2679 	EXC_CONTRIBUTORY,
2680 	EXC_PAGEFAULT
2681 };
2682 
2683 #define	IDT_VE	20	/* Virtualization Exception (Intel specific) */
2684 
2685 static enum exc_class
exception_class(uint64_t info)2686 exception_class(uint64_t info)
2687 {
2688 	ASSERT(VM_INTINFO_PENDING(info));
2689 
2690 	/* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */
2691 	switch (VM_INTINFO_TYPE(info)) {
2692 	case VM_INTINFO_HWINTR:
2693 	case VM_INTINFO_SWINTR:
2694 	case VM_INTINFO_NMI:
2695 		return (EXC_BENIGN);
2696 	default:
2697 		/*
2698 		 * Hardware exception.
2699 		 *
2700 		 * SVM and VT-x use identical type values to represent NMI,
2701 		 * hardware interrupt and software interrupt.
2702 		 *
2703 		 * SVM uses type '3' for all exceptions. VT-x uses type '3'
2704 		 * for exceptions except #BP and #OF. #BP and #OF use a type
2705 		 * value of '5' or '6'. Therefore we don't check for explicit
2706 		 * values of 'type' to classify 'intinfo' into a hardware
2707 		 * exception.
2708 		 */
2709 		break;
2710 	}
2711 
2712 	switch (VM_INTINFO_VECTOR(info)) {
2713 	case IDT_PF:
2714 	case IDT_VE:
2715 		return (EXC_PAGEFAULT);
2716 	case IDT_DE:
2717 	case IDT_TS:
2718 	case IDT_NP:
2719 	case IDT_SS:
2720 	case IDT_GP:
2721 		return (EXC_CONTRIBUTORY);
2722 	default:
2723 		return (EXC_BENIGN);
2724 	}
2725 }
2726 
2727 /*
2728  * Fetch event pending injection into the guest, if one exists.
2729  *
2730  * Returns true if an event is to be injected (which is placed in `retinfo`).
2731  */
2732 bool
vm_entry_intinfo(struct vm * vm,int vcpuid,uint64_t * retinfo)2733 vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo)
2734 {
2735 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2736 	const uint64_t info1 = vcpu->exit_intinfo;
2737 	vcpu->exit_intinfo = 0;
2738 	const uint64_t info2 = vcpu->exc_pending;
2739 	vcpu->exc_pending = 0;
2740 
2741 	if (VM_INTINFO_PENDING(info1) && VM_INTINFO_PENDING(info2)) {
2742 		/*
2743 		 * If an exception occurs while attempting to call the
2744 		 * double-fault handler the processor enters shutdown mode
2745 		 * (aka triple fault).
2746 		 */
2747 		if (VM_INTINFO_TYPE(info1) == VM_INTINFO_HWEXCP &&
2748 		    VM_INTINFO_VECTOR(info1) == IDT_DF) {
2749 			(void) vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT, vcpuid);
2750 			*retinfo = 0;
2751 			return (false);
2752 		}
2753 		/*
2754 		 * "Conditions for Generating a Double Fault"
2755 		 *  Intel SDM, Vol3, Table 6-5
2756 		 */
2757 		const enum exc_class exc1 = exception_class(info1);
2758 		const enum exc_class exc2 = exception_class(info2);
2759 		if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) ||
2760 		    (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) {
2761 			/* Convert nested fault into a double fault. */
2762 			*retinfo =
2763 			    VM_INTINFO_VALID |
2764 			    VM_INTINFO_DEL_ERRCODE |
2765 			    VM_INTINFO_HWEXCP |
2766 			    IDT_DF;
2767 		} else {
2768 			/* Handle exceptions serially */
2769 			vcpu->exit_intinfo = info1;
2770 			*retinfo = info2;
2771 		}
2772 		return (true);
2773 	} else if (VM_INTINFO_PENDING(info1)) {
2774 		*retinfo = info1;
2775 		return (true);
2776 	} else if (VM_INTINFO_PENDING(info2)) {
2777 		*retinfo = info2;
2778 		return (true);
2779 	}
2780 
2781 	return (false);
2782 }
2783 
2784 int
vm_get_intinfo(struct vm * vm,int vcpuid,uint64_t * info1,uint64_t * info2)2785 vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2)
2786 {
2787 	struct vcpu *vcpu;
2788 
2789 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2790 		return (EINVAL);
2791 
2792 	vcpu = &vm->vcpu[vcpuid];
2793 	*info1 = vcpu->exit_intinfo;
2794 	*info2 = vcpu->exc_pending;
2795 	return (0);
2796 }
2797 
2798 int
vm_inject_exception(struct vm * vm,int vcpuid,uint8_t vector,bool errcode_valid,uint32_t errcode,bool restart_instruction)2799 vm_inject_exception(struct vm *vm, int vcpuid, uint8_t vector,
2800     bool errcode_valid, uint32_t errcode, bool restart_instruction)
2801 {
2802 	struct vcpu *vcpu;
2803 	uint64_t regval;
2804 	int error;
2805 
2806 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2807 		return (EINVAL);
2808 
2809 	if (vector >= 32)
2810 		return (EINVAL);
2811 
2812 	/*
2813 	 * NMIs are to be injected via their own specialized path using
2814 	 * vm_inject_nmi().
2815 	 */
2816 	if (vector == IDT_NMI) {
2817 		return (EINVAL);
2818 	}
2819 
2820 	/*
2821 	 * A double fault exception should never be injected directly into
2822 	 * the guest. It is a derived exception that results from specific
2823 	 * combinations of nested faults.
2824 	 */
2825 	if (vector == IDT_DF) {
2826 		return (EINVAL);
2827 	}
2828 
2829 	vcpu = &vm->vcpu[vcpuid];
2830 
2831 	if (VM_INTINFO_PENDING(vcpu->exc_pending)) {
2832 		/* Unable to inject exception due to one already pending */
2833 		return (EBUSY);
2834 	}
2835 
2836 	if (errcode_valid) {
2837 		/*
2838 		 * Exceptions don't deliver an error code in real mode.
2839 		 */
2840 		error = vm_get_register(vm, vcpuid, VM_REG_GUEST_CR0, &regval);
2841 		VERIFY0(error);
2842 		if ((regval & CR0_PE) == 0) {
2843 			errcode_valid = false;
2844 		}
2845 	}
2846 
2847 	/*
2848 	 * From section 26.6.1 "Interruptibility State" in Intel SDM:
2849 	 *
2850 	 * Event blocking by "STI" or "MOV SS" is cleared after guest executes
2851 	 * one instruction or incurs an exception.
2852 	 */
2853 	error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0);
2854 	VERIFY0(error);
2855 
2856 	if (restart_instruction) {
2857 		VERIFY0(vm_restart_instruction(vm, vcpuid));
2858 	}
2859 
2860 	uint64_t val = VM_INTINFO_VALID | VM_INTINFO_HWEXCP | vector;
2861 	if (errcode_valid) {
2862 		val |= VM_INTINFO_DEL_ERRCODE;
2863 		val |= (uint64_t)errcode << VM_INTINFO_SHIFT_ERRCODE;
2864 	}
2865 	vcpu->exc_pending = val;
2866 	return (0);
2867 }
2868 
2869 void
vm_inject_ud(struct vm * vm,int vcpuid)2870 vm_inject_ud(struct vm *vm, int vcpuid)
2871 {
2872 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_UD, false, 0, true));
2873 }
2874 
2875 void
vm_inject_gp(struct vm * vm,int vcpuid)2876 vm_inject_gp(struct vm *vm, int vcpuid)
2877 {
2878 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_GP, true, 0, true));
2879 }
2880 
2881 void
vm_inject_ac(struct vm * vm,int vcpuid,uint32_t errcode)2882 vm_inject_ac(struct vm *vm, int vcpuid, uint32_t errcode)
2883 {
2884 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_AC, true, errcode, true));
2885 }
2886 
2887 void
vm_inject_ss(struct vm * vm,int vcpuid,uint32_t errcode)2888 vm_inject_ss(struct vm *vm, int vcpuid, uint32_t errcode)
2889 {
2890 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_SS, true, errcode, true));
2891 }
2892 
2893 void
vm_inject_pf(struct vm * vm,int vcpuid,uint32_t errcode,uint64_t cr2)2894 vm_inject_pf(struct vm *vm, int vcpuid, uint32_t errcode, uint64_t cr2)
2895 {
2896 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2));
2897 	VERIFY0(vm_inject_exception(vm, vcpuid, IDT_PF, true, errcode, true));
2898 }
2899 
2900 static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
2901 
2902 int
vm_inject_nmi(struct vm * vm,int vcpuid)2903 vm_inject_nmi(struct vm *vm, int vcpuid)
2904 {
2905 	struct vcpu *vcpu;
2906 
2907 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2908 		return (EINVAL);
2909 
2910 	vcpu = &vm->vcpu[vcpuid];
2911 
2912 	vcpu->nmi_pending = true;
2913 	vcpu_notify_event(vm, vcpuid);
2914 	return (0);
2915 }
2916 
2917 bool
vm_nmi_pending(struct vm * vm,int vcpuid)2918 vm_nmi_pending(struct vm *vm, int vcpuid)
2919 {
2920 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2921 
2922 	return (vcpu->nmi_pending);
2923 }
2924 
2925 void
vm_nmi_clear(struct vm * vm,int vcpuid)2926 vm_nmi_clear(struct vm *vm, int vcpuid)
2927 {
2928 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2929 
2930 	ASSERT(vcpu->nmi_pending);
2931 
2932 	vcpu->nmi_pending = false;
2933 	vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1);
2934 }
2935 
2936 static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu");
2937 
2938 int
vm_inject_extint(struct vm * vm,int vcpuid)2939 vm_inject_extint(struct vm *vm, int vcpuid)
2940 {
2941 	struct vcpu *vcpu;
2942 
2943 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2944 		return (EINVAL);
2945 
2946 	vcpu = &vm->vcpu[vcpuid];
2947 
2948 	vcpu->extint_pending = true;
2949 	vcpu_notify_event(vm, vcpuid);
2950 	return (0);
2951 }
2952 
2953 bool
vm_extint_pending(struct vm * vm,int vcpuid)2954 vm_extint_pending(struct vm *vm, int vcpuid)
2955 {
2956 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2957 
2958 	return (vcpu->extint_pending);
2959 }
2960 
2961 void
vm_extint_clear(struct vm * vm,int vcpuid)2962 vm_extint_clear(struct vm *vm, int vcpuid)
2963 {
2964 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
2965 
2966 	ASSERT(vcpu->extint_pending);
2967 
2968 	vcpu->extint_pending = false;
2969 	vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1);
2970 }
2971 
2972 int
vm_inject_init(struct vm * vm,int vcpuid)2973 vm_inject_init(struct vm *vm, int vcpuid)
2974 {
2975 	struct vcpu *vcpu;
2976 
2977 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
2978 		return (EINVAL);
2979 
2980 	vcpu = &vm->vcpu[vcpuid];
2981 	vcpu_lock(vcpu);
2982 	vcpu->run_state |= VRS_PEND_INIT;
2983 	/*
2984 	 * As part of queuing the INIT request, clear any pending SIPI.  It
2985 	 * would not otherwise survive across the reset of the vCPU when it
2986 	 * undergoes the requested INIT.  We would not want it to linger when it
2987 	 * could be mistaken as a subsequent (after the INIT) SIPI request.
2988 	 */
2989 	vcpu->run_state &= ~VRS_PEND_SIPI;
2990 	vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
2991 
2992 	vcpu_unlock(vcpu);
2993 	return (0);
2994 }
2995 
2996 int
vm_inject_sipi(struct vm * vm,int vcpuid,uint8_t vector)2997 vm_inject_sipi(struct vm *vm, int vcpuid, uint8_t vector)
2998 {
2999 	struct vcpu *vcpu;
3000 
3001 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3002 		return (EINVAL);
3003 
3004 	vcpu = &vm->vcpu[vcpuid];
3005 	vcpu_lock(vcpu);
3006 	vcpu->run_state |= VRS_PEND_SIPI;
3007 	vcpu->sipi_vector = vector;
3008 	/* SIPI is only actionable if the CPU is waiting in INIT state */
3009 	if ((vcpu->run_state & (VRS_INIT | VRS_RUN)) == VRS_INIT) {
3010 		vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
3011 	}
3012 	vcpu_unlock(vcpu);
3013 	return (0);
3014 }
3015 
3016 bool
vcpu_run_state_pending(struct vm * vm,int vcpuid)3017 vcpu_run_state_pending(struct vm *vm, int vcpuid)
3018 {
3019 	struct vcpu *vcpu;
3020 
3021 	ASSERT(vcpuid >= 0 && vcpuid < vm->maxcpus);
3022 	vcpu = &vm->vcpu[vcpuid];
3023 
3024 	/* Of interest: vCPU not in running state or with pending INIT */
3025 	return ((vcpu->run_state & (VRS_RUN | VRS_PEND_INIT)) != VRS_RUN);
3026 }
3027 
3028 int
vcpu_arch_reset(struct vm * vm,int vcpuid,bool init_only)3029 vcpu_arch_reset(struct vm *vm, int vcpuid, bool init_only)
3030 {
3031 	struct seg_desc desc;
3032 	const enum vm_reg_name clear_regs[] = {
3033 		VM_REG_GUEST_CR2,
3034 		VM_REG_GUEST_CR3,
3035 		VM_REG_GUEST_CR4,
3036 		VM_REG_GUEST_RAX,
3037 		VM_REG_GUEST_RBX,
3038 		VM_REG_GUEST_RCX,
3039 		VM_REG_GUEST_RSI,
3040 		VM_REG_GUEST_RDI,
3041 		VM_REG_GUEST_RBP,
3042 		VM_REG_GUEST_RSP,
3043 		VM_REG_GUEST_R8,
3044 		VM_REG_GUEST_R9,
3045 		VM_REG_GUEST_R10,
3046 		VM_REG_GUEST_R11,
3047 		VM_REG_GUEST_R12,
3048 		VM_REG_GUEST_R13,
3049 		VM_REG_GUEST_R14,
3050 		VM_REG_GUEST_R15,
3051 		VM_REG_GUEST_DR0,
3052 		VM_REG_GUEST_DR1,
3053 		VM_REG_GUEST_DR2,
3054 		VM_REG_GUEST_DR3,
3055 		VM_REG_GUEST_EFER,
3056 	};
3057 	const enum vm_reg_name data_segs[] = {
3058 		VM_REG_GUEST_SS,
3059 		VM_REG_GUEST_DS,
3060 		VM_REG_GUEST_ES,
3061 		VM_REG_GUEST_FS,
3062 		VM_REG_GUEST_GS,
3063 	};
3064 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3065 
3066 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3067 		return (EINVAL);
3068 
3069 	for (uint_t i = 0; i < nitems(clear_regs); i++) {
3070 		VERIFY0(vm_set_register(vm, vcpuid, clear_regs[i], 0));
3071 	}
3072 
3073 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RFLAGS, 2));
3074 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RIP, 0xfff0));
3075 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CR0, 0x60000010));
3076 
3077 	/*
3078 	 * The prescribed contents of %rdx differ slightly between the Intel and
3079 	 * AMD architectural definitions.  The former expects the Extended Model
3080 	 * in bits 16-19 where the latter expects all the Family, Model, and
3081 	 * Stepping be there.  Common boot ROMs appear to disregard this
3082 	 * anyways, so we stick with a compromise value similar to what is
3083 	 * spelled out in the Intel SDM.
3084 	 */
3085 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RDX, 0x600));
3086 
3087 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_DR6, 0xffff0ff0));
3088 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_DR7, 0x400));
3089 
3090 	/* CS: Present, R/W, Accessed */
3091 	desc.access = 0x0093;
3092 	desc.base = 0xffff0000;
3093 	desc.limit = 0xffff;
3094 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_CS, &desc));
3095 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CS, 0xf000));
3096 
3097 	/* SS, DS, ES, FS, GS: Present, R/W, Accessed */
3098 	desc.access = 0x0093;
3099 	desc.base = 0;
3100 	desc.limit = 0xffff;
3101 	for (uint_t i = 0; i < nitems(data_segs); i++) {
3102 		VERIFY0(vm_set_seg_desc(vm, vcpuid, data_segs[i], &desc));
3103 		VERIFY0(vm_set_register(vm, vcpuid, data_segs[i], 0));
3104 	}
3105 
3106 	/* GDTR, IDTR */
3107 	desc.base = 0;
3108 	desc.limit = 0xffff;
3109 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_GDTR, &desc));
3110 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_IDTR, &desc));
3111 
3112 	/* LDTR: Present, LDT */
3113 	desc.access = 0x0082;
3114 	desc.base = 0;
3115 	desc.limit = 0xffff;
3116 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_LDTR, &desc));
3117 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_LDTR, 0));
3118 
3119 	/* TR: Present, 32-bit TSS */
3120 	desc.access = 0x008b;
3121 	desc.base = 0;
3122 	desc.limit = 0xffff;
3123 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_TR, &desc));
3124 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_TR, 0));
3125 
3126 	vlapic_reset(vm_lapic(vm, vcpuid));
3127 
3128 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0));
3129 
3130 	vcpu->exit_intinfo = 0;
3131 	vcpu->exc_pending = 0;
3132 	vcpu->nmi_pending = false;
3133 	vcpu->extint_pending = 0;
3134 
3135 	/*
3136 	 * A CPU reset caused by power-on or system reset clears more state than
3137 	 * one which is trigged from an INIT IPI.
3138 	 */
3139 	if (!init_only) {
3140 		vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
3141 		(void) hma_fpu_init(vcpu->guestfpu);
3142 
3143 		/* XXX: clear MSRs and other pieces */
3144 		bzero(&vcpu->mtrr, sizeof (vcpu->mtrr));
3145 	}
3146 
3147 	return (0);
3148 }
3149 
3150 static int
vcpu_vector_sipi(struct vm * vm,int vcpuid,uint8_t vector)3151 vcpu_vector_sipi(struct vm *vm, int vcpuid, uint8_t vector)
3152 {
3153 	struct seg_desc desc;
3154 
3155 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3156 		return (EINVAL);
3157 
3158 	/* CS: Present, R/W, Accessed */
3159 	desc.access = 0x0093;
3160 	desc.base = (uint64_t)vector << 12;
3161 	desc.limit = 0xffff;
3162 	VERIFY0(vm_set_seg_desc(vm, vcpuid, VM_REG_GUEST_CS, &desc));
3163 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_CS,
3164 	    (uint64_t)vector << 8));
3165 
3166 	VERIFY0(vm_set_register(vm, vcpuid, VM_REG_GUEST_RIP, 0));
3167 
3168 	return (0);
3169 }
3170 
3171 int
vm_get_capability(struct vm * vm,int vcpu,int type,int * retval)3172 vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
3173 {
3174 	if (vcpu < 0 || vcpu >= vm->maxcpus)
3175 		return (EINVAL);
3176 
3177 	if (type < 0 || type >= VM_CAP_MAX)
3178 		return (EINVAL);
3179 
3180 	return (VMGETCAP(vm->cookie, vcpu, type, retval));
3181 }
3182 
3183 int
vm_set_capability(struct vm * vm,int vcpu,int type,int val)3184 vm_set_capability(struct vm *vm, int vcpu, int type, int val)
3185 {
3186 	if (vcpu < 0 || vcpu >= vm->maxcpus)
3187 		return (EINVAL);
3188 
3189 	if (type < 0 || type >= VM_CAP_MAX)
3190 		return (EINVAL);
3191 
3192 	return (VMSETCAP(vm->cookie, vcpu, type, val));
3193 }
3194 
3195 vcpu_cpuid_config_t *
vm_cpuid_config(struct vm * vm,int vcpuid)3196 vm_cpuid_config(struct vm *vm, int vcpuid)
3197 {
3198 	ASSERT3S(vcpuid, >=, 0);
3199 	ASSERT3S(vcpuid, <, VM_MAXCPU);
3200 
3201 	return (&vm->vcpu[vcpuid].cpuid_cfg);
3202 }
3203 
3204 struct vlapic *
vm_lapic(struct vm * vm,int cpu)3205 vm_lapic(struct vm *vm, int cpu)
3206 {
3207 	ASSERT3S(cpu, >=, 0);
3208 	ASSERT3S(cpu, <, VM_MAXCPU);
3209 
3210 	return (vm->vcpu[cpu].vlapic);
3211 }
3212 
3213 struct vioapic *
vm_ioapic(struct vm * vm)3214 vm_ioapic(struct vm *vm)
3215 {
3216 
3217 	return (vm->vioapic);
3218 }
3219 
3220 struct vhpet *
vm_hpet(struct vm * vm)3221 vm_hpet(struct vm *vm)
3222 {
3223 
3224 	return (vm->vhpet);
3225 }
3226 
3227 void *
vm_iommu_domain(struct vm * vm)3228 vm_iommu_domain(struct vm *vm)
3229 {
3230 
3231 	return (vm->iommu);
3232 }
3233 
3234 int
vcpu_set_state(struct vm * vm,int vcpuid,enum vcpu_state newstate,bool from_idle)3235 vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate,
3236     bool from_idle)
3237 {
3238 	int error;
3239 	struct vcpu *vcpu;
3240 
3241 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3242 		panic("vcpu_set_state: invalid vcpuid %d", vcpuid);
3243 
3244 	vcpu = &vm->vcpu[vcpuid];
3245 
3246 	vcpu_lock(vcpu);
3247 	error = vcpu_set_state_locked(vm, vcpuid, newstate, from_idle);
3248 	vcpu_unlock(vcpu);
3249 
3250 	return (error);
3251 }
3252 
3253 enum vcpu_state
vcpu_get_state(struct vm * vm,int vcpuid,int * hostcpu)3254 vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
3255 {
3256 	struct vcpu *vcpu;
3257 	enum vcpu_state state;
3258 
3259 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3260 		panic("vcpu_get_state: invalid vcpuid %d", vcpuid);
3261 
3262 	vcpu = &vm->vcpu[vcpuid];
3263 
3264 	vcpu_lock(vcpu);
3265 	state = vcpu->state;
3266 	if (hostcpu != NULL)
3267 		*hostcpu = vcpu->hostcpu;
3268 	vcpu_unlock(vcpu);
3269 
3270 	return (state);
3271 }
3272 
3273 /*
3274  * Calculate the TSC offset for a vCPU, applying physical CPU adjustments if
3275  * requested. The offset calculations include the VM-wide TSC offset.
3276  */
3277 uint64_t
vcpu_tsc_offset(struct vm * vm,int vcpuid,bool phys_adj)3278 vcpu_tsc_offset(struct vm *vm, int vcpuid, bool phys_adj)
3279 {
3280 	ASSERT(vcpuid >= 0 && vcpuid < vm->maxcpus);
3281 
3282 	uint64_t vcpu_off = vm->tsc_offset + vm->vcpu[vcpuid].tsc_offset;
3283 
3284 	if (phys_adj) {
3285 		/* Include any offset for the current physical CPU too */
3286 		vcpu_off += vmm_host_tsc_delta();
3287 	}
3288 
3289 	return (vcpu_off);
3290 }
3291 
3292 uint64_t
vm_get_freq_multiplier(struct vm * vm)3293 vm_get_freq_multiplier(struct vm *vm)
3294 {
3295 	return (vm->freq_multiplier);
3296 }
3297 
3298 /* Normalize hrtime against the boot time for a VM */
3299 hrtime_t
vm_normalize_hrtime(struct vm * vm,hrtime_t hrt)3300 vm_normalize_hrtime(struct vm *vm, hrtime_t hrt)
3301 {
3302 	/* To avoid underflow/overflow UB, perform math as unsigned */
3303 	return ((hrtime_t)((uint64_t)hrt - (uint64_t)vm->boot_hrtime));
3304 }
3305 
3306 /* Denormalize hrtime against the boot time for a VM */
3307 hrtime_t
vm_denormalize_hrtime(struct vm * vm,hrtime_t hrt)3308 vm_denormalize_hrtime(struct vm *vm, hrtime_t hrt)
3309 {
3310 	/* To avoid underflow/overflow UB, perform math as unsigned */
3311 	return ((hrtime_t)((uint64_t)hrt + (uint64_t)vm->boot_hrtime));
3312 }
3313 
3314 int
vm_activate_cpu(struct vm * vm,int vcpuid)3315 vm_activate_cpu(struct vm *vm, int vcpuid)
3316 {
3317 
3318 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3319 		return (EINVAL);
3320 
3321 	if (CPU_ISSET(vcpuid, &vm->active_cpus))
3322 		return (EBUSY);
3323 
3324 	if (vm_is_suspended(vm, NULL)) {
3325 		return (EBUSY);
3326 	}
3327 
3328 	CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
3329 
3330 	/*
3331 	 * It is possible that this vCPU was undergoing activation at the same
3332 	 * time that the VM was being suspended.
3333 	 */
3334 	if (vm_is_suspended(vm, NULL)) {
3335 		return (EBUSY);
3336 	}
3337 
3338 	return (0);
3339 }
3340 
3341 int
vm_suspend_cpu(struct vm * vm,int vcpuid)3342 vm_suspend_cpu(struct vm *vm, int vcpuid)
3343 {
3344 	int i;
3345 
3346 	if (vcpuid < -1 || vcpuid >= vm->maxcpus)
3347 		return (EINVAL);
3348 
3349 	if (vcpuid == -1) {
3350 		vm->debug_cpus = vm->active_cpus;
3351 		for (i = 0; i < vm->maxcpus; i++) {
3352 			if (CPU_ISSET(i, &vm->active_cpus))
3353 				vcpu_notify_event(vm, i);
3354 		}
3355 	} else {
3356 		if (!CPU_ISSET(vcpuid, &vm->active_cpus))
3357 			return (EINVAL);
3358 
3359 		CPU_SET_ATOMIC(vcpuid, &vm->debug_cpus);
3360 		vcpu_notify_event(vm, vcpuid);
3361 	}
3362 	return (0);
3363 }
3364 
3365 int
vm_resume_cpu(struct vm * vm,int vcpuid)3366 vm_resume_cpu(struct vm *vm, int vcpuid)
3367 {
3368 
3369 	if (vcpuid < -1 || vcpuid >= vm->maxcpus)
3370 		return (EINVAL);
3371 
3372 	if (vcpuid == -1) {
3373 		CPU_ZERO(&vm->debug_cpus);
3374 	} else {
3375 		if (!CPU_ISSET(vcpuid, &vm->debug_cpus))
3376 			return (EINVAL);
3377 
3378 		CPU_CLR_ATOMIC(vcpuid, &vm->debug_cpus);
3379 	}
3380 	return (0);
3381 }
3382 
3383 static bool
vcpu_bailout_checks(struct vm * vm,int vcpuid)3384 vcpu_bailout_checks(struct vm *vm, int vcpuid)
3385 {
3386 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3387 	struct vm_exit *vme = &vcpu->exitinfo;
3388 
3389 	ASSERT(vcpuid >= 0 && vcpuid < vm->maxcpus);
3390 
3391 	/*
3392 	 * Check if VM is suspended, only passing the 'vm_exit *' to be
3393 	 * populated if this check is being performed as part of entry.
3394 	 */
3395 	if (vm_is_suspended(vm, vme)) {
3396 		/* Confirm exit details are as expected */
3397 		VERIFY3S(vme->exitcode, ==, VM_EXITCODE_SUSPENDED);
3398 		VERIFY(vme->u.suspended.how > VM_SUSPEND_NONE &&
3399 		    vme->u.suspended.how < VM_SUSPEND_LAST);
3400 
3401 		return (true);
3402 	}
3403 	if (vcpu->reqidle) {
3404 		/*
3405 		 * Another thread is trying to lock this vCPU and is waiting for
3406 		 * it to enter the VCPU_IDLE state.  Take a lap with a BOGUS
3407 		 * exit to allow other thread(s) access to this vCPU.
3408 		 */
3409 		vme->exitcode = VM_EXITCODE_BOGUS;
3410 		vmm_stat_incr(vm, vcpuid, VMEXIT_REQIDLE, 1);
3411 		return (true);
3412 	}
3413 	if (vcpu->reqbarrier) {
3414 		/*
3415 		 * Similar to 'reqidle', userspace has requested that this vCPU
3416 		 * be pushed to a barrier by exiting to userspace.  Take that
3417 		 * lap with BOGUS and clear the flag.
3418 		 */
3419 		vme->exitcode = VM_EXITCODE_BOGUS;
3420 		vcpu->reqbarrier = false;
3421 		return (true);
3422 	}
3423 	if (vcpu->reqconsist) {
3424 		/*
3425 		 * We only expect exit-when-consistent requests to be asserted
3426 		 * during entry, not as an otherwise spontaneous condition.  As
3427 		 * such, we do not count it among the exit statistics, and emit
3428 		 * the expected BOGUS exitcode, while clearing the request.
3429 		 */
3430 		vme->exitcode = VM_EXITCODE_BOGUS;
3431 		vcpu->reqconsist = false;
3432 		return (true);
3433 	}
3434 	if (vcpu_should_yield(vm, vcpuid)) {
3435 		vme->exitcode = VM_EXITCODE_BOGUS;
3436 		vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1);
3437 		return (true);
3438 	}
3439 	if (CPU_ISSET(vcpuid, &vm->debug_cpus)) {
3440 		vme->exitcode = VM_EXITCODE_DEBUG;
3441 		return (true);
3442 	}
3443 
3444 	return (false);
3445 }
3446 
3447 static bool
vcpu_sleep_bailout_checks(struct vm * vm,int vcpuid)3448 vcpu_sleep_bailout_checks(struct vm *vm, int vcpuid)
3449 {
3450 	if (vcpu_bailout_checks(vm, vcpuid)) {
3451 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
3452 		struct vm_exit *vme = &vcpu->exitinfo;
3453 
3454 		/*
3455 		 * Bail-out check done prior to sleeping (in vCPU contexts like
3456 		 * HLT or wait-for-SIPI) expect that %rip is already populated
3457 		 * in the vm_exit structure, and we would only modify the
3458 		 * exitcode and clear the inst_length.
3459 		 */
3460 		vme->inst_length = 0;
3461 		return (true);
3462 	}
3463 	return (false);
3464 }
3465 
3466 bool
vcpu_entry_bailout_checks(struct vm * vm,int vcpuid,uint64_t rip)3467 vcpu_entry_bailout_checks(struct vm *vm, int vcpuid, uint64_t rip)
3468 {
3469 	if (vcpu_bailout_checks(vm, vcpuid)) {
3470 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
3471 		struct vm_exit *vme = &vcpu->exitinfo;
3472 
3473 		/*
3474 		 * Bail-out checks done as part of VM entry require an updated
3475 		 * %rip to populate the vm_exit struct if any of the conditions
3476 		 * of interest are matched in the check.
3477 		 */
3478 		vme->rip = rip;
3479 		vme->inst_length = 0;
3480 		return (true);
3481 	}
3482 	return (false);
3483 }
3484 
3485 int
vm_vcpu_barrier(struct vm * vm,int vcpuid)3486 vm_vcpu_barrier(struct vm *vm, int vcpuid)
3487 {
3488 	if (vcpuid >= 0 && vcpuid < vm->maxcpus) {
3489 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
3490 
3491 		/* Push specified vCPU to barrier */
3492 		vcpu_lock(vcpu);
3493 		if (CPU_ISSET(vcpuid, &vm->active_cpus)) {
3494 			vcpu->reqbarrier = true;
3495 			vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
3496 		}
3497 		vcpu_unlock(vcpu);
3498 
3499 		return (0);
3500 	} else if (vcpuid == -1) {
3501 		/* Push all (active) vCPUs to barrier */
3502 		for (int i = 0; i < vm->maxcpus; i++) {
3503 			struct vcpu *vcpu = &vm->vcpu[i];
3504 
3505 			vcpu_lock(vcpu);
3506 			if (CPU_ISSET(vcpuid, &vm->active_cpus)) {
3507 				vcpu->reqbarrier = true;
3508 				vcpu_notify_event_locked(vcpu,
3509 				    VCPU_NOTIFY_EXIT);
3510 			}
3511 			vcpu_unlock(vcpu);
3512 		}
3513 
3514 		return (0);
3515 	} else {
3516 		return (EINVAL);
3517 	}
3518 }
3519 
3520 cpuset_t
vm_active_cpus(struct vm * vm)3521 vm_active_cpus(struct vm *vm)
3522 {
3523 	return (vm->active_cpus);
3524 }
3525 
3526 cpuset_t
vm_debug_cpus(struct vm * vm)3527 vm_debug_cpus(struct vm *vm)
3528 {
3529 	return (vm->debug_cpus);
3530 }
3531 
3532 void *
vcpu_stats(struct vm * vm,int vcpuid)3533 vcpu_stats(struct vm *vm, int vcpuid)
3534 {
3535 
3536 	return (vm->vcpu[vcpuid].stats);
3537 }
3538 
3539 int
vm_get_x2apic_state(struct vm * vm,int vcpuid,enum x2apic_state * state)3540 vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
3541 {
3542 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3543 		return (EINVAL);
3544 
3545 	*state = vm->vcpu[vcpuid].x2apic_state;
3546 
3547 	return (0);
3548 }
3549 
3550 int
vm_set_x2apic_state(struct vm * vm,int vcpuid,enum x2apic_state state)3551 vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
3552 {
3553 	if (vcpuid < 0 || vcpuid >= vm->maxcpus)
3554 		return (EINVAL);
3555 
3556 	if (state >= X2APIC_STATE_LAST)
3557 		return (EINVAL);
3558 
3559 	vm->vcpu[vcpuid].x2apic_state = state;
3560 
3561 	vlapic_set_x2apic_state(vm, vcpuid, state);
3562 
3563 	return (0);
3564 }
3565 
3566 /*
3567  * This function is called to ensure that a vcpu "sees" a pending event
3568  * as soon as possible:
3569  * - If the vcpu thread is sleeping then it is woken up.
3570  * - If the vcpu is running on a different host_cpu then an IPI will be directed
3571  *   to the host_cpu to cause the vcpu to trap into the hypervisor.
3572  */
3573 static void
vcpu_notify_event_locked(struct vcpu * vcpu,vcpu_notify_t ntype)3574 vcpu_notify_event_locked(struct vcpu *vcpu, vcpu_notify_t ntype)
3575 {
3576 	int hostcpu;
3577 
3578 	ASSERT(ntype == VCPU_NOTIFY_APIC || VCPU_NOTIFY_EXIT);
3579 
3580 	hostcpu = vcpu->hostcpu;
3581 	if (vcpu->state == VCPU_RUNNING) {
3582 		KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
3583 		if (hostcpu != curcpu) {
3584 			if (ntype == VCPU_NOTIFY_APIC) {
3585 				vlapic_post_intr(vcpu->vlapic, hostcpu);
3586 			} else {
3587 				poke_cpu(hostcpu);
3588 			}
3589 		} else {
3590 			/*
3591 			 * If the 'vcpu' is running on 'curcpu' then it must
3592 			 * be sending a notification to itself (e.g. SELF_IPI).
3593 			 * The pending event will be picked up when the vcpu
3594 			 * transitions back to guest context.
3595 			 */
3596 		}
3597 	} else {
3598 		KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
3599 		    "with hostcpu %d", vcpu->state, hostcpu));
3600 		if (vcpu->state == VCPU_SLEEPING) {
3601 			cv_signal(&vcpu->vcpu_cv);
3602 		}
3603 	}
3604 }
3605 
3606 void
vcpu_notify_event(struct vm * vm,int vcpuid)3607 vcpu_notify_event(struct vm *vm, int vcpuid)
3608 {
3609 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3610 
3611 	vcpu_lock(vcpu);
3612 	vcpu_notify_event_locked(vcpu, VCPU_NOTIFY_EXIT);
3613 	vcpu_unlock(vcpu);
3614 }
3615 
3616 void
vcpu_notify_event_type(struct vm * vm,int vcpuid,vcpu_notify_t ntype)3617 vcpu_notify_event_type(struct vm *vm, int vcpuid, vcpu_notify_t ntype)
3618 {
3619 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3620 
3621 	if (ntype == VCPU_NOTIFY_NONE) {
3622 		return;
3623 	}
3624 
3625 	vcpu_lock(vcpu);
3626 	vcpu_notify_event_locked(vcpu, ntype);
3627 	vcpu_unlock(vcpu);
3628 }
3629 
3630 void
vcpu_ustate_change(struct vm * vm,int vcpuid,enum vcpu_ustate ustate)3631 vcpu_ustate_change(struct vm *vm, int vcpuid, enum vcpu_ustate ustate)
3632 {
3633 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3634 	const hrtime_t now = gethrtime();
3635 
3636 	ASSERT3S(ustate, <, VU_MAX);
3637 	ASSERT3S(ustate, >=, VU_INIT);
3638 
3639 	if (ustate == vcpu->ustate) {
3640 		return;
3641 	}
3642 
3643 	const hrtime_t delta = now - vcpu->ustate_when;
3644 	vcpu->ustate_total[vcpu->ustate] += delta;
3645 
3646 	membar_producer();
3647 
3648 	vcpu->ustate_when = now;
3649 	vcpu->ustate = ustate;
3650 }
3651 
3652 struct vmspace *
vm_get_vmspace(struct vm * vm)3653 vm_get_vmspace(struct vm *vm)
3654 {
3655 
3656 	return (vm->vmspace);
3657 }
3658 
3659 struct vm_client *
vm_get_vmclient(struct vm * vm,int vcpuid)3660 vm_get_vmclient(struct vm *vm, int vcpuid)
3661 {
3662 	return (vm->vcpu[vcpuid].vmclient);
3663 }
3664 
3665 int
vm_apicid2vcpuid(struct vm * vm,int apicid)3666 vm_apicid2vcpuid(struct vm *vm, int apicid)
3667 {
3668 	/*
3669 	 * XXX apic id is assumed to be numerically identical to vcpu id
3670 	 */
3671 	return (apicid);
3672 }
3673 
3674 struct vatpic *
vm_atpic(struct vm * vm)3675 vm_atpic(struct vm *vm)
3676 {
3677 	return (vm->vatpic);
3678 }
3679 
3680 struct vatpit *
vm_atpit(struct vm * vm)3681 vm_atpit(struct vm *vm)
3682 {
3683 	return (vm->vatpit);
3684 }
3685 
3686 struct vpmtmr *
vm_pmtmr(struct vm * vm)3687 vm_pmtmr(struct vm *vm)
3688 {
3689 
3690 	return (vm->vpmtmr);
3691 }
3692 
3693 struct vrtc *
vm_rtc(struct vm * vm)3694 vm_rtc(struct vm *vm)
3695 {
3696 
3697 	return (vm->vrtc);
3698 }
3699 
3700 enum vm_reg_name
vm_segment_name(int seg)3701 vm_segment_name(int seg)
3702 {
3703 	static enum vm_reg_name seg_names[] = {
3704 		VM_REG_GUEST_ES,
3705 		VM_REG_GUEST_CS,
3706 		VM_REG_GUEST_SS,
3707 		VM_REG_GUEST_DS,
3708 		VM_REG_GUEST_FS,
3709 		VM_REG_GUEST_GS
3710 	};
3711 
3712 	KASSERT(seg >= 0 && seg < nitems(seg_names),
3713 	    ("%s: invalid segment encoding %d", __func__, seg));
3714 	return (seg_names[seg]);
3715 }
3716 
3717 void
vm_copy_teardown(struct vm * vm,int vcpuid,struct vm_copyinfo * copyinfo,uint_t num_copyinfo)3718 vm_copy_teardown(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo,
3719     uint_t num_copyinfo)
3720 {
3721 	for (uint_t idx = 0; idx < num_copyinfo; idx++) {
3722 		if (copyinfo[idx].cookie != NULL) {
3723 			(void) vmp_release((vm_page_t *)copyinfo[idx].cookie);
3724 		}
3725 	}
3726 	bzero(copyinfo, num_copyinfo * sizeof (struct vm_copyinfo));
3727 }
3728 
3729 int
vm_copy_setup(struct vm * vm,int vcpuid,struct vm_guest_paging * paging,uint64_t gla,size_t len,int prot,struct vm_copyinfo * copyinfo,uint_t num_copyinfo,int * fault)3730 vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging,
3731     uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo,
3732     uint_t num_copyinfo, int *fault)
3733 {
3734 	uint_t idx, nused;
3735 	size_t n, off, remaining;
3736 	vm_client_t *vmc = vm_get_vmclient(vm, vcpuid);
3737 
3738 	bzero(copyinfo, sizeof (struct vm_copyinfo) * num_copyinfo);
3739 
3740 	nused = 0;
3741 	remaining = len;
3742 	while (remaining > 0) {
3743 		uint64_t gpa;
3744 		int error;
3745 
3746 		KASSERT(nused < num_copyinfo, ("insufficient vm_copyinfo"));
3747 		error = vm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa, fault);
3748 		if (error || *fault)
3749 			return (error);
3750 		off = gpa & PAGEOFFSET;
3751 		n = min(remaining, PAGESIZE - off);
3752 		copyinfo[nused].gpa = gpa;
3753 		copyinfo[nused].len = n;
3754 		remaining -= n;
3755 		gla += n;
3756 		nused++;
3757 	}
3758 
3759 	for (idx = 0; idx < nused; idx++) {
3760 		vm_page_t *vmp;
3761 		caddr_t hva;
3762 
3763 		vmp = vmc_hold(vmc, copyinfo[idx].gpa & PAGEMASK, prot);
3764 		if (vmp == NULL) {
3765 			break;
3766 		}
3767 		if ((prot & PROT_WRITE) != 0) {
3768 			hva = (caddr_t)vmp_get_writable(vmp);
3769 		} else {
3770 			hva = (caddr_t)vmp_get_readable(vmp);
3771 		}
3772 		copyinfo[idx].hva = hva + (copyinfo[idx].gpa & PAGEOFFSET);
3773 		copyinfo[idx].cookie = vmp;
3774 		copyinfo[idx].prot = prot;
3775 	}
3776 
3777 	if (idx != nused) {
3778 		vm_copy_teardown(vm, vcpuid, copyinfo, num_copyinfo);
3779 		return (EFAULT);
3780 	} else {
3781 		*fault = 0;
3782 		return (0);
3783 	}
3784 }
3785 
3786 void
vm_copyin(struct vm * vm,int vcpuid,struct vm_copyinfo * copyinfo,void * kaddr,size_t len)3787 vm_copyin(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, void *kaddr,
3788     size_t len)
3789 {
3790 	char *dst;
3791 	int idx;
3792 
3793 	dst = kaddr;
3794 	idx = 0;
3795 	while (len > 0) {
3796 		ASSERT(copyinfo[idx].prot & PROT_READ);
3797 
3798 		bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len);
3799 		len -= copyinfo[idx].len;
3800 		dst += copyinfo[idx].len;
3801 		idx++;
3802 	}
3803 }
3804 
3805 void
vm_copyout(struct vm * vm,int vcpuid,const void * kaddr,struct vm_copyinfo * copyinfo,size_t len)3806 vm_copyout(struct vm *vm, int vcpuid, const void *kaddr,
3807     struct vm_copyinfo *copyinfo, size_t len)
3808 {
3809 	const char *src;
3810 	int idx;
3811 
3812 	src = kaddr;
3813 	idx = 0;
3814 	while (len > 0) {
3815 		ASSERT(copyinfo[idx].prot & PROT_WRITE);
3816 
3817 		bcopy(src, copyinfo[idx].hva, copyinfo[idx].len);
3818 		len -= copyinfo[idx].len;
3819 		src += copyinfo[idx].len;
3820 		idx++;
3821 	}
3822 }
3823 
3824 /*
3825  * Return the amount of in-use and wired memory for the VM. Since
3826  * these are global stats, only return the values with for vCPU 0
3827  */
3828 VMM_STAT_DECLARE(VMM_MEM_RESIDENT);
3829 
3830 static void
vm_get_rescnt(struct vm * vm,int vcpu,struct vmm_stat_type * stat)3831 vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat)
3832 {
3833 	if (vcpu == 0) {
3834 		vmm_stat_set(vm, vcpu, VMM_MEM_RESIDENT,
3835 		    PAGE_SIZE * vmspace_resident_count(vm->vmspace));
3836 	}
3837 }
3838 
3839 VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt);
3840 
3841 int
vm_ioport_access(struct vm * vm,int vcpuid,bool in,uint16_t port,uint8_t bytes,uint32_t * val)3842 vm_ioport_access(struct vm *vm, int vcpuid, bool in, uint16_t port,
3843     uint8_t bytes, uint32_t *val)
3844 {
3845 	return (vm_inout_access(&vm->ioports, in, port, bytes, val));
3846 }
3847 
3848 /*
3849  * bhyve-internal interfaces to attach or detach IO port handlers.
3850  * Must be called with VM write lock held for safety.
3851  */
3852 int
vm_ioport_attach(struct vm * vm,uint16_t port,ioport_handler_t func,void * arg,void ** cookie)3853 vm_ioport_attach(struct vm *vm, uint16_t port, ioport_handler_t func, void *arg,
3854     void **cookie)
3855 {
3856 	int err;
3857 	err = vm_inout_attach(&vm->ioports, port, IOPF_DEFAULT, func, arg);
3858 	if (err == 0) {
3859 		*cookie = (void *)IOP_GEN_COOKIE(func, arg, port);
3860 	}
3861 	return (err);
3862 }
3863 int
vm_ioport_detach(struct vm * vm,void ** cookie,ioport_handler_t * old_func,void ** old_arg)3864 vm_ioport_detach(struct vm *vm, void **cookie, ioport_handler_t *old_func,
3865     void **old_arg)
3866 {
3867 	uint16_t port = IOP_PORT_FROM_COOKIE((uintptr_t)*cookie);
3868 	int err;
3869 
3870 	err = vm_inout_detach(&vm->ioports, port, false, old_func, old_arg);
3871 	if (err == 0) {
3872 		*cookie = NULL;
3873 	}
3874 	return (err);
3875 }
3876 
3877 /*
3878  * External driver interfaces to attach or detach IO port handlers.
3879  * Must be called with VM write lock held for safety.
3880  */
3881 int
vm_ioport_hook(struct vm * vm,uint16_t port,ioport_handler_t func,void * arg,void ** cookie)3882 vm_ioport_hook(struct vm *vm, uint16_t port, ioport_handler_t func,
3883     void *arg, void **cookie)
3884 {
3885 	int err;
3886 
3887 	if (port == 0) {
3888 		return (EINVAL);
3889 	}
3890 
3891 	err = vm_inout_attach(&vm->ioports, port, IOPF_DRV_HOOK, func, arg);
3892 	if (err == 0) {
3893 		*cookie = (void *)IOP_GEN_COOKIE(func, arg, port);
3894 	}
3895 	return (err);
3896 }
3897 void
vm_ioport_unhook(struct vm * vm,void ** cookie)3898 vm_ioport_unhook(struct vm *vm, void **cookie)
3899 {
3900 	uint16_t port = IOP_PORT_FROM_COOKIE((uintptr_t)*cookie);
3901 	ioport_handler_t old_func;
3902 	void *old_arg;
3903 	int err;
3904 
3905 	err = vm_inout_detach(&vm->ioports, port, true, &old_func, &old_arg);
3906 
3907 	/* ioport-hook-using drivers are expected to be well-behaved */
3908 	VERIFY0(err);
3909 	VERIFY(IOP_GEN_COOKIE(old_func, old_arg, port) == (uintptr_t)*cookie);
3910 
3911 	*cookie = NULL;
3912 }
3913 
3914 int
vmm_kstat_update_vcpu(struct kstat * ksp,int rw)3915 vmm_kstat_update_vcpu(struct kstat *ksp, int rw)
3916 {
3917 	struct vm *vm = ksp->ks_private;
3918 	vmm_vcpu_kstats_t *vvk = ksp->ks_data;
3919 	const int vcpuid = vvk->vvk_vcpu.value.ui32;
3920 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
3921 
3922 	ASSERT3U(vcpuid, <, VM_MAXCPU);
3923 
3924 	vvk->vvk_time_init.value.ui64 = vcpu->ustate_total[VU_INIT];
3925 	vvk->vvk_time_run.value.ui64 = vcpu->ustate_total[VU_RUN];
3926 	vvk->vvk_time_idle.value.ui64 = vcpu->ustate_total[VU_IDLE];
3927 	vvk->vvk_time_emu_kern.value.ui64 = vcpu->ustate_total[VU_EMU_KERN];
3928 	vvk->vvk_time_emu_user.value.ui64 = vcpu->ustate_total[VU_EMU_USER];
3929 	vvk->vvk_time_sched.value.ui64 = vcpu->ustate_total[VU_SCHED];
3930 
3931 	return (0);
3932 }
3933 
3934 SET_DECLARE(vmm_data_version_entries, const vmm_data_version_entry_t);
3935 
3936 static int
vmm_data_find(const vmm_data_req_t * req,const vmm_data_version_entry_t ** resp)3937 vmm_data_find(const vmm_data_req_t *req, const vmm_data_version_entry_t **resp)
3938 {
3939 	const vmm_data_version_entry_t **vdpp, *vdp;
3940 
3941 	ASSERT(resp != NULL);
3942 	ASSERT(req->vdr_result_len != NULL);
3943 
3944 	SET_FOREACH(vdpp, vmm_data_version_entries) {
3945 		vdp = *vdpp;
3946 		if (vdp->vdve_class != req->vdr_class ||
3947 		    vdp->vdve_version != req->vdr_version) {
3948 			continue;
3949 		}
3950 
3951 		/*
3952 		 * Enforce any data length expectation expressed by the provider
3953 		 * for this data.
3954 		 */
3955 		if (vdp->vdve_len_expect != 0 &&
3956 		    vdp->vdve_len_expect > req->vdr_len) {
3957 			*req->vdr_result_len = vdp->vdve_len_expect;
3958 			return (ENOSPC);
3959 		}
3960 
3961 		/*
3962 		 * Make sure that the provided vcpuid is acceptable for the
3963 		 * backend handler.
3964 		 */
3965 		if (vdp->vdve_readf != NULL || vdp->vdve_writef != NULL) {
3966 			/*
3967 			 * While it is tempting to demand the -1 sentinel value
3968 			 * in vcpuid here, that expectation was not established
3969 			 * for early consumers, so it is ignored.
3970 			 */
3971 		} else if (vdp->vdve_vcpu_readf != NULL ||
3972 		    vdp->vdve_vcpu_writef != NULL) {
3973 			/*
3974 			 * Per-vCPU handlers which permit "wildcard" access will
3975 			 * accept a vcpuid of -1 (for VM-wide data), while all
3976 			 * others expect vcpuid [0, VM_MAXCPU).
3977 			 */
3978 			const int llimit = vdp->vdve_vcpu_wildcard ? -1 : 0;
3979 			if (req->vdr_vcpuid < llimit ||
3980 			    req->vdr_vcpuid >= VM_MAXCPU) {
3981 				return (EINVAL);
3982 			}
3983 		} else {
3984 			/*
3985 			 * A provider with neither VM-wide nor per-vCPU handlers
3986 			 * is completely unexpected.  Such a situation should be
3987 			 * made into a compile-time error.  Bail out for now,
3988 			 * rather than punishing the user with a panic.
3989 			 */
3990 			return (EINVAL);
3991 		}
3992 
3993 
3994 		*resp = vdp;
3995 		return (0);
3996 	}
3997 	return (EINVAL);
3998 }
3999 
4000 static void *
vmm_data_from_class(const vmm_data_req_t * req,struct vm * vm)4001 vmm_data_from_class(const vmm_data_req_t *req, struct vm *vm)
4002 {
4003 	switch (req->vdr_class) {
4004 	case VDC_REGISTER:
4005 	case VDC_MSR:
4006 	case VDC_FPU:
4007 	case VDC_LAPIC:
4008 	case VDC_VMM_ARCH:
4009 		/*
4010 		 * These have per-CPU handling which is dispatched outside
4011 		 * vmm_data_version_entries listing.
4012 		 */
4013 		panic("Unexpected per-vcpu class %u", req->vdr_class);
4014 		break;
4015 
4016 	case VDC_IOAPIC:
4017 		return (vm->vioapic);
4018 	case VDC_ATPIT:
4019 		return (vm->vatpit);
4020 	case VDC_ATPIC:
4021 		return (vm->vatpic);
4022 	case VDC_HPET:
4023 		return (vm->vhpet);
4024 	case VDC_PM_TIMER:
4025 		return (vm->vpmtmr);
4026 	case VDC_RTC:
4027 		return (vm->vrtc);
4028 	case VDC_VMM_TIME:
4029 		return (vm);
4030 	case VDC_VERSION:
4031 		/*
4032 		 * Play along with all of the other classes which need backup
4033 		 * data, even though version info does not require it.
4034 		 */
4035 		return (vm);
4036 
4037 	default:
4038 		/* The data class will have been validated by now */
4039 		panic("Unexpected class %u", req->vdr_class);
4040 	}
4041 }
4042 
4043 const uint32_t default_msr_iter[] = {
4044 	/*
4045 	 * Although EFER is also available via the get/set-register interface,
4046 	 * we include it in the default list of emitted MSRs.
4047 	 */
4048 	MSR_EFER,
4049 
4050 	/*
4051 	 * While gsbase and fsbase are accessible via the MSR accessors, they
4052 	 * are not included in MSR iteration since they are covered by the
4053 	 * segment descriptor interface too.
4054 	 */
4055 	MSR_KGSBASE,
4056 
4057 	MSR_STAR,
4058 	MSR_LSTAR,
4059 	MSR_CSTAR,
4060 	MSR_SF_MASK,
4061 
4062 	MSR_SYSENTER_CS_MSR,
4063 	MSR_SYSENTER_ESP_MSR,
4064 	MSR_SYSENTER_EIP_MSR,
4065 
4066 	MSR_PAT,
4067 
4068 	MSR_TSC,
4069 
4070 	MSR_MTRRcap,
4071 	MSR_MTRRdefType,
4072 	MSR_MTRR4kBase, MSR_MTRR4kBase + 1, MSR_MTRR4kBase + 2,
4073 	MSR_MTRR4kBase + 3, MSR_MTRR4kBase + 4, MSR_MTRR4kBase + 5,
4074 	MSR_MTRR4kBase + 6, MSR_MTRR4kBase + 7,
4075 	MSR_MTRR16kBase, MSR_MTRR16kBase + 1,
4076 	MSR_MTRR64kBase,
4077 };
4078 
4079 static int
vmm_data_read_msr(struct vm * vm,int vcpuid,uint32_t msr,uint64_t * value)4080 vmm_data_read_msr(struct vm *vm, int vcpuid, uint32_t msr, uint64_t *value)
4081 {
4082 	int err = 0;
4083 
4084 	switch (msr) {
4085 	case MSR_TSC:
4086 		/*
4087 		 * The vmm-data interface for MSRs provides access to the
4088 		 * per-vCPU offset of the TSC, when reading/writing MSR_TSC.
4089 		 *
4090 		 * The VM-wide offset (and scaling) of the guest TSC is accessed
4091 		 * via the VMM_TIME data class.
4092 		 */
4093 		*value = vm->vcpu[vcpuid].tsc_offset;
4094 		return (0);
4095 
4096 	default:
4097 		if (is_mtrr_msr(msr)) {
4098 			err = vm_rdmtrr(&vm->vcpu[vcpuid].mtrr, msr, value);
4099 		} else {
4100 			err = ops->vmgetmsr(vm->cookie, vcpuid, msr, value);
4101 		}
4102 		break;
4103 	}
4104 
4105 	return (err);
4106 }
4107 
4108 static int
vmm_data_write_msr(struct vm * vm,int vcpuid,uint32_t msr,uint64_t value)4109 vmm_data_write_msr(struct vm *vm, int vcpuid, uint32_t msr, uint64_t value)
4110 {
4111 	int err = 0;
4112 
4113 	switch (msr) {
4114 	case MSR_TSC:
4115 		/* See vmm_data_read_msr() for more detail */
4116 		vm->vcpu[vcpuid].tsc_offset = value;
4117 		return (0);
4118 	case MSR_MTRRcap: {
4119 		/*
4120 		 * MTRRcap is read-only.  If the desired value matches the
4121 		 * existing one, consider it a success.
4122 		 */
4123 		uint64_t comp;
4124 		err = vm_rdmtrr(&vm->vcpu[vcpuid].mtrr, msr, &comp);
4125 		if (err == 0 && comp != value) {
4126 			return (EINVAL);
4127 		}
4128 		break;
4129 	}
4130 	default:
4131 		if (is_mtrr_msr(msr)) {
4132 			/* MTRRcap is already handled above */
4133 			ASSERT3U(msr, !=, MSR_MTRRcap);
4134 
4135 			err = vm_wrmtrr(&vm->vcpu[vcpuid].mtrr, msr, value);
4136 		} else {
4137 			err = ops->vmsetmsr(vm->cookie, vcpuid, msr, value);
4138 		}
4139 		break;
4140 	}
4141 
4142 	return (err);
4143 }
4144 
4145 static int
vmm_data_read_msrs(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4146 vmm_data_read_msrs(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4147 {
4148 	VERIFY3U(req->vdr_class, ==, VDC_MSR);
4149 	VERIFY3U(req->vdr_version, ==, 1);
4150 
4151 	struct vdi_field_entry_v1 *entryp = req->vdr_data;
4152 
4153 	/* Specific MSRs requested */
4154 	if ((req->vdr_flags & VDX_FLAG_READ_COPYIN) != 0) {
4155 		const uint_t count =
4156 		    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4157 
4158 		for (uint_t i = 0; i < count; i++, entryp++) {
4159 			int err = vmm_data_read_msr(vm, vcpuid,
4160 			    entryp->vfe_ident, &entryp->vfe_value);
4161 
4162 			if (err != 0) {
4163 				return (err);
4164 			}
4165 		}
4166 
4167 		*req->vdr_result_len =
4168 		    count * sizeof (struct vdi_field_entry_v1);
4169 		return (0);
4170 	}
4171 
4172 	/*
4173 	 * If specific MSRs are not requested, try to provide all those which we
4174 	 * know about instead.
4175 	 */
4176 	const uint_t num_msrs = nitems(default_msr_iter) +
4177 	    (VMM_MTRR_VAR_MAX * 2);
4178 	const uint32_t output_len =
4179 	    num_msrs * sizeof (struct vdi_field_entry_v1);
4180 
4181 	*req->vdr_result_len = output_len;
4182 	if (req->vdr_len < output_len) {
4183 		return (ENOSPC);
4184 	}
4185 
4186 	/* Output the MSRs in the default list */
4187 	for (uint_t i = 0; i < nitems(default_msr_iter); i++, entryp++) {
4188 		entryp->vfe_ident = default_msr_iter[i];
4189 
4190 		/* All of these MSRs are expected to work */
4191 		VERIFY0(vmm_data_read_msr(vm, vcpuid, entryp->vfe_ident,
4192 		    &entryp->vfe_value));
4193 	}
4194 
4195 	/* Output the variable MTRRs */
4196 	for (uint_t i = 0; i < (VMM_MTRR_VAR_MAX * 2); i++, entryp++) {
4197 		entryp->vfe_ident = MSR_MTRRVarBase + i;
4198 
4199 		/* All of these MSRs are expected to work */
4200 		VERIFY0(vmm_data_read_msr(vm, vcpuid, entryp->vfe_ident,
4201 		    &entryp->vfe_value));
4202 	}
4203 	return (0);
4204 }
4205 
4206 static int
vmm_data_write_msrs(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4207 vmm_data_write_msrs(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4208 {
4209 	VERIFY3U(req->vdr_class, ==, VDC_MSR);
4210 	VERIFY3U(req->vdr_version, ==, 1);
4211 
4212 	const struct vdi_field_entry_v1 *entryp = req->vdr_data;
4213 	const uint_t entry_count =
4214 	    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4215 
4216 	/*
4217 	 * First make sure that all of the MSRs can be manipulated.
4218 	 * For now, this check is done by going though the getmsr handler
4219 	 */
4220 	for (uint_t i = 0; i < entry_count; i++, entryp++) {
4221 		const uint64_t msr = entryp->vfe_ident;
4222 		uint64_t val;
4223 
4224 		if (vmm_data_read_msr(vm, vcpuid, msr, &val) != 0) {
4225 			return (EINVAL);
4226 		}
4227 	}
4228 
4229 	/*
4230 	 * Fairly confident that all of the 'set' operations are at least
4231 	 * targeting valid MSRs, continue on.
4232 	 */
4233 	entryp = req->vdr_data;
4234 	for (uint_t i = 0; i < entry_count; i++, entryp++) {
4235 		int err = vmm_data_write_msr(vm, vcpuid, entryp->vfe_ident,
4236 		    entryp->vfe_value);
4237 
4238 		if (err != 0) {
4239 			return (err);
4240 		}
4241 	}
4242 	*req->vdr_result_len = entry_count * sizeof (struct vdi_field_entry_v1);
4243 
4244 	return (0);
4245 }
4246 
4247 static const vmm_data_version_entry_t msr_v1 = {
4248 	.vdve_class = VDC_MSR,
4249 	.vdve_version = 1,
4250 	.vdve_len_per_item = sizeof (struct vdi_field_entry_v1),
4251 	.vdve_vcpu_readf = vmm_data_read_msrs,
4252 	.vdve_vcpu_writef = vmm_data_write_msrs,
4253 };
4254 VMM_DATA_VERSION(msr_v1);
4255 
4256 static const uint32_t vmm_arch_v1_fields[] = {
4257 	VAI_VM_IS_PAUSED,
4258 };
4259 
4260 static const uint32_t vmm_arch_v1_vcpu_fields[] = {
4261 	VAI_PEND_NMI,
4262 	VAI_PEND_EXTINT,
4263 	VAI_PEND_EXCP,
4264 	VAI_PEND_INTINFO,
4265 };
4266 
4267 static bool
vmm_read_arch_field(struct vm * vm,int vcpuid,uint32_t ident,uint64_t * valp)4268 vmm_read_arch_field(struct vm *vm, int vcpuid, uint32_t ident, uint64_t *valp)
4269 {
4270 	ASSERT(valp != NULL);
4271 
4272 	if (vcpuid == -1) {
4273 		switch (ident) {
4274 		case VAI_VM_IS_PAUSED:
4275 			*valp = vm->is_paused ? 1 : 0;
4276 			return (true);
4277 		default:
4278 			break;
4279 		}
4280 	} else {
4281 		VERIFY(vcpuid >= 0 && vcpuid <= VM_MAXCPU);
4282 
4283 		struct vcpu *vcpu = &vm->vcpu[vcpuid];
4284 		switch (ident) {
4285 		case VAI_PEND_NMI:
4286 			*valp = vcpu->nmi_pending != 0 ? 1 : 0;
4287 			return (true);
4288 		case VAI_PEND_EXTINT:
4289 			*valp = vcpu->extint_pending != 0 ? 1 : 0;
4290 			return (true);
4291 		case VAI_PEND_EXCP:
4292 			*valp = vcpu->exc_pending;
4293 			return (true);
4294 		case VAI_PEND_INTINFO:
4295 			*valp = vcpu->exit_intinfo;
4296 			return (true);
4297 		default:
4298 			break;
4299 		}
4300 	}
4301 	return (false);
4302 }
4303 
4304 static int
vmm_data_read_varch(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4305 vmm_data_read_varch(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4306 {
4307 	VERIFY3U(req->vdr_class, ==, VDC_VMM_ARCH);
4308 	VERIFY3U(req->vdr_version, ==, 1);
4309 
4310 	/* per-vCPU fields are handled separately from VM-wide ones */
4311 	if (vcpuid != -1 && (vcpuid < 0 || vcpuid >= VM_MAXCPU)) {
4312 		return (EINVAL);
4313 	}
4314 
4315 	struct vdi_field_entry_v1 *entryp = req->vdr_data;
4316 
4317 	/* Specific fields requested */
4318 	if ((req->vdr_flags & VDX_FLAG_READ_COPYIN) != 0) {
4319 		const uint_t count =
4320 		    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4321 
4322 		for (uint_t i = 0; i < count; i++, entryp++) {
4323 			if (!vmm_read_arch_field(vm, vcpuid, entryp->vfe_ident,
4324 			    &entryp->vfe_value)) {
4325 				return (EINVAL);
4326 			}
4327 		}
4328 		*req->vdr_result_len =
4329 		    count * sizeof (struct vdi_field_entry_v1);
4330 		return (0);
4331 	}
4332 
4333 	/* Emit all of the possible values */
4334 	const uint32_t *idents;
4335 	uint_t ident_count;
4336 
4337 	if (vcpuid == -1) {
4338 		idents = vmm_arch_v1_fields;
4339 		ident_count = nitems(vmm_arch_v1_fields);
4340 	} else {
4341 		idents = vmm_arch_v1_vcpu_fields;
4342 		ident_count = nitems(vmm_arch_v1_vcpu_fields);
4343 
4344 	}
4345 
4346 	const uint32_t total_size =
4347 	    ident_count * sizeof (struct vdi_field_entry_v1);
4348 
4349 	*req->vdr_result_len = total_size;
4350 	if (req->vdr_len < total_size) {
4351 		return (ENOSPC);
4352 	}
4353 	for (uint_t i = 0; i < ident_count; i++, entryp++) {
4354 		entryp->vfe_ident = idents[i];
4355 		VERIFY(vmm_read_arch_field(vm, vcpuid, entryp->vfe_ident,
4356 		    &entryp->vfe_value));
4357 	}
4358 	return (0);
4359 }
4360 
4361 static int
vmm_data_write_varch_vcpu(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4362 vmm_data_write_varch_vcpu(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4363 {
4364 	VERIFY3U(req->vdr_class, ==, VDC_VMM_ARCH);
4365 	VERIFY3U(req->vdr_version, ==, 1);
4366 
4367 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU) {
4368 		return (EINVAL);
4369 	}
4370 
4371 	const struct vdi_field_entry_v1 *entryp = req->vdr_data;
4372 	const uint_t entry_count =
4373 	    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4374 	struct vcpu *vcpu = &vm->vcpu[vcpuid];
4375 
4376 	for (uint_t i = 0; i < entry_count; i++, entryp++) {
4377 		const uint64_t val = entryp->vfe_value;
4378 
4379 		switch (entryp->vfe_ident) {
4380 		case VAI_PEND_NMI:
4381 			vcpu->nmi_pending = (val != 0);
4382 			break;
4383 		case VAI_PEND_EXTINT:
4384 			vcpu->extint_pending = (val != 0);
4385 			break;
4386 		case VAI_PEND_EXCP:
4387 			if (!VM_INTINFO_PENDING(val)) {
4388 				vcpu->exc_pending = 0;
4389 			} else if (VM_INTINFO_TYPE(val) != VM_INTINFO_HWEXCP ||
4390 			    (val & VM_INTINFO_MASK_RSVD) != 0) {
4391 				/* reject improperly-formed hw exception */
4392 				return (EINVAL);
4393 			} else {
4394 				vcpu->exc_pending = val;
4395 			}
4396 			break;
4397 		case VAI_PEND_INTINFO:
4398 			if (vm_exit_intinfo(vm, vcpuid, val) != 0) {
4399 				return (EINVAL);
4400 			}
4401 			break;
4402 		default:
4403 			return (EINVAL);
4404 		}
4405 	}
4406 
4407 	*req->vdr_result_len = entry_count * sizeof (struct vdi_field_entry_v1);
4408 	return (0);
4409 }
4410 
4411 static int
vmm_data_write_varch(struct vm * vm,int vcpuid,const vmm_data_req_t * req)4412 vmm_data_write_varch(struct vm *vm, int vcpuid, const vmm_data_req_t *req)
4413 {
4414 	VERIFY3U(req->vdr_class, ==, VDC_VMM_ARCH);
4415 	VERIFY3U(req->vdr_version, ==, 1);
4416 
4417 	/* per-vCPU fields are handled separately from VM-wide ones */
4418 	if (vcpuid != -1) {
4419 		return (vmm_data_write_varch_vcpu(vm, vcpuid, req));
4420 	}
4421 
4422 	const struct vdi_field_entry_v1 *entryp = req->vdr_data;
4423 	const uint_t entry_count =
4424 	    req->vdr_len / sizeof (struct vdi_field_entry_v1);
4425 
4426 	if (entry_count > 0) {
4427 		if (entryp->vfe_ident == VAI_VM_IS_PAUSED) {
4428 			/*
4429 			 * The VM_PAUSE and VM_RESUME ioctls are the officially
4430 			 * sanctioned mechanisms for setting the is-paused state
4431 			 * of the VM.
4432 			 */
4433 			return (EPERM);
4434 		} else {
4435 			/* no other valid arch entries at this time */
4436 			return (EINVAL);
4437 		}
4438 	}
4439 
4440 	*req->vdr_result_len = entry_count * sizeof (struct vdi_field_entry_v1);
4441 	return (0);
4442 }
4443 
4444 static const vmm_data_version_entry_t vmm_arch_v1 = {
4445 	.vdve_class = VDC_VMM_ARCH,
4446 	.vdve_version = 1,
4447 	.vdve_len_per_item = sizeof (struct vdi_field_entry_v1),
4448 	.vdve_vcpu_readf = vmm_data_read_varch,
4449 	.vdve_vcpu_writef = vmm_data_write_varch,
4450 
4451 	/*
4452 	 * Handlers for VMM_ARCH can process VM-wide (vcpuid == -1) entries in
4453 	 * addition to vCPU specific ones.
4454 	 */
4455 	.vdve_vcpu_wildcard = true,
4456 };
4457 VMM_DATA_VERSION(vmm_arch_v1);
4458 
4459 
4460 /*
4461  * GUEST TIME SUPPORT
4462  *
4463  * Broadly, there are two categories of functionality related to time passing in
4464  * the guest: the guest's TSC and timers used by emulated devices.
4465  *
4466  * ---------------------------
4467  * GUEST TSC "VIRTUALIZATION"
4468  * ---------------------------
4469  *
4470  * The TSC can be read either via an instruction (rdtsc/rdtscp) or by reading
4471  * the TSC MSR.
4472  *
4473  * When a guest reads the TSC via its MSR, the guest will exit and we emulate
4474  * the rdmsr. More typically, the guest reads the TSC via a rdtsc(p)
4475  * instruction. Both SVM and VMX support virtualizing the guest TSC in hardware
4476  * -- that is, a guest will not generally exit on a rdtsc instruction.
4477  *
4478  * To support hardware-virtualized guest TSC, both SVM and VMX provide two knobs
4479  * for the hypervisor to adjust the guest's view of the TSC:
4480  * - TSC offset
4481  * - TSC frequency multiplier (also called "frequency ratio")
4482  *
4483  * When a guest calls rdtsc(p), the TSC value it sees is the sum of:
4484  *     guest_tsc = (host TSC, scaled according to frequency multiplier)
4485  *		    + (TSC offset, programmed by hypervisor)
4486  *
4487  * See the discussions of the TSC offset and frequency multiplier below for more
4488  * details on each of these.
4489  *
4490  * --------------------
4491  * TSC OFFSET OVERVIEW
4492  * --------------------
4493  *
4494  * The TSC offset is a value added to the host TSC (which may be scaled first)
4495  * to provide the guest TSC. This offset addition is generally done by hardware,
4496  * but may be used in emulating the TSC if necessary.
4497  *
4498  * Recall that general formula for calculating the guest TSC is:
4499  *
4500  *	guest_tsc = (host TSC, scaled if needed) + TSC offset
4501  *
4502  * Intuitively, the TSC offset is simply an offset of the host's TSC to make the
4503  * guest's view of the TSC appear correct: The guest TSC should be 0 at boot and
4504  * monotonically increase at a roughly constant frequency. Thus in the simplest
4505  * case, the TSC offset is just the negated value of the host TSC when the guest
4506  * was booted, assuming they have the same frequencies.
4507  *
4508  * In practice, there are several factors that can make calculating the TSC
4509  * offset more complicated, including:
4510  *
4511  * (1) the physical CPU the guest is running on
4512  * (2) whether the guest has written to the TSC of that vCPU
4513  * (3) differing host and guest frequencies, like after a live migration
4514  * (4) a guest running on a different system than where it was booted, like
4515  *     after a live migration
4516  *
4517  * We will explore each of these factors individually. See below for a
4518  * summary.
4519  *
4520  *
4521  * (1) Physical CPU offsets
4522  *
4523  * The system maintains a set of per-CPU offsets to the TSC to provide a
4524  * consistent view of the TSC regardless of the CPU a thread is running on.
4525  * These offsets are included automatically as a part of rdtsc_offset().
4526  *
4527  * The per-CPU offset must be included as a part reading the host TSC when
4528  * calculating the offset before running the guest on a given CPU.
4529  *
4530  *
4531  * (2) Guest TSC writes (vCPU offsets)
4532  *
4533  * The TSC is a writable MSR. When a guest writes to the TSC, this operation
4534  * should result in the TSC, when read from that vCPU, shows the value written,
4535  * plus whatever time has elapsed since the read.
4536  *
4537  * To support this, when the guest writes to the TSC, we store an additional
4538  * vCPU offset calculated to make future reads of the TSC map to what the guest
4539  * expects.
4540  *
4541  *
4542  * (3) Differing host and guest frequencies (host TSC scaling)
4543  *
4544  * A guest has the same frequency of its host when it boots, but it may be
4545  * migrated to a machine with a different TSC frequency. Systems expect that
4546  * their TSC frequency does not change. To support this fiction in which a guest
4547  * is running on hardware of a different TSC frequency, the hypervisor  can
4548  * program a "frequency multiplier" that represents the ratio of guest/host
4549  * frequency.
4550  *
4551  * Any time a host TSC is used in calculations for the offset, it should be
4552  * "scaled" according to this multiplier, and the hypervisor should program the
4553  * multiplier before running a guest so that the hardware virtualization of the
4554  * TSC functions properly. Similarly, the multiplier should be used in any TSC
4555  * emulation.
4556  *
4557  * See below for more details about the frequency multiplier.
4558  *
4559  *
4560  * (4) Guest running on a system it did not boot on ("base guest TSC")
4561  *
4562  * When a guest boots, its TSC offset is simply the negated host TSC at the time
4563  * it booted. If a guest is migrated from a source host to a target host, the
4564  * TSC offset from the source host is no longer useful for several reasons:
4565  * - the target host TSC has no relationship to the source host TSC
4566  * - the guest did not boot on the target system, so the TSC of the target host
4567  *   is not sufficient to describe how long the guest has been running prior to
4568  *   migration
4569  * - the target system may have a different TSC frequency than the source system
4570  *
4571  * Ignoring the issue of frequency differences for a moment, let's consider how
4572  * to re-align the guest TSC with the host TSC of the target host. Intuitively,
4573  * for the guest to see the correct TSC, we still want to add some offset to the
4574  * host TSC that offsets how long this guest has been running on
4575  * the system.
4576  *
4577  * An example here might be helpful. Consider a source host and target host,
4578  * both with TSC frequencies of 1GHz. On the source host, the guest and host TSC
4579  * values might look like:
4580  *
4581  *  +----------------------------------------------------------------------+
4582  *  | Event                 | source host TSC  | guest TSC                 |
4583  *  ------------------------------------------------------------------------
4584  *  | guest boot  (t=0s)    | 5000000000       | 5000000000 + -5000000000  |
4585  *  |                       |                  | 0			   |
4586  *  ------------------------------------------------------------------------
4587  *  | guest rdtsc (t=10s))  | 15000000000      | 15000000000 + -5000000000 |
4588  *  |                       |                  | 10000000000		   |
4589  *  ------------------------------------------------------------------------
4590  *  | migration   (t=15s)   | 20000000000      | 20000000000 + -5000000000 |
4591  *  |                       |                  | 15000000000		   |
4592  *  +----------------------------------------------------------------------+
4593  *
4594  * Ignoring the time it takes for a guest to physically migrate machines, on the
4595  * target host, we would expect the TSC to continue functioning as such:
4596  *
4597  *  +----------------------------------------------------------------------+
4598  *  | Event                 | target host TSC  | guest TSC                 |
4599  *  ------------------------------------------------------------------------
4600  *  | guest migrate (t=15s) | 300000000000     | 15000000000		   |
4601  *  ------------------------------------------------------------------------
4602  *  | guest rdtsc (t=20s))  | 305000000000     | 20000000000		   |
4603  *  ------------------------------------------------------------------------
4604  *
4605  * In order to produce a correct TSC value here, we can calculate a new
4606  * "effective" boot TSC that maps to what the host TSC would've been had it been
4607  * booted on the target. We add that to the guest TSC when it began to run on
4608  * this machine, and negate them both to get a new offset. In this example, the
4609  * effective boot TSC is: -(300000000000 - 15000000000) = -285000000000.
4610  *
4611  *  +-------------------------------------------------------------------------+
4612  *  | Event                 | target host TSC  | guest TSC                    |
4613  *  ---------------------------------------------------------------------------
4614  *  | guest "boot" (t=0s)   | 285000000000     | 285000000000 + -285000000000 |
4615  *  |                       |                  | 0			      |
4616  *  ---------------------------------------------------------------------------
4617  *  | guest migrate (t=15s) | 300000000000     | 300000000000 + -285000000000 |
4618  *  |                       |                  | 15000000000		      |
4619  *  ---------------------------------------------------------------------------
4620  *  | guest rdtsc (t=20s))  | 305000000000     | 305000000000 + -285000000000 |
4621  *  |                       |                  | 20000000000		      |
4622  *  --------------------------------------------------------------------------+
4623  *
4624  * To support the offset calculation following a migration, the VMM data time
4625  * interface allows callers to set a "base guest TSC", which is the TSC value of
4626  * the guest when it began running on the host. The current guest TSC can be
4627  * requested via a read of the time data. See below for details on that
4628  * interface.
4629  *
4630  * Frequency differences between the host and the guest are accounted for when
4631  * scaling the host TSC. See below for details on the frequency multiplier.
4632  *
4633  *
4634  * --------------------
4635  * TSC OFFSET SUMMARY
4636  * --------------------
4637  *
4638  * Factoring in all of the components to the TSC above, the TSC offset that is
4639  * programmed by the hypervisor before running a given vCPU is:
4640  *
4641  * offset = -((base host TSC, scaled if needed) - base_guest_tsc) + vCPU offset
4642  *
4643  * This offset is stored in two pieces. Per-vCPU offsets are stored with the
4644  * given vCPU and added in when programming the offset. The rest of the offset
4645  * is stored as a VM-wide offset, and computed either at boot or when the time
4646  * data is written to.
4647  *
4648  * It is safe to add the vCPU offset and the VM-wide offsets together because
4649  * the vCPU offset is in terms of the guest TSC. The host TSC is scaled before
4650  * using it in calculations, so all TSC values are applicable to the same
4651  * frequency.
4652  *
4653  * Note: Though both the VM-wide offset and per-vCPU offsets may be negative, we
4654  * store them as unsigned values and perform all offsetting math unsigned. This
4655  * is to avoid UB from signed overflow.
4656  *
4657  * -------------------------
4658  * TSC FREQUENCY MULTIPLIER
4659  * -------------------------
4660  *
4661  * In order to account for frequency differences between the host and guest, SVM
4662  * and VMX provide an interface to set a "frequency multiplier" (or "frequency
4663  * ratio") representing guest to host frequency. In a hardware-virtualized read
4664  * of the TSC, the host TSC is scaled using this multiplier prior to adding the
4665  * programmed TSC offset.
4666  *
4667  * Both platforms represent the ratio as a fixed point number, where the lower
4668  * bits are used as a fractional component, and some number of the upper bits
4669  * are used as the integer component.
4670  *
4671  * Some example multipliers, for a platform with FRAC fractional bits in the
4672  * multiplier:
4673  * - guest frequency == host: 1 << FRAC
4674  * - guest frequency is 2x host: 1 << (FRAC + 1)
4675  * - guest frequency is 0.5x host: 1 << (FRAC - 1), as the highest-order
4676  *   fractional bit represents 1/2
4677  * - guest frequency is 2.5x host: (1 << FRAC) | (1 << (FRAC - 1))
4678  * and so on.
4679  *
4680  * In general, the frequency multiplier is calculated as follows:
4681  *		(guest_hz * (1 << FRAC_SIZE)) / host_hz
4682  *
4683  * The multiplier should be used any time the host TSC value is used in
4684  * calculations with the guest TSC (and their frequencies differ). The function
4685  * `vmm_scale_tsc` is intended to be used for these purposes, as it will scale
4686  * the host TSC only if needed.
4687  *
4688  * The multiplier should also be programmed by the hypervisor before the guest
4689  * is run.
4690  *
4691  *
4692  * ----------------------------
4693  * DEVICE TIMERS (BOOT_HRTIME)
4694  * ----------------------------
4695  *
4696  * Emulated devices use timers to do things such as scheduling periodic events.
4697  * These timers are scheduled relative to the hrtime of the host. When device
4698  * state is exported or imported, we use boot_hrtime to normalize these timers
4699  * against the host hrtime. The boot_hrtime represents the hrtime of the host
4700  * when the guest was booted.
4701  *
4702  * If a guest is migrated to a different machine, boot_hrtime must be adjusted
4703  * to match the hrtime of when the guest was effectively booted on the target
4704  * host. This allows timers to continue functioning when device state is
4705  * imported on the target.
4706  *
4707  *
4708  * ------------------------
4709  * VMM DATA TIME INTERFACE
4710  * ------------------------
4711  *
4712  * In order to facilitate live migrations of guests, we provide an interface,
4713  * via the VMM data read/write ioctls, for userspace to make changes to the
4714  * guest's view of the TSC and device timers, allowing these features to
4715  * continue functioning after a migration.
4716  *
4717  * The interface was designed to expose the minimal amount of data needed for a
4718  * userspace component to make adjustments to the guest's view of time (e.g., to
4719  * account for time passing in a live migration). At a minimum, such a program
4720  * needs:
4721  * - the current guest TSC
4722  * - guest TSC frequency
4723  * - guest's boot_hrtime
4724  * - timestamps of when this data was taken (hrtime for hrtime calculations, and
4725  *   wall clock time for computing time deltas between machines)
4726  *
4727  * The wall clock time is provided for consumers to make adjustments to the
4728  * guest TSC and boot_hrtime based on deltas observed during migrations. It may
4729  * be prudent for consumers to use this data only in circumstances where the
4730  * source and target have well-synchronized wall clocks, but nothing in the
4731  * interface depends on this assumption.
4732  *
4733  * On writes, consumers write back:
4734  * - the base guest TSC (used for TSC offset calculations)
4735  * - desired boot_hrtime
4736  * - guest_frequency (cannot change)
4737  * - hrtime of when this data was adjusted
4738  * - (wall clock time on writes is ignored)
4739  *
4740  * The interface will adjust the input guest TSC slightly, based on the input
4741  * hrtime, to account for latency between userspace calculations and application
4742  * of the data on the kernel side. This amounts to adding a small amount of
4743  * additional "uptime" for the guest.
4744  *
4745  * After the adjustments, the interface updates the VM-wide TSC offset and
4746  * boot_hrtime. Per-vCPU offsets are not adjusted, as those are already in terms
4747  * of the guest TSC and can be exported/imported via the MSR VMM data interface.
4748  *
4749  *
4750  * --------------------------------
4751  * SUPPORTED PLATFORMS AND CAVEATS
4752  * --------------------------------
4753  *
4754  * While both VMX and SVM offer TSC scaling as a feature, at this time only SVM
4755  * is supported by bhyve.
4756  *
4757  * The time data interface is designed such that Intel support can be added
4758  * easily, and all other aspects of the time interface should work on Intel.
4759  * (Without frequency control though, in practice, doing live migrations of
4760  * guests on Intel will not work for time-related things, as two machines
4761  * rarely have exactly the same frequency).
4762  *
4763  * Additionally, while on both SVM and VMX the frequency multiplier is a fixed
4764  * point number, each uses a different number of fractional and integer bits for
4765  * the multiplier. As such, calculating the multiplier and fractional bit size
4766  * is requested via the vmm_ops.
4767  *
4768  * Care should be taken to set reasonable limits for ratios based on the
4769  * platform, as the difference in fractional bits can lead to slightly different
4770  * tradeoffs in terms of representable ratios and potentially overflowing
4771  * calculations.
4772  */
4773 
4774 /*
4775  * Scales the TSC if needed, based on the input frequency multiplier.
4776  */
4777 static uint64_t
vmm_scale_tsc(uint64_t tsc,uint64_t mult)4778 vmm_scale_tsc(uint64_t tsc, uint64_t mult)
4779 {
4780 	const uint32_t frac_size = ops->fr_fracsize;
4781 
4782 	if (mult != VM_TSCM_NOSCALE) {
4783 		VERIFY3U(frac_size, >, 0);
4784 		return (scale_tsc(tsc, mult, frac_size));
4785 	} else {
4786 		return (tsc);
4787 	}
4788 }
4789 
4790 /*
4791  * Calculate the frequency multiplier, which represents the ratio of
4792  * guest_hz / host_hz. The frequency multiplier is a fixed point number with
4793  * `frac_sz` fractional bits (fractional bits begin at bit 0).
4794  *
4795  * See comment for "calc_freq_multiplier" in "vmm_time_support.S" for more
4796  * information about valid input to this function.
4797  */
4798 uint64_t
vmm_calc_freq_multiplier(uint64_t guest_hz,uint64_t host_hz,uint32_t frac_size)4799 vmm_calc_freq_multiplier(uint64_t guest_hz, uint64_t host_hz,
4800     uint32_t frac_size)
4801 {
4802 	VERIFY3U(guest_hz, !=, 0);
4803 	VERIFY3U(frac_size, >, 0);
4804 	VERIFY3U(frac_size, <, 64);
4805 
4806 	return (calc_freq_multiplier(guest_hz, host_hz, frac_size));
4807 }
4808 
4809 /*
4810  * Calculate the guest VM-wide TSC offset.
4811  *
4812  * offset = - ((base host TSC, scaled if needed) - base_guest_tsc)
4813  *
4814  * The base_host_tsc and the base_guest_tsc are the TSC values of the host
4815  * (read on the system) and the guest (calculated) at the same point in time.
4816  * This allows us to fix the guest TSC at this point in time as a base, either
4817  * following boot (guest TSC = 0), or a change to the guest's time data from
4818  * userspace (such as in the case of a migration).
4819  */
4820 static uint64_t
calc_tsc_offset(uint64_t base_host_tsc,uint64_t base_guest_tsc,uint64_t mult)4821 calc_tsc_offset(uint64_t base_host_tsc, uint64_t base_guest_tsc, uint64_t mult)
4822 {
4823 	const uint64_t htsc_scaled = vmm_scale_tsc(base_host_tsc, mult);
4824 	if (htsc_scaled > base_guest_tsc) {
4825 		return ((uint64_t)(- (int64_t)(htsc_scaled - base_guest_tsc)));
4826 	} else {
4827 		return (base_guest_tsc - htsc_scaled);
4828 	}
4829 }
4830 
4831 /*
4832  * Calculate an estimate of the guest TSC.
4833  *
4834  * guest_tsc = (host TSC, scaled if needed) + offset
4835  */
4836 static uint64_t
calc_guest_tsc(uint64_t host_tsc,uint64_t mult,uint64_t offset)4837 calc_guest_tsc(uint64_t host_tsc, uint64_t mult, uint64_t offset)
4838 {
4839 	return (vmm_scale_tsc(host_tsc, mult) + offset);
4840 }
4841 
4842 /*
4843  * Take a non-atomic "snapshot" of the current:
4844  * - TSC
4845  * - hrtime
4846  * - wall clock time
4847  */
4848 static void
vmm_time_snapshot(uint64_t * tsc,hrtime_t * hrtime,timespec_t * hrestime)4849 vmm_time_snapshot(uint64_t *tsc, hrtime_t *hrtime, timespec_t *hrestime)
4850 {
4851 	/*
4852 	 * Disable interrupts while we take the readings: In the absence of a
4853 	 * mechanism to convert hrtime to hrestime, we want the time between
4854 	 * each of these measurements to be as small as possible.
4855 	 */
4856 	ulong_t iflag = intr_clear();
4857 
4858 	hrtime_t hrt = gethrtimeunscaledf();
4859 	*tsc = (uint64_t)hrt;
4860 	*hrtime = hrt;
4861 	scalehrtime(hrtime);
4862 	gethrestime(hrestime);
4863 
4864 	intr_restore(iflag);
4865 }
4866 
4867 /*
4868  * Read VMM Time data
4869  *
4870  * Provides:
4871  * - the current guest TSC and TSC frequency
4872  * - guest boot_hrtime
4873  * - timestamps of the read (hrtime and wall clock time)
4874  */
4875 static int
vmm_data_read_vmm_time(void * arg,const vmm_data_req_t * req)4876 vmm_data_read_vmm_time(void *arg, const vmm_data_req_t *req)
4877 {
4878 	VERIFY3U(req->vdr_class, ==, VDC_VMM_TIME);
4879 	VERIFY3U(req->vdr_version, ==, 1);
4880 	VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_time_info_v1));
4881 
4882 	struct vm *vm = arg;
4883 	struct vdi_time_info_v1 *out = req->vdr_data;
4884 
4885 	/*
4886 	 * Since write operations on VMM_TIME data are strict about vcpuid
4887 	 * (see: vmm_data_write_vmm_time()), read operations should be as well.
4888 	 */
4889 	if (req->vdr_vcpuid != -1) {
4890 		return (EINVAL);
4891 	}
4892 
4893 	/* Take a snapshot of this point in time */
4894 	uint64_t tsc;
4895 	hrtime_t hrtime;
4896 	timespec_t hrestime;
4897 	vmm_time_snapshot(&tsc, &hrtime, &hrestime);
4898 
4899 	/* Write the output values */
4900 	out->vt_guest_freq = vm->guest_freq;
4901 
4902 	/*
4903 	 * Use only the VM-wide TSC offset for calculating the guest TSC,
4904 	 * ignoring per-vCPU offsets. This value is provided as a "base" guest
4905 	 * TSC at the time of the read; per-vCPU offsets are factored in as
4906 	 * needed elsewhere, either when running the vCPU or if the guest reads
4907 	 * the TSC via rdmsr.
4908 	 */
4909 	out->vt_guest_tsc = calc_guest_tsc(tsc, vm->freq_multiplier,
4910 	    vm->tsc_offset);
4911 	out->vt_boot_hrtime = vm->boot_hrtime;
4912 	out->vt_hrtime = hrtime;
4913 	out->vt_hres_sec = hrestime.tv_sec;
4914 	out->vt_hres_ns = hrestime.tv_nsec;
4915 
4916 	return (0);
4917 }
4918 
4919 /*
4920  * Modify VMM Time data related values
4921  *
4922  * This interface serves to allow guests' TSC and device timers to continue
4923  * functioning across live migrations. On a successful write, the VM-wide TSC
4924  * offset and boot_hrtime of the guest are updated.
4925  *
4926  * The interface requires an hrtime of the system at which the caller wrote
4927  * this data; this allows us to adjust the TSC and boot_hrtime slightly to
4928  * account for time passing between the userspace call and application
4929  * of the data here.
4930  *
4931  * There are several possibilities for invalid input, including:
4932  * - a requested guest frequency of 0, or a frequency otherwise unsupported by
4933  *   the underlying platform
4934  * - hrtime or boot_hrtime values that appear to be from the future
4935  * - the requested frequency does not match the host, and this system does not
4936  *   have hardware TSC scaling support
4937  */
4938 static int
vmm_data_write_vmm_time(void * arg,const vmm_data_req_t * req)4939 vmm_data_write_vmm_time(void *arg, const vmm_data_req_t *req)
4940 {
4941 	VERIFY3U(req->vdr_class, ==, VDC_VMM_TIME);
4942 	VERIFY3U(req->vdr_version, ==, 1);
4943 	VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_time_info_v1));
4944 
4945 	struct vm *vm = arg;
4946 	const struct vdi_time_info_v1 *src = req->vdr_data;
4947 
4948 	/*
4949 	 * While vcpuid values != -1 are tolerated by the vmm_data machinery for
4950 	 * VM-wide endpoints, the time-related data is more strict: It relies on
4951 	 * write-locking the VM (implied by the vcpuid -1) to prevent vCPUs or
4952 	 * other bits from observing inconsistent values while the state is
4953 	 * being written.
4954 	 */
4955 	if (req->vdr_vcpuid != -1) {
4956 		return (EINVAL);
4957 	}
4958 
4959 	/*
4960 	 * Platform-specific checks will verify the requested frequency against
4961 	 * the supported range further, but a frequency of 0 is never valid.
4962 	 */
4963 	if (src->vt_guest_freq == 0) {
4964 		return (EINVAL);
4965 	}
4966 
4967 	/*
4968 	 * Check whether the request frequency is supported and get the
4969 	 * frequency multiplier.
4970 	 */
4971 	uint64_t mult = VM_TSCM_NOSCALE;
4972 	freqratio_res_t res = ops->vmfreqratio(src->vt_guest_freq,
4973 	    vmm_host_freq, &mult);
4974 	switch (res) {
4975 	case FR_SCALING_NOT_SUPPORTED:
4976 		/*
4977 		 * This system doesn't support TSC scaling, and the guest/host
4978 		 * frequencies differ
4979 		 */
4980 		return (EPERM);
4981 	case FR_OUT_OF_RANGE:
4982 		/* Requested frequency ratio is too small/large */
4983 		return (EINVAL);
4984 	case FR_SCALING_NOT_NEEDED:
4985 		/* Host and guest frequencies are the same */
4986 		VERIFY3U(mult, ==, VM_TSCM_NOSCALE);
4987 		break;
4988 	case FR_VALID:
4989 		VERIFY3U(mult, !=, VM_TSCM_NOSCALE);
4990 		break;
4991 	}
4992 
4993 	/*
4994 	 * Find (and validate) the hrtime delta between the input request and
4995 	 * when we received it so that we can bump the TSC to account for time
4996 	 * passing.
4997 	 *
4998 	 * We ignore the hrestime as input, as this is a field that
4999 	 * exists for reads.
5000 	 */
5001 	uint64_t tsc;
5002 	hrtime_t hrtime;
5003 	timespec_t hrestime;
5004 	vmm_time_snapshot(&tsc, &hrtime, &hrestime);
5005 	if ((src->vt_hrtime > hrtime) || (src->vt_boot_hrtime > hrtime)) {
5006 		/*
5007 		 * The caller has passed in an hrtime / boot_hrtime from the
5008 		 * future.
5009 		 */
5010 		return (EINVAL);
5011 	}
5012 	hrtime_t hrt_delta = hrtime - src->vt_hrtime;
5013 
5014 	/* Calculate guest TSC adjustment */
5015 	const uint64_t host_ticks = unscalehrtime(hrt_delta);
5016 	const uint64_t guest_ticks = vmm_scale_tsc(host_ticks,
5017 	    vm->freq_multiplier);
5018 	const uint64_t base_guest_tsc = src->vt_guest_tsc + guest_ticks;
5019 
5020 	/* Update guest time data */
5021 	vm->freq_multiplier = mult;
5022 	vm->guest_freq = src->vt_guest_freq;
5023 	vm->boot_hrtime = src->vt_boot_hrtime;
5024 	vm->tsc_offset = calc_tsc_offset(tsc, base_guest_tsc,
5025 	    vm->freq_multiplier);
5026 
5027 	return (0);
5028 }
5029 
5030 static const vmm_data_version_entry_t vmm_time_v1 = {
5031 	.vdve_class = VDC_VMM_TIME,
5032 	.vdve_version = 1,
5033 	.vdve_len_expect = sizeof (struct vdi_time_info_v1),
5034 	.vdve_readf = vmm_data_read_vmm_time,
5035 	.vdve_writef = vmm_data_write_vmm_time,
5036 };
5037 VMM_DATA_VERSION(vmm_time_v1);
5038 
5039 
5040 static int
vmm_data_read_versions(void * arg,const vmm_data_req_t * req)5041 vmm_data_read_versions(void *arg, const vmm_data_req_t *req)
5042 {
5043 	VERIFY3U(req->vdr_class, ==, VDC_VERSION);
5044 	VERIFY3U(req->vdr_version, ==, 1);
5045 
5046 	const uint32_t total_size = SET_COUNT(vmm_data_version_entries) *
5047 	    sizeof (struct vdi_version_entry_v1);
5048 
5049 	/* Make sure there is room for all of the entries */
5050 	*req->vdr_result_len = total_size;
5051 	if (req->vdr_len < *req->vdr_result_len) {
5052 		return (ENOSPC);
5053 	}
5054 
5055 	struct vdi_version_entry_v1 *entryp = req->vdr_data;
5056 	const vmm_data_version_entry_t **vdpp;
5057 	SET_FOREACH(vdpp, vmm_data_version_entries) {
5058 		const vmm_data_version_entry_t *vdp = *vdpp;
5059 
5060 		entryp->vve_class = vdp->vdve_class;
5061 		entryp->vve_version = vdp->vdve_version;
5062 		entryp->vve_len_expect = vdp->vdve_len_expect;
5063 		entryp->vve_len_per_item = vdp->vdve_len_per_item;
5064 		entryp++;
5065 	}
5066 	return (0);
5067 }
5068 
5069 static int
vmm_data_write_versions(void * arg,const vmm_data_req_t * req)5070 vmm_data_write_versions(void *arg, const vmm_data_req_t *req)
5071 {
5072 	/* Writing to the version information makes no sense */
5073 	return (EPERM);
5074 }
5075 
5076 static const vmm_data_version_entry_t versions_v1 = {
5077 	.vdve_class = VDC_VERSION,
5078 	.vdve_version = 1,
5079 	.vdve_len_per_item = sizeof (struct vdi_version_entry_v1),
5080 	.vdve_readf = vmm_data_read_versions,
5081 	.vdve_writef = vmm_data_write_versions,
5082 };
5083 VMM_DATA_VERSION(versions_v1);
5084 
5085 int
vmm_data_read(struct vm * vm,const vmm_data_req_t * req)5086 vmm_data_read(struct vm *vm, const vmm_data_req_t *req)
5087 {
5088 	int err = 0;
5089 
5090 	const vmm_data_version_entry_t *entry = NULL;
5091 	err = vmm_data_find(req, &entry);
5092 	if (err != 0) {
5093 		return (err);
5094 	}
5095 	ASSERT(entry != NULL);
5096 
5097 	if (entry->vdve_readf != NULL) {
5098 		void *datap = vmm_data_from_class(req, vm);
5099 
5100 		err = entry->vdve_readf(datap, req);
5101 	} else if (entry->vdve_vcpu_readf != NULL) {
5102 		err = entry->vdve_vcpu_readf(vm, req->vdr_vcpuid, req);
5103 	} else {
5104 		err = EINVAL;
5105 	}
5106 
5107 	/*
5108 	 * Successful reads of fixed-length data should populate the length of
5109 	 * that result.
5110 	 */
5111 	if (err == 0 && entry->vdve_len_expect != 0) {
5112 		*req->vdr_result_len = entry->vdve_len_expect;
5113 	}
5114 
5115 	return (err);
5116 }
5117 
5118 int
vmm_data_write(struct vm * vm,const vmm_data_req_t * req)5119 vmm_data_write(struct vm *vm, const vmm_data_req_t *req)
5120 {
5121 	int err = 0;
5122 
5123 	const vmm_data_version_entry_t *entry = NULL;
5124 	err = vmm_data_find(req, &entry);
5125 	if (err != 0) {
5126 		return (err);
5127 	}
5128 	ASSERT(entry != NULL);
5129 
5130 	if (entry->vdve_writef != NULL) {
5131 		void *datap = vmm_data_from_class(req, vm);
5132 
5133 		err = entry->vdve_writef(datap, req);
5134 	} else if (entry->vdve_vcpu_writef != NULL) {
5135 		err = entry->vdve_vcpu_writef(vm, req->vdr_vcpuid, req);
5136 	} else {
5137 		err = EINVAL;
5138 	}
5139 
5140 	/*
5141 	 * Successful writes of fixed-length data should populate the length of
5142 	 * that result.
5143 	 */
5144 	if (err == 0 && entry->vdve_len_expect != 0) {
5145 		*req->vdr_result_len = entry->vdve_len_expect;
5146 	}
5147 
5148 	return (err);
5149 }
5150