xref: /illumos-gate/usr/src/uts/i86pc/io/vmm/intel/vmx.c (revision d1c02647)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011 NetApp, Inc.
5  * All rights reserved.
6  * Copyright (c) 2018 Joyent, Inc.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 /*
32  * This file and its contents are supplied under the terms of the
33  * Common Development and Distribution License ("CDDL"), version 1.0.
34  * You may only use this file in accordance with the terms of version
35  * 1.0 of the CDDL.
36  *
37  * A full copy of the text of the CDDL should have accompanied this
38  * source.  A copy of the CDDL is also available via the Internet at
39  * http://www.illumos.org/license/CDDL.
40  *
41  * Copyright 2015 Pluribus Networks Inc.
42  * Copyright 2018 Joyent, Inc.
43  * Copyright 2021 Oxide Computer Company
44  */
45 
46 #include <sys/cdefs.h>
47 __FBSDID("$FreeBSD$");
48 
49 #include <sys/param.h>
50 #include <sys/systm.h>
51 #include <sys/kernel.h>
52 #include <sys/malloc.h>
53 #include <sys/pcpu.h>
54 #include <sys/proc.h>
55 #include <sys/sysctl.h>
56 
57 #include <sys/x86_archext.h>
58 #include <sys/smp_impldefs.h>
59 #include <sys/smt.h>
60 #include <sys/hma.h>
61 #include <sys/trap.h>
62 #include <sys/archsystm.h>
63 
64 #include <machine/psl.h>
65 #include <machine/cpufunc.h>
66 #include <machine/md_var.h>
67 #include <machine/reg.h>
68 #include <machine/segments.h>
69 #include <machine/specialreg.h>
70 #include <machine/vmparam.h>
71 #include <sys/vmm_vm.h>
72 
73 #include <machine/vmm.h>
74 #include <machine/vmm_dev.h>
75 #include <sys/vmm_instruction_emul.h>
76 #include "vmm_lapic.h"
77 #include "vmm_host.h"
78 #include "vmm_ioport.h"
79 #include "vmm_ktr.h"
80 #include "vmm_stat.h"
81 #include "vatpic.h"
82 #include "vlapic.h"
83 #include "vlapic_priv.h"
84 
85 #include "vmcs.h"
86 #include "vmx.h"
87 #include "vmx_msr.h"
88 #include "x86.h"
89 #include "vmx_controls.h"
90 
91 #define	PINBASED_CTLS_ONE_SETTING					\
92 	(PINBASED_EXTINT_EXITING	|				\
93 	PINBASED_NMI_EXITING		|				\
94 	PINBASED_VIRTUAL_NMI)
95 #define	PINBASED_CTLS_ZERO_SETTING	0
96 
97 #define	PROCBASED_CTLS_WINDOW_SETTING					\
98 	(PROCBASED_INT_WINDOW_EXITING	|				\
99 	PROCBASED_NMI_WINDOW_EXITING)
100 
101 /* We consider TSC offset a necessity for unsynched TSC handling */
102 #define	PROCBASED_CTLS_ONE_SETTING					\
103 	(PROCBASED_SECONDARY_CONTROLS	|				\
104 	PROCBASED_TSC_OFFSET		|				\
105 	PROCBASED_MWAIT_EXITING		|				\
106 	PROCBASED_MONITOR_EXITING	|				\
107 	PROCBASED_IO_EXITING		|				\
108 	PROCBASED_MSR_BITMAPS		|				\
109 	PROCBASED_CTLS_WINDOW_SETTING	|				\
110 	PROCBASED_CR8_LOAD_EXITING	|				\
111 	PROCBASED_CR8_STORE_EXITING)
112 
113 #define	PROCBASED_CTLS_ZERO_SETTING	\
114 	(PROCBASED_CR3_LOAD_EXITING |	\
115 	PROCBASED_CR3_STORE_EXITING |	\
116 	PROCBASED_IO_BITMAPS)
117 
118 /*
119  * EPT and Unrestricted Guest are considered necessities.  The latter is not a
120  * requirement on FreeBSD, where grub2-bhyve is used to load guests directly
121  * without a bootrom starting in real mode.
122  */
123 #define	PROCBASED_CTLS2_ONE_SETTING		\
124 	(PROCBASED2_ENABLE_EPT |		\
125 	PROCBASED2_UNRESTRICTED_GUEST)
126 #define	PROCBASED_CTLS2_ZERO_SETTING	0
127 
128 #define	VM_EXIT_CTLS_ONE_SETTING					\
129 	(VM_EXIT_SAVE_DEBUG_CONTROLS		|			\
130 	VM_EXIT_HOST_LMA			|			\
131 	VM_EXIT_LOAD_PAT			|			\
132 	VM_EXIT_SAVE_EFER			|			\
133 	VM_EXIT_LOAD_EFER			|			\
134 	VM_EXIT_ACKNOWLEDGE_INTERRUPT)
135 
136 #define	VM_EXIT_CTLS_ZERO_SETTING	0
137 
138 #define	VM_ENTRY_CTLS_ONE_SETTING					\
139 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
140 	VM_ENTRY_LOAD_EFER)
141 
142 #define	VM_ENTRY_CTLS_ZERO_SETTING					\
143 	(VM_ENTRY_INTO_SMM			|			\
144 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
145 
146 /*
147  * Cover the EPT capabilities used by bhyve at present:
148  * - 4-level page walks
149  * - write-back memory type
150  * - INVEPT operations (all types)
151  * - INVVPID operations (single-context only)
152  */
153 #define	EPT_CAPS_REQUIRED			\
154 	(IA32_VMX_EPT_VPID_PWL4 |		\
155 	IA32_VMX_EPT_VPID_TYPE_WB |		\
156 	IA32_VMX_EPT_VPID_INVEPT |		\
157 	IA32_VMX_EPT_VPID_INVEPT_SINGLE |	\
158 	IA32_VMX_EPT_VPID_INVEPT_ALL |		\
159 	IA32_VMX_EPT_VPID_INVVPID |		\
160 	IA32_VMX_EPT_VPID_INVVPID_SINGLE)
161 
162 #define	HANDLED		1
163 #define	UNHANDLED	0
164 
165 static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
166 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
167 
168 SYSCTL_DECL(_hw_vmm);
169 SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
170     NULL);
171 
172 static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
173 static uint32_t exit_ctls, entry_ctls;
174 
175 static uint64_t cr0_ones_mask, cr0_zeros_mask;
176 
177 static uint64_t cr4_ones_mask, cr4_zeros_mask;
178 
179 static int vmx_initialized;
180 
181 /* Do not flush RSB upon vmexit */
182 static int no_flush_rsb;
183 
184 /*
185  * Optional capabilities
186  */
187 
188 /* HLT triggers a VM-exit */
189 static int cap_halt_exit;
190 
191 /* PAUSE triggers a VM-exit */
192 static int cap_pause_exit;
193 
194 /* Monitor trap flag */
195 static int cap_monitor_trap;
196 
197 /* Guests are allowed to use INVPCID */
198 static int cap_invpcid;
199 
200 /* Extra capabilities (VMX_CAP_*) beyond the minimum */
201 static enum vmx_caps vmx_capabilities;
202 
203 /* APICv posted interrupt vector */
204 static int pirvec = -1;
205 
206 static uint_t vpid_alloc_failed;
207 
208 int guest_l1d_flush;
209 int guest_l1d_flush_sw;
210 
211 /* MSR save region is composed of an array of 'struct msr_entry' */
212 struct msr_entry {
213 	uint32_t	index;
214 	uint32_t	reserved;
215 	uint64_t	val;
216 };
217 
218 static struct msr_entry msr_load_list[1] __aligned(16);
219 
220 /*
221  * The definitions of SDT probes for VMX.
222  */
223 
224 /* BEGIN CSTYLED */
225 SDT_PROBE_DEFINE3(vmm, vmx, exit, entry,
226     "struct vmx *", "int", "struct vm_exit *");
227 
228 SDT_PROBE_DEFINE4(vmm, vmx, exit, taskswitch,
229     "struct vmx *", "int", "struct vm_exit *", "struct vm_task_switch *");
230 
231 SDT_PROBE_DEFINE4(vmm, vmx, exit, craccess,
232     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
233 
234 SDT_PROBE_DEFINE4(vmm, vmx, exit, rdmsr,
235     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
236 
237 SDT_PROBE_DEFINE5(vmm, vmx, exit, wrmsr,
238     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "uint64_t");
239 
240 SDT_PROBE_DEFINE3(vmm, vmx, exit, halt,
241     "struct vmx *", "int", "struct vm_exit *");
242 
243 SDT_PROBE_DEFINE3(vmm, vmx, exit, mtrap,
244     "struct vmx *", "int", "struct vm_exit *");
245 
246 SDT_PROBE_DEFINE3(vmm, vmx, exit, pause,
247     "struct vmx *", "int", "struct vm_exit *");
248 
249 SDT_PROBE_DEFINE3(vmm, vmx, exit, intrwindow,
250     "struct vmx *", "int", "struct vm_exit *");
251 
252 SDT_PROBE_DEFINE4(vmm, vmx, exit, interrupt,
253     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
254 
255 SDT_PROBE_DEFINE3(vmm, vmx, exit, nmiwindow,
256     "struct vmx *", "int", "struct vm_exit *");
257 
258 SDT_PROBE_DEFINE3(vmm, vmx, exit, inout,
259     "struct vmx *", "int", "struct vm_exit *");
260 
261 SDT_PROBE_DEFINE3(vmm, vmx, exit, cpuid,
262     "struct vmx *", "int", "struct vm_exit *");
263 
264 SDT_PROBE_DEFINE5(vmm, vmx, exit, exception,
265     "struct vmx *", "int", "struct vm_exit *", "uint32_t", "int");
266 
267 SDT_PROBE_DEFINE5(vmm, vmx, exit, nestedfault,
268     "struct vmx *", "int", "struct vm_exit *", "uint64_t", "uint64_t");
269 
270 SDT_PROBE_DEFINE4(vmm, vmx, exit, mmiofault,
271     "struct vmx *", "int", "struct vm_exit *", "uint64_t");
272 
273 SDT_PROBE_DEFINE3(vmm, vmx, exit, eoi,
274     "struct vmx *", "int", "struct vm_exit *");
275 
276 SDT_PROBE_DEFINE3(vmm, vmx, exit, apicaccess,
277     "struct vmx *", "int", "struct vm_exit *");
278 
279 SDT_PROBE_DEFINE4(vmm, vmx, exit, apicwrite,
280     "struct vmx *", "int", "struct vm_exit *", "struct vlapic *");
281 
282 SDT_PROBE_DEFINE3(vmm, vmx, exit, xsetbv,
283     "struct vmx *", "int", "struct vm_exit *");
284 
285 SDT_PROBE_DEFINE3(vmm, vmx, exit, monitor,
286     "struct vmx *", "int", "struct vm_exit *");
287 
288 SDT_PROBE_DEFINE3(vmm, vmx, exit, mwait,
289     "struct vmx *", "int", "struct vm_exit *");
290 
291 SDT_PROBE_DEFINE3(vmm, vmx, exit, vminsn,
292     "struct vmx *", "int", "struct vm_exit *");
293 
294 SDT_PROBE_DEFINE4(vmm, vmx, exit, unknown,
295     "struct vmx *", "int", "struct vm_exit *", "uint32_t");
296 
297 SDT_PROBE_DEFINE4(vmm, vmx, exit, return,
298     "struct vmx *", "int", "struct vm_exit *", "int");
299 /* END CSTYLED */
300 
301 static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc);
302 static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval);
303 static void vmx_apply_tsc_adjust(struct vmx *, int);
304 static void vmx_apicv_sync_tmr(struct vlapic *vlapic);
305 static void vmx_tpr_shadow_enter(struct vlapic *vlapic);
306 static void vmx_tpr_shadow_exit(struct vlapic *vlapic);
307 
308 static void
vmx_allow_x2apic_msrs(struct vmx * vmx,int vcpuid)309 vmx_allow_x2apic_msrs(struct vmx *vmx, int vcpuid)
310 {
311 	/*
312 	 * Allow readonly access to the following x2APIC MSRs from the guest.
313 	 */
314 	guest_msr_ro(vmx, vcpuid, MSR_APIC_ID);
315 	guest_msr_ro(vmx, vcpuid, MSR_APIC_VERSION);
316 	guest_msr_ro(vmx, vcpuid, MSR_APIC_LDR);
317 	guest_msr_ro(vmx, vcpuid, MSR_APIC_SVR);
318 
319 	for (uint_t i = 0; i < 8; i++) {
320 		guest_msr_ro(vmx, vcpuid, MSR_APIC_ISR0 + i);
321 		guest_msr_ro(vmx, vcpuid, MSR_APIC_TMR0 + i);
322 		guest_msr_ro(vmx, vcpuid, MSR_APIC_IRR0 + i);
323 	}
324 
325 	guest_msr_ro(vmx, vcpuid, MSR_APIC_ESR);
326 	guest_msr_ro(vmx, vcpuid, MSR_APIC_LVT_TIMER);
327 	guest_msr_ro(vmx, vcpuid, MSR_APIC_LVT_THERMAL);
328 	guest_msr_ro(vmx, vcpuid, MSR_APIC_LVT_PCINT);
329 	guest_msr_ro(vmx, vcpuid, MSR_APIC_LVT_LINT0);
330 	guest_msr_ro(vmx, vcpuid, MSR_APIC_LVT_LINT1);
331 	guest_msr_ro(vmx, vcpuid, MSR_APIC_LVT_ERROR);
332 	guest_msr_ro(vmx, vcpuid, MSR_APIC_ICR_TIMER);
333 	guest_msr_ro(vmx, vcpuid, MSR_APIC_DCR_TIMER);
334 	guest_msr_ro(vmx, vcpuid, MSR_APIC_ICR);
335 
336 	/*
337 	 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
338 	 *
339 	 * These registers get special treatment described in the section
340 	 * "Virtualizing MSR-Based APIC Accesses".
341 	 */
342 	guest_msr_rw(vmx, vcpuid, MSR_APIC_TPR);
343 	guest_msr_rw(vmx, vcpuid, MSR_APIC_EOI);
344 	guest_msr_rw(vmx, vcpuid, MSR_APIC_SELF_IPI);
345 }
346 
347 static ulong_t
vmx_fix_cr0(ulong_t cr0)348 vmx_fix_cr0(ulong_t cr0)
349 {
350 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
351 }
352 
353 /*
354  * Given a live (VMCS-active) cr0 value, and its shadow counterpart, calculate
355  * the value observable from the guest.
356  */
357 static ulong_t
vmx_unshadow_cr0(uint64_t cr0,uint64_t shadow)358 vmx_unshadow_cr0(uint64_t cr0, uint64_t shadow)
359 {
360 	return ((cr0 & ~cr0_ones_mask) |
361 	    (shadow & (cr0_zeros_mask | cr0_ones_mask)));
362 }
363 
364 static ulong_t
vmx_fix_cr4(ulong_t cr4)365 vmx_fix_cr4(ulong_t cr4)
366 {
367 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
368 }
369 
370 /*
371  * Given a live (VMCS-active) cr4 value, and its shadow counterpart, calculate
372  * the value observable from the guest.
373  */
374 static ulong_t
vmx_unshadow_cr4(uint64_t cr4,uint64_t shadow)375 vmx_unshadow_cr4(uint64_t cr4, uint64_t shadow)
376 {
377 	return ((cr4 & ~cr4_ones_mask) |
378 	    (shadow & (cr4_zeros_mask | cr4_ones_mask)));
379 }
380 
381 static void
vpid_free(int vpid)382 vpid_free(int vpid)
383 {
384 	if (vpid < 0 || vpid > 0xffff)
385 		panic("vpid_free: invalid vpid %d", vpid);
386 
387 	/*
388 	 * VPIDs [0,VM_MAXCPU] are special and are not allocated from
389 	 * the unit number allocator.
390 	 */
391 
392 	if (vpid > VM_MAXCPU)
393 		hma_vmx_vpid_free((uint16_t)vpid);
394 }
395 
396 static void
vpid_alloc(uint16_t * vpid,int num)397 vpid_alloc(uint16_t *vpid, int num)
398 {
399 	int i, x;
400 
401 	if (num <= 0 || num > VM_MAXCPU)
402 		panic("invalid number of vpids requested: %d", num);
403 
404 	/*
405 	 * If the "enable vpid" execution control is not enabled then the
406 	 * VPID is required to be 0 for all vcpus.
407 	 */
408 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
409 		for (i = 0; i < num; i++)
410 			vpid[i] = 0;
411 		return;
412 	}
413 
414 	/*
415 	 * Allocate a unique VPID for each vcpu from the unit number allocator.
416 	 */
417 	for (i = 0; i < num; i++) {
418 		uint16_t tmp;
419 
420 		tmp = hma_vmx_vpid_alloc();
421 		x = (tmp == 0) ? -1 : tmp;
422 
423 		if (x == -1)
424 			break;
425 		else
426 			vpid[i] = x;
427 	}
428 
429 	if (i < num) {
430 		atomic_add_int(&vpid_alloc_failed, 1);
431 
432 		/*
433 		 * If the unit number allocator does not have enough unique
434 		 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
435 		 *
436 		 * These VPIDs are not be unique across VMs but this does not
437 		 * affect correctness because the combined mappings are also
438 		 * tagged with the EP4TA which is unique for each VM.
439 		 *
440 		 * It is still sub-optimal because the invvpid will invalidate
441 		 * combined mappings for a particular VPID across all EP4TAs.
442 		 */
443 		while (i-- > 0)
444 			vpid_free(vpid[i]);
445 
446 		for (i = 0; i < num; i++)
447 			vpid[i] = i + 1;
448 	}
449 }
450 
451 static int
vmx_cleanup(void)452 vmx_cleanup(void)
453 {
454 	/* This is taken care of by the hma registration */
455 	return (0);
456 }
457 
458 static void
vmx_restore(void)459 vmx_restore(void)
460 {
461 	/* No-op on illumos */
462 }
463 
464 static int
vmx_init(void)465 vmx_init(void)
466 {
467 	int error;
468 	uint64_t fixed0, fixed1;
469 	uint32_t tmp;
470 	enum vmx_caps avail_caps = VMX_CAP_NONE;
471 
472 	/* Check support for primary processor-based VM-execution controls */
473 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
474 	    MSR_VMX_TRUE_PROCBASED_CTLS,
475 	    PROCBASED_CTLS_ONE_SETTING,
476 	    PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
477 	if (error) {
478 		printf("vmx_init: processor does not support desired primary "
479 		    "processor-based controls\n");
480 		return (error);
481 	}
482 
483 	/* Clear the processor-based ctl bits that are set on demand */
484 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
485 
486 	/* Check support for secondary processor-based VM-execution controls */
487 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
488 	    MSR_VMX_PROCBASED_CTLS2,
489 	    PROCBASED_CTLS2_ONE_SETTING,
490 	    PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
491 	if (error) {
492 		printf("vmx_init: processor does not support desired secondary "
493 		    "processor-based controls\n");
494 		return (error);
495 	}
496 
497 	/* Check support for VPID */
498 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
499 	    MSR_VMX_PROCBASED_CTLS2,
500 	    PROCBASED2_ENABLE_VPID,
501 	    0, &tmp);
502 	if (error == 0)
503 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
504 
505 	/* Check support for pin-based VM-execution controls */
506 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
507 	    MSR_VMX_TRUE_PINBASED_CTLS,
508 	    PINBASED_CTLS_ONE_SETTING,
509 	    PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
510 	if (error) {
511 		printf("vmx_init: processor does not support desired "
512 		    "pin-based controls\n");
513 		return (error);
514 	}
515 
516 	/* Check support for VM-exit controls */
517 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
518 	    VM_EXIT_CTLS_ONE_SETTING,
519 	    VM_EXIT_CTLS_ZERO_SETTING,
520 	    &exit_ctls);
521 	if (error) {
522 		printf("vmx_init: processor does not support desired "
523 		    "exit controls\n");
524 		return (error);
525 	}
526 
527 	/* Check support for VM-entry controls */
528 	error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS,
529 	    VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING,
530 	    &entry_ctls);
531 	if (error) {
532 		printf("vmx_init: processor does not support desired "
533 		    "entry controls\n");
534 		return (error);
535 	}
536 
537 	/*
538 	 * Check support for optional features by testing them
539 	 * as individual bits
540 	 */
541 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
542 	    MSR_VMX_TRUE_PROCBASED_CTLS,
543 	    PROCBASED_HLT_EXITING, 0,
544 	    &tmp) == 0);
545 
546 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
547 	    MSR_VMX_PROCBASED_CTLS,
548 	    PROCBASED_MTF, 0,
549 	    &tmp) == 0);
550 
551 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
552 	    MSR_VMX_TRUE_PROCBASED_CTLS,
553 	    PROCBASED_PAUSE_EXITING, 0,
554 	    &tmp) == 0);
555 
556 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
557 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
558 	    &tmp) == 0);
559 
560 	/*
561 	 * Check for APIC virtualization capabilities:
562 	 * - TPR shadowing
563 	 * - Full APICv (with or without x2APIC support)
564 	 * - Posted interrupt handling
565 	 */
566 	if (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, MSR_VMX_TRUE_PROCBASED_CTLS,
567 	    PROCBASED_USE_TPR_SHADOW, 0, &tmp) == 0) {
568 		avail_caps |= VMX_CAP_TPR_SHADOW;
569 
570 		const uint32_t apicv_bits =
571 		    PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
572 		    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
573 		    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
574 		    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY;
575 		if (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
576 		    MSR_VMX_PROCBASED_CTLS2, apicv_bits, 0, &tmp) == 0) {
577 			avail_caps |= VMX_CAP_APICV;
578 
579 			/*
580 			 * It may make sense in the future to differentiate
581 			 * hardware (or software) configurations with APICv but
582 			 * no support for accelerating x2APIC mode.
583 			 */
584 			avail_caps |= VMX_CAP_APICV_X2APIC;
585 
586 			error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
587 			    MSR_VMX_TRUE_PINBASED_CTLS,
588 			    PINBASED_POSTED_INTERRUPT, 0, &tmp);
589 			if (error == 0) {
590 				/*
591 				 * If the PSM-provided interfaces for requesting
592 				 * and using a PIR IPI vector are present, use
593 				 * them for posted interrupts.
594 				 */
595 				if (psm_get_pir_ipivect != NULL &&
596 				    psm_send_pir_ipi != NULL) {
597 					pirvec = psm_get_pir_ipivect();
598 					avail_caps |= VMX_CAP_APICV_PIR;
599 				}
600 			}
601 		}
602 	}
603 
604 	/*
605 	 * Check for necessary EPT capabilities
606 	 *
607 	 * TODO: Properly handle when IA32_VMX_EPT_VPID_HW_AD is missing and the
608 	 * hypervisor intends to utilize dirty page tracking.
609 	 */
610 	uint64_t ept_caps = rdmsr(MSR_IA32_VMX_EPT_VPID_CAP);
611 	if ((ept_caps & EPT_CAPS_REQUIRED) != EPT_CAPS_REQUIRED) {
612 		cmn_err(CE_WARN, "!Inadequate EPT capabilities: %lx", ept_caps);
613 		return (EINVAL);
614 	}
615 
616 #ifdef __FreeBSD__
617 	guest_l1d_flush = (cpu_ia32_arch_caps &
618 	    IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) == 0;
619 	TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush);
620 
621 	/*
622 	 * L1D cache flush is enabled.  Use IA32_FLUSH_CMD MSR when
623 	 * available.  Otherwise fall back to the software flush
624 	 * method which loads enough data from the kernel text to
625 	 * flush existing L1D content, both on VMX entry and on NMI
626 	 * return.
627 	 */
628 	if (guest_l1d_flush) {
629 		if ((cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) == 0) {
630 			guest_l1d_flush_sw = 1;
631 			TUNABLE_INT_FETCH("hw.vmm.l1d_flush_sw",
632 			    &guest_l1d_flush_sw);
633 		}
634 		if (guest_l1d_flush_sw) {
635 			if (nmi_flush_l1d_sw <= 1)
636 				nmi_flush_l1d_sw = 1;
637 		} else {
638 			msr_load_list[0].index = MSR_IA32_FLUSH_CMD;
639 			msr_load_list[0].val = IA32_FLUSH_CMD_L1D;
640 		}
641 	}
642 #else
643 	/* L1D flushing is taken care of by smt_acquire() and friends */
644 	guest_l1d_flush = 0;
645 #endif /* __FreeBSD__ */
646 
647 	/*
648 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
649 	 */
650 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
651 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
652 	cr0_ones_mask = fixed0 & fixed1;
653 	cr0_zeros_mask = ~fixed0 & ~fixed1;
654 
655 	/*
656 	 * Since Unrestricted Guest was already verified present, CR0_PE and
657 	 * CR0_PG are allowed to be set to zero in VMX non-root operation
658 	 */
659 	cr0_ones_mask &= ~(CR0_PG | CR0_PE);
660 
661 	/*
662 	 * Do not allow the guest to set CR0_NW or CR0_CD.
663 	 */
664 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
665 
666 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
667 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
668 	cr4_ones_mask = fixed0 & fixed1;
669 	cr4_zeros_mask = ~fixed0 & ~fixed1;
670 
671 	vmx_msr_init();
672 
673 	vmx_capabilities = avail_caps;
674 	vmx_initialized = 1;
675 
676 	return (0);
677 }
678 
679 static void
vmx_trigger_hostintr(int vector)680 vmx_trigger_hostintr(int vector)
681 {
682 	VERIFY(vector >= 32 && vector <= 255);
683 	vmx_call_isr(vector - 32);
684 }
685 
686 static void *
vmx_vminit(struct vm * vm)687 vmx_vminit(struct vm *vm)
688 {
689 	uint16_t vpid[VM_MAXCPU];
690 	int i, error, datasel;
691 	struct vmx *vmx;
692 	uint32_t exc_bitmap;
693 	uint16_t maxcpus;
694 	uint32_t proc_ctls, proc2_ctls, pin_ctls;
695 	uint64_t apic_access_pa = UINT64_MAX;
696 
697 	vmx = malloc(sizeof (struct vmx), M_VMX, M_WAITOK | M_ZERO);
698 	if ((uintptr_t)vmx & PAGE_MASK) {
699 		panic("malloc of struct vmx not aligned on %d byte boundary",
700 		    PAGE_SIZE);
701 	}
702 	vmx->vm = vm;
703 
704 	vmx->eptp = vmspace_table_root(vm_get_vmspace(vm));
705 
706 	/*
707 	 * Clean up EP4TA-tagged guest-physical and combined mappings
708 	 *
709 	 * VMX transitions are not required to invalidate any guest physical
710 	 * mappings. So, it may be possible for stale guest physical mappings
711 	 * to be present in the processor TLBs.
712 	 *
713 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
714 	 */
715 	hma_vmx_invept_allcpus((uintptr_t)vmx->eptp);
716 
717 	vmx_msr_bitmap_initialize(vmx);
718 
719 	vpid_alloc(vpid, VM_MAXCPU);
720 
721 	/* Grab the established defaults */
722 	proc_ctls = procbased_ctls;
723 	proc2_ctls = procbased_ctls2;
724 	pin_ctls = pinbased_ctls;
725 	/* For now, default to the available capabilities */
726 	vmx->vmx_caps = vmx_capabilities;
727 
728 	if (vmx_cap_en(vmx, VMX_CAP_TPR_SHADOW)) {
729 		proc_ctls |= PROCBASED_USE_TPR_SHADOW;
730 		proc_ctls &= ~PROCBASED_CR8_LOAD_EXITING;
731 		proc_ctls &= ~PROCBASED_CR8_STORE_EXITING;
732 	}
733 	if (vmx_cap_en(vmx, VMX_CAP_APICV)) {
734 		ASSERT(vmx_cap_en(vmx, VMX_CAP_TPR_SHADOW));
735 
736 		proc2_ctls |= (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
737 		    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
738 		    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
739 
740 		/*
741 		 * Allocate a page of memory to back the APIC access address for
742 		 * when APICv features are in use.  Guest MMIO accesses should
743 		 * never actually reach this page, but rather be intercepted.
744 		 */
745 		vmx->apic_access_page = kmem_zalloc(PAGESIZE, KM_SLEEP);
746 		VERIFY3U((uintptr_t)vmx->apic_access_page & PAGEOFFSET, ==, 0);
747 		apic_access_pa = vtophys(vmx->apic_access_page);
748 
749 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
750 		    apic_access_pa);
751 		/* XXX this should really return an error to the caller */
752 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
753 	}
754 	if (vmx_cap_en(vmx, VMX_CAP_APICV_PIR)) {
755 		ASSERT(vmx_cap_en(vmx, VMX_CAP_APICV));
756 
757 		pin_ctls |= PINBASED_POSTED_INTERRUPT;
758 	}
759 
760 	maxcpus = vm_get_maxcpus(vm);
761 	datasel = vmm_get_host_datasel();
762 	for (i = 0; i < maxcpus; i++) {
763 		/*
764 		 * Cache physical address lookups for various components which
765 		 * may be required inside the critical_enter() section implied
766 		 * by VMPTRLD() below.
767 		 */
768 		vm_paddr_t msr_bitmap_pa = vtophys(vmx->msr_bitmap[i]);
769 		vm_paddr_t apic_page_pa = vtophys(&vmx->apic_page[i]);
770 		vm_paddr_t pir_desc_pa = vtophys(&vmx->pir_desc[i]);
771 
772 		vmx->vmcs_pa[i] = (uintptr_t)vtophys(&vmx->vmcs[i]);
773 		vmcs_initialize(&vmx->vmcs[i], vmx->vmcs_pa[i]);
774 
775 		vmx_msr_guest_init(vmx, i);
776 
777 		vmcs_load(vmx->vmcs_pa[i]);
778 
779 		vmcs_write(VMCS_HOST_IA32_PAT, vmm_get_host_pat());
780 		vmcs_write(VMCS_HOST_IA32_EFER, vmm_get_host_efer());
781 
782 		/* Load the control registers */
783 		vmcs_write(VMCS_HOST_CR0, vmm_get_host_cr0());
784 		vmcs_write(VMCS_HOST_CR4, vmm_get_host_cr4() | CR4_VMXE);
785 
786 		/* Load the segment selectors */
787 		vmcs_write(VMCS_HOST_CS_SELECTOR, vmm_get_host_codesel());
788 
789 		vmcs_write(VMCS_HOST_ES_SELECTOR, datasel);
790 		vmcs_write(VMCS_HOST_SS_SELECTOR, datasel);
791 		vmcs_write(VMCS_HOST_DS_SELECTOR, datasel);
792 
793 		vmcs_write(VMCS_HOST_FS_SELECTOR, vmm_get_host_fssel());
794 		vmcs_write(VMCS_HOST_GS_SELECTOR, vmm_get_host_gssel());
795 		vmcs_write(VMCS_HOST_TR_SELECTOR, vmm_get_host_tsssel());
796 
797 		/*
798 		 * Configure host sysenter MSRs to be restored on VM exit.
799 		 * The thread-specific MSR_INTC_SEP_ESP value is loaded in
800 		 * vmx_run.
801 		 */
802 		vmcs_write(VMCS_HOST_IA32_SYSENTER_CS, KCS_SEL);
803 		vmcs_write(VMCS_HOST_IA32_SYSENTER_EIP,
804 		    rdmsr(MSR_SYSENTER_EIP_MSR));
805 
806 		/* instruction pointer */
807 		if (no_flush_rsb) {
808 			vmcs_write(VMCS_HOST_RIP, (uint64_t)vmx_exit_guest);
809 		} else {
810 			vmcs_write(VMCS_HOST_RIP,
811 			    (uint64_t)vmx_exit_guest_flush_rsb);
812 		}
813 
814 		/* link pointer */
815 		vmcs_write(VMCS_LINK_POINTER, ~0);
816 
817 		vmcs_write(VMCS_EPTP, vmx->eptp);
818 		vmcs_write(VMCS_PIN_BASED_CTLS, pin_ctls);
819 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc_ctls);
820 		vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc2_ctls);
821 		vmcs_write(VMCS_EXIT_CTLS, exit_ctls);
822 		vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
823 		vmcs_write(VMCS_MSR_BITMAP, msr_bitmap_pa);
824 		vmcs_write(VMCS_VPID, vpid[i]);
825 
826 		if (guest_l1d_flush && !guest_l1d_flush_sw) {
827 			vmcs_write(VMCS_ENTRY_MSR_LOAD,
828 			    vtophys(&msr_load_list[0]));
829 			vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT,
830 			    nitems(msr_load_list));
831 			vmcs_write(VMCS_EXIT_MSR_STORE, 0);
832 			vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0);
833 		}
834 
835 		/* exception bitmap */
836 		if (vcpu_trace_exceptions(vm, i))
837 			exc_bitmap = 0xffffffff;
838 		else
839 			exc_bitmap = 1 << IDT_MC;
840 		vmcs_write(VMCS_EXCEPTION_BITMAP, exc_bitmap);
841 
842 		vmx->ctx[i].guest_dr6 = DBREG_DR6_RESERVED1;
843 		vmcs_write(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1);
844 
845 		if (vmx_cap_en(vmx, VMX_CAP_TPR_SHADOW)) {
846 			vmcs_write(VMCS_VIRTUAL_APIC, apic_page_pa);
847 		}
848 
849 		if (vmx_cap_en(vmx, VMX_CAP_APICV)) {
850 			vmcs_write(VMCS_APIC_ACCESS, apic_access_pa);
851 			vmcs_write(VMCS_EOI_EXIT0, 0);
852 			vmcs_write(VMCS_EOI_EXIT1, 0);
853 			vmcs_write(VMCS_EOI_EXIT2, 0);
854 			vmcs_write(VMCS_EOI_EXIT3, 0);
855 		}
856 		if (vmx_cap_en(vmx, VMX_CAP_APICV_PIR)) {
857 			vmcs_write(VMCS_PIR_VECTOR, pirvec);
858 			vmcs_write(VMCS_PIR_DESC, pir_desc_pa);
859 		}
860 
861 		/*
862 		 * Set up the CR0/4 masks and configure the read shadow state
863 		 * to the power-on register value from the Intel Sys Arch.
864 		 *  CR0 - 0x60000010
865 		 *  CR4 - 0
866 		 */
867 		vmcs_write(VMCS_CR0_MASK, cr0_ones_mask | cr0_zeros_mask);
868 		vmcs_write(VMCS_CR0_SHADOW, 0x60000010);
869 		vmcs_write(VMCS_CR4_MASK, cr4_ones_mask | cr4_zeros_mask);
870 		vmcs_write(VMCS_CR4_SHADOW, 0);
871 
872 		vmcs_clear(vmx->vmcs_pa[i]);
873 
874 		vmx->cap[i].set = 0;
875 		vmx->cap[i].proc_ctls = proc_ctls;
876 		vmx->cap[i].proc_ctls2 = proc2_ctls;
877 		vmx->cap[i].exc_bitmap = exc_bitmap;
878 
879 		vmx->state[i].nextrip = ~0;
880 		vmx->state[i].lastcpu = NOCPU;
881 		vmx->state[i].vpid = vpid[i];
882 	}
883 
884 	return (vmx);
885 }
886 
887 static int
vmx_handle_cpuid(struct vm * vm,int vcpu,struct vmxctx * vmxctx)888 vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
889 {
890 	int handled;
891 
892 	handled = x86_emulate_cpuid(vm, vcpu, (uint64_t *)&vmxctx->guest_rax,
893 	    (uint64_t *)&vmxctx->guest_rbx, (uint64_t *)&vmxctx->guest_rcx,
894 	    (uint64_t *)&vmxctx->guest_rdx);
895 	return (handled);
896 }
897 
898 static __inline void
vmx_run_trace(struct vmx * vmx,int vcpu)899 vmx_run_trace(struct vmx *vmx, int vcpu)
900 {
901 #ifdef KTR
902 	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %lx", vmcs_guest_rip());
903 #endif
904 }
905 
906 static __inline void
vmx_astpending_trace(struct vmx * vmx,int vcpu,uint64_t rip)907 vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
908 {
909 #ifdef KTR
910 	VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
911 #endif
912 }
913 
914 static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
915 static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done");
916 
917 #define	INVVPID_TYPE_ADDRESS		0UL
918 #define	INVVPID_TYPE_SINGLE_CONTEXT	1UL
919 #define	INVVPID_TYPE_ALL_CONTEXTS	2UL
920 
921 struct invvpid_desc {
922 	uint16_t	vpid;
923 	uint16_t	_res1;
924 	uint32_t	_res2;
925 	uint64_t	linear_addr;
926 };
927 CTASSERT(sizeof (struct invvpid_desc) == 16);
928 
929 static __inline void
invvpid(uint64_t type,struct invvpid_desc desc)930 invvpid(uint64_t type, struct invvpid_desc desc)
931 {
932 	int error;
933 
934 	__asm __volatile("invvpid %[desc], %[type];"
935 	    VMX_SET_ERROR_CODE_ASM
936 	    : [error] "=r" (error)
937 	    : [desc] "m" (desc), [type] "r" (type)
938 	    : "memory");
939 
940 	if (error)
941 		panic("invvpid error %d", error);
942 }
943 
944 /*
945  * Invalidate guest mappings identified by its VPID from the TLB.
946  *
947  * This is effectively a flush of the guest TLB, removing only "combined
948  * mappings" (to use the VMX parlance).  Actions which modify the EPT structures
949  * for the instance (such as unmapping GPAs) would require an 'invept' flush.
950  */
951 static __inline void
vmx_invvpid(struct vmx * vmx,int vcpu,int running)952 vmx_invvpid(struct vmx *vmx, int vcpu, int running)
953 {
954 	struct vmxstate *vmxstate;
955 	struct invvpid_desc invvpid_desc;
956 	struct vmspace *vms;
957 
958 	vmxstate = &vmx->state[vcpu];
959 	if (vmxstate->vpid == 0)
960 		return;
961 	vms = vm_get_vmspace(vmx->vm);
962 
963 	if (!running) {
964 		/*
965 		 * Set the 'lastcpu' to an invalid host cpu.
966 		 *
967 		 * This will invalidate TLB entries tagged with the vcpu's
968 		 * vpid the next time it runs via vmx_set_pcpu_defaults().
969 		 */
970 		vmxstate->lastcpu = NOCPU;
971 		return;
972 	}
973 
974 	/*
975 	 * Invalidate all mappings tagged with 'vpid'
976 	 *
977 	 * This is done when a vCPU moves between host CPUs, where there may be
978 	 * stale TLB entries for this VPID on the target, or if emulated actions
979 	 * in the guest CPU have incurred an explicit TLB flush.
980 	 */
981 	if (vmspace_table_gen(vms) == vmx->eptgen[curcpu]) {
982 		invvpid_desc._res1 = 0;
983 		invvpid_desc._res2 = 0;
984 		invvpid_desc.vpid = vmxstate->vpid;
985 		invvpid_desc.linear_addr = 0;
986 		invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
987 		vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1);
988 	} else {
989 		/*
990 		 * The INVVPID can be skipped if an INVEPT is going to be
991 		 * performed before entering the guest.  The INVEPT will
992 		 * invalidate combined mappings for the EP4TA associated with
993 		 * this guest, in all VPIDs.
994 		 */
995 		vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
996 	}
997 }
998 
999 static __inline void
invept(uint64_t type,uint64_t eptp)1000 invept(uint64_t type, uint64_t eptp)
1001 {
1002 	int error;
1003 	struct invept_desc {
1004 		uint64_t eptp;
1005 		uint64_t _resv;
1006 	} desc = { eptp, 0 };
1007 
1008 	__asm __volatile("invept %[desc], %[type];"
1009 	    VMX_SET_ERROR_CODE_ASM
1010 	    : [error] "=r" (error)
1011 	    : [desc] "m" (desc), [type] "r" (type)
1012 	    : "memory");
1013 
1014 	if (error != 0) {
1015 		panic("invvpid error %d", error);
1016 	}
1017 }
1018 
1019 static void
vmx_set_pcpu_defaults(struct vmx * vmx,int vcpu)1020 vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu)
1021 {
1022 	struct vmxstate *vmxstate;
1023 
1024 	/*
1025 	 * Regardless of whether the VM appears to have migrated between CPUs,
1026 	 * save the host sysenter stack pointer.  As it points to the kernel
1027 	 * stack of each thread, the correct value must be maintained for every
1028 	 * trip into the critical section.
1029 	 */
1030 	vmcs_write(VMCS_HOST_IA32_SYSENTER_ESP, rdmsr(MSR_SYSENTER_ESP_MSR));
1031 
1032 	/*
1033 	 * Perform any needed TSC_OFFSET adjustment based on TSC_MSR writes or
1034 	 * migration between host CPUs with differing TSC values.
1035 	 */
1036 	vmx_apply_tsc_adjust(vmx, vcpu);
1037 
1038 	vmxstate = &vmx->state[vcpu];
1039 	if (vmxstate->lastcpu == curcpu)
1040 		return;
1041 
1042 	vmxstate->lastcpu = curcpu;
1043 
1044 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
1045 
1046 	/* Load the per-CPU IDT address */
1047 	vmcs_write(VMCS_HOST_IDTR_BASE, vmm_get_host_idtrbase());
1048 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
1049 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
1050 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
1051 	vmx_invvpid(vmx, vcpu, 1);
1052 }
1053 
1054 /*
1055  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1056  */
1057 CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1058 
1059 static __inline void
vmx_set_int_window_exiting(struct vmx * vmx,int vcpu)1060 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1061 {
1062 
1063 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1064 		vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
1065 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1066 		VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
1067 	}
1068 }
1069 
1070 static __inline void
vmx_clear_int_window_exiting(struct vmx * vmx,int vcpu)1071 vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1072 {
1073 
1074 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
1075 	    ("intr_window_exiting not set: %x", vmx->cap[vcpu].proc_ctls));
1076 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
1077 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1078 	VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1079 }
1080 
1081 static __inline bool
vmx_nmi_window_exiting(struct vmx * vmx,int vcpu)1082 vmx_nmi_window_exiting(struct vmx *vmx, int vcpu)
1083 {
1084 	return ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0);
1085 }
1086 
1087 static __inline void
vmx_set_nmi_window_exiting(struct vmx * vmx,int vcpu)1088 vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1089 {
1090 	if (!vmx_nmi_window_exiting(vmx, vcpu)) {
1091 		vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
1092 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1093 	}
1094 }
1095 
1096 static __inline void
vmx_clear_nmi_window_exiting(struct vmx * vmx,int vcpu)1097 vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1098 {
1099 	ASSERT(vmx_nmi_window_exiting(vmx, vcpu));
1100 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
1101 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1102 }
1103 
1104 /*
1105  * Set the TSC adjustment, taking into account the offsets measured between
1106  * host physical CPUs.  This is required even if the guest has not set a TSC
1107  * offset since vCPUs inherit the TSC offset of whatever physical CPU it has
1108  * migrated onto.  Without this mitigation, un-synched host TSCs will convey
1109  * the appearance of TSC time-travel to the guest as its vCPUs migrate.
1110  */
1111 static void
vmx_apply_tsc_adjust(struct vmx * vmx,int vcpu)1112 vmx_apply_tsc_adjust(struct vmx *vmx, int vcpu)
1113 {
1114 	const uint64_t offset = vcpu_tsc_offset(vmx->vm, vcpu, true);
1115 
1116 	ASSERT(vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET);
1117 
1118 	if (vmx->tsc_offset_active[vcpu] != offset) {
1119 		vmcs_write(VMCS_TSC_OFFSET, offset);
1120 		vmx->tsc_offset_active[vcpu] = offset;
1121 	}
1122 }
1123 
1124 #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
1125 			VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1126 #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
1127 			VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
1128 
1129 static void
vmx_inject_nmi(struct vmx * vmx,int vcpu)1130 vmx_inject_nmi(struct vmx *vmx, int vcpu)
1131 {
1132 	ASSERT0(vmcs_read(VMCS_GUEST_INTERRUPTIBILITY) & NMI_BLOCKING);
1133 	ASSERT0(vmcs_read(VMCS_ENTRY_INTR_INFO) & VMCS_INTR_VALID);
1134 
1135 	/*
1136 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1137 	 * or the VMCS entry check will fail.
1138 	 */
1139 	vmcs_write(VMCS_ENTRY_INTR_INFO,
1140 	    IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID);
1141 
1142 	/* Clear the request */
1143 	vm_nmi_clear(vmx->vm, vcpu);
1144 }
1145 
1146 /*
1147  * Inject exceptions, NMIs, and ExtINTs.
1148  *
1149  * The logic behind these are complicated and may involve mutex contention, so
1150  * the injection is performed without the protection of host CPU interrupts
1151  * being disabled.  This means a racing notification could be "lost",
1152  * necessitating a later call to vmx_inject_recheck() to close that window
1153  * of opportunity.
1154  */
1155 static enum event_inject_state
vmx_inject_events(struct vmx * vmx,int vcpu,uint64_t rip)1156 vmx_inject_events(struct vmx *vmx, int vcpu, uint64_t rip)
1157 {
1158 	uint64_t entryinfo;
1159 	uint32_t gi, info;
1160 	int vector;
1161 	enum event_inject_state state;
1162 
1163 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1164 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1165 	state = EIS_CAN_INJECT;
1166 
1167 	/* Clear any interrupt blocking if the guest %rip has changed */
1168 	if (vmx->state[vcpu].nextrip != rip && (gi & HWINTR_BLOCKING) != 0) {
1169 		gi &= ~HWINTR_BLOCKING;
1170 		vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1171 	}
1172 
1173 	/*
1174 	 * It could be that an interrupt is already pending for injection from
1175 	 * the VMCS.  This would be the case if the vCPU exited for conditions
1176 	 * such as an AST before a vm-entry delivered the injection.
1177 	 */
1178 	if ((info & VMCS_INTR_VALID) != 0) {
1179 		return (EIS_EV_EXISTING | EIS_REQ_EXIT);
1180 	}
1181 
1182 	if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) {
1183 		ASSERT(entryinfo & VMCS_INTR_VALID);
1184 
1185 		info = entryinfo;
1186 		vector = info & 0xff;
1187 		if (vector == IDT_BP || vector == IDT_OF) {
1188 			/*
1189 			 * VT-x requires #BP and #OF to be injected as software
1190 			 * exceptions.
1191 			 */
1192 			info &= ~VMCS_INTR_T_MASK;
1193 			info |= VMCS_INTR_T_SWEXCEPTION;
1194 		}
1195 
1196 		if (info & VMCS_INTR_DEL_ERRCODE) {
1197 			vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32);
1198 		}
1199 
1200 		vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1201 		state = EIS_EV_INJECTED;
1202 	}
1203 
1204 	if (vm_nmi_pending(vmx->vm, vcpu)) {
1205 		/*
1206 		 * If there are no conditions blocking NMI injection then inject
1207 		 * it directly here otherwise enable "NMI window exiting" to
1208 		 * inject it as soon as we can.
1209 		 *
1210 		 * According to the Intel manual, some CPUs do not allow NMI
1211 		 * injection when STI_BLOCKING is active.  That check is
1212 		 * enforced here, regardless of CPU capability.  If running on a
1213 		 * CPU without such a restriction it will immediately exit and
1214 		 * the NMI will be injected in the "NMI window exiting" handler.
1215 		 */
1216 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
1217 			if (state == EIS_CAN_INJECT) {
1218 				vmx_inject_nmi(vmx, vcpu);
1219 				state = EIS_EV_INJECTED;
1220 			} else {
1221 				return (state | EIS_REQ_EXIT);
1222 			}
1223 		} else {
1224 			vmx_set_nmi_window_exiting(vmx, vcpu);
1225 		}
1226 	}
1227 
1228 	if (vm_extint_pending(vmx->vm, vcpu)) {
1229 		if (state != EIS_CAN_INJECT) {
1230 			return (state | EIS_REQ_EXIT);
1231 		}
1232 		if ((gi & HWINTR_BLOCKING) != 0 ||
1233 		    (vmcs_read(VMCS_GUEST_RFLAGS) & PSL_I) == 0) {
1234 			return (EIS_GI_BLOCK);
1235 		}
1236 
1237 		/* Ask the legacy pic for a vector to inject */
1238 		vatpic_pending_intr(vmx->vm, &vector);
1239 
1240 		/*
1241 		 * From the Intel SDM, Volume 3, Section "Maskable
1242 		 * Hardware Interrupts":
1243 		 * - maskable interrupt vectors [0,255] can be delivered
1244 		 *   through the INTR pin.
1245 		 */
1246 		KASSERT(vector >= 0 && vector <= 255,
1247 		    ("invalid vector %d from INTR", vector));
1248 
1249 		/* Inject the interrupt */
1250 		vmcs_write(VMCS_ENTRY_INTR_INFO,
1251 		    VMCS_INTR_T_HWINTR | VMCS_INTR_VALID | vector);
1252 
1253 		vm_extint_clear(vmx->vm, vcpu);
1254 		vatpic_intr_accepted(vmx->vm, vector);
1255 		state = EIS_EV_INJECTED;
1256 	}
1257 
1258 	return (state);
1259 }
1260 
1261 /*
1262  * Inject any interrupts pending on the vLAPIC.
1263  *
1264  * This is done with host CPU interrupts disabled so notification IPIs, either
1265  * from the standard vCPU notification or APICv posted interrupts, will be
1266  * queued on the host APIC and recognized when entering VMX context.
1267  */
1268 static enum event_inject_state
vmx_inject_vlapic(struct vmx * vmx,int vcpu,struct vlapic * vlapic)1269 vmx_inject_vlapic(struct vmx *vmx, int vcpu, struct vlapic *vlapic)
1270 {
1271 	int vector;
1272 
1273 	if (!vlapic_pending_intr(vlapic, &vector)) {
1274 		return (EIS_CAN_INJECT);
1275 	}
1276 
1277 	/*
1278 	 * From the Intel SDM, Volume 3, Section "Maskable
1279 	 * Hardware Interrupts":
1280 	 * - maskable interrupt vectors [16,255] can be delivered
1281 	 *   through the local APIC.
1282 	 */
1283 	KASSERT(vector >= 16 && vector <= 255,
1284 	    ("invalid vector %d from local APIC", vector));
1285 
1286 	if (vmx_cap_en(vmx, VMX_CAP_APICV)) {
1287 		uint16_t status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
1288 		uint16_t status_new = (status_old & 0xff00) | vector;
1289 
1290 		/*
1291 		 * The APICv state will have been synced into the vLAPIC
1292 		 * as part of vlapic_pending_intr().  Prepare the VMCS
1293 		 * for the to-be-injected pending interrupt.
1294 		 */
1295 		if (status_new > status_old) {
1296 			vmcs_write(VMCS_GUEST_INTR_STATUS, status_new);
1297 			VCPU_CTR2(vlapic->vm, vlapic->vcpuid,
1298 			    "vmx_inject_interrupts: guest_intr_status "
1299 			    "changed from 0x%04x to 0x%04x",
1300 			    status_old, status_new);
1301 		}
1302 
1303 		/*
1304 		 * Ensure VMCS state regarding EOI traps is kept in sync
1305 		 * with the TMRs in the vlapic.
1306 		 */
1307 		vmx_apicv_sync_tmr(vlapic);
1308 
1309 		/*
1310 		 * The rest of the injection process for injecting the
1311 		 * interrupt(s) is handled by APICv. It does not preclude other
1312 		 * event injection from occurring.
1313 		 */
1314 		return (EIS_CAN_INJECT);
1315 	}
1316 
1317 	ASSERT0(vmcs_read(VMCS_ENTRY_INTR_INFO) & VMCS_INTR_VALID);
1318 
1319 	/* Does guest interruptability block injection? */
1320 	if ((vmcs_read(VMCS_GUEST_INTERRUPTIBILITY) & HWINTR_BLOCKING) != 0 ||
1321 	    (vmcs_read(VMCS_GUEST_RFLAGS) & PSL_I) == 0) {
1322 		return (EIS_GI_BLOCK);
1323 	}
1324 
1325 	/* Inject the interrupt */
1326 	vmcs_write(VMCS_ENTRY_INTR_INFO,
1327 	    VMCS_INTR_T_HWINTR | VMCS_INTR_VALID | vector);
1328 
1329 	/* Update the Local APIC ISR */
1330 	vlapic_intr_accepted(vlapic, vector);
1331 
1332 	return (EIS_EV_INJECTED);
1333 }
1334 
1335 /*
1336  * Re-check for events to be injected.
1337  *
1338  * Once host CPU interrupts are disabled, check for the presence of any events
1339  * which require injection processing.  If an exit is required upon injection,
1340  * or once the guest becomes interruptable, that will be configured too.
1341  */
1342 static bool
vmx_inject_recheck(struct vmx * vmx,int vcpu,enum event_inject_state state)1343 vmx_inject_recheck(struct vmx *vmx, int vcpu, enum event_inject_state state)
1344 {
1345 	if (state == EIS_CAN_INJECT) {
1346 		if (vm_nmi_pending(vmx->vm, vcpu) &&
1347 		    !vmx_nmi_window_exiting(vmx, vcpu)) {
1348 			/* queued NMI not blocked by NMI-window-exiting */
1349 			return (true);
1350 		}
1351 		if (vm_extint_pending(vmx->vm, vcpu)) {
1352 			/* queued ExtINT not blocked by existing injection */
1353 			return (true);
1354 		}
1355 	} else {
1356 		if ((state & EIS_REQ_EXIT) != 0) {
1357 			/*
1358 			 * Use a self-IPI to force an immediate exit after
1359 			 * event injection has occurred.
1360 			 */
1361 			poke_cpu(CPU->cpu_id);
1362 		} else {
1363 			/*
1364 			 * If any event is being injected, an exit immediately
1365 			 * upon becoming interruptable again will allow pending
1366 			 * or newly queued events to be injected in a timely
1367 			 * manner.
1368 			 */
1369 			vmx_set_int_window_exiting(vmx, vcpu);
1370 		}
1371 	}
1372 	return (false);
1373 }
1374 
1375 /*
1376  * If the Virtual NMIs execution control is '1' then the logical processor
1377  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1378  * the VMCS. An IRET instruction in VMX non-root operation will remove any
1379  * virtual-NMI blocking.
1380  *
1381  * This unblocking occurs even if the IRET causes a fault. In this case the
1382  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1383  */
1384 static void
vmx_restore_nmi_blocking(struct vmx * vmx,int vcpuid)1385 vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
1386 {
1387 	uint32_t gi;
1388 
1389 	VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
1390 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1391 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1392 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1393 }
1394 
1395 static void
vmx_clear_nmi_blocking(struct vmx * vmx,int vcpuid)1396 vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
1397 {
1398 	uint32_t gi;
1399 
1400 	VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
1401 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1402 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
1403 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
1404 }
1405 
1406 static void
vmx_assert_nmi_blocking(struct vmx * vmx,int vcpuid)1407 vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid)
1408 {
1409 	uint32_t gi;
1410 
1411 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1412 	KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING,
1413 	    ("NMI blocking is not in effect %x", gi));
1414 }
1415 
1416 static int
vmx_emulate_xsetbv(struct vmx * vmx,int vcpu,struct vm_exit * vmexit)1417 vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1418 {
1419 	struct vmxctx *vmxctx;
1420 	uint64_t xcrval;
1421 	const struct xsave_limits *limits;
1422 
1423 	vmxctx = &vmx->ctx[vcpu];
1424 	limits = vmm_get_xsave_limits();
1425 
1426 	/*
1427 	 * Note that the processor raises a GP# fault on its own if
1428 	 * xsetbv is executed for CPL != 0, so we do not have to
1429 	 * emulate that fault here.
1430 	 */
1431 
1432 	/* Only xcr0 is supported. */
1433 	if (vmxctx->guest_rcx != 0) {
1434 		vm_inject_gp(vmx->vm, vcpu);
1435 		return (HANDLED);
1436 	}
1437 
1438 	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
1439 	if (!limits->xsave_enabled ||
1440 	    !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1441 		vm_inject_ud(vmx->vm, vcpu);
1442 		return (HANDLED);
1443 	}
1444 
1445 	xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1446 	if ((xcrval & ~limits->xcr0_allowed) != 0) {
1447 		vm_inject_gp(vmx->vm, vcpu);
1448 		return (HANDLED);
1449 	}
1450 
1451 	if (!(xcrval & XFEATURE_ENABLED_X87)) {
1452 		vm_inject_gp(vmx->vm, vcpu);
1453 		return (HANDLED);
1454 	}
1455 
1456 	/* AVX (YMM_Hi128) requires SSE. */
1457 	if (xcrval & XFEATURE_ENABLED_AVX &&
1458 	    (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
1459 		vm_inject_gp(vmx->vm, vcpu);
1460 		return (HANDLED);
1461 	}
1462 
1463 	/*
1464 	 * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
1465 	 * ZMM_Hi256, and Hi16_ZMM.
1466 	 */
1467 	if (xcrval & XFEATURE_AVX512 &&
1468 	    (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
1469 	    (XFEATURE_AVX512 | XFEATURE_AVX)) {
1470 		vm_inject_gp(vmx->vm, vcpu);
1471 		return (HANDLED);
1472 	}
1473 
1474 	/*
1475 	 * Intel MPX requires both bound register state flags to be
1476 	 * set.
1477 	 */
1478 	if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
1479 	    ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
1480 		vm_inject_gp(vmx->vm, vcpu);
1481 		return (HANDLED);
1482 	}
1483 
1484 	/*
1485 	 * This runs "inside" vmrun() with the guest's FPU state, so
1486 	 * modifying xcr0 directly modifies the guest's xcr0, not the
1487 	 * host's.
1488 	 */
1489 	load_xcr(0, xcrval);
1490 	return (HANDLED);
1491 }
1492 
1493 static uint64_t
vmx_get_guest_reg(struct vmx * vmx,int vcpu,int ident)1494 vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident)
1495 {
1496 	const struct vmxctx *vmxctx;
1497 
1498 	vmxctx = &vmx->ctx[vcpu];
1499 
1500 	switch (ident) {
1501 	case 0:
1502 		return (vmxctx->guest_rax);
1503 	case 1:
1504 		return (vmxctx->guest_rcx);
1505 	case 2:
1506 		return (vmxctx->guest_rdx);
1507 	case 3:
1508 		return (vmxctx->guest_rbx);
1509 	case 4:
1510 		return (vmcs_read(VMCS_GUEST_RSP));
1511 	case 5:
1512 		return (vmxctx->guest_rbp);
1513 	case 6:
1514 		return (vmxctx->guest_rsi);
1515 	case 7:
1516 		return (vmxctx->guest_rdi);
1517 	case 8:
1518 		return (vmxctx->guest_r8);
1519 	case 9:
1520 		return (vmxctx->guest_r9);
1521 	case 10:
1522 		return (vmxctx->guest_r10);
1523 	case 11:
1524 		return (vmxctx->guest_r11);
1525 	case 12:
1526 		return (vmxctx->guest_r12);
1527 	case 13:
1528 		return (vmxctx->guest_r13);
1529 	case 14:
1530 		return (vmxctx->guest_r14);
1531 	case 15:
1532 		return (vmxctx->guest_r15);
1533 	default:
1534 		panic("invalid vmx register %d", ident);
1535 	}
1536 }
1537 
1538 static void
vmx_set_guest_reg(struct vmx * vmx,int vcpu,int ident,uint64_t regval)1539 vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval)
1540 {
1541 	struct vmxctx *vmxctx;
1542 
1543 	vmxctx = &vmx->ctx[vcpu];
1544 
1545 	switch (ident) {
1546 	case 0:
1547 		vmxctx->guest_rax = regval;
1548 		break;
1549 	case 1:
1550 		vmxctx->guest_rcx = regval;
1551 		break;
1552 	case 2:
1553 		vmxctx->guest_rdx = regval;
1554 		break;
1555 	case 3:
1556 		vmxctx->guest_rbx = regval;
1557 		break;
1558 	case 4:
1559 		vmcs_write(VMCS_GUEST_RSP, regval);
1560 		break;
1561 	case 5:
1562 		vmxctx->guest_rbp = regval;
1563 		break;
1564 	case 6:
1565 		vmxctx->guest_rsi = regval;
1566 		break;
1567 	case 7:
1568 		vmxctx->guest_rdi = regval;
1569 		break;
1570 	case 8:
1571 		vmxctx->guest_r8 = regval;
1572 		break;
1573 	case 9:
1574 		vmxctx->guest_r9 = regval;
1575 		break;
1576 	case 10:
1577 		vmxctx->guest_r10 = regval;
1578 		break;
1579 	case 11:
1580 		vmxctx->guest_r11 = regval;
1581 		break;
1582 	case 12:
1583 		vmxctx->guest_r12 = regval;
1584 		break;
1585 	case 13:
1586 		vmxctx->guest_r13 = regval;
1587 		break;
1588 	case 14:
1589 		vmxctx->guest_r14 = regval;
1590 		break;
1591 	case 15:
1592 		vmxctx->guest_r15 = regval;
1593 		break;
1594 	default:
1595 		panic("invalid vmx register %d", ident);
1596 	}
1597 }
1598 
1599 static int
vmx_emulate_cr0_access(struct vmx * vmx,int vcpu,uint64_t exitqual)1600 vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1601 {
1602 	uint64_t crval, regval;
1603 
1604 	/* We only handle mov to %cr0 at this time */
1605 	if ((exitqual & 0xf0) != 0x00)
1606 		return (UNHANDLED);
1607 
1608 	regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1609 
1610 	vmcs_write(VMCS_CR0_SHADOW, regval);
1611 
1612 	crval = regval | cr0_ones_mask;
1613 	crval &= ~cr0_zeros_mask;
1614 
1615 	const uint64_t old = vmcs_read(VMCS_GUEST_CR0);
1616 	const uint64_t diff = crval ^ old;
1617 	/* Flush the TLB if the paging or write-protect bits are changing */
1618 	if ((diff & CR0_PG) != 0 || (diff & CR0_WP) != 0) {
1619 		vmx_invvpid(vmx, vcpu, 1);
1620 	}
1621 
1622 	vmcs_write(VMCS_GUEST_CR0, crval);
1623 
1624 	if (regval & CR0_PG) {
1625 		uint64_t efer, entry_ctls;
1626 
1627 		/*
1628 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
1629 		 * the "IA-32e mode guest" bit in VM-entry control must be
1630 		 * equal.
1631 		 */
1632 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
1633 		if (efer & EFER_LME) {
1634 			efer |= EFER_LMA;
1635 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
1636 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
1637 			entry_ctls |= VM_ENTRY_GUEST_LMA;
1638 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
1639 		}
1640 	}
1641 
1642 	return (HANDLED);
1643 }
1644 
1645 static int
vmx_emulate_cr4_access(struct vmx * vmx,int vcpu,uint64_t exitqual)1646 vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1647 {
1648 	uint64_t crval, regval;
1649 
1650 	/* We only handle mov to %cr4 at this time */
1651 	if ((exitqual & 0xf0) != 0x00)
1652 		return (UNHANDLED);
1653 
1654 	regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
1655 
1656 	vmcs_write(VMCS_CR4_SHADOW, regval);
1657 
1658 	crval = regval | cr4_ones_mask;
1659 	crval &= ~cr4_zeros_mask;
1660 	vmcs_write(VMCS_GUEST_CR4, crval);
1661 
1662 	return (HANDLED);
1663 }
1664 
1665 static int
vmx_emulate_cr8_access(struct vmx * vmx,int vcpu,uint64_t exitqual)1666 vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1667 {
1668 	struct vlapic *vlapic;
1669 	uint64_t cr8;
1670 	int regnum;
1671 
1672 	/* We only handle mov %cr8 to/from a register at this time. */
1673 	if ((exitqual & 0xe0) != 0x00) {
1674 		return (UNHANDLED);
1675 	}
1676 
1677 	vlapic = vm_lapic(vmx->vm, vcpu);
1678 	regnum = (exitqual >> 8) & 0xf;
1679 	if (exitqual & 0x10) {
1680 		cr8 = vlapic_get_cr8(vlapic);
1681 		vmx_set_guest_reg(vmx, vcpu, regnum, cr8);
1682 	} else {
1683 		cr8 = vmx_get_guest_reg(vmx, vcpu, regnum);
1684 		vlapic_set_cr8(vlapic, cr8);
1685 	}
1686 
1687 	return (HANDLED);
1688 }
1689 
1690 /*
1691  * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
1692  */
1693 static int
vmx_cpl(void)1694 vmx_cpl(void)
1695 {
1696 	uint32_t ssar;
1697 
1698 	ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS);
1699 	return ((ssar >> 5) & 0x3);
1700 }
1701 
1702 static enum vm_cpu_mode
vmx_cpu_mode(void)1703 vmx_cpu_mode(void)
1704 {
1705 	uint32_t csar;
1706 
1707 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) {
1708 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1709 		if (csar & 0x2000)
1710 			return (CPU_MODE_64BIT);	/* CS.L = 1 */
1711 		else
1712 			return (CPU_MODE_COMPATIBILITY);
1713 	} else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) {
1714 		return (CPU_MODE_PROTECTED);
1715 	} else {
1716 		return (CPU_MODE_REAL);
1717 	}
1718 }
1719 
1720 static enum vm_paging_mode
vmx_paging_mode(void)1721 vmx_paging_mode(void)
1722 {
1723 
1724 	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
1725 		return (PAGING_MODE_FLAT);
1726 	if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
1727 		return (PAGING_MODE_32);
1728 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
1729 		return (PAGING_MODE_64);
1730 	else
1731 		return (PAGING_MODE_PAE);
1732 }
1733 
1734 static void
vmx_paging_info(struct vm_guest_paging * paging)1735 vmx_paging_info(struct vm_guest_paging *paging)
1736 {
1737 	paging->cr3 = vmcs_guest_cr3();
1738 	paging->cpl = vmx_cpl();
1739 	paging->cpu_mode = vmx_cpu_mode();
1740 	paging->paging_mode = vmx_paging_mode();
1741 }
1742 
1743 static void
vmexit_mmio_emul(struct vm_exit * vmexit,struct vie * vie,uint64_t gpa,uint64_t gla)1744 vmexit_mmio_emul(struct vm_exit *vmexit, struct vie *vie, uint64_t gpa,
1745     uint64_t gla)
1746 {
1747 	struct vm_guest_paging paging;
1748 	uint32_t csar;
1749 
1750 	vmexit->exitcode = VM_EXITCODE_MMIO_EMUL;
1751 	vmexit->inst_length = 0;
1752 	vmexit->u.mmio_emul.gpa = gpa;
1753 	vmexit->u.mmio_emul.gla = gla;
1754 	vmx_paging_info(&paging);
1755 
1756 	switch (paging.cpu_mode) {
1757 	case CPU_MODE_REAL:
1758 		vmexit->u.mmio_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1759 		vmexit->u.mmio_emul.cs_d = 0;
1760 		break;
1761 	case CPU_MODE_PROTECTED:
1762 	case CPU_MODE_COMPATIBILITY:
1763 		vmexit->u.mmio_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
1764 		csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS);
1765 		vmexit->u.mmio_emul.cs_d = SEG_DESC_DEF32(csar);
1766 		break;
1767 	default:
1768 		vmexit->u.mmio_emul.cs_base = 0;
1769 		vmexit->u.mmio_emul.cs_d = 0;
1770 		break;
1771 	}
1772 
1773 	vie_init_mmio(vie, NULL, 0, &paging, gpa);
1774 }
1775 
1776 static void
vmexit_inout(struct vm_exit * vmexit,struct vie * vie,uint64_t qual,uint32_t eax)1777 vmexit_inout(struct vm_exit *vmexit, struct vie *vie, uint64_t qual,
1778     uint32_t eax)
1779 {
1780 	struct vm_guest_paging paging;
1781 	struct vm_inout *inout;
1782 
1783 	inout = &vmexit->u.inout;
1784 
1785 	inout->bytes = (qual & 0x7) + 1;
1786 	inout->flags = 0;
1787 	inout->flags |= (qual & 0x8) ? INOUT_IN : 0;
1788 	inout->flags |= (qual & 0x10) ? INOUT_STR : 0;
1789 	inout->flags |= (qual & 0x20) ? INOUT_REP : 0;
1790 	inout->port = (uint16_t)(qual >> 16);
1791 	inout->eax = eax;
1792 	if (inout->flags & INOUT_STR) {
1793 		uint64_t inst_info;
1794 
1795 		inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
1796 
1797 		/*
1798 		 * According to the SDM, bits 9:7 encode the address size of the
1799 		 * ins/outs operation, but only values 0/1/2 are expected,
1800 		 * corresponding to 16/32/64 bit sizes.
1801 		 */
1802 		inout->addrsize = 2 << BITX(inst_info, 9, 7);
1803 		VERIFY(inout->addrsize == 2 || inout->addrsize == 4 ||
1804 		    inout->addrsize == 8);
1805 
1806 		if (inout->flags & INOUT_IN) {
1807 			/*
1808 			 * The bits describing the segment in INSTRUCTION_INFO
1809 			 * are not defined for ins, leaving it to system
1810 			 * software to assume %es (encoded as 0)
1811 			 */
1812 			inout->segment = 0;
1813 		} else {
1814 			/*
1815 			 * Bits 15-17 encode the segment for OUTS.
1816 			 * This value follows the standard x86 segment order.
1817 			 */
1818 			inout->segment = (inst_info >> 15) & 0x7;
1819 		}
1820 	}
1821 
1822 	vmexit->exitcode = VM_EXITCODE_INOUT;
1823 	vmx_paging_info(&paging);
1824 	vie_init_inout(vie, inout, vmexit->inst_length, &paging);
1825 
1826 	/* The in/out emulation will handle advancing %rip */
1827 	vmexit->inst_length = 0;
1828 }
1829 
1830 static int
ept_fault_type(uint64_t ept_qual)1831 ept_fault_type(uint64_t ept_qual)
1832 {
1833 	int fault_type;
1834 
1835 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1836 		fault_type = PROT_WRITE;
1837 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1838 		fault_type = PROT_EXEC;
1839 	else
1840 		fault_type = PROT_READ;
1841 
1842 	return (fault_type);
1843 }
1844 
1845 static bool
ept_emulation_fault(uint64_t ept_qual)1846 ept_emulation_fault(uint64_t ept_qual)
1847 {
1848 	int read, write;
1849 
1850 	/* EPT fault on an instruction fetch doesn't make sense here */
1851 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
1852 		return (false);
1853 
1854 	/* EPT fault must be a read fault or a write fault */
1855 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1856 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
1857 	if ((read | write) == 0)
1858 		return (false);
1859 
1860 	/*
1861 	 * The EPT violation must have been caused by accessing a
1862 	 * guest-physical address that is a translation of a guest-linear
1863 	 * address.
1864 	 */
1865 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1866 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1867 		return (false);
1868 	}
1869 
1870 	return (true);
1871 }
1872 
1873 static __inline int
apic_access_virtualization(struct vmx * vmx,int vcpuid)1874 apic_access_virtualization(struct vmx *vmx, int vcpuid)
1875 {
1876 	uint32_t proc_ctls2;
1877 
1878 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1879 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
1880 }
1881 
1882 static __inline int
x2apic_virtualization(struct vmx * vmx,int vcpuid)1883 x2apic_virtualization(struct vmx *vmx, int vcpuid)
1884 {
1885 	uint32_t proc_ctls2;
1886 
1887 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
1888 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
1889 }
1890 
1891 static int
vmx_handle_apic_write(struct vmx * vmx,int vcpuid,struct vlapic * vlapic,uint64_t qual)1892 vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
1893     uint64_t qual)
1894 {
1895 	int handled, offset;
1896 	uint32_t *apic_regs, vector;
1897 
1898 	handled = HANDLED;
1899 	offset = APIC_WRITE_OFFSET(qual);
1900 
1901 	if (!apic_access_virtualization(vmx, vcpuid)) {
1902 		/*
1903 		 * In general there should not be any APIC write VM-exits
1904 		 * unless APIC-access virtualization is enabled.
1905 		 *
1906 		 * However self-IPI virtualization can legitimately trigger
1907 		 * an APIC-write VM-exit so treat it specially.
1908 		 */
1909 		if (x2apic_virtualization(vmx, vcpuid) &&
1910 		    offset == APIC_OFFSET_SELF_IPI) {
1911 			apic_regs = (uint32_t *)(vlapic->apic_page);
1912 			vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
1913 			vlapic_self_ipi_handler(vlapic, vector);
1914 			return (HANDLED);
1915 		} else
1916 			return (UNHANDLED);
1917 	}
1918 
1919 	switch (offset) {
1920 	case APIC_OFFSET_ID:
1921 		vlapic_id_write_handler(vlapic);
1922 		break;
1923 	case APIC_OFFSET_LDR:
1924 		vlapic_ldr_write_handler(vlapic);
1925 		break;
1926 	case APIC_OFFSET_DFR:
1927 		vlapic_dfr_write_handler(vlapic);
1928 		break;
1929 	case APIC_OFFSET_SVR:
1930 		vlapic_svr_write_handler(vlapic);
1931 		break;
1932 	case APIC_OFFSET_ESR:
1933 		vlapic_esr_write_handler(vlapic);
1934 		break;
1935 	case APIC_OFFSET_ICR_LOW:
1936 		if (vlapic_icrlo_write_handler(vlapic) != 0) {
1937 			handled = UNHANDLED;
1938 		}
1939 		break;
1940 	case APIC_OFFSET_CMCI_LVT:
1941 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1942 		vlapic_lvt_write_handler(vlapic, offset);
1943 		break;
1944 	case APIC_OFFSET_TIMER_ICR:
1945 		vlapic_icrtmr_write_handler(vlapic);
1946 		break;
1947 	case APIC_OFFSET_TIMER_DCR:
1948 		vlapic_dcr_write_handler(vlapic);
1949 		break;
1950 	default:
1951 		handled = UNHANDLED;
1952 		break;
1953 	}
1954 	return (handled);
1955 }
1956 
1957 static bool
apic_access_fault(struct vmx * vmx,int vcpuid,uint64_t gpa)1958 apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
1959 {
1960 
1961 	if (apic_access_virtualization(vmx, vcpuid) &&
1962 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
1963 		return (true);
1964 	else
1965 		return (false);
1966 }
1967 
1968 static int
vmx_handle_apic_access(struct vmx * vmx,int vcpuid,struct vm_exit * vmexit)1969 vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
1970 {
1971 	uint64_t qual;
1972 	int access_type, offset, allowed;
1973 	struct vie *vie;
1974 
1975 	if (!apic_access_virtualization(vmx, vcpuid))
1976 		return (UNHANDLED);
1977 
1978 	qual = vmexit->u.vmx.exit_qualification;
1979 	access_type = APIC_ACCESS_TYPE(qual);
1980 	offset = APIC_ACCESS_OFFSET(qual);
1981 
1982 	allowed = 0;
1983 	if (access_type == 0) {
1984 		/*
1985 		 * Read data access to the following registers is expected.
1986 		 */
1987 		switch (offset) {
1988 		case APIC_OFFSET_APR:
1989 		case APIC_OFFSET_PPR:
1990 		case APIC_OFFSET_RRR:
1991 		case APIC_OFFSET_CMCI_LVT:
1992 		case APIC_OFFSET_TIMER_CCR:
1993 			allowed = 1;
1994 			break;
1995 		default:
1996 			break;
1997 		}
1998 	} else if (access_type == 1) {
1999 		/*
2000 		 * Write data access to the following registers is expected.
2001 		 */
2002 		switch (offset) {
2003 		case APIC_OFFSET_VER:
2004 		case APIC_OFFSET_APR:
2005 		case APIC_OFFSET_PPR:
2006 		case APIC_OFFSET_RRR:
2007 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
2008 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
2009 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
2010 		case APIC_OFFSET_CMCI_LVT:
2011 		case APIC_OFFSET_TIMER_CCR:
2012 			allowed = 1;
2013 			break;
2014 		default:
2015 			break;
2016 		}
2017 	}
2018 
2019 	if (allowed) {
2020 		vie = vm_vie_ctx(vmx->vm, vcpuid);
2021 		vmexit_mmio_emul(vmexit, vie, DEFAULT_APIC_BASE + offset,
2022 		    VIE_INVALID_GLA);
2023 	}
2024 
2025 	/*
2026 	 * Regardless of whether the APIC-access is allowed this handler
2027 	 * always returns UNHANDLED:
2028 	 * - if the access is allowed then it is handled by emulating the
2029 	 *   instruction that caused the VM-exit (outside the critical section)
2030 	 * - if the access is not allowed then it will be converted to an
2031 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
2032 	 */
2033 	return (UNHANDLED);
2034 }
2035 
2036 static enum task_switch_reason
vmx_task_switch_reason(uint64_t qual)2037 vmx_task_switch_reason(uint64_t qual)
2038 {
2039 	int reason;
2040 
2041 	reason = (qual >> 30) & 0x3;
2042 	switch (reason) {
2043 	case 0:
2044 		return (TSR_CALL);
2045 	case 1:
2046 		return (TSR_IRET);
2047 	case 2:
2048 		return (TSR_JMP);
2049 	case 3:
2050 		return (TSR_IDT_GATE);
2051 	default:
2052 		panic("%s: invalid reason %d", __func__, reason);
2053 	}
2054 }
2055 
2056 static int
emulate_wrmsr(struct vmx * vmx,int vcpuid,uint_t num,uint64_t val)2057 emulate_wrmsr(struct vmx *vmx, int vcpuid, uint_t num, uint64_t val)
2058 {
2059 	int error;
2060 
2061 	if (lapic_msr(num))
2062 		error = lapic_wrmsr(vmx->vm, vcpuid, num, val);
2063 	else
2064 		error = vmx_wrmsr(vmx, vcpuid, num, val);
2065 
2066 	return (error);
2067 }
2068 
2069 static int
emulate_rdmsr(struct vmx * vmx,int vcpuid,uint_t num)2070 emulate_rdmsr(struct vmx *vmx, int vcpuid, uint_t num)
2071 {
2072 	uint64_t result;
2073 	int error;
2074 
2075 	if (lapic_msr(num))
2076 		error = lapic_rdmsr(vmx->vm, vcpuid, num, &result);
2077 	else
2078 		error = vmx_rdmsr(vmx, vcpuid, num, &result);
2079 
2080 	if (error == 0) {
2081 		vmx->ctx[vcpuid].guest_rax = (uint32_t)result;
2082 		vmx->ctx[vcpuid].guest_rdx = result >> 32;
2083 	}
2084 
2085 	return (error);
2086 }
2087 
2088 static int
vmx_exit_process(struct vmx * vmx,int vcpu,struct vm_exit * vmexit)2089 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
2090 {
2091 	int error, errcode, errcode_valid, handled;
2092 	struct vmxctx *vmxctx;
2093 	struct vie *vie;
2094 	struct vlapic *vlapic;
2095 	struct vm_task_switch *ts;
2096 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info;
2097 	uint32_t intr_type, intr_vec, reason;
2098 	uint64_t exitintinfo, qual, gpa;
2099 
2100 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
2101 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
2102 
2103 	handled = UNHANDLED;
2104 	vmxctx = &vmx->ctx[vcpu];
2105 
2106 	qual = vmexit->u.vmx.exit_qualification;
2107 	reason = vmexit->u.vmx.exit_reason;
2108 	vmexit->exitcode = VM_EXITCODE_BOGUS;
2109 
2110 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
2111 	SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpu, vmexit);
2112 
2113 	/*
2114 	 * VM-entry failures during or after loading guest state.
2115 	 *
2116 	 * These VM-exits are uncommon but must be handled specially
2117 	 * as most VM-exit fields are not populated as usual.
2118 	 */
2119 	if (reason == EXIT_REASON_MCE_DURING_ENTRY) {
2120 		VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry");
2121 		vmm_call_trap(T_MCE);
2122 		return (1);
2123 	}
2124 
2125 	/*
2126 	 * VM exits that can be triggered during event delivery need to
2127 	 * be handled specially by re-injecting the event if the IDT
2128 	 * vectoring information field's valid bit is set.
2129 	 *
2130 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
2131 	 * for details.
2132 	 */
2133 	idtvec_info = vmcs_idt_vectoring_info();
2134 	if (idtvec_info & VMCS_IDT_VEC_VALID) {
2135 		idtvec_info &= ~(1 << 12); /* clear undefined bit */
2136 		exitintinfo = idtvec_info;
2137 		if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2138 			idtvec_err = vmcs_idt_vectoring_err();
2139 			exitintinfo |= (uint64_t)idtvec_err << 32;
2140 		}
2141 		error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo);
2142 		KASSERT(error == 0, ("%s: vm_set_intinfo error %d",
2143 		    __func__, error));
2144 
2145 		/*
2146 		 * If 'virtual NMIs' are being used and the VM-exit
2147 		 * happened while injecting an NMI during the previous
2148 		 * VM-entry, then clear "blocking by NMI" in the
2149 		 * Guest Interruptibility-State so the NMI can be
2150 		 * reinjected on the subsequent VM-entry.
2151 		 *
2152 		 * However, if the NMI was being delivered through a task
2153 		 * gate, then the new task must start execution with NMIs
2154 		 * blocked so don't clear NMI blocking in this case.
2155 		 */
2156 		intr_type = idtvec_info & VMCS_INTR_T_MASK;
2157 		if (intr_type == VMCS_INTR_T_NMI) {
2158 			if (reason != EXIT_REASON_TASK_SWITCH)
2159 				vmx_clear_nmi_blocking(vmx, vcpu);
2160 			else
2161 				vmx_assert_nmi_blocking(vmx, vcpu);
2162 		}
2163 
2164 		/*
2165 		 * Update VM-entry instruction length if the event being
2166 		 * delivered was a software interrupt or software exception.
2167 		 */
2168 		if (intr_type == VMCS_INTR_T_SWINTR ||
2169 		    intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION ||
2170 		    intr_type == VMCS_INTR_T_SWEXCEPTION) {
2171 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2172 		}
2173 	}
2174 
2175 	switch (reason) {
2176 	case EXIT_REASON_TASK_SWITCH:
2177 		ts = &vmexit->u.task_switch;
2178 		ts->tsssel = qual & 0xffff;
2179 		ts->reason = vmx_task_switch_reason(qual);
2180 		ts->ext = 0;
2181 		ts->errcode_valid = 0;
2182 		vmx_paging_info(&ts->paging);
2183 		/*
2184 		 * If the task switch was due to a CALL, JMP, IRET, software
2185 		 * interrupt (INT n) or software exception (INT3, INTO),
2186 		 * then the saved %rip references the instruction that caused
2187 		 * the task switch. The instruction length field in the VMCS
2188 		 * is valid in this case.
2189 		 *
2190 		 * In all other cases (e.g., NMI, hardware exception) the
2191 		 * saved %rip is one that would have been saved in the old TSS
2192 		 * had the task switch completed normally so the instruction
2193 		 * length field is not needed in this case and is explicitly
2194 		 * set to 0.
2195 		 */
2196 		if (ts->reason == TSR_IDT_GATE) {
2197 			KASSERT(idtvec_info & VMCS_IDT_VEC_VALID,
2198 			    ("invalid idtvec_info %x for IDT task switch",
2199 			    idtvec_info));
2200 			intr_type = idtvec_info & VMCS_INTR_T_MASK;
2201 			if (intr_type != VMCS_INTR_T_SWINTR &&
2202 			    intr_type != VMCS_INTR_T_SWEXCEPTION &&
2203 			    intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) {
2204 				/* Task switch triggered by external event */
2205 				ts->ext = 1;
2206 				vmexit->inst_length = 0;
2207 				if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
2208 					ts->errcode_valid = 1;
2209 					ts->errcode = vmcs_idt_vectoring_err();
2210 				}
2211 			}
2212 		}
2213 		vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
2214 		SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpu, vmexit, ts);
2215 		VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, "
2216 		    "%s errcode 0x%016lx", ts->reason, ts->tsssel,
2217 		    ts->ext ? "external" : "internal",
2218 		    ((uint64_t)ts->errcode << 32) | ts->errcode_valid);
2219 		break;
2220 	case EXIT_REASON_CR_ACCESS:
2221 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
2222 		SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpu, vmexit, qual);
2223 		switch (qual & 0xf) {
2224 		case 0:
2225 			handled = vmx_emulate_cr0_access(vmx, vcpu, qual);
2226 			break;
2227 		case 4:
2228 			handled = vmx_emulate_cr4_access(vmx, vcpu, qual);
2229 			break;
2230 		case 8:
2231 			handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
2232 			break;
2233 		}
2234 		break;
2235 	case EXIT_REASON_RDMSR:
2236 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
2237 		ecx = vmxctx->guest_rcx;
2238 		VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx);
2239 		SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpu, vmexit, ecx);
2240 		error = emulate_rdmsr(vmx, vcpu, ecx);
2241 		if (error == 0) {
2242 			handled = HANDLED;
2243 		} else if (error > 0) {
2244 			vmexit->exitcode = VM_EXITCODE_RDMSR;
2245 			vmexit->u.msr.code = ecx;
2246 		} else {
2247 			/* Return to userspace with a valid exitcode */
2248 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2249 			    ("emulate_rdmsr retu with bogus exitcode"));
2250 		}
2251 		break;
2252 	case EXIT_REASON_WRMSR:
2253 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
2254 		eax = vmxctx->guest_rax;
2255 		ecx = vmxctx->guest_rcx;
2256 		edx = vmxctx->guest_rdx;
2257 		VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx",
2258 		    ecx, (uint64_t)edx << 32 | eax);
2259 		SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpu, ecx,
2260 		    (uint64_t)edx << 32 | eax);
2261 		error = emulate_wrmsr(vmx, vcpu, ecx,
2262 		    (uint64_t)edx << 32 | eax);
2263 		if (error == 0) {
2264 			handled = HANDLED;
2265 		} else if (error > 0) {
2266 			vmexit->exitcode = VM_EXITCODE_WRMSR;
2267 			vmexit->u.msr.code = ecx;
2268 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
2269 		} else {
2270 			/* Return to userspace with a valid exitcode */
2271 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2272 			    ("emulate_wrmsr retu with bogus exitcode"));
2273 		}
2274 		break;
2275 	case EXIT_REASON_HLT:
2276 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
2277 		SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpu, vmexit);
2278 		vmexit->exitcode = VM_EXITCODE_HLT;
2279 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2280 		break;
2281 	case EXIT_REASON_MTF:
2282 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
2283 		SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpu, vmexit);
2284 		vmexit->exitcode = VM_EXITCODE_MTRAP;
2285 		vmexit->inst_length = 0;
2286 		break;
2287 	case EXIT_REASON_PAUSE:
2288 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
2289 		SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpu, vmexit);
2290 		vmexit->exitcode = VM_EXITCODE_PAUSE;
2291 		break;
2292 	case EXIT_REASON_INTR_WINDOW:
2293 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
2294 		SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpu, vmexit);
2295 		vmx_clear_int_window_exiting(vmx, vcpu);
2296 		return (1);
2297 	case EXIT_REASON_EXT_INTR:
2298 		/*
2299 		 * External interrupts serve only to cause VM exits and allow
2300 		 * the host interrupt handler to run.
2301 		 *
2302 		 * If this external interrupt triggers a virtual interrupt
2303 		 * to a VM, then that state will be recorded by the
2304 		 * host interrupt handler in the VM's softc. We will inject
2305 		 * this virtual interrupt during the subsequent VM enter.
2306 		 */
2307 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2308 		SDT_PROBE4(vmm, vmx, exit, interrupt,
2309 		    vmx, vcpu, vmexit, intr_info);
2310 
2311 		/*
2312 		 * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
2313 		 * This appears to be a bug in VMware Fusion?
2314 		 */
2315 		if (!(intr_info & VMCS_INTR_VALID))
2316 			return (1);
2317 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
2318 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
2319 		    ("VM exit interruption info invalid: %x", intr_info));
2320 		vmx_trigger_hostintr(intr_info & 0xff);
2321 
2322 		/*
2323 		 * This is special. We want to treat this as an 'handled'
2324 		 * VM-exit but not increment the instruction pointer.
2325 		 */
2326 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
2327 		return (1);
2328 	case EXIT_REASON_NMI_WINDOW:
2329 		SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpu, vmexit);
2330 		/* Exit to allow the pending virtual NMI to be injected */
2331 		if (vm_nmi_pending(vmx->vm, vcpu))
2332 			vmx_inject_nmi(vmx, vcpu);
2333 		vmx_clear_nmi_window_exiting(vmx, vcpu);
2334 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
2335 		return (1);
2336 	case EXIT_REASON_INOUT:
2337 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
2338 		vie = vm_vie_ctx(vmx->vm, vcpu);
2339 		vmexit_inout(vmexit, vie, qual, (uint32_t)vmxctx->guest_rax);
2340 		SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpu, vmexit);
2341 		break;
2342 	case EXIT_REASON_CPUID:
2343 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
2344 		SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpu, vmexit);
2345 		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
2346 		break;
2347 	case EXIT_REASON_EXCEPTION:
2348 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
2349 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2350 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
2351 		    ("VM exit interruption info invalid: %x", intr_info));
2352 
2353 		intr_vec = intr_info & 0xff;
2354 		intr_type = intr_info & VMCS_INTR_T_MASK;
2355 
2356 		/*
2357 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
2358 		 * fault encountered during the execution of IRET then we must
2359 		 * restore the state of "virtual-NMI blocking" before resuming
2360 		 * the guest.
2361 		 *
2362 		 * See "Resuming Guest Software after Handling an Exception".
2363 		 * See "Information for VM Exits Due to Vectored Events".
2364 		 */
2365 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2366 		    (intr_vec != IDT_DF) &&
2367 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
2368 			vmx_restore_nmi_blocking(vmx, vcpu);
2369 
2370 		/*
2371 		 * The NMI has already been handled in vmx_exit_handle_nmi().
2372 		 */
2373 		if (intr_type == VMCS_INTR_T_NMI)
2374 			return (1);
2375 
2376 		/*
2377 		 * Call the machine check handler by hand. Also don't reflect
2378 		 * the machine check back into the guest.
2379 		 */
2380 		if (intr_vec == IDT_MC) {
2381 			VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler");
2382 			vmm_call_trap(T_MCE);
2383 			return (1);
2384 		}
2385 
2386 		/*
2387 		 * If the hypervisor has requested user exits for
2388 		 * debug exceptions, bounce them out to userland.
2389 		 */
2390 		if (intr_type == VMCS_INTR_T_SWEXCEPTION &&
2391 		    intr_vec == IDT_BP &&
2392 		    (vmx->cap[vcpu].set & (1 << VM_CAP_BPT_EXIT))) {
2393 			vmexit->exitcode = VM_EXITCODE_BPT;
2394 			vmexit->u.bpt.inst_length = vmexit->inst_length;
2395 			vmexit->inst_length = 0;
2396 			break;
2397 		}
2398 
2399 		if (intr_vec == IDT_PF) {
2400 			vmxctx->guest_cr2 = qual;
2401 		}
2402 
2403 		/*
2404 		 * Software exceptions exhibit trap-like behavior. This in
2405 		 * turn requires populating the VM-entry instruction length
2406 		 * so that the %rip in the trap frame is past the INT3/INTO
2407 		 * instruction.
2408 		 */
2409 		if (intr_type == VMCS_INTR_T_SWEXCEPTION)
2410 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2411 
2412 		/* Reflect all other exceptions back into the guest */
2413 		errcode_valid = errcode = 0;
2414 		if (intr_info & VMCS_INTR_DEL_ERRCODE) {
2415 			errcode_valid = 1;
2416 			errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE);
2417 		}
2418 		VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%x into "
2419 		    "the guest", intr_vec, errcode);
2420 		SDT_PROBE5(vmm, vmx, exit, exception,
2421 		    vmx, vcpu, vmexit, intr_vec, errcode);
2422 		error = vm_inject_exception(vmx->vm, vcpu, intr_vec,
2423 		    errcode_valid, errcode, 0);
2424 		KASSERT(error == 0, ("%s: vm_inject_exception error %d",
2425 		    __func__, error));
2426 		return (1);
2427 
2428 	case EXIT_REASON_EPT_FAULT:
2429 		/*
2430 		 * If 'gpa' lies within the address space allocated to
2431 		 * memory then this must be a nested page fault otherwise
2432 		 * this must be an instruction that accesses MMIO space.
2433 		 */
2434 		gpa = vmcs_gpa();
2435 		if (vm_mem_allocated(vmx->vm, vcpu, gpa) ||
2436 		    apic_access_fault(vmx, vcpu, gpa)) {
2437 			vmexit->exitcode = VM_EXITCODE_PAGING;
2438 			vmexit->inst_length = 0;
2439 			vmexit->u.paging.gpa = gpa;
2440 			vmexit->u.paging.fault_type = ept_fault_type(qual);
2441 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
2442 			SDT_PROBE5(vmm, vmx, exit, nestedfault,
2443 			    vmx, vcpu, vmexit, gpa, qual);
2444 		} else if (ept_emulation_fault(qual)) {
2445 			vie = vm_vie_ctx(vmx->vm, vcpu);
2446 			vmexit_mmio_emul(vmexit, vie, gpa, vmcs_gla());
2447 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MMIO_EMUL, 1);
2448 			SDT_PROBE4(vmm, vmx, exit, mmiofault,
2449 			    vmx, vcpu, vmexit, gpa);
2450 		}
2451 		/*
2452 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
2453 		 * EPT fault during the execution of IRET then we must restore
2454 		 * the state of "virtual-NMI blocking" before resuming.
2455 		 *
2456 		 * See description of "NMI unblocking due to IRET" in
2457 		 * "Exit Qualification for EPT Violations".
2458 		 */
2459 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
2460 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
2461 			vmx_restore_nmi_blocking(vmx, vcpu);
2462 		break;
2463 	case EXIT_REASON_VIRTUALIZED_EOI:
2464 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
2465 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
2466 		SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpu, vmexit);
2467 		vmexit->inst_length = 0;	/* trap-like */
2468 		break;
2469 	case EXIT_REASON_APIC_ACCESS:
2470 		SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpu, vmexit);
2471 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
2472 		break;
2473 	case EXIT_REASON_APIC_WRITE:
2474 		/*
2475 		 * APIC-write VM exit is trap-like so the %rip is already
2476 		 * pointing to the next instruction.
2477 		 */
2478 		vmexit->inst_length = 0;
2479 		vlapic = vm_lapic(vmx->vm, vcpu);
2480 		SDT_PROBE4(vmm, vmx, exit, apicwrite,
2481 		    vmx, vcpu, vmexit, vlapic);
2482 		handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
2483 		break;
2484 	case EXIT_REASON_XSETBV:
2485 		SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpu, vmexit);
2486 		handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
2487 		break;
2488 	case EXIT_REASON_MONITOR:
2489 		SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpu, vmexit);
2490 		vmexit->exitcode = VM_EXITCODE_MONITOR;
2491 		break;
2492 	case EXIT_REASON_MWAIT:
2493 		SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpu, vmexit);
2494 		vmexit->exitcode = VM_EXITCODE_MWAIT;
2495 		break;
2496 	case EXIT_REASON_TPR:
2497 		vlapic = vm_lapic(vmx->vm, vcpu);
2498 		vlapic_sync_tpr(vlapic);
2499 		vmexit->inst_length = 0;
2500 		handled = HANDLED;
2501 		break;
2502 	case EXIT_REASON_VMCALL:
2503 	case EXIT_REASON_VMCLEAR:
2504 	case EXIT_REASON_VMLAUNCH:
2505 	case EXIT_REASON_VMPTRLD:
2506 	case EXIT_REASON_VMPTRST:
2507 	case EXIT_REASON_VMREAD:
2508 	case EXIT_REASON_VMRESUME:
2509 	case EXIT_REASON_VMWRITE:
2510 	case EXIT_REASON_VMXOFF:
2511 	case EXIT_REASON_VMXON:
2512 		SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpu, vmexit);
2513 		vmexit->exitcode = VM_EXITCODE_VMINSN;
2514 		break;
2515 	default:
2516 		SDT_PROBE4(vmm, vmx, exit, unknown,
2517 		    vmx, vcpu, vmexit, reason);
2518 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
2519 		break;
2520 	}
2521 
2522 	if (handled) {
2523 		/*
2524 		 * It is possible that control is returned to userland
2525 		 * even though we were able to handle the VM exit in the
2526 		 * kernel.
2527 		 *
2528 		 * In such a case we want to make sure that the userland
2529 		 * restarts guest execution at the instruction *after*
2530 		 * the one we just processed. Therefore we update the
2531 		 * guest rip in the VMCS and in 'vmexit'.
2532 		 */
2533 		vmexit->rip += vmexit->inst_length;
2534 		vmexit->inst_length = 0;
2535 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2536 	} else {
2537 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2538 			/*
2539 			 * If this VM exit was not claimed by anybody then
2540 			 * treat it as a generic VMX exit.
2541 			 */
2542 			vmexit->exitcode = VM_EXITCODE_VMX;
2543 			vmexit->u.vmx.status = VM_SUCCESS;
2544 			vmexit->u.vmx.inst_type = 0;
2545 			vmexit->u.vmx.inst_error = 0;
2546 		} else {
2547 			/*
2548 			 * The exitcode and collateral have been populated.
2549 			 * The VM exit will be processed further in userland.
2550 			 */
2551 		}
2552 	}
2553 
2554 	SDT_PROBE4(vmm, vmx, exit, return,
2555 	    vmx, vcpu, vmexit, handled);
2556 	return (handled);
2557 }
2558 
2559 static void
vmx_exit_inst_error(struct vmxctx * vmxctx,int rc,struct vm_exit * vmexit)2560 vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
2561 {
2562 
2563 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
2564 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
2565 	    vmxctx->inst_fail_status));
2566 
2567 	vmexit->inst_length = 0;
2568 	vmexit->exitcode = VM_EXITCODE_VMX;
2569 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
2570 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
2571 	vmexit->u.vmx.exit_reason = ~0;
2572 	vmexit->u.vmx.exit_qualification = ~0;
2573 
2574 	switch (rc) {
2575 	case VMX_VMRESUME_ERROR:
2576 	case VMX_VMLAUNCH_ERROR:
2577 	case VMX_INVEPT_ERROR:
2578 	case VMX_VMWRITE_ERROR:
2579 		vmexit->u.vmx.inst_type = rc;
2580 		break;
2581 	default:
2582 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
2583 	}
2584 }
2585 
2586 /*
2587  * If the NMI-exiting VM execution control is set to '1' then an NMI in
2588  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
2589  * sufficient to simply vector to the NMI handler via a software interrupt.
2590  * However, this must be done before maskable interrupts are enabled
2591  * otherwise the "iret" issued by an interrupt handler will incorrectly
2592  * clear NMI blocking.
2593  */
2594 static __inline void
vmx_exit_handle_possible_nmi(struct vm_exit * vmexit)2595 vmx_exit_handle_possible_nmi(struct vm_exit *vmexit)
2596 {
2597 	ASSERT(!interrupts_enabled());
2598 
2599 	if (vmexit->u.vmx.exit_reason == EXIT_REASON_EXCEPTION) {
2600 		uint32_t intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
2601 		ASSERT(intr_info & VMCS_INTR_VALID);
2602 
2603 		if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
2604 			ASSERT3U(intr_info & 0xff, ==, IDT_NMI);
2605 			vmm_call_trap(T_NMIFLT);
2606 		}
2607 	}
2608 }
2609 
2610 static __inline void
vmx_dr_enter_guest(struct vmxctx * vmxctx)2611 vmx_dr_enter_guest(struct vmxctx *vmxctx)
2612 {
2613 	uint64_t rflags;
2614 
2615 	/* Save host control debug registers. */
2616 	vmxctx->host_dr7 = rdr7();
2617 	vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR);
2618 
2619 	/*
2620 	 * Disable debugging in DR7 and DEBUGCTL to avoid triggering
2621 	 * exceptions in the host based on the guest DRx values.  The
2622 	 * guest DR7 and DEBUGCTL are saved/restored in the VMCS.
2623 	 */
2624 	load_dr7(0);
2625 	wrmsr(MSR_DEBUGCTLMSR, 0);
2626 
2627 	/*
2628 	 * Disable single stepping the kernel to avoid corrupting the
2629 	 * guest DR6.  A debugger might still be able to corrupt the
2630 	 * guest DR6 by setting a breakpoint after this point and then
2631 	 * single stepping.
2632 	 */
2633 	rflags = read_rflags();
2634 	vmxctx->host_tf = rflags & PSL_T;
2635 	write_rflags(rflags & ~PSL_T);
2636 
2637 	/* Save host debug registers. */
2638 	vmxctx->host_dr0 = rdr0();
2639 	vmxctx->host_dr1 = rdr1();
2640 	vmxctx->host_dr2 = rdr2();
2641 	vmxctx->host_dr3 = rdr3();
2642 	vmxctx->host_dr6 = rdr6();
2643 
2644 	/* Restore guest debug registers. */
2645 	load_dr0(vmxctx->guest_dr0);
2646 	load_dr1(vmxctx->guest_dr1);
2647 	load_dr2(vmxctx->guest_dr2);
2648 	load_dr3(vmxctx->guest_dr3);
2649 	load_dr6(vmxctx->guest_dr6);
2650 }
2651 
2652 static __inline void
vmx_dr_leave_guest(struct vmxctx * vmxctx)2653 vmx_dr_leave_guest(struct vmxctx *vmxctx)
2654 {
2655 
2656 	/* Save guest debug registers. */
2657 	vmxctx->guest_dr0 = rdr0();
2658 	vmxctx->guest_dr1 = rdr1();
2659 	vmxctx->guest_dr2 = rdr2();
2660 	vmxctx->guest_dr3 = rdr3();
2661 	vmxctx->guest_dr6 = rdr6();
2662 
2663 	/*
2664 	 * Restore host debug registers.  Restore DR7, DEBUGCTL, and
2665 	 * PSL_T last.
2666 	 */
2667 	load_dr0(vmxctx->host_dr0);
2668 	load_dr1(vmxctx->host_dr1);
2669 	load_dr2(vmxctx->host_dr2);
2670 	load_dr3(vmxctx->host_dr3);
2671 	load_dr6(vmxctx->host_dr6);
2672 	wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl);
2673 	load_dr7(vmxctx->host_dr7);
2674 	write_rflags(read_rflags() | vmxctx->host_tf);
2675 }
2676 
2677 static int
vmx_run(void * arg,int vcpu,uint64_t rip)2678 vmx_run(void *arg, int vcpu, uint64_t rip)
2679 {
2680 	int rc, handled, launched;
2681 	struct vmx *vmx;
2682 	struct vm *vm;
2683 	struct vmxctx *vmxctx;
2684 	uintptr_t vmcs_pa;
2685 	struct vm_exit *vmexit;
2686 	struct vlapic *vlapic;
2687 	uint32_t exit_reason;
2688 	bool tpr_shadow_active;
2689 	vm_client_t *vmc;
2690 
2691 	vmx = arg;
2692 	vm = vmx->vm;
2693 	vmcs_pa = vmx->vmcs_pa[vcpu];
2694 	vmxctx = &vmx->ctx[vcpu];
2695 	vlapic = vm_lapic(vm, vcpu);
2696 	vmexit = vm_exitinfo(vm, vcpu);
2697 	vmc = vm_get_vmclient(vm, vcpu);
2698 	launched = 0;
2699 	tpr_shadow_active = vmx_cap_en(vmx, VMX_CAP_TPR_SHADOW) &&
2700 	    !vmx_cap_en(vmx, VMX_CAP_APICV) &&
2701 	    (vmx->cap[vcpu].proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0;
2702 
2703 	vmx_msr_guest_enter(vmx, vcpu);
2704 
2705 	vmcs_load(vmcs_pa);
2706 
2707 	VERIFY(vmx->vmcs_state[vcpu] == VS_NONE && curthread->t_preempt != 0);
2708 	vmx->vmcs_state[vcpu] = VS_LOADED;
2709 
2710 	/*
2711 	 * XXX
2712 	 * We do this every time because we may setup the virtual machine
2713 	 * from a different process than the one that actually runs it.
2714 	 *
2715 	 * If the life of a virtual machine was spent entirely in the context
2716 	 * of a single process we could do this once in vmx_vminit().
2717 	 */
2718 	vmcs_write(VMCS_HOST_CR3, rcr3());
2719 
2720 	vmcs_write(VMCS_GUEST_RIP, rip);
2721 	vmx_set_pcpu_defaults(vmx, vcpu);
2722 	do {
2723 		enum event_inject_state inject_state;
2724 		uint64_t eptgen;
2725 
2726 		KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch "
2727 		    "%lx/%lx", __func__, vmcs_guest_rip(), rip));
2728 
2729 		handled = UNHANDLED;
2730 
2731 		/*
2732 		 * Perform initial event/exception/interrupt injection before
2733 		 * host CPU interrupts are disabled.
2734 		 */
2735 		inject_state = vmx_inject_events(vmx, vcpu, rip);
2736 
2737 		/*
2738 		 * Interrupts are disabled from this point on until the
2739 		 * guest starts executing. This is done for the following
2740 		 * reasons:
2741 		 *
2742 		 * If an AST is asserted on this thread after the check below,
2743 		 * then the IPI_AST notification will not be lost, because it
2744 		 * will cause a VM exit due to external interrupt as soon as
2745 		 * the guest state is loaded.
2746 		 *
2747 		 * A posted interrupt after vmx_inject_vlapic() will not be
2748 		 * "lost" because it will be held pending in the host APIC
2749 		 * because interrupts are disabled. The pending interrupt will
2750 		 * be recognized as soon as the guest state is loaded.
2751 		 *
2752 		 * The same reasoning applies to the IPI generated by vmspace
2753 		 * invalidation.
2754 		 */
2755 		disable_intr();
2756 
2757 		/*
2758 		 * If not precluded by existing events, inject any interrupt
2759 		 * pending on the vLAPIC.  As a lock-less operation, it is safe
2760 		 * (and prudent) to perform with host CPU interrupts disabled.
2761 		 */
2762 		if (inject_state == EIS_CAN_INJECT) {
2763 			inject_state = vmx_inject_vlapic(vmx, vcpu, vlapic);
2764 		}
2765 
2766 		/*
2767 		 * Check for vCPU bail-out conditions.  This must be done after
2768 		 * vmx_inject_events() to detect a triple-fault condition.
2769 		 */
2770 		if (vcpu_entry_bailout_checks(vmx->vm, vcpu, rip)) {
2771 			enable_intr();
2772 			break;
2773 		}
2774 
2775 		if (vcpu_run_state_pending(vm, vcpu)) {
2776 			enable_intr();
2777 			vm_exit_run_state(vmx->vm, vcpu, rip);
2778 			break;
2779 		}
2780 
2781 		/*
2782 		 * If subsequent activity queued events which require injection
2783 		 * handling, take another lap to handle them.
2784 		 */
2785 		if (vmx_inject_recheck(vmx, vcpu, inject_state)) {
2786 			enable_intr();
2787 			handled = HANDLED;
2788 			continue;
2789 		}
2790 
2791 		if ((rc = smt_acquire()) != 1) {
2792 			enable_intr();
2793 			vmexit->rip = rip;
2794 			vmexit->inst_length = 0;
2795 			if (rc == -1) {
2796 				vmexit->exitcode = VM_EXITCODE_HT;
2797 			} else {
2798 				vmexit->exitcode = VM_EXITCODE_BOGUS;
2799 				handled = HANDLED;
2800 			}
2801 			break;
2802 		}
2803 
2804 		/*
2805 		 * If this thread has gone off-cpu due to mutex operations
2806 		 * during vmx_run, the VMCS will have been unloaded, forcing a
2807 		 * re-VMLAUNCH as opposed to VMRESUME.
2808 		 */
2809 		launched = (vmx->vmcs_state[vcpu] & VS_LAUNCHED) != 0;
2810 		/*
2811 		 * Restoration of the GDT limit is taken care of by
2812 		 * vmx_savectx().  Since the maximum practical index for the
2813 		 * IDT is 255, restoring its limits from the post-VMX-exit
2814 		 * default of 0xffff is not a concern.
2815 		 *
2816 		 * Only 64-bit hypervisor callers are allowed, which forgoes
2817 		 * the need to restore any LDT descriptor.  Toss an error to
2818 		 * anyone attempting to break that rule.
2819 		 */
2820 		if (curproc->p_model != DATAMODEL_LP64) {
2821 			smt_release();
2822 			enable_intr();
2823 			bzero(vmexit, sizeof (*vmexit));
2824 			vmexit->rip = rip;
2825 			vmexit->exitcode = VM_EXITCODE_VMX;
2826 			vmexit->u.vmx.status = VM_FAIL_INVALID;
2827 			handled = UNHANDLED;
2828 			break;
2829 		}
2830 
2831 		if (tpr_shadow_active) {
2832 			vmx_tpr_shadow_enter(vlapic);
2833 		}
2834 
2835 		/*
2836 		 * Indicate activation of vmspace (EPT) table just prior to VMX
2837 		 * entry, checking for the necessity of an invept invalidation.
2838 		 */
2839 		eptgen = vmc_table_enter(vmc);
2840 		if (vmx->eptgen[curcpu] != eptgen) {
2841 			/*
2842 			 * VMspace generation does not match what was previously
2843 			 * used on this host CPU, so all mappings associated
2844 			 * with this EP4TA must be invalidated.
2845 			 */
2846 			invept(1, vmx->eptp);
2847 			vmx->eptgen[curcpu] = eptgen;
2848 		}
2849 
2850 		vmx_run_trace(vmx, vcpu);
2851 		vcpu_ustate_change(vm, vcpu, VU_RUN);
2852 		vmx_dr_enter_guest(vmxctx);
2853 
2854 		/* Perform VMX entry */
2855 		rc = vmx_enter_guest(vmxctx, vmx, launched);
2856 
2857 		vmx_dr_leave_guest(vmxctx);
2858 		vcpu_ustate_change(vm, vcpu, VU_EMU_KERN);
2859 
2860 		vmx->vmcs_state[vcpu] |= VS_LAUNCHED;
2861 		smt_release();
2862 
2863 		if (tpr_shadow_active) {
2864 			vmx_tpr_shadow_exit(vlapic);
2865 		}
2866 
2867 		/* Collect some information for VM exit processing */
2868 		vmexit->rip = rip = vmcs_guest_rip();
2869 		vmexit->inst_length = vmexit_instruction_length();
2870 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
2871 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
2872 		/* Update 'nextrip' */
2873 		vmx->state[vcpu].nextrip = rip;
2874 
2875 		if (rc == VMX_GUEST_VMEXIT) {
2876 			vmx_exit_handle_possible_nmi(vmexit);
2877 		}
2878 		enable_intr();
2879 		vmc_table_exit(vmc);
2880 
2881 		if (rc == VMX_GUEST_VMEXIT) {
2882 			handled = vmx_exit_process(vmx, vcpu, vmexit);
2883 		} else {
2884 			vmx_exit_inst_error(vmxctx, rc, vmexit);
2885 		}
2886 		DTRACE_PROBE3(vmm__vexit, int, vcpu, uint64_t, rip,
2887 		    uint32_t, exit_reason);
2888 		rip = vmexit->rip;
2889 	} while (handled);
2890 
2891 	/* If a VM exit has been handled then the exitcode must be BOGUS */
2892 	if (handled && vmexit->exitcode != VM_EXITCODE_BOGUS) {
2893 		panic("Non-BOGUS exitcode (%d) unexpected for handled VM exit",
2894 		    vmexit->exitcode);
2895 	}
2896 
2897 	VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
2898 	    vmexit->exitcode);
2899 
2900 	vmcs_clear(vmcs_pa);
2901 	vmx_msr_guest_exit(vmx, vcpu);
2902 
2903 	VERIFY(vmx->vmcs_state != VS_NONE && curthread->t_preempt != 0);
2904 	vmx->vmcs_state[vcpu] = VS_NONE;
2905 
2906 	return (0);
2907 }
2908 
2909 static void
vmx_vmcleanup(void * arg)2910 vmx_vmcleanup(void *arg)
2911 {
2912 	int i;
2913 	struct vmx *vmx = arg;
2914 	uint16_t maxcpus;
2915 
2916 	if (vmx_cap_en(vmx, VMX_CAP_APICV)) {
2917 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
2918 		kmem_free(vmx->apic_access_page, PAGESIZE);
2919 	} else {
2920 		VERIFY3P(vmx->apic_access_page, ==, NULL);
2921 	}
2922 
2923 	vmx_msr_bitmap_destroy(vmx);
2924 
2925 	maxcpus = vm_get_maxcpus(vmx->vm);
2926 	for (i = 0; i < maxcpus; i++)
2927 		vpid_free(vmx->state[i].vpid);
2928 
2929 	free(vmx, M_VMX);
2930 }
2931 
2932 static uint64_t *
vmxctx_regptr(struct vmxctx * vmxctx,int reg)2933 vmxctx_regptr(struct vmxctx *vmxctx, int reg)
2934 {
2935 	switch (reg) {
2936 	case VM_REG_GUEST_RAX:
2937 		return (&vmxctx->guest_rax);
2938 	case VM_REG_GUEST_RBX:
2939 		return (&vmxctx->guest_rbx);
2940 	case VM_REG_GUEST_RCX:
2941 		return (&vmxctx->guest_rcx);
2942 	case VM_REG_GUEST_RDX:
2943 		return (&vmxctx->guest_rdx);
2944 	case VM_REG_GUEST_RSI:
2945 		return (&vmxctx->guest_rsi);
2946 	case VM_REG_GUEST_RDI:
2947 		return (&vmxctx->guest_rdi);
2948 	case VM_REG_GUEST_RBP:
2949 		return (&vmxctx->guest_rbp);
2950 	case VM_REG_GUEST_R8:
2951 		return (&vmxctx->guest_r8);
2952 	case VM_REG_GUEST_R9:
2953 		return (&vmxctx->guest_r9);
2954 	case VM_REG_GUEST_R10:
2955 		return (&vmxctx->guest_r10);
2956 	case VM_REG_GUEST_R11:
2957 		return (&vmxctx->guest_r11);
2958 	case VM_REG_GUEST_R12:
2959 		return (&vmxctx->guest_r12);
2960 	case VM_REG_GUEST_R13:
2961 		return (&vmxctx->guest_r13);
2962 	case VM_REG_GUEST_R14:
2963 		return (&vmxctx->guest_r14);
2964 	case VM_REG_GUEST_R15:
2965 		return (&vmxctx->guest_r15);
2966 	case VM_REG_GUEST_CR2:
2967 		return (&vmxctx->guest_cr2);
2968 	case VM_REG_GUEST_DR0:
2969 		return (&vmxctx->guest_dr0);
2970 	case VM_REG_GUEST_DR1:
2971 		return (&vmxctx->guest_dr1);
2972 	case VM_REG_GUEST_DR2:
2973 		return (&vmxctx->guest_dr2);
2974 	case VM_REG_GUEST_DR3:
2975 		return (&vmxctx->guest_dr3);
2976 	case VM_REG_GUEST_DR6:
2977 		return (&vmxctx->guest_dr6);
2978 	default:
2979 		break;
2980 	}
2981 	return (NULL);
2982 }
2983 
2984 static int
vmx_getreg(void * arg,int vcpu,int reg,uint64_t * retval)2985 vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
2986 {
2987 	int running, hostcpu, err;
2988 	struct vmx *vmx = arg;
2989 	uint64_t *regp;
2990 
2991 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
2992 	if (running && hostcpu != curcpu)
2993 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
2994 
2995 	/* VMCS access not required for ctx reads */
2996 	if ((regp = vmxctx_regptr(&vmx->ctx[vcpu], reg)) != NULL) {
2997 		*retval = *regp;
2998 		return (0);
2999 	}
3000 
3001 	if (!running) {
3002 		vmcs_load(vmx->vmcs_pa[vcpu]);
3003 	}
3004 
3005 	err = 0;
3006 	if (reg == VM_REG_GUEST_INTR_SHADOW) {
3007 		uint64_t gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
3008 		*retval = (gi & HWINTR_BLOCKING) ? 1 : 0;
3009 	} else {
3010 		uint32_t encoding;
3011 
3012 		encoding = vmcs_field_encoding(reg);
3013 		switch (encoding) {
3014 		case VMCS_GUEST_CR0:
3015 			/* Take the shadow bits into account */
3016 			*retval = vmx_unshadow_cr0(vmcs_read(encoding),
3017 			    vmcs_read(VMCS_CR0_SHADOW));
3018 			break;
3019 		case VMCS_GUEST_CR4:
3020 			/* Take the shadow bits into account */
3021 			*retval = vmx_unshadow_cr4(vmcs_read(encoding),
3022 			    vmcs_read(VMCS_CR4_SHADOW));
3023 			break;
3024 		case VMCS_INVALID_ENCODING:
3025 			err = EINVAL;
3026 			break;
3027 		default:
3028 			*retval = vmcs_read(encoding);
3029 			break;
3030 		}
3031 	}
3032 
3033 	if (!running) {
3034 		vmcs_clear(vmx->vmcs_pa[vcpu]);
3035 	}
3036 
3037 	return (err);
3038 }
3039 
3040 static int
vmx_setreg(void * arg,int vcpu,int reg,uint64_t val)3041 vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
3042 {
3043 	int running, hostcpu, error;
3044 	struct vmx *vmx = arg;
3045 	uint64_t *regp;
3046 
3047 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3048 	if (running && hostcpu != curcpu)
3049 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
3050 
3051 	/* VMCS access not required for ctx writes */
3052 	if ((regp = vmxctx_regptr(&vmx->ctx[vcpu], reg)) != NULL) {
3053 		*regp = val;
3054 		return (0);
3055 	}
3056 
3057 	if (!running) {
3058 		vmcs_load(vmx->vmcs_pa[vcpu]);
3059 	}
3060 
3061 	if (reg == VM_REG_GUEST_INTR_SHADOW) {
3062 		if (val != 0) {
3063 			/*
3064 			 * Forcing the vcpu into an interrupt shadow is not
3065 			 * presently supported.
3066 			 */
3067 			error = EINVAL;
3068 		} else {
3069 			uint64_t gi;
3070 
3071 			gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
3072 			gi &= ~HWINTR_BLOCKING;
3073 			vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
3074 			error = 0;
3075 		}
3076 	} else {
3077 		uint32_t encoding;
3078 
3079 		error = 0;
3080 		encoding = vmcs_field_encoding(reg);
3081 		switch (encoding) {
3082 		case VMCS_GUEST_IA32_EFER:
3083 			/*
3084 			 * If the "load EFER" VM-entry control is 1 then the
3085 			 * value of EFER.LMA must be identical to "IA-32e mode
3086 			 * guest" bit in the VM-entry control.
3087 			 */
3088 			if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0) {
3089 				uint64_t ctls;
3090 
3091 				ctls = vmcs_read(VMCS_ENTRY_CTLS);
3092 				if (val & EFER_LMA) {
3093 					ctls |= VM_ENTRY_GUEST_LMA;
3094 				} else {
3095 					ctls &= ~VM_ENTRY_GUEST_LMA;
3096 				}
3097 				vmcs_write(VMCS_ENTRY_CTLS, ctls);
3098 			}
3099 			vmcs_write(encoding, val);
3100 			break;
3101 		case VMCS_GUEST_CR0:
3102 			/*
3103 			 * The guest is not allowed to modify certain bits in
3104 			 * %cr0 and %cr4.  To maintain the illusion of full
3105 			 * control, they have shadow versions which contain the
3106 			 * guest-perceived (via reads from the register) values
3107 			 * as opposed to the guest-effective values.
3108 			 *
3109 			 * This is detailed in the SDM: Vol. 3 Ch. 24.6.6.
3110 			 */
3111 			vmcs_write(VMCS_CR0_SHADOW, val);
3112 			vmcs_write(encoding, vmx_fix_cr0(val));
3113 			break;
3114 		case VMCS_GUEST_CR4:
3115 			/* See above for detail on %cr4 shadowing */
3116 			vmcs_write(VMCS_CR4_SHADOW, val);
3117 			vmcs_write(encoding, vmx_fix_cr4(val));
3118 			break;
3119 		case VMCS_GUEST_CR3:
3120 			vmcs_write(encoding, val);
3121 			/*
3122 			 * Invalidate the guest vcpu's TLB mappings to emulate
3123 			 * the behavior of updating %cr3.
3124 			 *
3125 			 * XXX the processor retains global mappings when %cr3
3126 			 * is updated but vmx_invvpid() does not.
3127 			 */
3128 			vmx_invvpid(vmx, vcpu, running);
3129 			break;
3130 		case VMCS_INVALID_ENCODING:
3131 			error = EINVAL;
3132 			break;
3133 		default:
3134 			vmcs_write(encoding, val);
3135 			break;
3136 		}
3137 	}
3138 
3139 	if (!running) {
3140 		vmcs_clear(vmx->vmcs_pa[vcpu]);
3141 	}
3142 
3143 	return (error);
3144 }
3145 
3146 static int
vmx_getdesc(void * arg,int vcpu,int seg,struct seg_desc * desc)3147 vmx_getdesc(void *arg, int vcpu, int seg, struct seg_desc *desc)
3148 {
3149 	int hostcpu, running;
3150 	struct vmx *vmx = arg;
3151 	uint32_t base, limit, access;
3152 
3153 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3154 	if (running && hostcpu != curcpu)
3155 		panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3156 
3157 	if (!running) {
3158 		vmcs_load(vmx->vmcs_pa[vcpu]);
3159 	}
3160 
3161 	vmcs_seg_desc_encoding(seg, &base, &limit, &access);
3162 	desc->base = vmcs_read(base);
3163 	desc->limit = vmcs_read(limit);
3164 	if (access != VMCS_INVALID_ENCODING) {
3165 		desc->access = vmcs_read(access);
3166 	} else {
3167 		desc->access = 0;
3168 	}
3169 
3170 	if (!running) {
3171 		vmcs_clear(vmx->vmcs_pa[vcpu]);
3172 	}
3173 	return (0);
3174 }
3175 
3176 static int
vmx_setdesc(void * arg,int vcpu,int seg,const struct seg_desc * desc)3177 vmx_setdesc(void *arg, int vcpu, int seg, const struct seg_desc *desc)
3178 {
3179 	int hostcpu, running;
3180 	struct vmx *vmx = arg;
3181 	uint32_t base, limit, access;
3182 
3183 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
3184 	if (running && hostcpu != curcpu)
3185 		panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu);
3186 
3187 	if (!running) {
3188 		vmcs_load(vmx->vmcs_pa[vcpu]);
3189 	}
3190 
3191 	vmcs_seg_desc_encoding(seg, &base, &limit, &access);
3192 	vmcs_write(base, desc->base);
3193 	vmcs_write(limit, desc->limit);
3194 	if (access != VMCS_INVALID_ENCODING) {
3195 		vmcs_write(access, desc->access);
3196 	}
3197 
3198 	if (!running) {
3199 		vmcs_clear(vmx->vmcs_pa[vcpu]);
3200 	}
3201 	return (0);
3202 }
3203 
3204 static int
vmx_getcap(void * arg,int vcpu,int type,int * retval)3205 vmx_getcap(void *arg, int vcpu, int type, int *retval)
3206 {
3207 	struct vmx *vmx = arg;
3208 	int vcap;
3209 	int ret;
3210 
3211 	ret = ENOENT;
3212 
3213 	vcap = vmx->cap[vcpu].set;
3214 
3215 	switch (type) {
3216 	case VM_CAP_HALT_EXIT:
3217 		if (cap_halt_exit)
3218 			ret = 0;
3219 		break;
3220 	case VM_CAP_PAUSE_EXIT:
3221 		if (cap_pause_exit)
3222 			ret = 0;
3223 		break;
3224 	case VM_CAP_MTRAP_EXIT:
3225 		if (cap_monitor_trap)
3226 			ret = 0;
3227 		break;
3228 	case VM_CAP_ENABLE_INVPCID:
3229 		if (cap_invpcid)
3230 			ret = 0;
3231 		break;
3232 	case VM_CAP_BPT_EXIT:
3233 		ret = 0;
3234 		break;
3235 	default:
3236 		break;
3237 	}
3238 
3239 	if (ret == 0)
3240 		*retval = (vcap & (1 << type)) ? 1 : 0;
3241 
3242 	return (ret);
3243 }
3244 
3245 static int
vmx_setcap(void * arg,int vcpu,int type,int val)3246 vmx_setcap(void *arg, int vcpu, int type, int val)
3247 {
3248 	struct vmx *vmx = arg;
3249 	uint32_t baseval, reg, flag;
3250 	uint32_t *pptr;
3251 	int error;
3252 
3253 	error = ENOENT;
3254 	pptr = NULL;
3255 
3256 	switch (type) {
3257 	case VM_CAP_HALT_EXIT:
3258 		if (cap_halt_exit) {
3259 			error = 0;
3260 			pptr = &vmx->cap[vcpu].proc_ctls;
3261 			baseval = *pptr;
3262 			flag = PROCBASED_HLT_EXITING;
3263 			reg = VMCS_PRI_PROC_BASED_CTLS;
3264 		}
3265 		break;
3266 	case VM_CAP_MTRAP_EXIT:
3267 		if (cap_monitor_trap) {
3268 			error = 0;
3269 			pptr = &vmx->cap[vcpu].proc_ctls;
3270 			baseval = *pptr;
3271 			flag = PROCBASED_MTF;
3272 			reg = VMCS_PRI_PROC_BASED_CTLS;
3273 		}
3274 		break;
3275 	case VM_CAP_PAUSE_EXIT:
3276 		if (cap_pause_exit) {
3277 			error = 0;
3278 			pptr = &vmx->cap[vcpu].proc_ctls;
3279 			baseval = *pptr;
3280 			flag = PROCBASED_PAUSE_EXITING;
3281 			reg = VMCS_PRI_PROC_BASED_CTLS;
3282 		}
3283 		break;
3284 	case VM_CAP_ENABLE_INVPCID:
3285 		if (cap_invpcid) {
3286 			error = 0;
3287 			pptr = &vmx->cap[vcpu].proc_ctls2;
3288 			baseval = *pptr;
3289 			flag = PROCBASED2_ENABLE_INVPCID;
3290 			reg = VMCS_SEC_PROC_BASED_CTLS;
3291 		}
3292 		break;
3293 	case VM_CAP_BPT_EXIT:
3294 		error = 0;
3295 
3296 		/* Don't change the bitmap if we are tracing all exceptions. */
3297 		if (vmx->cap[vcpu].exc_bitmap != 0xffffffff) {
3298 			pptr = &vmx->cap[vcpu].exc_bitmap;
3299 			baseval = *pptr;
3300 			flag = (1 << IDT_BP);
3301 			reg = VMCS_EXCEPTION_BITMAP;
3302 		}
3303 		break;
3304 	default:
3305 		break;
3306 	}
3307 
3308 	if (error != 0) {
3309 		return (error);
3310 	}
3311 
3312 	if (pptr != NULL) {
3313 		if (val) {
3314 			baseval |= flag;
3315 		} else {
3316 			baseval &= ~flag;
3317 		}
3318 		vmcs_load(vmx->vmcs_pa[vcpu]);
3319 		vmcs_write(reg, baseval);
3320 		vmcs_clear(vmx->vmcs_pa[vcpu]);
3321 
3322 		/*
3323 		 * Update optional stored flags, and record
3324 		 * setting
3325 		 */
3326 		*pptr = baseval;
3327 	}
3328 
3329 	if (val) {
3330 		vmx->cap[vcpu].set |= (1 << type);
3331 	} else {
3332 		vmx->cap[vcpu].set &= ~(1 << type);
3333 	}
3334 
3335 	return (0);
3336 }
3337 
3338 struct vlapic_vtx {
3339 	struct vlapic	vlapic;
3340 
3341 	/* Align to the nearest cacheline */
3342 	uint8_t		_pad[64 - (sizeof (struct vlapic) % 64)];
3343 
3344 	/* TMR handling state for posted interrupts */
3345 	uint32_t	tmr_active[8];
3346 	uint32_t	pending_level[8];
3347 	uint32_t	pending_edge[8];
3348 
3349 	struct pir_desc	*pir_desc;
3350 	struct vmx	*vmx;
3351 	uint_t	pending_prio;
3352 	boolean_t	tmr_sync;
3353 };
3354 
3355 CTASSERT((offsetof(struct vlapic_vtx, tmr_active) & 63) == 0);
3356 
3357 #define	VPR_PRIO_BIT(vpr)	(1 << ((vpr) >> 4))
3358 
3359 static vcpu_notify_t
vmx_apicv_set_ready(struct vlapic * vlapic,int vector,bool level)3360 vmx_apicv_set_ready(struct vlapic *vlapic, int vector, bool level)
3361 {
3362 	struct vlapic_vtx *vlapic_vtx;
3363 	struct pir_desc *pir_desc;
3364 	uint32_t mask, tmrval;
3365 	int idx;
3366 	vcpu_notify_t notify = VCPU_NOTIFY_NONE;
3367 
3368 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3369 	pir_desc = vlapic_vtx->pir_desc;
3370 	idx = vector / 32;
3371 	mask = 1UL << (vector % 32);
3372 
3373 	/*
3374 	 * If the currently asserted TMRs do not match the state requested by
3375 	 * the incoming interrupt, an exit will be required to reconcile those
3376 	 * bits in the APIC page.  This will keep the vLAPIC behavior in line
3377 	 * with the architecturally defined expectations.
3378 	 *
3379 	 * If actors of mixed types (edge and level) are racing against the same
3380 	 * vector (toggling its TMR bit back and forth), the results could
3381 	 * inconsistent.  Such circumstances are considered a rare edge case and
3382 	 * are never expected to be found in the wild.
3383 	 */
3384 	tmrval = atomic_load_acq_int(&vlapic_vtx->tmr_active[idx]);
3385 	if (!level) {
3386 		if ((tmrval & mask) != 0) {
3387 			/* Edge-triggered interrupt needs TMR de-asserted */
3388 			atomic_set_int(&vlapic_vtx->pending_edge[idx], mask);
3389 			atomic_store_rel_long(&pir_desc->pending, 1);
3390 			return (VCPU_NOTIFY_EXIT);
3391 		}
3392 	} else {
3393 		if ((tmrval & mask) == 0) {
3394 			/* Level-triggered interrupt needs TMR asserted */
3395 			atomic_set_int(&vlapic_vtx->pending_level[idx], mask);
3396 			atomic_store_rel_long(&pir_desc->pending, 1);
3397 			return (VCPU_NOTIFY_EXIT);
3398 		}
3399 	}
3400 
3401 	/*
3402 	 * If the interrupt request does not require manipulation of the TMRs
3403 	 * for delivery, set it in PIR descriptor.  It cannot be inserted into
3404 	 * the APIC page while the vCPU might be running.
3405 	 */
3406 	atomic_set_int(&pir_desc->pir[idx], mask);
3407 
3408 	/*
3409 	 * A notification is required whenever the 'pending' bit makes a
3410 	 * transition from 0->1.
3411 	 *
3412 	 * Even if the 'pending' bit is already asserted, notification about
3413 	 * the incoming interrupt may still be necessary.  For example, if a
3414 	 * vCPU is HLTed with a high PPR, a low priority interrupt would cause
3415 	 * the 0->1 'pending' transition with a notification, but the vCPU
3416 	 * would ignore the interrupt for the time being.  The same vCPU would
3417 	 * need to then be notified if a high-priority interrupt arrived which
3418 	 * satisfied the PPR.
3419 	 *
3420 	 * The priorities of interrupts injected while 'pending' is asserted
3421 	 * are tracked in a custom bitfield 'pending_prio'.  Should the
3422 	 * to-be-injected interrupt exceed the priorities already present, the
3423 	 * notification is sent.  The priorities recorded in 'pending_prio' are
3424 	 * cleared whenever the 'pending' bit makes another 0->1 transition.
3425 	 */
3426 	if (atomic_cmpset_long(&pir_desc->pending, 0, 1) != 0) {
3427 		notify = VCPU_NOTIFY_APIC;
3428 		vlapic_vtx->pending_prio = 0;
3429 	} else {
3430 		const uint_t old_prio = vlapic_vtx->pending_prio;
3431 		const uint_t prio_bit = VPR_PRIO_BIT(vector & APIC_TPR_INT);
3432 
3433 		if ((old_prio & prio_bit) == 0 && prio_bit > old_prio) {
3434 			atomic_set_int(&vlapic_vtx->pending_prio, prio_bit);
3435 			notify = VCPU_NOTIFY_APIC;
3436 		}
3437 	}
3438 
3439 	return (notify);
3440 }
3441 
3442 static void
vmx_apicv_accepted(struct vlapic * vlapic,int vector)3443 vmx_apicv_accepted(struct vlapic *vlapic, int vector)
3444 {
3445 	/*
3446 	 * When APICv is enabled for an instance, the traditional interrupt
3447 	 * injection method (populating ENTRY_INTR_INFO in the VMCS) is not
3448 	 * used and the CPU does the heavy lifting of virtual interrupt
3449 	 * delivery.  For that reason vmx_intr_accepted() should never be called
3450 	 * when APICv is enabled.
3451 	 */
3452 	panic("vmx_intr_accepted: not expected to be called");
3453 }
3454 
3455 static void
vmx_apicv_sync_tmr(struct vlapic * vlapic)3456 vmx_apicv_sync_tmr(struct vlapic *vlapic)
3457 {
3458 	struct vlapic_vtx *vlapic_vtx;
3459 	const uint32_t *tmrs;
3460 
3461 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
3462 	tmrs = &vlapic_vtx->tmr_active[0];
3463 
3464 	if (!vlapic_vtx->tmr_sync) {
3465 		return;
3466 	}
3467 
3468 	vmcs_write(VMCS_EOI_EXIT0, ((uint64_t)